made the embedding procedure more robust

app/MindWork AI Studio/Assistants/I18N/allTexts.lua

@@ -3826,14 +3826,20 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1133561850"
 -- Describe what data this directory contains to help the AI select it.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1136409150"] = "Describe what data this directory contains to help the AI select it."
 
+-- Default tokenizer
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1220918127"] = "Default tokenizer"
+
 -- Select a root directory for this data source. All data in this directory and all its subdirectories will be processed for this data source.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1265737624"] = "Select a root directory for this data source. All data in this directory and all its subdirectories will be processed for this data source."
 
 -- Selected base directory for this data source
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1312296210"] = "Selected base directory for this data source"
 
--- Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting.
+-- No embedding selected
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1529596777"] = "Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting."
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1359179968"] = "No embedding selected"
+
+-- Tokenizer
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1696723386"] = "Tokenizer"
 
 -- Description
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1725856265"] = "Description"
@@ -3841,23 +3847,26 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1725856265"
 -- How many matches do you want at most per query?
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1827669611"] = "How many matches do you want at most per query?"
 
+-- This data source already has indexed embeddings. Delete and recreate it to change the folder path or embedding model.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1845247453"] = "This data source already has indexed embeddings. Delete and recreate it to change the folder path or embedding model."
+
 -- Update
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1847791252"] = "Update"
 
 -- In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1948697886"] = "In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method."
 
--- The data source max chunk length must be less than or equal to the embedding provider limit ({0}).
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2166221195"] = "The data source max chunk length must be less than or equal to the embedding provider limit ({0})."
-
 -- Number of tokens repeated at the start of the next chunk. Use 0 to disable overlap.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2263831524"] = "Number of tokens repeated at the start of the next chunk. Use 0 to disable overlap."
 
+-- Please enter 0 or a positive max embedding size.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2571963846"] = "Please enter 0 or a positive max embedding size."
+
 -- Add
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2646845972"] = "Add"
 
--- Chunk overlap length
+-- Max embedding size
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2801813827"] = "Chunk overlap length"
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2812251281"] = "Max embedding size"
 
 -- The embedding you selected runs locally or in your organization. Your data is not sent to the cloud.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2814869210"] = "The embedding you selected runs locally or in your organization. Your data is not sent to the cloud."
@@ -3865,8 +3874,11 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2814869210"
 -- Embedding
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2838542994"] = "Embedding"
 
--- Please enter 0 or a positive max chunk length.
+-- Maximum number of tokens per embedded chunk for this data source. Use 0 to use the embedding provider setting.
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3130373769"] = "Please enter 0 or a positive max chunk length."
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T287150655"] = "Maximum number of tokens per embedded chunk for this data source. Use 0 to use the embedding provider setting."
+
+-- The overlap must be smaller than the effective max embedding size.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3100817060"] = "The overlap must be smaller than the effective max embedding size."
 
 -- Please enter 0 or a positive overlap length.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3242265813"] = "Please enter 0 or a positive overlap length."
@@ -3877,8 +3889,8 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3359366900"
 -- Show Expert Settings
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3361153305"] = "Show Expert Settings"
 
--- Max chunk length
+-- Embedding overlap
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3621670500"] = "Max chunk length"
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T375910361"] = "Embedding overlap"
 
 -- Your security policy
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T4081226330"] = "Your security policy"
@@ -3886,8 +3898,8 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T4081226330"
 -- Select the base directory
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T562479068"] = "Select the base directory"
 
--- The overlap must be smaller than the effective max chunk length.
+-- The data source max embedding size must be smaller than the embedding provider limit ({0}). Use 0 to use the provider setting.
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T57492807"] = "The overlap must be smaller than the effective max chunk length."
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T607909514"] = "The data source max embedding size must be smaller than the embedding provider limit ({0}). Use 0 to use the provider setting."
 
 -- Data Source Name
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T813773421"] = "Data Source Name"
@@ -3976,8 +3988,14 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1133561850"] = "
 -- Select a file for this data source. The content of this file will be processed for the data source.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1190880267"] = "Select a file for this data source. The content of this file will be processed for the data source."
 
--- Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting.
+-- Default tokenizer
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1529596777"] = "Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting."
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1220918127"] = "Default tokenizer"
+
+-- No embedding selected
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1359179968"] = "No embedding selected"
+
+-- Tokenizer
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1696723386"] = "Tokenizer"
 
 -- Description
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1725856265"] = "Description"
@@ -3991,17 +4009,17 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1847791252"] = "
 -- In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1948697886"] = "In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method."
 
--- The data source max chunk length must be less than or equal to the embedding provider limit ({0}).
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2166221195"] = "The data source max chunk length must be less than or equal to the embedding provider limit ({0})."
-
 -- Number of tokens repeated at the start of the next chunk. Use 0 to disable overlap.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2263831524"] = "Number of tokens repeated at the start of the next chunk. Use 0 to disable overlap."
 
+-- Please enter 0 or a positive max embedding size.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2571963846"] = "Please enter 0 or a positive max embedding size."
+
 -- Add
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2646845972"] = "Add"
 
--- Chunk overlap length
+-- Max embedding size
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2801813827"] = "Chunk overlap length"
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2812251281"] = "Max embedding size"
 
 -- The embedding you selected runs locally or in your organization. Your data is not sent to the cloud.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2814869210"] = "The embedding you selected runs locally or in your organization. Your data is not sent to the cloud."
@@ -4012,8 +4030,11 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2838542994"] = "
 -- Describe what data this file contains to help the AI select it.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2859265837"] = "Describe what data this file contains to help the AI select it."
 
--- Please enter 0 or a positive max chunk length.
+-- Maximum number of tokens per embedded chunk for this data source. Use 0 to use the embedding provider setting.
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3130373769"] = "Please enter 0 or a positive max chunk length."
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T287150655"] = "Maximum number of tokens per embedded chunk for this data source. Use 0 to use the embedding provider setting."
+
+-- The overlap must be smaller than the effective max embedding size.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3100817060"] = "The overlap must be smaller than the effective max embedding size."
 
 -- Please enter 0 or a positive overlap length.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3242265813"] = "Please enter 0 or a positive overlap length."
@@ -4024,17 +4045,20 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3359366900"] = "
 -- Show Expert Settings
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3361153305"] = "Show Expert Settings"
 
--- Max chunk length
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3621670500"] = "Max chunk length"
-
 -- Select the file
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3740148848"] = "Select the file"
 
+-- Embedding overlap
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T375910361"] = "Embedding overlap"
+
 -- Your security policy
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T4081226330"] = "Your security policy"
 
--- The overlap must be smaller than the effective max chunk length.
+-- This data source already has indexed embeddings. Delete and recreate it to change the file path or embedding model.
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T57492807"] = "The overlap must be smaller than the effective max chunk length."
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T458267883"] = "This data source already has indexed embeddings. Delete and recreate it to change the file path or embedding model."
+
+-- The data source max embedding size must be smaller than the embedding provider limit ({0}). Use 0 to use the provider setting.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T607909514"] = "The data source max embedding size must be smaller than the embedding provider limit ({0}). Use 0 to use the provider setting."
 
 -- Data Source Name
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T813773421"] = "Data Source Name"
@@ -4246,6 +4270,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2331453405"] = "(O
 -- Invalid tokenizer:
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2448302543"] = "Invalid tokenizer:"
 
+-- Please enter a max embedding size greater than 0.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2608328491"] = "Please enter a max embedding size greater than 0."
+
 -- Add
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2646845972"] = "Add"
 
@@ -4255,18 +4282,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T278585345"] = "Sel
 -- No models loaded or available.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2810182573"] = "No models loaded or available."
 
+-- Max embedding size
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2812251281"] = "Max embedding size"
+
 -- Instance Name
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2842060373"] = "Instance Name"
 
 -- Currently, we cannot query the embedding models for the selected provider and/or host. Therefore, please enter the model name manually.
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T290547799"] = "Currently, we cannot query the embedding models for the selected provider and/or host. Therefore, please enter the model name manually."
 
--- Token limit
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2961294165"] = "Token limit"
-
--- Please enter a token limit greater than 0.
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T3316544737"] = "Please enter a token limit greater than 0."
-
 -- Show Expert Settings
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T3361153305"] = "Show Expert Settings"
 

app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor

@@ -41,12 +41,18 @@
             <MudJustifiedText Typo="Typo.body1" Class="mb-3">
                 @T("Select a root directory for this data source. All data in this directory and all its subdirectories will be processed for this data source.")
             </MudJustifiedText>
-            <SelectDirectory @bind-Directory="@this.dataPath" Label="@T("Selected base directory for this data source")" DirectoryDialogTitle="@T("Select the base directory")" Validation="@this.dataSourceValidation.ValidatePath" />
+            @if (!this.CanChangeSourceAndEmbedding)
+            {
+                <MudAlert Severity="Severity.Info" Variant="Variant.Outlined" Class="mb-3">
+                    @T("This data source already has indexed embeddings. Delete and recreate it to change the folder path or embedding model.")
+                </MudAlert>
+            }
+            <SelectDirectory @bind-Directory="@this.dataPath" Label="@T("Selected base directory for this data source")" DirectoryDialogTitle="@T("Select the base directory")" Validation="@this.dataSourceValidation.ValidatePath" Disabled="@(!this.CanChangeSourceAndEmbedding)" />
 
             <MudJustifiedText Typo="Typo.body1" Class="mb-3">
                 @T("In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method.")
             </MudJustifiedText>
-            <MudSelect @bind-Value="@this.dataEmbeddingId" Label="@T("Embedding")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateEmbeddingId">
+            <MudSelect @bind-Value="@this.dataEmbeddingId" Label="@T("Embedding")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateEmbeddingId" Disabled="@(!this.CanChangeSourceAndEmbedding)">
                 @foreach (var embedding in this.AvailableEmbeddings)
                 {
                     <MudSelectItem Value="@embedding.Value">
@@ -57,6 +63,18 @@
             
             @if (!string.IsNullOrWhiteSpace(this.dataEmbeddingId))
             {
+                <MudTextField
+                    T="string"
+                    Text="@this.SelectedEmbeddingTokenizerText"
+                    Label="@T("Tokenizer")"
+                    Class="mb-3"
+                    ReadOnly="@true"
+                    Variant="Variant.Outlined"
+                    Adornment="Adornment.Start"
+                    AdornmentIcon="@Icons.Material.Filled.DataObject"
+                    AdornmentColor="Color.Info"
+                />
+
                 if (this.SelectedCloudEmbedding)
                 {
                     <DataSourceCloudEmbeddingWarning DataSourceType="DataSourceType.LOCAL_DIRECTORY" SourcePath="@this.dataPath" @bind-UserAcknowledged="@this.dataUserAcknowledgedCloudEmbedding" Validation="@this.dataSourceValidation.ValidateUserAcknowledgedCloudEmbedding"/>
@@ -80,7 +98,7 @@
                         <MudNumericField
                             T="int"
                             @bind-Value="@this.dataMaxChunkTokenLength"
-                            Label="@T("Max chunk length")"
+                            Label="@T("Token limit")"
                             Class="mb-3"
                             Min="0"
                             Immediate="@true"
@@ -94,7 +112,7 @@
                         <MudNumericField
                             T="int"
                             @bind-Value="@this.dataChunkOverlapTokenLength"
-                            Label="@T("Chunk overlap length")"
+                            Label="@T("Token overlap")"
                             Min="0"
                             Immediate="@true"
                             Validation="@this.ValidateChunkOverlapTokenLength"

app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs

@@ -18,6 +18,9 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
     [Parameter]
     public DataSourceLocalDirectory DataSource { get; set; }
 
+    [Parameter]
+    public bool LockSourceAndEmbedding { get; set; }
+
     [Parameter]
     public IReadOnlyList<ConfigurationSelectData<string>> AvailableEmbeddings { get; set; } = [];
     
@@ -107,6 +110,14 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
 
     private bool SelectedCloudEmbedding => this.SelectedEmbedding is { IsSelfHosted: false };
 
+    private bool CanChangeSourceAndEmbedding => !this.IsEditing || !this.LockSourceAndEmbedding;
+
+    private string SelectedEmbeddingTokenizerText => this.SelectedEmbedding is null
+        ? T("No embedding selected")
+        : string.IsNullOrWhiteSpace(this.SelectedEmbedding.TokenizerPath)
+            ? T("Default tokenizer")
+            : System.IO.Path.GetFileName(this.SelectedEmbedding.TokenizerPath);
+
     private DataSourceLocalDirectory CreateDataSource() => new()
     {
         Id = this.dataId,
@@ -114,8 +125,8 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
         Name = this.dataName,
         Description = this.dataDescription,
         Type = DataSourceType.LOCAL_DIRECTORY,
-        EmbeddingId = this.dataEmbeddingId,
+        EmbeddingId = this.CanChangeSourceAndEmbedding ? this.dataEmbeddingId : this.DataSource.EmbeddingId,
-        Path = this.dataPath,
+        Path = this.CanChangeSourceAndEmbedding ? this.dataPath : this.DataSource.Path,
         MaxChunkTokenLength = this.dataMaxChunkTokenLength,
         ChunkOverlapTokenLength = this.dataChunkOverlapTokenLength,
         SecurityPolicy = this.dataSecurityPolicy,
@@ -139,11 +150,11 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
     private string? ValidateMaxChunkTokenLength(int maxChunkTokenLength)
     {
         if (maxChunkTokenLength < 0)
-            return T("Please enter 0 or a positive max chunk length.");
+            return T("Please enter 0 or a positive token limit.");
 
         var providerMaxChunkTokenLength = this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT;
-        if (maxChunkTokenLength > providerMaxChunkTokenLength)
+        if (maxChunkTokenLength > 0 && maxChunkTokenLength >= providerMaxChunkTokenLength)
-            return string.Format(T("The data source max chunk length must be less than or equal to the embedding provider limit ({0})."), providerMaxChunkTokenLength);
+            return string.Format(T("The data source token limit must be smaller than the embedding provider token limit ({0}). Use 0 to use the provider setting."), providerMaxChunkTokenLength);
 
         return null;
     }
@@ -157,7 +168,7 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
             ? this.dataMaxChunkTokenLength
             : this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT;
         if (chunkOverlapTokenLength >= effectiveMaxChunkTokenLength)
-            return T("The overlap must be smaller than the effective max chunk length.");
+            return T("The overlap must be smaller than the effective token limit.");
 
         return null;
     }

app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor

@@ -41,12 +41,18 @@
             <MudJustifiedText Typo="Typo.body1" Class="mb-3">
                 @T("Select a file for this data source. The content of this file will be processed for the data source.")
             </MudJustifiedText>
-            <SelectFile @bind-File="@this.dataFilePath" Label="@T("Selected file path for this data source")" FileDialogTitle="@T("Select the file")" Validation="@this.dataSourceValidation.ValidateFilePath" />
+            @if (!this.CanChangeSourceAndEmbedding)
+            {
+                <MudAlert Severity="Severity.Info" Variant="Variant.Outlined" Class="mb-3">
+                    @T("This data source already has indexed embeddings. Delete and recreate it to change the file path or embedding model.")
+                </MudAlert>
+            }
+            <SelectFile @bind-File="@this.dataFilePath" Label="@T("Selected file path for this data source")" FileDialogTitle="@T("Select the file")" Validation="@this.dataSourceValidation.ValidateFilePath" Disabled="@(!this.CanChangeSourceAndEmbedding)" />
 
             <MudJustifiedText Typo="Typo.body1" Class="mb-3">
                 @T("In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method.")
             </MudJustifiedText>
-            <MudSelect @bind-Value="@this.dataEmbeddingId" Label="@T("Embedding")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateEmbeddingId">
+            <MudSelect @bind-Value="@this.dataEmbeddingId" Label="@T("Embedding")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateEmbeddingId" Disabled="@(!this.CanChangeSourceAndEmbedding)">
                 @foreach (var embedding in this.AvailableEmbeddings)
                 {
                     <MudSelectItem Value="@embedding.Value">
@@ -57,6 +63,18 @@
             
             @if (!string.IsNullOrWhiteSpace(this.dataEmbeddingId))
             {
+                <MudTextField
+                    T="string"
+                    Text="@this.SelectedEmbeddingTokenizerText"
+                    Label="@T("Tokenizer")"
+                    Class="mb-3"
+                    ReadOnly="@true"
+                    Variant="Variant.Outlined"
+                    Adornment="Adornment.Start"
+                    AdornmentIcon="@Icons.Material.Filled.DataObject"
+                    AdornmentColor="Color.Info"
+                />
+
                 if (this.SelectedCloudEmbedding)
                 {
                     <DataSourceCloudEmbeddingWarning DataSourceType="DataSourceType.LOCAL_FILE" SourcePath="@this.dataFilePath" @bind-UserAcknowledged="@this.dataUserAcknowledgedCloudEmbedding" Validation="@this.dataSourceValidation.ValidateUserAcknowledgedCloudEmbedding"/>
@@ -80,7 +98,7 @@
                         <MudNumericField
                             T="int"
                             @bind-Value="@this.dataMaxChunkTokenLength"
-                            Label="@T("Max chunk length")"
+                            Label="@T("Token limit")"
                             Class="mb-3"
                             Min="0"
                             Immediate="@true"
@@ -94,7 +112,7 @@
                         <MudNumericField
                             T="int"
                             @bind-Value="@this.dataChunkOverlapTokenLength"
-                            Label="@T("Chunk overlap length")"
+                            Label="@T("Token overlap")"
                             Min="0"
                             Immediate="@true"
                             Validation="@this.ValidateChunkOverlapTokenLength"

app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs

@@ -17,6 +17,9 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
     
     [Parameter]
     public DataSourceLocalFile DataSource { get; set; }
+
+    [Parameter]
+    public bool LockSourceAndEmbedding { get; set; }
     
     [Parameter]
     public IReadOnlyList<ConfigurationSelectData<string>> AvailableEmbeddings { get; set; } = [];
@@ -107,6 +110,14 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
 
     private bool SelectedCloudEmbedding => this.SelectedEmbedding is { IsSelfHosted: false };
 
+    private bool CanChangeSourceAndEmbedding => !this.IsEditing || !this.LockSourceAndEmbedding;
+
+    private string SelectedEmbeddingTokenizerText => this.SelectedEmbedding is null
+        ? T("No embedding selected")
+        : string.IsNullOrWhiteSpace(this.SelectedEmbedding.TokenizerPath)
+            ? T("Default tokenizer")
+            : System.IO.Path.GetFileName(this.SelectedEmbedding.TokenizerPath);
+
     private DataSourceLocalFile CreateDataSource() => new()
     {
         Id = this.dataId,
@@ -114,8 +125,8 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
         Name = this.dataName,
         Description = this.dataDescription,
         Type = DataSourceType.LOCAL_FILE,
-        EmbeddingId = this.dataEmbeddingId,
+        EmbeddingId = this.CanChangeSourceAndEmbedding ? this.dataEmbeddingId : this.DataSource.EmbeddingId,
-        FilePath = this.dataFilePath,
+        FilePath = this.CanChangeSourceAndEmbedding ? this.dataFilePath : this.DataSource.FilePath,
         MaxChunkTokenLength = this.dataMaxChunkTokenLength,
         ChunkOverlapTokenLength = this.dataChunkOverlapTokenLength,
         SecurityPolicy = this.dataSecurityPolicy,
@@ -139,11 +150,11 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
     private string? ValidateMaxChunkTokenLength(int maxChunkTokenLength)
     {
         if (maxChunkTokenLength < 0)
-            return T("Please enter 0 or a positive max chunk length.");
+            return T("Please enter 0 or a positive token limit.");
 
         var providerMaxChunkTokenLength = this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT;
-        if (maxChunkTokenLength > providerMaxChunkTokenLength)
+        if (maxChunkTokenLength > 0 && maxChunkTokenLength >= providerMaxChunkTokenLength)
-            return string.Format(T("The data source max chunk length must be less than or equal to the embedding provider limit ({0})."), providerMaxChunkTokenLength);
+            return string.Format(T("The data source token limit must be smaller than the embedding provider token limit ({0}). Use 0 to use the provider setting."), providerMaxChunkTokenLength);
 
         return null;
     }
@@ -157,7 +168,7 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
             ? this.dataMaxChunkTokenLength
             : this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT;
         if (chunkOverlapTokenLength >= effectiveMaxChunkTokenLength)
-            return T("The overlap must be smaller than the effective max chunk length.");
+            return T("The overlap must be smaller than the effective token limit.");
 
         return null;
     }

app/MindWork AI Studio/Dialogs/Settings/SettingsDialogDataSources.razor.cs

@@ -227,6 +227,8 @@ public partial class SettingsDialogDataSources : SettingsDialogBase
             return;
 
         IDataSource? editedDataSource = null;
+        var lockDataSourceIdentity = dataSource is IInternalDataSource
+            && await this.DataSourceEmbeddingService.ShouldLockDataSourceIdentityAsync(dataSource.Id);
         switch (dataSource)
         {
             case DataSourceLocalFile localFile:
@@ -234,6 +236,7 @@ public partial class SettingsDialogDataSources : SettingsDialogBase
                 {
                     { x => x.IsEditing, true },
                     { x => x.DataSource, localFile },
+                    { x => x.LockSourceAndEmbedding, lockDataSourceIdentity },
                     { x => x.AvailableEmbeddings, this.AvailableEmbeddingProviders }
                 };
         
@@ -250,6 +253,7 @@ public partial class SettingsDialogDataSources : SettingsDialogBase
                 {
                     { x => x.IsEditing, true },
                     { x => x.DataSource, localDirectory },
+                    { x => x.LockSourceAndEmbedding, lockDataSourceIdentity },
                     { x => x.AvailableEmbeddings, this.AvailableEmbeddingProviders }
                 };
         

app/MindWork AI Studio/Pages/Embeddings.razor.cs

@@ -64,7 +64,7 @@ public partial class Embeddings : MSGComponentBase
 
     private async Task RefreshDataSource(DataSourceEmbeddingStatus status)
     {
-        await this.DataSourceEmbeddingService.QueueDataSourceAsync(status.DataSourceId);
+        await this.DataSourceEmbeddingService.RetryDataSourceAsync(status.DataSourceId);
         this.ReloadStatuses();
         await this.InvokeAsync(this.StateHasChanged);
     }

app/MindWork AI Studio/Tools/Databases/EmbeddingState/EmbeddingStateClient.cs

@@ -12,11 +12,14 @@ public abstract class EmbeddingStateClient(string name, string path) : DatabaseC
         string dataSourceType,
         string embeddingProviderId,
         string embeddingSignature,
+        string sourceHash,
         int vectorSize,
         CancellationToken token);
 
     public abstract Task UpdateVectorSizeAsync(string dataSourceId, int vectorSize, CancellationToken token);
 
+    public abstract Task UpdateDataSourceHashAsync(string dataSourceId, string sourceHash, CancellationToken token);
+
     public abstract Task UpsertFileAsync(string dataSourceId, EmbeddingStateFile file, CancellationToken token);
 
     public abstract Task DeleteFileAsync(string dataSourceId, string filePath, CancellationToken token);

app/MindWork AI Studio/Tools/Databases/EmbeddingState/NoEmbeddingStateClient.cs

@@ -32,11 +32,14 @@ public sealed class NoEmbeddingStateClient(string name, string? unavailableReaso
         string dataSourceType,
         string embeddingProviderId,
         string embeddingSignature,
+        string sourceHash,
         int vectorSize,
         CancellationToken token) => Task.CompletedTask;
 
     public override Task UpdateVectorSizeAsync(string dataSourceId, int vectorSize, CancellationToken token) => Task.CompletedTask;
 
+    public override Task UpdateDataSourceHashAsync(string dataSourceId, string sourceHash, CancellationToken token) => Task.CompletedTask;
+
     public override Task UpsertFileAsync(string dataSourceId, EmbeddingStateFile file, CancellationToken token) => Task.CompletedTask;
 
     public override Task DeleteFileAsync(string dataSourceId, string filePath, CancellationToken token) => Task.CompletedTask;

app/MindWork AI Studio/Tools/Databases/EmbeddingState/SqliteEmbeddingStateClientImplementation.cs

@@ -77,7 +77,7 @@ public sealed class SqliteEmbeddingStateClientImplementation(
         var manifest = new DataSourceEmbeddingManifest();
 
         await using (var command = CreateCommand(connection, """
-                                                            SELECT embedding_provider_id, embedding_signature, vector_size
+                                                            SELECT embedding_provider_id, embedding_signature, source_hash, vector_size
                                                             FROM data_sources
                                                             WHERE data_source_id = $dataSourceId
                                                             """))
@@ -88,7 +88,8 @@ public sealed class SqliteEmbeddingStateClientImplementation(
             {
                 manifest.EmbeddingProviderId = reader.GetString(0);
                 manifest.EmbeddingSignature = reader.GetString(1);
-                manifest.VectorSize = reader.GetInt32(2);
+                manifest.SourceHash = reader.GetString(2);
+                manifest.VectorSize = reader.GetInt32(3);
             }
         }
 
@@ -120,6 +121,7 @@ public sealed class SqliteEmbeddingStateClientImplementation(
         string dataSourceType,
         string embeddingProviderId,
         string embeddingSignature,
+        string sourceHash,
         int vectorSize,
         CancellationToken token)
     {
@@ -131,6 +133,7 @@ public sealed class SqliteEmbeddingStateClientImplementation(
                                                   data_source_type,
                                                   embedding_provider_id,
                                                   embedding_signature,
+                                                  source_hash,
                                                   vector_size,
                                                   updated_at_utc)
                                               VALUES (
@@ -139,6 +142,7 @@ public sealed class SqliteEmbeddingStateClientImplementation(
                                                   $dataSourceType,
                                                   $embeddingProviderId,
                                                   $embeddingSignature,
+                                                  $sourceHash,
                                                   $vectorSize,
                                                   $updatedAtUtc)
                                               ON CONFLICT(data_source_id) DO UPDATE SET
@@ -146,6 +150,7 @@ public sealed class SqliteEmbeddingStateClientImplementation(
                                                   data_source_type = excluded.data_source_type,
                                                   embedding_provider_id = excluded.embedding_provider_id,
                                                   embedding_signature = excluded.embedding_signature,
+                                                  source_hash = excluded.source_hash,
                                                   vector_size = excluded.vector_size,
                                                   updated_at_utc = excluded.updated_at_utc
                                               """, token,
@@ -154,6 +159,7 @@ public sealed class SqliteEmbeddingStateClientImplementation(
             ("$dataSourceType", dataSourceType),
             ("$embeddingProviderId", embeddingProviderId),
             ("$embeddingSignature", embeddingSignature),
+            ("$sourceHash", sourceHash),
             ("$vectorSize", vectorSize),
             ("$updatedAtUtc", ToUtcText(DateTime.UtcNow)));
     }
@@ -172,6 +178,20 @@ public sealed class SqliteEmbeddingStateClientImplementation(
             ("$updatedAtUtc", ToUtcText(DateTime.UtcNow)));
     }
 
+    public override async Task UpdateDataSourceHashAsync(string dataSourceId, string sourceHash, CancellationToken token)
+    {
+        await using var connection = await this.OpenConnectionAsync(token);
+        await ExecuteNonQueryAsync(connection, """
+                                              UPDATE data_sources
+                                              SET source_hash = $sourceHash,
+                                                  updated_at_utc = $updatedAtUtc
+                                              WHERE data_source_id = $dataSourceId
+                                              """, token,
+            ("$dataSourceId", dataSourceId),
+            ("$sourceHash", sourceHash),
+            ("$updatedAtUtc", ToUtcText(DateTime.UtcNow)));
+    }
+
     public override async Task UpsertFileAsync(string dataSourceId, EmbeddingStateFile file, CancellationToken token)
     {
         await using var connection = await this.OpenConnectionAsync(token);
@@ -255,6 +275,7 @@ public sealed class SqliteEmbeddingStateClientImplementation(
                                                   data_source_type TEXT NOT NULL,
                                                   embedding_provider_id TEXT NOT NULL,
                                                   embedding_signature TEXT NOT NULL,
+                                                  source_hash TEXT NOT NULL DEFAULT '',
                                                   vector_size INTEGER NOT NULL DEFAULT 0,
                                                   updated_at_utc TEXT NOT NULL
                                               );
@@ -278,6 +299,8 @@ public sealed class SqliteEmbeddingStateClientImplementation(
                                               CREATE INDEX IF NOT EXISTS idx_embedded_files_data_source
                                                   ON embedded_files(data_source_id);
                                               """, token);
+
+        await EnsureColumnAsync(connection, "data_sources", "source_hash", "TEXT NOT NULL DEFAULT ''", token);
     }
 
     private async Task<string> GetSqliteVersionAsync(CancellationToken token)
@@ -296,6 +319,21 @@ public sealed class SqliteEmbeddingStateClientImplementation(
         return Convert.ToInt64(countObject, CultureInfo.InvariantCulture);
     }
 
+    private static async Task EnsureColumnAsync(SqliteConnection connection, string tableName, string columnName, string columnDefinition, CancellationToken token)
+    {
+        await using (var command = CreateCommand(connection, $"PRAGMA table_info({tableName})"))
+        {
+            await using var reader = await command.ExecuteReaderAsync(token);
+            while (await reader.ReadAsync(token))
+            {
+                if (string.Equals(reader.GetString(1), columnName, StringComparison.OrdinalIgnoreCase))
+                    return;
+            }
+        }
+
+        await ExecuteNonQueryAsync(connection, $"ALTER TABLE {tableName} ADD COLUMN {columnName} {columnDefinition}", token);
+    }
+
     private async Task<SqliteConnection> OpenConnectionAsync(CancellationToken token)
     {
         var connection = new SqliteConnection(this.connectionString);

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingModels.cs

@@ -82,6 +82,8 @@ public sealed class DataSourceEmbeddingManifest
 
     public string EmbeddingSignature { get; set; } = string.Empty;
 
+    public string SourceHash { get; set; } = string.Empty;
+
     public int VectorSize { get; set; }
 
     public Dictionary<string, EmbeddedFileRecord> Files { get; init; } = new(StringComparer.OrdinalIgnoreCase);

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs

@@ -33,6 +33,8 @@ public sealed partial class DataSourceEmbeddingService
 
     private sealed record ChunkingRule(string Name, Func<string, IReadOnlyList<string>, IReadOnlyList<string>>? Split);
 
+    private sealed record DataSourceMetadataSnapshot(string SourceHash, IReadOnlyDictionary<string, string> FileHashes);
+
     private async IAsyncEnumerable<string> StreamEmbeddingChunksAsync(string filePath, IDataSource dataSource, EmbeddingProvider embeddingProvider, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token)
     {
         var options = this.GetChunkingOptions(dataSource, embeddingProvider);
@@ -752,18 +754,96 @@ public sealed partial class DataSourceEmbeddingService
             chunkingOptions.OverlapTokenLength);
     }
 
-    private async Task<string> BuildFingerprintAsync(FileInfo file, CancellationToken token)
+    private DataSourceMetadataSnapshot BuildDataSourceMetadataSnapshot(IDataSource dataSource, IReadOnlyList<FileInfo> indexedFiles)
     {
-        await using var stream = new FileStream(
+        var fileHashes = indexedFiles
+            .OrderBy(file => file.FullName, StringComparer.OrdinalIgnoreCase)
+            .ToDictionary(file => file.FullName, BuildFileMetadataHash, StringComparer.OrdinalIgnoreCase);
+
+        var sourceHash = dataSource switch
+        {
+            DataSourceLocalFile localFile => indexedFiles.Count > 0
+                ? fileHashes[indexedFiles[0].FullName]
+                : BuildMetadataHash("file", localFile.FilePath, Path.GetFileName(localFile.FilePath) ?? string.Empty, "missing", "0"),
+
+            DataSourceLocalDirectory localDirectory => this.BuildDirectoryMetadataHash(localDirectory, indexedFiles, fileHashes),
+
+            _ => BuildMetadataHash(dataSource.Type.ToString(), dataSource.Id, dataSource.Name)
+        };
+
+        return new(sourceHash, fileHashes);
+    }
+
+    private string BuildDirectoryMetadataHash(DataSourceLocalDirectory dataSource, IReadOnlyList<FileInfo> indexedFiles, IReadOnlyDictionary<string, string> fileHashes)
+    {
+        var directory = new DirectoryInfo(dataSource.Path);
+        directory.Refresh();
+
+        var totalSize = 0L;
+        var latestFileWriteTicks = 0L;
+        foreach (var file in indexedFiles)
+        {
+            file.Refresh();
+            if (!file.Exists)
+                continue;
+
+            totalSize += file.Length;
+            latestFileWriteTicks = Math.Max(latestFileWriteTicks, file.LastWriteTimeUtc.Ticks);
+        }
+
+        var latestWriteTicks = Math.Max(directory.LastWriteTimeUtc.Ticks, latestFileWriteTicks);
+        var parts = new List<string>
+        {
+            "directory",
+            directory.FullName,
+            directory.Name,
+            latestWriteTicks.ToString(),
+            totalSize.ToString(),
+            indexedFiles.Count.ToString()
+        };
+
+        foreach (var file in indexedFiles.OrderBy(file => file.FullName, StringComparer.OrdinalIgnoreCase))
+        {
+            parts.Add(this.TryGetRelativePath(dataSource, file));
+            parts.Add(fileHashes[file.FullName]);
+        }
+
+        return BuildMetadataHash(parts);
+    }
+
+    private static string BuildFileMetadataHash(FileInfo file)
+    {
+        file.Refresh();
+        if (!file.Exists)
+        {
+            return BuildMetadataHash(
+                "file",
+                file.FullName,
+                file.Name,
+                "missing",
+                "0");
+        }
+
+        return BuildMetadataHash(
+            "file",
             file.FullName,
-            FileMode.Open,
+            file.Name,
-            FileAccess.Read,
+            file.LastWriteTimeUtc.Ticks.ToString(),
-            FileShare.ReadWrite | FileShare.Delete,
+            file.Length.ToString());
-            1024 * 128,
+    }
-            FileOptions.Asynchronous | FileOptions.SequentialScan);
+
-        var contentHash = await SHA256.HashDataAsync(stream, token);
+    private static string BuildMetadataHash(params string[] parts)
-        var fingerprintSource = $"{file.FullName}|{Convert.ToHexString(contentHash)}";
+    {
-        var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(fingerprintSource));
+        return BuildMetadataHash((IEnumerable<string>)parts);
+    }
+
+    private static string BuildMetadataHash(IEnumerable<string> parts)
+    {
+        var fingerprintSource = new StringBuilder();
+        foreach (var part in parts)
+            fingerprintSource.Append(part.Length).Append(':').Append(part).Append('|');
+
+        var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(fingerprintSource.ToString()));
         return Convert.ToHexString(bytes);
     }
 

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Watchers.cs

@@ -1,4 +1,5 @@
 using System.Collections.Concurrent;
+using System.Threading;
 using AIStudio.Settings;
 using AIStudio.Settings.DataModel;
 
@@ -20,6 +21,13 @@ public sealed partial class DataSourceEmbeddingService
             return;
         }
 
+        if (Volatile.Read(ref this.startupHashCheckCompleted) == 0)
+        {
+            logger.LogDebug("File watchers are not activated yet because the startup persisted hash check has not completed.");
+            this.RemoveAllWatchers();
+            return;
+        }
+
         var supportedSources = settingsManager.ConfigurationData.DataSources
             .Where(this.IsSupportedInternalDataSource)
             .ToDictionary(source => source.Id, StringComparer.OrdinalIgnoreCase);
@@ -161,8 +169,8 @@ public sealed partial class DataSourceEmbeddingService
 
                 if (dataSource is not null)
                 {
-                    logger.LogInformation("Queueing data source '{DataSourceName}' ({DataSourceId}) after file system changes settled.", dataSource.Name, dataSource.Id);
+                    logger.LogInformation("Queueing data source '{DataSourceName}' ({DataSourceId}) after file system changes settled. The hash pipeline will reindex only changed files.", dataSource.Name, dataSource.Id);
-                    await this.QueueDataSourceAsync(dataSource, true);
+                    await this.QueueDataSourceAsync(dataSource, true, DataSourceEmbeddingRefreshMode.WATCHER_HASH_CHECK);
                 }
             }
             catch (OperationCanceledException)

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs

@@ -1,5 +1,6 @@
 using System.Collections.Concurrent;
 using System.Diagnostics.CodeAnalysis;
+using System.Threading;
 using System.Threading.Channels;
 
 using AIStudio.Provider;
@@ -15,12 +16,15 @@ namespace AIStudio.Tools.Services;
 public sealed partial class DataSourceEmbeddingService(SettingsManager settingsManager, RustService rustService, DatabaseClientProvider databaseClientProvider, ILogger<DataSourceEmbeddingService> logger)
     : BackgroundService
 {
-    private readonly Channel<string> queue = Channel.CreateUnbounded<string>();
+    private readonly Channel<DataSourceEmbeddingQueueItem> queue = Channel.CreateUnbounded<DataSourceEmbeddingQueueItem>();
     private readonly ConcurrentDictionary<string, byte> queuedIds = new(StringComparer.OrdinalIgnoreCase);
     private readonly ConcurrentDictionary<string, byte> runningIds = new(StringComparer.OrdinalIgnoreCase);
     private readonly ConcurrentDictionary<string, byte> pendingQueueIds = new(StringComparer.OrdinalIgnoreCase);
+    private readonly ConcurrentDictionary<string, DataSourceRunControl> activeRuns = new(StringComparer.OrdinalIgnoreCase);
     private readonly ConcurrentDictionary<string, DataSourceEmbeddingStatus> statuses = new(StringComparer.OrdinalIgnoreCase);
     private readonly object queueStateLock = new();
+    private int startupHashCheckStarted;
+    private int startupHashCheckCompleted;
 
     private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceEmbeddingService).Namespace, nameof(DataSourceEmbeddingService));
 
@@ -32,6 +36,18 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
         RUNNING_MARKED_PENDING,
     }
 
+    private enum DataSourceEmbeddingRefreshMode
+    {
+        STARTUP_HASH_CHECK,
+        HASH_CHECK,
+        WATCHER_HASH_CHECK,
+        MANUAL_RETRY,
+    }
+
+    private sealed record DataSourceEmbeddingQueueItem(string DataSourceId, DataSourceEmbeddingRefreshMode RefreshMode);
+
+    private sealed record DataSourceRunControl(CancellationTokenSource TokenSource, TaskCompletionSource<object?> Completion);
+
     public IReadOnlyList<DataSourceEmbeddingStatus> GetStatuses()
     {
         return this.statuses.Values
@@ -75,9 +91,16 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
     {
         this.RefreshWatchers();
 
-        var tasks = settingsManager.ConfigurationData.DataSources
+        var supportedDataSources = settingsManager.ConfigurationData.DataSources
             .Where(this.IsSupportedInternalDataSource)
-            .Select(dataSource => this.QueueDataSourceAsync(dataSource, queueAfterCurrentRun));
+            .ToList();
+
+        logger.LogInformation(
+            "Queueing {DataSourceCount} supported internal data source(s) for background embedding hash checks. QueueAfterCurrentRun={QueueAfterCurrentRun}.",
+            supportedDataSources.Count,
+            queueAfterCurrentRun);
+
+        var tasks = supportedDataSources.Select(dataSource => this.QueueDataSourceAsync(dataSource, queueAfterCurrentRun, DataSourceEmbeddingRefreshMode.HASH_CHECK));
 
         return Task.WhenAll(tasks);
     }
@@ -90,11 +113,36 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
             return Task.CompletedTask;
         }
 
-        return this.QueueAllInternalDataSourcesAsync(false);
+        logger.LogDebug("Automatic startup embedding hash check is handled by the background service. Ignoring duplicate startup queue request.");
+        return Task.CompletedTask;
     }
 
     public void RefreshAutomaticWatchers()
     {
+        if (!settingsManager.ConfigurationData.DataSourceIndexing.AutomaticRefresh)
+        {
+            Volatile.Write(ref this.startupHashCheckCompleted, 0);
+            Interlocked.Exchange(ref this.startupHashCheckStarted, 0);
+            this.RemoveAllWatchers();
+            return;
+        }
+
+        if (Volatile.Read(ref this.startupHashCheckCompleted) == 0)
+        {
+            _ = Task.Run(async () =>
+            {
+                try
+                {
+                    await this.RunInitialDataSourceHashCheckAsync(CancellationToken.None);
+                }
+                catch (Exception exception)
+                {
+                    logger.LogWarning(exception, "Failed to run the initial data source hash check after automatic refresh was enabled.");
+                }
+            });
+            return;
+        }
+
         this.RefreshWatchers();
     }
 
@@ -109,9 +157,26 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
             this.CanRefreshDataSource(dataSource);
     }
 
+    public async Task<bool> ShouldLockDataSourceIdentityAsync(string dataSourceId, CancellationToken token = default)
+    {
+        var embeddingState = await databaseClientProvider.GetEmbeddingStateAsync(token);
+        if (!embeddingState.IsAvailable)
+        {
+            logger.LogWarning("Locking identity settings for data source '{DataSourceId}' because the embedding state database '{DatabaseName}' is unavailable.", dataSourceId, embeddingState.Name);
+            return true;
+        }
+
+        var manifest = await embeddingState.GetManifestAsync(dataSourceId, token);
+        return !string.IsNullOrWhiteSpace(manifest.EmbeddingProviderId)
+               || !string.IsNullOrWhiteSpace(manifest.EmbeddingSignature)
+               || !string.IsNullOrWhiteSpace(manifest.SourceHash)
+               || manifest.VectorSize > 0
+               || manifest.Files.Count > 0;
+    }
+
     public Task QueueDataSourceAsync(IDataSource dataSource)
     {
-        return this.QueueDataSourceAsync(dataSource, true);
+        return this.QueueDataSourceAsync(dataSource, true, DataSourceEmbeddingRefreshMode.HASH_CHECK);
     }
 
     public Task QueueDataSourceAsync(string dataSourceId)
@@ -121,13 +186,20 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
             : Task.CompletedTask;
     }
 
-    private async Task QueueDataSourceAsync(IDataSource dataSource, bool queueAfterCurrentRun)
+    public Task RetryDataSourceAsync(string dataSourceId)
+    {
+        return this.TryGetConfiguredDataSource(dataSourceId, out var dataSource)
+            ? this.QueueDataSourceAsync(dataSource, true, DataSourceEmbeddingRefreshMode.MANUAL_RETRY)
+            : Task.CompletedTask;
+    }
+
+    private async Task QueueDataSourceAsync(IDataSource dataSource, bool queueAfterCurrentRun, DataSourceEmbeddingRefreshMode refreshMode)
     {
         if (!this.IsSupportedInternalDataSource(dataSource))
             return;
 
         this.RefreshWatchers();
-        logger.LogDebug("Ensured watcher for data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id);
+        logger.LogDebug("Refreshed watcher state for data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id);
 
         var queueRequestResult = this.TryReserveDataSourceQueueSlot(dataSource.Id, queueAfterCurrentRun);
         switch (queueRequestResult)
@@ -145,7 +217,11 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
                 return;
         }
 
-        logger.LogInformation("Queueing data source '{DataSourceName}' ({DataSourceId}) for background embeddings.", dataSource.Name, dataSource.Id);
+        logger.LogInformation(
+            "Queueing data source '{DataSourceName}' ({DataSourceId}) for background embedding hash check. RefreshMode={RefreshMode}.",
+            dataSource.Name,
+            dataSource.Id,
+            refreshMode);
         if (!this.statuses.TryGetValue(dataSource.Id, out var currentStatus) || currentStatus.State is not DataSourceEmbeddingState.RUNNING)
         {
             this.UpsertStatus(this.CreateStatus(
@@ -157,7 +233,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
                 failures: currentStatus?.Failures ?? []));
         }
         logger.LogDebug("Upserting status for data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id);
-        await this.queue.Writer.WriteAsync(dataSource.Id);
+        await this.queue.Writer.WriteAsync(new DataSourceEmbeddingQueueItem(dataSource.Id, refreshMode));
         logger.LogDebug("Queued data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id);
     }
 
@@ -167,8 +243,21 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
             return;
 
         this.RemoveWatcher(dataSource.Id);
+        var activeRun = this.CancelActiveDataSourceRun(dataSource);
+        this.ClearQueuedDataSourceState(dataSource.Id);
+        this.statuses.TryRemove(dataSource.Id, out _);
+        if (activeRun is not null)
+        {
+            logger.LogInformation(
+                "Waiting for the active embedding run for deleted data source '{DataSourceName}' ({DataSourceId}) to stop before deleting persisted embeddings.",
+                dataSource.Name,
+                dataSource.Id);
+            await activeRun.Completion.Task;
+        }
+
         this.statuses.TryRemove(dataSource.Id, out _);
         await this.ResetPersistedStateAsync(dataSource.Name, dataSource.Id, null, null, CancellationToken.None);
+        this.statuses.TryRemove(dataSource.Id, out _);
         this.PublishStatusChanged();
     }
 
@@ -178,7 +267,8 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
 
         while (!stoppingToken.IsCancellationRequested)
         {
-            var dataSourceId = await this.queue.Reader.ReadAsync(stoppingToken);
+            var queueItem = await this.queue.Reader.ReadAsync(stoppingToken);
+            var dataSourceId = queueItem.DataSourceId;
             this.MarkDataSourceRunStarted(dataSourceId);
 
             IDataSource? dataSource = null;
@@ -191,7 +281,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
                 if (dataSource is null || !this.IsSupportedInternalDataSource(dataSource))
                     continue;
 
-                await this.ProcessDataSourceAsync(dataSource, stoppingToken);
+                await this.ProcessDataSourceRunAsync(dataSource, queueItem.RefreshMode, stoppingToken);
             }
             catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
             {
@@ -222,12 +312,68 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
         base.Dispose();
     }
 
-    private async Task ProcessDataSourceAsync(IDataSource dataSource, CancellationToken token)
+    private async Task ProcessDataSourceRunAsync(IDataSource dataSource, DataSourceEmbeddingRefreshMode refreshMode, CancellationToken parentToken)
     {
-        logger.LogInformation("Starting background embeddings for data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id);
+        if (!this.TryGetConfiguredDataSource(dataSource.Id, out var configuredDataSource) ||
+            !this.IsSupportedInternalDataSource(configuredDataSource))
+        {
+            logger.LogDebug(
+                "Skipping embedding run for data source '{DataSourceName}' ({DataSourceId}) because it is no longer configured. RefreshMode={RefreshMode}.",
+                dataSource.Name,
+                dataSource.Id,
+                refreshMode);
+            return;
+        }
+
+        dataSource = configuredDataSource;
+        var runTokenSource = CancellationTokenSource.CreateLinkedTokenSource(parentToken);
+        var runControl = new DataSourceRunControl(
+            runTokenSource,
+            new TaskCompletionSource<object?>(TaskCreationOptions.RunContinuationsAsynchronously));
+
+        if (!this.activeRuns.TryAdd(dataSource.Id, runControl))
+        {
+            runTokenSource.Dispose();
+            logger.LogDebug(
+                "Data source '{DataSourceName}' ({DataSourceId}) already has an active embedding run. Skipping duplicate process request. RefreshMode={RefreshMode}.",
+                dataSource.Name,
+                dataSource.Id,
+                refreshMode);
+            return;
+        }
+
+        try
+        {
+            await this.ProcessDataSourceAsync(dataSource, refreshMode, runTokenSource.Token);
+        }
+        catch (OperationCanceledException) when (!parentToken.IsCancellationRequested && runTokenSource.IsCancellationRequested)
+        {
+            logger.LogInformation(
+                "Stopped background embeddings for data source '{DataSourceName}' ({DataSourceId}) because the data source was removed or canceled. RefreshMode={RefreshMode}.",
+                dataSource.Name,
+                dataSource.Id,
+                refreshMode);
+        }
+        finally
+        {
+            this.activeRuns.TryRemove(dataSource.Id, out _);
+            runControl.Completion.TrySetResult(null);
+            runTokenSource.Dispose();
+        }
+    }
+
+    private async Task ProcessDataSourceAsync(IDataSource dataSource, DataSourceEmbeddingRefreshMode refreshMode, CancellationToken token)
+    {
+        logger.LogInformation(
+            "Starting background embedding hash check for data source '{DataSourceName}' ({DataSourceId}). RefreshMode={RefreshMode}.",
+            dataSource.Name,
+            dataSource.Id,
+            refreshMode);
+        token.ThrowIfCancellationRequested();
 
         var vectorStore = await databaseClientProvider.GetVectorStoreAsync(token);
         var embeddingState = await databaseClientProvider.GetEmbeddingStateAsync(token);
+        token.ThrowIfCancellationRequested();
 
         if (!vectorStore.IsAvailable)
         {
@@ -236,6 +382,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
                 dataSource.Name,
                 dataSource.Id,
                 vectorStore.Name);
+            token.ThrowIfCancellationRequested();
             this.UpsertStatus(this.GetFallbackStatus(dataSource, "The vector database is not available."));
             return;
         }
@@ -247,12 +394,14 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
                 dataSource.Name,
                 dataSource.Id,
                 embeddingState.Name);
+            token.ThrowIfCancellationRequested();
             this.UpsertStatus(this.GetFallbackStatus(dataSource, "The SQLite embedding state database is not available."));
             return;
         }
 
         if (!this.TryResolveEmbeddingProvider(dataSource, out var embeddingProvider))
         {
+            token.ThrowIfCancellationRequested();
             this.UpsertStatus(this.GetFallbackStatus(dataSource, "The selected embedding provider is not available."));
             return;
         }
@@ -267,6 +416,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
 
         var collectionName = this.GetCollectionName(dataSource.Name, dataSource.Id);
         var manifest = await this.EnsureCompatibleManifestAsync(dataSource, embeddingProvider, collectionName, vectorStore, embeddingState, token);
+        token.ThrowIfCancellationRequested();
         var inputFiles = this.GetInputFiles(dataSource);
         var indexedFiles = inputFiles.Files;
         var totalFiles = indexedFiles.Count + inputFiles.FailedFiles;
@@ -279,8 +429,35 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
             inputFiles.FailedFiles,
             collectionName);
 
-        await this.RemoveMissingFileEmbeddingsAsync(vectorStore, embeddingState, dataSource, collectionName, manifest, indexedFiles, token);
+        var metadataSnapshot = this.BuildDataSourceMetadataSnapshot(dataSource, indexedFiles);
+        var removedMissingFiles = await this.RemoveMissingFileEmbeddingsAsync(vectorStore, embeddingState, dataSource, collectionName, manifest, indexedFiles, token);
+        token.ThrowIfCancellationRequested();
 
+        logger.LogInformation(
+            "Compared data source hash for '{DataSourceName}' ({DataSourceId}). StoredSourceHashPrefix={StoredSourceHashPrefix}, CurrentSourceHashPrefix={CurrentSourceHashPrefix}, StoredFileRecords={StoredFileRecords}, CurrentFiles={CurrentFiles}, RemovedMissingFiles={RemovedMissingFiles}.",
+            dataSource.Name,
+            dataSource.Id,
+            ShortHash(manifest.SourceHash),
+            ShortHash(metadataSnapshot.SourceHash),
+            manifest.Files.Count,
+            indexedFiles.Count,
+            removedMissingFiles);
+
+        if (this.CanSkipDataSourceByHash(manifest, metadataSnapshot, indexedFiles))
+        {
+            logger.LogInformation(
+                "Skipping data source '{DataSourceName}' ({DataSourceId}) because the persisted data source hash and all persisted file hashes match. RefreshMode={RefreshMode}.",
+                dataSource.Name,
+                dataSource.Id,
+                refreshMode);
+
+            token.ThrowIfCancellationRequested();
+            await embeddingState.UpdateDataSourceHashAsync(dataSource.Id, metadataSnapshot.SourceHash, token);
+            this.UpsertStatus(this.CreateCompletedStatus(dataSource, totalFiles, indexedFiles.Count, inputFiles.FailedFiles, inputFiles.LastError, inputFiles.Failures));
+            return;
+        }
+
+        token.ThrowIfCancellationRequested();
         this.UpsertStatus(this.CreateStatus(
             dataSource,
             DataSourceEmbeddingState.RUNNING,
@@ -293,6 +470,8 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
         var provider = embeddingProvider.CreateProvider();
         var skippedFiles = 0;
         var completedFiles = 0;
+        var newFiles = 0;
+        var changedFiles = 0;
         var failedFiles = inputFiles.FailedFiles;
         var lastError = inputFiles.LastError;
         var failureDetails = inputFiles.Failures.ToList();
@@ -301,15 +480,18 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
         {
             token.ThrowIfCancellationRequested();
 
-            var fingerprint = await this.BuildFingerprintAsync(file, token);
+            var fingerprint = metadataSnapshot.FileHashes[file.FullName];
             if (manifest.Files.TryGetValue(file.FullName, out var existingRecord) &&
                 string.Equals(existingRecord.Fingerprint, fingerprint, StringComparison.Ordinal))
             {
                 logger.LogDebug(
-                    "Skipping unchanged file '{FilePath}' for data source '{DataSourceName}' ({DataSourceId}).",
+                    "Skipping unchanged file '{FilePath}' for data source '{DataSourceName}' ({DataSourceId}) because the persisted metadata hash matches. MetadataHashPrefix={MetadataHashPrefix}, LastWriteUtc={LastWriteUtc:O}, FileSize={FileSize}.",
                     file.FullName,
                     dataSource.Name,
-                    dataSource.Id);
+                    dataSource.Id,
+                    ShortHash(fingerprint),
+                    file.LastWriteTimeUtc,
+                    file.Length);
                 skippedFiles++;
                 this.UpsertStatus(this.CreateStatus(dataSource, DataSourceEmbeddingState.RUNNING, totalFiles, skippedFiles + completedFiles, failedFiles, lastError: lastError, failures: failureDetails));
                 continue;
@@ -320,14 +502,17 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
             try
             {
                 logger.LogInformation(
-                    "Embedding file '{FilePath}' for data source '{DataSourceName}' ({DataSourceId}). Progress={CompletedFiles}/{TotalFiles}.",
+                    "Embedding file '{FilePath}' for data source '{DataSourceName}' ({DataSourceId}) because {EmbeddingReason}. CurrentMetadataHashPrefix={CurrentMetadataHashPrefix}. Progress={CompletedFiles}/{TotalFiles}.",
                     file.FullName,
                     dataSource.Name,
                     dataSource.Id,
+                    GetFileEmbeddingReason(file, fingerprint, existingRecord),
+                    ShortHash(fingerprint),
                     skippedFiles + completedFiles + 1,
                     totalFiles);
                 var startedAtUtc = DateTime.UtcNow;
                 var chunkCount = await this.IndexOneFileAsync(embeddingState, vectorStore, dataSource, file, fingerprint, embeddingProvider, provider, manifest, token);
+                token.ThrowIfCancellationRequested();
                 var embeddedAtUtc = DateTime.UtcNow;
                 var record = new EmbeddedFileRecord(
                     fingerprint,
@@ -349,6 +534,11 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
                     token);
                 manifest.Files[file.FullName] = record;
                 completedFiles++;
+                if (existingRecord is null)
+                    newFiles++;
+                else
+                    changedFiles++;
+
                 logger.LogInformation(
                     "Embedded file '{FilePath}' for data source '{DataSourceName}' ({DataSourceId}) successfully. Chunks={ChunkCount}, DurationMs={DurationMs}.",
                     file.FullName,
@@ -357,6 +547,10 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
                     chunkCount,
                     (DateTime.UtcNow - startedAtUtc).TotalMilliseconds);
             }
+            catch (OperationCanceledException) when (token.IsCancellationRequested)
+            {
+                throw;
+            }
             catch (Exception exception)
             {
                 failedFiles++;
@@ -371,14 +565,25 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
             }
         }
 
+        manifest.SourceHash = metadataSnapshot.SourceHash;
+        token.ThrowIfCancellationRequested();
+        await embeddingState.UpdateDataSourceHashAsync(dataSource.Id, metadataSnapshot.SourceHash, token);
+        token.ThrowIfCancellationRequested();
+
         this.UpsertStatus(this.CreateCompletedStatus(dataSource, totalFiles, skippedFiles + completedFiles, failedFiles, lastError, failureDetails));
         logger.LogInformation(
-            "Finished background embeddings for data source '{DataSourceName}' ({DataSourceId}). Indexed={IndexedFiles}, Failed={FailedFiles}, Total={TotalFiles}.",
+            "Finished background embeddings for data source '{DataSourceName}' ({DataSourceId}). RefreshMode={RefreshMode}, Embedded={EmbeddedFiles}, New={NewFiles}, Changed={ChangedFiles}, Skipped={SkippedFiles}, RemovedMissing={RemovedMissingFiles}, Failed={FailedFiles}, Total={TotalFiles}, SourceHashPrefix={SourceHashPrefix}.",
             dataSource.Name,
             dataSource.Id,
-            skippedFiles + completedFiles,
+            refreshMode,
+            completedFiles,
+            newFiles,
+            changedFiles,
+            skippedFiles,
+            removedMissingFiles,
             failedFiles,
-            totalFiles);
+            totalFiles,
+            ShortHash(metadataSnapshot.SourceHash));
     }
 
     private async Task<int> IndexOneFileAsync(
@@ -453,6 +658,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
         try
         {
             vectors = await provider.EmbedTextAsync(embeddingProvider.Model, settingsManager, token, texts);
+            token.ThrowIfCancellationRequested();
         }
         catch (OperationCanceledException) when (token.IsCancellationRequested)
         {
@@ -475,6 +681,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
 
         if (manifest.VectorSize == 0)
         {
+            token.ThrowIfCancellationRequested();
             manifest.VectorSize = vectorSize;
             await this.EnsureCollectionExistsAsync(vectorStore, collectionName, vectorSize, token);
             await embeddingState.UpdateVectorSizeAsync(dataSource.Id, vectorSize, token);
@@ -486,6 +693,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
                 dataSource.Id);
         }
 
+        token.ThrowIfCancellationRequested();
         await this.UpsertPointsAsync(
             vectorStore,
             collectionName,
@@ -574,8 +782,62 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
 
         token.ThrowIfCancellationRequested();
 
-        logger.LogInformation("Embedding background service is ready. Checking whether automatic data source refresh is enabled.");
+        logger.LogInformation("Embedding background service is ready. Running the initial persisted hash check before activating file watchers.");
-        await this.QueueAllInternalDataSourcesIfAutomaticRefreshAsync();
+        await this.RunInitialDataSourceHashCheckAsync(token);
+    }
+
+    private async Task RunInitialDataSourceHashCheckAsync(CancellationToken token)
+    {
+        if (!settingsManager.ConfigurationData.DataSourceIndexing.AutomaticRefresh)
+        {
+            logger.LogInformation("Automatic local data source refresh is disabled. Startup hash checks and file watchers are disabled.");
+            this.RemoveAllWatchers();
+            return;
+        }
+
+        if (Interlocked.Exchange(ref this.startupHashCheckStarted, 1) == 1)
+            return;
+
+        this.RemoveAllWatchers();
+
+        var supportedDataSources = settingsManager.ConfigurationData.DataSources
+            .Where(this.IsSupportedInternalDataSource)
+            .ToList();
+
+        logger.LogInformation(
+            "Starting initial persisted hash check for {DataSourceCount} supported internal data source(s). Incomplete or failed embedding state will be retried during this pass. File watchers will be activated after this check completes.",
+            supportedDataSources.Count);
+
+        foreach (var dataSource in supportedDataSources)
+        {
+            token.ThrowIfCancellationRequested();
+            try
+            {
+                await this.ProcessDataSourceRunAsync(dataSource, DataSourceEmbeddingRefreshMode.STARTUP_HASH_CHECK, token);
+            }
+            catch (OperationCanceledException) when (token.IsCancellationRequested)
+            {
+                throw;
+            }
+            catch (Exception exception)
+            {
+                logger.LogError(exception, "Initial embedding hash check failed for data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id);
+                this.UpsertStatus(this.GetFallbackStatus(dataSource, exception.Message));
+            }
+        }
+
+        if (!settingsManager.ConfigurationData.DataSourceIndexing.AutomaticRefresh)
+        {
+            Volatile.Write(ref this.startupHashCheckCompleted, 0);
+            Interlocked.Exchange(ref this.startupHashCheckStarted, 0);
+            logger.LogInformation("Automatic local data source refresh was disabled before the initial hash check completed. File watchers remain inactive.");
+            this.RemoveAllWatchers();
+            return;
+        }
+
+        Volatile.Write(ref this.startupHashCheckCompleted, 1);
+        logger.LogInformation("Completed initial persisted hash check. Activating file watchers for automatic local data source refresh.");
+        this.RefreshWatchers();
     }
 
     private bool IsSupportedInternalDataSource(IDataSource dataSource)
@@ -612,6 +874,15 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
         var embeddingSignature = this.BuildEmbeddingSignature(dataSource, embeddingProvider, chunkingOptions);
         var manifest = await embeddingState.GetManifestAsync(dataSource.Id, token);
 
+        logger.LogInformation(
+            "Loaded persisted embedding manifest for data source '{DataSourceName}' ({DataSourceId}). StoredFiles={StoredFiles}, StoredSourceHashPrefix={StoredSourceHashPrefix}, StoredSignaturePrefix={StoredSignaturePrefix}, CurrentSignaturePrefix={CurrentSignaturePrefix}.",
+            dataSource.Name,
+            dataSource.Id,
+            manifest.Files.Count,
+            ShortHash(manifest.SourceHash),
+            ShortHash(manifest.EmbeddingSignature),
+            ShortHash(embeddingSignature));
+
         if (!string.Equals(manifest.EmbeddingSignature, embeddingSignature, StringComparison.Ordinal))
         {
             logger.LogInformation(
@@ -642,13 +913,14 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
             dataSource.Type.ToString(),
             manifest.EmbeddingProviderId,
             manifest.EmbeddingSignature,
+            manifest.SourceHash,
             manifest.VectorSize,
             token);
 
         return manifest;
     }
 
-    private async Task RemoveMissingFileEmbeddingsAsync(
+    private async Task<int> RemoveMissingFileEmbeddingsAsync(
         VectorStoreClient vectorStore,
         EmbeddingStateClient embeddingState,
         IDataSource dataSource,
@@ -661,17 +933,72 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
             .Select(file => file.FullName)
             .ToHashSet(StringComparer.OrdinalIgnoreCase);
 
+        var removedFiles = 0;
         foreach (var removedFilePath in manifest.Files.Keys.Except(existingPaths, StringComparer.OrdinalIgnoreCase).ToList())
         {
             await this.DeleteFilePointsAsync(vectorStore, collectionName, removedFilePath, token);
             await embeddingState.DeleteFileAsync(dataSource.Id, removedFilePath, token);
             manifest.Files.Remove(removedFilePath);
+            removedFiles++;
             logger.LogInformation(
                 "Removed stale embeddings for deleted file '{FilePath}' from data source '{DataSourceName}' ({DataSourceId}).",
                 removedFilePath,
                 dataSource.Name,
                 dataSource.Id);
         }
+
+        return removedFiles;
+    }
+
+    private bool CanSkipDataSourceByHash(DataSourceEmbeddingManifest manifest, DataSourceMetadataSnapshot metadataSnapshot, IReadOnlyCollection<FileInfo> indexedFiles)
+    {
+        if (!string.Equals(manifest.SourceHash, metadataSnapshot.SourceHash, StringComparison.Ordinal))
+            return false;
+
+        if (manifest.Files.Count != indexedFiles.Count)
+            return false;
+
+        foreach (var file in indexedFiles)
+        {
+            if (!metadataSnapshot.FileHashes.TryGetValue(file.FullName, out var currentHash))
+                return false;
+
+            if (!manifest.Files.TryGetValue(file.FullName, out var existingRecord))
+                return false;
+
+            if (!string.Equals(existingRecord.Fingerprint, currentHash, StringComparison.Ordinal))
+                return false;
+        }
+
+        return true;
+    }
+
+    private static string GetFileEmbeddingReason(FileInfo file, string currentHash, EmbeddedFileRecord? existingRecord)
+    {
+        if (existingRecord is null)
+            return "no stored file hash exists";
+
+        var reasons = new List<string>();
+        if (!string.Equals(existingRecord.Fingerprint, currentHash, StringComparison.Ordinal))
+            reasons.Add($"stored hash {ShortHash(existingRecord.Fingerprint)} differs from current hash {ShortHash(currentHash)}");
+
+        if (existingRecord.FileSize != file.Length)
+            reasons.Add($"file size changed from {existingRecord.FileSize} to {file.Length} bytes");
+
+        if (existingRecord.LastWriteUtc != file.LastWriteTimeUtc)
+            reasons.Add($"last modified time changed from {existingRecord.LastWriteUtc:O} to {file.LastWriteTimeUtc:O}");
+
+        return reasons.Count == 0
+            ? "the file hash changed"
+            : string.Join("; ", reasons);
+    }
+
+    private static string ShortHash(string value)
+    {
+        if (string.IsNullOrWhiteSpace(value))
+            return "<empty>";
+
+        return value.Length <= 12 ? value : value[..12];
     }
 
     private DataSourceEmbeddingStatus CreateStatus(
@@ -774,6 +1101,36 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
         }
     }
 
+    private void ClearQueuedDataSourceState(string dataSourceId)
+    {
+        lock (this.queueStateLock)
+        {
+            this.queuedIds.TryRemove(dataSourceId, out _);
+            this.pendingQueueIds.TryRemove(dataSourceId, out _);
+        }
+    }
+
+    private DataSourceRunControl? CancelActiveDataSourceRun(IDataSource dataSource)
+    {
+        if (!this.activeRuns.TryGetValue(dataSource.Id, out var activeRun))
+            return null;
+
+        logger.LogInformation(
+            "Canceling active embedding run for deleted data source '{DataSourceName}' ({DataSourceId}).",
+            dataSource.Name,
+            dataSource.Id);
+        try
+        {
+            activeRun.TokenSource.Cancel();
+        }
+        catch (ObjectDisposedException)
+        {
+            return null;
+        }
+
+        return activeRun;
+    }
+
     private async Task QueuePendingDataSourceRunAsync(string dataSourceId, CancellationToken token)
     {
         var dataSource = token.IsCancellationRequested
@@ -804,7 +1161,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
 
         try
         {
-            await this.queue.Writer.WriteAsync(dataSourceId, token);
+            await this.queue.Writer.WriteAsync(new DataSourceEmbeddingQueueItem(dataSourceId, DataSourceEmbeddingRefreshMode.HASH_CHECK), token);
         }
         catch (OperationCanceledException) when (token.IsCancellationRequested)
         {