MindWorkAI/AI-Studio
MindWorkAI/AI-Studio
1
0
Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2026-08-13 14:02:11 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

made the embedding procedure more robust

Browse Source
This commit is contained in:
PaulKoudelka 2026-08-03 17:53:31 +02:00
parent 89f7156fdf
commit b3ba945b98
14 changed files with 670 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

90
app/MindWork AI Studio/Assistants/I18N/allTexts.lua
View File

@ -3826,14 +3826,20 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1133561850"
-- Describe what data this directory contains to help the AI select it.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1136409150"] = "Describe what data this directory contains to help the AI select it."
-- Default tokenizer
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1220918127"] = "Default tokenizer"
-- Select a root directory for this data source. All data in this directory and all its subdirectories will be processed for this data source.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1265737624"] = "Select a root directory for this data source. All data in this directory and all its subdirectories will be processed for this data source."
-- Selected base directory for this data source
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1312296210"] = "Selected base directory for this data source"
-- Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1529596777"] = "Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting."
-- No embedding selected
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1359179968"] = "No embedding selected"
-- Tokenizer
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1696723386"] = "Tokenizer"
-- Description
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1725856265"] = "Description"
@ -3841,23 +3847,26 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1725856265"
-- How many matches do you want at most per query?
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1827669611"] = "How many matches do you want at most per query?"
-- This data source already has indexed embeddings. Delete and recreate it to change the folder path or embedding model.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1845247453"] = "This data source already has indexed embeddings. Delete and recreate it to change the folder path or embedding model."
-- Update
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1847791252"] = "Update"
-- In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1948697886"] = "In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method."
-- The data source max chunk length must be less than or equal to the embedding provider limit ({0}).
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2166221195"] = "The data source max chunk length must be less than or equal to the embedding provider limit ({0})."
-- Number of tokens repeated at the start of the next chunk. Use 0 to disable overlap.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2263831524"] = "Number of tokens repeated at the start of the next chunk. Use 0 to disable overlap."
-- Please enter 0 or a positive max embedding size.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2571963846"] = "Please enter 0 or a positive max embedding size."
-- Add
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2646845972"] = "Add"
-- Chunk overlap length
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2801813827"] = "Chunk overlap length"
-- Max embedding size
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2812251281"] = "Max embedding size"
-- The embedding you selected runs locally or in your organization. Your data is not sent to the cloud.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2814869210"] = "The embedding you selected runs locally or in your organization. Your data is not sent to the cloud."
@ -3865,8 +3874,11 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2814869210"
-- Embedding
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2838542994"] = "Embedding"
-- Please enter 0 or a positive max chunk length.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3130373769"] = "Please enter 0 or a positive max chunk length."
-- Maximum number of tokens per embedded chunk for this data source. Use 0 to use the embedding provider setting.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T287150655"] = "Maximum number of tokens per embedded chunk for this data source. Use 0 to use the embedding provider setting."
-- The overlap must be smaller than the effective max embedding size.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3100817060"] = "The overlap must be smaller than the effective max embedding size."
-- Please enter 0 or a positive overlap length.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3242265813"] = "Please enter 0 or a positive overlap length."
@ -3877,8 +3889,8 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3359366900"
-- Show Expert Settings
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3361153305"] = "Show Expert Settings"
-- Max chunk length
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3621670500"] = "Max chunk length"
-- Embedding overlap
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T375910361"] = "Embedding overlap"
-- Your security policy
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T4081226330"] = "Your security policy"
@ -3886,8 +3898,8 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T4081226330"
-- Select the base directory
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T562479068"] = "Select the base directory"
-- The overlap must be smaller than the effective max chunk length.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T57492807"] = "The overlap must be smaller than the effective max chunk length."
-- The data source max embedding size must be smaller than the embedding provider limit ({0}). Use 0 to use the provider setting.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T607909514"] = "The data source max embedding size must be smaller than the embedding provider limit ({0}). Use 0 to use the provider setting."
-- Data Source Name
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T813773421"] = "Data Source Name"
@ -3976,8 +3988,14 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1133561850"] = "
-- Select a file for this data source. The content of this file will be processed for the data source.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1190880267"] = "Select a file for this data source. The content of this file will be processed for the data source."
-- Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1529596777"] = "Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting."
-- Default tokenizer
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1220918127"] = "Default tokenizer"
-- No embedding selected
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1359179968"] = "No embedding selected"
-- Tokenizer
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1696723386"] = "Tokenizer"
-- Description
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1725856265"] = "Description"
@ -3991,17 +4009,17 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1847791252"] = "
-- In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1948697886"] = "In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method."
-- The data source max chunk length must be less than or equal to the embedding provider limit ({0}).
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2166221195"] = "The data source max chunk length must be less than or equal to the embedding provider limit ({0})."
-- Number of tokens repeated at the start of the next chunk. Use 0 to disable overlap.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2263831524"] = "Number of tokens repeated at the start of the next chunk. Use 0 to disable overlap."
-- Please enter 0 or a positive max embedding size.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2571963846"] = "Please enter 0 or a positive max embedding size."
-- Add
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2646845972"] = "Add"
-- Chunk overlap length
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2801813827"] = "Chunk overlap length"
-- Max embedding size
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2812251281"] = "Max embedding size"
-- The embedding you selected runs locally or in your organization. Your data is not sent to the cloud.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2814869210"] = "The embedding you selected runs locally or in your organization. Your data is not sent to the cloud."
@ -4012,8 +4030,11 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2838542994"] = "
-- Describe what data this file contains to help the AI select it.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2859265837"] = "Describe what data this file contains to help the AI select it."
-- Please enter 0 or a positive max chunk length.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3130373769"] = "Please enter 0 or a positive max chunk length."
-- Maximum number of tokens per embedded chunk for this data source. Use 0 to use the embedding provider setting.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T287150655"] = "Maximum number of tokens per embedded chunk for this data source. Use 0 to use the embedding provider setting."
-- The overlap must be smaller than the effective max embedding size.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3100817060"] = "The overlap must be smaller than the effective max embedding size."
-- Please enter 0 or a positive overlap length.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3242265813"] = "Please enter 0 or a positive overlap length."
@ -4024,17 +4045,20 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3359366900"] = "
-- Show Expert Settings
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3361153305"] = "Show Expert Settings"
-- Max chunk length
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3621670500"] = "Max chunk length"
-- Select the file
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3740148848"] = "Select the file"
-- Embedding overlap
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T375910361"] = "Embedding overlap"
-- Your security policy
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T4081226330"] = "Your security policy"
-- The overlap must be smaller than the effective max chunk length.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T57492807"] = "The overlap must be smaller than the effective max chunk length."
-- This data source already has indexed embeddings. Delete and recreate it to change the file path or embedding model.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T458267883"] = "This data source already has indexed embeddings. Delete and recreate it to change the file path or embedding model."
-- The data source max embedding size must be smaller than the embedding provider limit ({0}). Use 0 to use the provider setting.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T607909514"] = "The data source max embedding size must be smaller than the embedding provider limit ({0}). Use 0 to use the provider setting."
-- Data Source Name
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T813773421"] = "Data Source Name"
@ -4246,6 +4270,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2331453405"] = "(O
-- Invalid tokenizer:
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2448302543"] = "Invalid tokenizer:"
-- Please enter a max embedding size greater than 0.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2608328491"] = "Please enter a max embedding size greater than 0."
-- Add
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2646845972"] = "Add"
@ -4255,18 +4282,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T278585345"] = "Sel
-- No models loaded or available.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2810182573"] = "No models loaded or available."
-- Max embedding size
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2812251281"] = "Max embedding size"
-- Instance Name
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2842060373"] = "Instance Name"
-- Currently, we cannot query the embedding models for the selected provider and/or host. Therefore, please enter the model name manually.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T290547799"] = "Currently, we cannot query the embedding models for the selected provider and/or host. Therefore, please enter the model name manually."
-- Token limit
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2961294165"] = "Token limit"
-- Please enter a token limit greater than 0.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T3316544737"] = "Please enter a token limit greater than 0."
-- Show Expert Settings
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T3361153305"] = "Show Expert Settings"

26
app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor
View File

@ -41,12 +41,18 @@
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
@T("Select a root directory for this data source. All data in this directory and all its subdirectories will be processed for this data source.")
</MudJustifiedText>
<SelectDirectory @bind-Directory="@this.dataPath" Label="@T("Selected base directory for this data source")" DirectoryDialogTitle="@T("Select the base directory")" Validation="@this.dataSourceValidation.ValidatePath" />
@if (!this.CanChangeSourceAndEmbedding)
{
<MudAlert Severity="Severity.Info" Variant="Variant.Outlined" Class="mb-3">
@T("This data source already has indexed embeddings. Delete and recreate it to change the folder path or embedding model.")
</MudAlert>
}
<SelectDirectory @bind-Directory="@this.dataPath" Label="@T("Selected base directory for this data source")" DirectoryDialogTitle="@T("Select the base directory")" Validation="@this.dataSourceValidation.ValidatePath" Disabled="@(!this.CanChangeSourceAndEmbedding)" />
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
@T("In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method.")
</MudJustifiedText>
<MudSelect @bind-Value="@this.dataEmbeddingId" Label="@T("Embedding")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateEmbeddingId">
<MudSelect @bind-Value="@this.dataEmbeddingId" Label="@T("Embedding")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateEmbeddingId" Disabled="@(!this.CanChangeSourceAndEmbedding)">
@foreach (var embedding in this.AvailableEmbeddings)
{
<MudSelectItem Value="@embedding.Value">
@ -57,6 +63,18 @@
@if (!string.IsNullOrWhiteSpace(this.dataEmbeddingId))
{
<MudTextField
T="string"
Text="@this.SelectedEmbeddingTokenizerText"
Label="@T("Tokenizer")"
Class="mb-3"
ReadOnly="@true"
Variant="Variant.Outlined"
Adornment="Adornment.Start"
AdornmentIcon="@Icons.Material.Filled.DataObject"
AdornmentColor="Color.Info"
/>
if (this.SelectedCloudEmbedding)
{
<DataSourceCloudEmbeddingWarning DataSourceType="DataSourceType.LOCAL_DIRECTORY" SourcePath="@this.dataPath" @bind-UserAcknowledged="@this.dataUserAcknowledgedCloudEmbedding" Validation="@this.dataSourceValidation.ValidateUserAcknowledgedCloudEmbedding"/>
@ -80,7 +98,7 @@
<MudNumericField
T="int"
@bind-Value="@this.dataMaxChunkTokenLength"
Label="@T("Max chunk length")"
Label="@T("Token limit")"
Class="mb-3"
Min="0"
Immediate="@true"
@ -94,7 +112,7 @@
<MudNumericField
T="int"
@bind-Value="@this.dataChunkOverlapTokenLength"
Label="@T("Chunk overlap length")"
Label="@T("Token overlap")"
Min="0"
Immediate="@true"
Validation="@this.ValidateChunkOverlapTokenLength"

23
app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs
View File

@ -18,6 +18,9 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
[Parameter]
public DataSourceLocalDirectory DataSource { get; set; }
[Parameter]
public bool LockSourceAndEmbedding { get; set; }
[Parameter]
public IReadOnlyList<ConfigurationSelectData<string>> AvailableEmbeddings { get; set; } = [];
@ -107,6 +110,14 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
private bool SelectedCloudEmbedding => this.SelectedEmbedding is { IsSelfHosted: false };
private bool CanChangeSourceAndEmbedding => !this.IsEditing || !this.LockSourceAndEmbedding;
private string SelectedEmbeddingTokenizerText => this.SelectedEmbedding is null
? T("No embedding selected")
: string.IsNullOrWhiteSpace(this.SelectedEmbedding.TokenizerPath)
? T("Default tokenizer")
: System.IO.Path.GetFileName(this.SelectedEmbedding.TokenizerPath);
private DataSourceLocalDirectory CreateDataSource() => new()
{
Id = this.dataId,
@ -114,8 +125,8 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
Name = this.dataName,
Description = this.dataDescription,
Type = DataSourceType.LOCAL_DIRECTORY,
EmbeddingId = this.dataEmbeddingId,
Path = this.dataPath,
EmbeddingId = this.CanChangeSourceAndEmbedding ? this.dataEmbeddingId : this.DataSource.EmbeddingId,
Path = this.CanChangeSourceAndEmbedding ? this.dataPath : this.DataSource.Path,
MaxChunkTokenLength = this.dataMaxChunkTokenLength,
ChunkOverlapTokenLength = this.dataChunkOverlapTokenLength,
SecurityPolicy = this.dataSecurityPolicy,
@ -139,11 +150,11 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
private string? ValidateMaxChunkTokenLength(int maxChunkTokenLength)
{
if (maxChunkTokenLength < 0)
return T("Please enter 0 or a positive max chunk length.");
return T("Please enter 0 or a positive token limit.");
var providerMaxChunkTokenLength = this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT;
if (maxChunkTokenLength > providerMaxChunkTokenLength)
return string.Format(T("The data source max chunk length must be less than or equal to the embedding provider limit ({0})."), providerMaxChunkTokenLength);
if (maxChunkTokenLength > 0 && maxChunkTokenLength >= providerMaxChunkTokenLength)
return string.Format(T("The data source token limit must be smaller than the embedding provider token limit ({0}). Use 0 to use the provider setting."), providerMaxChunkTokenLength);
return null;
}
@ -157,7 +168,7 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
? this.dataMaxChunkTokenLength
: this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT;
if (chunkOverlapTokenLength >= effectiveMaxChunkTokenLength)
return T("The overlap must be smaller than the effective max chunk length.");
return T("The overlap must be smaller than the effective token limit.");
return null;
}

26
app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor
View File

@ -41,12 +41,18 @@
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
@T("Select a file for this data source. The content of this file will be processed for the data source.")
</MudJustifiedText>
<SelectFile @bind-File="@this.dataFilePath" Label="@T("Selected file path for this data source")" FileDialogTitle="@T("Select the file")" Validation="@this.dataSourceValidation.ValidateFilePath" />
@if (!this.CanChangeSourceAndEmbedding)
{
<MudAlert Severity="Severity.Info" Variant="Variant.Outlined" Class="mb-3">
@T("This data source already has indexed embeddings. Delete and recreate it to change the file path or embedding model.")
</MudAlert>
}
<SelectFile @bind-File="@this.dataFilePath" Label="@T("Selected file path for this data source")" FileDialogTitle="@T("Select the file")" Validation="@this.dataSourceValidation.ValidateFilePath" Disabled="@(!this.CanChangeSourceAndEmbedding)" />
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
@T("In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method.")
</MudJustifiedText>
<MudSelect @bind-Value="@this.dataEmbeddingId" Label="@T("Embedding")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateEmbeddingId">
<MudSelect @bind-Value="@this.dataEmbeddingId" Label="@T("Embedding")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateEmbeddingId" Disabled="@(!this.CanChangeSourceAndEmbedding)">
@foreach (var embedding in this.AvailableEmbeddings)
{
<MudSelectItem Value="@embedding.Value">
@ -57,6 +63,18 @@
@if (!string.IsNullOrWhiteSpace(this.dataEmbeddingId))
{
<MudTextField
T="string"
Text="@this.SelectedEmbeddingTokenizerText"
Label="@T("Tokenizer")"
Class="mb-3"
ReadOnly="@true"
Variant="Variant.Outlined"
Adornment="Adornment.Start"
AdornmentIcon="@Icons.Material.Filled.DataObject"
AdornmentColor="Color.Info"
/>
if (this.SelectedCloudEmbedding)
{
<DataSourceCloudEmbeddingWarning DataSourceType="DataSourceType.LOCAL_FILE" SourcePath="@this.dataFilePath" @bind-UserAcknowledged="@this.dataUserAcknowledgedCloudEmbedding" Validation="@this.dataSourceValidation.ValidateUserAcknowledgedCloudEmbedding"/>
@ -80,7 +98,7 @@
<MudNumericField
T="int"
@bind-Value="@this.dataMaxChunkTokenLength"
Label="@T("Max chunk length")"
Label="@T("Token limit")"
Class="mb-3"
Min="0"
Immediate="@true"
@ -94,7 +112,7 @@
<MudNumericField
T="int"
@bind-Value="@this.dataChunkOverlapTokenLength"
Label="@T("Chunk overlap length")"
Label="@T("Token overlap")"
Min="0"
Immediate="@true"
Validation="@this.ValidateChunkOverlapTokenLength"

23
app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs
View File

@ -17,6 +17,9 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
[Parameter]
public DataSourceLocalFile DataSource { get; set; }
[Parameter]
public bool LockSourceAndEmbedding { get; set; }
[Parameter]
public IReadOnlyList<ConfigurationSelectData<string>> AvailableEmbeddings { get; set; } = [];
@ -107,6 +110,14 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
private bool SelectedCloudEmbedding => this.SelectedEmbedding is { IsSelfHosted: false };
private bool CanChangeSourceAndEmbedding => !this.IsEditing || !this.LockSourceAndEmbedding;
private string SelectedEmbeddingTokenizerText => this.SelectedEmbedding is null
? T("No embedding selected")
: string.IsNullOrWhiteSpace(this.SelectedEmbedding.TokenizerPath)
? T("Default tokenizer")
: System.IO.Path.GetFileName(this.SelectedEmbedding.TokenizerPath);
private DataSourceLocalFile CreateDataSource() => new()
{
Id = this.dataId,
@ -114,8 +125,8 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
Name = this.dataName,
Description = this.dataDescription,
Type = DataSourceType.LOCAL_FILE,
EmbeddingId = this.dataEmbeddingId,
FilePath = this.dataFilePath,
EmbeddingId = this.CanChangeSourceAndEmbedding ? this.dataEmbeddingId : this.DataSource.EmbeddingId,
FilePath = this.CanChangeSourceAndEmbedding ? this.dataFilePath : this.DataSource.FilePath,
MaxChunkTokenLength = this.dataMaxChunkTokenLength,
ChunkOverlapTokenLength = this.dataChunkOverlapTokenLength,
SecurityPolicy = this.dataSecurityPolicy,
@ -139,11 +150,11 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
private string? ValidateMaxChunkTokenLength(int maxChunkTokenLength)
{
if (maxChunkTokenLength < 0)
return T("Please enter 0 or a positive max chunk length.");
return T("Please enter 0 or a positive token limit.");
var providerMaxChunkTokenLength = this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT;
if (maxChunkTokenLength > providerMaxChunkTokenLength)
return string.Format(T("The data source max chunk length must be less than or equal to the embedding provider limit ({0})."), providerMaxChunkTokenLength);
if (maxChunkTokenLength > 0 && maxChunkTokenLength >= providerMaxChunkTokenLength)
return string.Format(T("The data source token limit must be smaller than the embedding provider token limit ({0}). Use 0 to use the provider setting."), providerMaxChunkTokenLength);
return null;
}
@ -157,7 +168,7 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
? this.dataMaxChunkTokenLength
: this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT;
if (chunkOverlapTokenLength >= effectiveMaxChunkTokenLength)
return T("The overlap must be smaller than the effective max chunk length.");
return T("The overlap must be smaller than the effective token limit.");
return null;
}

4
app/MindWork AI Studio/Dialogs/Settings/SettingsDialogDataSources.razor.cs
View File

@ -227,6 +227,8 @@ public partial class SettingsDialogDataSources : SettingsDialogBase
return;
IDataSource? editedDataSource = null;
var lockDataSourceIdentity = dataSource is IInternalDataSource
&& await this.DataSourceEmbeddingService.ShouldLockDataSourceIdentityAsync(dataSource.Id);
switch (dataSource)
{
case DataSourceLocalFile localFile:
@ -234,6 +236,7 @@ public partial class SettingsDialogDataSources : SettingsDialogBase
{
{ x => x.IsEditing, true },
{ x => x.DataSource, localFile },
{ x => x.LockSourceAndEmbedding, lockDataSourceIdentity },
{ x => x.AvailableEmbeddings, this.AvailableEmbeddingProviders }
};
@ -250,6 +253,7 @@ public partial class SettingsDialogDataSources : SettingsDialogBase
{
{ x => x.IsEditing, true },
{ x => x.DataSource, localDirectory },
{ x => x.LockSourceAndEmbedding, lockDataSourceIdentity },
{ x => x.AvailableEmbeddings, this.AvailableEmbeddingProviders }
};

2
app/MindWork AI Studio/Pages/Embeddings.razor.cs
View File

@ -64,7 +64,7 @@ public partial class Embeddings : MSGComponentBase
private async Task RefreshDataSource(DataSourceEmbeddingStatus status)
{
await this.DataSourceEmbeddingService.QueueDataSourceAsync(status.DataSourceId);
await this.DataSourceEmbeddingService.RetryDataSourceAsync(status.DataSourceId);
this.ReloadStatuses();
await this.InvokeAsync(this.StateHasChanged);
}

3
app/MindWork AI Studio/Tools/Databases/EmbeddingState/EmbeddingStateClient.cs
View File

@ -12,11 +12,14 @@ public abstract class EmbeddingStateClient(string name, string path) : DatabaseC
string dataSourceType,
string embeddingProviderId,
string embeddingSignature,
string sourceHash,
int vectorSize,
CancellationToken token);
public abstract Task UpdateVectorSizeAsync(string dataSourceId, int vectorSize, CancellationToken token);
public abstract Task UpdateDataSourceHashAsync(string dataSourceId, string sourceHash, CancellationToken token);
public abstract Task UpsertFileAsync(string dataSourceId, EmbeddingStateFile file, CancellationToken token);
public abstract Task DeleteFileAsync(string dataSourceId, string filePath, CancellationToken token);

3
app/MindWork AI Studio/Tools/Databases/EmbeddingState/NoEmbeddingStateClient.cs
View File

@ -32,11 +32,14 @@ public sealed class NoEmbeddingStateClient(string name, string? unavailableReaso
string dataSourceType,
string embeddingProviderId,
string embeddingSignature,
string sourceHash,
int vectorSize,
CancellationToken token) => Task.CompletedTask;
public override Task UpdateVectorSizeAsync(string dataSourceId, int vectorSize, CancellationToken token) => Task.CompletedTask;
public override Task UpdateDataSourceHashAsync(string dataSourceId, string sourceHash, CancellationToken token) => Task.CompletedTask;
public override Task UpsertFileAsync(string dataSourceId, EmbeddingStateFile file, CancellationToken token) => Task.CompletedTask;
public override Task DeleteFileAsync(string dataSourceId, string filePath, CancellationToken token) => Task.CompletedTask;

42
app/MindWork AI Studio/Tools/Databases/EmbeddingState/SqliteEmbeddingStateClientImplementation.cs
View File

@ -77,7 +77,7 @@ public sealed class SqliteEmbeddingStateClientImplementation(
var manifest = new DataSourceEmbeddingManifest();
await using (var command = CreateCommand(connection, """
SELECT embedding_provider_id, embedding_signature, vector_size
SELECT embedding_provider_id, embedding_signature, source_hash, vector_size
FROM data_sources
WHERE data_source_id = $dataSourceId
"""))
@ -88,7 +88,8 @@ public sealed class SqliteEmbeddingStateClientImplementation(
{
manifest.EmbeddingProviderId = reader.GetString(0);
manifest.EmbeddingSignature = reader.GetString(1);
manifest.VectorSize = reader.GetInt32(2);
manifest.SourceHash = reader.GetString(2);
manifest.VectorSize = reader.GetInt32(3);
}
}
@ -120,6 +121,7 @@ public sealed class SqliteEmbeddingStateClientImplementation(
string dataSourceType,
string embeddingProviderId,
string embeddingSignature,
string sourceHash,
int vectorSize,
CancellationToken token)
{
@ -131,6 +133,7 @@ public sealed class SqliteEmbeddingStateClientImplementation(
data_source_type,
embedding_provider_id,
embedding_signature,
source_hash,
vector_size,
updated_at_utc)
VALUES (
@ -139,6 +142,7 @@ public sealed class SqliteEmbeddingStateClientImplementation(
$dataSourceType,
$embeddingProviderId,
$embeddingSignature,
$sourceHash,
$vectorSize,
$updatedAtUtc)
ON CONFLICT(data_source_id) DO UPDATE SET
@ -146,6 +150,7 @@ public sealed class SqliteEmbeddingStateClientImplementation(
data_source_type = excluded.data_source_type,
embedding_provider_id = excluded.embedding_provider_id,
embedding_signature = excluded.embedding_signature,
source_hash = excluded.source_hash,
vector_size = excluded.vector_size,
updated_at_utc = excluded.updated_at_utc
""", token,
@ -154,6 +159,7 @@ public sealed class SqliteEmbeddingStateClientImplementation(
("$dataSourceType", dataSourceType),
("$embeddingProviderId", embeddingProviderId),
("$embeddingSignature", embeddingSignature),
("$sourceHash", sourceHash),
("$vectorSize", vectorSize),
("$updatedAtUtc", ToUtcText(DateTime.UtcNow)));
}
@ -172,6 +178,20 @@ public sealed class SqliteEmbeddingStateClientImplementation(
("$updatedAtUtc", ToUtcText(DateTime.UtcNow)));
}
public override async Task UpdateDataSourceHashAsync(string dataSourceId, string sourceHash, CancellationToken token)
{
await using var connection = await this.OpenConnectionAsync(token);
await ExecuteNonQueryAsync(connection, """
UPDATE data_sources
SET source_hash = $sourceHash,
updated_at_utc = $updatedAtUtc
WHERE data_source_id = $dataSourceId
""", token,
("$dataSourceId", dataSourceId),
("$sourceHash", sourceHash),
("$updatedAtUtc", ToUtcText(DateTime.UtcNow)));
}
public override async Task UpsertFileAsync(string dataSourceId, EmbeddingStateFile file, CancellationToken token)
{
await using var connection = await this.OpenConnectionAsync(token);
@ -255,6 +275,7 @@ public sealed class SqliteEmbeddingStateClientImplementation(
data_source_type TEXT NOT NULL,
embedding_provider_id TEXT NOT NULL,
embedding_signature TEXT NOT NULL,
source_hash TEXT NOT NULL DEFAULT '',
vector_size INTEGER NOT NULL DEFAULT 0,
updated_at_utc TEXT NOT NULL
);
@ -278,6 +299,8 @@ public sealed class SqliteEmbeddingStateClientImplementation(
CREATE INDEX IF NOT EXISTS idx_embedded_files_data_source
ON embedded_files(data_source_id);
""", token);
await EnsureColumnAsync(connection, "data_sources", "source_hash", "TEXT NOT NULL DEFAULT ''", token);
}
private async Task<string> GetSqliteVersionAsync(CancellationToken token)
@ -296,6 +319,21 @@ public sealed class SqliteEmbeddingStateClientImplementation(
return Convert.ToInt64(countObject, CultureInfo.InvariantCulture);
}
private static async Task EnsureColumnAsync(SqliteConnection connection, string tableName, string columnName, string columnDefinition, CancellationToken token)
{
await using (var command = CreateCommand(connection, $"PRAGMA table_info({tableName})"))
{
await using var reader = await command.ExecuteReaderAsync(token);
while (await reader.ReadAsync(token))
{
if (string.Equals(reader.GetString(1), columnName, StringComparison.OrdinalIgnoreCase))
return;
}
}
await ExecuteNonQueryAsync(connection, $"ALTER TABLE {tableName} ADD COLUMN {columnName} {columnDefinition}", token);
}
private async Task<SqliteConnection> OpenConnectionAsync(CancellationToken token)
{
var connection = new SqliteConnection(this.connectionString);

2
app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingModels.cs
View File

@ -82,6 +82,8 @@ public sealed class DataSourceEmbeddingManifest
public string EmbeddingSignature { get; set; } = string.Empty;
public string SourceHash { get; set; } = string.Empty;
public int VectorSize { get; set; }
public Dictionary<string, EmbeddedFileRecord> Files { get; init; } = new(StringComparer.OrdinalIgnoreCase);

100
app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs
View File

@ -33,6 +33,8 @@ public sealed partial class DataSourceEmbeddingService
private sealed record ChunkingRule(string Name, Func<string, IReadOnlyList<string>, IReadOnlyList<string>>? Split);
private sealed record DataSourceMetadataSnapshot(string SourceHash, IReadOnlyDictionary<string, string> FileHashes);
private async IAsyncEnumerable<string> StreamEmbeddingChunksAsync(string filePath, IDataSource dataSource, EmbeddingProvider embeddingProvider, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token)
{
var options = this.GetChunkingOptions(dataSource, embeddingProvider);
@ -752,18 +754,96 @@ public sealed partial class DataSourceEmbeddingService
chunkingOptions.OverlapTokenLength);
}
private async Task<string> BuildFingerprintAsync(FileInfo file, CancellationToken token)
private DataSourceMetadataSnapshot BuildDataSourceMetadataSnapshot(IDataSource dataSource, IReadOnlyList<FileInfo> indexedFiles)
{
await using var stream = new FileStream(
var fileHashes = indexedFiles
.OrderBy(file => file.FullName, StringComparer.OrdinalIgnoreCase)
.ToDictionary(file => file.FullName, BuildFileMetadataHash, StringComparer.OrdinalIgnoreCase);
var sourceHash = dataSource switch
{
DataSourceLocalFile localFile => indexedFiles.Count > 0
? fileHashes[indexedFiles[0].FullName]
: BuildMetadataHash("file", localFile.FilePath, Path.GetFileName(localFile.FilePath) ?? string.Empty, "missing", "0"),
DataSourceLocalDirectory localDirectory => this.BuildDirectoryMetadataHash(localDirectory, indexedFiles, fileHashes),
_ => BuildMetadataHash(dataSource.Type.ToString(), dataSource.Id, dataSource.Name)
};
return new(sourceHash, fileHashes);
}
private string BuildDirectoryMetadataHash(DataSourceLocalDirectory dataSource, IReadOnlyList<FileInfo> indexedFiles, IReadOnlyDictionary<string, string> fileHashes)
{
var directory = new DirectoryInfo(dataSource.Path);
directory.Refresh();
var totalSize = 0L;
var latestFileWriteTicks = 0L;
foreach (var file in indexedFiles)
{
file.Refresh();
if (!file.Exists)
continue;
totalSize += file.Length;
latestFileWriteTicks = Math.Max(latestFileWriteTicks, file.LastWriteTimeUtc.Ticks);
}
var latestWriteTicks = Math.Max(directory.LastWriteTimeUtc.Ticks, latestFileWriteTicks);
var parts = new List<string>
{
"directory",
directory.FullName,
directory.Name,
latestWriteTicks.ToString(),
totalSize.ToString(),
indexedFiles.Count.ToString()
};
foreach (var file in indexedFiles.OrderBy(file => file.FullName, StringComparer.OrdinalIgnoreCase))
{
parts.Add(this.TryGetRelativePath(dataSource, file));
parts.Add(fileHashes[file.FullName]);
}
return BuildMetadataHash(parts);
}
private static string BuildFileMetadataHash(FileInfo file)
{
file.Refresh();
if (!file.Exists)
{
return BuildMetadataHash(
"file",
file.FullName,
file.Name,
"missing",
"0");
}
return BuildMetadataHash(
"file",
file.FullName,
FileMode.Open,
FileAccess.Read,
FileShare.ReadWrite | FileShare.Delete,
1024 * 128,
FileOptions.Asynchronous | FileOptions.SequentialScan);
var contentHash = await SHA256.HashDataAsync(stream, token);
var fingerprintSource = $"{file.FullName}|{Convert.ToHexString(contentHash)}";
var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(fingerprintSource));
file.Name,
file.LastWriteTimeUtc.Ticks.ToString(),
file.Length.ToString());
}
private static string BuildMetadataHash(params string[] parts)
{
return BuildMetadataHash((IEnumerable<string>)parts);
}
private static string BuildMetadataHash(IEnumerable<string> parts)
{
var fingerprintSource = new StringBuilder();
foreach (var part in parts)
fingerprintSource.Append(part.Length).Append(':').Append(part).Append('|');
var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(fingerprintSource.ToString()));
return Convert.ToHexString(bytes);
}

12
app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Watchers.cs
View File

@ -1,4 +1,5 @@
using System.Collections.Concurrent;
using System.Threading;
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
@ -20,6 +21,13 @@ public sealed partial class DataSourceEmbeddingService
return;
}
if (Volatile.Read(ref this.startupHashCheckCompleted) == 0)
{
logger.LogDebug("File watchers are not activated yet because the startup persisted hash check has not completed.");
this.RemoveAllWatchers();
return;
}
var supportedSources = settingsManager.ConfigurationData.DataSources
.Where(this.IsSupportedInternalDataSource)
.ToDictionary(source => source.Id, StringComparer.OrdinalIgnoreCase);
@ -161,8 +169,8 @@ public sealed partial class DataSourceEmbeddingService
if (dataSource is not null)
{
logger.LogInformation("Queueing data source '{DataSourceName}' ({DataSourceId}) after file system changes settled.", dataSource.Name, dataSource.Id);
await this.QueueDataSourceAsync(dataSource, true);
logger.LogInformation("Queueing data source '{DataSourceName}' ({DataSourceId}) after file system changes settled. The hash pipeline will reindex only changed files.", dataSource.Name, dataSource.Id);
await this.QueueDataSourceAsync(dataSource, true, DataSourceEmbeddingRefreshMode.WATCHER_HASH_CHECK);
}
}
catch (OperationCanceledException)

407
app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs
View File

@ -1,5 +1,6 @@
using System.Collections.Concurrent;
using System.Diagnostics.CodeAnalysis;
using System.Threading;
using System.Threading.Channels;
using AIStudio.Provider;
@ -15,12 +16,15 @@ namespace AIStudio.Tools.Services;
public sealed partial class DataSourceEmbeddingService(SettingsManager settingsManager, RustService rustService, DatabaseClientProvider databaseClientProvider, ILogger<DataSourceEmbeddingService> logger)
: BackgroundService
{
private readonly Channel<string> queue = Channel.CreateUnbounded<string>();
private readonly Channel<DataSourceEmbeddingQueueItem> queue = Channel.CreateUnbounded<DataSourceEmbeddingQueueItem>();
private readonly ConcurrentDictionary<string, byte> queuedIds = new(StringComparer.OrdinalIgnoreCase);
private readonly ConcurrentDictionary<string, byte> runningIds = new(StringComparer.OrdinalIgnoreCase);
private readonly ConcurrentDictionary<string, byte> pendingQueueIds = new(StringComparer.OrdinalIgnoreCase);
private readonly ConcurrentDictionary<string, DataSourceRunControl> activeRuns = new(StringComparer.OrdinalIgnoreCase);
private readonly ConcurrentDictionary<string, DataSourceEmbeddingStatus> statuses = new(StringComparer.OrdinalIgnoreCase);
private readonly object queueStateLock = new();
private int startupHashCheckStarted;
private int startupHashCheckCompleted;
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceEmbeddingService).Namespace, nameof(DataSourceEmbeddingService));
@ -32,6 +36,18 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
RUNNING_MARKED_PENDING,
}
private enum DataSourceEmbeddingRefreshMode
{
STARTUP_HASH_CHECK,
HASH_CHECK,
WATCHER_HASH_CHECK,
MANUAL_RETRY,
}
private sealed record DataSourceEmbeddingQueueItem(string DataSourceId, DataSourceEmbeddingRefreshMode RefreshMode);
private sealed record DataSourceRunControl(CancellationTokenSource TokenSource, TaskCompletionSource<object?> Completion);
public IReadOnlyList<DataSourceEmbeddingStatus> GetStatuses()
{
return this.statuses.Values
@ -75,9 +91,16 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
{
this.RefreshWatchers();
var tasks = settingsManager.ConfigurationData.DataSources
var supportedDataSources = settingsManager.ConfigurationData.DataSources
.Where(this.IsSupportedInternalDataSource)
.Select(dataSource => this.QueueDataSourceAsync(dataSource, queueAfterCurrentRun));
.ToList();
logger.LogInformation(
"Queueing {DataSourceCount} supported internal data source(s) for background embedding hash checks. QueueAfterCurrentRun={QueueAfterCurrentRun}.",
supportedDataSources.Count,
queueAfterCurrentRun);
var tasks = supportedDataSources.Select(dataSource => this.QueueDataSourceAsync(dataSource, queueAfterCurrentRun, DataSourceEmbeddingRefreshMode.HASH_CHECK));
return Task.WhenAll(tasks);
}
@ -90,11 +113,36 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
return Task.CompletedTask;
}
return this.QueueAllInternalDataSourcesAsync(false);
logger.LogDebug("Automatic startup embedding hash check is handled by the background service. Ignoring duplicate startup queue request.");
return Task.CompletedTask;
}
public void RefreshAutomaticWatchers()
{
if (!settingsManager.ConfigurationData.DataSourceIndexing.AutomaticRefresh)
{
Volatile.Write(ref this.startupHashCheckCompleted, 0);
Interlocked.Exchange(ref this.startupHashCheckStarted, 0);
this.RemoveAllWatchers();
return;
}
if (Volatile.Read(ref this.startupHashCheckCompleted) == 0)
{
_ = Task.Run(async () =>
{
try
{
await this.RunInitialDataSourceHashCheckAsync(CancellationToken.None);
}
catch (Exception exception)
{
logger.LogWarning(exception, "Failed to run the initial data source hash check after automatic refresh was enabled.");
}
});
return;
}
this.RefreshWatchers();
}
@ -109,9 +157,26 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
this.CanRefreshDataSource(dataSource);
}
public async Task<bool> ShouldLockDataSourceIdentityAsync(string dataSourceId, CancellationToken token = default)
{
var embeddingState = await databaseClientProvider.GetEmbeddingStateAsync(token);
if (!embeddingState.IsAvailable)
{
logger.LogWarning("Locking identity settings for data source '{DataSourceId}' because the embedding state database '{DatabaseName}' is unavailable.", dataSourceId, embeddingState.Name);
return true;
}
var manifest = await embeddingState.GetManifestAsync(dataSourceId, token);
return !string.IsNullOrWhiteSpace(manifest.EmbeddingProviderId)
|| !string.IsNullOrWhiteSpace(manifest.EmbeddingSignature)
|| !string.IsNullOrWhiteSpace(manifest.SourceHash)
|| manifest.VectorSize > 0
|| manifest.Files.Count > 0;
}
public Task QueueDataSourceAsync(IDataSource dataSource)
{
return this.QueueDataSourceAsync(dataSource, true);
return this.QueueDataSourceAsync(dataSource, true, DataSourceEmbeddingRefreshMode.HASH_CHECK);
}
public Task QueueDataSourceAsync(string dataSourceId)
@ -121,13 +186,20 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
: Task.CompletedTask;
}
private async Task QueueDataSourceAsync(IDataSource dataSource, bool queueAfterCurrentRun)
public Task RetryDataSourceAsync(string dataSourceId)
{
return this.TryGetConfiguredDataSource(dataSourceId, out var dataSource)
? this.QueueDataSourceAsync(dataSource, true, DataSourceEmbeddingRefreshMode.MANUAL_RETRY)
: Task.CompletedTask;
}
private async Task QueueDataSourceAsync(IDataSource dataSource, bool queueAfterCurrentRun, DataSourceEmbeddingRefreshMode refreshMode)
{
if (!this.IsSupportedInternalDataSource(dataSource))
return;
this.RefreshWatchers();
logger.LogDebug("Ensured watcher for data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id);
logger.LogDebug("Refreshed watcher state for data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id);
var queueRequestResult = this.TryReserveDataSourceQueueSlot(dataSource.Id, queueAfterCurrentRun);
switch (queueRequestResult)
@ -145,7 +217,11 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
return;
}
logger.LogInformation("Queueing data source '{DataSourceName}' ({DataSourceId}) for background embeddings.", dataSource.Name, dataSource.Id);
logger.LogInformation(
"Queueing data source '{DataSourceName}' ({DataSourceId}) for background embedding hash check. RefreshMode={RefreshMode}.",
dataSource.Name,
dataSource.Id,
refreshMode);
if (!this.statuses.TryGetValue(dataSource.Id, out var currentStatus) || currentStatus.State is not DataSourceEmbeddingState.RUNNING)
{
this.UpsertStatus(this.CreateStatus(
@ -157,7 +233,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
failures: currentStatus?.Failures ?? []));
}
logger.LogDebug("Upserting status for data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id);
await this.queue.Writer.WriteAsync(dataSource.Id);
await this.queue.Writer.WriteAsync(new DataSourceEmbeddingQueueItem(dataSource.Id, refreshMode));
logger.LogDebug("Queued data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id);
}
@ -167,8 +243,21 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
return;
this.RemoveWatcher(dataSource.Id);
var activeRun = this.CancelActiveDataSourceRun(dataSource);
this.ClearQueuedDataSourceState(dataSource.Id);
this.statuses.TryRemove(dataSource.Id, out _);
if (activeRun is not null)
{
logger.LogInformation(
"Waiting for the active embedding run for deleted data source '{DataSourceName}' ({DataSourceId}) to stop before deleting persisted embeddings.",
dataSource.Name,
dataSource.Id);
await activeRun.Completion.Task;
}
this.statuses.TryRemove(dataSource.Id, out _);
await this.ResetPersistedStateAsync(dataSource.Name, dataSource.Id, null, null, CancellationToken.None);
this.statuses.TryRemove(dataSource.Id, out _);
this.PublishStatusChanged();
}
@ -178,7 +267,8 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
while (!stoppingToken.IsCancellationRequested)
{
var dataSourceId = await this.queue.Reader.ReadAsync(stoppingToken);
var queueItem = await this.queue.Reader.ReadAsync(stoppingToken);
var dataSourceId = queueItem.DataSourceId;
this.MarkDataSourceRunStarted(dataSourceId);
IDataSource? dataSource = null;
@ -191,7 +281,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
if (dataSource is null || !this.IsSupportedInternalDataSource(dataSource))
continue;
await this.ProcessDataSourceAsync(dataSource, stoppingToken);
await this.ProcessDataSourceRunAsync(dataSource, queueItem.RefreshMode, stoppingToken);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
@ -222,12 +312,68 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
base.Dispose();
}
private async Task ProcessDataSourceAsync(IDataSource dataSource, CancellationToken token)
private async Task ProcessDataSourceRunAsync(IDataSource dataSource, DataSourceEmbeddingRefreshMode refreshMode, CancellationToken parentToken)
{
logger.LogInformation("Starting background embeddings for data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id);
if (!this.TryGetConfiguredDataSource(dataSource.Id, out var configuredDataSource) ||
!this.IsSupportedInternalDataSource(configuredDataSource))
{
logger.LogDebug(
"Skipping embedding run for data source '{DataSourceName}' ({DataSourceId}) because it is no longer configured. RefreshMode={RefreshMode}.",
dataSource.Name,
dataSource.Id,
refreshMode);
return;
}
dataSource = configuredDataSource;
var runTokenSource = CancellationTokenSource.CreateLinkedTokenSource(parentToken);
var runControl = new DataSourceRunControl(
runTokenSource,
new TaskCompletionSource<object?>(TaskCreationOptions.RunContinuationsAsynchronously));
if (!this.activeRuns.TryAdd(dataSource.Id, runControl))
{
runTokenSource.Dispose();
logger.LogDebug(
"Data source '{DataSourceName}' ({DataSourceId}) already has an active embedding run. Skipping duplicate process request. RefreshMode={RefreshMode}.",
dataSource.Name,
dataSource.Id,
refreshMode);
return;
}
try
{
await this.ProcessDataSourceAsync(dataSource, refreshMode, runTokenSource.Token);
}
catch (OperationCanceledException) when (!parentToken.IsCancellationRequested && runTokenSource.IsCancellationRequested)
{
logger.LogInformation(
"Stopped background embeddings for data source '{DataSourceName}' ({DataSourceId}) because the data source was removed or canceled. RefreshMode={RefreshMode}.",
dataSource.Name,
dataSource.Id,
refreshMode);
}
finally
{
this.activeRuns.TryRemove(dataSource.Id, out _);
runControl.Completion.TrySetResult(null);
runTokenSource.Dispose();
}
}
private async Task ProcessDataSourceAsync(IDataSource dataSource, DataSourceEmbeddingRefreshMode refreshMode, CancellationToken token)
{
logger.LogInformation(
"Starting background embedding hash check for data source '{DataSourceName}' ({DataSourceId}). RefreshMode={RefreshMode}.",
dataSource.Name,
dataSource.Id,
refreshMode);
token.ThrowIfCancellationRequested();
var vectorStore = await databaseClientProvider.GetVectorStoreAsync(token);
var embeddingState = await databaseClientProvider.GetEmbeddingStateAsync(token);
token.ThrowIfCancellationRequested();
if (!vectorStore.IsAvailable)
{
@ -236,6 +382,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
dataSource.Name,
dataSource.Id,
vectorStore.Name);
token.ThrowIfCancellationRequested();
this.UpsertStatus(this.GetFallbackStatus(dataSource, "The vector database is not available."));
return;
}
@ -247,12 +394,14 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
dataSource.Name,
dataSource.Id,
embeddingState.Name);
token.ThrowIfCancellationRequested();
this.UpsertStatus(this.GetFallbackStatus(dataSource, "The SQLite embedding state database is not available."));
return;
}
if (!this.TryResolveEmbeddingProvider(dataSource, out var embeddingProvider))
{
token.ThrowIfCancellationRequested();
this.UpsertStatus(this.GetFallbackStatus(dataSource, "The selected embedding provider is not available."));
return;
}
@ -267,6 +416,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
var collectionName = this.GetCollectionName(dataSource.Name, dataSource.Id);
var manifest = await this.EnsureCompatibleManifestAsync(dataSource, embeddingProvider, collectionName, vectorStore, embeddingState, token);
token.ThrowIfCancellationRequested();
var inputFiles = this.GetInputFiles(dataSource);
var indexedFiles = inputFiles.Files;
var totalFiles = indexedFiles.Count + inputFiles.FailedFiles;
@ -279,8 +429,35 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
inputFiles.FailedFiles,
collectionName);
await this.RemoveMissingFileEmbeddingsAsync(vectorStore, embeddingState, dataSource, collectionName, manifest, indexedFiles, token);
var metadataSnapshot = this.BuildDataSourceMetadataSnapshot(dataSource, indexedFiles);
var removedMissingFiles = await this.RemoveMissingFileEmbeddingsAsync(vectorStore, embeddingState, dataSource, collectionName, manifest, indexedFiles, token);
token.ThrowIfCancellationRequested();
logger.LogInformation(
"Compared data source hash for '{DataSourceName}' ({DataSourceId}). StoredSourceHashPrefix={StoredSourceHashPrefix}, CurrentSourceHashPrefix={CurrentSourceHashPrefix}, StoredFileRecords={StoredFileRecords}, CurrentFiles={CurrentFiles}, RemovedMissingFiles={RemovedMissingFiles}.",
dataSource.Name,
dataSource.Id,
ShortHash(manifest.SourceHash),
ShortHash(metadataSnapshot.SourceHash),
manifest.Files.Count,
indexedFiles.Count,
removedMissingFiles);
if (this.CanSkipDataSourceByHash(manifest, metadataSnapshot, indexedFiles))
{
logger.LogInformation(
"Skipping data source '{DataSourceName}' ({DataSourceId}) because the persisted data source hash and all persisted file hashes match. RefreshMode={RefreshMode}.",
dataSource.Name,
dataSource.Id,
refreshMode);
token.ThrowIfCancellationRequested();
await embeddingState.UpdateDataSourceHashAsync(dataSource.Id, metadataSnapshot.SourceHash, token);
this.UpsertStatus(this.CreateCompletedStatus(dataSource, totalFiles, indexedFiles.Count, inputFiles.FailedFiles, inputFiles.LastError, inputFiles.Failures));
return;
}
token.ThrowIfCancellationRequested();
this.UpsertStatus(this.CreateStatus(
dataSource,
DataSourceEmbeddingState.RUNNING,
@ -293,6 +470,8 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
var provider = embeddingProvider.CreateProvider();
var skippedFiles = 0;
var completedFiles = 0;
var newFiles = 0;
var changedFiles = 0;
var failedFiles = inputFiles.FailedFiles;
var lastError = inputFiles.LastError;
var failureDetails = inputFiles.Failures.ToList();
@ -301,15 +480,18 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
{
token.ThrowIfCancellationRequested();
var fingerprint = await this.BuildFingerprintAsync(file, token);
var fingerprint = metadataSnapshot.FileHashes[file.FullName];
if (manifest.Files.TryGetValue(file.FullName, out var existingRecord) &&
string.Equals(existingRecord.Fingerprint, fingerprint, StringComparison.Ordinal))
{
logger.LogDebug(
"Skipping unchanged file '{FilePath}' for data source '{DataSourceName}' ({DataSourceId}).",
"Skipping unchanged file '{FilePath}' for data source '{DataSourceName}' ({DataSourceId}) because the persisted metadata hash matches. MetadataHashPrefix={MetadataHashPrefix}, LastWriteUtc={LastWriteUtc:O}, FileSize={FileSize}.",
file.FullName,
dataSource.Name,
dataSource.Id);
dataSource.Id,
ShortHash(fingerprint),
file.LastWriteTimeUtc,
file.Length);
skippedFiles++;
this.UpsertStatus(this.CreateStatus(dataSource, DataSourceEmbeddingState.RUNNING, totalFiles, skippedFiles + completedFiles, failedFiles, lastError: lastError, failures: failureDetails));
continue;
@ -320,14 +502,17 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
try
{
logger.LogInformation(
"Embedding file '{FilePath}' for data source '{DataSourceName}' ({DataSourceId}). Progress={CompletedFiles}/{TotalFiles}.",
"Embedding file '{FilePath}' for data source '{DataSourceName}' ({DataSourceId}) because {EmbeddingReason}. CurrentMetadataHashPrefix={CurrentMetadataHashPrefix}. Progress={CompletedFiles}/{TotalFiles}.",
file.FullName,
dataSource.Name,
dataSource.Id,
GetFileEmbeddingReason(file, fingerprint, existingRecord),
ShortHash(fingerprint),
skippedFiles + completedFiles + 1,
totalFiles);
var startedAtUtc = DateTime.UtcNow;
var chunkCount = await this.IndexOneFileAsync(embeddingState, vectorStore, dataSource, file, fingerprint, embeddingProvider, provider, manifest, token);
token.ThrowIfCancellationRequested();
var embeddedAtUtc = DateTime.UtcNow;
var record = new EmbeddedFileRecord(
fingerprint,
@ -349,6 +534,11 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
token);
manifest.Files[file.FullName] = record;
completedFiles++;
if (existingRecord is null)
newFiles++;
else
changedFiles++;
logger.LogInformation(
"Embedded file '{FilePath}' for data source '{DataSourceName}' ({DataSourceId}) successfully. Chunks={ChunkCount}, DurationMs={DurationMs}.",
file.FullName,
@ -357,6 +547,10 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
chunkCount,
(DateTime.UtcNow - startedAtUtc).TotalMilliseconds);
}
catch (OperationCanceledException) when (token.IsCancellationRequested)
{
throw;
}
catch (Exception exception)
{
failedFiles++;
@ -371,14 +565,25 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
}
}
manifest.SourceHash = metadataSnapshot.SourceHash;
token.ThrowIfCancellationRequested();
await embeddingState.UpdateDataSourceHashAsync(dataSource.Id, metadataSnapshot.SourceHash, token);
token.ThrowIfCancellationRequested();
this.UpsertStatus(this.CreateCompletedStatus(dataSource, totalFiles, skippedFiles + completedFiles, failedFiles, lastError, failureDetails));
logger.LogInformation(
"Finished background embeddings for data source '{DataSourceName}' ({DataSourceId}). Indexed={IndexedFiles}, Failed={FailedFiles}, Total={TotalFiles}.",
"Finished background embeddings for data source '{DataSourceName}' ({DataSourceId}). RefreshMode={RefreshMode}, Embedded={EmbeddedFiles}, New={NewFiles}, Changed={ChangedFiles}, Skipped={SkippedFiles}, RemovedMissing={RemovedMissingFiles}, Failed={FailedFiles}, Total={TotalFiles}, SourceHashPrefix={SourceHashPrefix}.",
dataSource.Name,
dataSource.Id,
skippedFiles + completedFiles,
refreshMode,
completedFiles,
newFiles,
changedFiles,
skippedFiles,
removedMissingFiles,
failedFiles,
totalFiles);
totalFiles,
ShortHash(metadataSnapshot.SourceHash));
}
private async Task<int> IndexOneFileAsync(
@ -453,6 +658,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
try
{
vectors = await provider.EmbedTextAsync(embeddingProvider.Model, settingsManager, token, texts);
token.ThrowIfCancellationRequested();
}
catch (OperationCanceledException) when (token.IsCancellationRequested)
{
@ -475,6 +681,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
if (manifest.VectorSize == 0)
{
token.ThrowIfCancellationRequested();
manifest.VectorSize = vectorSize;
await this.EnsureCollectionExistsAsync(vectorStore, collectionName, vectorSize, token);
await embeddingState.UpdateVectorSizeAsync(dataSource.Id, vectorSize, token);
@ -486,6 +693,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
dataSource.Id);
}
token.ThrowIfCancellationRequested();
await this.UpsertPointsAsync(
vectorStore,
collectionName,
@ -574,8 +782,62 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
token.ThrowIfCancellationRequested();
logger.LogInformation("Embedding background service is ready. Checking whether automatic data source refresh is enabled.");
await this.QueueAllInternalDataSourcesIfAutomaticRefreshAsync();
logger.LogInformation("Embedding background service is ready. Running the initial persisted hash check before activating file watchers.");
await this.RunInitialDataSourceHashCheckAsync(token);
}
private async Task RunInitialDataSourceHashCheckAsync(CancellationToken token)
{
if (!settingsManager.ConfigurationData.DataSourceIndexing.AutomaticRefresh)
{
logger.LogInformation("Automatic local data source refresh is disabled. Startup hash checks and file watchers are disabled.");
this.RemoveAllWatchers();
return;
}
if (Interlocked.Exchange(ref this.startupHashCheckStarted, 1) == 1)
return;
this.RemoveAllWatchers();
var supportedDataSources = settingsManager.ConfigurationData.DataSources
.Where(this.IsSupportedInternalDataSource)
.ToList();
logger.LogInformation(
"Starting initial persisted hash check for {DataSourceCount} supported internal data source(s). Incomplete or failed embedding state will be retried during this pass. File watchers will be activated after this check completes.",
supportedDataSources.Count);
foreach (var dataSource in supportedDataSources)
{
token.ThrowIfCancellationRequested();
try
{
await this.ProcessDataSourceRunAsync(dataSource, DataSourceEmbeddingRefreshMode.STARTUP_HASH_CHECK, token);
}
catch (OperationCanceledException) when (token.IsCancellationRequested)
{
throw;
}
catch (Exception exception)
{
logger.LogError(exception, "Initial embedding hash check failed for data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id);
this.UpsertStatus(this.GetFallbackStatus(dataSource, exception.Message));
}
}
if (!settingsManager.ConfigurationData.DataSourceIndexing.AutomaticRefresh)
{
Volatile.Write(ref this.startupHashCheckCompleted, 0);
Interlocked.Exchange(ref this.startupHashCheckStarted, 0);
logger.LogInformation("Automatic local data source refresh was disabled before the initial hash check completed. File watchers remain inactive.");
this.RemoveAllWatchers();
return;
}
Volatile.Write(ref this.startupHashCheckCompleted, 1);
logger.LogInformation("Completed initial persisted hash check. Activating file watchers for automatic local data source refresh.");
this.RefreshWatchers();
}
private bool IsSupportedInternalDataSource(IDataSource dataSource)
@ -612,6 +874,15 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
var embeddingSignature = this.BuildEmbeddingSignature(dataSource, embeddingProvider, chunkingOptions);
var manifest = await embeddingState.GetManifestAsync(dataSource.Id, token);
logger.LogInformation(
"Loaded persisted embedding manifest for data source '{DataSourceName}' ({DataSourceId}). StoredFiles={StoredFiles}, StoredSourceHashPrefix={StoredSourceHashPrefix}, StoredSignaturePrefix={StoredSignaturePrefix}, CurrentSignaturePrefix={CurrentSignaturePrefix}.",
dataSource.Name,
dataSource.Id,
manifest.Files.Count,
ShortHash(manifest.SourceHash),
ShortHash(manifest.EmbeddingSignature),
ShortHash(embeddingSignature));
if (!string.Equals(manifest.EmbeddingSignature, embeddingSignature, StringComparison.Ordinal))
{
logger.LogInformation(
@ -642,13 +913,14 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
dataSource.Type.ToString(),
manifest.EmbeddingProviderId,
manifest.EmbeddingSignature,
manifest.SourceHash,
manifest.VectorSize,
token);
return manifest;
}
private async Task RemoveMissingFileEmbeddingsAsync(
private async Task<int> RemoveMissingFileEmbeddingsAsync(
VectorStoreClient vectorStore,
EmbeddingStateClient embeddingState,
IDataSource dataSource,
@ -661,17 +933,72 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
.Select(file => file.FullName)
.ToHashSet(StringComparer.OrdinalIgnoreCase);
var removedFiles = 0;
foreach (var removedFilePath in manifest.Files.Keys.Except(existingPaths, StringComparer.OrdinalIgnoreCase).ToList())
{
await this.DeleteFilePointsAsync(vectorStore, collectionName, removedFilePath, token);
await embeddingState.DeleteFileAsync(dataSource.Id, removedFilePath, token);
manifest.Files.Remove(removedFilePath);
removedFiles++;
logger.LogInformation(
"Removed stale embeddings for deleted file '{FilePath}' from data source '{DataSourceName}' ({DataSourceId}).",
removedFilePath,
dataSource.Name,
dataSource.Id);
}
return removedFiles;
}
private bool CanSkipDataSourceByHash(DataSourceEmbeddingManifest manifest, DataSourceMetadataSnapshot metadataSnapshot, IReadOnlyCollection<FileInfo> indexedFiles)
{
if (!string.Equals(manifest.SourceHash, metadataSnapshot.SourceHash, StringComparison.Ordinal))
return false;
if (manifest.Files.Count != indexedFiles.Count)
return false;
foreach (var file in indexedFiles)
{
if (!metadataSnapshot.FileHashes.TryGetValue(file.FullName, out var currentHash))
return false;
if (!manifest.Files.TryGetValue(file.FullName, out var existingRecord))
return false;
if (!string.Equals(existingRecord.Fingerprint, currentHash, StringComparison.Ordinal))
return false;
}
return true;
}
private static string GetFileEmbeddingReason(FileInfo file, string currentHash, EmbeddedFileRecord? existingRecord)
{
if (existingRecord is null)
return "no stored file hash exists";
var reasons = new List<string>();
if (!string.Equals(existingRecord.Fingerprint, currentHash, StringComparison.Ordinal))
reasons.Add($"stored hash {ShortHash(existingRecord.Fingerprint)} differs from current hash {ShortHash(currentHash)}");
if (existingRecord.FileSize != file.Length)
reasons.Add($"file size changed from {existingRecord.FileSize} to {file.Length} bytes");
if (existingRecord.LastWriteUtc != file.LastWriteTimeUtc)
reasons.Add($"last modified time changed from {existingRecord.LastWriteUtc:O} to {file.LastWriteTimeUtc:O}");
return reasons.Count == 0
? "the file hash changed"
: string.Join("; ", reasons);
}
private static string ShortHash(string value)
{
if (string.IsNullOrWhiteSpace(value))
return "<empty>";
return value.Length <= 12 ? value : value[..12];
}
private DataSourceEmbeddingStatus CreateStatus(
@ -774,6 +1101,36 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
}
}
private void ClearQueuedDataSourceState(string dataSourceId)
{
lock (this.queueStateLock)
{
this.queuedIds.TryRemove(dataSourceId, out _);
this.pendingQueueIds.TryRemove(dataSourceId, out _);
}
}
private DataSourceRunControl? CancelActiveDataSourceRun(IDataSource dataSource)
{
if (!this.activeRuns.TryGetValue(dataSource.Id, out var activeRun))
return null;
logger.LogInformation(
"Canceling active embedding run for deleted data source '{DataSourceName}' ({DataSourceId}).",
dataSource.Name,
dataSource.Id);
try
{
activeRun.TokenSource.Cancel();
}
catch (ObjectDisposedException)
{
return null;
}
return activeRun;
}
private async Task QueuePendingDataSourceRunAsync(string dataSourceId, CancellationToken token)
{
var dataSource = token.IsCancellationRequested
@ -804,7 +1161,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
try
{
await this.queue.Writer.WriteAsync(dataSourceId, token);
await this.queue.Writer.WriteAsync(new DataSourceEmbeddingQueueItem(dataSourceId, DataSourceEmbeddingRefreshMode.HASH_CHECK), token);
}
catch (OperationCanceledException) when (token.IsCancellationRequested)
{