MindWorkAI/AI-Studio
MindWorkAI/AI-Studio
1
0
Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2026-09-27 03:53:36 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Let people change the embedding of an indexed data source

Browse Source
This commit is contained in:
Thorsten Sommer 2026-09-18 18:11:00 +02:00
parent 946ae1a5a5
commit 3a7f9b4deb
Signed by untrusted user who does not match committer: tsommer
GPG Key ID: 371BBA77A02C0108
6 changed files with 88 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
app/MindWork AI Studio/Components/DataSourceManagement.razor.cs
View File

@ -335,8 +335,8 @@ public partial class DataSourceManagement : MSGComponentBase
return;
IDataSource? editedDataSource = null;
var lockDataSourceIdentity = dataSource is IInternalDataSource
&& await this.DataSourceEmbeddingService.ShouldLockDataSourceIdentityAsync(dataSource.Id);
var lockDataSourceOrigin = dataSource is IInternalDataSource
&& await this.DataSourceEmbeddingService.ShouldLockDataSourceOriginAsync(dataSource.Id);
switch (dataSource)
{
case DataSourceLocalFile localFile:
@ -344,7 +344,7 @@ public partial class DataSourceManagement : MSGComponentBase
{
{ x => x.IsEditing, true },
{ x => x.DataSource, localFile },
{ x => x.LockSourceAndEmbedding, lockDataSourceIdentity },
{ x => x.LockSource, lockDataSourceOrigin },
{ x => x.AvailableEmbeddings, this.availableEmbeddingProviders }
};
@ -361,7 +361,7 @@ public partial class DataSourceManagement : MSGComponentBase
{
{ x => x.IsEditing, true },
{ x => x.DataSource, localDirectory },
{ x => x.LockSourceAndEmbedding, lockDataSourceIdentity },
{ x => x.LockSource, lockDataSourceOrigin },
{ x => x.AvailableEmbeddings, this.availableEmbeddingProviders }
};

53
app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor
View File

@ -45,13 +45,13 @@
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
@T("Select a root directory for this data source. All data in this directory and all its subdirectories will be processed for this data source.")
</MudJustifiedText>
@if (!this.CanChangeSourceAndEmbedding)
@if (!this.CanChangeSource)
{
<MudAlert Severity="Severity.Info" Variant="Variant.Outlined" Class="mb-3">
@T("This data source already has indexed embeddings. Delete and recreate it to change the folder path or embedding model.")
@T("The documents of this data source are already prepared, so its folder cannot be changed. Another folder holds other documents, which makes it another data source: please add one for it. The embedding method below can be changed.")
</MudAlert>
}
@if (this.CanChangeSourceAndEmbedding)
@if (this.CanChangeSource)
{
<SelectDirectory @bind-Directory="@this.dataPath" Label="@T("Selected base directory for this data source")" DirectoryDialogTitle="@T("Select the base directory")" Validation="@this.dataSourceValidation.ValidatePath" EnableDragDrop="true" CatchAllDocuments="true" />
}
@ -74,38 +74,21 @@
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
@T("In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method.")
</MudJustifiedText>
@if (this.CanChangeSourceAndEmbedding)
{
<MudSelect @bind-Value="@this.dataEmbeddingId" Label="@T("Embedding")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateEmbeddingProviderAccess">
@foreach (var embedding in this.AvailableEmbeddings)
{
<MudSelectItem Value="@embedding.Value">
@if (this.GetEmbeddingProvider(embedding.Value) is { } provider)
{
<ProviderLabel ProviderType="@provider.UsedLLMProvider" CustomIconDataUrl="@provider.CustomIconDataUrl" Text="@embedding.Name" />
}
else
{
@embedding.Name
}
</MudSelectItem>
}
</MudSelect>
}
else
{
<MudTextField
T="string"
Text="@this.SelectedEmbeddingNameText"
Label="@T("Embedding")"
Class="mb-3"
ReadOnly="@true"
Adornment="Adornment.Start"
AdornmentIcon="@Icons.Material.Filled.DataObject"
AdornmentColor="Color.Info"
Variant="Variant.Outlined"
/>
}
<MudSelect @bind-Value="@this.dataEmbeddingId" Label="@T("Embedding")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateEmbeddingProviderAccess">
@foreach (var embedding in this.AvailableEmbeddings)
{
<MudSelectItem Value="@embedding.Value">
@if (this.GetEmbeddingProvider(embedding.Value) is { } provider)
{
<ProviderLabel ProviderType="@provider.UsedLLMProvider" CustomIconDataUrl="@provider.CustomIconDataUrl" Text="@embedding.Name" />
}
else
{
@embedding.Name
}
</MudSelectItem>
}
</MudSelect>
@if (!string.IsNullOrWhiteSpace(this.dataEmbeddingId))
{

27
app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs
View File

@ -20,8 +20,15 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
[Parameter]
public DataSourceLocalDirectory DataSource { get; set; }
/// <summary>
/// Whether the folder this data source reads must stay as it is.
/// </summary>
/// <remarks>
/// Set once the index holds something for this data source. The embedding is not locked along
/// with it: it can be changed, and DataSourceReindexWarning asks what that costs.
/// </remarks>
[Parameter]
public bool LockSourceAndEmbedding { get; set; }
public bool LockSource { get; set; }
[Parameter]
public IReadOnlyList<ConfigurationSelectData<string>> AvailableEmbeddings { get; set; } = [];
@ -126,19 +133,10 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
private bool SelectedCloudEmbedding => this.SelectedEmbedding is { IsSelfHosted: false };
private bool CanChangeSourceAndEmbedding => !this.IsEditing || !this.LockSourceAndEmbedding;
private bool CanChangeSource => !this.IsEditing || !this.LockSource;
private IEnumerable<ConfigurationSelectData<ConfidenceLevel>> ConfidenceLevels => ConfigurationSelectDataFactory.GetDataSourceConfidenceLevelsData();
private string SelectedEmbeddingNameText
{
get
{
var selectedEmbedding = this.AvailableEmbeddings.FirstOrDefault(x => x.Value == this.dataEmbeddingId);
return string.IsNullOrWhiteSpace(selectedEmbedding.Name) ? T("Unknown") : selectedEmbedding.Name;
}
}
private string SelectedEmbeddingTokenizerText => this.SelectedEmbedding is null
? T("No embedding selected")
: string.IsNullOrWhiteSpace(this.SelectedEmbedding.TokenizerPath)
@ -162,8 +160,11 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
Name = this.dataName,
Description = this.dataDescription,
Type = DataSourceType.LOCAL_DIRECTORY,
EmbeddingId = this.CanChangeSourceAndEmbedding ? this.dataEmbeddingId : this.DataSource.EmbeddingId,
Path = this.CanChangeSourceAndEmbedding ? this.dataPath : this.DataSource.Path,
EmbeddingId = this.dataEmbeddingId,
// Kept out of reach of the form while the source is locked, so a stale field cannot point an
// indexed data source somewhere else:
Path = this.CanChangeSource ? this.dataPath : this.DataSource.Path,
MaxChunkTokenLength = this.dataMaxChunkTokenLength,
ChunkOverlapTokenLength = this.dataChunkOverlapTokenLength,
ConfidenceLevel = this.dataConfidenceLevel,

53
app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor
View File

@ -45,13 +45,13 @@
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
@T("Select a file for this data source. The content of this file will be processed for the data source.")
</MudJustifiedText>
@if (!this.CanChangeSourceAndEmbedding)
@if (!this.CanChangeSource)
{
<MudAlert Severity="Severity.Info" Variant="Variant.Outlined" Class="mb-3">
@T("This data source already has indexed embeddings. Delete and recreate it to change the file path or embedding model.")
@T("The documents of this data source are already prepared, so its file cannot be changed. Another file holds other content, which makes it another data source: please add one for it. The embedding method below can be changed.")
</MudAlert>
}
@if (this.CanChangeSourceAndEmbedding)
@if (this.CanChangeSource)
{
<SelectFile @bind-File="@this.dataFilePath" Label="@T("Selected file path for this data source")" FileDialogTitle="@T("Select the file")" Validation="@this.dataSourceValidation.ValidateFilePath" EnableDragDrop="true" CatchAllDocuments="true" />
}
@ -74,38 +74,21 @@
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
@T("In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method.")
</MudJustifiedText>
@if (this.CanChangeSourceAndEmbedding)
{
<MudSelect @bind-Value="@this.dataEmbeddingId" Label="@T("Embedding")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateEmbeddingProviderAccess">
@foreach (var embedding in this.AvailableEmbeddings)
{
<MudSelectItem Value="@embedding.Value">
@if (this.GetEmbeddingProvider(embedding.Value) is { } provider)
{
<ProviderLabel ProviderType="@provider.UsedLLMProvider" CustomIconDataUrl="@provider.CustomIconDataUrl" Text="@embedding.Name" />
}
else
{
@embedding.Name
}
</MudSelectItem>
}
</MudSelect>
}
else
{
<MudTextField
T="string"
Text="@this.SelectedEmbeddingNameText"
Label="@T("Embedding")"
Class="mb-3"
ReadOnly="@true"
Adornment="Adornment.Start"
AdornmentIcon="@Icons.Material.Filled.DataObject"
AdornmentColor="Color.Info"
Variant="Variant.Outlined"
/>
}
<MudSelect @bind-Value="@this.dataEmbeddingId" Label="@T("Embedding")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateEmbeddingProviderAccess">
@foreach (var embedding in this.AvailableEmbeddings)
{
<MudSelectItem Value="@embedding.Value">
@if (this.GetEmbeddingProvider(embedding.Value) is { } provider)
{
<ProviderLabel ProviderType="@provider.UsedLLMProvider" CustomIconDataUrl="@provider.CustomIconDataUrl" Text="@embedding.Name" />
}
else
{
@embedding.Name
}
</MudSelectItem>
}
</MudSelect>
@if (!string.IsNullOrWhiteSpace(this.dataEmbeddingId))
{

27
app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs
View File

@ -20,8 +20,15 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
[Parameter]
public DataSourceLocalFile DataSource { get; set; }
/// <summary>
/// Whether the file this data source reads must stay as it is.
/// </summary>
/// <remarks>
/// Set once the index holds something for this data source. The embedding is not locked along
/// with it: it can be changed, and DataSourceReindexWarning asks what that costs.
/// </remarks>
[Parameter]
public bool LockSourceAndEmbedding { get; set; }
public bool LockSource { get; set; }
[Parameter]
public IReadOnlyList<ConfigurationSelectData<string>> AvailableEmbeddings { get; set; } = [];
@ -126,19 +133,10 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
private bool SelectedCloudEmbedding => this.SelectedEmbedding is { IsSelfHosted: false };
private bool CanChangeSourceAndEmbedding => !this.IsEditing || !this.LockSourceAndEmbedding;
private bool CanChangeSource => !this.IsEditing || !this.LockSource;
private IEnumerable<ConfigurationSelectData<ConfidenceLevel>> ConfidenceLevels => ConfigurationSelectDataFactory.GetDataSourceConfidenceLevelsData();
private string SelectedEmbeddingNameText
{
get
{
var selectedEmbedding = this.AvailableEmbeddings.FirstOrDefault(x => x.Value == this.dataEmbeddingId);
return string.IsNullOrWhiteSpace(selectedEmbedding.Name) ? T("Unknown") : selectedEmbedding.Name;
}
}
private string SelectedEmbeddingTokenizerText => this.SelectedEmbedding is null
? T("No embedding selected")
: string.IsNullOrWhiteSpace(this.SelectedEmbedding.TokenizerPath)
@ -162,8 +160,11 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
Name = this.dataName,
Description = this.dataDescription,
Type = DataSourceType.LOCAL_FILE,
EmbeddingId = this.CanChangeSourceAndEmbedding ? this.dataEmbeddingId : this.DataSource.EmbeddingId,
FilePath = this.CanChangeSourceAndEmbedding ? this.dataFilePath : this.DataSource.FilePath,
EmbeddingId = this.dataEmbeddingId,
// Kept out of reach of the form while the source is locked, so a stale field cannot point an
// indexed data source somewhere else:
FilePath = this.CanChangeSource ? this.dataFilePath : this.DataSource.FilePath,
MaxChunkTokenLength = this.dataMaxChunkTokenLength,
ChunkOverlapTokenLength = this.dataChunkOverlapTokenLength,
ConfidenceLevel = this.dataConfidenceLevel,

22
app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs
View File

@ -199,12 +199,30 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
this.CanRefreshDataSource(dataSource);
}
public async Task<bool> ShouldLockDataSourceIdentityAsync(string dataSourceId, CancellationToken token = default)
/// <summary>
/// Whether the file or folder a data source reads must stay as it is.
/// </summary>
/// <remarks>
/// Locked as soon as the index holds anything, because where a data source reads from is what it
/// is: another folder is another data source, and the path reaches no signature, so swapping it
/// would leave the stored index describing documents nobody points at any more.
///
/// The embedding provider used to be locked along with it and no longer is. It does reach the
/// signature, so changing it rebuilds the index cleanly -- and DataSourceReindexWarning asks
/// before it does. Locking it as well left a data source whose provider was deleted stuck on
/// keyword search for good, with no way back.
///
/// Unclear counts as locked: an unavailable index database says nothing about what is stored.
/// </remarks>
/// <param name="dataSourceId">The data source to ask about.</param>
/// <param name="token">The cancellation token.</param>
/// <returns>True when the source must not be changed.</returns>
public async Task<bool> ShouldLockDataSourceOriginAsync(string dataSourceId, CancellationToken token = default)
{
var indexStore = await databaseClientProvider.GetIndexStoreAsync(token);
if (!indexStore.IsAvailable)
{
logger.LogWarning("Locking identity settings for data source '{DataSourceId}' because the local RAG index database '{DatabaseName}' is unavailable.", dataSourceId, indexStore.Name);
logger.LogWarning("Locking the source of data source '{DataSourceId}' because the local RAG index database '{DatabaseName}' is unavailable.", dataSourceId, indexStore.Name);
return true;
}