diff --git a/app/MindWork AI Studio/Components/DataSourceManagement.razor.cs b/app/MindWork AI Studio/Components/DataSourceManagement.razor.cs index 47461d8f..2aaacbb7 100644 --- a/app/MindWork AI Studio/Components/DataSourceManagement.razor.cs +++ b/app/MindWork AI Studio/Components/DataSourceManagement.razor.cs @@ -335,8 +335,8 @@ public partial class DataSourceManagement : MSGComponentBase return; IDataSource? editedDataSource = null; - var lockDataSourceIdentity = dataSource is IInternalDataSource - && await this.DataSourceEmbeddingService.ShouldLockDataSourceIdentityAsync(dataSource.Id); + var lockDataSourceOrigin = dataSource is IInternalDataSource + && await this.DataSourceEmbeddingService.ShouldLockDataSourceOriginAsync(dataSource.Id); switch (dataSource) { case DataSourceLocalFile localFile: @@ -344,7 +344,7 @@ public partial class DataSourceManagement : MSGComponentBase { { x => x.IsEditing, true }, { x => x.DataSource, localFile }, - { x => x.LockSourceAndEmbedding, lockDataSourceIdentity }, + { x => x.LockSource, lockDataSourceOrigin }, { x => x.AvailableEmbeddings, this.availableEmbeddingProviders } }; @@ -361,7 +361,7 @@ public partial class DataSourceManagement : MSGComponentBase { { x => x.IsEditing, true }, { x => x.DataSource, localDirectory }, - { x => x.LockSourceAndEmbedding, lockDataSourceIdentity }, + { x => x.LockSource, lockDataSourceOrigin }, { x => x.AvailableEmbeddings, this.availableEmbeddingProviders } }; diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor index ad4f7c0f..600fd819 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor @@ -45,13 +45,13 @@ @T("Select a root directory for this data source. All data in this directory and all its subdirectories will be processed for this data source.") - @if (!this.CanChangeSourceAndEmbedding) + @if (!this.CanChangeSource) { - @T("This data source already has indexed embeddings. Delete and recreate it to change the folder path or embedding model.") + @T("The documents of this data source are already prepared, so its folder cannot be changed. Another folder holds other documents, which makes it another data source: please add one for it. The embedding method below can be changed.") } - @if (this.CanChangeSourceAndEmbedding) + @if (this.CanChangeSource) { } @@ -74,38 +74,21 @@ @T("In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method.") - @if (this.CanChangeSourceAndEmbedding) - { - - @foreach (var embedding in this.AvailableEmbeddings) - { - - @if (this.GetEmbeddingProvider(embedding.Value) is { } provider) - { - - } - else - { - @embedding.Name - } - - } - - } - else - { - - } + + @foreach (var embedding in this.AvailableEmbeddings) + { + + @if (this.GetEmbeddingProvider(embedding.Value) is { } provider) + { + + } + else + { + @embedding.Name + } + + } + @if (!string.IsNullOrWhiteSpace(this.dataEmbeddingId)) { diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs index 56c6c7d5..ebd4a374 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs @@ -20,8 +20,15 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase [Parameter] public DataSourceLocalDirectory DataSource { get; set; } + /// + /// Whether the folder this data source reads must stay as it is. + /// + /// + /// Set once the index holds something for this data source. The embedding is not locked along + /// with it: it can be changed, and DataSourceReindexWarning asks what that costs. + /// [Parameter] - public bool LockSourceAndEmbedding { get; set; } + public bool LockSource { get; set; } [Parameter] public IReadOnlyList> AvailableEmbeddings { get; set; } = []; @@ -126,19 +133,10 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase private bool SelectedCloudEmbedding => this.SelectedEmbedding is { IsSelfHosted: false }; - private bool CanChangeSourceAndEmbedding => !this.IsEditing || !this.LockSourceAndEmbedding; + private bool CanChangeSource => !this.IsEditing || !this.LockSource; private IEnumerable> ConfidenceLevels => ConfigurationSelectDataFactory.GetDataSourceConfidenceLevelsData(); - private string SelectedEmbeddingNameText - { - get - { - var selectedEmbedding = this.AvailableEmbeddings.FirstOrDefault(x => x.Value == this.dataEmbeddingId); - return string.IsNullOrWhiteSpace(selectedEmbedding.Name) ? T("Unknown") : selectedEmbedding.Name; - } - } - private string SelectedEmbeddingTokenizerText => this.SelectedEmbedding is null ? T("No embedding selected") : string.IsNullOrWhiteSpace(this.SelectedEmbedding.TokenizerPath) @@ -162,8 +160,11 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase Name = this.dataName, Description = this.dataDescription, Type = DataSourceType.LOCAL_DIRECTORY, - EmbeddingId = this.CanChangeSourceAndEmbedding ? this.dataEmbeddingId : this.DataSource.EmbeddingId, - Path = this.CanChangeSourceAndEmbedding ? this.dataPath : this.DataSource.Path, + EmbeddingId = this.dataEmbeddingId, + + // Kept out of reach of the form while the source is locked, so a stale field cannot point an + // indexed data source somewhere else: + Path = this.CanChangeSource ? this.dataPath : this.DataSource.Path, MaxChunkTokenLength = this.dataMaxChunkTokenLength, ChunkOverlapTokenLength = this.dataChunkOverlapTokenLength, ConfidenceLevel = this.dataConfidenceLevel, diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor index 3a94c100..6d26a3d4 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor @@ -45,13 +45,13 @@ @T("Select a file for this data source. The content of this file will be processed for the data source.") - @if (!this.CanChangeSourceAndEmbedding) + @if (!this.CanChangeSource) { - @T("This data source already has indexed embeddings. Delete and recreate it to change the file path or embedding model.") + @T("The documents of this data source are already prepared, so its file cannot be changed. Another file holds other content, which makes it another data source: please add one for it. The embedding method below can be changed.") } - @if (this.CanChangeSourceAndEmbedding) + @if (this.CanChangeSource) { } @@ -74,38 +74,21 @@ @T("In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method.") - @if (this.CanChangeSourceAndEmbedding) - { - - @foreach (var embedding in this.AvailableEmbeddings) - { - - @if (this.GetEmbeddingProvider(embedding.Value) is { } provider) - { - - } - else - { - @embedding.Name - } - - } - - } - else - { - - } + + @foreach (var embedding in this.AvailableEmbeddings) + { + + @if (this.GetEmbeddingProvider(embedding.Value) is { } provider) + { + + } + else + { + @embedding.Name + } + + } + @if (!string.IsNullOrWhiteSpace(this.dataEmbeddingId)) { diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs index 278db390..cdbe26ab 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs @@ -20,8 +20,15 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase [Parameter] public DataSourceLocalFile DataSource { get; set; } + /// + /// Whether the file this data source reads must stay as it is. + /// + /// + /// Set once the index holds something for this data source. The embedding is not locked along + /// with it: it can be changed, and DataSourceReindexWarning asks what that costs. + /// [Parameter] - public bool LockSourceAndEmbedding { get; set; } + public bool LockSource { get; set; } [Parameter] public IReadOnlyList> AvailableEmbeddings { get; set; } = []; @@ -126,19 +133,10 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase private bool SelectedCloudEmbedding => this.SelectedEmbedding is { IsSelfHosted: false }; - private bool CanChangeSourceAndEmbedding => !this.IsEditing || !this.LockSourceAndEmbedding; + private bool CanChangeSource => !this.IsEditing || !this.LockSource; private IEnumerable> ConfidenceLevels => ConfigurationSelectDataFactory.GetDataSourceConfidenceLevelsData(); - private string SelectedEmbeddingNameText - { - get - { - var selectedEmbedding = this.AvailableEmbeddings.FirstOrDefault(x => x.Value == this.dataEmbeddingId); - return string.IsNullOrWhiteSpace(selectedEmbedding.Name) ? T("Unknown") : selectedEmbedding.Name; - } - } - private string SelectedEmbeddingTokenizerText => this.SelectedEmbedding is null ? T("No embedding selected") : string.IsNullOrWhiteSpace(this.SelectedEmbedding.TokenizerPath) @@ -162,8 +160,11 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase Name = this.dataName, Description = this.dataDescription, Type = DataSourceType.LOCAL_FILE, - EmbeddingId = this.CanChangeSourceAndEmbedding ? this.dataEmbeddingId : this.DataSource.EmbeddingId, - FilePath = this.CanChangeSourceAndEmbedding ? this.dataFilePath : this.DataSource.FilePath, + EmbeddingId = this.dataEmbeddingId, + + // Kept out of reach of the form while the source is locked, so a stale field cannot point an + // indexed data source somewhere else: + FilePath = this.CanChangeSource ? this.dataFilePath : this.DataSource.FilePath, MaxChunkTokenLength = this.dataMaxChunkTokenLength, ChunkOverlapTokenLength = this.dataChunkOverlapTokenLength, ConfidenceLevel = this.dataConfidenceLevel, diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs index ccf326af..642ef9a1 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs @@ -199,12 +199,30 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM this.CanRefreshDataSource(dataSource); } - public async Task ShouldLockDataSourceIdentityAsync(string dataSourceId, CancellationToken token = default) + /// + /// Whether the file or folder a data source reads must stay as it is. + /// + /// + /// Locked as soon as the index holds anything, because where a data source reads from is what it + /// is: another folder is another data source, and the path reaches no signature, so swapping it + /// would leave the stored index describing documents nobody points at any more. + /// + /// The embedding provider used to be locked along with it and no longer is. It does reach the + /// signature, so changing it rebuilds the index cleanly -- and DataSourceReindexWarning asks + /// before it does. Locking it as well left a data source whose provider was deleted stuck on + /// keyword search for good, with no way back. + /// + /// Unclear counts as locked: an unavailable index database says nothing about what is stored. + /// + /// The data source to ask about. + /// The cancellation token. + /// True when the source must not be changed. + public async Task ShouldLockDataSourceOriginAsync(string dataSourceId, CancellationToken token = default) { var indexStore = await databaseClientProvider.GetIndexStoreAsync(token); if (!indexStore.IsAvailable) { - logger.LogWarning("Locking identity settings for data source '{DataSourceId}' because the local RAG index database '{DatabaseName}' is unavailable.", dataSourceId, indexStore.Name); + logger.LogWarning("Locking the source of data source '{DataSourceId}' because the local RAG index database '{DatabaseName}' is unavailable.", dataSourceId, indexStore.Name); return true; }