Ask before local data sources are indexed again (#983)

app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor

@@ -45,13 +45,13 @@
             <MudJustifiedText Typo="Typo.body1" Class="mb-3">
                 @T("Select a root directory for this data source. All data in this directory and all its subdirectories will be processed for this data source.")
             </MudJustifiedText>
-            @if (!this.CanChangeSourceAndEmbedding)
+            @if (!this.CanChangeSource)
             {
                 <MudAlert Severity="Severity.Info" Variant="Variant.Outlined" Class="mb-3">
-                    @T("This data source already has indexed embeddings. Delete and recreate it to change the folder path or embedding model.")
+                    @T("The documents of this data source are already prepared, so its folder cannot be changed. Another folder holds other documents, which makes it another data source: please add one for it. The embedding method below can be changed.")
                 </MudAlert>
             }
-            @if (this.CanChangeSourceAndEmbedding)
+            @if (this.CanChangeSource)
             {
                 <SelectDirectory @bind-Directory="@this.dataPath" Label="@T("Selected base directory for this data source")" DirectoryDialogTitle="@T("Select the base directory")" Validation="@this.dataSourceValidation.ValidatePath" EnableDragDrop="true" CatchAllDocuments="true" />
             }
@@ -74,38 +74,21 @@
             <MudJustifiedText Typo="Typo.body1" Class="mb-3">
                 @T("In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method.")
             </MudJustifiedText>
-            @if (this.CanChangeSourceAndEmbedding)
-            {
-                <MudSelect @bind-Value="@this.dataEmbeddingId" Label="@T("Embedding")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateEmbeddingProviderAccess">
-                    @foreach (var embedding in this.AvailableEmbeddings)
-                    {
-                        <MudSelectItem Value="@embedding.Value">
-                            @if (this.GetEmbeddingProvider(embedding.Value) is { } provider)
-                            {
-                                <ProviderLabel ProviderType="@provider.UsedLLMProvider" CustomIconDataUrl="@provider.CustomIconDataUrl" Text="@embedding.Name" />
-                            }
-                            else
-                            {
-                                @embedding.Name
-                            }
-                        </MudSelectItem>
-                    }
-                </MudSelect>
-            }
-            else
-            {
-                <MudTextField
-                    T="string"
-                    Text="@this.SelectedEmbeddingNameText"
-                    Label="@T("Embedding")"
-                    Class="mb-3"
-                    ReadOnly="@true"
-                    Adornment="Adornment.Start"
-                    AdornmentIcon="@Icons.Material.Filled.DataObject"
-                    AdornmentColor="Color.Info"
-                    Variant="Variant.Outlined"
-                />
-            }
+            <MudSelect @bind-Value="@this.dataEmbeddingId" Label="@T("Embedding")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateEmbeddingProviderAccess">
+                @foreach (var embedding in this.AvailableEmbeddings)
+                {
+                    <MudSelectItem Value="@embedding.Value">
+                        @if (this.GetEmbeddingProvider(embedding.Value) is { } provider)
+                        {
+                            <ProviderLabel ProviderType="@provider.UsedLLMProvider" CustomIconDataUrl="@provider.CustomIconDataUrl" Text="@embedding.Name" />
+                        }
+                        else
+                        {
+                            @embedding.Name
+                        }
+                    </MudSelectItem>
+                }
+            </MudSelect>
             
             @if (!string.IsNullOrWhiteSpace(this.dataEmbeddingId))
             {

app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs

@@ -20,12 +20,25 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
     [Parameter]
     public DataSourceLocalDirectory DataSource { get; set; }
 
+    /// <summary>
+    /// Whether the folder this data source reads must stay as it is.
+    /// </summary>
+    /// <remarks>
+    /// Set once the index holds something for this data source. The embedding is not locked along
+    /// with it: it can be changed, and DataSourceReindexWarning asks what that costs.
+    /// </remarks>
     [Parameter]
-    public bool LockSourceAndEmbedding { get; set; }
+    public bool LockSource { get; set; }
 
     [Parameter]
     public IReadOnlyList<ConfigurationSelectData<string>> AvailableEmbeddings { get; set; } = [];
-    
+
+    [Inject]
+    private IDialogService DialogService { get; init; } = null!;
+
+    [Inject]
+    private DataSourceEmbeddingService DataSourceEmbeddingService { get; init; } = null!;
+
     private static readonly Dictionary<string, object?> SPELLCHECK_ATTRIBUTES = new();
     
     private readonly DataSourceValidation dataSourceValidation;
@@ -120,19 +133,10 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
 
     private bool SelectedCloudEmbedding => this.SelectedEmbedding is { IsSelfHosted: false };
 
-    private bool CanChangeSourceAndEmbedding => !this.IsEditing || !this.LockSourceAndEmbedding;
+    private bool CanChangeSource => !this.IsEditing || !this.LockSource;
 
     private IEnumerable<ConfigurationSelectData<ConfidenceLevel>> ConfidenceLevels => ConfigurationSelectDataFactory.GetDataSourceConfidenceLevelsData();
 
-    private string SelectedEmbeddingNameText
-    {
-        get
-        {
-            var selectedEmbedding = this.AvailableEmbeddings.FirstOrDefault(x => x.Value == this.dataEmbeddingId);
-            return string.IsNullOrWhiteSpace(selectedEmbedding.Name) ? T("Unknown") : selectedEmbedding.Name;
-        }
-    }
-
     private string SelectedEmbeddingTokenizerText => this.SelectedEmbedding is null
         ? T("No embedding selected")
         : string.IsNullOrWhiteSpace(this.SelectedEmbedding.TokenizerPath)
@@ -156,8 +160,11 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
         Name = this.dataName,
         Description = this.dataDescription,
         Type = DataSourceType.LOCAL_DIRECTORY,
-        EmbeddingId = this.CanChangeSourceAndEmbedding ? this.dataEmbeddingId : this.DataSource.EmbeddingId,
-        Path = this.CanChangeSourceAndEmbedding ? this.dataPath : this.DataSource.Path,
+        EmbeddingId = this.dataEmbeddingId,
+
+        // Kept out of reach of the form while the source is locked, so a stale field cannot point an
+        // indexed data source somewhere else:
+        Path = this.CanChangeSource ? this.dataPath : this.DataSource.Path,
         MaxChunkTokenLength = this.dataMaxChunkTokenLength,
         ChunkOverlapTokenLength = this.dataChunkOverlapTokenLength,
         ConfidenceLevel = this.dataConfidenceLevel,
@@ -171,8 +178,20 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
         // When the data is not valid, we don't store it:
         if (!this.dataIsValid)
             return;
-        
+
         var addedDataSource = this.CreateDataSource();
+
+        //
+        // Ask while the dialog is still open, so a token limit which would have cost the prepared
+        // documents can be corrected right away. Asking in DataSourceManagement instead would have
+        // to be written once per data source kind, and by then the numbers are out of reach.
+        //
+        // Only when editing: while adding, DataSource is still default -- both local data sources
+        // are record structs -- and nothing has been prepared for a source which does not exist yet.
+        //
+        if (this.IsEditing && !await DataSourceReindexWarning.ConfirmDataSourceChangeAsync(this.DialogService, this.SettingsManager, this.DataSourceEmbeddingService, this.DataSource, addedDataSource))
+            return;
+
         this.MudDialog.Close(DialogResult.Ok(addedDataSource));
     }
     

app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor

@@ -45,13 +45,13 @@
             <MudJustifiedText Typo="Typo.body1" Class="mb-3">
                 @T("Select a file for this data source. The content of this file will be processed for the data source.")
             </MudJustifiedText>
-            @if (!this.CanChangeSourceAndEmbedding)
+            @if (!this.CanChangeSource)
             {
                 <MudAlert Severity="Severity.Info" Variant="Variant.Outlined" Class="mb-3">
-                    @T("This data source already has indexed embeddings. Delete and recreate it to change the file path or embedding model.")
+                    @T("The documents of this data source are already prepared, so its file cannot be changed. Another file holds other content, which makes it another data source: please add one for it. The embedding method below can be changed.")
                 </MudAlert>
             }
-            @if (this.CanChangeSourceAndEmbedding)
+            @if (this.CanChangeSource)
             {
                 <SelectFile @bind-File="@this.dataFilePath" Label="@T("Selected file path for this data source")" FileDialogTitle="@T("Select the file")" Validation="@this.dataSourceValidation.ValidateFilePath" EnableDragDrop="true" CatchAllDocuments="true" />
             }
@@ -74,38 +74,21 @@
             <MudJustifiedText Typo="Typo.body1" Class="mb-3">
                 @T("In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method.")
             </MudJustifiedText>
-            @if (this.CanChangeSourceAndEmbedding)
-            {
-                <MudSelect @bind-Value="@this.dataEmbeddingId" Label="@T("Embedding")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateEmbeddingProviderAccess">
-                    @foreach (var embedding in this.AvailableEmbeddings)
-                    {
-                        <MudSelectItem Value="@embedding.Value">
-                            @if (this.GetEmbeddingProvider(embedding.Value) is { } provider)
-                            {
-                                <ProviderLabel ProviderType="@provider.UsedLLMProvider" CustomIconDataUrl="@provider.CustomIconDataUrl" Text="@embedding.Name" />
-                            }
-                            else
-                            {
-                                @embedding.Name
-                            }
-                        </MudSelectItem>
-                    }
-                </MudSelect>
-            }
-            else
-            {
-                <MudTextField
-                    T="string"
-                    Text="@this.SelectedEmbeddingNameText"
-                    Label="@T("Embedding")"
-                    Class="mb-3"
-                    ReadOnly="@true"
-                    Adornment="Adornment.Start"
-                    AdornmentIcon="@Icons.Material.Filled.DataObject"
-                    AdornmentColor="Color.Info"
-                    Variant="Variant.Outlined"
-                />
-            }
+            <MudSelect @bind-Value="@this.dataEmbeddingId" Label="@T("Embedding")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateEmbeddingProviderAccess">
+                @foreach (var embedding in this.AvailableEmbeddings)
+                {
+                    <MudSelectItem Value="@embedding.Value">
+                        @if (this.GetEmbeddingProvider(embedding.Value) is { } provider)
+                        {
+                            <ProviderLabel ProviderType="@provider.UsedLLMProvider" CustomIconDataUrl="@provider.CustomIconDataUrl" Text="@embedding.Name" />
+                        }
+                        else
+                        {
+                            @embedding.Name
+                        }
+                    </MudSelectItem>
+                }
+            </MudSelect>
             
             @if (!string.IsNullOrWhiteSpace(this.dataEmbeddingId))
             {

app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs

@@ -20,12 +20,25 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
     [Parameter]
     public DataSourceLocalFile DataSource { get; set; }
 
+    /// <summary>
+    /// Whether the file this data source reads must stay as it is.
+    /// </summary>
+    /// <remarks>
+    /// Set once the index holds something for this data source. The embedding is not locked along
+    /// with it: it can be changed, and DataSourceReindexWarning asks what that costs.
+    /// </remarks>
     [Parameter]
-    public bool LockSourceAndEmbedding { get; set; }
+    public bool LockSource { get; set; }
     
     [Parameter]
     public IReadOnlyList<ConfigurationSelectData<string>> AvailableEmbeddings { get; set; } = [];
-    
+
+    [Inject]
+    private IDialogService DialogService { get; init; } = null!;
+
+    [Inject]
+    private DataSourceEmbeddingService DataSourceEmbeddingService { get; init; } = null!;
+
     private static readonly Dictionary<string, object?> SPELLCHECK_ATTRIBUTES = new();
     
     private readonly DataSourceValidation dataSourceValidation;
@@ -120,19 +133,10 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
 
     private bool SelectedCloudEmbedding => this.SelectedEmbedding is { IsSelfHosted: false };
 
-    private bool CanChangeSourceAndEmbedding => !this.IsEditing || !this.LockSourceAndEmbedding;
+    private bool CanChangeSource => !this.IsEditing || !this.LockSource;
 
     private IEnumerable<ConfigurationSelectData<ConfidenceLevel>> ConfidenceLevels => ConfigurationSelectDataFactory.GetDataSourceConfidenceLevelsData();
 
-    private string SelectedEmbeddingNameText
-    {
-        get
-        {
-            var selectedEmbedding = this.AvailableEmbeddings.FirstOrDefault(x => x.Value == this.dataEmbeddingId);
-            return string.IsNullOrWhiteSpace(selectedEmbedding.Name) ? T("Unknown") : selectedEmbedding.Name;
-        }
-    }
-
     private string SelectedEmbeddingTokenizerText => this.SelectedEmbedding is null
         ? T("No embedding selected")
         : string.IsNullOrWhiteSpace(this.SelectedEmbedding.TokenizerPath)
@@ -156,8 +160,11 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
         Name = this.dataName,
         Description = this.dataDescription,
         Type = DataSourceType.LOCAL_FILE,
-        EmbeddingId = this.CanChangeSourceAndEmbedding ? this.dataEmbeddingId : this.DataSource.EmbeddingId,
-        FilePath = this.CanChangeSourceAndEmbedding ? this.dataFilePath : this.DataSource.FilePath,
+        EmbeddingId = this.dataEmbeddingId,
+
+        // Kept out of reach of the form while the source is locked, so a stale field cannot point an
+        // indexed data source somewhere else:
+        FilePath = this.CanChangeSource ? this.dataFilePath : this.DataSource.FilePath,
         MaxChunkTokenLength = this.dataMaxChunkTokenLength,
         ChunkOverlapTokenLength = this.dataChunkOverlapTokenLength,
         ConfidenceLevel = this.dataConfidenceLevel,
@@ -171,8 +178,20 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
         // When the data is not valid, we don't store it:
         if (!this.dataIsValid)
             return;
-        
+
         var addedDataSource = this.CreateDataSource();
+
+        //
+        // Ask while the dialog is still open, so a token limit which would have cost the prepared
+        // documents can be corrected right away. Asking in DataSourceManagement instead would have
+        // to be written once per data source kind, and by then the numbers are out of reach.
+        //
+        // Only when editing: while adding, DataSource is still default -- both local data sources
+        // are record structs -- and nothing has been prepared for a source which does not exist yet.
+        //
+        if (this.IsEditing && !await DataSourceReindexWarning.ConfirmDataSourceChangeAsync(this.DialogService, this.SettingsManager, this.DataSourceEmbeddingService, this.DataSource, addedDataSource))
+            return;
+
         this.MudDialog.Close(DialogResult.Ok(addedDataSource));
     }
     

app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs

@@ -86,6 +86,17 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
     [Parameter]
     public string DataTokenizerPath { get; set; } = string.Empty;
 
+    /// <summary>
+    /// The fingerprint of the tokenizer this provider was stored with.
+    /// </summary>
+    /// <remarks>
+    /// Carried through the dialog untouched as long as the user leaves the tokenizer alone. Rebuilding
+    /// it from the path on every open would read a file for nothing, and an unreadable one would look
+    /// like another tokenizer and cost every data source of this provider its index.
+    /// </remarks>
+    [Parameter]
+    public string DataTokenizerFingerprint { get; set; } = string.Empty;
+
     [Parameter]
     public int DataTokenLimit { get; set; } = EmbeddingProvider.DEFAULT_TOKEN_LIMIT;
 
@@ -105,6 +116,12 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
     [Inject]
     private ILogger<EmbeddingProviderDialog> Logger { get; init; } = null!;
 
+    [Inject]
+    private IDialogService DialogService { get; init; } = null!;
+
+    [Inject]
+    private DataSourceEmbeddingService DataSourceEmbeddingService { get; init; } = null!;
+
     private static readonly Dictionary<string, object?> SPELLCHECK_ATTRIBUTES = new();
 
     /// <summary>
@@ -121,6 +138,7 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
     private string dataEditingPreviousInstanceName = string.Empty;
     private string dataLoadingModelsIssue = string.Empty;
     private string dataFilePath = string.Empty;
+    private string dataTokenizerFingerprint = string.Empty;
     private string dataCustomTokenizerValidationIssue = string.Empty;
     private Task dataTokenizerValidationTask = Task.CompletedTask;
     private bool dataStoreWasAttempted;
@@ -176,6 +194,7 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
             IsEnterpriseConfiguration = this.IsEnterpriseConfiguration,
             EnterpriseConfigurationPluginId = Guid.Empty,
             TokenizerPath = this.dataFilePath,
+            TokenizerFingerprint = this.dataTokenizerFingerprint,
             EmbeddingBatchSize = this.DataEmbeddingBatchSize,
             TokenLimit = this.DataTokenLimit,
             CustomIconDataUrl = this.DataCustomIconDataUrl,
@@ -201,6 +220,7 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
         {
             this.dataEditingPreviousInstanceName = this.DataName.ToLowerInvariant();
             this.dataFilePath = this.DataTokenizerPath;
+            this.dataTokenizerFingerprint = this.DataTokenizerFingerprint;
             this.showExpertSettings = !string.IsNullOrWhiteSpace(this.DataTokenizerPath)
                                       || this.DataTokenLimit != EmbeddingProvider.DEFAULT_TOKEN_LIMIT
                                       || this.DataEmbeddingBatchSize != EmbeddingProvider.DEFAULT_EMBEDDING_BATCH_SIZE;
@@ -279,7 +299,22 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
         // When the data is not valid, we don't store it:
         if (!this.dataIsValid)
             return;
-        
+
+        //
+        // Ask before anything is written. Storing a tokenizer deletes the previous one before it
+        // copies, and the API key goes into the OS keyring right after, so asking any later would
+        // leave those changes behind even when the user says no. Saying no also keeps this dialog
+        // open, which is the point: the value which would have cost the index can be corrected
+        // right away.
+        //
+        // Enterprise-managed providers are left out. Every field which reaches the embedding
+        // signature is locked for them, and their data sources are not queued for indexing either.
+        //
+        if (this.IsEditing && !this.IsEnterpriseConfiguration && !await DataSourceReindexWarning.ConfirmEmbeddingProviderChangeAsync(
+                this.DialogService, this.SettingsManager, this.DataSourceEmbeddingService,
+                this.SettingsManager.GetEmbeddingProviderById(this.DataId), this.CreateEmbeddingProviderSettings()))
+            return;
+
         var response = await this.StoreOrDeleteTokenizerAsync();
         if (!response.Success)
         {
@@ -413,9 +448,18 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
         this.dataTokenizerValidationTask = this.ValidateCustomTokenizer(filePath, validationRevision);
         await this.dataTokenizerValidationTask;
 
+        //
+        // The embedding signature carries the tokenizer's content, so it has to be read while we have
+        // the file the user just picked. Reading it here rather than while storing also keeps a large
+        // file off that path, where it would stall the circuit.
+        //
+        var tokenizerFingerprint = await TokenizerFingerprint.ForFileAsync(filePath);
+
         if (validationRevision != this.dataTokenizerValidationRevision)
             return;
 
+        this.dataTokenizerFingerprint = tokenizerFingerprint;
+
         if (this.dataStoreWasAttempted)
             await this.form.Validate();
         else