Ask before local data sources are indexed again (#983)

AGENTS.md

@@ -344,3 +344,12 @@ following words:
 - Upgraded
 
 The entire changelog is sorted by these categories in the order shown above. The language used for the changelog is US English.
+
+**Every entry has to stand on its own.** Never refer back to another entry, neither by wording such
+as "the same question", "that dialog", or "as described above", nor by relying on one read just
+before it. Readers pick out the entries which concern them; an entry which only makes sense after
+reading its neighbors turns the changelog into something nobody reads at all. Name the context
+inside the entry instead, even when that repeats a few words from another one.
+
+**Split a topic into several short entries** rather than growing a single long one, and address the
+reader with "you".

app/MindWork AI Studio/Assistants/I18N/allTexts.lua

@@ -6088,9 +6088,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1725856265"
 -- How many matches do you want at most per query?
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1827669611"] = "How many matches do you want at most per query?"
 
--- This data source already has indexed embeddings. Delete and recreate it to change the folder path or embedding model.
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1845247453"] = "This data source already has indexed embeddings. Delete and recreate it to change the folder path or embedding model."
-
 -- Update
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1847791252"] = "Update"
 
@@ -6127,9 +6124,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3359366900"
 -- Show Expert Settings
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3361153305"] = "Show Expert Settings"
 
--- Unknown
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3424652889"] = "Unknown"
-
 -- Select the base directory
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T562479068"] = "Select the base directory"
 
@@ -6139,6 +6133,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T787118522"]
 -- Data Source Name
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T813773421"] = "Data Source Name"
 
+-- The documents of this data source are already prepared, so its folder cannot be changed. Another folder holds other documents, which makes it another data source: please add one for it. The embedding method below can be changed.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T870152265"] = "The documents of this data source are already prepared, so its folder cannot be changed. Another folder holds other documents, which makes it another data source: please add one for it. The embedding method below can be changed."
+
 -- Cancel
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T900713019"] = "Cancel"
 
@@ -6286,15 +6283,12 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3359366900"] = "
 -- Show Expert Settings
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3361153305"] = "Show Expert Settings"
 
--- Unknown
+-- The documents of this data source are already prepared, so its file cannot be changed. Another file holds other content, which makes it another data source: please add one for it. The embedding method below can be changed.
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3424652889"] = "Unknown"
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3731767732"] = "The documents of this data source are already prepared, so its file cannot be changed. Another file holds other content, which makes it another data source: please add one for it. The embedding method below can be changed."
 
 -- Select the file
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3740148848"] = "Select the file"
 
--- This data source already has indexed embeddings. Delete and recreate it to change the file path or embedding model.
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T458267883"] = "This data source already has indexed embeddings. Delete and recreate it to change the file path or embedding model."
-
 -- The data source token limit must not be larger than the embedding provider token limit ({0}).
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T787118522"] = "The data source token limit must not be larger than the embedding provider token limit ({0})."
 
@@ -10822,6 +10816,30 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::QDRANTEDGECLIENTIMPLEM
 -- Qdrant Edge is not available.
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::QDRANTEDGECLIENTIMPLEMENTATION::T744445696"] = "Qdrant Edge is not available."
 
+-- They keep answering keyword searches, but searching them by meaning stops working, and no further documents can be prepared for them. The ones which are already prepared stay tied to this provider as well, so you cannot simply move them to another one.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T2343773457"] = "They keep answering keyword searches, but searching them by meaning stops working, and no further documents can be prepared for them. The ones which are already prepared stay tied to this provider as well, so you cannot simply move them to another one."
+
+-- and {0} more.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T2519847121"] = "and {0} more."
+
+-- This change makes the prepared documents of the following data sources unusable ({0}):
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T3337378891"] = "This change makes the prepared documents of the following data sources unusable ({0}):"
+
+-- Do you want to apply this change anyway?
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T3419411838"] = "Do you want to apply this change anyway?"
+
+-- Documents Will Be Prepared Again
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T737291513"] = "Documents Will Be Prepared Again"
+
+-- Your embedding provider runs in the cloud, so preparing everything again costs money.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T774305382"] = "Your embedding provider runs in the cloud, so preparing everything again costs money."
+
+-- These data sources are set up with this embedding provider ({0}):
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T858000918"] = "These data sources are set up with this embedding provider ({0}):"
+
+-- Everything prepared for them is thrown away, and every one of their documents goes to your embedding provider once more. With a large data source, this takes a while.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T874850580"] = "Everything prepared for them is thrown away, and every one of their documents goes to your embedding provider once more. With a large data source, this takes a while."
+
 -- Repair Data Source
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREPAIR::T4175865785"] = "Repair Data Source"
 

app/MindWork AI Studio/Components/DataSourceCloudEmbeddingWarning.razor

@@ -1,9 +1,9 @@
 @inherits MSGComponentBase
 
 <MudAlert Severity="Severity.Warning" Variant="Variant.Outlined" Dense="@true" Class="mb-3">
-    <MudText Typo="Typo.body2">
+    <MudJustifiedText Typo="Typo.body2">
         @this.WarningText
-    </MudText>
+    </MudJustifiedText>
 </MudAlert>
 
 <MudTextSwitch Value="@this.UserAcknowledged" ValueChanged="@this.UserAcknowledgedChanged" Label="@T("I confirm that I have read and understood the above")" LabelOn="@T("Yes, please send my data to the external embedding provider")" LabelOff="@T("No, I will choose another embedding")" Validation="@this.Validation"/>

app/MindWork AI Studio/Components/DataSourceManagement.razor.cs

@@ -335,8 +335,8 @@ public partial class DataSourceManagement : MSGComponentBase
             return;
 
         IDataSource? editedDataSource = null;
-        var lockDataSourceIdentity = dataSource is IInternalDataSource
+        var lockDataSourceOrigin = dataSource is IInternalDataSource
-            && await this.DataSourceEmbeddingService.ShouldLockDataSourceIdentityAsync(dataSource.Id);
+            && await this.DataSourceEmbeddingService.ShouldLockDataSourceOriginAsync(dataSource.Id);
         switch (dataSource)
         {
             case DataSourceLocalFile localFile:
@@ -344,7 +344,7 @@ public partial class DataSourceManagement : MSGComponentBase
                 {
                     { x => x.IsEditing, true },
                     { x => x.DataSource, localFile },
-                    { x => x.LockSourceAndEmbedding, lockDataSourceIdentity },
+                    { x => x.LockSource, lockDataSourceOrigin },
                     { x => x.AvailableEmbeddings, this.availableEmbeddingProviders }
                 };
 
@@ -361,7 +361,7 @@ public partial class DataSourceManagement : MSGComponentBase
                 {
                     { x => x.IsEditing, true },
                     { x => x.DataSource, localDirectory },
-                    { x => x.LockSourceAndEmbedding, lockDataSourceIdentity },
+                    { x => x.LockSource, lockDataSourceOrigin },
                     { x => x.AvailableEmbeddings, this.availableEmbeddingProviders }
                 };
 

app/MindWork AI Studio/Components/Settings/SettingsPanelEmbeddings.razor.cs

@@ -94,6 +94,7 @@ public partial class SettingsPanelEmbeddings : SettingsPanelProviderBase
             { x => x.IsEditing, true },
             { x => x.DataHost, embeddingProvider.Host },
             { x => x.DataTokenizerPath, embeddingProvider.TokenizerPath },
+            { x => x.DataTokenizerFingerprint, embeddingProvider.TokenizerFingerprint },
             { x => x.DataTokenLimit, embeddingProvider.EffectiveTokenLimit },
             { x => x.DataEmbeddingBatchSize, embeddingProvider.EffectiveEmbeddingBatchSize },
             { x => x.HFInferenceProviderId, embeddingProvider.HFInferenceProvider },
@@ -132,10 +133,17 @@ public partial class SettingsPanelEmbeddings : SettingsPanelProviderBase
 
     private async Task DeleteEmbeddingProvider(EmbeddingProvider provider)
     {
-        var dialogParameters = new DialogParameters<ConfirmDialog>
+        var question = string.Format(T("Are you sure you want to delete the embedding provider '{0}'?"), provider.Name);
-        {
+        var affectedDataSources = DataSourceReindexWarning.DescribeDataSourcesLosingTheirProvider(this.SettingsManager, provider);
-            { x => x.Message, string.Format(T("Are you sure you want to delete the embedding provider '{0}'?"), provider.Name) },
+
-        };
+        //
+        // The names arrive as a Markdown list, so the question travels as Markdown as well as soon
+        // as there is something to name. With no data source behind the provider, the plain message
+        // stays what it always was:
+        //
+        var dialogParameters = string.IsNullOrEmpty(affectedDataSources)
+            ? new DialogParameters<ConfirmDialog> { { x => x.Message, question } }
+            : new DialogParameters<ConfirmDialog> { { x => x.MarkdownBody, $"{affectedDataSources}{Environment.NewLine}{question}" } };
 
         var dialogReference = await this.DialogService.ShowAsync<ConfirmDialog>(T("Delete Embedding Provider"), dialogParameters, DialogOptions.FULLSCREEN);
         var dialogResult = await dialogReference.Result;

app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor

@@ -45,13 +45,13 @@
             <MudJustifiedText Typo="Typo.body1" Class="mb-3">
                 @T("Select a root directory for this data source. All data in this directory and all its subdirectories will be processed for this data source.")
             </MudJustifiedText>
-            @if (!this.CanChangeSourceAndEmbedding)
+            @if (!this.CanChangeSource)
             {
                 <MudAlert Severity="Severity.Info" Variant="Variant.Outlined" Class="mb-3">
-                    @T("This data source already has indexed embeddings. Delete and recreate it to change the folder path or embedding model.")
+                    @T("The documents of this data source are already prepared, so its folder cannot be changed. Another folder holds other documents, which makes it another data source: please add one for it. The embedding method below can be changed.")
                 </MudAlert>
             }
-            @if (this.CanChangeSourceAndEmbedding)
+            @if (this.CanChangeSource)
             {
                 <SelectDirectory @bind-Directory="@this.dataPath" Label="@T("Selected base directory for this data source")" DirectoryDialogTitle="@T("Select the base directory")" Validation="@this.dataSourceValidation.ValidatePath" EnableDragDrop="true" CatchAllDocuments="true" />
             }
@@ -74,8 +74,6 @@
             <MudJustifiedText Typo="Typo.body1" Class="mb-3">
                 @T("In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method.")
             </MudJustifiedText>
-            @if (this.CanChangeSourceAndEmbedding)
-            {
             <MudSelect @bind-Value="@this.dataEmbeddingId" Label="@T("Embedding")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateEmbeddingProviderAccess">
                 @foreach (var embedding in this.AvailableEmbeddings)
                 {
@@ -91,21 +89,6 @@
                     </MudSelectItem>
                 }
             </MudSelect>
-            }
-            else
-            {
-                <MudTextField
-                    T="string"
-                    Text="@this.SelectedEmbeddingNameText"
-                    Label="@T("Embedding")"
-                    Class="mb-3"
-                    ReadOnly="@true"
-                    Adornment="Adornment.Start"
-                    AdornmentIcon="@Icons.Material.Filled.DataObject"
-                    AdornmentColor="Color.Info"
-                    Variant="Variant.Outlined"
-                />
-            }
             
             @if (!string.IsNullOrWhiteSpace(this.dataEmbeddingId))
             {

app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs

@@ -20,12 +20,25 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
     [Parameter]
     public DataSourceLocalDirectory DataSource { get; set; }
 
+    /// <summary>
+    /// Whether the folder this data source reads must stay as it is.
+    /// </summary>
+    /// <remarks>
+    /// Set once the index holds something for this data source. The embedding is not locked along
+    /// with it: it can be changed, and DataSourceReindexWarning asks what that costs.
+    /// </remarks>
     [Parameter]
-    public bool LockSourceAndEmbedding { get; set; }
+    public bool LockSource { get; set; }
 
     [Parameter]
     public IReadOnlyList<ConfigurationSelectData<string>> AvailableEmbeddings { get; set; } = [];
 
+    [Inject]
+    private IDialogService DialogService { get; init; } = null!;
+
+    [Inject]
+    private DataSourceEmbeddingService DataSourceEmbeddingService { get; init; } = null!;
+
     private static readonly Dictionary<string, object?> SPELLCHECK_ATTRIBUTES = new();
     
     private readonly DataSourceValidation dataSourceValidation;
@@ -120,19 +133,10 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
 
     private bool SelectedCloudEmbedding => this.SelectedEmbedding is { IsSelfHosted: false };
 
-    private bool CanChangeSourceAndEmbedding => !this.IsEditing || !this.LockSourceAndEmbedding;
+    private bool CanChangeSource => !this.IsEditing || !this.LockSource;
 
     private IEnumerable<ConfigurationSelectData<ConfidenceLevel>> ConfidenceLevels => ConfigurationSelectDataFactory.GetDataSourceConfidenceLevelsData();
 
-    private string SelectedEmbeddingNameText
-    {
-        get
-        {
-            var selectedEmbedding = this.AvailableEmbeddings.FirstOrDefault(x => x.Value == this.dataEmbeddingId);
-            return string.IsNullOrWhiteSpace(selectedEmbedding.Name) ? T("Unknown") : selectedEmbedding.Name;
-        }
-    }
-
     private string SelectedEmbeddingTokenizerText => this.SelectedEmbedding is null
         ? T("No embedding selected")
         : string.IsNullOrWhiteSpace(this.SelectedEmbedding.TokenizerPath)
@@ -156,8 +160,11 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
         Name = this.dataName,
         Description = this.dataDescription,
         Type = DataSourceType.LOCAL_DIRECTORY,
-        EmbeddingId = this.CanChangeSourceAndEmbedding ? this.dataEmbeddingId : this.DataSource.EmbeddingId,
+        EmbeddingId = this.dataEmbeddingId,
-        Path = this.CanChangeSourceAndEmbedding ? this.dataPath : this.DataSource.Path,
+
+        // Kept out of reach of the form while the source is locked, so a stale field cannot point an
+        // indexed data source somewhere else:
+        Path = this.CanChangeSource ? this.dataPath : this.DataSource.Path,
         MaxChunkTokenLength = this.dataMaxChunkTokenLength,
         ChunkOverlapTokenLength = this.dataChunkOverlapTokenLength,
         ConfidenceLevel = this.dataConfidenceLevel,
@@ -173,6 +180,18 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
             return;
 
         var addedDataSource = this.CreateDataSource();
+
+        //
+        // Ask while the dialog is still open, so a token limit which would have cost the prepared
+        // documents can be corrected right away. Asking in DataSourceManagement instead would have
+        // to be written once per data source kind, and by then the numbers are out of reach.
+        //
+        // Only when editing: while adding, DataSource is still default -- both local data sources
+        // are record structs -- and nothing has been prepared for a source which does not exist yet.
+        //
+        if (this.IsEditing && !await DataSourceReindexWarning.ConfirmDataSourceChangeAsync(this.DialogService, this.SettingsManager, this.DataSourceEmbeddingService, this.DataSource, addedDataSource))
+            return;
+
         this.MudDialog.Close(DialogResult.Ok(addedDataSource));
     }
     

app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor

@@ -45,13 +45,13 @@
             <MudJustifiedText Typo="Typo.body1" Class="mb-3">
                 @T("Select a file for this data source. The content of this file will be processed for the data source.")
             </MudJustifiedText>
-            @if (!this.CanChangeSourceAndEmbedding)
+            @if (!this.CanChangeSource)
             {
                 <MudAlert Severity="Severity.Info" Variant="Variant.Outlined" Class="mb-3">
-                    @T("This data source already has indexed embeddings. Delete and recreate it to change the file path or embedding model.")
+                    @T("The documents of this data source are already prepared, so its file cannot be changed. Another file holds other content, which makes it another data source: please add one for it. The embedding method below can be changed.")
                 </MudAlert>
             }
-            @if (this.CanChangeSourceAndEmbedding)
+            @if (this.CanChangeSource)
             {
                 <SelectFile @bind-File="@this.dataFilePath" Label="@T("Selected file path for this data source")" FileDialogTitle="@T("Select the file")" Validation="@this.dataSourceValidation.ValidateFilePath" EnableDragDrop="true" CatchAllDocuments="true" />
             }
@@ -74,8 +74,6 @@
             <MudJustifiedText Typo="Typo.body1" Class="mb-3">
                 @T("In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method.")
             </MudJustifiedText>
-            @if (this.CanChangeSourceAndEmbedding)
-            {
             <MudSelect @bind-Value="@this.dataEmbeddingId" Label="@T("Embedding")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateEmbeddingProviderAccess">
                 @foreach (var embedding in this.AvailableEmbeddings)
                 {
@@ -91,21 +89,6 @@
                     </MudSelectItem>
                 }
             </MudSelect>
-            }
-            else
-            {
-                <MudTextField
-                    T="string"
-                    Text="@this.SelectedEmbeddingNameText"
-                    Label="@T("Embedding")"
-                    Class="mb-3"
-                    ReadOnly="@true"
-                    Adornment="Adornment.Start"
-                    AdornmentIcon="@Icons.Material.Filled.DataObject"
-                    AdornmentColor="Color.Info"
-                    Variant="Variant.Outlined"
-                />
-            }
             
             @if (!string.IsNullOrWhiteSpace(this.dataEmbeddingId))
             {

app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs

@@ -20,12 +20,25 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
     [Parameter]
     public DataSourceLocalFile DataSource { get; set; }
 
+    /// <summary>
+    /// Whether the file this data source reads must stay as it is.
+    /// </summary>
+    /// <remarks>
+    /// Set once the index holds something for this data source. The embedding is not locked along
+    /// with it: it can be changed, and DataSourceReindexWarning asks what that costs.
+    /// </remarks>
     [Parameter]
-    public bool LockSourceAndEmbedding { get; set; }
+    public bool LockSource { get; set; }
     
     [Parameter]
     public IReadOnlyList<ConfigurationSelectData<string>> AvailableEmbeddings { get; set; } = [];
 
+    [Inject]
+    private IDialogService DialogService { get; init; } = null!;
+
+    [Inject]
+    private DataSourceEmbeddingService DataSourceEmbeddingService { get; init; } = null!;
+
     private static readonly Dictionary<string, object?> SPELLCHECK_ATTRIBUTES = new();
     
     private readonly DataSourceValidation dataSourceValidation;
@@ -120,19 +133,10 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
 
     private bool SelectedCloudEmbedding => this.SelectedEmbedding is { IsSelfHosted: false };
 
-    private bool CanChangeSourceAndEmbedding => !this.IsEditing || !this.LockSourceAndEmbedding;
+    private bool CanChangeSource => !this.IsEditing || !this.LockSource;
 
     private IEnumerable<ConfigurationSelectData<ConfidenceLevel>> ConfidenceLevels => ConfigurationSelectDataFactory.GetDataSourceConfidenceLevelsData();
 
-    private string SelectedEmbeddingNameText
-    {
-        get
-        {
-            var selectedEmbedding = this.AvailableEmbeddings.FirstOrDefault(x => x.Value == this.dataEmbeddingId);
-            return string.IsNullOrWhiteSpace(selectedEmbedding.Name) ? T("Unknown") : selectedEmbedding.Name;
-        }
-    }
-
     private string SelectedEmbeddingTokenizerText => this.SelectedEmbedding is null
         ? T("No embedding selected")
         : string.IsNullOrWhiteSpace(this.SelectedEmbedding.TokenizerPath)
@@ -156,8 +160,11 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
         Name = this.dataName,
         Description = this.dataDescription,
         Type = DataSourceType.LOCAL_FILE,
-        EmbeddingId = this.CanChangeSourceAndEmbedding ? this.dataEmbeddingId : this.DataSource.EmbeddingId,
+        EmbeddingId = this.dataEmbeddingId,
-        FilePath = this.CanChangeSourceAndEmbedding ? this.dataFilePath : this.DataSource.FilePath,
+
+        // Kept out of reach of the form while the source is locked, so a stale field cannot point an
+        // indexed data source somewhere else:
+        FilePath = this.CanChangeSource ? this.dataFilePath : this.DataSource.FilePath,
         MaxChunkTokenLength = this.dataMaxChunkTokenLength,
         ChunkOverlapTokenLength = this.dataChunkOverlapTokenLength,
         ConfidenceLevel = this.dataConfidenceLevel,
@@ -173,6 +180,18 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
             return;
 
         var addedDataSource = this.CreateDataSource();
+
+        //
+        // Ask while the dialog is still open, so a token limit which would have cost the prepared
+        // documents can be corrected right away. Asking in DataSourceManagement instead would have
+        // to be written once per data source kind, and by then the numbers are out of reach.
+        //
+        // Only when editing: while adding, DataSource is still default -- both local data sources
+        // are record structs -- and nothing has been prepared for a source which does not exist yet.
+        //
+        if (this.IsEditing && !await DataSourceReindexWarning.ConfirmDataSourceChangeAsync(this.DialogService, this.SettingsManager, this.DataSourceEmbeddingService, this.DataSource, addedDataSource))
+            return;
+
         this.MudDialog.Close(DialogResult.Ok(addedDataSource));
     }
     

app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs

@@ -86,6 +86,17 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
     [Parameter]
     public string DataTokenizerPath { get; set; } = string.Empty;
 
+    /// <summary>
+    /// The fingerprint of the tokenizer this provider was stored with.
+    /// </summary>
+    /// <remarks>
+    /// Carried through the dialog untouched as long as the user leaves the tokenizer alone. Rebuilding
+    /// it from the path on every open would read a file for nothing, and an unreadable one would look
+    /// like another tokenizer and cost every data source of this provider its index.
+    /// </remarks>
+    [Parameter]
+    public string DataTokenizerFingerprint { get; set; } = string.Empty;
+
     [Parameter]
     public int DataTokenLimit { get; set; } = EmbeddingProvider.DEFAULT_TOKEN_LIMIT;
 
@@ -105,6 +116,12 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
     [Inject]
     private ILogger<EmbeddingProviderDialog> Logger { get; init; } = null!;
 
+    [Inject]
+    private IDialogService DialogService { get; init; } = null!;
+
+    [Inject]
+    private DataSourceEmbeddingService DataSourceEmbeddingService { get; init; } = null!;
+
     private static readonly Dictionary<string, object?> SPELLCHECK_ATTRIBUTES = new();
 
     /// <summary>
@@ -121,6 +138,7 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
     private string dataEditingPreviousInstanceName = string.Empty;
     private string dataLoadingModelsIssue = string.Empty;
     private string dataFilePath = string.Empty;
+    private string dataTokenizerFingerprint = string.Empty;
     private string dataCustomTokenizerValidationIssue = string.Empty;
     private Task dataTokenizerValidationTask = Task.CompletedTask;
     private bool dataStoreWasAttempted;
@@ -176,6 +194,7 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
             IsEnterpriseConfiguration = this.IsEnterpriseConfiguration,
             EnterpriseConfigurationPluginId = Guid.Empty,
             TokenizerPath = this.dataFilePath,
+            TokenizerFingerprint = this.dataTokenizerFingerprint,
             EmbeddingBatchSize = this.DataEmbeddingBatchSize,
             TokenLimit = this.DataTokenLimit,
             CustomIconDataUrl = this.DataCustomIconDataUrl,
@@ -201,6 +220,7 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
         {
             this.dataEditingPreviousInstanceName = this.DataName.ToLowerInvariant();
             this.dataFilePath = this.DataTokenizerPath;
+            this.dataTokenizerFingerprint = this.DataTokenizerFingerprint;
             this.showExpertSettings = !string.IsNullOrWhiteSpace(this.DataTokenizerPath)
                                       || this.DataTokenLimit != EmbeddingProvider.DEFAULT_TOKEN_LIMIT
                                       || this.DataEmbeddingBatchSize != EmbeddingProvider.DEFAULT_EMBEDDING_BATCH_SIZE;
@@ -280,6 +300,21 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
         if (!this.dataIsValid)
             return;
 
+        //
+        // Ask before anything is written. Storing a tokenizer deletes the previous one before it
+        // copies, and the API key goes into the OS keyring right after, so asking any later would
+        // leave those changes behind even when the user says no. Saying no also keeps this dialog
+        // open, which is the point: the value which would have cost the index can be corrected
+        // right away.
+        //
+        // Enterprise-managed providers are left out. Every field which reaches the embedding
+        // signature is locked for them, and their data sources are not queued for indexing either.
+        //
+        if (this.IsEditing && !this.IsEnterpriseConfiguration && !await DataSourceReindexWarning.ConfirmEmbeddingProviderChangeAsync(
+                this.DialogService, this.SettingsManager, this.DataSourceEmbeddingService,
+                this.SettingsManager.GetEmbeddingProviderById(this.DataId), this.CreateEmbeddingProviderSettings()))
+            return;
+
         var response = await this.StoreOrDeleteTokenizerAsync();
         if (!response.Success)
         {
@@ -413,9 +448,18 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
         this.dataTokenizerValidationTask = this.ValidateCustomTokenizer(filePath, validationRevision);
         await this.dataTokenizerValidationTask;
 
+        //
+        // The embedding signature carries the tokenizer's content, so it has to be read while we have
+        // the file the user just picked. Reading it here rather than while storing also keeps a large
+        // file off that path, where it would stall the circuit.
+        //
+        var tokenizerFingerprint = await TokenizerFingerprint.ForFileAsync(filePath);
+
         if (validationRevision != this.dataTokenizerValidationRevision)
             return;
 
+        this.dataTokenizerFingerprint = tokenizerFingerprint;
+
         if (this.dataStoreWasAttempted)
             await this.form.Validate();
         else

app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua

@@ -3810,6 +3810,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2675917723"] = "Ni
 -- No valid embedding
 UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2698203405"] = "Keine gültige Einbettung"
 
+-- Repair this data source by indexing it anew
+UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2771708618"] = "Diese Datenquelle durch erneutes Indexieren reparieren"
+
 -- Embedding
 UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2838542994"] = "Einbettung"
 
@@ -3957,6 +3960,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTION::T86053874"] = "Verf
 -- This data source is waiting to be indexed again. Until that is finished, it cannot be searched.
 UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTIONROW::T1692539409"] = "Diese Datenquelle wartet darauf, erneut indexiert zu werden. Bis dies abgeschlossen ist, kann sie nicht durchsucht werden."
 
+-- The index of this data source cannot be read anymore. Open your data source settings with the gear icon above, then use the repair action there.
+UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTIONROW::T4047623216"] = "Der Index dieser Datenquelle kann nicht mehr gelesen werden. Öffnen Sie die Einstellungen Ihrer Datenquelle über das Zahnradsymbol oben und führen Sie dort die Reparaturaktion aus."
+
 -- Tools (Optional)
 UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DIRECTCHATLAUNCHERFORM::T1019749907"] = "Werkzeuge (optional)"
 
@@ -6084,9 +6090,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1725856265"
 -- How many matches do you want at most per query?
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1827669611"] = "Wie viele Treffer möchten Sie maximal pro Abfrage erhalten?"
 
--- This data source already has indexed embeddings. Delete and recreate it to change the folder path or embedding model.
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1845247453"] = "Diese Datenquelle verfügt bereits über indexierte Einbettungen. Löschen und erstellen Sie sie neu, um den Ordnerpfad oder das Einbettungsmodell zu ändern."
-
 -- Update
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1847791252"] = "Aktualisieren"
 
@@ -6123,9 +6126,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3359366900"
 -- Show Expert Settings
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3361153305"] = "Experten-Einstellungen anzeigen"
 
--- Unknown
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3424652889"] = "Unbekannt"
-
 -- Select the base directory
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T562479068"] = "Wählen Sie das Stammverzeichnis aus"
 
@@ -6135,6 +6135,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T787118522"]
 -- Data Source Name
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T813773421"] = "Name der Datenquelle"
 
+-- The documents of this data source are already prepared, so its folder cannot be changed. Another folder holds other documents, which makes it another data source: please add one for it. The embedding method below can be changed.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T870152265"] = "Die Dokumente dieser Datenquelle sind bereits vorbereitet, daher kann ihr Ordner nicht geändert werden. Ein anderer Ordner enthält andere Dokumente und ist damit eine andere Datenquelle: Bitte fügen Sie dafür eine neue hinzu. Die Einbettungsmethode darunter können Sie ändern."
+
 -- Cancel
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T900713019"] = "Abbrechen"
 
@@ -6282,15 +6285,12 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3359366900"] = "
 -- Show Expert Settings
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3361153305"] = "Experten-Einstellungen anzeigen"
 
--- Unknown
+-- The documents of this data source are already prepared, so its file cannot be changed. Another file holds other content, which makes it another data source: please add one for it. The embedding method below can be changed.
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3424652889"] = "Unbekannt"
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3731767732"] = "Die Dokumente dieser Datenquelle sind bereits vorbereitet, daher kann ihre Datei nicht geändert werden. Eine andere Datei enthält andere Inhalte und ist damit eine andere Datenquelle: Bitte fügen Sie dafür eine neue hinzu. Die Einbettungsmethode darunter können Sie ändern."
 
 -- Select the file
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3740148848"] = "Datei auswählen"
 
--- This data source already has indexed embeddings. Delete and recreate it to change the file path or embedding model.
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T458267883"] = "Diese Datenquelle verfügt bereits über indexierte Einbettungen. Löschen und erstellen Sie sie neu, um den Dateipfad oder das Einbettungsmodell zu ändern."
-
 -- The data source token limit must not be larger than the embedding provider token limit ({0}).
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T787118522"] = "Das Token-Limit der Datenquelle darf nicht größer sein als das Token-Limit des Einbettungsanbieters ({0})."
 
@@ -9141,6 +9141,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T2525374657"] = "{0} von {1} Datei
 -- Background embeddings
 UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T2547971789"] = "Einbettungen im Hintergrund"
 
+-- Repair this data source by indexing it anew
+UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T2771708618"] = "Diese Datenquelle durch erneutes Indexieren reparieren"
+
 -- Refresh this data source
 UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T2901874229"] = "Diese Datenquelle aktualisieren"
 
@@ -10815,6 +10818,36 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::QDRANTEDGECLIENTIMPLEM
 -- Qdrant Edge is not available.
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::QDRANTEDGECLIENTIMPLEMENTATION::T744445696"] = "Qdrant Edge ist nicht verfügbar."
 
+-- They keep answering keyword searches, but searching them by meaning stops working, and no further documents can be prepared for them. The ones which are already prepared stay tied to this provider as well, so you cannot simply move them to another one.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T2343773457"] = "Sie beantworten weiterhin Stichwortsuchen, aber die Suche nach Bedeutung funktioniert nicht mehr, und es können keine weiteren Dokumente für sie vorbereitet werden. Bereits vorbereitete Datenquellen bleiben zudem an diesen Anbieter gebunden, sodass Sie sie nicht einfach auf einen anderen umstellen können."
+
+-- and {0} more.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T2519847121"] = "und {0} weitere."
+
+-- This change makes the prepared documents of the following data sources unusable ({0}):
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T3337378891"] = "Durch diese Änderung werden die vorbereiteten Dokumente der folgenden Datenquellen unbrauchbar ({0}):"
+
+-- Do you want to apply this change anyway?
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T3419411838"] = "Möchten Sie diese Änderung trotzdem übernehmen?"
+
+-- Documents Will Be Prepared Again
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T737291513"] = "Dokumente werden erneut vorbereitet"
+
+-- Your embedding provider runs in the cloud, so preparing everything again costs money.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T774305382"] = "Ihr Einbettungsanbieter läuft in der Cloud, daher kostet es Geld, alles erneut vorzubereiten."
+
+-- These data sources are set up with this embedding provider ({0}):
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T858000918"] = "Diese Datenquellen sind mit diesem Einbettungsanbieter eingerichtet ({0}):"
+
+-- Everything prepared for them is thrown away, and every one of their documents goes to your embedding provider once more. With a large data source, this takes a while.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T874850580"] = "Alles, was für sie vorbereitet wurde, wird verworfen, und jedes ihrer Dokumente wird erneut an Ihren Einbettungsanbieter gesendet. Bei einer großen Datenquelle dauert dies eine Weile."
+
+-- Repair Data Source
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREPAIR::T4175865785"] = "Datenquelle reparieren"
+
+-- The index of the data source '{0}' cannot be read anymore. Repairing it means building the index from scratch: everything indexed so far is thrown away, and every document of this data source is sent to your embedding provider once more. With a cloud provider, this costs money, and with a large data source it takes a while. Do you want to repair this data source now?
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREPAIR::T857336889"] = "Der Index der Datenquelle „{0}“ kann nicht mehr gelesen werden. Bei einer Reparatur wird der Index vollständig neu erstellt: Alle bisher indexierten Daten werden verworfen und jedes Dokument dieser Datenquelle erneut an Ihren Einbettungsanbieter gesendet. Bei einem Cloud-Anbieter entstehen dadurch Kosten, und bei einer großen Datenquelle kann dies einige Zeit dauern. Möchten Sie diese Datenquelle jetzt reparieren?"
+
 -- The related data is not allowed to be sent to any LLM provider. This means that this data source cannot be used at the moment.
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::ERICLIENT::DATAMODEL::PROVIDERTYPEEXTENSIONS::T1555790630"] = "Die zugehörigen Daten dürfen an keinen LLM-Anbieter gesendet werden. Das bedeutet, dass diese Datenquelle momentan nicht verwendet werden kann."
 
@@ -12009,6 +12042,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T4515612
 -- The embedding provider answered with {0} vectors for {1} parts of the file '{2}'. Please select another embedding model or provider.
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T667058890"] = "Der Einbettungsanbieter hat für {1} Teile der Datei „{2}“ {0} Vektoren zurückgegeben. Bitte wählen Sie ein anderes Einbettungsmodell oder einen anderen Anbieter aus."
 
+-- The index of the data source '{0}' cannot be read anymore. The data source stays out of your chats until its index was built anew. Use the repair action to start that.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T831900720"] = "Der Index der Datenquelle „{0}“ kann nicht mehr gelesen werden. Die Datenquelle wird in Ihren Chats nicht verwendet, bis ihr Index neu erstellt wurde. Verwenden Sie die Aktion „Reparieren“, um dies zu starten."
+
 -- The folder '{0}' does not exist.
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T871336081"] = "Der Ordner „{0}“ existiert nicht."
 
@@ -12057,6 +12093,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T40
 -- Page {0}
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T4127287940"] = "Seite {0}"
 
+-- The data source '{0}' was left out of the answer: its index cannot be read anymore. You can repair it in your data source settings.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T59210871"] = "Die Datenquelle „{0}“ wurde aus der Antwort weggelassen, weil ihr Index nicht mehr gelesen werden kann. Sie können ihn in den Einstellungen der Datenquelle reparieren."
+
 -- The data source '{0}' was left out of the answer because searching it failed.
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T934856625"] = "Die Datenquelle „{0}“ wurde aus der Antwort weggelassen, weil die Suche darin fehlgeschlagen ist."
 

app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua

@@ -3810,6 +3810,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2675917723"] = "No
 -- No valid embedding
 UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2698203405"] = "No valid embedding"
 
+-- Repair this data source by indexing it anew
+UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2771708618"] = "Repair this data source by indexing it anew"
+
 -- Embedding
 UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2838542994"] = "Embedding"
 
@@ -3957,6 +3960,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTION::T86053874"] = "Avail
 -- This data source is waiting to be indexed again. Until that is finished, it cannot be searched.
 UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTIONROW::T1692539409"] = "This data source is waiting to be indexed again. Until that is finished, it cannot be searched."
 
+-- The index of this data source cannot be read anymore. Open your data source settings with the gear icon above, then use the repair action there.
+UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTIONROW::T4047623216"] = "The index of this data source cannot be read anymore. Open your data source settings with the gear icon above, then use the repair action there."
+
 -- Tools (Optional)
 UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DIRECTCHATLAUNCHERFORM::T1019749907"] = "Tools (Optional)"
 
@@ -6084,9 +6090,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1725856265"
 -- How many matches do you want at most per query?
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1827669611"] = "How many matches do you want at most per query?"
 
--- This data source already has indexed embeddings. Delete and recreate it to change the folder path or embedding model.
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1845247453"] = "This data source already has indexed embeddings. Delete and recreate it to change the folder path or embedding model."
-
 -- Update
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1847791252"] = "Update"
 
@@ -6123,9 +6126,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3359366900"
 -- Show Expert Settings
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3361153305"] = "Show Expert Settings"
 
--- Unknown
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3424652889"] = "Unknown"
-
 -- Select the base directory
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T562479068"] = "Select the base directory"
 
@@ -6135,6 +6135,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T787118522"]
 -- Data Source Name
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T813773421"] = "Data Source Name"
 
+-- The documents of this data source are already prepared, so its folder cannot be changed. Another folder holds other documents, which makes it another data source: please add one for it. The embedding method below can be changed.
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T870152265"] = "The documents of this data source are already prepared, so its folder cannot be changed. Another folder holds other documents, which makes it another data source: please add one for it. The embedding method below can be changed."
+
 -- Cancel
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T900713019"] = "Cancel"
 
@@ -6282,15 +6285,12 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3359366900"] = "
 -- Show Expert Settings
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3361153305"] = "Show Expert Settings"
 
--- Unknown
+-- The documents of this data source are already prepared, so its file cannot be changed. Another file holds other content, which makes it another data source: please add one for it. The embedding method below can be changed.
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3424652889"] = "Unknown"
+UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3731767732"] = "The documents of this data source are already prepared, so its file cannot be changed. Another file holds other content, which makes it another data source: please add one for it. The embedding method below can be changed."
 
 -- Select the file
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3740148848"] = "Select the file"
 
--- This data source already has indexed embeddings. Delete and recreate it to change the file path or embedding model.
-UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T458267883"] = "This data source already has indexed embeddings. Delete and recreate it to change the file path or embedding model."
-
 -- The data source token limit must not be larger than the embedding provider token limit ({0}).
 UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T787118522"] = "The data source token limit must not be larger than the embedding provider token limit ({0})."
 
@@ -9141,6 +9141,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T2525374657"] = "{0} of {1} files
 -- Background embeddings
 UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T2547971789"] = "Background embeddings"
 
+-- Repair this data source by indexing it anew
+UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T2771708618"] = "Repair this data source by indexing it anew"
+
 -- Refresh this data source
 UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T2901874229"] = "Refresh this data source"
 
@@ -10815,6 +10818,36 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::QDRANTEDGECLIENTIMPLEM
 -- Qdrant Edge is not available.
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::QDRANTEDGECLIENTIMPLEMENTATION::T744445696"] = "Qdrant Edge is not available."
 
+-- They keep answering keyword searches, but searching them by meaning stops working, and no further documents can be prepared for them. The ones which are already prepared stay tied to this provider as well, so you cannot simply move them to another one.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T2343773457"] = "They keep answering keyword searches, but searching them by meaning stops working, and no further documents can be prepared for them. The ones which are already prepared stay tied to this provider as well, so you cannot simply move them to another one."
+
+-- and {0} more.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T2519847121"] = "and {0} more."
+
+-- This change makes the prepared documents of the following data sources unusable ({0}):
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T3337378891"] = "This change makes the prepared documents of the following data sources unusable ({0}):"
+
+-- Do you want to apply this change anyway?
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T3419411838"] = "Do you want to apply this change anyway?"
+
+-- Documents Will Be Prepared Again
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T737291513"] = "Documents Will Be Prepared Again"
+
+-- Your embedding provider runs in the cloud, so preparing everything again costs money.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T774305382"] = "Your embedding provider runs in the cloud, so preparing everything again costs money."
+
+-- These data sources are set up with this embedding provider ({0}):
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T858000918"] = "These data sources are set up with this embedding provider ({0}):"
+
+-- Everything prepared for them is thrown away, and every one of their documents goes to your embedding provider once more. With a large data source, this takes a while.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T874850580"] = "Everything prepared for them is thrown away, and every one of their documents goes to your embedding provider once more. With a large data source, this takes a while."
+
+-- Repair Data Source
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREPAIR::T4175865785"] = "Repair Data Source"
+
+-- The index of the data source '{0}' cannot be read anymore. Repairing it means building the index from scratch: everything indexed so far is thrown away, and every document of this data source is sent to your embedding provider once more. With a cloud provider, this costs money, and with a large data source it takes a while. Do you want to repair this data source now?
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREPAIR::T857336889"] = "The index of the data source '{0}' cannot be read anymore. Repairing it means building the index from scratch: everything indexed so far is thrown away, and every document of this data source is sent to your embedding provider once more. With a cloud provider, this costs money, and with a large data source it takes a while. Do you want to repair this data source now?"
+
 -- The related data is not allowed to be sent to any LLM provider. This means that this data source cannot be used at the moment.
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::ERICLIENT::DATAMODEL::PROVIDERTYPEEXTENSIONS::T1555790630"] = "The related data is not allowed to be sent to any LLM provider. This means that this data source cannot be used at the moment."
 
@@ -12009,6 +12042,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T4515612
 -- The embedding provider answered with {0} vectors for {1} parts of the file '{2}'. Please select another embedding model or provider.
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T667058890"] = "The embedding provider answered with {0} vectors for {1} parts of the file '{2}'. Please select another embedding model or provider."
 
+-- The index of the data source '{0}' cannot be read anymore. The data source stays out of your chats until its index was built anew. Use the repair action to start that.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T831900720"] = "The index of the data source '{0}' cannot be read anymore. The data source stays out of your chats until its index was built anew. Use the repair action to start that."
+
 -- The folder '{0}' does not exist.
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T871336081"] = "The folder '{0}' does not exist."
 
@@ -12057,6 +12093,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T40
 -- Page {0}
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T4127287940"] = "Page {0}"
 
+-- The data source '{0}' was left out of the answer: its index cannot be read anymore. You can repair it in your data source settings.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T59210871"] = "The data source '{0}' was left out of the answer: its index cannot be read anymore. You can repair it in your data source settings."
+
 -- The data source '{0}' was left out of the answer because searching it failed.
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T934856625"] = "The data source '{0}' was left out of the answer because searching it failed."
 

app/MindWork AI Studio/Settings/EmbeddingProvider.cs

@@ -23,6 +23,7 @@ public sealed record EmbeddingProvider(
     string Hostname = "http://localhost:1234",
     Host Host = Host.NONE,
     string TokenizerPath = "",
+    string TokenizerFingerprint = "",
     int EmbeddingBatchSize = 0,
     int TokenLimit = 0,
     bool AllowUserProvidedAPIKey = false,
@@ -113,14 +114,14 @@ public sealed record EmbeddingProvider(
         }
 
         var tokenLimit = DEFAULT_TOKEN_LIMIT;
-        if (table.TryGetValue("TokenLimit", out var tokenLimitValue) && (!tokenLimitValue.TryRead<int>(out tokenLimit) || tokenLimit < 1))
+        if (table.TryGetValue("TokenLimit", out var tokenLimitValue) && (!tokenLimitValue.TryRead(out tokenLimit) || tokenLimit < 1))
         {
             LOGGER.LogWarning($"The configured embedding provider {idx} does not contain a valid token limit. Falling back to {DEFAULT_TOKEN_LIMIT}. (Plugin ID: {configPluginId})");
             tokenLimit = DEFAULT_TOKEN_LIMIT;
         }
 
         var embeddingBatchSize = DEFAULT_EMBEDDING_BATCH_SIZE;
-        if (table.TryGetValue("EmbeddingBatchSize", out var embeddingBatchSizeValue) && (!embeddingBatchSizeValue.TryRead<int>(out embeddingBatchSize) || embeddingBatchSize < 1))
+        if (table.TryGetValue("EmbeddingBatchSize", out var embeddingBatchSizeValue) && (!embeddingBatchSizeValue.TryRead(out embeddingBatchSize) || embeddingBatchSize < 1))
         {
             LOGGER.LogWarning($"The configured embedding provider {idx} does not contain a valid embedding batch size. Falling back to {DEFAULT_EMBEDDING_BATCH_SIZE}. (Plugin ID: {configPluginId})");
             embeddingBatchSize = DEFAULT_EMBEDDING_BATCH_SIZE;

app/MindWork AI Studio/Tools/DataSourceReindexWarning.cs

@@ -0,0 +1,194 @@
+using System.Text;
+
+using AIStudio.Dialogs;
+using AIStudio.Settings;
+using AIStudio.Tools.PluginSystem;
+using AIStudio.Tools.Services;
+
+namespace AIStudio.Tools;
+
+/// <summary>
+/// Asks before an edit makes the prepared documents of data sources useless, and names the data
+/// sources which depend on an embedding provider somebody is about to delete.
+/// </summary>
+/// <remarks>
+/// Kept here rather than in the dialogs which ask -- the embedding provider dialog and the two data
+/// source dialogs -- so the sentence naming what a rebuild costs cannot drift apart between them.
+/// That is the same reason DataSourceRepair sits next to it, and both name the same two costs.
+///
+/// Nothing is asked when nothing is lost. A data source only reaches the question when the edit
+/// really changes its embedding signature and when the index already holds something for it, so
+/// renaming an embedding provider or editing a data source nobody has indexed yet stays silent.
+/// </remarks>
+public static class DataSourceReindexWarning
+{
+    /// <summary>
+    /// How many data sources are named before the rest is only counted.
+    /// </summary>
+    private const int MAX_NAMED_DATA_SOURCES = 10;
+
+    private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceReindexWarning).Namespace, nameof(DataSourceReindexWarning));
+
+    /// <summary>
+    /// Asks before an edited embedding provider is saved.
+    /// </summary>
+    /// <param name="dialogService">The dialog service to ask with.</param>
+    /// <param name="settingsManager">The settings, read for the data sources behind the provider.</param>
+    /// <param name="embeddingService">The service which knows what the index holds.</param>
+    /// <param name="before">The embedding provider as it is stored.</param>
+    /// <param name="after">The embedding provider as it would be stored.</param>
+    /// <param name="token">The cancellation token.</param>
+    /// <returns>True when the edit may be saved.</returns>
+    public static async Task<bool> ConfirmEmbeddingProviderChangeAsync(IDialogService dialogService, SettingsManager settingsManager, DataSourceEmbeddingService embeddingService,
+        EmbeddingProvider before, EmbeddingProvider after, CancellationToken token = default)
+    {
+        // Nothing was stored under this id, so no data source can point at it:
+        if (before == EmbeddingProvider.NONE)
+            return true;
+
+        var candidates = GetDataSourcesUsing(settingsManager, before.Id)
+            .Where(dataSource => EmbeddingChangeImpact.AffectsStoredIndex(dataSource, before, after))
+            .Cast<IDataSource>()
+            .ToList();
+
+        if (candidates.Count == 0)
+            return true;
+
+        var affected = await embeddingService.GetDataSourcesWithStoredIndexAsync(candidates, token);
+        return await ConfirmAsync(dialogService, affected, !after.IsSelfHosted);
+    }
+
+    /// <summary>
+    /// Asks before an edited data source is saved.
+    /// </summary>
+    /// <param name="dialogService">The dialog service to ask with.</param>
+    /// <param name="settingsManager">The settings, read for the embedding provider of the data source.</param>
+    /// <param name="embeddingService">The service which knows what the index holds.</param>
+    /// <param name="before">The data source as it is stored.</param>
+    /// <param name="after">The data source as it would be stored.</param>
+    /// <param name="token">The cancellation token.</param>
+    /// <returns>True when the edit may be saved.</returns>
+    public static async Task<bool> ConfirmDataSourceChangeAsync(IDialogService dialogService, SettingsManager settingsManager, DataSourceEmbeddingService embeddingService,
+        IInternalDataSource before, IInternalDataSource after, CancellationToken token = default)
+    {
+        // Without a provider nothing is embedded at all, so nothing can be lost:
+        if (!DataSourceEmbeddingProviders.TryResolve(settingsManager, after, out var afterProvider))
+            return true;
+
+        //
+        // The provider a data source points at today may be gone -- somebody deleted it, and this
+        // edit is how the source is put back to work. Standing in for it with NONE gives a signature
+        // of its own, so that edit is asked about as well, which is right: what is stored was made by
+        // a provider nobody can reach any more.
+        //
+        DataSourceEmbeddingProviders.TryResolve(settingsManager, before, out var resolvedBeforeProvider);
+        var beforeProvider = resolvedBeforeProvider ?? EmbeddingProvider.NONE;
+
+        if (!EmbeddingChangeImpact.AffectsStoredIndex(before, beforeProvider, after, afterProvider))
+            return true;
+
+        var affected = await embeddingService.GetDataSourcesWithStoredIndexAsync([after], token);
+        return await ConfirmAsync(dialogService, affected, !afterProvider.IsSelfHosted);
+    }
+
+    /// <summary>
+    /// Names the data sources which would lose their embedding provider, for the deletion question.
+    /// </summary>
+    /// <remarks>
+    /// Deleting is the one case where nothing prepared is thrown away: the documents stay where they
+    /// are, but nothing can reach them by meaning any more, and nothing new can be prepared either.
+    /// The names come from the same place as the ones in the questions above so that both lists read
+    /// alike, which is also why this returns the text instead of asking on its own -- the deletion
+    /// question has more to say than this.
+    ///
+    /// Every data source pointing at the provider is named, prepared or not. A source which was never
+    /// indexed loses just as much: it can no longer be prepared at all.
+    /// </remarks>
+    /// <param name="settingsManager">The settings holding the data sources.</param>
+    /// <param name="embeddingProvider">The embedding provider which is about to be deleted.</param>
+    /// <returns>The Markdown text, or an empty string when no data source uses that provider.</returns>
+    public static string DescribeDataSourcesLosingTheirProvider(SettingsManager settingsManager, EmbeddingProvider embeddingProvider)
+    {
+        if (embeddingProvider == EmbeddingProvider.NONE)
+            return string.Empty;
+
+        var affected = GetDataSourcesUsing(settingsManager, embeddingProvider.Id).Cast<IDataSource>().ToList();
+        if (affected.Count == 0)
+            return string.Empty;
+
+        var body = new StringBuilder();
+
+        // Counted rather than put into a plural form: the I18N has no mechanism for one.
+        body.AppendLine(string.Format(TB("These data sources are set up with this embedding provider ({0}):"), affected.Count.CompactCount()));
+        body.AppendLine();
+        body.AppendLine(FormatDataSourceNames(affected));
+        body.AppendLine();
+        body.AppendLine(TB("They keep answering keyword searches, but searching them by meaning stops working, and no further documents can be prepared for them. The ones which are already prepared stay tied to this provider as well, so you cannot simply move them to another one."));
+
+        return body.ToString();
+    }
+
+    /// <summary>
+    /// The data sources which are indexed with a given embedding provider.
+    /// </summary>
+    /// <param name="settingsManager">The settings holding the data sources.</param>
+    /// <param name="embeddingProviderId">The id of the embedding provider.</param>
+    /// <returns>The data sources pointing at that embedding provider.</returns>
+    private static IReadOnlyList<IInternalDataSource> GetDataSourcesUsing(SettingsManager settingsManager, string embeddingProviderId) =>
+        settingsManager.ConfigurationData.DataSources
+            .OfType<IInternalDataSource>()
+            .Where(dataSource => embeddingProviderId.Equals(dataSource.EmbeddingId, StringComparison.OrdinalIgnoreCase))
+            .ToList();
+
+    /// <summary>
+    /// Names data sources as a Markdown list, counting the rest when there are too many to name.
+    /// </summary>
+    /// <param name="dataSources">The data sources to name.</param>
+    /// <returns>The Markdown list.</returns>
+    private static string FormatDataSourceNames(IReadOnlyList<IDataSource> dataSources)
+    {
+        var names = dataSources
+            .Select(dataSource => dataSource.Name)
+            .OrderBy(name => name, StringComparer.OrdinalIgnoreCase)
+            .ToList();
+
+        var lines = names.Take(MAX_NAMED_DATA_SOURCES).Select(name => $"- {name}").ToList();
+        if (names.Count > MAX_NAMED_DATA_SOURCES)
+            lines.Add($"- {string.Format(TB("and {0} more."), (names.Count - MAX_NAMED_DATA_SOURCES).CompactCount())}");
+
+        return string.Join(Environment.NewLine, lines);
+    }
+
+    private static async Task<bool> ConfirmAsync(IDialogService dialogService, IReadOnlyList<IDataSource> affected, bool usesCloudEmbedding)
+    {
+        if (affected.Count == 0)
+            return true;
+
+        var body = new StringBuilder();
+
+        // Counted rather than put into a plural form: the I18N has no mechanism for one.
+        body.AppendLine(string.Format(TB("This change makes the prepared documents of the following data sources unusable ({0}):"), affected.Count.CompactCount()));
+        body.AppendLine();
+        body.AppendLine(FormatDataSourceNames(affected));
+        body.AppendLine();
+        body.AppendLine(TB("Everything prepared for them is thrown away, and every one of their documents goes to your embedding provider once more. With a large data source, this takes a while."));
+
+        if (usesCloudEmbedding)
+        {
+            body.AppendLine();
+            body.AppendLine(TB("Your embedding provider runs in the cloud, so preparing everything again costs money."));
+        }
+
+        body.AppendLine();
+        body.AppendLine(TB("Do you want to apply this change anyway?"));
+
+        var dialogParameters = new DialogParameters<ConfirmDialog>
+        {
+            { x => x.MarkdownBody, body.ToString() },
+        };
+
+        var dialogReference = await dialogService.ShowAsync<ConfirmDialog>(TB("Documents Will Be Prepared Again"), dialogParameters, Dialogs.DialogOptions.FULLSCREEN);
+        var dialogResult = await dialogReference.Result;
+        return dialogResult is not null && !dialogResult.Canceled;
+    }
+}

app/MindWork AI Studio/Tools/PluginSystem/PluginConfigurationObject.cs

@@ -497,7 +497,17 @@ public sealed record PluginConfigurationObject
             TokenizerModelId.ForEmbeddingProvider(provider),
             $"embedding provider '{provider.Name}'");
 
-        return provider with { TokenizerPath = syncedTokenizerPath };
+        //
+        // The embedding signature is built from the tokenizer's content, so the fingerprint travels
+        // with the provider. An unreadable file yields nothing, and writing that would look like
+        // another tokenizer and cost every data source of this provider its index -- so in that case
+        // the previous fingerprint is kept rather than cleared.
+        //
+        var syncedTokenizerFingerprint = await TokenizerFingerprint.ForFileAsync(syncedTokenizerPath);
+        if (string.IsNullOrEmpty(syncedTokenizerFingerprint) && !string.IsNullOrWhiteSpace(syncedTokenizerPath))
+            syncedTokenizerFingerprint = provider.TokenizerFingerprint;
+
+        return provider with { TokenizerPath = syncedTokenizerPath, TokenizerFingerprint = syncedTokenizerFingerprint };
     }
 
     private static async Task<string> SyncTokenizerAsync(string configuredTokenizerPath, string pluginPath, string modelId, string logName)

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs

@@ -67,7 +67,7 @@ public sealed partial class DataSourceEmbeddingService
 
     private async IAsyncEnumerable<EmbeddingChunk> StreamEmbeddingChunksAsync(string filePath, IDataSource dataSource, EmbeddingProvider embeddingProvider, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token)
     {
-        var options = this.GetChunkingOptions(dataSource, embeddingProvider);
+        var options = GetChunkingOptions(dataSource, embeddingProvider);
         var strategy = this.GetChunkingStrategy(filePath);
         var content = await this.ReadExtractedFileContentAsync(filePath, embeddingProvider, token);
 
@@ -582,7 +582,18 @@ public sealed partial class DataSourceEmbeddingService
         throw new InvalidOperationException(string.Format(TB("The tokens of the text could not be counted for the embedding provider '{0}'. {1}"), embeddingProvider.Name, message));
     }
 
-    private ChunkingOptions GetChunkingOptions(IDataSource dataSource, EmbeddingProvider embeddingProvider)
+    /// <summary>
+    /// Works out how the text of a data source is cut for a given embedding provider.
+    /// </summary>
+    /// <remarks>
+    /// Static, because the answer follows from its two arguments alone. That lets the embedding
+    /// signature be built for a configuration which is not stored yet, which is what the dialogs ask
+    /// before they save a change.
+    /// </remarks>
+    /// <param name="dataSource">The data source whose own chunk settings apply.</param>
+    /// <param name="embeddingProvider">The embedding provider whose token limit caps them.</param>
+    /// <returns>The chunk size and overlap which are actually used.</returns>
+    internal static ChunkingOptions GetChunkingOptions(IDataSource dataSource, EmbeddingProvider embeddingProvider)
     {
         var providerMaxChunkTokenLength = Math.Max(1, embeddingProvider.EffectiveTokenLimit);
         var dataSourceMaxChunkTokenLength = dataSource is IInternalDataSource { MaxChunkTokenLength: > 0 } internalDataSource
@@ -994,6 +1005,19 @@ public sealed partial class DataSourceEmbeddingService
     /// where it runs, how the text was cut for it, and the chunk metadata version — the things a
     /// vector actually depends on.
     ///
+    /// Two of them are less obvious than they look. The Hugging Face inference provider belongs to
+    /// where the model runs: the same model name served by another backend is another vector source.
+    /// And a custom tokenizer enters through its content, not through its path, because a tokenizer
+    /// is stored under the name it came with — almost always tokenizer.json — so swapping one for
+    /// another lands on the identical path, while moving the data directory changes every path
+    /// without changing a single tokenizer.
+    ///
+    /// The chunk settings enter only as what they amount to, never as what somebody typed. A data
+    /// source storing 0 means "follow the embedding provider", and writing that provider's own limit
+    /// into the field changes nothing about how the text is cut. Carrying the typed numbers as well
+    /// made that a different signature, so opening the expert settings of a data source — which
+    /// fills an empty limit with the provider's — threw the whole index away for nothing.
+    ///
     /// The confidence level a data source asks of a provider is deliberately not among them. It
     /// changes no vector, and it is enforced live on every request anyway: DataSourceService checks
     /// it against the participating chat providers and against the embedding provider, and this
@@ -1010,14 +1034,22 @@ public sealed partial class DataSourceEmbeddingService
             embeddingProvider.Model.Id,
             embeddingProvider.Host,
             embeddingProvider.Hostname,
-            embeddingProvider.TokenizerPath,
+            embeddingProvider.HFInferenceProvider,
+            embeddingProvider.TokenizerFingerprint,
             embeddingProvider.EffectiveTokenLimit,
-            dataSource is IInternalDataSource internalDataSource ? internalDataSource.MaxChunkTokenLength : 0,
-            dataSource is IInternalDataSource overlapDataSource ? overlapDataSource.ChunkOverlapTokenLength : DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH,
             chunkingOptions.MaxChunkTokenLength,
             chunkingOptions.OverlapTokenLength);
     }
 
+    /// <summary>
+    /// Describes how the vectors of a data source were made, working the chunking out along the way.
+    /// </summary>
+    /// <param name="dataSource">The data source the vectors belong to.</param>
+    /// <param name="embeddingProvider">The embedding provider which makes them.</param>
+    /// <returns>The signature of this pairing.</returns>
+    internal static string BuildEmbeddingSignature(IDataSource dataSource, EmbeddingProvider embeddingProvider) =>
+        BuildEmbeddingSignature(dataSource, embeddingProvider, GetChunkingOptions(dataSource, embeddingProvider));
+
     private DataSourceMetadataSnapshot BuildDataSourceMetadataSnapshot(IDataSource dataSource, IReadOnlyList<FileInfo> indexedFiles)
     {
         var fileHashes = indexedFiles

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs

@@ -199,16 +199,44 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
             this.CanRefreshDataSource(dataSource);
     }
 
-    public async Task<bool> ShouldLockDataSourceIdentityAsync(string dataSourceId, CancellationToken token = default)
+    /// <summary>
+    /// Whether the file or folder a data source reads must stay as it is.
+    /// </summary>
+    /// <remarks>
+    /// Locked as soon as the index holds anything, because where a data source reads from is what it
+    /// is: another folder is another data source, and the path reaches no signature, so swapping it
+    /// would leave the stored index describing documents nobody points at any more.
+    ///
+    /// The embedding provider used to be locked along with it and no longer is. It does reach the
+    /// signature, so changing it rebuilds the index cleanly -- and DataSourceReindexWarning asks
+    /// before it does. Locking it as well left a data source whose provider was deleted stuck on
+    /// keyword search for good, with no way back.
+    ///
+    /// Unclear counts as locked: an unavailable index database says nothing about what is stored.
+    /// </remarks>
+    /// <param name="dataSourceId">The data source to ask about.</param>
+    /// <param name="token">The cancellation token.</param>
+    /// <returns>True when the source must not be changed.</returns>
+    public async Task<bool> ShouldLockDataSourceOriginAsync(string dataSourceId, CancellationToken token = default)
     {
         var indexStore = await databaseClientProvider.GetIndexStoreAsync(token);
         if (!indexStore.IsAvailable)
         {
-            logger.LogWarning("Locking identity settings for data source '{DataSourceId}' because the local RAG index database '{DatabaseName}' is unavailable.", dataSourceId, indexStore.Name);
+            logger.LogWarning("Locking the source of data source '{DataSourceId}' because the local RAG index database '{DatabaseName}' is unavailable.", dataSourceId, indexStore.Name);
             return true;
         }
 
         var manifest = await indexStore.GetManifestAsync(dataSourceId, token);
+        return HasStoredIndexState(manifest);
+    }
+
+    /// <summary>
+    /// Whether the index holds anything at all about a data source.
+    /// </summary>
+    /// <param name="manifest">What the index store returned for it.</param>
+    /// <returns>True when there is stored index state.</returns>
+    private static bool HasStoredIndexState(DataSourceEmbeddingManifest manifest)
+    {
         return !string.IsNullOrWhiteSpace(manifest.EmbeddingProviderId)
                || !string.IsNullOrWhiteSpace(manifest.EmbeddingSignature)
                || !string.IsNullOrWhiteSpace(manifest.SourceHash)
@@ -220,6 +248,59 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
                || manifest.PermanentFailures.Count > 0;
     }
 
+    /// <summary>
+    /// Picks the data sources which already hold something in the index.
+    /// </summary>
+    /// <remarks>
+    /// Asked before a setting is saved which would throw those indexes away, so the question can be
+    /// put to the user with the names in it. Anything unclear counts as holding something — the
+    /// opposite of IsAwaitingReindexAsync, and for the opposite reason: there, a wrongly greyed-out
+    /// row would stay wrong for good, while a question asked once too often costs a click, and one
+    /// skipped costs whatever a cloud provider charges for embedding everything again.
+    /// </remarks>
+    /// <param name="dataSources">The data sources to ask about.</param>
+    /// <param name="token">The cancellation token.</param>
+    /// <returns>Those of them which have stored index state.</returns>
+    public async Task<IReadOnlyList<IDataSource>> GetDataSourcesWithStoredIndexAsync(IReadOnlyCollection<IDataSource> dataSources, CancellationToken token = default)
+    {
+        //
+        // Filtering first also keeps the index database from being created while local RAG is off:
+        // asking for the store runs its migrations on the first call, which must not happen because
+        // somebody opened a dialog.
+        //
+        var candidates = dataSources.Where(this.IsSupportedInternalDataSource).ToList();
+        if (candidates.Count == 0)
+            return [];
+
+        try
+        {
+            using var timeout = CancellationTokenSource.CreateLinkedTokenSource(token);
+            timeout.CancelAfter(REINDEX_CHECK_TIMEOUT);
+
+            var indexStore = await databaseClientProvider.GetIndexStoreAsync(timeout.Token);
+            if (!indexStore.IsAvailable)
+            {
+                logger.LogWarning("Could not tell which data sources hold a stored index because the local RAG index database '{DatabaseName}' is unavailable. Treating all {DataSourceCount} of them as affected.", indexStore.Name, candidates.Count);
+                return candidates;
+            }
+
+            var affected = new List<IDataSource>(candidates.Count);
+            foreach (var dataSource in candidates)
+            {
+                var manifest = await indexStore.GetManifestAsync(dataSource.Id, timeout.Token);
+                if (HasStoredIndexState(manifest))
+                    affected.Add(dataSource);
+            }
+
+            return affected;
+        }
+        catch (Exception exception)
+        {
+            logger.LogWarning(exception, "Could not tell which of {DataSourceCount} data source(s) hold a stored index. Treating all of them as affected.", candidates.Count);
+            return candidates;
+        }
+    }
+
     /// <summary>
     /// Whether a data source cannot answer a search right now because its index has to be built anew.
     /// </summary>
@@ -269,7 +350,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
                 return false;
 
             var indexState = await indexStore.GetDataSourceStateAsync(dataSource.Id, timeout.Token);
-            var chunkingOptions = this.GetChunkingOptions(dataSource, embeddingProvider);
+            var chunkingOptions = GetChunkingOptions(dataSource, embeddingProvider);
             var embeddingSignature = BuildEmbeddingSignature(dataSource, embeddingProvider, chunkingOptions);
             var runState = this.statuses.TryGetValue(dataSource.Id, out var status) ? status.State : (DataSourceEmbeddingState?)null;
 
@@ -1444,7 +1525,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
         IndexStoreClient indexStore,
         CancellationToken token)
     {
-        var chunkingOptions = this.GetChunkingOptions(dataSource, embeddingProvider);
+        var chunkingOptions = GetChunkingOptions(dataSource, embeddingProvider);
         var embeddingSignature = BuildEmbeddingSignature(dataSource, embeddingProvider, chunkingOptions);
         var manifest = await indexStore.GetManifestAsync(dataSource.Id, token);
 

app/MindWork AI Studio/Tools/Services/EmbeddingChangeImpact.cs

@@ -0,0 +1,47 @@
+using AIStudio.Settings;
+
+namespace AIStudio.Tools.Services;
+
+/// <summary>
+/// Answers whether an edit throws the stored index of a data source away.
+/// </summary>
+/// <remarks>
+/// Nothing here knows which settings matter. Both questions are answered by building the embedding
+/// signature twice and comparing the two, so the single place which decides stays
+/// BuildEmbeddingSignature and this cannot drift away from what an indexing run then does.
+/// </remarks>
+internal static class EmbeddingChangeImpact
+{
+    /// <summary>
+    /// Whether an edited embedding provider invalidates what is stored for one of its data sources.
+    /// </summary>
+    /// <param name="dataSource">The data source, which the edit leaves alone.</param>
+    /// <param name="before">The embedding provider as it is stored.</param>
+    /// <param name="after">The embedding provider as it would be stored.</param>
+    /// <returns>True when the stored index would be discarded.</returns>
+    public static bool AffectsStoredIndex(IDataSource dataSource, EmbeddingProvider before, EmbeddingProvider after) =>
+        !string.Equals(
+            DataSourceEmbeddingService.BuildEmbeddingSignature(dataSource, before),
+            DataSourceEmbeddingService.BuildEmbeddingSignature(dataSource, after),
+            StringComparison.Ordinal);
+
+    /// <summary>
+    /// Whether an edited data source invalidates what is stored for it.
+    /// </summary>
+    /// <remarks>
+    /// Each side is asked with the embedding provider it points at, never both with the same one. A
+    /// data source carries only the id of its provider, while the signature carries what that provider
+    /// is, so comparing both sides against one of them would report a changed embedding as no change
+    /// at all -- and the next indexing run would then rebuild everything unannounced.
+    /// </remarks>
+    /// <param name="before">The data source as it is stored.</param>
+    /// <param name="beforeProvider">The embedding provider it points at today.</param>
+    /// <param name="after">The data source as it would be stored.</param>
+    /// <param name="afterProvider">The embedding provider it would point at.</param>
+    /// <returns>True when the stored index would be discarded.</returns>
+    public static bool AffectsStoredIndex(IDataSource before, EmbeddingProvider beforeProvider, IDataSource after, EmbeddingProvider afterProvider) =>
+        !string.Equals(
+            DataSourceEmbeddingService.BuildEmbeddingSignature(before, beforeProvider),
+            DataSourceEmbeddingService.BuildEmbeddingSignature(after, afterProvider),
+            StringComparison.Ordinal);
+}

app/MindWork AI Studio/Tools/TokenizerFingerprint.cs

@@ -0,0 +1,46 @@
+using System.Security.Cryptography;
+
+namespace AIStudio.Tools;
+
+/// <summary>
+/// Identifies a tokenizer by what is inside its file, not by where the file lies.
+/// </summary>
+/// <remarks>
+/// The embedding signature asks this to decide whether stored vectors still belong to the current
+/// configuration, and the path cannot answer it. A tokenizer is stored below the data directory under
+/// the model it belongs to, keeping the name it came with -- and the usual name for one is
+/// tokenizer.json. Picking a different tokenizer with that name lands on the identical path, so the
+/// index would be kept although the chunk boundaries moved. The other way round, moving the data
+/// directory changes every path without changing a single tokenizer.
+/// </remarks>
+public static class TokenizerFingerprint
+{
+    /// <summary>
+    /// Reads a tokenizer file and returns a fingerprint of its content.
+    /// </summary>
+    /// <param name="tokenizerPath">The tokenizer file to read. May be empty when no tokenizer is set.</param>
+    /// <param name="token">The cancellation token.</param>
+    /// <returns>The fingerprint, or an empty string when there is no readable file.</returns>
+    public static async Task<string> ForFileAsync(string tokenizerPath, CancellationToken token = default)
+    {
+        if (string.IsNullOrWhiteSpace(tokenizerPath))
+            return string.Empty;
+
+        try
+        {
+            await using var stream = File.OpenRead(tokenizerPath);
+            return Convert.ToHexString(await SHA256.HashDataAsync(stream, token));
+        }
+        catch
+        {
+            //
+            // An unreadable tokenizer is not this method's problem to report: the dialog validates the
+            // file before it ever gets here, and an indexing run says so again when it cannot tokenize
+            // anything. Whoever stores a provider has to decide what an empty answer means for them,
+            // because writing it into the settings would look like another tokenizer and throw the
+            // stored vectors away.
+            //
+            return string.Empty;
+        }
+    }
+}

app/MindWork AI Studio/wwwroot/changelog/v26.9.1.md

@@ -20,10 +20,10 @@
 - Added a way to open the sources of your own documents: click a source below an answer, and the document opens in the program your system uses for it.
 - Added a jump to the right page for the sources of your own documents (RAG), so a PDF opens directly where the passage was found, wherever your system and its program support it.
 - Added a way to show a source of your own documents (RAG) in your file manager.
-- Added the page of a passage to what the AI is told when it answers from your own documents (RAG), so it can name the page an answer rests on.
 - Added the details of the two databases behind local RAG to the information page: which versions they run, how much space they use on your disk, and how much they hold.
 - Added a repair for your local data sources. Should the index of a data source ever become unreadable, AI Studio now says so instead of quietly finding nothing and leaves that source out of your chats until it works again.
 - Added the repair itself as a button next to each of your data sources. Rebuilding an index sends your documents to your embedding provider once more, so AI Studio asks you first and never starts it on its own.
+- Added a check that recognizes your own tokenizer by its content rather than by its file name. Tokenizers are almost always named alike, so swapping one for another is now noticed, and you are asked about it.
 - Added support for several drop areas on the same page. More complex assistants can now receive files or folders by drag and drop at more than one place.
 - Added drag and drop to the input and output folder of the Batch Processing assistant: drop a folder onto either field to choose it.
 - Added ways to load text from a file and drop zones for them, throughout the assistants and dialogs. We went through them one by one, so many fields that used to accept typed text only now take the content of a file as well.
@@ -34,6 +34,10 @@
 - Improved the list of your attached files: every file now appears under the folder it came from, and each folder is named only once, no matter in which order you attached your files.
 - Improved organization-wide provider management: IT departments can now separately prevent users from adding chat, transcription, or embedding providers. The existing master setting still overrides all three provider-specific settings.
 - Improved the provider selection throughout the assistants: when there is nothing to choose from, it now says why. Either you have not set up a provider yet, or none of yours is trusted enough for what you are doing. Before, the list was simply empty.
+- Improved the settings of your embedding providers and data sources. Some of them decide how your documents are read, so changing one means preparing every document all over again. AI Studio now asks before that happens, names the data sources it would affect, and says when a cloud provider charges you for it.
+- Improved the dialogs of your embedding providers and data sources: when a change would mean preparing all your documents again, and you decide against it, nothing is saved and the dialog stays open with your change in front of you, ready to be corrected.
+- Improved the question AI Studio asks before you delete an embedding provider. It now names the data sources depending on that provider, together with what they can still do without it.
+- Improved what the AI is told when it answers from your own documents (RAG): it now learns which page a passage came from, so it can name the page an answer rests on.
 - Changed how provider trust and provider confidence work together. Marking a provider as trustworthy in a configuration no longer also satisfies a required confidence level: one says who runs the provider, the other how confidential it is. Organizations raise a provider's level in their own confidence scheme instead. This applies beyond local data sources, for example, when a model reads a page from your intranet.
 - Fixed the abilities AI Studio assumed for many models. We checked the families against their documentation: some models gained image input, reasoning, or tool calling, others lost an ability they never had.
 - Fixed the Document Analysis assistant freezing while you edited a policy. It needed a change of yours to be saved in the background just as you were making the next one — picking a provider, for instance — which is why it hit some of you again and again and others never at all.

app/Tests/Tools/EmbeddingChangeImpactTests.cs

@@ -0,0 +1,247 @@
+using AIStudio.Provider;
+using AIStudio.Provider.HuggingFace;
+using AIStudio.Settings;
+using AIStudio.Settings.DataModel;
+using AIStudio.Tools.Services;
+
+using Host = AIStudio.Provider.SelfHosted.Host;
+
+namespace AIStudio.Tests.Tools;
+
+/// <summary>
+/// Checks which edits have to be asked about before they are saved.
+/// </summary>
+/// <remarks>
+/// An edit which changes the embedding signature throws away everything indexed for the data sources
+/// behind it, and sends every one of their documents to the embedding provider again. Asking about an
+/// edit which costs nothing trains people to click the question away; not asking about one which does
+/// costs them money at a cloud provider. So both directions are pinned down here.
+/// </remarks>
+[TestFixture]
+public sealed class EmbeddingChangeImpactTests
+{
+    [Test]
+    public void HarmlessEmbeddingProviderEditsKeepTheStoredIndex()
+    {
+        var dataSource = StoredDataSource();
+        var stored = StoredEmbeddingProvider();
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(EmbeddingChangeImpact.AffectsStoredIndex(dataSource, stored, stored with { Name = "Another name" }), Is.False, "The name of an embedding provider reaches no vector.");
+            Assert.That(EmbeddingChangeImpact.AffectsStoredIndex(dataSource, stored, stored with { Num = 42 }), Is.False, "The number is there to sort the list with.");
+            Assert.That(EmbeddingChangeImpact.AffectsStoredIndex(dataSource, stored, stored with { EmbeddingBatchSize = 16 }), Is.False, "How many chunks travel in one request says nothing about the vectors which come back.");
+            Assert.That(EmbeddingChangeImpact.AffectsStoredIndex(dataSource, stored, stored with { CustomIconDataUrl = "data:image/png;base64,AAAA" }), Is.False, "An icon is an icon.");
+        });
+    }
+
+    [Test]
+    public void ChangingTheModelDropsTheStoredIndex()
+    {
+        var dataSource = StoredDataSource();
+        var stored = StoredEmbeddingProvider();
+
+        Assert.That(
+            EmbeddingChangeImpact.AffectsStoredIndex(dataSource, stored, stored with { Model = new("text-embedding-3-large", "text-embedding-3-large") }),
+            Is.True,
+            "Another model means another vector space.");
+    }
+
+    [Test]
+    public void ChangingTheTokenLimitDropsTheStoredIndex()
+    {
+        var dataSource = StoredDataSource();
+        var stored = StoredEmbeddingProvider();
+
+        Assert.That(
+            EmbeddingChangeImpact.AffectsStoredIndex(dataSource, stored, stored with { TokenLimit = 4096 }),
+            Is.True,
+            "The token limit decides where the text is cut, and other chunks are other vectors.");
+    }
+
+    [Test]
+    public void ChangingWhereTheProviderRunsDropsTheStoredIndex()
+    {
+        var dataSource = StoredDataSource();
+        var stored = StoredEmbeddingProvider();
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(EmbeddingChangeImpact.AffectsStoredIndex(dataSource, stored, stored with { Hostname = "http://localhost:9999" }), Is.True, "Another server can serve another model under the same name.");
+            Assert.That(EmbeddingChangeImpact.AffectsStoredIndex(dataSource, stored, stored with { Host = Host.LM_STUDIO }), Is.True, "Another kind of host speaks another API.");
+            Assert.That(EmbeddingChangeImpact.AffectsStoredIndex(dataSource, stored, stored with { HFInferenceProvider = HFInferenceProvider.GROQ }), Is.True, "The same model name served by another backend is another vector source.");
+        });
+    }
+
+    [Test]
+    public void TheTokenizerIsComparedByItsContentNotItsPath()
+    {
+        var dataSource = StoredDataSource();
+        var stored = StoredEmbeddingProvider() with { TokenizerPath = "/data/tokenizers/embeddings/tokenizer.json", TokenizerFingerprint = "AAAA" };
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(EmbeddingChangeImpact.AffectsStoredIndex(dataSource, stored, stored with { TokenizerFingerprint = "BBBB" }), Is.True, "Another tokenizer counts tokens differently, so the text is cut elsewhere.");
+            Assert.That(EmbeddingChangeImpact.AffectsStoredIndex(dataSource, stored, stored with { TokenizerPath = "/somewhere/else/tokenizer.json" }), Is.False, "It is the same tokenizer under another path.");
+        });
+    }
+
+    [Test]
+    public void ChangingTheChunkSettingsOfADataSourceDropsItsStoredIndex()
+    {
+        var embeddingProvider = StoredEmbeddingProvider();
+        var stored = StoredDataSource();
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(EditKeepingTheProvider(embeddingProvider, stored, stored with { MaxChunkTokenLength = 256 }), Is.True, "Other chunk boundaries mean other vectors.");
+            Assert.That(EditKeepingTheProvider(embeddingProvider, stored, stored with { ChunkOverlapTokenLength = 50 }), Is.True, "Another overlap changes what every chunk starts with.");
+        });
+    }
+
+    /// <summary>
+    /// Writing out what a data source already follows must not cost it its index.
+    /// </summary>
+    /// <remarks>
+    /// A token limit of 0 means "follow the embedding provider", and opening the expert settings of a
+    /// data source fills that empty field with exactly the provider's limit. Both are the same cut,
+    /// so nobody may be asked about it -- and above all, nothing may be re-embedded for it. Somebody
+    /// who only wanted to change how many matches an answer may use paid for a full rebuild.
+    /// </remarks>
+    [Test]
+    public void SpellingOutWhatTheProviderAlreadyDictatesKeepsTheStoredIndex()
+    {
+        var embeddingProvider = StoredEmbeddingProvider() with { TokenLimit = 8192 };
+        var followingTheProvider = StoredDataSource() with { MaxChunkTokenLength = 0 };
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(
+                EditKeepingTheProvider(embeddingProvider, followingTheProvider, followingTheProvider with { MaxChunkTokenLength = 8192 }),
+                Is.False,
+                "The provider limit typed into the field is the cut the data source already had.");
+
+            Assert.That(
+                EditKeepingTheProvider(embeddingProvider, followingTheProvider, followingTheProvider with { MaxChunkTokenLength = 4096 }),
+                Is.True,
+                "Anything below the provider limit really does cut the text elsewhere.");
+        });
+    }
+
+    /// <summary>
+    /// An overlap larger than the chunk is capped, so several of them are the same cut.
+    /// </summary>
+    /// <remarks>
+    /// The same reasoning as for the token limit: what counts is where the text is cut, not what
+    /// somebody typed into the field.
+    /// </remarks>
+    [Test]
+    public void AnOverlapWhichIsCappedAnywayKeepsTheStoredIndex()
+    {
+        var embeddingProvider = StoredEmbeddingProvider();
+        var stored = StoredDataSource() with { MaxChunkTokenLength = 512, ChunkOverlapTokenLength = 600 };
+
+        Assert.That(
+            EditKeepingTheProvider(embeddingProvider, stored, stored with { ChunkOverlapTokenLength = 700 }),
+            Is.False,
+            "Both overlaps are capped to the chunk size, so the text is cut identically.");
+    }
+
+    [Test]
+    public void HarmlessDataSourceEditsKeepTheStoredIndex()
+    {
+        var embeddingProvider = StoredEmbeddingProvider();
+        var stored = StoredDataSource();
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(EditKeepingTheProvider(embeddingProvider, stored, stored with { Name = "Another name" }), Is.False, "The name is how the data source is offered, not how it was read.");
+            Assert.That(EditKeepingTheProvider(embeddingProvider, stored, stored with { Description = "Another description" }), Is.False, "The description is there for the agent which picks data sources.");
+            Assert.That(EditKeepingTheProvider(embeddingProvider, stored, stored with { MaxMatches = 42 }), Is.False, "How many matches an answer may use is decided per query.");
+            Assert.That(EditKeepingTheProvider(embeddingProvider, stored, stored with { ConfidenceLevel = ConfidenceLevel.HIGH }), Is.False, "The confidence level is enforced live on every request and changes no vector.");
+        });
+    }
+
+    /// <summary>
+    /// Checks what changing the embedding of a data source costs.
+    /// </summary>
+    /// <remarks>
+    /// This is why each side has to be asked with its own provider: a data source carries only the id
+    /// of its embedding provider, and that id is nowhere in the signature. Asking both sides with the
+    /// same provider would call this edit harmless, while the next indexing run throws everything away.
+    /// </remarks>
+    [Test]
+    public void ChangingTheEmbeddingOfADataSourceDropsItsStoredIndex()
+    {
+        var storedProvider = StoredEmbeddingProvider();
+        var anotherProvider = AnotherEmbeddingProvider();
+        var stored = StoredDataSource();
+        var moved = stored with { EmbeddingId = anotherProvider.Id };
+
+        Assert.That(
+            EmbeddingChangeImpact.AffectsStoredIndex(stored, storedProvider, moved, anotherProvider),
+            Is.True,
+            "Another embedding provider means another vector space, so nothing stored survives it.");
+    }
+
+    /// <summary>
+    /// Putting a data source back to work after its provider was deleted is a rebuild as well.
+    /// </summary>
+    /// <remarks>
+    /// The provider a data source points at can be gone. What is stored was made by it, so pointing
+    /// the source at any provider at all discards that -- and nobody may be surprised by it.
+    /// </remarks>
+    [Test]
+    public void RepointingADataSourceWhoseProviderIsGoneDropsItsStoredIndex()
+    {
+        var stored = StoredDataSource();
+        var anotherProvider = AnotherEmbeddingProvider();
+
+        Assert.That(
+            EmbeddingChangeImpact.AffectsStoredIndex(stored, EmbeddingProvider.NONE, stored with { EmbeddingId = anotherProvider.Id }, anotherProvider),
+            Is.True,
+            "A provider which cannot be resolved stands in as NONE, which is a signature of its own.");
+    }
+
+    [Test]
+    public void KeepingTheEmbeddingKeepsTheStoredIndex()
+    {
+        var storedProvider = StoredEmbeddingProvider();
+        var stored = StoredDataSource();
+
+        Assert.That(
+            EmbeddingChangeImpact.AffectsStoredIndex(stored, storedProvider, stored with { Name = "Another name" }, storedProvider with { Name = "Renamed provider" }),
+            Is.False,
+            "Neither name reaches a vector, and the data source still points at the same provider.");
+    }
+
+    /// <summary>
+    /// Asks the question for an edit which leaves the embedding provider of the data source alone.
+    /// </summary>
+    /// <param name="embeddingProvider">The provider both sides point at.</param>
+    /// <param name="before">The data source as it is stored.</param>
+    /// <param name="after">The data source as it would be stored.</param>
+    /// <returns>True when the stored index would be discarded.</returns>
+    private static bool EditKeepingTheProvider(EmbeddingProvider embeddingProvider, IDataSource before, IDataSource after) =>
+        EmbeddingChangeImpact.AffectsStoredIndex(before, embeddingProvider, after, embeddingProvider);
+
+    private static DataSourceLocalDirectory StoredDataSource() => new()
+    {
+        Num = 1,
+        Id = "6f1d6a4e-6a5e-4c62-9a4f-0f2d2c8b7a11",
+        Name = "Test data",
+        Description = "Documents used by the tests.",
+        Type = DataSourceType.LOCAL_DIRECTORY,
+        EmbeddingId = "b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01",
+        MaxChunkTokenLength = 512,
+        ChunkOverlapTokenLength = 100,
+        ConfidenceLevel = ConfidenceLevel.LOW,
+        Path = "/tmp/test-data",
+    };
+
+    private static EmbeddingProvider StoredEmbeddingProvider() =>
+        new(1, "b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01", "Test embeddings", LLMProviders.OPEN_AI, new("text-embedding-3-small", "text-embedding-3-small"));
+
+    private static EmbeddingProvider AnotherEmbeddingProvider() =>
+        new(2, "c1b5d5e3-2a4f-4b1b-9dae-6b7c8d9e0f12", "Other embeddings", LLMProviders.MISTRAL, new("mistral-embed", "mistral-embed"));
+}

app/Tests/Tools/EmbeddingSignatureTests.cs

@@ -1,4 +1,5 @@
 using AIStudio.Provider;
+using AIStudio.Provider.HuggingFace;
 using AIStudio.Settings;
 using AIStudio.Settings.DataModel;
 using AIStudio.Tools.Services;
@@ -47,11 +48,68 @@ public sealed class EmbeddingSignatureTests
             "Another model means another vector space, so nothing stored may be kept.");
     }
 
+    [Test]
+    public void ChangingTheTokenizerContentDropsTheStoredEmbeddings()
+    {
+        var dataSource = DataSource(ConfidenceLevel.LOW);
+        var oneTokenizer = TokenizerAt("/data/tokenizers/embeddings/tokenizer.json", "AAAA");
+        var anotherTokenizer = oneTokenizer with { TokenizerFingerprint = "BBBB" };
+
+        Assert.That(
+            Signature(dataSource, anotherTokenizer),
+            Is.Not.EqualTo(Signature(dataSource, oneTokenizer)),
+            "Another tokenizer cuts the text at other places. A tokenizer is stored under the name it came with, almost always tokenizer.json, so the path alone would not notice the swap.");
+    }
+
+    [Test]
+    public void MovingTheTokenizerFileKeepsTheStoredEmbeddings()
+    {
+        var dataSource = DataSource(ConfidenceLevel.LOW);
+        var here = TokenizerAt("/data/tokenizers/embeddings/tokenizer.json", "AAAA");
+        var there = here with { TokenizerPath = "/somewhere/else/tokenizers/embeddings/tokenizer.json" };
+
+        Assert.That(
+            Signature(dataSource, there),
+            Is.EqualTo(Signature(dataSource, here)),
+            "It is the same tokenizer and only the data directory moved, so embedding everything again would buy nothing.");
+    }
+
+    [Test]
+    public void ChangingTheHuggingFaceInferenceProviderDropsTheStoredEmbeddings()
+    {
+        var dataSource = DataSource(ConfidenceLevel.LOW);
+        var oneBackend = EmbeddingProviderFor("text-embedding-3-small") with { HFInferenceProvider = HFInferenceProvider.GROQ };
+        var anotherBackend = oneBackend with { HFInferenceProvider = HFInferenceProvider.CEREBRAS };
+
+        Assert.That(
+            Signature(dataSource, anotherBackend),
+            Is.Not.EqualTo(Signature(dataSource, oneBackend)),
+            "The same model name served by another backend is another vector source.");
+    }
+
+    [Test]
+    public void TheSignatureOfAKnownConfigurationIsPinned()
+    {
+        Assert.That(
+            Signature(DataSource(ConfidenceLevel.LOW)),
+            Is.EqualTo("2|b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01|OPEN_AI|text-embedding-3-small|NONE|http://localhost:1234|NONE||8192|512|100"),
+            "Reordering or extending the signature throws away every index anybody has. This test makes that a decision somebody takes rather than something which happens on the way past.");
+    }
+
+    /// <summary>
+    /// Builds the signature the way an indexing run does, working the chunking out along the way.
+    /// </summary>
+    /// <remarks>
+    /// Handing in fixed chunking options instead would hide exactly what these tests are here for:
+    /// the signature would then no longer notice a data source being cut differently.
+    /// </remarks>
+    /// <param name="dataSource">The data source to build the signature for.</param>
+    /// <param name="embeddingProvider">The embedding provider, or the test default.</param>
+    /// <returns>The signature of that pairing.</returns>
     private static string Signature(DataSourceLocalDirectory dataSource, EmbeddingProvider? embeddingProvider = null) =>
         DataSourceEmbeddingService.BuildEmbeddingSignature(
             dataSource,
-            embeddingProvider ?? EmbeddingProviderFor("text-embedding-3-small"),
+            embeddingProvider ?? EmbeddingProviderFor("text-embedding-3-small"));
-            new(512, 100));
 
     private static DataSourceLocalDirectory DataSource(ConfidenceLevel confidenceLevel) => new()
     {
@@ -67,6 +125,9 @@ public sealed class EmbeddingSignatureTests
         Path = "/tmp/test-data",
     };
 
+    private static EmbeddingProvider TokenizerAt(string tokenizerPath, string tokenizerFingerprint) =>
+        EmbeddingProviderFor("text-embedding-3-small") with { TokenizerPath = tokenizerPath, TokenizerFingerprint = tokenizerFingerprint };
+
     private static EmbeddingProvider EmbeddingProviderFor(string modelId) =>
         new(1, "b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01", "Test embeddings", LLMProviders.OPEN_AI, new(modelId, modelId));
 }