made the embedding procedure more robust

app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor

@@ -41,12 +41,18 @@
             <MudJustifiedText Typo="Typo.body1" Class="mb-3">
                 @T("Select a root directory for this data source. All data in this directory and all its subdirectories will be processed for this data source.")
             </MudJustifiedText>
-            <SelectDirectory @bind-Directory="@this.dataPath" Label="@T("Selected base directory for this data source")" DirectoryDialogTitle="@T("Select the base directory")" Validation="@this.dataSourceValidation.ValidatePath" />
+            @if (!this.CanChangeSourceAndEmbedding)
+            {
+                <MudAlert Severity="Severity.Info" Variant="Variant.Outlined" Class="mb-3">
+                    @T("This data source already has indexed embeddings. Delete and recreate it to change the folder path or embedding model.")
+                </MudAlert>
+            }
+            <SelectDirectory @bind-Directory="@this.dataPath" Label="@T("Selected base directory for this data source")" DirectoryDialogTitle="@T("Select the base directory")" Validation="@this.dataSourceValidation.ValidatePath" Disabled="@(!this.CanChangeSourceAndEmbedding)" />
 
             <MudJustifiedText Typo="Typo.body1" Class="mb-3">
                 @T("In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method.")
             </MudJustifiedText>
-            <MudSelect @bind-Value="@this.dataEmbeddingId" Label="@T("Embedding")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateEmbeddingId">
+            <MudSelect @bind-Value="@this.dataEmbeddingId" Label="@T("Embedding")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateEmbeddingId" Disabled="@(!this.CanChangeSourceAndEmbedding)">
                 @foreach (var embedding in this.AvailableEmbeddings)
                 {
                     <MudSelectItem Value="@embedding.Value">
@@ -57,6 +63,18 @@
             
             @if (!string.IsNullOrWhiteSpace(this.dataEmbeddingId))
             {
+                <MudTextField
+                    T="string"
+                    Text="@this.SelectedEmbeddingTokenizerText"
+                    Label="@T("Tokenizer")"
+                    Class="mb-3"
+                    ReadOnly="@true"
+                    Variant="Variant.Outlined"
+                    Adornment="Adornment.Start"
+                    AdornmentIcon="@Icons.Material.Filled.DataObject"
+                    AdornmentColor="Color.Info"
+                />
+
                 if (this.SelectedCloudEmbedding)
                 {
                     <DataSourceCloudEmbeddingWarning DataSourceType="DataSourceType.LOCAL_DIRECTORY" SourcePath="@this.dataPath" @bind-UserAcknowledged="@this.dataUserAcknowledgedCloudEmbedding" Validation="@this.dataSourceValidation.ValidateUserAcknowledgedCloudEmbedding"/>
@@ -80,7 +98,7 @@
                         <MudNumericField
                             T="int"
                             @bind-Value="@this.dataMaxChunkTokenLength"
-                            Label="@T("Max chunk length")"
+                            Label="@T("Token limit")"
                             Class="mb-3"
                             Min="0"
                             Immediate="@true"
@@ -94,7 +112,7 @@
                         <MudNumericField
                             T="int"
                             @bind-Value="@this.dataChunkOverlapTokenLength"
-                            Label="@T("Chunk overlap length")"
+                            Label="@T("Token overlap")"
                             Min="0"
                             Immediate="@true"
                             Validation="@this.ValidateChunkOverlapTokenLength"

app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs

@@ -18,6 +18,9 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
     [Parameter]
     public DataSourceLocalDirectory DataSource { get; set; }
 
+    [Parameter]
+    public bool LockSourceAndEmbedding { get; set; }
+
     [Parameter]
     public IReadOnlyList<ConfigurationSelectData<string>> AvailableEmbeddings { get; set; } = [];
     
@@ -107,6 +110,14 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
 
     private bool SelectedCloudEmbedding => this.SelectedEmbedding is { IsSelfHosted: false };
 
+    private bool CanChangeSourceAndEmbedding => !this.IsEditing || !this.LockSourceAndEmbedding;
+
+    private string SelectedEmbeddingTokenizerText => this.SelectedEmbedding is null
+        ? T("No embedding selected")
+        : string.IsNullOrWhiteSpace(this.SelectedEmbedding.TokenizerPath)
+            ? T("Default tokenizer")
+            : System.IO.Path.GetFileName(this.SelectedEmbedding.TokenizerPath);
+
     private DataSourceLocalDirectory CreateDataSource() => new()
     {
         Id = this.dataId,
@@ -114,8 +125,8 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
         Name = this.dataName,
         Description = this.dataDescription,
         Type = DataSourceType.LOCAL_DIRECTORY,
-        EmbeddingId = this.dataEmbeddingId,
-        Path = this.dataPath,
+        EmbeddingId = this.CanChangeSourceAndEmbedding ? this.dataEmbeddingId : this.DataSource.EmbeddingId,
+        Path = this.CanChangeSourceAndEmbedding ? this.dataPath : this.DataSource.Path,
         MaxChunkTokenLength = this.dataMaxChunkTokenLength,
         ChunkOverlapTokenLength = this.dataChunkOverlapTokenLength,
         SecurityPolicy = this.dataSecurityPolicy,
@@ -139,11 +150,11 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
     private string? ValidateMaxChunkTokenLength(int maxChunkTokenLength)
     {
         if (maxChunkTokenLength < 0)
-            return T("Please enter 0 or a positive max chunk length.");
+            return T("Please enter 0 or a positive token limit.");
 
         var providerMaxChunkTokenLength = this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT;
-        if (maxChunkTokenLength > providerMaxChunkTokenLength)
-            return string.Format(T("The data source max chunk length must be less than or equal to the embedding provider limit ({0})."), providerMaxChunkTokenLength);
+        if (maxChunkTokenLength > 0 && maxChunkTokenLength >= providerMaxChunkTokenLength)
+            return string.Format(T("The data source token limit must be smaller than the embedding provider token limit ({0}). Use 0 to use the provider setting."), providerMaxChunkTokenLength);
 
         return null;
     }
@@ -157,7 +168,7 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
             ? this.dataMaxChunkTokenLength
             : this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT;
         if (chunkOverlapTokenLength >= effectiveMaxChunkTokenLength)
-            return T("The overlap must be smaller than the effective max chunk length.");
+            return T("The overlap must be smaller than the effective token limit.");
 
         return null;
     }

app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor

@@ -41,12 +41,18 @@
             <MudJustifiedText Typo="Typo.body1" Class="mb-3">
                 @T("Select a file for this data source. The content of this file will be processed for the data source.")
             </MudJustifiedText>
-            <SelectFile @bind-File="@this.dataFilePath" Label="@T("Selected file path for this data source")" FileDialogTitle="@T("Select the file")" Validation="@this.dataSourceValidation.ValidateFilePath" />
+            @if (!this.CanChangeSourceAndEmbedding)
+            {
+                <MudAlert Severity="Severity.Info" Variant="Variant.Outlined" Class="mb-3">
+                    @T("This data source already has indexed embeddings. Delete and recreate it to change the file path or embedding model.")
+                </MudAlert>
+            }
+            <SelectFile @bind-File="@this.dataFilePath" Label="@T("Selected file path for this data source")" FileDialogTitle="@T("Select the file")" Validation="@this.dataSourceValidation.ValidateFilePath" Disabled="@(!this.CanChangeSourceAndEmbedding)" />
 
             <MudJustifiedText Typo="Typo.body1" Class="mb-3">
                 @T("In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method.")
             </MudJustifiedText>
-            <MudSelect @bind-Value="@this.dataEmbeddingId" Label="@T("Embedding")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateEmbeddingId">
+            <MudSelect @bind-Value="@this.dataEmbeddingId" Label="@T("Embedding")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateEmbeddingId" Disabled="@(!this.CanChangeSourceAndEmbedding)">
                 @foreach (var embedding in this.AvailableEmbeddings)
                 {
                     <MudSelectItem Value="@embedding.Value">
@@ -57,6 +63,18 @@
             
             @if (!string.IsNullOrWhiteSpace(this.dataEmbeddingId))
             {
+                <MudTextField
+                    T="string"
+                    Text="@this.SelectedEmbeddingTokenizerText"
+                    Label="@T("Tokenizer")"
+                    Class="mb-3"
+                    ReadOnly="@true"
+                    Variant="Variant.Outlined"
+                    Adornment="Adornment.Start"
+                    AdornmentIcon="@Icons.Material.Filled.DataObject"
+                    AdornmentColor="Color.Info"
+                />
+
                 if (this.SelectedCloudEmbedding)
                 {
                     <DataSourceCloudEmbeddingWarning DataSourceType="DataSourceType.LOCAL_FILE" SourcePath="@this.dataFilePath" @bind-UserAcknowledged="@this.dataUserAcknowledgedCloudEmbedding" Validation="@this.dataSourceValidation.ValidateUserAcknowledgedCloudEmbedding"/>
@@ -80,7 +98,7 @@
                         <MudNumericField
                             T="int"
                             @bind-Value="@this.dataMaxChunkTokenLength"
-                            Label="@T("Max chunk length")"
+                            Label="@T("Token limit")"
                             Class="mb-3"
                             Min="0"
                             Immediate="@true"
@@ -94,7 +112,7 @@
                         <MudNumericField
                             T="int"
                             @bind-Value="@this.dataChunkOverlapTokenLength"
-                            Label="@T("Chunk overlap length")"
+                            Label="@T("Token overlap")"
                             Min="0"
                             Immediate="@true"
                             Validation="@this.ValidateChunkOverlapTokenLength"

app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs

@@ -17,6 +17,9 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
     
     [Parameter]
     public DataSourceLocalFile DataSource { get; set; }
+
+    [Parameter]
+    public bool LockSourceAndEmbedding { get; set; }
     
     [Parameter]
     public IReadOnlyList<ConfigurationSelectData<string>> AvailableEmbeddings { get; set; } = [];
@@ -107,6 +110,14 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
 
     private bool SelectedCloudEmbedding => this.SelectedEmbedding is { IsSelfHosted: false };
 
+    private bool CanChangeSourceAndEmbedding => !this.IsEditing || !this.LockSourceAndEmbedding;
+
+    private string SelectedEmbeddingTokenizerText => this.SelectedEmbedding is null
+        ? T("No embedding selected")
+        : string.IsNullOrWhiteSpace(this.SelectedEmbedding.TokenizerPath)
+            ? T("Default tokenizer")
+            : System.IO.Path.GetFileName(this.SelectedEmbedding.TokenizerPath);
+
     private DataSourceLocalFile CreateDataSource() => new()
     {
         Id = this.dataId,
@@ -114,8 +125,8 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
         Name = this.dataName,
         Description = this.dataDescription,
         Type = DataSourceType.LOCAL_FILE,
-        EmbeddingId = this.dataEmbeddingId,
-        FilePath = this.dataFilePath,
+        EmbeddingId = this.CanChangeSourceAndEmbedding ? this.dataEmbeddingId : this.DataSource.EmbeddingId,
+        FilePath = this.CanChangeSourceAndEmbedding ? this.dataFilePath : this.DataSource.FilePath,
         MaxChunkTokenLength = this.dataMaxChunkTokenLength,
         ChunkOverlapTokenLength = this.dataChunkOverlapTokenLength,
         SecurityPolicy = this.dataSecurityPolicy,
@@ -139,11 +150,11 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
     private string? ValidateMaxChunkTokenLength(int maxChunkTokenLength)
     {
         if (maxChunkTokenLength < 0)
-            return T("Please enter 0 or a positive max chunk length.");
+            return T("Please enter 0 or a positive token limit.");
 
         var providerMaxChunkTokenLength = this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT;
-        if (maxChunkTokenLength > providerMaxChunkTokenLength)
-            return string.Format(T("The data source max chunk length must be less than or equal to the embedding provider limit ({0})."), providerMaxChunkTokenLength);
+        if (maxChunkTokenLength > 0 && maxChunkTokenLength >= providerMaxChunkTokenLength)
+            return string.Format(T("The data source token limit must be smaller than the embedding provider token limit ({0}). Use 0 to use the provider setting."), providerMaxChunkTokenLength);
 
         return null;
     }
@@ -157,7 +168,7 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
             ? this.dataMaxChunkTokenLength
             : this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT;
         if (chunkOverlapTokenLength >= effectiveMaxChunkTokenLength)
-            return T("The overlap must be smaller than the effective max chunk length.");
+            return T("The overlap must be smaller than the effective token limit.");
 
         return null;
     }

app/MindWork AI Studio/Dialogs/Settings/SettingsDialogDataSources.razor.cs

@@ -227,6 +227,8 @@ public partial class SettingsDialogDataSources : SettingsDialogBase
             return;
 
         IDataSource? editedDataSource = null;
+        var lockDataSourceIdentity = dataSource is IInternalDataSource
+            && await this.DataSourceEmbeddingService.ShouldLockDataSourceIdentityAsync(dataSource.Id);
         switch (dataSource)
         {
             case DataSourceLocalFile localFile:
@@ -234,6 +236,7 @@ public partial class SettingsDialogDataSources : SettingsDialogBase
                 {
                     { x => x.IsEditing, true },
                     { x => x.DataSource, localFile },
+                    { x => x.LockSourceAndEmbedding, lockDataSourceIdentity },
                     { x => x.AvailableEmbeddings, this.AvailableEmbeddingProviders }
                 };
         
@@ -250,6 +253,7 @@ public partial class SettingsDialogDataSources : SettingsDialogBase
                 {
                     { x => x.IsEditing, true },
                     { x => x.DataSource, localDirectory },
+                    { x => x.LockSourceAndEmbedding, lockDataSourceIdentity },
                     { x => x.AvailableEmbeddings, this.AvailableEmbeddingProviders }
                 };