Added local RAG (#756)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Dialogs/DataSourceERI_V1Dialog.razor

@@ -1,4 +1,5 @@
 @using AIStudio.Settings.DataModel
+@using AIStudio.Tools.Validation
 @using AIStudio.Tools.ERIClient.DataModel
 @inherits MSGComponentBase
 
@@ -11,8 +12,8 @@
                 @bind-Text="@this.dataName"
                 Label="@T("Data Source Name")"
                 Class="mb-6"
-                MaxLength="40"
-                Counter="40"
+                MaxLength="@DataSourceValidation.MAX_NAME_LENGTH"
+                Counter="@DataSourceValidation.MAX_NAME_LENGTH"
                 Immediate="@true"
                 Validation="@this.dataSourceValidation.ValidatingName"
                 Adornment="Adornment.Start"
@@ -119,7 +120,7 @@
                     </MudSelectItem>
                 }
             </MudSelect>
-            
+
             <MudNumericField T="ushort" Min="10" @bind-Value="@this.dataMaxMatches" Label="@T("How many matches do you want at most per query?")" Variant="Variant.Outlined" Step="10" />
             
         </MudForm>

app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor

@@ -1,4 +1,6 @@
 @using AIStudio.Settings.DataModel
+@using AIStudio.Tools.Validation
+@using AIStudio.Provider
 @inherits MSGComponentBase
 
 <MudDialog>
@@ -10,8 +12,8 @@
                 @bind-Text="@this.dataName"
                 Label="@T("Data Source Name")"
                 Class="mb-6"
-                MaxLength="40"
-                Counter="40"
+                MaxLength="@DataSourceValidation.MAX_NAME_LENGTH"
+                Counter="@DataSourceValidation.MAX_NAME_LENGTH"
                 Immediate="@true"
                 Validation="@this.dataSourceValidation.ValidatingName"
                 Adornment="Adornment.Start"
@@ -41,42 +43,73 @@
             <MudJustifiedText Typo="Typo.body1" Class="mb-3">
                 @T("Select a root directory for this data source. All data in this directory and all its subdirectories will be processed for this data source.")
             </MudJustifiedText>
-            <SelectDirectory @bind-Directory="@this.dataPath" Label="@T("Selected base directory for this data source")" DirectoryDialogTitle="@T("Select the base directory")" Validation="@this.dataSourceValidation.ValidatePath" />
+            @if (!this.CanChangeSourceAndEmbedding)
+            {
+                <MudAlert Severity="Severity.Info" Variant="Variant.Outlined" Class="mb-3">
+                    @T("This data source already has indexed embeddings. Delete and recreate it to change the folder path or embedding model.")
+                </MudAlert>
+            }
+            @if (this.CanChangeSourceAndEmbedding)
+            {
+                <SelectDirectory @bind-Directory="@this.dataPath" Label="@T("Selected base directory for this data source")" DirectoryDialogTitle="@T("Select the base directory")" Validation="@this.dataSourceValidation.ValidatePath" EnableDragDrop="true" Layer="@DropLayers.DIALOGS" CatchAllDocuments="true" />
+            }
+            else
+            {
+                <MudTextField
+                    T="string"
+                    Text="@this.dataPath"
+                    Label="@T("Selected base directory for this data source")"
+                    Class="mb-3"
+                    ReadOnly="@true"
+                    Validation="@this.dataSourceValidation.ValidatePath"
+                    Adornment="Adornment.Start"
+                    AdornmentIcon="@Icons.Material.Filled.Folder"
+                    UserAttributes="@SPELLCHECK_ATTRIBUTES"
+                    Variant="Variant.Outlined"
+                />
+            }
 
             <MudJustifiedText Typo="Typo.body1" Class="mb-3">
                 @T("In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method.")
             </MudJustifiedText>
-            <MudSelect @bind-Value="@this.dataEmbeddingId" Label="@T("Embedding")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateEmbeddingId">
-                @foreach (var embedding in this.AvailableEmbeddings)
-                {
-                    <MudSelectItem Value="@embedding.Value">
-                        @if (this.GetEmbeddingProvider(embedding.Value) is { } provider)
-                        {
-                            <ProviderLabel ProviderType="@provider.UsedLLMProvider" CustomIconDataUrl="@provider.CustomIconDataUrl" Text="@embedding.Name" />
-                        }
-                        else
-                        {
-                            @embedding.Name
-                        }
-                    </MudSelectItem>
-                }
-            </MudSelect>
+            @if (this.CanChangeSourceAndEmbedding)
+            {
+                <MudSelect @bind-Value="@this.dataEmbeddingId" Label="@T("Embedding")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateEmbeddingProviderAccess">
+                    @foreach (var embedding in this.AvailableEmbeddings)
+                    {
+                        <MudSelectItem Value="@embedding.Value">
+                            @if (this.GetEmbeddingProvider(embedding.Value) is { } provider)
+                            {
+                                <ProviderLabel ProviderType="@provider.UsedLLMProvider" CustomIconDataUrl="@provider.CustomIconDataUrl" Text="@embedding.Name" />
+                            }
+                            else
+                            {
+                                @embedding.Name
+                            }
+                        </MudSelectItem>
+                    }
+                </MudSelect>
+            }
+            else
+            {
+                <MudTextField
+                    T="string"
+                    Text="@this.SelectedEmbeddingNameText"
+                    Label="@T("Embedding")"
+                    Class="mb-3"
+                    ReadOnly="@true"
+                    Adornment="Adornment.Start"
+                    AdornmentIcon="@Icons.Material.Filled.DataObject"
+                    AdornmentColor="Color.Info"
+                    Variant="Variant.Outlined"
+                />
+            }
             
             @if (!string.IsNullOrWhiteSpace(this.dataEmbeddingId))
             {
                 if (this.SelectedCloudEmbedding)
                 {
-                    <MudJustifiedText Typo="Typo.body1" Color="Color.Error" Class="mb-3">
-                        @if (string.IsNullOrWhiteSpace(this.dataPath))
-                        {
-                            @T("Please note: the embedding you selected runs in the cloud. All your data will be sent to the cloud. Please confirm that you have read and understood this.")
-                        }
-                        else
-                        {
-                            @string.Format(T("Please note: the embedding you selected runs in the cloud. All your data from the folder '{0}' and all its subdirectories will be sent to the cloud. Please confirm that you have read and understood this."), this.dataPath)
-                        }
-                    </MudJustifiedText>
-                    <MudTextSwitch @bind-Value="@this.dataUserAcknowledgedCloudEmbedding" Label="@T("I confirm that I have read and understood the above")" LabelOn="@T("Yes, please send my data to the cloud")" LabelOff="@T("No, I will chose another embedding")" Validation="@this.dataSourceValidation.ValidateUserAcknowledgedCloudEmbedding"/>
+                    <DataSourceCloudEmbeddingWarning DataSourceType="DataSourceType.LOCAL_DIRECTORY" SourcePath="@this.dataPath" @bind-UserAcknowledged="@this.dataUserAcknowledgedCloudEmbedding" Validation="@this.dataSourceValidation.ValidateUserAcknowledgedCloudEmbedding"/>
                 }
                 else
                 {
@@ -88,16 +121,74 @@
             
             <ManagePandocDependency IntroText="@T("For some data types, such as Office files, MindWork AI Studio requires the open-source application Pandoc.")"/>
 
-            <MudSelect @bind-Value="@this.dataSecurityPolicy" Text="@this.dataSecurityPolicy.ToSelectionText()" Label="@T("Your security policy")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateSecurityPolicy">
-                @foreach (var policy in Enum.GetValues<DataSourceSecurity>())
+            <MudSelect @bind-Value="@this.dataConfidenceLevel" Text="@this.dataConfidenceLevel.GetName()" Label="@T("Required provider confidence level")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateDataSourceConfidenceLevel">
+                @foreach (var level in this.ConfidenceLevels)
                 {
-                    <MudSelectItem Value="@policy">
-                        @policy.ToSelectionText()
+                    <MudSelectItem Value="@level.Value">
+                        @level.Name
                     </MudSelectItem>
                 }
             </MudSelect>
             
-            <MudNumericField T="ushort" Min="10" @bind-Value="@this.dataMaxMatches" Label="@T("How many matches do you want at most per query?")" Variant="Variant.Outlined" Step="10" />
+            <MudStack Class="mb-3">
+                <MudButton OnClick="@this.ToggleExpertSettings" Variant="Variant.Text" Color="Color.Primary" StartIcon="@Icons.Material.Filled.Tune">
+                    @(this.showExpertSettings ? T("Hide Expert Settings") : T("Show Expert Settings"))
+                </MudButton>
+                <MudDivider/>
+                <MudCollapse Expanded="@this.showExpertSettings" Class="@this.GetExpertStyles">
+                    @if (!string.IsNullOrWhiteSpace(this.dataEmbeddingId))
+                    {
+                        <MudTextField
+                            T="string"
+                            Text="@this.SelectedEmbeddingTokenizerText"
+                            Label="@T("Tokenizer")"
+                            Class="mb-3"
+                            ReadOnly="@true"
+                            Variant="Variant.Outlined"
+                            Adornment="Adornment.Start"
+                            AdornmentIcon="@Icons.Material.Filled.TextFields"
+                            AdornmentColor="Color.Info"/>
+                    }
+                    <MudJustifiedText Typo="Typo.body1" Class="mb-3">
+                        @T("Optional expert settings for how this data source is split before embedding.")
+                    </MudJustifiedText>
+                    <MudNumericField
+                        T="int"
+                        @bind-Value="@this.dataMaxChunkTokenLength"
+                        Label="@T("Token limit")"
+                        Class="mb-3"
+                        Min="1"
+                        Immediate="@true"
+                        Validation="@this.ValidateMaxChunkTokenLength"
+                        HelperText="@this.MaxChunkTokenLengthHelperText"
+                        Variant="Variant.Outlined"
+                        Adornment="Adornment.Start"
+                        AdornmentIcon="@Icons.Material.Filled.FormatListNumbered"
+                        AdornmentColor="Color.Info"/>
+                    <MudNumericField
+                        T="int"
+                        @bind-Value="@this.dataChunkOverlapTokenLength"
+                        Label="@T("Token overlap")"
+                        Min="0"
+                        Immediate="@true"
+                        Validation="@this.ValidateChunkOverlapTokenLength"
+                        HelperText="@this.ChunkOverlapTokenLengthHelperText"
+                        Variant="Variant.Outlined"
+                        Adornment="Adornment.Start"
+                        AdornmentIcon="@Icons.Material.Filled.CompareArrows"
+                        AdornmentColor="Color.Info"/>
+                    <MudNumericField
+                        T="ushort"
+                        Min="10"
+                        @bind-Value="@this.dataMaxMatches"
+                        Label="@T("How many matches do you want at most per query?")"
+                        Variant="Variant.Outlined"
+                        Step="10"
+                        Adornment="Adornment.Start"
+                        AdornmentIcon="@Icons.Material.Filled.Search"
+                        AdornmentColor="Color.Info"/>
+                </MudCollapse>
+            </MudStack>
         </MudForm>
         <Issues IssuesData="@this.dataIssues"/>
     </DialogContent>

app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs

@@ -1,6 +1,8 @@
 using AIStudio.Components;
+using AIStudio.Provider;
 using AIStudio.Settings;
 using AIStudio.Settings.DataModel;
+using AIStudio.Tools.Services;
 using AIStudio.Tools.Validation;
 
 using Microsoft.AspNetCore.Components;
@@ -18,6 +20,9 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
     [Parameter]
     public DataSourceLocalDirectory DataSource { get; set; }
 
+    [Parameter]
+    public bool LockSourceAndEmbedding { get; set; }
+
     [Parameter]
     public IReadOnlyList<ConfigurationSelectData<string>> AvailableEmbeddings { get; set; } = [];
     
@@ -41,8 +46,11 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
     private bool dataUserAcknowledgedCloudEmbedding;
     private string dataEmbeddingId = string.Empty;
     private string dataPath = string.Empty;
+    private int dataMaxChunkTokenLength;
+    private int dataChunkOverlapTokenLength = DataSourceEmbeddingService.DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH;
     private ushort dataMaxMatches = 10;
-    private DataSourceSecurity dataSecurityPolicy;
+    private bool showExpertSettings;
+    private ConfidenceLevel dataConfidenceLevel = ConfidenceLevel.UNKNOWN;
     
     // We get the form reference from Blazor code to validate it manually:
     private MudForm form = null!;
@@ -52,6 +60,9 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
         this.dataSourceValidation = new()
         {
             GetSelectedCloudEmbedding = () => this.SelectedCloudEmbedding,
+            GetSelectedEmbeddingProvider = () => this.SelectedEmbedding,
+            GetConfidenceLevel = () => this.dataConfidenceLevel,
+            GetSettingsManager = () => this.SettingsManager,
             GetPreviousDataSourceName = () => this.dataEditingPreviousInstanceName,
             GetUsedDataSourceNames = () => this.UsedDataSourcesNames,
         };
@@ -77,7 +88,9 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
             this.dataDescription = this.DataSource.Description;
             this.dataEmbeddingId = this.DataSource.EmbeddingId;
             this.dataPath = this.DataSource.Path;
-            this.dataSecurityPolicy = this.DataSource.SecurityPolicy;
+            this.dataMaxChunkTokenLength = this.DataSource.MaxChunkTokenLength;
+            this.dataChunkOverlapTokenLength = this.DataSource.ChunkOverlapTokenLength;
+            this.dataConfidenceLevel = this.DataSource.ConfidenceLevel;
             this.dataMaxMatches = this.DataSource.MaxMatches;
         }
         
@@ -102,7 +115,39 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
         return provider == EmbeddingProvider.NONE ? null : provider;
     }
     
-    private bool SelectedCloudEmbedding => !(this.SettingsManager.ConfigurationData.EmbeddingProviders.FirstOrDefault(x => x.Id == this.dataEmbeddingId)?.IsTrustedForDataSourceSecurityChecks(this.SettingsManager) ?? false);
+    private EmbeddingProvider? SelectedEmbedding => this.SettingsManager.ConfigurationData.EmbeddingProviders
+        .FirstOrDefault(x => x.Id == this.dataEmbeddingId);
+
+    private bool SelectedCloudEmbedding => this.SelectedEmbedding is { IsSelfHosted: false };
+
+    private bool CanChangeSourceAndEmbedding => !this.IsEditing || !this.LockSourceAndEmbedding;
+
+    private IEnumerable<ConfigurationSelectData<ConfidenceLevel>> ConfidenceLevels => ConfigurationSelectDataFactory.GetDataSourceConfidenceLevelsData();
+
+    private string SelectedEmbeddingNameText
+    {
+        get
+        {
+            var selectedEmbedding = this.AvailableEmbeddings.FirstOrDefault(x => x.Value == this.dataEmbeddingId);
+            return string.IsNullOrWhiteSpace(selectedEmbedding.Name) ? T("Unknown") : selectedEmbedding.Name;
+        }
+    }
+
+    private string SelectedEmbeddingTokenizerText => this.SelectedEmbedding is null
+        ? T("No embedding selected")
+        : string.IsNullOrWhiteSpace(this.SelectedEmbedding.TokenizerPath)
+            ? T("Default tokenizer")
+            : Path.GetFileName(this.SelectedEmbedding.TokenizerPath);
+
+    private int ProviderMaxChunkTokenLength => this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT;
+
+    private string MaxChunkTokenLengthHelperText => string.Format(
+        T("Maximum number of tokens per chunk for this data source. The embedding provider default is {0} tokens."),
+        this.ProviderMaxChunkTokenLength);
+
+    private string ChunkOverlapTokenLengthHelperText => string.Format(
+        T("Number of tokens repeated at the start of the next chunk. The default overlap is {0} tokens."),
+        DataSourceEmbeddingService.DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH);
 
     private DataSourceLocalDirectory CreateDataSource() => new()
     {
@@ -111,9 +156,11 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
         Name = this.dataName,
         Description = this.dataDescription,
         Type = DataSourceType.LOCAL_DIRECTORY,
-        EmbeddingId = this.dataEmbeddingId,
-        Path = this.dataPath,
-        SecurityPolicy = this.dataSecurityPolicy,
+        EmbeddingId = this.CanChangeSourceAndEmbedding ? this.dataEmbeddingId : this.DataSource.EmbeddingId,
+        Path = this.CanChangeSourceAndEmbedding ? this.dataPath : this.DataSource.Path,
+        MaxChunkTokenLength = this.dataMaxChunkTokenLength,
+        ChunkOverlapTokenLength = this.dataChunkOverlapTokenLength,
+        ConfidenceLevel = this.dataConfidenceLevel,
         MaxMatches = this.dataMaxMatches,
     };
 
@@ -130,4 +177,45 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
     }
     
     private void Cancel() => this.MudDialog.Cancel();
+
+    private string? ValidateMaxChunkTokenLength(int maxChunkTokenLength)
+    {
+        if (!this.showExpertSettings)
+            return null;
+
+        if (maxChunkTokenLength < 1)
+            return T("Please enter a token limit of at least 1.");
+
+        var providerMaxChunkTokenLength = this.ProviderMaxChunkTokenLength;
+        if (maxChunkTokenLength > providerMaxChunkTokenLength)
+            return string.Format(T("The data source token limit must not be larger than the embedding provider token limit ({0})."), providerMaxChunkTokenLength);
+
+        return null;
+    }
+
+    private string? ValidateChunkOverlapTokenLength(int chunkOverlapTokenLength)
+    {
+        if (!this.showExpertSettings)
+            return null;
+
+        if (chunkOverlapTokenLength < 0)
+            return T("Please enter 0 or a positive overlap length.");
+
+        var effectiveMaxChunkTokenLength = this.showExpertSettings && this.dataMaxChunkTokenLength > 0
+            ? this.dataMaxChunkTokenLength
+            : this.ProviderMaxChunkTokenLength;
+        if (chunkOverlapTokenLength >= effectiveMaxChunkTokenLength)
+            return T("The overlap must be smaller than the effective token limit.");
+
+        return null;
+    }
+
+    private void ToggleExpertSettings()
+    {
+        this.showExpertSettings = !this.showExpertSettings;
+        if (this.showExpertSettings && this.dataMaxChunkTokenLength < 1)
+            this.dataMaxChunkTokenLength = this.ProviderMaxChunkTokenLength;
+    }
+
+    private string GetExpertStyles => this.showExpertSettings ? "border-2 border-dashed rounded pa-2" : string.Empty;
 }

app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryInfoDialog.razor

@@ -1,4 +1,4 @@
-@using AIStudio.Settings.DataModel
+@using AIStudio.Provider
 @inherits MSGComponentBase
 
 <MudDialog>
@@ -37,7 +37,7 @@
             </MudJustifiedText>
         }
 
-        <TextInfoLines Label="@T("Your security policy")" MaxLines="3" Value="@this.DataSource.SecurityPolicy.ToInfoText()" Color="@this.DataSource.SecurityPolicy.GetColor()" ClipboardTooltipSubject="@T("your security policy")"/>
+        <TextInfoLine Label="@T("Required provider confidence level")" Value="@this.DataSource.ConfidenceLevel.GetName()" ClipboardTooltipSubject="@T("the required provider confidence level")"/>
         <TextInfoLine Label="@T("Maximum matches per query")" Value="@this.DataSource.MaxMatches.ToString()" ClipboardTooltipSubject="@T("the maximum number of matches per query")"/>
 
         <TextInfoLine Icon="@Icons.Material.Filled.SquareFoot" Label="@T("Number of files")" Value="@this.NumberFilesInDirectory" ClipboardTooltipSubject="@T("the number of files in the directory")"/>

app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor

@@ -1,4 +1,6 @@
 @using AIStudio.Settings.DataModel
+@using AIStudio.Tools.Validation
+@using AIStudio.Provider
 @inherits MSGComponentBase
 
 <MudDialog>
@@ -10,8 +12,8 @@
                 @bind-Text="@this.dataName"
                 Label="@T("Data Source Name")"
                 Class="mb-6"
-                MaxLength="40"
-                Counter="40"
+                MaxLength="@DataSourceValidation.MAX_NAME_LENGTH"
+                Counter="@DataSourceValidation.MAX_NAME_LENGTH"
                 Immediate="@true"
                 Validation="@this.dataSourceValidation.ValidatingName"
                 Adornment="Adornment.Start"
@@ -41,42 +43,73 @@
             <MudJustifiedText Typo="Typo.body1" Class="mb-3">
                 @T("Select a file for this data source. The content of this file will be processed for the data source.")
             </MudJustifiedText>
-            <SelectFile @bind-File="@this.dataFilePath" Label="@T("Selected file path for this data source")" FileDialogTitle="@T("Select the file")" Validation="@this.dataSourceValidation.ValidateFilePath" />
+            @if (!this.CanChangeSourceAndEmbedding)
+            {
+                <MudAlert Severity="Severity.Info" Variant="Variant.Outlined" Class="mb-3">
+                    @T("This data source already has indexed embeddings. Delete and recreate it to change the file path or embedding model.")
+                </MudAlert>
+            }
+            @if (this.CanChangeSourceAndEmbedding)
+            {
+                <SelectFile @bind-File="@this.dataFilePath" Label="@T("Selected file path for this data source")" FileDialogTitle="@T("Select the file")" Validation="@this.dataSourceValidation.ValidateFilePath" EnableDragDrop="true" Layer="@DropLayers.DIALOGS" CatchAllDocuments="true" />
+            }
+            else
+            {
+                <MudTextField
+                    T="string"
+                    Text="@this.dataFilePath"
+                    Label="@T("Selected file path for this data source")"
+                    Class="mb-3"
+                    ReadOnly="@true"
+                    Validation="@this.dataSourceValidation.ValidateFilePath"
+                    Adornment="Adornment.Start"
+                    AdornmentIcon="@Icons.Material.Filled.AttachFile"
+                    UserAttributes="@SPELLCHECK_ATTRIBUTES"
+                    Variant="Variant.Outlined"
+                />
+            }
 
             <MudJustifiedText Typo="Typo.body1" Class="mb-3">
                 @T("In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method.")
             </MudJustifiedText>
-            <MudSelect @bind-Value="@this.dataEmbeddingId" Label="@T("Embedding")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.GetSelectedCloudEmbedding">
-                @foreach (var embedding in this.AvailableEmbeddings)
-                {
-                    <MudSelectItem Value="@embedding.Value">
-                        @if (this.GetEmbeddingProvider(embedding.Value) is { } provider)
-                        {
-                            <ProviderLabel ProviderType="@provider.UsedLLMProvider" CustomIconDataUrl="@provider.CustomIconDataUrl" Text="@embedding.Name" />
-                        }
-                        else
-                        {
-                            @embedding.Name
-                        }
-                    </MudSelectItem>
-                }
-            </MudSelect>
+            @if (this.CanChangeSourceAndEmbedding)
+            {
+                <MudSelect @bind-Value="@this.dataEmbeddingId" Label="@T("Embedding")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateEmbeddingProviderAccess">
+                    @foreach (var embedding in this.AvailableEmbeddings)
+                    {
+                        <MudSelectItem Value="@embedding.Value">
+                            @if (this.GetEmbeddingProvider(embedding.Value) is { } provider)
+                            {
+                                <ProviderLabel ProviderType="@provider.UsedLLMProvider" CustomIconDataUrl="@provider.CustomIconDataUrl" Text="@embedding.Name" />
+                            }
+                            else
+                            {
+                                @embedding.Name
+                            }
+                        </MudSelectItem>
+                    }
+                </MudSelect>
+            }
+            else
+            {
+                <MudTextField
+                    T="string"
+                    Text="@this.SelectedEmbeddingNameText"
+                    Label="@T("Embedding")"
+                    Class="mb-3"
+                    ReadOnly="@true"
+                    Adornment="Adornment.Start"
+                    AdornmentIcon="@Icons.Material.Filled.DataObject"
+                    AdornmentColor="Color.Info"
+                    Variant="Variant.Outlined"
+                />
+            }
             
             @if (!string.IsNullOrWhiteSpace(this.dataEmbeddingId))
             {
                 if (this.SelectedCloudEmbedding)
                 {
-                    <MudJustifiedText Typo="Typo.body1" Color="Color.Error" Class="mb-3">
-                        @if (string.IsNullOrWhiteSpace(this.dataFilePath))
-                        {
-                            @T("Please note: the embedding you selected runs in the cloud. All your data will be sent to the cloud. Please confirm that you have read and understood this.")
-                        }
-                        else
-                        {
-                            @string.Format(T("Please note: the embedding you selected runs in the cloud. All your data within the file '{0}' will be sent to the cloud. Please confirm that you have read and understood this."), this.dataFilePath)
-                        }
-                    </MudJustifiedText>
-                    <MudTextSwitch @bind-Value="@this.dataUserAcknowledgedCloudEmbedding" Label="@T("I confirm that I have read and understood the above")" LabelOn="@T("Yes, please send my data to the cloud")" LabelOff="@T("No, I will chose another embedding")" Validation="@this.dataSourceValidation.ValidateUserAcknowledgedCloudEmbedding"/>
+                    <DataSourceCloudEmbeddingWarning DataSourceType="DataSourceType.LOCAL_FILE" SourcePath="@this.dataFilePath" @bind-UserAcknowledged="@this.dataUserAcknowledgedCloudEmbedding" Validation="@this.dataSourceValidation.ValidateUserAcknowledgedCloudEmbedding"/>
                 }
                 else
                 {
@@ -88,16 +121,74 @@
             
             <ManagePandocDependency IntroText="@T("For some data types, such as Office files, MindWork AI Studio requires the open-source application Pandoc.")"/>
 
-            <MudSelect @bind-Value="@this.dataSecurityPolicy" Text="@this.dataSecurityPolicy.ToSelectionText()" Label="@T("Your security policy")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateSecurityPolicy">
-                @foreach (var policy in Enum.GetValues<DataSourceSecurity>())
+            <MudSelect @bind-Value="@this.dataConfidenceLevel" Text="@this.dataConfidenceLevel.GetName()" Label="@T("Required provider confidence level")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateDataSourceConfidenceLevel">
+                @foreach (var level in this.ConfidenceLevels)
                 {
-                    <MudSelectItem Value="@policy">
-                        @policy.ToSelectionText()
+                    <MudSelectItem Value="@level.Value">
+                        @level.Name
                     </MudSelectItem>
                 }
             </MudSelect>
-            
-            <MudNumericField T="ushort" Min="10" @bind-Value="@this.dataMaxMatches" Label="@T("How many matches do you want at most per query?")" Variant="Variant.Outlined" Step="10" />
+
+            <MudStack Class="mb-3">
+                <MudButton OnClick="@this.ToggleExpertSettings" Variant="Variant.Text" Color="Color.Primary" StartIcon="@Icons.Material.Filled.Tune">
+                    @(this.showExpertSettings ? T("Hide Expert Settings") : T("Show Expert Settings"))
+                </MudButton>
+                <MudDivider/>
+                <MudCollapse Expanded="@this.showExpertSettings" Class="@this.GetExpertStyles">
+                    @if (!string.IsNullOrWhiteSpace(this.dataEmbeddingId))
+                    {
+                        <MudTextField
+                            T="string"
+                            Text="@this.SelectedEmbeddingTokenizerText"
+                            Label="@T("Tokenizer")"
+                            Class="mb-3"
+                            ReadOnly="@true"
+                            Variant="Variant.Outlined"
+                            Adornment="Adornment.Start"
+                            AdornmentIcon="@Icons.Material.Filled.TextFields"
+                            AdornmentColor="Color.Info"/>
+                    }
+                    <MudJustifiedText Typo="Typo.body1" Class="mb-3">
+                        @T("Optional expert settings for how this data source is split before embedding.")
+                    </MudJustifiedText>
+                    <MudNumericField
+                        T="int"
+                        @bind-Value="@this.dataMaxChunkTokenLength"
+                        Label="@T("Token limit")"
+                        Class="mb-3"
+                        Min="1"
+                        Immediate="@true"
+                        Validation="@this.ValidateMaxChunkTokenLength"
+                        HelperText="@this.MaxChunkTokenLengthHelperText"
+                        Variant="Variant.Outlined"
+                        Adornment="Adornment.Start"
+                        AdornmentIcon="@Icons.Material.Filled.FormatListNumbered"
+                        AdornmentColor="Color.Info"/>
+                    <MudNumericField
+                        T="int"
+                        @bind-Value="@this.dataChunkOverlapTokenLength"
+                        Label="@T("Token overlap")"
+                        Min="0"
+                        Immediate="@true"
+                        Validation="@this.ValidateChunkOverlapTokenLength"
+                        HelperText="@this.ChunkOverlapTokenLengthHelperText"
+                        Variant="Variant.Outlined"
+                        Adornment="Adornment.Start"
+                        AdornmentIcon="@Icons.Material.Filled.CompareArrows"
+                        AdornmentColor="Color.Info"/>
+                    <MudNumericField
+                        T="ushort"
+                        Min="10"
+                        @bind-Value="@this.dataMaxMatches"
+                        Label="@T("How many matches do you want at most per query?")"
+                        Variant="Variant.Outlined"
+                        Step="10"
+                        Adornment="Adornment.Start"
+                        AdornmentIcon="@Icons.Material.Filled.Search"
+                        AdornmentColor="Color.Info"/>
+                </MudCollapse>
+            </MudStack>
         </MudForm>
         <Issues IssuesData="@this.dataIssues"/>
     </DialogContent>

app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs

@@ -1,6 +1,8 @@
 using AIStudio.Components;
+using AIStudio.Provider;
 using AIStudio.Settings;
 using AIStudio.Settings.DataModel;
+using AIStudio.Tools.Services;
 using AIStudio.Tools.Validation;
 
 using Microsoft.AspNetCore.Components;
@@ -17,6 +19,9 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
     
     [Parameter]
     public DataSourceLocalFile DataSource { get; set; }
+
+    [Parameter]
+    public bool LockSourceAndEmbedding { get; set; }
     
     [Parameter]
     public IReadOnlyList<ConfigurationSelectData<string>> AvailableEmbeddings { get; set; } = [];
@@ -41,8 +46,11 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
     private bool dataUserAcknowledgedCloudEmbedding;
     private string dataEmbeddingId = string.Empty;
     private string dataFilePath = string.Empty;
+    private int dataMaxChunkTokenLength;
+    private int dataChunkOverlapTokenLength = DataSourceEmbeddingService.DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH;
     private ushort dataMaxMatches = 10;
-    private DataSourceSecurity dataSecurityPolicy;
+    private bool showExpertSettings;
+    private ConfidenceLevel dataConfidenceLevel = ConfidenceLevel.UNKNOWN;
     
     // We get the form reference from Blazor code to validate it manually:
     private MudForm form = null!;
@@ -52,6 +60,9 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
         this.dataSourceValidation = new()
         {
             GetSelectedCloudEmbedding = () => this.SelectedCloudEmbedding,
+            GetSelectedEmbeddingProvider = () => this.SelectedEmbedding,
+            GetConfidenceLevel = () => this.dataConfidenceLevel,
+            GetSettingsManager = () => this.SettingsManager,
             GetPreviousDataSourceName = () => this.dataEditingPreviousInstanceName,
             GetUsedDataSourceNames = () => this.UsedDataSourcesNames,
         };
@@ -77,7 +88,9 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
             this.dataDescription = this.DataSource.Description;
             this.dataEmbeddingId = this.DataSource.EmbeddingId;
             this.dataFilePath = this.DataSource.FilePath;
-            this.dataSecurityPolicy = this.DataSource.SecurityPolicy;
+            this.dataMaxChunkTokenLength = this.DataSource.MaxChunkTokenLength;
+            this.dataChunkOverlapTokenLength = this.DataSource.ChunkOverlapTokenLength;
+            this.dataConfidenceLevel = this.DataSource.ConfidenceLevel;
             this.dataMaxMatches = this.DataSource.MaxMatches;
         }
         
@@ -102,7 +115,39 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
         return provider == EmbeddingProvider.NONE ? null : provider;
     }
     
-    private bool SelectedCloudEmbedding => !(this.SettingsManager.ConfigurationData.EmbeddingProviders.FirstOrDefault(x => x.Id == this.dataEmbeddingId)?.IsTrustedForDataSourceSecurityChecks(this.SettingsManager) ?? false);
+    private EmbeddingProvider? SelectedEmbedding => this.SettingsManager.ConfigurationData.EmbeddingProviders
+        .FirstOrDefault(x => x.Id == this.dataEmbeddingId);
+
+    private bool SelectedCloudEmbedding => this.SelectedEmbedding is { IsSelfHosted: false };
+
+    private bool CanChangeSourceAndEmbedding => !this.IsEditing || !this.LockSourceAndEmbedding;
+
+    private IEnumerable<ConfigurationSelectData<ConfidenceLevel>> ConfidenceLevels => ConfigurationSelectDataFactory.GetDataSourceConfidenceLevelsData();
+
+    private string SelectedEmbeddingNameText
+    {
+        get
+        {
+            var selectedEmbedding = this.AvailableEmbeddings.FirstOrDefault(x => x.Value == this.dataEmbeddingId);
+            return string.IsNullOrWhiteSpace(selectedEmbedding.Name) ? T("Unknown") : selectedEmbedding.Name;
+        }
+    }
+
+    private string SelectedEmbeddingTokenizerText => this.SelectedEmbedding is null
+        ? T("No embedding selected")
+        : string.IsNullOrWhiteSpace(this.SelectedEmbedding.TokenizerPath)
+            ? T("Default tokenizer")
+            : Path.GetFileName(this.SelectedEmbedding.TokenizerPath);
+
+    private int ProviderMaxChunkTokenLength => this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT;
+
+    private string MaxChunkTokenLengthHelperText => string.Format(
+        T("Maximum number of tokens per chunk for this data source. The embedding provider default is {0} tokens."),
+        this.ProviderMaxChunkTokenLength);
+
+    private string ChunkOverlapTokenLengthHelperText => string.Format(
+        T("Number of tokens repeated at the start of the next chunk. The default overlap is {0} tokens."),
+        DataSourceEmbeddingService.DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH);
 
     private DataSourceLocalFile CreateDataSource() => new()
     {
@@ -111,9 +156,11 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
         Name = this.dataName,
         Description = this.dataDescription,
         Type = DataSourceType.LOCAL_FILE,
-        EmbeddingId = this.dataEmbeddingId,
-        FilePath = this.dataFilePath,
-        SecurityPolicy = this.dataSecurityPolicy,
+        EmbeddingId = this.CanChangeSourceAndEmbedding ? this.dataEmbeddingId : this.DataSource.EmbeddingId,
+        FilePath = this.CanChangeSourceAndEmbedding ? this.dataFilePath : this.DataSource.FilePath,
+        MaxChunkTokenLength = this.dataMaxChunkTokenLength,
+        ChunkOverlapTokenLength = this.dataChunkOverlapTokenLength,
+        ConfidenceLevel = this.dataConfidenceLevel,
         MaxMatches = this.dataMaxMatches,
     };
     
@@ -130,4 +177,45 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
     }
     
     private void Cancel() => this.MudDialog.Cancel();
+
+    private string? ValidateMaxChunkTokenLength(int maxChunkTokenLength)
+    {
+        if (!this.showExpertSettings)
+            return null;
+
+        if (maxChunkTokenLength < 1)
+            return T("Please enter a token limit of at least 1.");
+
+        var providerMaxChunkTokenLength = this.ProviderMaxChunkTokenLength;
+        if (maxChunkTokenLength > providerMaxChunkTokenLength)
+            return string.Format(T("The data source token limit must not be larger than the embedding provider token limit ({0})."), providerMaxChunkTokenLength);
+
+        return null;
+    }
+
+    private string? ValidateChunkOverlapTokenLength(int chunkOverlapTokenLength)
+    {
+        if (!this.showExpertSettings)
+            return null;
+
+        if (chunkOverlapTokenLength < 0)
+            return T("Please enter 0 or a positive overlap length.");
+
+        var effectiveMaxChunkTokenLength = this.showExpertSettings && this.dataMaxChunkTokenLength > 0
+            ? this.dataMaxChunkTokenLength
+            : this.ProviderMaxChunkTokenLength;
+        if (chunkOverlapTokenLength >= effectiveMaxChunkTokenLength)
+            return T("The overlap must be smaller than the effective token limit.");
+
+        return null;
+    }
+
+    private void ToggleExpertSettings()
+    {
+        this.showExpertSettings = !this.showExpertSettings;
+        if (this.showExpertSettings && this.dataMaxChunkTokenLength < 1)
+            this.dataMaxChunkTokenLength = this.ProviderMaxChunkTokenLength;
+    }
+
+    private string GetExpertStyles => this.showExpertSettings ? "border-2 border-dashed rounded pa-2" : string.Empty;
 }

app/MindWork AI Studio/Dialogs/DataSourceLocalFileInfoDialog.razor

@@ -1,4 +1,4 @@
-@using AIStudio.Settings.DataModel
+@using AIStudio.Provider
 @inherits MSGComponentBase
 
 <MudDialog>
@@ -37,7 +37,7 @@
             </MudJustifiedText>
         }
 
-        <TextInfoLines Label="@T("Your security policy")" MaxLines="3" Value="@this.DataSource.SecurityPolicy.ToInfoText()" Color="@this.DataSource.SecurityPolicy.GetColor()" ClipboardTooltipSubject="@T("your security policy")"/>
+        <TextInfoLine Label="@T("Required provider confidence level")" Value="@this.DataSource.ConfidenceLevel.GetName()" ClipboardTooltipSubject="@T("the required provider confidence level")"/>
         <TextInfoLine Label="@T("Maximum matches per query")" Value="@this.DataSource.MaxMatches.ToString()" ClipboardTooltipSubject="@T("the maximum number of matches per query")"/>
         <TextInfoLine Icon="@Icons.Material.Filled.SquareFoot" Label="@T("File size")" Value="@this.FileSize" ClipboardTooltipSubject="@T("the file size")"/>
     </DialogContent>

app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor

@@ -14,7 +14,7 @@
         <MudForm @ref="@this.form" @bind-IsValid="@this.dataIsValid" @bind-Errors="@this.dataIssues">
             <MudStack Row="@true" AlignItems="AlignItems.Center">
                 @* ReSharper disable once CSharpWarnings::CS8974 *@
-                <MudSelect @bind-Value="@this.DataLLMProvider" Label="@T("Provider")" Class="mb-3" Disabled="@this.IsEnterpriseConfiguration" Validation="@this.providerValidation.ValidatingProvider">
+                <MudSelect @bind-Value="@this.DataLLMProvider" Label="@T("Provider")" Class="mb-3" OpenIcon="@Icons.Material.Filled.AccountBalance" AdornmentColor="Color.Info" Adornment="Adornment.Start" Disabled="@this.IsEnterpriseConfiguration" Validation="@this.providerValidation.ValidatingProvider">
                     @foreach (LLMProviders provider in Enum.GetValues(typeof(LLMProviders)))
                     {
                         if (provider.ProvideEmbeddingAPI() || provider is LLMProviders.NONE)
@@ -29,7 +29,7 @@
                     @T("Create account")
                 </MudButton>
             </MudStack>
-            
+
             @if (this.DataLLMProvider.IsAPIKeyNeeded(this.DataHost))
             {
                 <SecretInputField Secret="@this.dataAPIKey" SecretChanged="@this.OnAPIKeyChanged" Label="@this.APIKeyText" Validation="@this.providerValidation.ValidatingAPIKey"/>
@@ -98,15 +98,14 @@
                             Disabled="@this.IsEnterpriseConfiguration"
                             Validation="@this.ValidateManuallyModel"
                             UserAttributes="@SPELLCHECK_ATTRIBUTES"
-                            HelperText="@T("Currently, we cannot query the embedding models for the selected provider and/or host. Therefore, please enter the model name manually.")"
-                        />
+                            HelperText="@T("Currently, we cannot query the embedding models for the selected provider and/or host. Therefore, please enter the model name manually.")"/>
                     }
                     else
                     {
                         <MudButton Disabled="@(this.IsEnterpriseConfiguration || !this.DataLLMProvider.CanLoadModels(this.DataHost, this.dataAPIKey))" Variant="Variant.Filled" Size="Size.Small" StartIcon="@Icons.Material.Filled.Refresh" OnClick="@this.ReloadModels">
                             @T("Load")
                         </MudButton>
-                        @if(this.availableModels.Count is 0)
+                        @if (this.availableModels.Count is 0)
                         {
                             <MudText Typo="Typo.body1">
                                 @T("No models loaded or available.")
@@ -150,18 +149,77 @@
                 AdornmentColor="Color.Info"
                 Disabled="@this.IsEnterpriseConfiguration"
                 Validation="@this.providerValidation.ValidatingInstanceName"
-                UserAttributes="@SPELLCHECK_ATTRIBUTES"
-            />
-            
+            UserAttributes="@SPELLCHECK_ATTRIBUTES"/>
+            @if (this.DataLLMProvider != LLMProviders.NONE)
+            {
+                <MudStack>
+                    <MudButton OnClick="@this.ToggleExpertSettings">
+                        @(this.showExpertSettings ? T("Hide Expert Settings") : T("Show Expert Settings"))
+                    </MudButton>
+                    <MudDivider />
+                    <MudCollapse Expanded="@this.showExpertSettings" Class="@this.GetExpertStyles">
+                        <MudJustifiedText Typo="Typo.body1" Class="mb-3">
+                            @T("Please be aware: This section is for experts only. For cloud providers, the selected tokenizer and chunk settings may not match the real embedding model limits exactly.")
+                        </MudJustifiedText>
+                        <MudNumericField
+                            T="int"
+                            @bind-Value="@this.DataTokenLimit"
+                            Label="@T("Token limit")"
+                            Class="mb-3"
+                            Min="1"
+                            Immediate="@true"
+                            Adornment="Adornment.Start"
+                            AdornmentIcon="@Icons.Material.Filled.Numbers"
+                            AdornmentColor="Color.Info"
+                            Validation="@this.ValidateTokenLimit"
+                            HelperText="@T("Maximum number of tokens sent to the embedding model per chunk. The default is 8,192.")"/>
+                        <MudNumericField
+                            T="int"
+                            @bind-Value="@this.DataEmbeddingBatchSize"
+                            Label="@T("Embedding batch size")"
+                            Class="mb-3"
+                            Min="1"
+                            Immediate="@true"
+                            Adornment="Adornment.Start"
+                            AdornmentIcon="@Icons.Material.Filled.FormatListNumbered"
+                            AdornmentColor="Color.Info"
+                            Validation="@this.ValidateEmbeddingBatchSize"
+                            HelperText="@T("How many chunks are sent to the embedding provider at once. The default is 1.")"/>
+                        <MudStack Row="@true" Spacing="3" Class="mb-3" StretchItems="StretchItems.None" AlignItems="AlignItems.Center">
+                            <MudTextField
+                                T="string"
+                                Text="@this.dataFilePath"
+                                TextChanged="@this.OnDataFilePathChanged"
+                                Label="@T("Selected file path for the custom tokenizer")"
+                                Validation="@this.providerValidation.ValidatingCustomTokenizer"
+                                Adornment="Adornment.Start"
+                                AdornmentIcon="@Icons.Material.Filled.AttachFile"
+                                UserAttributes="@SPELLCHECK_ATTRIBUTES"
+                                Variant="Variant.Outlined"
+                                Clearable="@true"
+                                Error="@(!string.IsNullOrWhiteSpace(this.dataCustomTokenizerValidationIssue))"
+                                ErrorText="@this.dataCustomTokenizerValidationIssue"
+                                OnClearButtonClick="@this.ClearPathTokenizer"
+                            />
+                            <MudButton StartIcon="@Icons.Material.Filled.FolderOpen" Variant="Variant.Outlined" Color="Color.Primary" Disabled="@this.isTokenizerFileDialogOpen" OnClick="@this.OpenTokenizerFileDialog">
+                                @T("Choose File")
+                            </MudButton>
+                        </MudStack>
+                    </MudCollapse>
+                </MudStack>
+            }
         </MudForm>
-        <Issues IssuesData="@this.dataIssues"/>
+        @if (this.dataStoreWasAttempted)
+        {
+            <Issues IssuesData="@this.dataIssues"/>
+        }
     </DialogContent>
     <DialogActions>
         <MudButton OnClick="@this.Cancel" Variant="Variant.Filled">
             @T("Cancel")
         </MudButton>
         <MudButton OnClick="@this.Store" Variant="Variant.Filled" Color="Color.Primary">
-            @if(this.IsEditing)
+            @if (this.IsEditing)
             {
                 @T("Update")
             }

app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs

@@ -2,11 +2,12 @@ using AIStudio.Components;
 using AIStudio.Provider;
 using AIStudio.Provider.HuggingFace;
 using AIStudio.Settings;
+using AIStudio.Tools.Rust;
 using AIStudio.Tools.Services;
 using AIStudio.Tools.Validation;
 
 using Microsoft.AspNetCore.Components;
-
+using Microsoft.AspNetCore.Components.Web;
 using Host = AIStudio.Provider.SelfHosted.Host;
 
 namespace AIStudio.Dialogs;
@@ -82,6 +83,15 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
     [Parameter]
     public bool IsEditing { get; init; }
 
+    [Parameter]
+    public string DataTokenizerPath { get; set; } = string.Empty;
+
+    [Parameter]
+    public int DataTokenLimit { get; set; } = EmbeddingProvider.DEFAULT_TOKEN_LIMIT;
+
+    [Parameter]
+    public int DataEmbeddingBatchSize { get; set; } = EmbeddingProvider.DEFAULT_EMBEDDING_BATCH_SIZE;
+
     /// <summary>
     /// Whether this embedding provider is managed by an enterprise configuration plugin. When true,
     /// every field except the API key is locked, matching Settings.EmbeddingProvider.IsEnterpriseConfiguration.
@@ -110,6 +120,13 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
     private string dataAPIKeyStorageIssue = string.Empty;
     private string dataEditingPreviousInstanceName = string.Empty;
     private string dataLoadingModelsIssue = string.Empty;
+    private string dataFilePath = string.Empty;
+    private string dataCustomTokenizerValidationIssue = string.Empty;
+    private Task dataTokenizerValidationTask = Task.CompletedTask;
+    private bool dataStoreWasAttempted;
+    private bool isTokenizerFileDialogOpen;
+    private bool showExpertSettings;
+    private int dataTokenizerValidationRevision;
 
     // We get the form reference from Blazor code to validate it manually:
     private MudForm form = null!;
@@ -117,7 +134,7 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
     private readonly List<Model> availableModels = new();
     private readonly Encryption encryption = Program.ENCRYPTION;
     private readonly ProviderValidation providerValidation;
-    
+
     public EmbeddingProviderDialog()
     {
         this.providerValidation = new()
@@ -127,7 +144,8 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
             GetPreviousInstanceName = () => this.dataEditingPreviousInstanceName,
             GetUsedInstanceNames = () => this.UsedInstanceNames,
             GetHost = () => this.DataHost,
-            IsModelProvidedManually = () => this.DataLLMProvider is LLMProviders.SELF_HOSTED && this.DataHost is Host.OLLAMA,
+            IsModelProvidedManually = () => this.DataLLMProvider.IsEmbeddingModelProvidedManually(this.DataHost),
+            GetCustomTokenizerValidationIssue = () => this.dataCustomTokenizerValidationIssue,
         };
     }
     
@@ -137,7 +155,7 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
         Model model = default;
         if(this.DataLLMProvider is LLMProviders.SELF_HOSTED)
         {
-            if (this.DataHost is Host.OLLAMA)
+            if (this.DataLLMProvider.IsEmbeddingModelProvidedManually(this.DataHost))
                 model = new Model(this.dataManuallyModel, null);
             else if (this.DataHost is Host.LM_STUDIO)
                 model = this.DataModel;
@@ -157,6 +175,9 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
             Host = this.DataHost,
             IsEnterpriseConfiguration = this.IsEnterpriseConfiguration,
             EnterpriseConfigurationPluginId = Guid.Empty,
+            TokenizerPath = this.dataFilePath,
+            EmbeddingBatchSize = this.DataEmbeddingBatchSize,
+            TokenLimit = this.DataTokenLimit,
             CustomIconDataUrl = this.DataCustomIconDataUrl,
             HFInferenceProvider = this.HFInferenceProviderId,
         };
@@ -179,6 +200,10 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
         if(this.IsEditing)
         {
             this.dataEditingPreviousInstanceName = this.DataName.ToLowerInvariant();
+            this.dataFilePath = this.DataTokenizerPath;
+            this.showExpertSettings = !string.IsNullOrWhiteSpace(this.DataTokenizerPath)
+                                      || this.DataTokenLimit != EmbeddingProvider.DEFAULT_TOKEN_LIMIT
+                                      || this.DataEmbeddingBatchSize != EmbeddingProvider.DEFAULT_EMBEDDING_BATCH_SIZE;
             
             // When using self-hosted embedding, we must copy the model name:
             if (this.DataLLMProvider is LLMProviders.SELF_HOSTED)
@@ -187,7 +212,7 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
             //
             // We cannot load the API key for self-hosted providers:
             //
-            if (this.DataLLMProvider is LLMProviders.SELF_HOSTED && this.DataHost is not Host.OLLAMA)
+            if (this.DataLLMProvider is LLMProviders.SELF_HOSTED && this.DataHost is not Host.OLLAMA && this.DataHost is not Host.VLLM)
             {
                 await this.ReloadModels();
                 await base.OnInitializedAsync();
@@ -244,6 +269,8 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
     
     private async Task Store()
     {
+        this.dataStoreWasAttempted = true;
+        await this.dataTokenizerValidationTask;
         await this.form.Validate();
         this.dataAPIKeyStorageIssue = string.Empty;
 
@@ -260,6 +287,15 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
         if (!this.dataIsValid)
             return;
         
+        var response = await this.StoreOrDeleteTokenizerAsync();
+        if (!response.Success)
+        {
+            this.dataCustomTokenizerValidationIssue = string.IsNullOrWhiteSpace(response.Message) ? string.Empty : response.Message;
+            await this.form.Validate();
+            return;
+        }
+        this.dataFilePath = response.StoredPath;
+        
         // Use the data model to store the provider.
         // We just return this data to the parent component:
         var addedProviderSettings = this.CreateEmbeddingProviderSettings();
@@ -302,6 +338,22 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
         return null;
     }
 
+    private string? ValidateTokenLimit(int tokenLimit)
+    {
+        if (tokenLimit < 1)
+            return T("Please enter a token limit greater than 0.");
+
+        return null;
+    }
+
+    private string? ValidateEmbeddingBatchSize(int embeddingBatchSize)
+    {
+        if (embeddingBatchSize < 1)
+            return T("Please enter an embedding batch size greater than 0.");
+
+        return null;
+    }
+
     private void Cancel() => this.MudDialog.Cancel();
 
     private async Task OnAPIKeyChanged(string apiKey)
@@ -314,6 +366,90 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
         }
     }
 
+    private async Task OpenTokenizerFileDialog()
+    {
+        if (this.isTokenizerFileDialogOpen)
+            return;
+
+        this.isTokenizerFileDialogOpen = true;
+        try
+        {
+            var response = await this.RustService.SelectFile(T("Choose a custom tokenizer here"), [ FileTypes.JSON ], string.IsNullOrWhiteSpace(this.dataFilePath) ? null : this.dataFilePath);
+            if (!response.UserCancelled)
+                await this.OnDataFilePathChanged(response.SelectedFilePath);
+        }
+        finally
+        {
+            this.isTokenizerFileDialogOpen = false;
+        }
+    }
+
+    private Task ClearPathTokenizer(MouseEventArgs _)
+    {
+        return this.OnDataFilePathChanged(string.Empty);
+    }
+    
+    private async Task OnDataFilePathChanged(string filePath)
+    {
+        this.dataFilePath = filePath;
+        var validationRevision = ++this.dataTokenizerValidationRevision;
+        this.dataTokenizerValidationTask = this.ValidateCustomTokenizer(filePath, validationRevision);
+        await this.dataTokenizerValidationTask;
+
+        if (validationRevision != this.dataTokenizerValidationRevision)
+            return;
+
+        if (this.dataStoreWasAttempted)
+            await this.form.Validate();
+        else
+            this.form.ResetValidation();
+    }
+
+    private async Task ValidateCustomTokenizer(string filePath, int validationRevision)
+    {
+        if (string.IsNullOrWhiteSpace(filePath))
+        {
+            if (validationRevision == this.dataTokenizerValidationRevision)
+                this.dataCustomTokenizerValidationIssue = string.Empty;
+
+            return;
+        }
+
+        try
+        {
+            var response = await this.RustService.ValidateTokenizer(filePath);
+            if (validationRevision != this.dataTokenizerValidationRevision)
+                return;
+
+            if (response.Success)
+                this.dataCustomTokenizerValidationIssue = string.Empty;
+            else
+                this.dataCustomTokenizerValidationIssue = T("Invalid tokenizer: ") + response.Message;
+        }
+        catch (Exception e)
+        {
+            if (validationRevision != this.dataTokenizerValidationRevision)
+                return;
+
+            this.Logger.LogError(e, "Failed to validate custom tokenizer.");
+            this.dataCustomTokenizerValidationIssue = T("Failed to validate the selected tokenizer. Please try again.");
+        }
+    }
+
+    /// <summary>
+    /// Stores a new tokenizer or deletes the existing one, based on the specified tokenizer path.
+    /// If the path is null or empty, any existing tokenizer is removed.
+    /// Otherwise, the tokenizer is stored at the specified path.
+    /// </summary>
+    private Task<TokenizerResponse> StoreOrDeleteTokenizerAsync()
+    {
+        var tokenizerId = TokenizerModelId.ForEmbeddingProviderId(this.DataId);
+        if (string.IsNullOrWhiteSpace(this.dataFilePath))
+            return this.RustService.DeleteTokenizer(tokenizerId);
+
+        return this.RustService.StoreTokenizer(tokenizerId, this.dataFilePath);
+    }
+
     private void OnHostChanged(Host selectedHost)
     {
         // When the host changes, reset the model selection state:
@@ -374,4 +510,8 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
     };
     
     private bool IsNoneProvider => this.DataLLMProvider is LLMProviders.NONE;
+
+    private void ToggleExpertSettings() => this.showExpertSettings = !this.showExpertSettings;
+
+    private string GetExpertStyles => this.showExpertSettings ? "border-2 border-dashed rounded pa-2" : string.Empty;
 }

app/MindWork AI Studio/Dialogs/ProviderDialog.razor

@@ -174,6 +174,33 @@
                 Validation="@this.providerValidation.ValidatingInstanceName"
                 UserAttributes="@SPELLCHECK_ATTRIBUTES"
             />
+
+            @if (this.DataLLMProvider != LLMProviders.NONE)
+            {
+                <MudJustifiedText Typo="Typo.body1" Class="mb-3">
+                    @T("For better token estimates, you can configure a custom tokenizer for this provider.")
+                </MudJustifiedText>
+                <MudStack Row="@true" Spacing="3" Class="mb-3" StretchItems="StretchItems.None" AlignItems="AlignItems.Center">
+                    <MudTextField
+                        T="string"
+                        Text="@this.dataFilePath"
+                        TextChanged="@this.OnDataFilePathChanged"
+                        Label="@T("Selected file path for the custom tokenizer")"
+                        Validation="@this.providerValidation.ValidatingCustomTokenizer"
+                        Adornment="Adornment.Start"
+                        AdornmentIcon="@Icons.Material.Filled.AttachFile"
+                        UserAttributes="@SPELLCHECK_ATTRIBUTES"
+                        Variant="Variant.Outlined"
+                        Clearable="@true"
+                        Error="@(!string.IsNullOrWhiteSpace(this.dataCustomTokenizerValidationIssue))"
+                        ErrorText="@this.dataCustomTokenizerValidationIssue"
+                        OnClearButtonClick="@this.ClearPathTokenizer"
+                    />
+                    <MudButton StartIcon="@Icons.Material.Filled.FolderOpen" Variant="Variant.Outlined" Color="Color.Primary" Disabled="@this.isTokenizerFileDialogOpen" OnClick="@this.OpenTokenizerFileDialog">
+                        @T("Choose File")
+                    </MudButton>
+                </MudStack>
+            }
             
             <MudStack>
                 <MudButton OnClick="@this.ToggleExpertSettings">

app/MindWork AI Studio/Dialogs/ProviderDialog.razor.cs

@@ -4,11 +4,13 @@ using System.Text.Json;
 using AIStudio.Components;
 using AIStudio.Provider;
 using AIStudio.Provider.HuggingFace;
+using AIStudio.Tools.Rust;
 using AIStudio.Settings;
 using AIStudio.Tools.Services;
 using AIStudio.Tools.Validation;
 
 using Microsoft.AspNetCore.Components;
+using Microsoft.AspNetCore.Components.Web;
 
 using Host = AIStudio.Provider.SelfHosted.Host;
 
@@ -107,6 +109,9 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
     [Parameter]
     public string AdditionalJsonApiParameters { get; set; } = string.Empty;
 
+    [Parameter]
+    public string DataTokenizerPath { get; set; } = string.Empty;
+
     [Parameter]
     public ProviderCapabilityOverrides? DataCapabilityOverrides { get; set; }
     
@@ -134,7 +139,7 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
         ReasoningOverrideMode.ON_BY_DEFAULT,
         ReasoningOverrideMode.ALWAYS_ON
     ];
-    
+
     /// <summary>
     /// The list of used instance names. We need this to check for uniqueness.
     /// </summary>
@@ -148,6 +153,12 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
     private string dataAPIKeyStorageIssue = string.Empty;
     private string dataEditingPreviousInstanceName = string.Empty;
     private string dataLoadingModelsIssue = string.Empty;
+    private string dataFilePath = string.Empty;
+    private string dataCustomTokenizerValidationIssue = string.Empty;
+    private Task dataTokenizerValidationTask = Task.CompletedTask;
+    private bool dataStoreWasAttempted;
+    private bool isTokenizerFileDialogOpen;
+    private int dataTokenizerValidationRevision;
     private bool usesLegacySystemModelFallback;
     private bool showExpertSettings;
     private ProviderCapabilityOverrides capabilityOverrides = new();
@@ -169,6 +180,7 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
             GetUsedInstanceNames = () => this.UsedInstanceNames,
             GetHost = () => this.DataHost,
             IsModelProvidedManually = () => this.DataLLMProvider.IsLLMModelProvidedManually(),
+            GetCustomTokenizerValidationIssue = () => this.dataCustomTokenizerValidationIssue,
             IsModelSelectionHidden = () => this.IsLLMModelSelectionHidden,
         };
     }
@@ -190,6 +202,7 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
             Host = this.DataHost,
             HFInferenceProvider = this.HFInferenceProviderId,
             AdditionalJsonApiParameters = this.AdditionalJsonApiParameters,
+            TokenizerPath = this.dataFilePath,
             CapabilityOverrides = this.capabilityOverrides.HasOverrides ? this.capabilityOverrides : null,
             CustomIconDataUrl = this.DataCustomIconDataUrl,
         };
@@ -226,6 +239,7 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
         if(this.IsEditing)
         {
             this.dataEditingPreviousInstanceName = this.DataInstanceName.ToLowerInvariant();
+            this.dataFilePath = this.DataTokenizerPath;
             
             // When using Fireworks, we must copy the model name:
             if (this.DataLLMProvider.IsLLMModelProvidedManually())
@@ -291,6 +305,8 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
 
     private async Task Store()
     {
+        this.dataStoreWasAttempted = true;
+        await this.dataTokenizerValidationTask;
         await this.form.Validate();
         if (!string.IsNullOrWhiteSpace(this.dataAPIKeyStorageIssue))
             this.dataAPIKeyStorageIssue = string.Empty;
@@ -307,6 +323,27 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
         // When the data is not valid, we don't store it:
         if (!this.dataIsValid)
             return;
+
+        var tokenizerResponse = await this.StoreOrDeleteTokenizerAsync();
+        if (!tokenizerResponse.Success)
+        {
+            //
+            // Storing a tokenizer the user has chosen must succeed: otherwise the provider would
+            // silently work without the tokenizer the user asked for. Removing a tokenizer the
+            // user has cleared is best effort, though. A failed cleanup leaves an unused file
+            // behind, which is no reason to refuse saving the provider itself.
+            //
+            if (!string.IsNullOrWhiteSpace(this.dataFilePath))
+            {
+                this.dataCustomTokenizerValidationIssue = tokenizerResponse.Message;
+                await this.form.Validate();
+                return;
+            }
+
+            this.Logger.LogWarning($"Failed to remove the tokenizer of provider '{this.DataInstanceName}'. The provider is stored anyway. The message was: {tokenizerResponse.Message}");
+        }
+
+        this.dataFilePath = tokenizerResponse.StoredPath;
         
         // Use the data model to store the provider.
         // We just return this data to the parent component:
@@ -362,6 +399,98 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
         }
     }
 
+    private async Task OpenTokenizerFileDialog()
+    {
+        if (this.isTokenizerFileDialogOpen)
+            return;
+
+        this.isTokenizerFileDialogOpen = true;
+        try
+        {
+            var response = await this.RustService.SelectFile(T("Choose a custom tokenizer here"), [ FileTypes.JSON ], string.IsNullOrWhiteSpace(this.dataFilePath) ? null : this.dataFilePath);
+            if (!response.UserCancelled)
+                await this.OnDataFilePathChanged(response.SelectedFilePath);
+        }
+        finally
+        {
+            this.isTokenizerFileDialogOpen = false;
+        }
+    }
+
+    private Task ClearPathTokenizer(MouseEventArgs _)
+    {
+        return this.OnDataFilePathChanged(string.Empty);
+    }
+
+    private async Task OnDataFilePathChanged(string filePath)
+    {
+        this.dataFilePath = filePath;
+        var validationRevision = ++this.dataTokenizerValidationRevision;
+        this.dataTokenizerValidationTask = this.ValidateCustomTokenizer(filePath, validationRevision);
+        await this.dataTokenizerValidationTask;
+
+        if (validationRevision != this.dataTokenizerValidationRevision)
+            return;
+
+        if (this.dataStoreWasAttempted)
+            await this.form.Validate();
+        else
+            this.form.ResetValidation();
+    }
+
+    private async Task ValidateCustomTokenizer(string filePath, int validationRevision)
+    {
+        if (string.IsNullOrWhiteSpace(filePath))
+        {
+            if (validationRevision == this.dataTokenizerValidationRevision)
+                this.dataCustomTokenizerValidationIssue = string.Empty;
+
+            return;
+        }
+
+        try
+        {
+            var response = await this.RustService.ValidateTokenizer(filePath);
+            if (validationRevision != this.dataTokenizerValidationRevision)
+                return;
+
+            if (response.Success)
+                this.dataCustomTokenizerValidationIssue = string.Empty;
+            else
+                this.dataCustomTokenizerValidationIssue = T("Invalid tokenizer: ") + response.Message;
+        }
+        catch (Exception e)
+        {
+            if (validationRevision != this.dataTokenizerValidationRevision)
+                return;
+
+            this.Logger.LogError(e, "Failed to validate custom tokenizer.");
+            this.dataCustomTokenizerValidationIssue = T("Failed to validate the selected tokenizer. Please try again.");
+        }
+    }
+
+    /// <summary>
+    /// Stores a new tokenizer or deletes the existing one, based on the specified tokenizer path.
+    /// If the path is null or empty, any existing tokenizer is removed.
+    /// Otherwise, the tokenizer is stored at the specified path.
+    /// </summary>
+    private Task<TokenizerResponse> StoreOrDeleteTokenizerAsync()
+    {
+        var tokenizerId = TokenizerModelId.ForProviderId(this.DataId);
+        if (!string.IsNullOrWhiteSpace(this.dataFilePath))
+            return this.RustService.StoreTokenizer(tokenizerId, this.dataFilePath);
+
+        //
+        // A provider which never had a tokenizer has nothing to clean up. Calling the runtime
+        // anyway could only fail here, and that failure would block saving a provider which has
+        // nothing to do with tokenizers at all.
+        //
+        if (string.IsNullOrWhiteSpace(this.DataTokenizerPath))
+            return Task.FromResult(new TokenizerResponse(true, 0, string.Empty));
+
+        return this.RustService.DeleteTokenizer(tokenizerId);
+    }
+
     private void OnProviderChanged(LLMProviders selectedProvider)
     {
         this.DataLLMProvider = selectedProvider;
@@ -795,10 +924,10 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
 
         return true;
     }
-    
+
     private string GetExpertStyles => this.showExpertSettings ? "border-2 border-dashed rounded pa-2" : string.Empty;
-    
-    private static string GetPlaceholderExpertSettings => 
+
+    private static string GetPlaceholderExpertSettings =>
       """
       "temperature": 0.5,
       "top_p": 0.9,

app/MindWork AI Studio/Dialogs/Settings/SettingsDialogDataSources.razor

@@ -1,85 +1,15 @@
-@using AIStudio.Settings.DataModel
 @inherits SettingsDialogBase
 
 <MudDialog>
     <TitleContent>
-        <PreviewPrototype/>
+        <PreviewBeta/>
         <MudText Typo="Typo.h6" Class="d-flex align-center">
-            <MudIcon Icon="@Icons.Material.Filled.IntegrationInstructions" Class="mr-2"/>
+            <MudIcon Icon="@AppIcons.DATABASE" Class="mr-2"/>
             @T("Configured Data Sources")
         </MudText>
     </TitleContent>
     <DialogContent>
-        <MudJustifiedText Typo="Typo.body1" Class="mb-3">
-            @T("You might configure different data sources. A data source can include one file, all files in a directory, or data from your company. Later, you can incorporate these data sources as needed when the AI requires this data to complete a certain task.")
-        </MudJustifiedText>
-
-        <MudTable Items="@this.SettingsManager.ConfigurationData.DataSources" Hover="@true" Class="border-dashed border rounded-lg">
-            <ColGroup>
-                <col style="width: 3em;"/>
-                <col/>
-                <col style="width: 12em;"/>
-                <col style="width: 12em;"/>
-                <col style="width: 40em;"/>
-            </ColGroup>
-            <HeaderContent>
-                <MudTh>#</MudTh>
-                <MudTh>@T("Name")</MudTh>
-                <MudTh>@T("Type")</MudTh>
-                <MudTh>@T("Embedding")</MudTh>
-                <MudTh>@T("Actions")</MudTh>
-            </HeaderContent>
-            <RowTemplate>
-                <MudTd>@context.Num</MudTd>
-                <MudTd>@context.Name</MudTd>
-                <MudTd>@context.Type.GetDisplayName()</MudTd>
-                <MudTd>@this.GetEmbeddingName(context)</MudTd>
-                
-                <MudTd>
-                    <MudStack Row="true" Class="mb-2 mt-2" Wrap="Wrap.Wrap">
-                        <MudIconButton Variant="Variant.Filled" Color="Color.Info" Icon="@Icons.Material.Filled.Info" OnClick="() => this.ShowInformation(context)"/>
-                        @if (context.IsEnterpriseConfiguration)
-                        {
-                            <MudTooltip Text="@T("This data source is managed by your organization.")">
-                                <MudIconButton Color="Color.Info" Icon="@Icons.Material.Filled.Business" Disabled="true"/>
-                            </MudTooltip>
-                        }
-                        else
-                        {
-                            <MudButton Variant="Variant.Filled" Color="Color.Info" StartIcon="@Icons.Material.Filled.Edit" OnClick="() => this.EditDataSource(context)">
-                                @T("Edit")
-                            </MudButton>
-                            @if (context is DataSourceERI_V1)
-                            {
-                                <AdminExportButton Variant="Variant.Filled" OnClick="@(() => this.ExportDataSource(context))" />
-                            }
-                            <MudButton Variant="Variant.Filled" Color="Color.Error" StartIcon="@Icons.Material.Filled.Delete" OnClick="() => this.DeleteDataSource(context)">
-                                @T("Delete")
-                            </MudButton>
-                        }
-                    </MudStack>
-                </MudTd>
-            </RowTemplate>
-        </MudTable>
-
-        @if (this.SettingsManager.ConfigurationData.DataSources.Count == 0)
-        {
-            <MudText Typo="Typo.h6" Class="mt-3">
-                @T("No data sources configured yet.")
-            </MudText>
-        }
-        
-        <MudMenu EndIcon="@Icons.Material.Filled.KeyboardArrowDown" Label="@T("Add Data Source")" Color="Color.Primary" Variant="Variant.Filled" AnchorOrigin="Origin.CenterCenter" TransformOrigin="Origin.TopLeft" Class="mt-3 mb-6">
-            <MudMenuItem OnClick="() => this.AddDataSource(DataSourceType.ERI_V1)">
-                @T("External Data (ERI-Server v1)")
-            </MudMenuItem>
-            <MudMenuItem OnClick="() => this.AddDataSource(DataSourceType.LOCAL_DIRECTORY)">
-                @T("Local Directory")
-            </MudMenuItem>
-            <MudMenuItem OnClick="() => this.AddDataSource(DataSourceType.LOCAL_FILE)">
-                @T("Local File")
-            </MudMenuItem>
-        </MudMenu>
+        <DataSourceManagement/>
     </DialogContent>
     <DialogActions>
         <MudButton OnClick="@this.Close" Variant="Variant.Filled">

app/MindWork AI Studio/Dialogs/Settings/SettingsDialogDataSources.razor.cs

@@ -1,324 +1,3 @@
-using AIStudio.Settings;
-using AIStudio.Settings.DataModel;
-using AIStudio.Tools.ERIClient.DataModel;
-using AIStudio.Tools.PluginSystem;
-
 namespace AIStudio.Dialogs.Settings;
 
-public partial class SettingsDialogDataSources : SettingsDialogBase
-{
-    private string GetEmbeddingName(IDataSource dataSource)
-    {
-        if(dataSource is IInternalDataSource internalDataSource)
-        {
-            var matchedEmbedding = this.SettingsManager.ConfigurationData.EmbeddingProviders.FirstOrDefault(x => x.Id == internalDataSource.EmbeddingId);
-            if(matchedEmbedding == default)
-                return T("No valid embedding");
-            
-            return matchedEmbedding.Name;
-        }
-        
-        if(dataSource is IExternalDataSource)
-            return T("External (ERI)");
-        
-        return T("Unknown");
-    }
-    
-    private async Task AddDataSource(DataSourceType type)
-    {
-        IDataSource? addedDataSource = null;
-        switch (type)
-        {
-            case DataSourceType.LOCAL_FILE:
-                var localFileDialogParameters = new DialogParameters<DataSourceLocalFileDialog>
-                {
-                    { x => x.IsEditing, false },
-                    { x => x.AvailableEmbeddings, this.AvailableEmbeddingProviders }
-                };
-        
-                var localFileDialogReference = await this.DialogService.ShowAsync<DataSourceLocalFileDialog>(T("Add Local File as Data Source"), localFileDialogParameters, DialogOptions.FULLSCREEN);
-                var localFileDialogResult = await localFileDialogReference.Result;
-                if (localFileDialogResult is null || localFileDialogResult.Canceled)
-                    return;
-                
-                var localFile = (DataSourceLocalFile)localFileDialogResult.Data!;
-                localFile = localFile with { Num = this.SettingsManager.ConfigurationData.NextDataSourceNum++ };
-                addedDataSource = localFile;
-                break;
-            
-            case DataSourceType.LOCAL_DIRECTORY:
-                var localDirectoryDialogParameters = new DialogParameters<DataSourceLocalDirectoryDialog>
-                {
-                    { x => x.IsEditing, false },
-                    { x => x.AvailableEmbeddings, this.AvailableEmbeddingProviders }
-                };
-        
-                var localDirectoryDialogReference = await this.DialogService.ShowAsync<DataSourceLocalDirectoryDialog>(T("Add Local Directory as Data Source"), localDirectoryDialogParameters, DialogOptions.FULLSCREEN);
-                var localDirectoryDialogResult = await localDirectoryDialogReference.Result;
-                if (localDirectoryDialogResult is null || localDirectoryDialogResult.Canceled)
-                    return;
-                
-                var localDirectory = (DataSourceLocalDirectory)localDirectoryDialogResult.Data!;
-                localDirectory = localDirectory with { Num = this.SettingsManager.ConfigurationData.NextDataSourceNum++ };
-                addedDataSource = localDirectory;
-                break;
-            
-            case DataSourceType.ERI_V1:
-                var eriDialogParameters = new DialogParameters<DataSourceERI_V1Dialog>
-                {
-                    { x => x.IsEditing, false },
-                };
-                
-                var eriDialogReference = await this.DialogService.ShowAsync<DataSourceERI_V1Dialog>(T("Add ERI v1 Data Source"), eriDialogParameters, DialogOptions.FULLSCREEN);
-                var eriDialogResult = await eriDialogReference.Result;
-                if (eriDialogResult is null || eriDialogResult.Canceled)
-                    return;
-                
-                var eriDataSource = (DataSourceERI_V1)eriDialogResult.Data!;
-                eriDataSource = eriDataSource with { Num = this.SettingsManager.ConfigurationData.NextDataSourceNum++ };
-                addedDataSource = eriDataSource;
-                break;
-        }
-        
-        if(addedDataSource is null)
-            return;
-        
-        this.SettingsManager.ConfigurationData.DataSources.Add(addedDataSource);
-        await this.SettingsManager.StoreSettings();
-        await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
-    }
-
-    private async Task ExportDataSource(IDataSource dataSource)
-    {
-        if (!this.SettingsManager.ConfigurationData.App.ShowAdminSettings)
-            return;
-
-        if (dataSource is not DataSourceERI_V1 eriDataSource)
-            return;
-
-        if (eriDataSource.AuthMethod is AuthMethod.KERBEROS)
-        {
-            await this.DialogService.ShowMessageBox(
-                T("Export ERI Data Source"),
-                T("Kerberos/SSO ERI data sources cannot be exported yet. Please configure them manually in the configuration plugin."),
-                T("Close"));
-            return;
-        }
-
-        var needsSecret = eriDataSource.AuthMethod is AuthMethod.TOKEN or AuthMethod.USERNAME_PASSWORD;
-        if (!needsSecret)
-        {
-            var publicLuaCode = eriDataSource.ExportAsConfigurationSection();
-            if (!string.IsNullOrWhiteSpace(publicLuaCode))
-                await this.RustService.CopyText2Clipboard(publicLuaCode);
-
-            return;
-        }
-
-        var secretResponse = await this.RustService.GetSecret(eriDataSource, SecretStoreType.DATA_SOURCE, isTrying: true);
-        if (!secretResponse.Success)
-        {
-            await this.DialogService.ShowMessageBox(
-                T("Export ERI Data Source"),
-                string.Format(T("Cannot export this ERI data source because no authentication secret is configured. The issue was: {0}"), secretResponse.Issue),
-                T("Close"));
-            return;
-        }
-
-        var encryption = PluginFactory.EnterpriseEncryption;
-        if (encryption?.IsAvailable != true)
-        {
-            await this.DialogService.ShowMessageBox(
-                T("Export ERI Data Source"),
-                T("Cannot export this ERI data source because no enterprise encryption secret is configured."),
-                T("Close"));
-            return;
-        }
-
-        var usernamePasswordMode = DataSourceERIUsernamePasswordMode.USER_MANAGED;
-        if (eriDataSource.AuthMethod is AuthMethod.TOKEN)
-        {
-            var dialogParameters = new DialogParameters<ConfirmDialog>
-            {
-                { x => x.Message, T("This ERI data source has an access token configured. Do you want to include the encrypted access token in the export? Note: The recipient will need the same encryption secret to use the access token.") },
-            };
-
-            var dialogReference = await this.DialogService.ShowAsync<ConfirmDialog>(T("Export Access Token?"), dialogParameters, DialogOptions.FULLSCREEN);
-            var dialogResult = await dialogReference.Result;
-            if (dialogResult is null || dialogResult.Canceled)
-                return;
-        }
-        else if (eriDataSource.AuthMethod is AuthMethod.USERNAME_PASSWORD)
-        {
-            var dialogParameters = new DialogParameters<DataSourceERIV1UsernamePasswordExportDialog>
-            {
-                { x => x.DataSource, eriDataSource },
-            };
-
-            var dialogReference = await this.DialogService.ShowAsync<DataSourceERIV1UsernamePasswordExportDialog>(T("Export ERI Data Source"), dialogParameters, DialogOptions.FULLSCREEN);
-            var dialogResult = await dialogReference.Result;
-            if (dialogResult is null || dialogResult.Canceled || dialogResult.Data is not DataSourceERIV1UsernamePasswordExportDialogResult exportResult)
-                return;
-
-            usernamePasswordMode = exportResult.UsernamePasswordMode;
-        }
-
-        var decryptedSecret = await secretResponse.Secret.Decrypt(Program.ENCRYPTION);
-        if (!encryption.TryEncrypt(decryptedSecret, out var encryptedSecret))
-        {
-            await this.DialogService.ShowMessageBox(
-                T("Export ERI Data Source"),
-                T("Cannot export this ERI data source because the authentication secret could not be encrypted."),
-                T("Close"));
-            return;
-        }
-
-        var luaCode = eriDataSource.ExportAsConfigurationSection(
-            encryptedSecret,
-            usernamePasswordMode);
-        if (string.IsNullOrWhiteSpace(luaCode))
-            return;
-
-        await this.RustService.CopyText2Clipboard(luaCode);
-    }
-    
-    private async Task EditDataSource(IDataSource dataSource)
-    {
-        if (dataSource.IsEnterpriseConfiguration)
-            return;
-
-        IDataSource? editedDataSource = null;
-        switch (dataSource)
-        {
-            case DataSourceLocalFile localFile:
-                var localFileDialogParameters = new DialogParameters<DataSourceLocalFileDialog>
-                {
-                    { x => x.IsEditing, true },
-                    { x => x.DataSource, localFile },
-                    { x => x.AvailableEmbeddings, this.AvailableEmbeddingProviders }
-                };
-        
-                var localFileDialogReference = await this.DialogService.ShowAsync<DataSourceLocalFileDialog>(T("Edit Local File Data Source"), localFileDialogParameters, DialogOptions.FULLSCREEN);
-                var localFileDialogResult = await localFileDialogReference.Result;
-                if (localFileDialogResult is null || localFileDialogResult.Canceled)
-                    return;
-                
-                editedDataSource = (DataSourceLocalFile)localFileDialogResult.Data!;
-                break;
-            
-            case DataSourceLocalDirectory localDirectory:
-                var localDirectoryDialogParameters = new DialogParameters<DataSourceLocalDirectoryDialog>
-                {
-                    { x => x.IsEditing, true },
-                    { x => x.DataSource, localDirectory },
-                    { x => x.AvailableEmbeddings, this.AvailableEmbeddingProviders }
-                };
-        
-                var localDirectoryDialogReference = await this.DialogService.ShowAsync<DataSourceLocalDirectoryDialog>(T("Edit Local Directory Data Source"), localDirectoryDialogParameters, DialogOptions.FULLSCREEN);
-                var localDirectoryDialogResult = await localDirectoryDialogReference.Result;
-                if (localDirectoryDialogResult is null || localDirectoryDialogResult.Canceled)
-                    return;
-                
-                editedDataSource = (DataSourceLocalDirectory)localDirectoryDialogResult.Data!;
-                break;
-            
-            case DataSourceERI_V1 eriDataSource:
-                var eriDialogParameters = new DialogParameters<DataSourceERI_V1Dialog>
-                {
-                    { x => x.IsEditing, true },
-                    { x => x.DataSource, eriDataSource },
-                };
-                
-                var eriDialogReference = await this.DialogService.ShowAsync<DataSourceERI_V1Dialog>(T("Edit ERI v1 Data Source"), eriDialogParameters, DialogOptions.FULLSCREEN);
-                var eriDialogResult = await eriDialogReference.Result;
-                if (eriDialogResult is null || eriDialogResult.Canceled)
-                    return;
-                
-                editedDataSource = (DataSourceERI_V1)eriDialogResult.Data!;
-                break;
-        }
-        
-        if(editedDataSource is null)
-            return;
-        
-        this.SettingsManager.ConfigurationData.DataSources[this.SettingsManager.ConfigurationData.DataSources.IndexOf(dataSource)] = editedDataSource;
-
-        await this.SettingsManager.StoreSettings();
-        await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
-    }
-    
-    private async Task DeleteDataSource(IDataSource dataSource)
-    {
-        if (dataSource.IsEnterpriseConfiguration)
-            return;
-
-        var dialogParameters = new DialogParameters<ConfirmDialog>
-        {
-            { x => x.Message, string.Format(T("Are you sure you want to delete the data source '{0}' of type {1}?"), dataSource.Name, dataSource.Type.GetDisplayName()) },
-        };
-        
-        var dialogReference = await this.DialogService.ShowAsync<ConfirmDialog>(T("Delete Data Source"), dialogParameters, DialogOptions.FULLSCREEN);
-        var dialogResult = await dialogReference.Result;
-        if (dialogResult is null || dialogResult.Canceled)
-            return;
-
-        var applyChanges = dataSource is IInternalDataSource;
-        
-        // External data sources may need a secret for authentication:
-        if (dataSource is IExternalDataSource externalDataSource)
-        {
-            // When the auth method is NONE or KERBEROS, we don't need to delete a secret.
-            // In the case of KERBEROS, we don't store the Kerberos ticket in the secret store.
-            if(dataSource is IERIDataSource { AuthMethod: AuthMethod.NONE or AuthMethod.KERBEROS })
-                applyChanges = true;
-            
-            // All other auth methods require a secret, which we need to delete now:
-            else
-            {
-                var deleteSecretResponse = await this.RustService.DeleteSecret(externalDataSource, SecretStoreType.DATA_SOURCE);
-                if (deleteSecretResponse.Success)
-                    applyChanges = true;
-            }
-        }
-        
-        if(applyChanges)
-        {
-            this.SettingsManager.ConfigurationData.DataSources.Remove(dataSource);
-            await this.SettingsManager.StoreSettings();
-            await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
-        }
-    }
-    
-    private async Task ShowInformation(IDataSource dataSource)
-    {
-        switch (dataSource)
-        {
-            case DataSourceLocalFile localFile:
-                var localFileDialogParameters = new DialogParameters<DataSourceLocalFileInfoDialog>
-                {
-                    { x => x.DataSource, localFile },
-                };
-
-                await this.DialogService.ShowAsync<DataSourceLocalFileInfoDialog>(T("Local File Data Source Information"), localFileDialogParameters, DialogOptions.FULLSCREEN);
-                break;
-            
-            case DataSourceLocalDirectory localDirectory:
-                var localDirectoryDialogParameters = new DialogParameters<DataSourceLocalDirectoryInfoDialog>
-                {
-                    { x => x.DataSource, localDirectory },
-                };
-
-                await this.DialogService.ShowAsync<DataSourceLocalDirectoryInfoDialog>(T("Local Directory Data Source Information"), localDirectoryDialogParameters, DialogOptions.FULLSCREEN);
-                break;
-            
-            case DataSourceERI_V1 eriV1DataSource:
-                var eriV1DialogParameters = new DialogParameters<DataSourceERI_V1InfoDialog>
-                {
-                    { x => x.DataSource, eriV1DataSource },
-                };
-                
-                await this.DialogService.ShowAsync<DataSourceERI_V1InfoDialog>(T("ERI v1 Data Source Information"), eriV1DialogParameters, DialogOptions.FULLSCREEN);
-                break;
-        }
-    }
-}
+public partial class SettingsDialogDataSources : SettingsDialogBase;

app/MindWork AI Studio/Dialogs/Settings/SettingsDialogERIServer.razor

@@ -3,7 +3,7 @@
 
 <MudDialog>
     <TitleContent>
-        <PreviewPrototype/>
+        <PreviewBeta/>
         <MudText Typo="Typo.h6" Class="d-flex align-center">
             <MudIcon Icon="@Icons.Material.Filled.PrivateConnectivity" Class="mr-2"/>
             @T("Assistant: ERI Server Options")