Added local RAG (#756)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio.sln.DotSettings

@@ -1,4 +1,8 @@
 <wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
+	<!-- The local RAG index store is SQLite, where TEXT is stored dynamically. HasMaxLength() has no
+	     effect there, while picking arbitrary limits for paths or chunk texts would turn into real
+	     storage errors as soon as somebody indexes a deeply nested folder or a long document. -->
+	<s:String x:Key="/Default/CodeInspection/Highlighting/InspectionSeverities/=EntityFramework_002EModelValidation_002EUnlimitedStringLength/@EntryIndexedValue">DO_NOT_SHOW</s:String>
 	<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=AI/@EntryIndexedValue">AI</s:String>
 	<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=EDI/@EntryIndexedValue">EDI</s:String>
 	<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=ERI/@EntryIndexedValue">ERI</s:String>

app/MindWork AI Studio/Agents/AgentDataSourceSelection.cs

@@ -140,13 +140,21 @@ public sealed class AgentDataSourceSelection (ILogger<AgentDataSourceSelection>
         //
 
         // We start with the provider currently selected by the user:
-        var agentProvider = this.SettingsManager.GetPreselectedProvider(Tools.Components.AGENT_DATA_SOURCE_SELECTION, provider.Id, true);
+        var requiredDataSecurity = dataSources.AllowedDataSources.GetRequiredSecurityPolicy();
+        var requiredConfidenceLevel = dataSources.AllowedDataSources.GetRequiredConfidenceLevel();
+        var agentProvider = this.SettingsManager.GetPreselectedProvider(Tools.Components.AGENT_DATA_SOURCE_SELECTION, provider.ConfiguredProviderId, true);
         if (agentProvider == Settings.Provider.NONE)
         {
             logger.LogWarning("No provider is selected for the agent. The agent cannot select data sources.");
             return [];
         }
 
+        if (!agentProvider.AllowsDataSourceAccess(this.SettingsManager, requiredDataSecurity, requiredConfidenceLevel))
+        {
+            logger.LogWarning($"The agent for data source selection uses provider '{agentProvider.InstanceName}' with confidence '{agentProvider.GetConfidenceLevel(this.SettingsManager).GetName()}', but the available data sources require data security '{requiredDataSecurity}' and provider confidence '{requiredConfidenceLevel.GetName()}'. The agent cannot select data sources.");
+            return [];
+        }
+
         // Assign the provider settings to the agent:
         logger.LogInformation($"The agent for the data source selection uses the provider '{agentProvider.InstanceName}' ({agentProvider.UsedLLMProvider.ToName()}, confidence={agentProvider.UsedLLMProvider.GetConfidence(this.SettingsManager).Level.GetName()}).");
         this.ProviderSettings = agentProvider;

app/MindWork AI Studio/Agents/AgentRetrievalContextValidation.cs

@@ -3,6 +3,7 @@ using System.Text.Json;
 using AIStudio.Chat;
 using AIStudio.Provider;
 using AIStudio.Settings;
+using AIStudio.Settings.DataModel;
 using AIStudio.Tools.RAG;
 using AIStudio.Tools.Services;
 
@@ -129,19 +130,30 @@ public sealed class AgentRetrievalContextValidation (ILogger<AgentRetrievalConte
     /// you can set the provider once and then call the validation method in parallel.
     /// </remarks>
     /// <param name="provider">The current LLM provider. When the user doesn't preselect an agent provider, the agent uses this provider.</param>
-    public void SetLLMProvider(IProvider provider)
+    /// <param name="requiredDataSecurity">The data security required by the retrieved data.</param>
+    /// <param name="requiredConfidenceLevel">The minimum provider confidence required by the retrieved data.</param>
+    public bool SetLLMProvider(IProvider provider, DataSourceSecurity requiredDataSecurity = DataSourceSecurity.NOT_SPECIFIED, ConfidenceLevel requiredConfidenceLevel = ConfidenceLevel.NONE)
     {
         // We start with the provider currently selected by the user:
-        var agentProvider = this.SettingsManager.GetPreselectedProvider(Tools.Components.AGENT_RETRIEVAL_CONTEXT_VALIDATION, provider.Id, true);
+        var agentProvider = this.SettingsManager.GetPreselectedProvider(Tools.Components.AGENT_RETRIEVAL_CONTEXT_VALIDATION, provider.ConfiguredProviderId, true);
         if (agentProvider == Settings.Provider.NONE)
         {
             logger.LogWarning("No provider is selected for the agent.");
-            return;
+            this.ProviderSettings = Settings.Provider.NONE;
+            return false;
+        }
+
+        if (!agentProvider.AllowsDataSourceAccess(this.SettingsManager, requiredDataSecurity, requiredConfidenceLevel))
+        {
+            logger.LogWarning($"The agent for retrieval context validation uses provider '{agentProvider.InstanceName}' with confidence '{agentProvider.GetConfidenceLevel(this.SettingsManager).GetName()}', but the retrieved data requires data security '{requiredDataSecurity}' and provider confidence '{requiredConfidenceLevel.GetName()}'. The agent cannot validate retrieval contexts.");
+            this.ProviderSettings = Settings.Provider.NONE;
+            return false;
         }
         
         // Assign the provider settings to the agent:
         logger.LogInformation($"The agent for the retrieval context validation uses the provider '{agentProvider.InstanceName}' ({agentProvider.UsedLLMProvider.ToName()}, confidence={agentProvider.UsedLLMProvider.GetConfidence(this.SettingsManager).Level.GetName()}).");
         this.ProviderSettings = agentProvider;
+        return true;
     }
     
     /// <summary>

app/MindWork AI Studio/Assistants/ERI/AssistantERI.razor

@@ -22,7 +22,7 @@
     </MudListItem>
 </MudList>
 
-<PreviewPrototype ApplyInnerScrollingFix="true"/>
+<PreviewBeta ApplyInnerScrollingFix="true"/>
 <div class="mb-6"></div>
 
 <MudText Typo="Typo.h4" Class="mb-3">
@@ -345,4 +345,4 @@ else
 </MudJustifiedText>
 
 <MudTextSwitch Label="@T("Should we write the generated code to the file system?")" Disabled="@this.IsERIInputDisabled" @bind-Value="@this.writeToFilesystem" LabelOn="@T("Yes, please write or update all generated code to the file system")" LabelOff="@T("No, just show me the code")" />
-<SelectDirectory Label="@T("Base directory where to write the code")" @bind-Directory="@this.baseDirectory" Disabled="@this.IsBaseDirectorySelectionDisabled" DirectoryDialogTitle="@T("Select the target directory for the ERI server")" Validation="@this.ValidateDirectory" />
+<SelectDirectory Label="@T("Base directory where to write the code")" @bind-Directory="@this.baseDirectory" Disabled="@this.IsBaseDirectorySelectionDisabled" DirectoryDialogTitle="@T("Select the target directory for the ERI server")" Validation="@this.ValidateDirectory" EnableDragDrop="true" Layer="@DropLayers.ASSISTANTS" CatchAllDocuments="true" />

app/MindWork AI Studio/Chat/ChatThread.cs

@@ -92,7 +92,10 @@ public sealed record ChatThread
     public DataSourceSecurity DataSecurity { get; set; } = DataSourceSecurity.NOT_SPECIFIED;
 
     /// <summary>
-    /// The minimum confidence required for providers that continue this chat after a tool returned sensitive data.
+    /// The minimum confidence required for providers that continue this chat. It is raised whenever
+    /// a tool returned sensitive data, and whenever a data source was used which demands a higher
+    /// level. Both cases share one rule: once such data is in the thread, every provider which
+    /// continues it must meet the level.
     /// </summary>
     [JsonInclude]
     public ConfidenceLevel RequiredProviderConfidence { get; private set; } = ConfidenceLevel.NONE;

app/MindWork AI Studio/Chat/ChatThreadExtensions.cs

@@ -11,12 +11,12 @@ public static class ChatThreadExtensions
     /// </summary>
     /// <remarks>
     /// We don't check if the provider is allowed to use the data sources of the chat thread.
-    /// That kind of check is done in the RAG process itself.<br/><br/>
+    /// That kind of check is done when the available data sources are resolved.<br/><br/>
     /// 
     /// One thing which is not so obvious: after RAG was used on this thread, the entire chat
     /// thread is kind of a data source by itself. Why? Because the augmentation data collected
     /// from the data sources is stored in the chat thread. This means we must check if the
-    /// selected provider is allowed to use this thread's data.
+    /// selected provider is allowed to use this thread's data security and confidence level.
     /// </remarks>
     /// <param name="chatThread">The chat thread to check.</param>
     /// <param name="provider">The provider to check.</param>
@@ -26,23 +26,24 @@ public static class ChatThreadExtensions
         // No chat thread available means we have a new chat. That's fine:
         if (chatThread is null)
             return true;
-        
+
         var settingsManager = Program.SERVICE_PROVIDER.GetRequiredService<SettingsManager>();
         var providerConfidence = provider switch
         {
-            IProvider p => p.Provider.GetConfidence(settingsManager).Level,
-            AIStudio.Settings.Provider p => p.UsedLLMProvider.GetConfidence(settingsManager).Level,
+            IProvider p => p.GetConfidenceLevel(settingsManager),
+            AIStudio.Settings.Provider p => p.GetConfidenceLevel(settingsManager),
 
             _ => ConfidenceLevel.UNKNOWN,
         };
-        var isTrustedByConfiguration = provider switch
-        {
-            IProvider p => p.IsTrustedByConfiguration(settingsManager),
-            AIStudio.Settings.Provider p => p.IsTrustedByConfiguration(settingsManager),
 
-            _ => false,
-        };
-        if (providerConfidence < chatThread.RequiredProviderConfidence && !isTrustedByConfiguration)
+        //
+        // The confidence axis is checked on its own: a provider trusted by configuration counts as
+        // self-hosted for data-source security, which is the check further down, but that trust
+        // says nothing about how confidential the provider is. An organization which wants its
+        // contractually covered cloud provider to pass here raises its level through the custom
+        // confidence scheme instead.
+        //
+        if (providerConfidence < chatThread.RequiredProviderConfidence)
             return false;
 
         // The chat thread is available, but the data security is not specified.

app/MindWork AI Studio/Chat/ContentText.cs

@@ -66,7 +66,7 @@ public sealed class ContentText : IContent
         
         if(!chatThread.IsLLMProviderAllowed(provider))
         {
-            LOGGER.LogError("The provider is not allowed for this chat thread due to data security reasons. Skipping the AI process.");
+            LOGGER.LogError("The provider is not allowed for this chat thread due to data security or confidence-level requirements. Skipping the AI process.");
             await this.CompleteWithoutStreaming();
             return chatThread;
         }

app/MindWork AI Studio/Components/ChatComponent.razor

@@ -35,7 +35,7 @@
     <FooterContent>
         <MediaTranscriptionStatus Owner="@this.CurrentMediaImportOwner"/>
         <MudElement Style="flex: 0 0 auto;">
-            <MudTextField
+            <UserPromptComponent
                 T="string"
                 @ref="@this.inputField"
                 @bind-Text="@this.UserInput"
@@ -51,8 +51,11 @@
                 Disabled="@this.IsInputForbidden()"
                 Immediate="@true"
                 OnKeyUp="@this.InputKeyEvent"
+                WhenTextChangedAsync="@(_ =>this.CalculateTokenCount())"
                 UserAttributes="@USER_INPUT_ATTRIBUTES"
                 Class="@this.UserInputClass"
+                DebounceTime="TimeSpan.FromSeconds(1)" 
+                HelperText="@this.TokenCountMessage"
                 Style="@this.UserInputStyle"/>
         </MudElement>
         <MudToolBar WrapContent="true" Gutters="@false" Class="border border-solid rounded" Style="border-color: lightgrey; gap: 2px;">
@@ -132,7 +135,7 @@
             
             @if (PreviewFeatures.PRE_RAG_2024.IsEnabled(this.SettingsManager))
             {
-                <DataSourceSelection @ref="@this.dataSourceSelectionComponent" PopoverTriggerMode="PopoverTriggerMode.BUTTON" LLMProvider="@this.Provider" DataSourceOptions="@this.GetCurrentDataSourceOptions()" DataSourceOptionsChanged="@(async options => await this.SetCurrentDataSourceOptions(options))" DataSourcesAISelected="@this.GetAgentSelectedDataSources()"/>
+                <DataSourceSelection @ref="@this.dataSourceSelectionComponent" PopoverTriggerMode="PopoverTriggerMode.ICON" LLMProvider="@this.Provider" DataSourceOptions="@this.GetCurrentDataSourceOptions()" DataSourceOptionsChanged="@(async options => await this.SetCurrentDataSourceOptions(options))" DataSourcesAISelected="@this.GetAgentSelectedDataSources()"/>
             }
             
             @if (this.SettingsManager.ConfigurationData.Confidence.ShowProviderConfidence)
@@ -149,7 +152,7 @@
 
             @if (!this.ChatThread.IsLLMProviderAllowed(this.Provider))
             {
-                <MudTooltip Text="@T("The selected provider is not allowed in this chat due to data security reasons.")" Placement="@TOOLBAR_TOOLTIP_PLACEMENT">
+                <MudTooltip Text="@T("The selected provider is not allowed in this chat due to data security or confidence-level requirements.")" Placement="@TOOLBAR_TOOLTIP_PLACEMENT">
                     <MudIconButton Icon="@Icons.Material.Filled.Error" Color="Color.Error"/>
                 </MudTooltip>
             }

app/MindWork AI Studio/Components/ChatComponent.razor.cs

@@ -55,6 +55,8 @@ public partial class ChatComponent : MSGComponentBase
     [Inject]
     private IDialogService DialogService { get; init; } = null!;
     
+    [Inject] 
+    private RustService RustService { get; init; } = null!;
     [Inject]
     private IJSRuntime JsRuntime { get; init; } = null!;
 
@@ -91,12 +93,17 @@ public partial class ChatComponent : MSGComponentBase
     private Guid loadedParameterWorkspaceId = Guid.Empty;
     private Guid foregroundChatId = Guid.Empty;
     private int workspaceHeaderSyncVersion;
+    private string tokenCount = "0";
+    private bool HasCustomTokenizer => !string.IsNullOrWhiteSpace(this.Provider.TokenizerPath);
+    private string TokenCountMessage => this.HasCustomTokenizer
+        ? $"{this.T("Estimated amount of tokens:")} {this.tokenCount}"
+        : string.Empty;
 
     private MediaImportOwner CurrentMediaImportOwner => MediaImportOwner.ForChat(this.ChatThread?.ChatId ?? this.draftMediaOwnerId);
 
     // Unfortunately, we need the input field reference to blur the focus away. Without
     // this, we cannot clear the input field.
-    private MudTextField<string> inputField = null!;
+    private UserPromptComponent<string> inputField = null!;
 
     /// <summary>
     /// Represents the user's input in the chat interface.
@@ -373,15 +380,22 @@ public partial class ChatComponent : MSGComponentBase
     protected override async Task OnParametersSetAsync()
     {
         var incomingChatId = this.ChatThread?.ChatId ?? Guid.Empty;
+        var providerChanged = this.Provider != this.lastSeenProvider;
         if (incomingChatId != this.lastSeenChatId || this.Provider != this.lastSeenProvider)
         {
             this.lastSeenChatId = incomingChatId;
             this.lastSeenProvider = this.Provider;
+            if (providerChanged)
+                this.tokenCount = "0";
+
             this.previousInputForbidden = true;
         }
 
         await this.ApplyLoadedChatParameterAsync();
         await this.SyncForegroundChatAsync();
+        if (providerChanged && this.HasCustomTokenizer)
+            await this.CalculateTokenCount();
+
         await this.ConsumeMediaOutcomeAsync();
         await base.OnParametersSetAsync();
     }
@@ -713,6 +727,9 @@ public partial class ChatComponent : MSGComponentBase
         // Was a modifier key pressed as well?
         var isModifier = keyEvent.AltKey || keyEvent.CtrlKey || keyEvent.MetaKey || keyEvent.ShiftKey;
         
+        if (isEnter)
+            await this.CalculateTokenCount();
+        
         // Depending on the user's settings, might react to shortcuts:
         switch (this.SettingsManager.ConfigurationData.Chat.ShortcutSendBehavior)
         {
@@ -899,6 +916,7 @@ public partial class ChatComponent : MSGComponentBase
         this.ComposerState.Clear();
 
         await this.inputField.BlurAsync();
+        this.tokenCount = "0";
         
         // Enable the stream state for the chat component:
         this.hasUnsavedChanges = true;
@@ -1291,6 +1309,43 @@ public partial class ChatComponent : MSGComponentBase
         this.ComposerState.RestoreFromTextBlock(textBlock);
     }
     
+    private async Task CalculateTokenCount()
+    {
+        if (!this.HasCustomTokenizer)
+        {
+            if (this.tokenCount != "0")
+            {
+                this.tokenCount = "0";
+                this.StateHasChanged();
+            }
+
+            return;
+        }
+
+        //
+        // Read the text from the bound property rather than from the input field: the field is a
+        // component reference, which is only set once the component has rendered. Counting is also
+        // triggered while parameters are set, which happens before that.
+        //
+        var currentInput = this.UserInput;
+        if (string.IsNullOrEmpty(currentInput))
+        {
+            this.tokenCount = "0";
+            return;
+        }
+
+        var response = await this.RustService.GetTokenCount(this.Provider, currentInput);
+        if (response is null)
+            return;
+        if (!response.Value.Success)
+        {
+            this.Logger.LogWarning("Failed to calculate token count: reason='{Reason}'", response.Value.Message);
+            return;
+        }
+        this.tokenCount = response.Value.TokenCount.ToString();
+        this.StateHasChanged();
+    }
+    
     #region Overrides of MSGComponentBase
     
     protected override async Task ProcessIncomingMessage<T>(ComponentBase? sendingComponent, Event triggeredEvent, T? data) where T : default

app/MindWork AI Studio/Components/DataSourceCloudEmbeddingWarning.razor

@@ -0,0 +1,9 @@
+@inherits MSGComponentBase
+
+<MudAlert Severity="Severity.Warning" Variant="Variant.Outlined" Dense="@true" Class="mb-3">
+    <MudText Typo="Typo.body2">
+        @this.WarningText
+    </MudText>
+</MudAlert>
+
+<MudTextSwitch Value="@this.UserAcknowledged" ValueChanged="@this.UserAcknowledgedChanged" Label="@T("I confirm that I have read and understood the above")" LabelOn="@T("Yes, please send my data to the external embedding provider")" LabelOff="@T("No, I will choose another embedding")" Validation="@this.Validation"/>

app/MindWork AI Studio/Components/DataSourceCloudEmbeddingWarning.razor.cs

@@ -0,0 +1,52 @@
+using AIStudio.Settings.DataModel;
+
+using Microsoft.AspNetCore.Components;
+
+namespace AIStudio.Components;
+
+public partial class DataSourceCloudEmbeddingWarning : MSGComponentBase
+{
+    [Parameter]
+    public DataSourceType DataSourceType { get; set; }
+
+    [Parameter]
+    public string SourcePath { get; set; } = string.Empty;
+
+    [Parameter]
+    public bool UserAcknowledged { get; set; }
+
+    [Parameter]
+    public EventCallback<bool> UserAcknowledgedChanged { get; set; }
+
+    [Parameter]
+    public Func<bool, string?> Validation { get; set; } = _ => null;
+
+    private string WarningText
+    {
+        get
+        {
+            var subject = this.GetSubjectText();
+            return string.Format(
+                T("Warning: The selected embedding provider is not self-hosted. Creating embeddings can cost money and may need to run multiple times, for example after errors or file changes. {0} will be sent to an external third party. MindWork AI Studio has no control over what that third party does with the data after it is sent."),
+                subject);
+        }
+    }
+
+    private string GetSubjectText()
+    {
+        if (string.IsNullOrWhiteSpace(this.SourcePath))
+            return this.DataSourceType switch
+            {
+                DataSourceType.LOCAL_DIRECTORY => T("All files in this folder and its subfolders"),
+                DataSourceType.LOCAL_FILE => T("The selected file"),
+                _ => T("The selected data")
+            };
+
+        return this.DataSourceType switch
+        {
+            DataSourceType.LOCAL_DIRECTORY => string.Format(T("All files in the folder '{0}' and its subfolders"), this.SourcePath),
+            DataSourceType.LOCAL_FILE => string.Format(T("The file '{0}'"), this.SourcePath),
+            _ => string.Format(T("The data source '{0}'"), this.SourcePath)
+        };
+    }
+}

app/MindWork AI Studio/Components/DataSourceManagement.razor

@@ -0,0 +1,106 @@
+@using AIStudio.Settings
+@using AIStudio.Settings.DataModel
+@inherits MSGComponentBase
+
+<MudJustifiedText Typo="Typo.body1" Class="mb-3">
+    @T("You might configure different data sources. A data source can include one file, all files in a directory, or data from your company. Later, you can incorporate these data sources as needed when the AI requires this data to complete a certain task.")
+</MudJustifiedText>
+
+<MudStack Row="true" AlignItems="AlignItems.Center" Justify="Justify.SpaceBetween" Class="mb-3" Wrap="Wrap.Wrap">
+    <MudTextSwitch Label="@T("Automatic local data source refresh")" Value="@this.SettingsManager.ConfigurationData.App.DataSourceIndexing.AutomaticRefresh" LabelOn="@T("Local data sources refresh when files change.")" LabelOff="@T("Local data sources refresh only when triggered manually.")" ValueChanged="@this.AutomaticRefreshChanged"/>
+    <MudTooltip Text="@T("Refresh all")">
+        <MudIconButton Color="Color.Primary" Icon="@Icons.Material.Filled.Sync" Disabled="@(!this.HasRefreshableDataSources())" OnClick="@this.RefreshAllDataSources"/>
+    </MudTooltip>
+</MudStack>
+
+@{ var embeddingStatuses = this.DataSourceEmbeddingService.GetStatuses().ToDictionary(status => status.DataSourceId, StringComparer.OrdinalIgnoreCase); }
+<MudTable Items="@this.SettingsManager.ConfigurationData.DataSources" Hover="@true" Class="border-dashed border rounded-lg">
+    <ColGroup>
+        <col style="width: 3em;"/>
+        <col/>
+        <col style="width: 9em;"/>
+        <col style="width: 10em;"/>
+        <col style="width: 8em;"/>
+        <col style="width: 16em;"/>
+    </ColGroup>
+    <HeaderContent>
+        <MudTh>#</MudTh>
+        <MudTh>@T("Name")</MudTh>
+        <MudTh>@T("Type")</MudTh>
+        <MudTh>@T("Embedding")</MudTh>
+        <MudTh>@T("Indexed files")</MudTh>
+        <MudTh>@T("Actions")</MudTh>
+    </HeaderContent>
+    <RowTemplate>
+        @{ var embeddingStatus = embeddingStatuses.GetValueOrDefault(context.Id); }
+        <MudTd>@context.Num</MudTd>
+        <MudTd Style="white-space: normal; overflow-wrap: anywhere;">@context.Name</MudTd>
+        <MudTd Style="white-space: nowrap;">@context.Type.GetDisplayName()</MudTd>
+        <MudTd Style="white-space: nowrap;">@this.GetEmbeddingName(context)</MudTd>
+        <MudTd>
+            @if (context is IInternalDataSource)
+            {
+                <MudTooltip Text="@this.GetIndexingStatusTooltip(embeddingStatus)">
+                    <MudStack Row="true" AlignItems="AlignItems.Center" Spacing="1">
+                        <MudIcon Icon="@Icons.Material.Filled.Circle" Color="@GetIndexingStatusColor(embeddingStatus)" Size="Size.Small" Style="width: 0.75rem; height: 0.75rem; flex-shrink: 0;"/>
+                        <MudText Typo="Typo.body2">@(embeddingStatus is null ? T("Not available") : string.Format(T("{0} of {1}"), embeddingStatus.IndexedFiles, embeddingStatus.TotalFiles))</MudText>
+                    </MudStack>
+                </MudTooltip>
+            }
+            else
+            {
+                @* Deliberately muted instead of colored: there is nothing to index and nothing to fix here. *@
+                <MudText Typo="Typo.body2" Class="mud-text-secondary">@T("Not applicable")</MudText>
+            }
+        </MudTd>
+
+        <MudTd>
+            <MudStack Row="true" Class="mb-2 mt-2" Spacing="1" Wrap="Wrap.NoWrap">
+                <MudTooltip Text="@T("Information")">
+                    <MudIconButton Color="Color.Info" Icon="@Icons.Material.Outlined.Info" OnClick="@(() => this.ShowInformation(context))"/>
+                </MudTooltip>
+                @if (context.IsEnterpriseConfiguration)
+                {
+                    <MudTooltip Text="@T("This data source is managed by your organization.")">
+                        <MudIconButton Color="Color.Info" Icon="@Icons.Material.Filled.Business" Disabled="true"/>
+                    </MudTooltip>
+                }
+                else
+                {
+                    <MudTooltip Text="@T("Edit")">
+                        <MudIconButton Color="Color.Info" Icon="@Icons.Material.Filled.Edit" OnClick="@(() => this.EditDataSource(context))"/>
+                    </MudTooltip>
+                    <MudTooltip Text="@T("Refresh")">
+                        <MudIconButton Color="Color.Primary" Icon="@Icons.Material.Filled.Sync" Disabled="@(!this.CanRefreshDataSource(context))" OnClick="@(() => this.RefreshDataSource(context))"/>
+                    </MudTooltip>
+                    @if (context is DataSourceERI_V1)
+                    {
+                        <AdminExportButton OnClick="@(() => this.ExportDataSource(context))" />
+                    }
+                    <MudTooltip Text="@T("Delete")">
+                        <MudIconButton Color="Color.Error" Icon="@Icons.Material.Filled.Delete" OnClick="@(() => this.DeleteDataSource(context))"/>
+                    </MudTooltip>
+                }
+            </MudStack>
+        </MudTd>
+    </RowTemplate>
+</MudTable>
+
+@if (this.SettingsManager.ConfigurationData.DataSources.Count == 0)
+{
+    <MudText Typo="Typo.h6" Class="mt-3">
+        @T("No data sources configured yet.")
+    </MudText>
+}
+
+<MudMenu EndIcon="@Icons.Material.Filled.KeyboardArrowDown" Label="@T("Add Data Source")" Color="Color.Primary" Variant="Variant.Filled" AnchorOrigin="Origin.TopLeft" TransformOrigin="Origin.BottomLeft" Class="mt-3 mb-6">
+    <MudMenuItem OnClick="@(() => this.AddDataSource(DataSourceType.ERI_V1))">
+        @T("External Data (ERI-Server v1)")
+    </MudMenuItem>
+    <MudMenuItem OnClick="@(() => this.AddDataSource(DataSourceType.LOCAL_DIRECTORY))">
+        @T("Local Directory")
+    </MudMenuItem>
+    <MudMenuItem OnClick="@(() => this.AddDataSource(DataSourceType.LOCAL_FILE))">
+        @T("Local File")
+    </MudMenuItem>
+</MudMenu>

app/MindWork AI Studio/Components/DataSourceManagement.razor.cs

@@ -0,0 +1,458 @@
+using AIStudio.Dialogs;
+using AIStudio.Settings;
+using AIStudio.Settings.DataModel;
+using AIStudio.Tools.ERIClient.DataModel;
+using AIStudio.Tools.PluginSystem;
+using AIStudio.Tools.Services;
+
+using Microsoft.AspNetCore.Components;
+
+using DialogOptions = AIStudio.Dialogs.DialogOptions;
+
+namespace AIStudio.Components;
+
+/// <summary>
+/// Manages the configured data sources. Used by the data source settings dialog, which the chat
+/// opens, and by the data source panel in the app settings.
+/// </summary>
+public partial class DataSourceManagement : MSGComponentBase
+{
+    [Inject]
+    private DataSourceEmbeddingService DataSourceEmbeddingService { get; init; } = null!;
+
+    [Inject]
+    private IDialogService DialogService { get; init; } = null!;
+
+    [Inject]
+    private RustService RustService { get; init; } = null!;
+
+    private readonly List<ConfigurationSelectData<string>> availableEmbeddingProviders = new();
+
+    #region Overrides of ComponentBase
+
+    protected override async Task OnInitializedAsync()
+    {
+        await base.OnInitializedAsync();
+        this.ApplyFilters([], [ Event.CONFIGURATION_CHANGED, Event.RAG_EMBEDDING_STATUS_CHANGED ]);
+        this.UpdateEmbeddingProviders();
+    }
+
+    #endregion
+
+    #region Overrides of MSGComponentBase
+
+    protected override Task ProcessIncomingMessage<T>(ComponentBase? sendingComponent, Event triggeredEvent, T? data) where T : default
+    {
+        switch (triggeredEvent)
+        {
+            case Event.CONFIGURATION_CHANGED:
+            case Event.PLUGINS_RELOADED:
+                this.UpdateEmbeddingProviders();
+                this.StateHasChanged();
+                break;
+
+            case Event.RAG_EMBEDDING_STATUS_CHANGED:
+                this.StateHasChanged();
+                break;
+        }
+
+        return Task.CompletedTask;
+    }
+
+    #endregion
+
+    private void UpdateEmbeddingProviders()
+    {
+        this.availableEmbeddingProviders.Clear();
+        foreach (var provider in this.SettingsManager.GetAllEmbeddingProviders())
+            this.availableEmbeddingProviders.Add(new (provider.Name, provider.Id));
+    }
+
+    /// <remarks>
+    /// Files which were skipped for good are none of the failed ones, so a data source made of
+    /// scanned documents stays green: there is nothing here for the user to fix.
+    /// </remarks>
+    private static Color GetIndexingStatusColor(DataSourceEmbeddingStatus? status)
+    {
+        if (status is null || status.State is DataSourceEmbeddingState.IDLE or DataSourceEmbeddingState.QUEUED or DataSourceEmbeddingState.RUNNING)
+            return Color.Warning;
+
+        return status.State is DataSourceEmbeddingState.FAILED || status.FailedFiles > 0
+            ? Color.Error
+            : Color.Success;
+    }
+
+    /// <summary>
+    /// Explains the indexing dot, including the files which stay out of the index.
+    /// </summary>
+    /// <remarks>
+    /// The column shows the indexed files against the total, which reads as unfinished for a data
+    /// source whose remaining files were skipped for good. The tooltip is where that gap gets its
+    /// explanation.
+    /// </remarks>
+    private string GetIndexingStatusTooltip(DataSourceEmbeddingStatus? status)
+    {
+        if (status is null)
+            return T("Waiting for indexing status");
+
+        if (status.PermanentlySkippedFiles == 0)
+            return status.StateLabel;
+
+        return $"{status.StateLabel} — {string.Format(T("{0} files were skipped because they contain no readable text. AI Studio reads them again once they change."), status.PermanentlySkippedFiles)}";
+    }
+
+    private string GetEmbeddingName(IDataSource dataSource)
+    {
+        if(dataSource is IInternalDataSource internalDataSource)
+        {
+            var matchedEmbedding = this.SettingsManager.ConfigurationData.EmbeddingProviders.FirstOrDefault(x => x.Id == internalDataSource.EmbeddingId);
+            if(matchedEmbedding == default)
+                return T("No valid embedding");
+
+            return matchedEmbedding.Name;
+        }
+
+        if(dataSource is IExternalDataSource)
+            return T("External (ERI)");
+
+        return T("Unknown");
+    }
+
+    private bool CanRefreshDataSource(IDataSource dataSource)
+    {
+        return this.DataSourceEmbeddingService.CanRefreshDataSource(dataSource);
+    }
+
+    private bool HasRefreshableDataSources()
+    {
+        return this.SettingsManager.ConfigurationData.DataSources.Any(this.CanRefreshDataSource);
+    }
+
+    private async Task AutomaticRefreshChanged(bool enabled)
+    {
+        this.SettingsManager.ConfigurationData.App.DataSourceIndexing.AutomaticRefresh = enabled;
+        await this.SettingsManager.StoreSettings();
+        this.DataSourceEmbeddingService.RefreshAutomaticWatchers();
+        await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
+    }
+
+    private async Task RefreshAllDataSources()
+    {
+        await this.DataSourceEmbeddingService.QueueAllInternalDataSourcesAsync();
+        await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
+    }
+
+    private async Task RefreshDataSource(IDataSource dataSource)
+    {
+        if (!this.CanRefreshDataSource(dataSource))
+            return;
+
+        await this.DataSourceEmbeddingService.QueueDataSourceAsync(dataSource);
+        await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
+    }
+
+    private async Task AddDataSource(DataSourceType type)
+    {
+        IDataSource? addedDataSource = null;
+        switch (type)
+        {
+            case DataSourceType.LOCAL_FILE:
+                var localFileDialogParameters = new DialogParameters<DataSourceLocalFileDialog>
+                {
+                    { x => x.IsEditing, false },
+                    { x => x.AvailableEmbeddings, this.availableEmbeddingProviders }
+                };
+
+                var localFileDialogReference = await this.DialogService.ShowAsync<DataSourceLocalFileDialog>(T("Add Local File as Data Source"), localFileDialogParameters, DialogOptions.FULLSCREEN);
+                var localFileDialogResult = await localFileDialogReference.Result;
+                if (localFileDialogResult is null || localFileDialogResult.Canceled)
+                    return;
+
+                var localFile = (DataSourceLocalFile)localFileDialogResult.Data!;
+                localFile = localFile with { Num = this.SettingsManager.ConfigurationData.NextDataSourceNum++ };
+                addedDataSource = localFile;
+                break;
+
+            case DataSourceType.LOCAL_DIRECTORY:
+                var localDirectoryDialogParameters = new DialogParameters<DataSourceLocalDirectoryDialog>
+                {
+                    { x => x.IsEditing, false },
+                    { x => x.AvailableEmbeddings, this.availableEmbeddingProviders }
+                };
+
+                var localDirectoryDialogReference = await this.DialogService.ShowAsync<DataSourceLocalDirectoryDialog>(T("Add Local Directory as Data Source"), localDirectoryDialogParameters, DialogOptions.FULLSCREEN);
+                var localDirectoryDialogResult = await localDirectoryDialogReference.Result;
+                if (localDirectoryDialogResult is null || localDirectoryDialogResult.Canceled)
+                    return;
+
+                var localDirectory = (DataSourceLocalDirectory)localDirectoryDialogResult.Data!;
+                localDirectory = localDirectory with { Num = this.SettingsManager.ConfigurationData.NextDataSourceNum++ };
+                addedDataSource = localDirectory;
+                break;
+
+            case DataSourceType.ERI_V1:
+                var eriDialogParameters = new DialogParameters<DataSourceERI_V1Dialog>
+                {
+                    { x => x.IsEditing, false },
+                };
+
+                var eriDialogReference = await this.DialogService.ShowAsync<DataSourceERI_V1Dialog>(T("Add ERI v1 Data Source"), eriDialogParameters, DialogOptions.FULLSCREEN);
+                var eriDialogResult = await eriDialogReference.Result;
+                if (eriDialogResult is null || eriDialogResult.Canceled)
+                    return;
+
+                var eriDataSource = (DataSourceERI_V1)eriDialogResult.Data!;
+                eriDataSource = eriDataSource with { Num = this.SettingsManager.ConfigurationData.NextDataSourceNum++ };
+                addedDataSource = eriDataSource;
+                break;
+        }
+
+        if(addedDataSource is null)
+            return;
+
+        this.SettingsManager.ConfigurationData.DataSources.Add(addedDataSource);
+        await this.SettingsManager.StoreSettings();
+        await this.DataSourceEmbeddingService.QueueDataSourceAsync(addedDataSource);
+        await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
+    }
+
+    private async Task ExportDataSource(IDataSource dataSource)
+    {
+        if (!this.SettingsManager.ConfigurationData.App.ShowAdminSettings)
+            return;
+
+        if (dataSource is not DataSourceERI_V1 eriDataSource)
+            return;
+
+        if (eriDataSource.AuthMethod is AuthMethod.KERBEROS)
+        {
+            await this.DialogService.ShowMessageBox(
+                T("Export ERI Data Source"),
+                T("Kerberos/SSO ERI data sources cannot be exported yet. Please configure them manually in the configuration plugin."),
+                T("Close"));
+            return;
+        }
+
+        var needsSecret = eriDataSource.AuthMethod is AuthMethod.TOKEN or AuthMethod.USERNAME_PASSWORD;
+        if (!needsSecret)
+        {
+            var publicLuaCode = eriDataSource.ExportAsConfigurationSection();
+            if (!string.IsNullOrWhiteSpace(publicLuaCode))
+                await this.RustService.CopyText2Clipboard(publicLuaCode);
+
+            return;
+        }
+
+        var secretResponse = await this.RustService.GetSecret(eriDataSource, SecretStoreType.DATA_SOURCE, isTrying: true);
+        if (!secretResponse.Success)
+        {
+            await this.DialogService.ShowMessageBox(
+                T("Export ERI Data Source"),
+                string.Format(T("Cannot export this ERI data source because no authentication secret is configured. The issue was: {0}"), secretResponse.Issue),
+                T("Close"));
+            return;
+        }
+
+        var encryption = PluginFactory.EnterpriseEncryption;
+        if (encryption?.IsAvailable != true)
+        {
+            await this.DialogService.ShowMessageBox(
+                T("Export ERI Data Source"),
+                T("Cannot export this ERI data source because no enterprise encryption secret is configured."),
+                T("Close"));
+            return;
+        }
+
+        var usernamePasswordMode = DataSourceERIUsernamePasswordMode.USER_MANAGED;
+        if (eriDataSource.AuthMethod is AuthMethod.TOKEN)
+        {
+            var dialogParameters = new DialogParameters<ConfirmDialog>
+            {
+                { x => x.Message, T("This ERI data source has an access token configured. Do you want to include the encrypted access token in the export? Note: The recipient will need the same encryption secret to use the access token.") },
+            };
+
+            var dialogReference = await this.DialogService.ShowAsync<ConfirmDialog>(T("Export Access Token?"), dialogParameters, DialogOptions.FULLSCREEN);
+            var dialogResult = await dialogReference.Result;
+            if (dialogResult is null || dialogResult.Canceled)
+                return;
+        }
+        else if (eriDataSource.AuthMethod is AuthMethod.USERNAME_PASSWORD)
+        {
+            var dialogParameters = new DialogParameters<DataSourceERIV1UsernamePasswordExportDialog>
+            {
+                { x => x.DataSource, eriDataSource },
+            };
+
+            var dialogReference = await this.DialogService.ShowAsync<DataSourceERIV1UsernamePasswordExportDialog>(T("Export ERI Data Source"), dialogParameters, DialogOptions.FULLSCREEN);
+            var dialogResult = await dialogReference.Result;
+            if (dialogResult is null || dialogResult.Canceled || dialogResult.Data is not DataSourceERIV1UsernamePasswordExportDialogResult exportResult)
+                return;
+
+            usernamePasswordMode = exportResult.UsernamePasswordMode;
+        }
+
+        var decryptedSecret = await secretResponse.Secret.Decrypt(Program.ENCRYPTION);
+        if (!encryption.TryEncrypt(decryptedSecret, out var encryptedSecret))
+        {
+            await this.DialogService.ShowMessageBox(
+                T("Export ERI Data Source"),
+                T("Cannot export this ERI data source because the authentication secret could not be encrypted."),
+                T("Close"));
+            return;
+        }
+
+        var luaCode = eriDataSource.ExportAsConfigurationSection(
+            encryptedSecret,
+            usernamePasswordMode);
+        if (string.IsNullOrWhiteSpace(luaCode))
+            return;
+
+        await this.RustService.CopyText2Clipboard(luaCode);
+    }
+
+    private async Task EditDataSource(IDataSource dataSource)
+    {
+        if (dataSource.IsEnterpriseConfiguration)
+            return;
+
+        IDataSource? editedDataSource = null;
+        var lockDataSourceIdentity = dataSource is IInternalDataSource
+            && await this.DataSourceEmbeddingService.ShouldLockDataSourceIdentityAsync(dataSource.Id);
+        switch (dataSource)
+        {
+            case DataSourceLocalFile localFile:
+                var localFileDialogParameters = new DialogParameters<DataSourceLocalFileDialog>
+                {
+                    { x => x.IsEditing, true },
+                    { x => x.DataSource, localFile },
+                    { x => x.LockSourceAndEmbedding, lockDataSourceIdentity },
+                    { x => x.AvailableEmbeddings, this.availableEmbeddingProviders }
+                };
+
+                var localFileDialogReference = await this.DialogService.ShowAsync<DataSourceLocalFileDialog>(T("Edit Local File Data Source"), localFileDialogParameters, DialogOptions.FULLSCREEN);
+                var localFileDialogResult = await localFileDialogReference.Result;
+                if (localFileDialogResult is null || localFileDialogResult.Canceled)
+                    return;
+
+                editedDataSource = (DataSourceLocalFile)localFileDialogResult.Data!;
+                break;
+
+            case DataSourceLocalDirectory localDirectory:
+                var localDirectoryDialogParameters = new DialogParameters<DataSourceLocalDirectoryDialog>
+                {
+                    { x => x.IsEditing, true },
+                    { x => x.DataSource, localDirectory },
+                    { x => x.LockSourceAndEmbedding, lockDataSourceIdentity },
+                    { x => x.AvailableEmbeddings, this.availableEmbeddingProviders }
+                };
+
+                var localDirectoryDialogReference = await this.DialogService.ShowAsync<DataSourceLocalDirectoryDialog>(T("Edit Local Directory Data Source"), localDirectoryDialogParameters, DialogOptions.FULLSCREEN);
+                var localDirectoryDialogResult = await localDirectoryDialogReference.Result;
+                if (localDirectoryDialogResult is null || localDirectoryDialogResult.Canceled)
+                    return;
+
+                editedDataSource = (DataSourceLocalDirectory)localDirectoryDialogResult.Data!;
+                break;
+
+            case DataSourceERI_V1 eriDataSource:
+                var eriDialogParameters = new DialogParameters<DataSourceERI_V1Dialog>
+                {
+                    { x => x.IsEditing, true },
+                    { x => x.DataSource, eriDataSource },
+                };
+
+                var eriDialogReference = await this.DialogService.ShowAsync<DataSourceERI_V1Dialog>(T("Edit ERI v1 Data Source"), eriDialogParameters, DialogOptions.FULLSCREEN);
+                var eriDialogResult = await eriDialogReference.Result;
+                if (eriDialogResult is null || eriDialogResult.Canceled)
+                    return;
+
+                editedDataSource = (DataSourceERI_V1)eriDialogResult.Data!;
+                break;
+        }
+
+        if(editedDataSource is null)
+            return;
+
+        this.SettingsManager.ConfigurationData.DataSources[this.SettingsManager.ConfigurationData.DataSources.IndexOf(dataSource)] = editedDataSource;
+
+        await this.SettingsManager.StoreSettings();
+        await this.DataSourceEmbeddingService.QueueDataSourceAsync(editedDataSource);
+        await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
+    }
+
+    private async Task DeleteDataSource(IDataSource dataSource)
+    {
+        if (dataSource.IsEnterpriseConfiguration)
+            return;
+
+        var dialogParameters = new DialogParameters<ConfirmDialog>
+        {
+            { x => x.Message, string.Format(T("Are you sure you want to delete the data source '{0}' of type '{1}'?"), dataSource.Name, dataSource.Type.GetDisplayName()) },
+        };
+
+        var dialogReference = await this.DialogService.ShowAsync<ConfirmDialog>(T("Delete Data Source"), dialogParameters, DialogOptions.FULLSCREEN);
+        var dialogResult = await dialogReference.Result;
+        if (dialogResult is null || dialogResult.Canceled)
+            return;
+
+        var applyChanges = dataSource is IInternalDataSource;
+
+        // External data sources may need a secret for authentication:
+        if (dataSource is IExternalDataSource externalDataSource)
+        {
+            // When the auth method is NONE or KERBEROS, we don't need to delete a secret.
+            // In the case of KERBEROS, we don't store the Kerberos ticket in the secret store.
+            if(dataSource is IERIDataSource { AuthMethod: AuthMethod.NONE or AuthMethod.KERBEROS })
+                applyChanges = true;
+
+            // All other auth methods require a secret, which we need to delete now:
+            else
+            {
+                var deleteSecretResponse = await this.RustService.DeleteSecret(externalDataSource, SecretStoreType.DATA_SOURCE);
+                if (deleteSecretResponse.Success)
+                    applyChanges = true;
+            }
+        }
+
+        if(applyChanges)
+        {
+            this.SettingsManager.ConfigurationData.DataSources.Remove(dataSource);
+            await this.SettingsManager.StoreSettings();
+            await this.DataSourceEmbeddingService.RemoveDataSourceAsync(dataSource);
+            await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
+        }
+    }
+
+    private async Task ShowInformation(IDataSource dataSource)
+    {
+        switch (dataSource)
+        {
+            case DataSourceLocalFile localFile:
+                var localFileDialogParameters = new DialogParameters<DataSourceLocalFileInfoDialog>
+                {
+                    { x => x.DataSource, localFile },
+                };
+
+                await this.DialogService.ShowAsync<DataSourceLocalFileInfoDialog>(T("Local File Data Source Information"), localFileDialogParameters, DialogOptions.FULLSCREEN);
+                break;
+
+            case DataSourceLocalDirectory localDirectory:
+                var localDirectoryDialogParameters = new DialogParameters<DataSourceLocalDirectoryInfoDialog>
+                {
+                    { x => x.DataSource, localDirectory },
+                };
+
+                await this.DialogService.ShowAsync<DataSourceLocalDirectoryInfoDialog>(T("Local Directory Data Source Information"), localDirectoryDialogParameters, DialogOptions.FULLSCREEN);
+                break;
+
+            case DataSourceERI_V1 eriV1DataSource:
+                var eriV1DialogParameters = new DialogParameters<DataSourceERI_V1InfoDialog>
+                {
+                    { x => x.DataSource, eriV1DataSource },
+                };
+
+                await this.DialogService.ShowAsync<DataSourceERI_V1InfoDialog>(T("ERI v1 Data Source Information"), eriV1DialogParameters, DialogOptions.FULLSCREEN);
+                break;
+        }
+    }
+}

app/MindWork AI Studio/Components/DataSourceSelection.razor

@@ -1,4 +1,5 @@
 @using AIStudio.Settings
+@using AIStudio.Provider
 @inherits MSGComponentBase
 @if (this.SelectionMode is DataSourceSelectionMode.SELECTION_MODE)
 {
@@ -6,11 +7,11 @@
         <MudTooltip Text="@T("Select the data you want to use here.")" Placement="Placement.Top">
             @if (this.PopoverTriggerMode is PopoverTriggerMode.ICON)
             {
-                <MudIconButton Icon="@Icons.Material.Filled.Source" Class="@this.PopoverButtonClasses" OnClick="@(() => this.ToggleDataSourceSelection())"/>
+                <MudIconButton Icon="@AppIcons.DATABASE" Class="@this.PopoverButtonClasses" OnClick="@(() => this.ToggleDataSourceSelection())"/>
             }
             else
             {
-                <MudButton Variant="Variant.Filled" StartIcon="@Icons.Material.Filled.Source" Class="@this.PopoverButtonClasses" OnClick="@(() => this.ToggleDataSourceSelection())">
+                <MudButton Variant="Variant.Filled" StartIcon="@AppIcons.DATABASE" Class="@this.PopoverButtonClasses" OnClick="@(() => this.ToggleDataSourceSelection())">
                     @T("Select data")
                 </MudButton>
             }
@@ -20,7 +21,7 @@
             <MudCard>
                 <MudCardHeader>
                     <CardHeaderContent>
-                        <PreviewPrototype/>
+                        <PreviewBeta/>
                         <MudStack Row="true" AlignItems="AlignItems.Center">
                             <MudText Typo="Typo.h5">
                                 @T("Data Source Selection")
@@ -70,7 +71,7 @@
                             {
                                 case true when this.availableDataSources.Count == 0:
                                     <MudText Typo="Typo.body1" Class="mb-3">
-                                        @T("Your data sources cannot be used with the LLM provider you selected due to data privacy, or they are currently unavailable.")
+                                        @T("Your data sources cannot be used with the selected providers due to data privacy or confidence-level requirements, or they are currently unavailable.")
                                     </MudText>
                                     break; 
                                     
@@ -82,7 +83,7 @@
                                     
                                 case false when this.availableDataSources.Count == 0:
                                     <MudText Typo="Typo.body1" Class="mb-3">
-                                        @T("Your data sources cannot be used with the LLM provider you selected due to data privacy, or they are currently unavailable.")
+                                        @T("Your data sources cannot be used with the selected providers due to data privacy or confidence-level requirements, or they are currently unavailable.")
                                     </MudText>
                                     break;
                                     
@@ -92,7 +93,18 @@
                                             @foreach (var source in this.availableDataSources)
                                             {
                                                 <MudListItem Value="@source">
-                                                    @source.Name
+                                                    <MudStack Row="true" AlignItems="AlignItems.Center" Spacing="1" Style="min-width: 0; width: 100%;">
+                                                        <MudText Typo="Typo.body1" Style="min-width: 0; white-space: normal; overflow-wrap: anywhere;">
+                                                            @source.Name
+                                                        </MudText>
+                                                        @if (source is IInternalDataSource internalSource)
+                                                        {
+                                                            <MudSpacer/>
+                                                            <MudTooltip Text="@internalSource.ConfidenceLevel.GetName()">
+                                                                <MudIcon Icon="@Icons.Material.Filled.Security" Class="confidence-icon" Style="@this.GetConfidenceIconStyle(internalSource)"/>
+                                                            </MudTooltip>
+                                                        }
+                                                    </MudStack>
                                                 </MudListItem>
                                             }
                                         </MudList>
@@ -106,7 +118,18 @@
                                                 @foreach (var source in this.availableDataSources)
                                                 {
                                                     <MudListItem Value="@source">
-                                                        @source.Name
+                                                        <MudStack Row="true" AlignItems="AlignItems.Center" Spacing="1" Style="min-width: 0; width: 100%;">
+                                                            <MudText Typo="Typo.body1" Style="min-width: 0; white-space: normal; overflow-wrap: anywhere;">
+                                                                @source.Name
+                                                            </MudText>
+                                                            @if (source is IInternalDataSource internalSource)
+                                                            {
+                                                                <MudSpacer/>
+                                                                <MudTooltip Text="@internalSource.ConfidenceLevel.GetName()">
+                                                                    <MudIcon Icon="@Icons.Material.Filled.Security" Class="confidence-icon" Style="@this.GetConfidenceIconStyle(internalSource)"/>
+                                                                </MudTooltip>
+                                                            }
+                                                        </MudStack>
                                                     </MudListItem>
                                                 }
                                             </MudList>
@@ -117,9 +140,18 @@
                                                 {
                                                     <MudListItem Value="@source">
                                                         <ChildContent>
-                                                            <MudText Typo="Typo.body1">
-                                                                @source.DataSource.Name
-                                                            </MudText>
+                                                            <MudStack Row="true" AlignItems="AlignItems.Center" Spacing="1" Style="min-width: 0; width: 100%;">
+                                                                <MudText Typo="Typo.body1" Style="min-width: 0; white-space: normal; overflow-wrap: anywhere;">
+                                                                    @source.DataSource.Name
+                                                                </MudText>
+                                                                @if (source.DataSource is IInternalDataSource internalSource)
+                                                                {
+                                                                    <MudSpacer/>
+                                                                    <MudTooltip Text="@internalSource.ConfidenceLevel.GetName()">
+                                                                        <MudIcon Icon="@Icons.Material.Filled.Security" Class="confidence-icon" Style="@this.GetConfidenceIconStyle(internalSource)"/>
+                                                                    </MudTooltip>
+                                                                }
+                                                            </MudStack>
                                                         
                                                             <MudProgressLinear Color="Color.Info" Min="0" Max="1" Value="@source.AIDecision.Confidence"/>
                                                             <MudJustifiedText Typo="Typo.body2">
@@ -148,7 +180,7 @@
 else if (this.SelectionMode is DataSourceSelectionMode.CONFIGURATION_MODE)
 {
     <MudPaper Class="pa-3 mb-8 mt-3 border-dashed border rounded-lg">
-        <PreviewPrototype/>
+        <PreviewBeta/>
         <MudText Typo="Typo.h5">
             @T("Data Source Selection")
         </MudText>
@@ -170,11 +202,22 @@ else if (this.SelectionMode is DataSourceSelectionMode.CONFIGURATION_MODE)
                     @foreach (var source in this.availableDataSources)
                     {
                         <MudListItem Value="@source">
-                            @source.Name
+                            <MudStack Row="true" AlignItems="AlignItems.Center" Spacing="1" Style="min-width: 0; width: 100%;">
+                                <MudText Typo="Typo.body1" Style="min-width: 0; white-space: normal; overflow-wrap: anywhere;">
+                                    @source.Name
+                                </MudText>
+                                @if (source is IInternalDataSource internalSource)
+                                {
+                                    <MudSpacer/>
+                                    <MudTooltip Text="@internalSource.ConfidenceLevel.GetName()">
+                                        <MudIcon Icon="@Icons.Material.Filled.Security" Class="confidence-icon" Style="@this.GetConfidenceIconStyle(internalSource)"/>
+                                    </MudTooltip>
+                                }
+                            </MudStack>
                         </MudListItem>
                     }
                 </MudList>
             </MudField>
         }
     </MudPaper>
-}
+}

app/MindWork AI Studio/Components/DataSourceSelection.razor.cs

@@ -1,4 +1,5 @@
 using AIStudio.Dialogs.Settings;
+using AIStudio.Provider;
 using AIStudio.Settings;
 using AIStudio.Settings.DataModel;
 using AIStudio.Tools.Services;
@@ -141,6 +142,8 @@ public partial class DataSourceSelection : MSGComponentBase
     private IReadOnlyCollection<DataSourceAgentSelected> GetSelectedDataSourcesWithAI() => this.DataSourcesAISelected.Where(n => n.Selected).ToList();
 
     private string GetAIReasoning(DataSourceAgentSelected source) => $"AI reasoning (confidence {source.AIDecision.Confidence:P0}): {source.AIDecision.Reason}";
+
+    private string GetConfidenceIconStyle(IInternalDataSource source) => $"{source.ConfidenceLevel.SetColorStyle(this.SettingsManager)} flex-shrink: 0;";
     
     public void ChangeOptionWithoutSaving(DataSourceOptions options, IReadOnlyList<DataSourceAgentSelected>? aiSelectedDataSources = null)
     {
@@ -198,7 +201,7 @@ public partial class DataSourceSelection : MSGComponentBase
         this.StateHasChanged();
             
         // Load the data sources:
-        var sources = await this.DataSourceService.GetDataSources(this.LLMProvider, this.selectedDataSources);
+        var sources = await this.DataSourceService.GetDataSources(this.LLMProvider, this.DataSourceOptions, this.selectedDataSources);
         if (generation != this.loadAndApplyFiltersGeneration)
             return;
 
@@ -222,7 +225,8 @@ public partial class DataSourceSelection : MSGComponentBase
     {
         this.aiBasedSourceSelection = state;
         this.DataSourceOptions.AutomaticDataSourceSelection = this.aiBasedSourceSelection;
-        
+
+        await this.LoadAndApplyFilters();
         await this.OptionsChanged();
     }
     
@@ -230,7 +234,8 @@ public partial class DataSourceSelection : MSGComponentBase
     {
         this.aiBasedValidation = state;
         this.DataSourceOptions.AutomaticValidation = this.aiBasedValidation;
-        
+
+        await this.LoadAndApplyFilters();
         await this.OptionsChanged();
     }
     
@@ -308,4 +313,4 @@ public partial class DataSourceSelection : MSGComponentBase
     }
 
     #endregion
-}
+}

app/MindWork AI Studio/Components/DirectChatLauncherForm.razor

@@ -36,7 +36,7 @@
         <MudSelectItem T="string" Value="@chatTemplate.Id">@chatTemplate.GetSafeName()</MudSelectItem>
     }
 </MudSelect>
-<MudSelect T="string" Label="@T("Data sources (Optional)")" MultiSelection="@true" SelectedValues="@this.DataSourceIds" SelectedValuesChanged="@this.SetDataSourceIds" MultiSelectionTextFunc="@this.GetSelectedDataSourceText" Variant="Variant.Outlined" Margin="Margin.Dense" Class="mb-3 rounded-lg" Adornment="Adornment.Start" AdornmentIcon="@Icons.Material.Filled.Source">
+<MudSelect T="string" Label="@T("Data sources (Optional)")" MultiSelection="@true" SelectedValues="@this.DataSourceIds" SelectedValuesChanged="@this.SetDataSourceIds" MultiSelectionTextFunc="@this.GetSelectedDataSourceText" Variant="Variant.Outlined" Margin="Margin.Dense" Class="mb-3 rounded-lg" Adornment="Adornment.Start" AdornmentIcon="@AppIcons.DATABASE">
     @foreach (var dataSource in this.SettingsManager.ConfigurationData.DataSources)
     {
         <MudSelectItem T="string" Value="@dataSource.Id">@dataSource.Name</MudSelectItem>

app/MindWork AI Studio/Components/ExpansionPanel.razor

@@ -1,8 +1,8 @@
-<MudExpansionPanel Class="border-solid border rounded-lg" Expanded="@this.IsExpanded" MaxHeight="@this.MaxHeight" ExpandedChanged="async s => await this.ExpandedChanged(s)">
+<MudExpansionPanel Class="border-solid border rounded-lg" HeaderClass="@this.HeaderClass" Expanded="@this.IsExpanded" MaxHeight="@this.MaxHeight" ExpandedChanged="async s => await this.ExpandedChanged(s)">
     <TitleContent>
         <div class="d-flex align-center">
             <MudIcon Icon="@this.HeaderIcon" Size="@this.IconSize" Color="@this.IconColor" class="mr-3"/>
-            <MudText Typo="Typo.h6">
+            <MudText Typo="@this.HeaderTypo">
                 @this.HeaderText
             </MudText>
             @if (this.ShowEndButton)

app/MindWork AI Studio/Components/ExpansionPanel.razor.cs

@@ -15,7 +15,27 @@ public partial class ExpansionPanel : ComponentBase
     
     [Parameter]
     public string HeaderText { get; set; } = "n/a";
-    
+
+    /// <summary>
+    /// The typography of the header text.
+    /// </summary>
+    /// <remarks>
+    /// Worth lowering for a panel which sits inside another one, together with the compact header
+    /// class below: two headers of the same size give no clue about which one contains the other.
+    /// </remarks>
+    [Parameter]
+    public Typo HeaderTypo { get; set; } = Typo.h6;
+
+    /// <summary>
+    /// Additional class names for the header, separated by space.
+    /// </summary>
+    /// <remarks>
+    /// The one this exists for is expansion-panel-header-compact, which takes the height of the
+    /// header down for a nested panel.
+    /// </remarks>
+    [Parameter]
+    public string HeaderClass { get; set; } = string.Empty;
+
     [Parameter]
     public int? MaxHeight { get; set; }
     

app/MindWork AI Studio/Components/PathDropZone.razor

@@ -0,0 +1,7 @@
+@inherits MSGComponentBase
+
+<div @onmouseenter="@this.OnMouseEnter" @onmouseleave="@this.OnMouseLeave">
+    <MudPaper Outlined="@true" Class="@this.dragClass">
+        @this.ChildContent
+    </MudPaper>
+</div>

app/MindWork AI Studio/Components/PathDropZone.razor.cs

@@ -0,0 +1,178 @@
+using AIStudio.Tools.Rust;
+
+using Microsoft.AspNetCore.Components;
+
+namespace AIStudio.Components;
+
+/// <summary>
+/// A drop zone which reports the paths of whatever was dropped on it, and nothing else.
+/// </summary>
+/// <remarks>
+/// Dropping is a native matter in AI Studio: the Tauri runtime reports real paths, which is why
+/// this zone can hand out folders just as well as files. What those paths mean is the consumer's
+/// business — this component reads no content and does not care whether a path leads to a file or
+/// to a folder.
+/// </remarks>
+public partial class PathDropZone : MSGComponentBase
+{
+    /// <summary>
+    /// The content shown inside the zone.
+    /// </summary>
+    [Parameter]
+    public RenderFragment? ChildContent { get; set; }
+
+    /// <summary>
+    /// Reports the dropped paths, in the order the runtime delivered them.
+    /// </summary>
+    [Parameter]
+    public EventCallback<List<string>> OnPathsDropped { get; set; }
+
+    /// <summary>
+    /// On which layer to register the drop area. Higher layers have priority over lower layers.
+    /// </summary>
+    [Parameter]
+    public int Layer { get; set; } = DropLayers.ROOT;
+
+    /// <summary>
+    /// Catch all documents that are hovered over the AI Studio window and not only over the drop zone.
+    /// </summary>
+    /// <remarks>
+    /// Practically every zone needs this today. Hovering is detected through mouse events, and no
+    /// webview delivers those while a native drag is in progress, so a zone without this flag
+    /// hardly ever catches anything. The consequence is that two zones of the same layer cannot be
+    /// told apart: the one carrying this flag takes every drop, including the ones meant for the
+    /// other. A page may therefore hold only one zone per layer. Lifting that limit needs the
+    /// cursor position, which the runtime receives from Tauri and currently discards in
+    /// app_window.rs.
+    /// </remarks>
+    [Parameter]
+    public bool CatchAllDocuments { get; set; }
+
+    /// <summary>
+    /// When true, the zone ignores drops and is not highlighted.
+    /// </summary>
+    /// <remarks>
+    /// The drop area stays registered nevertheless. Releasing it during the lifetime of the
+    /// component would lower the count of every zone below this one, and those zones would then
+    /// catch files while this one is still on screen.
+    /// </remarks>
+    [Parameter]
+    public bool Disabled { get; set; }
+
+    [Inject]
+    private ILogger<PathDropZone> Logger { get; init; } = null!;
+
+    private const string DEFAULT_DRAG_CLASS = "relative rounded-lg border-2 border-dashed pa-3 mb-3 mud-width-full";
+
+    private string dragClass = DEFAULT_DRAG_CLASS;
+    private uint numDropAreasAboveThis;
+    private bool isComponentHovered;
+
+    #region Overrides of MSGComponentBase
+
+    protected override async Task OnInitializedAsync()
+    {
+        this.ApplyFilters([], [ Event.TAURI_EVENT_RECEIVED, Event.REGISTER_FILE_DROP_AREA, Event.UNREGISTER_FILE_DROP_AREA ]);
+        await this.MessageBus.SendMessage(this, Event.REGISTER_FILE_DROP_AREA, this.Layer);
+
+        await base.OnInitializedAsync();
+    }
+
+    /// <summary>
+    /// Releases the drop area.
+    /// </summary>
+    protected override void DisposeResources()
+    {
+        // Without this, drop areas below this one would count this component forever and would
+        // stop catching dropped files:
+        this.MessageBus.SendMessage(this, Event.UNREGISTER_FILE_DROP_AREA, this.Layer).Observe($"{nameof(PathDropZone)}: releasing the drop area");
+
+        base.DisposeResources();
+    }
+
+    protected override async Task ProcessIncomingMessage<T>(ComponentBase? sendingComponent, Event triggeredEvent, T? data) where T : default
+    {
+        // A disabled zone takes no files. It keeps track of the zones above it, though, because
+        // those come and go while this one is disabled:
+        if (this.Disabled && triggeredEvent == Event.TAURI_EVENT_RECEIVED)
+            return;
+
+        switch (triggeredEvent)
+        {
+            case Event.REGISTER_FILE_DROP_AREA when sendingComponent != this:
+            {
+                if(data is int layer && layer > this.Layer)
+                {
+                    this.numDropAreasAboveThis++;
+                    this.ClearDragClass();
+                }
+
+                break;
+            }
+
+            case Event.UNREGISTER_FILE_DROP_AREA when sendingComponent != this:
+            {
+                if(data is int layer && layer > this.Layer && this.numDropAreasAboveThis > 0)
+                    this.numDropAreasAboveThis--;
+
+                break;
+            }
+
+            case Event.TAURI_EVENT_RECEIVED when data is TauriEvent { EventType: TauriEventType.FILE_DROP_HOVERED }:
+                if(!this.CanCatchDroppedPath())
+                    return;
+
+                this.SetDragClass();
+                this.StateHasChanged();
+                break;
+
+            case Event.TAURI_EVENT_RECEIVED when data is TauriEvent { EventType: TauriEventType.FILE_DROP_CANCELED }:
+            case Event.TAURI_EVENT_RECEIVED when data is TauriEvent { EventType: TauriEventType.WINDOW_NOT_FOCUSED }:
+                this.isComponentHovered = false;
+                this.ClearDragClass();
+                this.StateHasChanged();
+                break;
+
+            case Event.TAURI_EVENT_RECEIVED when data is TauriEvent { EventType: TauriEventType.FILE_DROP_DROPPED, Payload: var paths }:
+                if(!this.CanCatchDroppedPath())
+                    return;
+
+                this.Logger.LogDebug("The path drop zone on layer {Layer} caught {Count} path(s).", this.Layer, paths.Count);
+                await this.OnPathsDropped.InvokeAsync(paths);
+                this.ClearDragClass();
+                this.StateHasChanged();
+                break;
+        }
+    }
+
+    #endregion
+
+    private bool CanCatchDroppedPath() => this.numDropAreasAboveThis is 0 && (this.isComponentHovered || this.CatchAllDocuments);
+
+    private void SetDragClass() => this.dragClass = $"{DEFAULT_DRAG_CLASS} mud-border-primary border-2";
+
+    private void ClearDragClass() => this.dragClass = DEFAULT_DRAG_CLASS;
+
+    private void OnMouseEnter(EventArgs _)
+    {
+        if(this.Disabled || this.numDropAreasAboveThis > 0)
+            return;
+
+        // A native drag delivers no DOM events at all, mouse events included. This fires before a
+        // drag begins, while the pointer still moves freely, which makes it a hint about where the
+        // user is aiming rather than a reliable signal. See the remarks on CatchAllDocuments:
+        this.isComponentHovered = true;
+        this.SetDragClass();
+        this.StateHasChanged();
+    }
+
+    private void OnMouseLeave(EventArgs _)
+    {
+        if(this.Disabled)
+            return;
+
+        this.isComponentHovered = false;
+        this.ClearDragClass();
+        this.StateHasChanged();
+    }
+}

app/MindWork AI Studio/Components/SelectDirectory.razor

@@ -1,19 +1,37 @@
 @inherits MSGComponentBase
 
-<MudStack Row="@true" Spacing="3" Class="mb-3" StretchItems="StretchItems.None" AlignItems="AlignItems.Center">
-    <MudTextField
-        T="string"
-        Text="@this.Directory"
-        Label="@this.Label"
-        ReadOnly="@true"
-        Validation="@this.Validation"
-        Adornment="Adornment.Start"
-        AdornmentIcon="@Icons.Material.Filled.Folder"
-        UserAttributes="@SPELLCHECK_ATTRIBUTES"
-        Variant="Variant.Outlined"
-    />
-    
-    <MudButton StartIcon="@Icons.Material.Filled.FolderOpen" Variant="Variant.Outlined" Color="Color.Primary" Disabled="@(this.Disabled || this.isDirectoryDialogOpen)" OnClick="@this.OpenDirectoryDialog">
-        @T("Choose Directory")
-    </MudButton>
-</MudStack>
+@if (this.EnableDragDrop)
+{
+    <PathDropZone Layer="@this.Layer" CatchAllDocuments="@this.CatchAllDocuments" Disabled="@this.Disabled" OnPathsDropped="@this.PathsDropped">
+        @this.Picker
+        <MudText Typo="Typo.body2">
+            @T("You can also drag & drop the folder here.")
+        </MudText>
+    </PathDropZone>
+}
+else
+{
+    @this.Picker
+}
+
+@code {
+
+    private RenderFragment Picker =>
+        @<MudStack Row="@true" Spacing="3" Class="mb-3" StretchItems="StretchItems.None" AlignItems="AlignItems.Center">
+            <MudTextField
+                T="string"
+                Text="@this.Directory"
+                Label="@this.Label"
+                ReadOnly="@true"
+                Validation="@this.Validation"
+                Adornment="Adornment.Start"
+                AdornmentIcon="@Icons.Material.Filled.Folder"
+                UserAttributes="@SPELLCHECK_ATTRIBUTES"
+                Variant="Variant.Outlined"
+            />
+
+            <MudButton StartIcon="@Icons.Material.Filled.FolderOpen" Variant="Variant.Outlined" Color="Color.Primary" Disabled="@(this.Disabled || this.isDirectoryDialogOpen)" OnClick="@this.OpenDirectoryDialog">
+                @T("Choose Directory")
+            </MudButton>
+        </MudStack>;
+}

app/MindWork AI Studio/Components/SelectDirectory.razor.cs

@@ -2,6 +2,9 @@ using AIStudio.Tools.Services;
 
 using Microsoft.AspNetCore.Components;
 
+// This component has a parameter called Directory, which would shadow the file system's Directory class:
+using IODirectory = System.IO.Directory;
+
 namespace AIStudio.Components;
 
 public partial class SelectDirectory : MSGComponentBase
@@ -24,6 +27,24 @@ public partial class SelectDirectory : MSGComponentBase
     [Parameter]
     public Func<string, string?> Validation { get; set; } = _ => null;
 
+    /// <summary>
+    /// When true, the folder can also be chosen by dropping it onto this component.
+    /// </summary>
+    [Parameter]
+    public bool EnableDragDrop { get; set; }
+
+    /// <summary>
+    /// On which layer to register the drop area. Higher layers have priority over lower layers.
+    /// </summary>
+    [Parameter]
+    public int Layer { get; set; } = DropLayers.ROOT;
+
+    /// <summary>
+    /// Catch all documents that are hovered over the AI Studio window and not only over the drop zone.
+    /// </summary>
+    [Parameter]
+    public bool CatchAllDocuments { get; set; }
+
     [Inject]
     public RustService RustService { get; set; } = null!;
     
@@ -69,4 +90,39 @@ public partial class SelectDirectory : MSGComponentBase
             this.isDirectoryDialogOpen = false;
         }
     }
+
+    /// <summary>
+    /// Takes the first dropped path which leads to a folder.
+    /// </summary>
+    /// <remarks>
+    /// A dropped file is rejected instead of being taken as its parent folder. Everything a folder
+    /// contains is processed, so guessing the parent of a mistakenly dropped file could pull in far
+    /// more data than the user meant to hand over.
+    /// </remarks>
+    /// <param name="paths">The dropped paths.</param>
+    private async Task PathsDropped(List<string> paths)
+    {
+        foreach (var path in paths)
+        {
+            if (!IODirectory.Exists(path))
+                continue;
+
+            this.Logger.LogInformation("The user dropped the directory '{DroppedDirectory}'.", path);
+            this.InternalDirectoryChanged(path);
+            return;
+        }
+
+        this.Logger.LogWarning("None of the {Count} dropped path(s) could be used as a directory.", paths.Count);
+        await this.MessageBus.SendWarning(new(Icons.Material.Filled.Warning, this.GetDropWarning(paths)));
+    }
+
+    private string GetDropWarning(List<string> paths)
+    {
+        // Naming the actual mistake beats a generic "that did not work". Dropping a file onto a
+        // folder picker is the likeliest of them:
+        if (paths.Any(File.Exists))
+            return T("Please drop a folder, not a file.");
+
+        return T("The dropped folder could not be accessed. Please choose it with the folder chooser instead.");
+    }
 }

app/MindWork AI Studio/Components/SelectFile.razor

@@ -1,19 +1,37 @@
 @inherits MSGComponentBase
 
-<MudStack Row="@true" Spacing="3" Class="mb-3" StretchItems="StretchItems.None" AlignItems="AlignItems.Center">
-    <MudTextField
-        T="string"
-        Text="@this.File"
-        Label="@this.Label"
-        ReadOnly="@true"
-        Validation="@this.Validation"
-        Adornment="Adornment.Start"
-        AdornmentIcon="@Icons.Material.Filled.AttachFile"
-        UserAttributes="@SPELLCHECK_ATTRIBUTES"
-        Variant="Variant.Outlined"
-    />
-    
-    <MudButton StartIcon="@Icons.Material.Filled.FolderOpen" Variant="Variant.Outlined" Color="Color.Primary" Disabled="@(this.Disabled || this.isFileDialogOpen)" OnClick="@this.OpenFileDialog">
-        @T("Choose File")
-    </MudButton>
-</MudStack>
+@if (this.EnableDragDrop)
+{
+    <PathDropZone Layer="@this.Layer" CatchAllDocuments="@this.CatchAllDocuments" Disabled="@this.Disabled" OnPathsDropped="@this.PathsDropped">
+        @this.Picker
+        <MudText Typo="Typo.body2">
+            @T("You can also drag & drop the file here.")
+        </MudText>
+    </PathDropZone>
+}
+else
+{
+    @this.Picker
+}
+
+@code {
+
+    private RenderFragment Picker =>
+        @<MudStack Row="@true" Spacing="3" Class="mb-3" StretchItems="StretchItems.None" AlignItems="AlignItems.Center">
+            <MudTextField
+                T="string"
+                Text="@this.File"
+                Label="@this.Label"
+                ReadOnly="@true"
+                Validation="@this.Validation"
+                Adornment="Adornment.Start"
+                AdornmentIcon="@Icons.Material.Filled.AttachFile"
+                UserAttributes="@SPELLCHECK_ATTRIBUTES"
+                Variant="Variant.Outlined"
+            />
+
+            <MudButton StartIcon="@Icons.Material.Filled.FolderOpen" Variant="Variant.Outlined" Color="Color.Primary" Disabled="@(this.Disabled || this.isFileDialogOpen)" OnClick="@this.OpenFileDialog">
+                @T("Choose File")
+            </MudButton>
+        </MudStack>;
+}

app/MindWork AI Studio/Components/SelectFile.razor.cs

@@ -3,6 +3,9 @@ using AIStudio.Tools.Services;
 
 using Microsoft.AspNetCore.Components;
 
+// This component has a parameter called File, which would shadow the file system's File class:
+using IOFile = System.IO.File;
+
 namespace AIStudio.Components;
 
 public partial class SelectFile : MSGComponentBase
@@ -28,6 +31,24 @@ public partial class SelectFile : MSGComponentBase
     [Parameter]
     public Func<string, string?> Validation { get; set; } = _ => null;
 
+    /// <summary>
+    /// When true, the file can also be chosen by dropping it onto this component.
+    /// </summary>
+    [Parameter]
+    public bool EnableDragDrop { get; set; }
+
+    /// <summary>
+    /// On which layer to register the drop area. Higher layers have priority over lower layers.
+    /// </summary>
+    [Parameter]
+    public int Layer { get; set; } = DropLayers.ROOT;
+
+    /// <summary>
+    /// Catch all documents that are hovered over the AI Studio window and not only over the drop zone.
+    /// </summary>
+    [Parameter]
+    public bool CatchAllDocuments { get; set; }
+
     [Inject]
     public RustService RustService { get; set; } = null!;
     
@@ -73,4 +94,46 @@ public partial class SelectFile : MSGComponentBase
             this.isFileDialogOpen = false;
         }
     }
+
+    /// <summary>
+    /// Takes the first dropped path which leads to a usable file.
+    /// </summary>
+    /// <remarks>
+    /// This component carries exactly one file, so a multi-selection cannot be honored as a whole.
+    /// A dropped folder is rejected instead of being read as "the first file inside it": the user
+    /// was asked for a file, and picking one for them would be a surprise.
+    /// </remarks>
+    /// <param name="paths">The dropped paths.</param>
+    private async Task PathsDropped(List<string> paths)
+    {
+        foreach (var path in paths)
+        {
+            if (!IOFile.Exists(path))
+                continue;
+
+            if (this.Filter is { Length: > 0 } && !FileTypes.IsAllowedPath(path, this.Filter))
+                continue;
+
+            this.Logger.LogInformation("The user dropped the file '{DroppedFilePath}'.", path);
+            this.InternalFileChanged(path);
+            return;
+        }
+
+        this.Logger.LogWarning("None of the {Count} dropped path(s) could be used as a file.", paths.Count);
+        await this.MessageBus.SendWarning(new(Icons.Material.Filled.Warning, this.GetDropWarning(paths)));
+    }
+
+    private string GetDropWarning(List<string> paths)
+    {
+        // Naming the actual mistake beats a generic "that did not work". Dropping a folder onto a
+        // file picker is the likeliest of them:
+        if (paths.Any(Directory.Exists))
+            return T("Please drop a file, not a folder.");
+
+        // The file exists, so the file type filter is what turned it down:
+        if (paths.Any(IOFile.Exists))
+            return T("Please drop a file with a supported file type.");
+
+        return T("The dropped file could not be accessed. Please choose it with the file chooser instead.");
+    }
 }

app/MindWork AI Studio/Components/Settings/SettingsPanelDataSources.razor

@@ -0,0 +1,10 @@
+@using AIStudio.Settings.DataModel
+@inherits SettingsPanelBase
+
+@if (PreviewFeatures.PRE_RAG_2024.IsEnabled(this.SettingsManager))
+{
+    <ExpansionPanel HeaderIcon="@AppIcons.DATABASE" HeaderText="@T("Configure Data Sources")">
+        <PreviewBeta ApplyInnerScrollingFix="true"/>
+        <DataSourceManagement/>
+    </ExpansionPanel>
+}

app/MindWork AI Studio/Components/Settings/SettingsPanelDataSources.razor.cs

@@ -0,0 +1,3 @@
+namespace AIStudio.Components.Settings;
+
+public partial class SettingsPanelDataSources : SettingsPanelBase;

app/MindWork AI Studio/Components/Settings/SettingsPanelEmbeddings.razor

@@ -6,7 +6,7 @@
 @if (PreviewFeatures.PRE_RAG_2024.IsEnabled(this.SettingsManager))
 {
     <ExpansionPanel HeaderIcon="@Icons.Material.Filled.IntegrationInstructions" HeaderText="@T("Configure Embedding Providers")">
-        <PreviewPrototype ApplyInnerScrollingFix="true"/>
+        <PreviewBeta ApplyInnerScrollingFix="true"/>
         <MudText Typo="Typo.h4" Class="mb-3">
             @T("Configured Embedding Providers")
         </MudText>

app/MindWork AI Studio/Components/Settings/SettingsPanelEmbeddings.razor.cs

@@ -2,6 +2,7 @@ using System.Globalization;
 using AIStudio.Dialogs;
 using AIStudio.Provider;
 using AIStudio.Settings;
+using AIStudio.Tools.Services;
 
 using Microsoft.AspNetCore.Components;
 
@@ -11,6 +12,9 @@ namespace AIStudio.Components.Settings;
 
 public partial class SettingsPanelEmbeddings : SettingsPanelProviderBase
 {
+    [Inject]
+    private DataSourceEmbeddingService DataSourceEmbeddingService { get; init; } = null!;
+
     /// <summary>
     /// Groups the table by the used LLM provider. The embedding provider list is already sorted by
     /// that provider, so all instances of one LLM provider form a single, coherent group.
@@ -68,6 +72,7 @@ public partial class SettingsPanelEmbeddings : SettingsPanelProviderBase
         await this.UpdateEmbeddingProviders();
         
         await this.SettingsManager.StoreSettings();
+        await this.DataSourceEmbeddingService.QueueAllInternalDataSourcesAsync();
         await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
     }
     
@@ -88,6 +93,9 @@ public partial class SettingsPanelEmbeddings : SettingsPanelProviderBase
             { x => x.IsSelfHosted, embeddingProvider.IsSelfHosted },
             { x => x.IsEditing, true },
             { x => x.DataHost, embeddingProvider.Host },
+            { x => x.DataTokenizerPath, embeddingProvider.TokenizerPath },
+            { x => x.DataTokenLimit, embeddingProvider.EffectiveTokenLimit },
+            { x => x.DataEmbeddingBatchSize, embeddingProvider.EffectiveEmbeddingBatchSize },
             { x => x.HFInferenceProviderId, embeddingProvider.HFInferenceProvider },
             { x => x.IsEnterpriseConfiguration, embeddingProvider.IsEnterpriseConfiguration },
         };
@@ -118,6 +126,7 @@ public partial class SettingsPanelEmbeddings : SettingsPanelProviderBase
         await this.UpdateEmbeddingProviders();
         
         await this.SettingsManager.StoreSettings();
+        await this.DataSourceEmbeddingService.QueueAllInternalDataSourcesAsync();
         await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
     }
 
@@ -134,13 +143,36 @@ public partial class SettingsPanelEmbeddings : SettingsPanelProviderBase
             return;
         
         var deleteSecretResponse = await this.RustService.DeleteAPIKey(provider, SecretStoreType.EMBEDDING_PROVIDER);
+
+        //
+        // Removing the tokenizer is best effort: it leaves an unused file behind when it fails,
+        // which is not worth bothering the user about while they are deleting the provider. The
+        // API key is different, though, because a leftover secret is a secret we promised to remove.
+        //
+        _ = await this.RustService.DeleteTokenizer(TokenizerModelId.ForEmbeddingProvider(provider));
         if(deleteSecretResponse.Success)
         {
             this.SettingsManager.ConfigurationData.EmbeddingProviders.Remove(provider);
             await this.SettingsManager.StoreSettings();
         }
+        else
+        {
+            var issueDialogParameters = new DialogParameters<ConfirmDialog>
+            {
+                { x => x.Message, string.Format(T("Couldn't delete the embedding provider '{0}'. The issue: {1}. We can ignore this issue and delete the embedding provider anyway. Do you want to ignore it and delete this embedding provider?"), provider.Name, deleteSecretResponse.Issue) },
+            };
+
+            var issueDialogReference = await this.DialogService.ShowAsync<ConfirmDialog>(T("Delete Embedding Provider"), issueDialogParameters, DialogOptions.FULLSCREEN);
+            var issueDialogResult = await issueDialogReference.Result;
+            if (issueDialogResult is null || issueDialogResult.Canceled)
+                return;
+
+            this.SettingsManager.ConfigurationData.EmbeddingProviders.Remove(provider);
+            await this.SettingsManager.StoreSettings();
+        }
 
         await this.UpdateEmbeddingProviders();
+        await this.DataSourceEmbeddingService.QueueAllInternalDataSourcesAsync();
         await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
     }
 
@@ -183,7 +215,21 @@ public partial class SettingsPanelEmbeddings : SettingsPanelProviderBase
             return;
 
         var embeddingProvider = provider.CreateProvider();
-        var embeddings = await embeddingProvider.EmbedTextAsync(provider.Model, this.SettingsManager, default, new List<string> { inputText });
+        IReadOnlyList<IReadOnlyList<float>> embeddings;
+        try
+        {
+            embeddings = await embeddingProvider.EmbedTextAsync(provider.Model, this.SettingsManager, CancellationToken.None, inputText);
+        }
+        catch (ProviderRequestException exception)
+        {
+            //
+            // The provider named what went wrong and what to do about it. Showing that beats the
+            // sentence below, which used to be the same one for a missing API key, an unreachable
+            // provider and a provider which cannot embed anything at all:
+            //
+            await this.DialogService.ShowMessageBox(T("Embedding Result"), exception.UserMessage, T("Close"));
+            return;
+        }
 
         if (embeddings.Count == 0)
         {

app/MindWork AI Studio/Components/Settings/SettingsPanelProviders.razor.cs

@@ -2,6 +2,7 @@ using System.Diagnostics.CodeAnalysis;
 
 using AIStudio.Dialogs;
 using AIStudio.Settings;
+using AIStudio.Tools.Services;
 
 using Microsoft.AspNetCore.Components;
 
@@ -84,6 +85,7 @@ public partial class SettingsPanelProviders : SettingsPanelProviderBase
             { x => x.DataHost, provider.Host },
             { x => x.HFInferenceProviderId, provider.HFInferenceProvider },
             { x => x.AdditionalJsonApiParameters, provider.AdditionalJsonApiParameters },
+            { x => x.DataTokenizerPath, provider.TokenizerPath },
             { x => x.DataCapabilityOverrides, provider.CapabilityOverrides },
             { x => x.IsEnterpriseConfiguration, provider.IsEnterpriseConfiguration },
         };
@@ -131,6 +133,13 @@ public partial class SettingsPanelProviders : SettingsPanelProviderBase
             return;
         
         var deleteSecretResponse = await this.RustService.DeleteAPIKey(provider, SecretStoreType.LLM_PROVIDER);
+
+        //
+        // Removing the tokenizer is best effort: it leaves an unused file behind when it fails,
+        // which is not worth bothering the user about while they are deleting the provider. The
+        // API key is different, though, because a leftover secret is a secret we promised to remove.
+        //
+        _ = await this.RustService.DeleteTokenizer(TokenizerModelId.ForProvider(provider));
         if(deleteSecretResponse.Success)
         {
             this.SettingsManager.ConfigurationData.Providers.Remove(provider);

app/MindWork AI Studio/Components/UserPromptComponent.cs

@@ -0,0 +1,108 @@
+using Microsoft.AspNetCore.Components;
+using Timer = System.Timers.Timer;
+
+namespace AIStudio.Components;
+
+/// <summary>
+/// Debounced multi-line text input built on <see cref="MudTextField{T}"/>.
+/// Keeps the base API while adding a debounce timer.
+/// Callers can override any property as usual.
+/// </summary>
+public class UserPromptComponent<T> : MudTextField<T>, IDisposable
+{
+    [Parameter]
+    public TimeSpan DebounceTime { get; set; } = TimeSpan.FromMilliseconds(800);
+
+    [Parameter]
+    public Func<string, Task> WhenTextChangedAsync { get; set; } = _ => Task.CompletedTask;
+
+    private readonly Timer debounceTimer = new();
+    private string text = string.Empty;
+    private string lastParameterText = string.Empty;
+    private string lastNotifiedText = string.Empty;
+    private bool isInitialized;
+    private bool isDisposed;
+
+    protected override async Task OnInitializedAsync()
+    {
+        this.text = this.Text ?? string.Empty;
+        this.lastParameterText = this.text;
+        this.lastNotifiedText = this.text;
+        this.debounceTimer.AutoReset = false;
+        this.debounceTimer.Interval = this.DebounceTime.TotalMilliseconds;
+        this.debounceTimer.Elapsed += this.WhenDebounceElapsed;
+
+        this.isInitialized = true;
+        await base.OnInitializedAsync();
+    }
+
+    protected override async Task OnParametersSetAsync()
+    {
+        // Ensure the timer uses the latest debouncing interval:
+        if (!this.isInitialized || this.isDisposed)
+        {
+            await base.OnParametersSetAsync();
+            return;
+        }
+
+        if(Math.Abs(this.debounceTimer.Interval - this.DebounceTime.TotalMilliseconds) > 1)
+            this.debounceTimer.Interval = this.DebounceTime.TotalMilliseconds;
+
+        // Only sync when the parent's parameter actually changed since the last change:
+        if (this.Text != this.lastParameterText)
+        {
+            this.text = this.Text ?? string.Empty;
+            this.lastParameterText = this.text;
+        }
+
+        this.debounceTimer.Stop();
+        this.debounceTimer.Start();
+
+        await base.OnParametersSetAsync();
+    }
+
+    private void WhenDebounceElapsed(object? sender, System.Timers.ElapsedEventArgs args)
+    {
+        this.debounceTimer.Stop();
+
+        //
+        // The timer runs on its own thread and may still fire while this component is being torn
+        // down. Notifying a renderer which is already gone would throw on that thread, where no
+        // caller is left to handle it.
+        //
+        if (this.isDisposed || this.text == this.lastNotifiedText)
+            return;
+
+        this.lastNotifiedText = this.text;
+        this.InvokeAsync(async () => await this.TextChanged.InvokeAsync(this.text)).Observe($"{nameof(UserPromptComponent<T>)}: notifying about changed text");
+        this.InvokeAsync(async () => await this.WhenTextChangedAsync(this.text)).Observe($"{nameof(UserPromptComponent<T>)}: handling changed text asynchronously");
+    }
+
+    #region IDisposable
+
+    public void Dispose()
+    {
+        if (this.isDisposed)
+            return;
+
+        //
+        // Set before stopping the timer: the handler might be running on the timer thread right
+        // now, and this is what tells it to leave the gone renderer alone.
+        //
+        this.isDisposed = true;
+        try
+        {
+            this.debounceTimer.Elapsed -= this.WhenDebounceElapsed;
+            this.debounceTimer.Stop();
+            this.debounceTimer.Dispose();
+        }
+        catch
+        {
+            // ignore
+        }
+
+        GC.SuppressFinalize(this);
+    }
+
+    #endregion
+}

app/MindWork AI Studio/Dialogs/DataSourceERI_V1Dialog.razor

@@ -1,4 +1,5 @@
 @using AIStudio.Settings.DataModel
+@using AIStudio.Tools.Validation
 @using AIStudio.Tools.ERIClient.DataModel
 @inherits MSGComponentBase
 
@@ -11,8 +12,8 @@
                 @bind-Text="@this.dataName"
                 Label="@T("Data Source Name")"
                 Class="mb-6"
-                MaxLength="40"
-                Counter="40"
+                MaxLength="@DataSourceValidation.MAX_NAME_LENGTH"
+                Counter="@DataSourceValidation.MAX_NAME_LENGTH"
                 Immediate="@true"
                 Validation="@this.dataSourceValidation.ValidatingName"
                 Adornment="Adornment.Start"
@@ -119,7 +120,7 @@
                     </MudSelectItem>
                 }
             </MudSelect>
-            
+
             <MudNumericField T="ushort" Min="10" @bind-Value="@this.dataMaxMatches" Label="@T("How many matches do you want at most per query?")" Variant="Variant.Outlined" Step="10" />
             
         </MudForm>

app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor

@@ -1,4 +1,6 @@
 @using AIStudio.Settings.DataModel
+@using AIStudio.Tools.Validation
+@using AIStudio.Provider
 @inherits MSGComponentBase
 
 <MudDialog>
@@ -10,8 +12,8 @@
                 @bind-Text="@this.dataName"
                 Label="@T("Data Source Name")"
                 Class="mb-6"
-                MaxLength="40"
-                Counter="40"
+                MaxLength="@DataSourceValidation.MAX_NAME_LENGTH"
+                Counter="@DataSourceValidation.MAX_NAME_LENGTH"
                 Immediate="@true"
                 Validation="@this.dataSourceValidation.ValidatingName"
                 Adornment="Adornment.Start"
@@ -41,42 +43,73 @@
             <MudJustifiedText Typo="Typo.body1" Class="mb-3">
                 @T("Select a root directory for this data source. All data in this directory and all its subdirectories will be processed for this data source.")
             </MudJustifiedText>
-            <SelectDirectory @bind-Directory="@this.dataPath" Label="@T("Selected base directory for this data source")" DirectoryDialogTitle="@T("Select the base directory")" Validation="@this.dataSourceValidation.ValidatePath" />
+            @if (!this.CanChangeSourceAndEmbedding)
+            {
+                <MudAlert Severity="Severity.Info" Variant="Variant.Outlined" Class="mb-3">
+                    @T("This data source already has indexed embeddings. Delete and recreate it to change the folder path or embedding model.")
+                </MudAlert>
+            }
+            @if (this.CanChangeSourceAndEmbedding)
+            {
+                <SelectDirectory @bind-Directory="@this.dataPath" Label="@T("Selected base directory for this data source")" DirectoryDialogTitle="@T("Select the base directory")" Validation="@this.dataSourceValidation.ValidatePath" EnableDragDrop="true" Layer="@DropLayers.DIALOGS" CatchAllDocuments="true" />
+            }
+            else
+            {
+                <MudTextField
+                    T="string"
+                    Text="@this.dataPath"
+                    Label="@T("Selected base directory for this data source")"
+                    Class="mb-3"
+                    ReadOnly="@true"
+                    Validation="@this.dataSourceValidation.ValidatePath"
+                    Adornment="Adornment.Start"
+                    AdornmentIcon="@Icons.Material.Filled.Folder"
+                    UserAttributes="@SPELLCHECK_ATTRIBUTES"
+                    Variant="Variant.Outlined"
+                />
+            }
 
             <MudJustifiedText Typo="Typo.body1" Class="mb-3">
                 @T("In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method.")
             </MudJustifiedText>
-            <MudSelect @bind-Value="@this.dataEmbeddingId" Label="@T("Embedding")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateEmbeddingId">
-                @foreach (var embedding in this.AvailableEmbeddings)
-                {
-                    <MudSelectItem Value="@embedding.Value">
-                        @if (this.GetEmbeddingProvider(embedding.Value) is { } provider)
-                        {
-                            <ProviderLabel ProviderType="@provider.UsedLLMProvider" CustomIconDataUrl="@provider.CustomIconDataUrl" Text="@embedding.Name" />
-                        }
-                        else
-                        {
-                            @embedding.Name
-                        }
-                    </MudSelectItem>
-                }
-            </MudSelect>
+            @if (this.CanChangeSourceAndEmbedding)
+            {
+                <MudSelect @bind-Value="@this.dataEmbeddingId" Label="@T("Embedding")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateEmbeddingProviderAccess">
+                    @foreach (var embedding in this.AvailableEmbeddings)
+                    {
+                        <MudSelectItem Value="@embedding.Value">
+                            @if (this.GetEmbeddingProvider(embedding.Value) is { } provider)
+                            {
+                                <ProviderLabel ProviderType="@provider.UsedLLMProvider" CustomIconDataUrl="@provider.CustomIconDataUrl" Text="@embedding.Name" />
+                            }
+                            else
+                            {
+                                @embedding.Name
+                            }
+                        </MudSelectItem>
+                    }
+                </MudSelect>
+            }
+            else
+            {
+                <MudTextField
+                    T="string"
+                    Text="@this.SelectedEmbeddingNameText"
+                    Label="@T("Embedding")"
+                    Class="mb-3"
+                    ReadOnly="@true"
+                    Adornment="Adornment.Start"
+                    AdornmentIcon="@Icons.Material.Filled.DataObject"
+                    AdornmentColor="Color.Info"
+                    Variant="Variant.Outlined"
+                />
+            }
             
             @if (!string.IsNullOrWhiteSpace(this.dataEmbeddingId))
             {
                 if (this.SelectedCloudEmbedding)
                 {
-                    <MudJustifiedText Typo="Typo.body1" Color="Color.Error" Class="mb-3">
-                        @if (string.IsNullOrWhiteSpace(this.dataPath))
-                        {
-                            @T("Please note: the embedding you selected runs in the cloud. All your data will be sent to the cloud. Please confirm that you have read and understood this.")
-                        }
-                        else
-                        {
-                            @string.Format(T("Please note: the embedding you selected runs in the cloud. All your data from the folder '{0}' and all its subdirectories will be sent to the cloud. Please confirm that you have read and understood this."), this.dataPath)
-                        }
-                    </MudJustifiedText>
-                    <MudTextSwitch @bind-Value="@this.dataUserAcknowledgedCloudEmbedding" Label="@T("I confirm that I have read and understood the above")" LabelOn="@T("Yes, please send my data to the cloud")" LabelOff="@T("No, I will chose another embedding")" Validation="@this.dataSourceValidation.ValidateUserAcknowledgedCloudEmbedding"/>
+                    <DataSourceCloudEmbeddingWarning DataSourceType="DataSourceType.LOCAL_DIRECTORY" SourcePath="@this.dataPath" @bind-UserAcknowledged="@this.dataUserAcknowledgedCloudEmbedding" Validation="@this.dataSourceValidation.ValidateUserAcknowledgedCloudEmbedding"/>
                 }
                 else
                 {
@@ -88,16 +121,74 @@
             
             <ManagePandocDependency IntroText="@T("For some data types, such as Office files, MindWork AI Studio requires the open-source application Pandoc.")"/>
 
-            <MudSelect @bind-Value="@this.dataSecurityPolicy" Text="@this.dataSecurityPolicy.ToSelectionText()" Label="@T("Your security policy")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateSecurityPolicy">
-                @foreach (var policy in Enum.GetValues<DataSourceSecurity>())
+            <MudSelect @bind-Value="@this.dataConfidenceLevel" Text="@this.dataConfidenceLevel.GetName()" Label="@T("Required provider confidence level")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateDataSourceConfidenceLevel">
+                @foreach (var level in this.ConfidenceLevels)
                 {
-                    <MudSelectItem Value="@policy">
-                        @policy.ToSelectionText()
+                    <MudSelectItem Value="@level.Value">
+                        @level.Name
                     </MudSelectItem>
                 }
             </MudSelect>
             
-            <MudNumericField T="ushort" Min="10" @bind-Value="@this.dataMaxMatches" Label="@T("How many matches do you want at most per query?")" Variant="Variant.Outlined" Step="10" />
+            <MudStack Class="mb-3">
+                <MudButton OnClick="@this.ToggleExpertSettings" Variant="Variant.Text" Color="Color.Primary" StartIcon="@Icons.Material.Filled.Tune">
+                    @(this.showExpertSettings ? T("Hide Expert Settings") : T("Show Expert Settings"))
+                </MudButton>
+                <MudDivider/>
+                <MudCollapse Expanded="@this.showExpertSettings" Class="@this.GetExpertStyles">
+                    @if (!string.IsNullOrWhiteSpace(this.dataEmbeddingId))
+                    {
+                        <MudTextField
+                            T="string"
+                            Text="@this.SelectedEmbeddingTokenizerText"
+                            Label="@T("Tokenizer")"
+                            Class="mb-3"
+                            ReadOnly="@true"
+                            Variant="Variant.Outlined"
+                            Adornment="Adornment.Start"
+                            AdornmentIcon="@Icons.Material.Filled.TextFields"
+                            AdornmentColor="Color.Info"/>
+                    }
+                    <MudJustifiedText Typo="Typo.body1" Class="mb-3">
+                        @T("Optional expert settings for how this data source is split before embedding.")
+                    </MudJustifiedText>
+                    <MudNumericField
+                        T="int"
+                        @bind-Value="@this.dataMaxChunkTokenLength"
+                        Label="@T("Token limit")"
+                        Class="mb-3"
+                        Min="1"
+                        Immediate="@true"
+                        Validation="@this.ValidateMaxChunkTokenLength"
+                        HelperText="@this.MaxChunkTokenLengthHelperText"
+                        Variant="Variant.Outlined"
+                        Adornment="Adornment.Start"
+                        AdornmentIcon="@Icons.Material.Filled.FormatListNumbered"
+                        AdornmentColor="Color.Info"/>
+                    <MudNumericField
+                        T="int"
+                        @bind-Value="@this.dataChunkOverlapTokenLength"
+                        Label="@T("Token overlap")"
+                        Min="0"
+                        Immediate="@true"
+                        Validation="@this.ValidateChunkOverlapTokenLength"
+                        HelperText="@this.ChunkOverlapTokenLengthHelperText"
+                        Variant="Variant.Outlined"
+                        Adornment="Adornment.Start"
+                        AdornmentIcon="@Icons.Material.Filled.CompareArrows"
+                        AdornmentColor="Color.Info"/>
+                    <MudNumericField
+                        T="ushort"
+                        Min="10"
+                        @bind-Value="@this.dataMaxMatches"
+                        Label="@T("How many matches do you want at most per query?")"
+                        Variant="Variant.Outlined"
+                        Step="10"
+                        Adornment="Adornment.Start"
+                        AdornmentIcon="@Icons.Material.Filled.Search"
+                        AdornmentColor="Color.Info"/>
+                </MudCollapse>
+            </MudStack>
         </MudForm>
         <Issues IssuesData="@this.dataIssues"/>
     </DialogContent>

app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs

@@ -1,6 +1,8 @@
 using AIStudio.Components;
+using AIStudio.Provider;
 using AIStudio.Settings;
 using AIStudio.Settings.DataModel;
+using AIStudio.Tools.Services;
 using AIStudio.Tools.Validation;
 
 using Microsoft.AspNetCore.Components;
@@ -18,6 +20,9 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
     [Parameter]
     public DataSourceLocalDirectory DataSource { get; set; }
 
+    [Parameter]
+    public bool LockSourceAndEmbedding { get; set; }
+
     [Parameter]
     public IReadOnlyList<ConfigurationSelectData<string>> AvailableEmbeddings { get; set; } = [];
     
@@ -41,8 +46,11 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
     private bool dataUserAcknowledgedCloudEmbedding;
     private string dataEmbeddingId = string.Empty;
     private string dataPath = string.Empty;
+    private int dataMaxChunkTokenLength;
+    private int dataChunkOverlapTokenLength = DataSourceEmbeddingService.DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH;
     private ushort dataMaxMatches = 10;
-    private DataSourceSecurity dataSecurityPolicy;
+    private bool showExpertSettings;
+    private ConfidenceLevel dataConfidenceLevel = ConfidenceLevel.UNKNOWN;
     
     // We get the form reference from Blazor code to validate it manually:
     private MudForm form = null!;
@@ -52,6 +60,9 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
         this.dataSourceValidation = new()
         {
             GetSelectedCloudEmbedding = () => this.SelectedCloudEmbedding,
+            GetSelectedEmbeddingProvider = () => this.SelectedEmbedding,
+            GetConfidenceLevel = () => this.dataConfidenceLevel,
+            GetSettingsManager = () => this.SettingsManager,
             GetPreviousDataSourceName = () => this.dataEditingPreviousInstanceName,
             GetUsedDataSourceNames = () => this.UsedDataSourcesNames,
         };
@@ -77,7 +88,9 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
             this.dataDescription = this.DataSource.Description;
             this.dataEmbeddingId = this.DataSource.EmbeddingId;
             this.dataPath = this.DataSource.Path;
-            this.dataSecurityPolicy = this.DataSource.SecurityPolicy;
+            this.dataMaxChunkTokenLength = this.DataSource.MaxChunkTokenLength;
+            this.dataChunkOverlapTokenLength = this.DataSource.ChunkOverlapTokenLength;
+            this.dataConfidenceLevel = this.DataSource.ConfidenceLevel;
             this.dataMaxMatches = this.DataSource.MaxMatches;
         }
         
@@ -102,7 +115,39 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
         return provider == EmbeddingProvider.NONE ? null : provider;
     }
     
-    private bool SelectedCloudEmbedding => !(this.SettingsManager.ConfigurationData.EmbeddingProviders.FirstOrDefault(x => x.Id == this.dataEmbeddingId)?.IsTrustedForDataSourceSecurityChecks(this.SettingsManager) ?? false);
+    private EmbeddingProvider? SelectedEmbedding => this.SettingsManager.ConfigurationData.EmbeddingProviders
+        .FirstOrDefault(x => x.Id == this.dataEmbeddingId);
+
+    private bool SelectedCloudEmbedding => this.SelectedEmbedding is { IsSelfHosted: false };
+
+    private bool CanChangeSourceAndEmbedding => !this.IsEditing || !this.LockSourceAndEmbedding;
+
+    private IEnumerable<ConfigurationSelectData<ConfidenceLevel>> ConfidenceLevels => ConfigurationSelectDataFactory.GetDataSourceConfidenceLevelsData();
+
+    private string SelectedEmbeddingNameText
+    {
+        get
+        {
+            var selectedEmbedding = this.AvailableEmbeddings.FirstOrDefault(x => x.Value == this.dataEmbeddingId);
+            return string.IsNullOrWhiteSpace(selectedEmbedding.Name) ? T("Unknown") : selectedEmbedding.Name;
+        }
+    }
+
+    private string SelectedEmbeddingTokenizerText => this.SelectedEmbedding is null
+        ? T("No embedding selected")
+        : string.IsNullOrWhiteSpace(this.SelectedEmbedding.TokenizerPath)
+            ? T("Default tokenizer")
+            : Path.GetFileName(this.SelectedEmbedding.TokenizerPath);
+
+    private int ProviderMaxChunkTokenLength => this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT;
+
+    private string MaxChunkTokenLengthHelperText => string.Format(
+        T("Maximum number of tokens per chunk for this data source. The embedding provider default is {0} tokens."),
+        this.ProviderMaxChunkTokenLength);
+
+    private string ChunkOverlapTokenLengthHelperText => string.Format(
+        T("Number of tokens repeated at the start of the next chunk. The default overlap is {0} tokens."),
+        DataSourceEmbeddingService.DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH);
 
     private DataSourceLocalDirectory CreateDataSource() => new()
     {
@@ -111,9 +156,11 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
         Name = this.dataName,
         Description = this.dataDescription,
         Type = DataSourceType.LOCAL_DIRECTORY,
-        EmbeddingId = this.dataEmbeddingId,
-        Path = this.dataPath,
-        SecurityPolicy = this.dataSecurityPolicy,
+        EmbeddingId = this.CanChangeSourceAndEmbedding ? this.dataEmbeddingId : this.DataSource.EmbeddingId,
+        Path = this.CanChangeSourceAndEmbedding ? this.dataPath : this.DataSource.Path,
+        MaxChunkTokenLength = this.dataMaxChunkTokenLength,
+        ChunkOverlapTokenLength = this.dataChunkOverlapTokenLength,
+        ConfidenceLevel = this.dataConfidenceLevel,
         MaxMatches = this.dataMaxMatches,
     };
 
@@ -130,4 +177,45 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
     }
     
     private void Cancel() => this.MudDialog.Cancel();
+
+    private string? ValidateMaxChunkTokenLength(int maxChunkTokenLength)
+    {
+        if (!this.showExpertSettings)
+            return null;
+
+        if (maxChunkTokenLength < 1)
+            return T("Please enter a token limit of at least 1.");
+
+        var providerMaxChunkTokenLength = this.ProviderMaxChunkTokenLength;
+        if (maxChunkTokenLength > providerMaxChunkTokenLength)
+            return string.Format(T("The data source token limit must not be larger than the embedding provider token limit ({0})."), providerMaxChunkTokenLength);
+
+        return null;
+    }
+
+    private string? ValidateChunkOverlapTokenLength(int chunkOverlapTokenLength)
+    {
+        if (!this.showExpertSettings)
+            return null;
+
+        if (chunkOverlapTokenLength < 0)
+            return T("Please enter 0 or a positive overlap length.");
+
+        var effectiveMaxChunkTokenLength = this.showExpertSettings && this.dataMaxChunkTokenLength > 0
+            ? this.dataMaxChunkTokenLength
+            : this.ProviderMaxChunkTokenLength;
+        if (chunkOverlapTokenLength >= effectiveMaxChunkTokenLength)
+            return T("The overlap must be smaller than the effective token limit.");
+
+        return null;
+    }
+
+    private void ToggleExpertSettings()
+    {
+        this.showExpertSettings = !this.showExpertSettings;
+        if (this.showExpertSettings && this.dataMaxChunkTokenLength < 1)
+            this.dataMaxChunkTokenLength = this.ProviderMaxChunkTokenLength;
+    }
+
+    private string GetExpertStyles => this.showExpertSettings ? "border-2 border-dashed rounded pa-2" : string.Empty;
 }

app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryInfoDialog.razor

@@ -1,4 +1,4 @@
-@using AIStudio.Settings.DataModel
+@using AIStudio.Provider
 @inherits MSGComponentBase
 
 <MudDialog>
@@ -37,7 +37,7 @@
             </MudJustifiedText>
         }
 
-        <TextInfoLines Label="@T("Your security policy")" MaxLines="3" Value="@this.DataSource.SecurityPolicy.ToInfoText()" Color="@this.DataSource.SecurityPolicy.GetColor()" ClipboardTooltipSubject="@T("your security policy")"/>
+        <TextInfoLine Label="@T("Required provider confidence level")" Value="@this.DataSource.ConfidenceLevel.GetName()" ClipboardTooltipSubject="@T("the required provider confidence level")"/>
         <TextInfoLine Label="@T("Maximum matches per query")" Value="@this.DataSource.MaxMatches.ToString()" ClipboardTooltipSubject="@T("the maximum number of matches per query")"/>
 
         <TextInfoLine Icon="@Icons.Material.Filled.SquareFoot" Label="@T("Number of files")" Value="@this.NumberFilesInDirectory" ClipboardTooltipSubject="@T("the number of files in the directory")"/>

app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor

@@ -1,4 +1,6 @@
 @using AIStudio.Settings.DataModel
+@using AIStudio.Tools.Validation
+@using AIStudio.Provider
 @inherits MSGComponentBase
 
 <MudDialog>
@@ -10,8 +12,8 @@
                 @bind-Text="@this.dataName"
                 Label="@T("Data Source Name")"
                 Class="mb-6"
-                MaxLength="40"
-                Counter="40"
+                MaxLength="@DataSourceValidation.MAX_NAME_LENGTH"
+                Counter="@DataSourceValidation.MAX_NAME_LENGTH"
                 Immediate="@true"
                 Validation="@this.dataSourceValidation.ValidatingName"
                 Adornment="Adornment.Start"
@@ -41,42 +43,73 @@
             <MudJustifiedText Typo="Typo.body1" Class="mb-3">
                 @T("Select a file for this data source. The content of this file will be processed for the data source.")
             </MudJustifiedText>
-            <SelectFile @bind-File="@this.dataFilePath" Label="@T("Selected file path for this data source")" FileDialogTitle="@T("Select the file")" Validation="@this.dataSourceValidation.ValidateFilePath" />
+            @if (!this.CanChangeSourceAndEmbedding)
+            {
+                <MudAlert Severity="Severity.Info" Variant="Variant.Outlined" Class="mb-3">
+                    @T("This data source already has indexed embeddings. Delete and recreate it to change the file path or embedding model.")
+                </MudAlert>
+            }
+            @if (this.CanChangeSourceAndEmbedding)
+            {
+                <SelectFile @bind-File="@this.dataFilePath" Label="@T("Selected file path for this data source")" FileDialogTitle="@T("Select the file")" Validation="@this.dataSourceValidation.ValidateFilePath" EnableDragDrop="true" Layer="@DropLayers.DIALOGS" CatchAllDocuments="true" />
+            }
+            else
+            {
+                <MudTextField
+                    T="string"
+                    Text="@this.dataFilePath"
+                    Label="@T("Selected file path for this data source")"
+                    Class="mb-3"
+                    ReadOnly="@true"
+                    Validation="@this.dataSourceValidation.ValidateFilePath"
+                    Adornment="Adornment.Start"
+                    AdornmentIcon="@Icons.Material.Filled.AttachFile"
+                    UserAttributes="@SPELLCHECK_ATTRIBUTES"
+                    Variant="Variant.Outlined"
+                />
+            }
 
             <MudJustifiedText Typo="Typo.body1" Class="mb-3">
                 @T("In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method.")
             </MudJustifiedText>
-            <MudSelect @bind-Value="@this.dataEmbeddingId" Label="@T("Embedding")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.GetSelectedCloudEmbedding">
-                @foreach (var embedding in this.AvailableEmbeddings)
-                {
-                    <MudSelectItem Value="@embedding.Value">
-                        @if (this.GetEmbeddingProvider(embedding.Value) is { } provider)
-                        {
-                            <ProviderLabel ProviderType="@provider.UsedLLMProvider" CustomIconDataUrl="@provider.CustomIconDataUrl" Text="@embedding.Name" />
-                        }
-                        else
-                        {
-                            @embedding.Name
-                        }
-                    </MudSelectItem>
-                }
-            </MudSelect>
+            @if (this.CanChangeSourceAndEmbedding)
+            {
+                <MudSelect @bind-Value="@this.dataEmbeddingId" Label="@T("Embedding")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateEmbeddingProviderAccess">
+                    @foreach (var embedding in this.AvailableEmbeddings)
+                    {
+                        <MudSelectItem Value="@embedding.Value">
+                            @if (this.GetEmbeddingProvider(embedding.Value) is { } provider)
+                            {
+                                <ProviderLabel ProviderType="@provider.UsedLLMProvider" CustomIconDataUrl="@provider.CustomIconDataUrl" Text="@embedding.Name" />
+                            }
+                            else
+                            {
+                                @embedding.Name
+                            }
+                        </MudSelectItem>
+                    }
+                </MudSelect>
+            }
+            else
+            {
+                <MudTextField
+                    T="string"
+                    Text="@this.SelectedEmbeddingNameText"
+                    Label="@T("Embedding")"
+                    Class="mb-3"
+                    ReadOnly="@true"
+                    Adornment="Adornment.Start"
+                    AdornmentIcon="@Icons.Material.Filled.DataObject"
+                    AdornmentColor="Color.Info"
+                    Variant="Variant.Outlined"
+                />
+            }
             
             @if (!string.IsNullOrWhiteSpace(this.dataEmbeddingId))
             {
                 if (this.SelectedCloudEmbedding)
                 {
-                    <MudJustifiedText Typo="Typo.body1" Color="Color.Error" Class="mb-3">
-                        @if (string.IsNullOrWhiteSpace(this.dataFilePath))
-                        {
-                            @T("Please note: the embedding you selected runs in the cloud. All your data will be sent to the cloud. Please confirm that you have read and understood this.")
-                        }
-                        else
-                        {
-                            @string.Format(T("Please note: the embedding you selected runs in the cloud. All your data within the file '{0}' will be sent to the cloud. Please confirm that you have read and understood this."), this.dataFilePath)
-                        }
-                    </MudJustifiedText>
-                    <MudTextSwitch @bind-Value="@this.dataUserAcknowledgedCloudEmbedding" Label="@T("I confirm that I have read and understood the above")" LabelOn="@T("Yes, please send my data to the cloud")" LabelOff="@T("No, I will chose another embedding")" Validation="@this.dataSourceValidation.ValidateUserAcknowledgedCloudEmbedding"/>
+                    <DataSourceCloudEmbeddingWarning DataSourceType="DataSourceType.LOCAL_FILE" SourcePath="@this.dataFilePath" @bind-UserAcknowledged="@this.dataUserAcknowledgedCloudEmbedding" Validation="@this.dataSourceValidation.ValidateUserAcknowledgedCloudEmbedding"/>
                 }
                 else
                 {
@@ -88,16 +121,74 @@
             
             <ManagePandocDependency IntroText="@T("For some data types, such as Office files, MindWork AI Studio requires the open-source application Pandoc.")"/>
 
-            <MudSelect @bind-Value="@this.dataSecurityPolicy" Text="@this.dataSecurityPolicy.ToSelectionText()" Label="@T("Your security policy")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateSecurityPolicy">
-                @foreach (var policy in Enum.GetValues<DataSourceSecurity>())
+            <MudSelect @bind-Value="@this.dataConfidenceLevel" Text="@this.dataConfidenceLevel.GetName()" Label="@T("Required provider confidence level")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateDataSourceConfidenceLevel">
+                @foreach (var level in this.ConfidenceLevels)
                 {
-                    <MudSelectItem Value="@policy">
-                        @policy.ToSelectionText()
+                    <MudSelectItem Value="@level.Value">
+                        @level.Name
                     </MudSelectItem>
                 }
             </MudSelect>
-            
-            <MudNumericField T="ushort" Min="10" @bind-Value="@this.dataMaxMatches" Label="@T("How many matches do you want at most per query?")" Variant="Variant.Outlined" Step="10" />
+
+            <MudStack Class="mb-3">
+                <MudButton OnClick="@this.ToggleExpertSettings" Variant="Variant.Text" Color="Color.Primary" StartIcon="@Icons.Material.Filled.Tune">
+                    @(this.showExpertSettings ? T("Hide Expert Settings") : T("Show Expert Settings"))
+                </MudButton>
+                <MudDivider/>
+                <MudCollapse Expanded="@this.showExpertSettings" Class="@this.GetExpertStyles">
+                    @if (!string.IsNullOrWhiteSpace(this.dataEmbeddingId))
+                    {
+                        <MudTextField
+                            T="string"
+                            Text="@this.SelectedEmbeddingTokenizerText"
+                            Label="@T("Tokenizer")"
+                            Class="mb-3"
+                            ReadOnly="@true"
+                            Variant="Variant.Outlined"
+                            Adornment="Adornment.Start"
+                            AdornmentIcon="@Icons.Material.Filled.TextFields"
+                            AdornmentColor="Color.Info"/>
+                    }
+                    <MudJustifiedText Typo="Typo.body1" Class="mb-3">
+                        @T("Optional expert settings for how this data source is split before embedding.")
+                    </MudJustifiedText>
+                    <MudNumericField
+                        T="int"
+                        @bind-Value="@this.dataMaxChunkTokenLength"
+                        Label="@T("Token limit")"
+                        Class="mb-3"
+                        Min="1"
+                        Immediate="@true"
+                        Validation="@this.ValidateMaxChunkTokenLength"
+                        HelperText="@this.MaxChunkTokenLengthHelperText"
+                        Variant="Variant.Outlined"
+                        Adornment="Adornment.Start"
+                        AdornmentIcon="@Icons.Material.Filled.FormatListNumbered"
+                        AdornmentColor="Color.Info"/>
+                    <MudNumericField
+                        T="int"
+                        @bind-Value="@this.dataChunkOverlapTokenLength"
+                        Label="@T("Token overlap")"
+                        Min="0"
+                        Immediate="@true"
+                        Validation="@this.ValidateChunkOverlapTokenLength"
+                        HelperText="@this.ChunkOverlapTokenLengthHelperText"
+                        Variant="Variant.Outlined"
+                        Adornment="Adornment.Start"
+                        AdornmentIcon="@Icons.Material.Filled.CompareArrows"
+                        AdornmentColor="Color.Info"/>
+                    <MudNumericField
+                        T="ushort"
+                        Min="10"
+                        @bind-Value="@this.dataMaxMatches"
+                        Label="@T("How many matches do you want at most per query?")"
+                        Variant="Variant.Outlined"
+                        Step="10"
+                        Adornment="Adornment.Start"
+                        AdornmentIcon="@Icons.Material.Filled.Search"
+                        AdornmentColor="Color.Info"/>
+                </MudCollapse>
+            </MudStack>
         </MudForm>
         <Issues IssuesData="@this.dataIssues"/>
     </DialogContent>

app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs

@@ -1,6 +1,8 @@
 using AIStudio.Components;
+using AIStudio.Provider;
 using AIStudio.Settings;
 using AIStudio.Settings.DataModel;
+using AIStudio.Tools.Services;
 using AIStudio.Tools.Validation;
 
 using Microsoft.AspNetCore.Components;
@@ -17,6 +19,9 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
     
     [Parameter]
     public DataSourceLocalFile DataSource { get; set; }
+
+    [Parameter]
+    public bool LockSourceAndEmbedding { get; set; }
     
     [Parameter]
     public IReadOnlyList<ConfigurationSelectData<string>> AvailableEmbeddings { get; set; } = [];
@@ -41,8 +46,11 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
     private bool dataUserAcknowledgedCloudEmbedding;
     private string dataEmbeddingId = string.Empty;
     private string dataFilePath = string.Empty;
+    private int dataMaxChunkTokenLength;
+    private int dataChunkOverlapTokenLength = DataSourceEmbeddingService.DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH;
     private ushort dataMaxMatches = 10;
-    private DataSourceSecurity dataSecurityPolicy;
+    private bool showExpertSettings;
+    private ConfidenceLevel dataConfidenceLevel = ConfidenceLevel.UNKNOWN;
     
     // We get the form reference from Blazor code to validate it manually:
     private MudForm form = null!;
@@ -52,6 +60,9 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
         this.dataSourceValidation = new()
         {
             GetSelectedCloudEmbedding = () => this.SelectedCloudEmbedding,
+            GetSelectedEmbeddingProvider = () => this.SelectedEmbedding,
+            GetConfidenceLevel = () => this.dataConfidenceLevel,
+            GetSettingsManager = () => this.SettingsManager,
             GetPreviousDataSourceName = () => this.dataEditingPreviousInstanceName,
             GetUsedDataSourceNames = () => this.UsedDataSourcesNames,
         };
@@ -77,7 +88,9 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
             this.dataDescription = this.DataSource.Description;
             this.dataEmbeddingId = this.DataSource.EmbeddingId;
             this.dataFilePath = this.DataSource.FilePath;
-            this.dataSecurityPolicy = this.DataSource.SecurityPolicy;
+            this.dataMaxChunkTokenLength = this.DataSource.MaxChunkTokenLength;
+            this.dataChunkOverlapTokenLength = this.DataSource.ChunkOverlapTokenLength;
+            this.dataConfidenceLevel = this.DataSource.ConfidenceLevel;
             this.dataMaxMatches = this.DataSource.MaxMatches;
         }
         
@@ -102,7 +115,39 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
         return provider == EmbeddingProvider.NONE ? null : provider;
     }
     
-    private bool SelectedCloudEmbedding => !(this.SettingsManager.ConfigurationData.EmbeddingProviders.FirstOrDefault(x => x.Id == this.dataEmbeddingId)?.IsTrustedForDataSourceSecurityChecks(this.SettingsManager) ?? false);
+    private EmbeddingProvider? SelectedEmbedding => this.SettingsManager.ConfigurationData.EmbeddingProviders
+        .FirstOrDefault(x => x.Id == this.dataEmbeddingId);
+
+    private bool SelectedCloudEmbedding => this.SelectedEmbedding is { IsSelfHosted: false };
+
+    private bool CanChangeSourceAndEmbedding => !this.IsEditing || !this.LockSourceAndEmbedding;
+
+    private IEnumerable<ConfigurationSelectData<ConfidenceLevel>> ConfidenceLevels => ConfigurationSelectDataFactory.GetDataSourceConfidenceLevelsData();
+
+    private string SelectedEmbeddingNameText
+    {
+        get
+        {
+            var selectedEmbedding = this.AvailableEmbeddings.FirstOrDefault(x => x.Value == this.dataEmbeddingId);
+            return string.IsNullOrWhiteSpace(selectedEmbedding.Name) ? T("Unknown") : selectedEmbedding.Name;
+        }
+    }
+
+    private string SelectedEmbeddingTokenizerText => this.SelectedEmbedding is null
+        ? T("No embedding selected")
+        : string.IsNullOrWhiteSpace(this.SelectedEmbedding.TokenizerPath)
+            ? T("Default tokenizer")
+            : Path.GetFileName(this.SelectedEmbedding.TokenizerPath);
+
+    private int ProviderMaxChunkTokenLength => this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT;
+
+    private string MaxChunkTokenLengthHelperText => string.Format(
+        T("Maximum number of tokens per chunk for this data source. The embedding provider default is {0} tokens."),
+        this.ProviderMaxChunkTokenLength);
+
+    private string ChunkOverlapTokenLengthHelperText => string.Format(
+        T("Number of tokens repeated at the start of the next chunk. The default overlap is {0} tokens."),
+        DataSourceEmbeddingService.DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH);
 
     private DataSourceLocalFile CreateDataSource() => new()
     {
@@ -111,9 +156,11 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
         Name = this.dataName,
         Description = this.dataDescription,
         Type = DataSourceType.LOCAL_FILE,
-        EmbeddingId = this.dataEmbeddingId,
-        FilePath = this.dataFilePath,
-        SecurityPolicy = this.dataSecurityPolicy,
+        EmbeddingId = this.CanChangeSourceAndEmbedding ? this.dataEmbeddingId : this.DataSource.EmbeddingId,
+        FilePath = this.CanChangeSourceAndEmbedding ? this.dataFilePath : this.DataSource.FilePath,
+        MaxChunkTokenLength = this.dataMaxChunkTokenLength,
+        ChunkOverlapTokenLength = this.dataChunkOverlapTokenLength,
+        ConfidenceLevel = this.dataConfidenceLevel,
         MaxMatches = this.dataMaxMatches,
     };
     
@@ -130,4 +177,45 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
     }
     
     private void Cancel() => this.MudDialog.Cancel();
+
+    private string? ValidateMaxChunkTokenLength(int maxChunkTokenLength)
+    {
+        if (!this.showExpertSettings)
+            return null;
+
+        if (maxChunkTokenLength < 1)
+            return T("Please enter a token limit of at least 1.");
+
+        var providerMaxChunkTokenLength = this.ProviderMaxChunkTokenLength;
+        if (maxChunkTokenLength > providerMaxChunkTokenLength)
+            return string.Format(T("The data source token limit must not be larger than the embedding provider token limit ({0})."), providerMaxChunkTokenLength);
+
+        return null;
+    }
+
+    private string? ValidateChunkOverlapTokenLength(int chunkOverlapTokenLength)
+    {
+        if (!this.showExpertSettings)
+            return null;
+
+        if (chunkOverlapTokenLength < 0)
+            return T("Please enter 0 or a positive overlap length.");
+
+        var effectiveMaxChunkTokenLength = this.showExpertSettings && this.dataMaxChunkTokenLength > 0
+            ? this.dataMaxChunkTokenLength
+            : this.ProviderMaxChunkTokenLength;
+        if (chunkOverlapTokenLength >= effectiveMaxChunkTokenLength)
+            return T("The overlap must be smaller than the effective token limit.");
+
+        return null;
+    }
+
+    private void ToggleExpertSettings()
+    {
+        this.showExpertSettings = !this.showExpertSettings;
+        if (this.showExpertSettings && this.dataMaxChunkTokenLength < 1)
+            this.dataMaxChunkTokenLength = this.ProviderMaxChunkTokenLength;
+    }
+
+    private string GetExpertStyles => this.showExpertSettings ? "border-2 border-dashed rounded pa-2" : string.Empty;
 }

app/MindWork AI Studio/Dialogs/DataSourceLocalFileInfoDialog.razor

@@ -1,4 +1,4 @@
-@using AIStudio.Settings.DataModel
+@using AIStudio.Provider
 @inherits MSGComponentBase
 
 <MudDialog>
@@ -37,7 +37,7 @@
             </MudJustifiedText>
         }
 
-        <TextInfoLines Label="@T("Your security policy")" MaxLines="3" Value="@this.DataSource.SecurityPolicy.ToInfoText()" Color="@this.DataSource.SecurityPolicy.GetColor()" ClipboardTooltipSubject="@T("your security policy")"/>
+        <TextInfoLine Label="@T("Required provider confidence level")" Value="@this.DataSource.ConfidenceLevel.GetName()" ClipboardTooltipSubject="@T("the required provider confidence level")"/>
         <TextInfoLine Label="@T("Maximum matches per query")" Value="@this.DataSource.MaxMatches.ToString()" ClipboardTooltipSubject="@T("the maximum number of matches per query")"/>
         <TextInfoLine Icon="@Icons.Material.Filled.SquareFoot" Label="@T("File size")" Value="@this.FileSize" ClipboardTooltipSubject="@T("the file size")"/>
     </DialogContent>

app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor

@@ -14,7 +14,7 @@
         <MudForm @ref="@this.form" @bind-IsValid="@this.dataIsValid" @bind-Errors="@this.dataIssues">
             <MudStack Row="@true" AlignItems="AlignItems.Center">
                 @* ReSharper disable once CSharpWarnings::CS8974 *@
-                <MudSelect @bind-Value="@this.DataLLMProvider" Label="@T("Provider")" Class="mb-3" Disabled="@this.IsEnterpriseConfiguration" Validation="@this.providerValidation.ValidatingProvider">
+                <MudSelect @bind-Value="@this.DataLLMProvider" Label="@T("Provider")" Class="mb-3" OpenIcon="@Icons.Material.Filled.AccountBalance" AdornmentColor="Color.Info" Adornment="Adornment.Start" Disabled="@this.IsEnterpriseConfiguration" Validation="@this.providerValidation.ValidatingProvider">
                     @foreach (LLMProviders provider in Enum.GetValues(typeof(LLMProviders)))
                     {
                         if (provider.ProvideEmbeddingAPI() || provider is LLMProviders.NONE)
@@ -29,7 +29,7 @@
                     @T("Create account")
                 </MudButton>
             </MudStack>
-            
+
             @if (this.DataLLMProvider.IsAPIKeyNeeded(this.DataHost))
             {
                 <SecretInputField Secret="@this.dataAPIKey" SecretChanged="@this.OnAPIKeyChanged" Label="@this.APIKeyText" Validation="@this.providerValidation.ValidatingAPIKey"/>
@@ -98,15 +98,14 @@
                             Disabled="@this.IsEnterpriseConfiguration"
                             Validation="@this.ValidateManuallyModel"
                             UserAttributes="@SPELLCHECK_ATTRIBUTES"
-                            HelperText="@T("Currently, we cannot query the embedding models for the selected provider and/or host. Therefore, please enter the model name manually.")"
-                        />
+                            HelperText="@T("Currently, we cannot query the embedding models for the selected provider and/or host. Therefore, please enter the model name manually.")"/>
                     }
                     else
                     {
                         <MudButton Disabled="@(this.IsEnterpriseConfiguration || !this.DataLLMProvider.CanLoadModels(this.DataHost, this.dataAPIKey))" Variant="Variant.Filled" Size="Size.Small" StartIcon="@Icons.Material.Filled.Refresh" OnClick="@this.ReloadModels">
                             @T("Load")
                         </MudButton>
-                        @if(this.availableModels.Count is 0)
+                        @if (this.availableModels.Count is 0)
                         {
                             <MudText Typo="Typo.body1">
                                 @T("No models loaded or available.")
@@ -150,18 +149,77 @@
                 AdornmentColor="Color.Info"
                 Disabled="@this.IsEnterpriseConfiguration"
                 Validation="@this.providerValidation.ValidatingInstanceName"
-                UserAttributes="@SPELLCHECK_ATTRIBUTES"
-            />
-            
+            UserAttributes="@SPELLCHECK_ATTRIBUTES"/>
+            @if (this.DataLLMProvider != LLMProviders.NONE)
+            {
+                <MudStack>
+                    <MudButton OnClick="@this.ToggleExpertSettings">
+                        @(this.showExpertSettings ? T("Hide Expert Settings") : T("Show Expert Settings"))
+                    </MudButton>
+                    <MudDivider />
+                    <MudCollapse Expanded="@this.showExpertSettings" Class="@this.GetExpertStyles">
+                        <MudJustifiedText Typo="Typo.body1" Class="mb-3">
+                            @T("Please be aware: This section is for experts only. For cloud providers, the selected tokenizer and chunk settings may not match the real embedding model limits exactly.")
+                        </MudJustifiedText>
+                        <MudNumericField
+                            T="int"
+                            @bind-Value="@this.DataTokenLimit"
+                            Label="@T("Token limit")"
+                            Class="mb-3"
+                            Min="1"
+                            Immediate="@true"
+                            Adornment="Adornment.Start"
+                            AdornmentIcon="@Icons.Material.Filled.Numbers"
+                            AdornmentColor="Color.Info"
+                            Validation="@this.ValidateTokenLimit"
+                            HelperText="@T("Maximum number of tokens sent to the embedding model per chunk. The default is 8,192.")"/>
+                        <MudNumericField
+                            T="int"
+                            @bind-Value="@this.DataEmbeddingBatchSize"
+                            Label="@T("Embedding batch size")"
+                            Class="mb-3"
+                            Min="1"
+                            Immediate="@true"
+                            Adornment="Adornment.Start"
+                            AdornmentIcon="@Icons.Material.Filled.FormatListNumbered"
+                            AdornmentColor="Color.Info"
+                            Validation="@this.ValidateEmbeddingBatchSize"
+                            HelperText="@T("How many chunks are sent to the embedding provider at once. The default is 1.")"/>
+                        <MudStack Row="@true" Spacing="3" Class="mb-3" StretchItems="StretchItems.None" AlignItems="AlignItems.Center">
+                            <MudTextField
+                                T="string"
+                                Text="@this.dataFilePath"
+                                TextChanged="@this.OnDataFilePathChanged"
+                                Label="@T("Selected file path for the custom tokenizer")"
+                                Validation="@this.providerValidation.ValidatingCustomTokenizer"
+                                Adornment="Adornment.Start"
+                                AdornmentIcon="@Icons.Material.Filled.AttachFile"
+                                UserAttributes="@SPELLCHECK_ATTRIBUTES"
+                                Variant="Variant.Outlined"
+                                Clearable="@true"
+                                Error="@(!string.IsNullOrWhiteSpace(this.dataCustomTokenizerValidationIssue))"
+                                ErrorText="@this.dataCustomTokenizerValidationIssue"
+                                OnClearButtonClick="@this.ClearPathTokenizer"
+                            />
+                            <MudButton StartIcon="@Icons.Material.Filled.FolderOpen" Variant="Variant.Outlined" Color="Color.Primary" Disabled="@this.isTokenizerFileDialogOpen" OnClick="@this.OpenTokenizerFileDialog">
+                                @T("Choose File")
+                            </MudButton>
+                        </MudStack>
+                    </MudCollapse>
+                </MudStack>
+            }
         </MudForm>
-        <Issues IssuesData="@this.dataIssues"/>
+        @if (this.dataStoreWasAttempted)
+        {
+            <Issues IssuesData="@this.dataIssues"/>
+        }
     </DialogContent>
     <DialogActions>
         <MudButton OnClick="@this.Cancel" Variant="Variant.Filled">
             @T("Cancel")
         </MudButton>
         <MudButton OnClick="@this.Store" Variant="Variant.Filled" Color="Color.Primary">
-            @if(this.IsEditing)
+            @if (this.IsEditing)
             {
                 @T("Update")
             }

app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs

@@ -2,11 +2,12 @@ using AIStudio.Components;
 using AIStudio.Provider;
 using AIStudio.Provider.HuggingFace;
 using AIStudio.Settings;
+using AIStudio.Tools.Rust;
 using AIStudio.Tools.Services;
 using AIStudio.Tools.Validation;
 
 using Microsoft.AspNetCore.Components;
-
+using Microsoft.AspNetCore.Components.Web;
 using Host = AIStudio.Provider.SelfHosted.Host;
 
 namespace AIStudio.Dialogs;
@@ -82,6 +83,15 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
     [Parameter]
     public bool IsEditing { get; init; }
 
+    [Parameter]
+    public string DataTokenizerPath { get; set; } = string.Empty;
+
+    [Parameter]
+    public int DataTokenLimit { get; set; } = EmbeddingProvider.DEFAULT_TOKEN_LIMIT;
+
+    [Parameter]
+    public int DataEmbeddingBatchSize { get; set; } = EmbeddingProvider.DEFAULT_EMBEDDING_BATCH_SIZE;
+
     /// <summary>
     /// Whether this embedding provider is managed by an enterprise configuration plugin. When true,
     /// every field except the API key is locked, matching Settings.EmbeddingProvider.IsEnterpriseConfiguration.
@@ -110,6 +120,13 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
     private string dataAPIKeyStorageIssue = string.Empty;
     private string dataEditingPreviousInstanceName = string.Empty;
     private string dataLoadingModelsIssue = string.Empty;
+    private string dataFilePath = string.Empty;
+    private string dataCustomTokenizerValidationIssue = string.Empty;
+    private Task dataTokenizerValidationTask = Task.CompletedTask;
+    private bool dataStoreWasAttempted;
+    private bool isTokenizerFileDialogOpen;
+    private bool showExpertSettings;
+    private int dataTokenizerValidationRevision;
 
     // We get the form reference from Blazor code to validate it manually:
     private MudForm form = null!;
@@ -117,7 +134,7 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
     private readonly List<Model> availableModels = new();
     private readonly Encryption encryption = Program.ENCRYPTION;
     private readonly ProviderValidation providerValidation;
-    
+
     public EmbeddingProviderDialog()
     {
         this.providerValidation = new()
@@ -127,7 +144,8 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
             GetPreviousInstanceName = () => this.dataEditingPreviousInstanceName,
             GetUsedInstanceNames = () => this.UsedInstanceNames,
             GetHost = () => this.DataHost,
-            IsModelProvidedManually = () => this.DataLLMProvider is LLMProviders.SELF_HOSTED && this.DataHost is Host.OLLAMA,
+            IsModelProvidedManually = () => this.DataLLMProvider.IsEmbeddingModelProvidedManually(this.DataHost),
+            GetCustomTokenizerValidationIssue = () => this.dataCustomTokenizerValidationIssue,
         };
     }
     
@@ -137,7 +155,7 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
         Model model = default;
         if(this.DataLLMProvider is LLMProviders.SELF_HOSTED)
         {
-            if (this.DataHost is Host.OLLAMA)
+            if (this.DataLLMProvider.IsEmbeddingModelProvidedManually(this.DataHost))
                 model = new Model(this.dataManuallyModel, null);
             else if (this.DataHost is Host.LM_STUDIO)
                 model = this.DataModel;
@@ -157,6 +175,9 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
             Host = this.DataHost,
             IsEnterpriseConfiguration = this.IsEnterpriseConfiguration,
             EnterpriseConfigurationPluginId = Guid.Empty,
+            TokenizerPath = this.dataFilePath,
+            EmbeddingBatchSize = this.DataEmbeddingBatchSize,
+            TokenLimit = this.DataTokenLimit,
             CustomIconDataUrl = this.DataCustomIconDataUrl,
             HFInferenceProvider = this.HFInferenceProviderId,
         };
@@ -179,6 +200,10 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
         if(this.IsEditing)
         {
             this.dataEditingPreviousInstanceName = this.DataName.ToLowerInvariant();
+            this.dataFilePath = this.DataTokenizerPath;
+            this.showExpertSettings = !string.IsNullOrWhiteSpace(this.DataTokenizerPath)
+                                      || this.DataTokenLimit != EmbeddingProvider.DEFAULT_TOKEN_LIMIT
+                                      || this.DataEmbeddingBatchSize != EmbeddingProvider.DEFAULT_EMBEDDING_BATCH_SIZE;
             
             // When using self-hosted embedding, we must copy the model name:
             if (this.DataLLMProvider is LLMProviders.SELF_HOSTED)
@@ -187,7 +212,7 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
             //
             // We cannot load the API key for self-hosted providers:
             //
-            if (this.DataLLMProvider is LLMProviders.SELF_HOSTED && this.DataHost is not Host.OLLAMA)
+            if (this.DataLLMProvider is LLMProviders.SELF_HOSTED && this.DataHost is not Host.OLLAMA && this.DataHost is not Host.VLLM)
             {
                 await this.ReloadModels();
                 await base.OnInitializedAsync();
@@ -244,6 +269,8 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
     
     private async Task Store()
     {
+        this.dataStoreWasAttempted = true;
+        await this.dataTokenizerValidationTask;
         await this.form.Validate();
         this.dataAPIKeyStorageIssue = string.Empty;
 
@@ -260,6 +287,15 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
         if (!this.dataIsValid)
             return;
         
+        var response = await this.StoreOrDeleteTokenizerAsync();
+        if (!response.Success)
+        {
+            this.dataCustomTokenizerValidationIssue = string.IsNullOrWhiteSpace(response.Message) ? string.Empty : response.Message;
+            await this.form.Validate();
+            return;
+        }
+        this.dataFilePath = response.StoredPath;
+        
         // Use the data model to store the provider.
         // We just return this data to the parent component:
         var addedProviderSettings = this.CreateEmbeddingProviderSettings();
@@ -302,6 +338,22 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
         return null;
     }
 
+    private string? ValidateTokenLimit(int tokenLimit)
+    {
+        if (tokenLimit < 1)
+            return T("Please enter a token limit greater than 0.");
+
+        return null;
+    }
+
+    private string? ValidateEmbeddingBatchSize(int embeddingBatchSize)
+    {
+        if (embeddingBatchSize < 1)
+            return T("Please enter an embedding batch size greater than 0.");
+
+        return null;
+    }
+
     private void Cancel() => this.MudDialog.Cancel();
 
     private async Task OnAPIKeyChanged(string apiKey)
@@ -314,6 +366,90 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
         }
     }
 
+    private async Task OpenTokenizerFileDialog()
+    {
+        if (this.isTokenizerFileDialogOpen)
+            return;
+
+        this.isTokenizerFileDialogOpen = true;
+        try
+        {
+            var response = await this.RustService.SelectFile(T("Choose a custom tokenizer here"), [ FileTypes.JSON ], string.IsNullOrWhiteSpace(this.dataFilePath) ? null : this.dataFilePath);
+            if (!response.UserCancelled)
+                await this.OnDataFilePathChanged(response.SelectedFilePath);
+        }
+        finally
+        {
+            this.isTokenizerFileDialogOpen = false;
+        }
+    }
+
+    private Task ClearPathTokenizer(MouseEventArgs _)
+    {
+        return this.OnDataFilePathChanged(string.Empty);
+    }
+    
+    private async Task OnDataFilePathChanged(string filePath)
+    {
+        this.dataFilePath = filePath;
+        var validationRevision = ++this.dataTokenizerValidationRevision;
+        this.dataTokenizerValidationTask = this.ValidateCustomTokenizer(filePath, validationRevision);
+        await this.dataTokenizerValidationTask;
+
+        if (validationRevision != this.dataTokenizerValidationRevision)
+            return;
+
+        if (this.dataStoreWasAttempted)
+            await this.form.Validate();
+        else
+            this.form.ResetValidation();
+    }
+
+    private async Task ValidateCustomTokenizer(string filePath, int validationRevision)
+    {
+        if (string.IsNullOrWhiteSpace(filePath))
+        {
+            if (validationRevision == this.dataTokenizerValidationRevision)
+                this.dataCustomTokenizerValidationIssue = string.Empty;
+
+            return;
+        }
+
+        try
+        {
+            var response = await this.RustService.ValidateTokenizer(filePath);
+            if (validationRevision != this.dataTokenizerValidationRevision)
+                return;
+
+            if (response.Success)
+                this.dataCustomTokenizerValidationIssue = string.Empty;
+            else
+                this.dataCustomTokenizerValidationIssue = T("Invalid tokenizer: ") + response.Message;
+        }
+        catch (Exception e)
+        {
+            if (validationRevision != this.dataTokenizerValidationRevision)
+                return;
+
+            this.Logger.LogError(e, "Failed to validate custom tokenizer.");
+            this.dataCustomTokenizerValidationIssue = T("Failed to validate the selected tokenizer. Please try again.");
+        }
+    }
+
+    /// <summary>
+    /// Stores a new tokenizer or deletes the existing one, based on the specified tokenizer path.
+    /// If the path is null or empty, any existing tokenizer is removed.
+    /// Otherwise, the tokenizer is stored at the specified path.
+    /// </summary>
+    private Task<TokenizerResponse> StoreOrDeleteTokenizerAsync()
+    {
+        var tokenizerId = TokenizerModelId.ForEmbeddingProviderId(this.DataId);
+        if (string.IsNullOrWhiteSpace(this.dataFilePath))
+            return this.RustService.DeleteTokenizer(tokenizerId);
+
+        return this.RustService.StoreTokenizer(tokenizerId, this.dataFilePath);
+    }
+
     private void OnHostChanged(Host selectedHost)
     {
         // When the host changes, reset the model selection state:
@@ -374,4 +510,8 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId
     };
     
     private bool IsNoneProvider => this.DataLLMProvider is LLMProviders.NONE;
+
+    private void ToggleExpertSettings() => this.showExpertSettings = !this.showExpertSettings;
+
+    private string GetExpertStyles => this.showExpertSettings ? "border-2 border-dashed rounded pa-2" : string.Empty;
 }

app/MindWork AI Studio/Dialogs/ProviderDialog.razor

@@ -174,6 +174,33 @@
                 Validation="@this.providerValidation.ValidatingInstanceName"
                 UserAttributes="@SPELLCHECK_ATTRIBUTES"
             />
+
+            @if (this.DataLLMProvider != LLMProviders.NONE)
+            {
+                <MudJustifiedText Typo="Typo.body1" Class="mb-3">
+                    @T("For better token estimates, you can configure a custom tokenizer for this provider.")
+                </MudJustifiedText>
+                <MudStack Row="@true" Spacing="3" Class="mb-3" StretchItems="StretchItems.None" AlignItems="AlignItems.Center">
+                    <MudTextField
+                        T="string"
+                        Text="@this.dataFilePath"
+                        TextChanged="@this.OnDataFilePathChanged"
+                        Label="@T("Selected file path for the custom tokenizer")"
+                        Validation="@this.providerValidation.ValidatingCustomTokenizer"
+                        Adornment="Adornment.Start"
+                        AdornmentIcon="@Icons.Material.Filled.AttachFile"
+                        UserAttributes="@SPELLCHECK_ATTRIBUTES"
+                        Variant="Variant.Outlined"
+                        Clearable="@true"
+                        Error="@(!string.IsNullOrWhiteSpace(this.dataCustomTokenizerValidationIssue))"
+                        ErrorText="@this.dataCustomTokenizerValidationIssue"
+                        OnClearButtonClick="@this.ClearPathTokenizer"
+                    />
+                    <MudButton StartIcon="@Icons.Material.Filled.FolderOpen" Variant="Variant.Outlined" Color="Color.Primary" Disabled="@this.isTokenizerFileDialogOpen" OnClick="@this.OpenTokenizerFileDialog">
+                        @T("Choose File")
+                    </MudButton>
+                </MudStack>
+            }
             
             <MudStack>
                 <MudButton OnClick="@this.ToggleExpertSettings">

app/MindWork AI Studio/Dialogs/ProviderDialog.razor.cs

@@ -4,11 +4,13 @@ using System.Text.Json;
 using AIStudio.Components;
 using AIStudio.Provider;
 using AIStudio.Provider.HuggingFace;
+using AIStudio.Tools.Rust;
 using AIStudio.Settings;
 using AIStudio.Tools.Services;
 using AIStudio.Tools.Validation;
 
 using Microsoft.AspNetCore.Components;
+using Microsoft.AspNetCore.Components.Web;
 
 using Host = AIStudio.Provider.SelfHosted.Host;
 
@@ -107,6 +109,9 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
     [Parameter]
     public string AdditionalJsonApiParameters { get; set; } = string.Empty;
 
+    [Parameter]
+    public string DataTokenizerPath { get; set; } = string.Empty;
+
     [Parameter]
     public ProviderCapabilityOverrides? DataCapabilityOverrides { get; set; }
     
@@ -134,7 +139,7 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
         ReasoningOverrideMode.ON_BY_DEFAULT,
         ReasoningOverrideMode.ALWAYS_ON
     ];
-    
+
     /// <summary>
     /// The list of used instance names. We need this to check for uniqueness.
     /// </summary>
@@ -148,6 +153,12 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
     private string dataAPIKeyStorageIssue = string.Empty;
     private string dataEditingPreviousInstanceName = string.Empty;
     private string dataLoadingModelsIssue = string.Empty;
+    private string dataFilePath = string.Empty;
+    private string dataCustomTokenizerValidationIssue = string.Empty;
+    private Task dataTokenizerValidationTask = Task.CompletedTask;
+    private bool dataStoreWasAttempted;
+    private bool isTokenizerFileDialogOpen;
+    private int dataTokenizerValidationRevision;
     private bool usesLegacySystemModelFallback;
     private bool showExpertSettings;
     private ProviderCapabilityOverrides capabilityOverrides = new();
@@ -169,6 +180,7 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
             GetUsedInstanceNames = () => this.UsedInstanceNames,
             GetHost = () => this.DataHost,
             IsModelProvidedManually = () => this.DataLLMProvider.IsLLMModelProvidedManually(),
+            GetCustomTokenizerValidationIssue = () => this.dataCustomTokenizerValidationIssue,
             IsModelSelectionHidden = () => this.IsLLMModelSelectionHidden,
         };
     }
@@ -190,6 +202,7 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
             Host = this.DataHost,
             HFInferenceProvider = this.HFInferenceProviderId,
             AdditionalJsonApiParameters = this.AdditionalJsonApiParameters,
+            TokenizerPath = this.dataFilePath,
             CapabilityOverrides = this.capabilityOverrides.HasOverrides ? this.capabilityOverrides : null,
             CustomIconDataUrl = this.DataCustomIconDataUrl,
         };
@@ -226,6 +239,7 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
         if(this.IsEditing)
         {
             this.dataEditingPreviousInstanceName = this.DataInstanceName.ToLowerInvariant();
+            this.dataFilePath = this.DataTokenizerPath;
             
             // When using Fireworks, we must copy the model name:
             if (this.DataLLMProvider.IsLLMModelProvidedManually())
@@ -291,6 +305,8 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
 
     private async Task Store()
     {
+        this.dataStoreWasAttempted = true;
+        await this.dataTokenizerValidationTask;
         await this.form.Validate();
         if (!string.IsNullOrWhiteSpace(this.dataAPIKeyStorageIssue))
             this.dataAPIKeyStorageIssue = string.Empty;
@@ -307,6 +323,27 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
         // When the data is not valid, we don't store it:
         if (!this.dataIsValid)
             return;
+
+        var tokenizerResponse = await this.StoreOrDeleteTokenizerAsync();
+        if (!tokenizerResponse.Success)
+        {
+            //
+            // Storing a tokenizer the user has chosen must succeed: otherwise the provider would
+            // silently work without the tokenizer the user asked for. Removing a tokenizer the
+            // user has cleared is best effort, though. A failed cleanup leaves an unused file
+            // behind, which is no reason to refuse saving the provider itself.
+            //
+            if (!string.IsNullOrWhiteSpace(this.dataFilePath))
+            {
+                this.dataCustomTokenizerValidationIssue = tokenizerResponse.Message;
+                await this.form.Validate();
+                return;
+            }
+
+            this.Logger.LogWarning($"Failed to remove the tokenizer of provider '{this.DataInstanceName}'. The provider is stored anyway. The message was: {tokenizerResponse.Message}");
+        }
+
+        this.dataFilePath = tokenizerResponse.StoredPath;
         
         // Use the data model to store the provider.
         // We just return this data to the parent component:
@@ -362,6 +399,98 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
         }
     }
 
+    private async Task OpenTokenizerFileDialog()
+    {
+        if (this.isTokenizerFileDialogOpen)
+            return;
+
+        this.isTokenizerFileDialogOpen = true;
+        try
+        {
+            var response = await this.RustService.SelectFile(T("Choose a custom tokenizer here"), [ FileTypes.JSON ], string.IsNullOrWhiteSpace(this.dataFilePath) ? null : this.dataFilePath);
+            if (!response.UserCancelled)
+                await this.OnDataFilePathChanged(response.SelectedFilePath);
+        }
+        finally
+        {
+            this.isTokenizerFileDialogOpen = false;
+        }
+    }
+
+    private Task ClearPathTokenizer(MouseEventArgs _)
+    {
+        return this.OnDataFilePathChanged(string.Empty);
+    }
+
+    private async Task OnDataFilePathChanged(string filePath)
+    {
+        this.dataFilePath = filePath;
+        var validationRevision = ++this.dataTokenizerValidationRevision;
+        this.dataTokenizerValidationTask = this.ValidateCustomTokenizer(filePath, validationRevision);
+        await this.dataTokenizerValidationTask;
+
+        if (validationRevision != this.dataTokenizerValidationRevision)
+            return;
+
+        if (this.dataStoreWasAttempted)
+            await this.form.Validate();
+        else
+            this.form.ResetValidation();
+    }
+
+    private async Task ValidateCustomTokenizer(string filePath, int validationRevision)
+    {
+        if (string.IsNullOrWhiteSpace(filePath))
+        {
+            if (validationRevision == this.dataTokenizerValidationRevision)
+                this.dataCustomTokenizerValidationIssue = string.Empty;
+
+            return;
+        }
+
+        try
+        {
+            var response = await this.RustService.ValidateTokenizer(filePath);
+            if (validationRevision != this.dataTokenizerValidationRevision)
+                return;
+
+            if (response.Success)
+                this.dataCustomTokenizerValidationIssue = string.Empty;
+            else
+                this.dataCustomTokenizerValidationIssue = T("Invalid tokenizer: ") + response.Message;
+        }
+        catch (Exception e)
+        {
+            if (validationRevision != this.dataTokenizerValidationRevision)
+                return;
+
+            this.Logger.LogError(e, "Failed to validate custom tokenizer.");
+            this.dataCustomTokenizerValidationIssue = T("Failed to validate the selected tokenizer. Please try again.");
+        }
+    }
+
+    /// <summary>
+    /// Stores a new tokenizer or deletes the existing one, based on the specified tokenizer path.
+    /// If the path is null or empty, any existing tokenizer is removed.
+    /// Otherwise, the tokenizer is stored at the specified path.
+    /// </summary>
+    private Task<TokenizerResponse> StoreOrDeleteTokenizerAsync()
+    {
+        var tokenizerId = TokenizerModelId.ForProviderId(this.DataId);
+        if (!string.IsNullOrWhiteSpace(this.dataFilePath))
+            return this.RustService.StoreTokenizer(tokenizerId, this.dataFilePath);
+
+        //
+        // A provider which never had a tokenizer has nothing to clean up. Calling the runtime
+        // anyway could only fail here, and that failure would block saving a provider which has
+        // nothing to do with tokenizers at all.
+        //
+        if (string.IsNullOrWhiteSpace(this.DataTokenizerPath))
+            return Task.FromResult(new TokenizerResponse(true, 0, string.Empty));
+
+        return this.RustService.DeleteTokenizer(tokenizerId);
+    }
+
     private void OnProviderChanged(LLMProviders selectedProvider)
     {
         this.DataLLMProvider = selectedProvider;
@@ -795,10 +924,10 @@ public partial class ProviderDialog : MSGComponentBase, ISecretId
 
         return true;
     }
-    
+
     private string GetExpertStyles => this.showExpertSettings ? "border-2 border-dashed rounded pa-2" : string.Empty;
-    
-    private static string GetPlaceholderExpertSettings => 
+
+    private static string GetPlaceholderExpertSettings =>
       """
       "temperature": 0.5,
       "top_p": 0.9,

app/MindWork AI Studio/Dialogs/Settings/SettingsDialogDataSources.razor

@@ -1,85 +1,15 @@
-@using AIStudio.Settings.DataModel
 @inherits SettingsDialogBase
 
 <MudDialog>
     <TitleContent>
-        <PreviewPrototype/>
+        <PreviewBeta/>
         <MudText Typo="Typo.h6" Class="d-flex align-center">
-            <MudIcon Icon="@Icons.Material.Filled.IntegrationInstructions" Class="mr-2"/>
+            <MudIcon Icon="@AppIcons.DATABASE" Class="mr-2"/>
             @T("Configured Data Sources")
         </MudText>
     </TitleContent>
     <DialogContent>
-        <MudJustifiedText Typo="Typo.body1" Class="mb-3">
-            @T("You might configure different data sources. A data source can include one file, all files in a directory, or data from your company. Later, you can incorporate these data sources as needed when the AI requires this data to complete a certain task.")
-        </MudJustifiedText>
-
-        <MudTable Items="@this.SettingsManager.ConfigurationData.DataSources" Hover="@true" Class="border-dashed border rounded-lg">
-            <ColGroup>
-                <col style="width: 3em;"/>
-                <col/>
-                <col style="width: 12em;"/>
-                <col style="width: 12em;"/>
-                <col style="width: 40em;"/>
-            </ColGroup>
-            <HeaderContent>
-                <MudTh>#</MudTh>
-                <MudTh>@T("Name")</MudTh>
-                <MudTh>@T("Type")</MudTh>
-                <MudTh>@T("Embedding")</MudTh>
-                <MudTh>@T("Actions")</MudTh>
-            </HeaderContent>
-            <RowTemplate>
-                <MudTd>@context.Num</MudTd>
-                <MudTd>@context.Name</MudTd>
-                <MudTd>@context.Type.GetDisplayName()</MudTd>
-                <MudTd>@this.GetEmbeddingName(context)</MudTd>
-                
-                <MudTd>
-                    <MudStack Row="true" Class="mb-2 mt-2" Wrap="Wrap.Wrap">
-                        <MudIconButton Variant="Variant.Filled" Color="Color.Info" Icon="@Icons.Material.Filled.Info" OnClick="() => this.ShowInformation(context)"/>
-                        @if (context.IsEnterpriseConfiguration)
-                        {
-                            <MudTooltip Text="@T("This data source is managed by your organization.")">
-                                <MudIconButton Color="Color.Info" Icon="@Icons.Material.Filled.Business" Disabled="true"/>
-                            </MudTooltip>
-                        }
-                        else
-                        {
-                            <MudButton Variant="Variant.Filled" Color="Color.Info" StartIcon="@Icons.Material.Filled.Edit" OnClick="() => this.EditDataSource(context)">
-                                @T("Edit")
-                            </MudButton>
-                            @if (context is DataSourceERI_V1)
-                            {
-                                <AdminExportButton Variant="Variant.Filled" OnClick="@(() => this.ExportDataSource(context))" />
-                            }
-                            <MudButton Variant="Variant.Filled" Color="Color.Error" StartIcon="@Icons.Material.Filled.Delete" OnClick="() => this.DeleteDataSource(context)">
-                                @T("Delete")
-                            </MudButton>
-                        }
-                    </MudStack>
-                </MudTd>
-            </RowTemplate>
-        </MudTable>
-
-        @if (this.SettingsManager.ConfigurationData.DataSources.Count == 0)
-        {
-            <MudText Typo="Typo.h6" Class="mt-3">
-                @T("No data sources configured yet.")
-            </MudText>
-        }
-        
-        <MudMenu EndIcon="@Icons.Material.Filled.KeyboardArrowDown" Label="@T("Add Data Source")" Color="Color.Primary" Variant="Variant.Filled" AnchorOrigin="Origin.CenterCenter" TransformOrigin="Origin.TopLeft" Class="mt-3 mb-6">
-            <MudMenuItem OnClick="() => this.AddDataSource(DataSourceType.ERI_V1)">
-                @T("External Data (ERI-Server v1)")
-            </MudMenuItem>
-            <MudMenuItem OnClick="() => this.AddDataSource(DataSourceType.LOCAL_DIRECTORY)">
-                @T("Local Directory")
-            </MudMenuItem>
-            <MudMenuItem OnClick="() => this.AddDataSource(DataSourceType.LOCAL_FILE)">
-                @T("Local File")
-            </MudMenuItem>
-        </MudMenu>
+        <DataSourceManagement/>
     </DialogContent>
     <DialogActions>
         <MudButton OnClick="@this.Close" Variant="Variant.Filled">

app/MindWork AI Studio/Dialogs/Settings/SettingsDialogDataSources.razor.cs

@@ -1,324 +1,3 @@
-using AIStudio.Settings;
-using AIStudio.Settings.DataModel;
-using AIStudio.Tools.ERIClient.DataModel;
-using AIStudio.Tools.PluginSystem;
-
 namespace AIStudio.Dialogs.Settings;
 
-public partial class SettingsDialogDataSources : SettingsDialogBase
-{
-    private string GetEmbeddingName(IDataSource dataSource)
-    {
-        if(dataSource is IInternalDataSource internalDataSource)
-        {
-            var matchedEmbedding = this.SettingsManager.ConfigurationData.EmbeddingProviders.FirstOrDefault(x => x.Id == internalDataSource.EmbeddingId);
-            if(matchedEmbedding == default)
-                return T("No valid embedding");
-            
-            return matchedEmbedding.Name;
-        }
-        
-        if(dataSource is IExternalDataSource)
-            return T("External (ERI)");
-        
-        return T("Unknown");
-    }
-    
-    private async Task AddDataSource(DataSourceType type)
-    {
-        IDataSource? addedDataSource = null;
-        switch (type)
-        {
-            case DataSourceType.LOCAL_FILE:
-                var localFileDialogParameters = new DialogParameters<DataSourceLocalFileDialog>
-                {
-                    { x => x.IsEditing, false },
-                    { x => x.AvailableEmbeddings, this.AvailableEmbeddingProviders }
-                };
-        
-                var localFileDialogReference = await this.DialogService.ShowAsync<DataSourceLocalFileDialog>(T("Add Local File as Data Source"), localFileDialogParameters, DialogOptions.FULLSCREEN);
-                var localFileDialogResult = await localFileDialogReference.Result;
-                if (localFileDialogResult is null || localFileDialogResult.Canceled)
-                    return;
-                
-                var localFile = (DataSourceLocalFile)localFileDialogResult.Data!;
-                localFile = localFile with { Num = this.SettingsManager.ConfigurationData.NextDataSourceNum++ };
-                addedDataSource = localFile;
-                break;
-            
-            case DataSourceType.LOCAL_DIRECTORY:
-                var localDirectoryDialogParameters = new DialogParameters<DataSourceLocalDirectoryDialog>
-                {
-                    { x => x.IsEditing, false },
-                    { x => x.AvailableEmbeddings, this.AvailableEmbeddingProviders }
-                };
-        
-                var localDirectoryDialogReference = await this.DialogService.ShowAsync<DataSourceLocalDirectoryDialog>(T("Add Local Directory as Data Source"), localDirectoryDialogParameters, DialogOptions.FULLSCREEN);
-                var localDirectoryDialogResult = await localDirectoryDialogReference.Result;
-                if (localDirectoryDialogResult is null || localDirectoryDialogResult.Canceled)
-                    return;
-                
-                var localDirectory = (DataSourceLocalDirectory)localDirectoryDialogResult.Data!;
-                localDirectory = localDirectory with { Num = this.SettingsManager.ConfigurationData.NextDataSourceNum++ };
-                addedDataSource = localDirectory;
-                break;
-            
-            case DataSourceType.ERI_V1:
-                var eriDialogParameters = new DialogParameters<DataSourceERI_V1Dialog>
-                {
-                    { x => x.IsEditing, false },
-                };
-                
-                var eriDialogReference = await this.DialogService.ShowAsync<DataSourceERI_V1Dialog>(T("Add ERI v1 Data Source"), eriDialogParameters, DialogOptions.FULLSCREEN);
-                var eriDialogResult = await eriDialogReference.Result;
-                if (eriDialogResult is null || eriDialogResult.Canceled)
-                    return;
-                
-                var eriDataSource = (DataSourceERI_V1)eriDialogResult.Data!;
-                eriDataSource = eriDataSource with { Num = this.SettingsManager.ConfigurationData.NextDataSourceNum++ };
-                addedDataSource = eriDataSource;
-                break;
-        }
-        
-        if(addedDataSource is null)
-            return;
-        
-        this.SettingsManager.ConfigurationData.DataSources.Add(addedDataSource);
-        await this.SettingsManager.StoreSettings();
-        await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
-    }
-
-    private async Task ExportDataSource(IDataSource dataSource)
-    {
-        if (!this.SettingsManager.ConfigurationData.App.ShowAdminSettings)
-            return;
-
-        if (dataSource is not DataSourceERI_V1 eriDataSource)
-            return;
-
-        if (eriDataSource.AuthMethod is AuthMethod.KERBEROS)
-        {
-            await this.DialogService.ShowMessageBox(
-                T("Export ERI Data Source"),
-                T("Kerberos/SSO ERI data sources cannot be exported yet. Please configure them manually in the configuration plugin."),
-                T("Close"));
-            return;
-        }
-
-        var needsSecret = eriDataSource.AuthMethod is AuthMethod.TOKEN or AuthMethod.USERNAME_PASSWORD;
-        if (!needsSecret)
-        {
-            var publicLuaCode = eriDataSource.ExportAsConfigurationSection();
-            if (!string.IsNullOrWhiteSpace(publicLuaCode))
-                await this.RustService.CopyText2Clipboard(publicLuaCode);
-
-            return;
-        }
-
-        var secretResponse = await this.RustService.GetSecret(eriDataSource, SecretStoreType.DATA_SOURCE, isTrying: true);
-        if (!secretResponse.Success)
-        {
-            await this.DialogService.ShowMessageBox(
-                T("Export ERI Data Source"),
-                string.Format(T("Cannot export this ERI data source because no authentication secret is configured. The issue was: {0}"), secretResponse.Issue),
-                T("Close"));
-            return;
-        }
-
-        var encryption = PluginFactory.EnterpriseEncryption;
-        if (encryption?.IsAvailable != true)
-        {
-            await this.DialogService.ShowMessageBox(
-                T("Export ERI Data Source"),
-                T("Cannot export this ERI data source because no enterprise encryption secret is configured."),
-                T("Close"));
-            return;
-        }
-
-        var usernamePasswordMode = DataSourceERIUsernamePasswordMode.USER_MANAGED;
-        if (eriDataSource.AuthMethod is AuthMethod.TOKEN)
-        {
-            var dialogParameters = new DialogParameters<ConfirmDialog>
-            {
-                { x => x.Message, T("This ERI data source has an access token configured. Do you want to include the encrypted access token in the export? Note: The recipient will need the same encryption secret to use the access token.") },
-            };
-
-            var dialogReference = await this.DialogService.ShowAsync<ConfirmDialog>(T("Export Access Token?"), dialogParameters, DialogOptions.FULLSCREEN);
-            var dialogResult = await dialogReference.Result;
-            if (dialogResult is null || dialogResult.Canceled)
-                return;
-        }
-        else if (eriDataSource.AuthMethod is AuthMethod.USERNAME_PASSWORD)
-        {
-            var dialogParameters = new DialogParameters<DataSourceERIV1UsernamePasswordExportDialog>
-            {
-                { x => x.DataSource, eriDataSource },
-            };
-
-            var dialogReference = await this.DialogService.ShowAsync<DataSourceERIV1UsernamePasswordExportDialog>(T("Export ERI Data Source"), dialogParameters, DialogOptions.FULLSCREEN);
-            var dialogResult = await dialogReference.Result;
-            if (dialogResult is null || dialogResult.Canceled || dialogResult.Data is not DataSourceERIV1UsernamePasswordExportDialogResult exportResult)
-                return;
-
-            usernamePasswordMode = exportResult.UsernamePasswordMode;
-        }
-
-        var decryptedSecret = await secretResponse.Secret.Decrypt(Program.ENCRYPTION);
-        if (!encryption.TryEncrypt(decryptedSecret, out var encryptedSecret))
-        {
-            await this.DialogService.ShowMessageBox(
-                T("Export ERI Data Source"),
-                T("Cannot export this ERI data source because the authentication secret could not be encrypted."),
-                T("Close"));
-            return;
-        }
-
-        var luaCode = eriDataSource.ExportAsConfigurationSection(
-            encryptedSecret,
-            usernamePasswordMode);
-        if (string.IsNullOrWhiteSpace(luaCode))
-            return;
-
-        await this.RustService.CopyText2Clipboard(luaCode);
-    }
-    
-    private async Task EditDataSource(IDataSource dataSource)
-    {
-        if (dataSource.IsEnterpriseConfiguration)
-            return;
-
-        IDataSource? editedDataSource = null;
-        switch (dataSource)
-        {
-            case DataSourceLocalFile localFile:
-                var localFileDialogParameters = new DialogParameters<DataSourceLocalFileDialog>
-                {
-                    { x => x.IsEditing, true },
-                    { x => x.DataSource, localFile },
-                    { x => x.AvailableEmbeddings, this.AvailableEmbeddingProviders }
-                };
-        
-                var localFileDialogReference = await this.DialogService.ShowAsync<DataSourceLocalFileDialog>(T("Edit Local File Data Source"), localFileDialogParameters, DialogOptions.FULLSCREEN);
-                var localFileDialogResult = await localFileDialogReference.Result;
-                if (localFileDialogResult is null || localFileDialogResult.Canceled)
-                    return;
-                
-                editedDataSource = (DataSourceLocalFile)localFileDialogResult.Data!;
-                break;
-            
-            case DataSourceLocalDirectory localDirectory:
-                var localDirectoryDialogParameters = new DialogParameters<DataSourceLocalDirectoryDialog>
-                {
-                    { x => x.IsEditing, true },
-                    { x => x.DataSource, localDirectory },
-                    { x => x.AvailableEmbeddings, this.AvailableEmbeddingProviders }
-                };
-        
-                var localDirectoryDialogReference = await this.DialogService.ShowAsync<DataSourceLocalDirectoryDialog>(T("Edit Local Directory Data Source"), localDirectoryDialogParameters, DialogOptions.FULLSCREEN);
-                var localDirectoryDialogResult = await localDirectoryDialogReference.Result;
-                if (localDirectoryDialogResult is null || localDirectoryDialogResult.Canceled)
-                    return;
-                
-                editedDataSource = (DataSourceLocalDirectory)localDirectoryDialogResult.Data!;
-                break;
-            
-            case DataSourceERI_V1 eriDataSource:
-                var eriDialogParameters = new DialogParameters<DataSourceERI_V1Dialog>
-                {
-                    { x => x.IsEditing, true },
-                    { x => x.DataSource, eriDataSource },
-                };
-                
-                var eriDialogReference = await this.DialogService.ShowAsync<DataSourceERI_V1Dialog>(T("Edit ERI v1 Data Source"), eriDialogParameters, DialogOptions.FULLSCREEN);
-                var eriDialogResult = await eriDialogReference.Result;
-                if (eriDialogResult is null || eriDialogResult.Canceled)
-                    return;
-                
-                editedDataSource = (DataSourceERI_V1)eriDialogResult.Data!;
-                break;
-        }
-        
-        if(editedDataSource is null)
-            return;
-        
-        this.SettingsManager.ConfigurationData.DataSources[this.SettingsManager.ConfigurationData.DataSources.IndexOf(dataSource)] = editedDataSource;
-
-        await this.SettingsManager.StoreSettings();
-        await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
-    }
-    
-    private async Task DeleteDataSource(IDataSource dataSource)
-    {
-        if (dataSource.IsEnterpriseConfiguration)
-            return;
-
-        var dialogParameters = new DialogParameters<ConfirmDialog>
-        {
-            { x => x.Message, string.Format(T("Are you sure you want to delete the data source '{0}' of type {1}?"), dataSource.Name, dataSource.Type.GetDisplayName()) },
-        };
-        
-        var dialogReference = await this.DialogService.ShowAsync<ConfirmDialog>(T("Delete Data Source"), dialogParameters, DialogOptions.FULLSCREEN);
-        var dialogResult = await dialogReference.Result;
-        if (dialogResult is null || dialogResult.Canceled)
-            return;
-
-        var applyChanges = dataSource is IInternalDataSource;
-        
-        // External data sources may need a secret for authentication:
-        if (dataSource is IExternalDataSource externalDataSource)
-        {
-            // When the auth method is NONE or KERBEROS, we don't need to delete a secret.
-            // In the case of KERBEROS, we don't store the Kerberos ticket in the secret store.
-            if(dataSource is IERIDataSource { AuthMethod: AuthMethod.NONE or AuthMethod.KERBEROS })
-                applyChanges = true;
-            
-            // All other auth methods require a secret, which we need to delete now:
-            else
-            {
-                var deleteSecretResponse = await this.RustService.DeleteSecret(externalDataSource, SecretStoreType.DATA_SOURCE);
-                if (deleteSecretResponse.Success)
-                    applyChanges = true;
-            }
-        }
-        
-        if(applyChanges)
-        {
-            this.SettingsManager.ConfigurationData.DataSources.Remove(dataSource);
-            await this.SettingsManager.StoreSettings();
-            await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
-        }
-    }
-    
-    private async Task ShowInformation(IDataSource dataSource)
-    {
-        switch (dataSource)
-        {
-            case DataSourceLocalFile localFile:
-                var localFileDialogParameters = new DialogParameters<DataSourceLocalFileInfoDialog>
-                {
-                    { x => x.DataSource, localFile },
-                };
-
-                await this.DialogService.ShowAsync<DataSourceLocalFileInfoDialog>(T("Local File Data Source Information"), localFileDialogParameters, DialogOptions.FULLSCREEN);
-                break;
-            
-            case DataSourceLocalDirectory localDirectory:
-                var localDirectoryDialogParameters = new DialogParameters<DataSourceLocalDirectoryInfoDialog>
-                {
-                    { x => x.DataSource, localDirectory },
-                };
-
-                await this.DialogService.ShowAsync<DataSourceLocalDirectoryInfoDialog>(T("Local Directory Data Source Information"), localDirectoryDialogParameters, DialogOptions.FULLSCREEN);
-                break;
-            
-            case DataSourceERI_V1 eriV1DataSource:
-                var eriV1DialogParameters = new DialogParameters<DataSourceERI_V1InfoDialog>
-                {
-                    { x => x.DataSource, eriV1DataSource },
-                };
-                
-                await this.DialogService.ShowAsync<DataSourceERI_V1InfoDialog>(T("ERI v1 Data Source Information"), eriV1DialogParameters, DialogOptions.FULLSCREEN);
-                break;
-        }
-    }
-}
+public partial class SettingsDialogDataSources : SettingsDialogBase;

app/MindWork AI Studio/Dialogs/Settings/SettingsDialogERIServer.razor

@@ -3,7 +3,7 @@
 
 <MudDialog>
     <TitleContent>
-        <PreviewPrototype/>
+        <PreviewBeta/>
         <MudText Typo="Typo.h6" Class="d-flex align-center">
             <MudIcon Icon="@Icons.Material.Filled.PrivateConnectivity" Class="mr-2"/>
             @T("Assistant: ERI Server Options")

app/MindWork AI Studio/Layout/MainLayout.razor

@@ -25,7 +25,17 @@
                         
                         <MudSpacer/>
                         
-                        <MudStack AlignItems="AlignItems.Center">
+                        @if (this.showEmbeddingStatusIcon)
+                        {
+                            <MudNavMenu>
+                                <MudTooltip Text="@this.EmbeddingNavigationTooltip" Placement="Placement.Right">
+                                    <MudNavLink Href="@this.embeddingItem.Path" Match="@(this.embeddingItem.MatchAll ? NavLinkMatch.All : NavLinkMatch.Prefix)" Icon="@this.embeddingItem.Icon" Style="@this.embeddingItem.SetColorStyle(this.SettingsManager)" Class="custom-icon-color">
+                                        @T("Data sync")
+                                    </MudNavLink>
+                                </MudTooltip>
+                            </MudNavMenu>
+                        }
+                        <MudStack AlignItems="AlignItems.Center" Class="pb-2">
                             <MudToolBar WrapContent="true">
                                 <VoiceRecorder />
                             </MudToolBar>
@@ -53,8 +63,23 @@
                     </MudNavMenu>
 
                     <MudSpacer/>
-
-                    <MudStack AlignItems="AlignItems.Center">
+                    
+                    @if (this.showEmbeddingStatusIcon)
+                    {
+                        <MudNavMenu>
+                            @if (this.SettingsManager.ConfigurationData.App.NavigationBehavior is NavBehavior.NEVER_EXPAND_USE_TOOLTIPS)
+                            {
+                                <MudTooltip Text="@this.EmbeddingNavigationTooltip" Placement="Placement.Right">
+                                    <MudNavLink Href="@this.embeddingItem.Path" Match="@(this.embeddingItem.MatchAll ? NavLinkMatch.All : NavLinkMatch.Prefix)" Icon="@this.embeddingItem.Icon" Style="@this.embeddingItem.SetColorStyle(this.SettingsManager)" Class="custom-icon-color"/>
+                                </MudTooltip>
+                            }
+                            else
+                            {
+                            <MudNavLink Href="@this.embeddingItem.Path" Match="@(this.embeddingItem.MatchAll ? NavLinkMatch.All : NavLinkMatch.Prefix)" Icon="@this.embeddingItem.Icon" Style="@this.embeddingItem.SetColorStyle(this.SettingsManager)" Class="custom-icon-color"/>
+                            }
+                        </MudNavMenu>
+                    }
+                    <MudStack AlignItems="AlignItems.Center" Class="pb-2">
                         <MudToolBar WrapContent="true">
                             <VoiceRecorder />
                         </MudToolBar>

app/MindWork AI Studio/Layout/MainLayout.razor.cs

@@ -55,6 +55,9 @@ public partial class MainLayout : LayoutComponentBase, IMessageBusReceiver, ILan
     [Inject]
     private MudTheme ColorTheme { get; init; } = null!;
 
+    [Inject]
+    private DataSourceEmbeddingService DataSourceEmbeddingService { get; init; } = null!;
+
     [Inject]
     private CircuitStateService CircuitState { get; init; } = null!;
     
@@ -77,7 +80,10 @@ public partial class MainLayout : LayoutComponentBase, IMessageBusReceiver, ILan
     private readonly SemaphoreSlim mandatoryInfoDialogSemaphore = new(1, 1);
     private readonly SemaphoreSlim promptInjectionDialogSemaphore = new(1, 1);
 
+    private DataSourceEmbeddingOverview embeddingOverview = new(DataSourceEmbeddingState.COMPLETED, 0, 0, 0);
     private IReadOnlyCollection<NavBarItem> navItems = [];
+    private NavBarItem embeddingItem = new (string.Empty, string.Empty, string.Empty, string.Empty, string.Empty, false);
+    private bool showEmbeddingStatusIcon;
     
     #region Overrides of ComponentBase
 
@@ -111,6 +117,7 @@ public partial class MainLayout : LayoutComponentBase, IMessageBusReceiver, ILan
         
         // Ensure that all settings are loaded:
         await this.SettingsManager.LoadSettings();
+        await this.DataSourceEmbeddingService.QueueAllInternalDataSourcesIfAutomaticRefreshAsync();
         
         // Register this component with the message bus:
         this.MessageBus.RegisterComponent(this, this.CircuitState);
@@ -119,7 +126,8 @@ public partial class MainLayout : LayoutComponentBase, IMessageBusReceiver, ILan
             Event.UPDATE_AVAILABLE, Event.CONFIGURATION_CHANGED, Event.COLOR_THEME_CHANGED, Event.SHOW_ERROR,
             Event.SHOW_WARNING, Event.SHOW_SUCCESS, Event.SHOW_INFO, Event.SHOW_PROMPT_INJECTION_ALERT, Event.STARTUP_PLUGIN_SYSTEM, Event.PLUGINS_RELOADED,
             Event.INSTALL_UPDATE, Event.STARTUP_COMPLETED, Event.AI_JOB_CHANGED, Event.AI_JOB_FINISHED,
-            Event.CHAT_GENERATION_CHANGED, Event.ASSISTANT_SESSION_CHANGED, Event.ASSISTANT_SESSION_FINISHED,
+            Event.CHAT_GENERATION_CHANGED, Event.RAG_EMBEDDING_STATUS_CHANGED,Event.ASSISTANT_SESSION_CHANGED, 
+            Event.ASSISTANT_SESSION_FINISHED,
         ]);
         
         // Set the snackbar for the update service:
@@ -139,6 +147,7 @@ public partial class MainLayout : LayoutComponentBase, IMessageBusReceiver, ILan
         await this.themeProvider.WatchSystemDarkModeAsync(this.SystemeThemeChanged);
         await this.UpdateThemeConfiguration();
         this.LoadNavItems();
+        this.LoadEmbeddingItem();
 
         await base.OnInitializedAsync();
     }
@@ -235,6 +244,7 @@ public partial class MainLayout : LayoutComponentBase, IMessageBusReceiver, ILan
 
                     await this.UpdateThemeConfiguration();
                     this.LoadNavItems();
+                    this.LoadEmbeddingItem();
                     this.StateHasChanged();
                     if (this.startupCompleted)
                         this.EnsureMandatoryInfosAcceptedAsync().Observe($"{nameof(MainLayout)}: mandatory infos after a configuration change");
@@ -347,6 +357,7 @@ public partial class MainLayout : LayoutComponentBase, IMessageBusReceiver, ILan
                     I18N.Init(this.Lang);
                     this.ShowSettingsWriteProtectionWarning();
                     this.LoadNavItems();
+                    this.LoadEmbeddingItem();
 
                     await this.InvokeAsync(this.StateHasChanged);
                     if (this.startupCompleted)
@@ -357,6 +368,12 @@ public partial class MainLayout : LayoutComponentBase, IMessageBusReceiver, ILan
                     this.startupCompleted = true;
                     this.EnsureMandatoryInfosAcceptedAsync().Observe($"{nameof(MainLayout)}: mandatory infos after the startup");
                     break;
+
+                case Event.RAG_EMBEDDING_STATUS_CHANGED:
+                    this.LoadNavItems();
+                    this.LoadEmbeddingItem();
+                    this.StateHasChanged();
+                    break;
             }
         });
     }
@@ -439,6 +456,52 @@ public partial class MainLayout : LayoutComponentBase, IMessageBusReceiver, ILan
         yield return new(T("Settings"), Icons.Material.Filled.Settings, defaultLightColor, defaultDarkColor, Routes.SETTINGS, false);
     }
 
+    private void LoadEmbeddingItem()
+    {
+        this.embeddingOverview = this.DataSourceEmbeddingService.GetOverview();
+
+        //
+        // The entry is shown whenever local RAG is available, in every state. Hiding it while
+        // nothing was running looked tidier, but a data source which was just added has no status
+        // yet: the service creates one when the run begins. The icon was therefore missing during
+        // the very moment the user was waiting for it. What the entry does communicate is its
+        // state, through the icon below.
+        //
+        // The preview feature is what gates it now. The route itself is not gated, so without this
+        // check, users who have no RAG at all would get a navigation entry for it.
+        //
+        this.showEmbeddingStatusIcon = PreviewFeatures.PRE_RAG_2024.IsEnabled(this.SettingsManager);
+
+        var palette = this.ColorTheme.GetCurrentPalette(this.SettingsManager);
+        (string icon, string lightcolor, string darkcolor) embeddingIcon = this.embeddingOverview.State switch
+        {
+            DataSourceEmbeddingState.FAILED => (Icons.Material.Filled.Warning, palette.Error.Value, "#d32f2f"),
+            DataSourceEmbeddingState.QUEUED => (Icons.Material.Filled.Sync, palette.Info.Value, "#1976d2"),
+            DataSourceEmbeddingState.RUNNING => (Icons.Material.Filled.Sync, palette.Warning.Value, "#d29f00"),
+
+            // Nothing to do: the entry keeps the colors of its neighbors, so a permanently visible
+            // icon does not draw attention while there is nothing to attend to:
+            _ => (Icons.Material.Filled.CloudDone, palette.DarkLighten, palette.GrayLight),
+        };
+        this.embeddingItem = new NavBarItem(T("Embeddings"), embeddingIcon.icon, embeddingIcon.lightcolor, embeddingIcon.darkcolor, Routes.EMBEDDINGS, false);
+    }
+    
+    private string EmbeddingNavigationTooltip => this.embeddingOverview.State switch
+    {
+        DataSourceEmbeddingState.QUEUED => T("Embeddings are waiting to be processed."),
+        DataSourceEmbeddingState.RUNNING => string.Format(
+            T("Embeddings are running: {0} of {1} files are indexed."),
+            this.embeddingOverview.IndexedFiles,
+            this.embeddingOverview.TotalFiles),
+        DataSourceEmbeddingState.FAILED => this.embeddingOverview.FailedFiles > 0
+            ? string.Format(T("Some embeddings failed. {0} file(s) need attention."), this.embeddingOverview.FailedFiles)
+            : T("Some embeddings failed and need attention."),
+
+        // The entry is always visible, so its resting state needs words as well. An empty tooltip
+        // would leave the user guessing what the icon is there for:
+        _ => T("All data sources are up to date.")
+    };
+
     private async Task ShowUpdateDialog()
     {
         if (!this.UpdatePolicy.AllowsInstallations)

app/MindWork AI Studio/MindWork AI Studio.csproj

@@ -62,10 +62,13 @@
     <ItemGroup>
       <PackageReference Include="CodeBeam.MudBlazor.Extensions" Version="8.3.0" />
       <PackageReference Include="HtmlAgilityPack" Version="1.12.4" />
+      <PackageReference Include="Microsoft.Data.Sqlite.Core" Version="9.0.19" />
+      <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="9.0.19" />
       <PackageReference Include="Microsoft.Extensions.FileProviders.Embedded" Version="9.0.19" />
       <PackageReference Include="MudBlazor" Version="8.15.0" />
       <PackageReference Include="MudBlazor.Markdown" Version="8.11.0" />
       <PackageReference Include="ReverseMarkdown" Version="5.0.0" />
+      <PackageReference Include="SQLitePCLRaw.bundle_e_sqlite3" Version="3.0.5" />
       <PackageReference Include="LuaCSharp" Version="0.5.6" />
     </ItemGroup>
 

app/MindWork AI Studio/Pages/Embeddings.razor

@@ -0,0 +1,176 @@
+@attribute [Route(Routes.EMBEDDINGS)]
+@inherits MSGComponentBase
+
+<MudStack Spacing="3" Class="pr-2 pb-4" Style="height: 100%; min-height: 0; overflow-y: auto;">
+    <MudPaper Class="pa-4 border-dashed border rounded-lg" Elevation="0">
+        <MudStack Row="true" AlignItems="AlignItems.Center">
+            <MudText Typo="Typo.h5">@T("Background embeddings")</MudText>
+            <MudSpacer/>
+            <MudTooltip Text="@T("Manage your data sources")" Placement="Placement.Top">
+                <MudIconButton Icon="@Icons.Material.Filled.Settings" OnClick="@this.OpenDataSourceSettings"/>
+            </MudTooltip>
+        </MudStack>
+        <MudText Typo="Typo.body1" Class="mt-2">
+            @T("AI Studio indexes local RAG data sources in the background. Finished files stay recorded so unchanged files can be skipped after a restart, while added or deleted files are detected during the next run. The same applies to documents without readable text, such as scanned pages: AI Studio remembers them and reads them again only once they change.")
+        </MudText>
+        <MudStack Row="true" Class="mt-3" Wrap="Wrap.Wrap" Spacing="2">
+            <MudChip T="string" Color="Color.Success" Variant="Variant.Filled">@string.Format(T("Indexed files: {0}"), this.TotalIndexedFiles)</MudChip>
+            <MudChip T="string" Color="Color.Info" Variant="Variant.Filled">@string.Format(T("Pending files: {0}"), this.TotalPendingFiles)</MudChip>
+            <MudChip T="string" Color="Color.Default" Variant="Variant.Filled">@string.Format(T("Skipped files: {0}"), this.TotalPermanentlySkippedFiles)</MudChip>
+            <MudChip T="string" Color="Color.Error" Variant="Variant.Filled">@string.Format(T("Failed files: {0}"), this.TotalFailedFiles)</MudChip>
+        </MudStack>
+    </MudPaper>
+
+    @if (this.Statuses.Count == 0)
+    {
+        <MudAlert Severity="Severity.Info" Variant="Variant.Outlined">
+            @T("No local data source has been queued for embedding yet.")
+        </MudAlert>
+    }
+    else
+    {
+        @*
+            One panel per data source, and only one of them open: a folder with thousands of files
+            fills the page by itself. What a closed panel still has to say stays in its header, so
+            nobody has to open every one of them to see how their data sources are doing.
+        *@
+        <MudExpansionPanels>
+            @foreach (var status in this.Statuses)
+            {
+                <MudExpansionPanel Class="border-solid border rounded-lg" Expanded="@(this.expandedDataSourceId == status.DataSourceId)" ExpandedChanged="@(isExpanded => this.DataSourcePanelExpandedChanged(status, isExpanded))">
+                    <TitleContent>
+                        <div class="d-flex align-center">
+                            <MudText Typo="Typo.h6">@string.Format(T("Data source: {0}"), status.DataSourceName)</MudText>
+                            <MudSpacer/>
+                            <MudChip T="string" Color="@GetStatusColor(status)" Variant="Variant.Filled">@status.StateLabel</MudChip>
+                            @if (CanRefresh(status))
+                            {
+                                @*
+                                    This opens or closes the panel along the way. The header is what
+                                    toggles it, and neither stopping the event nor swallowing the
+                                    click works in this project — see the end button of our own
+                                    ExpansionPanel component, which behaves the same way.
+                                *@
+                                <MudTooltip Text="@T("Refresh this data source")">
+                                    <MudIconButton Icon="@Icons.Material.Filled.Sync" Color="Color.Info" Size="Size.Small" OnClick="@(async () => await this.RefreshDataSource(status))" />
+                                </MudTooltip>
+                            }
+                        </div>
+                    </TitleContent>
+                    <ChildContent>
+                        <MudStack Spacing="2">
+                            <MudProgressLinear Value="@status.ProgressPercent" Rounded="@true" Color="@GetStatusColor(status)" />
+
+                            <MudText Typo="Typo.body2">
+                                @string.Format(T("{0} of {1} files are indexed."), status.IndexedFiles, status.TotalFiles)
+                            </MudText>
+
+                            @if (status.PermanentlySkippedFiles > 0)
+                            {
+                                <MudText Typo="Typo.body2">
+                                    @string.Format(T("Skipped files: {0}. AI Studio reads them again once they change."), status.PermanentlySkippedFiles)
+                                </MudText>
+                            }
+
+                            @if (status.FailedFiles > 0)
+                            {
+                                <MudText Typo="Typo.body2">
+                                    @string.Format(T("Failed files: {0}"), status.FailedFiles)
+                                </MudText>
+                            }
+
+                            @if (!string.IsNullOrWhiteSpace(status.CurrentFile))
+                            {
+                                <MudText Typo="Typo.body2">
+                                    @string.Format(T("Current file: {0}"), status.CurrentFile)
+                                </MudText>
+                            }
+
+                            @*
+                                One panel per cause, the most pressing one first. A folder in which nine
+                                hundred scanned documents were skipped otherwise buries the handful of
+                                files somebody has to look at.
+                            *@
+                            @if (status.Failures.Count > 0)
+                            {
+                                <MudExpansionPanels MultiExpansion="@true" Class="mt-2">
+                                    @foreach (var group in this.GetFailureGroups(status))
+                                    {
+                                        <ExpansionPanel HeaderIcon="@group.Cause.Icon" IconColor="@group.Cause.Color" IconSize="Size.Small" HeaderTypo="Typo.subtitle1" HeaderClass="expansion-panel-header-compact" HeaderText="@GetGroupHeader(group)">
+                                            <MudStack Spacing="3">
+                                                <MudStack Row="@true" Wrap="Wrap.Wrap" Spacing="2" AlignItems="AlignItems.Center">
+                                                    <MudChip T="string" Size="Size.Small" Color="@group.Cause.Color" Variant="Variant.Outlined">
+                                                        @(group.Cause.IsPermanent ? T("Skipped until the file changes") : T("Tried again during the next run"))
+                                                    </MudChip>
+                                                    @if (!string.IsNullOrWhiteSpace(group.EmbeddingProviderName))
+                                                    {
+                                                        <MudText Typo="Typo.caption">@string.Format(T("Embedding provider: {0}"), group.EmbeddingProviderName)</MudText>
+                                                    }
+                                                    @if (group.Cause.NeedsProviderSettings)
+                                                    {
+                                                        <MudButton Variant="Variant.Text" Size="Size.Small" Color="Color.Primary" StartIcon="@Icons.Material.Filled.Settings" OnClick="@this.OpenEmbeddingProviderSettings">
+                                                            @T("Open the settings")
+                                                        </MudButton>
+                                                    }
+                                                </MudStack>
+                                                <MudTable Items="@group.Failures" Dense="@true" Hover="@true" Elevation="0" Breakpoint="Breakpoint.None" RowsPerPage="10" Class="border-dashed border rounded-lg">
+                                                    <ColGroup>
+                                                        <col/>
+                                                        <col style="width: 11em;"/>
+                                                        <col style="width: 7em;"/>
+                                                    </ColGroup>
+                                                    <HeaderContent>
+                                                        <MudTh>@T("File")</MudTh>
+                                                        <MudTh>@T("Noticed")</MudTh>
+                                                        <MudTh>@T("Actions")</MudTh>
+                                                    </HeaderContent>
+                                                    <RowTemplate>
+                                                        <MudTd Style="white-space: normal; overflow-wrap: anywhere;">
+                                                            @* Captions render as spans, so each of them needs to be told to take its own line. *@
+                                                            <MudText Typo="Typo.body2">@GetFileName(context.FilePath)</MudText>
+                                                            <MudText Typo="Typo.caption" Class="mud-text-secondary d-block">@context.FilePath</MudText>
+                                                            @if (group.Cause.ShowsMessagePerFile)
+                                                            {
+                                                                <MudText Typo="Typo.caption" Color="Color.Warning" Class="d-block">@context.Reason</MudText>
+                                                            }
+                                                        </MudTd>
+                                                        <MudTd Style="white-space: nowrap;">
+                                                            <MudText Typo="Typo.caption">@GetOccurrenceText(context)</MudText>
+                                                        </MudTd>
+                                                        <MudTd>
+                                                            @if (CanShowInFileManager(context))
+                                                            {
+                                                                <MudTooltip Text="@T("Show this file in the file browser of your system")">
+                                                                    <MudIconButton Icon="@Icons.Material.Filled.FolderOpen" Color="Color.Info" Size="Size.Small" OnClick="@(async () => await this.ShowInFileManager(context))"/>
+                                                                </MudTooltip>
+                                                            }
+                                                        </MudTd>
+                                                    </RowTemplate>
+                                                    <PagerContent>
+                                                        <MudTablePager PageSizeOptions="@PAGE_SIZE_OPTIONS"/>
+                                                    </PagerContent>
+                                                </MudTable>
+                                            </MudStack>
+                                        </ExpansionPanel>
+                                    }
+                                </MudExpansionPanels>
+                            }
+
+                            @*
+                                Shown next to the list, not instead of it: the list says which files failed,
+                                while this is the one sentence about the data source as a whole. Hiding it
+                                as soon as a single file failed is what made it invisible in practice.
+                            *@
+                            @if (!string.IsNullOrWhiteSpace(status.LastError))
+                            {
+                                <MudAlert Severity="Severity.Warning" Variant="Variant.Text">
+                                    @status.LastError
+                                </MudAlert>
+                            }
+                        </MudStack>
+                    </ChildContent>
+                </MudExpansionPanel>
+            }
+        </MudExpansionPanels>
+    }
+</MudStack>

app/MindWork AI Studio/Pages/Embeddings.razor.cs

@@ -0,0 +1,277 @@
+using AIStudio.Components;
+using AIStudio.Dialogs.Settings;
+using AIStudio.Provider;
+using AIStudio.Settings.DataModel;
+using AIStudio.Tools.Rust;
+using AIStudio.Tools.Services;
+
+using Microsoft.AspNetCore.Components;
+
+using DialogOptions = AIStudio.Dialogs.DialogOptions;
+
+namespace AIStudio.Pages;
+
+public partial class Embeddings : MSGComponentBase
+{
+    private static readonly int[] PAGE_SIZE_OPTIONS = [10, 25, 50, 100];
+
+    [Inject]
+    private DataSourceEmbeddingService DataSourceEmbeddingService { get; init; } = null!;
+
+    [Inject]
+    private NavigationManager NavigationManager { get; init; } = null!;
+
+    [Inject]
+    private RustService RustService { get; init; } = null!;
+
+    [Inject]
+    private IDialogService DialogService { get; init; } = null!;
+
+    [Inject]
+    private ILogger<Embeddings> Logger { get; init; } = null!;
+
+    private IReadOnlyList<DataSourceEmbeddingStatus> Statuses { get; set; } = [];
+
+    private string? expandedDataSourceId;
+    private bool userChoseExpansion;
+
+    private int TotalIndexedFiles => this.Statuses.Sum(status => status.IndexedFiles);
+
+    private int TotalPendingFiles => this.Statuses.Sum(status => Math.Max(0, status.TotalFiles - status.IndexedFiles - status.FailedFiles - status.PermanentlySkippedFiles));
+
+    private int TotalFailedFiles => this.Statuses.Sum(status => status.FailedFiles);
+
+    private int TotalPermanentlySkippedFiles => this.Statuses.Sum(status => status.PermanentlySkippedFiles);
+
+    protected override async Task OnInitializedAsync()
+    {
+        //
+        // This page belongs to the local RAG preview feature. Unlike the other preview pages, it
+        // has a route of its own, so it can be reached by typing the address even while the feature
+        // is switched off. There is nothing to show in that case.
+        //
+        if (!PreviewFeatures.PRE_RAG_2024.IsEnabled(this.SettingsManager))
+        {
+            this.NavigationManager.NavigateTo(Routes.HOME);
+            return;
+        }
+
+        this.ApplyFilters([], [ Event.RAG_EMBEDDING_STATUS_CHANGED, Event.CONFIGURATION_CHANGED ]);
+        await base.OnInitializedAsync();
+        this.ReloadStatuses();
+    }
+
+    protected override Task ProcessIncomingMessage<T>(ComponentBase? sendingComponent, Event triggeredEvent, T? data) where T : default
+    {
+        if (triggeredEvent is Event.RAG_EMBEDDING_STATUS_CHANGED or Event.CONFIGURATION_CHANGED)
+        {
+            this.ReloadStatuses();
+            this.StateHasChanged();
+        }
+
+        return Task.CompletedTask;
+    }
+
+    private void ReloadStatuses()
+    {
+        this.Statuses = this.DataSourceEmbeddingService
+            .GetStatuses()
+            .OrderBy(status => status.SortOrder)
+            .ThenBy(status => status.DataSourceName, StringComparer.OrdinalIgnoreCase)
+            .ToList();
+
+        this.UpdateAutoExpansion();
+    }
+
+    /// <summary>
+    /// Opens the data source which is worth reading, as long as the user has not chosen one.
+    /// </summary>
+    /// <remarks>
+    /// It never closes what is open. A data source which finishes its run while somebody is reading
+    /// it would otherwise fold up at the very moment its result becomes interesting. From the first
+    /// click on, the page stops rearranging itself at all.
+    /// </remarks>
+    private void UpdateAutoExpansion()
+    {
+        if (this.userChoseExpansion)
+            return;
+
+        // The list is sorted by state, so the first match is the most pressing one: a running data
+        // source before a queued one, and a failed one before a completed one:
+        var worthOpening = this.Statuses.FirstOrDefault(IsWorthOpening);
+        if (worthOpening is null)
+            return;
+
+        this.expandedDataSourceId = worthOpening.DataSourceId;
+    }
+
+    /// <remarks>
+    /// Whoever opens this page does so because a run is under way or because something went wrong.
+    /// Meeting nothing but closed panels would be a step back from the version which showed every
+    /// data source at once.
+    /// </remarks>
+    private static bool IsWorthOpening(DataSourceEmbeddingStatus status) =>
+        status.State is DataSourceEmbeddingState.RUNNING or DataSourceEmbeddingState.QUEUED or DataSourceEmbeddingState.FAILED || status.FailedFiles > 0;
+
+    /// <remarks>
+    /// MudBlazor keeps track of which panel is open on its own, so this only records the decision.
+    /// Both events of a switch arrive, in either order — the one closing the old panel and the one
+    /// opening the new one — which is why the closing event only clears what it actually named.
+    /// </remarks>
+    private void DataSourcePanelExpandedChanged(DataSourceEmbeddingStatus status, bool isExpanded)
+    {
+        this.userChoseExpansion = true;
+
+        if (isExpanded)
+            this.expandedDataSourceId = status.DataSourceId;
+        else if (this.expandedDataSourceId == status.DataSourceId)
+            this.expandedDataSourceId = null;
+    }
+
+    /// <summary>
+    /// Opens the data source settings, the same dialog the chat offers next to its data source selection.
+    /// </summary>
+    /// <remarks>
+    /// Nothing is left to do once it closes: the dialog writes the settings itself and publishes
+    /// CONFIGURATION_CHANGED, which this page already listens to.
+    /// </remarks>
+    private async Task OpenDataSourceSettings()
+    {
+        var dialogParameters = new DialogParameters();
+        var dialogReference = await this.DialogService.ShowAsync<SettingsDialogDataSources>(null, dialogParameters, DialogOptions.FULLSCREEN);
+        await dialogReference.Result;
+    }
+
+    private static Color GetStatusColor(DataSourceEmbeddingStatus status) => status.State switch
+    {
+        DataSourceEmbeddingState.RUNNING => Color.Warning,
+        DataSourceEmbeddingState.QUEUED => Color.Info,
+        DataSourceEmbeddingState.FAILED => Color.Error,
+        DataSourceEmbeddingState.COMPLETED when status.FailedFiles > 0 => Color.Warning,
+        DataSourceEmbeddingState.COMPLETED => Color.Success,
+        _ => Color.Default,
+    };
+
+    /// <summary>
+    /// What a group of failures has in common.
+    /// </summary>
+    /// <remarks>
+    /// Also the key the failures are grouped by: two of them belong together exactly when the
+    /// list would say the same thing about both.
+    /// </remarks>
+    private sealed record FailureCause(int Priority, string Title, string Icon, Color Color, bool IsPermanent, bool NeedsProviderSettings, bool ShowsMessagePerFile);
+
+    private sealed record FailureGroup(FailureCause Cause, string EmbeddingProviderName, IReadOnlyList<DataSourceEmbeddingFailure> Failures);
+
+    /// <summary>
+    /// Puts the failures of a data source into one group per cause, the most pressing one first.
+    /// </summary>
+    /// <remarks>
+    /// Within a priority, the largest group comes first: it is the one telling the user the most
+    /// about their folder.
+    /// </remarks>
+    private IReadOnlyList<FailureGroup> GetFailureGroups(DataSourceEmbeddingStatus status) => status.Failures
+        .GroupBy(this.GetFailureCause)
+        .Select(group => new FailureGroup(group.Key, GetEmbeddingProviderName(group), group.OrderBy(failure => failure.FilePath, StringComparer.OrdinalIgnoreCase).ToList()))
+        .OrderBy(group => group.Cause.Priority)
+        .ThenByDescending(group => group.Failures.Count)
+        .ThenBy(group => group.Cause.Title, StringComparer.OrdinalIgnoreCase)
+        .ToList();
+
+    private FailureCause GetFailureCause(DataSourceEmbeddingFailure failure)
+    {
+        //
+        // Anything the provider answered comes first: it stops the entire data source, while a
+        // file nobody can read costs that one file:
+        //
+        if (failure.FailureReason is not ProviderRequestFailureReason.NONE)
+        {
+            var isFixedInSettings = failure.FailureReason.IsFixedInProviderSettings();
+            return new FailureCause(isFixedInSettings ? 0 : 1, failure.FailureReason.GetName(), isFixedInSettings ? Icons.Material.Filled.Key : Icons.Material.Filled.CloudOff, Color.Error, false, isFixedInSettings, true);
+        }
+
+        //
+        // Codes without a name of their own carry everything they know in the message of the
+        // single file, which is why those groups show that message per file:
+        //
+        var causeName = failure.ExtractionCode.GetIndexingCauseName();
+        var hasCauseName = !string.IsNullOrWhiteSpace(causeName);
+        var title = hasCauseName ? causeName : T("Other cause");
+
+        return failure.IsPermanent
+            ? new FailureCause(3, title, Icons.Material.Filled.SkipNext, Color.Default, true, false, !hasCauseName)
+            : new FailureCause(2, title, Icons.Material.Filled.ReportProblem, Color.Warning, false, false, !hasCauseName);
+    }
+
+    private static string GetEmbeddingProviderName(IEnumerable<DataSourceEmbeddingFailure> failures) => failures
+        .Select(failure => failure.EmbeddingProviderName)
+        .FirstOrDefault(name => !string.IsNullOrWhiteSpace(name)) ?? string.Empty;
+
+    private static string GetGroupHeader(FailureGroup group) => $"{group.Cause.Title} ({group.Failures.Count})";
+
+    private static string GetFileName(string filePath)
+    {
+        var fileName = Path.GetFileName(filePath);
+        return string.IsNullOrWhiteSpace(fileName) ? filePath : fileName;
+    }
+
+    private static string GetOccurrenceText(DataSourceEmbeddingFailure failure) => failure.OccurredAtUtc > DateTimeOffset.MinValue ? failure.OccurredAtUtc.ToLocalTime().ToString("g") : string.Empty;
+
+    /// <remarks>
+    /// A failure which was not about one file, such as a folder which is gone, carries the name of
+    /// the data source instead of a path. There is nothing to show for those.
+    /// </remarks>
+    private static bool CanShowInFileManager(DataSourceEmbeddingFailure failure) => !string.IsNullOrWhiteSpace(failure.FilePath) && Path.IsPathRooted(failure.FilePath);
+
+    /// <summary>
+    /// Opens the file browser of the system and selects the file in it.
+    /// </summary>
+    /// <remarks>
+    /// Reading that a file could not be indexed is where the work starts, not where it ends: the
+    /// file has to be opened, replaced, or run through an OCR. This is the same way out the log
+    /// viewer offers for the log files.
+    /// </remarks>
+    private async Task ShowInFileManager(DataSourceEmbeddingFailure failure)
+    {
+        OpenPathResponse response;
+        try
+        {
+            response = await this.RustService.TryOpenPathInRuntimeFileManager(failure.FilePath);
+        }
+        catch (Exception e)
+        {
+            this.Logger.LogWarning(e, "Could not show a file of the embedding failure list in the file manager.");
+            await this.MessageBus.SendError(new(Icons.Material.Filled.Folder, T("Could not open the file location.")));
+            return;
+        }
+
+        if (response.Success)
+            return;
+
+        var issue = string.IsNullOrWhiteSpace(response.Issue) ? T("Unknown error") : response.Issue;
+        await this.MessageBus.SendError(new(Icons.Material.Filled.Folder, string.Format(T("Could not open the file location: {0}"), issue)));
+    }
+
+    private bool CanRefresh(DataSourceEmbeddingStatus status)
+    {
+        return this.DataSourceEmbeddingService.CanRefreshDataSource(status.DataSourceId) &&
+            status.State is not DataSourceEmbeddingState.RUNNING and not DataSourceEmbeddingState.QUEUED &&
+            (status.State is DataSourceEmbeddingState.FAILED || status.FailedFiles > 0);
+    }
+
+    /// <summary>
+    /// Takes the user to the settings, where the embedding providers are configured.
+    /// </summary>
+    /// <remarks>
+    /// Offered only for the failures a setting fixes, such as a rejected API key. Reading what
+    /// went wrong and then having to find the right page is where people give up.
+    /// </remarks>
+    private void OpenEmbeddingProviderSettings() => this.NavigationManager.NavigateTo(Routes.SETTINGS);
+
+    private async Task RefreshDataSource(DataSourceEmbeddingStatus status)
+    {
+        await this.DataSourceEmbeddingService.RetryDataSourceAsync(status.DataSourceId);
+        this.ReloadStatuses();
+        await this.InvokeAsync(this.StateHasChanged);
+    }
+}

app/MindWork AI Studio/Pages/Information.razor

@@ -351,6 +351,7 @@
                     }
                     
                     <ThirdPartyComponent Name="Qdrant Edge" Developer="Andrey Vasnetsov, Tim Visée, Arnaud Gourlay, Luis Cossío, Ivan Pleshkov, Roman Titov, xzfc, JojiiOfficial & Open Source Community" LicenseName="Apache-2.0" LicenseUrl="https://github.com/qdrant/qdrant/blob/master/LICENSE" RepositoryUrl="https://github.com/qdrant/qdrant" UseCase="@T("Qdrant Edge is an embedded vector database and vector similarity search engine. We use it to realize local RAG—retrieval-augmented generation—within AI Studio. Thanks for the effort and great work that has been and is being put into Qdrant.")"/>
+                    <ThirdPartyComponent Name="SQLite" Developer="Richard Hipp &  Team" LicenseName="None (public domain)" LicenseUrl="https://www.sqlite.org/copyright.html" RepositoryUrl="https://www.sqlite.org/src/" UseCase="@T("SQLite stores local RAG indexing metadata, searchable chunk text, and the file fingerprints used to decide whether local files need to be indexed again, without requiring a separate database server or a system SQLite installation.")"/>
                     <ThirdPartyComponent Name="regex" Developer="Andrew Gallant, Alex Crichton, The Rust Project Developers & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/rust-lang/regex/blob/master/LICENSE-MIT" RepositoryUrl="https://github.com/rust-lang/regex" UseCase="@T("The regex crate detects structural and obfuscated prompt-injection patterns in untrusted document content. Its linear-time matching without backtracking keeps these scans predictable, even for hostile input.")"/>
                     <ThirdPartyComponent Name="aho-corasick" Developer="Alfred V. Aho, Margaret J. Corasick, Andrew Gallant & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/BurntSushi/aho-corasick/blob/master/LICENSE-MIT" RepositoryUrl="https://github.com/BurntSushi/aho-corasick" UseCase="@T("The aho-corasick crate searches the fixed catalog of prompt-injection phrases in one pass, allowing AI Studio to scan large documents efficiently. We thank Alfred V. Aho and Margaret J. Corasick for publishing the algorithm in 1975, and Andrew Gallant and the Open Source Community for bringing it to Rust.")"/>
                     <ThirdPartyComponent Name="toml" Developer="Ed Page, Alex Crichton, ordian, Eric Huss & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/toml-rs/toml/blob/main/LICENSE-MIT" RepositoryUrl="https://github.com/toml-rs/toml" UseCase="@T("The toml crate parses the embedded prompt-injection phrase catalog when the runtime starts. This keeps the detection rules separate from the Rust implementation and easier to maintain.")"/>
@@ -389,6 +390,8 @@
                     <ThirdPartyComponent Name="sysinfo" Developer="Guillaume Gomez & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/GuillaumeGomez/sysinfo/blob/main/LICENSE" RepositoryUrl="https://github.com/GuillaumeGomez/sysinfo" UseCase="@T("This library is used to manage sidecar processes and to ensure that stale or zombie sidecars are detected and terminated.")"/>
                     <ThirdPartyComponent Name="tempfile" Developer="Steven Allen, Ashley Mannix & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/Stebalien/tempfile/blob/master/LICENSE-MIT" RepositoryUrl="https://github.com/Stebalien/tempfile" UseCase="@T("This library is used to create temporary folders in runtime tests and supporting filesystem operations.")"/>
                     <ThirdPartyComponent Name="Lua-CSharp" Developer="Yusuke Nakada & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/nuskey8/Lua-CSharp/blob/main/LICENSE" RepositoryUrl="https://github.com/nuskey8/Lua-CSharp" UseCase="@T("We use Lua as the language for plugins. Lua-CSharp lets Lua scripts communicate with AI Studio and vice versa. Thank you, Yusuke Nakada, for this great library.")" />
+                    <ThirdPartyComponent Name="DeepSeek-V3.2 Tokenizer" Developer="DeepSeek-AI" LicenseName="MIT" LicenseUrl="https://huggingface.co/datasets/choosealicense/licenses/blob/main/markdown/mit.md" RepositoryUrl="https://huggingface.co/deepseek-ai/DeepSeek-V3.2/tree/main" UseCase="@T("We use the DeepSeek Tokenizer to estimate the number of tokens an input will generate.")" />
+                    <ThirdPartyComponent Name="Tokenizer" Developer="Anthony Moi, Nicolas Patry, Pierric Cistac, Arthur Zucker & Open Source Community" LicenseName="Apache-2.0" LicenseUrl="https://github.com/huggingface/tokenizers/blob/main/LICENSE" RepositoryUrl="https://github.com/huggingface/tokenizers" UseCase="@T("The Tokenizer library serves as the base framework for integrating the DeepSeek tokenizer.")" />
                     <ThirdPartyComponent Name="HtmlAgilityPack" Developer="ZZZ Projects & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/zzzprojects/html-agility-pack/blob/master/LICENSE" RepositoryUrl="https://github.com/zzzprojects/html-agility-pack" UseCase="@T("We use the HtmlAgilityPack to extract content from the web. This is necessary, e.g., when you provide a URL as input for an assistant.")"/>
                     <ThirdPartyComponent Name="ReverseMarkdown" Developer="Babu Annamalai & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/mysticmind/reversemarkdown-net/blob/master/LICENSE" RepositoryUrl="https://github.com/mysticmind/reversemarkdown-net" UseCase="@T("This library is used to convert HTML to Markdown. This is necessary, e.g., when you provide a URL as input for an assistant.")"/>
                     <ThirdPartyComponent Name="wikEd diff" Developer="Cacycle & Open Source Community" LicenseName="None (public domain)" LicenseUrl="https://en.wikipedia.org/wiki/User:Cacycle/diff#License" RepositoryUrl="https://en.wikipedia.org/wiki/User:Cacycle/diff" UseCase="@T("This library is used to display the differences between two texts. This is necessary, e.g., for the grammar and spelling assistant.")"/>

app/MindWork AI Studio/Pages/Settings.razor

@@ -24,6 +24,10 @@
             <SettingsPanelApp AvailableLLMProvidersFunc="@(() => this.availableLLMProviders)"/>
 
             <SettingsPanelTools />
+            @if (PreviewFeatures.PRE_RAG_2024.IsEnabled(this.SettingsManager))
+            {
+                <SettingsPanelDataSources/>
+            }
 
             @if (PreviewFeatures.PRE_RAG_2024.IsEnabled(this.SettingsManager))
             {
@@ -35,4 +39,4 @@
             <SettingsPanelAgentAssistantAudit AvailableLLMProvidersFunc="@(() => this.availableLLMProviders)"/>
         </MudExpansionPanels>
     </InnerScrolling>
-</div>
+</div>

app/MindWork AI Studio/Plugins/configuration/plugin.lua

@@ -94,6 +94,9 @@ CONFIG["LLM_PROVIDERS"] = {}
 --     -- Please do not add the enclosing curly braces {} here. Also, no trailing comma is allowed.
 --     ["AdditionalJsonApiParameters"] = "",
 --
+--     -- Optional: tokenizer path for this provider relative to the plugin directory.
+--     -- ["TokenizerPath"] = "",
+--
 --     -- Optional: replace the built-in provider logo with a project-specific icon.
 --     -- The path is relative to this plugin.lua and must point to an SVG file inside
 --     -- this plugin directory, for example: assets/project-icon.svg. Absolute paths,
@@ -211,6 +214,15 @@ CONFIG["EMBEDDING_PROVIDERS"] = {}
 --     -- Optional: Encrypted API key (see LLM_PROVIDERS example for details)
 --     -- ["APIKey"] = "ENC:v1:<base64-encoded encrypted data>",
 --
+--     -- Optional: tokenizer path for this provider relative to the plugin directory.
+--     -- ["TokenizerPath"] = "",
+--
+--     -- Optional: maximum number of tokens per embedding chunk. If omitted, AI Studio uses its default.
+--     -- ["TokenLimit"] = 8192,
+--
+--     -- Optional: number of chunks sent to the embedding provider in one request. If omitted, AI Studio sends one chunk per request.
+--     -- ["EmbeddingBatchSize"] = 1,
+--
 --     -- Optional: let each user set their own API key for this otherwise locked embedding
 --     -- provider (see LLM_PROVIDERS example for details). Mutually exclusive with "APIKey"
 --     -- above: when both are set, the embedded key is ignored and a warning is logged.

app/MindWork AI Studio/Program.cs

@@ -199,6 +199,8 @@ internal sealed class Program
         builder.Services.AddSingleton<UpdatePolicy>();
         builder.Services.AddSingleton<AssistantPluginGenerationService>();
         builder.Services.AddSingleton<DataSourceService>();
+        builder.Services.AddSingleton<DataSourceEmbeddingService>();
+        builder.Services.AddSingleton<DataSourceLocalRetrievalService>();
         builder.Services.AddSingleton<DirectChatService>();
         builder.Services.AddScoped<PandocAvailabilityService>();
         
@@ -213,6 +215,7 @@ internal sealed class Program
         builder.Services.AddHostedService<TemporaryChatService>();
         builder.Services.AddHostedService<TranscriptStagingCleanupService>();
         builder.Services.AddHostedService<EnterpriseEnvironmentService>();
+        builder.Services.AddHostedService(sp => sp.GetRequiredService<DataSourceEmbeddingService>());
         builder.Services.AddSingleton<DatabaseClientProvider>();
         builder.Services.AddHostedService<GlobalShortcutService>(serviceProvider => serviceProvider.GetRequiredService<GlobalShortcutService>());
         builder.Services.AddHostedService<RustAvailabilityMonitorService>();
@@ -308,6 +311,7 @@ internal sealed class Program
 
         RUST_SERVICE = rust;
         ENCRYPTION = encryption;
+        
         DATABASE_CLIENT_PROVIDER = app.Services.GetRequiredService<DatabaseClientProvider>();
 
         programLogger.LogInformation("Initialize internal file system.");

app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs

@@ -194,7 +194,7 @@ public sealed class ProviderAnthropic() : BaseProvider(LLMProviders.ANTHROPIC, n
     /// <inhertidoc />
     public override Task<IReadOnlyList<IReadOnlyList<float>>> EmbedTextAsync(Model embeddingModel, SettingsManager settingsManager, CancellationToken token = default, params List<string> texts)
     {
-        return Task.FromResult<IReadOnlyList<IReadOnlyList<float>>>([]);
+        throw this.CreateEmbeddingsNotSupportedException();
     }
 
     /// <inheritdoc />

app/MindWork AI Studio/Provider/BaseProvider.cs

@@ -91,6 +91,9 @@ public abstract class BaseProvider : IProvider, ISecretId
 
     internal ProviderCapabilityOverrides? CapabilityOverrides { get; set; }
 
+    /// <inheritdoc />
+    public string TokenizerPath { get; init; } = string.Empty;
+
     /// <inheritdoc />
     public abstract bool HasModelLoadingCapability { get; }
 
@@ -235,9 +238,106 @@ public abstract class BaseProvider : IProvider, ISecretId
     protected virtual string GetProviderRequestFailureUserMessage(ProviderRequestFailureReason failureReason) => failureReason switch
     {
         ProviderRequestFailureReason.TOO_MANY_REQUESTS => TB("The provider rejected the request because too many requests were sent. Please wait a moment and try again."),
+        ProviderRequestFailureReason.INVALID_OR_MISSING_API_KEY => string.Format(TB("The API key for the provider '{0}' is missing or was rejected. Please check the key in the settings."), this.InstanceName),
+        ProviderRequestFailureReason.AUTHENTICATION_OR_PERMISSION_ERROR => string.Format(TB("The provider '{0}' refused the request. Your account might not be allowed to use the selected model, or the provider might not serve your region."), this.InstanceName),
+        ProviderRequestFailureReason.PROVIDER_UNAVAILABLE => string.Format(TB("The provider '{0}' could not be reached. Please check whether it is running and reachable, then try again."), this.InstanceName),
+        ProviderRequestFailureReason.MODEL_NOT_FOUND => string.Format(TB("The provider '{0}' does not know the selected model. Please select another model."), this.InstanceName),
+        ProviderRequestFailureReason.CONTEXT_LENGTH_EXCEEDED => TB("The text was longer than the selected model accepts. Please select a model which takes longer texts, or reduce the chunk size of the data source."),
+        ProviderRequestFailureReason.EMBEDDINGS_NOT_SUPPORTED => string.Format(TB("The provider '{0}' cannot create embeddings. Please select a provider which offers an embedding model."), this.InstanceName),
+        ProviderRequestFailureReason.INVALID_RESPONSE => string.Format(TB("The provider '{0}' sent an answer AI Studio was not able to read."), this.InstanceName),
         _ => string.Empty,
     };
 
+    /// <summary>
+    /// Builds the failure a provider reports when it offers no embeddings at all.
+    /// </summary>
+    /// <remarks>
+    /// Such a provider used to answer with an empty list, which the caller was not able to tell
+    /// apart from a provider which simply produced nothing this time. Saying it outright is what
+    /// lets the user go and pick a provider which can do the job.
+    /// </remarks>
+    protected ProviderRequestException CreateEmbeddingsNotSupportedException() => new(ProviderRequestFailureReason.EMBEDDINGS_NOT_SUPPORTED,
+        this.GetProviderRequestFailureUserMessage(ProviderRequestFailureReason.EMBEDDINGS_NOT_SUPPORTED));
+
+    /// <summary>
+    /// Builds the failure of an embedding request the provider answered with an error.
+    /// </summary>
+    /// <remarks>
+    /// Shared with the providers which talk to an embedding endpoint of their own: what the user
+    /// needs to know does not depend on which route the request took.
+    /// </remarks>
+    protected ProviderRequestException CreateEmbeddingRequestException(HttpStatusCode statusCode, string reasonPhrase, string responseBody)
+    {
+        var failureReason = this.ClassifyEmbeddingRequestFailure(statusCode, responseBody);
+        var userMessage = this.GetProviderRequestFailureUserMessage(failureReason);
+
+        // We know nothing about this failure, so we pass on what the provider said about it:
+        if (string.IsNullOrWhiteSpace(userMessage))
+        {
+            var providerMessage = ReadProviderErrorMessage(responseBody);
+            userMessage = string.IsNullOrWhiteSpace(providerMessage)
+                ? string.Format(TB("The provider '{0}' rejected the embedding request with the status code {1}."), this.InstanceName, (int)statusCode)
+                : string.Format(TB("The provider '{0}' reported an error: {1}"), this.InstanceName, providerMessage);
+        }
+
+        return new(failureReason, userMessage, statusCode, reasonPhrase, responseBody);
+    }
+
+    /// <summary>
+    /// Builds the failure of an embedding request which did not get an answer at all.
+    /// </summary>
+    /// <param name="exception">What went wrong while the request was on its way.</param>
+    /// <param name="isTimeout">Whether the provider took longer than we were willing to wait.</param>
+    protected ProviderRequestException CreateEmbeddingRequestException(Exception exception, bool isTimeout)
+    {
+        if (isTimeout)
+            return new(ProviderRequestFailureReason.PROVIDER_UNAVAILABLE, this.GetProviderRequestFailureUserMessage(ProviderRequestFailureReason.PROVIDER_UNAVAILABLE), responseBody: exception.Message);
+
+        return new(ProviderRequestFailureReason.UNKNOWN, string.Format(TB("The embedding request to the provider '{0}' failed: {1}"), this.InstanceName, exception.Message), responseBody: exception.Message);
+    }
+
+    /// <summary>
+    /// Classifies why an embedding request failed.
+    /// </summary>
+    /// <remarks>
+    /// Kept apart from the chat classification on purpose. The chat path turns most failures into
+    /// a message and carries on, so classifying more cases there would change what every user
+    /// sees. The embedding path has no such fallback: it either produces vectors or it fails, and
+    /// then the caller has to be able to say why.
+    /// </remarks>
+    private ProviderRequestFailureReason ClassifyEmbeddingRequestFailure(HttpStatusCode statusCode, string responseBody)
+    {
+        //
+        // Whatever the shared classification recognizes wins: it knows what a provider says about
+        // quota and rate limits, and several providers refine it for their own error format.
+        //
+        var sharedFailureReason = this.ClassifyProviderRequestFailure(statusCode, responseBody);
+        if (sharedFailureReason is not ProviderRequestFailureReason.NONE)
+            return sharedFailureReason;
+
+        return statusCode switch
+        {
+            HttpStatusCode.Unauthorized => ProviderRequestFailureReason.INVALID_OR_MISSING_API_KEY,
+            HttpStatusCode.Forbidden => ProviderRequestFailureReason.AUTHENTICATION_OR_PERMISSION_ERROR,
+            HttpStatusCode.NotFound => ProviderRequestFailureReason.MODEL_NOT_FOUND,
+            HttpStatusCode.RequestEntityTooLarge => ProviderRequestFailureReason.CONTEXT_LENGTH_EXCEEDED,
+            HttpStatusCode.BadRequest when IsContextLengthFailure(responseBody) => ProviderRequestFailureReason.CONTEXT_LENGTH_EXCEEDED,
+            HttpStatusCode.RequestTimeout or HttpStatusCode.InternalServerError or HttpStatusCode.BadGateway or HttpStatusCode.ServiceUnavailable or HttpStatusCode.GatewayTimeout => ProviderRequestFailureReason.PROVIDER_UNAVAILABLE,
+            _ => ProviderRequestFailureReason.UNKNOWN,
+        };
+    }
+
+    /// <summary>
+    /// Recognizes the answer a provider gives when the text was longer than the model accepts.
+    /// </summary>
+    /// <remarks>
+    /// There is no common error code for this. What the answers have in common is that they talk
+    /// about the context and about tokens, which is the same hint the chat path goes by.
+    /// </remarks>
+    private static bool IsContextLengthFailure(string responseBody) =>
+        responseBody.Contains("context", StringComparison.InvariantCultureIgnoreCase) &&
+        responseBody.Contains("token", StringComparison.InvariantCultureIgnoreCase);
+
     protected virtual ProviderRequestFailureReason ClassifyProviderRequestFailure(HttpStatusCode statusCode, string responseBody)
     {
         if (statusCode is not HttpStatusCode.TooManyRequests)
@@ -398,7 +498,14 @@ public abstract class BaseProvider : IProvider, ISecretId
     /// </remarks>
     /// <param name="responseBody">The body of the failed response.</param>
     /// <returns>The message, or an empty string when the body carries none.</returns>
-    private static string ReadProviderErrorMessage(string responseBody)
+    /// <summary>
+    /// Reads what the provider itself said about a failure out of its error response.
+    /// </summary>
+    /// <remarks>
+    /// Available to the providers because some of them talk to an endpoint of their own rather
+    /// than through the shared request methods, and their users deserve the same explanation.
+    /// </remarks>
+    protected static string ReadProviderErrorMessage(string responseBody)
     {
         if (string.IsNullOrWhiteSpace(responseBody))
             return string.Empty;
@@ -1336,9 +1443,9 @@ public abstract class BaseProvider : IProvider, ISecretId
                     if(!requestedSecret.Success)
                     {
                         this.logger.LogError("No valid API key available for embedding request.");
-                        return [];
+                        throw new ProviderRequestException(ProviderRequestFailureReason.INVALID_OR_MISSING_API_KEY, this.GetProviderRequestFailureUserMessage(ProviderRequestFailureReason.INVALID_OR_MISSING_API_KEY));
                     }
-                    
+
                     request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await requestedSecret.Secret.Decrypt(Program.ENCRYPTION));
                     break;
             }
@@ -1351,21 +1458,13 @@ public abstract class BaseProvider : IProvider, ISecretId
             if (!response.IsSuccessStatusCode)
             {
                 this.logger.LogError("Embedding request failed with status code {ResponseStatusCode} and body: '{ResponseBody}'.", response.StatusCode, responseBody);
-                var providerRequestFailure = this.ClassifyProviderRequestFailure(response.StatusCode, responseBody);
-                var userMessage = this.GetProviderRequestFailureUserMessage(providerRequestFailure);
 
-                // We know nothing about this failure, so we pass on what the provider said about it:
-                if (string.IsNullOrWhiteSpace(userMessage))
-                {
-                    var providerMessage = ReadProviderErrorMessage(responseBody);
-                    if (!string.IsNullOrWhiteSpace(providerMessage))
-                        userMessage = string.Format(TB("The provider '{0}' reported an error: {1}"), this.InstanceName, providerMessage);
-                }
-
-                if (!string.IsNullOrWhiteSpace(userMessage))
-                    await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.CloudOff, userMessage));
-
-                return [];
+                //
+                // Thrown instead of shown: the caller knows whether this is one file out of
+                // thousands being indexed in the background or the one thing the user just asked
+                // for, and only it can decide how often the user should hear about it.
+                //
+                throw this.CreateEmbeddingRequestException(response.StatusCode, response.ReasonPhrase ?? string.Empty, responseBody);
             }
 
             var embeddingResponse = JsonSerializer.Deserialize<EmbeddingResponse>(responseBody, JSON_SERIALIZER_OPTIONS);
@@ -1379,16 +1478,32 @@ public abstract class BaseProvider : IProvider, ISecretId
             else
             {
                 this.logger.LogError("Was not able to deserialize the embedding response.");
-                return [];
+                throw new ProviderRequestException(ProviderRequestFailureReason.INVALID_RESPONSE, this.GetProviderRequestFailureUserMessage(ProviderRequestFailureReason.INVALID_RESPONSE));
             }
         }
+        catch (ProviderRequestException)
+        {
+            // Already classified and carrying its user message. Wrapping it again would only
+            // replace what we know with the fact that something went wrong:
+            throw;
+        }
+        catch (OperationCanceledException) when (token.IsCancellationRequested)
+        {
+            //
+            // The caller stopped the work, e.g. because the user removed the data source while it
+            // was being indexed. That is not a failure of the provider and must not be recorded
+            // as one:
+            //
+            throw;
+        }
         catch (Exception e)
         {
-            if (this.IsTimeoutException(e, token))
+            var isTimeout = this.IsTimeoutException(e, token);
+            if (isTimeout)
                 await this.SendTimeoutError("creating embeddings");
 
             this.logger.LogError("Failed to perform embedding request: '{Message}'.", e.Message);
-            return [];
+            throw this.CreateEmbeddingRequestException(e, isTimeout);
         }
     }
     

app/MindWork AI Studio/Provider/ConfidenceLevelExtensions.cs

@@ -12,6 +12,7 @@ public static class ConfidenceLevelExtensions
         ConfidenceLevel.NONE => TB("No provider selected"),
         
         ConfidenceLevel.UNTRUSTED => TB("Untrusted"),
+        ConfidenceLevel.UNKNOWN => TB("Unknown"),
         ConfidenceLevel.VERY_LOW => TB("Very Low"),
         ConfidenceLevel.LOW => TB("Low"),
         ConfidenceLevel.MODERATE => TB("Moderate"),
@@ -24,6 +25,9 @@ public static class ConfidenceLevelExtensions
     public static string GetColor(this ConfidenceLevel level, SettingsManager settingsManager) => (level, settingsManager.IsDarkMode) switch
     {
         (ConfidenceLevel.NONE, _) => "#cccccc",
+
+        (ConfidenceLevel.UNKNOWN, false) => "#777777",
+        (ConfidenceLevel.UNKNOWN, true) => "#aaaaaa",
         
         (ConfidenceLevel.UNTRUSTED, false) => "#ff0000",
         (ConfidenceLevel.UNTRUSTED, true) => "#800000",

app/MindWork AI Studio/Provider/DeepSeek/ProviderDeepSeek.cs

@@ -69,7 +69,7 @@ public sealed class ProviderDeepSeek() : BaseProvider(LLMProviders.DEEP_SEEK, ne
     /// <inhertidoc />
     public override Task<IReadOnlyList<IReadOnlyList<float>>> EmbedTextAsync(Model embeddingModel, SettingsManager settingsManager, CancellationToken token = default, params List<string> texts)
     {
-        return Task.FromResult<IReadOnlyList<IReadOnlyList<float>>>([]);
+        throw this.CreateEmbeddingsNotSupportedException();
     }
 
     /// <inheritdoc />

app/MindWork AI Studio/Provider/Fireworks/ProviderFireworks.cs

@@ -71,7 +71,7 @@ public class ProviderFireworks() : BaseProvider(LLMProviders.FIREWORKS, new Uri(
     /// <inhertidoc />
     public override Task<IReadOnlyList<IReadOnlyList<float>>> EmbedTextAsync(Model embeddingModel, SettingsManager settingsManager, CancellationToken token = default, params List<string> texts)
     {
-        return Task.FromResult<IReadOnlyList<IReadOnlyList<float>>>([]);
+        throw this.CreateEmbeddingsNotSupportedException();
     }
 
     /// <inheritdoc />

app/MindWork AI Studio/Provider/Google/ProviderGoogle.cs

@@ -79,16 +79,16 @@ public class ProviderGoogle() : BaseProvider(LLMProviders.GOOGLE, new Uri("https
             if (string.IsNullOrWhiteSpace(modelName))
             {
                 LOGGER.LogError("No model name provided for embedding request.");
-                return [];
+                throw new ProviderRequestException(ProviderRequestFailureReason.MODEL_NOT_FOUND, this.GetProviderRequestFailureUserMessage(ProviderRequestFailureReason.MODEL_NOT_FOUND));
             }
 
             if (modelName.StartsWith("models/", StringComparison.OrdinalIgnoreCase))
-                modelName = modelName.Substring("models/".Length);
+                modelName = modelName["models/".Length..];
 
             if (!requestedSecret.Success)
             {
                 LOGGER.LogError("No valid API key available for embedding request.");
-                return [];
+                throw new ProviderRequestException(ProviderRequestFailureReason.INVALID_OR_MISSING_API_KEY, this.GetProviderRequestFailureUserMessage(ProviderRequestFailureReason.INVALID_OR_MISSING_API_KEY));
             }
             
             // Prepare the Google Gemini embedding request:
@@ -116,7 +116,7 @@ public class ProviderGoogle() : BaseProvider(LLMProviders.GOOGLE, new Uri("https
             if (!response.IsSuccessStatusCode)
             {
                 LOGGER.LogError("Embedding request failed with status code {ResponseStatusCode} and body: '{ResponseBody}'.", response.StatusCode, responseBody);
-                return [];
+                throw this.CreateEmbeddingRequestException(response.StatusCode, response.ReasonPhrase ?? string.Empty, responseBody);
             }
 
             var embeddingResponse = JsonSerializer.Deserialize<GoogleEmbeddingResponse>(responseBody, JSON_SERIALIZER_OPTIONS);
@@ -130,17 +130,33 @@ public class ProviderGoogle() : BaseProvider(LLMProviders.GOOGLE, new Uri("https
             else
             {
                 LOGGER.LogError("Was not able to deserialize the embedding response.");
-                return [];
+                throw new ProviderRequestException(ProviderRequestFailureReason.INVALID_RESPONSE, this.GetProviderRequestFailureUserMessage(ProviderRequestFailureReason.INVALID_RESPONSE));
             }
-            
+
+        }
+        catch (ProviderRequestException)
+        {
+            // Already classified and carrying its user message. Wrapping it again would only
+            // replace what we know with the fact that something went wrong:
+            throw;
+        }
+        catch (OperationCanceledException) when (token.IsCancellationRequested)
+        {
+            //
+            // The caller stopped the work, e.g. because the user removed the data source while it
+            // was being indexed. That is not a failure of the provider and must not be recorded
+            // as one:
+            //
+            throw;
         }
         catch (Exception e)
         {
-            if (this.IsTimeoutException(e, token))
+            var isTimeout = this.IsTimeoutException(e, token);
+            if (isTimeout)
                 await this.SendTimeoutError("creating embeddings");
 
             LOGGER.LogError("Failed to perform embedding request: '{Message}'.", e.Message);
-            return [];
+            throw this.CreateEmbeddingRequestException(e, isTimeout);
         }
     }
 

app/MindWork AI Studio/Provider/Groq/ProviderGroq.cs

@@ -74,7 +74,7 @@ public class ProviderGroq() : BaseProvider(LLMProviders.GROQ, new Uri("https://a
     /// <inhertidoc />
     public override Task<IReadOnlyList<IReadOnlyList<float>>> EmbedTextAsync(Model embeddingModel, SettingsManager settingsManager, CancellationToken token = default, params List<string> texts)
     {
-        return Task.FromResult<IReadOnlyList<IReadOnlyList<float>>>([]);
+        throw this.CreateEmbeddingsNotSupportedException();
     }
 
     /// <inheritdoc />

app/MindWork AI Studio/Provider/Hetzner/ProviderHetzner.cs

@@ -63,7 +63,7 @@ public sealed class ProviderHetzner() : BaseProvider(LLMProviders.HETZNER, new U
     /// <inheritdoc />
     public override Task<IReadOnlyList<IReadOnlyList<float>>> EmbedTextAsync(Model embeddingModel, SettingsManager settingsManager, CancellationToken token = default, params List<string> texts)
     {
-        return Task.FromResult<IReadOnlyList<IReadOnlyList<float>>>([]);
+        throw this.CreateEmbeddingsNotSupportedException();
     }
 
     /// <inheritdoc />

app/MindWork AI Studio/Provider/IProvider.cs

@@ -34,6 +34,11 @@ public interface IProvider
     /// </summary>
     public string AdditionalJsonApiParameters { get; }
 
+    /// <summary>
+    /// The tokenizer path associated with this provider configuration.
+    /// </summary>
+    public string TokenizerPath { get; }
+
     /// <summary>
     /// Whether this provider instance can load available models from the backend/API.
     /// This capability may differ by provider type, host, or modality.

app/MindWork AI Studio/Provider/LLMProvidersExtensions.cs

@@ -278,7 +278,7 @@ public static class LLMProvidersExtensions
     /// <returns>The provider instance.</returns>
     public static IProvider CreateProvider(this AIStudio.Settings.Provider providerSettings)
     {
-        return providerSettings.UsedLLMProvider.CreateProvider(providerSettings.InstanceName, providerSettings.Host, providerSettings.Hostname, providerSettings.HFInferenceProvider, providerSettings.Id, providerSettings.AdditionalJsonApiParameters, providerSettings.IsEnterpriseConfiguration, capabilityOverrides: providerSettings.CapabilityOverrides);
+        return providerSettings.UsedLLMProvider.CreateProvider(providerSettings.InstanceName, providerSettings.Host, providerSettings.Hostname, providerSettings.HFInferenceProvider, providerSettings.Id, providerSettings.AdditionalJsonApiParameters, providerSettings.IsEnterpriseConfiguration, capabilityOverrides: providerSettings.CapabilityOverrides, tokenizerPath: providerSettings.TokenizerPath);
     }
     
     /// <summary>
@@ -288,7 +288,7 @@ public static class LLMProvidersExtensions
     /// <returns>The provider instance.</returns>
     public static IProvider CreateProvider(this EmbeddingProvider embeddingProviderSettings)
     {
-        return embeddingProviderSettings.UsedLLMProvider.CreateProvider(embeddingProviderSettings.Name, embeddingProviderSettings.Host, embeddingProviderSettings.Hostname, embeddingProviderSettings.HFInferenceProvider, configuredProviderId: embeddingProviderSettings.Id, isEnterpriseConfiguration: embeddingProviderSettings.IsEnterpriseConfiguration, hfEndpointKind: HFEndpointKind.EMBEDDING);
+        return embeddingProviderSettings.UsedLLMProvider.CreateProvider(embeddingProviderSettings.Name, embeddingProviderSettings.Host, embeddingProviderSettings.Hostname, embeddingProviderSettings.HFInferenceProvider, configuredProviderId: embeddingProviderSettings.Id, isEnterpriseConfiguration: embeddingProviderSettings.IsEnterpriseConfiguration, hfEndpointKind: HFEndpointKind.EMBEDDING, tokenizerPath: embeddingProviderSettings.TokenizerPath);
     }
     
     /// <summary>
@@ -300,34 +300,34 @@ public static class LLMProvidersExtensions
     {
         return transcriptionProviderSettings.UsedLLMProvider.CreateProvider(transcriptionProviderSettings.Name, transcriptionProviderSettings.Host, transcriptionProviderSettings.Hostname, transcriptionProviderSettings.HFInferenceProvider, configuredProviderId: transcriptionProviderSettings.Id, isEnterpriseConfiguration: transcriptionProviderSettings.IsEnterpriseConfiguration, hfEndpointKind: HFEndpointKind.TRANSCRIPTION);
     }
-    
-    private static IProvider CreateProvider(this LLMProviders provider, string instanceName, Host host, string hostname, HFInferenceProvider inferenceProvider, string configuredProviderId = "", string expertProviderApiParameter = "", bool isEnterpriseConfiguration = false, HFEndpointKind hfEndpointKind = HFEndpointKind.CHAT, ProviderCapabilityOverrides? capabilityOverrides = null)
+
+    private static IProvider CreateProvider(this LLMProviders provider, string instanceName, Host host, string hostname, HFInferenceProvider inferenceProvider, string configuredProviderId = "", string expertProviderApiParameter = "", bool isEnterpriseConfiguration = false, HFEndpointKind hfEndpointKind = HFEndpointKind.CHAT, ProviderCapabilityOverrides? capabilityOverrides = null, string tokenizerPath = "")
     {
         try
         {
             IProvider providerInstance = provider switch
             {
-                LLMProviders.OPEN_AI => new ProviderOpenAI { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, IsEnterpriseConfiguration = isEnterpriseConfiguration },
-                LLMProviders.ANTHROPIC => new ProviderAnthropic { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, IsEnterpriseConfiguration = isEnterpriseConfiguration },
-                LLMProviders.MISTRAL => new ProviderMistral { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, IsEnterpriseConfiguration = isEnterpriseConfiguration },
-                LLMProviders.GOOGLE => new ProviderGoogle { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, IsEnterpriseConfiguration = isEnterpriseConfiguration },
-                LLMProviders.X => new ProviderX { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, IsEnterpriseConfiguration = isEnterpriseConfiguration },
-                LLMProviders.DEEP_SEEK => new ProviderDeepSeek { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, IsEnterpriseConfiguration = isEnterpriseConfiguration },
-                LLMProviders.ALIBABA_CLOUD => new ProviderAlibabaCloud { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, IsEnterpriseConfiguration = isEnterpriseConfiguration },
-                LLMProviders.PERPLEXITY => new ProviderPerplexity { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, IsEnterpriseConfiguration = isEnterpriseConfiguration },
-                LLMProviders.OPEN_ROUTER => new ProviderOpenRouter { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, IsEnterpriseConfiguration = isEnterpriseConfiguration },
-                LLMProviders.HETZNER => new ProviderHetzner { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, IsEnterpriseConfiguration = isEnterpriseConfiguration },
-                LLMProviders.IONOS => new ProviderIONOS { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, IsEnterpriseConfiguration = isEnterpriseConfiguration },
-                LLMProviders.LITE_LLM => new ProviderLiteLLM(hostname) { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, IsEnterpriseConfiguration = isEnterpriseConfiguration },
+                LLMProviders.OPEN_AI => new ProviderOpenAI { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, TokenizerPath = tokenizerPath, IsEnterpriseConfiguration = isEnterpriseConfiguration },
+                LLMProviders.ANTHROPIC => new ProviderAnthropic { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, TokenizerPath = tokenizerPath, IsEnterpriseConfiguration = isEnterpriseConfiguration },
+                LLMProviders.MISTRAL => new ProviderMistral { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, TokenizerPath = tokenizerPath, IsEnterpriseConfiguration = isEnterpriseConfiguration },
+                LLMProviders.GOOGLE => new ProviderGoogle { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, TokenizerPath = tokenizerPath, IsEnterpriseConfiguration = isEnterpriseConfiguration },
+                LLMProviders.X => new ProviderX { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, TokenizerPath = tokenizerPath, IsEnterpriseConfiguration = isEnterpriseConfiguration },
+                LLMProviders.DEEP_SEEK => new ProviderDeepSeek { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, TokenizerPath = tokenizerPath, IsEnterpriseConfiguration = isEnterpriseConfiguration },
+                LLMProviders.ALIBABA_CLOUD => new ProviderAlibabaCloud { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, TokenizerPath = tokenizerPath, IsEnterpriseConfiguration = isEnterpriseConfiguration },
+                LLMProviders.PERPLEXITY => new ProviderPerplexity { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, TokenizerPath = tokenizerPath, IsEnterpriseConfiguration = isEnterpriseConfiguration },
+                LLMProviders.OPEN_ROUTER => new ProviderOpenRouter { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, TokenizerPath = tokenizerPath, IsEnterpriseConfiguration = isEnterpriseConfiguration },
+                LLMProviders.HETZNER => new ProviderHetzner { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, TokenizerPath = tokenizerPath, IsEnterpriseConfiguration = isEnterpriseConfiguration },
+                LLMProviders.IONOS => new ProviderIONOS { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, TokenizerPath = tokenizerPath, IsEnterpriseConfiguration = isEnterpriseConfiguration },
+                LLMProviders.LITE_LLM => new ProviderLiteLLM(hostname) { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, TokenizerPath = tokenizerPath, IsEnterpriseConfiguration = isEnterpriseConfiguration },
 
-                LLMProviders.GROQ => new ProviderGroq { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, IsEnterpriseConfiguration = isEnterpriseConfiguration },
-                LLMProviders.FIREWORKS => new ProviderFireworks { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, IsEnterpriseConfiguration = isEnterpriseConfiguration },
-                LLMProviders.HUGGINGFACE => new ProviderHuggingFace(inferenceProvider, hfEndpointKind) { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, IsEnterpriseConfiguration = isEnterpriseConfiguration },
+                LLMProviders.GROQ => new ProviderGroq { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, TokenizerPath = tokenizerPath, IsEnterpriseConfiguration = isEnterpriseConfiguration },
+                LLMProviders.FIREWORKS => new ProviderFireworks { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, TokenizerPath = tokenizerPath, IsEnterpriseConfiguration = isEnterpriseConfiguration },
+                LLMProviders.HUGGINGFACE => new ProviderHuggingFace(inferenceProvider, hfEndpointKind) { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, TokenizerPath = tokenizerPath, IsEnterpriseConfiguration = isEnterpriseConfiguration },
 
-                LLMProviders.SELF_HOSTED => new ProviderSelfHosted(host, hostname) { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, IsEnterpriseConfiguration = isEnterpriseConfiguration },
+                LLMProviders.SELF_HOSTED => new ProviderSelfHosted(host, hostname) { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, TokenizerPath = tokenizerPath, IsEnterpriseConfiguration = isEnterpriseConfiguration },
 
-                LLMProviders.HELMHOLTZ => new ProviderHelmholtz { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, IsEnterpriseConfiguration = isEnterpriseConfiguration },
-                LLMProviders.GWDG => new ProviderGWDG { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, IsEnterpriseConfiguration = isEnterpriseConfiguration },
+                LLMProviders.HELMHOLTZ => new ProviderHelmholtz { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, TokenizerPath = tokenizerPath, IsEnterpriseConfiguration = isEnterpriseConfiguration },
+                LLMProviders.GWDG => new ProviderGWDG { InstanceName = instanceName, ConfiguredProviderId = configuredProviderId, AdditionalJsonApiParameters = expertProviderApiParameter, TokenizerPath = tokenizerPath, IsEnterpriseConfiguration = isEnterpriseConfiguration },
 
                 _ => new NoProvider(),
             };

app/MindWork AI Studio/Provider/NoProvider.cs

@@ -21,6 +21,8 @@ public class NoProvider : IProvider
     public string AdditionalJsonApiParameters { get; init; } = string.Empty;
 
     /// <inheritdoc />
+    public string TokenizerPath { get; init; } = string.Empty;
+
     public bool HasModelLoadingCapability => false;
 
     public Task<ModelLoadResult> GetTextModels(string? apiKeyProvisional = null, CancellationToken token = default) => Task.FromResult(ModelLoadResult.FromModels([]));

app/MindWork AI Studio/Provider/Perplexity/ProviderPerplexity.cs

@@ -77,7 +77,7 @@ public sealed class ProviderPerplexity() : BaseProvider(LLMProviders.PERPLEXITY,
     /// <inhertidoc />
     public override Task<IReadOnlyList<IReadOnlyList<float>>> EmbedTextAsync(Model embeddingModel, SettingsManager settingsManager, CancellationToken token = default, params List<string> texts)
     {
-        return Task.FromResult<IReadOnlyList<IReadOnlyList<float>>>([]);
+        throw this.CreateEmbeddingsNotSupportedException();
     }
 
     /// <inheritdoc />

app/MindWork AI Studio/Provider/ProviderRequestFailureReason.cs

@@ -14,4 +14,56 @@ public enum ProviderRequestFailureReason
     /// meant to answer for it does not offer it.
     /// </remarks>
     MODEL_NOT_SUPPORTED_BY_PROVIDER,
+
+    /// <summary>
+    /// No usable API key was available, or the provider rejected the one we sent.
+    /// </summary>
+    /// <remarks>
+    /// Both cases lead to the same place for the user: the key stored for this provider is not
+    /// one the provider works with, and the settings are where they fix it.
+    /// </remarks>
+    INVALID_OR_MISSING_API_KEY,
+
+    /// <summary>
+    /// The key was accepted, but the account is not allowed to do what we asked for.
+    /// </summary>
+    /// <remarks>
+    /// Typical causes are a key without the required scope, a model the account has no access
+    /// to, and providers which refuse requests from the user's region.
+    /// </remarks>
+    AUTHENTICATION_OR_PERMISSION_ERROR,
+
+    /// <summary>
+    /// The provider could not be reached, or said that it cannot serve requests right now.
+    /// </summary>
+    PROVIDER_UNAVAILABLE,
+
+    /// <summary>
+    /// The provider does not know the requested model at all.
+    /// </summary>
+    MODEL_NOT_FOUND,
+
+    /// <summary>
+    /// The text we sent was longer than the model accepts.
+    /// </summary>
+    CONTEXT_LENGTH_EXCEEDED,
+
+    /// <summary>
+    /// The provider cannot create embeddings at all.
+    /// </summary>
+    EMBEDDINGS_NOT_SUPPORTED,
+
+    /// <summary>
+    /// The provider answered successfully, but with something we were not able to read.
+    /// </summary>
+    INVALID_RESPONSE,
+
+    /// <summary>
+    /// The request failed and we were not able to tell why.
+    /// </summary>
+    /// <remarks>
+    /// Deliberately without a user message of its own: what the provider itself said about the
+    /// failure tells the user more than a sentence which says nothing.
+    /// </remarks>
+    UNKNOWN,
 }

app/MindWork AI Studio/Provider/ProviderRequestFailureReasonExtensions.cs

@@ -0,0 +1,47 @@
+using AIStudio.Tools.PluginSystem;
+
+namespace AIStudio.Provider;
+
+public static class ProviderRequestFailureReasonExtensions
+{
+    private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(ProviderRequestFailureReasonExtensions).Namespace, nameof(ProviderRequestFailureReasonExtensions));
+
+    /// <summary>
+    /// Names the kind of failure in a few words.
+    /// </summary>
+    /// <remarks>
+    /// Meant as a label beside the full message, so a list of failures can be scanned instead of
+    /// read: twenty entries which all say API key are one problem, not twenty.
+    /// </remarks>
+    public static string GetName(this ProviderRequestFailureReason failureReason) => failureReason switch
+    {
+        ProviderRequestFailureReason.INSUFFICIENT_QUOTA => TB("No credits left"),
+        ProviderRequestFailureReason.TOO_MANY_REQUESTS => TB("Too many requests"),
+        ProviderRequestFailureReason.MODEL_NOT_SUPPORTED_BY_PROVIDER => TB("Model not offered"),
+        ProviderRequestFailureReason.INVALID_OR_MISSING_API_KEY => TB("API key problem"),
+        ProviderRequestFailureReason.AUTHENTICATION_OR_PERMISSION_ERROR => TB("Not permitted"),
+        ProviderRequestFailureReason.PROVIDER_UNAVAILABLE => TB("Provider unreachable"),
+        ProviderRequestFailureReason.MODEL_NOT_FOUND => TB("Model unknown"),
+        ProviderRequestFailureReason.CONTEXT_LENGTH_EXCEEDED => TB("Text too long"),
+        ProviderRequestFailureReason.EMBEDDINGS_NOT_SUPPORTED => TB("No embeddings"),
+        ProviderRequestFailureReason.INVALID_RESPONSE => TB("Unreadable answer"),
+        ProviderRequestFailureReason.UNKNOWN => TB("Unknown cause"),
+
+        _ => string.Empty,
+    };
+
+    /// <summary>
+    /// Gets a value indicating whether the way out of this failure is in the provider settings.
+    /// </summary>
+    /// <remarks>
+    /// Only for the failures a setting actually fixes. Pointing at the settings for a provider
+    /// which is merely overloaded would send the user looking for a mistake they never made.
+    /// </remarks>
+    public static bool IsFixedInProviderSettings(this ProviderRequestFailureReason failureReason) => failureReason is
+        ProviderRequestFailureReason.INVALID_OR_MISSING_API_KEY or
+        ProviderRequestFailureReason.AUTHENTICATION_OR_PERMISSION_ERROR or
+        ProviderRequestFailureReason.MODEL_NOT_FOUND or
+        ProviderRequestFailureReason.MODEL_NOT_SUPPORTED_BY_PROVIDER or
+        ProviderRequestFailureReason.EMBEDDINGS_NOT_SUPPORTED or
+        ProviderRequestFailureReason.CONTEXT_LENGTH_EXCEEDED;
+}

app/MindWork AI Studio/Provider/X/ProviderX.cs

@@ -70,7 +70,7 @@ public sealed class ProviderX() : BaseProvider(LLMProviders.X, new Uri("https://
     /// <inhertidoc />
     public override Task<IReadOnlyList<IReadOnlyList<float>>> EmbedTextAsync(Model embeddingModel, SettingsManager settingsManager, CancellationToken token = default, params List<string> texts)
     {
-        return Task.FromResult<IReadOnlyList<IReadOnlyList<float>>>([]);
+        throw this.CreateEmbeddingsNotSupportedException();
     }
     
     /// <inheritdoc />

app/MindWork AI Studio/Routes.razor.cs

@@ -4,6 +4,7 @@ public sealed partial class Routes
 {
     public const string HOME = "/";
     public const string CHAT = "/chat";
+    public const string EMBEDDINGS = "/embeddings";
     public const string ABOUT = "/about";
     public const string ASSISTANTS = "/assistants";
     public const string SETTINGS = "/settings";

app/MindWork AI Studio/Settings/ConfigurationSelectDataFactory.cs

@@ -295,6 +295,17 @@ public static class ConfigurationSelectDataFactory
             }
         }
     }
+
+    public static IEnumerable<ConfigurationSelectData<ConfidenceLevel>> GetDataSourceConfidenceLevelsData()
+    {
+        foreach (var level in Enum.GetValues<ConfidenceLevel>())
+        {
+            if (level is ConfidenceLevel.NONE)
+                continue;
+
+            yield return new(level.GetName(), level);
+        }
+    }
     
     public static IEnumerable<ConfigurationSelectData<Themes>> GetThemesData()
     {

app/MindWork AI Studio/Settings/DataModel/DataApp.cs

@@ -179,6 +179,11 @@ public sealed class DataApp(Expression<Func<Data, DataApp>>? configSelection = n
     /// </summary>
     public bool ShowAdminSettings { get; set; } = ManagedConfiguration.Register(configSelection, n => n.ShowAdminSettings, false);
 
+    /// <summary>
+    /// Settings for indexing local data sources.
+    /// </summary>
+    public DataDataSourceIndexing DataSourceIndexing { get; init; } = new();
+
     /// <summary>
     /// List of assistants that should be hidden from the UI.
     /// </summary>

app/MindWork AI Studio/Settings/DataModel/DataDataSourceIndexing.cs

@@ -0,0 +1,9 @@
+namespace AIStudio.Settings.DataModel;
+
+public sealed class DataDataSourceIndexing
+{
+    /// <summary>
+    /// Whether local data source embeddings should refresh automatically when files change.
+    /// </summary>
+    public bool AutomaticRefresh { get; set; } = true;
+}

app/MindWork AI Studio/Settings/DataModel/DataSourceERI_V1.cs

@@ -7,6 +7,7 @@ using AIStudio.Tools.ERIClient.DataModel;
 using AIStudio.Tools.PluginSystem;
 using AIStudio.Tools.RAG;
 using AIStudio.Tools.Services;
+using AIStudio.Tools.Validation;
 
 using SharedTools;
 
@@ -164,9 +165,9 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
             return false;
         }
 
-        if (!table.TryGetValue("Name", out var nameValue) || !nameValue.TryRead<string>(out var name) || string.IsNullOrWhiteSpace(name))
+        if (!table.TryGetValue("Name", out var nameValue) || !nameValue.TryRead<string>(out var name) || !DataSourceValidation.IsNameValid(name))
         {
-            LOGGER.LogWarning($"The configured data source {idx} does not contain a valid name. (Plugin ID: {configPluginId})");
+            LOGGER.LogWarning($"The configured data source {idx} does not contain a valid name of at most {DataSourceValidation.MAX_NAME_LENGTH} characters without control characters. (Plugin ID: {configPluginId})");
             return false;
         }
 
@@ -390,4 +391,4 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
         var cleanedHostname = hostname.Trim();
         return cleanedHostname.EndsWith('/') ? cleanedHostname[..^1] : cleanedHostname;
     }
-}
+}

app/MindWork AI Studio/Settings/DataModel/DataSourceLocalDirectory.cs

@@ -1,5 +1,7 @@
 using AIStudio.Chat;
+using AIStudio.Provider;
 using AIStudio.Tools.RAG;
+using AIStudio.Tools.Services;
 
 namespace AIStudio.Settings.DataModel;
 
@@ -32,9 +34,15 @@ public readonly record struct DataSourceLocalDirectory : IInternalDataSource
     
     /// <inheritdoc />
     public string EmbeddingId { get; init; } = Guid.Empty.ToString();
+
+    /// <inheritdoc />
+    public int MaxChunkTokenLength { get; init; }
+
+    /// <inheritdoc />
+    public int ChunkOverlapTokenLength { get; init; } = DataSourceEmbeddingService.DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH;
     
     /// <inheritdoc />
-    public DataSourceSecurity SecurityPolicy { get; init; } = DataSourceSecurity.NOT_SPECIFIED;
+    public ConfidenceLevel ConfidenceLevel { get; init; } = ConfidenceLevel.UNKNOWN;
 
     /// <inheritdoc />
     public bool IsEnterpriseConfiguration { get; init; }
@@ -46,11 +54,8 @@ public readonly record struct DataSourceLocalDirectory : IInternalDataSource
     public ushort MaxMatches { get; init; } = 10;
     
     /// <inheritdoc />
-    public Task<IReadOnlyList<IRetrievalContext>> RetrieveDataAsync(IContent lastUserPrompt, ChatThread thread, CancellationToken token = default)
-    {
-        IReadOnlyList<IRetrievalContext> retrievalContext = new List<IRetrievalContext>();
-        return Task.FromResult(retrievalContext);
-    }
+    public Task<IReadOnlyList<IRetrievalContext>> RetrieveDataAsync(IContent lastUserPrompt, ChatThread thread, CancellationToken token = default) =>
+        Program.SERVICE_PROVIDER.GetRequiredService<DataSourceLocalRetrievalService>().RetrieveDataAsync(this, lastUserPrompt, thread, token);
     
     /// <summary>
     /// The path to the directory.

app/MindWork AI Studio/Settings/DataModel/DataSourceLocalFile.cs

@@ -1,5 +1,7 @@
 using AIStudio.Chat;
+using AIStudio.Provider;
 using AIStudio.Tools.RAG;
+using AIStudio.Tools.Services;
 
 namespace AIStudio.Settings.DataModel;
 
@@ -32,9 +34,15 @@ public readonly record struct DataSourceLocalFile : IInternalDataSource
     
     /// <inheritdoc />
     public string EmbeddingId { get; init; } = Guid.Empty.ToString();
+
+    /// <inheritdoc />
+    public int MaxChunkTokenLength { get; init; }
+
+    /// <inheritdoc />
+    public int ChunkOverlapTokenLength { get; init; } = DataSourceEmbeddingService.DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH;
     
     /// <inheritdoc />
-    public DataSourceSecurity SecurityPolicy { get; init; } = DataSourceSecurity.NOT_SPECIFIED;
+    public ConfidenceLevel ConfidenceLevel { get; init; } = ConfidenceLevel.UNKNOWN;
 
     /// <inheritdoc />
     public bool IsEnterpriseConfiguration { get; init; }
@@ -46,11 +54,8 @@ public readonly record struct DataSourceLocalFile : IInternalDataSource
     public ushort MaxMatches { get; init; } = 10;
     
     /// <inheritdoc />
-    public Task<IReadOnlyList<IRetrievalContext>> RetrieveDataAsync(IContent lastUserPrompt, ChatThread thread, CancellationToken token = default)
-    {
-        IReadOnlyList<IRetrievalContext> retrievalContext = new List<IRetrievalContext>();
-        return Task.FromResult(retrievalContext);
-    }
+    public Task<IReadOnlyList<IRetrievalContext>> RetrieveDataAsync(IContent lastUserPrompt, ChatThread thread, CancellationToken token = default) =>
+        Program.SERVICE_PROVIDER.GetRequiredService<DataSourceLocalRetrievalService>().RetrieveDataAsync(this, lastUserPrompt, thread, token);
     
     /// <summary>
     /// The path to the file.

app/MindWork AI Studio/Settings/DataModel/PreviewVisibilityExtensions.cs

@@ -13,6 +13,7 @@ public static class PreviewVisibilityExtensions
         {
             features.Add(PreviewFeatures.PRE_DOCUMENT_ANALYSIS_2025);
             features.Add(PreviewFeatures.PRE_META_ASSISTANT_V1);
+            features.Add(PreviewFeatures.PRE_RAG_2024);
         }
         
         if (visibility >= PreviewVisibility.ALPHA)
@@ -21,7 +22,6 @@ public static class PreviewVisibilityExtensions
         
         if (visibility >= PreviewVisibility.PROTOTYPE)
         {
-            features.Add(PreviewFeatures.PRE_RAG_2024);
             features.Add(PreviewFeatures.PRE_VISUAL_BRIEFING_ASSISTANT_2026);
         }
         

app/MindWork AI Studio/Settings/DataSourceSecurityTrustExtensions.cs

@@ -1,4 +1,5 @@
 using AIStudio.Provider;
+using AIStudio.Settings.DataModel;
 
 namespace AIStudio.Settings;
 
@@ -36,6 +37,88 @@ public static class DataSourceSecurityTrustExtensions
         return provider.Provider is LLMProviders.SELF_HOSTED || IsTrustedProviderId(provider.ConfiguredProviderId, settingsManager);
     }
 
+    public static ConfidenceLevel GetConfidenceLevel(this Provider provider, SettingsManager settingsManager)
+    {
+        if (provider == Provider.NONE)
+            return ConfidenceLevel.NONE;
+
+        return provider.UsedLLMProvider.GetConfidence(settingsManager).Level;
+    }
+
+    public static ConfidenceLevel GetConfidenceLevel(this EmbeddingProvider provider, SettingsManager settingsManager)
+    {
+        if (provider == EmbeddingProvider.NONE)
+            return ConfidenceLevel.NONE;
+
+        return provider.UsedLLMProvider.GetConfidence(settingsManager).Level;
+    }
+
+    public static ConfidenceLevel GetConfidenceLevel(this IProvider provider, SettingsManager settingsManager)
+    {
+        if (provider is NoProvider)
+            return ConfidenceLevel.NONE;
+
+        return provider.Provider.GetConfidence(settingsManager).Level;
+    }
+
+    public static bool AllowsDataSourceAccess(this Provider provider, SettingsManager settingsManager, DataSourceSecurity dataSourceSecurity, ConfidenceLevel requiredConfidenceLevel)
+    {
+        return provider.AllowsDataSourceSecurity(dataSourceSecurity, settingsManager)
+            && provider.GetConfidenceLevel(settingsManager).AllowsDataSourceConfidenceLevel(requiredConfidenceLevel);
+    }
+
+    public static bool AllowsDataSourceAccess(this IProvider provider, SettingsManager settingsManager, DataSourceSecurity dataSourceSecurity, ConfidenceLevel requiredConfidenceLevel)
+    {
+        return provider.AllowsDataSourceSecurity(dataSourceSecurity, settingsManager)
+            && provider.GetConfidenceLevel(settingsManager).AllowsDataSourceConfidenceLevel(requiredConfidenceLevel);
+    }
+
+    public static bool AllowsDataSourceSecurity(this Provider provider, DataSourceSecurity dataSourceSecurity, SettingsManager settingsManager)
+        => provider.IsTrustedForDataSourceSecurityChecks(settingsManager).AllowsDataSourceSecurity(dataSourceSecurity);
+
+    public static bool AllowsDataSourceSecurity(this IProvider provider, DataSourceSecurity dataSourceSecurity, SettingsManager settingsManager)
+        => provider.IsTrustedForDataSourceSecurityChecks(settingsManager).AllowsDataSourceSecurity(dataSourceSecurity);
+
+    public static bool AllowsDataSourceSecurity(this bool usingTrustedProvider, DataSourceSecurity dataSourceSecurity) => dataSourceSecurity switch
+    {
+        DataSourceSecurity.ALLOW_ANY => true,
+        DataSourceSecurity.SELF_HOSTED => usingTrustedProvider,
+        _ => false,
+    };
+
+    public static bool AllowsDataSourceConfidenceLevel(this ConfidenceLevel providerConfidenceLevel, ConfidenceLevel requiredConfidenceLevel)
+    {
+        if (requiredConfidenceLevel is ConfidenceLevel.NONE)
+            return true;
+
+        return providerConfidenceLevel >= requiredConfidenceLevel;
+    }
+
+    public static ConfidenceLevel GetRequiredConfidenceLevel(this IEnumerable<IDataSource> dataSources)
+    {
+        var requiredConfidenceLevel = ConfidenceLevel.NONE;
+        foreach (var dataSource in dataSources.OfType<IInternalDataSource>())
+            if (dataSource.ConfidenceLevel > requiredConfidenceLevel)
+                requiredConfidenceLevel = dataSource.ConfidenceLevel;
+
+        return requiredConfidenceLevel;
+    }
+
+    public static DataSourceSecurity GetRequiredSecurityPolicy(this IEnumerable<IDataSource> dataSources)
+    {
+        var requiredSecurityPolicy = DataSourceSecurity.ALLOW_ANY;
+        foreach (var dataSource in dataSources.OfType<IExternalDataSource>())
+        {
+            if (dataSource.SecurityPolicy is DataSourceSecurity.NOT_SPECIFIED)
+                return DataSourceSecurity.NOT_SPECIFIED;
+
+            if (dataSource.SecurityPolicy is DataSourceSecurity.SELF_HOSTED)
+                requiredSecurityPolicy = DataSourceSecurity.SELF_HOSTED;
+        }
+
+        return requiredSecurityPolicy;
+    }
+
     public static bool IsTrustedByConfiguration(this Provider provider, SettingsManager settingsManager) => IsTrustedProviderId(provider.Id, settingsManager);
 
     public static bool IsTrustedByConfiguration(this EmbeddingProvider provider, SettingsManager settingsManager) => IsTrustedProviderId(provider.Id, settingsManager);

app/MindWork AI Studio/Settings/EmbeddingProvider.cs

@@ -22,23 +22,21 @@ public sealed record EmbeddingProvider(
     Guid EnterpriseConfigurationPluginId = default,
     string Hostname = "http://localhost:1234",
     Host Host = Host.NONE,
+    string TokenizerPath = "",
+    int EmbeddingBatchSize = 0,
+    int TokenLimit = 0,
     bool AllowUserProvidedAPIKey = false,
     string CustomIconDataUrl = "",
     HFInferenceProvider HFInferenceProvider = HFInferenceProvider.NONE) : ConfigurationBaseObject, ISecretId, IUserProvidedAPIKey
 {
+    public const int DEFAULT_TOKEN_LIMIT = 8192;
+    public const int DEFAULT_EMBEDDING_BATCH_SIZE = 1;
+
     private static readonly ILogger<EmbeddingProvider> LOGGER = Program.LOGGER_FACTORY.CreateLogger<EmbeddingProvider>();
 
     public static readonly EmbeddingProvider NONE = new();
 
-    public EmbeddingProvider() : this(
-        0,
-        Guid.Empty.ToString(),
-        string.Empty,
-        LLMProviders.NONE,
-        default,
-        false,
-        false,
-        Guid.Empty)
+    public EmbeddingProvider() : this(0, Guid.Empty.ToString(), string.Empty, LLMProviders.NONE, default, false, false, Guid.Empty)
     {
     }
 
@@ -54,6 +52,12 @@ public sealed record EmbeddingProvider(
     [JsonIgnore]
     public string SecretName => this.Name;
 
+    [JsonIgnore]
+    public int EffectiveTokenLimit => this.TokenLimit > 0 ? this.TokenLimit : DEFAULT_TOKEN_LIMIT;
+
+    [JsonIgnore]
+    public int EffectiveEmbeddingBatchSize => this.EmbeddingBatchSize > 0 ? this.EmbeddingBatchSize : DEFAULT_EMBEDDING_BATCH_SIZE;
+
     #endregion
 
     public static bool TryParseEmbeddingProviderTable(int idx, LuaTable table, Guid configPluginId, string pluginPath, out ConfigurationBaseObject provider)
@@ -101,6 +105,27 @@ public sealed record EmbeddingProvider(
             return false;
         }
 
+        var tokenizerPath = string.Empty;
+        if (table.TryGetValue("TokenizerPath", out var tokenizerPathValue) && !tokenizerPathValue.TryRead<string>(out tokenizerPath))
+        {
+            LOGGER.LogWarning($"The configured embedding provider {idx} does not contain a valid tokenizer path. (Plugin ID: {configPluginId})");
+            tokenizerPath = string.Empty;
+        }
+
+        var tokenLimit = DEFAULT_TOKEN_LIMIT;
+        if (table.TryGetValue("TokenLimit", out var tokenLimitValue) && (!tokenLimitValue.TryRead<int>(out tokenLimit) || tokenLimit < 1))
+        {
+            LOGGER.LogWarning($"The configured embedding provider {idx} does not contain a valid token limit. Falling back to {DEFAULT_TOKEN_LIMIT}. (Plugin ID: {configPluginId})");
+            tokenLimit = DEFAULT_TOKEN_LIMIT;
+        }
+
+        var embeddingBatchSize = DEFAULT_EMBEDDING_BATCH_SIZE;
+        if (table.TryGetValue("EmbeddingBatchSize", out var embeddingBatchSizeValue) && (!embeddingBatchSizeValue.TryRead<int>(out embeddingBatchSize) || embeddingBatchSize < 1))
+        {
+            LOGGER.LogWarning($"The configured embedding provider {idx} does not contain a valid embedding batch size. Falling back to {DEFAULT_EMBEDDING_BATCH_SIZE}. (Plugin ID: {configPluginId})");
+            embeddingBatchSize = DEFAULT_EMBEDDING_BATCH_SIZE;
+        }
+
         var allowUserProvidedApiKey = false;
         if (table.TryGetValue("AllowUserProvidedAPIKey", out var allowUserProvidedApiKeyValue) && allowUserProvidedApiKeyValue.TryRead<bool>(out var allowUserProvidedApiKeyBool))
             allowUserProvidedApiKey = allowUserProvidedApiKeyBool;
@@ -136,6 +161,9 @@ public sealed record EmbeddingProvider(
             EnterpriseConfigurationPluginId = configPluginId,
             Hostname = hostname,
             Host = host,
+            TokenizerPath = tokenizerPath,
+            EmbeddingBatchSize = embeddingBatchSize,
+            TokenLimit = tokenLimit,
             AllowUserProvidedAPIKey = allowUserProvidedApiKey,
             CustomIconDataUrl = customIconDataUrl,
             HFInferenceProvider = hfInferenceProvider,
@@ -227,6 +255,10 @@ public sealed record EmbeddingProvider(
                     ["Name"] = "{{LuaTools.EscapeLuaString(this.Name)}}",
                     ["UsedLLMProvider"] = "{{this.UsedLLMProvider}}",
 
+                    ["TokenizerPath"] = "{{this.TokenizerPath}}",
+                    ["TokenLimit"] = {{this.EffectiveTokenLimit}},
+                    ["EmbeddingBatchSize"] = {{this.EffectiveEmbeddingBatchSize}},
+
                     ["Host"] = "{{this.Host}}",
                     ["Hostname"] = "{{LuaTools.EscapeLuaString(this.Hostname)}}",
                     {{hfInferenceProviderLine}}

app/MindWork AI Studio/Settings/IDataSource.cs

@@ -21,11 +21,6 @@ public interface IDataSource : IConfigurationObject
     /// </summary>
     public DataSourceType Type { get; init; }
 
-    /// <summary>
-    /// Which data security policy is applied to this data source?
-    /// </summary>
-    public DataSourceSecurity SecurityPolicy { get; init; }
-    
     /// <summary>
     /// The maximum number of matches to return when retrieving data from the ERI server.
     /// </summary>

app/MindWork AI Studio/Settings/IExternalDataSource.cs

@@ -1,9 +1,16 @@
 using System.Text.Json.Serialization;
 
+using AIStudio.Settings.DataModel;
+
 namespace AIStudio.Settings;
 
 public interface IExternalDataSource : IDataSource, ISecretId
 {
+    /// <summary>
+    /// Which data security policy is applied to this external data source?
+    /// </summary>
+    public DataSourceSecurity SecurityPolicy { get; init; }
+
     #region Implementation of ISecretId
 
     [JsonIgnore]

app/MindWork AI Studio/Settings/IInternalDataSource.cs

@@ -1,9 +1,27 @@
+using AIStudio.Provider;
+
 namespace AIStudio.Settings;
 
 public interface IInternalDataSource : IDataSource
 {
+    /// <summary>
+    /// Which provider confidence level is required by this internal data source?
+    /// </summary>
+    public ConfidenceLevel ConfidenceLevel { get; init; }
+
     /// <summary>
     /// The unique identifier of the embedding method used by this internal data source.
     /// </summary>
     public string EmbeddingId { get; init; }
+
+    /// <summary>
+    /// Optional maximum number of tokens per embedding chunk for this data source.
+    /// A value of 0 means the embedding provider's setting is used.
+    /// </summary>
+    public int MaxChunkTokenLength { get; init; }
+
+    /// <summary>
+    /// Optional number of tokens to overlap between consecutive chunks.
+    /// </summary>
+    public int ChunkOverlapTokenLength { get; init; }
 }

app/MindWork AI Studio/Settings/Provider.cs

@@ -36,6 +36,7 @@ public sealed record Provider(
     Host Host = Host.NONE,
     HFInferenceProvider HFInferenceProvider = HFInferenceProvider.NONE,
     string AdditionalJsonApiParameters = "",
+    string TokenizerPath = "",
     ProviderCapabilityOverrides? CapabilityOverrides = null,
     bool AllowUserProvidedAPIKey = false,
     string CustomIconDataUrl = "") : ConfigurationBaseObject, ISecretId, IUserProvidedAPIKey
@@ -44,15 +45,7 @@ public sealed record Provider(
     
     public static readonly Provider NONE = new();
 
-    public Provider() : this(
-        0,
-        Guid.Empty.ToString(),
-        string.Empty,
-        LLMProviders.NONE,
-        default, 
-        false,
-        false,
-        Guid.Empty)
+    public Provider() : this(0, Guid.Empty.ToString(), string.Empty, LLMProviders.NONE, default, false, false, Guid.Empty)
     {
     }
     
@@ -157,6 +150,12 @@ public sealed record Provider(
             additionalJsonApiParameters = string.Empty;
         }
 
+        var tokenizerPath = string.Empty;
+        if (table.TryGetValue("TokenizerPath", out var tokenizerPathValue) && !tokenizerPathValue.TryRead<string>(out tokenizerPath))
+        {
+            LOGGER.LogWarning($"The configured provider {idx} does not contain a valid tokenizer path. (Plugin ID: {configPluginId})");
+            tokenizerPath = string.Empty;
+        }
         var capabilityOverrides = ProviderCapabilityOverrides.TryParseFromLuaTable(idx, table, configPluginId, LOGGER);
 
         var allowUserProvidedApiKey = false;
@@ -186,6 +185,7 @@ public sealed record Provider(
             Host = host,
             HFInferenceProvider = hfInferenceProvider,
             AdditionalJsonApiParameters = additionalJsonApiParameters,
+            TokenizerPath = tokenizerPath,
             CapabilityOverrides = capabilityOverrides,
             AllowUserProvidedAPIKey = allowUserProvidedApiKey,
             CustomIconDataUrl = customIconDataUrl,
@@ -278,6 +278,8 @@ public sealed record Provider(
                     ["Id"] = "{{Guid.NewGuid().ToString()}}",
                     ["InstanceName"] = "{{LuaTools.EscapeLuaString(this.InstanceName)}}",
                     ["UsedLLMProvider"] = "{{this.UsedLLMProvider}}",
+                    
+                    ["TokenizerPath"] = "{{this.TokenizerPath}}",
 
                     ["Host"] = "{{this.Host}}",
                     ["Hostname"] = "{{LuaTools.EscapeLuaString(this.Hostname)}}",

app/MindWork AI Studio/Tools/AppIcons.cs

@@ -0,0 +1,20 @@
+namespace AIStudio.Tools;
+
+/// <summary>
+/// Icons we draw ourselves, because the Material icon set MudBlazor ships does not contain them.
+/// </summary>
+/// <remarks>
+/// The strings follow the same convention as the MudBlazor icons: they contain the SVG child
+/// elements only, drawn on a 24 by 24 canvas. MudIcon and every component taking an icon wrap them
+/// into the svg element themselves, which is why there must be no svg root element here.
+/// </remarks>
+public static class AppIcons
+{
+    /// <summary>
+    /// The classic database symbol: a cylinder made of three stacked discs.
+    /// </summary>
+    public const string DATABASE =
+        """
+        <path d="M5 4.6A7 2.6 0 0 1 19 4.6L19 8.9A7 2.6 0 0 1 5 8.9Z"/><path d="M5 9.9A7 2.6 0 0 0 19 9.9L19 14.1A7 2.6 0 0 1 5 14.1Z"/><path d="M5 15.1A7 2.6 0 0 0 19 15.1L19 19.4A7 2.6 0 0 1 5 19.4Z"/>
+        """;
+}

app/MindWork AI Studio/Tools/ContentStreamPendingContent.cs

@@ -0,0 +1,28 @@
+namespace AIStudio.Tools;
+
+/// <summary>
+/// Content which a reader held back, together with the token count of exactly that content.
+/// </summary>
+/// <remarks>
+/// Readers which assemble a page or a slide from several stream events cannot pass their content
+/// on right away. Its token count has to travel with it: the count describes the content, not the
+/// event which happened to arrive at the moment the content was released. Keeping the two together
+/// is what stops a page from being sized by the text of the page after it.
+/// </remarks>
+/// <param name="Content">The assembled content.</param>
+/// <param name="TokenCount">The number of tokens of that content, or null when it is unknown.</param>
+public readonly record struct ContentStreamPendingContent(string Content, int? TokenCount)
+{
+    /// <summary>
+    /// Adds up two token counts, where an unknown count makes the sum unknown as well.
+    /// </summary>
+    /// <remarks>
+    /// A partial sum would understate the whole and would let the chunking size a chunk by a part
+    /// of what it holds. Reporting the count as unknown is the honest answer, because the caller
+    /// can still count the content itself.
+    /// </remarks>
+    /// <param name="left">The first count, or null when it is unknown.</param>
+    /// <param name="right">The second count, or null when it is unknown.</param>
+    /// <returns>The sum, or null when either count is unknown.</returns>
+    public static int? AddTokenCounts(int? left, int? right) => left is null || right is null ? null : left + right;
+}

app/MindWork AI Studio/Tools/ContentStreamProcessedEvent.cs

@@ -11,14 +11,25 @@ namespace AIStudio.Tools;
 /// <param name="Content">The content to append, or null when this event carries none.</param>
 /// <param name="Error">The reported failure, or null when the event was processed successfully.</param>
 /// <param name="PromptInjection">What the runtime filtered out of the content, or null when it filtered nothing.</param>
-public readonly record struct ContentStreamProcessedEvent(string? Content, ContentStreamErrorDetails? Error, ContentStreamPromptInjectionDetails? PromptInjection = null)
+/// <param name="TokenCount">The number of tokens of the content, or null when it is unknown.</param>
+public readonly record struct ContentStreamProcessedEvent(string? Content, ContentStreamErrorDetails? Error, ContentStreamPromptInjectionDetails? PromptInjection = null, int? TokenCount = null)
 {
     /// <summary>
     /// An event which neither produced content nor reported a failure.
     /// </summary>
     public static readonly ContentStreamProcessedEvent NOTHING = new(null, null);
 
-    public static ContentStreamProcessedEvent FromContent(string? content) => new(content, null);
+    /// <summary>
+    /// An event which produced content, with the token count of that very content.
+    /// </summary>
+    /// <remarks>
+    /// The count travels with the content because a reader may hold content back across several
+    /// events: pairing it with the count of the event which released it would size it by the
+    /// wrong text.
+    /// </remarks>
+    /// <param name="content">The content to append.</param>
+    /// <param name="tokenCount">The number of tokens of that content, or null when it is unknown.</param>
+    public static ContentStreamProcessedEvent FromContent(string? content, int? tokenCount = null) => new(content, null, TokenCount: tokenCount);
 
     public static ContentStreamProcessedEvent FromError(ContentStreamErrorDetails? error) => new(null, error);
 

app/MindWork AI Studio/Tools/ContentStreamSseEvent.cs

@@ -12,4 +12,7 @@ public sealed class ContentStreamSseEvent
 
     [JsonPropertyName("metadata")] 
     public ContentStreamSseMetadata? Metadata { get; init; }
+
+    [JsonPropertyName("token_count")]
+    public int? TokenCount { get; init; }
 }

app/MindWork AI Studio/Tools/ContentStreamSseHandler.cs

@@ -17,7 +17,7 @@ public static class ContentStreamSseHandler
                 switch (sseEvent.Metadata)
                 {
                     case ContentStreamTextMetadata:
-                        return ContentStreamProcessedEvent.FromContent(sseEvent.Content);
+                        return ContentStreamProcessedEvent.FromContent(sseEvent.Content, sseEvent.TokenCount);
 
                     case ContentStreamPdfMetadata pdfMetadata:
                         var pageNumber = pdfMetadata.Pdf?.PageNumber ?? 0;
@@ -25,7 +25,7 @@ public static class ContentStreamSseHandler
                                 # Page {pageNumber}
                                 {sseEvent.Content}
 
-                                """);
+                                """, sseEvent.TokenCount);
 
                     case ContentStreamSpreadsheetMetadata spreadsheetMetadata:
                         var sheetName = spreadsheetMetadata.Spreadsheet?.SheetName;
@@ -38,31 +38,43 @@ public static class ContentStreamSseHandler
                         }
 
                         spreadSheetResult.Append(sseEvent.Content);
-                        return ContentStreamProcessedEvent.FromContent(spreadSheetResult.ToString());
+                        return ContentStreamProcessedEvent.FromContent(spreadSheetResult.ToString(), sseEvent.TokenCount);
 
                     //
                     // Documents which the runtime reads page by page are buffered, so the images of
                     // a page can follow its Markdown. Documents converted as a whole, e.g. by Pandoc,
                     // carry no page number and are passed on unchanged.
                     //
+                    // The buffering is why the count comes back from the reader rather than from
+                    // this event: the page which is released here arrived one event ago, and this
+                    // event's count belongs to the page which is now being buffered.
+                    //
                     case ContentStreamDocumentMetadata documentMetadata:
                         if (documentMetadata.Document?.PageNumber is not > 0)
-                            return ContentStreamProcessedEvent.FromContent(sseEvent.Content);
+                            return ContentStreamProcessedEvent.FromContent(sseEvent.Content, sseEvent.TokenCount);
 
                         var documentManager = DOCUMENT_MANAGERS.GetOrAdd(sseEvent.StreamId!, _ => new());
-                        var documentContent = documentManager.AddPage(documentMetadata, sseEvent.Content, extractImages);
-                        return documentContent is null ? ContentStreamProcessedEvent.NOTHING : ContentStreamProcessedEvent.FromContent(documentContent);
+                        var documentContent = documentManager.AddPage(documentMetadata, sseEvent.Content, sseEvent.TokenCount, extractImages);
+                        return documentContent is null ? ContentStreamProcessedEvent.NOTHING : ContentStreamProcessedEvent.FromContent(documentContent.Value.Content, documentContent.Value.TokenCount);
 
                     case ContentStreamImageMetadata:
-                        return ContentStreamProcessedEvent.FromContent(sseEvent.Content);
+                        return ContentStreamProcessedEvent.FromContent(sseEvent.Content, sseEvent.TokenCount);
 
                     case ContentStreamPresentationMetadata presentationMetadata:
+                        if (!extractImages)
+                        {
+                            var slideNumber = presentationMetadata.Presentation?.SlideNumber ?? 0;
+                            return ContentStreamProcessedEvent.FromContent(slideNumber > 0
+                                ? $"# Slide {slideNumber}\n{sseEvent.Content}"
+                                : sseEvent.Content, sseEvent.TokenCount);
+                        }
+
                         var slideManager = SLIDE_MANAGERS.GetOrAdd(
                             sseEvent.StreamId!,
                             _ => new()
                         );
 
-                        slideManager.AddSlide(presentationMetadata, sseEvent.Content, extractImages);
+                        slideManager.AddSlide(presentationMetadata, sseEvent.Content, sseEvent.TokenCount, extractImages);
                         return ContentStreamProcessedEvent.NOTHING;
 
                     //
@@ -82,11 +94,11 @@ public static class ContentStreamSseHandler
                         return ContentStreamProcessedEvent.FromPromptInjection(promptInjectionMetadata.PromptInjection);
 
                     default:
-                        return ContentStreamProcessedEvent.FromContent(sseEvent.Content);
+                        return ContentStreamProcessedEvent.FromContent(sseEvent.Content, sseEvent.TokenCount);
                 }
 
             case { Content: not null, Metadata: null }:
-                return ContentStreamProcessedEvent.FromContent(sseEvent.Content);
+                return ContentStreamProcessedEvent.FromContent(sseEvent.Content, sseEvent.TokenCount);
 
             default:
                 return ContentStreamProcessedEvent.NOTHING;
@@ -166,32 +178,45 @@ public static class ContentStreamSseHandler
         return $"![Image](data:{imageMediaType};base64,{base64Image})";
     }
 
-    public static string? Clear(string streamId)
+    /// <summary>
+    /// Releases what the readers of a stream still hold back and forgets the stream.
+    /// </summary>
+    /// <remarks>
+    /// The readers which assemble pages or slides always keep the last one of them: nothing tells
+    /// them that no further image is coming. It is released here, and it carries its own token
+    /// count, because a chunk without one cannot be sized by the caller.
+    /// </remarks>
+    /// <param name="streamId">The stream to release and forget.</param>
+    /// <returns>The content which was held back, or null when there was none.</returns>
+    public static ContentStreamPendingContent? Clear(string streamId)
     {
         if (string.IsNullOrWhiteSpace(streamId))
             return null;
- 
+
         var finalContentChunk = new StringBuilder();
-        if(SLIDE_MANAGERS.TryGetValue(streamId, out var slideManager))
+        int? tokenCount = 0;
+        if(SLIDE_MANAGERS.TryGetValue(streamId, out var slideManager)
+           && slideManager.GetAllSlidesInOrder() is { } slides
+           && !string.IsNullOrWhiteSpace(slides.Content))
         {
-            var result = slideManager.GetAllSlidesInOrder();
-            if (!string.IsNullOrWhiteSpace(result))
-                finalContentChunk.Append(result);
+            finalContentChunk.Append(slides.Content);
+            tokenCount = ContentStreamPendingContent.AddTokenCounts(tokenCount, slides.TokenCount);
         }
 
-        if (DOCUMENT_MANAGERS.TryGetValue(streamId, out var documentManager))
+        if (DOCUMENT_MANAGERS.TryGetValue(streamId, out var documentManager)
+            && documentManager.Flush() is { } page
+            && !string.IsNullOrWhiteSpace(page.Content))
         {
-            var result = documentManager.Flush();
-            if (!string.IsNullOrWhiteSpace(result))
-                finalContentChunk.Append(result);
+            finalContentChunk.Append(page.Content);
+            tokenCount = ContentStreamPendingContent.AddTokenCounts(tokenCount, page.TokenCount);
         }
-        
+
         SLIDE_MANAGERS.TryRemove(streamId, out _);
         DOCUMENT_MANAGERS.TryRemove(streamId, out _);
         var imageIdPrefix = $"{streamId}-";
         foreach (var key in CHUNKED_IMAGES.Keys.Where(k => k.StartsWith(imageIdPrefix, StringComparison.InvariantCultureIgnoreCase)))
             CHUNKED_IMAGES.TryRemove(key, out _);
-        
-        return finalContentChunk.Length > 0 ? finalContentChunk.ToString() : null;
+
+        return finalContentChunk.Length > 0 ? new ContentStreamPendingContent(finalContentChunk.ToString(), tokenCount) : null;
     }
-}
+}

app/MindWork AI Studio/Tools/Databases/DatabaseClient.cs

@@ -39,14 +39,15 @@ public abstract class DatabaseClient(string name, string path)
     {
         string[] suffixes = { "B", "KB", "MB", "GB", "TB", "PB" };
         int suffixIndex = 0;
+        double convertedSize = size;
     
-        while (size >= 1024 && suffixIndex < suffixes.Length - 1)
+        while (convertedSize >= 1024 && suffixIndex < suffixes.Length - 1)
         {
-            size /= 1024;
+            convertedSize /= 1024;
             suffixIndex++;
         }
     
-        return $"{size:0##} {suffixes[suffixIndex]}";
+        return $"{convertedSize:0.##} {suffixes[suffixIndex]}";
     }
     
     public void SetLogger(ILogger<DatabaseClient> logService)

app/MindWork AI Studio/Tools/Databases/DatabaseClientProvider.cs

@@ -1,5 +1,6 @@
-using AIStudio.Tools.Services;
+using AIStudio.Tools.Databases.IndexStore;
 using AIStudio.Tools.Databases.VectorStore;
+using AIStudio.Tools.Services;
 
 namespace AIStudio.Tools.Databases;
 
@@ -44,10 +45,10 @@ public sealed class DatabaseClientProvider(RustService rustService, ILoggerFacto
         }
     }
 
-    public async Task<IVectorStoreClient> GetVectorStoreAsync(CancellationToken cancellationToken = default)
+    public async Task<VectorStoreClient> GetVectorStoreAsync(CancellationToken cancellationToken = default)
     {
         var client = await this.GetClientAsync(DatabaseRole.VECTOR_STORE, cancellationToken);
-        if (client is IVectorStoreClient vectorStore)
+        if (client is VectorStoreClient vectorStore)
             return vectorStore;
 
         return new NoVectorStoreClient(
@@ -56,6 +57,18 @@ public sealed class DatabaseClientProvider(RustService rustService, ILoggerFacto
             client.Status);
     }
 
+    public async Task<IndexStoreClient> GetIndexStoreAsync(CancellationToken cancellationToken = default)
+    {
+        var client = await this.GetClientAsync(DatabaseRole.INDEX_STORE, cancellationToken);
+        if (client is IndexStoreClient indexStore)
+            return indexStore;
+
+        return new NoIndexStoreClient(
+            client.Name,
+            "The configured database client does not support local RAG index operations.",
+            client.Status);
+    }
+
     private DatabaseClient CacheIfAvailable(DatabaseRole databaseRole, DatabaseClient client)
     {
         if (!client.IsAvailable)
@@ -92,6 +105,7 @@ public sealed class DatabaseClientProvider(RustService rustService, ILoggerFacto
     private async Task<DatabaseClient> CreateClientAsync(DatabaseRole databaseRole, CancellationToken cancellationToken) => databaseRole switch
     {
         DatabaseRole.VECTOR_STORE => await QdrantEdgeClientImplementation.CreateAsync(rustService, this.logger, this.databaseClientLogger, cancellationToken),
+        DatabaseRole.INDEX_STORE => await SqliteIndexStoreClientImplementation.CreateAsync(this.logger, this.databaseClientLogger, cancellationToken),
         _ => new NoDatabaseClient(databaseRole.ToString(), "The requested database role is not supported.")
     };
 

app/MindWork AI Studio/Tools/Databases/DatabaseRole.cs

@@ -3,4 +3,5 @@ namespace AIStudio.Tools.Databases;
 public enum DatabaseRole
 {
     VECTOR_STORE,
+    INDEX_STORE,
 }

app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateChunk.cs

@@ -0,0 +1,3 @@
+namespace AIStudio.Tools.Databases.IndexStore;
+
+public sealed record EmbeddingStateChunk(string ChunkId, string ParentFileId, int? PageNumber, int ChunkIndex, string ChunkText, DateTimeOffset EmbeddedAtUtc);

app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateChunkEntity.cs

@@ -0,0 +1,20 @@
+namespace AIStudio.Tools.Databases.IndexStore;
+
+internal sealed class EmbeddingStateChunkEntity
+{
+    public int Id { get; set; }
+
+    public string ChunkId { get; set; } = string.Empty;
+
+    public string ParentFileId { get; set; } = string.Empty;
+
+    public int? PageNumber { get; set; }
+
+    public int ChunkIndex { get; set; }
+
+    public string ChunkText { get; set; } = string.Empty;
+
+    public DateTimeOffset EmbeddedAtUtc { get; set; }
+
+    public EmbeddingStateFileEntity? File { get; set; }
+}

app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateDataSourceEntity.cs

@@ -0,0 +1,24 @@
+namespace AIStudio.Tools.Databases.IndexStore;
+
+internal sealed class EmbeddingStateDataSourceEntity
+{
+    public string DataSourceId { get; set; } = string.Empty;
+
+    public string DataSourceName { get; set; } = string.Empty;
+
+    public string DataSourceType { get; set; } = string.Empty;
+
+    public string EmbeddingProviderId { get; set; } = string.Empty;
+
+    public string EmbeddingSignature { get; set; } = string.Empty;
+
+    public string SourceHash { get; set; } = string.Empty;
+
+    public int VectorSize { get; set; }
+
+    public DateTimeOffset UpdatedAtUtc { get; set; }
+
+    public List<EmbeddingStateFileEntity> Files { get; set; } = [];
+
+    public List<IndexingFailureEntity> PermanentIndexingFailures { get; set; } = [];
+}

app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFile.cs

@@ -0,0 +1,16 @@
+namespace AIStudio.Tools.Databases.IndexStore;
+
+public sealed record EmbeddingStateFile(
+    string ParentFileId,
+    string AbsolutePath,
+    string FileName,
+    string RelativePath,
+    string FileType,
+    string Fingerprint,
+    long FileSize,
+    DateTimeOffset CreationUtc,
+    DateTimeOffset LastWriteUtc,
+    DateTimeOffset EmbeddedAtUtc,
+    int ChunkCount,
+    string ConfidenceLevel,
+    int ConfidenceLevelRank);

app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFileEntity.cs

@@ -0,0 +1,36 @@
+namespace AIStudio.Tools.Databases.IndexStore;
+
+internal sealed class EmbeddingStateFileEntity
+{
+    public string ParentFileId { get; set; } = string.Empty;
+
+    public string DataSourceId { get; set; } = string.Empty;
+
+    public string AbsolutePath { get; set; } = string.Empty;
+
+    public string FileName { get; set; } = string.Empty;
+
+    public string RelativePath { get; set; } = string.Empty;
+
+    public string FileType { get; set; } = string.Empty;
+
+    public string Fingerprint { get; set; } = string.Empty;
+
+    public long FileSize { get; set; }
+
+    public DateTimeOffset CreationUtc { get; set; }
+
+    public DateTimeOffset LastWriteUtc { get; set; }
+
+    public DateTimeOffset EmbeddedAtUtc { get; set; }
+
+    public int ChunkCount { get; set; }
+
+    public string ConfidenceLevel { get; set; } = string.Empty;
+
+    public int ConfidenceLevelRank { get; set; }
+
+    public EmbeddingStateDataSourceEntity? DataSource { get; set; }
+
+    public List<EmbeddingStateChunkEntity> Chunks { get; set; } = [];
+}

app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreClient.cs

@@ -0,0 +1,36 @@
+using AIStudio.Tools.Services;
+
+namespace AIStudio.Tools.Databases.IndexStore;
+
+public abstract class IndexStoreClient(string name, string path) : DatabaseClient(name, path)
+{
+    public abstract Task<DataSourceEmbeddingManifest> GetManifestAsync(string dataSourceId, CancellationToken token);
+
+    public abstract Task UpsertDataSourceAsync(
+        string dataSourceId,
+        string dataSourceName,
+        string dataSourceType,
+        string embeddingProviderId,
+        string embeddingSignature,
+        string sourceHash,
+        int vectorSize,
+        CancellationToken token);
+
+    public abstract Task UpdateVectorSizeAsync(string dataSourceId, int vectorSize, CancellationToken token);
+
+    public abstract Task UpdateDataSourceHashAsync(string dataSourceId, string sourceHash, CancellationToken token);
+
+    public abstract Task UpsertFileAsync(string dataSourceId, EmbeddingStateFile file, CancellationToken token);
+
+    public abstract Task DeleteFileAsync(string dataSourceId, string filePath, CancellationToken token);
+
+    public abstract Task UpsertPermanentFailureAsync(string dataSourceId, PermanentIndexingFailure failure, CancellationToken token);
+
+    public abstract Task DeletePermanentFailureAsync(string dataSourceId, string filePath, CancellationToken token);
+
+    public abstract Task UpsertChunksAsync(string dataSourceId, IReadOnlyList<EmbeddingStateChunk> chunks, CancellationToken token);
+
+    public abstract Task<IReadOnlyList<IndexStoreSearchResult>> SearchChunksAsync(string dataSourceId, string query, int maxMatches, CancellationToken token);
+
+    public abstract Task DeleteDataSourceAsync(string dataSourceId, CancellationToken token);
+}

app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDateTimeOffset.cs

@@ -0,0 +1,18 @@
+using System.Globalization;
+
+namespace AIStudio.Tools.Databases.IndexStore;
+
+internal static class IndexStoreDateTimeOffset
+{
+    public static string ToUtcText(DateTimeOffset dateTime)
+    {
+        return dateTime.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture);
+    }
+
+    public static DateTimeOffset ParseUtc(string value)
+    {
+        return DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.None, out var dateTime)
+            ? dateTime.ToUniversalTime()
+            : DateTimeOffset.UnixEpoch;
+    }
+}