Improved adding local data sources without an embedding provider (#1084)

app/MindWork AI Studio/Components/DataSourceManagement.razor

@@ -14,6 +14,8 @@
     }
 </MudJustifiedText>
 
+<DataSourceMissingEmbeddingWarning/>
+
 <MudStack Row="true" AlignItems="AlignItems.Center" Justify="Justify.SpaceBetween" Class="mb-3" Wrap="Wrap.Wrap">
     <MudTextSwitch Label="@T("Automatic local data source refresh")" Value="@this.SettingsManager.ConfigurationData.App.DataSourceIndexing.AutomaticRefresh" LabelOn="@(this.AreMailboxesEnabled ? T("Local data sources refresh when files change, mailboxes every 16 minutes.") : T("Local data sources refresh when files change."))" LabelOff="@T("Local data sources refresh only when triggered manually.")" ValueChanged="@this.AutomaticRefreshChanged"/>
     <MudTooltip Text="@T("Refresh all")">
@@ -130,17 +132,21 @@
         <MudMenuItem OnClick="@(() => this.AddDataSource(DataSourceType.ERI_V1))">
             @T("External Data (ERI-Server v1)")
         </MudMenuItem>
-        <MudMenuItem OnClick="@(() => this.AddDataSource(DataSourceType.LOCAL_DIRECTORY))">
-            @T("Local Directory")
+        @*
+            Without an embedding provider, the dialogs of the local data sources could never be saved.
+            They stay listed with the reason, so nobody concludes that AI Studio cannot read local files:
+        *@
+        <MudMenuItem OnClick="@(() => this.AddDataSource(DataSourceType.LOCAL_DIRECTORY))" Disabled="@(!this.HasEmbeddingProvider)">
+            @(this.HasEmbeddingProvider ? T("Local Directory") : T("Local Directory, requires an embedding provider"))
         </MudMenuItem>
-        <MudMenuItem OnClick="@(() => this.AddDataSource(DataSourceType.LOCAL_FILE))">
-            @T("Local File")
+        <MudMenuItem OnClick="@(() => this.AddDataSource(DataSourceType.LOCAL_FILE))" Disabled="@(!this.HasEmbeddingProvider)">
+            @(this.HasEmbeddingProvider ? T("Local File") : T("Local File, requires an embedding provider"))
         </MudMenuItem>
         @if (PreviewFeatures.PRE_MAILBOXES_2026.IsEnabled(this.SettingsManager))
         {
             @* Shown with a lock rather than left out, like the locked buttons for adding providers: *@
-            <MudMenuItem OnClick="@this.AddMailbox" Disabled="@(!this.MayAddMailbox)" Icon="@(this.MayAddMailbox ? null : Icons.Material.Filled.Lock)">
-                @(this.MayAddMailbox ? T("Mailbox (IMAP)") : T("Mailbox (IMAP), not allowed by your organization"))
+            <MudMenuItem OnClick="@this.AddMailbox" Disabled="@(!this.MayAddMailbox || !this.HasEmbeddingProvider)" Icon="@(this.MayAddMailbox ? null : Icons.Material.Filled.Lock)">
+                @this.MailboxMenuText
             </MudMenuItem>
         }
     </MudMenu>

app/MindWork AI Studio/Components/DataSourceManagement.razor.cs

@@ -30,8 +30,6 @@ public partial class DataSourceManagement : MSGComponentBase
     [Inject]
     private DatabaseClientProvider DatabaseClientProvider { get; init; } = null!;
 
-    private readonly List<ConfigurationSelectData<string>> availableEmbeddingProviders = new();
-
     /// <summary>
     /// The data sources and then the mailboxes, which the table shows together.
     /// </summary>
@@ -42,13 +40,26 @@ public partial class DataSourceManagement : MSGComponentBase
 
     private bool MayAddMailbox => this.SettingsManager.ConfigurationData.App.AllowUserToAddMailbox;
 
+    /// <summary>
+    /// Whether an embedding provider exists, which every local data source needs.
+    /// </summary>
+    private bool HasEmbeddingProvider => !EmbeddingProviderSetup.IsMissing(this.SettingsManager);
+
+    /// <remarks>
+    /// The rule of the organization comes first: an embedding provider would not make the mailbox possible.
+    /// </remarks>
+    private string MailboxMenuText => !this.MayAddMailbox
+        ? T("Mailbox (IMAP), not allowed by your organization")
+        : this.HasEmbeddingProvider
+            ? T("Mailbox (IMAP)")
+            : T("Mailbox (IMAP), requires an embedding provider");
+
     #region Overrides of ComponentBase
 
     protected override async Task OnInitializedAsync()
     {
         await base.OnInitializedAsync();
         this.ApplyFilters([], [ Event.CONFIGURATION_CHANGED, Event.RAG_EMBEDDING_STATUS_CHANGED ]);
-        this.UpdateEmbeddingProviders();
     }
 
     #endregion
@@ -61,10 +72,6 @@ public partial class DataSourceManagement : MSGComponentBase
         {
             case Event.CONFIGURATION_CHANGED:
             case Event.PLUGINS_RELOADED:
-                this.UpdateEmbeddingProviders();
-                this.StateHasChanged();
-                break;
-
             case Event.RAG_EMBEDDING_STATUS_CHANGED:
                 this.StateHasChanged();
                 break;
@@ -75,13 +82,6 @@ public partial class DataSourceManagement : MSGComponentBase
 
     #endregion
 
-    private void UpdateEmbeddingProviders()
-    {
-        this.availableEmbeddingProviders.Clear();
-        foreach (var provider in this.SettingsManager.GetAllEmbeddingProviders())
-            this.availableEmbeddingProviders.Add(new (provider.Name, provider.Id));
-    }
-
     /// <remarks>
     /// Files which were skipped for good are none of the failed ones, so an otherwise usable data
     /// source stays green. The shared health policy warns when none of the selected documents could
@@ -222,6 +222,10 @@ public partial class DataSourceManagement : MSGComponentBase
 
     private async Task AddDataSource(DataSourceType type)
     {
+        // The menu offers no local data source without an embedding provider, but the last one may have gone since it was drawn:
+        if (type is DataSourceType.LOCAL_FILE or DataSourceType.LOCAL_DIRECTORY && !this.HasEmbeddingProvider)
+            return;
+
         IDataSource? addedDataSource = null;
         switch (type)
         {
@@ -229,7 +233,6 @@ public partial class DataSourceManagement : MSGComponentBase
                 var localFileDialogParameters = new DialogParameters<DataSourceLocalFileDialog>
                 {
                     { x => x.IsEditing, false },
-                    { x => x.AvailableEmbeddings, this.availableEmbeddingProviders }
                 };
 
                 var localFileDialogReference = await this.DialogService.ShowAsync<DataSourceLocalFileDialog>(T("Add Local File as Data Source"), localFileDialogParameters, DialogOptions.FULLSCREEN);
@@ -246,7 +249,6 @@ public partial class DataSourceManagement : MSGComponentBase
                 var localDirectoryDialogParameters = new DialogParameters<DataSourceLocalDirectoryDialog>
                 {
                     { x => x.IsEditing, false },
-                    { x => x.AvailableEmbeddings, this.availableEmbeddingProviders }
                 };
 
                 var localDirectoryDialogReference = await this.DialogService.ShowAsync<DataSourceLocalDirectoryDialog>(T("Add Local Directory as Data Source"), localDirectoryDialogParameters, DialogOptions.FULLSCREEN);
@@ -400,7 +402,6 @@ public partial class DataSourceManagement : MSGComponentBase
                     { x => x.IsEditing, true },
                     { x => x.DataSource, localFile },
                     { x => x.LockSource, lockDataSourceOrigin },
-                    { x => x.AvailableEmbeddings, this.availableEmbeddingProviders }
                 };
 
                 var localFileDialogReference = await this.DialogService.ShowAsync<DataSourceLocalFileDialog>(T("Edit Local File Data Source"), localFileDialogParameters, DialogOptions.FULLSCREEN);
@@ -417,7 +418,6 @@ public partial class DataSourceManagement : MSGComponentBase
                     { x => x.IsEditing, true },
                     { x => x.DataSource, localDirectory },
                     { x => x.LockSource, lockDataSourceOrigin },
-                    { x => x.AvailableEmbeddings, this.availableEmbeddingProviders }
                 };
 
                 var localDirectoryDialogReference = await this.DialogService.ShowAsync<DataSourceLocalDirectoryDialog>(T("Edit Local Directory Data Source"), localDirectoryDialogParameters, DialogOptions.FULLSCREEN);
@@ -539,13 +539,12 @@ public partial class DataSourceManagement : MSGComponentBase
 
     private async Task AddMailbox()
     {
-        if (!this.MayAddMailbox)
+        if (!this.MayAddMailbox || !this.HasEmbeddingProvider)
             return;
 
         var dialogParameters = new DialogParameters<DataSourceMailboxDialog>
         {
             { x => x.IsEditing, false },
-            { x => x.AvailableEmbeddings, this.availableEmbeddingProviders }
         };
 
         var dialogReference = await this.DialogService.ShowAsync<DataSourceMailboxDialog>(T("Add Mailbox"), dialogParameters, DialogOptions.FULLSCREEN);

app/MindWork AI Studio/Components/DataSourceMissingEmbeddingWarning.razor

@@ -0,0 +1,28 @@
+@inherits MSGComponentBase
+
+@*
+    The condition lives here rather than with the parents: the component stays in place while the
+    alert comes and goes, so it can still report the provider it just added.
+*@
+@if (this.IsMissing)
+{
+    <MudAlert Severity="@(this.MayAdd ? Severity.Warning : Severity.Info)" Variant="Variant.Outlined" Class="mb-3">
+        <MudJustifiedText Typo="Typo.body1">
+            @this.Text
+        </MudJustifiedText>
+        @if (this.MayAdd)
+        {
+            <MudStack Row="true" Wrap="Wrap.Wrap" Class="mt-3">
+                <MudButton Variant="Variant.Filled" Color="Color.Warning" StartIcon="@Icons.Material.Filled.AddRoad" Disabled="@this.isAdding" OnClick="@this.AddEmbeddingProvider">
+                    @T("Add Embedding")
+                </MudButton>
+                @if (this.MayImport)
+                {
+                    <MudButton Variant="Variant.Outlined" Color="Color.Warning" StartIcon="@Icons.Material.Filled.FileDownload" Disabled="@this.isAdding" OnClick="@this.ImportEmbeddingProvider">
+                        @T("Import Embedding")
+                    </MudButton>
+                }
+            </MudStack>
+        }
+    </MudAlert>
+}

app/MindWork AI Studio/Components/DataSourceMissingEmbeddingWarning.razor.cs

@@ -0,0 +1,114 @@
+using AIStudio.Settings;
+using AIStudio.Tools.Services;
+
+using Microsoft.AspNetCore.Components;
+
+namespace AIStudio.Components;
+
+/// <summary>
+/// Tells that no embedding provider exists, which every local data source needs, and offers to add
+/// one right there.
+/// </summary>
+/// <remarks>
+/// Shown where a local data source is set up: above the data source table, and in the dialogs for
+/// adding or editing one. Without it, the missing provider only came up when the user saved the
+/// dialog, after filling in everything else, and leaving the dialog to add one lost all of it.
+/// </remarks>
+public partial class DataSourceMissingEmbeddingWarning : MSGComponentBase
+{
+    /// <summary>
+    /// Whether the warning sits in the dialog of a data source rather than above the data source table.
+    /// </summary>
+    [Parameter]
+    public bool IsInDataSourceDialog { get; set; }
+
+    /// <summary>
+    /// Receives the embedding provider the user just added, so that a dialog can select it.
+    /// </summary>
+    [Parameter]
+    public EventCallback<EmbeddingProvider> EmbeddingProviderAdded { get; set; }
+
+    [Inject]
+    private IDialogService DialogService { get; init; } = null!;
+
+    [Inject]
+    private DataSourceEmbeddingService DataSourceEmbeddingService { get; init; } = null!;
+
+    /// <summary>
+    /// Whether the dialog for adding a provider is open, so that a second click does not open another one.
+    /// </summary>
+    private bool isAdding;
+
+    private bool IsMissing => EmbeddingProviderSetup.IsMissing(this.SettingsManager);
+
+    private bool MayAdd => EmbeddingProviderSetup.MayAdd(this.SettingsManager);
+
+    private bool MayImport => EmbeddingProviderSetup.MayImport(this.SettingsManager);
+
+    private string Text
+    {
+        get
+        {
+            var problem = this.IsInDataSourceDialog
+                ? T("No embedding provider is configured yet, but this data source needs one.")
+                : T("No embedding provider is configured yet. Local data sources, such as your files and folders, need one: it prepares your data so that the AI can search it.");
+
+            if (!this.MayAdd)
+                return $"{problem} {T("Your organization manages the embedding providers and has not provided one yet. Please ask your IT department.")}";
+
+            // Above the table, the buttons speak for themselves. In a dialog, the worry is about what was entered so far:
+            return this.IsInDataSourceDialog
+                ? $"{problem} {T("You can add one right here; everything you entered in this dialog stays as it is.")}"
+                : problem;
+        }
+    }
+
+    #region Overrides of ComponentBase
+
+    protected override async Task OnInitializedAsync()
+    {
+        await base.OnInitializedAsync();
+
+        // The warning goes away as soon as a provider exists, also one which a configuration plugin brings along:
+        this.ApplyFilters([], [ Event.CONFIGURATION_CHANGED ]);
+    }
+
+    #endregion
+
+    #region Overrides of MSGComponentBase
+
+    protected override Task ProcessIncomingMessage<T>(ComponentBase? sendingComponent, Event triggeredEvent, T? data) where T : default
+    {
+        if (triggeredEvent is Event.CONFIGURATION_CHANGED)
+            this.StateHasChanged();
+
+        return base.ProcessIncomingMessage(sendingComponent, triggeredEvent, data);
+    }
+
+    #endregion
+
+    private Task AddEmbeddingProvider() => this.AddOnceAsync(() => EmbeddingProviderSetup.AddAsync(this.DialogService, this.SettingsManager, this.DataSourceEmbeddingService));
+
+    private Task ImportEmbeddingProvider() => this.AddOnceAsync(() => EmbeddingProviderSetup.ImportAsync(this.DialogService, this.SettingsManager, this.DataSourceEmbeddingService));
+
+    private async Task AddOnceAsync(Func<Task<EmbeddingProvider?>> addProvider)
+    {
+        if (this.isAdding)
+            return;
+
+        this.isAdding = true;
+        try
+        {
+            var addedProvider = await addProvider();
+            if (addedProvider is null)
+                return;
+
+            await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
+            await this.EmbeddingProviderAdded.InvokeAsync(addedProvider);
+        }
+        finally
+        {
+            this.isAdding = false;
+        }
+    }
+}

app/MindWork AI Studio/Components/Settings/SettingsPanelEmbeddings.razor

@@ -1,7 +1,6 @@
 @using AIStudio.Provider
 @using AIStudio.Settings
 @using AIStudio.Settings.DataModel
-@using AIStudio.Tools.PluginSystem
 @inherits SettingsPanelProviderBase
 
 @if (PreviewFeatures.PRE_RAG_2024.IsEnabled(this.SettingsManager))
@@ -95,8 +94,8 @@
         }
 
         <MudStack Row="true" Wrap="Wrap.Wrap" Class="mt-3 mb-6">
-            <LockableButton Text="@T("Add Embedding")" IsLocked="@(() => !this.SettingsManager.ConfigurationData.App.AllowUserToAddProvider || !this.SettingsManager.ConfigurationData.App.AllowUserToAddEmbeddingProvider)" Icon="@Icons.Material.Filled.AddRoad" OnClickAsync="@this.AddEmbeddingProvider" Class="flex-grow-0" NoMargin="true" />
-            <LockableButton Text="@T("Import Embedding")" IsLocked="@(() => !this.SettingsManager.ConfigurationData.App.CanImportConfigurationSnippet(PluginConfigurationObjectType.EMBEDDING_PROVIDER))" Icon="@Icons.Material.Filled.FileDownload" OnClickAsync="@this.ImportEmbeddingProvider" ButtonVariant="Variant.Outlined" Class="flex-grow-0" NoMargin="true" />
+            <LockableButton Text="@T("Add Embedding")" IsLocked="@(() => !EmbeddingProviderSetup.MayAdd(this.SettingsManager))" Icon="@Icons.Material.Filled.AddRoad" OnClickAsync="@this.AddEmbeddingProvider" Class="flex-grow-0" NoMargin="true" />
+            <LockableButton Text="@T("Import Embedding")" IsLocked="@(() => !EmbeddingProviderSetup.MayImport(this.SettingsManager))" Icon="@Icons.Material.Filled.FileDownload" OnClickAsync="@this.ImportEmbeddingProvider" ButtonVariant="Variant.Outlined" Class="flex-grow-0" NoMargin="true" />
         </MudStack>
     </ExpansionPanel>
 }

app/MindWork AI Studio/Components/Settings/SettingsPanelEmbeddings.razor.cs

@@ -2,7 +2,6 @@ using System.Globalization;
 using AIStudio.Dialogs;
 using AIStudio.Provider;
 using AIStudio.Settings;
-using AIStudio.Tools.PluginSystem;
 using AIStudio.Tools.Services;
 
 using Microsoft.AspNetCore.Components;
@@ -56,57 +55,28 @@ public partial class SettingsPanelEmbeddings : SettingsPanelProviderBase
 
     private async Task AddEmbeddingProvider()
     {
-        var dialogParameters = new DialogParameters<EmbeddingProviderDialog>
-        {
-            { x => x.IsEditing, false },
-        };
+        if (await EmbeddingProviderSetup.AddAsync(this.DialogService, this.SettingsManager, this.DataSourceEmbeddingService) is null)
+            return;
 
-        await this.AddEmbeddingProvider(dialogParameters, isImport: false);
+        await this.UpdateEmbeddingProviders();
+        await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
     }
 
     private async Task ImportEmbeddingProvider()
     {
-        var import = await ConfigurationSnippetImportDialog.ShowAsync(this.DialogService, PluginConfigurationObjectType.EMBEDDING_PROVIDER, T("Import Embedding"));
-        if (import?.ConfigurationObject is not EmbeddingProvider embeddingProvider)
+        if (await EmbeddingProviderSetup.ImportAsync(this.DialogService, this.SettingsManager, this.DataSourceEmbeddingService) is null)
             return;
 
-        var dialogParameters = CreateDialogParameters(embeddingProvider, isEditing: false);
-        dialogParameters.Add(x => x.ImportedAPIKey, import.Secret);
-        await this.AddEmbeddingProvider(dialogParameters, isImport: true);
-    }
-
-    private async Task AddEmbeddingProvider(DialogParameters<EmbeddingProviderDialog> dialogParameters, bool isImport)
-    {
-        var dialogReference = await this.DialogService.ShowAsync<EmbeddingProviderDialog>(T("Add Embedding Provider"), dialogParameters, DialogOptions.FULLSCREEN);
-        var dialogResult = await dialogReference.Result;
-        if (dialogResult is null || dialogResult.Canceled)
-            return;
-
-        //
-        // An organization can lock the import while the user is still filling in the form, e.g.,
-        // with a new version of its configuration plugin. Hence, the permission is checked once
-        // more before the imported embedding provider is stored:
-        //
-        if (isImport && !this.SettingsManager.ConfigurationData.App.CanImportConfigurationSnippet(PluginConfigurationObjectType.EMBEDDING_PROVIDER))
-            return;
-
-        var addedEmbedding = (EmbeddingProvider)dialogResult.Data!;
-        addedEmbedding = addedEmbedding with { Num = this.SettingsManager.ConfigurationData.NextEmbeddingNum++ };
-        
-        this.SettingsManager.ConfigurationData.EmbeddingProviders.Add(addedEmbedding);
         await this.UpdateEmbeddingProviders();
-        
-        await this.SettingsManager.StoreSettings();
-        await this.DataSourceEmbeddingService.QueueAllInternalDataSourcesAsync();
         await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
     }
-    
+
     private async Task EditEmbeddingProvider(EmbeddingProvider embeddingProvider)
     {
         if (embeddingProvider.IsEnterpriseConfiguration && !embeddingProvider.AllowUserProvidedAPIKey)
             return;
 
-        var dialogReference = await this.DialogService.ShowAsync<EmbeddingProviderDialog>(T("Edit Embedding Provider"), CreateDialogParameters(embeddingProvider, isEditing: true), DialogOptions.FULLSCREEN);
+        var dialogReference = await this.DialogService.ShowAsync<EmbeddingProviderDialog>(T("Edit Embedding Provider"), EmbeddingProviderSetup.CreateDialogParameters(embeddingProvider, isEditing: true), DialogOptions.FULLSCREEN);
         var dialogResult = await dialogReference.Result;
         if (dialogResult is null || dialogResult.Canceled)
             return;
@@ -136,26 +106,6 @@ public partial class SettingsPanelEmbeddings : SettingsPanelProviderBase
         await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
     }
 
-    private static DialogParameters<EmbeddingProviderDialog> CreateDialogParameters(EmbeddingProvider embeddingProvider, bool isEditing) => new()
-    {
-        { x => x.DataNum, embeddingProvider.Num },
-        { x => x.DataId, embeddingProvider.Id },
-        { x => x.DataName, embeddingProvider.Name },
-        { x => x.DataLLMProvider, embeddingProvider.UsedLLMProvider },
-        { x => x.DataCustomIconDataUrl, embeddingProvider.CustomIconDataUrl },
-        { x => x.DataModel, embeddingProvider.Model },
-        { x => x.DataHostname, embeddingProvider.Hostname },
-        { x => x.IsSelfHosted, embeddingProvider.IsSelfHosted },
-        { x => x.IsEditing, isEditing },
-        { x => x.DataHost, embeddingProvider.Host },
-        { x => x.DataTokenizerPath, embeddingProvider.TokenizerPath },
-        { x => x.DataTokenizerFingerprint, embeddingProvider.TokenizerFingerprint },
-        { x => x.DataTokenLimit, embeddingProvider.EffectiveTokenLimit },
-        { x => x.DataEmbeddingBatchSize, embeddingProvider.EffectiveEmbeddingBatchSize },
-        { x => x.HFInferenceProviderId, embeddingProvider.HFInferenceProvider },
-        { x => x.IsEnterpriseConfiguration, embeddingProvider.IsEnterpriseConfiguration },
-    };
-
     private async Task DeleteEmbeddingProvider(EmbeddingProvider provider)
     {
         var question = string.Format(T("Are you sure you want to delete the embedding provider '{0}'?"), provider.Name);