Configure data sources (#259)

app/MindWork AI Studio/Components/SelectDirectory.razor

@@ -4,6 +4,7 @@
         Text="@this.Directory"
         Label="@this.Label"
         ReadOnly="@true"
+        Validation="@this.Validation"
         Adornment="Adornment.Start"
         AdornmentIcon="@Icons.Material.Filled.Folder"
         UserAttributes="@SPELLCHECK_ATTRIBUTES"

app/MindWork AI Studio/Components/SelectDirectory.razor.cs

@@ -21,6 +21,9 @@ public partial class SelectDirectory : ComponentBase
     [Parameter]
     public string DirectoryDialogTitle { get; set; } = "Select Directory";
     
+    [Parameter]
+    public Func<string, string?> Validation { get; set; } = _ => null;
+    
     [Inject]
     private SettingsManager SettingsManager { get; init; } = null!;
 

app/MindWork AI Studio/Components/SelectFile.razor

@@ -0,0 +1,17 @@
+<MudStack Row="@true" Spacing="3" Class="mb-3" StretchItems="StretchItems.None" AlignItems="AlignItems.Center">
+    <MudTextField
+        T="string"
+        Text="@this.File"
+        Label="@this.Label"
+        ReadOnly="@true"
+        Validation="@this.Validation"
+        Adornment="Adornment.Start"
+        AdornmentIcon="@Icons.Material.Filled.AttachFile"
+        UserAttributes="@SPELLCHECK_ATTRIBUTES"
+        Variant="Variant.Outlined"
+    />
+    
+    <MudButton StartIcon="@Icons.Material.Filled.FolderOpen" Variant="Variant.Outlined" Color="Color.Primary" Disabled="this.Disabled" OnClick="@this.OpenFileDialog">
+        Choose File
+    </MudButton>
+</MudStack>

app/MindWork AI Studio/Components/SelectFile.razor.cs

@@ -0,0 +1,63 @@
+using AIStudio.Settings;
+
+using Microsoft.AspNetCore.Components;
+
+namespace AIStudio.Components;
+
+public partial class SelectFile : ComponentBase
+{
+    [Parameter]
+    public string File { get; set; } = string.Empty;
+    
+    [Parameter]
+    public EventCallback<string> FileChanged { get; set; }
+
+    [Parameter]
+    public bool Disabled { get; set; }
+
+    [Parameter]
+    public string Label { get; set; } = string.Empty;
+
+    [Parameter]
+    public string FileDialogTitle { get; set; } = "Select File";
+    
+    [Parameter]
+    public Func<string, string?> Validation { get; set; } = _ => null;
+    
+    [Inject]
+    private SettingsManager SettingsManager { get; init; } = null!;
+
+    [Inject]
+    public RustService RustService { get; set; } = null!;
+    
+    [Inject]
+    protected ILogger<SelectDirectory> Logger { get; init; } = null!;
+    
+    private static readonly Dictionary<string, object?> SPELLCHECK_ATTRIBUTES = new();
+    
+    #region Overrides of ComponentBase
+
+    protected override async Task OnInitializedAsync()
+    {
+        // Configure the spellchecking for the instance name input:
+        this.SettingsManager.InjectSpellchecking(SPELLCHECK_ATTRIBUTES);
+        await base.OnInitializedAsync();
+    }
+
+    #endregion
+
+    private void InternalFileChanged(string file)
+    {
+        this.File = file;
+        this.FileChanged.InvokeAsync(file);
+    }
+
+    private async Task OpenFileDialog()
+    {
+        var response = await this.RustService.SelectFile(this.FileDialogTitle, string.IsNullOrWhiteSpace(this.File) ? null : this.File);
+        this.Logger.LogInformation($"The user selected the file '{response.SelectedFilePath}'.");
+
+        if (!response.UserCancelled)
+            this.InternalFileChanged(response.SelectedFilePath);
+    }
+}

app/MindWork AI Studio/Components/Settings/SettingsPanelDataSources.razor

@@ -0,0 +1,67 @@
+@using AIStudio.Settings
+@using AIStudio.Settings.DataModel
+@inherits SettingsPanelBase
+
+@if (PreviewFeatures.PRE_RAG_2024.IsEnabled(this.SettingsManager))
+{
+    <ExpansionPanel HeaderIcon="@Icons.Material.Filled.IntegrationInstructions" HeaderText="Configure Data Sources">
+        <PreviewPrototype/>
+        <MudText Typo="Typo.h4" Class="mb-3">
+            Configured Data Sources
+        </MudText>
+        <MudJustifiedText Typo="Typo.body1" Class="mb-3">
+            You might configure different data sources. A data source can include one file, all files
+            in a directory, or data from your company. Later, you can incorporate these data sources
+            as needed when the AI requires this data to complete a certain task.
+        </MudJustifiedText>
+
+        <MudTable Items="@this.SettingsManager.ConfigurationData.DataSources" Hover="@true" Class="border-dashed border rounded-lg">
+            <ColGroup>
+                <col style="width: 3em;"/>
+                <col/>
+                <col style="width: 12em;"/>
+                <col style="width: 12em;"/>
+                <col style="width: 40em;"/>
+            </ColGroup>
+            <HeaderContent>
+                <MudTh>#</MudTh>
+                <MudTh>Name</MudTh>
+                <MudTh>Type</MudTh>
+                <MudTh>Embedding</MudTh>
+                <MudTh Style="text-align: left;">Actions</MudTh>
+            </HeaderContent>
+            <RowTemplate>
+                <MudTd>@context.Num</MudTd>
+                <MudTd>@context.Name</MudTd>
+                <MudTd>@context.Type.GetDisplayName()</MudTd>
+                <MudTd>@this.GetEmbeddingName(context)</MudTd>
+                
+                <MudTd Style="text-align: left;">
+                    @if (context is IERIDataSource)
+                    {
+                        @* <MudButton Variant="Variant.Filled" Color="Color.Info" StartIcon="@Icons.Material.Filled.Info" Class="ma-2" OnClick="() => this.ShowInformation(context)"> *@
+                        @*     Show Information *@
+                        @* </MudButton> *@
+                    }
+                    <MudButton Variant="Variant.Filled" Color="Color.Info" StartIcon="@Icons.Material.Filled.Edit" Class="ma-2" OnClick="() => this.EditDataSource(context)">
+                        Edit
+                    </MudButton>
+                    <MudButton Variant="Variant.Filled" Color="Color.Error" StartIcon="@Icons.Material.Filled.Delete" Class="ma-2" OnClick="() => this.DeleteDataSource(context)">
+                        Delete
+                    </MudButton>
+                </MudTd>
+            </RowTemplate>
+        </MudTable>
+
+        @if (this.SettingsManager.ConfigurationData.DataSources.Count == 0)
+        {
+            <MudText Typo="Typo.h6" Class="mt-3">No data sources configured yet.</MudText>
+        }
+        
+        <MudMenu EndIcon="@Icons.Material.Filled.KeyboardArrowDown" Label="Add Data Source" Color="Color.Primary" Variant="Variant.Filled" AnchorOrigin="Origin.CenterCenter" TransformOrigin="Origin.TopLeft" Class="mt-3 mb-6">
+            <MudMenuItem OnClick="() => this.AddDataSource(DataSourceType.ERI_V1)">External Data (ERI-Server v1)</MudMenuItem>
+            <MudMenuItem OnClick="() => this.AddDataSource(DataSourceType.LOCAL_DIRECTORY)">Local Directory</MudMenuItem>
+            <MudMenuItem OnClick="() => this.AddDataSource(DataSourceType.LOCAL_FILE)">Local File</MudMenuItem>
+        </MudMenu>
+    </ExpansionPanel>
+}

app/MindWork AI Studio/Components/Settings/SettingsPanelDataSources.razor.cs

@@ -0,0 +1,233 @@
+using AIStudio.Dialogs;
+using AIStudio.Settings;
+using AIStudio.Settings.DataModel;
+
+using ERI_Client.V1;
+
+using Microsoft.AspNetCore.Components;
+
+using DialogOptions = AIStudio.Dialogs.DialogOptions;
+
+namespace AIStudio.Components.Settings;
+
+public partial class SettingsPanelDataSources : SettingsPanelBase
+{
+    [Parameter]
+    public List<ConfigurationSelectData<string>> AvailableDataSources { get; set; } = new();
+    
+    [Parameter]
+    public EventCallback<List<ConfigurationSelectData<string>>> AvailableDataSourcesChanged { get; set; }
+    
+    [Parameter]
+    public Func<IReadOnlyList<ConfigurationSelectData<string>>> AvailableEmbeddingsFunc { get; set; } = () => [];
+
+    #region Overrides of ComponentBase
+
+    protected override async Task OnInitializedAsync()
+    {
+        await this.UpdateDataSources();
+        await base.OnInitializedAsync();
+    }
+
+    #endregion
+
+    private string GetEmbeddingName(IDataSource dataSource)
+    {
+        if(dataSource is IInternalDataSource internalDataSource)
+        {
+            var matchedEmbedding = this.SettingsManager.ConfigurationData.EmbeddingProviders.FirstOrDefault(x => x.Id == internalDataSource.EmbeddingId);
+            if(matchedEmbedding == default)
+                return "No valid embedding";
+            
+            return matchedEmbedding.Name;
+        }
+        
+        if(dataSource is IExternalDataSource)
+            return "External (ERI)";
+        
+        return "Unknown";
+    }
+    
+    private async Task AddDataSource(DataSourceType type)
+    {
+        IDataSource? addedDataSource = null;
+        switch (type)
+        {
+            case DataSourceType.LOCAL_FILE:
+                var localFileDialogParameters = new DialogParameters<DataSourceLocalFileDialog>
+                {
+                    { x => x.IsEditing, false },
+                    { x => x.AvailableEmbeddings, this.AvailableEmbeddingsFunc() }
+                };
+        
+                var localFileDialogReference = await this.DialogService.ShowAsync<DataSourceLocalFileDialog>("Add Local File as Data Source", localFileDialogParameters, DialogOptions.FULLSCREEN);
+                var localFileDialogResult = await localFileDialogReference.Result;
+                if (localFileDialogResult is null || localFileDialogResult.Canceled)
+                    return;
+                
+                var localFile = (DataSourceLocalFile)localFileDialogResult.Data!;
+                localFile = localFile with { Num = this.SettingsManager.ConfigurationData.NextDataSourceNum++ };
+                addedDataSource = localFile;
+                break;
+            
+            case DataSourceType.LOCAL_DIRECTORY:
+                var localDirectoryDialogParameters = new DialogParameters<DataSourceLocalDirectoryDialog>
+                {
+                    { x => x.IsEditing, false },
+                    { x => x.AvailableEmbeddings, this.AvailableEmbeddingsFunc() }
+                };
+        
+                var localDirectoryDialogReference = await this.DialogService.ShowAsync<DataSourceLocalDirectoryDialog>("Add Local Directory as Data Source", localDirectoryDialogParameters, DialogOptions.FULLSCREEN);
+                var localDirectoryDialogResult = await localDirectoryDialogReference.Result;
+                if (localDirectoryDialogResult is null || localDirectoryDialogResult.Canceled)
+                    return;
+                
+                var localDirectory = (DataSourceLocalDirectory)localDirectoryDialogResult.Data!;
+                localDirectory = localDirectory with { Num = this.SettingsManager.ConfigurationData.NextDataSourceNum++ };
+                addedDataSource = localDirectory;
+                break;
+            
+            case DataSourceType.ERI_V1:
+                var eriDialogParameters = new DialogParameters<DataSourceERI_V1Dialog>
+                {
+                    { x => x.IsEditing, false },
+                };
+                
+                var eriDialogReference = await this.DialogService.ShowAsync<DataSourceERI_V1Dialog>("Add ERI v1 Data Source", eriDialogParameters, DialogOptions.FULLSCREEN);
+                var eriDialogResult = await eriDialogReference.Result;
+                if (eriDialogResult is null || eriDialogResult.Canceled)
+                    return;
+                
+                var eriDataSource = (DataSourceERI_V1)eriDialogResult.Data!;
+                eriDataSource = eriDataSource with { Num = this.SettingsManager.ConfigurationData.NextDataSourceNum++ };
+                addedDataSource = eriDataSource;
+                break;
+        }
+        
+        if(addedDataSource is null)
+            return;
+        
+        this.SettingsManager.ConfigurationData.DataSources.Add(addedDataSource);
+        await this.UpdateDataSources();
+        await this.SettingsManager.StoreSettings();
+        await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
+    }
+    
+    private async Task EditDataSource(IDataSource dataSource)
+    {
+        IDataSource? editedDataSource = null;
+        switch (dataSource)
+        {
+            case DataSourceLocalFile localFile:
+                var localFileDialogParameters = new DialogParameters<DataSourceLocalFileDialog>
+                {
+                    { x => x.IsEditing, true },
+                    { x => x.DataSource, localFile },
+                    { x => x.AvailableEmbeddings, this.AvailableEmbeddingsFunc() }
+                };
+        
+                var localFileDialogReference = await this.DialogService.ShowAsync<DataSourceLocalFileDialog>("Edit Local File Data Source", localFileDialogParameters, DialogOptions.FULLSCREEN);
+                var localFileDialogResult = await localFileDialogReference.Result;
+                if (localFileDialogResult is null || localFileDialogResult.Canceled)
+                    return;
+                
+                editedDataSource = (DataSourceLocalFile)localFileDialogResult.Data!;
+                break;
+            
+            case DataSourceLocalDirectory localDirectory:
+                var localDirectoryDialogParameters = new DialogParameters<DataSourceLocalDirectoryDialog>
+                {
+                    { x => x.IsEditing, true },
+                    { x => x.DataSource, localDirectory },
+                    { x => x.AvailableEmbeddings, this.AvailableEmbeddingsFunc() }
+                };
+        
+                var localDirectoryDialogReference = await this.DialogService.ShowAsync<DataSourceLocalDirectoryDialog>("Edit Local Directory Data Source", localDirectoryDialogParameters, DialogOptions.FULLSCREEN);
+                var localDirectoryDialogResult = await localDirectoryDialogReference.Result;
+                if (localDirectoryDialogResult is null || localDirectoryDialogResult.Canceled)
+                    return;
+                
+                editedDataSource = (DataSourceLocalDirectory)localDirectoryDialogResult.Data!;
+                break;
+            
+            case DataSourceERI_V1 eriDataSource:
+                var eriDialogParameters = new DialogParameters<DataSourceERI_V1Dialog>
+                {
+                    { x => x.IsEditing, true },
+                    { x => x.DataSource, eriDataSource },
+                };
+                
+                var eriDialogReference = await this.DialogService.ShowAsync<DataSourceERI_V1Dialog>("Edit ERI v1 Data Source", eriDialogParameters, DialogOptions.FULLSCREEN);
+                var eriDialogResult = await eriDialogReference.Result;
+                if (eriDialogResult is null || eriDialogResult.Canceled)
+                    return;
+                
+                editedDataSource = (DataSourceERI_V1)eriDialogResult.Data!;
+                break;
+        }
+        
+        if(editedDataSource is null)
+            return;
+        
+        this.SettingsManager.ConfigurationData.DataSources[this.SettingsManager.ConfigurationData.DataSources.IndexOf(dataSource)] = editedDataSource;
+
+        await this.UpdateDataSources();
+        await this.SettingsManager.StoreSettings();
+        await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
+    }
+    
+    private async Task DeleteDataSource(IDataSource dataSource)
+    {
+        var dialogParameters = new DialogParameters
+        {
+            { "Message", $"Are you sure you want to delete the data source '{dataSource.Name}' of type {dataSource.Type.GetDisplayName()}?" },
+        };
+        
+        var dialogReference = await this.DialogService.ShowAsync<ConfirmDialog>("Delete Data Source", dialogParameters, DialogOptions.FULLSCREEN);
+        var dialogResult = await dialogReference.Result;
+        if (dialogResult is null || dialogResult.Canceled)
+            return;
+
+        var applyChanges = dataSource is IInternalDataSource;
+        
+        // External data sources may need a secret for authentication:
+        if (dataSource is IExternalDataSource externalDataSource)
+        {
+            // When the auth method is NONE or KERBEROS, we don't need to delete a secret.
+            // In the case of KERBEROS, we don't store the Kerberos ticket in the secret store.
+            if(dataSource is IERIDataSource { AuthMethod: AuthMethod.NONE or AuthMethod.KERBEROS })
+                applyChanges = true;
+            
+            // All other auth methods require a secret, which we need to delete now:
+            else
+            {
+                var deleteSecretResponse = await this.RustService.DeleteSecret(externalDataSource);
+                if (deleteSecretResponse.Success)
+                    applyChanges = true;
+            }
+        }
+        
+        if(applyChanges)
+        {
+            this.SettingsManager.ConfigurationData.DataSources.Remove(dataSource);
+            await this.SettingsManager.StoreSettings();
+            await this.UpdateDataSources();
+            await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
+        }
+    }
+    
+    private Task ShowInformation(IDataSource dataSource)
+    {
+        #warning Implement the information dialog for ERI data sources.
+        return Task.CompletedTask;
+    }
+    
+    private async Task UpdateDataSources()
+    {
+        this.AvailableDataSources.Clear();
+        foreach (var dataSource in this.SettingsManager.ConfigurationData.DataSources)
+            this.AvailableDataSources.Add(new (dataSource.Name, dataSource.Id));
+        
+        await this.AvailableDataSourcesChanged.InvokeAsync(this.AvailableDataSources);
+    }
+}

app/MindWork AI Studio/Components/Settings/SettingsPanelEmbeddings.razor.cs

@@ -21,7 +21,17 @@ public partial class SettingsPanelEmbeddings : SettingsPanelBase
         var modelName = provider.Model.ToString();
         return modelName.Length > MAX_LENGTH ? "[...] " + modelName[^Math.Min(MAX_LENGTH, modelName.Length)..] : modelName;
     }
-    
+
+    #region Overrides of ComponentBase
+
+    protected override async Task OnInitializedAsync()
+    {
+        await this.UpdateEmbeddingProviders();
+        await base.OnInitializedAsync();
+    }
+
+    #endregion
+
     private async Task AddEmbeddingProvider()
     {
         var dialogParameters = new DialogParameters<EmbeddingProviderDialog>