mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-10-13 19:53:47 +00:00
Configure data sources (#259)
This commit is contained in:
Thorsten Sommer authored and 
GitHub committed
1 parent
06f66fdab2
commit
63be312bb4
47 files changed
+3388
-283
No files matched your search
@@ -0,0 +1,67 @@
|
||||
@using AIStudio.Settings
|
||||
@using AIStudio.Settings.DataModel
|
||||
@inherits SettingsPanelBase
|
||||
|
||||
@if (PreviewFeatures.PRE_RAG_2024.IsEnabled(this.SettingsManager))
|
||||
{
|
||||
<ExpansionPanel HeaderIcon="@Icons.Material.Filled.IntegrationInstructions" HeaderText="Configure Data Sources">
|
||||
<PreviewPrototype/>
|
||||
<MudText Typo="Typo.h4" Class="mb-3">
|
||||
Configured Data Sources
|
||||
</MudText>
|
||||
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
|
||||
You might configure different data sources. A data source can include one file, all files
|
||||
in a directory, or data from your company. Later, you can incorporate these data sources
|
||||
as needed when the AI requires this data to complete a certain task.
|
||||
</MudJustifiedText>
|
||||
|
||||
<MudTable Items="@this.SettingsManager.ConfigurationData.DataSources" Hover="@true" Class="border-dashed border rounded-lg">
|
||||
<ColGroup>
|
||||
<col style="width: 3em;"/>
|
||||
<col/>
|
||||
<col style="width: 12em;"/>
|
||||
<col style="width: 12em;"/>
|
||||
<col style="width: 40em;"/>
|
||||
</ColGroup>
|
||||
<HeaderContent>
|
||||
<MudTh>#</MudTh>
|
||||
<MudTh>Name</MudTh>
|
||||
<MudTh>Type</MudTh>
|
||||
<MudTh>Embedding</MudTh>
|
||||
<MudTh Style="text-align: left;">Actions</MudTh>
|
||||
</HeaderContent>
|
||||
<RowTemplate>
|
||||
<MudTd>@context.Num</MudTd>
|
||||
<MudTd>@context.Name</MudTd>
|
||||
<MudTd>@context.Type.GetDisplayName()</MudTd>
|
||||
<MudTd>@this.GetEmbeddingName(context)</MudTd>
|
||||
|
||||
<MudTd Style="text-align: left;">
|
||||
@if (context is IERIDataSource)
|
||||
{
|
||||
@* <MudButton Variant="Variant.Filled" Color="Color.Info" StartIcon="@Icons.Material.Filled.Info" Class="ma-2" OnClick="() => this.ShowInformation(context)"> *@
|
||||
@* Show Information *@
|
||||
@* </MudButton> *@
|
||||
}
|
||||
<MudButton Variant="Variant.Filled" Color="Color.Info" StartIcon="@Icons.Material.Filled.Edit" Class="ma-2" OnClick="() => this.EditDataSource(context)">
|
||||
Edit
|
||||
</MudButton>
|
||||
<MudButton Variant="Variant.Filled" Color="Color.Error" StartIcon="@Icons.Material.Filled.Delete" Class="ma-2" OnClick="() => this.DeleteDataSource(context)">
|
||||
Delete
|
||||
</MudButton>
|
||||
</MudTd>
|
||||
</RowTemplate>
|
||||
</MudTable>
|
||||
|
||||
@if (this.SettingsManager.ConfigurationData.DataSources.Count == 0)
|
||||
{
|
||||
<MudText Typo="Typo.h6" Class="mt-3">No data sources configured yet.</MudText>
|
||||
}
|
||||
|
||||
<MudMenu EndIcon="@Icons.Material.Filled.KeyboardArrowDown" Label="Add Data Source" Color="Color.Primary" Variant="Variant.Filled" AnchorOrigin="Origin.CenterCenter" TransformOrigin="Origin.TopLeft" Class="mt-3 mb-6">
|
||||
<MudMenuItem OnClick="() => this.AddDataSource(DataSourceType.ERI_V1)">External Data (ERI-Server v1)</MudMenuItem>
|
||||
<MudMenuItem OnClick="() => this.AddDataSource(DataSourceType.LOCAL_DIRECTORY)">Local Directory</MudMenuItem>
|
||||
<MudMenuItem OnClick="() => this.AddDataSource(DataSourceType.LOCAL_FILE)">Local File</MudMenuItem>
|
||||
</MudMenu>
|
||||
</ExpansionPanel>
|
||||
}
|
||||
@@ -0,0 +1,233 @@
|
||||
using AIStudio.Dialogs;
|
||||
using AIStudio.Settings;
|
||||
using AIStudio.Settings.DataModel;
|
||||
|
||||
using ERI_Client.V1;
|
||||
|
||||
using Microsoft.AspNetCore.Components;
|
||||
|
||||
using DialogOptions = AIStudio.Dialogs.DialogOptions;
|
||||
|
||||
namespace AIStudio.Components.Settings;
|
||||
|
||||
public partial class SettingsPanelDataSources : SettingsPanelBase
|
||||
{
|
||||
[Parameter]
|
||||
public List<ConfigurationSelectData<string>> AvailableDataSources { get; set; } = new();
|
||||
|
||||
[Parameter]
|
||||
public EventCallback<List<ConfigurationSelectData<string>>> AvailableDataSourcesChanged { get; set; }
|
||||
|
||||
[Parameter]
|
||||
public Func<IReadOnlyList<ConfigurationSelectData<string>>> AvailableEmbeddingsFunc { get; set; } = () => [];
|
||||
|
||||
#region Overrides of ComponentBase
|
||||
|
||||
protected override async Task OnInitializedAsync()
|
||||
{
|
||||
await this.UpdateDataSources();
|
||||
await base.OnInitializedAsync();
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
private string GetEmbeddingName(IDataSource dataSource)
|
||||
{
|
||||
if(dataSource is IInternalDataSource internalDataSource)
|
||||
{
|
||||
var matchedEmbedding = this.SettingsManager.ConfigurationData.EmbeddingProviders.FirstOrDefault(x => x.Id == internalDataSource.EmbeddingId);
|
||||
if(matchedEmbedding == default)
|
||||
return "No valid embedding";
|
||||
|
||||
return matchedEmbedding.Name;
|
||||
}
|
||||
|
||||
if(dataSource is IExternalDataSource)
|
||||
return "External (ERI)";
|
||||
|
||||
return "Unknown";
|
||||
}
|
||||
|
||||
private async Task AddDataSource(DataSourceType type)
|
||||
{
|
||||
IDataSource? addedDataSource = null;
|
||||
switch (type)
|
||||
{
|
||||
case DataSourceType.LOCAL_FILE:
|
||||
var localFileDialogParameters = new DialogParameters<DataSourceLocalFileDialog>
|
||||
{
|
||||
{ x => x.IsEditing, false },
|
||||
{ x => x.AvailableEmbeddings, this.AvailableEmbeddingsFunc() }
|
||||
};
|
||||
|
||||
var localFileDialogReference = await this.DialogService.ShowAsync<DataSourceLocalFileDialog>("Add Local File as Data Source", localFileDialogParameters, DialogOptions.FULLSCREEN);
|
||||
var localFileDialogResult = await localFileDialogReference.Result;
|
||||
if (localFileDialogResult is null || localFileDialogResult.Canceled)
|
||||
return;
|
||||
|
||||
var localFile = (DataSourceLocalFile)localFileDialogResult.Data!;
|
||||
localFile = localFile with { Num = this.SettingsManager.ConfigurationData.NextDataSourceNum++ };
|
||||
addedDataSource = localFile;
|
||||
break;
|
||||
|
||||
case DataSourceType.LOCAL_DIRECTORY:
|
||||
var localDirectoryDialogParameters = new DialogParameters<DataSourceLocalDirectoryDialog>
|
||||
{
|
||||
{ x => x.IsEditing, false },
|
||||
{ x => x.AvailableEmbeddings, this.AvailableEmbeddingsFunc() }
|
||||
};
|
||||
|
||||
var localDirectoryDialogReference = await this.DialogService.ShowAsync<DataSourceLocalDirectoryDialog>("Add Local Directory as Data Source", localDirectoryDialogParameters, DialogOptions.FULLSCREEN);
|
||||
var localDirectoryDialogResult = await localDirectoryDialogReference.Result;
|
||||
if (localDirectoryDialogResult is null || localDirectoryDialogResult.Canceled)
|
||||
return;
|
||||
|
||||
var localDirectory = (DataSourceLocalDirectory)localDirectoryDialogResult.Data!;
|
||||
localDirectory = localDirectory with { Num = this.SettingsManager.ConfigurationData.NextDataSourceNum++ };
|
||||
addedDataSource = localDirectory;
|
||||
break;
|
||||
|
||||
case DataSourceType.ERI_V1:
|
||||
var eriDialogParameters = new DialogParameters<DataSourceERI_V1Dialog>
|
||||
{
|
||||
{ x => x.IsEditing, false },
|
||||
};
|
||||
|
||||
var eriDialogReference = await this.DialogService.ShowAsync<DataSourceERI_V1Dialog>("Add ERI v1 Data Source", eriDialogParameters, DialogOptions.FULLSCREEN);
|
||||
var eriDialogResult = await eriDialogReference.Result;
|
||||
if (eriDialogResult is null || eriDialogResult.Canceled)
|
||||
return;
|
||||
|
||||
var eriDataSource = (DataSourceERI_V1)eriDialogResult.Data!;
|
||||
eriDataSource = eriDataSource with { Num = this.SettingsManager.ConfigurationData.NextDataSourceNum++ };
|
||||
addedDataSource = eriDataSource;
|
||||
break;
|
||||
}
|
||||
|
||||
if(addedDataSource is null)
|
||||
return;
|
||||
|
||||
this.SettingsManager.ConfigurationData.DataSources.Add(addedDataSource);
|
||||
await this.UpdateDataSources();
|
||||
await this.SettingsManager.StoreSettings();
|
||||
await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
|
||||
}
|
||||
|
||||
private async Task EditDataSource(IDataSource dataSource)
|
||||
{
|
||||
IDataSource? editedDataSource = null;
|
||||
switch (dataSource)
|
||||
{
|
||||
case DataSourceLocalFile localFile:
|
||||
var localFileDialogParameters = new DialogParameters<DataSourceLocalFileDialog>
|
||||
{
|
||||
{ x => x.IsEditing, true },
|
||||
{ x => x.DataSource, localFile },
|
||||
{ x => x.AvailableEmbeddings, this.AvailableEmbeddingsFunc() }
|
||||
};
|
||||
|
||||
var localFileDialogReference = await this.DialogService.ShowAsync<DataSourceLocalFileDialog>("Edit Local File Data Source", localFileDialogParameters, DialogOptions.FULLSCREEN);
|
||||
var localFileDialogResult = await localFileDialogReference.Result;
|
||||
if (localFileDialogResult is null || localFileDialogResult.Canceled)
|
||||
return;
|
||||
|
||||
editedDataSource = (DataSourceLocalFile)localFileDialogResult.Data!;
|
||||
break;
|
||||
|
||||
case DataSourceLocalDirectory localDirectory:
|
||||
var localDirectoryDialogParameters = new DialogParameters<DataSourceLocalDirectoryDialog>
|
||||
{
|
||||
{ x => x.IsEditing, true },
|
||||
{ x => x.DataSource, localDirectory },
|
||||
{ x => x.AvailableEmbeddings, this.AvailableEmbeddingsFunc() }
|
||||
};
|
||||
|
||||
var localDirectoryDialogReference = await this.DialogService.ShowAsync<DataSourceLocalDirectoryDialog>("Edit Local Directory Data Source", localDirectoryDialogParameters, DialogOptions.FULLSCREEN);
|
||||
var localDirectoryDialogResult = await localDirectoryDialogReference.Result;
|
||||
if (localDirectoryDialogResult is null || localDirectoryDialogResult.Canceled)
|
||||
return;
|
||||
|
||||
editedDataSource = (DataSourceLocalDirectory)localDirectoryDialogResult.Data!;
|
||||
break;
|
||||
|
||||
case DataSourceERI_V1 eriDataSource:
|
||||
var eriDialogParameters = new DialogParameters<DataSourceERI_V1Dialog>
|
||||
{
|
||||
{ x => x.IsEditing, true },
|
||||
{ x => x.DataSource, eriDataSource },
|
||||
};
|
||||
|
||||
var eriDialogReference = await this.DialogService.ShowAsync<DataSourceERI_V1Dialog>("Edit ERI v1 Data Source", eriDialogParameters, DialogOptions.FULLSCREEN);
|
||||
var eriDialogResult = await eriDialogReference.Result;
|
||||
if (eriDialogResult is null || eriDialogResult.Canceled)
|
||||
return;
|
||||
|
||||
editedDataSource = (DataSourceERI_V1)eriDialogResult.Data!;
|
||||
break;
|
||||
}
|
||||
|
||||
if(editedDataSource is null)
|
||||
return;
|
||||
|
||||
this.SettingsManager.ConfigurationData.DataSources[this.SettingsManager.ConfigurationData.DataSources.IndexOf(dataSource)] = editedDataSource;
|
||||
|
||||
await this.UpdateDataSources();
|
||||
await this.SettingsManager.StoreSettings();
|
||||
await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
|
||||
}
|
||||
|
||||
private async Task DeleteDataSource(IDataSource dataSource)
|
||||
{
|
||||
var dialogParameters = new DialogParameters
|
||||
{
|
||||
{ "Message", $"Are you sure you want to delete the data source '{dataSource.Name}' of type {dataSource.Type.GetDisplayName()}?" },
|
||||
};
|
||||
|
||||
var dialogReference = await this.DialogService.ShowAsync<ConfirmDialog>("Delete Data Source", dialogParameters, DialogOptions.FULLSCREEN);
|
||||
var dialogResult = await dialogReference.Result;
|
||||
if (dialogResult is null || dialogResult.Canceled)
|
||||
return;
|
||||
|
||||
var applyChanges = dataSource is IInternalDataSource;
|
||||
|
||||
// External data sources may need a secret for authentication:
|
||||
if (dataSource is IExternalDataSource externalDataSource)
|
||||
{
|
||||
// When the auth method is NONE or KERBEROS, we don't need to delete a secret.
|
||||
// In the case of KERBEROS, we don't store the Kerberos ticket in the secret store.
|
||||
if(dataSource is IERIDataSource { AuthMethod: AuthMethod.NONE or AuthMethod.KERBEROS })
|
||||
applyChanges = true;
|
||||
|
||||
// All other auth methods require a secret, which we need to delete now:
|
||||
else
|
||||
{
|
||||
var deleteSecretResponse = await this.RustService.DeleteSecret(externalDataSource);
|
||||
if (deleteSecretResponse.Success)
|
||||
applyChanges = true;
|
||||
}
|
||||
}
|
||||
|
||||
if(applyChanges)
|
||||
{
|
||||
this.SettingsManager.ConfigurationData.DataSources.Remove(dataSource);
|
||||
await this.SettingsManager.StoreSettings();
|
||||
await this.UpdateDataSources();
|
||||
await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
|
||||
}
|
||||
}
|
||||
|
||||
private Task ShowInformation(IDataSource dataSource)
|
||||
{
|
||||
#warning Implement the information dialog for ERI data sources.
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
private async Task UpdateDataSources()
|
||||
{
|
||||
this.AvailableDataSources.Clear();
|
||||
foreach (var dataSource in this.SettingsManager.ConfigurationData.DataSources)
|
||||
this.AvailableDataSources.Add(new (dataSource.Name, dataSource.Id));
|
||||
|
||||
await this.AvailableDataSourcesChanged.InvokeAsync(this.AvailableDataSources);
|
||||
}
|
||||
}
|
||||
@@ -21,7 +21,17 @@ public partial class SettingsPanelEmbeddings : SettingsPanelBase
|
||||
var modelName = provider.Model.ToString();
|
||||
return modelName.Length > MAX_LENGTH ? "[...] " + modelName[^Math.Min(MAX_LENGTH, modelName.Length)..] : modelName;
|
||||
}
|
||||
|
||||
|
||||
#region Overrides of ComponentBase
|
||||
|
||||
protected override async Task OnInitializedAsync()
|
||||
{
|
||||
await this.UpdateEmbeddingProviders();
|
||||
await base.OnInitializedAsync();
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
private async Task AddEmbeddingProvider()
|
||||
{
|
||||
var dialogParameters = new DialogParameters<EmbeddingProviderDialog>
|
||||
|
||||
Reference in new issue
Block a user