mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-10-12 20:53:48 +00:00
Added local RAG (#756)
Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>
This commit is contained in:
Paul KoudelkaandThorsten Sommer authored and 
GitHub committed
1 parent
d043fbc8f0
commit
c7b42bee96
179 files changed
+276977
-1421
No files matched your search
@@ -1,85 +1,15 @@
|
||||
@using AIStudio.Settings.DataModel
|
||||
@inherits SettingsDialogBase
|
||||
|
||||
<MudDialog>
|
||||
<TitleContent>
|
||||
<PreviewPrototype/>
|
||||
<PreviewBeta/>
|
||||
<MudText Typo="Typo.h6" Class="d-flex align-center">
|
||||
<MudIcon Icon="@Icons.Material.Filled.IntegrationInstructions" Class="mr-2"/>
|
||||
<MudIcon Icon="@AppIcons.DATABASE" Class="mr-2"/>
|
||||
@T("Configured Data Sources")
|
||||
</MudText>
|
||||
</TitleContent>
|
||||
<DialogContent>
|
||||
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
|
||||
@T("You might configure different data sources. A data source can include one file, all files in a directory, or data from your company. Later, you can incorporate these data sources as needed when the AI requires this data to complete a certain task.")
|
||||
</MudJustifiedText>
|
||||
|
||||
<MudTable Items="@this.SettingsManager.ConfigurationData.DataSources" Hover="@true" Class="border-dashed border rounded-lg">
|
||||
<ColGroup>
|
||||
<col style="width: 3em;"/>
|
||||
<col/>
|
||||
<col style="width: 12em;"/>
|
||||
<col style="width: 12em;"/>
|
||||
<col style="width: 40em;"/>
|
||||
</ColGroup>
|
||||
<HeaderContent>
|
||||
<MudTh>#</MudTh>
|
||||
<MudTh>@T("Name")</MudTh>
|
||||
<MudTh>@T("Type")</MudTh>
|
||||
<MudTh>@T("Embedding")</MudTh>
|
||||
<MudTh>@T("Actions")</MudTh>
|
||||
</HeaderContent>
|
||||
<RowTemplate>
|
||||
<MudTd>@context.Num</MudTd>
|
||||
<MudTd>@context.Name</MudTd>
|
||||
<MudTd>@context.Type.GetDisplayName()</MudTd>
|
||||
<MudTd>@this.GetEmbeddingName(context)</MudTd>
|
||||
|
||||
<MudTd>
|
||||
<MudStack Row="true" Class="mb-2 mt-2" Wrap="Wrap.Wrap">
|
||||
<MudIconButton Variant="Variant.Filled" Color="Color.Info" Icon="@Icons.Material.Filled.Info" OnClick="() => this.ShowInformation(context)"/>
|
||||
@if (context.IsEnterpriseConfiguration)
|
||||
{
|
||||
<MudTooltip Text="@T("This data source is managed by your organization.")">
|
||||
<MudIconButton Color="Color.Info" Icon="@Icons.Material.Filled.Business" Disabled="true"/>
|
||||
</MudTooltip>
|
||||
}
|
||||
else
|
||||
{
|
||||
<MudButton Variant="Variant.Filled" Color="Color.Info" StartIcon="@Icons.Material.Filled.Edit" OnClick="() => this.EditDataSource(context)">
|
||||
@T("Edit")
|
||||
</MudButton>
|
||||
@if (context is DataSourceERI_V1)
|
||||
{
|
||||
<AdminExportButton Variant="Variant.Filled" OnClick="@(() => this.ExportDataSource(context))" />
|
||||
}
|
||||
<MudButton Variant="Variant.Filled" Color="Color.Error" StartIcon="@Icons.Material.Filled.Delete" OnClick="() => this.DeleteDataSource(context)">
|
||||
@T("Delete")
|
||||
</MudButton>
|
||||
}
|
||||
</MudStack>
|
||||
</MudTd>
|
||||
</RowTemplate>
|
||||
</MudTable>
|
||||
|
||||
@if (this.SettingsManager.ConfigurationData.DataSources.Count == 0)
|
||||
{
|
||||
<MudText Typo="Typo.h6" Class="mt-3">
|
||||
@T("No data sources configured yet.")
|
||||
</MudText>
|
||||
}
|
||||
|
||||
<MudMenu EndIcon="@Icons.Material.Filled.KeyboardArrowDown" Label="@T("Add Data Source")" Color="Color.Primary" Variant="Variant.Filled" AnchorOrigin="Origin.CenterCenter" TransformOrigin="Origin.TopLeft" Class="mt-3 mb-6">
|
||||
<MudMenuItem OnClick="() => this.AddDataSource(DataSourceType.ERI_V1)">
|
||||
@T("External Data (ERI-Server v1)")
|
||||
</MudMenuItem>
|
||||
<MudMenuItem OnClick="() => this.AddDataSource(DataSourceType.LOCAL_DIRECTORY)">
|
||||
@T("Local Directory")
|
||||
</MudMenuItem>
|
||||
<MudMenuItem OnClick="() => this.AddDataSource(DataSourceType.LOCAL_FILE)">
|
||||
@T("Local File")
|
||||
</MudMenuItem>
|
||||
</MudMenu>
|
||||
<DataSourceManagement/>
|
||||
</DialogContent>
|
||||
<DialogActions>
|
||||
<MudButton OnClick="@this.Close" Variant="Variant.Filled">
|
||||
|
||||
@@ -1,324 +1,3 @@
|
||||
using AIStudio.Settings;
|
||||
using AIStudio.Settings.DataModel;
|
||||
using AIStudio.Tools.ERIClient.DataModel;
|
||||
using AIStudio.Tools.PluginSystem;
|
||||
|
||||
namespace AIStudio.Dialogs.Settings;
|
||||
|
||||
public partial class SettingsDialogDataSources : SettingsDialogBase
|
||||
{
|
||||
private string GetEmbeddingName(IDataSource dataSource)
|
||||
{
|
||||
if(dataSource is IInternalDataSource internalDataSource)
|
||||
{
|
||||
var matchedEmbedding = this.SettingsManager.ConfigurationData.EmbeddingProviders.FirstOrDefault(x => x.Id == internalDataSource.EmbeddingId);
|
||||
if(matchedEmbedding == default)
|
||||
return T("No valid embedding");
|
||||
|
||||
return matchedEmbedding.Name;
|
||||
}
|
||||
|
||||
if(dataSource is IExternalDataSource)
|
||||
return T("External (ERI)");
|
||||
|
||||
return T("Unknown");
|
||||
}
|
||||
|
||||
private async Task AddDataSource(DataSourceType type)
|
||||
{
|
||||
IDataSource? addedDataSource = null;
|
||||
switch (type)
|
||||
{
|
||||
case DataSourceType.LOCAL_FILE:
|
||||
var localFileDialogParameters = new DialogParameters<DataSourceLocalFileDialog>
|
||||
{
|
||||
{ x => x.IsEditing, false },
|
||||
{ x => x.AvailableEmbeddings, this.AvailableEmbeddingProviders }
|
||||
};
|
||||
|
||||
var localFileDialogReference = await this.DialogService.ShowAsync<DataSourceLocalFileDialog>(T("Add Local File as Data Source"), localFileDialogParameters, DialogOptions.FULLSCREEN);
|
||||
var localFileDialogResult = await localFileDialogReference.Result;
|
||||
if (localFileDialogResult is null || localFileDialogResult.Canceled)
|
||||
return;
|
||||
|
||||
var localFile = (DataSourceLocalFile)localFileDialogResult.Data!;
|
||||
localFile = localFile with { Num = this.SettingsManager.ConfigurationData.NextDataSourceNum++ };
|
||||
addedDataSource = localFile;
|
||||
break;
|
||||
|
||||
case DataSourceType.LOCAL_DIRECTORY:
|
||||
var localDirectoryDialogParameters = new DialogParameters<DataSourceLocalDirectoryDialog>
|
||||
{
|
||||
{ x => x.IsEditing, false },
|
||||
{ x => x.AvailableEmbeddings, this.AvailableEmbeddingProviders }
|
||||
};
|
||||
|
||||
var localDirectoryDialogReference = await this.DialogService.ShowAsync<DataSourceLocalDirectoryDialog>(T("Add Local Directory as Data Source"), localDirectoryDialogParameters, DialogOptions.FULLSCREEN);
|
||||
var localDirectoryDialogResult = await localDirectoryDialogReference.Result;
|
||||
if (localDirectoryDialogResult is null || localDirectoryDialogResult.Canceled)
|
||||
return;
|
||||
|
||||
var localDirectory = (DataSourceLocalDirectory)localDirectoryDialogResult.Data!;
|
||||
localDirectory = localDirectory with { Num = this.SettingsManager.ConfigurationData.NextDataSourceNum++ };
|
||||
addedDataSource = localDirectory;
|
||||
break;
|
||||
|
||||
case DataSourceType.ERI_V1:
|
||||
var eriDialogParameters = new DialogParameters<DataSourceERI_V1Dialog>
|
||||
{
|
||||
{ x => x.IsEditing, false },
|
||||
};
|
||||
|
||||
var eriDialogReference = await this.DialogService.ShowAsync<DataSourceERI_V1Dialog>(T("Add ERI v1 Data Source"), eriDialogParameters, DialogOptions.FULLSCREEN);
|
||||
var eriDialogResult = await eriDialogReference.Result;
|
||||
if (eriDialogResult is null || eriDialogResult.Canceled)
|
||||
return;
|
||||
|
||||
var eriDataSource = (DataSourceERI_V1)eriDialogResult.Data!;
|
||||
eriDataSource = eriDataSource with { Num = this.SettingsManager.ConfigurationData.NextDataSourceNum++ };
|
||||
addedDataSource = eriDataSource;
|
||||
break;
|
||||
}
|
||||
|
||||
if(addedDataSource is null)
|
||||
return;
|
||||
|
||||
this.SettingsManager.ConfigurationData.DataSources.Add(addedDataSource);
|
||||
await this.SettingsManager.StoreSettings();
|
||||
await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
|
||||
}
|
||||
|
||||
private async Task ExportDataSource(IDataSource dataSource)
|
||||
{
|
||||
if (!this.SettingsManager.ConfigurationData.App.ShowAdminSettings)
|
||||
return;
|
||||
|
||||
if (dataSource is not DataSourceERI_V1 eriDataSource)
|
||||
return;
|
||||
|
||||
if (eriDataSource.AuthMethod is AuthMethod.KERBEROS)
|
||||
{
|
||||
await this.DialogService.ShowMessageBox(
|
||||
T("Export ERI Data Source"),
|
||||
T("Kerberos/SSO ERI data sources cannot be exported yet. Please configure them manually in the configuration plugin."),
|
||||
T("Close"));
|
||||
return;
|
||||
}
|
||||
|
||||
var needsSecret = eriDataSource.AuthMethod is AuthMethod.TOKEN or AuthMethod.USERNAME_PASSWORD;
|
||||
if (!needsSecret)
|
||||
{
|
||||
var publicLuaCode = eriDataSource.ExportAsConfigurationSection();
|
||||
if (!string.IsNullOrWhiteSpace(publicLuaCode))
|
||||
await this.RustService.CopyText2Clipboard(publicLuaCode);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
var secretResponse = await this.RustService.GetSecret(eriDataSource, SecretStoreType.DATA_SOURCE, isTrying: true);
|
||||
if (!secretResponse.Success)
|
||||
{
|
||||
await this.DialogService.ShowMessageBox(
|
||||
T("Export ERI Data Source"),
|
||||
string.Format(T("Cannot export this ERI data source because no authentication secret is configured. The issue was: {0}"), secretResponse.Issue),
|
||||
T("Close"));
|
||||
return;
|
||||
}
|
||||
|
||||
var encryption = PluginFactory.EnterpriseEncryption;
|
||||
if (encryption?.IsAvailable != true)
|
||||
{
|
||||
await this.DialogService.ShowMessageBox(
|
||||
T("Export ERI Data Source"),
|
||||
T("Cannot export this ERI data source because no enterprise encryption secret is configured."),
|
||||
T("Close"));
|
||||
return;
|
||||
}
|
||||
|
||||
var usernamePasswordMode = DataSourceERIUsernamePasswordMode.USER_MANAGED;
|
||||
if (eriDataSource.AuthMethod is AuthMethod.TOKEN)
|
||||
{
|
||||
var dialogParameters = new DialogParameters<ConfirmDialog>
|
||||
{
|
||||
{ x => x.Message, T("This ERI data source has an access token configured. Do you want to include the encrypted access token in the export? Note: The recipient will need the same encryption secret to use the access token.") },
|
||||
};
|
||||
|
||||
var dialogReference = await this.DialogService.ShowAsync<ConfirmDialog>(T("Export Access Token?"), dialogParameters, DialogOptions.FULLSCREEN);
|
||||
var dialogResult = await dialogReference.Result;
|
||||
if (dialogResult is null || dialogResult.Canceled)
|
||||
return;
|
||||
}
|
||||
else if (eriDataSource.AuthMethod is AuthMethod.USERNAME_PASSWORD)
|
||||
{
|
||||
var dialogParameters = new DialogParameters<DataSourceERIV1UsernamePasswordExportDialog>
|
||||
{
|
||||
{ x => x.DataSource, eriDataSource },
|
||||
};
|
||||
|
||||
var dialogReference = await this.DialogService.ShowAsync<DataSourceERIV1UsernamePasswordExportDialog>(T("Export ERI Data Source"), dialogParameters, DialogOptions.FULLSCREEN);
|
||||
var dialogResult = await dialogReference.Result;
|
||||
if (dialogResult is null || dialogResult.Canceled || dialogResult.Data is not DataSourceERIV1UsernamePasswordExportDialogResult exportResult)
|
||||
return;
|
||||
|
||||
usernamePasswordMode = exportResult.UsernamePasswordMode;
|
||||
}
|
||||
|
||||
var decryptedSecret = await secretResponse.Secret.Decrypt(Program.ENCRYPTION);
|
||||
if (!encryption.TryEncrypt(decryptedSecret, out var encryptedSecret))
|
||||
{
|
||||
await this.DialogService.ShowMessageBox(
|
||||
T("Export ERI Data Source"),
|
||||
T("Cannot export this ERI data source because the authentication secret could not be encrypted."),
|
||||
T("Close"));
|
||||
return;
|
||||
}
|
||||
|
||||
var luaCode = eriDataSource.ExportAsConfigurationSection(
|
||||
encryptedSecret,
|
||||
usernamePasswordMode);
|
||||
if (string.IsNullOrWhiteSpace(luaCode))
|
||||
return;
|
||||
|
||||
await this.RustService.CopyText2Clipboard(luaCode);
|
||||
}
|
||||
|
||||
private async Task EditDataSource(IDataSource dataSource)
|
||||
{
|
||||
if (dataSource.IsEnterpriseConfiguration)
|
||||
return;
|
||||
|
||||
IDataSource? editedDataSource = null;
|
||||
switch (dataSource)
|
||||
{
|
||||
case DataSourceLocalFile localFile:
|
||||
var localFileDialogParameters = new DialogParameters<DataSourceLocalFileDialog>
|
||||
{
|
||||
{ x => x.IsEditing, true },
|
||||
{ x => x.DataSource, localFile },
|
||||
{ x => x.AvailableEmbeddings, this.AvailableEmbeddingProviders }
|
||||
};
|
||||
|
||||
var localFileDialogReference = await this.DialogService.ShowAsync<DataSourceLocalFileDialog>(T("Edit Local File Data Source"), localFileDialogParameters, DialogOptions.FULLSCREEN);
|
||||
var localFileDialogResult = await localFileDialogReference.Result;
|
||||
if (localFileDialogResult is null || localFileDialogResult.Canceled)
|
||||
return;
|
||||
|
||||
editedDataSource = (DataSourceLocalFile)localFileDialogResult.Data!;
|
||||
break;
|
||||
|
||||
case DataSourceLocalDirectory localDirectory:
|
||||
var localDirectoryDialogParameters = new DialogParameters<DataSourceLocalDirectoryDialog>
|
||||
{
|
||||
{ x => x.IsEditing, true },
|
||||
{ x => x.DataSource, localDirectory },
|
||||
{ x => x.AvailableEmbeddings, this.AvailableEmbeddingProviders }
|
||||
};
|
||||
|
||||
var localDirectoryDialogReference = await this.DialogService.ShowAsync<DataSourceLocalDirectoryDialog>(T("Edit Local Directory Data Source"), localDirectoryDialogParameters, DialogOptions.FULLSCREEN);
|
||||
var localDirectoryDialogResult = await localDirectoryDialogReference.Result;
|
||||
if (localDirectoryDialogResult is null || localDirectoryDialogResult.Canceled)
|
||||
return;
|
||||
|
||||
editedDataSource = (DataSourceLocalDirectory)localDirectoryDialogResult.Data!;
|
||||
break;
|
||||
|
||||
case DataSourceERI_V1 eriDataSource:
|
||||
var eriDialogParameters = new DialogParameters<DataSourceERI_V1Dialog>
|
||||
{
|
||||
{ x => x.IsEditing, true },
|
||||
{ x => x.DataSource, eriDataSource },
|
||||
};
|
||||
|
||||
var eriDialogReference = await this.DialogService.ShowAsync<DataSourceERI_V1Dialog>(T("Edit ERI v1 Data Source"), eriDialogParameters, DialogOptions.FULLSCREEN);
|
||||
var eriDialogResult = await eriDialogReference.Result;
|
||||
if (eriDialogResult is null || eriDialogResult.Canceled)
|
||||
return;
|
||||
|
||||
editedDataSource = (DataSourceERI_V1)eriDialogResult.Data!;
|
||||
break;
|
||||
}
|
||||
|
||||
if(editedDataSource is null)
|
||||
return;
|
||||
|
||||
this.SettingsManager.ConfigurationData.DataSources[this.SettingsManager.ConfigurationData.DataSources.IndexOf(dataSource)] = editedDataSource;
|
||||
|
||||
await this.SettingsManager.StoreSettings();
|
||||
await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
|
||||
}
|
||||
|
||||
private async Task DeleteDataSource(IDataSource dataSource)
|
||||
{
|
||||
if (dataSource.IsEnterpriseConfiguration)
|
||||
return;
|
||||
|
||||
var dialogParameters = new DialogParameters<ConfirmDialog>
|
||||
{
|
||||
{ x => x.Message, string.Format(T("Are you sure you want to delete the data source '{0}' of type {1}?"), dataSource.Name, dataSource.Type.GetDisplayName()) },
|
||||
};
|
||||
|
||||
var dialogReference = await this.DialogService.ShowAsync<ConfirmDialog>(T("Delete Data Source"), dialogParameters, DialogOptions.FULLSCREEN);
|
||||
var dialogResult = await dialogReference.Result;
|
||||
if (dialogResult is null || dialogResult.Canceled)
|
||||
return;
|
||||
|
||||
var applyChanges = dataSource is IInternalDataSource;
|
||||
|
||||
// External data sources may need a secret for authentication:
|
||||
if (dataSource is IExternalDataSource externalDataSource)
|
||||
{
|
||||
// When the auth method is NONE or KERBEROS, we don't need to delete a secret.
|
||||
// In the case of KERBEROS, we don't store the Kerberos ticket in the secret store.
|
||||
if(dataSource is IERIDataSource { AuthMethod: AuthMethod.NONE or AuthMethod.KERBEROS })
|
||||
applyChanges = true;
|
||||
|
||||
// All other auth methods require a secret, which we need to delete now:
|
||||
else
|
||||
{
|
||||
var deleteSecretResponse = await this.RustService.DeleteSecret(externalDataSource, SecretStoreType.DATA_SOURCE);
|
||||
if (deleteSecretResponse.Success)
|
||||
applyChanges = true;
|
||||
}
|
||||
}
|
||||
|
||||
if(applyChanges)
|
||||
{
|
||||
this.SettingsManager.ConfigurationData.DataSources.Remove(dataSource);
|
||||
await this.SettingsManager.StoreSettings();
|
||||
await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ShowInformation(IDataSource dataSource)
|
||||
{
|
||||
switch (dataSource)
|
||||
{
|
||||
case DataSourceLocalFile localFile:
|
||||
var localFileDialogParameters = new DialogParameters<DataSourceLocalFileInfoDialog>
|
||||
{
|
||||
{ x => x.DataSource, localFile },
|
||||
};
|
||||
|
||||
await this.DialogService.ShowAsync<DataSourceLocalFileInfoDialog>(T("Local File Data Source Information"), localFileDialogParameters, DialogOptions.FULLSCREEN);
|
||||
break;
|
||||
|
||||
case DataSourceLocalDirectory localDirectory:
|
||||
var localDirectoryDialogParameters = new DialogParameters<DataSourceLocalDirectoryInfoDialog>
|
||||
{
|
||||
{ x => x.DataSource, localDirectory },
|
||||
};
|
||||
|
||||
await this.DialogService.ShowAsync<DataSourceLocalDirectoryInfoDialog>(T("Local Directory Data Source Information"), localDirectoryDialogParameters, DialogOptions.FULLSCREEN);
|
||||
break;
|
||||
|
||||
case DataSourceERI_V1 eriV1DataSource:
|
||||
var eriV1DialogParameters = new DialogParameters<DataSourceERI_V1InfoDialog>
|
||||
{
|
||||
{ x => x.DataSource, eriV1DataSource },
|
||||
};
|
||||
|
||||
await this.DialogService.ShowAsync<DataSourceERI_V1InfoDialog>(T("ERI v1 Data Source Information"), eriV1DialogParameters, DialogOptions.FULLSCREEN);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
public partial class SettingsDialogDataSources : SettingsDialogBase;
|
||||
@@ -3,7 +3,7 @@
|
||||
|
||||
<MudDialog>
|
||||
<TitleContent>
|
||||
<PreviewPrototype/>
|
||||
<PreviewBeta/>
|
||||
<MudText Typo="Typo.h6" Class="d-flex align-center">
|
||||
<MudIcon Icon="@Icons.Material.Filled.PrivateConnectivity" Class="mr-2"/>
|
||||
@T("Assistant: ERI Server Options")
|
||||
|
||||
Reference in new issue
Block a user