From e85a75aa9668172b8214106510f5aee931e3fe06 Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Mon, 10 Feb 2025 22:19:27 +0100 Subject: [PATCH] Added an option to all data sources to select a local security policy --- .../Dialogs/DataSourceERI-V1InfoDialog.razor | 5 ++- .../Dialogs/DataSourceERI_V1Dialog.razor | 8 +++++ .../Dialogs/DataSourceERI_V1Dialog.razor.cs | 26 +++++++++++++++ .../DataSourceLocalDirectoryDialog.razor | 9 ++++++ .../DataSourceLocalDirectoryDialog.razor.cs | 2 ++ .../DataSourceLocalDirectoryInfoDialog.razor | 4 +++ .../Dialogs/DataSourceLocalFileDialog.razor | 8 +++++ .../DataSourceLocalFileDialog.razor.cs | 2 ++ .../DataSourceLocalFileInfoDialog.razor | 3 ++ .../Settings/DataModel/DataSourceERI_V1.cs | 3 ++ .../DataModel/DataSourceLocalDirectory.cs | 3 ++ .../Settings/DataModel/DataSourceLocalFile.cs | 3 ++ .../Settings/DataModel/DataSourceSecurity.cs | 19 +++++++++++ .../DataModel/DataSourceSecurityExtensions.cs | 32 +++++++++++++++++++ .../Settings/IDataSource.cs | 5 +++ .../Tools/Validation/DataSourceValidation.cs | 18 +++++++++++ .../wwwroot/changelog/v0.9.29.md | 3 +- 17 files changed, 151 insertions(+), 2 deletions(-) create mode 100644 app/MindWork AI Studio/Settings/DataModel/DataSourceSecurity.cs create mode 100644 app/MindWork AI Studio/Settings/DataModel/DataSourceSecurityExtensions.cs diff --git a/app/MindWork AI Studio/Dialogs/DataSourceERI-V1InfoDialog.razor b/app/MindWork AI Studio/Dialogs/DataSourceERI-V1InfoDialog.razor index a1ebe82d..984d2e40 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceERI-V1InfoDialog.razor +++ b/app/MindWork AI Studio/Dialogs/DataSourceERI-V1InfoDialog.razor @@ -1,4 +1,6 @@ +@using AIStudio.Settings.DataModel @using AIStudio.Tools.ERIClient.DataModel + @@ -25,7 +27,8 @@ } - + + Retrieval information diff --git a/app/MindWork AI Studio/Dialogs/DataSourceERI_V1Dialog.razor b/app/MindWork AI Studio/Dialogs/DataSourceERI_V1Dialog.razor index ac40ffa5..fcc9fff0 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceERI_V1Dialog.razor +++ b/app/MindWork AI Studio/Dialogs/DataSourceERI_V1Dialog.razor @@ -1,3 +1,4 @@ +@using AIStudio.Settings.DataModel @using AIStudio.Tools.ERIClient.DataModel @@ -108,6 +109,13 @@ InputType="InputType.Password" UserAttributes="@SPELLCHECK_ATTRIBUTES"/> } + + + @foreach (var policy in Enum.GetValues()) + { + @policy.ToSelectionText() + } + diff --git a/app/MindWork AI Studio/Dialogs/DataSourceERI_V1Dialog.razor.cs b/app/MindWork AI Studio/Dialogs/DataSourceERI_V1Dialog.razor.cs index aaa4dac1..03867de5 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceERI_V1Dialog.razor.cs +++ b/app/MindWork AI Studio/Dialogs/DataSourceERI_V1Dialog.razor.cs @@ -45,6 +45,8 @@ public partial class DataSourceERI_V1Dialog : ComponentBase, ISecretId private string dataSecretStorageIssue = string.Empty; private string dataEditingPreviousInstanceName = string.Empty; private List availableAuthMethods = []; + private DataSourceSecurity dataSecurityPolicy; + private SecurityRequirements dataSourceSecurityRequirements; private bool connectionTested; private bool connectionSuccessfulTested; @@ -71,6 +73,7 @@ public partial class DataSourceERI_V1Dialog : ComponentBase, ISecretId GetTestedConnection = () => this.connectionTested, GetTestedConnectionResult = () => this.connectionSuccessfulTested, GetAvailableAuthMethods = () => this.availableAuthMethods, + GetSecurityRequirements = () => this.dataSourceSecurityRequirements, }; } @@ -147,6 +150,7 @@ public partial class DataSourceERI_V1Dialog : ComponentBase, ISecretId AuthMethod = this.dataAuthMethod, Username = this.dataUsername, Type = DataSourceType.ERI_V1, + SecurityPolicy = this.dataSecurityPolicy, }; } @@ -196,6 +200,28 @@ public partial class DataSourceERI_V1Dialog : ComponentBase, ISecretId this.availableAuthMethods = authSchemes.Data!.Select(n => n.AuthMethod).ToList(); + var loginResult = await client.AuthenticateAsync(this.DataSource, this.RustService, cts.Token); + if (!loginResult.Successful) + { + await this.form.Validate(); + + Array.Resize(ref this.dataIssues, this.dataIssues.Length + 1); + this.dataIssues[^1] = loginResult.Message; + return; + } + + var securityRequirementsRequest = await client.GetSecurityRequirementsAsync(cts.Token); + if (!securityRequirementsRequest.Successful) + { + await this.form.Validate(); + + Array.Resize(ref this.dataIssues, this.dataIssues.Length + 1); + this.dataIssues[^1] = securityRequirementsRequest.Message; + return; + } + + this.dataSourceSecurityRequirements = securityRequirementsRequest.Data; + this.connectionTested = true; this.connectionSuccessfulTested = true; this.Logger.LogInformation("Connection to the ERI v1 server was successful tested."); diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor index 22d1c984..e7863285 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor @@ -1,3 +1,5 @@ +@using AIStudio.Settings.DataModel + @@ -60,6 +62,13 @@ } } + + + @foreach (var policy in Enum.GetValues()) + { + @policy.ToSelectionText() + } + diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs index 9f1b4f1f..ffbd7307 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs @@ -42,6 +42,7 @@ public partial class DataSourceLocalDirectoryDialog : ComponentBase private bool dataUserAcknowledgedCloudEmbedding; private string dataEmbeddingId = string.Empty; private string dataPath = string.Empty; + private DataSourceSecurity dataSecurityPolicy; // We get the form reference from Blazor code to validate it manually: private MudForm form = null!; @@ -102,6 +103,7 @@ public partial class DataSourceLocalDirectoryDialog : ComponentBase Type = DataSourceType.LOCAL_DIRECTORY, EmbeddingId = this.dataEmbeddingId, Path = this.dataPath, + SecurityPolicy = this.dataSecurityPolicy, }; private async Task Store() diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryInfoDialog.razor b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryInfoDialog.razor index cbd951dc..a4b647b0 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryInfoDialog.razor +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryInfoDialog.razor @@ -1,3 +1,5 @@ +@using AIStudio.Settings.DataModel + @@ -31,6 +33,8 @@ } + + @if (this.directorySizeNumFiles > 100) diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor index c60d32dc..ebd0a5bc 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor @@ -1,3 +1,4 @@ +@using AIStudio.Settings.DataModel @@ -59,6 +60,13 @@ } } + + + @foreach (var policy in Enum.GetValues()) + { + @policy.ToSelectionText() + } + diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs index 6a2cd14d..051155a1 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs @@ -42,6 +42,7 @@ public partial class DataSourceLocalFileDialog : ComponentBase private bool dataUserAcknowledgedCloudEmbedding; private string dataEmbeddingId = string.Empty; private string dataFilePath = string.Empty; + private DataSourceSecurity dataSecurityPolicy; // We get the form reference from Blazor code to validate it manually: private MudForm form = null!; @@ -102,6 +103,7 @@ public partial class DataSourceLocalFileDialog : ComponentBase Type = DataSourceType.LOCAL_FILE, EmbeddingId = this.dataEmbeddingId, FilePath = this.dataFilePath, + SecurityPolicy = this.dataSecurityPolicy, }; private async Task Store() diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileInfoDialog.razor b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileInfoDialog.razor index 6fb4b12c..0605ff93 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileInfoDialog.razor +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileInfoDialog.razor @@ -1,3 +1,5 @@ +@using AIStudio.Settings.DataModel + @@ -31,6 +33,7 @@ } + diff --git a/app/MindWork AI Studio/Settings/DataModel/DataSourceERI_V1.cs b/app/MindWork AI Studio/Settings/DataModel/DataSourceERI_V1.cs index addec256..a9931e38 100644 --- a/app/MindWork AI Studio/Settings/DataModel/DataSourceERI_V1.cs +++ b/app/MindWork AI Studio/Settings/DataModel/DataSourceERI_V1.cs @@ -36,4 +36,7 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource /// public string Username { get; init; } = string.Empty; + + /// + public DataSourceSecurity SecurityPolicy { get; init; } = DataSourceSecurity.NOT_SPECIFIED; } \ No newline at end of file diff --git a/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalDirectory.cs b/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalDirectory.cs index 963b9ba1..61c30d93 100644 --- a/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalDirectory.cs +++ b/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalDirectory.cs @@ -24,6 +24,9 @@ public readonly record struct DataSourceLocalDirectory : IInternalDataSource /// public string EmbeddingId { get; init; } = Guid.Empty.ToString(); + /// + public DataSourceSecurity SecurityPolicy { get; init; } = DataSourceSecurity.NOT_SPECIFIED; + /// /// The path to the directory. /// diff --git a/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalFile.cs b/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalFile.cs index a6088190..571fb0a8 100644 --- a/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalFile.cs +++ b/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalFile.cs @@ -24,6 +24,9 @@ public readonly record struct DataSourceLocalFile : IInternalDataSource /// public string EmbeddingId { get; init; } = Guid.Empty.ToString(); + /// + public DataSourceSecurity SecurityPolicy { get; init; } = DataSourceSecurity.NOT_SPECIFIED; + /// /// The path to the file. /// diff --git a/app/MindWork AI Studio/Settings/DataModel/DataSourceSecurity.cs b/app/MindWork AI Studio/Settings/DataModel/DataSourceSecurity.cs new file mode 100644 index 00000000..58caf457 --- /dev/null +++ b/app/MindWork AI Studio/Settings/DataModel/DataSourceSecurity.cs @@ -0,0 +1,19 @@ +namespace AIStudio.Settings.DataModel; + +public enum DataSourceSecurity +{ + /// + /// The security of the data source is not specified yet. + /// + NOT_SPECIFIED, + + /// + /// This data can be used with any LLM provider. + /// + ALLOW_ANY, + + /// + /// This data can only be used for self-hosted LLM providers. + /// + SELF_HOSTED, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Settings/DataModel/DataSourceSecurityExtensions.cs b/app/MindWork AI Studio/Settings/DataModel/DataSourceSecurityExtensions.cs new file mode 100644 index 00000000..6e52d0fd --- /dev/null +++ b/app/MindWork AI Studio/Settings/DataModel/DataSourceSecurityExtensions.cs @@ -0,0 +1,32 @@ +namespace AIStudio.Settings.DataModel; + +public static class DataSourceSecurityExtensions +{ + public static string ToSelectionText(this DataSourceSecurity security) => security switch + { + DataSourceSecurity.NOT_SPECIFIED => "Please select a security policy", + + DataSourceSecurity.ALLOW_ANY => "This data source can be used with any LLM provider. Your data may be sent to a cloud-based provider.", + DataSourceSecurity.SELF_HOSTED => "This data source can only be used with a self-hosted LLM provider. Your data will not be sent to any cloud-based provider.", + + _ => "Unknown security policy" + }; + + public static string ToInfoText(this DataSourceSecurity security) => security switch + { + DataSourceSecurity.NOT_SPECIFIED => "The security of the data source is not specified yet. You cannot use this data source until you specify a security policy.", + + DataSourceSecurity.ALLOW_ANY => "This data source can be used with any LLM provider. Your data may be sent to a cloud-based provider.", + DataSourceSecurity.SELF_HOSTED => "This data source can only be used with a self-hosted LLM provider. Your data will not be sent to any cloud-based provider.", + + _ => "Unknown security policy" + }; + + public static TextColor GetColor(this DataSourceSecurity security) => security switch + { + DataSourceSecurity.ALLOW_ANY => TextColor.WARN, + DataSourceSecurity.SELF_HOSTED => TextColor.SUCCESS, + + _ => TextColor.ERROR + }; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Settings/IDataSource.cs b/app/MindWork AI Studio/Settings/IDataSource.cs index 28bc3b9c..72f4ad3c 100644 --- a/app/MindWork AI Studio/Settings/IDataSource.cs +++ b/app/MindWork AI Studio/Settings/IDataSource.cs @@ -32,4 +32,9 @@ public interface IDataSource /// Which type of data source is this? /// public DataSourceType Type { get; init; } + + /// + /// Which data security policy is applied to this data source? + /// + public DataSourceSecurity SecurityPolicy { get; init; } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Validation/DataSourceValidation.cs b/app/MindWork AI Studio/Tools/Validation/DataSourceValidation.cs index cdd2f131..b3f85426 100644 --- a/app/MindWork AI Studio/Tools/Validation/DataSourceValidation.cs +++ b/app/MindWork AI Studio/Tools/Validation/DataSourceValidation.cs @@ -1,3 +1,4 @@ +using AIStudio.Settings.DataModel; using AIStudio.Tools.ERIClient.DataModel; namespace AIStudio.Tools.Validation; @@ -11,6 +12,8 @@ public sealed class DataSourceValidation public Func> GetUsedDataSourceNames { get; init; } = () => []; public Func GetAuthMethod { get; init; } = () => AuthMethod.NONE; + + public Func GetSecurityRequirements { get; init; } = () => null; public Func GetSelectedCloudEmbedding { get; init; } = () => false; @@ -42,6 +45,21 @@ public sealed class DataSourceValidation return null; } + public string? ValidateSecurityPolicy(DataSourceSecurity securityPolicy) + { + if(securityPolicy is DataSourceSecurity.NOT_SPECIFIED) + return "Please select your security policy."; + + var dataSourceSecurity = this.GetSecurityRequirements(); + if (dataSourceSecurity is null) + return null; + + if(dataSourceSecurity.Value.AllowedProviderType is ProviderType.SELF_HOSTED && securityPolicy is not DataSourceSecurity.SELF_HOSTED) + return "This data source can only be used with a self-hosted LLM provider. Please change the security policy."; + + return null; + } + public string? ValidateUsername(string username) { if(this.GetAuthMethod() is not AuthMethod.USERNAME_PASSWORD) diff --git a/app/MindWork AI Studio/wwwroot/changelog/v0.9.29.md b/app/MindWork AI Studio/wwwroot/changelog/v0.9.29.md index f98b4187..5dfb690f 100644 --- a/app/MindWork AI Studio/wwwroot/changelog/v0.9.29.md +++ b/app/MindWork AI Studio/wwwroot/changelog/v0.9.29.md @@ -1,2 +1,3 @@ # v0.9.29, build 204 (2025-02-xx xx:xx UTC) -- Added the possibility to select data sources for chats. This preview feature is hidden behind the RAG feature flag, check your app options in case you want to enable it. \ No newline at end of file +- Added the possibility to select data sources for chats. This preview feature is hidden behind the RAG feature flag, check your app options in case you want to enable it. +- Added an option to all data sources to select a local security policy. This preview feature is hidden behind the RAG feature flag. \ No newline at end of file