diff --git a/app/MindWork AI Studio/Agents/AgentDataSourceSelection.cs b/app/MindWork AI Studio/Agents/AgentDataSourceSelection.cs index f7947462..c7743252 100644 --- a/app/MindWork AI Studio/Agents/AgentDataSourceSelection.cs +++ b/app/MindWork AI Studio/Agents/AgentDataSourceSelection.cs @@ -140,6 +140,8 @@ public sealed class AgentDataSourceSelection (ILogger // // We start with the provider currently selected by the user: + var requiredDataSecurity = dataSources.AllowedDataSources.GetRequiredSecurityPolicy(); + var requiredComplianceLevel = dataSources.AllowedDataSources.GetRequiredComplianceLevel(); var agentProvider = this.SettingsManager.GetPreselectedProvider(Tools.Components.AGENT_DATA_SOURCE_SELECTION, provider.Id, true); if (agentProvider == Settings.Provider.NONE) { @@ -147,6 +149,12 @@ public sealed class AgentDataSourceSelection (ILogger return []; } + if (!agentProvider.AllowsDataSourceAccess(this.SettingsManager, requiredDataSecurity, requiredComplianceLevel)) + { + logger.LogWarning($"The agent for data source selection uses provider '{agentProvider.InstanceName}' with confidence '{agentProvider.GetConfidenceLevel(this.SettingsManager).GetName()}', but the available data sources require data security '{requiredDataSecurity}' and provider confidence '{requiredComplianceLevel.GetName()}'. The agent cannot select data sources."); + return []; + } + // Assign the provider settings to the agent: logger.LogInformation($"The agent for the data source selection uses the provider '{agentProvider.InstanceName}' ({agentProvider.UsedLLMProvider.ToName()}, confidence={agentProvider.UsedLLMProvider.GetConfidence(this.SettingsManager).Level.GetName()})."); this.ProviderSettings = agentProvider; diff --git a/app/MindWork AI Studio/Agents/AgentRetrievalContextValidation.cs b/app/MindWork AI Studio/Agents/AgentRetrievalContextValidation.cs index ee2437d9..a0fd5be6 100644 --- a/app/MindWork AI Studio/Agents/AgentRetrievalContextValidation.cs +++ b/app/MindWork AI Studio/Agents/AgentRetrievalContextValidation.cs @@ -3,6 +3,7 @@ using System.Text.Json; using AIStudio.Chat; using AIStudio.Provider; using AIStudio.Settings; +using AIStudio.Settings.DataModel; using AIStudio.Tools.RAG; using AIStudio.Tools.Services; @@ -129,19 +130,30 @@ public sealed class AgentRetrievalContextValidation (ILogger /// The current LLM provider. When the user doesn't preselect an agent provider, the agent uses this provider. - public void SetLLMProvider(IProvider provider) + /// The data security required by the retrieved data. + /// The minimum provider confidence required by the retrieved data. + public bool SetLLMProvider(IProvider provider, DataSourceSecurity requiredDataSecurity = DataSourceSecurity.NOT_SPECIFIED, ConfidenceLevel requiredComplianceLevel = ConfidenceLevel.NONE) { // We start with the provider currently selected by the user: var agentProvider = this.SettingsManager.GetPreselectedProvider(Tools.Components.AGENT_RETRIEVAL_CONTEXT_VALIDATION, provider.Id, true); if (agentProvider == Settings.Provider.NONE) { logger.LogWarning("No provider is selected for the agent."); - return; + this.ProviderSettings = Settings.Provider.NONE; + return false; + } + + if (!agentProvider.AllowsDataSourceAccess(this.SettingsManager, requiredDataSecurity, requiredComplianceLevel)) + { + logger.LogWarning($"The agent for retrieval context validation uses provider '{agentProvider.InstanceName}' with confidence '{agentProvider.GetConfidenceLevel(this.SettingsManager).GetName()}', but the retrieved data requires data security '{requiredDataSecurity}' and provider confidence '{requiredComplianceLevel.GetName()}'. The agent cannot validate retrieval contexts."); + this.ProviderSettings = Settings.Provider.NONE; + return false; } // Assign the provider settings to the agent: logger.LogInformation($"The agent for the retrieval context validation uses the provider '{agentProvider.InstanceName}' ({agentProvider.UsedLLMProvider.ToName()}, confidence={agentProvider.UsedLLMProvider.GetConfidence(this.SettingsManager).Level.GetName()})."); this.ProviderSettings = agentProvider; + return true; } /// diff --git a/app/MindWork AI Studio/Chat/ChatThread.cs b/app/MindWork AI Studio/Chat/ChatThread.cs index 3b00805a..7d13cd4d 100644 --- a/app/MindWork AI Studio/Chat/ChatThread.cs +++ b/app/MindWork AI Studio/Chat/ChatThread.cs @@ -1,6 +1,7 @@ using System.Globalization; using AIStudio.Components; +using AIStudio.Provider; using AIStudio.Settings; using AIStudio.Settings.DataModel; using AIStudio.Tools.ERIClient.DataModel; @@ -76,6 +77,11 @@ public sealed record ChatThread /// public DataSourceSecurity DataSecurity { get; set; } = DataSourceSecurity.NOT_SPECIFIED; + /// + /// The minimum provider confidence required by data sources used so far. + /// + public ConfidenceLevel DataComplianceLevel { get; set; } = ConfidenceLevel.NONE; + /// /// The name of the chat thread. Usually generated by an AI model or manually edited by the user. /// diff --git a/app/MindWork AI Studio/Chat/ChatThreadExtensions.cs b/app/MindWork AI Studio/Chat/ChatThreadExtensions.cs index 2eb5395b..4d673cfd 100644 --- a/app/MindWork AI Studio/Chat/ChatThreadExtensions.cs +++ b/app/MindWork AI Studio/Chat/ChatThreadExtensions.cs @@ -11,12 +11,12 @@ public static class ChatThreadExtensions /// /// /// We don't check if the provider is allowed to use the data sources of the chat thread. - /// That kind of check is done in the RAG process itself.

+ /// That kind of check is done when the available data sources are resolved.

/// /// One thing which is not so obvious: after RAG was used on this thread, the entire chat /// thread is kind of a data source by itself. Why? Because the augmentation data collected /// from the data sources is stored in the chat thread. This means we must check if the - /// selected provider is allowed to use this thread's data. + /// selected provider is allowed to use this thread's data security and compliance level. /// /// The chat thread to check. /// The provider to check. @@ -26,7 +26,19 @@ public static class ChatThreadExtensions // No chat thread available means we have a new chat. That's fine: if (chatThread is null) return true; - + + var settingsManager = Program.SERVICE_PROVIDER.GetRequiredService(); + var providerConfidenceLevel = provider switch + { + IProvider p => p.GetConfidenceLevel(settingsManager), + AIStudio.Settings.Provider p => p.GetConfidenceLevel(settingsManager), + + _ => ConfidenceLevel.NONE, + }; + + if (!providerConfidenceLevel.AllowsDataSourceComplianceLevel(chatThread.DataComplianceLevel)) + return false; + // The chat thread is available, but the data security is not specified. // Means, we never used RAG or RAG was enabled, but no data sources were selected. // That's fine as well: @@ -36,7 +48,6 @@ public static class ChatThreadExtensions // // Is the provider trusted for data-source security checks? // - var settingsManager = Program.SERVICE_PROVIDER.GetRequiredService(); var isTrustedProvider = provider switch { IProvider p => p.IsTrustedForDataSourceSecurityChecks(settingsManager), diff --git a/app/MindWork AI Studio/Chat/ContentText.cs b/app/MindWork AI Studio/Chat/ContentText.cs index 4c8be646..f57054e8 100644 --- a/app/MindWork AI Studio/Chat/ContentText.cs +++ b/app/MindWork AI Studio/Chat/ContentText.cs @@ -57,7 +57,7 @@ public sealed class ContentText : IContent if(!chatThread.IsLLMProviderAllowed(provider)) { - LOGGER.LogError("The provider is not allowed for this chat thread due to data security reasons. Skipping the AI process."); + LOGGER.LogError("The provider is not allowed for this chat thread due to data security or compliance reasons. Skipping the AI process."); await this.CompleteWithoutStreaming(); return chatThread; } diff --git a/app/MindWork AI Studio/Components/ChatComponent.razor b/app/MindWork AI Studio/Components/ChatComponent.razor index 322eaa94..bcd0c17e 100644 --- a/app/MindWork AI Studio/Components/ChatComponent.razor +++ b/app/MindWork AI Studio/Components/ChatComponent.razor @@ -147,7 +147,7 @@ @if (!this.ChatThread.IsLLMProviderAllowed(this.Provider)) { - + } diff --git a/app/MindWork AI Studio/Components/DataSourceSelection.razor b/app/MindWork AI Studio/Components/DataSourceSelection.razor index c5f1be6c..74566d7e 100644 --- a/app/MindWork AI Studio/Components/DataSourceSelection.razor +++ b/app/MindWork AI Studio/Components/DataSourceSelection.razor @@ -70,7 +70,7 @@ { case true when this.availableDataSources.Count == 0: - @T("Your data sources cannot be used with the LLM provider you selected due to data privacy, or they are currently unavailable.") + @T("Your data sources cannot be used with the LLM provider you selected due to data privacy or compliance requirements, or they are currently unavailable.") break; @@ -82,7 +82,7 @@ case false when this.availableDataSources.Count == 0: - @T("Your data sources cannot be used with the LLM provider you selected due to data privacy, or they are currently unavailable.") + @T("Your data sources cannot be used with the LLM provider you selected due to data privacy or compliance requirements, or they are currently unavailable.") break; @@ -177,4 +177,4 @@ else if (this.SelectionMode is DataSourceSelectionMode.CONFIGURATION_MODE) } -} \ No newline at end of file +} diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor index d4370433..954c1f56 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor @@ -73,7 +73,7 @@ @if (this.CanChangeSourceAndEmbedding) { - + @foreach (var embedding in this.AvailableEmbeddings) { @@ -99,18 +99,6 @@ @if (!string.IsNullOrWhiteSpace(this.dataEmbeddingId)) { - - if (this.SelectedCloudEmbedding) { @@ -121,59 +109,11 @@ @T("The embedding you selected runs locally or in your organization. Your data is not sent to the cloud.") } - - - - @(this.showExpertSettings ? T("Hide Expert Settings") : T("Show Expert Settings")) - - - - - @T("Optional expert settings for how this data source is split before embedding.") - - - - - } - - @foreach (var policy in Enum.GetValues()) - { - - @policy.ToSelectionText() - - } - - - + @foreach (var level in this.ComplianceLevels) { @@ -182,7 +122,56 @@ } - + + + @(this.showExpertSettings ? T("Hide Expert Settings") : T("Show Expert Settings")) + + + + @if (!string.IsNullOrWhiteSpace(this.dataEmbeddingId)) + { + + } + + @T("Optional expert settings for how this data source is split before embedding.") + + + + + + diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs index c179f210..ba88041e 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs @@ -49,7 +49,6 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase private int dataChunkOverlapTokenLength; private ushort dataMaxMatches = 10; private bool showExpertSettings; - private DataSourceSecurity dataSecurityPolicy; private ConfidenceLevel dataComplianceLevel = ConfidenceLevel.UNKNOWN; // We get the form reference from Blazor code to validate it manually: @@ -60,6 +59,10 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase this.dataSourceValidation = new() { GetSelectedCloudEmbedding = () => this.SelectedCloudEmbedding, + GetSelectedEmbeddingProvider = () => this.SelectedEmbedding, + GetSecurityPolicy = () => DataSourceSecurity.ALLOW_ANY, + GetComplianceLevel = () => this.dataComplianceLevel, + GetSettingsManager = () => this.SettingsManager, GetPreviousDataSourceName = () => this.dataEditingPreviousInstanceName, GetUsedDataSourceNames = () => this.UsedDataSourcesNames, }; @@ -87,7 +90,6 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase this.dataPath = this.DataSource.Path; this.dataMaxChunkTokenLength = this.DataSource.MaxChunkTokenLength; this.dataChunkOverlapTokenLength = this.DataSource.ChunkOverlapTokenLength; - this.dataSecurityPolicy = this.DataSource.SecurityPolicy; this.dataComplianceLevel = this.DataSource.ComplianceLevel; this.dataMaxMatches = this.DataSource.MaxMatches; this.showExpertSettings = this.dataMaxChunkTokenLength > 0 || this.dataChunkOverlapTokenLength > 0; @@ -143,7 +145,7 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase Path = this.CanChangeSourceAndEmbedding ? this.dataPath : this.DataSource.Path, MaxChunkTokenLength = this.dataMaxChunkTokenLength, ChunkOverlapTokenLength = this.dataChunkOverlapTokenLength, - SecurityPolicy = this.dataSecurityPolicy, + SecurityPolicy = DataSourceSecurity.ALLOW_ANY, ComplianceLevel = this.dataComplianceLevel, MaxMatches = this.dataMaxMatches, }; diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryInfoDialog.razor b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryInfoDialog.razor index b75c8fdd..0d4feace 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryInfoDialog.razor +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryInfoDialog.razor @@ -38,7 +38,6 @@ } - diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor index a8e3d09e..912c016b 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor @@ -73,7 +73,7 @@ @if (this.CanChangeSourceAndEmbedding) { - + @foreach (var embedding in this.AvailableEmbeddings) { @@ -164,16 +164,7 @@ - - @foreach (var policy in Enum.GetValues()) - { - - @policy.ToSelectionText() - - } - - - + @foreach (var level in this.ComplianceLevels) { diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs index 31c5e75e..a0491649 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs @@ -49,7 +49,6 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase private int dataChunkOverlapTokenLength; private ushort dataMaxMatches = 10; private bool showExpertSettings; - private DataSourceSecurity dataSecurityPolicy; private ConfidenceLevel dataComplianceLevel = ConfidenceLevel.UNKNOWN; // We get the form reference from Blazor code to validate it manually: @@ -60,6 +59,10 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase this.dataSourceValidation = new() { GetSelectedCloudEmbedding = () => this.SelectedCloudEmbedding, + GetSelectedEmbeddingProvider = () => this.SelectedEmbedding, + GetSecurityPolicy = () => DataSourceSecurity.ALLOW_ANY, + GetComplianceLevel = () => this.dataComplianceLevel, + GetSettingsManager = () => this.SettingsManager, GetPreviousDataSourceName = () => this.dataEditingPreviousInstanceName, GetUsedDataSourceNames = () => this.UsedDataSourcesNames, }; @@ -87,7 +90,6 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase this.dataFilePath = this.DataSource.FilePath; this.dataMaxChunkTokenLength = this.DataSource.MaxChunkTokenLength; this.dataChunkOverlapTokenLength = this.DataSource.ChunkOverlapTokenLength; - this.dataSecurityPolicy = this.DataSource.SecurityPolicy; this.dataComplianceLevel = this.DataSource.ComplianceLevel; this.dataMaxMatches = this.DataSource.MaxMatches; this.showExpertSettings = this.dataMaxChunkTokenLength > 0 || this.dataChunkOverlapTokenLength > 0; @@ -143,7 +145,7 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase FilePath = this.CanChangeSourceAndEmbedding ? this.dataFilePath : this.DataSource.FilePath, MaxChunkTokenLength = this.dataMaxChunkTokenLength, ChunkOverlapTokenLength = this.dataChunkOverlapTokenLength, - SecurityPolicy = this.dataSecurityPolicy, + SecurityPolicy = DataSourceSecurity.ALLOW_ANY, ComplianceLevel = this.dataComplianceLevel, MaxMatches = this.dataMaxMatches, }; diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileInfoDialog.razor b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileInfoDialog.razor index 9b997ec3..37e58dff 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileInfoDialog.razor +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileInfoDialog.razor @@ -38,7 +38,6 @@ } - diff --git a/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalDirectory.cs b/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalDirectory.cs index efed00aa..c7d37140 100644 --- a/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalDirectory.cs +++ b/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalDirectory.cs @@ -42,7 +42,7 @@ public readonly record struct DataSourceLocalDirectory : IInternalDataSource public int ChunkOverlapTokenLength { get; init; } /// - public DataSourceSecurity SecurityPolicy { get; init; } = DataSourceSecurity.NOT_SPECIFIED; + public DataSourceSecurity SecurityPolicy { get; init; } = DataSourceSecurity.ALLOW_ANY; /// public ConfidenceLevel ComplianceLevel { get; init; } = ConfidenceLevel.UNKNOWN; diff --git a/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalFile.cs b/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalFile.cs index 130c161a..20971a83 100644 --- a/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalFile.cs +++ b/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalFile.cs @@ -42,7 +42,7 @@ public readonly record struct DataSourceLocalFile : IInternalDataSource public int ChunkOverlapTokenLength { get; init; } /// - public DataSourceSecurity SecurityPolicy { get; init; } = DataSourceSecurity.NOT_SPECIFIED; + public DataSourceSecurity SecurityPolicy { get; init; } = DataSourceSecurity.ALLOW_ANY; /// public ConfidenceLevel ComplianceLevel { get; init; } = ConfidenceLevel.UNKNOWN; diff --git a/app/MindWork AI Studio/Settings/DataSourceSecurityTrustExtensions.cs b/app/MindWork AI Studio/Settings/DataSourceSecurityTrustExtensions.cs index bff1f898..f70216b9 100644 --- a/app/MindWork AI Studio/Settings/DataSourceSecurityTrustExtensions.cs +++ b/app/MindWork AI Studio/Settings/DataSourceSecurityTrustExtensions.cs @@ -1,4 +1,5 @@ using AIStudio.Provider; +using AIStudio.Settings.DataModel; namespace AIStudio.Settings; @@ -36,6 +37,97 @@ public static class DataSourceSecurityTrustExtensions return provider.Provider is LLMProviders.SELF_HOSTED || IsTrustedProviderId(provider.ConfiguredProviderId, settingsManager); } + public static ConfidenceLevel GetConfidenceLevel(this Provider provider, SettingsManager settingsManager) + { + if (provider == Provider.NONE) + return ConfidenceLevel.NONE; + + return provider.UsedLLMProvider.GetConfidence(settingsManager).Level; + } + + public static ConfidenceLevel GetConfidenceLevel(this EmbeddingProvider provider, SettingsManager settingsManager) + { + if (provider == EmbeddingProvider.NONE) + return ConfidenceLevel.NONE; + + return provider.UsedLLMProvider.GetConfidence(settingsManager).Level; + } + + public static ConfidenceLevel GetConfidenceLevel(this IProvider provider, SettingsManager settingsManager) + { + if (provider is NoProvider) + return ConfidenceLevel.NONE; + + return provider.Provider.GetConfidence(settingsManager).Level; + } + + public static bool AllowsDataSourceAccess(this Provider provider, SettingsManager settingsManager, DataSourceSecurity dataSourceSecurity, ConfidenceLevel dataSourceComplianceLevel) + { + return provider.AllowsDataSourceSecurity(dataSourceSecurity, settingsManager) + && provider.GetConfidenceLevel(settingsManager).AllowsDataSourceComplianceLevel(dataSourceComplianceLevel); + } + + public static bool AllowsDataSourceAccess(this EmbeddingProvider provider, SettingsManager settingsManager, DataSourceSecurity dataSourceSecurity, ConfidenceLevel dataSourceComplianceLevel) + { + return provider.AllowsDataSourceSecurity(dataSourceSecurity, settingsManager) + && provider.GetConfidenceLevel(settingsManager).AllowsDataSourceComplianceLevel(dataSourceComplianceLevel); + } + + public static bool AllowsDataSourceAccess(this IProvider provider, SettingsManager settingsManager, DataSourceSecurity dataSourceSecurity, ConfidenceLevel dataSourceComplianceLevel) + { + return provider.AllowsDataSourceSecurity(dataSourceSecurity, settingsManager) + && provider.GetConfidenceLevel(settingsManager).AllowsDataSourceComplianceLevel(dataSourceComplianceLevel); + } + + public static bool AllowsDataSourceSecurity(this Provider provider, DataSourceSecurity dataSourceSecurity, SettingsManager settingsManager) + => provider.IsTrustedForDataSourceSecurityChecks(settingsManager).AllowsDataSourceSecurity(dataSourceSecurity); + + public static bool AllowsDataSourceSecurity(this EmbeddingProvider provider, DataSourceSecurity dataSourceSecurity, SettingsManager settingsManager) + => provider.IsTrustedForDataSourceSecurityChecks(settingsManager).AllowsDataSourceSecurity(dataSourceSecurity); + + public static bool AllowsDataSourceSecurity(this IProvider provider, DataSourceSecurity dataSourceSecurity, SettingsManager settingsManager) + => provider.IsTrustedForDataSourceSecurityChecks(settingsManager).AllowsDataSourceSecurity(dataSourceSecurity); + + public static bool AllowsDataSourceSecurity(this bool usingTrustedProvider, DataSourceSecurity dataSourceSecurity) => dataSourceSecurity switch + { + DataSourceSecurity.ALLOW_ANY => true, + DataSourceSecurity.SELF_HOSTED => usingTrustedProvider, + _ => false, + }; + + public static bool AllowsDataSourceComplianceLevel(this ConfidenceLevel providerConfidenceLevel, ConfidenceLevel dataSourceComplianceLevel) + { + if (dataSourceComplianceLevel is ConfidenceLevel.NONE) + return true; + + return providerConfidenceLevel >= dataSourceComplianceLevel; + } + + public static ConfidenceLevel GetRequiredComplianceLevel(this IEnumerable dataSources) + { + var requiredComplianceLevel = ConfidenceLevel.NONE; + foreach (var dataSource in dataSources) + if (dataSource.ComplianceLevel > requiredComplianceLevel) + requiredComplianceLevel = dataSource.ComplianceLevel; + + return requiredComplianceLevel; + } + + public static DataSourceSecurity GetRequiredSecurityPolicy(this IEnumerable dataSources) + { + var requiredSecurityPolicy = DataSourceSecurity.ALLOW_ANY; + foreach (var dataSource in dataSources) + { + if (dataSource.SecurityPolicy is DataSourceSecurity.NOT_SPECIFIED) + return DataSourceSecurity.NOT_SPECIFIED; + + if (dataSource.SecurityPolicy is DataSourceSecurity.SELF_HOSTED) + requiredSecurityPolicy = DataSourceSecurity.SELF_HOSTED; + } + + return requiredSecurityPolicy; + } + public static bool IsTrustedByConfiguration(this Provider provider, SettingsManager settingsManager) => IsTrustedProviderId(provider.Id, settingsManager); public static bool IsTrustedByConfiguration(this EmbeddingProvider provider, SettingsManager settingsManager) => IsTrustedProviderId(provider.Id, settingsManager); diff --git a/app/MindWork AI Studio/Tools/RAG/AugmentationProcesses/AugmentationOne.cs b/app/MindWork AI Studio/Tools/RAG/AugmentationProcesses/AugmentationOne.cs index e6a63a3d..b3c56369 100644 --- a/app/MindWork AI Studio/Tools/RAG/AugmentationProcesses/AugmentationOne.cs +++ b/app/MindWork AI Studio/Tools/RAG/AugmentationProcesses/AugmentationOne.cs @@ -43,19 +43,22 @@ public sealed class AugmentationOne : IAugmentationProcess { // Let's get the validation agent & set up its provider: var validationAgent = Program.SERVICE_PROVIDER.GetService()!; - validationAgent.SetLLMProvider(provider); + if (validationAgent.SetLLMProvider(provider, chatThread.DataSecurity, chatThread.DataComplianceLevel)) + { + // Let's validate all retrieval contexts: + var validationResults = await validationAgent.ValidateRetrievalContextsAsync(lastUserPrompt, chatThread, retrievalContexts, token); + + // + // Now, filter the retrieval contexts to the most relevant ones: + // + var targetWindow = validationResults.DetermineTargetWindow(TargetWindowStrategy.TOP10_BETTER_THAN_GUESSING); + var threshold = validationResults.GetConfidenceThreshold(targetWindow); - // Let's validate all retrieval contexts: - var validationResults = await validationAgent.ValidateRetrievalContextsAsync(lastUserPrompt, chatThread, retrievalContexts, token); - - // - // Now, filter the retrieval contexts to the most relevant ones: - // - var targetWindow = validationResults.DetermineTargetWindow(TargetWindowStrategy.TOP10_BETTER_THAN_GUESSING); - var threshold = validationResults.GetConfidenceThreshold(targetWindow); - - // Filter the retrieval contexts: - retrievalContexts = validationResults.Where(x => x.RetrievalContext is not null && x.Confidence >= threshold).Select(x => x.RetrievalContext!).ToList(); + // Filter the retrieval contexts: + retrievalContexts = validationResults.Where(x => x.RetrievalContext is not null && x.Confidence >= threshold).Select(x => x.RetrievalContext!).ToList(); + } + else + LOGGER.LogWarning("Skipping retrieval context validation because no compliant validation agent provider is available."); } LOGGER.LogInformation($"Starting the augmentation process over {numTotalRetrievalContexts:###,###,###,###} retrieval contexts."); diff --git a/app/MindWork AI Studio/Tools/RAG/DataSourceSelectionProcesses/AgenticSrcSelWithDynHeur.cs b/app/MindWork AI Studio/Tools/RAG/DataSourceSelectionProcesses/AgenticSrcSelWithDynHeur.cs index 9d544e6e..094a4bee 100644 --- a/app/MindWork AI Studio/Tools/RAG/DataSourceSelectionProcesses/AgenticSrcSelWithDynHeur.cs +++ b/app/MindWork AI Studio/Tools/RAG/DataSourceSelectionProcesses/AgenticSrcSelWithDynHeur.cs @@ -31,11 +31,9 @@ public class AgenticSrcSelWithDynHeur : IDataSourceSelectionProcess IReadOnlyList selectedDataSources = []; IReadOnlyList finalAISelection = []; - // Get the settings manager: - var settings = Program.SERVICE_PROVIDER.GetService()!; - // Get the agent for the data source selection: var selectionAgent = Program.SERVICE_PROVIDER.GetService()!; + var allowedDataSources = dataSources.AllowedDataSources.ToDictionary(ds => ds.Id, StringComparer.Ordinal); try { @@ -61,14 +59,14 @@ public class AgenticSrcSelWithDynHeur : IDataSourceSelectionProcess var totalAISelectedDataSources = aiSelectedDataSources.Count; // Filter out the data sources that are not available: - aiSelectedDataSources = aiSelectedDataSources.Where(x => settings.ConfigurationData.DataSources.FirstOrDefault(ds => ds.Id == x.Id) is not null).ToList(); + aiSelectedDataSources = aiSelectedDataSources.Where(x => allowedDataSources.ContainsKey(x.Id)).ToList(); // Store the real AI-selected data sources: - finalAISelection = aiSelectedDataSources.Select(x => new DataSourceAgentSelected { DataSource = settings.ConfigurationData.DataSources.First(ds => ds.Id == x.Id), AIDecision = x, Selected = false }).ToList(); + finalAISelection = aiSelectedDataSources.Select(x => new DataSourceAgentSelected { DataSource = allowedDataSources[x.Id], AIDecision = x, Selected = false }).ToList(); var numHallucinatedSources = totalAISelectedDataSources - aiSelectedDataSources.Count; if (numHallucinatedSources > 0) - LOGGER.LogWarning($"The AI hallucinated {numHallucinatedSources} data source(s). We ignore them."); + LOGGER.LogWarning($"The AI selected {numHallucinatedSources} unavailable data source(s). We ignore them."); if (aiSelectedDataSources.Count > 3) { @@ -87,7 +85,7 @@ public class AgenticSrcSelWithDynHeur : IDataSourceSelectionProcess LOGGER.LogInformation($"The AI selected {aiSelectedDataSources.Count} data source(s) with a confidence of at least {threshold}."); // Transform the final data sources to the actual data sources: - selectedDataSources = aiSelectedDataSources.Select(x => settings.ConfigurationData.DataSources.FirstOrDefault(ds => ds.Id == x.Id)).Where(ds => ds is not null).ToList()!; + selectedDataSources = aiSelectedDataSources.Select(x => allowedDataSources[x.Id]).ToList(); return new(proceedWithRAG, selectedDataSources); } @@ -96,7 +94,7 @@ public class AgenticSrcSelWithDynHeur : IDataSourceSelectionProcess // // Transform the selected data sources to the actual data sources: - selectedDataSources = aiSelectedDataSources.Select(x => settings.ConfigurationData.DataSources.FirstOrDefault(ds => ds.Id == x.Id)).Where(ds => ds is not null).ToList()!; + selectedDataSources = aiSelectedDataSources.Select(x => allowedDataSources[x.Id]).ToList(); // Mark the data sources as selected: foreach (var dataSource in finalAISelection) diff --git a/app/MindWork AI Studio/Tools/RAG/RAGProcesses/AISrcSelWithRetCtxVal.cs b/app/MindWork AI Studio/Tools/RAG/RAGProcesses/AISrcSelWithRetCtxVal.cs index d24df2ea..6cd62f95 100644 --- a/app/MindWork AI Studio/Tools/RAG/RAGProcesses/AISrcSelWithRetCtxVal.cs +++ b/app/MindWork AI Studio/Tools/RAG/RAGProcesses/AISrcSelWithRetCtxVal.cs @@ -92,7 +92,7 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess // // No, the user made the choice manually: // - var selectedDataSourceInfo = selectedDataSources.Select(ds => ds.Name).Aggregate((a, b) => $"'{a}', '{b}'"); + var selectedDataSourceInfo = string.Join(", ", selectedDataSources.Select(ds => $"'{ds.Name}'")); LOGGER.LogInformation($"The user selected the data sources manually. {selectedDataSources.Count} data source(s) are selected: {selectedDataSourceInfo}."); } @@ -104,12 +104,13 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess else { var previousDataSecurity = chatThread.DataSecurity; + var previousDataComplianceLevel = chatThread.DataComplianceLevel; // // Update the data security of the chat thread. We consider the current data security // of the chat thread and the data security of the selected data sources: // - var dataSecurityRestrictedToSelfHosted = selectedDataSources.Any(x => x.SecurityPolicy is DataSourceSecurity.SELF_HOSTED); + var dataSecurityRestrictedToSelfHosted = selectedDataSources.Any(x => x is not IInternalDataSource && x.SecurityPolicy is DataSourceSecurity.SELF_HOSTED); chatThread.DataSecurity = dataSecurityRestrictedToSelfHosted switch { // @@ -150,6 +151,13 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess if (previousDataSecurity != chatThread.DataSecurity) LOGGER.LogInformation($"The data security of the chat thread was updated from '{previousDataSecurity}' to '{chatThread.DataSecurity}'."); + + foreach (var dataSource in selectedDataSources) + if (dataSource.ComplianceLevel > chatThread.DataComplianceLevel) + chatThread.DataComplianceLevel = dataSource.ComplianceLevel; + + if (previousDataComplianceLevel != chatThread.DataComplianceLevel) + LOGGER.LogInformation($"The data compliance level of the chat thread was updated from '{previousDataComplianceLevel.GetName()}' to '{chatThread.DataComplianceLevel.GetName()}'."); } // diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs index deada2ac..2a8837b2 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs @@ -434,7 +434,24 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM this.UpsertStatus(this.GetFallbackStatus(dataSource, "The selected embedding provider is not available.")); return; } - + + if (!embeddingProvider.AllowsDataSourceAccess(settingsManager, dataSource.SecurityPolicy, dataSource.ComplianceLevel)) + { + var errorMessage = $"The selected embedding provider is not allowed to embed this data source. The data source requires provider confidence '{dataSource.ComplianceLevel.GetName()}'. The embedding provider has confidence '{embeddingProvider.GetConfidenceLevel(settingsManager).GetName()}'."; + logger.LogWarning( + "Skipping background embeddings for data source '{DataSourceName}' ({DataSourceId}) because embedding provider '{EmbeddingProviderName}' ({EmbeddingProviderId}) is not allowed. RequiredDataSecurity={RequiredDataSecurity}, RequiredCompliance={RequiredCompliance}, EmbeddingProviderConfidence={EmbeddingProviderConfidence}.", + dataSource.Name, + dataSource.Id, + embeddingProvider.Name, + embeddingProvider.Id, + dataSource.SecurityPolicy, + dataSource.ComplianceLevel.GetName(), + embeddingProvider.GetConfidenceLevel(settingsManager).GetName()); + + token.ThrowIfCancellationRequested(); + this.UpsertStatus(this.GetFallbackStatus(dataSource, errorMessage)); + return; + } logger.LogInformation( "Using embedding provider '{EmbeddingProviderId}' with model '{EmbeddingModelId}' for data source '{DataSourceName}' ({DataSourceId}).", diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceService.cs index b86fd8fb..34747d93 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceService.cs @@ -42,7 +42,10 @@ public sealed class DataSourceService return new([], []); } - return await this.GetDataSources(selectedLLMProvider.IsTrustedForDataSourceSecurityChecks(this.settingsManager), previousSelectedDataSources); + return await this.GetDataSources( + selectedLLMProvider.IsTrustedForDataSourceSecurityChecks(this.settingsManager), + selectedLLMProvider.GetConfidenceLevel(this.settingsManager), + previousSelectedDataSources); } /// @@ -65,10 +68,13 @@ public sealed class DataSourceService return new([], []); } - return await this.GetDataSources(selectedLLMProvider.IsTrustedForDataSourceSecurityChecks(this.settingsManager), previousSelectedDataSources); + return await this.GetDataSources( + selectedLLMProvider.IsTrustedForDataSourceSecurityChecks(this.settingsManager), + selectedLLMProvider.GetConfidenceLevel(this.settingsManager), + previousSelectedDataSources); } - private async Task GetDataSources(bool usingTrustedProvider, IReadOnlyCollection? previousSelectedDataSources = null) + private async Task GetDataSources(bool usingTrustedProvider, ConfidenceLevel providerConfidenceLevel, IReadOnlyCollection? previousSelectedDataSources = null) { var allDataSources = this.settingsManager.ConfigurationData.DataSources.ToList(); var previousSelectedDataSourceIds = previousSelectedDataSources?.Select(source => source.Id).ToHashSet(StringComparer.Ordinal) ?? []; @@ -78,7 +84,7 @@ public sealed class DataSourceService // Start all checks in parallel: foreach (var source in allDataSources) - tasks.Add(this.CheckOneDataSource(source, usingTrustedProvider)); + tasks.Add(this.CheckOneDataSource(source, usingTrustedProvider, providerConfidenceLevel)); // Wait for all checks and collect the results: foreach (var task in tasks) @@ -95,8 +101,17 @@ public sealed class DataSourceService return new(filteredDataSources, filteredSelectedDataSources); } - private async Task CheckOneDataSource(IDataSource source, bool usingTrustedProvider) + private async Task CheckOneDataSource(IDataSource source, bool usingTrustedProvider, ConfidenceLevel providerConfidenceLevel) { + if (!providerConfidenceLevel.AllowsDataSourceComplianceLevel(source.ComplianceLevel)) + { + this.logger.LogWarning($"The data source '{source.Name}' (id={source.Id}) requires provider confidence '{source.ComplianceLevel.GetName()}'. The selected provider only has confidence '{providerConfidenceLevel.GetName()}'. We skip this source."); + return null; + } + + if (source is IInternalDataSource) + return source; + // // Unfortunately, we have to live-check any ERI source for its security requirements. // Because the ERI server operator might change the security requirements at any time. @@ -206,4 +221,4 @@ public sealed class DataSourceService return null; } } -} \ No newline at end of file +} diff --git a/app/MindWork AI Studio/Tools/Validation/DataSourceValidation.cs b/app/MindWork AI Studio/Tools/Validation/DataSourceValidation.cs index a761ed08..29b39f9b 100644 --- a/app/MindWork AI Studio/Tools/Validation/DataSourceValidation.cs +++ b/app/MindWork AI Studio/Tools/Validation/DataSourceValidation.cs @@ -1,3 +1,5 @@ +using AIStudio.Provider; +using AIStudio.Settings; using AIStudio.Settings.DataModel; using AIStudio.Tools.ERIClient.DataModel; using AIStudio.Tools.PluginSystem; @@ -19,6 +21,14 @@ public sealed class DataSourceValidation public Func GetSecurityRequirements { get; init; } = () => null; public Func GetSelectedCloudEmbedding { get; init; } = () => false; + + public Func GetSelectedEmbeddingProvider { get; init; } = () => null; + + public Func GetSecurityPolicy { get; init; } = () => DataSourceSecurity.ALLOW_ANY; + + public Func GetComplianceLevel { get; init; } = () => ConfidenceLevel.NONE; + + public Func GetSettingsManager { get; init; } = () => null; public Func GetTestedConnection { get; init; } = () => false; @@ -149,6 +159,20 @@ public sealed class DataSourceValidation return null; } + public string? ValidateEmbeddingProviderAccess(string embeddingId) + { + var embeddingIssue = this.ValidateEmbeddingId(embeddingId); + return embeddingIssue ?? this.ValidateSelectedEmbeddingProviderAccess(); + } + + public string? ValidateDataSourceComplianceLevel(ConfidenceLevel complianceLevel) + { + if(complianceLevel is ConfidenceLevel.NONE) + return TB("Please select a compliance level."); + + return this.ValidateSelectedEmbeddingProviderAccess(); + } + public string? ValidateUserAcknowledgedCloudEmbedding(bool value) { if(this.GetSelectedCloudEmbedding() && !value) @@ -175,4 +199,25 @@ public sealed class DataSourceValidation return null; } -} \ No newline at end of file + + private string? ValidateSelectedEmbeddingProviderAccess() + { + var selectedEmbedding = this.GetSelectedEmbeddingProvider(); + var settingsManager = this.GetSettingsManager(); + if(selectedEmbedding is null || settingsManager is null) + return null; + + var dataSecurity = this.GetSecurityPolicy(); + var complianceLevel = this.GetComplianceLevel(); + if(selectedEmbedding.AllowsDataSourceAccess(settingsManager, dataSecurity, complianceLevel)) + return null; + + if(!selectedEmbedding.AllowsDataSourceSecurity(dataSecurity, settingsManager)) + return TB("The selected embedding provider is not allowed to process this data source due to its data security policy. Select a self-hosted or organization-trusted embedding provider."); + + return string.Format( + TB("The selected embedding provider has confidence '{0}', but this data source requires provider confidence '{1}'. Select an embedding provider with equal or higher confidence or lower the compliance level."), + selectedEmbedding.GetConfidenceLevel(settingsManager).GetName(), + complianceLevel.GetName()); + } +}