Added security checks for confidence levels

app/MindWork AI Studio/Tools/RAG/AugmentationProcesses/AugmentationOne.cs

@@ -43,19 +43,22 @@ public sealed class AugmentationOne : IAugmentationProcess
         {
             // Let's get the validation agent & set up its provider:
             var validationAgent = Program.SERVICE_PROVIDER.GetService<AgentRetrievalContextValidation>()!;
-            validationAgent.SetLLMProvider(provider);
+            if (validationAgent.SetLLMProvider(provider, chatThread.DataSecurity, chatThread.DataComplianceLevel))
+            {
+                // Let's validate all retrieval contexts:
+                var validationResults = await validationAgent.ValidateRetrievalContextsAsync(lastUserPrompt, chatThread, retrievalContexts, token);
+
+                //
+                // Now, filter the retrieval contexts to the most relevant ones:
+                //
+                var targetWindow = validationResults.DetermineTargetWindow(TargetWindowStrategy.TOP10_BETTER_THAN_GUESSING);
+                var threshold = validationResults.GetConfidenceThreshold(targetWindow);
             
-            // Let's validate all retrieval contexts:
-            var validationResults = await validationAgent.ValidateRetrievalContextsAsync(lastUserPrompt, chatThread, retrievalContexts, token);
-         
-            //
-            // Now, filter the retrieval contexts to the most relevant ones:
-            //
-            var targetWindow = validationResults.DetermineTargetWindow(TargetWindowStrategy.TOP10_BETTER_THAN_GUESSING);
-            var threshold = validationResults.GetConfidenceThreshold(targetWindow);
-            
-            // Filter the retrieval contexts:
-            retrievalContexts = validationResults.Where(x => x.RetrievalContext is not null && x.Confidence >= threshold).Select(x => x.RetrievalContext!).ToList();
+                // Filter the retrieval contexts:
+                retrievalContexts = validationResults.Where(x => x.RetrievalContext is not null && x.Confidence >= threshold).Select(x => x.RetrievalContext!).ToList();
+            }
+            else
+                LOGGER.LogWarning("Skipping retrieval context validation because no compliant validation agent provider is available.");
         }
         
         LOGGER.LogInformation($"Starting the augmentation process over {numTotalRetrievalContexts:###,###,###,###} retrieval contexts.");

app/MindWork AI Studio/Tools/RAG/DataSourceSelectionProcesses/AgenticSrcSelWithDynHeur.cs

@@ -31,11 +31,9 @@ public class AgenticSrcSelWithDynHeur : IDataSourceSelectionProcess
         IReadOnlyList<IDataSource> selectedDataSources = [];
         IReadOnlyList<DataSourceAgentSelected> finalAISelection = [];
         
-        // Get the settings manager:
-        var settings = Program.SERVICE_PROVIDER.GetService<SettingsManager>()!;
-        
         // Get the agent for the data source selection:
         var selectionAgent = Program.SERVICE_PROVIDER.GetService<AgentDataSourceSelection>()!;
+        var allowedDataSources = dataSources.AllowedDataSources.ToDictionary(ds => ds.Id, StringComparer.Ordinal);
 
         try
         {
@@ -61,14 +59,14 @@ public class AgenticSrcSelWithDynHeur : IDataSourceSelectionProcess
             var totalAISelectedDataSources = aiSelectedDataSources.Count;
 
             // Filter out the data sources that are not available:
-            aiSelectedDataSources = aiSelectedDataSources.Where(x => settings.ConfigurationData.DataSources.FirstOrDefault(ds => ds.Id == x.Id) is not null).ToList();
+            aiSelectedDataSources = aiSelectedDataSources.Where(x => allowedDataSources.ContainsKey(x.Id)).ToList();
 
             // Store the real AI-selected data sources:
-            finalAISelection = aiSelectedDataSources.Select(x => new DataSourceAgentSelected { DataSource = settings.ConfigurationData.DataSources.First(ds => ds.Id == x.Id), AIDecision = x, Selected = false }).ToList();
+            finalAISelection = aiSelectedDataSources.Select(x => new DataSourceAgentSelected { DataSource = allowedDataSources[x.Id], AIDecision = x, Selected = false }).ToList();
 
             var numHallucinatedSources = totalAISelectedDataSources - aiSelectedDataSources.Count;
             if (numHallucinatedSources > 0)
-                LOGGER.LogWarning($"The AI hallucinated {numHallucinatedSources} data source(s). We ignore them.");
+                LOGGER.LogWarning($"The AI selected {numHallucinatedSources} unavailable data source(s). We ignore them.");
 
             if (aiSelectedDataSources.Count > 3)
             {
@@ -87,7 +85,7 @@ public class AgenticSrcSelWithDynHeur : IDataSourceSelectionProcess
                 LOGGER.LogInformation($"The AI selected {aiSelectedDataSources.Count} data source(s) with a confidence of at least {threshold}.");
 
                 // Transform the final data sources to the actual data sources:
-                selectedDataSources = aiSelectedDataSources.Select(x => settings.ConfigurationData.DataSources.FirstOrDefault(ds => ds.Id == x.Id)).Where(ds => ds is not null).ToList()!;
+                selectedDataSources = aiSelectedDataSources.Select(x => allowedDataSources[x.Id]).ToList();
                 return new(proceedWithRAG, selectedDataSources);
             }
 
@@ -96,7 +94,7 @@ public class AgenticSrcSelWithDynHeur : IDataSourceSelectionProcess
             //
 
             // Transform the selected data sources to the actual data sources:
-            selectedDataSources = aiSelectedDataSources.Select(x => settings.ConfigurationData.DataSources.FirstOrDefault(ds => ds.Id == x.Id)).Where(ds => ds is not null).ToList()!;
+            selectedDataSources = aiSelectedDataSources.Select(x => allowedDataSources[x.Id]).ToList();
 
             // Mark the data sources as selected:
             foreach (var dataSource in finalAISelection)

app/MindWork AI Studio/Tools/RAG/RAGProcesses/AISrcSelWithRetCtxVal.cs

@@ -92,7 +92,7 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
                 //
                 // No, the user made the choice manually:
                 //
-                var selectedDataSourceInfo = selectedDataSources.Select(ds => ds.Name).Aggregate((a, b) => $"'{a}', '{b}'");
+                var selectedDataSourceInfo = string.Join(", ", selectedDataSources.Select(ds => $"'{ds.Name}'"));
                 LOGGER.LogInformation($"The user selected the data sources manually. {selectedDataSources.Count} data source(s) are selected: {selectedDataSourceInfo}.");
             }
 
@@ -104,12 +104,13 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
             else
             {
                 var previousDataSecurity = chatThread.DataSecurity;
+                var previousDataComplianceLevel = chatThread.DataComplianceLevel;
                 
                 //
                 // Update the data security of the chat thread. We consider the current data security
                 // of the chat thread and the data security of the selected data sources:
                 //
-                var dataSecurityRestrictedToSelfHosted = selectedDataSources.Any(x => x.SecurityPolicy is DataSourceSecurity.SELF_HOSTED);
+                var dataSecurityRestrictedToSelfHosted = selectedDataSources.Any(x => x is not IInternalDataSource && x.SecurityPolicy is DataSourceSecurity.SELF_HOSTED);
                 chatThread.DataSecurity = dataSecurityRestrictedToSelfHosted switch
                 {
                     //
@@ -150,6 +151,13 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
                 
                 if (previousDataSecurity != chatThread.DataSecurity)
                     LOGGER.LogInformation($"The data security of the chat thread was updated from '{previousDataSecurity}' to '{chatThread.DataSecurity}'.");
+
+                foreach (var dataSource in selectedDataSources)
+                    if (dataSource.ComplianceLevel > chatThread.DataComplianceLevel)
+                        chatThread.DataComplianceLevel = dataSource.ComplianceLevel;
+
+                if (previousDataComplianceLevel != chatThread.DataComplianceLevel)
+                    LOGGER.LogInformation($"The data compliance level of the chat thread was updated from '{previousDataComplianceLevel.GetName()}' to '{chatThread.DataComplianceLevel.GetName()}'.");
             }
             
             //

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs

@@ -434,7 +434,24 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
             this.UpsertStatus(this.GetFallbackStatus(dataSource, "The selected embedding provider is not available."));
             return;
         }
-        
+
+        if (!embeddingProvider.AllowsDataSourceAccess(settingsManager, dataSource.SecurityPolicy, dataSource.ComplianceLevel))
+        {
+            var errorMessage = $"The selected embedding provider is not allowed to embed this data source. The data source requires provider confidence '{dataSource.ComplianceLevel.GetName()}'. The embedding provider has confidence '{embeddingProvider.GetConfidenceLevel(settingsManager).GetName()}'.";
+            logger.LogWarning(
+                "Skipping background embeddings for data source '{DataSourceName}' ({DataSourceId}) because embedding provider '{EmbeddingProviderName}' ({EmbeddingProviderId}) is not allowed. RequiredDataSecurity={RequiredDataSecurity}, RequiredCompliance={RequiredCompliance}, EmbeddingProviderConfidence={EmbeddingProviderConfidence}.",
+                dataSource.Name,
+                dataSource.Id,
+                embeddingProvider.Name,
+                embeddingProvider.Id,
+                dataSource.SecurityPolicy,
+                dataSource.ComplianceLevel.GetName(),
+                embeddingProvider.GetConfidenceLevel(settingsManager).GetName());
+
+            token.ThrowIfCancellationRequested();
+            this.UpsertStatus(this.GetFallbackStatus(dataSource, errorMessage));
+            return;
+        }
 
         logger.LogInformation(
             "Using embedding provider '{EmbeddingProviderId}' with model '{EmbeddingModelId}' for data source '{DataSourceName}' ({DataSourceId}).",

app/MindWork AI Studio/Tools/Services/DataSourceService.cs

@@ -42,7 +42,10 @@ public sealed class DataSourceService
             return new([], []);
         }
         
-        return await this.GetDataSources(selectedLLMProvider.IsTrustedForDataSourceSecurityChecks(this.settingsManager), previousSelectedDataSources);
+        return await this.GetDataSources(
+            selectedLLMProvider.IsTrustedForDataSourceSecurityChecks(this.settingsManager),
+            selectedLLMProvider.GetConfidenceLevel(this.settingsManager),
+            previousSelectedDataSources);
     }
     
     /// <summary>
@@ -65,10 +68,13 @@ public sealed class DataSourceService
             return new([], []);
         }
         
-        return await this.GetDataSources(selectedLLMProvider.IsTrustedForDataSourceSecurityChecks(this.settingsManager), previousSelectedDataSources);
+        return await this.GetDataSources(
+            selectedLLMProvider.IsTrustedForDataSourceSecurityChecks(this.settingsManager),
+            selectedLLMProvider.GetConfidenceLevel(this.settingsManager),
+            previousSelectedDataSources);
     }
     
-    private async Task<AllowedSelectedDataSources> GetDataSources(bool usingTrustedProvider, IReadOnlyCollection<IDataSource>? previousSelectedDataSources = null)
+    private async Task<AllowedSelectedDataSources> GetDataSources(bool usingTrustedProvider, ConfidenceLevel providerConfidenceLevel, IReadOnlyCollection<IDataSource>? previousSelectedDataSources = null)
     {
         var allDataSources = this.settingsManager.ConfigurationData.DataSources.ToList();
         var previousSelectedDataSourceIds = previousSelectedDataSources?.Select(source => source.Id).ToHashSet(StringComparer.Ordinal) ?? [];
@@ -78,7 +84,7 @@ public sealed class DataSourceService
         
         // Start all checks in parallel:
         foreach (var source in allDataSources)
-            tasks.Add(this.CheckOneDataSource(source, usingTrustedProvider));
+            tasks.Add(this.CheckOneDataSource(source, usingTrustedProvider, providerConfidenceLevel));
         
         // Wait for all checks and collect the results:
         foreach (var task in tasks)
@@ -95,8 +101,17 @@ public sealed class DataSourceService
         return new(filteredDataSources, filteredSelectedDataSources);
     }
     
-    private async Task<IDataSource?> CheckOneDataSource(IDataSource source, bool usingTrustedProvider)
+    private async Task<IDataSource?> CheckOneDataSource(IDataSource source, bool usingTrustedProvider, ConfidenceLevel providerConfidenceLevel)
     {
+        if (!providerConfidenceLevel.AllowsDataSourceComplianceLevel(source.ComplianceLevel))
+        {
+            this.logger.LogWarning($"The data source '{source.Name}' (id={source.Id}) requires provider confidence '{source.ComplianceLevel.GetName()}'. The selected provider only has confidence '{providerConfidenceLevel.GetName()}'. We skip this source.");
+            return null;
+        }
+
+        if (source is IInternalDataSource)
+            return source;
+
         //
         // Unfortunately, we have to live-check any ERI source for its security requirements.
         // Because the ERI server operator might change the security requirements at any time.
@@ -206,4 +221,4 @@ public sealed class DataSourceService
                 return null;
         }
     }
-}
+}

app/MindWork AI Studio/Tools/Validation/DataSourceValidation.cs

@@ -1,3 +1,5 @@
+using AIStudio.Provider;
+using AIStudio.Settings;
 using AIStudio.Settings.DataModel;
 using AIStudio.Tools.ERIClient.DataModel;
 using AIStudio.Tools.PluginSystem;
@@ -19,6 +21,14 @@ public sealed class DataSourceValidation
     public Func<SecurityRequirements?> GetSecurityRequirements { get; init; } = () => null;
     
     public Func<bool> GetSelectedCloudEmbedding { get; init; } = () => false;
+
+    public Func<EmbeddingProvider?> GetSelectedEmbeddingProvider { get; init; } = () => null;
+
+    public Func<DataSourceSecurity> GetSecurityPolicy { get; init; } = () => DataSourceSecurity.ALLOW_ANY;
+
+    public Func<ConfidenceLevel> GetComplianceLevel { get; init; } = () => ConfidenceLevel.NONE;
+
+    public Func<SettingsManager?> GetSettingsManager { get; init; } = () => null;
     
     public Func<bool> GetTestedConnection { get; init; } = () => false;
     
@@ -149,6 +159,20 @@ public sealed class DataSourceValidation
         return null;
     }
 
+    public string? ValidateEmbeddingProviderAccess(string embeddingId)
+    {
+        var embeddingIssue = this.ValidateEmbeddingId(embeddingId);
+        return embeddingIssue ?? this.ValidateSelectedEmbeddingProviderAccess();
+    }
+
+    public string? ValidateDataSourceComplianceLevel(ConfidenceLevel complianceLevel)
+    {
+        if(complianceLevel is ConfidenceLevel.NONE)
+            return TB("Please select a compliance level.");
+
+        return this.ValidateSelectedEmbeddingProviderAccess();
+    }
+
     public string? ValidateUserAcknowledgedCloudEmbedding(bool value)
     {
         if(this.GetSelectedCloudEmbedding() && !value)
@@ -175,4 +199,25 @@ public sealed class DataSourceValidation
         
         return null;
     }
-}
+
+    private string? ValidateSelectedEmbeddingProviderAccess()
+    {
+        var selectedEmbedding = this.GetSelectedEmbeddingProvider();
+        var settingsManager = this.GetSettingsManager();
+        if(selectedEmbedding is null || settingsManager is null)
+            return null;
+
+        var dataSecurity = this.GetSecurityPolicy();
+        var complianceLevel = this.GetComplianceLevel();
+        if(selectedEmbedding.AllowsDataSourceAccess(settingsManager, dataSecurity, complianceLevel))
+            return null;
+
+        if(!selectedEmbedding.AllowsDataSourceSecurity(dataSecurity, settingsManager))
+            return TB("The selected embedding provider is not allowed to process this data source due to its data security policy. Select a self-hosted or organization-trusted embedding provider.");
+
+        return string.Format(
+            TB("The selected embedding provider has confidence '{0}', but this data source requires provider confidence '{1}'. Select an embedding provider with equal or higher confidence or lower the compliance level."),
+            selectedEmbedding.GetConfidenceLevel(settingsManager).GetName(),
+            complianceLevel.GetName());
+    }
+}