refactored confidence levels and ensured checks work

app/MindWork AI Studio/Tools/Services/DataSourceService.cs

@@ -9,6 +9,8 @@ namespace AIStudio.Tools.Services;
 
 public sealed class DataSourceService
 {
+    private readonly record struct ParticipatingProvider(string Role, bool IsTrusted, ConfidenceLevel ConfidenceLevel);
+
     private readonly RustService rustService;
     private readonly SettingsManager settingsManager;
     private readonly ILogger<DataSourceService> logger;
@@ -27,9 +29,10 @@ public sealed class DataSourceService
     /// It also returns the data sources selected before when they are still allowed.
     /// </summary>
     /// <param name="selectedLLMProvider">The selected LLM provider.</param>
+    /// <param name="dataSourceOptions">The active data source options, which determine which agent providers participate.</param>
     /// <param name="previousSelectedDataSources">The data sources selected before.</param>
     /// <returns>The allowed data sources and the data sources selected before -- when they are still allowed.</returns>
-    public async Task<AllowedSelectedDataSources> GetDataSources(AIStudio.Settings.Provider selectedLLMProvider, IReadOnlyCollection<IDataSource>? previousSelectedDataSources = null)
+    public async Task<AllowedSelectedDataSources> GetDataSources(AIStudio.Settings.Provider selectedLLMProvider, DataSourceOptions dataSourceOptions, IReadOnlyCollection<IDataSource>? previousSelectedDataSources = null)
     {
         //
         // Case: Somehow the selected LLM provider was not set. The default provider
@@ -42,10 +45,10 @@ public sealed class DataSourceService
             return new([], []);
         }
         
-        return await this.GetDataSources(
-            selectedLLMProvider.IsTrustedForDataSourceSecurityChecks(this.settingsManager),
-            selectedLLMProvider.GetConfidenceLevel(this.settingsManager),
-            previousSelectedDataSources);
+        var usingTrustedProvider = selectedLLMProvider.IsTrustedForDataSourceSecurityChecks(this.settingsManager);
+        var participatingProviders = this.GetParticipatingProviders(selectedLLMProvider.Id, dataSourceOptions,
+            new("chat provider", usingTrustedProvider, selectedLLMProvider.GetConfidenceLevel(this.settingsManager)));
+        return await this.GetDataSources(usingTrustedProvider, participatingProviders, previousSelectedDataSources);
     }
     
     /// <summary>
@@ -53,9 +56,10 @@ public sealed class DataSourceService
     /// It also returns the data sources selected before when they are still allowed.
     /// </summary>
     /// <param name="selectedLLMProvider">The selected LLM provider.</param>
+    /// <param name="dataSourceOptions">The active data source options, which determine which agent providers participate.</param>
     /// <param name="previousSelectedDataSources">The data sources selected before.</param>
     /// <returns>The allowed data sources and the data sources selected before -- when they are still allowed.</returns>
-    public async Task<AllowedSelectedDataSources> GetDataSources(IProvider selectedLLMProvider, IReadOnlyCollection<IDataSource>? previousSelectedDataSources = null)
+    public async Task<AllowedSelectedDataSources> GetDataSources(IProvider selectedLLMProvider, DataSourceOptions dataSourceOptions, IReadOnlyCollection<IDataSource>? previousSelectedDataSources = null)
     {
         //
         // Case: Somehow the selected LLM provider was not set. The default provider
@@ -68,13 +72,42 @@ public sealed class DataSourceService
             return new([], []);
         }
         
-        return await this.GetDataSources(
-            selectedLLMProvider.IsTrustedForDataSourceSecurityChecks(this.settingsManager),
-            selectedLLMProvider.GetConfidenceLevel(this.settingsManager),
-            previousSelectedDataSources);
+        var usingTrustedProvider = selectedLLMProvider.IsTrustedForDataSourceSecurityChecks(this.settingsManager);
+        var participatingProviders = this.GetParticipatingProviders(selectedLLMProvider.ConfiguredProviderId, dataSourceOptions,
+            new("chat provider", usingTrustedProvider, selectedLLMProvider.GetConfidenceLevel(this.settingsManager)));
+        return await this.GetDataSources(usingTrustedProvider, participatingProviders, previousSelectedDataSources);
     }
     
-    private async Task<AllowedSelectedDataSources> GetDataSources(bool usingTrustedProvider, ConfidenceLevel providerConfidenceLevel, IReadOnlyCollection<IDataSource>? previousSelectedDataSources = null)
+    private IReadOnlyList<ParticipatingProvider> GetParticipatingProviders(string currentProviderId, DataSourceOptions dataSourceOptions, ParticipatingProvider currentProvider)
+    {
+        var providers = new List<ParticipatingProvider> { currentProvider };
+
+        if (dataSourceOptions.AutomaticDataSourceSelection)
+            this.AddAgentProvider(providers, Components.AGENT_DATA_SOURCE_SELECTION, currentProviderId, "data source selection agent");
+
+        if (dataSourceOptions.AutomaticValidation && this.settingsManager.ConfigurationData.AgentRetrievalContextValidation.EnableRetrievalContextValidation)
+            this.AddAgentProvider(providers, Components.AGENT_RETRIEVAL_CONTEXT_VALIDATION, currentProviderId, "retrieval context validation agent");
+
+        return providers;
+    }
+
+    private void AddAgentProvider(List<ParticipatingProvider> providers, Components component, string currentProviderId, string role)
+    {
+        var provider = this.settingsManager.GetPreselectedProvider(component, currentProviderId, true);
+        if (provider == Settings.Provider.NONE)
+        {
+            this.logger.LogWarning($"No provider is available for the {role}. Data sources cannot be made available while this agent is enabled.");
+            providers.Add(new(role, false, ConfidenceLevel.NONE));
+            return;
+        }
+
+        providers.Add(new(
+            role,
+            provider.IsTrustedForDataSourceSecurityChecks(this.settingsManager),
+            provider.GetConfidenceLevel(this.settingsManager)));
+    }
+
+    private async Task<AllowedSelectedDataSources> GetDataSources(bool usingTrustedProvider, IReadOnlyList<ParticipatingProvider> participatingProviders, IReadOnlyCollection<IDataSource>? previousSelectedDataSources = null)
     {
         var allDataSources = this.settingsManager.ConfigurationData.DataSources.ToList();
         var previousSelectedDataSourceIds = previousSelectedDataSources?.Select(source => source.Id).ToHashSet(StringComparer.Ordinal) ?? [];
@@ -84,7 +117,7 @@ public sealed class DataSourceService
         
         // Start all checks in parallel:
         foreach (var source in allDataSources)
-            tasks.Add(this.CheckOneDataSource(source, usingTrustedProvider, providerConfidenceLevel));
+            tasks.Add(this.CheckOneDataSource(source, usingTrustedProvider, participatingProviders));
         
         // Wait for all checks and collect the results:
         foreach (var task in tasks)
@@ -101,16 +134,34 @@ public sealed class DataSourceService
         return new(filteredDataSources, filteredSelectedDataSources);
     }
     
-    private async Task<IDataSource?> CheckOneDataSource(IDataSource source, bool usingTrustedProvider, ConfidenceLevel providerConfidenceLevel)
+    private async Task<IDataSource?> CheckOneDataSource(IDataSource source, bool usingTrustedProvider, IReadOnlyList<ParticipatingProvider> participatingProviders)
     {
-        if (!providerConfidenceLevel.AllowsDataSourceComplianceLevel(source.ComplianceLevel))
+        if (source is IInternalDataSource internalSource)
         {
-            this.logger.LogWarning($"The data source '{source.Name}' (id={source.Id}) requires provider confidence '{source.ComplianceLevel.GetName()}'. The selected provider only has confidence '{providerConfidenceLevel.GetName()}'. We skip this source.");
-            return null;
-        }
+            foreach (var provider in participatingProviders)
+            {
+                if (!provider.ConfidenceLevel.AllowsDataSourceConfidenceLevel(internalSource.ConfidenceLevel))
+                {
+                    this.logger.LogWarning($"The internal data source '{source.Name}' (id={source.Id}) requires provider confidence '{internalSource.ConfidenceLevel.GetName()}'. The {provider.Role} only has confidence '{provider.ConfidenceLevel.GetName()}'. We skip this source.");
+                    return null;
+                }
+            }
+
+            if (!DataSourceEmbeddingProviders.TryResolve(this.settingsManager, source, out var embeddingProvider))
+            {
+                this.logger.LogWarning($"The internal data source '{source.Name}' (id={source.Id}) has no usable embedding provider. We skip this source.");
+                return null;
+            }
+
+            var embeddingProviderConfidence = embeddingProvider.GetConfidenceLevel(this.settingsManager);
+            if (!embeddingProviderConfidence.AllowsDataSourceConfidenceLevel(internalSource.ConfidenceLevel))
+            {
+                this.logger.LogWarning($"The internal data source '{source.Name}' (id={source.Id}) requires provider confidence '{internalSource.ConfidenceLevel.GetName()}'. Its embedding provider '{embeddingProvider.Name}' only has confidence '{embeddingProviderConfidence.GetName()}'. We skip this source.");
+                return null;
+            }
 
-        if (source is IInternalDataSource)
             return source;
+        }
 
         //
         // Unfortunately, we have to live-check any ERI source for its security requirements.
@@ -146,8 +197,11 @@ public sealed class DataSourceService
             eriSourceRequirements = securityRequest.Data;
             this.logger.LogInformation($"Security requirements for ERI source '{source.Name}' (id={source.Id}) retrieved successfully.");
         }
-        
-        switch (source.SecurityPolicy)
+
+        if (source is not IExternalDataSource externalSource)
+            return source;
+
+        switch (externalSource.SecurityPolicy)
         {
             case DataSourceSecurity.ALLOW_ANY: