From a75ffc54a6231c83bc4df5e93d1c37b6dd90e0d5 Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Thu, 24 Sep 2026 16:32:01 +0200 Subject: [PATCH] Count the agents only where the classic RAG process runs them --- .../Components/DataSourceSelection.razor.cs | 2 +- .../DataModel/DataSourceRetrievalMode.cs | 21 ++++++++ .../RAG/RAGProcesses/AISrcSelWithRetCtxVal.cs | 2 +- .../Tools/Services/DataSourceService.cs | 54 ++++++++++++++----- .../Tools/Services/DirectChatService.cs | 2 +- .../DataSourceParticipatingAgentsTests.cs | 51 ++++++++++++++++++ 6 files changed, 116 insertions(+), 16 deletions(-) create mode 100644 app/MindWork AI Studio/Settings/DataModel/DataSourceRetrievalMode.cs create mode 100644 app/Tests/Tools/DataSourceParticipatingAgentsTests.cs diff --git a/app/MindWork AI Studio/Components/DataSourceSelection.razor.cs b/app/MindWork AI Studio/Components/DataSourceSelection.razor.cs index 170d6a0b..839a4850 100644 --- a/app/MindWork AI Studio/Components/DataSourceSelection.razor.cs +++ b/app/MindWork AI Studio/Components/DataSourceSelection.razor.cs @@ -246,7 +246,7 @@ public partial class DataSourceSelection : MSGComponentBase // that field holds what was usable the last time we looked, so a source filtered out once // would never come back, while the RAG process keeps reading it from the preselection. // - var sources = await this.DataSourceService.GetDataSources(this.LLMProvider, this.DataSourceOptions, this.GetDataSourcesFromConfiguredIds()); + var sources = await this.DataSourceService.GetDataSources(this.LLMProvider, this.DataSourceOptions, DataSourceRetrievalMode.EVERY_MESSAGE, this.GetDataSourcesFromConfiguredIds()); if (generation != this.loadAndApplyFiltersGeneration) return; diff --git a/app/MindWork AI Studio/Settings/DataModel/DataSourceRetrievalMode.cs b/app/MindWork AI Studio/Settings/DataModel/DataSourceRetrievalMode.cs new file mode 100644 index 00000000..8377aed4 --- /dev/null +++ b/app/MindWork AI Studio/Settings/DataModel/DataSourceRetrievalMode.cs @@ -0,0 +1,21 @@ +namespace AIStudio.Settings.DataModel; + +/// +/// How the data sources of a chat are searched. +/// +public enum DataSourceRetrievalMode +{ + /// + /// The model searches the data sources itself, through the tool semantic_search, whenever a + /// question calls for it. No agent takes part: the chat model picks the data sources and + /// judges what it found. + /// + SEMANTIC_SEARCH, + + /// + /// AI Studio searches the data sources with every message, before the model answers. This is + /// the classic RAG process, with its agents for selecting data sources and for validating what + /// was found. + /// + EVERY_MESSAGE, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/RAG/RAGProcesses/AISrcSelWithRetCtxVal.cs b/app/MindWork AI Studio/Tools/RAG/RAGProcesses/AISrcSelWithRetCtxVal.cs index 5bfaebc9..286ab29e 100644 --- a/app/MindWork AI Studio/Tools/RAG/RAGProcesses/AISrcSelWithRetCtxVal.cs +++ b/app/MindWork AI Studio/Tools/RAG/RAGProcesses/AISrcSelWithRetCtxVal.cs @@ -94,7 +94,7 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess // data sources changed its security requirements. // List preselectedDataSources = chatThread.DataSourceOptions.PreselectedDataSourceIds.Select(id => settings.ConfigurationData.DataSources.FirstOrDefault(ds => ds.Id == id)).Where(ds => ds is not null).ToList()!; - var dataSources = await dataSourceService.GetDataSources(provider, chatThread.DataSourceOptions, preselectedDataSources); + var dataSources = await dataSourceService.GetDataSources(provider, chatThread.DataSourceOptions, DataSourceRetrievalMode.EVERY_MESSAGE, preselectedDataSources); var selectedDataSources = dataSources.SelectedDataSources; // diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceService.cs index ce18b3b6..c74f1d50 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceService.cs @@ -39,9 +39,10 @@ public sealed class DataSourceService /// /// The selected LLM provider. /// The active data source options, which determine which agent providers participate. + /// How the data sources are searched in effect, which decides whether any agent participates at all. /// The data sources selected before. /// The allowed data sources and the data sources selected before -- when they are still allowed. - public async Task GetDataSources(AIStudio.Settings.Provider selectedLLMProvider, DataSourceOptions dataSourceOptions, IReadOnlyCollection? previousSelectedDataSources = null) + public async Task GetDataSources(AIStudio.Settings.Provider selectedLLMProvider, DataSourceOptions dataSourceOptions, DataSourceRetrievalMode retrievalMode, IReadOnlyCollection? previousSelectedDataSources = null) { // // Case: Somehow the selected LLM provider was not set. The default provider @@ -55,7 +56,7 @@ public sealed class DataSourceService } var usingTrustedProvider = selectedLLMProvider.IsTrustedForDataSourceSecurityChecks(this.settingsManager); - var participatingProviders = this.GetParticipatingProviders(selectedLLMProvider.Id, dataSourceOptions, + var participatingProviders = this.GetParticipatingProviders(selectedLLMProvider.Id, dataSourceOptions, retrievalMode, new("chat provider", usingTrustedProvider, selectedLLMProvider.GetConfidenceLevel(this.settingsManager))); return await this.GetDataSources(usingTrustedProvider, participatingProviders, previousSelectedDataSources); } @@ -67,9 +68,10 @@ public sealed class DataSourceService /// /// The selected LLM provider. /// The active data source options, which determine which agent providers participate. + /// How the data sources are searched in effect, which decides whether any agent participates at all. /// The data sources to check. /// The requested data sources that are allowed for the provider. - public async Task> GetAllowedDataSources(AIStudio.Settings.Provider selectedLLMProvider, DataSourceOptions dataSourceOptions, IReadOnlyCollection requestedDataSources) + public async Task> GetAllowedDataSources(AIStudio.Settings.Provider selectedLLMProvider, DataSourceOptions dataSourceOptions, DataSourceRetrievalMode retrievalMode, IReadOnlyCollection requestedDataSources) { if (selectedLLMProvider == Settings.Provider.NONE) { @@ -78,7 +80,7 @@ public sealed class DataSourceService } var usingTrustedProvider = selectedLLMProvider.IsTrustedForDataSourceSecurityChecks(this.settingsManager); - var participatingProviders = this.GetParticipatingProviders(selectedLLMProvider.Id, dataSourceOptions, + var participatingProviders = this.GetParticipatingProviders(selectedLLMProvider.Id, dataSourceOptions, retrievalMode, new("chat provider", usingTrustedProvider, selectedLLMProvider.GetConfidenceLevel(this.settingsManager))); var allowedDataSources = await this.GetAllowedDataSources(usingTrustedProvider, participatingProviders, requestedDataSources); @@ -98,9 +100,10 @@ public sealed class DataSourceService /// /// The selected LLM provider. /// The active data source options, which determine which agent providers participate. + /// How the data sources are searched in effect, which decides whether any agent participates at all. /// The data sources selected before. /// The allowed data sources and the data sources selected before -- when they are still allowed. - public async Task GetDataSources(IProvider selectedLLMProvider, DataSourceOptions dataSourceOptions, IReadOnlyCollection? previousSelectedDataSources = null) + public async Task GetDataSources(IProvider selectedLLMProvider, DataSourceOptions dataSourceOptions, DataSourceRetrievalMode retrievalMode, IReadOnlyCollection? previousSelectedDataSources = null) { // // Case: Somehow the selected LLM provider was not set. The default provider @@ -114,24 +117,49 @@ public sealed class DataSourceService } var usingTrustedProvider = selectedLLMProvider.IsTrustedForDataSourceSecurityChecks(this.settingsManager); - var participatingProviders = this.GetParticipatingProviders(selectedLLMProvider.ConfiguredProviderId, dataSourceOptions, + var participatingProviders = this.GetParticipatingProviders(selectedLLMProvider.ConfiguredProviderId, dataSourceOptions, retrievalMode, new("chat provider", usingTrustedProvider, selectedLLMProvider.GetConfidenceLevel(this.settingsManager))); return await this.GetDataSources(usingTrustedProvider, participatingProviders, previousSelectedDataSources); } - private IReadOnlyList GetParticipatingProviders(string currentProviderId, DataSourceOptions dataSourceOptions, ParticipatingProvider currentProvider) + private IReadOnlyList GetParticipatingProviders(string currentProviderId, DataSourceOptions dataSourceOptions, DataSourceRetrievalMode retrievalMode, ParticipatingProvider currentProvider) { var providers = new List { currentProvider }; - - if (dataSourceOptions.AutomaticDataSourceSelection) - this.AddAgentProvider(providers, Components.AGENT_DATA_SOURCE_SELECTION, currentProviderId, "data source selection agent"); - - if (dataSourceOptions.AutomaticValidation && this.settingsManager.ConfigurationData.AgentRetrievalContextValidation.EnableRetrievalContextValidation) - this.AddAgentProvider(providers, Components.AGENT_RETRIEVAL_CONTEXT_VALIDATION, currentProviderId, "retrieval context validation agent"); + var retrievalContextValidationEnabled = this.settingsManager.ConfigurationData.AgentRetrievalContextValidation.EnableRetrievalContextValidation; + foreach (var (component, role) in GetParticipatingAgents(dataSourceOptions, retrievalMode, retrievalContextValidationEnabled)) + this.AddAgentProvider(providers, component, currentProviderId, role); return providers; } + /// + /// Which agents get to see the data of the data sources, besides the chat provider. + /// + /// + /// Only the classic RAG process runs agents. With Semantic Search, the chat model picks the + /// data sources and judges what it found itself, so the data reaches no other provider. + /// Counting the providers of the agents there would hold back data sources which the chat + /// provider alone may use. + /// + /// The active data source options. + /// How the data sources are searched in effect. + /// Whether the validation of retrieval contexts is enabled in the settings. + /// The component of each participating agent, together with its role for the log. + internal static IReadOnlyList<(Components Component, string Role)> GetParticipatingAgents(DataSourceOptions dataSourceOptions, DataSourceRetrievalMode retrievalMode, bool retrievalContextValidationEnabled) + { + if (retrievalMode is DataSourceRetrievalMode.SEMANTIC_SEARCH) + return []; + + var agents = new List<(Components Component, string Role)>(2); + if (dataSourceOptions.AutomaticDataSourceSelection) + agents.Add((Components.AGENT_DATA_SOURCE_SELECTION, "data source selection agent")); + + if (dataSourceOptions.AutomaticValidation && retrievalContextValidationEnabled) + agents.Add((Components.AGENT_RETRIEVAL_CONTEXT_VALIDATION, "retrieval context validation agent")); + + return agents; + } + private void AddAgentProvider(List providers, Components component, string currentProviderId, string role) { var provider = this.settingsManager.GetPreselectedProvider(component, currentProviderId, true); diff --git a/app/MindWork AI Studio/Tools/Services/DirectChatService.cs b/app/MindWork AI Studio/Tools/Services/DirectChatService.cs index 0bbd5ffe..dfb50c27 100644 --- a/app/MindWork AI Studio/Tools/Services/DirectChatService.cs +++ b/app/MindWork AI Studio/Tools/Services/DirectChatService.cs @@ -274,7 +274,7 @@ public sealed class DirectChatService(SettingsManager settingsManager, DataSourc // decide which agent providers take part, and an agent with too little confidence makes // a data source unavailable. // - availableDataSources = await dataSourceService.GetAllowedDataSources(provider, chosenOptions, requestedDataSources); + availableDataSources = await dataSourceService.GetAllowedDataSources(provider, chosenOptions, DataSourceRetrievalMode.EVERY_MESSAGE, requestedDataSources); } catch (Exception exception) { diff --git a/app/Tests/Tools/DataSourceParticipatingAgentsTests.cs b/app/Tests/Tools/DataSourceParticipatingAgentsTests.cs new file mode 100644 index 00000000..e66a28ea --- /dev/null +++ b/app/Tests/Tools/DataSourceParticipatingAgentsTests.cs @@ -0,0 +1,51 @@ +using AIStudio.Settings.DataModel; +using AIStudio.Tools.Services; + +namespace AIStudio.Tests.Tools; + +/// +/// Checks which agents count as seeing the data of the data sources. +/// +/// +/// Every provider which sees the data must be trusted enough for every data source it sees. That +/// cuts both ways: leaving out an agent which does run would send data to a provider trusted too +/// little, while counting an agent which does not run holds back data sources for no reason. The +/// classic RAG process runs its agents; Semantic Search runs none, since the chat model picks the +/// data sources and judges the passages itself. +/// +[TestFixture] +public sealed class DataSourceParticipatingAgentsTests +{ + [Test] + public void SemanticSearchRunsNoAgent() + { + var agents = DataSourceService.GetParticipatingAgents(Options(automaticSelection: true, automaticValidation: true), DataSourceRetrievalMode.SEMANTIC_SEARCH, retrievalContextValidationEnabled: true); + + Assert.That(agents, Is.Empty, "The chat provider alone sees the data, so its trust alone decides which data sources it may search."); + } + + [Test] + public void TheClassicRAGProcessCountsTheAgentsItRuns() + { + var agents = DataSourceService.GetParticipatingAgents(Options(automaticSelection: true, automaticValidation: true), DataSourceRetrievalMode.EVERY_MESSAGE, retrievalContextValidationEnabled: true); + + Assert.That(agents.Select(agent => agent.Component), Is.EqualTo(new[] { AIStudio.Tools.Components.AGENT_DATA_SOURCE_SELECTION, AIStudio.Tools.Components.AGENT_RETRIEVAL_CONTEXT_VALIDATION })); + } + + [Test] + public void TheClassicRAGProcessCountsNoAgentItDoesNotRun() + { + Assert.Multiple(() => + { + Assert.That(DataSourceService.GetParticipatingAgents(Options(automaticSelection: false, automaticValidation: false), DataSourceRetrievalMode.EVERY_MESSAGE, retrievalContextValidationEnabled: true), Is.Empty); + Assert.That(DataSourceService.GetParticipatingAgents(Options(automaticSelection: false, automaticValidation: true), DataSourceRetrievalMode.EVERY_MESSAGE, retrievalContextValidationEnabled: false), Is.Empty, "The validation of this chat is on, but the settings switch it off everywhere."); + }); + } + + private static DataSourceOptions Options(bool automaticSelection, bool automaticValidation) => new() + { + DisableDataSources = false, + AutomaticDataSourceSelection = automaticSelection, + AutomaticValidation = automaticValidation, + }; +} \ No newline at end of file