Outbound data restrictions after searching data sources (#1106)

app/MindWork AI Studio/Tools/RAG/DataSourceRequirements.cs

@@ -0,0 +1,57 @@
+using AIStudio.Provider;
+using AIStudio.Settings;
+using AIStudio.Settings.DataModel;
+
+namespace AIStudio.Tools.RAG;
+
+/// <summary>
+/// What a chat has to require from now on, because passages of data sources reached the model.
+/// </summary>
+/// <remarks>
+/// Both ways of searching the data sources ask here, Semantic Search and the classic RAG process,
+/// so that both demand the same of the same data sources: two answers which drift apart would leave
+/// one of the ways open.
+/// </remarks>
+/// <param name="Confidence">The provider confidence the chat requires from now on.</param>
+/// <param name="Security">The data security the chat requires from now on; NOT_SPECIFIED leaves it as it was.</param>
+/// <param name="OutboundData">Where the chat may still send data, and which data source demands it.</param>
+public sealed record DataSourceRequirements(ConfidenceLevel Confidence, DataSourceSecurity Security, OutboundDataRequirement OutboundData)
+{
+    /// <summary>
+    /// Demands nothing: what a search requires which brought no passage into the chat.
+    /// </summary>
+    public static readonly DataSourceRequirements NONE = new(ConfidenceLevel.NONE, DataSourceSecurity.NOT_SPECIFIED, OutboundDataRequirement.NONE);
+
+    /// <summary>
+    /// What the passages of these data sources demand.
+    /// </summary>
+    /// <remarks>
+    /// Only the data sources with passages count: a data source which found nothing brought nothing
+    /// into the chat. A data source which only self-hosted providers may use restricts the chat to
+    /// those. Of the outbound data restrictions, the strictest wins, see
+    /// DataSourceOutboundDataExtensions.GetEffectiveOutboundDataRestriction. Of several equally strict
+    /// data sources, the first in their fixed order is named, see DataSourceOrderExtensions.InFixedOrder,
+    /// so that the name does not depend on the order in which one way of searching got them.
+    /// </remarks>
+    /// <param name="dataSources">The data sources searched.</param>
+    /// <param name="passageCounts">How many passages of each data source reached the model, in the same order.</param>
+    /// <param name="organizationMinimum">The least strict outbound data restriction the organization allows, see DataSourceSecuritySettings.MinimumOutboundDataRestriction.</param>
+    /// <returns>What the chat requires from now on; NONE when no data source brought a passage.</returns>
+    /// <exception cref="ArgumentException">The counts do not match the data sources.</exception>
+    public static DataSourceRequirements Of(IReadOnlyList<IDataSource> dataSources, IReadOnlyList<int> passageCounts, OutboundDataRestriction organizationMinimum)
+    {
+        if (passageCounts.Count != dataSources.Count)
+            throw new ArgumentException($"There are {passageCounts.Count} passage counts for {dataSources.Count} data sources.", nameof(passageCounts));
+
+        var contributingDataSources = dataSources.Where((_, index) => passageCounts[index] > 0).InFixedOrder();
+        if (contributingDataSources.Count == 0)
+            return NONE;
+
+        var outboundData = OutboundDataRequirement.NONE;
+        foreach (var dataSource in contributingDataSources)
+            outboundData = outboundData.StricterOf(new(dataSource.GetEffectiveOutboundDataRestriction(organizationMinimum), OutboundDataOrigin.DATA_SOURCE, dataSource.Id));
+
+        var requiresSelfHosted = contributingDataSources.OfType<IExternalDataSource>().Any(dataSource => dataSource.SecurityPolicy is DataSourceSecurity.SELF_HOSTED);
+        return new(contributingDataSources.GetRequiredConfidenceLevel(), requiresSelfHosted ? DataSourceSecurity.SELF_HOSTED : DataSourceSecurity.ALLOW_ANY, outboundData);
+    }
+}

app/MindWork AI Studio/Tools/RAG/RAGProcesses/AISrcSelWithRetCtxVal.cs

@@ -43,8 +43,9 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
         // them can be used right now, or the search found nothing -- would still send the
         // passages of the last message which did, cf. ChatThread.RollBackTo.
         //
-        // The data security and the required provider confidence stay as they are: both only
-        // ever tighten, because the data which raised them was seen by this thread.
+        // The data security, the required provider confidence, and where the chat may still send
+        // data stay as they are: they only ever tighten, because the data which raised them was
+        // seen by this thread.
         //
         chatThread.AugmentedData = string.Empty;
         chatThread.AISelectedDataSources = [];
@@ -187,13 +188,17 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
                     retrievalTasks.Add(dataSource.RetrieveDataAsync(lastUserPrompt, chatThreadWithoutWaitingAIBlock, token));
             
                 //
-                // Wait for all retrieval tasks to finish:
+                // Wait for all retrieval tasks to finish, and count the passages of each data source.
+                // The tasks are in the order of the selected data sources:
                 //
-                foreach (var retrievalTask in retrievalTasks)
+                var passageCounts = new int[retrievalTasks.Count];
+                for (var index = 0; index < retrievalTasks.Count; index++)
                 {
                     try
                     {
-                        dataContexts.AddRange(await retrievalTask);
+                        var retrievedContexts = await retrievalTasks[index];
+                        dataContexts.AddRange(retrievedContexts);
+                        passageCounts[index] = retrievedContexts.Count;
                     }
                     catch (OperationCanceledException) when (token.IsCancellationRequested)
                     {
@@ -205,6 +210,12 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
                         LOGGER.LogError(e, "An error occurred during the retrieval process.");
                     }
                 }
+
+                //
+                // Right after the retrieval, before the augmentation: the passages are in the chat
+                // from here on, and a step below which ends early must not leave it unrestricted:
+                //
+                RequireOutboundDataRestriction(chatThread, selectedDataSources, passageCounts, settings.ConfigurationData.DataSourceSecurity.MinimumOutboundDataRestriction);
             }
 
             //
@@ -250,6 +261,33 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
 
     #endregion
 
+    /// <summary>
+    /// Restricts where the chat may still send data, because of the passages the retrieval brought in.
+    /// </summary>
+    /// <remarks>
+    /// Semantic Search asks the same place, DataSourceRequirements, so that both ways of searching
+    /// demand the same of the same data sources. Only the data sources with passages count: one
+    /// which found nothing leaves the chat as it was. A passage the validation agent drops later
+    /// still counts, which is the safe side. The data security and the provider confidence were
+    /// raised before the retrieval already, for every selected data source, because the agents see
+    /// the passages as well; the data sources with passages are among them, so asking again would
+    /// change nothing.<br/><br/>
+    /// The answer which follows does not offer the web search any more: the request reads the
+    /// restriction from this very thread when it puts its tools together, after this process, see
+    /// ToolRegistry.GetRunnableToolsAsync.
+    /// </remarks>
+    /// <param name="chatThread">The chat thread.</param>
+    /// <param name="selectedDataSources">The data sources searched.</param>
+    /// <param name="passageCounts">How many passages each data source returned, in the same order.</param>
+    /// <param name="organizationMinimum">The least strict outbound data restriction the organization allows.</param>
+    internal static void RequireOutboundDataRestriction(ChatThread chatThread, IReadOnlyList<IDataSource> selectedDataSources, IReadOnlyList<int> passageCounts, OutboundDataRestriction organizationMinimum)
+    {
+        var previousRestriction = chatThread.RequiredOutboundDataRestriction;
+        chatThread.RequireOutboundDataRestriction(DataSourceRequirements.Of(selectedDataSources, passageCounts, organizationMinimum).OutboundData);
+        if (previousRestriction != chatThread.RequiredOutboundDataRestriction)
+            LOGGER.LogInformation($"The outbound data restriction of the chat thread was updated from '{previousRestriction.Restriction}' to '{chatThread.RequiredOutboundDataRestriction.Restriction}' because of the data source with id '{chatThread.RequiredOutboundDataRestriction.Reference}'.");
+    }
+
     /// <summary>
     /// Adds the note on the data sources whose server could not be reached to what the model reads along with the passages.
     /// </summary>