Outbound data restrictions after searching data sources (#1106)

app/MindWork AI Studio/Settings/ConfigurationSelectDataFactory.cs

@@ -336,9 +336,9 @@ public static class ConfigurationSelectDataFactory
     }
 
     /// <summary>
-    /// The restrictions a mailbox may have.
+    /// The restrictions a mailbox or a data source may have.
     /// </summary>
-    /// <param name="minimumRestriction">The least strict restriction the organization allows, see DataMailboxes.MinimumOutboundDataRestriction.</param>
+    /// <param name="minimumRestriction">The least strict restriction allowed, see DataMailboxes.MinimumOutboundDataRestriction and DataSourceOutboundDataExtensions.GetMinimumOutboundDataRestriction.</param>
     public static IEnumerable<ConfigurationSelectData<OutboundDataRestriction>> GetOutboundDataRestrictionData(OutboundDataRestriction minimumRestriction)
     {
         foreach (var restriction in Enum.GetValues<OutboundDataRestriction>())

app/MindWork AI Studio/Settings/DataModel/DataSourceERI_V1.cs

@@ -63,6 +63,9 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
     /// <inheritdoc />
     public DataSourceSecurity SecurityPolicy { get; init; } = DataSourceSecurity.NOT_SPECIFIED;
 
+    /// <inheritdoc />
+    public OutboundDataRestriction OutboundDataRestriction { get; init; } = DataSourceOutboundDataExtensions.DEFAULT_OUTBOUND_DATA_RESTRICTION;
+
     /// <inheritdoc />
     public bool IsEnterpriseConfiguration { get; init; }
 
@@ -262,6 +265,20 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
             return false;
         }
 
+        //
+        // Optional, so that configurations written before the setting existed stay valid; their data
+        // sources get the level of a new one. A value this version cannot read counts as the
+        // strictest level, as in the settings file. Only a name counts, never a number, see EnumNames.
+        // A data source for self-hosted providers only is not clamped here: the level applies when
+        // its passages reach a chat, see DataSourceOutboundDataExtensions.GetEffectiveOutboundDataRestriction.
+        //
+        var outboundDataRestriction = DataSourceOutboundDataExtensions.DEFAULT_OUTBOUND_DATA_RESTRICTION;
+        if (table.TryGetValue("OutboundDataRestriction", out var outboundDataRestrictionValue) && !(outboundDataRestrictionValue.TryRead<string>(out var outboundDataRestrictionText) && EnumNames.TryParse(outboundDataRestrictionText, out outboundDataRestriction)))
+        {
+            LOGGER.LogWarning($"The configured data source {idx} does not contain a valid outbound data restriction, so the strictest one applies. (Plugin ID: {configPluginId})");
+            outboundDataRestriction = OutboundDataRestriction.ONLY_CONFIGURED_SERVICES;
+        }
+
         if (!table.TryGetValue("SelectedRetrievalId", out var selectedRetrievalIdValue) || !selectedRetrievalIdValue.TryRead<string>(out var selectedRetrievalId) || string.IsNullOrWhiteSpace(selectedRetrievalId))
         {
             LOGGER.LogWarning($"The configured data source {idx} must specify a selected retrieval ID. (Plugin ID: {configPluginId})");
@@ -319,6 +336,7 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
             Username = username,
             UsernamePasswordMode = usernamePasswordMode,
             SecurityPolicy = securityPolicy,
+            OutboundDataRestriction = outboundDataRestriction,
             Version = ERIVersion.V1,
             SelectedRetrievalId = selectedRetrievalId,
             MaxMatches = (ushort)maxMatches,
@@ -379,6 +397,7 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
                     {{usernameLine}}
                     {{secretLine}}
                     ["SecurityPolicy"] = "{{this.SecurityPolicy}}",
+                    ["OutboundDataRestriction"] = "{{this.OutboundDataRestriction}}",
                     ["SelectedRetrievalId"] = "{{LuaTools.EscapeLuaString(this.SelectedRetrievalId)}}",
                     ["MaxMatches"] = {{this.MaxMatches}},
                 }

app/MindWork AI Studio/Settings/DataModel/DataSourceLocalDirectory.cs

@@ -64,7 +64,10 @@ public readonly record struct DataSourceLocalDirectory : IInternalDataSource
     
     /// <inheritdoc />
     public ushort MaxMatches { get; init; } = 10;
-    
+
+    /// <inheritdoc />
+    public OutboundDataRestriction OutboundDataRestriction { get; init; } = DataSourceOutboundDataExtensions.DEFAULT_OUTBOUND_DATA_RESTRICTION;
+
     /// <inheritdoc />
     public Task<IReadOnlyList<IRetrievalContext>> RetrieveDataAsync(IContent lastUserPrompt, ChatThread thread, CancellationToken token = default) =>
         Program.SERVICE_PROVIDER.GetRequiredService<DataSourceLocalRetrievalService>().RetrieveDataAsync(this, lastUserPrompt, thread, token);

app/MindWork AI Studio/Settings/DataModel/DataSourceLocalFile.cs

@@ -52,7 +52,10 @@ public readonly record struct DataSourceLocalFile : IInternalDataSource
     
     /// <inheritdoc />
     public ushort MaxMatches { get; init; } = 10;
-    
+
+    /// <inheritdoc />
+    public OutboundDataRestriction OutboundDataRestriction { get; init; } = DataSourceOutboundDataExtensions.DEFAULT_OUTBOUND_DATA_RESTRICTION;
+
     /// <inheritdoc />
     public Task<IReadOnlyList<IRetrievalContext>> RetrieveDataAsync(IContent lastUserPrompt, ChatThread thread, CancellationToken token = default) =>
         Program.SERVICE_PROVIDER.GetRequiredService<DataSourceLocalRetrievalService>().RetrieveDataAsync(this, lastUserPrompt, thread, token);

app/MindWork AI Studio/Settings/DataModel/OutboundDataOrigin.cs

@@ -26,4 +26,10 @@ public enum OutboundDataOrigin
     /// in the intranet or behind a VPN, or a page of the configured wiki, whatever its address.
     /// </summary>
     INTERNAL_WEB_PAGE,
+
+    /// <summary>
+    /// A configured data source, such as a local folder or an ERI server, whose passages reached the
+    /// model: found by Semantic Search or by the classic RAG process.
+    /// </summary>
+    DATA_SOURCE,
 }

app/MindWork AI Studio/Settings/DataModel/OutboundDataRequirement.cs

@@ -5,12 +5,12 @@ namespace AIStudio.Settings.DataModel;
 /// </summary>
 /// <remarks>
 /// The source travels along so a tool which is kept from running can name it, and the user
-/// learns what stands in the way. For a mailbox, the reference is its id rather than its name,
-/// because a mailbox may be renamed after a chat read from it.
+/// learns what stands in the way. For a mailbox or a data source, the reference is its id rather
+/// than its name, because either may be renamed after a chat read from it.
 /// </remarks>
 /// <param name="Restriction">Where the chat may still send data.</param>
 /// <param name="Origin">What kind of source demands the restriction, or NONE when nothing restricts the chat.</param>
-/// <param name="Reference">Which source of its kind demands the restriction: the id of a mailbox, or the host of a private web page. An empty text when the kind says it all, as for the wiki, or when nothing restricts the chat.</param>
+/// <param name="Reference">Which source of its kind demands the restriction: the id of a mailbox or of a data source, or the host of a private web page. An empty text when the kind says it all, as for the wiki, or when nothing restricts the chat.</param>
 public sealed record OutboundDataRequirement(OutboundDataRestriction Restriction, OutboundDataOrigin Origin, string Reference)
 {
     /// <summary>
@@ -25,9 +25,15 @@ public sealed record OutboundDataRequirement(OutboundDataRestriction Restriction
     /// <remarks>
     /// On a tie, this one is kept, whatever the origin of either, so the source which set a level
     /// first stays the one which is named, and a chat does not name another source with every
-    /// search.
+    /// search. A restriction this version does not know counts as the strictest one, and the
+    /// requirement which wins carries that one instead.
     /// </remarks>
     /// <param name="other">The other requirement.</param>
     /// <returns>The requirement whose restriction is the stricter one.</returns>
-    public OutboundDataRequirement StricterOf(OutboundDataRequirement other) => other.Restriction < this.Restriction ? other : this;
+    public OutboundDataRequirement StricterOf(OutboundDataRequirement other)
+    {
+        var stricter = other.Restriction.OrStrictestWhenUnknown() < this.Restriction.OrStrictestWhenUnknown() ? other : this;
+        var knownRestriction = stricter.Restriction.OrStrictestWhenUnknown();
+        return knownRestriction == stricter.Restriction ? stricter : stricter with { Restriction = knownRestriction };
+    }
 }

app/MindWork AI Studio/Settings/DataModel/OutboundDataRequirementExtensions.cs

@@ -16,13 +16,14 @@ public static class OutboundDataRequirementExtensions
     /// had when the chat read the content: a change made later does not loosen the chat.
     /// </remarks>
     /// <param name="requirement">What the chat demands.</param>
-    /// <param name="settings">The configuration, to find the name of the mailbox which demands it.</param>
+    /// <param name="settings">The configuration, to find the name of the mailbox or data source which demands it.</param>
     /// <returns>The text for the user.</returns>
     public static string GetToolBlockedMessage(this OutboundDataRequirement requirement, Data settings) => requirement.Origin switch
     {
         OutboundDataOrigin.MAILBOX => GetMailboxBlockedMessage(requirement, settings.Mailboxes),
         OutboundDataOrigin.CONFLUENCE_WIKI => string.Format(TB("This chat searched your Confluence wiki while the setting 'Where a chat may send data after reading internal pages' of the tool 'Read Web Page' was '{0}'. This tool would send data beyond that, so it is not available in this chat. A new chat can use it again."), requirement.Restriction.GetName()),
         OutboundDataOrigin.INTERNAL_WEB_PAGE => string.Format(TB("This chat read an internal web page on '{0}' while the setting 'Where a chat may send data after reading internal pages' of the tool 'Read Web Page' was '{1}'. This tool would send data beyond that, so it is not available in this chat. A new chat can use it again."), requirement.Reference, requirement.Restriction.GetName()),
+        OutboundDataOrigin.DATA_SOURCE => GetDataSourceBlockedMessage(requirement, settings.DataSources),
 
         // NONE keeps no tool from running, since nothing restricts such a chat. What is left is an
         // origin this version does not know:
@@ -40,4 +41,16 @@ public static class OutboundDataRequirementExtensions
             ? string.Format(TB("This chat read mails from a mailbox which has been removed since, while its setting 'Where a chat may send data after reading mails' was '{0}'. This tool would send data beyond that, so it is not available in this chat. A new chat can use it again."), requirement.Restriction.GetName())
             : string.Format(TB("This chat read mails from the mailbox '{0}' while its setting 'Where a chat may send data after reading mails' was '{1}'. This tool would send data beyond that, so it is not available in this chat. A new chat can use it again."), mailboxName, requirement.Restriction.GetName());
     }
+
+    private static string GetDataSourceBlockedMessage(OutboundDataRequirement requirement, IEnumerable<IDataSource> dataSources)
+    {
+        var dataSourceName = dataSources
+            .Where(dataSource => string.Equals(dataSource.Id, requirement.Reference, StringComparison.Ordinal))
+            .Select(dataSource => dataSource.Name)
+            .FirstOrDefault();
+
+        return string.IsNullOrWhiteSpace(dataSourceName)
+            ? string.Format(TB("This chat found content in a data source which has been removed since, while its setting 'Where a chat may send data after reading this data source' was '{0}'. This tool would send data beyond that, so it is not available in this chat. A new chat can use it again."), requirement.Restriction.GetName())
+            : string.Format(TB("This chat found content in the data source '{0}' while its setting 'Where a chat may send data after reading this data source' was '{1}'. This tool would send data beyond that, so it is not available in this chat. A new chat can use it again."), dataSourceName, requirement.Restriction.GetName());
+    }
 }

app/MindWork AI Studio/Settings/DataModel/OutboundDataRestriction.cs

@@ -2,18 +2,20 @@ namespace AIStudio.Settings.DataModel;
 
 /// <summary>
 /// Where a chat may still send data, once internal content entered it: mails, a search of the
-/// wiki, or an internal web page.
+/// wiki, an internal web page, or passages of a data source.
 /// </summary>
 /// <remarks>
 /// The members run from the strictest to the most permissive, so the stricter of two levels is the
 /// smaller one. Services which are configured in AI Studio, such as Confluence, the allowed private
-/// hosts, or the mailbox itself, stay allowed on every level.
+/// hosts, the data sources, or the mailbox itself, stay allowed on every level.
 /// </remarks>
 public enum OutboundDataRestriction
 {
     // The strictest level is deliberately the member with the underlying value 0: when the settings
     // file holds a value TolerantEnumConverter cannot read, it falls back to that member. Falling back
-    // to a more permissive level would open a way out for mail content the user meant to keep in.
+    // to a more permissive level would open a way out for internal content the user meant to keep in.
+    // A number which names no member is read as it is, so the comparisons count it as the strictest
+    // level, see OutboundDataRestrictionExtensions.OrStrictestWhenUnknown.
 
     /// <summary>
     /// Only the services configured in AI Studio.

app/MindWork AI Studio/Settings/DataModel/OutboundDataRestrictionExtensions.cs

@@ -9,10 +9,30 @@ public static class OutboundDataRestrictionExtensions
     /// <summary>
     /// The stricter of two restrictions.
     /// </summary>
+    /// <remarks>
+    /// A restriction this version does not know counts as the strictest one, see OrStrictestWhenUnknown.
+    /// </remarks>
     /// <param name="restriction">The one restriction.</param>
     /// <param name="other">The other restriction.</param>
     /// <returns>The restriction which allows less.</returns>
-    public static OutboundDataRestriction StricterOf(this OutboundDataRestriction restriction, OutboundDataRestriction other) => other < restriction ? other : restriction;
+    public static OutboundDataRestriction StricterOf(this OutboundDataRestriction restriction, OutboundDataRestriction other)
+    {
+        var knownRestriction = restriction.OrStrictestWhenUnknown();
+        var knownOther = other.OrStrictestWhenUnknown();
+        return knownOther < knownRestriction ? knownOther : knownRestriction;
+    }
+
+    /// <summary>
+    /// The restriction as this version applies it.
+    /// </summary>
+    /// <remarks>
+    /// The settings file and a saved chat may hold a number which names no member, e.g., one written
+    /// by hand, and the settings read it all the same. Compared as it is, such a number would rank
+    /// above UNRESTRICTED and lift every restriction it meets.
+    /// </remarks>
+    /// <param name="restriction">The restriction as it was read.</param>
+    /// <returns>The restriction itself, or the strictest one when this version does not know it.</returns>
+    public static OutboundDataRestriction OrStrictestWhenUnknown(this OutboundDataRestriction restriction) => Enum.IsDefined(restriction) ? restriction : OutboundDataRestriction.ONLY_CONFIGURED_SERVICES;
 
     public static string GetName(this OutboundDataRestriction restriction) => restriction switch
     {
@@ -28,9 +48,9 @@ public static class OutboundDataRestrictionExtensions
     /// </summary>
     public static string GetDescription(this OutboundDataRestriction restriction) => restriction switch
     {
-        OutboundDataRestriction.ONLY_CONFIGURED_SERVICES => TB("The chat may only use services configured in AI Studio: your mailboxes, your Confluence wiki, and the allowed private hosts of the tool 'Read Web Page'. It reads no other web pages, and it does not search the web."),
+        OutboundDataRestriction.ONLY_CONFIGURED_SERVICES => TB("The chat may only use services configured in AI Studio: your mailboxes, your data sources, your Confluence wiki, and the allowed private hosts of the tool 'Read Web Page'. It reads no other web pages, and it does not search the web."),
         OutboundDataRestriction.ONLY_LINKS_FROM_CHAT => TB("The chat may also read web pages whose addresses stand in the chat, written by you or returned by a tool, exactly as they stand there. It does not search the web, and the AI cannot choose addresses of its own outside the services configured in AI Studio."),
-        OutboundDataRestriction.UNRESTRICTED => TB("The chat may use every tool you chose, web search and any web page included. Internal content, such as your mails, can then reach third parties, e.g., inside a search query or the address of a web page."),
+        OutboundDataRestriction.UNRESTRICTED => TB("The chat may use every tool you chose, web search and any web page included. Internal content, such as your mails or the documents of your data sources, can then reach third parties, e.g., inside a search query or the address of a web page."),
 
         _ => TB("This version of AI Studio does not know this restriction, so it applies the strictest one."),
     };

app/MindWork AI Studio/Settings/DataSourceOrderExtensions.cs

@@ -0,0 +1,21 @@
+namespace AIStudio.Settings;
+
+public static class DataSourceOrderExtensions
+{
+    /// <summary>
+    /// Sorts data sources in a fixed order: by their number, then by their ID.
+    /// </summary>
+    /// <remarks>
+    /// The same data sources always come in the same order, however the settings list them, the
+    /// checks return them, or an agent selects them. Semantic Search offers them this way, because
+    /// the providers cache a request from its beginning, and the tools are part of that beginning,
+    /// see IToolImplementation.ResolveFunctionAsync. DataSourceRequirements names the first of
+    /// several equally strict data sources this way, so that both ways of searching name the same.
+    /// </remarks>
+    /// <param name="dataSources">The data sources to sort.</param>
+    /// <returns>The data sources in their fixed order.</returns>
+    public static IReadOnlyList<IDataSource> InFixedOrder(this IEnumerable<IDataSource> dataSources) => dataSources
+        .OrderBy(dataSource => dataSource.Num)
+        .ThenBy(dataSource => dataSource.Id, StringComparer.Ordinal)
+        .ToList();
+}

app/MindWork AI Studio/Settings/DataSourceOutboundDataExtensions.cs

@@ -0,0 +1,55 @@
+using AIStudio.Settings.DataModel;
+
+namespace AIStudio.Settings;
+
+public static class DataSourceOutboundDataExtensions
+{
+    /// <summary>
+    /// The level of a new data source, and of one stored before data sources had a level.
+    /// </summary>
+    /// <remarks>
+    /// The same level as for internal web pages: no web search, and no addresses the model chooses
+    /// itself outside the services configured in AI Studio, while the links which stand in the chat
+    /// stay readable. A data source holds the documents of the user or of their organization, so a
+    /// chat which found something in it must not carry that out in a search query.
+    /// </remarks>
+    public const OutboundDataRestriction DEFAULT_OUTBOUND_DATA_RESTRICTION = OutboundDataRestriction.ONLY_LINKS_FROM_CHAT;
+
+    /// <summary>
+    /// The least strict level a data source with this security policy may have.
+    /// </summary>
+    /// <remarks>
+    /// A data source which only self-hosted providers may see must not reach a third party through a
+    /// tool either, so it allows the services configured in AI Studio only. The internal pages among
+    /// them stay readable: the wiki, and the allowed private hosts of Read Web Page.
+    /// </remarks>
+    /// <param name="securityPolicy">The security policy of the data source; NOT_SPECIFIED for one which has none, such as a local data source.</param>
+    /// <param name="organizationMinimum">The least strict level the organization allows, see DataSourceSecuritySettings.MinimumOutboundDataRestriction.</param>
+    /// <returns>The least strict level allowed.</returns>
+    public static OutboundDataRestriction GetMinimumOutboundDataRestriction(DataSourceSecurity securityPolicy, OutboundDataRestriction organizationMinimum) => securityPolicy is DataSourceSecurity.SELF_HOSTED
+        ? organizationMinimum.StricterOf(OutboundDataRestriction.ONLY_CONFIGURED_SERVICES)
+        : organizationMinimum;
+
+    /// <summary>
+    /// The least strict level this data source may have.
+    /// </summary>
+    /// <param name="dataSource">The data source.</param>
+    /// <param name="organizationMinimum">The least strict level the organization allows, see DataSourceSecuritySettings.MinimumOutboundDataRestriction.</param>
+    /// <returns>The least strict level allowed.</returns>
+    public static OutboundDataRestriction GetMinimumOutboundDataRestriction(this IDataSource dataSource, OutboundDataRestriction organizationMinimum) =>
+        GetMinimumOutboundDataRestriction(dataSource is IExternalDataSource externalDataSource ? externalDataSource.SecurityPolicy : DataSourceSecurity.NOT_SPECIFIED, organizationMinimum);
+
+    /// <summary>
+    /// Where a chat may still send data, once passages of this data source reached it.
+    /// </summary>
+    /// <remarks>
+    /// The level of the data source, tightened to the least strict one it may have. This holds at
+    /// runtime, not only in the dialog: a data source stored before, or one a configuration plugin
+    /// provides, is never edited, so its stored level may be less strict.
+    /// </remarks>
+    /// <param name="dataSource">The data source.</param>
+    /// <param name="organizationMinimum">The least strict level the organization allows, see DataSourceSecuritySettings.MinimumOutboundDataRestriction.</param>
+    /// <returns>The level which applies.</returns>
+    public static OutboundDataRestriction GetEffectiveOutboundDataRestriction(this IDataSource dataSource, OutboundDataRestriction organizationMinimum) =>
+        dataSource.OutboundDataRestriction.StricterOf(dataSource.GetMinimumOutboundDataRestriction(organizationMinimum));
+}

app/MindWork AI Studio/Settings/DataSourceSecuritySettings.cs

@@ -17,4 +17,15 @@ public sealed class DataSourceSecuritySettings(Expression<Func<Data, DataSourceS
     /// Provider instance IDs trusted by an organization for data-source security checks.
     /// </summary>
     public HashSet<string> TrustedProviderIds { get; set; } = ManagedConfiguration.Register(configSelection, n => n.TrustedProviderIds, []);
+
+    /// <summary>
+    /// The least strict outbound data restriction a data source may have.
+    /// </summary>
+    /// <remarks>
+    /// Only an organization sets it; the default leaves every level to the user. A data source set to
+    /// a less strict level gets this one when its passages reach a chat, see
+    /// DataSourceOutboundDataExtensions.GetEffectiveOutboundDataRestriction, and its dialog no longer
+    /// offers the less strict levels.
+    /// </remarks>
+    public OutboundDataRestriction MinimumOutboundDataRestriction { get; set; } = ManagedConfiguration.Register(configSelection, n => n.MinimumOutboundDataRestriction, OutboundDataRestriction.UNRESTRICTED);
 }

app/MindWork AI Studio/Settings/IDataSource.cs

@@ -20,6 +20,17 @@ public interface IDataSource : IDataSourceBase
     /// </summary>
     public ushort MaxMatches { get; init; }
 
+    /// <summary>
+    /// Where a chat may still send data, once content of this data source reached it.
+    /// </summary>
+    /// <remarks>
+    /// Read it through DataSourceOutboundDataExtensions.GetEffectiveOutboundDataRestriction, never
+    /// on its own: an organization may demand a stricter level, see
+    /// DataSourceSecuritySettings.MinimumOutboundDataRestriction, and so does a data source which only
+    /// self-hosted providers may see.
+    /// </remarks>
+    public OutboundDataRestriction OutboundDataRestriction { get; init; }
+
     /// <summary>
     /// The last page which can be retrieved from this data source.
     /// </summary>