Outbound data restrictions after searching data sources (#1106)

app/MindWork AI Studio/Dialogs/DataSourceERI_V1Dialog.razor

@@ -125,6 +125,8 @@
                 }
             </MudSelect>
 
+            <OutboundDataRestrictionSelection @bind-Value="@this.SelectedOutboundDataRestriction" Label="@T("Where a chat may send data after reading this data source")" Explanation="@T("This setting decides where a chat may still send data once the AI found something in this data source, so that its content does not end up in a web search, for example. Choose no restriction only for content anybody may read anyway.")" OrganizationMinimum="@this.OrganizationMinimumOutboundDataRestriction" SecurityPolicy="@this.dataSecurityPolicy"/>
+
             <MudNumericField T="ushort" Min="10" @bind-Value="@this.dataMaxMatches" Label="@T("How many matches do you want at most per query?")" Variant="Variant.Outlined" Step="10" />
             
         </MudForm>

app/MindWork AI Studio/Dialogs/DataSourceERI_V1Dialog.razor.cs

@@ -1,5 +1,6 @@
 using AIStudio.Assistants.ERI;
 using AIStudio.Components;
+using AIStudio.Settings;
 using AIStudio.Settings.DataModel;
 using AIStudio.Tools.ERIClient;
 using AIStudio.Tools.ERIClient.DataModel;
@@ -61,6 +62,7 @@ public partial class DataSourceERI_V1Dialog : MSGComponentBase, ISecretId
     private string dataEditingPreviousInstanceName = string.Empty;
     private List<AuthMethod> availableAuthMethods = [];
     private DataSourceSecurity dataSecurityPolicy;
+    private OutboundDataRestriction dataOutboundDataRestriction = DataSourceOutboundDataExtensions.DEFAULT_OUTBOUND_DATA_RESTRICTION;
     private SecurityRequirements dataSourceSecurityRequirements;
     private ushort dataMaxMatches = 10;
     private bool connectionTested;
@@ -121,6 +123,7 @@ public partial class DataSourceERI_V1Dialog : MSGComponentBase, ISecretId
             this.dataAuthMethod = this.DataSource.AuthMethod;
             this.dataUsername = this.DataSource.Username;
             this.dataSecurityPolicy = this.DataSource.SecurityPolicy;
+            this.dataOutboundDataRestriction = this.DataSource.OutboundDataRestriction;
             this.dataMaxMatches = this.DataSource.MaxMatches;
             
             // We cannot load the retrieval processes now, since we have
@@ -199,11 +202,29 @@ public partial class DataSourceERI_V1Dialog : MSGComponentBase, ISecretId
             UsernamePasswordMode = DataSourceERIUsernamePasswordMode.USER_MANAGED,
             Type = DataSourceType.ERI_V1,
             SecurityPolicy = this.dataSecurityPolicy,
+            OutboundDataRestriction = this.SelectedOutboundDataRestriction,
             SelectedRetrievalId = this.dataSelectedRetrievalProcess.Id,
             MaxMatches = this.dataMaxMatches,
         };
     }
     
+    private OutboundDataRestriction OrganizationMinimumOutboundDataRestriction => this.SettingsManager.ConfigurationData.DataSourceSecurity.MinimumOutboundDataRestriction;
+
+    /// <summary>
+    /// The level the data source gets: the one chosen, as far as the organization and the security
+    /// policy allow it.
+    /// </summary>
+    /// <remarks>
+    /// Choosing the policy for self-hosted providers rules out every level but the strictest, see
+    /// DataSourceOutboundDataExtensions.GetMinimumOutboundDataRestriction. The choice itself stays,
+    /// so the dialog shows it again when the user chooses another policy.
+    /// </remarks>
+    private OutboundDataRestriction SelectedOutboundDataRestriction
+    {
+        get => this.dataOutboundDataRestriction.StricterOf(DataSourceOutboundDataExtensions.GetMinimumOutboundDataRestriction(this.dataSecurityPolicy, this.OrganizationMinimumOutboundDataRestriction));
+        set => this.dataOutboundDataRestriction = value;
+    }
+
     private void ClearImportRetrievalIssue() => this.importRetrievalIssue = string.Empty;
 
     private bool IsConnectionEncrypted() => this.dataHostname.StartsWith("https://", StringComparison.InvariantCultureIgnoreCase);

app/MindWork AI Studio/Dialogs/DataSourceERI_V1InfoDialog.razor

@@ -27,6 +27,7 @@
         <TextInfoLines Label="@T("Server description")" MaxLines="14" Value="@this.serverDescription" ClipboardTooltipSubject="@T("the server description")"/>
         <TextInfoLines Label="@T("Security requirements of the data provider")" MaxLines="3" Value="@this.securityRequirements.Explain()" ClipboardTooltipSubject="@T("the security requirements of the data provider")"/>
         <TextInfoLines Label="@T("Your security policy")" MaxLines="3" Value="@this.DataSource.SecurityPolicy.ToInfoText()" Color="@this.DataSource.SecurityPolicy.GetColor()" ClipboardTooltipSubject="@T("your security policy")"/>
+        <TextInfoLine Icon="@Icons.Material.Filled.Shield" Label="@T("Where a chat may send data after reading this data source")" Value="@this.OutboundDataRestrictionName" ClipboardTooltipSubject="@T("where a chat may send data after reading this data source")"/>
         <TextInfoLine Label="@T("Maximum matches per query")" Value="@this.DataSource.MaxMatches.ToString()" ClipboardTooltipSubject="@T("the maximum number of matches per query")"/>
 
         <MudText Typo="Typo.h5" Class="mt-6">

app/MindWork AI Studio/Dialogs/DataSourceERI_V1InfoDialog.razor.cs

@@ -4,6 +4,7 @@ using System.Text;
 
 using AIStudio.Assistants.ERI;
 using AIStudio.Components;
+using AIStudio.Settings;
 using AIStudio.Settings.DataModel;
 using AIStudio.Tools.ERIClient;
 using AIStudio.Tools.ERIClient.DataModel;
@@ -52,6 +53,11 @@ public partial class DataSourceERI_V1InfoDialog : MSGComponentBase, ISecretId
     
     private string Port => this.DataSource.Port == 0 ? string.Empty : $"{this.DataSource.Port}";
 
+    /// <summary>
+    /// The level which applies, including what the organization or the security policy demands.
+    /// </summary>
+    private string OutboundDataRestrictionName => this.DataSource.GetEffectiveOutboundDataRestriction(this.SettingsManager.ConfigurationData.DataSourceSecurity.MinimumOutboundDataRestriction).GetName();
+
     private async Task<(bool Success, DataSourceERI_V1 EffectiveDataSource)> CreateEffectiveDataSource()
     {
         this.effectiveUsername = this.DataSource.Username;

app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor

@@ -110,6 +110,8 @@
                 }
             </MudSelect>
 
+            <OutboundDataRestrictionSelection @bind-Value="@this.SelectedOutboundDataRestriction" Label="@T("Where a chat may send data after reading this data source")" Explanation="@T("This setting decides where a chat may still send data once the AI found something in this data source, so that its content does not end up in a web search, for example. Choose no restriction only for content anybody may read anyway.")" OrganizationMinimum="@this.OrganizationMinimumOutboundDataRestriction"/>
+
             @if (this.MediaTranscriptionHint is { } mediaTranscriptionHint)
             {
                 <MudAlert Severity="Severity.Info" Variant="Variant.Outlined" Dense="@true" Class="mb-3">

app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs

@@ -63,6 +63,7 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
     private bool showExpertSettings;
     private bool revalidateAfterRender;
     private ConfidenceLevel dataConfidenceLevel = ConfidenceLevel.UNKNOWN;
+    private OutboundDataRestriction dataOutboundDataRestriction = DataSourceOutboundDataExtensions.DEFAULT_OUTBOUND_DATA_RESTRICTION;
     
     // We get the form reference from Blazor code to validate it manually:
     private MudForm form = null!;
@@ -117,6 +118,7 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
             this.dataMaxChunkTokenLength = this.DataSource.MaxChunkTokenLength;
             this.dataChunkOverlapTokenLength = this.DataSource.ChunkOverlapTokenLength;
             this.dataConfidenceLevel = this.DataSource.ConfidenceLevel;
+            this.dataOutboundDataRestriction = this.DataSource.OutboundDataRestriction;
             this.dataMaxMatches = this.DataSource.MaxMatches;
         }
         
@@ -207,6 +209,22 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
 
     private IEnumerable<ConfigurationSelectData<ConfidenceLevel>> ConfidenceLevels => ConfigurationSelectDataFactory.GetDataSourceConfidenceLevelsData();
 
+    private OutboundDataRestriction OrganizationMinimumOutboundDataRestriction => this.SettingsManager.ConfigurationData.DataSourceSecurity.MinimumOutboundDataRestriction;
+
+    /// <summary>
+    /// The level the data source gets: the one chosen, as far as the organization allows it.
+    /// </summary>
+    /// <remarks>
+    /// The organization may rule out a level while the dialog is open, or after the data source was
+    /// stored with it. The choice itself stays, so the dialog shows it again once the organization
+    /// allows it again.
+    /// </remarks>
+    private OutboundDataRestriction SelectedOutboundDataRestriction
+    {
+        get => this.dataOutboundDataRestriction.StricterOf(this.OrganizationMinimumOutboundDataRestriction);
+        set => this.dataOutboundDataRestriction = value;
+    }
+
     private string SelectedEmbeddingTokenizerText => this.SelectedEmbedding is null
         ? T("No embedding selected")
         : string.IsNullOrWhiteSpace(this.SelectedEmbedding.TokenizerPath)
@@ -243,6 +261,7 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
         MaxChunkTokenLength = this.dataMaxChunkTokenLength,
         ChunkOverlapTokenLength = this.dataChunkOverlapTokenLength,
         ConfidenceLevel = this.dataConfidenceLevel,
+        OutboundDataRestriction = this.SelectedOutboundDataRestriction,
         MaxMatches = this.dataMaxMatches,
     };
 

app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryInfoDialog.razor

@@ -43,6 +43,7 @@
         }
 
         <TextInfoLine Label="@T("Required provider confidence level")" Value="@this.DataSource.ConfidenceLevel.GetName()" ClipboardTooltipSubject="@T("the required provider confidence level")"/>
+        <TextInfoLine Icon="@Icons.Material.Filled.Shield" Label="@T("Where a chat may send data after reading this data source")" Value="@this.OutboundDataRestrictionName" ClipboardTooltipSubject="@T("where a chat may send data after reading this data source")"/>
         <TextInfoLine Label="@T("Maximum matches per query")" Value="@this.DataSource.MaxMatches.ToString()" ClipboardTooltipSubject="@T("the maximum number of matches per query")"/>
 
         <TextInfoLine Icon="@Icons.Material.Filled.SquareFoot" Label="@T("Number of matching files")" Value="@this.NumberFilesInDirectory" ClipboardTooltipSubject="@T("the number of matching files in the directory")"/>

app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryInfoDialog.razor.cs

@@ -60,6 +60,11 @@ public partial class DataSourceLocalDirectoryInfoDialog : MSGComponentBase
 
     private bool IsDirectoryAvailable => this.directoryInfo.Exists;
 
+    /// <summary>
+    /// The level which applies, including what the organization demands.
+    /// </summary>
+    private string OutboundDataRestrictionName => this.DataSource.GetEffectiveOutboundDataRestriction(this.SettingsManager.ConfigurationData.DataSourceSecurity.MinimumOutboundDataRestriction).GetName();
+
     /// <summary>
     /// Takes the next file which the directory scan found.
     /// </summary>

app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor

@@ -83,6 +83,8 @@
                 }
             </MudSelect>
 
+            <OutboundDataRestrictionSelection @bind-Value="@this.SelectedOutboundDataRestriction" Label="@T("Where a chat may send data after reading this data source")" Explanation="@T("This setting decides where a chat may still send data once the AI found something in this data source, so that its content does not end up in a web search, for example. Choose no restriction only for content anybody may read anyway.")" OrganizationMinimum="@this.OrganizationMinimumOutboundDataRestriction"/>
+
             @if (this.MediaTranscriptionHint is { } mediaTranscriptionHint)
             {
                 <MudAlert Severity="Severity.Info" Variant="Variant.Outlined" Dense="@true" Class="mb-3">

app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs

@@ -59,6 +59,7 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
     private bool showExpertSettings;
     private bool revalidateAfterRender;
     private ConfidenceLevel dataConfidenceLevel = ConfidenceLevel.UNKNOWN;
+    private OutboundDataRestriction dataOutboundDataRestriction = DataSourceOutboundDataExtensions.DEFAULT_OUTBOUND_DATA_RESTRICTION;
     
     // We get the form reference from Blazor code to validate it manually:
     private MudForm form = null!;
@@ -108,6 +109,7 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
             this.dataMaxChunkTokenLength = this.DataSource.MaxChunkTokenLength;
             this.dataChunkOverlapTokenLength = this.DataSource.ChunkOverlapTokenLength;
             this.dataConfidenceLevel = this.DataSource.ConfidenceLevel;
+            this.dataOutboundDataRestriction = this.DataSource.OutboundDataRestriction;
             this.dataMaxMatches = this.DataSource.MaxMatches;
         }
         
@@ -198,6 +200,22 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
 
     private IEnumerable<ConfigurationSelectData<ConfidenceLevel>> ConfidenceLevels => ConfigurationSelectDataFactory.GetDataSourceConfidenceLevelsData();
 
+    private OutboundDataRestriction OrganizationMinimumOutboundDataRestriction => this.SettingsManager.ConfigurationData.DataSourceSecurity.MinimumOutboundDataRestriction;
+
+    /// <summary>
+    /// The level the data source gets: the one chosen, as far as the organization allows it.
+    /// </summary>
+    /// <remarks>
+    /// The organization may rule out a level while the dialog is open, or after the data source was
+    /// stored with it. The choice itself stays, so the dialog shows it again once the organization
+    /// allows it again.
+    /// </remarks>
+    private OutboundDataRestriction SelectedOutboundDataRestriction
+    {
+        get => this.dataOutboundDataRestriction.StricterOf(this.OrganizationMinimumOutboundDataRestriction);
+        set => this.dataOutboundDataRestriction = value;
+    }
+
     private string SelectedEmbeddingTokenizerText => this.SelectedEmbedding is null
         ? T("No embedding selected")
         : string.IsNullOrWhiteSpace(this.SelectedEmbedding.TokenizerPath)
@@ -227,6 +245,7 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
         MaxChunkTokenLength = this.dataMaxChunkTokenLength,
         ChunkOverlapTokenLength = this.dataChunkOverlapTokenLength,
         ConfidenceLevel = this.dataConfidenceLevel,
+        OutboundDataRestriction = this.SelectedOutboundDataRestriction,
         MaxMatches = this.dataMaxMatches,
     };
     

app/MindWork AI Studio/Dialogs/DataSourceLocalFileInfoDialog.razor

@@ -38,6 +38,7 @@
         }
 
         <TextInfoLine Label="@T("Required provider confidence level")" Value="@this.DataSource.ConfidenceLevel.GetName()" ClipboardTooltipSubject="@T("the required provider confidence level")"/>
+        <TextInfoLine Icon="@Icons.Material.Filled.Shield" Label="@T("Where a chat may send data after reading this data source")" Value="@this.OutboundDataRestrictionName" ClipboardTooltipSubject="@T("where a chat may send data after reading this data source")"/>
         <TextInfoLine Label="@T("Maximum matches per query")" Value="@this.DataSource.MaxMatches.ToString()" ClipboardTooltipSubject="@T("the maximum number of matches per query")"/>
         <TextInfoLine Icon="@Icons.Material.Filled.SquareFoot" Label="@T("File size")" Value="@this.FileSize" ClipboardTooltipSubject="@T("the file size")"/>
     </DialogContent>

app/MindWork AI Studio/Dialogs/DataSourceLocalFileInfoDialog.razor.cs

@@ -33,6 +33,11 @@ public partial class DataSourceLocalFileInfoDialog : MSGComponentBase
     private bool IsFileAvailable => this.fileInfo.Exists;
     
     private string FileSize => this.fileInfo.FileSize();
+
+    /// <summary>
+    /// The level which applies, including what the organization demands.
+    /// </summary>
+    private string OutboundDataRestrictionName => this.DataSource.GetEffectiveOutboundDataRestriction(this.SettingsManager.ConfigurationData.DataSourceSecurity.MinimumOutboundDataRestriction).GetName();
     
     private void Close() => this.MudDialog.Close();
 }

app/MindWork AI Studio/Dialogs/DataSourceMailboxDialog.razor

@@ -371,26 +371,7 @@
                 @T("Only providers with at least this confidence level see the content of your mails: the chat provider, the embedding provider, and every provider which checks a mail for you.")
             </MudJustifiedText>
 
-            <MudSelect T="OutboundDataRestriction" @bind-Value="@this.dataOutboundDataRestriction" Text="@this.dataOutboundDataRestriction.GetName()" Label="@T("Where a chat may send data after reading mails")" Class="mb-1" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" AdornmentIcon="@Icons.Material.Filled.Shield">
-                @foreach (var restriction in ConfigurationSelectDataFactory.GetOutboundDataRestrictionData(this.MinimumOutboundDataRestriction))
-                {
-                    <MudSelectItem T="OutboundDataRestriction" Value="@restriction.Value">
-                        @restriction.Name
-                    </MudSelectItem>
-                }
-            </MudSelect>
-            <MudJustifiedText Typo="Typo.body2" Class="mb-1">
-                @this.dataOutboundDataRestriction.GetDescription()
-            </MudJustifiedText>
-            @if (this.IsOutboundDataRestrictionLimited)
-            {
-                <MudJustifiedText Typo="Typo.body2" Class="mb-1">
-                    @T("Your organization does not allow less strict settings than the ones offered here.")
-                </MudJustifiedText>
-            }
-            <MudJustifiedText Typo="Typo.body2" Class="mb-6">
-                @T("Mails come from strangers and may contain instructions meant for the AI. This setting decides where a chat may still send data once it has read from this mailbox. Mind that opening a link can be enough to set something off, e.g., to confirm a subscription, an order, or a payment.")
-            </MudJustifiedText>
+            <OutboundDataRestrictionSelection @bind-Value="@this.dataOutboundDataRestriction" Label="@T("Where a chat may send data after reading mails")" Explanation="@T("Mails come from strangers and may contain instructions meant for the AI. This setting decides where a chat may still send data once it has read from this mailbox. Mind that opening a link can be enough to set something off, e.g., to confirm a subscription, an order, or a payment.")" OrganizationMinimum="@this.MinimumOutboundDataRestriction"/>
 
             <MudStack Class="mb-3">
                 <MudButton OnClick="@this.ToggleExpertSettings" Variant="Variant.Text" Color="Color.Primary" StartIcon="@Icons.Material.Filled.Tune">

app/MindWork AI Studio/Dialogs/DataSourceMailboxDialog.razor.cs

@@ -343,8 +343,6 @@ public partial class DataSourceMailboxDialog : MSGComponentBase
 
     private OutboundDataRestriction MinimumOutboundDataRestriction => this.SettingsManager.ConfigurationData.MailboxSettings.MinimumOutboundDataRestriction;
 
-    private bool IsOutboundDataRestrictionLimited => this.MinimumOutboundDataRestriction is not OutboundDataRestriction.UNRESTRICTED;
-
     private string TestResultText
     {
         get