Web search restrictions after internal content (#1074)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Tools/Web/RetrievedWebPage.cs

@@ -25,4 +25,20 @@ public sealed class RetrievedWebPage
     public required DateTimeOffset RetrievedAtUtc { get; init; }
 
     public ConfidenceLevel RequiredProviderConfidence { get; init; } = ConfidenceLevel.NONE;
+
+    /// <summary>
+    /// Whether the page was read from a non-public address, such as one in the intranet or behind
+    /// a VPN.
+    /// </summary>
+    /// <remarks>
+    /// Only the request which returned the page counts, the last one after every redirect: it is
+    /// where the content came from. A public address which redirects into the intranet therefore
+    /// counts, and an intranet address which redirects to a public page, such as a sign-in page,
+    /// does not.<br/><br/>
+    /// The content of such a page is internal, so a caller which hands it to a model has to keep
+    /// it from leaving the organization. Stated here rather than read from
+    /// RequiredProviderConfidence, which answers another question: it counts every request, so the
+    /// address of an intranet page does not reach a less trusted provider either.
+    /// </remarks>
+    public bool ReachedNonPublicAddress { get; init; }
 }

app/MindWork AI Studio/Tools/Web/WebPageAccessBlockReason.cs

@@ -9,4 +9,5 @@ public enum WebPageAccessBlockReason
     PRIVATE_HOST_NOT_ALLOWED,
     INSUFFICIENT_PROVIDER_CONFIDENCE,
     TARGET_NOT_ALLOWED,
+    PUBLIC_TARGET_NOT_ALLOWED,
 }

app/MindWork AI Studio/Tools/Web/WebPageRetrievalOptions.cs

@@ -22,6 +22,22 @@ public sealed class WebPageRetrievalOptions
 
     public bool PublicTargetsOnly { get; init; }
 
+    /// <summary>
+    /// Whether every request has to reach non-public addresses only, the counterpart of
+    /// PublicTargetsOnly.
+    /// </summary>
+    /// <remarks>
+    /// For an address the model chose on a host which counts as internal, such as one of the
+    /// allowed private hosts, while its chat may not send data to third parties. A host name says
+    /// nothing about where it leads: a pattern such as *.example.org matches public hosts as well,
+    /// and an intranet server may redirect anywhere. Every request is therefore checked, redirects
+    /// included, and one which would reach a public address is refused before anything is sent.
+    /// <br/><br/>
+    /// Holds whoever chose the address, because it guards what the address may carry rather than
+    /// where the user may go.
+    /// </remarks>
+    public bool NonPublicTargetsOnly { get; init; }
+
     public ConfidenceLevel ProviderConfidence { get; init; } = ConfidenceLevel.NONE;
 
     public bool UseOsSso { get; init; }

app/MindWork AI Studio/Tools/Web/WebPageRetrievalService.cs

@@ -16,6 +16,7 @@ public sealed class WebPageRetrievalService(HTMLParser htmlParser)
     {
         var triedOsSso = false;
         var requiredProviderConfidence = ConfidenceLevel.NONE;
+        var lastAddressIsNonPublic = false;
 
         // Always overwritten: the media type is validated before the body is read, so no page
         // arrives without the check having decided what it is.
@@ -29,7 +30,10 @@ public sealed class WebPageRetrievalService(HTMLParser htmlParser)
                 async (candidateUrl, validationToken) =>
                 {
                     var addresses = await ResolveValidatedUrlAddressesAsync(candidateUrl, options, validationToken);
-                    if (addresses.Any(IsNonPublicAddress))
+                    // Overwritten with every request, so it holds what the last one reached, the
+                    // one which returned the page:
+                    lastAddressIsNonPublic = addresses.Any(IsNonPublicAddress);
+                    if (lastAddressIsNonPublic)
                         requiredProviderConfidence = ConfidenceLevel.HIGH;
 
                     return addresses;
@@ -76,6 +80,7 @@ public sealed class WebPageRetrievalService(HTMLParser htmlParser)
             },
             RetrievedAtUtc = DateTimeOffset.UtcNow,
             RequiredProviderConfidence = requiredProviderConfidence,
+            ReachedNonPublicAddress = lastAddressIsNonPublic,
         };
     }
 
@@ -118,6 +123,12 @@ public sealed class WebPageRetrievalService(HTMLParser htmlParser)
         if (addresses.Count == 0)
             throw new InvalidOperationException($"The host '{url.Host}' did not resolve to an IP address.");
 
+        // Before the exemption for targets the user chose, because it guards what the address may
+        // carry. Every address has to be non-public, since the connection may go to any of them.
+        // The text names no address, so a caller may hand it on without repeating one:
+        if (options.NonPublicTargetsOnly && !addresses.All(IsNonPublicAddress))
+            throw new WebPageAccessBlockedException("The web page or one of its redirects leads to a public address, which this request may not reach.", WebPageAccessBlockReason.PUBLIC_TARGET_NOT_ALLOWED);
+
         //
         // Where the target came from decides which targets are acceptable. A URL a model produced
         // may not reach into the local network, because the model was talked into it by whatever