mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-06-27 14:36:27 +00:00
Added the option to authenticate with the OS settings for private or VPN web pages.
This commit is contained in:
parent
8901779342
commit
d0e1966e9e
@ -8005,8 +8005,8 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::ITOOLIMPLEMENTATION::T35170
|
|||||||
-- Tool description
|
-- Tool description
|
||||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::ITOOLIMPLEMENTATION::T4056470505"] = "Tool description"
|
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::ITOOLIMPLEMENTATION::T4056470505"] = "Tool description"
|
||||||
|
|
||||||
-- Load a single web page, extract its main HTML content, and return structured working material for the model. Use the result to synthesize a natural-language answer instead of exposing the raw payload to the user.
|
-- Load a single web page and extract its main HTML content.
|
||||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS::READWEBPAGETOOL::T1823236891"] = "Load a single web page, extract its main HTML content, and return structured working material for the model. Use the result to synthesize a natural-language answer instead of exposing the raw payload to the user."
|
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS::READWEBPAGETOOL::T204256540"] = "Load a single web page and extract its main HTML content."
|
||||||
|
|
||||||
-- Optional global truncation limit for extracted Markdown returned to the model.
|
-- Optional global truncation limit for extracted Markdown returned to the model.
|
||||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS::READWEBPAGETOOL::T2066580916"] = "Optional global truncation limit for extracted Markdown returned to the model."
|
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS::READWEBPAGETOOL::T2066580916"] = "Optional global truncation limit for extracted Markdown returned to the model."
|
||||||
@ -8014,12 +8014,12 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS:
|
|||||||
-- Allowed private hosts must be host names only, without scheme or path.
|
-- Allowed private hosts must be host names only, without scheme or path.
|
||||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS::READWEBPAGETOOL::T2196457612"] = "Allowed private hosts must be host names only, without scheme or path."
|
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS::READWEBPAGETOOL::T2196457612"] = "Allowed private hosts must be host names only, without scheme or path."
|
||||||
|
|
||||||
-- Optional host allowlist for private or VPN web pages. Separate host patterns with commas, such as example.de, *.example.de. Allowed private hosts require a High-confidence provider.
|
|
||||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS::READWEBPAGETOOL::T237631450"] = "Optional host allowlist for private or VPN web pages. Separate host patterns with commas, such as example.de, *.example.de. Allowed private hosts require a High-confidence provider."
|
|
||||||
|
|
||||||
-- Maximum Content Characters
|
-- Maximum Content Characters
|
||||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS::READWEBPAGETOOL::T2801581200"] = "Maximum Content Characters"
|
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS::READWEBPAGETOOL::T2801581200"] = "Maximum Content Characters"
|
||||||
|
|
||||||
|
-- Optional host allowlist for private or VPN web pages. For security reasons, private or VPN web pages aren't allowed to be read by default. Separate host patterns with commas, such as example.de, example.com. Allowed private hosts require a high-confidence provider. For allowed internal hosts, AI Studio also tries the operating system's default sign-in automatically when the server responds with integrated authentication.
|
||||||
|
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS::READWEBPAGETOOL::T2866833707"] = "Optional host allowlist for private or VPN web pages. For security reasons, private or VPN web pages aren't allowed to be read by default. Separate host patterns with commas, such as example.de, example.com. Allowed private hosts require a high-confidence provider. For allowed internal hosts, AI Studio also tries the operating system's default sign-in automatically when the server responds with integrated authentication."
|
||||||
|
|
||||||
-- Optional HTTP timeout for loading a web page in seconds.
|
-- Optional HTTP timeout for loading a web page in seconds.
|
||||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS::READWEBPAGETOOL::T2941521561"] = "Optional HTTP timeout for loading a web page in seconds."
|
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS::READWEBPAGETOOL::T2941521561"] = "Optional HTTP timeout for loading a web page in seconds."
|
||||||
|
|
||||||
|
|||||||
@ -272,6 +272,9 @@ CONFIG["SETTINGS"] = {}
|
|||||||
-- Configure private or VPN hosts that the Read Web Page tool may access.
|
-- Configure private or VPN hosts that the Read Web Page tool may access.
|
||||||
-- Public web pages do not need to be listed here.
|
-- Public web pages do not need to be listed here.
|
||||||
-- Private hosts listed here still require a provider with HIGH confidence before any page content is sent to the model.
|
-- Private hosts listed here still require a provider with HIGH confidence before any page content is sent to the model.
|
||||||
|
-- For hosts on this allowlist, AI Studio also tries the current user's operating-system sign-in
|
||||||
|
-- automatically when the server requests integrated authentication (for example Kerberos or NTLM).
|
||||||
|
-- This does not reuse Firefox cookies or an existing browser session.
|
||||||
-- Separate host patterns with commas. Wildcards only match subdomains, so add the root domain separately if needed.
|
-- Separate host patterns with commas. Wildcards only match subdomains, so add the root domain separately if needed.
|
||||||
-- Examples:
|
-- Examples:
|
||||||
-- CONFIG["SETTINGS"]["DataTools.ReadWebPageAllowedPrivateHosts"] = "dlr.de, *.dlr.de"
|
-- CONFIG["SETTINGS"]["DataTools.ReadWebPageAllowedPrivateHosts"] = "dlr.de, *.dlr.de"
|
||||||
|
|||||||
@ -44,13 +44,22 @@ public sealed class HTMLParser
|
|||||||
return innerHtml;
|
return innerHtml;
|
||||||
}
|
}
|
||||||
|
|
||||||
public async Task<HTMLParserWebPage> LoadWebPageAsync(Uri url, CancellationToken token = default, int timeoutSeconds = 30, Func<Uri, CancellationToken, Task<IReadOnlyList<IPAddress>>>? resolveUrlAddressesAsync = null, int maxResponseBytes = DEFAULT_MAX_RESPONSE_BYTES)
|
public async Task<HTMLParserWebPage> LoadWebPageAsync(
|
||||||
|
Uri url,
|
||||||
|
CancellationToken token = default,
|
||||||
|
int timeoutSeconds = 30,
|
||||||
|
Func<Uri, CancellationToken, Task<IReadOnlyList<IPAddress>>>? resolveUrlAddressesAsync = null,
|
||||||
|
int maxResponseBytes = DEFAULT_MAX_RESPONSE_BYTES,
|
||||||
|
ExternalWebAuthenticationMode authenticationMode = ExternalWebAuthenticationMode.NONE)
|
||||||
{
|
{
|
||||||
using var handler = new SocketsHttpHandler
|
using var handler = new SocketsHttpHandler
|
||||||
{
|
{
|
||||||
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate | DecompressionMethods.Brotli,
|
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate | DecompressionMethods.Brotli,
|
||||||
AllowAutoRedirect = false,
|
AllowAutoRedirect = false,
|
||||||
};
|
};
|
||||||
|
if (authenticationMode is ExternalWebAuthenticationMode.OS_DEFAULT_CREDENTIALS)
|
||||||
|
handler.Credentials = CreateDefaultCredentialCache(url);
|
||||||
|
|
||||||
if (resolveUrlAddressesAsync is not null)
|
if (resolveUrlAddressesAsync is not null)
|
||||||
{
|
{
|
||||||
// The callback binds the request to a vetted target IP; a proxy would change the endpoint being connected to.
|
// The callback binds the request to a vetted target IP; a proxy would change the endpoint being connected to.
|
||||||
@ -107,6 +116,16 @@ public sealed class HTMLParser
|
|||||||
throw new HttpRequestException($"The server returned more than {MAX_REDIRECTS} redirects for '{url}'.");
|
throw new HttpRequestException($"The server returned more than {MAX_REDIRECTS} redirects for '{url}'.");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static CredentialCache CreateDefaultCredentialCache(Uri url)
|
||||||
|
{
|
||||||
|
var credentialCache = new CredentialCache();
|
||||||
|
var uriPrefix = new UriBuilder(url.Scheme, url.Host, url.Port).Uri;
|
||||||
|
credentialCache.Add(uriPrefix, "Negotiate", CredentialCache.DefaultNetworkCredentials);
|
||||||
|
credentialCache.Add(uriPrefix, "NTLM", CredentialCache.DefaultNetworkCredentials);
|
||||||
|
credentialCache.Add(uriPrefix, "Kerberos", CredentialCache.DefaultNetworkCredentials);
|
||||||
|
return credentialCache;
|
||||||
|
}
|
||||||
|
|
||||||
private static void ValidateHttpOrHttpsUrl(Uri url)
|
private static void ValidateHttpOrHttpsUrl(Uri url)
|
||||||
{
|
{
|
||||||
if (url.Scheme.Equals(Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase) ||
|
if (url.Scheme.Equals(Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase) ||
|
||||||
@ -248,3 +267,9 @@ public sealed class HTMLParserWebPage
|
|||||||
|
|
||||||
public required HtmlDocument Document { get; init; }
|
public required HtmlDocument Document { get; init; }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public enum ExternalWebAuthenticationMode
|
||||||
|
{
|
||||||
|
NONE,
|
||||||
|
OS_DEFAULT_CREDENTIALS
|
||||||
|
}
|
||||||
|
|||||||
@ -57,7 +57,7 @@ public sealed class ReadWebPageTool(HTMLParser htmlParser, ILogger<ReadWebPageTo
|
|||||||
{
|
{
|
||||||
"timeoutSeconds" => TB("Optional HTTP timeout for loading a web page in seconds."),
|
"timeoutSeconds" => TB("Optional HTTP timeout for loading a web page in seconds."),
|
||||||
"maxContentCharacters" => TB("Optional global truncation limit for extracted Markdown returned to the model."),
|
"maxContentCharacters" => TB("Optional global truncation limit for extracted Markdown returned to the model."),
|
||||||
ALLOWED_PRIVATE_HOSTS_SETTING => TB("Optional host allowlist for private or VPN web pages. For security reasons, private or VPN web pages aren't allowed to be read by default. Separate host patterns with commas, such as example.de, example.com. Allowed private hosts require a high-confidence provider."),
|
ALLOWED_PRIVATE_HOSTS_SETTING => TB("Optional host allowlist for private or VPN web pages. For security reasons, private or VPN web pages aren't allowed to be read by default. Separate host patterns with commas, such as example.de, *.example.de. Allowed private hosts require a high-confidence provider. For allowed internal hosts, AI Studio also tries the operating system's default sign-in automatically when the server responds with integrated authentication."),
|
||||||
_ => TB(fieldDefinition.Description),
|
_ => TB(fieldDefinition.Description),
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -113,6 +113,7 @@ public sealed class ReadWebPageTool(HTMLParser htmlParser, ILogger<ReadWebPageTo
|
|||||||
var maxContentCharacters = Math.Min(ReadOptionalPositiveIntSetting(context.SettingsValues, "maxContentCharacters") ?? DEFAULT_MAX_CONTENT_CHARACTERS, MAX_CONTENT_CHARACTERS);
|
var maxContentCharacters = Math.Min(ReadOptionalPositiveIntSetting(context.SettingsValues, "maxContentCharacters") ?? DEFAULT_MAX_CONTENT_CHARACTERS, MAX_CONTENT_CHARACTERS);
|
||||||
if (!TryReadAllowedPrivateHostPatterns(context.SettingsValues.GetValueOrDefault(ALLOWED_PRIVATE_HOSTS_SETTING), out var allowedPrivateHosts, out var allowlistError))
|
if (!TryReadAllowedPrivateHostPatterns(context.SettingsValues.GetValueOrDefault(ALLOWED_PRIVATE_HOSTS_SETTING), out var allowedPrivateHosts, out var allowlistError))
|
||||||
throw new InvalidOperationException(allowlistError);
|
throw new InvalidOperationException(allowlistError);
|
||||||
|
var shouldTryOsSso = ShouldTryOsSso(url, allowedPrivateHosts, context.ProviderConfidence);
|
||||||
|
|
||||||
HTMLParserWebPage page;
|
HTMLParserWebPage page;
|
||||||
try
|
try
|
||||||
@ -122,7 +123,8 @@ public sealed class ReadWebPageTool(HTMLParser htmlParser, ILogger<ReadWebPageTo
|
|||||||
token,
|
token,
|
||||||
timeoutSeconds,
|
timeoutSeconds,
|
||||||
async (candidateUrl, validationToken) => await this.ResolveValidatedUrlAddressesAsync(candidateUrl, allowedPrivateHosts, context.ProviderConfidence, validationToken),
|
async (candidateUrl, validationToken) => await this.ResolveValidatedUrlAddressesAsync(candidateUrl, allowedPrivateHosts, context.ProviderConfidence, validationToken),
|
||||||
MAX_RESPONSE_BYTES);
|
MAX_RESPONSE_BYTES,
|
||||||
|
shouldTryOsSso ? ExternalWebAuthenticationMode.OS_DEFAULT_CREDENTIALS : ExternalWebAuthenticationMode.NONE);
|
||||||
}
|
}
|
||||||
catch (OperationCanceledException) when (!token.IsCancellationRequested)
|
catch (OperationCanceledException) when (!token.IsCancellationRequested)
|
||||||
{
|
{
|
||||||
@ -133,6 +135,13 @@ public sealed class ReadWebPageTool(HTMLParser htmlParser, ILogger<ReadWebPageTo
|
|||||||
if (FindBlockedException(exception) is { } blockedException)
|
if (FindBlockedException(exception) is { } blockedException)
|
||||||
throw blockedException;
|
throw blockedException;
|
||||||
|
|
||||||
|
if (shouldTryOsSso && exception.StatusCode is HttpStatusCode.Unauthorized)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException(
|
||||||
|
$"Loading the web page failed: The server returned HTTP 401 (Unauthorized) for '{url}'. The host is reachable and AI Studio already tried your operating system's default sign-in, but the server did not accept it or requires an additional browser session/cookies.",
|
||||||
|
exception);
|
||||||
|
}
|
||||||
|
|
||||||
throw new InvalidOperationException($"Loading the web page failed: {exception.Message}", exception);
|
throw new InvalidOperationException($"Loading the web page failed: {exception.Message}", exception);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -290,6 +299,14 @@ public sealed class ReadWebPageTool(HTMLParser htmlParser, ILogger<ReadWebPageTo
|
|||||||
return allowedPrivateHosts.Any(pattern => pattern.IsMatch(normalizedHost));
|
return allowedPrivateHosts.Any(pattern => pattern.IsMatch(normalizedHost));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static bool ShouldTryOsSso(
|
||||||
|
Uri url,
|
||||||
|
IReadOnlyList<AllowedPrivateHostPattern> allowedPrivateHosts,
|
||||||
|
ConfidenceLevel providerConfidence) =>
|
||||||
|
providerConfidence >= ConfidenceLevel.HIGH &&
|
||||||
|
!IsBlockedHostName(url.Host) &&
|
||||||
|
IsAllowedPrivateHost(url.Host, allowedPrivateHosts);
|
||||||
|
|
||||||
private static string NormalizeHost(string host) => host.Trim().TrimEnd('.').ToLowerInvariant();
|
private static string NormalizeHost(string host) => host.Trim().TrimEnd('.').ToLowerInvariant();
|
||||||
|
|
||||||
private static bool IsNeverAllowedAddress(IPAddress address)
|
private static bool IsNeverAllowedAddress(IPAddress address)
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user