diff --git a/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingImplementations/WebSearch/WebSearchResultRetrievalService.cs b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingImplementations/WebSearch/WebSearchResultRetrievalService.cs index c3a9f0ed..a8aab398 100644 --- a/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingImplementations/WebSearch/WebSearchResultRetrievalService.cs +++ b/app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingImplementations/WebSearch/WebSearchResultRetrievalService.cs @@ -4,6 +4,8 @@ namespace AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations.WebSearch; internal sealed class WebSearchResultRetrievalService(WebPageRetrievalService webPageRetrievalService) { + private static readonly ILogger LOGGER = Program.LOGGER_FACTORY.CreateLogger(); + private const int MAX_PARALLEL_RETRIEVALS = 4; /// @@ -110,9 +112,16 @@ internal sealed class WebSearchResultRetrievalService(WebPageRetrievalService we Interlocked.Increment(ref counters.PageTimedOut); return new(candidate, null, WebSearchPageRetrievalOutcome.PAGE_TIMED_OUT); } - catch (InvalidOperationException) + catch (InvalidOperationException exception) { + // + // The only outcome here which is not an expected one: a page was blocked on purpose, + // and a timeout is a limit the user set, but this is something going wrong. It is + // logged rather than only counted, because a search which quietly returns one result + // fewer is a search nobody can tell was incomplete. + // Interlocked.Increment(ref counters.Failed); + LOGGER.LogError(exception, "Reading a search result page failed. Url={Url}", candidate.RetrievalUrl); return new(candidate, null, WebSearchPageRetrievalOutcome.FAILED); } finally diff --git a/app/MindWork AI Studio/Tools/Web/WebPageContentExtractor.cs b/app/MindWork AI Studio/Tools/Web/WebPageContentExtractor.cs index 2576f11c..c4ff6342 100644 --- a/app/MindWork AI Studio/Tools/Web/WebPageContentExtractor.cs +++ b/app/MindWork AI Studio/Tools/Web/WebPageContentExtractor.cs @@ -76,7 +76,7 @@ internal static class WebPageContentExtractor .Select(x => LimitLength(x, MAX_OUTLINE_ITEM_CHARACTERS)) .Distinct(StringComparer.Ordinal) .ToList(); - var markdown = HTMLParser.ParseToMarkdown(contentRoot.InnerHtml) + var markdown = ConvertToMarkdown(contentRoot.InnerHtml, finalUrl) .Replace("\r\n", "\n", StringComparison.Ordinal) .Replace('\r', '\n') .Trim(); @@ -141,6 +141,30 @@ internal static class WebPageContentExtractor }; } + /// + /// Converts the readable part of the page to Markdown. + /// + /// + /// Only the call into the Markdown library is wrapped, not the extraction around it: a fault of + /// our own has to keep surfacing as what it is, instead of being filed away as an unreadable + /// page.

+ /// What the library throws depends on the HTML it was handed, and it says nothing beyond "this + /// page could not be converted". Reported as an InvalidOperationException, the retrieval treats + /// it like any other page it could not read, which costs this one page rather than the whole + /// search it belongs to. + ///
+ private static string ConvertToMarkdown(string html, Uri finalUrl) + { + try + { + return HTMLParser.ParseToMarkdown(html); + } + catch (Exception exception) when (exception is not OperationCanceledException) + { + throw new InvalidOperationException($"Converting the HTML of '{finalUrl}' to Markdown failed: {exception.Message}", exception); + } + } + private static JsonLdMetadata ExtractJsonLdMetadata(HtmlDocument document, Uri finalUrl) { JsonLdCandidate? bestCandidate = null;