Keep one unreadable page from failing the whole web search

app/MindWork AI Studio/Tools/ToolCallingSystem/ToolCallingImplementations/WebSearch/WebSearchResultRetrievalService.cs

@@ -4,6 +4,8 @@ namespace AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations.WebSearch;
 
 internal sealed class WebSearchResultRetrievalService(WebPageRetrievalService webPageRetrievalService)
 {
+    private static readonly ILogger<WebSearchResultRetrievalService> LOGGER = Program.LOGGER_FACTORY.CreateLogger<WebSearchResultRetrievalService>();
+
     private const int MAX_PARALLEL_RETRIEVALS = 4;
 
     /// <summary>
@@ -110,9 +112,16 @@ internal sealed class WebSearchResultRetrievalService(WebPageRetrievalService we
             Interlocked.Increment(ref counters.PageTimedOut);
             return new(candidate, null, WebSearchPageRetrievalOutcome.PAGE_TIMED_OUT);
         }
-        catch (InvalidOperationException)
+        catch (InvalidOperationException exception)
         {
+            //
+            // The only outcome here which is not an expected one: a page was blocked on purpose,
+            // and a timeout is a limit the user set, but this is something going wrong. It is
+            // logged rather than only counted, because a search which quietly returns one result
+            // fewer is a search nobody can tell was incomplete.
+            //
             Interlocked.Increment(ref counters.Failed);
+            LOGGER.LogError(exception, "Reading a search result page failed. Url={Url}", candidate.RetrievalUrl);
             return new(candidate, null, WebSearchPageRetrievalOutcome.FAILED);
         }
         finally

app/MindWork AI Studio/Tools/Web/WebPageContentExtractor.cs

@@ -76,7 +76,7 @@ internal static class WebPageContentExtractor
             .Select(x => LimitLength(x, MAX_OUTLINE_ITEM_CHARACTERS))
             .Distinct(StringComparer.Ordinal)
             .ToList();
-        var markdown = HTMLParser.ParseToMarkdown(contentRoot.InnerHtml)
+        var markdown = ConvertToMarkdown(contentRoot.InnerHtml, finalUrl)
             .Replace("\r\n", "\n", StringComparison.Ordinal)
             .Replace('\r', '\n')
             .Trim();
@@ -141,6 +141,30 @@ internal static class WebPageContentExtractor
         };
     }
 
+    /// <summary>
+    /// Converts the readable part of the page to Markdown.
+    /// </summary>
+    /// <remarks>
+    /// Only the call into the Markdown library is wrapped, not the extraction around it: a fault of
+    /// our own has to keep surfacing as what it is, instead of being filed away as an unreadable
+    /// page.<br/><br/>
+    /// What the library throws depends on the HTML it was handed, and it says nothing beyond "this
+    /// page could not be converted". Reported as an InvalidOperationException, the retrieval treats
+    /// it like any other page it could not read, which costs this one page rather than the whole
+    /// search it belongs to.
+    /// </remarks>
+    private static string ConvertToMarkdown(string html, Uri finalUrl)
+    {
+        try
+        {
+            return HTMLParser.ParseToMarkdown(html);
+        }
+        catch (Exception exception) when (exception is not OperationCanceledException)
+        {
+            throw new InvalidOperationException($"Converting the HTML of '{finalUrl}' to Markdown failed: {exception.Message}", exception);
+        }
+    }
+
     private static JsonLdMetadata ExtractJsonLdMetadata(HtmlDocument document, Uri finalUrl)
     {
         JsonLdCandidate? bestCandidate = null;