fix(html): recover from deeply nested Wikipedia and Bing pages

packages/markitdown/src/markitdown/converters/_bing_serp_converter.py

@@ -1,6 +1,7 @@
 import re
 import base64
 import binascii
+import warnings
 from urllib.parse import parse_qs, urlparse
 from typing import Any, BinaryIO
 from bs4 import BeautifulSoup
@@ -62,6 +63,10 @@ def convert(
     ) -> DocumentConverterResult:
         assert stream_info.url is not None
 
+        # Pop our own keyword before forwarding the rest to markdownify.
+        # strict=True raises RecursionError instead of falling back to plain text.
+        strict: bool = kwargs.pop("strict", False)
+
         # Parse the query parameters
         parsed_params = parse_qs(urlparse(stream_info.url).query)
         query = parsed_params.get("q", [""])[0]
@@ -105,7 +110,7 @@ def convert(
                         pass
 
             # Convert to markdown
-            md_result = _markdownify.convert_soup(result).strip()
+            md_result = self._convert_soup(result, _markdownify, strict=strict).strip()
             lines = [line.strip() for line in re.split(r"\n+", md_result)]
             results.append("\n".join([line for line in lines if len(line) > 0]))
 
@@ -118,3 +123,23 @@ def convert(
             markdown=webpage_text,
             title=None if soup.title is None else soup.title.string,
         )
+
+    def _convert_soup(
+        self, target: Any, markdownify: _CustomMarkdownify, *, strict: bool
+    ) -> str:
+        """Convert one result, tolerating markup too deep for markdownify."""
+        try:
+            return markdownify.convert_soup(target)
+        except RecursionError:
+            if strict:
+                raise
+            # Large or deeply-nested HTML can exceed Python's recursion limit
+            # during markdownify's recursive DOM traversal.  Fall back to
+            # BeautifulSoup's iterative get_text() so the caller still gets the
+            # result's text rather than losing the search-results extraction.
+            warnings.warn(
+                "HTML document is too deeply nested for markdown conversion "
+                "(RecursionError). Falling back to plain-text extraction.",
+                stacklevel=2,
+            )
+            return target.get_text("\n", strip=True)

packages/markitdown/src/markitdown/converters/_wikipedia_converter.py

@@ -1,4 +1,5 @@
 import re
+import warnings
 import bs4
 from typing import Any, BinaryIO
 
@@ -54,6 +55,10 @@ def convert(
         stream_info: StreamInfo,
         **kwargs: Any,  # Options to pass to the converter
     ) -> DocumentConverterResult:
+        # Pop our own keyword before forwarding the rest to markdownify.
+        # strict=True raises RecursionError instead of falling back to plain text.
+        strict: bool = kwargs.pop("strict", False)
+
         # Parse the stream
         encoding = "utf-8" if stream_info.charset is None else stream_info.charset
         soup = bs4.BeautifulSoup(file_stream, "html.parser", from_encoding=encoding)
@@ -81,11 +86,29 @@ def convert(
             # Convert the page
             webpage_text = (
                 f"# {main_title}\n\n" if main_title else ""
-            ) + _CustomMarkdownify(**kwargs).convert_soup(body_elm)
+            ) + self._convert_soup(body_elm, strict=strict, **kwargs)
         else:
-            webpage_text = _CustomMarkdownify(**kwargs).convert_soup(soup)
+            webpage_text = self._convert_soup(soup, strict=strict, **kwargs)
 
         return DocumentConverterResult(
             markdown=webpage_text,
             title=main_title,
         )
+
+    def _convert_soup(self, target: Any, *, strict: bool, **kwargs: Any) -> str:
+        """Convert a subtree, tolerating markup too deep for markdownify."""
+        try:
+            return _CustomMarkdownify(**kwargs).convert_soup(target)
+        except RecursionError:
+            if strict:
+                raise
+            # Large or deeply-nested HTML can exceed Python's recursion limit
+            # during markdownify's recursive DOM traversal.  Fall back to
+            # BeautifulSoup's iterative get_text() so the caller still gets the
+            # article's text rather than losing the Wikipedia extraction.
+            warnings.warn(
+                "HTML document is too deeply nested for markdown conversion "
+                "(RecursionError). Falling back to plain-text extraction.",
+                stacklevel=2,
+            )
+            return target.get_text("\n", strip=True)

packages/markitdown/tests/test_deeply_nested_site_html.py

@@ -0,0 +1,137 @@
+#!/usr/bin/env python3 -m pytest
+"""The site-specific HTML converters need the fallback the generic one has.
+
+``HtmlConverter`` (#1644) and ``RssConverter`` (#2333) recover from the
+``RecursionError`` that markdownify's recursive DOM traversal raises on deeply
+nested markup. ``WikipediaConverter`` and ``BingSerpConverter`` read the same
+kind of page and did not, so a deeply nested page silently lost the
+site-specific extraction and came back as the whole document instead.
+
+A lowered recursion limit keeps the test independent of the host's default.
+"""
+
+import io
+import sys
+import warnings
+from contextlib import contextmanager
+
+import pytest
+
+from markitdown import MarkItDown, StreamInfo
+from markitdown.converters import BingSerpConverter, WikipediaConverter
+
+WIKIPEDIA_URL = "https://en.wikipedia.org/wiki/Nested"
+BING_URL = "https://www.bing.com/search?q=nested"
+
+ARTICLE_TEXT = "Deep article content"
+SITE_NOTICE = "Site notice chrome"
+FOOTER = "Footer chrome"
+RESULT_TEXT = "Deep result content"
+
+DEPTH = 500
+LOW_LIMIT = 200  # well below markdownify's traversal depth for DEPTH nesting
+
+
+def _nest(text: str) -> str:
+    return "<div>" * DEPTH + f"<p>{text}</p>" + "</div>" * DEPTH
+
+
+WIKIPEDIA_HTML = (
+    "<html><head><title>Nested - Wikipedia</title></head><body>"
+    f'<div id="siteNotice">{SITE_NOTICE}</div>'
+    '<span class="mw-page-title-main">Nested</span>'
+    f'<div id="mw-content-text">{_nest(ARTICLE_TEXT)}</div>'
+    f'<div id="footer">{FOOTER}</div>'
+    "</body></html>"
+).encode("utf-8")
+
+BING_HTML = (
+    "<html><head><title>nested - Bing</title></head><body>"
+    f'<li class="b_algo">{_nest(RESULT_TEXT)}</li>'
+    "</body></html>"
+).encode("utf-8")
+
+
+@contextmanager
+def _low_recursion_limit():
+    original = sys.getrecursionlimit()
+    try:
+        sys.setrecursionlimit(LOW_LIMIT)
+        yield
+    finally:
+        sys.setrecursionlimit(original)
+
+
+def _convert(html: bytes, url: str) -> tuple[str, list[warnings.WarningMessage]]:
+    with _low_recursion_limit():
+        with warnings.catch_warnings(record=True) as caught:
+            warnings.simplefilter("always")
+            result = MarkItDown().convert_stream(
+                io.BytesIO(html),
+                stream_info=StreamInfo(
+                    extension=".html",
+                    mimetype="text/html",
+                    charset="utf-8",
+                    url=url,
+                ),
+            )
+    return result.markdown, [w for w in caught if "deeply nested" in str(w.message)]
+
+
+def test_deeply_nested_wikipedia_page_keeps_the_article_extraction() -> None:
+    markdown, recursion_warnings = _convert(WIKIPEDIA_HTML, WIKIPEDIA_URL)
+
+    assert len(recursion_warnings) > 0
+    assert ARTICLE_TEXT in markdown
+    # The page is still read as a Wikipedia page: its heading is emitted, and
+    # the chrome outside #mw-content-text stays out of the output.
+    assert markdown.startswith("# Nested")
+    assert SITE_NOTICE not in markdown
+    assert FOOTER not in markdown
+    assert "<div" not in markdown
+
+
+def test_deeply_nested_bing_serp_keeps_the_results_extraction() -> None:
+    markdown, recursion_warnings = _convert(BING_HTML, BING_URL)
+
+    assert len(recursion_warnings) > 0
+    assert RESULT_TEXT in markdown
+    # The page is still read as a Bing results page.
+    assert markdown.startswith("## A Bing search for 'nested' found")
+    assert "<div" not in markdown
+
+
+def test_strict_still_surfaces_the_recursion_error() -> None:
+    """strict=True is the documented escape hatch on the other converters."""
+    with _low_recursion_limit():
+        with pytest.raises(RecursionError):
+            WikipediaConverter().convert(
+                io.BytesIO(WIKIPEDIA_HTML),
+                StreamInfo(extension=".html", charset="utf-8", url=WIKIPEDIA_URL),
+                strict=True,
+            )
+        with pytest.raises(RecursionError):
+            BingSerpConverter().convert(
+                io.BytesIO(BING_HTML),
+                StreamInfo(extension=".html", charset="utf-8", url=BING_URL),
+                strict=True,
+            )
+
+
+def test_shallow_pages_are_unaffected() -> None:
+    shallow = (
+        "<html><head><title>Shallow - Wikipedia</title></head><body>"
+        '<span class="mw-page-title-main">Shallow</span>'
+        '<div id="mw-content-text"><p>Plain <b>article</b> text.</p></div>'
+        "</body></html>"
+    ).encode("utf-8")
+
+    result = MarkItDown().convert_stream(
+        io.BytesIO(shallow),
+        stream_info=StreamInfo(
+            extension=".html", mimetype="text/html", charset="utf-8", url=WIKIPEDIA_URL
+        ),
+    )
+
+    assert result.markdown.startswith("# Shallow")
+    assert "Plain **article** text." in result.markdown