fix(pdf): carry cookies across manual redirect hops

crawl4ai/processors/pdf/__init__.py

@@ -171,11 +171,17 @@ def _get_pdf_path(self, url: str) -> str:
                 # Redirects are followed manually so url_validator (when set) can vet every hop BEFORE it is fetched
                 from urllib.parse import urljoin
                 current_url = url
+                # One Session for the whole chain: a bare requests.get() per hop
+                # starts with an empty cookie jar, so a host that sets a cookie
+                # and then redirects (common for gated/CDN-signed PDFs) would
+                # get its own cookie back. allow_redirects=True used to carry
+                # them for us; following hops by hand means we carry them here.
+                session = requests.Session()
                 for _ in range(MAX_PDF_DOWNLOAD_REDIRECTS):
                     if self.url_validator:
                         self.url_validator(current_url)
-                    response = requests.get(current_url, stream=True, timeout=(20, 60 * 10),
-                                            allow_redirects=False)
+                    response = session.get(current_url, stream=True, timeout=(20, 60 * 10),
+                                           allow_redirects=False)
                     if response.is_redirect:
                         location = response.headers.get("location")
                         response.close()  # hop response holds its connection open (stream=True)

tests/test_docker_pdf_crawler_pairing.py

@@ -154,6 +154,21 @@ def do_GET(self):
                 self.send_response(302)
                 self.send_header("Location", "/loop")
                 self.end_headers()
+            elif self.path == "/gated":
+                # Sets a cookie, then redirects to a target that demands it.
+                self.send_response(302)
+                self.send_header("Set-Cookie", "sess=abc123; Path=/")
+                self.send_header("Location", "/gated-doc.pdf")
+                self.end_headers()
+            elif self.path == "/gated-doc.pdf":
+                if "sess=abc123" not in self.headers.get("Cookie", ""):
+                    self.send_response(403)
+                    self.end_headers()
+                    return
+                self.send_response(200)
+                self.send_header("Content-Type", "application/pdf")
+                self.end_headers()
+                self.wfile.write(pdf_bytes)
             else:
                 self.send_response(200)
                 self.send_header("Content-Type", "application/pdf")
@@ -199,6 +214,23 @@ def test_download_redirects_still_followed_without_validator(redirect_server):
         Path(path).unlink(missing_ok=True)
 
 
+def test_download_carries_cookies_across_redirect_hops(redirect_server):
+    """Cookies set by a redirecting host must reach the next hop.
+
+    Following redirects by hand (needed so url_validator can vet each hop)
+    means a bare requests.get() per hop starts with an empty cookie jar, so
+    a host that sets a cookie and then redirects never gets it back. That is
+    the normal shape for gated or CDN-signed PDFs, and allow_redirects=True
+    used to handle it implicitly.
+    """
+    strategy = PDFContentScrapingStrategy()
+    path = strategy._get_pdf_path(f"{redirect_server}/gated")
+    try:
+        assert Path(path).read_bytes().startswith(b"%PDF")
+    finally:
+        Path(path).unlink(missing_ok=True)
+
+
 def test_download_redirect_loop_aborts(redirect_server):
     """An endless redirect chain must abort after the cap, not hang."""
     strategy = PDFContentScrapingStrategy()