Log which urls timeouted

index.js

@@ -90,25 +90,25 @@ const defaults = userOptions => {
 
   let exit = false;
   if (!options.include || !options.include.length) {
-    console.log("⚠️  include option should be an non-empty array");
+    console.log("🔥  include option should be an non-empty array");
     exit = true;
   }
   if (options.preloadResources) {
     console.log(
-      "⚠️  preloadResources option deprecated. Use preloadImages or cacheAjaxRequests"
+      "🔥  preloadResources option deprecated. Use preloadImages or cacheAjaxRequests"
     );
     exit = true;
   }
   if (options.minifyOptions) {
-    console.log("⚠️  minifyOptions option renamed to minifyHtml");
+    console.log("🔥  minifyOptions option renamed to minifyHtml");
     options.minifyHtml = options.minifyOptions;
   }
   if (options.asyncJs) {
-    console.log("⚠️  asyncJs option renamed to asyncScriptTags");
+    console.log("🔥  asyncJs option renamed to asyncScriptTags");
     options.asyncScriptTags = options.asyncJs;
   }
   if (options.saveAs !== "html" && options.saveAs !== "png") {
-    console.log("⚠️  saveAs supported values are html and png");
+    console.log("🔥  saveAs supported values are html and png");
     exit = true;
   }
   if (exit) throw new Error();
@@ -293,7 +293,7 @@ const inlineCss = async opt => {
 
   if (cssSize > twentyKb)
     console.log(
-      `warning: inlining CSS more than 20kb (${cssSize / 1024}kb, ${cssStrategy})`
+      `⚠️  warning: inlining CSS more than 20kb (${cssSize / 1024}kb, ${cssStrategy})`
     );
 
   if (cssStrategy === "critical") {
@@ -443,11 +443,11 @@ const saveAsHtml = async ({ page, filePath, options, route, fs }) => {
   filePath = filePath.replace(/\//g, path.sep);
   if (route.endsWith(".html")) {
     if (route.endsWith("/404.html") && !title.includes("404"))
-      console.log('warning: 404 page title does not contain "404" string');
+      console.log('⚠️  warning: 404 page title does not contain "404" string');
     mkdirp.sync(path.dirname(filePath));
     fs.writeFileSync(filePath, minifiedContent);
   } else {
-    if (title.includes("404")) console.log(`warning: page not found ${route}`);
+    if (title.includes("404")) console.log(`⚠️  warning: page not found ${route}`);
     mkdirp.sync(filePath);
     fs.writeFileSync(path.join(filePath, "index.html"), minifiedContent);
   }
@@ -493,7 +493,7 @@ const run = async (userOptions, { fs } = { fs: nativeFs }) => {
     fs.existsSync(path.join(sourceDir, "200.html"))
   ) {
     console.log(
-      `200.html is present in the sourceDir (${sourceDir}). You can not run react-snap twice - this will break the build`
+      `🔥  200.html is present in the sourceDir (${sourceDir}). You can not run react-snap twice - this will break the build`
     );
     return Promise.reject("");
   }

src/puppeteer_utils.js

@@ -4,6 +4,7 @@ const url = require("url");
 const mapStackTrace = require("sourcemapped-stacktrace-node").default;
 const path = require("path");
 const fs = require("fs");
+const {createTracker, augmentTimeoutError} = require("./tracker");
 
 /**
  * @param {{page: Page, options: {skipThirdPartyRequests: true}, basePath: string }} opt
@@ -31,15 +32,15 @@ const enableLogging = opt => {
   page.on("console", msg => {
     const text = msg.text();
     if (text !== 'JSHandle@object') {
-      console.log(`console.log at ${route}:`, text)
+      console.log(`️️️💬  console.log at ${route}:`, text)
     } else {
       Promise.all(msg.args().map(x => x.jsonValue())).then(args =>
-        console.log(`console.log at ${route}:`, ...args)
+        console.log(`💬  console.log at ${route}:`, ...args)
       )
     }
   });
   page.on("error", msg => {
-    console.log(`error at ${route}:`, msg);
+    console.log(`🔥  error at ${route}:`, msg);
     onError && onError();
   });
   page.on("pageerror", e => {
@@ -56,16 +57,16 @@ const enableLogging = opt => {
             stackRows.length - 1;
 
           console.log(
-            `pageerror at ${route}: ${(e.stack || e.message).split("\n")[0] +
+            `🔥  pageerror at ${route}: ${(e.stack || e.message).split("\n")[0] +
               "\n"}${stackRows.slice(0, puppeteerLine).join("\n")}`
           );
         })
         .catch(e2 => {
-          console.log(`pageerror at ${route}:`, e);
-          console.log(`️️️warning at ${route} (error in source maps):`, e2.message);
+          console.log(`🔥  pageerror at ${route}:`, e);
+          console.log(`️️️⚠️  warning at ${route} (error in source maps):`, e2.message);
         });
     } else {
-      console.log(`pageerror at ${route}:`, e);
+      console.log(`🔥  pageerror at ${route}:`, e);
     }
     onError && onError();
   });
@@ -75,11 +76,11 @@ const enableLogging = opt => {
       try {
         route = response._request.headers().referer.replace(`http://localhost:${options.port}`, "");
       } catch (e) {}
-      console.log(`warning at ${route}: got ${response.status()} HTTP code for ${response.url()}`);
+      console.log(`️️️⚠️  warning at ${route}: got ${response.status()} HTTP code for ${response.url()}`);
     }
   });
   // page.on("requestfailed", msg =>
-  //   console.log(`${route} requestfailed:`, msg)
+  //   console.log(`️️️⚠️  ${route} requestfailed:`, msg)
   // );
 };
 
@@ -131,7 +132,7 @@ const crawl = async opt => {
   process.on("SIGINT", onSigint);
 
   const onUnhandledRejection = error => {
-    console.log("UnhandledPromiseRejectionWarning", error);
+    console.log("🔥  UnhandledPromiseRejectionWarning", error);
     shuttingDown = true;
   };
   process.on("unhandledRejection", onUnhandledRejection);
@@ -201,18 +202,26 @@ const crawl = async opt => {
         });
         beforeFetch && beforeFetch({ page, route });
         await page.setUserAgent(options.userAgent);
-        await page.goto(pageUrl, { waitUntil: "networkidle0" });
+        const tracker = createTracker(page)
+        try {
+          await page.goto(pageUrl, { waitUntil: "networkidle0" });
+        } catch (e) {
+          e.message = augmentTimeoutError(e.message, tracker);
+          throw e;
+        } finally {
+          tracker.dispose();
+        }
         if (options.waitFor) await page.waitFor(options.waitFor);
         if (options.crawl) {
           const links = await getLinks({ page });
           links.forEach(addToQueue);
         }
         afterFetch && (await afterFetch({ page, route, browser }));
         await page.close();
-        console.log(`crawled ${processed + 1} out of ${enqued} (${route})`);
+        console.log(`✅  crawled ${processed + 1} out of ${enqued} (${route})`);
       } catch (e) {
         if (!shuttingDown) {
-          console.log(`error at ${route}`, e);
+          console.log(`🔥  error at ${route}`, e);
         }
         shuttingDown = true;
       }

src/tracker.js

@@ -0,0 +1,48 @@
+/**
+ * Sets up event listeners on the Browser.Page instance to maintain a set
+ * of URLs that have started but never finished or failed.
+ *
+ * @param {Object} page
+ * @return Object
+ */
+const createTracker = page => {
+  const requests = new Set();
+  const onStarted = request => requests.add(request);
+  const onFinished = request => requests.delete(request);
+  page.on('request', onStarted);
+  page.on('requestfinished', onFinished);
+  page.on('requestfailed', onFinished);
+  return {
+    urls: () => Array.from(requests).map(r => r.url()),
+    dispose: () => {
+      page.removeListener('request', onStarted);
+      page.removeListener('requestfinished', onFinished);
+      page.removeListener('requestfailed', onFinished);
+    }
+  };
+};
+
+/**
+ * Adds information about timed out URLs if given message is about Timeout.
+ *
+ * @param {string} message error message
+ * @param {Object} tracker ConnectionTracker
+ * @returns {string}
+ */
+const augmentTimeoutError = (message, tracker) => {
+  if (message.startsWith('Navigation Timeout Exceeded')) {
+    const urls = tracker.urls();
+    if (urls.length > 1) {
+      message += `\nTracked URLs that have not finished: ${urls.join(
+        ', '
+      )}`;
+    } else if (urls.length > 0) {
+      message += `\nFor ${urls[0]}`;
+    } else {
+      message += `\nBut there are no pending connections`;
+    }
+  }
+  return message
+}
+
+module.exports = { createTracker, augmentTimeoutError };