Retries API and fix for Puppeteer (#936)

* retries API and fix for Puppeteer

* fixed linting

* added documentation on retries

* fixed retries issues, added tests, fixed within async/await

* fixed linting

* fixed puppeteer test

Author: DavertMik

CHANGELOG.md

@@ -1,3 +1,24 @@
+## 1.1.5
+
+* [Puppeteer] Rerun steps failed due to "Cannot find context with specified id" Error.
+* Added syntax to retry a single step:
+
+```js
+// retry action once on failure
+I.retry().see('Hello');
+// retry action 3 times on failure
+I.retry(3).see('Hello');
+// retry action 3 times waiting for 0.1 second before next try
+I.retry({ retries: 3, minTimeout: 100 }).see('Hello');
+// retry action 3 times waiting no more than 3 seconds for last retry
+I.retry({ retries: 3, maxTimeout: 3000 }).see('Hello');
+//
+I.retry({
+  retries: 2,
+  when: err => err.message === 'Node not visible'
+}).seeElement('#user');
+```
+
 ## 1.1.4
 
 * Removed `yarn` call in package.json

docs/basics.md

@@ -204,6 +204,8 @@ Scenario("Don't stop me", {timeout: 0}, (I) => {});
 
 ### Retries
 
+#### Retry Feature
+
 Browser tests can be very fragile and some time you need to re run the few times just to make them pass.
 This can be done with `retries` option added to `Feature` declaration.
 
@@ -212,25 +214,60 @@ You can set number of a retries for a feature:
 
 ```js
 Feature('Complex JS Stuff', {retries: 3})
+
+
+Scenario('Really complex', (I) => {
+  // test goes here
+});
 ```
 
 Every Scenario inside this feature will be rerun 3 times.
-You can make an exception for a specific scenario by passing `retries` option to it:
+You can make an exception for a specific scenario by passing `retries` option to a Scenario.
 
-```js
-Feature('Complex JS Stuff', {retries: 3})
+#### Retry Scenario
 
-Scenario('Not that complex', {retries: 1}, (I) => {
+```js
+Scenario('Really complex', {retries: 2}, (I) => {
   // test goes here
 });
 
-Scenario('Really complex', (I) => {
-  // test goes here
-});
 ```
 
-"Really complex" test will be restarted 3 times,
-while "Not that complex" test will be rerun only once.
+This scenario will be restarted two times on a failure
+
+#### Retry Step
+
+If you have a step which often fails you can retry execution for this single step.
+Use `retry()` function before an action to ask CodeceptJS to retry this step on failure:
+
+```js
+I.retry().see('Welcome');
+```
+
+If you'd like to retry step more than once pass the amount as parameter:
+
+```js
+I.retry(3).see('Welcome');
+```
+
+Additional options can be provided to retry so you can set the additional options (defined in [promise-retry](https://www.npmjs.com/package/promise-retry) library).
+
+
+```js
+// retry action 3 times waiting for 0.1 second before next try
+I.retry({ retries: 3, minTimeout: 100 }).see('Hello');
+
+// retry action 3 times waiting no more than 3 seconds for last retry
+I.retry({ retries: 3, maxTimeout: 3000 }).see('Hello');
+
+// retry 2 times if error with message 'Node not visible' happens
+I.retry({
+  retries: 2,
+  when: err => err.message === 'Node not visible'
+}).seeElement('#user');
+```
+
+Pass a function to `when` option to  retry only when error matches the expected one.
 
 ---
 

docs/helpers.md

@@ -228,4 +228,27 @@ class JSWait extends codecept_helper {
 module.exports = JSWait;
 ```
 
+## Conditional Retries
+
+It is possible to execute global conditional retries to handle unforseen errors.
+Lost connections and network issues are good candidates to be retried whenever they appear.
+
+This can be done inside a helper using the global [promise recorder](https://codecept.io/hooks/#api):
+
+Example: Retrying rendering errors in Puppeteer.
+
+```js
+_before() {
+  const recorder = require('codeceptjs').recorder;
+  recorder.retry({
+    retries: 2,
+    when: err => err.message.indexOf('Cannot find context with specified id') > -1,
+  });
+}
+```
+
+`recorder.retry` acts similarly to `I.retry()` and accepts the same parameters. It expects the `when` parameter to be set so it would handle only specific errors and not to retry for every failed step.
+
+Retry rules are available in array `recorder.retries`. The last retry rule can be disabled by running `recorder.retries.pop()`;
+
 ### done()

docs/hooks.md

@@ -251,6 +251,38 @@ module.exports = function() {
 
 Whenever you execute tests with `--verbose` option you will see registered events and promises executed by a recorder.
 
+
+### Conditional Retries with Recorder
+
+It is possible to execute global conditional retries to handle unforseen errors.
+Lost connections and network issues are good candidates to be retried whenever they appear.
+
+This can be done inside a [helper](https://codecept.io/helpers/) using `recorder`:
+
+Example: Retrying rendering errors in Puppeteer.
+
+```js
+_before() {
+  const recorder = require('codeceptjs').recorder;
+  recorder.retry({
+    retries: 2,
+    when: err => err.message.indexOf('Cannot find context with specified id') > -1,
+  });
+}
+```
+
+`recorder.retry` acts similarly to `I.retry()` and accepts the same parameters. It expects the `when` parameter to be set so it would handle only specific errors and not to retry for every failed step.
+
+Retry rules are available in array `recorder.retries`. The last retry rule can be disabled by running `recorder.retries.pop()`;
+
+```js
+// inside a helper
+disableLastRetryRule() {
+  const recorder = require('codeceptjs').recorder;
+  recorder.retries.pop();
+}
+```
+
 ### Output
 
 Output module provides 4 verbosity levels. Depending on the mode you can have different information printed using corresponding functions.

lib/actor.js

@@ -38,10 +38,19 @@ module.exports = function (obj) {
 
   // add print comment method`
   obj.say = msg => recorder.add(`say ${msg}`, () => output.say(msg));
+  obj.retry = retryStep;
 
   return obj;
 };
 
+function retryStep(opts) {
+  if (opts === undefined) opts = 1;
+  recorder.retry(opts);
+  // remove retry once the step passed
+  recorder.add(_ => event.dispatcher.once(event.step.passed, _ => recorder.retries.pop()));
+  return this;
+}
+
 function recordStep(step, args) {
   step.status = 'queued';
   step.setArguments(args);

lib/helper/Puppeteer.js

@@ -1,6 +1,7 @@
 const requireg = require('requireg');
 const Helper = require('../helper');
 const Locator = require('../locator');
+const recorder = require('../recorder');
 const stringIncludes = require('../assert/include').includes;
 const { urlEquals } = require('../assert/equal');
 const { equals } = require('../assert/equal');
@@ -19,8 +20,8 @@ const ElementNotFound = require('./errors/ElementNotFound');
 const Popup = require('./extras/Popup');
 const Console = require('./extras/Console');
 
-const puppeteer = requireg('puppeteer');
 
+let puppeteer;
 const popupStore = new Popup();
 const consoleLogStore = new Console();
 
@@ -58,6 +59,7 @@ const consoleLogStore = new Console();
 class Puppeteer extends Helper {
   constructor(config) {
     super(config);
+    puppeteer = requireg('puppeteer');
 
     // set defaults
     this.options = {
@@ -92,7 +94,7 @@ class Puppeteer extends Helper {
     try {
       requireg('puppeteer');
     } catch (e) {
-      return ['puppeteer'];
+      return ['puppeteer@^1.0.0'];
     }
   }
 
@@ -108,6 +110,10 @@ class Puppeteer extends Helper {
 
 
   async _before() {
+    recorder.retry({
+      retries: 2,
+      when: err => err.message.indexOf('Cannot find context with specified id') > -1,
+    });
     if (this.options.restart && !this.options.manualStart) return this._startBrowser();
     if (!this.isRunning && !this.options.manualStart) return this._startBrowser();
     return this.browser;
@@ -922,7 +928,7 @@ class Puppeteer extends Helper {
    * Get JS log from browser.
    *
    * ```js
-   * let logs = yield I.grabBrowserLogs();
+   * let logs = await I.grabBrowserLogs();
    * console.log(JSON.stringify(logs))
    * ```
    */

lib/recorder.js

@@ -1,20 +1,30 @@
+const promiseRetry = require('promise-retry');
+const log = require('./output').log;
+
 let promise;
 let running = false;
 let errFn;
 let queueId = 0;
 let sessionId = null;
 let asyncErr = null;
-const log = require('./output').log;
 
 let tasks = [];
-
 let oldPromises = [];
 
+const defaultRetryOptions = {
+  retries: 0,
+  minTimeout: 150,
+  maxTimeout: 10000,
+};
+
+
 /**
  * Singleton object to record all test steps as promises and run them in chain.
  */
 module.exports = {
 
+  retries: [],
+
   /**
    * Start recording promises
    *
@@ -63,6 +73,7 @@ module.exports = {
     oldPromises = [];
     tasks = [];
     this.session.running = false;
+    this.retries = [];
   },
 
   session: {
@@ -109,8 +120,38 @@ module.exports = {
     }
     tasks.push(taskName);
     log(`${currentQueue()}Queued | ${taskName}`);
-    // ensure a valid promise is always in chain
-    return promise = Promise.resolve(promise).then(fn);
+
+    return promise = Promise.resolve(promise).then((res) => {
+      const retryOpts = this.retries.slice(-1).pop();
+      // no retries or unnamed tasks
+      if (!retryOpts || !taskName) {
+        return Promise.resolve(res).then(fn);
+      }
+
+      return promiseRetry(Object.assign(defaultRetryOptions, retryOpts), (retry, number) => {
+        if (number > 1) log(`${currentQueue()}Retrying... Attempt #${number}`);
+        return Promise.resolve(res).then(fn).catch((err) => {
+          // does retry handled or should be thrown
+          for (const retryObj in this.retries.reverse()) {
+            if (!retryObj.when) return retry(err);
+            if (retryObj.when && retryObj.when(err)) return retry(err);
+          }
+          throw err;
+        });
+      });
+    });
+  },
+
+  retry(opts) {
+    if (!promise) return;
+
+    if (opts === null) {
+      opts = {};
+    }
+    if (Number.isInteger(opts)) {
+      opts = { retries: opts };
+    }
+    return promise.then(() => this.retries.push(opts));
   },
 
   catch(customErrFn) {