ci: fix chrome integration tests and split browser tests (dequelabs#3735)

Author: straker

Diff for: .circleci/config.yml

@@ -51,7 +51,7 @@ jobs:
       - run: npm run eslint
 
   # Run the test suite.
-  test_unix:
+  test_chrome:
     <<: *defaults
     <<: *unix_box
     steps:
@@ -60,8 +60,18 @@ jobs:
       - <<: *restore_dependency_cache_unix
       - run: npx browser-driver-manager install chromedriver --verbose
       - run: npm run build
-      - run: npm run test -- --browsers Chrome,Firefox
+      - run: npm run test -- --browsers Chrome
       - run: npm run test:integration:chrome
+
+  test_firefox:
+    <<: *defaults
+    <<: *unix_box
+    steps:
+      - browser-tools/install-browser-tools
+      - checkout
+      - <<: *restore_dependency_cache_unix
+      - run: npm run build
+      - run: npm run test -- --browsers Firefox
       - run: npm run test:integration:firefox
 
   # Run examples under `doc/examples`
@@ -276,38 +286,41 @@ workflows:
           requires:
             - dependencies_unix
       # Run tests on all commits, but after installing dependencies
-      - test_unix:
+      - test_chrome:
           requires:
             - lint
+      - test_firefox:
+          requires:
+            - test_chrome
       - test_examples:
           requires:
-            - test_unix
+            - test_chrome
       - test_act:
           requires:
-            - test_unix
+            - test_chrome
       - test_aria_practices:
           requires:
-            - test_unix
+            - test_chrome
       - test_locales:
           requires:
-            - test_unix
+            - test_chrome
       - test_virtual_rules:
           requires:
-            - test_unix
+            - test_chrome
       - build_api_docs:
           requires:
-            - test_unix
+            - test_chrome
       - test_rule_help_version:
           requires:
-            - test_unix
+            - test_chrome
       - test_node:
           requires:
-            - test_unix
+            - test_chrome
       # Hold for approval
       - hold:
           type: approval
           requires:
-            - test_unix
+            - test_chrome
             - test_examples
             - test_locales
             - test_virtual_rules
@@ -322,7 +335,8 @@ workflows:
       - next_release:
           requires:
             - dependencies_unix
-            - test_unix
+            - test_chrome
+            - test_firefox
             - test_examples
             - test_locales
             - test_virtual_rules
@@ -334,7 +348,8 @@ workflows:
       - release:
           requires:
             - dependencies_unix
-            - test_unix
+            - test_chrome
+            - test_firefox
             - test_examples
             - test_locales
             - test_virtual_rules

Diff for: test/integration/full/test-webdriver.js

@@ -1,18 +1,18 @@
 /*global window, Promise */
 
-var globby = require('globby');
-var WebDriver = require('selenium-webdriver');
-var chrome = require('selenium-webdriver/chrome');
-var chromedriver = require('chromedriver');
+const globby = require('globby');
+const chrome = require('selenium-webdriver/chrome');
+const firefox = require('selenium-webdriver/firefox');
+const chromedriver = require('chromedriver');
 
-var args = process.argv.slice(2);
+const args = process.argv.slice(2);
 
 // allow running certain browsers through command line args
 // (only one browser supported, run multiple times for more browsers)
-var browser = 'chrome';
+let browser = 'chrome';
 args.forEach(function (arg) {
   // pattern: browsers=Chrome
-  var parts = arg.split('=');
+  const parts = arg.split('=');
   if (parts[0] === 'browser') {
     browser = parts[1].toLowerCase();
   }
@@ -25,7 +25,7 @@ function collectTestResults(driver) {
   // inject a script that waits half a second
   return driver
     .executeAsyncScript(function () {
-      var callback = arguments[arguments.length - 1];
+      const callback = arguments[arguments.length - 1];
       setTimeout(function () {
         if (!window.mocha) {
           callback('mocha-missing;' + window.location.href);
@@ -57,7 +57,7 @@ function collectTestResults(driver) {
  * Test each URL
  */
 function runTestUrls(driver, isMobile, urls, errors) {
-  var url = urls.shift();
+  const url = urls.shift();
   errors = errors || [];
 
   return (
@@ -72,9 +72,9 @@ function runTestUrls(driver, isMobile, urls, errors) {
       })
       // And process them
       .then(function (promiseResults) {
-        var capabilities = promiseResults[0];
-        var result = promiseResults[1];
-        var browserName =
+        const capabilities = promiseResults[0];
+        const result = promiseResults[1];
+        const browserName =
           capabilities.get('browserName') +
           (capabilities.get('mobileEmulationEnabled') ? '-mobile' : '');
         console.log(url + ' [' + browserName + ']');
@@ -117,80 +117,45 @@ function runTestUrls(driver, isMobile, urls, errors) {
  * Build web driver depends whether REMOTE_SELENIUM_URL is set
  */
 function buildWebDriver(browser) {
-  // Pinned to [email protected]
-  // https://github.com/SeleniumHQ/selenium/pull/10796/files#diff-6c87d95a2288e92e15a6bb17710c763c01c2290e679beb26220858f3218b6a62L260
-
-  var capabilities;
-  var mobileBrowser = browser.split('-mobile');
-
-  if (mobileBrowser.length > 1) {
-    browser = mobileBrowser[0];
-    capabilities = {
-      browserName: mobileBrowser[0],
-      chromeOptions: {
-        mobileEmulation: {
-          deviceMetrics: {
-            width: 320,
-            height: 568,
-            pixelRatio: 2
-          }
-        }
-      }
-    };
-  }
+  let webdriver;
+  const mobileBrowser = browser.split('-mobile');
 
   // fix chrome DevToolsActivePort file doesn't exist in CricleCI (as well as a
   // host of other problems with starting Chrome). the only thing that seems to
   // allow Chrome to start without problems consistently is using ChromeHeadless
   // @see https://stackoverflow.com/questions/50642308/webdriverexception-unknown-error-devtoolsactiveport-file-doesnt-exist-while-t
   if (browser === 'chrome') {
-    var service = new chrome.ServiceBuilder(chromedriver.path).build();
-    chrome.setDefaultService(service);
+    const service = new chrome.ServiceBuilder(chromedriver.path).build();
 
-    capabilities = WebDriver.Capabilities.chrome();
-    capabilities.set('chromeOptions', {
-      args: [
+    const options = new chrome.Options()
+      .headless()
+      .addArguments([
         '--no-sandbox',
-        '--headless',
         '--disable-extensions',
         '--disable-dev-shm-usage'
-      ]
-    });
+      ]);
+    webdriver = chrome.Driver.createSession(options, service);
+  } else if (browser === 'firefox') {
+    const options = new firefox.Options().headless();
+    webdriver = firefox.Driver.createSession(options);
   }
 
-  var webdriver = new WebDriver.Builder()
-    .withCapabilities(capabilities)
-    .forBrowser(browser);
-
   if (process.env.REMOTE_SELENIUM_URL) {
     webdriver.usingServer(process.env.REMOTE_SELENIUM_URL);
   }
 
-  // @see https://github.com/SeleniumHQ/selenium/issues/6026
-  if (browser === 'safari') {
-    var safari = require('selenium-webdriver/safari');
-    var server = new safari.ServiceBuilder()
-      .addArguments('--legacy')
-      .build()
-      .start();
-
-    webdriver.usingServer(server);
-  }
-
   return {
-    driver: webdriver.build(),
+    driver: webdriver,
     isMobile: mobileBrowser.length > 1
   };
 }
 
 function start(options) {
-  var driver;
-  var isMobile = false;
   // yes, really, and this isn't documented anywhere either.
   options.browser =
     options.browser === 'edge' ? 'MicrosoftEdge' : options.browser;
 
-  var testUrls = globby
+  const testUrls = globby
     .sync([
       'test/integration/full/**/*.{html,xhtml}',
       '!**/frames/**/*.{html,xhtml}'
@@ -214,17 +179,9 @@ function start(options) {
   }
 
   // try to load the browser
-  try {
-    var webDriver = buildWebDriver(options.browser);
-    driver = webDriver.driver;
-    isMobile = webDriver.isMobile;
-    // If load fails, warn user and move to the next task
-  } catch (err) {
-    console.log();
-    console.log(err.message);
-    console.log('Aborted testing using ' + options.browser);
-    return process.exit();
-  }
+  const webDriver = buildWebDriver(options.browser);
+  const driver = webDriver.driver;
+  const isMobile = webDriver.isMobile;
 
   driver.manage().setTimeouts({
     pageLoad: 50000,