Automatically spread test file runs across parallel CI jobs

Author: jamiebuilds

api.js

@@ -83,7 +83,23 @@ class Api extends Emittery {
 		// Find all test files.
 		return new AvaFiles({cwd: apiOptions.resolveTestsFrom, files, extensions: this._allExtensions}).findTestFiles()
 			.then(files => {
-				runStatus = new RunStatus(files.length);
+				if (this.options.parallelRuns) {
+					// The files must be in the same order across all runs, so sort them.
+					files = files.sort((a, b) => a.localeCompare(b, [], {numeric: true}));
+
+					const {currentIndex, totalRuns} = this.options.parallelRuns;
+					const fileCount = files.length;
+					const each = Math.floor(fileCount / totalRuns);
+					const remainder = fileCount % totalRuns;
+
+					const offset = Math.min(currentIndex, remainder) + (currentIndex * each);
+					const currentFileCount = each + (currentIndex < remainder ? 1 : 0);
+
+					files = files.slice(offset, offset + currentFileCount);
+					runStatus = new RunStatus(fileCount, {currentFileCount, currentIndex, totalRuns});
+				} else {
+					runStatus = new RunStatus(files.length, null);
+				}
 
 				const emittedRun = this.emit('run', {
 					clearLogOnNextRun: runtimeOptions.clearLogOnNextRun === true,

lib/cli.js

@@ -155,6 +155,7 @@ exports.run = () => { // eslint-disable-line complexity
 		exit('The \'source\' option has been renamed. Use \'sources\' instead.');
 	}
 
+	const ciParallelVars = require('ci-parallel-vars');
 	const Api = require('../api');
 	const VerboseReporter = require('./reporters/verbose');
 	const MiniReporter = require('./reporters/mini');
@@ -180,6 +181,12 @@ exports.run = () => { // eslint-disable-line complexity
 	// Copy resultant cli.flags into conf for use with Api and elsewhere
 	Object.assign(conf, cli.flags);
 
+	let parallelRuns = null;
+	if (isCi && ciParallelVars) {
+		const {index: currentIndex, total: totalRuns} = ciParallelVars;
+		parallelRuns = {currentIndex, totalRuns};
+	}
+
 	const match = arrify(conf.match);
 	const api = new Api({
 		failFast: conf.failFast,
@@ -198,7 +205,8 @@ exports.run = () => { // eslint-disable-line complexity
 		updateSnapshots: conf.updateSnapshots,
 		snapshotDir: conf.snapshotDir ? path.resolve(projectDir, conf.snapshotDir) : null,
 		color: conf.color,
-		workerArgv: cli.flags['--']
+		workerArgv: cli.flags['--'],
+		parallelRuns
 	});
 
 	let reporter;

lib/reporters/tap.js

@@ -76,6 +76,11 @@ class TapReporter {
 				skipped: this.stats.skippedTests,
 				todo: this.stats.todoTests
 			}) + os.EOL);
+
+			if (this.stats.parallelRuns) {
+				const {currentFileCount, currentIndex, totalRuns} = this.stats.parallelRuns;
+				this.reportStream.write(`# Ran ${currentFileCount} test ${plur('file', currentFileCount)} out of ${this.stats.files} for job ${currentIndex + 1} of ${totalRuns}` + os.EOL + os.EOL);
+			}
 		} else {
 			this.reportStream.write(supertap.finish({
 				crashed: this.crashCount,

lib/reporters/verbose.js

@@ -296,6 +296,12 @@ class VerboseReporter {
 
 		this.lineWriter.writeLine();
 
+		if (this.stats.parallelRuns) {
+			const {currentFileCount, currentIndex, totalRuns} = this.stats.parallelRuns;
+			this.lineWriter.writeLine(colors.information(`Ran ${currentFileCount} test ${plur('file', currentFileCount)} out of ${this.stats.files} for job ${currentIndex + 1} of ${totalRuns}`));
+			this.lineWriter.writeLine();
+		}
+
 		let firstLinePostfix = this.watching ?
 			' ' + chalk.gray.dim('[' + new Date().toLocaleTimeString('en-US', {hour12: false}) + ']') :
 			'';

lib/run-status.js

@@ -3,7 +3,7 @@ const cloneDeep = require('lodash.clonedeep');
 const Emittery = require('./emittery');
 
 class RunStatus extends Emittery {
-	constructor(files) {
+	constructor(files, parallelRuns) {
 		super();
 
 		this.stats = {
@@ -13,6 +13,7 @@ class RunStatus extends Emittery {
 			failedTests: 0,
 			failedWorkers: 0,
 			files,
+			parallelRuns,
 			finishedWorkers: 0,
 			internalErrors: 0,
 			remainingTests: 0,

package-lock.json

@@ -77,6 +77,7 @@
 		"bluebird": "^3.5.1",
 		"chalk": "^2.4.1",
 		"chokidar": "^2.0.3",
+		"ci-parallel-vars": "^1.0.0",
 		"clean-stack": "^1.1.1",
 		"clean-yaml-object": "^0.1.0",
 		"cli-cursor": "^2.1.0",

package.json

@@ -96,7 +96,7 @@ const reporter = new VerboseReporter({
 	watching: false
 });
 
-const runStatus = new RunStatus([file]);
+const runStatus = new RunStatus(1, null);
 runStatus.observeWorker({
 	file,
 	onStateChange(listener) {

profile.js

@@ -53,6 +53,7 @@ Translations: [Español](https://github.com/avajs/ava-docs/blob/master/es_ES/rea
 - [Async function support](#async-function-support)
 - [Observable support](#observable-support)
 - [Enhanced assertion messages](#enhanced-assertion-messages)
+- [Automatic parallel test runs in CI](#parallel-runs-in-ci)
 - [TAP reporter](#tap-reporter)
 - [Automatic migration from other test runners](https://github.com/avajs/ava-codemods#migrating-to-ava)
 
@@ -819,6 +820,10 @@ $ ava --timeout=2m # 2 minutes
 $ ava --timeout=100 # 100 milliseconds
 ```
 
+### Parallel runs in CI
+
+AVA automatically detects whether your CI environment supports parallel builds. Each build will run a subset of all test files, while still making sure all tests get executed. See the [`ci-parallel-vars`](https://www.npmjs.com/package/ci-parallel-vars) package for a list of supported CI environments.
+
 ## API
 
 ### `test([title], implementation)`

readme.md

@@ -0,0 +1,5 @@
+import test from '../../..';
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '1');
+});

test/fixture/parallel-runs/10.js

@@ -0,0 +1,5 @@
+import test from '../../..';
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '0');
+});

test/fixture/parallel-runs/2.js

@@ -0,0 +1,5 @@
+import test from '../../..';
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '0');
+});

test/fixture/parallel-runs/2a.js

@@ -0,0 +1,5 @@
+import test from '../../..';
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '0');
+});

test/fixture/parallel-runs/9.js

@@ -0,0 +1,6 @@
+/* eslint-disable unicorn/filename-case */
+import test from '../../..';
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '1');
+});

test/fixture/parallel-runs/Ab.js

@@ -0,0 +1,5 @@
+import test from '../../..';
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '1');
+});

test/fixture/parallel-runs/a.js

@@ -0,0 +1,5 @@
+import test from '../../..';
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '2');
+});

test/fixture/parallel-runs/b.js

@@ -0,0 +1,5 @@
+import test from '../../..';
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '2');
+});

test/fixture/parallel-runs/c.js

@@ -0,0 +1,17 @@
+'use strict';
+const test = require('tap').test;
+const {execCli} = require('../helper/cli');
+
+test('correctly distributes the test files', t => {
+	t.plan(3);
+	for (let i = 0; i < 3; i++) {
+		execCli('*.js', {
+			dirname: 'fixture/parallel-runs',
+			env: {
+				CI: '1',
+				CI_NODE_INDEX: String(i),
+				CI_NODE_TOTAL: '3'
+			}
+		}, err => t.ifError(err));
+	}
+});