Merge pull request #83 from bocoup/json-reporter

Add CLI option: `--reporter-keys`

Author: rwaldron

README.md

@@ -30,10 +30,10 @@ Run `test262-harness --help` for details on the various configuration options.
 | `--hostPath` | Path to the host executable.
 | `--hostArgs` | Any additional arguments to pass to the host when invoking it (eg. `--harmony`, `--es6all`, etc). 
 | `-t`, `--threads` | Run this many tests in parallel. Note that the browser runners don't work great with t > 1.
-| `-r`, `--reporter` | Selects test case result format. Currently either `json` or `simple`. Default `simple`.
+| `-r`, `--reporter` | Format of data written to standard output. Currently either `json` or `simple`. Default `simple`.
 |`--test262Dir` | Optional. Root test262 directory and is used to locate the includes directory.
 |`--includesDir` | Includes directory. By default inferred from test262Dir or else detected by walking upward from the first test found.
 |`--prelude` | Path to a file to include before every test (useful for testing polyfills for example)
-
+|`--reporter-keys` | Comma-separated list of keys to include in output of `json` reporter.
 
 

bin/run.js

@@ -26,17 +26,29 @@ let includesDir = argv.includesDir;
 // print version of test262-harness
 if (argv.version) {
   printVersion();
-  process.exit(0);
+  return;
 }
 
 // initialize reporter by attempting to load lib/reporters/${reporter}
 // defaults to 'simple'
 let reporter;
+let reporterOpts = {};
 if (fs.existsSync(Path.join(__dirname, '../lib/reporters', `${argv.reporter}.js`))) {
   reporter = require(`../lib/reporters/${argv.reporter}.js`);
 } else {
   console.error(`Reporter ${argv.reporter} not found.`);
-  process.exit(1);
+  process.exitCode = 1;
+  return;
+}
+
+if (argv.reporterKeys) {
+  if (argv.reporter !== 'json') {
+    console.error('`--reporter-keys` option applies only to the `json` reporter.');
+    process.exitCode = 1;
+    return;
+  }
+
+  reporterOpts.reporterKeys = argv.reporterKeys.split(',');
 }
 
 // load preload contents
@@ -57,7 +69,8 @@ if (argv.hostType) {
 
   if (!argv.hostPath) {
     console.error('Missing host path. Pass --hostPath with a path to the host executable you want to test.');
-    process.exit(1);
+    process.exitCode = 1;
+    return;
   }
 
   hostPath = argv.hostPath;
@@ -76,7 +89,8 @@ argv.timeout = argv.timeout || DEFAULT_TEST_TIMEOUT;
 // Show help if no arguments provided
 if (!argv._.length) {
   cli.showHelp();
-  process.exit(1);
+  process.exitCode = 1;
+  return;
 }
 
 // Test Pipeline
@@ -95,7 +109,7 @@ const results = rawResults.map(test => {
   return test;
 });
 const resultEmitter = resultsEmitter(results);
-reporter(resultEmitter);
+reporter(resultEmitter, reporterOpts);
 
 function printVersion() {
   const p = require(Path.resolve(__dirname, "..", "package.json"));

lib/cli.js

@@ -15,10 +15,12 @@ const yargv = yargs
   .nargs('threads', 1)
   .default('threads', 1)
   .alias('threads', 't')
-  .describe('reporter', 'select a reporter to use (simple or json)')
+  .describe('reporter', 'format of data written to standard output')
+  .choices('reporter', ['simple', 'json'])
   .nargs('reporter', 1)
   .alias('reporter', 'r')
   .default('reporter', 'simple')
+  .describe('reporter-keys', 'comma-separated list of keys to include in JSON ouput')
   .help('help')
   .alias('help', 'h')
   .describe('timeout', 'test timeout (in ms, default 10000)')

lib/pick.js

@@ -0,0 +1,11 @@
+'use strict';
+
+module.exports = function(obj, keys) {
+  let picked = {};
+
+  keys.forEach(function(key) {
+      picked[key] = obj[key];
+    });
+
+  return picked;
+};

lib/reporters/json.js

@@ -1,6 +1,7 @@
 'use strict';
+const pick = require('../pick');
 
-function jsonReporter(results) {
+function jsonReporter(results, opts) {
   let started = false;
 
   results.on('start', function () {
@@ -12,13 +13,21 @@ function jsonReporter(results) {
   });
 
   results.on('test end', function (test) {
+    let toEmit;
+
     if (started) {
       process.stdout.write(',');
     } else {
       started = true;
     }
 
-    console.log(JSON.stringify(test));
+    if (opts.reporterKeys) {
+      toEmit = pick(test, opts.reporterKeys);
+    } else {
+      toEmit = test;
+    }
+
+    console.log(JSON.stringify(toEmit));
   });
 }
 

test/test.js

@@ -4,8 +4,12 @@ const path = require('path');
 const cp = require('child_process');
 
 Promise.all([
-  run(),
-  run({ prelude: true })
+  run(['test/collateral/**/*.js']),
+  run(['--prelude', './test/test-prelude.js', 'test/collateral/bothStrict.js']),
+  run(['--reporter-keys', 'attrs,result', 'test/collateral/bothStrict.js']),
+  run(['--reporter-keys', 'rawResult,attrs,result', 'test/collateral/bothStrict.js']),
+  run(['--reporter-keys', 'attrs,rawResult,result', 'test/collateral/bothStrict.js']),
+  run(['--reporter-keys', 'attrs,result,rawResult', 'test/collateral/bothStrict.js'])
 ])
 .then(validate)
 .catch(reportRunError);
@@ -16,26 +20,16 @@ function reportRunError(error) {
   process.exit(1);
 }
 
-function run(options = { prelude: false }) {
+function run(extraArgs) {
   return new Promise((resolve, reject) => {
     let stdout = '';
     let stderr = '';
     let args = [
-      '--hostType', 'node',
-      '--hostPath', process.execPath,
-      '-r', 'json',
-      '--includesDir', './test/test-includes',
-    ];
-
-    if (options.prelude) {
-      args.push(
-        '--prelude',
-        './test/test-prelude.js',
-        'test/collateral/bothStrict.js'
-      );
-    } else {
-      args.push('test/collateral/**/*.js');
-    }
+        '--hostType', 'node',
+        '--hostPath', process.execPath,
+        '-r', 'json',
+        '--includesDir', './test/test-includes',
+      ].concat(extraArgs);
 
     const child = cp.fork('bin/run.js', args, { silent: true });
 
@@ -56,9 +50,16 @@ function run(options = { prelude: false }) {
 }
 
 function validate(records) {
-  const [normal, prelude] = records;
+  const [
+    normal, prelude, withoutRawResult, withRawResult1, withRawResult2,
+    withRawResult3
+  ] = records;
   validateResultRecords(normal);
   validateResultRecords(prelude, { prelude: true });
+  validateResultRecords(withoutRawResult, { noRawResult: true });
+  validateResultRecords(withRawResult1);
+  validateResultRecords(withRawResult2);
+  validateResultRecords(withRawResult3);
 }
 
 function validateResultRecords(records, options = { prelude: false }) {
@@ -86,6 +87,12 @@ function validateResultRecords(records, options = { prelude: false }) {
         test.assert(record.rawResult.stdout.indexOf("prelude!") > -1, 'Has prelude content');
       }
 
+      if (options.noRawResult) {
+        test.equal(record.rawResult, undefined);
+      } else {
+        test.equal(typeof record.rawResult, 'object');
+      }
+
       test.end();
     });
   });