[testing] add an --artifacts-dir to store unittest coverage info

Drive-by: Add help text for command line arguments.

Bug: b:333423685
Change-Id: I7b91e4fac4f345ea66a60b93f83f34105a15343a
Reviewed-on: https://chromium-review.googlesource.com/c/devtools/devtools-frontend/+/5461358
Commit-Queue: Simon Zünd <[email protected]>
Auto-Submit: Philip Pfaffe <[email protected]>
Reviewed-by: Simon Zünd <[email protected]>

Author: pfaffe

test/conductor/commandline.ts

@@ -30,7 +30,8 @@ function validateDiffBehaviors(args: undefined|string|string[]) {
     }
   }
   if (failed.length > 0) {
-    throw new Error(`Invalid options for --on-diff: ${failed}`);
+    throw new Error(
+        `Invalid options for --on-diff: ${failed}. Valid options are: ${Object.values(DiffBehaviors).join(', ')}`);
   }
   return asArray(args);
 }
@@ -39,13 +40,21 @@ function validateDiffBehaviors(args: undefined|string|string[]) {
 export function commandLineArgs(yargs: any) {
   return yargs.parserConfiguration({'camel-case-expansion': false})
       .command('$0 [tests..]')
-      .option('debug', {type: 'boolean'})
-      .option('coverage', {type: 'boolean'})
-      .option('chrome-binary', {type: 'string'})
-      .option('repeat', {type: 'number', default: 1})
-      .option('on-diff', {type: 'string', coerce: validateDiffBehaviors})
-      .option('shuffle', {type: 'boolean'})
-      .option('grep', {type: 'string', conflicts: 'fgrep'})
-      .option('fgrep', {type: 'string', conflicts: 'grep'})
-      .option('invert-grep', {type: 'boolean'});
+      .option(
+          'artifacts-dir',
+          {type: 'string', desc: 'Path to a directory to store test artifacts in (e.g., coverage reports)'})
+      .option('debug', {type: 'boolean', desc: 'Execute tests in debug mode'})
+      .option('coverage', {type: 'boolean', desc: 'Enable coverage reporting'})
+      .option('chrome-binary', {type: 'string', desc: 'Run tests with a custom chrome binary'})
+      .option('repeat', {type: 'number', default: 1, desc: 'Repeat tests'})
+      .option('on-diff', {
+        type: 'string',
+        coerce: validateDiffBehaviors,
+        desc: `Define how to deal with diffs in snapshots/screenshots. Options are: ${
+            Object.values(DiffBehaviors).join(', ')}`,
+      })
+      .option('shuffle', {type: 'boolean', desc: 'Execute tests in random order'})
+      .option('grep', {type: 'string', conflicts: 'fgrep', desc: 'Filter tests by name using grep'})
+      .option('fgrep', {type: 'string', conflicts: 'grep', desc: 'Filter tests by name using fgrep'})
+      .option('invert-grep', {type: 'boolean', desc: 'Invert the grep/fgrep result'});
 }

test/conductor/test_config.ts

@@ -29,6 +29,7 @@ export const enum ServerType {
 
 interface Config {
   tests: string[];
+  artifactsDir: string;
   chromeBinary: string;
   serverType: ServerType;
   debug: boolean;
@@ -74,6 +75,7 @@ function mochaGrep(): Config['mochaGrep'] {
 
 export const TestConfig: Config = {
   tests: options['tests'],
+  artifactsDir: options['artifacts-dir'] || SOURCE_ROOT,
   chromeBinary: options['chrome-binary'] ?? chromePath(),
   serverType: ServerType.HostedMode,
   debug: options['debug'],

test/run.ts

@@ -14,7 +14,7 @@ const yargs = require('yargs');
 const unparse = require('yargs-unparser');
 const options = commandLineArgs(yargs(process.argv.slice(2)))
                     .options('skip-ninja', {type: 'boolean', desc: 'Skip rebuilding'})
-                    .options('debug-driver', {type: 'boolean', hidden: true})
+                    .options('debug-driver', {type: 'boolean', hidden: true, desc: 'Debug the driver part of tests'})
                     .positional('tests', {
                       type: 'string',
                       desc: 'Path to the test suite, starting from out/Target/gen directory.',

test/unit/karma.conf.ts

@@ -108,7 +108,7 @@ module.exports = function(config: any) {
     },
 
     coverageReporter: {
-      dir: path.join(GEN_DIR, COVERAGE_OUTPUT_DIRECTORY),
+      dir: path.join(TestConfig.artifactsDir, COVERAGE_OUTPUT_DIRECTORY),
       subdir: '.',
       reporters: [
         {type: 'json-summary'},