Fix stas feedback on perf tests (#349)

* Fix stas feedback on perf tests

* Address feedback

Author: zbraniecki

package.json

@@ -27,6 +27,7 @@
     "prettyjson": "^1.2.1",
     "rollup": "^0.59.1",
     "rollup-plugin-babel": "^4.0.0-beta.4",
-    "rollup-plugin-node-resolve": "^3.0.3"
+    "rollup-plugin-node-resolve": "^3.0.3",
+    "intl-pluralrules": "^1.0.0"
   }
 }

tools/perf/benchmark.common.js

@@ -1,19 +1,8 @@
-
-function runTests(env) {
-  const testData = JSON.parse(env.readFile("./fixtures/benchmarks.json"));
-
+function runTest(env) {
   const results = {};
-
-  for (let testName in testData) {
-    let test = testData[testName];
-    runTest(env, testName, test.args, test.functions, results);
-  }
-
-  return results;
-}
-
-function runTest(env, name, args, fncs, results) {
-  const ftlCode = env.readFile(`./fixtures/${name}.ftl`);
+  const testData = JSON.parse(env.readFile("./fixtures/benchmarks.json"));
+  const {args, functions} = testData[env.benchmarkName];
+  const ftlCode = env.readFile(`./fixtures/${env.benchmarkName}.ftl`);
 
   {
     const testName = "parse-syntax";
@@ -25,7 +14,7 @@ function runTest(env, name, args, fncs, results) {
       throw Error("Junk in syntax parser result!");
     }
 
-    results[`${testName}/"${name}"`] = env.ms(end) - env.ms(start);
+    results[`${testName}/${env.benchmarkName}`] = env.ms(end) - env.ms(start);
   }
 
   let resource;
@@ -39,18 +28,18 @@ function runTest(env, name, args, fncs, results) {
     // we'll rely on the syntax parser to verify that
     // the sample test syntax is correct.
 
-    results[`${testName}/"${name}"`] = env.ms(end) - env.ms(start);
+    results[`${testName}/${env.benchmarkName}`] = env.ms(end) - env.ms(start);
   }
 
   {
     const testName = "resolve-runtime";
-    const functions = {};
-    for (let fnName in fncs) {
-      let body = fncs[fnName];
-      functions[fnName] = new Function(body);
+    const fncs = {};
+    for (let fnName in functions) {
+      let body = functions[fnName];
+      fncs[fnName] = new Function(body);
     }
     const bundle = new env.Fluent.FluentBundle('en-US', {
-      functions
+      functions: fncs
     });
     const errors = bundle.addResource(resource);
 
@@ -69,11 +58,12 @@ function runTest(env, name, args, fncs, results) {
       throw new Error(`Errors accumulated while resolving ${name}.`);
     }
 
-    results[`${testName}/"${name}"`] = env.ms(end) - env.ms(start);
+    results[`${testName}/${env.benchmarkName}`] = env.ms(end) - env.ms(start);
   }
+
+  return results;
 }
 
 if (typeof exports !== "undefined") {
   exports.runTest = runTest;
-  exports.runTests = runTests;
 }

tools/perf/benchmark.d8.js

@@ -9,11 +9,12 @@ const env = {
   ms: (milliseconds) => {
     return milliseconds;
   },
+  benchmarkName: arguments[0],
   now: Date.now,
   FluentSyntax,
   Fluent,
 };
 
-const results = runTests(env);
+const results = runTest(env);
 
 print(JSON.stringify(results));

tools/perf/benchmark.jsshell.js

@@ -9,11 +9,12 @@ const env = {
   ms: (milliseconds) => {
     return milliseconds;
   },
+  benchmarkName: scriptArgs[0],
   now: performance.now,
   FluentSyntax,
   Fluent,
 };
 
-const results = runTests(env);
+const results = runTest(env);
 
 print(JSON.stringify(results));

tools/perf/benchmark.node.js

@@ -2,7 +2,8 @@ const fs = require('fs');
 
 const Fluent = require('../../fluent');
 const FluentSyntax = require('../../fluent-syntax');
-const { runTest, runTests } = require('./benchmark.common');
+const { runTest } = require('./benchmark.common');
+require('intl-pluralrules');
 
 const env = {
   readFile: (path) => {
@@ -11,11 +12,12 @@ const env = {
   ms: ([seconds, nanoseconds]) => {
     return seconds * 1e3 + nanoseconds / 1e6;
   },
+  benchmarkName: process.argv[2],
   now: process.hrtime,
   FluentSyntax,
   Fluent,
 };
 
-const results = runTests(env);
+const results = runTest(env);
 
 console.log(JSON.stringify(results));

tools/perf/test.js

@@ -28,7 +28,7 @@ var commands = {
     "node --harmony-async-iteration benchmark.node.js" :
     "node benchmark.node.js",
   jsshell: "js benchmark.jsshell.js",
-  d8: "d8 benchmark.d8.js",
+  d8: "d8 benchmark.d8.js -- ",
 };
 
 var command = program.args.length ?
@@ -47,47 +47,63 @@ function color(str, col) {
   return str;
 }
 
-function runAll(sample, callback) {
-  var results = {};
-  var times = {};
-  // run is recursive and thus sequential so that node doesn't spawn all the 
-  // processes at once
-  run();
-
-  function run() {
-    exec(command, { cwd: __dirname }, function (error, stdout, stderr) {
-      if (!program.raw && program.progress) {
-        process.stdout.write(color('.', INDICATOR));
-      }
-      if (error) {
-        console.log(error.toString());
-      }
-      var data = JSON.parse(stdout);
-      for (var scenario in data) {
-        if (!times[scenario]) {
-          times[scenario] = [];
-        }
-        times[scenario].push(data[scenario]);
-      }
-      if (times[scenario].length !== sample) {
-        run();
-      } else {
-        for (scenario in times) {
-          var mean = util.mean(times[scenario]);
-          results[scenario] = {
-            mean: mean,
-            stdev: util.stdev(times[scenario], mean),
-            sample: sample
-          };
-        }
-        callback(results);
-      }
-    });
+runAll(parseInt(program.sample)).then(printResults);
+
+async function runAll(sample) {
+  const results = {};
+
+  const testData = JSON.parse(fs.readFileSync(`${__dirname}/fixtures/benchmarks.json`).toString());
+  for (let benchmarkName in testData) {
+    if (!program.raw && program.progress) {
+      process.stdout.write(color(`\n${benchmarkName}: `, INDICATOR));
+    }
+    await runBenchmark(benchmarkName, sample, results);
   }
+  return results;
 }
 
-runAll(parseInt(program.sample), function(res) {
+function runBenchmark(benchmarkName, sample, results) {
+  return new Promise((resolve, reject) => {
+    const times = {};
+    const execCommand = `${command} ${benchmarkName}`;
+    // run is recursive and thus sequential so that node doesn't spawn all the
+    // processes at once
+    run();
 
+    function run() {
+      exec(execCommand, { cwd: __dirname }, function (error, stdout, stderr) {
+        if (!program.raw && program.progress) {
+          process.stdout.write(color('.', INDICATOR));
+        }
+        if (error) {
+          console.log(error.toString());
+        }
+        var data = JSON.parse(stdout);
+        for (var scenario in data) {
+          if (!times[scenario]) {
+            times[scenario] = [];
+          }
+          times[scenario].push(data[scenario]);
+        }
+        if (times[scenario].length !== sample) {
+          run();
+        } else {
+          for (scenario in times) {
+            var mean = util.mean(times[scenario]);
+            results[scenario] = {
+              mean: mean,
+              stdev: util.stdev(times[scenario], mean),
+              sample: sample
+            };
+          }
+          resolve(results);
+        }
+      });
+    }
+  });
+}
+
+function printResults(res) {
   for (var scenario in res) {
     if (program.compare) {
       var ref = JSON.parse(fs.readFileSync(program.compare).toString());
@@ -123,4 +139,4 @@ runAll(parseInt(program.sample), function(res) {
       dashColor: 'cyan',
     }));
   }
-});
+}