Update perf tests and unify them with fluent-rs benches (#346)

* Update perf tests and unify them with fluent-rs benches

* Address reviewers feedback

* Add missing ftl files

* Retain --harmny-async-iteration in node v8

* Address feedback

* Move data to benchmarks.json and into fixtures dir

Author: zbraniecki

tools/perf/benchmark.common.js

@@ -0,0 +1,79 @@
+
+function runTests(env) {
+  const testData = JSON.parse(env.readFile("./fixtures/benchmarks.json"));
+
+  const results = {};
+
+  for (let testName in testData) {
+    let test = testData[testName];
+    runTest(env, testName, test.args, test.functions, results);
+  }
+
+  return results;
+}
+
+function runTest(env, name, args, fncs, results) {
+  const ftlCode = env.readFile(`./fixtures/${name}.ftl`);
+
+  {
+    const testName = "parse-syntax";
+    let start = env.now();
+    const ast = env.FluentSyntax.parse(ftlCode);
+    let end = env.now();
+
+    if (ast.body.some(elem => elem.type == "Junk")) {
+      throw Error("Junk in syntax parser result!");
+    }
+
+    results[`${testName}/"${name}"`] = env.ms(end) - env.ms(start);
+  }
+
+  let resource;
+  {
+    const testName = "parse-runtime";
+    let start = env.now();
+    resource = env.Fluent.FluentResource.fromString(ftlCode);
+    let end = env.now();
+
+    // we don't report any runtime parser errors, so
+    // we'll rely on the syntax parser to verify that
+    // the sample test syntax is correct.
+
+    results[`${testName}/"${name}"`] = env.ms(end) - env.ms(start);
+  }
+
+  {
+    const testName = "resolve-runtime";
+    const functions = {};
+    for (let fnName in fncs) {
+      let body = fncs[fnName];
+      functions[fnName] = new Function(body);
+    }
+    const bundle = new env.Fluent.FluentBundle('en-US', {
+      functions
+    });
+    const errors = bundle.addResource(resource);
+
+    let start = env.now();
+    for (const [id, message] of bundle.messages) {
+      bundle.format(message, args, errors);
+      if (message.attrs) {
+        for (const attrName in message.attrs) {
+          bundle.format(message.attrs[attrName], args, errors)
+        }
+      }
+    }
+    let end = env.now();
+
+    if (errors.length > 0) {
+      throw new Error(`Errors accumulated while resolving ${name}.`);
+    }
+
+    results[`${testName}/"${name}"`] = env.ms(end) - env.ms(start);
+  }
+}
+
+if (typeof exports !== "undefined") {
+  exports.runTest = runTest;
+  exports.runTests = runTests;
+}

tools/perf/benchmark.d8.js

@@ -1,42 +1,19 @@
 load('../../fluent/fluent.js');
 load('../../fluent-syntax/fluent-syntax.js');
-
-var ftlCode = read('./workload-low.ftl');
-var args = {}
-
-function ms(time) {
-  // time is in milliseconds with decimals
-  return Math.round(time * 1e3) / 1e3;
-}
-
-var times = {};
-
-times.ftlParseStart = Date.now();
-var resource = FluentSyntax.parse(ftlCode);
-times.ftlParseEnd = Date.now();
-
-times.ftlEntriesParseStart = Date.now();
-var resource = Fluent.FluentResource.fromString(ftlCode);
-times.ftlEntriesParseEnd = Date.now();
-
-var bundle = new Fluent.FluentBundle('en-US');
-var errors = bundle.addMessages(ftlCode);
-
-times.format = Date.now();
-for (const [id, message] of bundle.messages) {
-  bundle.format(message, args, errors);
-  if (message.attrs) {
-    for (const name in message.attrs) {
-      bundle.format(message.attrs[name], args, errors)
-    }
-  }
-}
-times.formatEnd = Date.now();
-
-var results = {
-  "parse full AST (ms)": ms(times.ftlParseEnd - times.ftlParseStart),
-  "parse runtime AST (ms)": ms(times.ftlEntriesParseEnd - times.ftlEntriesParseStart),
-  "format (ms)": ms(times.formatEnd - times.format),
+load('./benchmark.common.js');
+
+const env = {
+  readFile: (path) => {
+    return read(path);
+  },
+  ms: (milliseconds) => {
+    return milliseconds;
+  },
+  now: Date.now,
+  FluentSyntax,
+  Fluent,
 };
 
+const results = runTests(env);
+
 print(JSON.stringify(results));

tools/perf/benchmark.jsshell.js

@@ -1,42 +1,19 @@
 load('../../fluent/fluent.js');
 load('../../fluent-syntax/fluent-syntax.js');
-
-var ftlCode = read('./workload-low.ftl');
-var args = {};
-
-function ms(time) {
-  // time is in milliseconds with decimals
-  return Math.round(time * 1e3) / 1e3;
-}
-
-var times = {};
-
-times.ftlParseStart = dateNow();
-var resource = FluentSyntax.parse(ftlCode);
-times.ftlParseEnd = dateNow();
-
-times.ftlEntriesParseStart = dateNow();
-var resource = Fluent.FluentResource.fromString(ftlCode);
-times.ftlEntriesParseEnd = dateNow();
-
-var bundle = new Fluent.FluentBundle('en-US');
-var errors = bundle.addMessages(ftlCode);
-
-times.format = dateNow();
-for (const [id, message] of bundle.messages) {
-  bundle.format(message, args, errors);
-  if (message.attrs) {
-    for (const name in message.attrs) {
-      bundle.format(message.attrs[name], args, errors)
-    }
-  }
-}
-times.formatEnd = dateNow();
-
-var results = {
-  "parse full AST (ms)": ms(times.ftlParseEnd - times.ftlParseStart),
-  "parse runtime AST (ms)": ms(times.ftlEntriesParseEnd - times.ftlEntriesParseStart),
-  "format (ms)": ms(times.formatEnd - times.format),
+load('./benchmark.common.js');
+
+const env = {
+  readFile: (path) => {
+    return read(path);
+  },
+  ms: (milliseconds) => {
+    return milliseconds;
+  },
+  now: performance.now,
+  FluentSyntax,
+  Fluent,
 };
 
+const results = runTests(env);
+
 print(JSON.stringify(results));

tools/perf/benchmark.node.js

@@ -1,45 +1,21 @@
-var fs = require('fs');
-
-require('../../fluent-intl-polyfill');
-var Fluent = require('../../fluent');
-var FluentSyntax = require('../../fluent-syntax');
-
-var ftlCode = fs.readFileSync(__dirname + '/workload-low.ftl').toString();
-
-var args = {};
-
-function ms([seconds, nanoseconds]) {
-  return Math.round((seconds * 1e9 + nanoseconds) / 1e3) / 1e3;
-}
-
-var cumulative = {};
-var start = process.hrtime();
-
-cumulative.ftlParseStart = process.hrtime(start);
-var resource = FluentSyntax.parse(ftlCode);
-cumulative.ftlParseEnd = process.hrtime(start);
-
-cumulative.ftlEntriesParseStart = process.hrtime(start);
-var resource = Fluent.FluentResource.fromString(ftlCode);
-cumulative.ftlEntriesParseEnd = process.hrtime(start);
-
-var bundle = new Fluent.FluentBundle('en-US');
-var errors = bundle.addMessages(ftlCode);
+const fs = require('fs');
+
+const Fluent = require('../../fluent');
+const FluentSyntax = require('../../fluent-syntax');
+const { runTest, runTests } = require('./benchmark.common');
+
+const env = {
+  readFile: (path) => {
+    return fs.readFileSync(`${__dirname}/${path}`).toString();
+  },
+  ms: ([seconds, nanoseconds]) => {
+    return seconds * 1e3 + nanoseconds / 1e6;
+  },
+  now: process.hrtime,
+  FluentSyntax,
+  Fluent,
+};
 
-cumulative.format = process.hrtime(start);
-for (const [id, message] of bundle.messages) {
-  bundle.format(message, args, errors);
-  if (message.attrs) {
-    for (const name in message.attrs) {
-      bundle.format(message.attrs[name], args, errors)
-    }
-  }
-}
-cumulative.formatEnd = process.hrtime(start);
+const results = runTests(env);
 
-var results = {
-  "parse full AST (ms)": ms(cumulative.ftlParseEnd) - ms(cumulative.ftlParseStart),
-  "parse runtime AST (ms)": ms(cumulative.ftlEntriesParseEnd) - ms(cumulative.ftlEntriesParseStart),
-  "format (ms)": ms(cumulative.formatEnd) - ms(cumulative.format),
-};
 console.log(JSON.stringify(results));

tools/perf/fixtures/benchmarks.json

@@ -0,0 +1,31 @@
+{
+  "simple": {
+    "args": {},
+    "functions": {}
+  },
+  "preferences": {
+    "args": {
+      "name": "John",
+      "tabCount": 5,
+      "count": 3,
+      "version": "65.0",
+      "path": "/tmp",
+      "num": 4,
+      "email": "[email protected]",
+      "value": 4.5,
+      "unit": "mb",
+      "service-name": "Mozilla Disk"
+    },
+    "functions": {
+      "PLATFORM": "return 'linux';"
+    }
+  },
+  "menubar": {
+    "args": {},
+    "functions": {}
+  },
+  "workload-low": {
+    "args": {},
+    "functions": {}
+  }
+}