Update the code, adds 2024 benchmark results

Author: atuchin-m

.gitignore

@@ -1,6 +1,7 @@
 .DS_Store
 *.tar.gz
 *.csv
-results/
 browser_profiles/
 browsertime/
+browsertime-results/
+__pycache__

add_results.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+# Copyright (c) 2024 The Brave Authors. All rights reserved.
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at https://mozilla.org/MPL/2.0/.
+
+import argparse
+from components.result_map import ResultMap
+def main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument('csv_file', type=str)
+  parser.add_argument('browser', type=str)
+  parser.add_argument('browser_version', type=str)
+  parser.add_argument('metric', type=str)
+  parser.add_argument('value_list', type=str)
+  parser.add_argument('--key', type=str)
+  args = parser.parse_args()
+
+  values = args.value_list.split(',')
+  results = ResultMap()
+  for v in values:
+    value = float(v)
+    results.addValue(args.browser, args.browser_version, args.metric, args.key, value)
+
+  results.write_csv(args.csv_file, True)
+
+main()

benchmark_scripts/basemark.js

@@ -8,7 +8,7 @@ async function perfTest(context, commands) {
       "return document.getElementsByClassName('overall-score')[1]?.textContent")
     if (value && value != '') {
       console.log('got result', value)
-      commands.measure.addObject({ 'basemark_score': parseFloat(value) });
+      commands.measure.addObject({ 'basemark': parseFloat(value) });
       break;
     }
   }

benchmark_scripts/jetstream.js

@@ -1,19 +1,18 @@
+const utils = require('./utils.js')
+
 async function perfTest(context, commands) {
   await commands.measure.start(
-    'https://browserbench.org/JetStream2.0/');
+    'https://browserbench.org/JetStream2.2/');
+
+  await commands.wait.byXpath('//a[text()="Start Test"]', 2 * 60 * 1000)
   await commands.js.run('JetStream.start()');
 
-  while (true) {
-    await commands.wait.byTime(3000)
-    raw = await commands.js.run(
-      "return document.getElementById('result-summary')?.childNodes[0]?.textContent")
+  const score = await utils.waitForThrottled(commands,
+    'document.getElementById("result-summary")?.childNodes[0]?.textContent',
+    10 * 60)
 
-    if (raw && raw != '') {
-      console.log('got result', raw)
-      commands.measure.addObject({ 'jetstream2_score': parseFloat(raw) });
-      break;
-    }
-  }
+  console.log('got result =', score)
+  commands.measure.addObject({ 'jetstream': parseFloat(score) });
   await commands.screenshot.take('result')
 };
 

benchmark_scripts/kraken.js

@@ -15,7 +15,10 @@ async function perfTest(context, commands) {
         console.error(raw)
       }
       console.log('got total', m[1], m[2])
-      commands.measure.addObject({ 'kraken_ms': parseFloat(m[1]) });
+      commands.measure.addObject({
+        'kraken_ms': parseFloat(m[1]),
+        'kraken_error%': parseFloat(m[2])
+      });
       break;
     }
   }

benchmark_scripts/motionmark.js

@@ -0,0 +1,29 @@
+const utils = require('./utils.js')
+
+async function perfTest(context, commands) {
+  await commands.measure.start(
+    'https://browserbench.org/MotionMark1.3');
+
+  await commands.wait.byXpath('//button[text()="Run Benchmark"]', 2 * 60 * 1000)
+  await commands.click.byIdAndWait('start-button');
+
+  const score = await utils.waitForThrottled(commands,
+    'document.querySelector(".score")?.textContent', 10 * 60)
+
+  const error = await commands.js.run(
+    "return document.querySelector('.confidence').textContent.slice(1, -1)")
+
+  console.log('got result =', score, 'std =', error)
+
+  commands.measure.addObject(
+    {
+      'motionmark': parseFloat(score.split(' @')[0]),
+      'motionmark_error': parseFloat(error)
+    });
+
+  await commands.screenshot.take('result')
+};
+
+module.exports = {
+  test: perfTest
+};

benchmark_scripts/speedometer2.js

@@ -1,18 +1,26 @@
+const utils = require('./utils.js')
+
 async function perfTest(context, commands) {
-  await commands.measure.start(
-    'https://browserbench.org/Speedometer2.0/?iterationCount=10');
+  const URL = 'https://www.browserbench.org/Speedometer2.1'
+  const getResults = 'document.getElementById("result-number")?.textContent';
+
+  // One warm up iteration:
+  await commands.navigate(`${URL}?iterationCount=1`);
+  await commands.js.run('startTest()');
+  await utils.waitForThrottled(commands, getResults);
+
+  await commands.measure.start(`${URL}?iterationCount=100'`);
   await commands.js.run('startTest()');
 
-  while (true) {
-    await commands.wait.byTime(3000)
-    value = await commands.js.run(
-      "return window.document.getElementById('result-number').innerHTML")
-    if (value && value != '') {
-      console.log('got result', value)
-      commands.measure.addObject({ 'speedometer2_score': parseFloat(value) });
-      break;
-    }
-  }
+  const value = await utils.waitForThrottled(commands, getResults);
+  const error = await commands.js.run(
+    'return document.getElementById("confidence-number").textContent.substr(2)')
+  console.log('got result = ', value, 'std =', error)
+  commands.measure.addObject(
+    {
+      'speedometer2.1': parseFloat(value),
+      'speedometer2.1_error': parseFloat(error)
+    });
   await commands.screenshot.take('result')
 };
 

benchmark_scripts/speedometer3.js

@@ -0,0 +1,34 @@
+const utils = require('./utils.js')
+
+async function perfTest(context, commands) {
+  const URL = 'https://www.browserbench.org/Speedometer3.0?startAutomatically=true'
+  const getResults = 'document.getElementById("result-number")?.textContent';
+
+  // One warm up iteration:
+  await commands.navigate(`${URL}&iterationCount=1`);
+  await utils.waitForThrottled(commands, getResults, 3 * 60);
+
+  await commands.measure.start(`${URL}&iterationCount=10`);
+
+  const value = await utils.waitForThrottled(commands, getResults, 10 * 60);
+  const error = await commands.js.run(
+    'return document.getElementById("confidence-number").textContent.substr(2)')
+
+  console.log('got result = ', value, 'error =', error)
+  commands.measure.addObject(
+    {
+      'speedometer3_avg': parseFloat(value),
+      'speedometer3_error': parseFloat(error)
+    });
+  const raw = await commands.js.run('return JSON.parse(window.benchmarkClient._formattedJSONResult({ modern: true })).Score.values')
+  commands.measure.addObject(
+    {
+      'speedometer3': raw,
+    });
+
+  await commands.screenshot.take('result')
+};
+
+module.exports = {
+  test: perfTest
+};

benchmark_scripts/utils.js

@@ -0,0 +1,13 @@
+async function waitForThrottled(commands, condition, timeoutSeconds = 15 * 60) {
+  for (let i = 0; i < timeoutSeconds; ++i) {
+    result = await commands.js.run(`return (${condition})`)
+    if (result != null && result != '')
+      return result
+    await commands.wait.byTime(1000)
+  }
+  return false
+}
+
+module.exports = {
+  waitForThrottled: waitForThrottled
+};

components/benchmark_measurement.py

@@ -23,7 +23,7 @@ def Run(
       browser = browser_class()
       script = os.path.join('benchmark_scripts', name)
       assert os.path.exists(script)
-      browser.prepare_profile(self.state.unsafe_use_profiles)
+      browser.prepare_profile()
       result_dir = f'browsertime/{browser.name()}/{index}_{name}/{iteration}/'
       preURLDelay = 1000 if self.state.low_delays_for_testing else 10000
 
@@ -33,9 +33,14 @@ def Run(
           '--timeouts.script',
           str(30 * 60 * 1000),
       ])
+      # browsertime/Chrome/0_speedometer2.js/0/screenshots/1
 
       js_metrics = output['extras'][0]
       for metric, value in js_metrics.items():
-        results.append((metric, None, value))
+        if isinstance(value, list):
+          for v in value:
+            results.append((metric, None, v))
+        else:
+          results.append((metric, None, float(value)))
 
     return results