Skip to content

Commit 3319253

Browse files
Rich-Harristrueadm
Rich-Harris
and
trueadm
authored
chore: add pnpm bench:compare script (#12099)
* WIP compare script * WIP * fix * add script * update * fix * tweak * fix * update * add TODO * compare * tmp * gah * revert * fix * fix * prevent div/0 * reenable benchmarks --------- Co-authored-by: Dominic Gannaway <[email protected]>
1 parent 7f087c4 commit 3319253

File tree

6 files changed

+151
-50
lines changed

6 files changed

+151
-50
lines changed

‎.gitignore

+2
Original file line numberDiff line numberDiff line change
@@ -22,3 +22,5 @@ coverage
2222
.DS_Store
2323

2424
tmp
25+
26+
benchmarking/compare/.results

‎benchmarking/benchmarks.js

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
import { kairo_avoidable } from './benchmarks/kairo/kairo_avoidable.js';
2+
import { kairo_broad } from './benchmarks/kairo/kairo_broad.js';
3+
import { kairo_deep } from './benchmarks/kairo/kairo_deep.js';
4+
import { kairo_diamond } from './benchmarks/kairo/kairo_diamond.js';
5+
import { kairo_mux } from './benchmarks/kairo/kairo_mux.js';
6+
import { kairo_repeated } from './benchmarks/kairo/kairo_repeated.js';
7+
import { kairo_triangle } from './benchmarks/kairo/kairo_triangle.js';
8+
import { kairo_unstable } from './benchmarks/kairo/kairo_unstable.js';
9+
import { mol_bench } from './benchmarks/mol_bench.js';
10+
11+
// This benchmark has been adapted from the js-reactivity-benchmark (https://github.com/milomg/js-reactivity-benchmark)
12+
// Not all tests are the same, and many parts have been tweaked to capture different data.
13+
14+
export const benchmarks = [
15+
kairo_avoidable,
16+
kairo_broad,
17+
kairo_deep,
18+
kairo_diamond,
19+
kairo_triangle,
20+
kairo_mux,
21+
kairo_repeated,
22+
kairo_unstable,
23+
mol_bench
24+
];

‎benchmarking/compare/index.js

+90
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
import fs from 'node:fs';
2+
import path from 'node:path';
3+
import { execSync, fork } from 'node:child_process';
4+
import { fileURLToPath } from 'node:url';
5+
import { benchmarks } from '../benchmarks.js';
6+
7+
// if (execSync('git status --porcelain').toString().trim()) {
8+
// console.error('Working directory is not clean');
9+
// process.exit(1);
10+
// }
11+
12+
const filename = fileURLToPath(import.meta.url);
13+
const runner = path.resolve(filename, '../runner.js');
14+
const outdir = path.resolve(filename, '../.results');
15+
16+
if (fs.existsSync(outdir)) fs.rmSync(outdir, { recursive: true });
17+
fs.mkdirSync(outdir);
18+
19+
const branches = [];
20+
21+
for (const arg of process.argv.slice(2)) {
22+
if (arg.startsWith('--')) continue;
23+
if (arg === filename) continue;
24+
25+
branches.push(arg);
26+
}
27+
28+
if (branches.length === 0) {
29+
branches.push(
30+
execSync('git symbolic-ref --short -q HEAD || git rev-parse --short HEAD').toString().trim()
31+
);
32+
}
33+
34+
if (branches.length === 1) {
35+
branches.push('main');
36+
}
37+
38+
process.on('exit', () => {
39+
execSync(`git checkout ${branches[0]}`);
40+
});
41+
42+
for (const branch of branches) {
43+
console.group(`Benchmarking ${branch}`);
44+
45+
execSync(`git checkout ${branch}`);
46+
47+
await new Promise((fulfil, reject) => {
48+
const child = fork(runner);
49+
50+
child.on('message', (results) => {
51+
fs.writeFileSync(`${outdir}/${branch}.json`, JSON.stringify(results, null, ' '));
52+
fulfil();
53+
});
54+
55+
child.on('error', reject);
56+
});
57+
58+
console.groupEnd();
59+
}
60+
61+
const results = branches.map((branch) => {
62+
return JSON.parse(fs.readFileSync(`${outdir}/${branch}.json`, 'utf-8'));
63+
});
64+
65+
for (let i = 0; i < results[0].length; i += 1) {
66+
console.group(`${results[0][i].benchmark}`);
67+
68+
for (const metric of ['time', 'gc_time']) {
69+
const times = results.map((result) => +result[i][metric]);
70+
let min = Infinity;
71+
let min_index = -1;
72+
73+
for (let b = 0; b < times.length; b += 1) {
74+
if (times[b] < min) {
75+
min = times[b];
76+
min_index = b;
77+
}
78+
}
79+
80+
if (min !== 0) {
81+
console.group(`${metric}: fastest is ${branches[min_index]}`);
82+
times.forEach((time, b) => {
83+
console.log(`${branches[b]}: ${time.toFixed(2)}ms (${((time / min) * 100).toFixed(2)}%)`);
84+
});
85+
console.groupEnd();
86+
}
87+
}
88+
89+
console.groupEnd();
90+
}

‎benchmarking/compare/runner.js

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
import { benchmarks } from '../benchmarks.js';
2+
3+
const results = [];
4+
for (const benchmark of benchmarks) {
5+
const result = await benchmark();
6+
console.error(result.benchmark);
7+
results.push(result);
8+
}
9+
10+
process.send(results);

‎benchmarking/run.js

+24-50
Original file line numberDiff line numberDiff line change
@@ -1,58 +1,32 @@
11
import * as $ from '../packages/svelte/src/internal/client/index.js';
2-
import { kairo_avoidable } from './benchmarks/kairo/kairo_avoidable.js';
3-
import { kairo_broad } from './benchmarks/kairo/kairo_broad.js';
4-
import { kairo_deep } from './benchmarks/kairo/kairo_deep.js';
5-
import { kairo_diamond } from './benchmarks/kairo/kairo_diamond.js';
6-
import { kairo_mux } from './benchmarks/kairo/kairo_mux.js';
7-
import { kairo_repeated } from './benchmarks/kairo/kairo_repeated.js';
8-
import { kairo_triangle } from './benchmarks/kairo/kairo_triangle.js';
9-
import { kairo_unstable } from './benchmarks/kairo/kairo_unstable.js';
10-
import { mol_bench } from './benchmarks/mol_bench.js';
2+
import { benchmarks } from './benchmarks.js';
113

12-
// This benchmark has been adapted from the js-reactivity-benchmark (https://github.com/milomg/js-reactivity-benchmark)
13-
// Not all tests are the same, and many parts have been tweaked to capture different data.
4+
let total_time = 0;
5+
let total_gc_time = 0;
146

15-
const benchmarks = [
16-
kairo_avoidable,
17-
kairo_broad,
18-
kairo_deep,
19-
kairo_diamond,
20-
kairo_triangle,
21-
kairo_mux,
22-
kairo_repeated,
23-
kairo_unstable,
24-
mol_bench
25-
];
26-
27-
async function run_benchmarks() {
28-
let total_time = 0;
29-
let total_gc_time = 0;
30-
// eslint-disable-next-line no-console
31-
console.log('-- Benchmarking Started --');
32-
$.push({}, true);
33-
try {
34-
for (const benchmark of benchmarks) {
35-
const results = await benchmark();
36-
// eslint-disable-next-line no-console
37-
console.log(results);
38-
total_time += Number(results.time);
39-
total_gc_time += Number(results.gc_time);
40-
}
41-
} catch (e) {
7+
// eslint-disable-next-line no-console
8+
console.log('-- Benchmarking Started --');
9+
$.push({}, true);
10+
try {
11+
for (const benchmark of benchmarks) {
12+
const results = await benchmark();
4213
// eslint-disable-next-line no-console
43-
console.error('-- Benchmarking Failed --');
44-
// eslint-disable-next-line no-console
45-
console.error(e);
46-
process.exit(1);
14+
console.log(results);
15+
total_time += Number(results.time);
16+
total_gc_time += Number(results.gc_time);
4717
}
48-
$.pop();
18+
} catch (e) {
4919
// eslint-disable-next-line no-console
50-
console.log(`-- Benchmarking Complete --`);
20+
console.error('-- Benchmarking Failed --');
5121
// eslint-disable-next-line no-console
52-
console.log({
53-
total_time: total_time.toFixed(2),
54-
total_gc_time: total_gc_time.toFixed(2)
55-
});
22+
console.error(e);
23+
process.exit(1);
5624
}
57-
58-
run_benchmarks();
25+
$.pop();
26+
// eslint-disable-next-line no-console
27+
console.log('-- Benchmarking Complete --');
28+
// eslint-disable-next-line no-console
29+
console.log({
30+
total_time: total_time.toFixed(2),
31+
total_gc_time: total_gc_time.toFixed(2)
32+
});

‎package.json

+1
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
"changeset:version": "changeset version && pnpm -r generate:version && git add --all",
2626
"changeset:publish": "changeset publish",
2727
"bench": "node --allow-natives-syntax ./benchmarking/run.js",
28+
"bench:compare": "node --allow-natives-syntax ./benchmarking/compare/index.js",
2829
"bench:debug": "node --allow-natives-syntax --inspect-brk ./benchmarking/run.js"
2930
},
3031
"devDependencies": {

0 commit comments

Comments
 (0)