Skip to content

Commit 035e76e

Browse files
steve-ssteve-s
committed
[GR-29724] Add interpreter benchmarks.
PullRequest: graalpython/1645
2 parents 4f22561 + 3868637 commit 035e76e

File tree

11 files changed

+109
-32
lines changed

11 files changed

+109
-32
lines changed

‎ci.jsonnet

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{ "overlay": "69d605b7f6f27a86c9e0e8e6c0991a61dbf09d3f" }
1+
{ "overlay": "96dbc8ee7d775b1067f7462f80c50274f11045f0" }

‎graalpython/com.oracle.graal.python.benchmarks/python/meso/raytrace-simple.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -365,5 +365,5 @@ def _main(width=400, height=400):
365365
s.addObject(Halfspace(Point(0,0,0), Vector.UP), CheckerboardSurface())
366366
s.render(c)
367367

368-
def __benchmark__():
369-
_main()
368+
def __benchmark__(*args):
369+
_main(*args)

‎graalpython/com.oracle.graal.python.benchmarks/python/micro/builtin-len.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,13 +33,13 @@ def call_len(num, ll):
3333
return length
3434

3535

36-
def measure():
36+
def measure(num=500000000):
3737
ll = [x * 2 for x in range(1000)]
38-
length = call_len(500000000, ll) # 1000000000
38+
length = call_len(num, ll) # 1000000000
3939

4040
print("Final length ", length)
4141

4242

4343

4444
def __benchmark__(*args):
45-
measure()
45+
measure(*args)

‎graalpython/com.oracle.graal.python.benchmarks/python/micro/generator-expression-sized.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -45,11 +45,11 @@ def call_generator_localvar(num, iteration):
4545
return item
4646

4747

48-
def measure(num):
49-
last_item = call_generator(num, 10000)
48+
def measure(num, iteration):
49+
last_item = call_generator(num, iteration)
5050

5151
print("Last item ", last_item)
5252

5353

54-
def __benchmark__(num):
55-
measure(num)
54+
def __benchmark__(num, it=10000):
55+
measure(num, it)

‎graalpython/com.oracle.graal.python.benchmarks/python/micro/generator-notaligned-sized.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -40,11 +40,11 @@ def call_generator(num, iteration):
4040
return item
4141

4242

43-
def measure(num):
44-
last_item = call_generator(num, 10000)
43+
def measure(num, iteration):
44+
last_item = call_generator(num, iteration)
4545

4646
print("Last item ", last_item)
4747

4848

49-
def __benchmark__(num):
50-
measure(num)
49+
def __benchmark__(num, it=10000):
50+
measure(num, it)

‎graalpython/com.oracle.graal.python.benchmarks/python/micro/generator-sized.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -39,11 +39,11 @@ def call_generator(num, iteration):
3939
return item
4040

4141

42-
def measure(num):
43-
last_item = call_generator(num, 10000)
42+
def measure(num, iteration):
43+
last_item = call_generator(num, iteration)
4444

4545
print("Last item ", last_item)
4646

4747

48-
def __benchmark__(num):
49-
measure(num)
48+
def __benchmark__(num, it=10000):
49+
measure(num, it)

‎graalpython/com.oracle.graal.python.benchmarks/python/micro/genexp-builtin-call-sized.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -55,10 +55,10 @@ def call_generator_localvar(num, iteration):
5555
return item
5656

5757

58-
def measure(num):
59-
last_item = call_generator(num, 10000)
58+
def measure(num, iteration):
59+
last_item = call_generator(num, iteration)
6060
print("Last item ", last_item)
6161

6262

63-
def __benchmark__(num=1000):
64-
measure(num)
63+
def __benchmark__(num=1000, it=10000):
64+
measure(num, it)

‎graalpython/com.oracle.graal.python.benchmarks/python/micro/special-len.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -47,12 +47,12 @@ def do_compute(num):
4747
return sum_
4848

4949

50-
def measure(num):
50+
def measure(num, iteration):
5151
for run in range(num):
52-
sum_ = do_compute(20000) # 10000
52+
sum_ = do_compute(iteration) # 10000
5353

5454
print("sum", sum_)
5555

5656

57-
def __benchmark__(num=5):
58-
measure(num)
57+
def __benchmark__(num=5, iteration=20000):
58+
measure(num, iteration)

‎mx.graalpython/mx_graalpython.py

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,14 +27,15 @@
2727
import contextlib
2828
import datetime
2929
import glob
30-
import itertools
3130
import json
3231
import os
3332
import pathlib
3433
import platform
3534
import re
3635
import shlex
3736
import shutil
37+
38+
import itertools
3839
import sys
3940

4041
HPY_IMPORT_ORPHAN_BRANCH_NAME = "hpy-import"
@@ -57,12 +58,14 @@
5758
import mx_graalpython_bisect
5859
from mx_gate import Task
5960
from mx_graalpython_bench_param import PATH_MESO, BENCHMARKS, WARMUP_BENCHMARKS, JBENCHMARKS, PARSER_BENCHMARKS
60-
from mx_graalpython_benchmark import PythonBenchmarkSuite, python_vm_registry, CPythonVm, PyPyVm, JythonVm, GraalPythonVm, \
61+
from mx_graalpython_benchmark import PythonBenchmarkSuite, python_vm_registry, CPythonVm, PyPyVm, JythonVm, \
62+
GraalPythonVm, \
6163
CONFIGURATION_DEFAULT, CONFIGURATION_SANDBOXED, CONFIGURATION_NATIVE, \
6264
CONFIGURATION_DEFAULT_MULTI, CONFIGURATION_SANDBOXED_MULTI, CONFIGURATION_NATIVE_MULTI, \
6365
CONFIGURATION_DEFAULT_MULTI_TIER, CONFIGURATION_NATIVE_MULTI_TIER, \
64-
PythonInteropBenchmarkSuite, PythonVmWarmupBenchmarkSuite, PythonParserBenchmarkSuite
65-
66+
PythonInteropBenchmarkSuite, PythonVmWarmupBenchmarkSuite, PythonParserBenchmarkSuite, \
67+
CONFIGURATION_INTERPRETER, CONFIGURATION_INTERPRETER_MULTI, CONFIGURATION_NATIVE_INTERPRETER, \
68+
CONFIGURATION_NATIVE_INTERPRETER_MULTI
6669

6770
if not sys.modules.get("__main__"):
6871
# workaround for pdb++
@@ -1490,9 +1493,15 @@ def _register_vms(namespace):
14901493

14911494
# graalpython
14921495
python_vm_registry.add_vm(GraalPythonVm(config_name=CONFIGURATION_DEFAULT), SUITE, 10)
1496+
python_vm_registry.add_vm(GraalPythonVm(config_name=CONFIGURATION_INTERPRETER, extra_polyglot_args=[
1497+
'--experimental-options', '--engine.Compilation=false'
1498+
]), SUITE, 10)
14931499
python_vm_registry.add_vm(GraalPythonVm(config_name=CONFIGURATION_DEFAULT_MULTI, extra_polyglot_args=[
14941500
'--experimental-options', '-multi-context',
14951501
]), SUITE, 10)
1502+
python_vm_registry.add_vm(GraalPythonVm(config_name=CONFIGURATION_INTERPRETER_MULTI, extra_polyglot_args=[
1503+
'--experimental-options', '-multi-context', '--engine.Compilation=false'
1504+
]), SUITE, 10)
14961505
python_vm_registry.add_vm(GraalPythonVm(config_name=CONFIGURATION_DEFAULT_MULTI_TIER, extra_polyglot_args=[
14971506
'--experimental-options', '--engine.MultiTier=true',
14981507
]), SUITE, 10)
@@ -1501,12 +1510,17 @@ def _register_vms(namespace):
15011510
]), SUITE, 10)
15021511
python_vm_registry.add_vm(GraalPythonVm(config_name=CONFIGURATION_NATIVE, extra_polyglot_args=[
15031512
]), SUITE, 10)
1513+
python_vm_registry.add_vm(GraalPythonVm(config_name=CONFIGURATION_NATIVE_INTERPRETER, extra_polyglot_args=[
1514+
'--experimental-options', '--engine.Compilation=false']), SUITE, 10)
15041515
python_vm_registry.add_vm(GraalPythonVm(config_name=CONFIGURATION_SANDBOXED_MULTI, extra_polyglot_args=[
15051516
'--experimental-options', '-multi-context', '--llvm.managed',
15061517
]), SUITE, 10)
15071518
python_vm_registry.add_vm(GraalPythonVm(config_name=CONFIGURATION_NATIVE_MULTI, extra_polyglot_args=[
15081519
'--experimental-options', '-multi-context',
15091520
]), SUITE, 10)
1521+
python_vm_registry.add_vm(GraalPythonVm(config_name=CONFIGURATION_NATIVE_INTERPRETER_MULTI, extra_polyglot_args=[
1522+
'--experimental-options', '-multi-context', '--engine.Compilation=false',
1523+
]), SUITE, 10)
15101524
python_vm_registry.add_vm(GraalPythonVm(config_name=CONFIGURATION_NATIVE_MULTI_TIER, extra_polyglot_args=[
15111525
'--experimental-options', '--engine.MultiTier=true',
15121526
]), SUITE, 10)

‎mx.graalpython/mx_graalpython_bench_param.py

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,8 @@
5252
ITER_20 = ['-i', '20']
5353
ITER_15 = ['-i', '15']
5454
ITER_10 = ['-i', '10']
55+
ITER_6 = ['-i', '6']
56+
WARMUP_2 = ['-w', '2']
5557

5658
MICRO_BENCHMARKS = {
5759
'arith-binop': ITER_10 + ['5'],
@@ -101,6 +103,52 @@
101103
'tuple-indexing-from-literal': ITER_10 + ['10000000'],
102104
}
103105

106+
MICRO_BENCHMARKS_SMALL = {
107+
'arith-modulo-sized': ITER_6 + WARMUP_2 + ['1'],
108+
'attribute-access-polymorphic': ITER_6 + WARMUP_2 + ['20'],
109+
'attribute-access': ITER_6 + WARMUP_2 + ['100'],
110+
'attribute-access-super': ITER_6 + WARMUP_2 + ['40'],
111+
'attribute-bool': ITER_6 + WARMUP_2 + ['2'],
112+
'boolean-logic-sized': ITER_6 + WARMUP_2 + ['10'],
113+
'builtin-len-tuple-sized': ITER_6 + WARMUP_2 + ['10_000_000'],
114+
'builtin-len': ITER_6 + WARMUP_2 + ['2_500_000'],
115+
'class-access': ITER_6 + WARMUP_2 + ['40'],
116+
'call-method-polymorphic': ITER_6 + WARMUP_2 + ['10'],
117+
'for-range': ITER_6 + WARMUP_2 + WARMUP_2 + ['50'],
118+
'function-call-sized': ITER_6 + WARMUP_2 + ['2_000_000'],
119+
'generator-expression-sized': ITER_6 + WARMUP_2 + ['3000', '500'],
120+
'generator-notaligned-sized': ITER_6 + WARMUP_2 + ['3000', '500'],
121+
'generator-sized': ITER_6 + WARMUP_2 + ['3000', '500'],
122+
'genexp-builtin-call-sized': ITER_6 + WARMUP_2 + ['3000', '500'],
123+
'list-comp': ITER_6 + WARMUP_2 + ['25'],
124+
'list-indexing': ITER_6 + WARMUP_2 + ['10_000'],
125+
'list-indexing-from-constructor': ITER_6 + WARMUP_2 + ['250_000'],
126+
'list-indexing-from-literal': ITER_6 + WARMUP_2 + ['250_000'],
127+
'list-iterating-explicit': ITER_6 + WARMUP_2 + ['10_000'],
128+
'list-iterating': ITER_6 + WARMUP_2 + ['25_000'],
129+
'list-iterating-obj-sized': ITER_6 + WARMUP_2 + ['1_000_000'],
130+
'list-constructions-sized': ITER_6 + WARMUP_2 + ['500'],
131+
'dict-getitem-sized': ITER_6 + WARMUP_2 + ['1_000_000'],
132+
'math-sqrt': ITER_6 + WARMUP_2 + ['20_000_000'],
133+
'object-allocate': ITER_6 + WARMUP_2 + ['50'],
134+
'object-layout-change': ITER_6 + WARMUP_2 + ['10_000'],
135+
'special-add-int-sized': ITER_6 + WARMUP_2 + ['1_000'],
136+
'special-add-sized': ITER_6 + WARMUP_2 + ['1_000'],
137+
'special-len': ITER_6 + WARMUP_2 + ['1', '1_000'],
138+
'member-access': ITER_6 + WARMUP_2 + ['10'],
139+
'magic-bool-sized': ITER_6 + WARMUP_2 + ['1_000_000'],
140+
'magic-iter': ITER_6 + WARMUP_2 + ['250000'],
141+
'call-classmethod-sized': ITER_6 + WARMUP_2 + ['1_000_000'],
142+
'mmap-anonymous-sized': ITER_6 + WARMUP_2 + ['1_000'],
143+
'mmap-file': ITER_6 + WARMUP_2 + ['100'],
144+
'generate-functions-sized': ITER_6 + WARMUP_2 + ['1_000_000'],
145+
'try-except-simple': ITER_6 + WARMUP_2 + ['2_500_000'],
146+
'try-except-store-simple': ITER_6 + WARMUP_2 + ['2_000_000'],
147+
'try-except-store-two-types': ITER_6 + WARMUP_2 + ['1_000_000'],
148+
'try-except-two-types': ITER_6 + WARMUP_2 + ['1_000_000'],
149+
'tuple-indexing-from-constructor': ITER_6 + WARMUP_2 + ['250_000'],
150+
'tuple-indexing-from-literal': ITER_6 + WARMUP_2 + ['400_000'],
151+
}
104152

105153
def _pickling_benchmarks(module='pickle'):
106154
return {
@@ -179,6 +227,15 @@ def _pickling_benchmarks(module='pickle'):
179227
}
180228

181229

230+
MESO_BENCHMARKS_SMALL = {
231+
'pads-eratosthenes': ITER_10 + WARMUP_2 + ['2000'],
232+
'richards3': ITER_10 + WARMUP_2 + ['2'],
233+
'chaos': ITER_10 + WARMUP_2 + ['2'],
234+
'image-magix': ITER_10 + WARMUP_2 + ['100'],
235+
'raytrace-simple': ITER_10 + WARMUP_2 + ['110', '110'],
236+
}
237+
238+
182239
MACRO_BENCHMARKS = {
183240
'gcbench': ITER_10 + ['10'],
184241
}
@@ -242,6 +299,8 @@ def _pickling_benchmarks(module='pickle'):
242299
"meso": [PATH_MESO, MESO_BENCHMARKS],
243300
"macro": [PATH_MACRO, MACRO_BENCHMARKS],
244301
"interop": [PATH_INTEROP, INTEROP_BENCHMARKS],
302+
"micro-small": [PATH_MICRO, MICRO_BENCHMARKS_SMALL],
303+
"meso-small": [PATH_MESO, MESO_BENCHMARKS_SMALL],
245304
}
246305

247306
WARMUP_BENCHMARKS = {

0 commit comments

Comments
 (0)