Skip to content

Commit de71487

Browse files
steve-ssteve-s
committed
Add Java embedding benchmarks driver and embedding suites for meso benchmarks
1 parent 87aeb8a commit de71487

File tree

9 files changed

+478
-35
lines changed

9 files changed

+478
-35
lines changed

‎graalpython/com.oracle.graal.python.benchmarks/java/com/oracle/graal/python/benchmarks/JavaBenchmarkDriver.java

Lines changed: 277 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,277 @@
1+
/*
2+
* Copyright (c) 2021, 2021, Oracle and/or its affiliates. All rights reserved.
3+
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4+
*
5+
* The Universal Permissive License (UPL), Version 1.0
6+
*
7+
* Subject to the condition set forth below, permission is hereby granted to any
8+
* person obtaining a copy of this software, associated documentation and/or
9+
* data (collectively the "Software"), free of charge and under any and all
10+
* copyright rights in the Software, and any and all patent rights owned or
11+
* freely licensable by each licensor hereunder covering either (i) the
12+
* unmodified Software as contributed to or provided by such licensor, or (ii)
13+
* the Larger Works (as defined below), to deal in both
14+
*
15+
* (a) the Software, and
16+
*
17+
* (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
18+
* one is included with the Software each a "Larger Work" to which the Software
19+
* is contributed by such licensors),
20+
*
21+
* without restriction, including without limitation the rights to copy, create
22+
* derivative works of, display, perform, and distribute the Software and make,
23+
* use, sell, offer for sale, import, export, have made, and have sold the
24+
* Software and the Larger Work(s), and to sublicense the foregoing rights on
25+
* either these or other terms.
26+
*
27+
* This license is subject to the following condition:
28+
*
29+
* The above copyright notice and either this complete permission notice or at a
30+
* minimum a reference to the UPL must be included in all copies or substantial
31+
* portions of the Software.
32+
*
33+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
34+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
35+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
36+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
37+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
38+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
39+
* SOFTWARE.
40+
*/
41+
package com.oracle.graal.python.benchmarks;
42+
43+
import java.io.IOException;
44+
import java.nio.file.Path;
45+
import java.nio.file.Paths;
46+
import java.util.ArrayList;
47+
import java.util.Arrays;
48+
import java.util.HashMap;
49+
import java.util.List;
50+
import java.util.Map;
51+
import java.util.stream.Collectors;
52+
53+
import org.graalvm.launcher.AbstractLanguageLauncher;
54+
import org.graalvm.options.OptionCategory;
55+
import org.graalvm.polyglot.Context;
56+
import org.graalvm.polyglot.Context.Builder;
57+
import org.graalvm.polyglot.Engine;
58+
import org.graalvm.polyglot.Source;
59+
import org.graalvm.polyglot.Value;
60+
61+
public class JavaBenchmarkDriver extends AbstractLanguageLauncher {
62+
private static final String LINE = "-------------------------------------------------------------------------------";
63+
private static final String BENCHMARK_ENTRY_POINT = "java_embedded_bench_entrypoint";
64+
65+
public static class BenchmarkOptions {
66+
public boolean sharedEngine;
67+
public boolean multiContext;
68+
public int iterations = 10;
69+
public int warmupIterations = 10;
70+
public String benchmarksPath;
71+
private String benchmarkName;
72+
private String[] benchmarkArgs = new String[0];
73+
}
74+
75+
private final Map<String, String> engineOptions = new HashMap<>();
76+
private final BenchmarkOptions options = new BenchmarkOptions();
77+
78+
public static void main(String[] args) {
79+
new JavaBenchmarkDriver().launch(args);
80+
}
81+
82+
@Override
83+
protected List<String> preprocessArguments(List<String> arguments, Map<String, String> polyglotOptions) {
84+
ArrayList<String> unrecognized = new ArrayList<>();
85+
int i = 0;
86+
optionsLoop: while (i < arguments.size()) {
87+
String arg = arguments.get(i);
88+
switch (arg) {
89+
case "-interpreter":
90+
engineOptions.put("engine.Compilation", "false");
91+
break;
92+
case "-shared-engine":
93+
options.sharedEngine = true;
94+
break;
95+
case "-multi-context":
96+
options.multiContext = true;
97+
break;
98+
case "-path":
99+
if (i + 1 >= arguments.size()) {
100+
System.err.println("Option -path is not followed by anything");
101+
System.exit(1);
102+
}
103+
options.benchmarksPath = arguments.get(i + 1);
104+
i += 1;
105+
break;
106+
case "-i":
107+
options.iterations = getIntOption(arguments, i);
108+
i += 1;
109+
break;
110+
case "-w":
111+
options.warmupIterations = getIntOption(arguments, i);
112+
i += 1;
113+
break;
114+
case "--":
115+
options.benchmarkArgs = arguments.subList(i + 1, arguments.size()).toArray(new String[0]);
116+
break optionsLoop;
117+
default:
118+
if (!arg.startsWith("-") && options.benchmarkName == null) {
119+
options.benchmarkName = arg;
120+
break;
121+
}
122+
unrecognized.add(arg);
123+
}
124+
i++;
125+
}
126+
return unrecognized;
127+
}
128+
129+
private static int getIntOption(List<String> arguments, int index) {
130+
if (index + 1 >= arguments.size()) {
131+
System.err.println("Option -i is not followed by anything");
132+
System.exit(1);
133+
}
134+
try {
135+
return Integer.parseInt(arguments.get(index + 1));
136+
} catch (NumberFormatException ex) {
137+
System.err.println("Option -i is not followed by a number");
138+
System.exit(1);
139+
}
140+
throw new AssertionError();
141+
}
142+
143+
@Override
144+
protected void launch(Builder contextBuilder) {
145+
contextBuilder.allowExperimentalOptions(true);
146+
contextBuilder.arguments("python", new String[]{"java_embedding_bench"});
147+
if (options.sharedEngine) {
148+
contextBuilder.engine(Engine.newBuilder().allowExperimentalOptions(true).options(engineOptions).build());
149+
} else {
150+
contextBuilder.options(engineOptions);
151+
}
152+
153+
System.out.println(LINE);
154+
System.out.printf("### %s, %d warmup iterations, %d bench iterations%n", options.benchmarkName, options.warmupIterations, options.iterations);
155+
System.out.printf("### args = %s%n", Arrays.toString(options.benchmarkArgs));
156+
157+
System.out.println(LINE);
158+
System.out.println("### setup ... ");
159+
Source source;
160+
Path path = Paths.get(options.benchmarksPath, options.benchmarkName + ".py");
161+
try {
162+
source = Source.newBuilder("python", path.toFile()).build();
163+
} catch (IOException e) {
164+
System.err.println("Cannot open the file: " + path);
165+
System.exit(1);
166+
return;
167+
}
168+
169+
System.out.println("### start benchmark ... ");
170+
if (options.multiContext) {
171+
runBenchmarkMultiContext(contextBuilder, source);
172+
} else {
173+
runBenchmarkSingleContext(contextBuilder, source);
174+
}
175+
}
176+
177+
private void runBenchmarkMultiContext(Builder contextBuilder, Source source) {
178+
System.out.println();
179+
System.out.printf("### (pre)warming up for %d iterations ... %n", options.warmupIterations);
180+
repeatBenchmarkMultiContext(contextBuilder, source, options.warmupIterations, true);
181+
182+
System.out.println();
183+
System.out.printf("### measure phase for %d iterations ... %n", options.iterations);
184+
long[] durations = repeatBenchmarkMultiContext(contextBuilder, source, options.iterations, false);
185+
186+
System.out.println(LINE);
187+
System.out.println("### benchmark complete");
188+
189+
System.out.println(LINE);
190+
System.out.printf("### BEST duration: %.4f%n", Arrays.stream(durations).min().getAsLong() / 1000_000_000.0);
191+
System.out.printf("### WORST duration: %.4f%n", Arrays.stream(durations).max().getAsLong() / 1000_000_000.0);
192+
System.out.printf("### AVG (no warmup) duration: %.4f%n", Arrays.stream(durations).average().getAsDouble() / 1000_000_000.0);
193+
194+
System.out.println(LINE);
195+
System.out.printf("### RAW DURATIONS: [%s]%n", Arrays.stream(durations).mapToObj(x -> String.format("%.4f", x / 1000_000_000.0)).collect(Collectors.joining(",")));
196+
}
197+
198+
private long[] repeatBenchmarkMultiContext(Builder contextBuilder, Source source, int iterations, boolean warmup) {
199+
long[] durations = new long[iterations];
200+
for (int i = 0; i < iterations; i++) {
201+
Context context = contextBuilder.build();
202+
Value benchmark = getBenchmark(source, context);
203+
204+
long start = System.nanoTime();
205+
benchmark.executeVoid((Object[]) options.benchmarkArgs);
206+
long end = System.nanoTime();
207+
durations[i] = end - start;
208+
reportIteration(i, durations[i], warmup);
209+
}
210+
return durations;
211+
}
212+
213+
private void runBenchmarkSingleContext(Builder contextBuilder, Source source) {
214+
Context context = contextBuilder.build();
215+
Value benchmark = getBenchmark(source, context);
216+
217+
System.out.println();
218+
System.out.printf("### (pre)warming up for %d iterations ... %n", options.warmupIterations);
219+
repeatBenchmarkSingleContext(benchmark, options.warmupIterations, true);
220+
221+
System.out.println();
222+
System.out.printf("### measure phase for %d iterations ... %n", options.iterations);
223+
repeatBenchmarkSingleContext(benchmark, options.iterations, false);
224+
}
225+
226+
private void repeatBenchmarkSingleContext(Value benchmark, int iterations, boolean warmup) {
227+
for (int i = 0; i < iterations; i++) {
228+
long start = System.nanoTime();
229+
benchmark.executeVoid((Object[]) options.benchmarkArgs);
230+
long end = System.nanoTime();
231+
long time = end - start;
232+
reportIteration(i, time, warmup);
233+
}
234+
}
235+
236+
private Value getBenchmark(Source source, Context context) {
237+
try {
238+
context.eval(source);
239+
Value result = context.getBindings("python").getMember(BENCHMARK_ENTRY_POINT);
240+
if (result == null) {
241+
throw new UnsupportedOperationException();
242+
}
243+
return result;
244+
} catch (UnsupportedOperationException ex) {
245+
System.err.printf("Benchmark in file '%s' does not define method named '%s'.%n", options.benchmarkName, BENCHMARK_ENTRY_POINT);
246+
System.exit(1);
247+
return null;
248+
}
249+
}
250+
251+
private void reportIteration(int iter, long nanoTime, boolean warmup) {
252+
System.out.printf("### %siteration=%d, name=%s, duration=%.4f%n", warmup ? "warmup " : "", iter, options.benchmarkName, nanoTime / 1_000_000_000.0);
253+
}
254+
255+
@Override
256+
protected String getLanguageId() {
257+
return "python";
258+
}
259+
260+
@Override
261+
protected void printHelp(OptionCategory maxCategory) {
262+
System.out.println("Python Java benchmarks driver");
263+
System.out.println();
264+
System.out.println("usage: java ... JavaBenchmarkDriver [options] benchmarkName -- [benchmark arguments]");
265+
System.out.println();
266+
System.out.println("Supported options:");
267+
System.out.println("-i Number of benchmark iterations");
268+
System.out.println("-w Number of warmup iterations");
269+
System.out.println("-interpreter Turn off Truffle compilations to benchmark interpreter performance");
270+
System.out.println("-shared-engine Use shared engine");
271+
System.out.println("-multi-context Run each iteration in a new context");
272+
System.out.println("-path /some/path Where to look for the Python scripts with the benchmarks");
273+
System.out.println();
274+
System.out.printf("The Python benchmark script must define function named '%s'.%n", BENCHMARK_ENTRY_POINT);
275+
System.out.println("The values from [benchmark arguments] will be passed to it as arguments, always as strings.");
276+
}
277+
}

‎graalpython/com.oracle.graal.python.benchmarks/python/meso/chaos.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -275,3 +275,7 @@ def main(iterations=10, w=1000, h=1200):
275275

276276
def __benchmark__(iterations=10):
277277
main(iterations)
278+
279+
def java_embedded_bench_entrypoint(iterations=2):
280+
__setup__(int(iterations))
281+
main(int(iterations))

‎graalpython/com.oracle.graal.python.benchmarks/python/meso/image-magix.py

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -125,15 +125,20 @@ def __benchmark__(num=10000):
125125
return measure(num)
126126

127127

128+
def java_embedded_bench_entrypoint(num=10000):
129+
measure(int(num))
130+
131+
128132
if __name__ == '__main__':
129133
import sys
130-
import time
131-
SZ = 5
132-
start = time.time()
133-
if len(sys.argv) >= 2:
134-
num = int(sys.argv[1])
135-
img = __benchmark__(num)
136-
else:
137-
img = __benchmark__(2)
138-
print(img.data)
139-
print("%s took %s s" % (__file__, time.time() - start))
134+
if not (len(sys.argv) == 1 and sys.argv[0] == 'java_embedding_bench'):
135+
import time
136+
SZ = 5
137+
start = time.time()
138+
if len(sys.argv) >= 2:
139+
num = int(sys.argv[1])
140+
img = __benchmark__(num)
141+
else:
142+
img = __benchmark__(2)
143+
print(img.data)
144+
print("%s took %s s" % (__file__, time.time() - start))

‎graalpython/com.oracle.graal.python.benchmarks/python/meso/raytrace-simple.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -367,3 +367,6 @@ def _main(width=400, height=400):
367367

368368
def __benchmark__(*args):
369369
_main(*args)
370+
371+
def java_embedded_bench_entrypoint(width=400, height=400):
372+
_main(int(width), int(height))

‎graalpython/com.oracle.graal.python.benchmarks/python/meso/richards3.py

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -448,14 +448,17 @@ def measure(iteration):
448448
def __benchmark__(num=200):
449449
measure(num)
450450

451+
def java_embedded_bench_entrypoint(num=200):
452+
measure(int(num))
451453

452454
if __name__ == '__main__':
453455
import sys
454-
import time
455-
start = time.time()
456-
if len(sys.argv) >= 2:
457-
num = int(sys.argv[1])
458-
__benchmark__(num)
459-
else:
460-
__benchmark__()
461-
print("%s took %s s" % (__file__, time.time() - start))
456+
if not (len(sys.argv) == 1 and sys.argv[0] == 'java_embedding_bench'):
457+
import time
458+
start = time.time()
459+
if len(sys.argv) >= 2:
460+
num = int(sys.argv[1])
461+
__benchmark__(num)
462+
else:
463+
__benchmark__()
464+
print("%s took %s s" % (__file__, time.time() - start))

‎mx.graalpython/mx_graalpython.py

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,15 +57,19 @@
5757
import mx_urlrewrites
5858
import mx_graalpython_bisect
5959
from mx_gate import Task
60-
from mx_graalpython_bench_param import PATH_MESO, BENCHMARKS, WARMUP_BENCHMARKS, JBENCHMARKS, PARSER_BENCHMARKS
60+
from mx_graalpython_bench_param import PATH_MESO, BENCHMARKS, WARMUP_BENCHMARKS, JBENCHMARKS, PARSER_BENCHMARKS, \
61+
JAVA_DRIVER_BENCHMARKS
6162
from mx_graalpython_benchmark import PythonBenchmarkSuite, python_vm_registry, CPythonVm, PyPyVm, JythonVm, \
6263
GraalPythonVm, \
6364
CONFIGURATION_DEFAULT, CONFIGURATION_SANDBOXED, CONFIGURATION_NATIVE, \
6465
CONFIGURATION_DEFAULT_MULTI, CONFIGURATION_SANDBOXED_MULTI, CONFIGURATION_NATIVE_MULTI, \
6566
CONFIGURATION_DEFAULT_MULTI_TIER, CONFIGURATION_NATIVE_MULTI_TIER, \
6667
PythonInteropBenchmarkSuite, PythonVmWarmupBenchmarkSuite, PythonParserBenchmarkSuite, \
6768
CONFIGURATION_INTERPRETER, CONFIGURATION_INTERPRETER_MULTI, CONFIGURATION_NATIVE_INTERPRETER, \
68-
CONFIGURATION_NATIVE_INTERPRETER_MULTI
69+
CONFIGURATION_NATIVE_INTERPRETER_MULTI, PythonJavaEmbeddingBenchmarkSuite, python_java_embedding_vm_registry, \
70+
GraalPythonJavaDriverVm, CONFIGURATION_JAVA_EMBEDDING_INTERPRETER_MULTI_SHARED, \
71+
CONFIGURATION_JAVA_EMBEDDING_INTERPRETER_MULTI, CONFIGURATION_JAVA_EMBEDDING_MULTI_SHARED, \
72+
CONFIGURATION_JAVA_EMBEDDING_MULTI
6973

7074
if not sys.modules.get("__main__"):
7175
# workaround for pdb++
@@ -1525,10 +1529,26 @@ def _register_vms(namespace):
15251529
'--experimental-options', '--engine.MultiTier=true',
15261530
]), SUITE, 10)
15271531

1532+
# java embedding driver
1533+
python_java_embedding_vm_registry.add_vm(
1534+
GraalPythonJavaDriverVm(config_name=CONFIGURATION_JAVA_EMBEDDING_MULTI,
1535+
extra_polyglot_args=['-multi-context']), SUITE, 10)
1536+
python_java_embedding_vm_registry.add_vm(
1537+
GraalPythonJavaDriverVm(config_name=CONFIGURATION_JAVA_EMBEDDING_MULTI_SHARED,
1538+
extra_polyglot_args=['-multi-context', '-shared-engine']), SUITE, 10)
1539+
python_java_embedding_vm_registry.add_vm(
1540+
GraalPythonJavaDriverVm(config_name=CONFIGURATION_JAVA_EMBEDDING_INTERPRETER_MULTI,
1541+
extra_polyglot_args=['-multi-context', '-interpreter']), SUITE, 10)
1542+
python_java_embedding_vm_registry.add_vm(
1543+
GraalPythonJavaDriverVm(config_name=CONFIGURATION_JAVA_EMBEDDING_INTERPRETER_MULTI_SHARED,
1544+
extra_polyglot_args=['-multi-context', '-interpreter', '-shared-engine']), SUITE, 10)
1545+
15281546

15291547
def _register_bench_suites(namespace):
15301548
for py_bench_suite in PythonBenchmarkSuite.get_benchmark_suites(BENCHMARKS):
15311549
mx_benchmark.add_bm_suite(py_bench_suite)
1550+
for py_bench_suite in PythonJavaEmbeddingBenchmarkSuite.get_benchmark_suites(JAVA_DRIVER_BENCHMARKS):
1551+
mx_benchmark.add_bm_suite(py_bench_suite)
15321552
for py_bench_suite in PythonVmWarmupBenchmarkSuite.get_benchmark_suites(WARMUP_BENCHMARKS):
15331553
mx_benchmark.add_bm_suite(py_bench_suite)
15341554
for java_bench_suite in PythonInteropBenchmarkSuite.get_benchmark_suites(JBENCHMARKS):

0 commit comments

Comments
 (0)