Skip to content

Commit 7062b23

Browse files
cosminbascacosminbasca
committed
[GR-12625] add cpython stats to running unittests
PullRequest: graalpython/284
2 parents 24948b5 + 603fb98 commit 7062b23

File tree

2 files changed

+108
-47
lines changed

2 files changed

+108
-47
lines changed

‎graalpython/com.oracle.graal.python.test/src/python_unittests.py

Lines changed: 75 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -62,17 +62,18 @@
6262
CSV_RESULTS_NAME = "{}.csv".format(_BASE_NAME)
6363
HTML_RESULTS_NAME = "{}.html".format(_BASE_NAME)
6464

65+
HR = "".join(['-' for _ in range(120)])
6566

6667
PTRN_ERROR = re.compile(r'^(?P<error>[A-Z][a-z][a-zA-Z]+):(?P<message>.*)$')
67-
PTRN_UNITTEST = re.compile(r'^#### running: graalpython/lib-python/3/test/(?P<unittest>.*)$')
68+
PTRN_UNITTEST = re.compile(r'^#### running: graalpython/lib-python/3/test/(?P<unittest>[\w.]+).*$', re.DOTALL)
6869
PTRN_NUM_TESTS = re.compile(r'^Ran (?P<num_tests>\d+) test.*$')
6970
PTRN_FAILED = re.compile(
7071
r'^FAILED \((failures=(?P<failures>\d+))?(, )?(errors=(?P<errors>\d+))?(, )?(skipped=(?P<skipped>\d+))?\)$')
7172
PTRN_OK = re.compile(
7273
r'^OK \((failures=(?P<failures>\d+))?(, )?(errors=(?P<errors>\d+))?(, )?(skipped=(?P<skipped>\d+))?\)$')
7374
PTRN_JAVA_EXCEPTION = re.compile(r'^(?P<exception>com\.oracle\.[^:]*):(?P<message>.*)')
7475
PTRN_MODULE_NOT_FOUND = re.compile(r'.*ModuleNotFound: \'(?P<module>.*)\'\..*', re.DOTALL)
75-
76+
PTRN_IMPORT_ERROR = re.compile(r".*cannot import name \'(?P<module>.*)\'.*", re.DOTALL)
7677

7778
# ----------------------------------------------------------------------------------------------------------------------
7879
#
@@ -126,8 +127,11 @@ def scp(results_file_path, destination_path, destination_name=None):
126127
return _run_cmd(cmd)[0]
127128

128129

129-
def _run_unittest(test_path):
130-
cmd = ["mx", "python3", "--python.CatchAllExceptions=true", test_path, "-v"]
130+
def _run_unittest(test_path, with_cpython=False):
131+
if with_cpython:
132+
cmd = ["python3", test_path, "-v"]
133+
else:
134+
cmd = ["mx", "python3", "--python.CatchAllExceptions=true", test_path, "-v"]
131135
success, output = _run_cmd(cmd)
132136
output = '''
133137
##############################################################
@@ -139,7 +143,7 @@ def _run_unittest(test_path):
139143
TIMEOUT = 60 * 20 # 20 mins per unittest wait time max ...
140144

141145

142-
def run_unittests(unittests, timeout):
146+
def run_unittests(unittests, timeout, with_cpython=False):
143147
assert isinstance(unittests, (list, tuple))
144148
num_unittests = len(unittests)
145149
log("[EXEC] running {} unittests ... ", num_unittests)
@@ -148,7 +152,7 @@ def run_unittests(unittests, timeout):
148152

149153
pool = Pool()
150154
for ut in unittests:
151-
results.append(pool.apply_async(_run_unittest, args=(ut, )))
155+
results.append(pool.apply_async(_run_unittest, args=(ut, with_cpython)))
152156
pool.close()
153157

154158
log("[INFO] collect results ... ")
@@ -163,11 +167,12 @@ def run_unittests(unittests, timeout):
163167
timed_out.append(unittests[i])
164168
log("[PROGRESS] {} / {}: \t {}%", i+1, num_unittests, int(((i+1) * 100.0) / num_unittests))
165169

166-
log("".join(['-' for i in range(120)]))
167-
for t in timed_out:
168-
log("[TIMEOUT] skipped: {}", t)
169-
log("".join(['-' for i in range(120)]))
170-
log("[STATS] processed {} out of {} unitttests", num_unittests - len(timed_out), num_unittests)
170+
if timed_out:
171+
log(HR)
172+
for t in timed_out:
173+
log("[TIMEOUT] skipped: {}", t)
174+
log(HR)
175+
log("[STATS] processed {} out of {} unittests", num_unittests - len(timed_out), num_unittests)
171176
pool.terminate()
172177
pool.join()
173178
return out
@@ -279,11 +284,14 @@ def process_output(output_lines):
279284
# python error processing
280285
#
281286
# ----------------------------------------------------------------------------------------------------------------------
282-
def process_errors(unittests, error_messages, error=None, msg_processor=None):
287+
def process_errors(unittests, error_messages, err=None, msg_processor=None):
288+
if isinstance(err, str):
289+
err = {err,}
290+
283291
def _err_filter(item):
284-
if not error:
292+
if not err:
285293
return True
286-
return item[0] == error
294+
return item[0] in err
287295

288296
def _processor(msg):
289297
if not msg_processor:
@@ -304,6 +312,11 @@ def get_missing_module(msg):
304312
return match.group('module') if match else None
305313

306314

315+
def get_cannot_import_module(msg):
316+
match = re.match(PTRN_IMPORT_ERROR, msg)
317+
return match.group('module') if match else None
318+
319+
307320
# ----------------------------------------------------------------------------------------------------------------------
308321
#
309322
# csv reporting
@@ -317,6 +330,11 @@ class Col(object):
317330
NUM_SKIPPED = 'num_skipped'
318331
NUM_PASSES = 'num_passes'
319332
PYTHON_ERRORS = 'python_errors'
333+
CPY_NUM_TESTS = 'cpy_num_tests'
334+
CPY_NUM_FAILS = 'cpy_num_fails'
335+
CPY_NUM_ERRORS = 'cpy_num_errors'
336+
CPY_NUM_SKIPPED = 'cpy_num_skipped'
337+
CPY_NUM_PASSES = 'cpy_num_passes'
320338

321339

322340
CSV_HEADER = [
@@ -326,6 +344,11 @@ class Col(object):
326344
Col.NUM_ERRORS,
327345
Col.NUM_SKIPPED,
328346
Col.NUM_PASSES,
347+
Col.CPY_NUM_TESTS,
348+
Col.CPY_NUM_FAILS,
349+
Col.CPY_NUM_ERRORS,
350+
Col.CPY_NUM_SKIPPED,
351+
Col.CPY_NUM_PASSES,
329352
Col.PYTHON_ERRORS
330353
]
331354

@@ -350,7 +373,7 @@ def save_as_txt(report_path, results):
350373
return output
351374

352375

353-
def save_as_csv(report_path, unittests, error_messages, java_exceptions, stats):
376+
def save_as_csv(report_path, unittests, error_messages, java_exceptions, stats, cpy_stats=None):
354377
rows = []
355378
with open(report_path, 'w') as CSV:
356379
totals = {
@@ -365,18 +388,27 @@ def save_as_csv(report_path, unittests, error_messages, java_exceptions, stats):
365388

366389
for unittest in unittests:
367390
unittest_stats = stats[unittest]
391+
cpy_unittest_stats = cpy_stats[unittest] if cpy_stats else None
368392
unittest_errmsg = error_messages[unittest]
369393
if not unittest_errmsg:
370394
unittest_errmsg = java_exceptions[unittest]
371395

372396
rows.append({
373397
Col.UNITTEST: unittest,
398+
# graalpython stats
374399
Col.NUM_TESTS: unittest_stats.num_tests,
375400
Col.NUM_FAILS: unittest_stats.num_fails,
376401
Col.NUM_ERRORS: unittest_stats.num_errors,
377402
Col.NUM_SKIPPED: unittest_stats.num_skipped,
378403
Col.NUM_PASSES: unittest_stats.num_passes,
379-
Col.PYTHON_ERRORS: dumps(list(unittest_errmsg))
404+
# cpython stats
405+
Col.CPY_NUM_TESTS: cpy_unittest_stats.num_tests if cpy_unittest_stats else None,
406+
Col.CPY_NUM_FAILS: cpy_unittest_stats.num_fails if cpy_unittest_stats else None,
407+
Col.CPY_NUM_ERRORS: cpy_unittest_stats.num_errors if cpy_unittest_stats else None,
408+
Col.CPY_NUM_SKIPPED: cpy_unittest_stats.num_skipped if cpy_unittest_stats else None,
409+
Col.CPY_NUM_PASSES: cpy_unittest_stats.num_passes if cpy_unittest_stats else None,
410+
# errors
411+
Col.PYTHON_ERRORS: dumps(list(unittest_errmsg)),
380412
})
381413

382414
# update totals that ran in some way
@@ -499,7 +531,7 @@ def save_as_csv(report_path, unittests, error_messages, java_exceptions, stats):
499531
'''
500532

501533

502-
def save_as_html(report_name, rows, totals, missing_modules, java_issues, current_date):
534+
def save_as_html(report_name, rows, totals, missing_modules, cannot_import_modules, java_issues, current_date):
503535
def grid(*components):
504536
def _fmt(cmp):
505537
if isinstance(cmp, tuple):
@@ -601,6 +633,11 @@ def format_val(row, k):
601633
for cnt, name in sorted(((cnt, name) for name, cnt in missing_modules.items()), reverse=True)
602634
])
603635

636+
cannot_import_modules_info = ul('modules which could not be imported', [
637+
'<b>{}</b>&nbsp;<span class="text-muted">could not be imported by {} unittests</span>'.format(name, cnt)
638+
for cnt, name in sorted(((cnt, name) for name, cnt in cannot_import_modules.items()), reverse=True)
639+
])
640+
604641
java_issues_info = ul('Java issues', [
605642
'<b>{}</b>&nbsp;<span class="text-muted">caused by {} unittests</span>'.format(name, cnt)
606643
for cnt, name in sorted(((cnt, name) for name, cnt in java_issues.items()), reverse=True)
@@ -619,7 +656,10 @@ def format_val(row, k):
619656

620657
table_stats = table('stats', CSV_HEADER, rows)
621658

622-
content = ' <br> '.join([total_stats_info, table_stats, missing_modules_info, java_issues_info])
659+
content = ' <br> '.join([total_stats_info, table_stats,
660+
missing_modules_info,
661+
cannot_import_modules_info,
662+
java_issues_info])
623663

624664
report = HTML_TEMPLATE.format(
625665
title='GraalPython Unittests Stats',
@@ -679,24 +719,37 @@ def _fmt(t):
679719
skip_tests = set([_fmt(test) for test in flags.skip_tests.split(",")]) if flags.skip_tests else None
680720
unittests = get_unittests(flags.tests_path, limit=flags.limit, skip_tests=skip_tests)
681721

682-
results = run_unittests(unittests, flags.timeout)
722+
# get cpython stats
723+
log(HR)
724+
log("[INFO] get cpython stats")
725+
cpy_results = run_unittests(unittests, 60 * 5, with_cpython=True)
726+
cpy_stats = process_output('\n'.join(cpy_results))[-1]
727+
728+
# get graalpython stats
729+
log(HR)
730+
log("[INFO] get graalpython stats")
731+
results = run_unittests(unittests, flags.timeout, with_cpython=False)
683732
txt_report_path = file_name(TXT_RESULTS_NAME, current_date)
684733
output = save_as_txt(txt_report_path, results)
685734

686735
unittests, error_messages, java_exceptions, stats = process_output(output)
687736

688737
csv_report_path = file_name(CSV_RESULTS_NAME, current_date)
689-
rows, totals = save_as_csv(csv_report_path, unittests, error_messages, java_exceptions, stats)
738+
rows, totals = save_as_csv(csv_report_path, unittests, error_messages, java_exceptions, stats, cpy_stats=cpy_stats)
690739

691-
missing_modules = process_errors(unittests, error_messages, error='ModuleNotFoundError',
740+
missing_modules = process_errors(unittests, error_messages, 'ModuleNotFoundError',
692741
msg_processor=get_missing_module)
693742
log("[MISSING MODULES] \n{}", pformat(dict(missing_modules)))
694743

744+
cannot_import_modules = process_errors(unittests, error_messages, err='ImportError',
745+
msg_processor=get_cannot_import_module)
746+
log("[CANNOT IMPORT MODULES] \n{}", pformat(dict(cannot_import_modules)))
747+
695748
java_issues = process_errors(unittests, java_exceptions)
696749
log("[JAVA ISSUES] \n{}", pformat(dict(java_issues)))
697750

698751
html_report_path = file_name(HTML_RESULTS_NAME, current_date)
699-
save_as_html(html_report_path, rows, totals, missing_modules, java_issues, current_date)
752+
save_as_html(html_report_path, rows, totals, missing_modules, cannot_import_modules, java_issues, current_date)
700753

701754
if flags.path:
702755
log("[SAVE] saving results to {} ... ", flags.path)

‎mx.graalpython/mx_graalpython_benchmark.py

Lines changed: 33 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -122,52 +122,60 @@ def run(self, cwd, args):
122122
return ret_code, out.data
123123

124124

125-
class CPythonVm(AbstractPythonVm):
126-
PYTHON_INTERPRETER = "python3"
127-
128-
def __init__(self, config_name, options=None, virtualenv=None, no_warmup=False):
129-
super(CPythonVm, self).__init__(config_name, options)
130-
self._virtualenv = virtualenv
131-
self._no_warmup = no_warmup
132-
133-
@property
134-
def interpreter(self):
135-
if self._virtualenv:
136-
return os.path.join(self._virtualenv, CPythonVm.PYTHON_INTERPRETER)
137-
return CPythonVm.PYTHON_INTERPRETER
125+
class AbstractPythonIterationsControlVm(AbstractPythonVm):
126+
__metaclass__ = ABCMeta
138127

139-
def name(self):
140-
return VM_NAME_CPYTHON
128+
def __init__(self, config_name, options=None, iterations=None):
129+
super(AbstractPythonIterationsControlVm, self).__init__(config_name, options)
130+
self._iterations = iterations
141131

142-
@staticmethod
143-
def remove_warmup_runs(args):
132+
def _override_iterations_args(self, args):
144133
_args = []
145134
i = 0
146135
while i < len(args):
147136
arg = args[i]
148137
_args.append(arg)
149138
if arg == '-i':
150-
_args.append('0')
139+
_args.append(str(self._iterations))
151140
i += 1
152141
i += 1
153142
return _args
154143

155144
def run(self, cwd, args):
156-
if self._no_warmup:
157-
args = CPythonVm.remove_warmup_runs(args)
158-
return super(CPythonVm, self).run(cwd, args)
145+
if isinstance(self._iterations, (int, long)):
146+
args = self._override_iterations_args(args)
147+
return super(AbstractPythonIterationsControlVm, self).run(cwd, args)
159148

160149

161-
class PyPyVm(AbstractPythonVm):
162-
def __init__(self, config_name, options=None):
163-
super(PyPyVm, self).__init__(config_name, options=options)
150+
class CPythonVm(AbstractPythonIterationsControlVm):
151+
PYTHON_INTERPRETER = "python3"
152+
153+
def __init__(self, config_name, options=None, virtualenv=None, iterations=False):
154+
super(CPythonVm, self).__init__(config_name, options=options, iterations=iterations)
155+
self._virtualenv = virtualenv
156+
157+
@property
158+
def interpreter(self):
159+
if self._virtualenv:
160+
return os.path.join(self._virtualenv, CPythonVm.PYTHON_INTERPRETER)
161+
return CPythonVm.PYTHON_INTERPRETER
162+
163+
def name(self):
164+
return VM_NAME_CPYTHON
165+
166+
167+
class PyPyVm(AbstractPythonIterationsControlVm):
168+
PYPY_INTERPRETER = "pypy3"
169+
170+
def __init__(self, config_name, options=None, iterations=False):
171+
super(PyPyVm, self).__init__(config_name, options=options, iterations=iterations)
164172

165173
@property
166174
def interpreter(self):
167175
home = mx.get_env(ENV_PYPY_HOME)
168176
if not home:
169177
mx.abort("{} is not set!".format(ENV_PYPY_HOME))
170-
return join(home, 'bin', 'pypy3')
178+
return join(home, 'bin', PyPyVm.PYPY_INTERPRETER)
171179

172180
def name(self):
173181
return VM_NAME_PYPY

0 commit comments

Comments
 (0)