Skip to content

Commit 7c13bce

Browse files
committed
unittests runner: extract cpython stats as well
1 parent 3f731dc commit 7c13bce

File tree

1 file changed

+47
-14
lines changed

1 file changed

+47
-14
lines changed

graalpython/com.oracle.graal.python.test/src/python_unittests.py

Lines changed: 47 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -62,9 +62,10 @@
6262
CSV_RESULTS_NAME = "{}.csv".format(_BASE_NAME)
6363
HTML_RESULTS_NAME = "{}.html".format(_BASE_NAME)
6464

65+
HR = "".join(['-' for _ in range(120)])
6566

6667
PTRN_ERROR = re.compile(r'^(?P<error>[A-Z][a-z][a-zA-Z]+):(?P<message>.*)$')
67-
PTRN_UNITTEST = re.compile(r'^#### running: graalpython/lib-python/3/test/(?P<unittest>.*)$')
68+
PTRN_UNITTEST = re.compile(r'^#### running: graalpython/lib-python/3/test/(?P<unittest>[\w.]+).*$', re.DOTALL)
6869
PTRN_NUM_TESTS = re.compile(r'^Ran (?P<num_tests>\d+) test.*$')
6970
PTRN_FAILED = re.compile(
7071
r'^FAILED \((failures=(?P<failures>\d+))?(, )?(errors=(?P<errors>\d+))?(, )?(skipped=(?P<skipped>\d+))?\)$')
@@ -126,8 +127,11 @@ def scp(results_file_path, destination_path, destination_name=None):
126127
return _run_cmd(cmd)[0]
127128

128129

129-
def _run_unittest(test_path):
130-
cmd = ["mx", "python3", "--python.CatchAllExceptions=true", test_path, "-v"]
130+
def _run_unittest(test_path, with_cpython=False):
131+
if with_cpython:
132+
cmd = ["python3", test_path, "-v"]
133+
else:
134+
cmd = ["mx", "python3", "--python.CatchAllExceptions=true", test_path, "-v"]
131135
success, output = _run_cmd(cmd)
132136
output = '''
133137
##############################################################
@@ -139,7 +143,7 @@ def _run_unittest(test_path):
139143
TIMEOUT = 60 * 20 # 20 mins per unittest wait time max ...
140144

141145

142-
def run_unittests(unittests, timeout):
146+
def run_unittests(unittests, timeout, with_cpython=False):
143147
assert isinstance(unittests, (list, tuple))
144148
num_unittests = len(unittests)
145149
log("[EXEC] running {} unittests ... ", num_unittests)
@@ -148,7 +152,7 @@ def run_unittests(unittests, timeout):
148152

149153
pool = Pool()
150154
for ut in unittests:
151-
results.append(pool.apply_async(_run_unittest, args=(ut, )))
155+
results.append(pool.apply_async(_run_unittest, args=(ut, with_cpython)))
152156
pool.close()
153157

154158
log("[INFO] collect results ... ")
@@ -163,11 +167,12 @@ def run_unittests(unittests, timeout):
163167
timed_out.append(unittests[i])
164168
log("[PROGRESS] {} / {}: \t {}%", i+1, num_unittests, int(((i+1) * 100.0) / num_unittests))
165169

166-
log("".join(['-' for i in range(120)]))
167-
for t in timed_out:
168-
log("[TIMEOUT] skipped: {}", t)
169-
log("".join(['-' for i in range(120)]))
170-
log("[STATS] processed {} out of {} unitttests", num_unittests - len(timed_out), num_unittests)
170+
if timed_out:
171+
log(HR)
172+
for t in timed_out:
173+
log("[TIMEOUT] skipped: {}", t)
174+
log(HR)
175+
log("[STATS] processed {} out of {} unittests", num_unittests - len(timed_out), num_unittests)
171176
pool.terminate()
172177
pool.join()
173178
return out
@@ -325,6 +330,11 @@ class Col(object):
325330
NUM_SKIPPED = 'num_skipped'
326331
NUM_PASSES = 'num_passes'
327332
PYTHON_ERRORS = 'python_errors'
333+
CPY_NUM_TESTS = 'cpy_num_tests'
334+
CPY_NUM_FAILS = 'cpy_num_fails'
335+
CPY_NUM_ERRORS = 'cpy_num_errors'
336+
CPY_NUM_SKIPPED = 'cpy_num_skipped'
337+
CPY_NUM_PASSES = 'cpy_num_passes'
328338

329339

330340
CSV_HEADER = [
@@ -334,6 +344,11 @@ class Col(object):
334344
Col.NUM_ERRORS,
335345
Col.NUM_SKIPPED,
336346
Col.NUM_PASSES,
347+
Col.CPY_NUM_TESTS,
348+
Col.CPY_NUM_FAILS,
349+
Col.CPY_NUM_ERRORS,
350+
Col.CPY_NUM_SKIPPED,
351+
Col.CPY_NUM_PASSES,
337352
Col.PYTHON_ERRORS
338353
]
339354

@@ -358,7 +373,7 @@ def save_as_txt(report_path, results):
358373
return output
359374

360375

361-
def save_as_csv(report_path, unittests, error_messages, java_exceptions, stats):
376+
def save_as_csv(report_path, unittests, error_messages, java_exceptions, stats, cpy_stats=None):
362377
rows = []
363378
with open(report_path, 'w') as CSV:
364379
totals = {
@@ -373,18 +388,27 @@ def save_as_csv(report_path, unittests, error_messages, java_exceptions, stats):
373388

374389
for unittest in unittests:
375390
unittest_stats = stats[unittest]
391+
cpy_unittest_stats = cpy_stats[unittest] if cpy_stats else None
376392
unittest_errmsg = error_messages[unittest]
377393
if not unittest_errmsg:
378394
unittest_errmsg = java_exceptions[unittest]
379395

380396
rows.append({
381397
Col.UNITTEST: unittest,
398+
# graalpython stats
382399
Col.NUM_TESTS: unittest_stats.num_tests,
383400
Col.NUM_FAILS: unittest_stats.num_fails,
384401
Col.NUM_ERRORS: unittest_stats.num_errors,
385402
Col.NUM_SKIPPED: unittest_stats.num_skipped,
386403
Col.NUM_PASSES: unittest_stats.num_passes,
387-
Col.PYTHON_ERRORS: dumps(list(unittest_errmsg))
404+
# cpython stats
405+
Col.CPY_NUM_TESTS: cpy_unittest_stats.num_tests if cpy_unittest_stats else None,
406+
Col.CPY_NUM_FAILS: cpy_unittest_stats.num_fails if cpy_unittest_stats else None,
407+
Col.CPY_NUM_ERRORS: cpy_unittest_stats.num_errors if cpy_unittest_stats else None,
408+
Col.CPY_NUM_SKIPPED: cpy_unittest_stats.num_skipped if cpy_unittest_stats else None,
409+
Col.CPY_NUM_PASSES: cpy_unittest_stats.num_passes if cpy_unittest_stats else None,
410+
# errors
411+
Col.PYTHON_ERRORS: dumps(list(unittest_errmsg)),
388412
})
389413

390414
# update totals that ran in some way
@@ -695,14 +719,23 @@ def _fmt(t):
695719
skip_tests = set([_fmt(test) for test in flags.skip_tests.split(",")]) if flags.skip_tests else None
696720
unittests = get_unittests(flags.tests_path, limit=flags.limit, skip_tests=skip_tests)
697721

698-
results = run_unittests(unittests, flags.timeout)
722+
# get cpython stats
723+
log(HR)
724+
log("[INFO] get cpython stats")
725+
cpy_results = run_unittests(unittests, flags.timeout, with_cpython=True)
726+
cpy_stats = process_output('\n'.join(cpy_results))[-1]
727+
728+
# get graalpython stats
729+
log(HR)
730+
log("[INFO] get graalpython stats")
731+
results = run_unittests(unittests, flags.timeout, with_cpython=False)
699732
txt_report_path = file_name(TXT_RESULTS_NAME, current_date)
700733
output = save_as_txt(txt_report_path, results)
701734

702735
unittests, error_messages, java_exceptions, stats = process_output(output)
703736

704737
csv_report_path = file_name(CSV_RESULTS_NAME, current_date)
705-
rows, totals = save_as_csv(csv_report_path, unittests, error_messages, java_exceptions, stats)
738+
rows, totals = save_as_csv(csv_report_path, unittests, error_messages, java_exceptions, stats, cpy_stats=cpy_stats)
706739

707740
missing_modules = process_errors(unittests, error_messages, 'ModuleNotFoundError',
708741
msg_processor=get_missing_module)

0 commit comments

Comments
 (0)