[GR-14051] gate unittests regressions

PullRequest: graalpython/417

Author: cosminbasca

ci.jsonnet

@@ -1,5 +1,5 @@
 {
-  overlay: "e08ca6c848943a04c5e46d54272ff927ff3d51f7",
+  overlay: "d20cc2abdeb3cfb022e1a8035e40e350e5cfe5fc",
 
   // ======================================================================================================
   // 

graalpython/com.oracle.graal.python.test/src/python_unittests.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # The Universal Permissive License (UPL), Version 1.0
@@ -74,6 +74,9 @@
 PTRN_JAVA_EXCEPTION = re.compile(r'^(?P<exception>com\.oracle\.[^:]*):(?P<message>.*)')
 PTRN_MODULE_NOT_FOUND = re.compile(r'.*ModuleNotFound: \'(?P<module>.*)\'\..*', re.DOTALL)
 PTRN_IMPORT_ERROR = re.compile(r".*cannot import name \'(?P<module>.*)\'.*", re.DOTALL)
+PTRN_REMOTE_HOST = re.compile(r"(?P<user>\w+)@(?P<host>[\w.]+):(?P<path>.+)")
+PTRN_VALID_CSV_NAME = re.compile(r"unittests-\d{4}-\d{2}-\d{2}.csv")
+
 
 # ----------------------------------------------------------------------------------------------------------------------
 #
@@ -192,6 +195,27 @@ def _sorter(iterable):
     return unittests[:limit] if limit else unittests
 
 
+def get_remote_host(scp_path):
+    match = re.match(PTRN_REMOTE_HOST, scp_path)
+    return match.group('user'), match.group('host'), match.group('path')
+
+
+def ssh_ls(scp_path):
+    user, host, path = get_remote_host(scp_path)
+    cmd = ['ssh', '{}@{}'.format(user, host), 'ls', '-l', path]
+    return map(lambda l: l.split()[-1], _run_cmd(cmd)[1].splitlines())
+
+
+def read_csv(path):
+    rows = []
+    with open(path, "r") as CSV_FILE:
+        reader = csv.reader(CSV_FILE)
+        headers = next(reader)[1:]
+        for row in reader:
+            rows.append(row)
+    return rows
+
+
 # ----------------------------------------------------------------------------------------------------------------------
 #
 # result (output processing)
@@ -597,7 +621,7 @@ def format_val(row, k):
             elif k == Col.NUM_TESTS:
                 _class = "text-dark"
             else:
-                _class = "text-danger" if value < 0 else "text-muted"
+                _class = "text-danger" if value and value < 0 else "text-muted"
             return '<span class="{} h6"><b>{}</b></span>'.format(_class, value)
 
         _tbody = '\n'.join([
@@ -694,6 +718,10 @@ def main(prog, args):
     parser.add_argument("-o", "--only_tests", help="Run only these unittests (comma sep values).", default=None)
     parser.add_argument("-s", "--skip_tests", help="Run all unittests except (comma sep values)."
                                                    "the only_tets option takes precedence", default=None)
+    parser.add_argument("-r", "--regression_running_tests", help="Regression threshold for running tests.", type=float,
+                        default=None)
+    parser.add_argument("-g", "--gate", help="Run in gate mode (Skip cpython runs; Do not upload results; "
+                                             "Detect regressions).", action="store_true")
     parser.add_argument("path", help="Path to store the csv output and logs to.", nargs='?', default=None)
 
     global flags
@@ -708,6 +736,12 @@ def main(prog, args):
     else:
         log("[INFO] results will not be saved remotely")
 
+    if flags.gate:
+        log("[INFO] running in gate mode")
+        if not flags.regression_running_tests:
+            log("[WARNING] --regression_running_tests not set while in gate mode. "
+                "Regression detection will not be performed")
+
     def _fmt(t):
         t = t.strip()
         return os.path.join(flags.tests_path, t if t.endswith(".py") else t + ".py")
@@ -720,15 +754,22 @@ def _fmt(t):
         unittests = get_unittests(flags.tests_path, limit=flags.limit, skip_tests=skip_tests)
 
     # get cpython stats
-    log(HR)
-    log("[INFO] get cpython stats")
-    cpy_results = run_unittests(unittests, 60 * 5, with_cpython=True)
-    cpy_stats = process_output('\n'.join(cpy_results))[-1]
+    if not flags.gate:
+        log(HR)
+        log("[INFO] get cpython stats")
+        cpy_results = run_unittests(unittests, 60 * 5, with_cpython=True)
+        cpy_stats = process_output('\n'.join(cpy_results))[-1]
+        # handle the timeout
+        timeout = flags.timeout if flags.timeout else None
+    else:
+        cpy_stats = None
+        # handle the timeout
+        timeout = flags.timeout if flags.timeout else 60 * 5  # 5 minutes if no value specified (in gate mode only)
 
     # get graalpython stats
     log(HR)
     log("[INFO] get graalpython stats")
-    results = run_unittests(unittests, flags.timeout, with_cpython=False)
+    results = run_unittests(unittests, timeout, with_cpython=False)
     txt_report_path = file_name(TXT_RESULTS_NAME, current_date)
     output = save_as_txt(txt_report_path, results)
 
@@ -749,15 +790,44 @@ def _fmt(t):
     log("[JAVA ISSUES] \n{}", pformat(dict(java_issues)))
 
     html_report_path = file_name(HTML_RESULTS_NAME, current_date)
-    save_as_html(html_report_path, rows, totals, missing_modules, cannot_import_modules, java_issues, current_date)
+    if not flags.gate:
+        save_as_html(html_report_path, rows, totals, missing_modules, cannot_import_modules, java_issues, current_date)
 
-    if flags.path:
+    if not flags.gate and flags.path:
         log("[SAVE] saving results to {} ... ", flags.path)
         scp(txt_report_path, flags.path)
         scp(csv_report_path, flags.path)
         scp(html_report_path, flags.path)
 
+    gate_failed = False
+    if flags.gate and flags.regression_running_tests:
+        log("[REGRESSION] detecting regression, acceptance threshold = {}%".format(
+            flags.regression_running_tests * 100))
+        csv_files = list(filter(lambda entry: True if PTRN_VALID_CSV_NAME.match(entry) else False, ssh_ls(flags.path)))
+        last_csv = csv_files[-1]
+        # log('\n'.join(csv_files))
+        # read the remote csv and extract stats
+        log("[REGRESSION] comparing against: {}".format(last_csv))
+        scp('{}/{}'.format(flags.path, last_csv), '.', destination_name=last_csv)
+        rows = read_csv(last_csv)
+        prev_totals = {
+            Col.NUM_TESTS: int(rows[-1][1]),
+            Col.NUM_FAILS: int(rows[-1][2]),
+            Col.NUM_ERRORS: int(rows[-1][3]),
+            Col.NUM_SKIPPED: int(rows[-1][4]),
+            Col.NUM_PASSES: int(rows[-1][5]),
+        }
+        print(prev_totals)
+        if float(totals[Col.NUM_TESTS]) < float(prev_totals[Col.NUM_TESTS]) * (1.0 - flags.regression_running_tests):
+            log("[REGRESSION] REGRESSION DETECTED, passed {} tests vs {} from {}".format(
+                totals[Col.NUM_TESTS], prev_totals[Col.NUM_TESTS], last_csv))
+            gate_failed = True
+        else:
+            log("[REGRESSION] no regression detected")
+
     log("[DONE]")
+    if flags.gate and gate_failed:
+        exit(1)
 
 
 if __name__ == "__main__":