1
- # Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
1
+ # Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
2
2
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
3
3
#
4
4
# The Universal Permissive License (UPL), Version 1.0
74
74
PTRN_JAVA_EXCEPTION = re .compile (r'^(?P<exception>com\.oracle\.[^:]*):(?P<message>.*)' )
75
75
PTRN_MODULE_NOT_FOUND = re .compile (r'.*ModuleNotFound: \'(?P<module>.*)\'\..*' , re .DOTALL )
76
76
PTRN_IMPORT_ERROR = re .compile (r".*cannot import name \'(?P<module>.*)\'.*" , re .DOTALL )
77
+ PTRN_REMOTE_HOST = re .compile (r"(?P<user>\w+)@(?P<host>[\w.]+):(?P<path>.+)" )
78
+ PTRN_VALID_CSV_NAME = re .compile (r"unittests-\d{4}-\d{2}-\d{2}.csv" )
79
+
77
80
78
81
# ----------------------------------------------------------------------------------------------------------------------
79
82
#
@@ -192,6 +195,27 @@ def _sorter(iterable):
192
195
return unittests [:limit ] if limit else unittests
193
196
194
197
198
+ def get_remote_host (scp_path ):
199
+ match = re .match (PTRN_REMOTE_HOST , scp_path )
200
+ return match .group ('user' ), match .group ('host' ), match .group ('path' )
201
+
202
+
203
+ def ssh_ls (scp_path ):
204
+ user , host , path = get_remote_host (scp_path )
205
+ cmd = ['ssh' , '{}@{}' .format (user , host ), 'ls' , '-l' , path ]
206
+ return map (lambda l : l .split ()[- 1 ], _run_cmd (cmd )[1 ].splitlines ())
207
+
208
+
209
+ def read_csv (path ):
210
+ rows = []
211
+ with open (path , "r" ) as CSV_FILE :
212
+ reader = csv .reader (CSV_FILE )
213
+ headers = next (reader )[1 :]
214
+ for row in reader :
215
+ rows .append (row )
216
+ return rows
217
+
218
+
195
219
# ----------------------------------------------------------------------------------------------------------------------
196
220
#
197
221
# result (output processing)
@@ -597,7 +621,7 @@ def format_val(row, k):
597
621
elif k == Col .NUM_TESTS :
598
622
_class = "text-dark"
599
623
else :
600
- _class = "text-danger" if value < 0 else "text-muted"
624
+ _class = "text-danger" if value and value < 0 else "text-muted"
601
625
return '<span class="{} h6"><b>{}</b></span>' .format (_class , value )
602
626
603
627
_tbody = '\n ' .join ([
@@ -694,6 +718,10 @@ def main(prog, args):
694
718
parser .add_argument ("-o" , "--only_tests" , help = "Run only these unittests (comma sep values)." , default = None )
695
719
parser .add_argument ("-s" , "--skip_tests" , help = "Run all unittests except (comma sep values)."
696
720
"the only_tets option takes precedence" , default = None )
721
+ parser .add_argument ("-r" , "--regression_running_tests" , help = "Regression threshold for running tests." , type = float ,
722
+ default = None )
723
+ parser .add_argument ("-g" , "--gate" , help = "Run in gate mode (Skip cpython runs; Do not upload results; "
724
+ "Detect regressions)." , action = "store_true" )
697
725
parser .add_argument ("path" , help = "Path to store the csv output and logs to." , nargs = '?' , default = None )
698
726
699
727
global flags
@@ -708,6 +736,12 @@ def main(prog, args):
708
736
else :
709
737
log ("[INFO] results will not be saved remotely" )
710
738
739
+ if flags .gate :
740
+ log ("[INFO] running in gate mode" )
741
+ if not flags .regression_running_tests :
742
+ log ("[WARNING] --regression_running_tests not set while in gate mode. "
743
+ "Regression detection will not be performed" )
744
+
711
745
def _fmt (t ):
712
746
t = t .strip ()
713
747
return os .path .join (flags .tests_path , t if t .endswith (".py" ) else t + ".py" )
@@ -720,15 +754,22 @@ def _fmt(t):
720
754
unittests = get_unittests (flags .tests_path , limit = flags .limit , skip_tests = skip_tests )
721
755
722
756
# get cpython stats
723
- log (HR )
724
- log ("[INFO] get cpython stats" )
725
- cpy_results = run_unittests (unittests , 60 * 5 , with_cpython = True )
726
- cpy_stats = process_output ('\n ' .join (cpy_results ))[- 1 ]
757
+ if not flags .gate :
758
+ log (HR )
759
+ log ("[INFO] get cpython stats" )
760
+ cpy_results = run_unittests (unittests , 60 * 5 , with_cpython = True )
761
+ cpy_stats = process_output ('\n ' .join (cpy_results ))[- 1 ]
762
+ # handle the timeout
763
+ timeout = flags .timeout if flags .timeout else None
764
+ else :
765
+ cpy_stats = None
766
+ # handle the timeout
767
+ timeout = flags .timeout if flags .timeout else 60 * 5 # 5 minutes if no value specified (in gate mode only)
727
768
728
769
# get graalpython stats
729
770
log (HR )
730
771
log ("[INFO] get graalpython stats" )
731
- results = run_unittests (unittests , flags . timeout , with_cpython = False )
772
+ results = run_unittests (unittests , timeout , with_cpython = False )
732
773
txt_report_path = file_name (TXT_RESULTS_NAME , current_date )
733
774
output = save_as_txt (txt_report_path , results )
734
775
@@ -749,15 +790,44 @@ def _fmt(t):
749
790
log ("[JAVA ISSUES] \n {}" , pformat (dict (java_issues )))
750
791
751
792
html_report_path = file_name (HTML_RESULTS_NAME , current_date )
752
- save_as_html (html_report_path , rows , totals , missing_modules , cannot_import_modules , java_issues , current_date )
793
+ if not flags .gate :
794
+ save_as_html (html_report_path , rows , totals , missing_modules , cannot_import_modules , java_issues , current_date )
753
795
754
- if flags .path :
796
+ if not flags . gate and flags .path :
755
797
log ("[SAVE] saving results to {} ... " , flags .path )
756
798
scp (txt_report_path , flags .path )
757
799
scp (csv_report_path , flags .path )
758
800
scp (html_report_path , flags .path )
759
801
802
+ gate_failed = False
803
+ if flags .gate and flags .regression_running_tests :
804
+ log ("[REGRESSION] detecting regression, acceptance threshold = {}%" .format (
805
+ flags .regression_running_tests * 100 ))
806
+ csv_files = list (filter (lambda entry : True if PTRN_VALID_CSV_NAME .match (entry ) else False , ssh_ls (flags .path )))
807
+ last_csv = csv_files [- 1 ]
808
+ # log('\n'.join(csv_files))
809
+ # read the remote csv and extract stats
810
+ log ("[REGRESSION] comparing against: {}" .format (last_csv ))
811
+ scp ('{}/{}' .format (flags .path , last_csv ), '.' , destination_name = last_csv )
812
+ rows = read_csv (last_csv )
813
+ prev_totals = {
814
+ Col .NUM_TESTS : int (rows [- 1 ][1 ]),
815
+ Col .NUM_FAILS : int (rows [- 1 ][2 ]),
816
+ Col .NUM_ERRORS : int (rows [- 1 ][3 ]),
817
+ Col .NUM_SKIPPED : int (rows [- 1 ][4 ]),
818
+ Col .NUM_PASSES : int (rows [- 1 ][5 ]),
819
+ }
820
+ print (prev_totals )
821
+ if float (totals [Col .NUM_TESTS ]) < float (prev_totals [Col .NUM_TESTS ]) * (1.0 - flags .regression_running_tests ):
822
+ log ("[REGRESSION] REGRESSION DETECTED, passed {} tests vs {} from {}" .format (
823
+ totals [Col .NUM_TESTS ], prev_totals [Col .NUM_TESTS ], last_csv ))
824
+ gate_failed = True
825
+ else :
826
+ log ("[REGRESSION] no regression detected" )
827
+
760
828
log ("[DONE]" )
829
+ if flags .gate and gate_failed :
830
+ exit (1 )
761
831
762
832
763
833
if __name__ == "__main__" :
0 commit comments