Skip to content

Commit dbc51fc

Browse files
committed
Merge remote-tracking branch 'origin/master' into hotfix/null-python-home
2 parents 563a414 + 9a00643 commit dbc51fc

File tree

9 files changed

+128
-22
lines changed

9 files changed

+128
-22
lines changed

CHANGELOG.md

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,15 @@
33
This changelog summarizes major changes between GraalVM versions of the Python
44
language runtime. The main focus is on user-observable behavior of the engine.
55

6+
## Version 1.0.0 RC13
7+
8+
* Support marshal.dumps and marshal.loads for code objects and some other built-in objects
9+
* Fix installation of NumPy in a venv
10+
* Initial support for module mmap
11+
* Support debugging with workspace files in the Chrome debugger
12+
* Support the PEP 553 breakpoint() message
13+
* Support running weak reference callbacks and signals on the main thread
14+
615
## Version 1.0.0 RC12
716

817
* Support the `__class__` variable in the class scope

ci.jsonnet

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
overlay: "e08ca6c848943a04c5e46d54272ff927ff3d51f7",
2+
overlay: "d20cc2abdeb3cfb022e1a8035e40e350e5cfe5fc",
33

44
// ======================================================================================================
55
//

graalpython/com.oracle.graal.python.test/src/python_unittests.py

Lines changed: 79 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
1+
# Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
22
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
33
#
44
# The Universal Permissive License (UPL), Version 1.0
@@ -74,6 +74,9 @@
7474
PTRN_JAVA_EXCEPTION = re.compile(r'^(?P<exception>com\.oracle\.[^:]*):(?P<message>.*)')
7575
PTRN_MODULE_NOT_FOUND = re.compile(r'.*ModuleNotFound: \'(?P<module>.*)\'\..*', re.DOTALL)
7676
PTRN_IMPORT_ERROR = re.compile(r".*cannot import name \'(?P<module>.*)\'.*", re.DOTALL)
77+
PTRN_REMOTE_HOST = re.compile(r"(?P<user>\w+)@(?P<host>[\w.]+):(?P<path>.+)")
78+
PTRN_VALID_CSV_NAME = re.compile(r"unittests-\d{4}-\d{2}-\d{2}.csv")
79+
7780

7881
# ----------------------------------------------------------------------------------------------------------------------
7982
#
@@ -192,6 +195,27 @@ def _sorter(iterable):
192195
return unittests[:limit] if limit else unittests
193196

194197

198+
def get_remote_host(scp_path):
199+
match = re.match(PTRN_REMOTE_HOST, scp_path)
200+
return match.group('user'), match.group('host'), match.group('path')
201+
202+
203+
def ssh_ls(scp_path):
204+
user, host, path = get_remote_host(scp_path)
205+
cmd = ['ssh', '{}@{}'.format(user, host), 'ls', '-l', path]
206+
return map(lambda l: l.split()[-1], _run_cmd(cmd)[1].splitlines())
207+
208+
209+
def read_csv(path):
210+
rows = []
211+
with open(path, "r") as CSV_FILE:
212+
reader = csv.reader(CSV_FILE)
213+
headers = next(reader)[1:]
214+
for row in reader:
215+
rows.append(row)
216+
return rows
217+
218+
195219
# ----------------------------------------------------------------------------------------------------------------------
196220
#
197221
# result (output processing)
@@ -597,7 +621,7 @@ def format_val(row, k):
597621
elif k == Col.NUM_TESTS:
598622
_class = "text-dark"
599623
else:
600-
_class = "text-danger" if value < 0 else "text-muted"
624+
_class = "text-danger" if value and value < 0 else "text-muted"
601625
return '<span class="{} h6"><b>{}</b></span>'.format(_class, value)
602626

603627
_tbody = '\n'.join([
@@ -694,6 +718,10 @@ def main(prog, args):
694718
parser.add_argument("-o", "--only_tests", help="Run only these unittests (comma sep values).", default=None)
695719
parser.add_argument("-s", "--skip_tests", help="Run all unittests except (comma sep values)."
696720
"the only_tets option takes precedence", default=None)
721+
parser.add_argument("-r", "--regression_running_tests", help="Regression threshold for running tests.", type=float,
722+
default=None)
723+
parser.add_argument("-g", "--gate", help="Run in gate mode (Skip cpython runs; Do not upload results; "
724+
"Detect regressions).", action="store_true")
697725
parser.add_argument("path", help="Path to store the csv output and logs to.", nargs='?', default=None)
698726

699727
global flags
@@ -708,6 +736,12 @@ def main(prog, args):
708736
else:
709737
log("[INFO] results will not be saved remotely")
710738

739+
if flags.gate:
740+
log("[INFO] running in gate mode")
741+
if not flags.regression_running_tests:
742+
log("[WARNING] --regression_running_tests not set while in gate mode. "
743+
"Regression detection will not be performed")
744+
711745
def _fmt(t):
712746
t = t.strip()
713747
return os.path.join(flags.tests_path, t if t.endswith(".py") else t + ".py")
@@ -720,15 +754,22 @@ def _fmt(t):
720754
unittests = get_unittests(flags.tests_path, limit=flags.limit, skip_tests=skip_tests)
721755

722756
# get cpython stats
723-
log(HR)
724-
log("[INFO] get cpython stats")
725-
cpy_results = run_unittests(unittests, 60 * 5, with_cpython=True)
726-
cpy_stats = process_output('\n'.join(cpy_results))[-1]
757+
if not flags.gate:
758+
log(HR)
759+
log("[INFO] get cpython stats")
760+
cpy_results = run_unittests(unittests, 60 * 5, with_cpython=True)
761+
cpy_stats = process_output('\n'.join(cpy_results))[-1]
762+
# handle the timeout
763+
timeout = flags.timeout if flags.timeout else None
764+
else:
765+
cpy_stats = None
766+
# handle the timeout
767+
timeout = flags.timeout if flags.timeout else 60 * 5 # 5 minutes if no value specified (in gate mode only)
727768

728769
# get graalpython stats
729770
log(HR)
730771
log("[INFO] get graalpython stats")
731-
results = run_unittests(unittests, flags.timeout, with_cpython=False)
772+
results = run_unittests(unittests, timeout, with_cpython=False)
732773
txt_report_path = file_name(TXT_RESULTS_NAME, current_date)
733774
output = save_as_txt(txt_report_path, results)
734775

@@ -749,15 +790,44 @@ def _fmt(t):
749790
log("[JAVA ISSUES] \n{}", pformat(dict(java_issues)))
750791

751792
html_report_path = file_name(HTML_RESULTS_NAME, current_date)
752-
save_as_html(html_report_path, rows, totals, missing_modules, cannot_import_modules, java_issues, current_date)
793+
if not flags.gate:
794+
save_as_html(html_report_path, rows, totals, missing_modules, cannot_import_modules, java_issues, current_date)
753795

754-
if flags.path:
796+
if not flags.gate and flags.path:
755797
log("[SAVE] saving results to {} ... ", flags.path)
756798
scp(txt_report_path, flags.path)
757799
scp(csv_report_path, flags.path)
758800
scp(html_report_path, flags.path)
759801

802+
gate_failed = False
803+
if flags.gate and flags.regression_running_tests:
804+
log("[REGRESSION] detecting regression, acceptance threshold = {}%".format(
805+
flags.regression_running_tests * 100))
806+
csv_files = list(filter(lambda entry: True if PTRN_VALID_CSV_NAME.match(entry) else False, ssh_ls(flags.path)))
807+
last_csv = csv_files[-1]
808+
# log('\n'.join(csv_files))
809+
# read the remote csv and extract stats
810+
log("[REGRESSION] comparing against: {}".format(last_csv))
811+
scp('{}/{}'.format(flags.path, last_csv), '.', destination_name=last_csv)
812+
rows = read_csv(last_csv)
813+
prev_totals = {
814+
Col.NUM_TESTS: int(rows[-1][1]),
815+
Col.NUM_FAILS: int(rows[-1][2]),
816+
Col.NUM_ERRORS: int(rows[-1][3]),
817+
Col.NUM_SKIPPED: int(rows[-1][4]),
818+
Col.NUM_PASSES: int(rows[-1][5]),
819+
}
820+
print(prev_totals)
821+
if float(totals[Col.NUM_TESTS]) < float(prev_totals[Col.NUM_TESTS]) * (1.0 - flags.regression_running_tests):
822+
log("[REGRESSION] REGRESSION DETECTED, passed {} tests vs {} from {}".format(
823+
totals[Col.NUM_TESTS], prev_totals[Col.NUM_TESTS], last_csv))
824+
gate_failed = True
825+
else:
826+
log("[REGRESSION] no regression detected")
827+
760828
log("[DONE]")
829+
if flags.gate and gate_failed:
830+
exit(1)
761831

762832

763833
if __name__ == "__main__":
Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
# Copyright (c) 2019, Oracle and/or its affiliates.
2+
# Copyright (C) 1996-2017 Python Software Foundation
3+
#
4+
# Licensed under the PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
5+
6+
import unittest
7+
from itertools import *
8+
9+
class CombinationsTests(unittest.TestCase):
10+
11+
def test_combinations_with_replacement(self):
12+
cwr = combinations_with_replacement
13+
self.assertRaises(TypeError, cwr, 'abc') # missing r argument
14+
self.assertRaises(TypeError, cwr, 'abc', 2, 1) # too many arguments
15+
self.assertRaises(TypeError, cwr, None) # pool is not iterable
16+
self.assertRaises(ValueError, cwr, 'abc', -2) # r is negative
17+
18+
result = list()
19+
for a in cwr('ABC', 2):
20+
result += a
21+
correct = [('A','A'), ('A','B'), ('A','C'), ('B','B'), ('B','C'), ('C','C')]
22+
compare = list();
23+
for a in correct:
24+
compare += a;
25+
self.assertEqual(result,compare)
26+

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/TruffleCextBuiltins.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1430,7 +1430,7 @@ Object doUnpackLong(long start, long stop, long step, long length) {
14301430
}
14311431

14321432
@Specialization(replaces = {"doUnpackLong", "doUnpack"})
1433-
Object doUnpackLongOvf(long start, long stop, long step, long length) {
1433+
Object doUnpackLongOvf0(long start, long stop, long step, long length) {
14341434
try {
14351435
PSlice tmpSlice = factory().createSlice(PInt.intValueExact(start), PInt.intValueExact(stop), PInt.intValueExact(step));
14361436
SliceInfo actualIndices = tmpSlice.computeIndices(length > Integer.MAX_VALUE ? Integer.MAX_VALUE : PInt.intValueExact(length));
@@ -1440,8 +1440,8 @@ Object doUnpackLongOvf(long start, long stop, long step, long length) {
14401440
}
14411441
}
14421442

1443-
@Specialization(replaces = {"doUnpackLongOvf"})
1444-
Object doUnpackLongOvf(Object start, Object stop, Object step, Object lengthObj,
1443+
@Specialization(replaces = {"doUnpackLongOvf0"})
1444+
Object doUnpackLongOvf1(Object start, Object stop, Object step, Object lengthObj,
14451445
@Cached("createOverflow()") CastToIndexNode castToIndexNode,
14461446
@Cached("create()") IsBuiltinClassProfile profile,
14471447
@Cached("create()") SliceLiteralNode sliceLiteralNode) {

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TypeBuiltins.java

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -230,24 +230,24 @@ protected Object doItUnboxedIndirect(VirtualFrame frame, @SuppressWarnings("unus
230230
}
231231

232232
@Specialization(limit = "getCallSiteInlineCacheMaxDepth()", guards = {"self == cachedSelf"})
233-
protected Object doIt(VirtualFrame frame, @SuppressWarnings("unused") PythonAbstractClass self, Object[] arguments, PKeyword[] keywords,
233+
protected Object doIt0(VirtualFrame frame, @SuppressWarnings("unused") PythonAbstractClass self, Object[] arguments, PKeyword[] keywords,
234234
@Cached("self") PythonAbstractClass cachedSelf) {
235235
return op(frame, cachedSelf, arguments, keywords, true);
236236
}
237237

238-
@Specialization(replaces = "doIt")
239-
protected Object doItIndirect(VirtualFrame frame, PythonAbstractClass self, Object[] arguments, PKeyword[] keywords) {
238+
@Specialization(replaces = "doIt0")
239+
protected Object doItIndirect0(VirtualFrame frame, PythonAbstractClass self, Object[] arguments, PKeyword[] keywords) {
240240
return op(frame, self, arguments, keywords, true);
241241
}
242242

243243
@Specialization(limit = "getCallSiteInlineCacheMaxDepth()", guards = {"self == cachedSelf"})
244-
protected Object doIt(VirtualFrame frame, @SuppressWarnings("unused") PythonNativeObject self, Object[] arguments, PKeyword[] keywords,
244+
protected Object doIt1(VirtualFrame frame, @SuppressWarnings("unused") PythonNativeObject self, Object[] arguments, PKeyword[] keywords,
245245
@Cached("self") PythonNativeObject cachedSelf) {
246246
return op(frame, PythonNativeClass.cast(cachedSelf), arguments, keywords, true);
247247
}
248248

249-
@Specialization(replaces = "doIt")
250-
protected Object doItIndirect(VirtualFrame frame, PythonNativeObject self, Object[] arguments, PKeyword[] keywords) {
249+
@Specialization(replaces = "doIt1")
250+
protected Object doItIndirect1(VirtualFrame frame, PythonNativeObject self, Object[] arguments, PKeyword[] keywords) {
251251
return op(frame, PythonNativeClass.cast(self), arguments, keywords, true);
252252
}
253253

graalpython/lib-graalpython/itertools.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -523,7 +523,7 @@ class combinations():
523523

524524
def __init__(self, pool, indices, r):
525525
self.pool = pool
526-
self.indices = range(indices)
526+
self.indices = indices
527527
if r < 0:
528528
raise ValueError("r must be non-negative")
529529
self.r = r
@@ -593,7 +593,7 @@ def __init__(self, iterable, r):
593593
if r < 0:
594594
raise ValueError("r must be non-negative")
595595
indices = [0] * r
596-
combinations.__init__(pool, indices, r)
596+
super().__init__(pool, indices, r)
597597
self.stopped = len(pool) == 0 and r > 0
598598

599599
def get_maximum(self, i):

graalpython/lib-python/3/venv/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ def create_if_needed(d):
135135
if sys.platform == "win32":
136136
f.write(" %*")
137137
else:
138-
f.write(" $@")
138+
f.write(" \"$@\"")
139139

140140
if sys.platform != "win32":
141141
os.chmod(script, 0o777)

mx.graalpython/copyrights/overrides

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -233,6 +233,7 @@ graalpython/com.oracle.graal.python.test/src/tests/test_getattribute-bimorphic-i
233233
graalpython/com.oracle.graal.python.test/src/tests/test_getattribute-bimorphic-inobject.py,zippy.copyright
234234
graalpython/com.oracle.graal.python.test/src/tests/test_if-class-none.py,zippy.copyright
235235
graalpython/com.oracle.graal.python.test/src/tests/test_if.py,zippy.copyright
236+
graalpython/com.oracle.graal.python.test/src/tests/test_itertools.py,python.copyright
236237
graalpython/com.oracle.graal.python.test/src/tests/test_list.py,python.copyright
237238
graalpython/com.oracle.graal.python.test/src/tests/test_builtin.py,python.copyright
238239
graalpython/com.oracle.graal.python.test/src/tests/test_mandelbrot3.py,benchmarks.copyright

0 commit comments

Comments
 (0)