[GR-13042] Remove arity matching macros in Python.h

PullRequest: graalpython/351

Author: fangerer

graalpython/com.oracle.graal.python.cext/include/Python.h

@@ -328,6 +328,9 @@ extern PyObject* wrapped_null;
 /* DICT */
 void* PyTruffle_Tuple_GetItem(void* jtuple, Py_ssize_t position);
 
+/* STR */
+PyObject* PyTruffle_Unicode_FromFormat(const char*, va_list, void**, int);
+
 /* BYTES, BYTEARRAY */
 int bytes_buffer_getbuffer(PyBytesObject *self, Py_buffer *view, int flags);
 int bytearray_getbuffer(PyByteArrayObject *obj, Py_buffer *view, int flags);
@@ -338,4 +341,35 @@ int bytes_copy2mem(char* target, char* source, size_t nbytes);
 /* MEMORYVIEW, BUFFERDECORATOR */
 int bufferdecorator_getbuffer(PyBufferDecorator *self, Py_buffer *view, int flags);
 
+#if 1
+/*
+ * (tfel): On native Sulong, using va_list will force all arguments to native
+ * memory, which hinders escape analysis and PE in a big way. To avoid this,
+ * when we have function called with var args (rather than already with a
+ * va_list), we allocate a managed array of void*, fill it with the arguments,
+ * and pass that one on. In the target functions, we use the macros below to
+ * access the variable arguments part depending on whether it is a va_list or a
+ * managed void* array. The assumption is that once everything is compiled
+ * together, the managed array with arguments will be escape analyzed away.
+ */
+#define CallWithPolyglotArgs(result, last, off, function, ...)          \
+    int __poly_argc = polyglot_get_arg_count();                         \
+    int __poly_args_s = sizeof(void*) * (__poly_argc - off);            \
+    void **__poly_args = truffle_managed_malloc(__poly_args_s);         \
+    for (int i = off; i < __poly_argc; i++) {                           \
+        __poly_args[i - off] = polyglot_get_arg(i);                     \
+    }                                                                   \
+    result = function(__VA_ARGS__, NULL, __poly_args, 0)
+#else
+/*
+ * (tfel): Just skip the optimization with using a managed malloc and use
+ * va_list always.
+ */
+#define CallWithPolyglotArgs(result, last, off, function, ...)          \
+    va_list __poly_args;                                                \
+    va_start(__poly_args, last);                                        \
+    result = function(__VA_ARGS__, __poly_args, NULL, 0);               \
+    va_end(__poly_args)
+#endif
+
 #endif

graalpython/com.oracle.graal.python.cext/src/capi.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * The Universal Permissive License (UPL), Version 1.0
@@ -159,8 +159,8 @@ int PyErr_ExceptionMatches(PyObject *exc) {
     return PyErr_GivenExceptionMatches(PyErr_Occurred(), exc);
 }
 
-PyObject* PyTruffle_Err_Format(PyObject* exception, const char* fmt, int s, void* v0, void* v1, void* v2, void* v3, void* v4, void* v5, void* v6, void* v7, void* v8, void* v9) {
-    PyObject *formatted_msg = PyTruffle_Unicode_FromFormat(fmt, s, v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
+PyObject* PyErr_Format(PyObject* exception, const char* fmt, ...) {
+    CallWithPolyglotArgs(PyObject* formatted_msg, fmt, 2, PyTruffle_Unicode_FromFormat, fmt);
     UPCALL_CEXT_VOID(_jls_PyErr_CreateAndSetException, native_to_java(exception), native_to_java(formatted_msg));
     return NULL;
 }

graalpython/com.oracle.graal.python.cext/src/errors.c

@@ -177,6 +177,10 @@ PyObject* PyObject_Str(PyObject* o) {
     return UPCALL_CEXT_O(_jls_PyObject_Str, native_to_java(o));
 }
 
+PyObject* PyObject_ASCII(PyObject* o) {
+    return UPCALL_O(PY_BUILTIN, polyglot_from_string("ascii", SRC_CS), native_to_java(o));
+}
+
 UPCALL_ID(PyObject_Repr);
 PyObject* PyObject_Repr(PyObject* o) {
     return UPCALL_CEXT_O(_jls_PyObject_Repr, native_to_java(o));

graalpython/com.oracle.graal.python.cext/src/modsupport.c

@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * The Universal Permissive License (UPL), Version 1.0
+ *
+ * Subject to the condition set forth below, permission is hereby granted to any
+ * person obtaining a copy of this software, associated documentation and/or
+ * data (collectively the "Software"), free of charge and under any and all
+ * copyright rights in the Software, and any and all patent rights owned or
+ * freely licensable by each licensor hereunder covering either (i) the
+ * unmodified Software as contributed to or provided by such licensor, or (ii)
+ * the Larger Works (as defined below), to deal in both
+ *
+ * (a) the Software, and
+ *
+ * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+ * one is included with the Software each a "Larger Work" to which the Software
+ * is contributed by such licensors),
+ *
+ * without restriction, including without limitation the rights to copy, create
+ * derivative works of, display, perform, and distribute the Software and make,
+ * use, sell, offer for sale, import, export, have made, and have sold the
+ * Software and the Larger Work(s), and to sublicense the foregoing rights on
+ * either these or other terms.
+ *
+ * This license is subject to the following condition:
+ *
+ * The above copyright notice and either this complete permission notice or at a
+ * minimum a reference to the UPL must be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include "capi.h"
+
+
+PyObject* _PyLong_FromTime_t(time_t sec) {
+    return PyLong_FromLong((long)sec);
+}

graalpython/com.oracle.graal.python.cext/src/object.c

@@ -112,89 +112,105 @@ PyObject * PyUnicode_FromStringAndSize(const char *u, Py_ssize_t size) {
     return to_sulong(polyglot_from_string_n(u, size, SRC_CS));
 }
 
-PyObject* PyTruffle_Unicode_FromFormat(const char* fmt, int s, void* v0, void* v1, void* v2, void* v3, void* v4, void* v5, void* v6, void* v7, void* v8, void* v9, void* v10, void* v11, void* v12, void* v13, void* v14, void* v15, void* v16, void* v17, void* v18, void* v19) {
-    char** allocated_strings = calloc(sizeof(char*), s);
-#   define ASSIGN(n, value)                     \
-    switch(n) {                                 \
-    case 0: v0 = value; break;                  \
-    case 1: v1 = value; break;                  \
-    case 2: v2 = value; break;                  \
-    case 3: v3 = value; break;                  \
-    case 4: v4 = value; break;                  \
-    case 5: v5 = value; break;                  \
-    case 6: v6 = value; break;                  \
-    case 7: v7 = value; break;                  \
-    case 8: v8 = value; break;                  \
-    case 9: v9 = value; break;                  \
-    case 10: v10 = value; break;                \
-    case 11: v11 = value; break;                \
-    case 12: v12 = value; break;                \
-    case 13: v13 = value; break;                \
-    case 14: v14 = value; break;                \
-    case 15: v15 = value; break;                \
-    case 16: v16 = value; break;                \
-    case 17: v17 = value; break;                \
-    case 18: v18 = value; break;                \
-    case 19: v19 = value; break;                \
-    }
+#define AS_I64(__arg__) (polyglot_fits_in_i64((__arg__)) ? polyglot_as_i64((__arg__)) : (int64_t)(__arg__))
 
+MUST_INLINE PyObject* PyTruffle_Unicode_FromFormat(const char *fmt, va_list va, void **args, int argc) {
     size_t fmt_size = strlen(fmt) + 1;
     // n.b. avoid using 'strdup' for compatiblity with MUSL libc
     char* fmtcpy = (char*) malloc(fmt_size*sizeof(char));
+    memcpy(fmtcpy, fmt, fmt_size);
     char* c = fmtcpy;
-    char* allocated;
-    int cnt = 0;
 
-    memcpy(fmtcpy, fmt, fmt_size);
+    int remaining_space = 2047;
+    char* buffer = (char*)calloc(sizeof(char), remaining_space + 1);
+    char* full_buffer = buffer;
 
-    while (c[0] && cnt < s) {
+    void *variable = NULL;
+    char *allocated = NULL; // points to the same as variable, if it has to be free'd
+
+    while (c[0]) {
         if (c[0] == '%') {
+            // we've reached the next directive, write until here
+            c[0] = '\0';
+            int bytes_written;
+            if (variable != NULL) {
+                bytes_written = snprintf(buffer, remaining_space, fmtcpy, AS_I64(variable));
+                if (allocated != NULL) {
+                    free(allocated);
+                    allocated = NULL;
+                }
+                variable = NULL;
+            } else if (va != NULL) {
+                bytes_written = vsnprintf(buffer, remaining_space, fmtcpy, va);
+            } else {
+                strncpy(buffer, fmtcpy, remaining_space);
+                bytes_written = strlen(fmtcpy);
+            }
+            remaining_space -= bytes_written;
+            buffer += bytes_written;
+            fmtcpy = c;
+            c[0] = '%';
+
+            // now decide if we need to do something special with this directive
+            PyObject* (*converter)(PyObject*) = NULL;
             switch (c[1]) {
-            case 'c':
-                c[1] = 'd';
-                break;
             case 'A':
-                c[1] = 's';
-                ;
-                allocated_strings[cnt] = allocated = as_char_pointer(UPCALL_O(PY_BUILTIN, polyglot_from_string("ascii", SRC_CS), native_to_java(polyglot_get_arg(cnt + 2))));
-                ASSIGN(cnt, allocated);
-                break;
+                // The conversion cases, these all use a function to convert the
+                // PyObject* to a char* and they fall through
+                converter = PyObject_ASCII;
             case 'U':
-                c[1] = 's';
-                allocated_strings[cnt] = allocated = as_char_pointer(PyObject_Str(polyglot_get_arg(cnt + 2)));
-                ASSIGN(cnt, allocated);
-                break;
+                if (converter == NULL) converter = PyObject_Str;
             case 'S':
-                c[1] = 's';
-                allocated_strings[cnt] = allocated = as_char_pointer(PyObject_Str(polyglot_get_arg(cnt + 2)));
-                ASSIGN(cnt, allocated);
-                break;
+                if (converter == NULL) converter = PyObject_Str;
             case 'R':
+                if (converter == NULL) converter = PyObject_Repr;
                 c[1] = 's';
-                allocated_strings[cnt] = allocated = as_char_pointer(PyObject_Repr(polyglot_get_arg(cnt + 2)));
-                ASSIGN(cnt, allocated);
+                allocated = variable = as_char_pointer(converter(args == NULL ? va_arg(va, PyObject*) : (PyObject*)(args[argc++])));
+                break;
+            case '%':
+                // literal %
                 break;
+            case 'c':
+                // This case should just treat it's argument as an integer
+                c[1] = 'd';
+            default:
+                // if we're reading args from a void* array, read it now,
+                // otherwise there's nothing to do
+                if (args != NULL) {
+                    variable = args[argc++];
+                }
             }
-            cnt++;
+            // skip over next char, we checked it
             c += 1;
         }
         c += 1;
     }
 
-    char buffer[2048] = {'\0'};
-    snprintf(buffer, 2047, fmtcpy, v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19);
-
-    for (int i = 0; i < s; i++) {
-        if (allocated_strings[i] != NULL) {
-            free(allocated_strings[i]);
+    // write the remaining buffer
+    if (variable != NULL) {
+        snprintf(buffer, remaining_space, fmtcpy, AS_I64(variable));
+        if (allocated) {
+            free(allocated);
         }
+    } else if (va != NULL) {
+        vsnprintf(buffer, remaining_space, fmtcpy, va);
+    } else {
+        strncpy(buffer, fmtcpy, remaining_space);
     }
 
-    return PyUnicode_FromString(buffer);
+    PyObject* result = PyUnicode_FromString(full_buffer);
+    free(full_buffer);
+    return result;
+}
 
-#   undef ASSIGN
+PyObject* PyUnicode_FromFormatV(const char* format, va_list va) {
+    return PyTruffle_Unicode_FromFormat(format, va, NULL, 0);
 }
 
+PyObject* PyUnicode_FromFormat(const char* format, ...) {
+    CallWithPolyglotArgs(PyObject* result, format, 1, PyTruffle_Unicode_FromFormat, format);
+    return result;
+}
 
 PyObject * PyUnicode_FromUnicode(const Py_UNICODE *u, Py_ssize_t size) {
     if (u == NULL) {

graalpython/com.oracle.graal.python.cext/src/pytime.c

@@ -171,7 +171,7 @@ private void parseOptions(String[] args) {
                             throw new RuntimeException("cannot mix source and compiled sources");
                         }
                         fileInputs.add(arg);
-                    } else if (arg.endsWith(".c") || arg.endsWith(".cpp") || arg.endsWith(".cxx")) {
+                    } else if (arg.endsWith(".c") || arg.endsWith(".cc") || arg.endsWith(".cpp") || arg.endsWith(".cxx")) {
                         if (arg.endsWith(".cpp") || arg.endsWith(".cxx")) {
                             isCpp = true;
                         }

graalpython/com.oracle.graal.python.cext/src/unicodeobject.c

@@ -48,6 +48,7 @@
 import com.oracle.graal.python.nodes.util.ExceptionStateNodes.SaveExceptionStateNode;
 import com.oracle.graal.python.runtime.exception.ExceptionHandledException;
 import com.oracle.graal.python.runtime.exception.PException;
+import com.oracle.truffle.api.CompilerAsserts;
 import com.oracle.truffle.api.frame.VirtualFrame;
 import com.oracle.truffle.api.nodes.ControlFlowException;
 import com.oracle.truffle.api.nodes.ExplodeLoop;
@@ -121,11 +122,8 @@ private void catchExceptionInGenerator(VirtualFrame frame, PException exception,
                 }
             }
         } else if (matchingExceptNodeIndex <= exceptNodes.length) {
-            // we already found the right except handler, jump back into
-            // it directly
-            ExceptNode exceptNode = exceptNodes[matchingExceptNodeIndex - 1];
-            runExceptionHandler(frame, exception, exceptNode, exceptionState);
-            wasHandled = true;
+            // we already found the right except handler, jump back into it directly
+            wasHandled = catchExceptionInGeneratorCached(frame, exceptNodes, exception, exceptionState, matchingExceptNodeIndex);
         }
         reset(frame);
         if (!wasHandled) {
@@ -135,6 +133,23 @@ private void catchExceptionInGenerator(VirtualFrame frame, PException exception,
         restoreExceptionState.execute(frame, exceptionState);
     }
 
+    @ExplodeLoop
+    private boolean catchExceptionInGeneratorCached(VirtualFrame frame, ExceptNode[] exceptNodes, PException exception, ExceptionState exceptionState, int matchingExceptNodeIndex) {
+        CompilerAsserts.compilationConstant(exceptNodes);
+        assert matchingExceptNodeIndex <= exceptNodes.length;
+        boolean wasHandled = false;
+        for (int i = 0; i < exceptNodes.length; i++) {
+            // we want a constant loop iteration count for ExplodeLoop to work,
+            // so we always run through all except handlers
+            if (i == matchingExceptNodeIndex - 1) {
+                runExceptionHandler(frame, exception, exceptNodes[i], exceptionState);
+                wasHandled = true;
+            }
+        }
+        assert wasHandled : "cached exception handler does not handle exception";
+        return wasHandled;
+    }
+
     private void runExceptionHandler(VirtualFrame frame, PException exception, ExceptNode exceptNode, ExceptionState exceptionState) {
         try {
             exceptNode.executeExcept(frame, exception);

graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonCC.java

@@ -51,6 +51,7 @@ public class AssertNode extends StatementNode {
     @Child private ExpressionNode message;
     @Child private LookupAndCallUnaryNode callNode;
     @CompilationFinal private Boolean assertionsEnabled = null;
+    @CompilationFinal private boolean javaExceptionsFailAssertions;
     @CompilationFinal private ContextReference<PythonContext> contextRef;
 
     private final ConditionProfile profile = ConditionProfile.createBinaryProfile();
@@ -64,7 +65,9 @@ public AssertNode(CastToBooleanNode condition, ExpressionNode message) {
     public void executeVoid(VirtualFrame frame) {
         if (assertionsEnabled == null) {
             CompilerDirectives.transferToInterpreterAndInvalidate();
-            assertionsEnabled = !PythonOptions.getOption(PythonLanguage.getContextRef().get(), PythonOptions.PythonOptimizeFlag);
+            PythonContext context = PythonLanguage.getContextRef().get();
+            assertionsEnabled = !PythonOptions.getOption(context, PythonOptions.PythonOptimizeFlag);
+            javaExceptionsFailAssertions = PythonOptions.getOption(context, PythonOptions.CatchAllExceptions);
         }
         if (assertionsEnabled) {
             try {
@@ -75,8 +78,12 @@ public void executeVoid(VirtualFrame frame) {
                 // Python exceptions just fall through
                 throw e;
             } catch (Exception e) {
-                // catch any other exception and convert to Python exception
-                throw assertionFailed(frame);
+                if (javaExceptionsFailAssertions) {
+                    // catch any other exception and convert to Python exception
+                    throw assertionFailed(frame);
+                } else {
+                    throw e;
+                }
             }
         }
     }