Merge pull request #564 from lf-lang/python-allow-async-threads-concurrently

Python target: Allow the execution of async Python threads concurrently with the C runtime

Author: lhstrh

org.lflang/src/lib/Python/pythontarget.c

@@ -254,7 +254,15 @@ static PyObject* py_request_stop(PyObject *self) {
 static PyObject* py_main(PyObject *self, PyObject *args) {
     DEBUG_PRINT("Initializing main.");
     const char *argv[] = {TOSTRING(MODULE_NAME), NULL };
+
+    // Initialize the Python interpreter
+    Py_Initialize();
+
+    DEBUG_PRINT("Initialized the Python interpreter.");
+
+    Py_BEGIN_ALLOW_THREADS
     lf_reactor_c_main(1, argv);
+    Py_END_ALLOW_THREADS
 
     Py_INCREF(Py_None);
     return Py_None;
@@ -966,25 +974,22 @@ PyObject* convert_C_action_to_py(void* action) {
  */
 PyObject*
 get_python_function(string module, string class, int instance_id, string func) {
-
-    // Set if the interpreter is already initialized
-    int is_initialized = 0;
-
-    if (Py_IsInitialized()) {
-        is_initialized = 1;
-    }
-
     DEBUG_PRINT("Starting the function start().");
 
     // Necessary PyObject variables to load the react() function from test.py
     PyObject *pFileName, *pModule, *pDict, *pClasses, *pClass, *pFunc;
 
     PyObject *rValue;
 
-    // Initialize the Python interpreter
-    Py_Initialize();
-
-    DEBUG_PRINT("Initialized the Python interpreter.");
+    // According to
+    // https://docs.python.org/3/c-api/init.html#non-python-created-threads
+    // the following code does the following:
+    // - Register this thread with the interpreter
+    // - Acquire the GIL (Global Interpreter Lock)
+    // - Store (return) the thread pointer
+    // When done, we should always call PyGILState_Release(gstate);
+    PyGILState_STATE gstate;
+    gstate = PyGILState_Ensure();
 
     // If the Python module is already loaded, skip this.
     if (globalPythonModule == NULL) {    
@@ -993,8 +998,7 @@ get_python_function(string module, string class, int instance_id, string func) {
 
         // Set the Python search path to be the current working directory
         char cwd[PATH_MAX];
-        if ( getcwd(cwd, sizeof(cwd)) == NULL)
-        {
+        if ( getcwd(cwd, sizeof(cwd)) == NULL) {
             error_print_and_exit("Failed to get the current working directory.");
         }
 
@@ -1004,7 +1008,7 @@ get_python_function(string module, string class, int instance_id, string func) {
 
         Py_SetPath(wcwd);
 
-    DEBUG_PRINT("Loading module %s in %s.", module, cwd);
+        DEBUG_PRINT("Loading module %s in %s.", module, cwd);
 
         pModule = PyImport_Import(pFileName);
 
@@ -1020,6 +1024,8 @@ get_python_function(string module, string class, int instance_id, string func) {
             if (pDict == NULL) {
                 PyErr_Print();
                 error_print("Failed to load contents of module %s.", module);
+                /* Release the thread. No Python API allowed beyond this point. */
+                PyGILState_Release(gstate);
                 return 1;
             }
 
@@ -1042,6 +1048,8 @@ get_python_function(string module, string class, int instance_id, string func) {
         if (pClasses == NULL){
             PyErr_Print();
             error_print("Failed to load class list \"%s\" in module %s.", class, module);
+            /* Release the thread. No Python API allowed beyond this point. */
+            PyGILState_Release(gstate);
             return 1;
         }
 
@@ -1051,6 +1059,8 @@ get_python_function(string module, string class, int instance_id, string func) {
         if (pClass == NULL) {
             PyErr_Print();
             error_print("Failed to load class \"%s[%d]\" in module %s.", class, instance_id, module);
+            /* Release the thread. No Python API allowed beyond this point. */
+            PyGILState_Release(gstate);
             return 1;
         }
 
@@ -1066,6 +1076,8 @@ get_python_function(string module, string class, int instance_id, string func) {
         if (pFunc && PyCallable_Check(pFunc)) {
             DEBUG_PRINT("Calling function %s from class %s[%d].", func , class, instance_id);
             Py_INCREF(pFunc);
+            /* Release the thread. No Python API allowed beyond this point. */
+            PyGILState_Release(gstate);
             return pFunc;
         }
         else {
@@ -1086,11 +1098,8 @@ get_python_function(string module, string class, int instance_id, string func) {
 
     DEBUG_PRINT("Done with start().");
 
-    if (is_initialized == 0) {
-        /* We are the first to initilize the Pyton interpreter. Destroy it when done. */
-        Py_FinalizeEx();
-    }
-
     Py_INCREF(Py_None);
+    /* Release the thread. No Python API allowed beyond this point. */
+    PyGILState_Release(gstate);
     return Py_None;
 }