From ed3421ef22d3a08a70e4d71f02e5fb62b6e8bad3 Mon Sep 17 00:00:00 2001
From: Jade Abraham <jade.abraham@hpe.com>
Date: Fri, 24 Jan 2025 10:44:58 -0800
Subject: [PATCH 01/13] implement sub interpreters

Signed-off-by: Jade Abraham <jade.abraham@hpe.com>
---
 modules/packages/Python.chpl | 70 +++++++++++++++++++++++++++++++++++-
 1 file changed, 69 insertions(+), 1 deletion(-)

diff --git a/modules/packages/Python.chpl b/modules/packages/Python.chpl
index 7ef9de10ebba..e68babfad938 100644
--- a/modules/packages/Python.chpl
+++ b/modules/packages/Python.chpl
@@ -299,10 +299,17 @@ module Python {
     var converters: List.list(owned TypeConverter);
     @chpldoc.nodoc
     var objgraph: PyObjectPtr = nil;
+    @chpldoc.nodoc
+    var isSubInterpreter: bool;
 
     @chpldoc.nodoc
-    proc init() throws {
+    proc init(isSubInterpreter: bool = false) {
+      this.isSubInterpreter = isSubInterpreter;
       init this;
+    }
+    @chpldoc.nodoc
+    proc postinit() throws {
+      if this.isSubInterpreter then return;
 
       // preinit
       var preconfig: PyPreConfig;
@@ -388,6 +395,7 @@ module Python {
     }
     @chpldoc.nodoc
     proc deinit()  {
+      if this.isSubInterpreter then return;
 
       if pyMemLeaks && this.objgraph != nil {
         // note: try! is used since we can't have a throwing deinit
@@ -903,6 +911,45 @@ module Python {
 
   }
 
+
+  class SubInterpreter: Interpreter {
+    @chpldoc.nodoc
+    var parent: borrowed Interpreter;
+    @chpldoc.nodoc
+    var tstate: PyThreadStatePtr;
+
+    @chpldoc.nodoc
+    proc init(parent: borrowed Interpreter) {
+      super.init(isSubInterpreter=true);
+      this.parent = parent;
+      init this;
+    }
+    @chpldoc.noddoc
+    proc postinit() throws {
+      if this.parent.isSubInterpreter {
+        throwChapelException("Parent interpreter cannot be a sub-interpreter");
+      }
+
+      var cfg: PyInterpreterConfig;
+      cfg.use_main_obmalloc = 0;
+      cfg.allow_fork = 0;
+      cfg.allow_exec = 0;
+      cfg.allow_threads = 1;
+      cfg.allow_daemon_threads = 0;
+      cfg.check_multi_interp_extensions = 1;
+      cfg.gil = PyInterpreterConfig_OWN_GIL;
+
+      var status = Py_NewInterpreterFromConfig(c_ptrTo(this.tstate), c_ptrTo(cfg));
+      if PyStatus_Exception(status) {
+        throwChapelException("Failed to create sub-interpreter");
+      }
+    }
+    @chpldoc.nodoc
+    proc deinit() {
+      Py_EndInterpreter(this.tstate);
+    }
+  }
+
   /*
     Represents a Python exception, either forwarded from Python
     (i.e. :proc:`Interpreter.checkException`) or thrown directly in Chapel code.
@@ -1853,6 +1900,25 @@ module Python {
     extern "chpl_PY_MICRO_VERSION" const PY_MICRO_VERSION: c_ulong;
 
 
+    // TODO: restrict to python 3.12+
+    /*
+      Sub Interpreters
+    */
+    extern var PyInterpreterConfig_DEFAULT_GIL: c_int;
+    extern var PyInterpreterConfig_SHARED_GIL: c_int;
+    extern var PyInterpreterConfig_OWN_GIL: c_int;
+    extern record PyInterpreterConfig {
+      var use_main_obmalloc: c_int;
+      var allow_fork: c_int;
+      var allow_exec: c_int;
+      var allow_threads: c_int;
+      var allow_daemon_threads: c_int;
+      var check_multi_interp_extensions: c_int;
+      var gil: c_int;
+    }
+    extern proc Py_NewInterpreterFromConfig(tstate_p: c_ptr(PyThreadStatePtr),
+                                            config_: c_ptr(PyInterpreterConfig)): PyStatus;
+    extern proc Py_EndInterpreter(tstate: PyThreadStatePtr);
 
     /*
       Global exec functions
@@ -1880,6 +1946,8 @@ module Python {
     */
     extern proc PyEval_SaveThread(): PyThreadStatePtr;
     extern proc PyEval_RestoreThread(state: PyThreadStatePtr);
+    extern proc PyThreadState_Get(): PyThreadStatePtr;
+    extern proc PyThreadState_Swap(state: PyThreadStatePtr): PyThreadStatePtr;
 
     extern proc PyGILState_Ensure(): PyGILState_STATE;
     extern proc PyGILState_Release(state: PyGILState_STATE);

From 743d8e3e008287de0a7490a15014c2a2afbb39ac Mon Sep 17 00:00:00 2001
From: Jade Abraham <jade.abraham@hpe.com>
Date: Fri, 24 Jan 2025 10:45:08 -0800
Subject: [PATCH 02/13] add standalone test

Signed-off-by: Jade Abraham <jade.abraham@hpe.com>
---
 .../Python/correctness/subInterp.chpl         | 25 +++++++++++++++++++
 1 file changed, 25 insertions(+)
 create mode 100644 test/library/packages/Python/correctness/subInterp.chpl

diff --git a/test/library/packages/Python/correctness/subInterp.chpl b/test/library/packages/Python/correctness/subInterp.chpl
new file mode 100644
index 000000000000..4d065aa089b7
--- /dev/null
+++ b/test/library/packages/Python/correctness/subInterp.chpl
@@ -0,0 +1,25 @@
+use Python;
+
+var code = """
+import sys
+def hello(tid, idx):
+  print("Hello from thread", tid, "index", idx)
+  sys.stdout.flush()
+""";
+
+proc main() {
+
+  var interp = new Interpreter();
+
+  coforall tid in 0..#here.maxTaskPar {
+    var localInterp = new SubInterpreter(interp);
+
+    var m = new Module(localInterp, '__empty__', code);
+    var f = new Function(m, 'hello');
+    for i in 1..100 {
+      f(NoneType, tid, i);
+    }
+  }
+
+
+}

From 8f854d7fec025a0978b79468a68c2bd189fd3ee6 Mon Sep 17 00:00:00 2001
From: Jade Abraham <jade.abraham@hpe.com>
Date: Fri, 24 Jan 2025 10:45:25 -0800
Subject: [PATCH 03/13] add comparison to iteration test

Signed-off-by: Jade Abraham <jade.abraham@hpe.com>
---
 .../correctness/compareIterationPatterns.chpl | 36 +++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/test/library/packages/Python/correctness/compareIterationPatterns.chpl b/test/library/packages/Python/correctness/compareIterationPatterns.chpl
index 40ee2364de04..6189855b7b27 100644
--- a/test/library/packages/Python/correctness/compareIterationPatterns.chpl
+++ b/test/library/packages/Python/correctness/compareIterationPatterns.chpl
@@ -34,6 +34,30 @@ proc parallelPythonApply(interp: borrowed, type t, arr, l) {
   ts.restore();
   return res;
 }
+//
+// Calling a python Lambda function from Chapel in parallel using SubInterpreter
+//
+proc parallelPythonApplySubInterp(interp: borrowed, type t, arr, l) {
+
+  record funcPair {
+    var interp: owned SubInterpreter?;
+    var func: owned Value?;
+    proc init(parent: borrowed, s) {
+      init this;
+      interp = try! (new SubInterpreter(parent));
+      func = try! (new Function(interp!, s));
+    }
+    inline proc this(type t, a) throws {
+      return func!(t, a);
+    }
+  }
+
+  var res: [arr.domain] t;
+  forall i in arr.domain with (var lambdaFunc = new funcPair(interp, l)) {
+    res(i) = lambdaFunc(t, arr(i));
+  }
+  return res;
+}
 
 
 //
@@ -88,6 +112,18 @@ proc main() {
       writeln("Parallel Python result: ", res);
   }
 
+  {
+    data = 1..#n;
+    var s = new stopwatch();
+    s.start();
+    var res = parallelPythonApplySubInterp(interp, int, data, lambdaStr);
+    s.stop();
+    if time then
+      writeln("Elapsed time (Parallel Python SubInterpreter): ", s.elapsed(), " seconds");
+    if print then
+      writeln("Parallel Python SubInterpreter result: ", res);
+  }
+
   {
     data = 1..#n;
     var s = new stopwatch();

From 2b65715ef5c9546f083b8ee434e8dd248d825a34 Mon Sep 17 00:00:00 2001
From: Jade Abraham <jade.abraham@hpe.com>
Date: Fri, 24 Jan 2025 10:48:23 -0800
Subject: [PATCH 04/13] add skipif

Signed-off-by: Jade Abraham <jade.abraham@hpe.com>
---
 .../correctness/compareIterationPatterns.skipif   | 15 +++++++++++++++
 .../packages/Python/correctness/subInterp.chpl    |  1 -
 .../packages/Python/correctness/subInterp.skipif  | 15 +++++++++++++++
 3 files changed, 30 insertions(+), 1 deletion(-)
 create mode 100755 test/library/packages/Python/correctness/compareIterationPatterns.skipif
 create mode 100755 test/library/packages/Python/correctness/subInterp.skipif

diff --git a/test/library/packages/Python/correctness/compareIterationPatterns.skipif b/test/library/packages/Python/correctness/compareIterationPatterns.skipif
new file mode 100755
index 000000000000..ab42bb68490b
--- /dev/null
+++ b/test/library/packages/Python/correctness/compareIterationPatterns.skipif
@@ -0,0 +1,15 @@
+# requires python 3.12+
+
+# respect CHPL_TEST_VENV_DIR if it is set and not none
+if [ -n "$CHPL_TEST_VENV_DIR" ] && [ "$CHPL_TEST_VENV_DIR" != "none" ]; then
+  chpl_python=$CHPL_TEST_VENV_DIR/bin/python3
+else
+  chpl_python=$($CHPL_HOME/util/config/find-python.sh)
+fi
+
+minor_version=$($chpl_python -c "import sys; print(sys.version_info.minor)")
+if [ $minor_version -ge 12 ]; then
+  echo "False"
+else
+  echo "True"
+fi
diff --git a/test/library/packages/Python/correctness/subInterp.chpl b/test/library/packages/Python/correctness/subInterp.chpl
index 4d065aa089b7..1dea7e36a3f8 100644
--- a/test/library/packages/Python/correctness/subInterp.chpl
+++ b/test/library/packages/Python/correctness/subInterp.chpl
@@ -21,5 +21,4 @@ proc main() {
     }
   }
 
-
 }
diff --git a/test/library/packages/Python/correctness/subInterp.skipif b/test/library/packages/Python/correctness/subInterp.skipif
new file mode 100755
index 000000000000..ab42bb68490b
--- /dev/null
+++ b/test/library/packages/Python/correctness/subInterp.skipif
@@ -0,0 +1,15 @@
+# requires python 3.12+
+
+# respect CHPL_TEST_VENV_DIR if it is set and not none
+if [ -n "$CHPL_TEST_VENV_DIR" ] && [ "$CHPL_TEST_VENV_DIR" != "none" ]; then
+  chpl_python=$CHPL_TEST_VENV_DIR/bin/python3
+else
+  chpl_python=$($CHPL_HOME/util/config/find-python.sh)
+fi
+
+minor_version=$($chpl_python -c "import sys; print(sys.version_info.minor)")
+if [ $minor_version -ge 12 ]; then
+  echo "False"
+else
+  echo "True"
+fi

From 3e3cb31124a89a391189c47f1919b38ae00ede02 Mon Sep 17 00:00:00 2001
From: Jade Abraham <jade.abraham@hpe.com>
Date: Fri, 24 Jan 2025 10:56:14 -0800
Subject: [PATCH 05/13] add shebang

Signed-off-by: Jade Abraham <jade.abraham@hpe.com>
---
 .../packages/Python/correctness/compareIterationPatterns.skipif  | 1 +
 test/library/packages/Python/correctness/subInterp.skipif        | 1 +
 2 files changed, 2 insertions(+)

diff --git a/test/library/packages/Python/correctness/compareIterationPatterns.skipif b/test/library/packages/Python/correctness/compareIterationPatterns.skipif
index ab42bb68490b..33ec676140d5 100755
--- a/test/library/packages/Python/correctness/compareIterationPatterns.skipif
+++ b/test/library/packages/Python/correctness/compareIterationPatterns.skipif
@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
 # requires python 3.12+
 
 # respect CHPL_TEST_VENV_DIR if it is set and not none
diff --git a/test/library/packages/Python/correctness/subInterp.skipif b/test/library/packages/Python/correctness/subInterp.skipif
index ab42bb68490b..33ec676140d5 100755
--- a/test/library/packages/Python/correctness/subInterp.skipif
+++ b/test/library/packages/Python/correctness/subInterp.skipif
@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
 # requires python 3.12+
 
 # respect CHPL_TEST_VENV_DIR if it is set and not none

From 23bfb82e5de01299542641ae39b5c302911c6a37 Mon Sep 17 00:00:00 2001
From: Jade Abraham <jade.abraham@hpe.com>
Date: Fri, 24 Jan 2025 11:04:50 -0800
Subject: [PATCH 06/13] cleanup test

Signed-off-by: Jade Abraham <jade.abraham@hpe.com>
---
 .../Python/correctness/subInterp.chpl         |  7 +-
 .../Python/correctness/subInterp.execopts     |  1 +
 .../Python/correctness/subInterp.good         | 80 +++++++++++++++++++
 .../Python/correctness/subInterp.prediff      |  3 +
 4 files changed, 89 insertions(+), 2 deletions(-)
 create mode 100644 test/library/packages/Python/correctness/subInterp.execopts
 create mode 100644 test/library/packages/Python/correctness/subInterp.good
 create mode 100755 test/library/packages/Python/correctness/subInterp.prediff

diff --git a/test/library/packages/Python/correctness/subInterp.chpl b/test/library/packages/Python/correctness/subInterp.chpl
index 1dea7e36a3f8..a585d6c7655b 100644
--- a/test/library/packages/Python/correctness/subInterp.chpl
+++ b/test/library/packages/Python/correctness/subInterp.chpl
@@ -1,5 +1,8 @@
 use Python;
 
+config const tasks = here.maxTaskPar;
+config const itersPerTask = 100;
+
 var code = """
 import sys
 def hello(tid, idx):
@@ -11,12 +14,12 @@ proc main() {
 
   var interp = new Interpreter();
 
-  coforall tid in 0..#here.maxTaskPar {
+  coforall tid in 0..#tasks {
     var localInterp = new SubInterpreter(interp);
 
     var m = new Module(localInterp, '__empty__', code);
     var f = new Function(m, 'hello');
-    for i in 1..100 {
+    for i in 1..#itersPerTask {
       f(NoneType, tid, i);
     }
   }
diff --git a/test/library/packages/Python/correctness/subInterp.execopts b/test/library/packages/Python/correctness/subInterp.execopts
new file mode 100644
index 000000000000..0cd751f24458
--- /dev/null
+++ b/test/library/packages/Python/correctness/subInterp.execopts
@@ -0,0 +1 @@
+--tasks=4 --itersPerTask=20
diff --git a/test/library/packages/Python/correctness/subInterp.good b/test/library/packages/Python/correctness/subInterp.good
new file mode 100644
index 000000000000..e5d59b8c03bf
--- /dev/null
+++ b/test/library/packages/Python/correctness/subInterp.good
@@ -0,0 +1,80 @@
+Hello from thread 0 index 1
+Hello from thread 0 index 10
+Hello from thread 0 index 11
+Hello from thread 0 index 12
+Hello from thread 0 index 13
+Hello from thread 0 index 14
+Hello from thread 0 index 15
+Hello from thread 0 index 16
+Hello from thread 0 index 17
+Hello from thread 0 index 18
+Hello from thread 0 index 19
+Hello from thread 0 index 2
+Hello from thread 0 index 20
+Hello from thread 0 index 3
+Hello from thread 0 index 4
+Hello from thread 0 index 5
+Hello from thread 0 index 6
+Hello from thread 0 index 7
+Hello from thread 0 index 8
+Hello from thread 0 index 9
+Hello from thread 1 index 1
+Hello from thread 1 index 10
+Hello from thread 1 index 11
+Hello from thread 1 index 12
+Hello from thread 1 index 13
+Hello from thread 1 index 14
+Hello from thread 1 index 15
+Hello from thread 1 index 16
+Hello from thread 1 index 17
+Hello from thread 1 index 18
+Hello from thread 1 index 19
+Hello from thread 1 index 2
+Hello from thread 1 index 20
+Hello from thread 1 index 3
+Hello from thread 1 index 4
+Hello from thread 1 index 5
+Hello from thread 1 index 6
+Hello from thread 1 index 7
+Hello from thread 1 index 8
+Hello from thread 1 index 9
+Hello from thread 2 index 1
+Hello from thread 2 index 10
+Hello from thread 2 index 11
+Hello from thread 2 index 12
+Hello from thread 2 index 13
+Hello from thread 2 index 14
+Hello from thread 2 index 15
+Hello from thread 2 index 16
+Hello from thread 2 index 17
+Hello from thread 2 index 18
+Hello from thread 2 index 19
+Hello from thread 2 index 2
+Hello from thread 2 index 20
+Hello from thread 2 index 3
+Hello from thread 2 index 4
+Hello from thread 2 index 5
+Hello from thread 2 index 6
+Hello from thread 2 index 7
+Hello from thread 2 index 8
+Hello from thread 2 index 9
+Hello from thread 3 index 1
+Hello from thread 3 index 10
+Hello from thread 3 index 11
+Hello from thread 3 index 12
+Hello from thread 3 index 13
+Hello from thread 3 index 14
+Hello from thread 3 index 15
+Hello from thread 3 index 16
+Hello from thread 3 index 17
+Hello from thread 3 index 18
+Hello from thread 3 index 19
+Hello from thread 3 index 2
+Hello from thread 3 index 20
+Hello from thread 3 index 3
+Hello from thread 3 index 4
+Hello from thread 3 index 5
+Hello from thread 3 index 6
+Hello from thread 3 index 7
+Hello from thread 3 index 8
+Hello from thread 3 index 9
diff --git a/test/library/packages/Python/correctness/subInterp.prediff b/test/library/packages/Python/correctness/subInterp.prediff
new file mode 100755
index 000000000000..e3eb2904a904
--- /dev/null
+++ b/test/library/packages/Python/correctness/subInterp.prediff
@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+sort $2 > $2.prediff.tmp
+mv $2.prediff.tmp $2

From 20e49c631021debfd885013e095eea9bdfc5fefb Mon Sep 17 00:00:00 2001
From: Jade Abraham <jade.abraham@hpe.com>
Date: Fri, 24 Jan 2025 11:06:15 -0800
Subject: [PATCH 07/13] update good file

Signed-off-by: Jade Abraham <jade.abraham@hpe.com>
---
 .../packages/Python/correctness/compareIterationPatterns.good    | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/library/packages/Python/correctness/compareIterationPatterns.good b/test/library/packages/Python/correctness/compareIterationPatterns.good
index b73a69445217..693d2cfc2c62 100644
--- a/test/library/packages/Python/correctness/compareIterationPatterns.good
+++ b/test/library/packages/Python/correctness/compareIterationPatterns.good
@@ -1,5 +1,6 @@
 data: 1 2 3 4 5 6 7 8 9 10
 Serial Python result: 2 2 4 4 6 6 8 8 10 10
 Parallel Python result: 2 2 4 4 6 6 8 8 10 10
+Parallel Python SubInterpreter result: 2 2 4 4 6 6 8 8 10 10
 Serial Chapel result: 2 2 4 4 6 6 8 8 10 10
 Parallel Chapel result: 2 2 4 4 6 6 8 8 10 10

From ad59f6e9de88d1d623a024f172ebcbfc49a16d11 Mon Sep 17 00:00:00 2001
From: Jade Abraham <jade.abraham@hpe.com>
Date: Fri, 24 Jan 2025 11:08:44 -0800
Subject: [PATCH 08/13] improve portability

Signed-off-by: Jade Abraham <jade.abraham@hpe.com>
---
 modules/packages/Python.chpl                  | 30 ++-----------------
 .../PythonHelper/ChapelPythonHelper.h         | 18 +++++++++++
 2 files changed, 20 insertions(+), 28 deletions(-)

diff --git a/modules/packages/Python.chpl b/modules/packages/Python.chpl
index e68babfad938..527700a29ed0 100644
--- a/modules/packages/Python.chpl
+++ b/modules/packages/Python.chpl
@@ -930,19 +930,7 @@ module Python {
         throwChapelException("Parent interpreter cannot be a sub-interpreter");
       }
 
-      var cfg: PyInterpreterConfig;
-      cfg.use_main_obmalloc = 0;
-      cfg.allow_fork = 0;
-      cfg.allow_exec = 0;
-      cfg.allow_threads = 1;
-      cfg.allow_daemon_threads = 0;
-      cfg.check_multi_interp_extensions = 1;
-      cfg.gil = PyInterpreterConfig_OWN_GIL;
-
-      var status = Py_NewInterpreterFromConfig(c_ptrTo(this.tstate), c_ptrTo(cfg));
-      if PyStatus_Exception(status) {
-        throwChapelException("Failed to create sub-interpreter");
-      }
+      checkPyStatus(chpl_Py_NewIsolatedInterpreter(c_ptrTo(this.tstate)));
     }
     @chpldoc.nodoc
     proc deinit() {
@@ -1900,24 +1888,10 @@ module Python {
     extern "chpl_PY_MICRO_VERSION" const PY_MICRO_VERSION: c_ulong;
 
 
-    // TODO: restrict to python 3.12+
     /*
       Sub Interpreters
     */
-    extern var PyInterpreterConfig_DEFAULT_GIL: c_int;
-    extern var PyInterpreterConfig_SHARED_GIL: c_int;
-    extern var PyInterpreterConfig_OWN_GIL: c_int;
-    extern record PyInterpreterConfig {
-      var use_main_obmalloc: c_int;
-      var allow_fork: c_int;
-      var allow_exec: c_int;
-      var allow_threads: c_int;
-      var allow_daemon_threads: c_int;
-      var check_multi_interp_extensions: c_int;
-      var gil: c_int;
-    }
-    extern proc Py_NewInterpreterFromConfig(tstate_p: c_ptr(PyThreadStatePtr),
-                                            config_: c_ptr(PyInterpreterConfig)): PyStatus;
+    extern proc chpl_Py_NewIsolatedInterpreter(tstate: c_ptr(PyThreadStatePtr)): PyStatus;
     extern proc Py_EndInterpreter(tstate: PyThreadStatePtr);
 
     /*
diff --git a/modules/packages/PythonHelper/ChapelPythonHelper.h b/modules/packages/PythonHelper/ChapelPythonHelper.h
index 59f1f9d47be2..c6d42bcfde90 100644
--- a/modules/packages/PythonHelper/ChapelPythonHelper.h
+++ b/modules/packages/PythonHelper/ChapelPythonHelper.h
@@ -67,4 +67,22 @@ static inline PyObject* chpl_Py_None(void) { return (PyObject*)Py_None; }
 static inline PyObject* chpl_Py_True(void) { return (PyObject*)Py_True; }
 static inline PyObject* chpl_Py_False(void) { return (PyObject*)Py_False; }
 
+
+static inline PyStatus chpl_Py_NewIsolatedInterpreter(PyThreadState** tstate) {
+#if PY_VERSION_HEX >= 0x030c0000 /* Python 3.12 */
+  PyInterpreterConfig config = {
+    .use_main_obmalloc = 0,
+    .allow_fork = 0,
+    .allow_exec = 0,
+    .allow_threads = 1,
+    .allow_daemon_threads = 0,
+    .check_multi_interp_extensions = 1,
+    .gil = PyInterpreterConfig_OWN_GIL,
+  };
+  return Py_NewInterpreterFromConfig(tstate, &config);
+#else
+  return PyStatus_Error("Sub-interpreters are not supported in Python " PY_VERSION);
+#endif
+}
+
 #endif

From 1ea0ed61c45ab73e5b4375ef3cb952affc3321ff Mon Sep 17 00:00:00 2001
From: Jade Abraham <jade.abraham@hpe.com>
Date: Fri, 24 Jan 2025 13:23:18 -0800
Subject: [PATCH 09/13] fix script for indentation

Signed-off-by: Jade Abraham <jade.abraham@hpe.com>
---
 .../packages/Python/doc-examples/parse_docs.py  | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/test/library/packages/Python/doc-examples/parse_docs.py b/test/library/packages/Python/doc-examples/parse_docs.py
index 18d4a3224bda..419e7e420d19 100644
--- a/test/library/packages/Python/doc-examples/parse_docs.py
+++ b/test/library/packages/Python/doc-examples/parse_docs.py
@@ -15,6 +15,19 @@ def basename(self):
         return os.path.splitext(self.filename)[0]
 
     def write(self, directory: str):
+
+        # find the first non-empty line, if it has leading whitespace, remove it
+        # then, remove the same amount of whitespace from all lines
+        whitespace = 0
+        for line in self.file_contents:
+            if line.strip() != "":
+                whitespace = len(line) - len(line.lstrip())
+                break
+        self.file_contents = [
+            line[whitespace:] if len(line) >= whitespace else line
+            for line in self.file_contents
+        ]
+
         with open(os.path.join(directory, self.filename), "w") as f:
             for line in self.file_contents:
                 f.write(line + "\n")
@@ -129,12 +142,12 @@ def has_more():
 
         # add the file lines to the good file
         if test and good_file:
-            good_file.file_contents.append(lines[cur_line].strip())
+            good_file.file_contents.append(lines[cur_line].rstrip())
             cur_line += 1
             continue
         # add the file lines to the test file
         if test and not good_file and test_file:
-            test_file.file_contents.append(lines[cur_line].strip())
+            test_file.file_contents.append(lines[cur_line].rstrip())
             cur_line += 1
             continue
 

From e860abb62d476ca932f90aa4136a9150384c0696 Mon Sep 17 00:00:00 2001
From: Jade Abraham <jade.abraham@hpe.com>
Date: Fri, 24 Jan 2025 13:23:36 -0800
Subject: [PATCH 10/13] expand docs

Signed-off-by: Jade Abraham <jade.abraham@hpe.com>
---
 modules/packages/Python.chpl | 147 +++++++++++++++++++++++++++++------
 1 file changed, 124 insertions(+), 23 deletions(-)

diff --git a/modules/packages/Python.chpl b/modules/packages/Python.chpl
index 527700a29ed0..6cb51255d2ea 100644
--- a/modules/packages/Python.chpl
+++ b/modules/packages/Python.chpl
@@ -62,10 +62,119 @@
   Parallel Execution
   ------------------
 
-  Running any Python code in parallel from Chapel requires special care. Before
-  any parallel execution with Python code can occur, the thread state needs to
-  be saved. After the parallel execution, the thread state must to be restored.
-  Then for each thread, the Global Interpreter Lock (GIL) must be acquired and
+  Running any Python code in parallel from Chapel requires special care, due
+  to the Global Interpreter Lock (GIL) in the Python interpreter.
+  This module supports two ways of doing this.
+
+  .. note::
+
+     Newer Python versions offer a free-threading mode that allows multiple
+     threads concurrently. This currently requires a custom build of Python. If
+     you are using a Python like this, you should be able to use this module
+     freely in parallel code.
+
+  Using Multiple Interpreters
+  ~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+  The most performant way to run Python code in parallel is to use multiple
+  sub-interpreters. Each sub-interpreter is isolated from the others with its
+  own GIL. This allows multiple threads to run Python code concurrently. Note
+  that communication between sub-interpreters is severely limited and it is
+  strongly recommend to limit the amount of data shared between
+  sub-interpreters.
+
+  The following demonstrates using sub-interpreters in a ``coforall`` loop:
+
+  ..
+     START_TEST
+     FILENAME: CoforallTestSub.chpl
+     START_GOOD
+     Hello from a task
+     Hello from a task
+     Hello from a task
+     Hello from a task
+     END_GOOD
+
+  .. code-block:: chapel
+
+     use Python;
+
+     var code = """
+     import sys
+     def hello():
+       print('Hello from a task')
+       sys.stdout.flush()
+     """;
+
+     proc main() {
+       var interpreter = new Interpreter();
+       coforall 0..#4 {
+         var subInterpreter = new SubInterpreter(interpreter);
+         var m = new Module(subInterpreter, 'myMod', code);
+         var hello = new Function(m, 'hello');
+         hello(NoneType);
+       }
+     }
+
+  ..
+     END_TEST
+
+  To make use of a sub-interpreter in a ``forall`` loop, the sub-interpreter
+  should be created as a task private variable. It is recommended that users
+  wrap the sub-interpreter in a ``record`` to initialize their Python objects,
+  to prevent duplicated work. For example, the following code creates multiple
+  sub-interpreters in a ``forall`` loop, where each task gets its own copy of
+  the module.
+
+  ..
+     START_TEST
+     FILENAME: TaskPrivateSubInterp.chpl
+     START_GOOD
+     10
+     10
+     10
+     10
+     10
+     10
+     10
+     10
+     10
+     10
+     END_GOOD
+
+  .. code-block:: chapel
+
+     use Python;
+
+     record perTaskModule {
+      var i: owned SubInterpreter?;
+      var m: owned Module?;
+      proc init(parent: borrowed Interpreter, code: string) {
+        init this;
+        i = try! (new SubInterpreter(parent));
+        m = try! (new Module(i!, "anon", code));
+      }
+     }
+
+     proc main() {
+       var interpreter = new Interpreter();
+       forall i in 1..10
+        with (var mod =
+          new perTaskModule(interpreter, "x = 10")) {
+          writeln(mod.m!.getAttr(int, "x"));
+       }
+     }
+
+  ..
+     END_TEST
+
+  Using A Single Interpreter
+  ~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+  A single Python interpreter can execute code in parallel, as long as the GIL
+  is properly handled. Before any parallel execution with Python code can occur,
+  the thread state needs to be saved. After the parallel execution, the thread
+  state must to be restored. Then for each thread, the GIL must be acquired and
   released. This is necessary to prevent segmentation faults and deadlocks in
   the Python interpreter.
 
@@ -135,20 +244,6 @@
   entirety of each task, these examples will be no faster than running the tasks
   serially.
 
-  .. note::
-
-     Newer Python versions offer a free-threading mode that allows multiple
-     threads concurrently, without the need for the GIL. In this mode, users can
-     either remove the GIL acquisition code or not. Without the GIL, the GIL
-     acquisition code will have no effect.
-
-  .. note::
-
-     In the future, it may be possible to achieve better parallelism with Python
-     by using sub-interpreters. However, sub-interpreters are not yet supported
-     in Chapel and attempting to have more than one :type:`Interpreter` instance
-     will likely result in segmentation faults.
-
   Using Python Modules With Distributed Code
   -------------------------------------------
 
@@ -279,8 +374,8 @@ module Python {
 
     .. warning::
 
-       Multiple/sub interpreters are not yet supported.
-       Do not create more than one instance of this class.
+       Do not create more than one instance of this class per locale. Multiple
+       interpreters can be created by using :type:`SubInterpreter` instances.
   */
   class Interpreter {
 
@@ -911,20 +1006,26 @@ module Python {
 
   }
 
-
+  /*
+    Represents an isolated Python sub-interpreter. This is useful for running
+    truly parallel Python code, without the GIL interferring.
+  */
   class SubInterpreter: Interpreter {
     @chpldoc.nodoc
     var parent: borrowed Interpreter;
     @chpldoc.nodoc
     var tstate: PyThreadStatePtr;
 
-    @chpldoc.nodoc
+    /*
+      Creates a new sub-interpreter with the given parent interpreter, which
+      must not be a sub-interpreter.
+    */
     proc init(parent: borrowed Interpreter) {
       super.init(isSubInterpreter=true);
       this.parent = parent;
       init this;
     }
-    @chpldoc.noddoc
+    @chpldoc.nodoc
     proc postinit() throws {
       if this.parent.isSubInterpreter {
         throwChapelException("Parent interpreter cannot be a sub-interpreter");

From c01442babd9eac9369ae2914edd1f47f4fed64e7 Mon Sep 17 00:00:00 2001
From: Jade Abraham <jade.abraham@hpe.com>
Date: Fri, 24 Jan 2025 13:24:16 -0800
Subject: [PATCH 11/13] remove generated file

Signed-off-by: Jade Abraham <jade.abraham@hpe.com>
---
 test/library/packages/Python/doc-examples/.gitignore            | 2 ++
 .../packages/Python/doc-examples/DistributedTest.execopts       | 1 -
 2 files changed, 2 insertions(+), 1 deletion(-)
 delete mode 100644 test/library/packages/Python/doc-examples/DistributedTest.execopts

diff --git a/test/library/packages/Python/doc-examples/.gitignore b/test/library/packages/Python/doc-examples/.gitignore
index f1f637982eb9..04ce433bf621 100644
--- a/test/library/packages/Python/doc-examples/.gitignore
+++ b/test/library/packages/Python/doc-examples/.gitignore
@@ -1,2 +1,4 @@
 *.chpl
 *.good
+*.execopts
+*.compopts
diff --git a/test/library/packages/Python/doc-examples/DistributedTest.execopts b/test/library/packages/Python/doc-examples/DistributedTest.execopts
deleted file mode 100644
index f728cc6dbf8b..000000000000
--- a/test/library/packages/Python/doc-examples/DistributedTest.execopts
+++ /dev/null
@@ -1 +0,0 @@
---n=10

From 1b671ca64bf497ee9a921cc0c0eb6b589219a251 Mon Sep 17 00:00:00 2001
From: Jade Abraham <jade.abraham@hpe.com>
Date: Fri, 24 Jan 2025 13:25:23 -0800
Subject: [PATCH 12/13] skip docs tests with python <3.12

Signed-off-by: Jade Abraham <jade.abraham@hpe.com>
---
 test/library/packages/Python/doc-examples/SKIPIF | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100755 test/library/packages/Python/doc-examples/SKIPIF

diff --git a/test/library/packages/Python/doc-examples/SKIPIF b/test/library/packages/Python/doc-examples/SKIPIF
new file mode 100755
index 000000000000..33ec676140d5
--- /dev/null
+++ b/test/library/packages/Python/doc-examples/SKIPIF
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+# requires python 3.12+
+
+# respect CHPL_TEST_VENV_DIR if it is set and not none
+if [ -n "$CHPL_TEST_VENV_DIR" ] && [ "$CHPL_TEST_VENV_DIR" != "none" ]; then
+  chpl_python=$CHPL_TEST_VENV_DIR/bin/python3
+else
+  chpl_python=$($CHPL_HOME/util/config/find-python.sh)
+fi
+
+minor_version=$($chpl_python -c "import sys; print(sys.version_info.minor)")
+if [ $minor_version -ge 12 ]; then
+  echo "False"
+else
+  echo "True"
+fi

From 35b61d44a518224f6bcc7369e35738337616c877 Mon Sep 17 00:00:00 2001
From: Jade Abraham <jade.abraham@hpe.com>
Date: Fri, 24 Jan 2025 13:26:23 -0800
Subject: [PATCH 13/13] add python 3.12 docs note

Signed-off-by: Jade Abraham <jade.abraham@hpe.com>
---
 modules/packages/Python.chpl | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/modules/packages/Python.chpl b/modules/packages/Python.chpl
index 6cb51255d2ea..8291789e01e8 100644
--- a/modules/packages/Python.chpl
+++ b/modules/packages/Python.chpl
@@ -83,6 +83,12 @@
   strongly recommend to limit the amount of data shared between
   sub-interpreters.
 
+  .. note::
+
+     This feature is only available in Python 3.12 and later. Attempting to use
+     sub-interpreters with earlier versions of Python will result in a runtime
+     exception.
+
   The following demonstrates using sub-interpreters in a ``coforall`` loop:
 
   ..