Add remote introspection support

Author: DinoV

Include/cpython/pyframe.h

@@ -43,4 +43,4 @@ PyAPI_FUNC(int) PyUnstable_InterpreterFrame_GetLine(struct _PyInterpreterFrame *
 #define PyUnstable_EXECUTABLE_KIND_JIT 5
 #define PyUnstable_EXECUTABLE_KINDS 6
 
-PyAPI_DATA(const PyTypeObject *) const PyUnstable_ExecutableKinds[PyUnstable_EXECUTABLE_KINDS+1];
+PyAPI_DATA(PyTypeObject *) PyUnstable_ExecutableKinds[PyUnstable_EXECUTABLE_KINDS+1];

Include/internal/pycore_debug_offsets.h

@@ -131,6 +131,10 @@ typedef struct _Py_DebugOffsets {
         uint64_t tlbc_index;
     } interpreter_frame;
 
+    struct _interpreter_frame_metadata {
+        uintptr_t executable_kinds;
+    } interpreter_frame_metadata;
+
     // Code object offset;
     struct _code_object {
         uint64_t size;
@@ -146,6 +150,10 @@ typedef struct _Py_DebugOffsets {
         uint64_t co_tlbc;
     } code_object;
 
+    struct _jit_executable {
+        uint64_t code;
+    } jit_executable;
+
     // PyObject offset;
     struct _pyobject {
         uint64_t size;
@@ -305,6 +313,9 @@ typedef struct _Py_DebugOffsets {
         .stackpointer = offsetof(_PyInterpreterFrame, stackpointer), \
         .tlbc_index = _Py_Debug_interpreter_frame_tlbc_index, \
     }, \
+    .interpreter_frame_metadata = { \
+        .executable_kinds = (uintptr_t)PyUnstable_ExecutableKinds, \
+    }, \
     .code_object = { \
         .size = sizeof(PyCodeObject), \
         .filename = offsetof(PyCodeObject, co_filename), \
@@ -318,6 +329,9 @@ typedef struct _Py_DebugOffsets {
         .co_code_adaptive = offsetof(PyCodeObject, co_code_adaptive), \
         .co_tlbc = _Py_Debug_code_object_co_tlbc, \
     }, \
+    .jit_executable = { \
+        .code = offsetof(PyUnstable_PyJitExecutable, je_code), \
+    }, \
     .pyobject = { \
         .size = sizeof(PyObject), \
         .ob_type = offsetof(PyObject, ob_type), \

Lib/test/test_external_inspection.py

@@ -3534,5 +3534,89 @@ def test_get_stats_disabled_raises(self):
             client_socket.sendall(b"done")
 
 
+class TestNonCodeExecutable(RemoteInspectionTestBase):
+    @skip_if_not_supported
+    @unittest.skipIf(
+        sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED,
+        "Test only runs on Linux with process_vm_readv support",
+    )
+    def test_remote_stack_trace(self):
+        port = find_unused_port()
+        script = textwrap.dedent(
+            f"""\
+            import time, sys, socket, threading
+            import _testinternalcapi
+            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            sock.connect(('localhost', {port}))
+
+            def bar():
+                for x in range(100):
+                    if x == 50:
+                        _testinternalcapi.call_with_jit_frame(baz, foo, ())
+
+            def baz():
+                pass
+
+            def foo():
+                sock.sendall(b"ready:thread\\n"); time.sleep(10_000)
+
+            t = threading.Thread(target=bar)
+            t.start()
+            sock.sendall(b"ready:main\\n"); t.join()
+            """
+        )
+
+        with os_helper.temp_dir() as work_dir:
+            script_dir = os.path.join(work_dir, "script_pkg")
+            os.mkdir(script_dir)
+
+            server_socket = _create_server_socket(port)
+            script_name = _make_test_script(script_dir, "script", script)
+            client_socket = None
+
+            try:
+                with _managed_subprocess([sys.executable, script_name]) as p:
+                    client_socket, _ = server_socket.accept()
+                    server_socket.close()
+                    server_socket = None
+
+                    _wait_for_signal(
+                        client_socket, [b"ready:main", b"ready:thread"]
+                    )
+
+                    try:
+                        stack_trace = get_stack_trace(p.pid)
+                    except PermissionError:
+                        self.skipTest(
+                            "Insufficient permissions to read the stack trace"
+                        )
+
+                    # Find expected thread stack by funcname
+                    found_thread = self._find_thread_with_frame(
+                        stack_trace,
+                        lambda f: f.funcname == "foo" and f.location.lineno == 15,
+                    )
+                    self.assertIsNotNone(
+                        found_thread, "Expected thread stack trace not found"
+                    )
+                    # Check the funcnames in order
+                    funcnames = [f.funcname for f in found_thread.frame_info]
+                    self.assertEqual(
+                        funcnames[:6],
+                        ["foo", "baz", "bar", "Thread.run", "Thread._bootstrap_inner", "Thread._bootstrap"]
+                    )
+
+                    # Check main thread
+                    found_main = self._find_frame_in_trace(
+                        stack_trace,
+                        lambda f: f.funcname == "<module>" and f.location.lineno == 19,
+                    )
+                    self.assertIsNotNone(
+                        found_main, "Main thread stack trace not found"
+                    )
+            finally:
+                _cleanup_sockets(client_socket, server_socket)
+
+
 if __name__ == "__main__":
     unittest.main()

Modules/_remote_debugging/_remote_debugging.h

@@ -238,6 +238,7 @@ typedef struct {
     RemoteDebuggingState *cached_state;
     FrameCacheEntry *frame_cache;  // preallocated array of FRAME_CACHE_MAX_THREADS entries
     UnwinderStats stats;  // statistics for performance analysis
+    uintptr_t frame_executable_types[PyUnstable_EXECUTABLE_KINDS];
 #ifdef Py_GIL_DISABLED
     uint32_t tlbc_generation;
     _Py_hashtable_t *tlbc_cache;
@@ -332,7 +333,7 @@ extern long read_py_long(RemoteUnwinderObject *unwinder, uintptr_t address);
  * CODE OBJECT FUNCTION DECLARATIONS
  * ============================================================================ */
 
-extern int parse_code_object(
+extern int parse_executable_object(
     RemoteUnwinderObject *unwinder,
     PyObject **result,
     uintptr_t address,
@@ -473,6 +474,8 @@ extern int populate_initial_state_data(
     uintptr_t *tstate
 );
 
+extern int populate_frame_executable_types(RemoteUnwinderObject *unwinder);
+
 extern int find_running_frame(
     RemoteUnwinderObject *unwinder,
     uintptr_t address_of_thread,

Modules/_remote_debugging/code_objects.c

@@ -275,12 +275,12 @@ make_frame_info(RemoteUnwinderObject *unwinder, PyObject *file, PyObject *locati
 }
 
 int
-parse_code_object(RemoteUnwinderObject *unwinder,
-                  PyObject **result,
-                  uintptr_t address,
-                  uintptr_t instruction_pointer,
-                  uintptr_t *previous_frame,
-                  int32_t tlbc_index)
+parse_executable_object(RemoteUnwinderObject *unwinder,
+                        PyObject **result,
+                        uintptr_t address,
+                        uintptr_t instruction_pointer,
+                        uintptr_t *previous_frame,
+                        int32_t tlbc_index)
 {
     void *key = (void *)address;
     CachedCodeMetadata *meta = NULL;
@@ -314,6 +314,23 @@ parse_code_object(RemoteUnwinderObject *unwinder,
             goto error;
         }
 
+        // Check for non code type executables
+        uintptr_t code_type = GET_MEMBER(uintptr_t, code_object, unwinder->debug_offsets.pyobject.ob_type);
+        if (code_type != unwinder->frame_executable_types[PyUnstable_EXECUTABLE_KIND_PY_FUNCTION]) {
+            if (code_type != unwinder->frame_executable_types[PyUnstable_EXECUTABLE_KIND_JIT]) {
+                // Unsupported executable type, report the frame as being invalid
+                return 0;
+            }
+            real_address = GET_MEMBER(uintptr_t, code_object, unwinder->debug_offsets.jit_executable.code);
+            if (_Py_RemoteDebug_PagedReadRemoteMemory(
+                    &unwinder->handle, real_address, SIZEOF_CODE_OBJ, code_object) < 0)
+            {
+                set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read code object");
+                goto error;
+            }
+        }
+
+
         func = read_py_str(unwinder,
             GET_MEMBER(uintptr_t, code_object, unwinder->debug_offsets.code_object.qualname), 1024);
         if (!func) {

Modules/_remote_debugging/frames.c

@@ -11,6 +11,26 @@
  * STACK CHUNK MANAGEMENT FUNCTIONS
  * ============================================================================ */
 
+int
+populate_frame_executable_types(RemoteUnwinderObject *unwinder)
+{
+    uintptr_t executable_kinds_addr =
+        (uintptr_t)unwinder->debug_offsets.interpreter_frame_metadata.executable_kinds;
+
+    int bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory(
+            &unwinder->handle,
+            executable_kinds_addr,
+            sizeof(uintptr_t) * PyUnstable_EXECUTABLE_KINDS,
+            (void *)unwinder->frame_executable_types);
+
+    if (bytes_read < 0) {
+        set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read frame executable types");
+        return -1;
+    }
+
+    return 0;
+}
+
 void
 cleanup_stack_chunks(StackChunkList *chunks)
 {
@@ -209,7 +229,7 @@ parse_frame_object(
 #endif
 
     *address_of_code_object = code_object;
-    return parse_code_object(unwinder, result, code_object, instruction_pointer, previous_frame, tlbc_index);
+    return parse_executable_object(unwinder, result, code_object, instruction_pointer, previous_frame, tlbc_index);
 }
 
 int
@@ -246,7 +266,7 @@ parse_frame_from_chunks(
     }
 #endif
 
-    return parse_code_object(unwinder, result, code_object, instruction_pointer, previous_frame, tlbc_index);
+    return parse_executable_object(unwinder, result, code_object, instruction_pointer, previous_frame, tlbc_index);
 }
 
 /* ============================================================================

Modules/_remote_debugging/module.c

@@ -370,6 +370,12 @@ _remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self,
         return -1;
     }
 
+    if (populate_frame_executable_types(self) < 0)
+    {
+        set_exception_cause(self, PyExc_RuntimeError, "Failed to populate initial state data");
+        return -1;
+    }
+
     self->code_object_cache = _Py_hashtable_new_full(
         _Py_hashtable_hash_ptr,
         _Py_hashtable_compare_direct,

Python/frame.c

@@ -248,7 +248,7 @@ PyUnstable_MakeJITExecutable(_PyFrame_Reifier reifier, PyCodeObject *code, PyObj
     return (PyObject *)jit_exec;
 }
 
-const PyTypeObject *const PyUnstable_ExecutableKinds[PyUnstable_EXECUTABLE_KINDS+1] = {
+PyTypeObject *PyUnstable_ExecutableKinds[PyUnstable_EXECUTABLE_KINDS+1] = {
     [PyUnstable_EXECUTABLE_KIND_SKIP] = &_PyNone_Type,
     [PyUnstable_EXECUTABLE_KIND_PY_FUNCTION] = &PyCode_Type,
     [PyUnstable_EXECUTABLE_KIND_BUILTIN_FUNCTION] = &PyMethod_Type,

Tools/c-analyzer/cpython/globals-to-fix.tsv

@@ -325,6 +325,7 @@ Objects/sliceobject.c	-	_Py_EllipsisObject	-
 Objects/typevarobject.c	-	_Py_NoDefaultStruct	-
 Python/instrumentation.c	-	_PyInstrumentation_DISABLE	-
 Python/instrumentation.c	-	_PyInstrumentation_MISSING	-
+Python/frame.c	-	PyUnstable_ExecutableKinds	-
 
 
 ##################################

Tools/check-c-api-docs/ignored_c_api.txt

@@ -81,6 +81,7 @@ PyLong_SHIFT
 # cpython/pyerrors.h
 PyException_HEAD
 # cpython/pyframe.h
+PyUnstable_ExecutableKinds
 PyUnstable_EXECUTABLE_KINDS
 PyUnstable_EXECUTABLE_KIND_BUILTIN_FUNCTION
 PyUnstable_EXECUTABLE_KIND_JIT