Skip to content

Commit 01e3184

Browse files
DinoVDinoV
committed
Add remote introspection support
1 parent 4f4ecb7 commit 01e3184

File tree

9 files changed

+171
-11
lines changed

9 files changed

+171
-11
lines changed

‎Include/cpython/pyframe.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,4 +43,4 @@ PyAPI_FUNC(int) PyUnstable_InterpreterFrame_GetLine(struct _PyInterpreterFrame *
4343
#define PyUnstable_EXECUTABLE_KIND_JIT 5
4444
#define PyUnstable_EXECUTABLE_KINDS 6
4545

46-
PyAPI_DATA(const PyTypeObject *) const PyUnstable_ExecutableKinds[PyUnstable_EXECUTABLE_KINDS+1];
46+
PyAPI_DATA(PyTypeObject *) PyUnstable_ExecutableKinds[PyUnstable_EXECUTABLE_KINDS+1];

‎Include/internal/pycore_debug_offsets.h‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,10 @@ typedef struct _Py_DebugOffsets {
131131
uint64_t tlbc_index;
132132
} interpreter_frame;
133133

134+
struct _interpreter_frame_metadata {
135+
uintptr_t executable_kinds;
136+
} interpreter_frame_metadata;
137+
134138
// Code object offset;
135139
struct _code_object {
136140
uint64_t size;
@@ -146,6 +150,10 @@ typedef struct _Py_DebugOffsets {
146150
uint64_t co_tlbc;
147151
} code_object;
148152

153+
struct _jit_executable {
154+
uint64_t code;
155+
} jit_executable;
156+
149157
// PyObject offset;
150158
struct _pyobject {
151159
uint64_t size;
@@ -305,6 +313,9 @@ typedef struct _Py_DebugOffsets {
305313
.stackpointer = offsetof(_PyInterpreterFrame, stackpointer), \
306314
.tlbc_index = _Py_Debug_interpreter_frame_tlbc_index, \
307315
}, \
316+
.interpreter_frame_metadata = { \
317+
.executable_kinds = (uintptr_t)PyUnstable_ExecutableKinds, \
318+
}, \
308319
.code_object = { \
309320
.size = sizeof(PyCodeObject), \
310321
.filename = offsetof(PyCodeObject, co_filename), \
@@ -318,6 +329,9 @@ typedef struct _Py_DebugOffsets {
318329
.co_code_adaptive = offsetof(PyCodeObject, co_code_adaptive), \
319330
.co_tlbc = _Py_Debug_code_object_co_tlbc, \
320331
}, \
332+
.jit_executable = { \
333+
.code = offsetof(PyUnstable_PyJitExecutable, je_code), \
334+
}, \
321335
.pyobject = { \
322336
.size = sizeof(PyObject), \
323337
.ob_type = offsetof(PyObject, ob_type), \

‎Lib/test/test_external_inspection.py‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3534,5 +3534,104 @@ def test_get_stats_disabled_raises(self):
35343534
client_socket.sendall(b"done")
35353535

35363536

3537+
class TestNonCodeExecutable(RemoteInspectionTestBase):
3538+
@skip_if_not_supported
3539+
@unittest.skipIf(
3540+
sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED,
3541+
"Test only runs on Linux with process_vm_readv support",
3542+
)
3543+
def test_remote_stack_trace(self):
3544+
port = find_unused_port()
3545+
script = textwrap.dedent(
3546+
f"""\
3547+
import time, sys, socket, threading
3548+
import _testinternalcapi
3549+
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
3550+
sock.connect(('localhost', {port}))
3551+
3552+
def bar():
3553+
for x in range(100):
3554+
if x == 50:
3555+
_testinternalcapi.call_with_jit_frame(baz, foo, ())
3556+
3557+
def baz():
3558+
pass
3559+
3560+
def foo():
3561+
sock.sendall(b"ready:thread\\n"); time.sleep(10_000)
3562+
3563+
t = threading.Thread(target=bar)
3564+
t.start()
3565+
sock.sendall(b"ready:main\\n"); t.join()
3566+
"""
3567+
)
3568+
3569+
with os_helper.temp_dir() as work_dir:
3570+
script_dir = os.path.join(work_dir, "script_pkg")
3571+
os.mkdir(script_dir)
3572+
3573+
server_socket = _create_server_socket(port)
3574+
script_name = _make_test_script(script_dir, "script", script)
3575+
client_socket = None
3576+
3577+
try:
3578+
with _managed_subprocess([sys.executable, script_name]) as p:
3579+
client_socket, _ = server_socket.accept()
3580+
server_socket.close()
3581+
server_socket = None
3582+
3583+
_wait_for_signal(
3584+
client_socket, [b"ready:main", b"ready:thread"]
3585+
)
3586+
3587+
try:
3588+
stack_trace = get_stack_trace(p.pid)
3589+
except PermissionError:
3590+
self.skipTest(
3591+
"Insufficient permissions to read the stack trace"
3592+
)
3593+
3594+
thread_expected_stack_trace = [
3595+
FrameInfo([script_name, 15, "foo"]),
3596+
# external frame line number is function start
3597+
FrameInfo([script_name, 11, "baz"]),
3598+
FrameInfo([script_name, 9, "bar"]),
3599+
FrameInfo([threading.__file__, ANY, "Thread.run"]),
3600+
FrameInfo(
3601+
[
3602+
threading.__file__,
3603+
ANY,
3604+
"Thread._bootstrap_inner",
3605+
]
3606+
),
3607+
FrameInfo(
3608+
[threading.__file__, ANY, "Thread._bootstrap"]
3609+
),
3610+
]
3611+
3612+
# Find expected thread stack
3613+
found_thread = self._find_thread_with_frame(
3614+
stack_trace,
3615+
lambda f: f.funcname == "foo" and f.lineno == 15,
3616+
)
3617+
self.assertIsNotNone(
3618+
found_thread, "Expected thread stack trace not found"
3619+
)
3620+
self.assertEqual(
3621+
found_thread.frame_info, thread_expected_stack_trace
3622+
)
3623+
3624+
# Check main thread
3625+
main_frame = FrameInfo([script_name, 19, "<module>"])
3626+
found_main = self._find_frame_in_trace(
3627+
stack_trace, lambda f: f == main_frame
3628+
)
3629+
self.assertIsNotNone(
3630+
found_main, "Main thread stack trace not found"
3631+
)
3632+
finally:
3633+
_cleanup_sockets(client_socket, server_socket)
3634+
3635+
35373636
if __name__ == "__main__":
35383637
unittest.main()

‎Modules/_remote_debugging/_remote_debugging.h‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -238,6 +238,7 @@ typedef struct {
238238
RemoteDebuggingState *cached_state;
239239
FrameCacheEntry *frame_cache; // preallocated array of FRAME_CACHE_MAX_THREADS entries
240240
UnwinderStats stats; // statistics for performance analysis
241+
uintptr_t frame_executable_types[PyUnstable_EXECUTABLE_KINDS];
241242
#ifdef Py_GIL_DISABLED
242243
uint32_t tlbc_generation;
243244
_Py_hashtable_t *tlbc_cache;
@@ -332,7 +333,7 @@ extern long read_py_long(RemoteUnwinderObject *unwinder, uintptr_t address);
332333
* CODE OBJECT FUNCTION DECLARATIONS
333334
* ============================================================================ */
334335

335-
extern int parse_code_object(
336+
extern int parse_executable_object(
336337
RemoteUnwinderObject *unwinder,
337338
PyObject **result,
338339
uintptr_t address,
@@ -473,6 +474,8 @@ extern int populate_initial_state_data(
473474
uintptr_t *tstate
474475
);
475476

477+
extern int populate_frame_executable_types(RemoteUnwinderObject *unwinder);
478+
476479
extern int find_running_frame(
477480
RemoteUnwinderObject *unwinder,
478481
uintptr_t address_of_thread,

‎Modules/_remote_debugging/code_objects.c‎

Lines changed: 23 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -275,12 +275,12 @@ make_frame_info(RemoteUnwinderObject *unwinder, PyObject *file, PyObject *locati
275275
}
276276

277277
int
278-
parse_code_object(RemoteUnwinderObject *unwinder,
279-
PyObject **result,
280-
uintptr_t address,
281-
uintptr_t instruction_pointer,
282-
uintptr_t *previous_frame,
283-
int32_t tlbc_index)
278+
parse_executable_object(RemoteUnwinderObject *unwinder,
279+
PyObject **result,
280+
uintptr_t address,
281+
uintptr_t instruction_pointer,
282+
uintptr_t *previous_frame,
283+
int32_t tlbc_index)
284284
{
285285
void *key = (void *)address;
286286
CachedCodeMetadata *meta = NULL;
@@ -314,6 +314,23 @@ parse_code_object(RemoteUnwinderObject *unwinder,
314314
goto error;
315315
}
316316

317+
// Check for non code type executables
318+
uintptr_t code_type = GET_MEMBER(uintptr_t, code_object, unwinder->debug_offsets.pyobject.ob_type);
319+
if (code_type != unwinder->frame_executable_types[PyUnstable_EXECUTABLE_KIND_PY_FUNCTION]) {
320+
if (code_type != unwinder->frame_executable_types[PyUnstable_EXECUTABLE_KIND_JIT]) {
321+
// Unsupported executable type, report the frame as being invalid
322+
return 0;
323+
}
324+
real_address = GET_MEMBER(uintptr_t, code_object, unwinder->debug_offsets.jit_executable.code);
325+
if (_Py_RemoteDebug_PagedReadRemoteMemory(
326+
&unwinder->handle, real_address, SIZEOF_CODE_OBJ, code_object) < 0)
327+
{
328+
set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read code object");
329+
goto error;
330+
}
331+
}
332+
333+
317334
func = read_py_str(unwinder,
318335
GET_MEMBER(uintptr_t, code_object, unwinder->debug_offsets.code_object.qualname), 1024);
319336
if (!func) {

‎Modules/_remote_debugging/frames.c‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,26 @@
1111
* STACK CHUNK MANAGEMENT FUNCTIONS
1212
* ============================================================================ */
1313

14+
int
15+
populate_frame_executable_types(RemoteUnwinderObject *unwinder)
16+
{
17+
uintptr_t executable_kinds_addr =
18+
(uintptr_t)unwinder->debug_offsets.interpreter_frame_metadata.executable_kinds;
19+
20+
int bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory(
21+
&unwinder->handle,
22+
executable_kinds_addr,
23+
sizeof(uintptr_t) * PyUnstable_EXECUTABLE_KINDS,
24+
(void *)unwinder->frame_executable_types);
25+
26+
if (bytes_read < 0) {
27+
set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read frame executable types");
28+
return -1;
29+
}
30+
31+
return 0;
32+
}
33+
1434
void
1535
cleanup_stack_chunks(StackChunkList *chunks)
1636
{
@@ -209,7 +229,7 @@ parse_frame_object(
209229
#endif
210230

211231
*address_of_code_object = code_object;
212-
return parse_code_object(unwinder, result, code_object, instruction_pointer, previous_frame, tlbc_index);
232+
return parse_executable_object(unwinder, result, code_object, instruction_pointer, previous_frame, tlbc_index);
213233
}
214234

215235
int
@@ -246,7 +266,7 @@ parse_frame_from_chunks(
246266
}
247267
#endif
248268

249-
return parse_code_object(unwinder, result, code_object, instruction_pointer, previous_frame, tlbc_index);
269+
return parse_executable_object(unwinder, result, code_object, instruction_pointer, previous_frame, tlbc_index);
250270
}
251271

252272
/* ============================================================================

‎Modules/_remote_debugging/module.c‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -370,6 +370,12 @@ _remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self,
370370
return -1;
371371
}
372372

373+
if (populate_frame_executable_types(self) < 0)
374+
{
375+
set_exception_cause(self, PyExc_RuntimeError, "Failed to populate initial state data");
376+
return -1;
377+
}
378+
373379
self->code_object_cache = _Py_hashtable_new_full(
374380
_Py_hashtable_hash_ptr,
375381
_Py_hashtable_compare_direct,

‎Python/frame.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -248,7 +248,7 @@ PyUnstable_MakeJITExecutable(_PyFrame_Reifier reifier, PyCodeObject *code, PyObj
248248
return (PyObject *)jit_exec;
249249
}
250250

251-
const PyTypeObject *const PyUnstable_ExecutableKinds[PyUnstable_EXECUTABLE_KINDS+1] = {
251+
PyTypeObject *PyUnstable_ExecutableKinds[PyUnstable_EXECUTABLE_KINDS+1] = {
252252
[PyUnstable_EXECUTABLE_KIND_SKIP] = &_PyNone_Type,
253253
[PyUnstable_EXECUTABLE_KIND_PY_FUNCTION] = &PyCode_Type,
254254
[PyUnstable_EXECUTABLE_KIND_BUILTIN_FUNCTION] = &PyMethod_Type,

‎Tools/c-analyzer/cpython/globals-to-fix.tsv‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -325,6 +325,7 @@ Objects/sliceobject.c - _Py_EllipsisObject -
325325
Objects/typevarobject.c - _Py_NoDefaultStruct -
326326
Python/instrumentation.c - _PyInstrumentation_DISABLE -
327327
Python/instrumentation.c - _PyInstrumentation_MISSING -
328+
Python/frame.c - PyUnstable_ExecutableKinds -
328329

329330

330331
##################################

0 commit comments

Comments
 (0)