[GR-54697] Implement debuginfo generation at image-runtime.

substratevm/CHANGELOG.md

@@ -16,6 +16,7 @@ This changelog summarizes major changes to GraalVM Native Image.
     1. `run-time-initialized-jdk` shifts away from build-time initialization of the JDK, instead initializing most of it at run time. This transition is gradual, with individual components of the JDK becoming run-time initialized in each release. This process should complete with JDK 29 when this option should not be needed anymore. Unless you store classes from the JDK in the image heap, this option should not affect you. In case this option breaks your build, follow the suggestions in the error messages.
 * (GR-63494) Recurring callback support is no longer enabled by default. If this feature is needed, please specify `-H:+SupportRecurringCallback` at image build-time.
 * (GR-60209) New syntax for configuration of the [Foreign Function & Memory API](https://github.com/oracle/graal/blob/master/docs/reference-manual/native-image/ForeignInterface.md)
+* (GR-54697) Parallelize debug info generation and add support for run-time debug info generation. `-H:+RuntimeDebugInfo` adds a run-time debug info generator into a native image for use with GDB.
 
 ## GraalVM for JDK 24 (Internal Version 24.2.0)
 * (GR-59717) Added `DuringSetupAccess.registerObjectReachabilityHandler` to allow registering a callback that is executed when an object of a specified type is marked as reachable during heap scanning.

substratevm/debug/include/gdb_jit_compilation_interface.h

@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SVM_NATIVE_GDBJITCOMPILATIONINTERFACE_H
+#define SVM_NATIVE_GDBJITCOMPILATIONINTERFACE_H
+
+#include <stdint.h>
+
+typedef enum
+{
+  JIT_NOACTION = 0,
+  JIT_REGISTER,
+  JIT_UNREGISTER
+} jit_actions_t;
+
+struct jit_code_entry
+{
+  struct jit_code_entry *next_entry;
+  struct jit_code_entry *prev_entry;
+  const char *symfile_addr;
+  uint64_t symfile_size;
+};
+
+struct jit_descriptor
+{
+  uint32_t version;
+  /* This type should be jit_actions_t, but we use uint32_t
+     to be explicit about the bitwidth.  */
+  uint32_t action_flag;
+  struct jit_code_entry *relevant_entry;
+  struct jit_code_entry *first_entry;
+};
+
+#endif

substratevm/mx.substratevm/gdb_utils.py

@@ -110,6 +110,8 @@ def check(self, text, skip_fails=True):
         for i in range(0, num_rexps):
             rexp = rexps[i]
             match = None
+            if skip_fails:
+                line_idx = 0
             while line_idx < num_lines and match is None:
                 line = lines[line_idx]
                 match = rexp.match(line)

substratevm/mx.substratevm/mx_substratevm.py

@@ -1145,7 +1145,7 @@ def run_debug_test(image_name: str, testfile: str, source_path: str, with_isolat
             else:
                 build_args += ['-o', join(build_dir, image_name)]
 
-            mx.log(f"native_image {' '.join(build_args)}")
+            mx.log(f"native-image {' '.join(build_args)}")
             native_image(build_args)
 
             if build_cinterfacetutorial:
@@ -1158,18 +1158,16 @@ def run_debug_test(image_name: str, testfile: str, source_path: str, with_isolat
                 else:
                     c_command = ['cl', '-MD', join(tutorial_c_source_dir, 'cinterfacetutorial.c'), '-I.',
                                  'libcinterfacetutorial.lib']
-                mx.log(' '.join(c_command))
                 mx.run(c_command, cwd=build_dir)
         if mx.get_os() == 'linux':
             logfile = join(path, pathlib.Path(testfile).stem + ('' if with_isolates else '_no_isolates') + '.log')
-            os.environ.update({'gdbdebughelperstest_logfile': logfile})
+            os.environ.update({'gdb_logfile': logfile})
             gdb_command = gdb_args + [
                 '-iex', f"set logging file {logfile}",
                 '-iex', 'set logging enabled on',
                 '-iex', f"set auto-load safe-path {join(build_dir, 'gdb-debughelpers.py')}",
                 '-x', testfile, join(build_dir, image_name)
             ]
-            mx.log(' '.join(gdb_command))
             # unittest may result in different exit code, nonZeroIsFatal ensures that we can go on with other test
             return mx.run(gdb_command, cwd=build_dir, nonZeroIsFatal=False)
         return 0
@@ -1196,6 +1194,91 @@ def run_debug_test(image_name: str, testfile: str, source_path: str, with_isolat
         mx.abort(status)
 
 
+def _runtimedebuginfotest(native_image, output_path, args=None):
+    """Build and run the runtimedebuginfotest"""
+
+    args = [] if args is None else args
+
+    test_proj = mx.dependency('com.oracle.svm.test')
+    test_source_path = test_proj.source_dirs()[0]
+
+    test_python_source_dir = join(test_source_path, 'com', 'oracle', 'svm', 'test', 'debug', 'helper')
+    test_runtime_compilation_py = join(test_python_source_dir, 'test_runtime_compilation.py')
+    test_runtime_deopt_py = join(test_python_source_dir, 'test_runtime_deopt.py')
+    testdeopt_js = join(suite.dir, 'mx.substratevm', 'testdeopt.js')
+
+    gdb_args = [
+        os.environ.get('GDB_BIN', 'gdb'),
+        '--nx',
+        '-q',  # do not print the introductory and copyright messages
+        '-iex', "set pagination off",  # messages from enabling logging could already cause pagination, so this must be done first
+        '-iex', "set logging redirect on",
+        '-iex', "set logging overwrite off",
+    ]
+
+    # clean / create output directory
+    if exists(output_path):
+        mx.rmtree(output_path)
+    mx_util.ensure_dir_exists(output_path)
+
+    # Build the native image from Java code
+    build_args = [
+        '-g', '-O0',
+        '-o', join(output_path, 'runtimedebuginfotest'),
+        '-cp', classpath('com.oracle.svm.test'),
+        # We do not want to step into class initializer, so initialize everything at build time.
+        '--initialize-at-build-time=com.oracle.svm.test.debug.helper',
+        '--features=com.oracle.svm.test.debug.helper.RuntimeCompileDebugInfoTest$RegisterMethodsFeature',
+        'com.oracle.svm.test.debug.helper.RuntimeCompileDebugInfoTest',
+    ] + svm_experimental_options([
+        '-H:DebugInfoSourceSearchPath=' + test_source_path,
+        '-H:+SourceLevelDebug',
+        '-H:+RuntimeDebugInfo',
+    ]) + args
+
+    mx.log(f"native-image {' '.join(build_args)}")
+    runtime_compile_binary = native_image(build_args)
+
+    logfile = join(output_path, 'test_runtime_compilation.log')
+    os.environ.update({'gdb_logfile': logfile})
+    gdb_command = gdb_args + [
+        '-iex', f"set logging file {logfile}",
+        '-iex', "set logging enabled on",
+        '-iex', f"set auto-load safe-path {join(output_path, 'gdb-debughelpers.py')}",
+        '-x', test_runtime_compilation_py, runtime_compile_binary
+    ]
+    # unittest may result in different exit code, nonZeroIsFatal ensures that we can go on with other test
+    status = mx.run(gdb_command, cwd=output_path, nonZeroIsFatal=False)
+
+    def run_js_test(eager: bool = False):
+        jslib = mx.add_lib_suffix(native_image(
+            args +
+            svm_experimental_options([
+                '-H:+SourceLevelDebug',
+                '-H:+RuntimeDebugInfo',
+                '-H:+LazyDeoptimization' if eager else '-H:-LazyDeoptimization',
+            ]) +
+            ['-g', '-O0', '--macro:jsvm-library']
+        ))
+        js_launcher = get_js_launcher(jslib)
+        logfile = join(output_path, 'test_runtime_deopt_' + ('eager' if eager else 'lazy') + '.log')
+        os.environ.update({'gdb_logfile': logfile})
+        gdb_command = gdb_args + [
+            '-iex', f"set logging file {logfile}",
+            '-iex', "set logging enabled on",
+            '-iex', f"set auto-load safe-path {join(output_path, 'gdb-debughelpers.py')}",
+            '-x', test_runtime_deopt_py, '--args', js_launcher, testdeopt_js
+        ]
+        # unittest may result in different exit code, nonZeroIsFatal ensures that we can go on with other test
+        return mx.run(gdb_command, cwd=output_path, nonZeroIsFatal=False)
+
+    status |= run_js_test()
+    status |= run_js_test(True)
+
+    if status != 0:
+        mx.abort(status)
+
+
 def _javac_image(native_image, path, args=None):
     args = [] if args is None else args
     mx_util.ensure_dir_exists(path)
@@ -1788,6 +1871,26 @@ def gdbdebughelperstest(args, config=None):
         config=config
     )
 
+
+@mx.command(suite.name, 'runtimedebuginfotest', 'Runs the runtime debuginfo generation test')
+def runtimedebuginfotest(args, config=None):
+    """
+    runs a native image that compiles code and creates debug info at runtime.
+    """
+    parser = ArgumentParser(prog='mx runtimedebuginfotest')
+    all_args = ['--output-path', '--with-isolates-only']
+    masked_args = [_mask(arg, all_args) for arg in args]
+    parser.add_argument(all_args[0], metavar='<output-path>', nargs=1, help='Path of the generated image', default=[join(svmbuild_dir(), "runtimedebuginfotest")])
+    parser.add_argument('image_args', nargs='*', default=[])
+    parsed = parser.parse_args(masked_args)
+    output_path = unmask(parsed.output_path)[0]
+    native_image_context_run(
+        lambda native_image, a:
+        _runtimedebuginfotest(native_image, output_path, a), unmask(parsed.image_args),
+        config=config
+    )
+
+
 @mx.command(suite_name=suite.name, command_name='helloworld', usage_msg='[options]')
 def helloworld(args, config=None):
     """
@@ -1880,6 +1983,7 @@ def build_and_test_java_agent_image(native_image, args):
 
     native_image_context_run(build_and_test_java_agent_image, args)
 
+
 @mx.command(suite.name, 'clinittest', 'Runs the ')
 def clinittest(args):
     def build_and_test_clinittest_image(native_image, args):

substratevm/mx.substratevm/suite.py

@@ -336,6 +336,7 @@
             "dependencies": [
                 "com.oracle.svm.common",
                 "com.oracle.svm.shaded.org.objectweb.asm",
+                "com.oracle.objectfile",
                 "SVM_CONFIGURE",
             ],
             "requires" : [
@@ -708,7 +709,6 @@
             "subDir": "src",
             "sourceDirs": ["src"],
             "dependencies": [
-                "com.oracle.objectfile",
                 "com.oracle.graal.reachability",
                 "com.oracle.svm.core.graal.amd64",
                 "com.oracle.svm.shaded.org.capnproto",
@@ -1119,6 +1119,10 @@
                     "jdk.internal.misc",
                     "sun.security.jca",
                 ],
+                "jdk.internal.vm.ci" : [
+                    "jdk.vm.ci.code",
+                    "jdk.vm.ci.meta",
+                ],
             },
             "checkstyle": "com.oracle.svm.test",
             "checkstyleVersion" : "10.21.0",
@@ -1930,6 +1934,8 @@
                 "com.oracle.objectfile",
                 "com.oracle.objectfile.io",
                 "com.oracle.objectfile.debuginfo",
+                "com.oracle.objectfile.debugentry",
+                "com.oracle.objectfile.debugentry.range",
                 "com.oracle.objectfile.macho",
               ],
 
@@ -2060,6 +2066,7 @@
                             "dependency:com.oracle.svm.native.libchelper/*",
                             "dependency:com.oracle.svm.native.jvm.posix/*",
                             "dependency:com.oracle.svm.native.libcontainer/*",
+                            "file:debug/include",
                         ],
                     },
                 },
@@ -2068,6 +2075,7 @@
                         # on all other os's we don't want libc specific subdirectories
                         "include/": [
                             "dependency:com.oracle.svm.native.libchelper/include/*",
+                            "file:debug/include/*",
                         ],
                         "<os>-<arch>/": [
                             "dependency:com.oracle.svm.native.libchelper/<os>-<arch>/default/*",

substratevm/mx.substratevm/testdeopt.js

@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+function add(a, b, test) {
+    if (test) {
+        a += b;
+    }
+    return a + b;
+}
+
+// trigger compilation add for ints and test = true
+for (let i = 0; i < 1000 * 1000; i++) {
+    add(i, i, true);
+}
+
+// deoptimize with failed assumption in compiled method
+// then trigger compilation again
+console.log("deopt1")
+for (let i = 0; i < 1000 * 1000; i++) {
+    add(i, i, false);
+}
+
+// deoptimize with different parameter types
+console.log("deopt2");
+add({f1: "test1", f2: 2}, {x: "x", y: {test: 42}}, false);

substratevm/mx.substratevm/testhello.py

@@ -147,25 +147,25 @@ def test():
 
     # check incoming parameters are bound to sensible values
     exec_string = execute("info args")
-    rexp = [fr"__0 = {digits_pattern}",
-            fr"__1 = 0x{hex_digits_pattern}"]
+    rexp = [fr"__int0 = {digits_pattern}",
+            fr"__long1 = 0x{hex_digits_pattern}"]
     checker = Checker(f"info args : {method_name}", rexp)
     checker.check(exec_string)
 
-    exec_string = execute("p __0")
+    exec_string = execute("p __int0")
     rexp = [fr"\${digits_pattern} = 1"]
-    checker = Checker("p __0", rexp)
+    checker = Checker("p __int0", rexp)
     checker.check(exec_string)
 
-    exec_string = execute("p __1")
+    exec_string = execute("p __long1")
     rexp = [fr"\${digits_pattern} = \(org\.graalvm\.nativeimage\.c\.type\.CCharPointerPointer\) 0x{hex_digits_pattern}"]
-    checker = Checker("p __1", rexp)
+    checker = Checker("p __long1", rexp)
     checker.check(exec_string)
 
-    exec_string = execute("p __1[0]")
+    exec_string = execute("p __long1[0]")
     rexp = [
         fr'\${digits_pattern} = \(org\.graalvm\.nativeimage\.c\.type\.CCharPointer\) 0x{hex_digits_pattern} "{wildcard_pattern}/hello_image"']
-    checker = Checker("p __1[0]", rexp)
+    checker = Checker("p __long1[0]", rexp)
     checker.check(exec_string)
 
     # set a break point at hello.Hello::main
@@ -971,7 +971,7 @@ def test():
     exec_string = execute("info types com.oracle.svm.test.debug.CStructTests\$")
     rexp = [
         fr"{spaces_pattern}typedef composite_struct \* com\.oracle\.svm\.test\.debug\.CStructTests\$CompositeStruct;",
-        fr"{spaces_pattern}typedef int32_t \* com\.oracle\.svm\.test\.debug\.CStructTests\$MyCIntPointer;",
+        fr"{spaces_pattern}typedef int \* com\.oracle\.svm\.test\.debug\.CStructTests\$MyCIntPointer;",
         fr"{spaces_pattern}typedef simple_struct \* com\.oracle\.svm\.test\.debug\.CStructTests\$SimpleStruct;",
         fr"{spaces_pattern}typedef simple_struct2 \* com\.oracle\.svm\.test\.debug\.CStructTests\$SimpleStruct2;",
         fr"{spaces_pattern}typedef weird \* com\.oracle\.svm\.test\.debug\.CStructTests\$Weird;"]
@@ -1012,8 +1012,8 @@ def test():
             fr"/\*{spaces_pattern}24{spaces_pattern}\|{spaces_pattern}4{spaces_pattern}\*/{spaces_pattern}float f_float;",
             fr"/\*{spaces_pattern}XXX{spaces_pattern}4-byte hole{spaces_pattern}\*/",
             fr"/\*{spaces_pattern}32{spaces_pattern}\|{spaces_pattern}8{spaces_pattern}\*/{spaces_pattern}double f_double;",
-            fr"/\*{spaces_pattern}40{spaces_pattern}\|{spaces_pattern}32{spaces_pattern}\*/{spaces_pattern}int32_t a_int\[8\];",
-            fr"/\*{spaces_pattern}72{spaces_pattern}\|{spaces_pattern}12{spaces_pattern}\*/{spaces_pattern}(u)?int8_t a_char\[12\];",
+            fr"/\*{spaces_pattern}40{spaces_pattern}\|{spaces_pattern}32{spaces_pattern}\*/{spaces_pattern}int a_int\[8\];",
+            fr"/\*{spaces_pattern}72{spaces_pattern}\|{spaces_pattern}12{spaces_pattern}\*/{spaces_pattern}char a_char\[12\];",
             fr"/\*{spaces_pattern}XXX{spaces_pattern}4-byte padding{spaces_pattern}\*/",
             fr"{spaces_pattern}/\* total size \(bytes\):{spaces_pattern}88 \*/",
             fr"{spaces_pattern}}} \*"]