[3.14] gh-145219: Add Emscripten cross-build and clean configurability (GH-145581) (#145654)

Modifies the Emscripten build script to allow for custom cross-build directory
names, and to only clean Emscripten-specific paths (optionally including the
build python).
(cherry picked from commit 0156133)

Co-authored-by: Hood Chatham <roberthoodchatham@gmail.com>
Co-authored-by: Russell Keith-Magee <russell@keith-magee.com>

Author: 3 people

.gitignore

@@ -135,7 +135,7 @@ Tools/unicode/data/
 /config.status
 /config.status.lineno
 /.ccache
-/cross-build/
+/cross-build*/
 /jit_stencils*.h
 /platform
 /profile-clean-stamp

Tools/wasm/emscripten/__main__.py

@@ -24,14 +24,25 @@
 CHECKOUT = EMSCRIPTEN_DIR.parent.parent.parent
 EMSCRIPTEN_VERSION_FILE = EMSCRIPTEN_DIR / "emscripten_version.txt"
 
-CROSS_BUILD_DIR = CHECKOUT / "cross-build"
-NATIVE_BUILD_DIR = CROSS_BUILD_DIR / "build"
+DEFAULT_CROSS_BUILD_DIR = CHECKOUT / "cross-build"
 HOST_TRIPLE = "wasm32-emscripten"
 
-DOWNLOAD_DIR = CROSS_BUILD_DIR / HOST_TRIPLE / "build"
-HOST_BUILD_DIR = CROSS_BUILD_DIR / HOST_TRIPLE / "build"
-HOST_DIR = HOST_BUILD_DIR / "python"
-PREFIX_DIR = CROSS_BUILD_DIR / HOST_TRIPLE / "prefix"
+
+def get_build_paths(cross_build_dir=None):
+    """Compute all build paths from the given cross-build directory."""
+    if cross_build_dir is None:
+        cross_build_dir = DEFAULT_CROSS_BUILD_DIR
+    cross_build_dir = Path(cross_build_dir).absolute()
+    host_triple_dir = cross_build_dir / HOST_TRIPLE
+    return {
+        "cross_build_dir": cross_build_dir,
+        "native_build_dir": cross_build_dir / "build",
+        "host_triple_dir": host_triple_dir,
+        "host_build_dir": host_triple_dir / "build",
+        "host_dir": host_triple_dir / "build" / "python",
+        "prefix_dir": host_triple_dir / "prefix",
+    }
+
 
 LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local"
 LOCAL_SETUP_MARKER = b"# Generated by Tools/wasm/emscripten.py\n"
@@ -115,12 +126,17 @@ def updated_env(updates, emsdk_cache):
     return environment
 
 
-def subdir(working_dir, *, clean_ok=False):
-    """Decorator to change to a working directory."""
+def subdir(path_key, *, clean_ok=False):
+    """Decorator to change to a working directory.
+
+    path_key is a key into context.build_paths, used to resolve the working
+    directory at call time.
+    """
 
     def decorator(func):
         @functools.wraps(func)
         def wrapper(context):
+            working_dir = context.build_paths[path_key]
             try:
                 tput_output = subprocess.check_output(
                     ["tput", "cols"], encoding="utf-8"
@@ -177,20 +193,21 @@ def build_platform():
     return sysconfig.get_config_var("BUILD_GNU_TYPE")
 
 
-def build_python_path():
+def build_python_path(context):
     """The path to the build Python binary."""
-    binary = NATIVE_BUILD_DIR / "python"
+    native_build_dir = context.build_paths["native_build_dir"]
+    binary = native_build_dir / "python"
     if not binary.is_file():
         binary = binary.with_suffix(".exe")
         if not binary.is_file():
             raise FileNotFoundError(
-                f"Unable to find `python(.exe)` in {NATIVE_BUILD_DIR}"
+                f"Unable to find `python(.exe)` in {native_build_dir}"
             )
 
     return binary
 
 
-@subdir(NATIVE_BUILD_DIR, clean_ok=True)
+@subdir("native_build_dir", clean_ok=True)
 def configure_build_python(context, working_dir):
     """Configure the build/host Python."""
     if LOCAL_SETUP.exists():
@@ -206,12 +223,12 @@ def configure_build_python(context, working_dir):
     call(configure, quiet=context.quiet)
 
 
-@subdir(NATIVE_BUILD_DIR)
+@subdir("native_build_dir")
 def make_build_python(context, working_dir):
     """Make/build the build Python."""
     call(["make", "--jobs", str(cpu_count()), "all"], quiet=context.quiet)
 
-    binary = build_python_path()
+    binary = build_python_path(context)
     cmd = [
         binary,
         "-c",
@@ -241,7 +258,7 @@ def download_and_unpack(working_dir: Path, url: str, expected_shasum: str):
         shutil.unpack_archive(tmp_file.name, working_dir)
 
 
-@subdir(HOST_BUILD_DIR, clean_ok=True)
+@subdir("host_build_dir", clean_ok=True)
 def make_emscripten_libffi(context, working_dir):
     ver = "3.4.6"
     libffi_dir = working_dir / f"libffi-{ver}"
@@ -253,13 +270,15 @@ def make_emscripten_libffi(context, working_dir):
     )
     call(
         [EMSCRIPTEN_DIR / "make_libffi.sh"],
-        env=updated_env({"PREFIX": PREFIX_DIR}, context.emsdk_cache),
+        env=updated_env(
+            {"PREFIX": context.build_paths["prefix_dir"]}, context.emsdk_cache
+        ),
         cwd=libffi_dir,
         quiet=context.quiet,
     )
 
 
-@subdir(HOST_BUILD_DIR, clean_ok=True)
+@subdir("host_build_dir", clean_ok=True)
 def make_mpdec(context, working_dir):
     ver = "4.0.1"
     mpdec_dir = working_dir / f"mpdecimal-{ver}"
@@ -275,7 +294,7 @@ def make_mpdec(context, working_dir):
             mpdec_dir / "configure",
             "CFLAGS=-fPIC",
             "--prefix",
-            PREFIX_DIR,
+            context.build_paths["prefix_dir"],
             "--disable-shared",
         ],
         cwd=mpdec_dir,
@@ -289,14 +308,15 @@ def make_mpdec(context, working_dir):
     )
 
 
-@subdir(HOST_DIR, clean_ok=True)
+@subdir("host_dir", clean_ok=True)
 def configure_emscripten_python(context, working_dir):
     """Configure the emscripten/host build."""
+    paths = context.build_paths
     config_site = os.fsdecode(EMSCRIPTEN_DIR / "config.site-wasm32-emscripten")
 
     emscripten_build_dir = working_dir.relative_to(CHECKOUT)
 
-    python_build_dir = NATIVE_BUILD_DIR / "build"
+    python_build_dir = paths["native_build_dir"] / "build"
     lib_dirs = list(python_build_dir.glob("lib.*"))
     assert len(lib_dirs) == 1, (
         f"Expected a single lib.* directory in {python_build_dir}"
@@ -322,13 +342,13 @@ def configure_emscripten_python(context, working_dir):
             capture_output=True,
         )
         host_runner = res.stdout.strip()
-    pkg_config_path_dir = (PREFIX_DIR / "lib/pkgconfig/").resolve()
+    pkg_config_path_dir = (paths["prefix_dir"] / "lib/pkgconfig/").resolve()
     env_additions = {
         "CONFIG_SITE": config_site,
         "HOSTRUNNER": host_runner,
         "EM_PKG_CONFIG_PATH": str(pkg_config_path_dir),
     }
-    build_python = os.fsdecode(build_python_path())
+    build_python = os.fsdecode(build_python_path(context))
     configure = [
         "emconfigure",
         os.path.relpath(CHECKOUT / "configure", working_dir),
@@ -342,7 +362,7 @@ def configure_emscripten_python(context, working_dir):
         "--disable-ipv6",
         "--enable-big-digits=30",
         "--enable-wasm-dynamic-linking",
-        f"--prefix={PREFIX_DIR}",
+        f"--prefix={paths['prefix_dir']}",
     ]
     if pydebug:
         configure.append("--with-pydebug")
@@ -403,7 +423,7 @@ def configure_emscripten_python(context, working_dir):
     sys.stdout.flush()
 
 
-@subdir(HOST_DIR)
+@subdir("host_dir")
 def make_emscripten_python(context, working_dir):
     """Run `make` for the emscripten/host build."""
     call(
@@ -432,9 +452,17 @@ def build_all(context):
 
 def clean_contents(context):
     """Delete all files created by this script."""
-    if CROSS_BUILD_DIR.exists():
-        print(f"🧹 Deleting {CROSS_BUILD_DIR} ...")
-        shutil.rmtree(CROSS_BUILD_DIR)
+    if context.target in {"all", "build"}:
+        build_dir = context.build_paths["native_build_dir"]
+        if build_dir.exists():
+            print(f"🧹 Deleting {build_dir} ...")
+            shutil.rmtree(build_dir)
+
+    if context.target in {"all", "host"}:
+        host_triple_dir = context.build_paths["host_triple_dir"]
+        if host_triple_dir.exists():
+            print(f"🧹 Deleting {host_triple_dir} ...")
+            shutil.rmtree(host_triple_dir)
 
     if LOCAL_SETUP.exists():
         with LOCAL_SETUP.open("rb") as file:
@@ -472,6 +500,17 @@ def main():
     clean = subcommands.add_parser(
         "clean", help="Delete files and directories created by this script"
     )
+    clean.add_argument(
+        "target",
+        nargs="?",
+        default="host",
+        choices=["all", "host", "build"],
+        help=(
+            "What should be cleaned. 'build' for just the build platform, or "
+            "'host' for the host platform, or 'all' for both. Defaults to 'host'."
+        ),
+    )
+
     for subcommand in (
         build,
         configure_build,
@@ -489,6 +528,14 @@ def main():
             dest="quiet",
             help="Redirect output from subprocesses to a log file",
         )
+        subcommand.add_argument(
+            "--cross-build-dir",
+            action="store",
+            default=None,
+            dest="cross_build_dir",
+            help="Path to the cross-build directory "
+            f"(default: {DEFAULT_CROSS_BUILD_DIR})",
+        )
         subcommand.add_argument(
             "--emsdk-cache",
             action="store",
@@ -521,6 +568,8 @@ def main():
 
     context = parser.parse_args()
 
+    context.build_paths = get_build_paths(context.cross_build_dir)
+
     if context.emsdk_cache:
         validate_emsdk_version(context.emsdk_cache)
         context.emsdk_cache = Path(context.emsdk_cache).absolute()