Skip to content

[3.14] gh-145219: Add Emscripten cross-build and clean configurability (GH-145581) #145654

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
freakboy3742 merged 1 commit into from
Mar 9, 2026
Merged

[3.14] gh-145219: Add Emscripten cross-build and clean configurability (GH-145581) #145654

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ Tools/unicode/data/
/config.status
/config.status.lineno
/.ccache
/cross-build/
/cross-build*/
/jit_stencils*.h
/platform
/profile-clean-stamp
Expand Down
103 changes: 76 additions & 27 deletions Tools/wasm/emscripten/__main__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,25 @@
CHECKOUT = EMSCRIPTEN_DIR.parent.parent.parent
EMSCRIPTEN_VERSION_FILE = EMSCRIPTEN_DIR / "emscripten_version.txt"

CROSS_BUILD_DIR = CHECKOUT / "cross-build"
NATIVE_BUILD_DIR = CROSS_BUILD_DIR / "build"
DEFAULT_CROSS_BUILD_DIR = CHECKOUT / "cross-build"
HOST_TRIPLE = "wasm32-emscripten"

DOWNLOAD_DIR = CROSS_BUILD_DIR / HOST_TRIPLE / "build"
HOST_BUILD_DIR = CROSS_BUILD_DIR / HOST_TRIPLE / "build"
HOST_DIR = HOST_BUILD_DIR / "python"
PREFIX_DIR = CROSS_BUILD_DIR / HOST_TRIPLE / "prefix"

def get_build_paths(cross_build_dir=None):
"""Compute all build paths from the given cross-build directory."""
if cross_build_dir is None:
cross_build_dir = DEFAULT_CROSS_BUILD_DIR
cross_build_dir = Path(cross_build_dir).absolute()
host_triple_dir = cross_build_dir / HOST_TRIPLE
return {
"cross_build_dir": cross_build_dir,
"native_build_dir": cross_build_dir / "build",
"host_triple_dir": host_triple_dir,
"host_build_dir": host_triple_dir / "build",
"host_dir": host_triple_dir / "build" / "python",
"prefix_dir": host_triple_dir / "prefix",
}


LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local"
LOCAL_SETUP_MARKER = b"# Generated by Tools/wasm/emscripten.py\n"
Expand Down Expand Up @@ -115,12 +126,17 @@ def updated_env(updates, emsdk_cache):
return environment


def subdir(working_dir, *, clean_ok=False):
"""Decorator to change to a working directory."""
def subdir(path_key, *, clean_ok=False):
"""Decorator to change to a working directory.
path_key is a key into context.build_paths, used to resolve the working
directory at call time.
"""

def decorator(func):
@functools.wraps(func)
def wrapper(context):
working_dir = context.build_paths[path_key]
try:
tput_output = subprocess.check_output(
["tput", "cols"], encoding="utf-8"
Expand Down Expand Up @@ -177,20 +193,21 @@ def build_platform():
return sysconfig.get_config_var("BUILD_GNU_TYPE")


def build_python_path():
def build_python_path(context):
"""The path to the build Python binary."""
binary = NATIVE_BUILD_DIR / "python"
native_build_dir = context.build_paths["native_build_dir"]
binary = native_build_dir / "python"
if not binary.is_file():
binary = binary.with_suffix(".exe")
if not binary.is_file():
raise FileNotFoundError(
f"Unable to find `python(.exe)` in {NATIVE_BUILD_DIR}"
f"Unable to find `python(.exe)` in {native_build_dir}"
)

return binary


@subdir(NATIVE_BUILD_DIR, clean_ok=True)
@subdir("native_build_dir", clean_ok=True)
def configure_build_python(context, working_dir):
"""Configure the build/host Python."""
if LOCAL_SETUP.exists():
Expand All @@ -206,12 +223,12 @@ def configure_build_python(context, working_dir):
call(configure, quiet=context.quiet)


@subdir(NATIVE_BUILD_DIR)
@subdir("native_build_dir")
def make_build_python(context, working_dir):
"""Make/build the build Python."""
call(["make", "--jobs", str(cpu_count()), "all"], quiet=context.quiet)

binary = build_python_path()
binary = build_python_path(context)
cmd = [
binary,
"-c",
Expand Down Expand Up @@ -241,7 +258,7 @@ def download_and_unpack(working_dir: Path, url: str, expected_shasum: str):
shutil.unpack_archive(tmp_file.name, working_dir)


@subdir(HOST_BUILD_DIR, clean_ok=True)
@subdir("host_build_dir", clean_ok=True)
def make_emscripten_libffi(context, working_dir):
ver = "3.4.6"
libffi_dir = working_dir / f"libffi-{ver}"
Expand All @@ -253,13 +270,15 @@ def make_emscripten_libffi(context, working_dir):
)
call(
[EMSCRIPTEN_DIR / "make_libffi.sh"],
env=updated_env({"PREFIX": PREFIX_DIR}, context.emsdk_cache),
env=updated_env(
{"PREFIX": context.build_paths["prefix_dir"]}, context.emsdk_cache
),
cwd=libffi_dir,
quiet=context.quiet,
)


@subdir(HOST_BUILD_DIR, clean_ok=True)
@subdir("host_build_dir", clean_ok=True)
def make_mpdec(context, working_dir):
ver = "4.0.1"
mpdec_dir = working_dir / f"mpdecimal-{ver}"
Expand All @@ -275,7 +294,7 @@ def make_mpdec(context, working_dir):
mpdec_dir / "configure",
"CFLAGS=-fPIC",
"--prefix",
PREFIX_DIR,
context.build_paths["prefix_dir"],
"--disable-shared",
],
cwd=mpdec_dir,
Expand All @@ -289,14 +308,15 @@ def make_mpdec(context, working_dir):
)


@subdir(HOST_DIR, clean_ok=True)
@subdir("host_dir", clean_ok=True)
def configure_emscripten_python(context, working_dir):
"""Configure the emscripten/host build."""
paths = context.build_paths
config_site = os.fsdecode(EMSCRIPTEN_DIR / "config.site-wasm32-emscripten")

emscripten_build_dir = working_dir.relative_to(CHECKOUT)

python_build_dir = NATIVE_BUILD_DIR / "build"
python_build_dir = paths["native_build_dir"] / "build"
lib_dirs = list(python_build_dir.glob("lib.*"))
assert len(lib_dirs) == 1, (
f"Expected a single lib.* directory in {python_build_dir}"
Expand All @@ -322,13 +342,13 @@ def configure_emscripten_python(context, working_dir):
capture_output=True,
)
host_runner = res.stdout.strip()
pkg_config_path_dir = (PREFIX_DIR / "lib/pkgconfig/").resolve()
pkg_config_path_dir = (paths["prefix_dir"] / "lib/pkgconfig/").resolve()
env_additions = {
"CONFIG_SITE": config_site,
"HOSTRUNNER": host_runner,
"EM_PKG_CONFIG_PATH": str(pkg_config_path_dir),
}
build_python = os.fsdecode(build_python_path())
build_python = os.fsdecode(build_python_path(context))
configure = [
"emconfigure",
os.path.relpath(CHECKOUT / "configure", working_dir),
Expand All @@ -342,7 +362,7 @@ def configure_emscripten_python(context, working_dir):
"--disable-ipv6",
"--enable-big-digits=30",
"--enable-wasm-dynamic-linking",
f"--prefix={PREFIX_DIR}",
f"--prefix={paths['prefix_dir']}",
]
if pydebug:
configure.append("--with-pydebug")
Expand Down Expand Up @@ -403,7 +423,7 @@ def configure_emscripten_python(context, working_dir):
sys.stdout.flush()


@subdir(HOST_DIR)
@subdir("host_dir")
def make_emscripten_python(context, working_dir):
"""Run `make` for the emscripten/host build."""
call(
Expand Down Expand Up @@ -432,9 +452,17 @@ def build_all(context):

def clean_contents(context):
"""Delete all files created by this script."""
if CROSS_BUILD_DIR.exists():
print(f"🧹 Deleting {CROSS_BUILD_DIR} ...")
shutil.rmtree(CROSS_BUILD_DIR)
if context.target in {"all", "build"}:
build_dir = context.build_paths["native_build_dir"]
if build_dir.exists():
print(f"🧹 Deleting {build_dir} ...")
shutil.rmtree(build_dir)

if context.target in {"all", "host"}:
host_triple_dir = context.build_paths["host_triple_dir"]
if host_triple_dir.exists():
print(f"🧹 Deleting {host_triple_dir} ...")
shutil.rmtree(host_triple_dir)

if LOCAL_SETUP.exists():
with LOCAL_SETUP.open("rb") as file:
Expand Down Expand Up @@ -472,6 +500,17 @@ def main():
clean = subcommands.add_parser(
"clean", help="Delete files and directories created by this script"
)
clean.add_argument(
"target",
nargs="?",
default="host",
choices=["all", "host", "build"],
help=(
"What should be cleaned. 'build' for just the build platform, or "
"'host' for the host platform, or 'all' for both. Defaults to 'host'."
),
)

for subcommand in (
build,
configure_build,
Expand All @@ -489,6 +528,14 @@ def main():
dest="quiet",
help="Redirect output from subprocesses to a log file",
)
subcommand.add_argument(
"--cross-build-dir",
action="store",
default=None,
dest="cross_build_dir",
help="Path to the cross-build directory "
f"(default: {DEFAULT_CROSS_BUILD_DIR})",
)
subcommand.add_argument(
"--emsdk-cache",
action="store",
Expand Down Expand Up @@ -521,6 +568,8 @@ def main():

context = parser.parse_args()

context.build_paths = get_build_paths(context.cross_build_dir)

if context.emsdk_cache:
validate_emsdk_version(context.emsdk_cache)
context.emsdk_cache = Path(context.emsdk_cache).absolute()
Expand Down
Loading