Modify handling of platform site to allow for venv usage. (#246)

Moves the platform site handling into the framework, and adds a mechanism to convert a macOS virtual environment into a cross-platform iOS build environment.

Author: freakboy3742

Makefile

@@ -129,10 +129,10 @@ ARCH-$(target)=$$(subst .,,$$(suffix $(target)))
 ifneq ($(os),macOS)
 	ifeq ($$(findstring simulator,$$(SDK-$(target))),)
 TARGET_TRIPLE-$(target)=$$(ARCH-$(target))-apple-$$(OS_LOWER-$(target))$$(VERSION_MIN-$(os))
-IS_SIMULATOR-$(target)="False"
+IS_SIMULATOR-$(target)=False
 	else
 TARGET_TRIPLE-$(target)=$$(ARCH-$(target))-apple-$$(OS_LOWER-$(target))$$(VERSION_MIN-$(os))-simulator
-IS_SIMULATOR-$(target)="True"
+IS_SIMULATOR-$(target)=True
 	endif
 endif
 
@@ -261,6 +261,9 @@ PYTHON_LIB-$(target)=$$(PYTHON_FRAMEWORK-$(target))/Python
 PYTHON_BIN-$(target)=$$(PYTHON_INSTALL-$(target))/bin
 PYTHON_INCLUDE-$(target)=$$(PYTHON_FRAMEWORK-$(target))/Headers
 PYTHON_STDLIB-$(target)=$$(PYTHON_INSTALL-$(target))/lib/python$(PYTHON_VER)
+PYTHON_PLATFORM_CONFIG-$(target)=$$(PYTHON_INSTALL-$(target))/platform-config/$$(ARCH-$(target))-$$(SDK-$(target))
+PYTHON_PLATFORM_SITECUSTOMIZE-$(target)=$$(PYTHON_PLATFORM_CONFIG-$(target))/sitecustomize.py
+
 
 $$(PYTHON_SRCDIR-$(target))/configure: \
 		downloads/Python-$(PYTHON_VERSION).tar.gz \
@@ -319,23 +322,35 @@ $$(PYTHON_LIB-$(target)): $$(PYTHON_SRCDIR-$(target))/python.exe
 		# Remove any .orig files produced by the compliance patching process
 		find $$(PYTHON_INSTALL-$(target)) -name "*.orig" -exec rm {} \;
 
-endif
-
-PYTHON_SITECUSTOMIZE-$(target)=$(PROJECT_DIR)/support/$(PYTHON_VER)/$(os)/platform-site/$(target)/sitecustomize.py
 
-$$(PYTHON_SITECUSTOMIZE-$(target)):
-	@echo ">>> Create cross-platform sitecustomize.py for $(target)"
-	mkdir -p $$(dir $$(PYTHON_SITECUSTOMIZE-$(target)))
-	cat $(PROJECT_DIR)/patch/Python/sitecustomize.$(os).py \
+$$(PYTHON_PLATFORM_SITECUSTOMIZE-$(target)):
+	@echo ">>> Create cross-plaform config for $(target)"
+	mkdir -p $$(PYTHON_PLATFORM_CONFIG-$(target))
+	# Create the cross-platform site definition
+	echo "import _cross_$$(ARCH-$(target))_$$(SDK-$(target)); import _cross_venv;" \
+		> $$(PYTHON_PLATFORM_CONFIG-$(target))/_cross_venv.pth
+	cp $(PROJECT_DIR)/patch/Python/make_cross_venv.py \
+		$$(PYTHON_PLATFORM_CONFIG-$(target))/make_cross_venv.py
+	cp $(PROJECT_DIR)/patch/Python/_cross_venv.py \
+		$$(PYTHON_PLATFORM_CONFIG-$(target))/_cross_venv.py
+	cp $$(PYTHON_STDLIB-$(target))/_sysconfig* \
+		$$(PYTHON_PLATFORM_CONFIG-$(target))
+	cat $(PROJECT_DIR)/patch/Python/_cross_target.py.tmpl \
 		| sed -e "s/{{os}}/$(os)/g" \
+		| sed -e "s/{{platform}}/$$(OS_LOWER-$(target))/g" \
 		| sed -e "s/{{arch}}/$$(ARCH-$(target))/g" \
+		| sed -e "s/{{sdk}}/$$(SDK-$(target))/g" \
 		| sed -e "s/{{version_min}}/$$(VERSION_MIN-$(os))/g" \
 		| sed -e "s/{{is_simulator}}/$$(IS_SIMULATOR-$(target))/g" \
-		| sed -e "s/{{multiarch}}/$$(ARCH-$(target))-$$(SDK-$(target))/g" \
-		| sed -e "s/{{tag}}/$$(OS_LOWER-$(target))-$$(VERSION_MIN-$(os))-$$(ARCH-$(target))-$$(SDK-$(target))/g" \
-		> $$(PYTHON_SITECUSTOMIZE-$(target))
+		> $$(PYTHON_PLATFORM_CONFIG-$(target))/_cross_$$(ARCH-$(target))_$$(SDK-$(target)).py
+	cat $(PROJECT_DIR)/patch/Python/sitecustomize.py.tmpl \
+		| sed -e "s/{{arch}}/$$(ARCH-$(target))/g" \
+		| sed -e "s/{{sdk}}/$$(SDK-$(target))/g" \
+		> $$(PYTHON_PLATFORM_SITECUSTOMIZE-$(target))
 
-$(target): $$(PYTHON_SITECUSTOMIZE-$(target)) $$(PYTHON_LIB-$(target))
+endif
+
+$(target): $$(PYTHON_PLATFORM_SITECUSTOMIZE-$(target)) $$(PYTHON_LIB-$(target))
 
 ###########################################################################
 # Target: Debug
@@ -364,6 +379,8 @@ vars-$(target):
 	@echo "PYTHON_BIN-$(target): $$(PYTHON_BIN-$(target))"
 	@echo "PYTHON_INCLUDE-$(target): $$(PYTHON_INCLUDE-$(target))"
 	@echo "PYTHON_STDLIB-$(target): $$(PYTHON_STDLIB-$(target))"
+	@echo "PYTHON_PLATFORM_CONFIG-$(target): $$(PYTHON_PLATFORM_CONFIG-$(target))"
+	@echo "PYTHON_PLATFORM_SITECUSTOMIZE-$(target): $$(PYTHON_PLATFORM_SITECUSTOMIZE-$(target))"
 	@echo
 
 endef # build-target
@@ -424,6 +441,7 @@ PYTHON_LIB-$(sdk)=$$(PYTHON_FRAMEWORK-$(sdk))/Python
 PYTHON_BIN-$(sdk)=$$(PYTHON_INSTALL-$(sdk))/bin
 PYTHON_INCLUDE-$(sdk)=$$(PYTHON_FRAMEWORK-$(sdk))/Headers
 PYTHON_STDLIB-$(sdk)=$$(PYTHON_INSTALL-$(sdk))/lib/python$(PYTHON_VER)
+PYTHON_PLATFORM_CONFIG-$(sdk)=$$(PYTHON_INSTALL-$(sdk))/platform-config
 
 $$(PYTHON_LIB-$(sdk)): $$(foreach target,$$(SDK_TARGETS-$(sdk)),$$(PYTHON_LIB-$$(target)))
 	@echo ">>> Build Python fat library for the $(sdk) SDK"
@@ -459,7 +477,7 @@ $$(PYTHON_INCLUDE-$(sdk))/pyconfig.h: $$(PYTHON_LIB-$(sdk))
 	cp $$(PYTHON_SRCDIR-$$(firstword $$(SDK_TARGETS-$(sdk))))/$(os)/Resources/pyconfig.h $$(PYTHON_INCLUDE-$(sdk))/pyconfig.h
 
 
-$$(PYTHON_STDLIB-$(sdk))/LICENSE.TXT: $$(PYTHON_LIB-$(sdk)) $$(PYTHON_FRAMEWORK-$(sdk))/Info.plist $$(PYTHON_INCLUDE-$(sdk))/pyconfig.h
+$$(PYTHON_STDLIB-$(sdk))/LICENSE.TXT: $$(PYTHON_LIB-$(sdk)) $$(PYTHON_FRAMEWORK-$(sdk))/Info.plist $$(PYTHON_INCLUDE-$(sdk))/pyconfig.h $$(foreach target,$$(SDK_TARGETS-$(sdk)),$$(PYTHON_PLATFORM_SITECUSTOMIZE-$$(target)))
 	@echo ">>> Build Python stdlib for the $(sdk) SDK"
 	mkdir -p $$(PYTHON_STDLIB-$(sdk))/lib-dynload
 	# Copy stdlib from the first target associated with the $(sdk) SDK
@@ -468,11 +486,17 @@ $$(PYTHON_STDLIB-$(sdk))/LICENSE.TXT: $$(PYTHON_LIB-$(sdk)) $$(PYTHON_FRAMEWORK-
 	# Delete the single-SDK parts of the standard library
 	rm -rf \
 		$$(PYTHON_STDLIB-$(sdk))/_sysconfigdata__*.py \
+		$$(PYTHON_STDLIB-$(sdk))/_sysconfig_vars__*.json \
 		$$(PYTHON_STDLIB-$(sdk))/config-* \
 		$$(PYTHON_STDLIB-$(sdk))/lib-dynload/*
 
 	# Copy the individual _sysconfigdata modules into names that include the architecture
 	$$(foreach target,$$(SDK_TARGETS-$(sdk)),cp $$(PYTHON_STDLIB-$$(target))/_sysconfigdata_* $$(PYTHON_STDLIB-$(sdk))/; )
+	$$(foreach target,$$(SDK_TARGETS-$(sdk)),cp $$(PYTHON_STDLIB-$$(target))/_sysconfig_vars_* $$(PYTHON_STDLIB-$(sdk))/; )
+
+	# Copy the platform site folders for each architecture
+	mkdir -p $$(PYTHON_PLATFORM_CONFIG-$(sdk))
+	$$(foreach target,$$(SDK_TARGETS-$(sdk)),cp -r $$(PYTHON_PLATFORM_CONFIG-$$(target)) $$(PYTHON_PLATFORM_CONFIG-$(sdk)); )
 
 	# Merge the binary modules from each target in the $(sdk) SDK into a single binary
 	$$(foreach module,$$(wildcard $$(PYTHON_STDLIB-$$(firstword $$(SDK_TARGETS-$(sdk))))/lib-dynload/*),lipo -create -output $$(PYTHON_STDLIB-$(sdk))/lib-dynload/$$(notdir $$(module)) $$(foreach target,$$(SDK_TARGETS-$(sdk)),$$(PYTHON_STDLIB-$$(target))/lib-dynload/$$(notdir $$(module))); )
@@ -581,7 +605,7 @@ support/$(PYTHON_VER)/macOS/VERSIONS:
 dist/Python-$(PYTHON_VER)-macOS-support.$(BUILD_NUMBER).tar.gz: \
 	$$(PYTHON_XCFRAMEWORK-macOS)/Info.plist \
 	support/$(PYTHON_VER)/macOS/VERSIONS \
-	$$(foreach target,$$(TARGETS-macOS), $$(PYTHON_SITECUSTOMIZE-$$(target)))
+	$$(foreach target,$$(TARGETS-macOS), $$(PYTHON_PLATFORM_SITECUSTOMIZE-$$(target)))
 
 	@echo ">>> Create final distribution artefact for macOS"
 	mkdir -p dist
@@ -604,9 +628,7 @@ $$(PYTHON_XCFRAMEWORK-$(os))/Info.plist: \
 	$$(foreach sdk,$$(SDKS-$(os)),cp -r $$(PYTHON_INSTALL-$$(sdk))/include $$(PYTHON_XCFRAMEWORK-$(os))/$$(SDK_SLICE-$$(sdk)); )
 	$$(foreach sdk,$$(SDKS-$(os)),cp -r $$(PYTHON_INSTALL-$$(sdk))/bin $$(PYTHON_XCFRAMEWORK-$(os))/$$(SDK_SLICE-$$(sdk)); )
 	$$(foreach sdk,$$(SDKS-$(os)),cp -r $$(PYTHON_INSTALL-$$(sdk))/lib $$(PYTHON_XCFRAMEWORK-$(os))/$$(SDK_SLICE-$$(sdk)); )
-
-	@echo ">>> Create helper links in XCframework for $(os)"
-	$$(foreach sdk,$$(SDKS-$(os)),ln -si $$(SDK_SLICE-$$(sdk)) $$(PYTHON_XCFRAMEWORK-$(os))/$$(sdk); )
+	$$(foreach sdk,$$(SDKS-$(os)),cp -r $$(PYTHON_INSTALL-$$(sdk))/platform-config $$(PYTHON_XCFRAMEWORK-$(os))/$$(SDK_SLICE-$$(sdk)); )
 
 ifeq ($(os),iOS)
 	@echo ">>> Clone testbed project for $(os)"
@@ -626,7 +648,7 @@ endif
 
 dist/Python-$(PYTHON_VER)-$(os)-support.$(BUILD_NUMBER).tar.gz: \
 	$$(PYTHON_XCFRAMEWORK-$(os))/Info.plist \
-	$$(foreach target,$$(TARGETS-$(os)), $$(PYTHON_SITECUSTOMIZE-$$(target)))
+	$$(foreach target,$$(TARGETS-$(os)), $$(PYTHON_PLATFORM_SITECUSTOMIZE-$$(target)))
 
 	@echo ">>> Create final distribution artefact for $(os)"
 	mkdir -p dist

patch/Python/_cross_target.py.tmpl

@@ -0,0 +1,71 @@
+# A site package that turns a macOS virtual environment
+# into an {{arch}} {{sdk}} cross-platform virtual environment
+import platform
+import subprocess
+import sys
+import sysconfig
+
+###########################################################################
+# sys module patches
+###########################################################################
+sys.cross_compiling = True
+sys.platform = "{{platform}}"
+sys.implementation._multiarch = "{{arch}}-{{sdk}}"
+
+###########################################################################
+# subprocess module patches
+###########################################################################
+subprocess._can_fork_exec = True
+
+
+###########################################################################
+# platform module patches
+###########################################################################
+
+def cross_system():
+    return "{{os}}"
+
+
+def cross_uname():
+    return platform.uname_result(
+        system="{{os}}",
+        node="build",
+        release="{{version_min}}",
+        version="",
+        machine="{{arch}}",
+    )
+
+
+def cross_ios_ver(system="", release="", model="", is_simulator=False):
+    if system == "":
+        system = "{{os}}"
+    if release == "":
+        release = "{{version_min}}"
+    if model == "":
+        model = "{{sdk}}"
+
+    return platform.IOSVersionInfo(system, release, model, {{is_simulator}})
+
+
+platform.system = cross_system
+platform.uname = cross_uname
+platform.ios_ver = cross_ios_ver
+
+
+###########################################################################
+# sysconfig module patches
+###########################################################################
+
+def cross_get_platform():
+    return "{{platform}}-{{version_min}}-{{arch}}-{{sdk}}"
+
+
+def cross_get_sysconfigdata_name():
+    return "_sysconfigdata__{{platform}}_{{arch}}-{{sdk}}"
+
+
+sysconfig.get_platform = cross_get_platform
+sysconfig._get_sysconfigdata_name = cross_get_sysconfigdata_name
+
+# Force sysconfig data to be loaded (and cached).
+sysconfig._init_config_vars()

patch/Python/_cross_venv.py

@@ -0,0 +1,105 @@
+import shutil
+import sys
+import sysconfig
+from pathlib import Path
+
+SITE_PACKAGE_PATH = Path(__file__).parent
+
+###########################################################################
+# importlib module patches
+###########################################################################
+
+
+def patch_env_create(env):
+    """
+    Patch the process of creating virtual environments to ensure that the cross
+    environment modification files are also copied as part of environment
+    creation.
+    """
+    old_pip_env_create = env._PipBackend.create
+
+    def pip_env_create(self, path, *args, **kwargs):
+        result = old_pip_env_create(self, path, *args, **kwargs)
+        # Copy any _cross_*.pth or _cross_*.py file, plus the cross-platform
+        # sysconfigdata module and sysconfig_vars JSON to the new environment.
+        data_name = sysconfig._get_sysconfigdata_name()
+        json_name = data_name.replace("_sysconfigdata", "_sysconfig_vars")
+        for filename in [
+            "_cross_venv.pth",
+            "_cross_venv.py",
+            f"_cross_{sys.implementation._multiarch.replace('-', '_')}.py",
+            f"{data_name}.py",
+            f"{json_name}.json",
+        ]:
+            src = SITE_PACKAGE_PATH / filename
+            target = Path(path) / src.relative_to(
+                SITE_PACKAGE_PATH.parent.parent.parent
+            )
+            if not target.exists():
+                shutil.copy(src, target)
+        return result
+
+    env._PipBackend.create = pip_env_create
+
+
+# Import hook that patches the creation of virtual environments by `build`
+#
+# The approach used here is the same as the one used by virtualenv to patch
+# distutils (but without support for the older load_module API).
+# https://docs.python.org/3/library/importlib.html#setting-up-an-importer
+_BUILD_PATCH = ("build.env",)
+
+
+class _Finder:
+    """A meta path finder that allows patching the imported build modules."""
+
+    fullname = None
+
+    # lock[0] is threading.Lock(), but initialized lazily to avoid importing
+    # threading very early at startup, because there are gevent-based
+    # applications that need to be first to import threading by themselves.
+    # See https://github.com/pypa/virtualenv/issues/1895 for details.
+    lock = []  # noqa: RUF012
+
+    def find_spec(self, fullname, path, target=None):
+        if fullname in _BUILD_PATCH and self.fullname is None:
+            # initialize lock[0] lazily
+            if len(self.lock) == 0:
+                import threading
+
+                lock = threading.Lock()
+                # there is possibility that two threads T1 and T2 are
+                # simultaneously running into find_spec, observing .lock as
+                # empty, and further going into hereby initialization. However
+                # due to the GIL, list.append() operation is atomic and this
+                # way only one of the threads will "win" to put the lock
+                # - that every thread will use - into .lock[0].
+                # https://docs.python.org/3/faq/library.html#what-kinds-of-global-value-mutation-are-thread-safe
+                self.lock.append(lock)
+
+            from functools import partial
+            from importlib.util import find_spec
+
+            with self.lock[0]:
+                self.fullname = fullname
+                try:
+                    spec = find_spec(fullname, path)
+                    if spec is not None:
+                        # https://www.python.org/dev/peps/pep-0451/#how-loading-will-work
+                        old = spec.loader.exec_module
+                        func = self.exec_module
+                        if old is not func:
+                            spec.loader.exec_module = partial(func, old)
+                        return spec
+                finally:
+                    self.fullname = None
+        return None
+
+    @staticmethod
+    def exec_module(old, module):
+        old(module)
+        if module.__name__ in _BUILD_PATCH:
+            patch_env_create(module)
+
+
+sys.meta_path.insert(0, _Finder())