Bump Python version to v3.8.11.

This incorporates macOS 11 compatibility for Apple Silicon.

Author: freakboy3742

Makefile

@@ -17,6 +17,9 @@
 # - XZ-iOS          - build XZ for iOS
 # - XZ-tvOS         - build XZ for tvOS
 # - XZ-watchOS      - build XZ for watchOS
+# - libFFI-iOS      - build libFFI for iOS
+# - libFFI-tvOS     - build libFFI for tvOS
+# - libFFI-watchOS  - build libFFI for watchOS
 # - Python-macOS    - build Python for macOS
 # - Python-iOS      - build Python for iOS
 # - Python-tvOS     - build Python for tvOS
@@ -30,7 +33,11 @@ BUILD_NUMBER=custom
 MACOSX_DEPLOYMENT_TARGET=10.8
 
 # Version of packages that will be compiled by this meta-package
-PYTHON_VERSION=3.8.9
+# PYTHON_VERSION is the full version number (e.g., 3.10.0b3)
+# PYTHON_MICRO_VERSION is the full version number, without any alpha/beta/rc suffix. (e.g., 3.10.0)
+# PYTHON_VER is the major/minor version (e.g., 3.10)
+PYTHON_VERSION=3.8.11
+PYTHON_MICRO_VERSION=$(shell echo $(PYTHON_VERSION) | grep -Eo "\d+\.\d+\.\d+")
 PYTHON_VER=$(basename $(PYTHON_VERSION))
 
 OPENSSL_VERSION_NUMBER=1.1.1
@@ -41,11 +48,13 @@ BZIP2_VERSION=1.0.8
 
 XZ_VERSION=5.2.5
 
+LIBFFI_VERSION=3.3
+
 # Supported OS
 OS=macOS iOS tvOS watchOS
 
 # macOS targets
-TARGETS-macOS=macosx.x86_64
+TARGETS-macOS=macosx.x86_64 macosx.arm64
 PYTHON_TARGETS-macOS=macOS
 CFLAGS-macOS=-mmacosx-version-min=$(MACOSX_DEPLOYMENT_TARGET)
 
@@ -83,7 +92,7 @@ clean:
 distclean: clean
 	rm -rf downloads
 
-downloads: downloads/openssl-$(OPENSSL_VERSION).tgz downloads/bzip2-$(BZIP2_VERSION).tgz downloads/xz-$(XZ_VERSION).tgz downloads/Python-$(PYTHON_VERSION).tgz
+downloads: downloads/openssl-$(OPENSSL_VERSION).tgz downloads/bzip2-$(BZIP2_VERSION).tgz downloads/xz-$(XZ_VERSION).tgz downloads/libffi-$(LIBFFI_VERSION).tgz downloads/Python-$(PYTHON_VERSION).tgz
 
 update-patch:
 	# Generate a diff from the clone of the python/cpython Github repository
@@ -141,6 +150,20 @@ downloads/xz-$(XZ_VERSION).tgz:
 	mkdir -p downloads
 	if [ ! -e downloads/xz-$(XZ_VERSION).tgz ]; then curl --fail -L http://tukaani.org/xz/xz-$(XZ_VERSION).tar.gz -o downloads/xz-$(XZ_VERSION).tgz; fi
 
+###########################################################################
+# LIBFFI
+###########################################################################
+
+# Clean the LibFFI project
+clean-libFFI:
+	rm -rf build/*/libffi-$(LIBFFI_VERSION) \
+		build/*/Support/libFFI
+
+# Download original XZ source code archive.
+downloads/libffi-$(LIBFFI_VERSION).tgz:
+	mkdir -p downloads
+	if [ ! -e downloads/libffi-$(LIBFFI_VERSION).tgz ]; then curl --fail -L http://github.com/libffi/libffi/releases/download/v$(LIBFFI_VERSION)/libffi-$(LIBFFI_VERSION).tar.gz -o downloads/libffi-$(LIBFFI_VERSION).tgz; fi
+
 ###########################################################################
 # Python
 ###########################################################################
@@ -156,7 +179,7 @@ clean-Python:
 # Download original Python source code archive.
 downloads/Python-$(PYTHON_VERSION).tgz:
 	mkdir -p downloads
-	if [ ! -e downloads/Python-$(PYTHON_VERSION).tgz ]; then curl -L https://www.python.org/ftp/python/$(PYTHON_VERSION)/Python-$(PYTHON_VERSION).tgz > downloads/Python-$(PYTHON_VERSION).tgz; fi
+	if [ ! -e downloads/Python-$(PYTHON_VERSION).tgz ]; then curl -L https://www.python.org/ftp/python/$(PYTHON_MICRO_VERSION)/Python-$(PYTHON_VERSION).tgz > downloads/Python-$(PYTHON_VERSION).tgz; fi
 
 # Some Python targets needed to identify the host build
 PYTHON_DIR-macOS=build/macOS/Python-$(PYTHON_VERSION)-macOS
@@ -191,6 +214,7 @@ LDFLAGS-$1=-arch $$(ARCH-$1) -isysroot=$$(SDK_ROOT-$1)
 OPENSSL_DIR-$1=build/$2/openssl-$(OPENSSL_VERSION)-$1
 BZIP2_DIR-$1=build/$2/bzip2-$(BZIP2_VERSION)-$1
 XZ_DIR-$1=build/$2/xz-$(XZ_VERSION)-$1
+LIBFFI_DIR-$1=build/$2/libffi-$(LIBFFI_VERSION)
 
 # Unpack OpenSSL
 $$(OPENSSL_DIR-$1)/Makefile: downloads/openssl-$(OPENSSL_VERSION).tgz
@@ -262,24 +286,32 @@ $$(XZ_DIR-$1)/Makefile: downloads/xz-$(XZ_VERSION).tgz
 $$(XZ_DIR-$1)/src/liblzma/.libs/liblzma.a: $$(XZ_DIR-$1)/Makefile
 	cd $$(XZ_DIR-$1) && make && make install
 
-# macOS builds are compiled as a single build. As a result, the macOS Python
-# build is configured in the `build` macro, rather than the `build-target` macro.
+# macOS builds use their own libFFI, and are compiled as a single
+# universal2 build. As a result, the macOS Python build is configured
+# in the `build` macro, rather than the `build-target` macro.
 ifneq ($2,macOS)
+LIBFFI_BUILD_DIR-$1=build_$$(SDK-$1)-$$(ARCH-$1)
 PYTHON_DIR-$1=build/$2/Python-$(PYTHON_VERSION)-$1
 pyconfig.h-$1=pyconfig-$$(ARCH-$1).h
 PYTHON_HOST-$1=$(PYTHON_HOST)
 
+# Build LibFFI
+$$(LIBFFI_DIR-$1)/libffi.$1.a: $$(LIBFFI_DIR-$1)/darwin_common
+	cd $$(LIBFFI_DIR-$1)/$$(LIBFFI_BUILD_DIR-$1) && make
+
+	# Copy in the lib to a non-BUILD_DIR dependent location;
+	# include the target in the final filename for disambiguation
+	cp $$(LIBFFI_DIR-$1)/$$(LIBFFI_BUILD_DIR-$1)/.libs/libffi.a $$(LIBFFI_DIR-$1)/libffi.$1.a
+
 # Unpack Python
 $$(PYTHON_DIR-$1)/Makefile: downloads/Python-$(PYTHON_VERSION).tgz $$(PYTHON_HOST-$1)
 	# Unpack target Python
 	mkdir -p $$(PYTHON_DIR-$1)
 	tar zxf downloads/Python-$(PYTHON_VERSION).tgz --strip-components 1 -C $$(PYTHON_DIR-$1)
 	# Apply target Python patches
 	cd $$(PYTHON_DIR-$1) && patch -p1 < $(PROJECT_DIR)/patch/Python/Python.patch
-	# Copy in the embedded and platform/arch configuration
-	cp -f $(PROJECT_DIR)/patch/Python/Setup.embedded $$(PYTHON_DIR-$1)/Modules/Setup.embedded
-	if [ -e "$(PROJECT_DIR)/patch/Python/Setup.$2-$$(ARCH-$1)" ]; then \
-		cp -f $(PROJECT_DIR)/patch/Python/Setup.$2-$$(ARCH-$1) $$(PYTHON_DIR-$1)/Modules/Setup.$2-$$(ARCH-$1); fi
+	# Copy in the embedded module configuration
+	cat $(PROJECT_DIR)/patch/Python/Setup.embedded $(PROJECT_DIR)/patch/Python/Setup.$2 > $$(PYTHON_DIR-$1)/Modules/Setup.local
 	# Configure target Python
 	cd $$(PYTHON_DIR-$1) && PATH=$(PROJECT_DIR)/$(PYTHON_DIR-macOS)/dist/bin:$(PATH) ./configure \
 		CC="$$(CC-$1)" LD="$$(CC-$1)" \
@@ -291,7 +323,7 @@ $$(PYTHON_DIR-$1)/Makefile: downloads/Python-$(PYTHON_VERSION).tgz $$(PYTHON_HOS
 		$$(PYTHON_CONFIGURE-$2)
 
 # Build Python
-$$(PYTHON_DIR-$1)/dist/lib/libpython$(PYTHON_VER).a: build/$2/Support/OpenSSL build/$2/Support/BZip2 build/$2/Support/XZ $$(PYTHON_DIR-$1)/Makefile
+$$(PYTHON_DIR-$1)/dist/lib/libpython$(PYTHON_VER).a: build/$2/Support/OpenSSL build/$2/Support/BZip2 build/$2/Support/XZ build/$2/Support/libFFI $$(PYTHON_DIR-$1)/Makefile
 	# Build target Python
 	cd $$(PYTHON_DIR-$1) && PATH="$(PROJECT_DIR)/$(PYTHON_DIR-macOS)/dist/bin:$(PATH)" make all install
 
@@ -307,9 +339,11 @@ vars-$1:
 	@echo "SDK-$1: $$(SDK-$1)"
 	@echo "SDK_ROOT-$1: $$(SDK_ROOT-$1)"
 	@echo "CC-$1: $$(CC-$1)"
+	@echo "LIBFFI_BUILD_DIR-$1: $$(LIBFFI_BUILD_DIR-$1)"
 	@echo "OPENSSL_DIR-$1: $$(OPENSSL_DIR-$1)"
 	@echo "BZIP2_DIR-$1: $$(BZIP2_DIR-$1)"
 	@echo "XZ_DIR-$1: $$(XZ_DIR-$1)"
+	@echo "LIBFFI_DIR-$1: $$(LIBFFI_DIR-$1)"
 	@echo "PYTHON_DIR-$1: $$(PYTHON_DIR-$1)"
 	@echo "pyconfig.h-$1: $$(pyconfig.h-$1)"
 
@@ -325,6 +359,7 @@ $$(foreach target,$$(TARGETS-$1),$$(eval $$(call build-target,$$(target),$1)))
 OPENSSL_FRAMEWORK-$1=build/$1/Support/OpenSSL
 BZIP2_FRAMEWORK-$1=build/$1/Support/BZip2
 XZ_FRAMEWORK-$1=build/$1/Support/XZ
+LIBFFI_FRAMEWORK-$1=build/$1/Support/libFFI
 PYTHON_FRAMEWORK-$1=build/$1/Support/Python
 PYTHON_RESOURCES-$1=$$(PYTHON_FRAMEWORK-$1)/Resources
 
@@ -333,11 +368,16 @@ $1: dist/Python-$(PYTHON_VER)-$1-support.$(BUILD_NUMBER).tar.gz
 clean-$1:
 	rm -rf build/$1
 
-dist/Python-$(PYTHON_VER)-$1-support.$(BUILD_NUMBER).tar.gz: $$(BZIP2_FRAMEWORK-$1) $$(XZ_FRAMEWORK-$1) $$(OPENSSL_FRAMEWORK-$1) $$(PYTHON_FRAMEWORK-$1)
+dist/Python-$(PYTHON_VER)-$1-support.$(BUILD_NUMBER).tar.gz: $$(BZIP2_FRAMEWORK-$1) $$(XZ_FRAMEWORK-$1) $$(OPENSSL_FRAMEWORK-$1) $$(LIBFFI_FRAMEWORK-$1) $$(PYTHON_FRAMEWORK-$1)
 	mkdir -p dist
 	echo "Python version: $(PYTHON_VERSION) " > build/$1/Support/VERSIONS
 	echo "Build: $(BUILD_NUMBER)" >> build/$1/Support/VERSIONS
 	echo "---------------------" >> build/$1/Support/VERSIONS
+ifeq ($1,macOS)
+	echo "libFFI: macOS native" >> build/$1/Support/VERSIONS
+else
+	echo "libFFI: $(LIBFFI_VERSION)" >> build/$1/Support/VERSIONS
+endif
 	echo "BZip2: $(BZIP2_VERSION)" >> build/$1/Support/VERSIONS
 	echo "OpenSSL: $(OPENSSL_VERSION)" >> build/$1/Support/VERSIONS
 	echo "XZ: $(XZ_VERSION)" >> build/$1/Support/VERSIONS
@@ -407,11 +447,18 @@ build/$1/xz/lib/liblzma.a: $$(foreach target,$$(TARGETS-$1),$$(XZ_DIR-$$(target)
 	mkdir -p build/$1
 	xcrun lipo -create -o $$@ $$^
 
-# macOS builds a single Python target; thus it needs to be
+# Build libFFI
+libFFI-$1: $$(LIBFFI_FRAMEWORK-$1)
+
+# macOS builds a single Python universal2 target; thus it needs to be
 # configured in the `build` macro, not the `build-target` macro.
+# macOS also uses the system-provided libFFI, so there's no need to package
+# a libFFI framework for macOS.
 ifeq ($1,macOS)
 # Some targets that are needed for consistency between macOS and other builds,
 # but are no-ops on macOS.
+$$(LIBFFI_FRAMEWORK-$1):
+
 build/$1/$$(pyconfig.h-$1):
 
 # Unpack Python
@@ -421,25 +468,54 @@ $$(PYTHON_DIR-$1)/Makefile: downloads/Python-$(PYTHON_VERSION).tgz
 	tar zxf downloads/Python-$(PYTHON_VERSION).tgz --strip-components 1 -C $$(PYTHON_DIR-$1)
 	# Apply target Python patches
 	cd $$(PYTHON_DIR-$1) && patch -p1 < $(PROJECT_DIR)/patch/Python/Python.patch
-	# A locally hosted Python requires a full Setup.local configuration
-	# because there's no PYTHON_HOST_PLATFORM to cause Setup.local to be
-	# generated
-	cat $(PROJECT_DIR)/patch/Python/Setup.embedded $(PROJECT_DIR)/patch/Python/Setup.macOS-x86_64 > $$(PYTHON_DIR-$1)/Modules/Setup.local
+	# Copy in the embedded module configuration
+	cat $(PROJECT_DIR)/patch/Python/Setup.embedded $(PROJECT_DIR)/patch/Python/Setup.$1 > $$(PYTHON_DIR-$1)/Modules/Setup.local
 	# Configure target Python
 	cd $$(PYTHON_DIR-$1) && MACOSX_DEPLOYMENT_TARGET=$$(MACOSX_DEPLOYMENT_TARGET) ./configure \
 		--prefix=$(PROJECT_DIR)/$$(PYTHON_DIR-$1)/dist \
-		--without-doc-strings --enable-ipv6 --without-ensurepip \
+		--without-doc-strings --enable-ipv6 --without-ensurepip --enable-universalsdk --with-universal-archs=universal2 \
 		$$(PYTHON_CONFIGURE-$1)
 
 # Build Python
-$$(PYTHON_DIR-$1)/dist/lib/libpython$(PYTHON_VER).a: build/$1/Support/OpenSSL build/$1/Support/BZip2 build/$1/Support/XZ $$(PYTHON_DIR-$1)/Makefile
+$$(PYTHON_DIR-$1)/dist/lib/libpython$(PYTHON_VER).a: build/$1/Support/OpenSSL build/$1/Support/BZip2 build/$1/Support/XZ build/$1/Support/libFFI $$(PYTHON_DIR-$1)/Makefile
 	# Build target Python
 	cd $$(PYTHON_DIR-$1) && PATH="$(PROJECT_DIR)/$(PYTHON_DIR-$1)/dist/bin:$(PATH)" make all install
 
 else
+# The LibFFI folder is shared between all architectures for the OS
+LIBFFI_DIR-$1=build/$1/libffi-$(LIBFFI_VERSION)
 # The Python targets are the same as they are for every other library
 PYTHON_TARGETS-$1=$$(TARGETS-$1)
 
+# Unpack LibFFI and generate source & headers
+$$(LIBFFI_DIR-$1)/darwin_common: downloads/libffi-$(LIBFFI_VERSION).tgz
+	# Unpack sources
+	mkdir -p $$(LIBFFI_DIR-$1)
+	tar zxf downloads/libffi-$(LIBFFI_VERSION).tgz --strip-components 1 -C $$(LIBFFI_DIR-$1)
+	# Apply libffi patches. Apple builds of libffi use a utility script; that
+	# script doesn't work with Python3 using the out-of-the-box version in
+	# libffi 3.3. This patch adds some changes that will be included in libFFI
+	# 3.4 (whenever that is released), plus some extra targets to support
+	# tvOS and watchOS.
+	cd $$(LIBFFI_DIR-$1) && patch -p1 < $(PROJECT_DIR)/patch/libffi/libffi.patch
+	# Configure the build
+	cd $$(LIBFFI_DIR-$1) && python generate-darwin-source-and-headers.py --only-$(shell echo $1 | tr '[:upper:]' '[:lower:]')
+
+$$(LIBFFI_FRAMEWORK-$1): $$(LIBFFI_DIR-$1)/libffi.a
+	# Create framework directory structure
+	mkdir -p $$(LIBFFI_FRAMEWORK-$1)
+
+	# Copy the headers.
+	cp -f -r $$(LIBFFI_DIR-$1)/darwin_common/include $$(LIBFFI_FRAMEWORK-$1)/Headers
+	cp -f -r $$(LIBFFI_DIR-$1)/darwin_$(shell echo $1 | tr '[:upper:]' '[:lower:]')/include/* $$(LIBFFI_FRAMEWORK-$1)/Headers
+
+	# Create the fat library
+	xcrun libtool -no_warning_for_no_symbols -static \
+		-o $$(LIBFFI_FRAMEWORK-$1)/libFFI.a $$^
+
+$$(LIBFFI_DIR-$1)/libffi.a: $$(foreach target,$$(TARGETS-$1),$$(LIBFFI_DIR-$1)/libffi.$$(target).a)
+	xcrun lipo -create -o $$@ $$^
+
 endif
 
 $1: Python-$1
@@ -479,7 +555,9 @@ vars-$1: $$(foreach target,$$(TARGETS-$1),vars-$$(target))
 	@echo "OPENSSL_FRAMEWORK-$1: $$(OPENSSL_FRAMEWORK-$1)"
 	@echo "BZIP2_FRAMEWORK-$1: $$(BZIP2_FRAMEWORK-$1)"
 	@echo "XZ_FRAMEWORK-$1: $$(XZ_FRAMEWORK-$1)"
+	@echo "LIBFFI_FRAMEWORK-$1: $$(LIBFFI_FRAMEWORK-$1)"
 	@echo "PYTHON_FRAMEWORK-$1: $$(PYTHON_FRAMEWORK-$1)"
+	@echo "LIBFFI_DIR-$1: $$(LIBFFI_DIR-$1)"
 	@echo "PYTHON_RESOURCES-$1: $$(PYTHON_RESOURCES-$1)"
 	@echo "PYTHON_TARGETS-$1: $$(PYTHON_TARGETS-$1)"