Merge branch 'dev' into 3.7

Author: freakboy3742

.github/workflows/publish.yaml

@@ -0,0 +1,48 @@
+name: Publish support package
+
+on:
+  release:
+    types: published
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python environment
+      uses: actions/setup-python@v1
+      with:
+        python-version: "3.X"
+    - name: Set build variables
+      env:
+        TAG_NAME: ${{ github.ref }}
+      run: |
+        export TAG=$(basename $TAG_NAME)
+        echo "TAG=${TAG}"
+        export TAG_VERSION="${TAG%-*}"
+        export TAG_BUILD="${TAG#*-}"
+        echo "PY_VERSION=${TAG_VERSION}"
+        echo "BUILD_NUMBER=${TAG_BUILD}"
+        echo "::set-env name=TAG::${TAG}"
+        echo "::set-env name=PY_VERSION::${TAG_VERSION}"
+        echo "::set-env name=BUILD_NUMBER::${TAG_BUILD}"
+    - name: Update Release Asset to S3
+      env:
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      run: |
+        python -m pip install -U pip
+        python -m pip install -U setuptools
+        python -m pip install awscli
+        # macOS build
+        curl -o macOS-artefact.tar.gz -L https://github.com/beeware/Python-Apple-support/releases/download/${{ env.TAG }}/Python-${{ env.PY_VERSION }}-macOS-support.${{ env.BUILD_NUMBER }}.tar.gz
+        aws s3 cp macOS-artefact.tar.gz s3://briefcase-support/python/${{ env.PY_VERSION }}/macOS/Python-${{ env.PY_VERSION }}-macOS-support.${{ env.BUILD_NUMBER }}.tar.gz
+        # iOS build
+        curl -o iOS-artefact.tar.gz -L https://github.com/beeware/Python-Apple-support/releases/download/${{ env.TAG }}/Python-${{ env.PY_VERSION }}-iOS-support.${{ env.BUILD_NUMBER }}.tar.gz
+        aws s3 cp iOS-artefact.tar.gz s3://briefcase-support/python/${{ env.PY_VERSION }}/iOS/Python-${{ env.PY_VERSION }}-iOS-support.${{ env.BUILD_NUMBER }}.tar.gz
+        # tvOS build
+        curl -o tvOS-artefact.tar.gz -L https://github.com/beeware/Python-Apple-support/releases/download/${{ env.TAG }}/Python-${{ env.PY_VERSION }}-tvOS-support.${{ env.BUILD_NUMBER }}.tar.gz
+        aws s3 cp tvOS-artefact.tar.gz s3://briefcase-support/python/${{ env.PY_VERSION }}/tvOS/Python-${{ env.PY_VERSION }}-tvOS-support.${{ env.BUILD_NUMBER }}.tar.gz
+        # watchOS build
+        curl -o watchOS-artefact.tar.gz -L https://github.com/beeware/Python-Apple-support/releases/download/${{ env.TAG }}/Python-${{ env.PY_VERSION }}-watchOS-support.${{ env.BUILD_NUMBER }}.tar.gz
+        aws s3 cp watchOS-artefact.tar.gz s3://briefcase-support/python/${{ env.PY_VERSION }}/watchOS/Python-${{ env.PY_VERSION }}-watchOS-support.${{ env.BUILD_NUMBER }}.tar.gz

.github/workflows/release.yaml

@@ -0,0 +1,92 @@
+# This is the DEV workflow.
+# Run this Action on creating a new tag matching "<version>-b<build>"
+# e.g., 3.7-b1
+name: Build support package
+on:
+  push:
+    tags:
+    - '*-b*'
+
+jobs:
+  make-release:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Set build variables
+      env:
+        TAG_NAME: ${{ github.ref }}
+      run: |
+        export TAG=$(basename $TAG_NAME)
+        echo "TAG=${TAG}"
+        export TAG_VERSION="${TAG%-*}"
+        export TAG_BUILD="${TAG#*-}"
+        echo "PY_VERSION=${TAG_VERSION}"
+        echo "BUILD_NUMBER=${TAG_BUILD}"
+        echo "::set-env name=TAG::${TAG}"
+        echo "::set-env name=PY_VERSION::${TAG_VERSION}"
+        echo "::set-env name=BUILD_NUMBER::${TAG_BUILD}"
+    - name: Create Release
+      id: create_release
+      uses: actions/create-release@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        tag_name: ${{ github.ref }}
+        release_name: ${{ github.ref }}
+        draft: true
+        prerelease: false
+        body: |
+          Build ${{ env.BUILD_NUMBER }} of the BeeWare support package for Python ${{ env.PY_VERSION }}.
+
+          Includes:
+          * Python ${{ env.PY_VERSION }}.?
+          * ...
+    outputs:
+      upload_url: ${{ steps.create_release.outputs.upload_url }}
+
+  build:
+    runs-on: macOS-latest
+    needs: make-release
+    strategy:
+      max-parallel: 4
+      matrix:
+        target: ['macOS', 'iOS', 'tvOS', 'watchOS']
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set build variables
+      env:
+        TAG_NAME: ${{ github.ref }}
+      run: |
+        export TAG=$(basename $TAG_NAME)
+        echo "TAG=${TAG}"
+        export TAG_VERSION="${TAG%-*}"
+        export TAG_BUILD="${TAG#*-}"
+        echo "PY_VERSION=${TAG_VERSION}"
+        echo "BUILD_NUMBER=${TAG_BUILD}"
+        echo "::set-env name=TAG::${TAG}"
+        echo "::set-env name=PY_VERSION::${TAG_VERSION}"
+        echo "::set-env name=BUILD_NUMBER::${TAG_BUILD}"
+    - name: Build ${{ matrix.target }}
+      env:
+        BUILD_NUMBER: ${{ env.BUILD_NUMBER }}
+      run: |
+        # 2020-05-29: The iOS 13.4 SDK changed the implementation of FD_SET
+        # to use a weak linked symbol (__darwin_check_fd_set_overflow).
+        # This means any support package built against that SDK is incompatible
+        # with linking against an earlier SDK.
+        # However, you can't get the iOS 13.4 SDK on macOS Mojave (10.14),
+        # because the latest XCode supported on Mojave (11.3.1) only provided
+        # SDK 13.2.
+        # So: We force the use of XCode 11.3.1, which ships with the 13.2 SDK.
+        # For a list of SDKs available on Github Actions, see:
+        # https://github.com/actions/virtual-environments/blob/master/images/macos/macos-10.15-Readme.md
+        sudo xcode-select --switch /Applications/Xcode_11.3.1.app
+        make -e ${{ matrix.target }}
+    - name: Upload ${{ matrix.target }} release asset
+      uses: actions/upload-release-asset@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ needs.make-release.outputs.upload_url }}
+        asset_path: ./dist/Python-${{ env.PY_VERSION }}-${{ matrix.target }}-support.${{ env.BUILD_NUMBER }}.tar.gz
+        asset_name: Python-${{ env.PY_VERSION }}-${{ matrix.target }}-support.${{ env.BUILD_NUMBER }}.tar.gz
+        asset_content_type: application/gzip

Makefile

@@ -25,7 +25,7 @@
 # Current director
 PROJECT_DIR=$(shell pwd)
 
-BUILD_NUMBER=3
+BUILD_NUMBER=custom
 
 MACOSX_DEPLOYMENT_TARGET=10.8
 
@@ -34,12 +34,12 @@ PYTHON_VERSION=3.7.6
 PYTHON_VER=$(basename $(PYTHON_VERSION))
 
 OPENSSL_VERSION_NUMBER=1.1.1
-OPENSSL_REVISION=d
+OPENSSL_REVISION=g
 OPENSSL_VERSION=$(OPENSSL_VERSION_NUMBER)$(OPENSSL_REVISION)
 
 BZIP2_VERSION=1.0.8
 
-XZ_VERSION=5.2.4
+XZ_VERSION=5.2.5
 
 # Supported OS
 OS=macOS iOS tvOS watchOS
@@ -50,20 +50,22 @@ CFLAGS-macOS=-mmacosx-version-min=$(MACOSX_DEPLOYMENT_TARGET)
 
 # iOS targets
 TARGETS-iOS=iphonesimulator.x86_64 iphoneos.arm64
-CFLAGS-iOS=-mios-version-min=8.0
-CFLAGS-iphoneos.arm64=-fembed-bitcode
-CFLAGS-iphonesimulator.x86_64=-fembed-bitcode
+CFLAGS-iOS=-mios-version-min=8.0 -fembed-bitcode
+CFLAGS-iphoneos.arm64=
+CFLAGS-iphonesimulator.x86_64=
 
 # tvOS targets
 TARGETS-tvOS=appletvsimulator.x86_64 appletvos.arm64
-CFLAGS-tvOS=-mtvos-version-min=9.0
-CFLAGS-appletvos.arm64=-fembed-bitcode
+CFLAGS-tvOS=-mtvos-version-min=9.0 -fembed-bitcode
+CFLAGS-appletvos.arm64=
+CFLAGS-appletvsimulator.x86_64=
 PYTHON_CONFIGURE-tvOS=ac_cv_func_sigaltstack=no
 
 # watchOS targets
 TARGETS-watchOS=watchsimulator.i386 watchos.armv7k
-CFLAGS-watchOS=-mwatchos-version-min=4.0
-CFLAGS-watchos.armv7k=-fembed-bitcode
+CFLAGS-watchOS=-mwatchos-version-min=4.0 -fembed-bitcode
+CFLAGS-watchsimulator.i386=
+CFLAGS-watchos.armv7k=
 PYTHON_CONFIGURE-watchOS=ac_cv_func_sigaltstack=no
 
 # override machine types for arm64
@@ -88,9 +90,6 @@ update-patch:
 	if [ -z "$(PYTHON_REPO_DIR)" ]; then echo "\n\nPYTHON_REPO_DIR must be set to the root of your Python github checkout\n\n"; fi
 	cd $(PYTHON_REPO_DIR) && git diff -D v$(PYTHON_VERSION) $(PYTHON_VER) | filterdiff -X $(PROJECT_DIR)/patch/Python/diff-exclude.lst -p 1 --clean > $(PROJECT_DIR)/patch/Python/Python.patch
 
-upload: $(foreach os,$(OS),$(os))
-	python tools/upload.py b$(BUILD_NUMBER)
-
 ###########################################################################
 # OpenSSL
 # These build instructions adapted from the scripts developed by
@@ -228,16 +227,12 @@ endif
 
 # Build OpenSSL
 $$(OPENSSL_DIR-$1)/libssl.a $$(OPENSSL_DIR-$1)/libcrypto.a: $$(OPENSSL_DIR-$1)/Makefile
-	# Installing multiple times causes problems with the man directory.
-	# Since we're just overwriting anyway, and we're not going to keep the man files
-	# delete the man directory (if it exists) first.
-	rm -rf $(PROJECT_DIR)/build/$2/openssl/man
-	# Make the build
+	# Make the build, and install just the software (not the docs)
 	cd $$(OPENSSL_DIR-$1) && \
 		CC="$$(CC-$1)" \
 		CROSS_TOP="$$(dir $$(SDK_ROOT-$1)).." \
 		CROSS_SDK="$$(notdir $$(SDK_ROOT-$1))" \
-		make all && make install
+		make all && make install_sw
 
 # Unpack BZip2
 $$(BZIP2_DIR-$1)/Makefile: downloads/bzip2-$(BZIP2_VERSION).tgz
@@ -277,17 +272,22 @@ $$(PYTHON_DIR-$1)/Makefile: downloads/Python-$(PYTHON_VERSION).tgz $$(PYTHON_HOS
 	tar zxf downloads/Python-$(PYTHON_VERSION).tgz --strip-components 1 -C $$(PYTHON_DIR-$1)
 	# Apply target Python patches
 	cd $$(PYTHON_DIR-$1) && patch -p1 < $(PROJECT_DIR)/patch/Python/Python.patch
-	cp -f $(PROJECT_DIR)/patch/Python/Setup.embedded $$(PYTHON_DIR-$1)/Modules/Setup.embedded
-
 	# Configure target Python
 ifeq ($2,macOS)
+	# A locally hosted Python requires a full Setup.local configuration
+	# because there's no PYTHON_HOST_PLATFORM to cause Setup.local to be
+	# generated
+	cat $(PROJECT_DIR)/patch/Python/Setup.embedded $(PROJECT_DIR)/patch/Python/Setup.macOS-x86_64 > $$(PYTHON_DIR-$1)/Modules/Setup.local
 	# Make a fully embedded macOS build
-	cat $$(PYTHON_DIR-$1)/Modules/Setup.embedded $(PROJECT_DIR)/patch/Python/Setup.macOS-x86_64 > $$(PYTHON_DIR-$1)/Modules/Setup.local
 	cd $$(PYTHON_DIR-$1) && MACOSX_DEPLOYMENT_TARGET=$$(MACOSX_DEPLOYMENT_TARGET) ./configure \
 		--prefix=$(PROJECT_DIR)/$$(PYTHON_DIR-$1)/dist \
 		--without-doc-strings --enable-ipv6 --without-ensurepip \
 		$$(PYTHON_CONFIGURE-$2)
 else
+	# Copy in the embedded and platform/arch configuration
+	cp -f $(PROJECT_DIR)/patch/Python/Setup.embedded $$(PYTHON_DIR-$1)/Modules/Setup.embedded
+	if [ -e "$(PROJECT_DIR)/patch/Python/Setup.$2-$$(ARCH-$1)" ]; then \
+		cp -f $(PROJECT_DIR)/patch/Python/Setup.$2-$$(ARCH-$1) $$(PYTHON_DIR-$1)/Modules/Setup.$2-$$(ARCH-$1); fi
 	cd $$(PYTHON_DIR-$1) && PATH=$(PROJECT_DIR)/$(PYTHON_DIR-macOS)/dist/bin:$(PATH) ./configure \
 		CC="$$(CC-$1)" LD="$$(CC-$1)" \
 		--host=$$(MACHINE_DETAILED-$1)-apple-$(shell echo $2 | tr '[:upper:]' '[:lower:]') \
@@ -328,12 +328,12 @@ XZ_FRAMEWORK-$1=build/$1/Support/XZ
 PYTHON_FRAMEWORK-$1=build/$1/Support/Python
 PYTHON_RESOURCES-$1=$$(PYTHON_FRAMEWORK-$1)/Resources
 
-$1: dist/Python-$(PYTHON_VER)-$1-support.b$(BUILD_NUMBER).tar.gz
+$1: dist/Python-$(PYTHON_VER)-$1-support.$(BUILD_NUMBER).tar.gz
 
 clean-$1:
 	rm -rf build/$1
 
-dist/Python-$(PYTHON_VER)-$1-support.b$(BUILD_NUMBER).tar.gz: $$(BZIP2_FRAMEWORK-$1) $$(XZ_FRAMEWORK-$1) $$(OPENSSL_FRAMEWORK-$1) $$(PYTHON_FRAMEWORK-$1)
+dist/Python-$(PYTHON_VER)-$1-support.$(BUILD_NUMBER).tar.gz: $$(BZIP2_FRAMEWORK-$1) $$(XZ_FRAMEWORK-$1) $$(OPENSSL_FRAMEWORK-$1) $$(PYTHON_FRAMEWORK-$1)
 	mkdir -p dist
 	echo "Python version: $(PYTHON_VERSION) " > build/$1/Support/VERSIONS
 	echo "Build: $(BUILD_NUMBER)" >> build/$1/Support/VERSIONS
@@ -344,9 +344,12 @@ dist/Python-$(PYTHON_VER)-$1-support.b$(BUILD_NUMBER).tar.gz: $$(BZIP2_FRAMEWORK
 ifeq ($1,macOS)
 	cp -r build/$1/Python-$(PYTHON_VERSION)-macosx.x86_64/dist build/$1/python
 	mv build/$1/Support/VERSIONS build/$1/python/VERSIONS
-	tar zcvf $$@ -C build/$1/python `ls -A build/$1/python`
+	tar zcvf $$@ -X patch/Python/exclude.macOS -C build/$1/python `ls -A build/$1/python`
 else
-	tar zcvf $$@ -C build/$1/Support `ls -A build/$1/Support`
+	# Build a "full" tarball with all content for test purposes
+	tar zcvf dist/Python-$(PYTHON_VER)-$1-support.test-$(BUILD_NUMBER).tar.gz -X patch/Python/test-exclude.embedded -C build/$1/Support `ls -A build/$1/Support`
+	# Build a distributable tarball
+	tar zcvf $$@ -X patch/Python/exclude.embedded -C build/$1/Support `ls -A build/$1/Support`
 endif
 
 # Build OpenSSL
@@ -411,7 +414,7 @@ build/$1/xz/lib/liblzma.a: $$(foreach target,$$(TARGETS-$1),$$(XZ_DIR-$$(target)
 
 $1: Python-$1
 
-Python-$1: dist/Python-$(PYTHON_VER)-$1-support.b$(BUILD_NUMBER).tar.gz
+Python-$1: dist/Python-$(PYTHON_VER)-$1-support.$(BUILD_NUMBER).tar.gz
 
 # Build Python
 $$(PYTHON_FRAMEWORK-$1): build/$1/libpython$(PYTHON_VER)m.a $$(foreach target,$$(TARGETS-$1),build/$1/$$(pyconfig.h-$$(target)))
@@ -432,16 +435,7 @@ endif
 	cp -f -r $$(PYTHON_FRAMEWORK-$1)/Headers/Python.h $$(PYTHON_RESOURCES-$1)/include/python$(PYTHON_VER)m
 
 	# Copy the standard library from the simulator build
-ifneq ($(TEST),)
 	cp -f -r $$(PYTHON_DIR-$$(firstword $$(TARGETS-$1)))/dist/lib $$(PYTHON_RESOURCES-$1)
-	# Remove the pieces of the resources directory that aren't needed:
-	rm -f $$(PYTHON_RESOURCES-$1)/lib/libpython$(PYTHON_VER)m.a
-	rm -rf $$(PYTHON_RESOURCES-$1)/lib/pkgconfig
-else
-	mkdir -p $$(PYTHON_RESOURCES-$1)/lib
-	cd $$(PYTHON_DIR-$$(firstword $$(TARGETS-$1)))/dist/lib/python$(PYTHON_VER) && \
-		zip -x@$(PROJECT_DIR)/patch/Python/lib-exclude.lst -r $(PROJECT_DIR)/$$(PYTHON_RESOURCES-$1)/lib/python$(subst .,,$(PYTHON_VER)) *
-endif
 
 	# Copy fat library
 	cp -f $$(filter %.a,$$^) $$(PYTHON_FRAMEWORK-$1)/libPython.a

patch/Python/exclude.embedded

@@ -0,0 +1,47 @@
+# This is a list of Python standard library path patterns
+# we exclude from the embedded device Python-Apple-support tarballs.
+# It is used by `tar -X` during the Makefile build.
+#
+# Remove binaries; not needed for
+Python/Resources/bin
+# Remove includes; use the version packaged in Headers
+Python/Resources/include
+# Remove the Resources version of libpython;
+# we use the version packaged at the root level
+Python/Resources/lib/libpython3.*.a
+# Remove lib/pkgconfig files. These are used for compiling C extension modules.
+Python/Resources/lib/pkgconfig
+# Remove standard library test suites.
+Python/Resources/lib/python*/ctypes/test
+Python/Resources/lib/python*/distutils/tests
+Python/Resources/lib/python*/lib2to3/tests
+Python/Resources/lib/python*/sqlite3/test
+Python/Resources/lib/python*/test
+# Remove compiled test and example modules.
+Python/Resources/lib/python*/lib-dynload/_test*.so
+Python/Resources/lib/python*/lib-dynload/_ctypes_test*.so
+Python/Resources/lib/python*/lib-dynload/xxlimited*.so
+Python/Resources/lib/python*/lib-dynload/_xxtestfuzz.so
+# Remove wsgiref web app module; it's unusual that mobile apps would
+# start a web app server with it.
+Python/Resources/lib/python*/wsgiref
+# Remove command-line curses toolkit.
+Python/Resources/lib/python*/curses
+# Remove config-* directory, which is used for compiling C extension modules.
+Python/Resources/lib/python*/config-*
+# Remove ensurepip. If user code needs pip, it can add it to
+Python/Resources/lib/python*/ensurepip
+# Remove Tcl/Tk GUI code. We don't build against Tcl/Tk at the moment, so this
+# will not work.
+Python/Resources/lib/python*/idlelib
+Python/Resources/lib/python*/tkinter
+Python/Resources/lib/python*/turtle.py
+Python/Resources/lib/python*/turtledemo
+# Remove site-packages directory. The template unpacks user code and
+# dependencies to a different path.
+Python/Resources/lib/python*/site-packages
+# Remove share/ directory, which contains user documentation (man pages).
+Python/Resources/share
+# Remove pyc files. These take up space, but since most stdlib modules are
+# never imported by user code, they mostly have no value.
+Python/Resources/*/*.pyc