Merge pull request #125 from vmware-labs/python3.12

assambar · web-flow · commit 13d5949eb077 · 2023-11-30T09:26:57.000+02:00
Python3.12
diff --git a/.github/workflows/build-python.yaml b/.github/workflows/build-python.yaml
@@ -13,7 +13,7 @@ jobs:
       fail-fast: false
       matrix:
         flavor: ["", "-wasmedge", "-aio", "-aio-wasmedge"]
-        version: ["3.11.3", "3.11.4"]
+        version: ["3.11.3", "3.11.4", "3.12.0"]
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
diff --git a/.gitignore b/.gitignore
@@ -5,6 +5,7 @@ build-staging
 node_modules
 target
 .vscode/**
+.python-version
 *.log
 # Except this file
 !.gitignore
diff --git a/Makefile b/Makefile
@@ -27,6 +27,7 @@ python/v*:
 $(eval $(call create_flavor_targets,python,v3.11.1,aio wasmedge aio-wasmedge))
 $(eval $(call create_flavor_targets,python,v3.11.3,aio wasmedge aio-wasmedge))
 $(eval $(call create_flavor_targets,python,v3.11.4,aio wasmedge aio-wasmedge))
+$(eval $(call create_flavor_targets,python,v3.12.0,aio wasmedge aio-wasmedge))
 
 .PHONY: oci-python-3.11.1
 oci-python-3.11.1: python/v3.11.1
diff --git a/python/Dockerfile b/python/Dockerfile
@@ -1,26 +1,26 @@
 ARG WASI_SDK_VERSION
-ARG PY_VERSION=3.11.0
-ARG PY_SHORT_VERSION=3.11
-
-FROM ghcr.io/vmware-labs/wasmlabs/wasi-builder:${WASI_SDK_VERSION} as build-host-python
-ARG PY_VERSION
-ARG PY_SHORT_VERSION
-RUN DEBIAN_FRONTEND=noninteractive apt install -y \
-      zlib1g-dev
-WORKDIR /tmp
-RUN wget -c https://www.python.org/ftp/python/${PY_VERSION}/Python-${PY_VERSION}.tar.xz && tar -Jxf Python-${PY_VERSION}.tar.xz
-WORKDIR /tmp/Python-${PY_VERSION}
-RUN ./configure --enable-optimizations --prefix=/opt/python${PY_SHORT_VERSION}
-RUN make -j4 && make install
+ARG PY_VERSIONS="3.12.0 3.11.4"
+ARG PY_DEFAULT_VERSION=3.12.0
 
 FROM ghcr.io/vmware-labs/wasmlabs/wasi-builder:${WASI_SDK_VERSION}
-ARG PY_SHORT_VERSION
+ARG PY_VERSIONS
+ARG PY_DEFAULT_VERSION
 # If more capabilities are required from the python-builder, consult this
 # github workflow configuration for a list of possible dependencies -
 # https://github.com/python/cpython/blob/main/.github/workflows/posix-deps-apt.sh
-RUN DEBIAN_FRONTEND=noninteractive apt install -y \
-      tcl
+RUN DEBIAN_FRONTEND=noninteractive apt update && \
+      apt install -y --no-install-recommends \
+      libssl-dev \
+      tcl \
+      zlib1g-dev
+
+ENV HOME="/root"
+WORKDIR ${HOME}
+RUN git clone --depth=1 https://github.com/pyenv/pyenv.git .pyenv
+ENV PYENV_ROOT="${HOME}/.pyenv"
+ENV PATH="${PYENV_ROOT}/shims:${PYENV_ROOT}/bin:${PATH}"
+
+RUN pyenv install ${PY_VERSIONS}
+RUN pyenv global ${PY_DEFAULT_VERSION}
 
-COPY --from=build-host-python /opt/python${PY_SHORT_VERSION}/ /opt/python${PY_SHORT_VERSION}/
-RUN update-alternatives --install /usr/bin/python3 python3 /opt/python${PY_SHORT_VERSION}/bin/python${PY_SHORT_VERSION} 110
-RUN update-alternatives --install /usr/bin/python${PY_SHORT_VERSION} python${PY_SHORT_VERSION} /opt/python${PY_SHORT_VERSION}/bin/python${PY_SHORT_VERSION} 110
+WORKDIR /wlr
diff --git a/python/test/run_me.sh b/python/test/run_me.sh
@@ -10,7 +10,7 @@ TESTS=$(for t in */test.py; do echo $(dirname $t); done)
 
 status=0
 
-for test_dir in "$TESTS"; do
+for test_dir in ${TESTS}; do
     echo "Running test in '${test_dir}'..."
     if diff ${test_dir}/test.stdout <($WLR_TEST_RUNTIME \
             --mapdir /usr::${WLR_OUTPUT}/usr \
diff --git a/python/v3.11.1/wlr-build.sh b/python/v3.11.1/wlr-build.sh
@@ -6,6 +6,8 @@ then
     exit 1
 fi
 
+pyenv local ${WLR_PY_BUILDER_VERSION}
+
 cd "${WLR_SOURCE_PATH}"
 
 if [[ "${WLR_BUILD_FLAVOR}" == *"aio"* ]]
diff --git a/python/v3.11.1/wlr-env-repo.sh b/python/v3.11.1/wlr-env-repo.sh
@@ -7,6 +7,7 @@ then
     unset WLR_ENV_NAME
     unset WLR_PACKAGE_VERSION
     unset WLR_PACKAGE_NAME
+    unset WLR_PY_BUILDER_VERSION
     return
 fi
 
@@ -15,3 +16,4 @@ export WLR_REPO_BRANCH=v3.11.1
 export WLR_ENV_NAME=python/v3.11.1
 export WLR_PACKAGE_VERSION=3.11.1
 export WLR_PACKAGE_NAME=python
+export WLR_PY_BUILDER_VERSION=3.11.4
diff --git a/python/v3.11.3/wlr-build.sh b/python/v3.11.3/wlr-build.sh
@@ -6,6 +6,8 @@ then
     exit 1
 fi
 
+pyenv local ${WLR_PY_BUILDER_VERSION}
+
 cd "${WLR_SOURCE_PATH}"
 
 if [[ "${WLR_BUILD_FLAVOR}" == *"aio"* ]]
diff --git a/python/v3.11.3/wlr-env-repo.sh b/python/v3.11.3/wlr-env-repo.sh
@@ -7,6 +7,7 @@ then
     unset WLR_ENV_NAME
     unset WLR_PACKAGE_VERSION
     unset WLR_PACKAGE_NAME
+    unset WLR_PY_BUILDER_VERSION
     return
 fi
 
@@ -15,3 +16,4 @@ export WLR_REPO_BRANCH=v3.11.3
 export WLR_ENV_NAME=python/v3.11.3
 export WLR_PACKAGE_VERSION=3.11.3
 export WLR_PACKAGE_NAME=python
+export WLR_PY_BUILDER_VERSION=3.11.4
diff --git a/python/v3.11.4/wlr-build.sh b/python/v3.11.4/wlr-build.sh
@@ -6,6 +6,8 @@ then
     exit 1
 fi
 
+pyenv local ${WLR_PY_BUILDER_VERSION}
+
 cd "${WLR_SOURCE_PATH}"
 
 if [[ "${WLR_BUILD_FLAVOR}" == *"aio"* ]]
diff --git a/python/v3.11.4/wlr-env-repo.sh b/python/v3.11.4/wlr-env-repo.sh
@@ -7,6 +7,7 @@ then
     unset WLR_ENV_NAME
     unset WLR_PACKAGE_VERSION
     unset WLR_PACKAGE_NAME
+    unset WLR_PY_BUILDER_VERSION
     return
 fi
 
@@ -15,3 +16,4 @@ export WLR_REPO_BRANCH=v3.11.4
 export WLR_ENV_NAME=python/v3.11.4
 export WLR_PACKAGE_VERSION=3.11.4
 export WLR_PACKAGE_NAME=python
+export WLR_PY_BUILDER_VERSION=3.11.4
diff --git a/python/v3.12.0/wlr-build.sh b/python/v3.12.0/wlr-build.sh
@@ -0,0 +1,144 @@
+#!/usr/bin/env bash
+
+if [[ ! -v WLR_ENV ]]
+then
+    echo "WLR build environment is not set"
+    exit 1
+fi
+
+pyenv local ${WLR_PY_BUILDER_VERSION}
+
+cd "${WLR_SOURCE_PATH}"
+
+if [[ "${WLR_BUILD_FLAVOR}" == *"aio"* ]]
+then
+    source ${WLR_REPO_ROOT}/scripts/build-helpers/wlr_wasi_vfs.sh
+    export LDFLAGS="$(wlr_wasi_vfs_get_link_flags) ${LDFLAGS}"
+fi
+
+source ${WLR_REPO_ROOT}/scripts/build-helpers/wlr_pkg_config.sh
+
+export CFLAGS_CONFIG="-O0"
+
+
+# This fails with upgraded clang for wasi-sdk19 and later. Disabled on cpython main.
+#
+# PyModule_AddIntMacro(module, CLOCK_MONOTONIC) and the like cause this.
+# In all POSIX variants CLOCK_MONOTONIC is a numeric constant, so python imports it as int macro
+# However, in wasi-libc clockid_t is defined as a pointer to struct __clockid.
+
+export CFLAGS_CONFIG="${CFLAGS_CONFIG} -Wno-int-conversion"
+
+export CFLAGS="${CFLAGS_CONFIG} ${CFLAGS_DEPENDENCIES} ${CFLAGS}"
+export LDFLAGS="${LDFLAGS_DEPENDENCIES} ${LDFLAGS}"
+
+export PYTHON_WASM_CONFIGURE="--with-build-python=python3"
+
+if [[ "${WLR_BUILD_FLAVOR}" == *"wasmedge"* ]]
+then
+    if [[ ! -v WABT_ROOT ]]
+    then
+        echo "WABT_ROOT is needed to patch imports for wasmedge"
+        exit 1
+    fi
+fi
+
+# By exporting WLR_SKIP_WASM_OPT envvar during the build, the
+# wasm-opt wrapper in the wasm-base image will be a dummy wrapper that
+# is effectively a NOP.
+#
+# This is due to https://github.com/llvm/llvm-project/issues/55781, so
+# that we get to choose which optimization passes are executed after
+# the artifacts have been built.
+export WLR_SKIP_WASM_OPT=1
+
+if [[ -z "$WLR_SKIP_CONFIGURE" ]]; then
+    logStatus "Configuring build with '${PYTHON_WASM_CONFIGURE}'... "
+    CONFIG_SITE=./Tools/wasm/config.site-wasm32-wasi ./configure -C --host=wasm32-wasi --build=$(./config.guess) ${PYTHON_WASM_CONFIGURE} || exit 1
+else
+    logStatus "Skipping configure..."
+fi
+
+export MAKE_TARGETS='python.wasm wasm_stdlib'
+
+logStatus "Building '${MAKE_TARGETS}'... "
+make -j ${MAKE_TARGETS} || exit 1
+
+unset WLR_SKIP_WASM_OPT
+
+if [[ "${WLR_BUILD_FLAVOR}" == *"aio"* ]]
+then
+    logStatus "Packing with wasi-vfs"
+    wlr_wasi_vfs_cli pack python.wasm --mapdir /usr::$PWD/usr -o python.wasm || exit 1
+fi
+
+logStatus "Optimizing python binary..."
+wasm-opt -O2 -o python-optimized.wasm python.wasm || exit 1
+
+if [[ "${WLR_BUILD_FLAVOR}" == *"wasmedge"* ]]
+then
+    logStatus "Patching python binary for wasmedge..."
+    ${WLR_REPO_ROOT}/scripts/build-helpers/patch_wasmedge_wat_sock_accept.sh python-optimized.wasm || exit 1
+fi
+
+logStatus "Preparing artifacts... "
+TARGET_PYTHON_BINARY=bin/python-${WLR_PACKAGE_VERSION}.wasm
+
+mkdir -p ${WLR_OUTPUT}/bin 2>/dev/null || exit 1
+
+if [[ "${WLR_BUILD_FLAVOR}" == *"aio"* ]]
+then
+    cp -v python-optimized.wasm ${WLR_OUTPUT}/${TARGET_PYTHON_BINARY} || exit 1
+else
+    mkdir -p ${WLR_OUTPUT}/usr 2>/dev/null || exit 1
+    cp -v python-optimized.wasm ${WLR_OUTPUT}/${TARGET_PYTHON_BINARY} || exit 1
+    cp -TRv usr ${WLR_OUTPUT}/usr || exit 1
+fi
+
+if [[ "${WLR_BUILD_FLAVOR}" != *"aio"* && "${WLR_BUILD_FLAVOR}" != *"wasmedge"* ]]
+then
+
+    logStatus "Install includes..."
+    make inclinstall \
+        prefix=${WLR_OUTPUT} \
+        libdir=${WLR_OUTPUT}/lib/wasm32-wasi \
+        pkgconfigdir=${WLR_OUTPUT}/lib/wasm32-wasi/pkgconfig || exit 1
+
+    logStatus "Create libpython3.12-aio.a"
+(${AR} -M <<EOF
+create libpython3.12-aio.a
+addlib libpython3.12.a
+addlib ${WLR_DEPS_ROOT}/build-output/lib/wasm32-wasi/libz.a
+addlib ${WLR_DEPS_ROOT}/build-output/lib/wasm32-wasi/libbz2.a
+addlib ${WLR_DEPS_ROOT}/build-output/lib/wasm32-wasi/libsqlite3.a
+addlib ${WLR_DEPS_ROOT}/build-output/lib/wasm32-wasi/libuuid.a
+addlib Modules/expat/libexpat.a
+addlib Modules/_decimal/libmpdec/libmpdec.a
+save
+end
+EOF
+) || echo exit 1
+
+    mkdir -p ${WLR_OUTPUT}/lib/wasm32-wasi/ 2>/dev/null || exit 1
+    cp -v libpython3.12-aio.a ${WLR_OUTPUT}/lib/wasm32-wasi/libpython3.12.a || exit 1
+
+    logStatus "Generating pkg-config file for libpython3.12.a"
+    DESCRIPTION="libpython3.12 allows embedding the CPython interpreter"
+    EXTRA_LINK_FLAGS="-lpython3.12 -Wl,-z,stack-size=524288 -Wl,--stack-first -Wl,--initial-memory=10485760 -lwasi-emulated-getpid -lwasi-emulated-signal -lwasi-emulated-process-clocks"
+
+    PC_INCLUDE_SUBDIR=python3.12 wlr_pkg_config_create_pc_file "libpython3.12" "${WLR_PACKAGE_VERSION}" "${DESCRIPTION}" "${EXTRA_LINK_FLAGS}" || exit 1
+
+    WLR_PACKAGE_EXTRA_DIRS=usr wlr_package_lib || exit 1
+    WLR_PACKAGE_LIST="${TARGET_PYTHON_BINARY} usr" wlr_package || exit 1
+
+elif [[ "${WLR_BUILD_FLAVOR}" == *"aio"* ]]; then
+    # skip 'aio' in the name
+    FLAVOR_SUFFIX=$(echo ${WLR_BUILD_FLAVOR} | sed 's/-\?aio//g')
+    PUBLISHED_PYTHON_BINARY=python-${WLR_PACKAGE_VERSION}${FLAVOR_SUFFIX}.wasm
+    cp -v ${WLR_OUTPUT}/${TARGET_PYTHON_BINARY} ${WLR_OUTPUT_BASE}/${PUBLISHED_PYTHON_BINARY} || exit 1
+
+else
+    WLR_PACKAGE_LIST="${TARGET_PYTHON_BINARY} usr" wlr_package || exit 1
+fi
+
+logStatus "DONE. Artifacts in ${WLR_OUTPUT}"
diff --git a/python/v3.12.0/wlr-env-repo.sh b/python/v3.12.0/wlr-env-repo.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+if [[ $1 == "--unset" ]]
+then
+    unset WLR_REPO
+    unset WLR_REPO_BRANCH
+    unset WLR_ENV_NAME
+    unset WLR_PACKAGE_VERSION
+    unset WLR_PACKAGE_NAME
+    unset WLR_PY_BUILDER_VERSION
+    return
+fi
+
+export WLR_REPO=https://github.com/python/cpython
+export WLR_REPO_BRANCH=v3.12.0
+export WLR_ENV_NAME=python/v3.12.0
+export WLR_PACKAGE_VERSION=3.12.0
+export WLR_PACKAGE_NAME=python
+export WLR_PY_BUILDER_VERSION=3.12.0
diff --git a/python/v3.12.0/wlr-info.json b/python/v3.12.0/wlr-info.json
@@ -0,0 +1,24 @@
+{
+    "deps": {
+        "libuuid": {
+            "build_target": "libs/libuuid/v1.0.3",
+            "required_file": "lib/wasm32-wasi/libuuid.a",
+            "url": "https://github.com/vmware-labs/webassembly-language-runtimes/releases/download/libs%2Flibuuid%2F1.0.3%2B20230623-2993864/libuuid-1.0.3-wasi-sdk-20.0.tar.gz"
+        },
+        "zlib": {
+            "build_target": "libs/zlib/v1.2.13",
+            "required_file": "lib/wasm32-wasi/libz.a",
+            "url": "https://github.com/vmware-labs/webassembly-language-runtimes/releases/download/libs%2Fzlib%2F1.2.13%2B20230623-2993864/libz-1.2.13-wasi-sdk-20.0.tar.gz"
+        },
+        "SQLite": {
+            "build_target": "libs/sqlite/v3.42.0",
+            "required_file": "lib/wasm32-wasi/libsqlite3.a",
+            "url": "https://github.com/vmware-labs/webassembly-language-runtimes/releases/download/libs%2Fsqlite%2F3.42.0%2B20230623-2993864/libsqlite-3.42.0-wasi-sdk-20.0.tar.gz"
+        },
+        "bzip2": {
+            "build_target": "libs/bzip2/v1.0.8",
+            "required_file": "lib/wasm32-wasi/libbz2.a",
+            "url": "https://github.com/vmware-labs/webassembly-language-runtimes/releases/download/libs%2Fbzip2%2F1.0.8%2B20230623-2993864/libbzip2-1.0.8-wasi-sdk-20.0.tar.gz"
+        }
+    }
+}
diff --git a/python/v3.12.0/wlr-tag.sh b/python/v3.12.0/wlr-tag.sh
@@ -0,0 +1 @@
+export WLR_TAG="python/3.12.0"
diff --git a/python/v3.12.0/wlr-test.sh b/python/v3.12.0/wlr-test.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+if [[ ! -v WLR_ENV ]]
+then
+    echo "WLR build environment is not set"
+    exit 1
+fi
+
+if [[ "${WLR_BUILD_FLAVOR}" == *"wasmedge"* ]]
+then
+    WLR_BINARY_WUFFIX=-wasmedge
+fi
+
+export WLR_TESTED_MODULE="${WLR_OUTPUT}/bin/python-${WLR_PACKAGE_VERSION}${WLR_BINARY_WUFFIX}.wasm"
+export TEST_ROOT=${WLR_ENV}/../test
+
+(cd ${TEST_ROOT}; ./run_me.sh)
