macOS semaphore pipeline for x64 and arm64

emasab · emasab · commit eba2d0d7bbf5 · 2022-07-29T23:16:40.000+02:00
diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml
@@ -0,0 +1,93 @@
+version: v1.0
+name: Test on PR or create and upload wheels on tag.
+agent:
+  machine:
+    type: s1-prod-mac-m1
+global_job_config:
+  secrets:
+    - name: vault_sem2_approle
+  env_vars:
+    - name: LIBRDKAFKA_VERSION
+      value: v1.9.2-RC3
+blocks:
+  - name: "Wheels: OSX x64"
+    run:
+      when: "tag =~ '.*'"
+    dependencies: []
+    task:
+      agent:
+        machine:
+          type: s1-prod-macos
+      env_vars:
+        - name: OS_NAME
+          value: osx
+        - name: ARCH
+          value: x64
+      jobs:
+        - name: Build
+          commands:
+            - cd $SEM_WORKSPACE
+            - export HOME=$SEM_WORKSPACE
+            - checkout
+            # needed on the self-hosted agent
+            - if [ ! -d ./tools ]; then cd $SEM_WORKSPACE/confluent-kafka-python; fi
+            - PIP_INSTALL_OPTIONS="--user" tools/wheels/build-wheels.sh "${LIBRDKAFKA_VERSION#v}" wheelhouse
+            - tar -czf wheelhouse-macOS-${ARCH}.tgz wheelhouse
+            - artifact push workflow wheelhouse-macOS-${ARCH}.tgz
+  - name: "Wheels: OSX arm64"
+    run:
+      when: "tag =~ '.*'"
+    dependencies: []
+    task:
+      env_vars:
+        - name: OS_NAME
+          value: osx
+        - name: CIBW_ARCHS
+          value: arm64
+        - name: ARCH
+          value: arm64
+      jobs:
+        - name: Build
+          commands:
+            - cd $SEM_WORKSPACE
+            - export HOME=$SEM_WORKSPACE
+            - checkout
+            # needed on the self-hosted agent
+            - if [ ! -d ./tools ]; then cd $SEM_WORKSPACE/confluent-kafka-python; fi
+            - PIP_INSTALL_OPTIONS="--user" tools/wheels/build-wheels.sh "${LIBRDKAFKA_VERSION#v}" wheelhouse
+            - tar -czf wheelhouse-macOS-${ARCH}.tgz wheelhouse
+            - artifact push workflow wheelhouse-macOS-${ARCH}.tgz
+  
+  - name: Source package verification with Python 3 (OSX x64) +docs
+    dependencies: []
+    task:
+      agent:
+        machine:
+          type: s1-prod-macos
+      env_vars:
+        - name: OS_NAME
+          value: osx
+        - name: ARCH
+          value: arm64
+      jobs:
+        - name: Build
+          commands:
+            - cd $SEM_WORKSPACE
+            - export HOME=$SEM_WORKSPACE
+            - checkout
+            # needed on the self-hosted agent
+            - if [ ! -d ./tools ]; then cd $SEM_WORKSPACE/confluent-kafka-python; fi
+            # use a virtualenv
+            - python3 -m venv _venv && source _venv/bin/activate
+            - pip install -r docs/requirements.txt
+            - pip install -U protobuf
+            # install librdkafka
+            - lib_dir=dest/runtimes/$OS_NAME-$ARCH/native
+            - tools/wheels/install-librdkafka.sh "${LIBRDKAFKA_VERSION#v}" dest
+            - export CFLAGS="$CFLAGS -I${PWD}/dest/build/native/include"
+            - export LDFLAGS="$LDFLAGS -L${PWD}/${lib_dir}"
+            - export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$PWD/$lib_dir"
+            - export DYLD_LIBRARY_PATH="$DYLD_LIBRARY_PATH:$PWD/$lib_dir"
+            # install confluent-kafka
+            - python setup.py build && python setup.py install
+            - make docs
diff --git a/.travis.yml b/.travis.yml
@@ -1,37 +1,17 @@
 env:
  global:
-  - LIBRDKAFKA_VERSION=v1.9.2-RC2
+  - LIBRDKAFKA_VERSION=v1.9.2-RC3
 
 jobs:
  include:
-  - name: "Source package verification with Python 2.7 (Linux)"
-    if: false
+  - name: "Source package verification with Python 3.8 (Linux)"
     os: linux
     language: python
     dist: xenial
-    python: "2.7"
-    env: LD_LIBRARY_PATH="$PWD/tmp-build/lib"
-    services: docker
-
-  - name: "Source package verification with Python 3.6 (Linux)"
-    os: linux
-    language: python
-    dist: xenial
-    python: "3.6"
+    python: "3.8"
     env: LD_LIBRARY_PATH="$PWD/tmp-build/lib"
     services: docker
 
-  - name: "Source package verification with Python 2.7 (OSX)"
-    if: false
-    os: osx
-    python: "2.7"
-    env: DYLD_LIBRARY_PATH="$PWD/tmp-build/lib" INTERPRETER_VERSION="2.7.17"
-
-  - name: "Source package verification with Python 3.6 (OSX) +docs"
-    os: osx
-    python: "3.6"
-    env: DYLD_LIBRARY_PATH="$PWD/tmp-build/lib" MK_DOCS="y" INTERPRETER_VERSION="3.6.5"
-
   - name: "Wheels: Windows x64"
     if: tag is present
     os: windows
@@ -70,19 +50,12 @@ jobs:
     env: BUILD_WHEELS=1
     script: tools/wheels/build-wheels.sh ${LIBRDKAFKA_VERSION#v} wheelhouse
 
-  - name: "Wheels: MacOSX x64"
-    if: tag is present
-    os: osx
-    language: shell
-    env: BUILD_WHEELS=1
-    script: tools/wheels/build-wheels.sh ${LIBRDKAFKA_VERSION#v} wheelhouse
-
-
-# Install test dependencies unconditionally
-# Travis OSX envs requires some setup; see tools/prepare-osx.sh
-# Install cibuildwheel if this is a tagged PR
-before_install:
-  - if [[ $TRAVIS_OS_NAME == "osx" && $BUILD_WHEELS != 1 ]]; then tools/prepare-osx.sh ${INTERPRETER_VERSION} /tmp/venv && source /tmp/venv/bin/activate; fi
+  # - name: "Wheels: MacOSX x64"
+  #   if: tag is present
+  #   os: osx
+  #   language: shell
+  #   env: BUILD_WHEELS=1
+  #   script: tools/wheels/build-wheels.sh ${LIBRDKAFKA_VERSION#v} wheelhouse
 
 install:
  # Install interceptors
diff --git a/examples/docker/Dockerfile.alpine b/examples/docker/Dockerfile.alpine
@@ -30,7 +30,7 @@ FROM alpine:3.12
 
 COPY . /usr/src/confluent-kafka-python
 
-ENV LIBRDKAFKA_VERSION v1.9.2-RC2
+ENV LIBRDKAFKA_VERSION v1.9.2-RC3
 ENV KAFKACAT_VERSION master
 
 
diff --git a/tools/prepare-osx.sh b/tools/prepare-osx.sh
diff --git a/tools/wheels/build-wheels.sh b/tools/wheels/build-wheels.sh
@@ -8,7 +8,7 @@ this_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
 
 # Skip PyPy, Python2, old Python3 versions, musl, and x86 builds.
-export CIBW_SKIP="pp* cp27-* cp35-* *i686 *musllinux*"
+export CIBW_SKIP="pp* cp27-* cp35-* *i686 *musllinux* $CIBW_SKIP"
 # Run a simple test suite
 export CIBW_TEST_REQUIRES="-r tests/requirements.txt"
 export CIBW_TEST_COMMAND="pytest {project}/tests/test_Producer.py"
@@ -26,19 +26,20 @@ set -ex
 
 [[ -d $wheeldir ]] || mkdir -p "$wheeldir"
 
+ARCH=${ARCH:-x64}
 
 case $OSTYPE in
     linux*)
         os=linux
         # Need to set up env vars (in docker) so that setup.py
         # finds librdkafka.
-        lib_dir=dest/runtimes/linux-x64/native
+        lib_dir=dest/runtimes/linux-$ARCH/native
         export CIBW_ENVIRONMENT="CFLAGS=-I\$PWD/dest/build/native/include LDFLAGS=-L\$PWD/$lib_dir LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:\$PWD/$lib_dir"
         ;;
     darwin*)
         os=macos
         # Need to set up env vars so that setup.py finds librdkafka.
-        lib_dir=dest/runtimes/osx-x64/native
+        lib_dir=dest/runtimes/osx-$ARCH/native
         export CFLAGS="-I${PWD}/dest/build/native/include"
         export LDFLAGS="-L${PWD}/$lib_dir"
         ;;
@@ -51,10 +52,10 @@ esac
 
 $this_dir/install-librdkafka.sh $librdkafka_version dest
 
-install_pkgs=cibuildwheel==2.7.0
+install_pkgs=cibuildwheel==2.8.1
 
-python3 -m pip install $install_pkgs ||
-    pip3 install $install_pkgs
+python3 -m pip install ${PIP_INSTALL_OPTS} $install_pkgs ||
+    pip3 install ${PIP_INSTALL_OPTS} $install_pkgs
 
 if [[ -z $TRAVIS ]]; then
     cibw_args="--platform $os"
diff --git a/tools/wheels/install-librdkafka.sh b/tools/wheels/install-librdkafka.sh
@@ -23,22 +23,23 @@ curl -L -o lrk$VER.zip https://www.nuget.org/api/v2/package/librdkafka.redist/$V
 
 unzip lrk$VER.zip
 
+ARCH=${ARCH:-x64}
 
 if [[ $OSTYPE == linux* ]]; then
     # Linux
 
     # Copy the librdkafka build with least dependencies to librdkafka.so.1
-    cp -v runtimes/linux-x64/native/{centos6-librdkafka.so,librdkafka.so.1}
-    ldd runtimes/linux-x64/native/librdkafka.so.1
+    cp -v runtimes/linux-$ARCH/native/{centos6-librdkafka.so,librdkafka.so.1}
+    ldd runtimes/linux-$ARCH/native/librdkafka.so.1
 
 elif [[ $OSTYPE == darwin* ]]; then
     # MacOS X
 
     # Change the library's self-referencing name from
     # /Users/travis/.....somelocation/librdkafka.1.dylib to its local path.
-    install_name_tool -id $PWD/runtimes/osx-x64/native/librdkafka.dylib runtimes/osx-x64/native/librdkafka.dylib
+    install_name_tool -id $PWD/runtimes/osx-$ARCH/native/librdkafka.dylib runtimes/osx-$ARCH/native/librdkafka.dylib
 
-    otool -L runtimes/osx-x64/native/librdkafka.dylib
+    otool -L runtimes/osx-$ARCH/native/librdkafka.dylib
 fi
 
 popd
