[HSTACK] Internal Adobe build tooling

Author: adragomir

.github/workflows/build.yml

@@ -17,11 +17,9 @@
 
 name: Python Release Build
 on:
-  pull_request:
-    branches: ["main"]
+  workflow_dispatch: {}
   push:
-    tags: ["*-rc*"]
-    branches: ["branch-*"]
+    branches: ["main*"]
 
 jobs:
   build:
@@ -43,28 +41,27 @@ jobs:
         run: uv sync --dev --no-install-package datafusion
 
       # Update output format to enable automatic inline annotations.
-      - name: Run Ruff
-        run: |
-          uv run --no-project ruff check --output-format=github python/
-          uv run --no-project ruff format --check python/
-
-  generate-license:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: astral-sh/setup-uv@v5
-        with:
-            enable-cache: true
+      # - name: Run Ruff
+      #   run: |
+      #     uv run --no-project ruff check --output-format=github python/
+      #     uv run --no-project ruff format --check python/
 
-      - name: Generate license file
-        run: uv run --no-project python ./dev/create_license.py
-      - uses: actions/upload-artifact@v4
-        with:
-          name: python-wheel-license
-          path: LICENSE.txt
+  # generate-license:
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #     - uses: astral-sh/setup-uv@v5
+  #       with:
+  #           enable-cache: true
+  #
+  #     - name: Generate license file
+  #       run: uv run --no-project python ./dev/create_license.py
+  #     - uses: actions/upload-artifact@v4
+  #       with:
+  #         name: python-wheel-license
+  #         path: LICENSE.txt
 
   build-python-mac-win:
-    needs: [generate-license]
     name: Mac/Win
     runs-on: ${{ matrix.os }}
     strategy:
@@ -81,13 +78,6 @@ jobs:
 
       - uses: dtolnay/rust-toolchain@stable
 
-      - run: rm LICENSE.txt
-      - name: Download LICENSE.txt
-        uses: actions/download-artifact@v4
-        with:
-          name: python-wheel-license
-          path: .
-
       - name: Install Protoc
         uses: arduino/setup-protoc@v3
         with:
@@ -121,7 +111,6 @@ jobs:
           path: target/wheels/*
 
   build-macos-x86_64:
-    needs: [generate-license]
     name: Mac x86_64
     runs-on: macos-13
     strategy:
@@ -137,13 +126,6 @@ jobs:
 
       - uses: dtolnay/rust-toolchain@stable
 
-      - run: rm LICENSE.txt
-      - name: Download LICENSE.txt
-        uses: actions/download-artifact@v4
-        with:
-          name: python-wheel-license
-          path: .
-
       - name: Install Protoc
         uses: arduino/setup-protoc@v3
         with:
@@ -169,17 +151,14 @@ jobs:
           path: target/wheels/*
 
   build-manylinux-x86_64:
-    needs: [generate-license]
     name: Manylinux x86_64
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - run: rm LICENSE.txt
-      - name: Download LICENSE.txt
-        uses: actions/download-artifact@v4
+      - uses: awalsh128/cache-apt-pkgs-action@latest
         with:
-          name: python-wheel-license
-          path: .
+          packages: libssl3 openssl
+          version: 1.0
       - run: cat LICENSE.txt
       - name: Build wheels
         uses: PyO3/maturin-action@v1
@@ -189,6 +168,20 @@ jobs:
           rust-toolchain: nightly
           target: x86_64
           manylinux: auto
+          before-script-linux: |
+            # If we're running on rhel centos, install needed packages.
+            if command -v yum &> /dev/null; then
+                yum update -y && yum install -y perl-core openssl openssl-devel pkgconfig libatomic
+
+                # If we're running on i686 we need to symlink libatomic
+                # in order to build openssl with -latomic flag.
+                if [[ ! -d "/usr/lib64" ]]; then
+                    ln -s /usr/lib/libatomic.so.1 /usr/lib/libatomic.so
+                fi
+            else
+                # If we're running on debian-based system.
+                apt update -y && apt-get install -y libssl-dev openssl pkg-config
+            fi
           rustup-components: rust-std rustfmt # Keep them in one line due to https://github.com/PyO3/maturin-action/issues/153
           args: --release --manylinux 2014 --features protoc,substrait
       - name: Archive wheels
@@ -197,74 +190,97 @@ jobs:
           name: dist-manylinux-x86_64
           path: target/wheels/*
 
-  build-manylinux-aarch64:
-    needs: [generate-license]
-    name: Manylinux arm64
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - run: rm LICENSE.txt
-      - name: Download LICENSE.txt
-        uses: actions/download-artifact@v4
-        with:
-          name: python-wheel-license
-          path: .
-      - run: cat LICENSE.txt
-      - name: Build wheels
-        uses: PyO3/maturin-action@v1
-        env:
-          RUST_BACKTRACE: 1
-        with:
-          rust-toolchain: nightly
-          target: aarch64
-          # Use manylinux_2_28-cross because the manylinux2014-cross has GCC 4.8.5, which causes the build to fail
-          manylinux: 2_28
-          rustup-components: rust-std rustfmt # Keep them in one line due to https://github.com/PyO3/maturin-action/issues/153
-          args: --release --features protoc,substrait
-      - name: Archive wheels
-        uses: actions/upload-artifact@v4
-        with:
-          name: dist-manylinux-aarch64
-          path: target/wheels/*
+  # build-manylinux-aarch64:
+  #   name: Manylinux arm64
+  #   runs-on: ubuntu-latest-arm
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #     - uses: awalsh128/cache-apt-pkgs-action@latest
+  #       with:
+  #         packages: libssl3 openssl
+  #         version: 1.0
+  #     - run: cat LICENSE.txt
+  #     - name: Build wheels
+  #       uses: PyO3/maturin-action@v1
+  #       env:
+  #         RUST_BACKTRACE: full
+  #       with:
+  #         rust-toolchain: nightly
+  #         target: aarch64
+  #         # Use manylinux_2_28-cross because the manylinux2014-cross has GCC 4.8.5, which causes the build to fail
+  #         manylinux: 2_28
+  #         before-script-linux: |
+  #           # If we're running on rhel centos, install needed packages.
+  #           if command -v yum &> /dev/null; then
+  #               yum update -y && yum install -y perl-core openssl openssl-devel pkgconfig libatomic
+  #
+  #               # If we're running on i686 we need to symlink libatomic
+  #               # in order to build openssl with -latomic flag.
+  #               if [[ ! -d "/usr/lib64" ]]; then
+  #                   ln -s /usr/lib/libatomic.so.1 /usr/lib/libatomic.so
+  #               fi
+  #           else
+  #               # If we're running on debian-based system.
+  #               apt update -y && apt-get install -y libssl-dev openssl pkg-config librust-openssl-sys-dev
+  #           fi
+  #           ls -lA /usr/include/openssl/
+  #         rustup-components: rust-std rustfmt # Keep them in one line due to https://github.com/PyO3/maturin-action/issues/153
+  #         args: --release --features protoc,substrait
+  #     - name: Archive wheels
+  #       uses: actions/upload-artifact@v4
+  #       with:
+  #         name: dist-manylinux-aarch64
+  #         path: target/wheels/*
 
-  build-sdist:
-    needs: [generate-license]
-    name: Source distribution
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - run: rm LICENSE.txt
-      - name: Download LICENSE.txt
-        uses: actions/download-artifact@v4
-        with:
-          name: python-wheel-license
-          path: .
-      - run: cat LICENSE.txt
-      - name: Build sdist
-        uses: PyO3/maturin-action@v1
-        with:
-          rust-toolchain: stable
-          manylinux: auto
-          rustup-components: rust-std rustfmt
-          args: --release --sdist --out dist --features protoc,substrait
-      - name: Assert sdist build does not generate wheels
-        run: |
-            if [ "$(ls -A target/wheels)" ]; then
-              echo "Error: Sdist build generated wheels"
-              exit 1
-            else
-              echo "Directory is clean"
-            fi
-        shell: bash
+  # build-sdist:
+  #   name: Source distribution
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #     - uses: awalsh128/cache-apt-pkgs-action@latest
+  #       with:
+  #         packages: libssl3 openssl
+  #         version: 1.0
+  #     - run: cat LICENSE.txt
+  #     - name: Build sdist
+  #       uses: PyO3/maturin-action@v1
+  #       with:
+  #         rust-toolchain: stable
+  #         manylinux: auto
+  #         before-script-linux: |
+  #           # If we're running on rhel centos, install needed packages.
+  #           if command -v yum &> /dev/null; then
+  #               yum update -y && yum install -y perl-core openssl openssl-devel pkgconfig libatomic
+  #
+  #               # If we're running on i686 we need to symlink libatomic
+  #               # in order to build openssl with -latomic flag.
+  #               if [[ ! -d "/usr/lib64" ]]; then
+  #                   ln -s /usr/lib/libatomic.so.1 /usr/lib/libatomic.so
+  #               fi
+  #           else
+  #               # If we're running on debian-based system.
+  #               apt update -y && apt-get install -y libssl-dev openssl pkg-config
+  #           fi
+  #         rustup-components: rust-std rustfmt
+  #         args: --release --sdist --out dist --features protoc,substrait
+  #     - name: Assert sdist build does not generate wheels
+  #       run: |
+  #           if [ "$(ls -A target/wheels)" ]; then
+  #             echo "Error: Sdist build generated wheels"
+  #             exit 1
+  #           else
+  #             echo "Directory is clean"
+  #           fi
+  #       shell: bash
 
   merge-build-artifacts:
     runs-on: ubuntu-latest
     needs:
       - build-python-mac-win
       - build-macos-x86_64
       - build-manylinux-x86_64
-      - build-manylinux-aarch64
-      - build-sdist
+      # - build-manylinux-aarch64
+      # - build-sdist
     steps:
       - name: Merge Build Artifacts
         uses: actions/upload-artifact/merge@v4

ARTIFACTORY.md

@@ -0,0 +1,16 @@
+# Publish to artifactory
+
+## MacOS binary
+```shell
+maturin publish --repository-url https://artifactory-uw2.adobeitc.com/artifactory/api/pypi/pypi-arrow-release/ --password $ARTIFACTORY_UW2_TOKEN --username $ARTIFACTORY_USER
+```
+
+## Linux ARM64
+```shell
+./build-linux-arm64.sh
+```
+
+## Linux X86_64
+```shell
+./build-linux-x86_64.sh
+```

Dockerfile-build-wheel.arm64

@@ -0,0 +1,56 @@
+# syntax=docker/dockerfile:1
+
+FROM quay.io/pypa/manylinux_2_28_aarch64 AS base
+
+WORKDIR /root
+
+RUN <<EOF
+  dnf install -y epel-release
+  dnf install -y curl pkg-config openssl ca-certificates openssl-devel patchelf autoconf automake make libtool unzip clang libatomic openssh-clients wget
+EOF
+
+RUN <<EOF
+  # echo 1
+  wget https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.12.zip
+  unzip v3.21.12.zip
+  cd protobuf-3.21.12
+  autoupdate
+  ./autogen.sh
+  ./configure --prefix=/usr --disable-dependency-tracking
+  make -j8
+  make install
+EOF
+
+RUN <<EOF
+  curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly -y
+EOF
+
+RUN --mount=type=ssh,mode=0666 \
+  <<EOF
+  mkdir -p -m 0700 ~/.ssh
+  ssh-keyscan git.corp.adobe.com >> ~/.ssh/known_hosts
+  ssh-keyscan github.com >> ~/.ssh/known_hosts
+EOF
+
+RUN --mount=type=bind,source=requirements.txt,target=requirements.txt \
+    --mount=type=bind,source=src,target=src,readwrite \
+    --mount=type=bind,source=python,target=python,readwrite \
+    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+    --mount=type=bind,source=build.rs,target=build.rs \
+    --mount=type=bind,source=Cargo.toml,target=Cargo.toml \
+    --mount=type=bind,source=Cargo.lock,target=Cargo.lock \
+    --mount=type=bind,source=README.md,target=README.md \
+    --mount=type=bind,source=LICENSE.txt,target=LICENSE.txt \
+    --mount=type=cache,mode=0777,target=/root/target \
+    --mount=type=cache,mode=0777,target=/usr/local/cargo/registry/ \
+    --mount=type=ssh,mode=0666 \
+    <<EOF
+  export PATH="/root/.cargo/bin:/opt/python/cp312-cp312/bin:$PATH"
+  python3 -m venv venv
+  source venv/bin/activate
+  source /root/.cargo/env
+  pip3 install maturin==1.8.1
+  pip3 install -r requirements.txt
+  maturin build --release --manylinux 2_28 --features protoc,substrait
+  cp ./target/wheels/*.whl /
+EOF