[HSTACK] Added docker builders for binary wheels

Author: adragomir

Dockerfile-build-wheel.arm64

@@ -0,0 +1,44 @@
+# syntax=docker/dockerfile:1
+
+FROM quay.io/pypa/manylinux_2_28_aarch64 AS base
+
+WORKDIR /root
+
+RUN <<EOF
+  dnf install -y epel-release
+  dnf install -y curl pkg-config openssl ca-certificates openssl-devel patchelf autoconf automake make libtool unzip clang libatomic protobuf-c-compiler openssh-clients
+EOF
+
+RUN <<EOF
+  curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly -y
+EOF
+
+RUN --mount=type=ssh,mode=0666 \
+  <<EOF
+  mkdir -p -m 0700 ~/.ssh
+  ssh-keyscan git.corp.adobe.com >> ~/.ssh/known_hosts
+  ssh-keyscan github.com >> ~/.ssh/known_hosts
+EOF
+
+RUN --mount=type=bind,source=requirements.txt,target=requirements.txt \
+    --mount=type=bind,source=src,target=src,readwrite \
+    --mount=type=bind,source=python,target=python,readwrite \
+    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+    --mount=type=bind,source=build.rs,target=build.rs \
+    --mount=type=bind,source=Cargo.toml,target=Cargo.toml \
+    --mount=type=bind,source=Cargo.lock,target=Cargo.lock \
+    --mount=type=bind,source=README.md,target=README.md \
+    --mount=type=bind,source=LICENSE.txt,target=LICENSE.txt \
+    --mount=type=cache,mode=0777,target=/root/target \
+    --mount=type=cache,mode=0777,target=/usr/local/cargo/registry/ \
+    --mount=type=ssh,mode=0666 \
+    <<EOF
+  export PATH="/root/.cargo/bin:/opt/python/cp312-cp312/bin:$PATH"
+  python3 -m venv venv
+  source venv/bin/activate
+  source /root/.cargo/env
+  pip3 install maturin==1.8.1
+  pip3 install -r requirements.txt
+  maturin build --release --manylinux 2_28 --features protoc,substrait
+  cp ./target/wheels/*.whl /
+EOF

Dockerfile-build-wheel.x86_64

@@ -0,0 +1,44 @@
+# syntax=docker/dockerfile:1
+
+FROM quay.io/pypa/manylinux_2_28_x86_64 AS base
+
+WORKDIR /root
+
+RUN <<EOF
+  dnf install -y epel-release
+  dnf install -y curl pkg-config openssl ca-certificates openssl-devel patchelf autoconf automake make libtool unzip clang libatomic protobuf-c-compiler openssh-clients
+EOF
+
+RUN <<EOF
+  curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly -y
+EOF
+
+RUN --mount=type=ssh,mode=0666 \
+  <<EOF
+  mkdir -p -m 0700 ~/.ssh
+  ssh-keyscan git.corp.adobe.com >> ~/.ssh/known_hosts
+  ssh-keyscan github.com >> ~/.ssh/known_hosts
+EOF
+
+RUN --mount=type=bind,source=requirements.txt,target=requirements.txt \
+    --mount=type=bind,source=src,target=src,readwrite \
+    --mount=type=bind,source=python,target=python,readwrite \
+    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+    --mount=type=bind,source=build.rs,target=build.rs \
+    --mount=type=bind,source=Cargo.toml,target=Cargo.toml \
+    --mount=type=bind,source=Cargo.lock,target=Cargo.lock \
+    --mount=type=bind,source=README.md,target=README.md \
+    --mount=type=bind,source=LICENSE.txt,target=LICENSE.txt \
+    --mount=type=cache,mode=0777,target=/root/target \
+    --mount=type=cache,mode=0777,target=/usr/local/cargo/registry/ \
+    --mount=type=ssh,mode=0666 \
+    <<EOF
+  export PATH="/root/.cargo/bin:/opt/python/cp312-cp312/bin:$PATH"
+  python3 -m venv venv
+  source venv/bin/activate
+  source /root/.cargo/env
+  pip3 install maturin==1.8.1
+  pip3 install -r requirements.txt
+  maturin build --release --manylinux 2_28 --features protoc,substrait
+  cp ./target/wheels/*.whl /
+EOF

build-linux-arm64.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+mkdir -p ./target-linux/wheels
+
+export DOCKER_BUILDKIT=1
+
+root_dir="$(dirname $0)"
+
+docker build \
+  --ssh default \
+  --progress=plain \
+  --platform linux/arm64 \
+  -f Dockerfile-build-wheel.arm64 \
+  -t datafusion-python-builder:latest \
+  "$root_dir"
+
+id=$(docker create --platform linux/arm64 datafusion-python-builder:latest)
+docker cp $id:/datafusion-44.0.0-cp38-abi3-manylinux_2_28_aarch64.whl .
+docker rm -v $id

build-linux-x86_64.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+export DOCKER_BUILDKIT=1
+
+root_dir="$(dirname $0)"
+
+docker build \
+  --ssh default \
+  --progress=plain \
+  --platform linux/x86_64 \
+  -f Dockerfile-build-wheel.x86_64 \
+  -t datafusion-python-builder:latest \
+  "$root_dir"
+
+id=$(docker create --platform linux/x86_64 datafusion-python-builder:latest)
+docker cp $id:/datafusion-44.0.0-cp38-abi3-manylinux_2_28_x86_64.whl .
+docker rm -v $id

requirements.txt

@@ -0,0 +1,10 @@
+#
+# This file is autogenerated by pip-compile with Python 3.12
+# by the following command:
+#
+#    pip-compile --output-file=requirements.txt pyproject.toml
+#
+pyarrow==19.0.1
+    # via datafusion (pyproject.toml)
+typing-extensions==4.12.2 ; python_full_version < "3.13"
+    # via datafusion (pyproject.toml)