Skip to content

Commit 191fdcb

Browse files
ianksianks
committed
[linux/arm64] Use own platform manifests
1 parent 63e23dd commit 191fdcb

File tree

5 files changed

+226
-16
lines changed

5 files changed

+226
-16
lines changed

‎.github/workflows/ci.yml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,9 +67,16 @@ jobs:
6767
- name: Build docker image
6868
id: buildx
6969
run: |
70+
echo "::group::Preparing docker build"
7071
docker buildx create --driver docker-container --use
7172
extra_tag="rcd-${PLATFORM}-${{ matrix.docker-platform-cpu }}-${{ github.sha }}"
7273
docker_build="docker buildx build --platform=$DOCKER_BUILD_PLATFORM --cache-from=type=local,src=tmp/build-cache --cache-to=type=local,dest=tmp/build-cache-new --load -t $extra_tag"
74+
if bundle exec rake -T | grep -q "prepare:${PLATFORM}"; then
75+
echo "::info::Preparing docker image for ${PLATFORM}"
76+
bundle exec rake prepare:${PLATFORM} RCD_DOCKER_BUILD="$docker_build"
77+
fi
78+
echo "::endgroup::"
79+
7380
bundle exec rake build:${PLATFORM} RCD_DOCKER_BUILD="$docker_build"
7481
7582
if [[ "${{ contains(github.event.head_commit.message, matrix.docker-platform-cpu) }}" == "true" ]]; then

‎Dockerfile.mri.erb

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,7 @@ RUN git clone -q --depth=1 https://github.com/tpoechtrager/osxcross.git /opt/osx
117117
curl -L -o MacOSX11.1.sdk.tar.xz https://github.com/larskanis/MacOSX-SDKs/releases/download/11.1/MacOSX11.1.sdk.tar.xz && \
118118
tar -xf MacOSX11.1.sdk.tar.xz -C . && \
119119
cp -rf /usr/lib/llvm-10/include/c++ MacOSX11.1.sdk/usr/include/c++ && \
120-
cp -rf /usr/include/<%= manylinux_cpu %>-linux-gnu/c++/9/bits/ MacOSX11.1.sdk/usr/include/c++/v1/bits && \
120+
cp -rf /usr/include/"$(uname -m)"-linux-gnu/c++/9/bits/ MacOSX11.1.sdk/usr/include/c++/v1/bits && \
121121
tar -cJf MacOSX11.1.sdk.tar.xz MacOSX11.1.sdk && \
122122
cd /opt/osxcross && \
123123
UNATTENDED=1 SDK_VERSION=11.1 OSX_VERSION_MIN=10.13 USE_CLANG_AS=1 ./build.sh && \

‎Dockerfile.mri.x64-mingw-ucrt

Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
FROM ubuntu:20.04
2+
ENV DEBIAN_FRONTEND noninteractive
3+
RUN apt-get -y update && \
4+
apt-get install -y curl git-core xz-utils build-essential zlib1g-dev libreadline-dev libssl-dev wget unzip sudo gnupg2 dirmngr cmake pkg-config autoconf && \
5+
rm -rf /var/lib/apt/lists/*
6+
7+
# Add "rvm" as system group, to avoid conflicts with host GIDs typically starting with 1000
8+
RUN groupadd -r rvm && useradd -r -g rvm -G sudo -p "" --create-home rvm
9+
10+
# Make sure rvm and later settings are available in interactive and non-interactive shells
11+
RUN echo "source /etc/profile.d/rvm.sh" >> /etc/rubybashrc && \
12+
echo "source /etc/rubybashrc" >> /etc/bashrc && \
13+
echo "source /etc/rubybashrc" >> /etc/bash.bashrc
14+
ENV BASH_ENV /etc/rubybashrc
15+
16+
USER rvm
17+
18+
RUN mkdir ~/.gnupg && \
19+
chmod 700 ~/.gnupg && \
20+
echo "disable-ipv6" >> ~/.gnupg/dirmngr.conf
21+
22+
# install rvm, RVM 1.26.0+ has signed releases, source rvm for usage outside of package scripts
23+
RUN gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB && \
24+
(curl -L http://get.rvm.io | sudo bash) && \
25+
bash -c " \
26+
source /etc/rubybashrc && \
27+
rvm autolibs disable && \
28+
rvmsudo rvm cleanup all "
29+
30+
# Import patch files for ruby and gems
31+
COPY build/patches /home/rvm/patches/
32+
33+
# install rubies and fix permissions on
34+
ENV RVM_RUBIES 2.5.9 3.1.0
35+
RUN bash -c " \
36+
export CFLAGS='-s -O3 -fno-fast-math -fPIC' && \
37+
for v in ${RVM_RUBIES} ; do \
38+
rvm install \$v --patch \$(echo ~/patches/ruby-\$v/* | tr ' ' ','); \
39+
done && \
40+
rvm cleanup all && \
41+
find /usr/local/rvm -type d -print0 | sudo xargs -0 chmod g+sw "
42+
43+
# Install rake-compiler and typical gems in all Rubies
44+
# do not generate documentation for gems
45+
# TODO: stop pinning rubygems to 3.3.20 once https://github.com/rake-compiler/rake-compiler/pull/209 is merged
46+
RUN echo "gem: --no-ri --no-rdoc" >> ~/.gemrc && \
47+
bash -c " \
48+
rvm all do gem update --system=3.3.20 --no-document && \
49+
rvm all do gem install --no-document bundler 'bundler:~>1.16' 'rake-compiler:1.1.6' hoe mini_portile rubygems-tasks mini_portile2 && \
50+
find /usr/local/rvm -type d -print0 | sudo xargs -0 chmod g+sw "
51+
52+
# Install rake-compiler's cross rubies in global dir instead of /root
53+
RUN sudo mkdir -p /usr/local/rake-compiler && \
54+
sudo chown rvm.rvm /usr/local/rake-compiler && \
55+
ln -s /usr/local/rake-compiler ~/.rake-compiler
56+
57+
# Add cross compilers for Windows and Linux
58+
USER root
59+
60+
COPY --from=larskanis/mingw64-ucrt:20.04 \
61+
/build/binutils-mingw-w64-x86-64_2.34-6ubuntu1.3+8.8_amd64.deb \
62+
/build/g++-mingw-w64-x86-64_9.3.0-17ubuntu1~20.04+22~exp1ubuntu4_amd64.deb \
63+
/build/gcc-mingw-w64-base_9.3.0-17ubuntu1~20.04+22~exp1ubuntu4_amd64.deb \
64+
/build/gcc-mingw-w64-x86-64_9.3.0-17ubuntu1~20.04+22~exp1ubuntu4_amd64.deb \
65+
/build/mingw-w64-common_7.0.0-2_all.deb \
66+
/build/mingw-w64-x86-64-dev_7.0.0-2_all.deb \
67+
/debs/
68+
RUN dpkg -i /debs/*.deb
69+
70+
71+
RUN bash -c " \
72+
rvm alias create default 3.1.0 && \
73+
rvm use default "
74+
75+
76+
77+
# Patch rake-compiler to build and install static libraries for Linux rubies
78+
USER rvm
79+
COPY build/patches2 /home/rvm/patches/
80+
RUN bash -c " \
81+
for v in ${RVM_RUBIES} ; do \
82+
cd /usr/local/rvm/gems/ruby-\$v/gems/rake-compiler-1.1.6 && \
83+
echo applying patches to ruby-\$v /home/rvm/patches/rake-compiler-1.1.6/*.patch && \
84+
( git apply /home/rvm/patches/rake-compiler-1.1.6/*.patch || true ) \
85+
done "
86+
87+
# Patch rubies for cross build
88+
#USER root
89+
#RUN bash -c " \
90+
# for v in 2.7.0 3.0.0 3.1.0 ; do \
91+
# curl -SL http://cache.ruby-lang.org/pub/ruby/\${v:0:3}/ruby-\$v.tar.xz | tar -xJC /root/ && \
92+
# cd /root/ruby-\$v && \
93+
# git apply /home/rvm/patches/ruby-\$v/*.patch && \
94+
# cd .. && \
95+
# mkdir -p /usr/local/rake-compiler/sources/ && \
96+
# tar cjf /usr/local/rake-compiler/sources/ruby-\$v.tar.bz2 ruby-\$v && \
97+
# chown rvm /usr/local/rake-compiler -R && \
98+
# rm -rf /root/ruby-\$v ; \
99+
# done "
100+
#USER rvm
101+
102+
ENV XRUBIES 3.2.0-rc1
103+
# Build xruby versions in parallel
104+
# Then cleanup all build artifacts
105+
RUN bash -c " \
106+
rvm use 3.1.0 && \
107+
export CPPFLAGS='-D__USE_MINGW_ANSI_STDIO=1' && \
108+
export CFLAGS='-O1 -fno-omit-frame-pointer -fno-fast-math -fstack-protector-strong -s' && \
109+
export LDFLAGS='-pipe -s' && \
110+
export LIBS='-l:libssp.a' && \
111+
\
112+
export MAKE='make V=1 -j`nproc`' && \
113+
rake-compiler cross-ruby VERSION=$XRUBIES HOST=x86_64-w64-mingw32 && \
114+
rm -rf ~/.rake-compiler/builds ~/.rake-compiler/sources && \
115+
find /usr/local/rvm -type d -print0 | sudo xargs -0 chmod g+sw "
116+
ENV XRUBIES 3.1.0
117+
# Build xruby versions in parallel
118+
# Then cleanup all build artifacts
119+
RUN bash -c " \
120+
rvm use 3.1.0 && \
121+
export CPPFLAGS='-D__USE_MINGW_ANSI_STDIO=1' && \
122+
export CFLAGS='-O1 -fno-omit-frame-pointer -fno-fast-math -fstack-protector-strong -s' && \
123+
export LDFLAGS='-pipe -s' && \
124+
export LIBS='-l:libssp.a' && \
125+
\
126+
export MAKE='make V=1 -j`nproc`' && \
127+
rake-compiler cross-ruby VERSION=$XRUBIES HOST=x86_64-w64-mingw32 && \
128+
rm -rf ~/.rake-compiler/builds ~/.rake-compiler/sources && \
129+
find /usr/local/rvm -type d -print0 | sudo xargs -0 chmod g+sw "
130+
131+
132+
# RubyInstaller doesn't install libgcc -> link it static.
133+
RUN find /usr/local/rake-compiler/ruby/*mingw*/ -name rbconfig.rb | while read f ; do sed -i 's/."LDFLAGS". = "/&-static-libgcc /' $f ; done
134+
# Raise Windows-API to Vista (affects ruby < 2.6 only)
135+
RUN find /usr/local/rake-compiler/ruby -name rbconfig.rb | while read f ; do sed -i 's/0x0501/0x0600/' $f ; done
136+
# Don't link to static libruby
137+
RUN find /usr/local/rake-compiler/ruby -name lib*-ruby*.dll.a | while read f ; do n=`echo $f | sed s/.dll//` ; mv $f $n ; done
138+
139+
USER root
140+
141+
# Fix paths in rake-compiler/config.yml
142+
RUN sed -i -- "s:/root/.rake-compiler:/usr/local/rake-compiler:g" /usr/local/rake-compiler/config.yml
143+
144+
145+
# Install wrappers for strip commands as a workaround for "Protocol error" in boot2docker.
146+
COPY build/strip_wrapper /root/
147+
RUN mv /usr/bin/x86_64-w64-mingw32-strip /usr/bin/x86_64-w64-mingw32-strip.bin && \
148+
ln /root/strip_wrapper /usr/bin/x86_64-w64-mingw32-strip
149+
150+
# Use posix pthread for mingw so that C++ standard library for thread could be
151+
# available such as std::thread, std::mutex, so on.
152+
# https://sourceware.org/pthreads-win32/
153+
RUN printf "1\n" | update-alternatives --config x86_64-w64-mingw32-gcc && \
154+
printf "1\n" | update-alternatives --config x86_64-w64-mingw32-g++
155+
156+
157+
158+
# Install SIGINT forwarder
159+
COPY build/sigfw.c /root/
160+
RUN gcc $HOME/sigfw.c -o /usr/bin/sigfw
161+
162+
# Install user mapper
163+
COPY build/runas /usr/bin/
164+
COPY build/rcd-env.sh /etc/profile.d/
165+
RUN echo "source /etc/profile.d/rcd-env.sh" >> /etc/rubybashrc
166+
167+
# Install sudoers configuration
168+
COPY build/sudoers /etc/sudoers.d/rake-compiler-dock
169+
170+
ENV RUBY_CC_VERSION 3.2.0:3.1.0:3.0.0:2.7.0:2.6.0:2.5.0:2.4.0
171+
172+
CMD bash

‎Rakefile

Lines changed: 14 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -26,27 +26,28 @@ namespace :build do
2626
platforms.each do |platform, target|
2727
sdf = "Dockerfile.mri.#{platform}"
2828

29-
# Native images to alleviate qemu slowness, and manylinux2014 provides per-arch
30-
# images. But they are not yet conformant to the Docker platform spec (i.e.
31-
# amd64/linux). We have to do some string manipulation to get the right for
32-
# now, but you should be able to nuke this code soon, and rely on only the
33-
# buildx `--platform` feature instead....
34-
#
35-
# See: https://github.com/pypa/manylinux/issues/1306
36-
manylinux_cpu, dpkg_arch = case ENV["DOCKER_BUILD_PLATFORM"]
29+
dpkg_arch = case ENV["DOCKER_BUILD_PLATFORM"]
3730
when /arm64/
38-
["aarch64", "arm64"]
31+
"arm64"
3932
when /amd64/
40-
["x86_64", "amd64"]
33+
"amd64"
4134
else
4235
if ENV["CI"]
43-
raise "Couldnt infer manylinux CPU for #{ENV["DOCKER_BUILD_PLATFORM"].inspect}"
36+
raise "Couldnt infer dpkg arch for #{ENV["DOCKER_BUILD_PLATFORM"].inspect}"
4437
else
45-
["x86_64", "amd64"]
38+
"amd64"
4639
end
4740
end
4841

49-
manylinux_image = "quay.io/pypa/manylinux2014_#{manylinux_cpu}"
42+
# Native images to alleviate qemu slowness, and manylinux2014 provides per-arch
43+
# images. But they are not yet conformant to the Docker platform spec (i.e.
44+
# amd64/linux). We generate our own platformed manifests now (using
45+
# scrip/remanifest-manylinux-multiplatform.sh), but you should be able to
46+
# nuke that code soon, and rely on only the buildx `--platform` feature once
47+
# manylinux finishes the feature.
48+
#
49+
# See: https://github.com/pypa/manylinux/issues/1306
50+
manylinux_image = "rbsys/manylinux2014:2022-12-11-145d107"
5051

5152
desc "Build image for platform #{platform}"
5253
task platform => sdf
@@ -84,8 +85,6 @@ namespace :prepare do
8485
end
8586
end
8687

87-
task "build:mingw64-ucrt" => "prepare:mingw64-ucrt"
88-
8988
desc "Run tests"
9089
task :test do
9190
sh "ruby -w -W2 -I. -Ilib -e \"#{Dir["test/test_*.rb"].map{|f| "require '#{f}';"}.join}\" -- -v"

‎script/remanifest-manylinux-multiplatform.sh

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
#!/bin/bash
2+
3+
set -euo pipefail
4+
IFS=$'\n\t'
5+
6+
# A script to build and push a multi-arch image to Docker Hub, based on
7+
# https://github.com/pypa/manylinux/issues/1306
8+
9+
repo="${DOCKERHUB_USER:-larskanis}"
10+
tags=("latest" "2022-12-11-145d107")
11+
base_images=("manylinux2014" "manylinux_2_24")
12+
13+
for base_image in "${base_images[@]}"; do
14+
for tag in "${tags[@]}"; do
15+
echo Re-manifesting "$base_image":"$tag" >&2
16+
17+
docker pull --quiet quay.io/pypa/"$base_image"_aarch64:"$tag"
18+
docker pull --quiet quay.io/pypa/"$base_image"_x86_64:"$tag"
19+
20+
docker tag quay.io/pypa/"$base_image"_x86_64:"$tag" "$repo"/"$base_image"_x86_64:"$tag"
21+
docker tag quay.io/pypa/"$base_image"_aarch64:"$tag" "$repo"/"$base_image"_aarch64:"$tag"
22+
23+
docker push --quiet "$repo"/"$base_image"_x86_64:"$tag"
24+
docker push --quiet "$repo"/"$base_image"_aarch64:"$tag"
25+
26+
docker manifest create "$repo"/"$base_image":"$tag" \
27+
--amend "$repo"/"$base_image"_x86_64:"$tag" \
28+
--amend "$repo"/"$base_image"_aarch64:"$tag"
29+
30+
docker manifest push "$repo"/"$base_image":"$tag"
31+
done
32+
done

0 commit comments

Comments
 (0)