Merge branch 'master' into linux/arm64

* master:
  doc: update History.md
  gitignore the ucrt Dockerfile
  reduce gems installed, move things around
  update base packages for psych and ffi requirements
  move osxcross building into a script
  update to rake-compiler 1.2.1
  update base images with yaml dev package, for psych 5
  turn off parallelism in windows 3.2.0-rc1 build

Author: ianks

.gitignore

@@ -4,6 +4,7 @@
 /Dockerfile.mri.arm-linux
 /Dockerfile.mri.arm64-darwin
 /Dockerfile.mri.arm64-linux
+/Dockerfile.mri.x64-mingw-ucrt
 /Dockerfile.mri.x64-mingw32
 /Dockerfile.mri.x86-linux
 /Dockerfile.mri.x86-mingw32

Dockerfile.mri.erb

@@ -10,9 +10,12 @@ manylinux = !!(image =~ /manylinux/)
 %>
 FROM <%= image %>
 
+##
+## RVM and native rubies
+##
+# Install packages which rvm will require
 <% if manylinux %>
-# install packages which rvm will require
-RUN yum install -y autoconf gcc-c++ libtool readline-devel sqlite-devel ruby openssl-devel xz cmake sudo less libffi-devel git wget
+RUN yum install -y sudo ruby less git wget curl autoconf libtool cmake gcc-c++ xz readline-devel sqlite-devel openssl-devel libffi-devel libyaml-devel
 
 # Prepare sudo and delete sudo as alias to gosu
 RUN rm -f /usr/local/bin/sudo && \
@@ -21,7 +24,7 @@ RUN rm -f /usr/local/bin/sudo && \
 <% else %>
 ENV DEBIAN_FRONTEND noninteractive
 RUN apt-get -y update && \
-    apt-get install -y curl git-core xz-utils build-essential zlib1g-dev libreadline-dev libssl-dev wget unzip sudo gnupg2 dirmngr cmake pkg-config autoconf && \
+    apt-get install -y sudo wget autoconf cmake curl git-core pkg-config build-essential xz-utils unzip gnupg2 dirmngr zlib1g-dev libreadline-dev libsqlite0-dev libssl-dev libyaml-dev libffi-dev && \
     rm -rf /var/lib/apt/lists/*
 <% end %>
 
@@ -44,38 +47,31 @@ RUN mkdir ~/.gnupg && \
 RUN gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB && \
     (curl -L http://get.rvm.io | sudo bash) && \
     bash -c " \
-        source /etc/rubybashrc && \
-        rvm autolibs disable && \
-        rvmsudo rvm cleanup all "
+      source /etc/rubybashrc && \
+      rvm autolibs disable && \
+      rvmsudo rvm cleanup all \
+    "
 
-# Import patch files for ruby and gems
+# Install native rubies and fix permissions
 COPY build/patches /home/rvm/patches/
-
-# install rubies and fix permissions on
 ENV RVM_RUBIES 2.5.9 3.1.0
 RUN bash -c " \
-    export CFLAGS='-s -O3 -fno-fast-math -fPIC' && \
-    for v in ${RVM_RUBIES} ; do \
+      export CFLAGS='-s -O3 -fno-fast-math -fPIC' && \
+      for v in ${RVM_RUBIES} ; do \
         rvm install \$v --patch \$(echo ~/patches/ruby-\$v/* | tr ' ' ','); \
-    done && \
-    rvm cleanup all && \
-    find /usr/local/rvm -type d -print0 | sudo xargs -0 chmod g+sw "
-
-# Install rake-compiler and typical gems in all Rubies
-# do not generate documentation for gems
-# TODO: stop pinning rubygems to 3.3.20 once https://github.com/rake-compiler/rake-compiler/pull/209 is merged
-RUN echo "gem: --no-ri --no-rdoc" >> ~/.gemrc && \
-    bash -c " \
-        rvm all do gem update --system=3.3.20 --no-document && \
-        rvm all do gem install --no-document bundler 'bundler:~>1.16' 'rake-compiler:1.1.6' hoe mini_portile rubygems-tasks mini_portile2 && \
-        find /usr/local/rvm -type d -print0 | sudo xargs -0 chmod g+sw "
-
-# Install rake-compiler's cross rubies in global dir instead of /root
-RUN sudo mkdir -p /usr/local/rake-compiler && \
-    sudo chown rvm.rvm /usr/local/rake-compiler && \
-    ln -s /usr/local/rake-compiler ~/.rake-compiler
+      done && \
+      rvm cleanup all \
+    "
 
-# Add cross compilers for Windows and Linux
+RUN bash -c " \
+      rvm all do gem update --system --no-document && \
+      rvm all do gem update --no-document && \
+      rvm all do gem install bundler --no-document \
+    "
+
+##
+## Cross compilers
+##
 USER root
 
 <% if platform=~/x64-mingw-ucrt/ %>
@@ -100,77 +96,46 @@ if platform=~/x64-mingw32/   %> gcc-mingw-w64-x86-64 g++-mingw-w64-x86-64 <% end
     rm -rf /var/lib/apt/lists/*
 <% end %>
 
-RUN bash -c " \
-        rvm alias create default 3.1.0 && \
-        rvm use default "
-
 <% if manylinux %>
 # Create dev tools x86-linux-*
 COPY build/mk_i686.rb /root/
-RUN bash -c " \
-        ruby /root/mk_i686.rb "
+RUN /root/mk_i686.rb
 <% end %>
 
 <% if platform=~/darwin/ %>
-RUN git clone -q --depth=1 https://github.com/tpoechtrager/osxcross.git /opt/osxcross && rm -rf /opt/osxcross/.git && \
-    cd /opt/osxcross/tarballs && \
-    curl -L -o MacOSX11.1.sdk.tar.xz https://github.com/larskanis/MacOSX-SDKs/releases/download/11.1/MacOSX11.1.sdk.tar.xz && \
-    tar -xf MacOSX11.1.sdk.tar.xz -C . && \
-    cp -rf /usr/lib/llvm-10/include/c++ MacOSX11.1.sdk/usr/include/c++ && \
-    cp -rf /usr/include/"$(uname -m)"-linux-gnu/c++/9/bits/ MacOSX11.1.sdk/usr/include/c++/v1/bits && \
-    tar -cJf MacOSX11.1.sdk.tar.xz MacOSX11.1.sdk && \
-    cd /opt/osxcross && \
-    UNATTENDED=1 SDK_VERSION=11.1 OSX_VERSION_MIN=10.13 USE_CLANG_AS=1 ./build.sh && \
-    ln -s /usr/bin/llvm-config-10 /usr/bin/llvm-config && \
-    ENABLE_COMPILER_RT_INSTALL=1 SDK_VERSION=11.1 ./build_compiler_rt.sh && \
-    rm -rf *~ build tarballs/*
-
-RUN echo "export PATH=/opt/osxcross/target/bin:\$PATH" >> /etc/rubybashrc && \
-    echo "export MACOSX_DEPLOYMENT_TARGET=10.13" >> /etc/rubybashrc && \
-    echo "export OSXCROSS_MP_INC=1" >> /etc/rubybashrc
-
-# Add links to build tools without target version kind of:
-#   arm64-apple-darwin-clang   =>   arm64-apple-darwin20.1-clang
-RUN rm /opt/osxcross/target/bin/*-apple-darwin-* ; \
-    find /opt/osxcross/target/bin/ -name '*-apple-darwin[0-9]*' | sort | while read f ; do d=`echo $f | sed s/darwin[0-9\.]*/darwin/`; echo $f '"$@"' | tee $d && chmod +x $d ; done
-
-# There's no objdump in osxcross but we can use llvm's
-RUN ln -s /usr/lib/llvm-10/bin/llvm-objdump /opt/osxcross/target/bin/x86_64-apple-darwin-objdump && \
-    ln -s /usr/lib/llvm-10/bin/llvm-objdump /opt/osxcross/target/bin/aarch64-apple-darwin-objdump
-
+COPY build/mk_osxcross.sh /home/rvm
+RUN /home/rvm/mk_osxcross.sh
 <% end %>
 
-# Patch rake-compiler to build and install static libraries for Linux rubies
+
+##
+## Cross-compile rubies
+##
 USER rvm
+
+RUN bash -c "rvm all do gem install --no-document rake-compiler:1.2.1"
+
+# Install rake-compiler's cross rubies in global dir instead of /root
+RUN sudo mkdir -p /usr/local/rake-compiler && \
+    sudo chown rvm.rvm /usr/local/rake-compiler && \
+    ln -s /usr/local/rake-compiler ~/.rake-compiler
+
+# Patch rake-compiler to build and install static libraries for Linux rubies
 COPY build/patches2 /home/rvm/patches/
 RUN bash -c " \
-    for v in ${RVM_RUBIES} ; do \
-      cd /usr/local/rvm/gems/ruby-\$v/gems/rake-compiler-1.1.6 && \
-      echo applying patches to ruby-\$v /home/rvm/patches/rake-compiler-1.1.6/*.patch && \
-      ( git apply /home/rvm/patches/rake-compiler-1.1.6/*.patch || true ) \
-    done "
-
-# Patch rubies for cross build
-#USER root
-#RUN bash -c " \
-#    for v in 2.7.0 3.0.0 3.1.0 ; do \
-#      curl -SL http://cache.ruby-lang.org/pub/ruby/\${v:0:3}/ruby-\$v.tar.xz | tar -xJC /root/ && \
-#      cd /root/ruby-\$v && \
-#      git apply /home/rvm/patches/ruby-\$v/*.patch && \
-#      cd .. && \
-#      mkdir -p /usr/local/rake-compiler/sources/ && \
-#      tar cjf /usr/local/rake-compiler/sources/ruby-\$v.tar.bz2 ruby-\$v && \
-#      chown rvm /usr/local/rake-compiler -R && \
-#      rm -rf /root/ruby-\$v ; \
-#    done "
-#USER rvm
+      for v in ${RVM_RUBIES} ; do \
+        cd /usr/local/rvm/gems/ruby-\$v/gems/rake-compiler-1.2.1 && \
+        echo applying patches to ruby-\$v /home/rvm/patches/rake-compiler-1.2.1/*.patch && \
+        ( git apply /home/rvm/patches/rake-compiler-1.2.1/*.patch || true ) \
+      done \
+    "
 
 <%
 axrubies = if platform =~ /x64-mingw-ucrt/
   [
     # Rubyinstaller-3.1.0+ is platform x64-mingw-ucrt
-    ["3.2.0-rc1", "3.1.0", true],
-    ["3.1.0", "3.1.0", true],
+    # parallel=false because 3.2.0-rc1 has undefined references when using a high -j arg
+    ["3.2.0-rc1:3.1.0", "3.1.0", false],
   ]
 elsif platform =~ /x64-mingw32/
   [
@@ -194,16 +159,16 @@ ENV XRUBIES <%= xrubies %>
 # Build xruby versions in parallel
 # Then cleanup all build artifacts
 RUN bash -c " \
-    rvm use <%= rvm %> && \
-    export CPPFLAGS='<%= "-D__USE_MINGW_ANSI_STDIO=1" if platform=~/x64-mingw-ucrt/ %>' && \
-    export CFLAGS='-O1 -fno-omit-frame-pointer -fno-fast-math -fstack-protector-strong <%= strip %>' && \
-    export LDFLAGS='-pipe <%= strip %>' && \
-    <%= "export LIBS='-l:libssp.a' &&" if platform =~ /mingw/ %> \
-    <%= "export CC=#{target}-clang &&" if platform =~ /darwin/ %> \
-    export MAKE='make V=1 <%= "-j`nproc`" if parallel %>' && \
-    rake-compiler cross-ruby VERSION=$XRUBIES HOST=<%= target %> && \
-    rm -rf ~/.rake-compiler/builds ~/.rake-compiler/sources && \
-    find /usr/local/rvm -type d -print0 | sudo xargs -0 chmod g+sw "
+      rvm use <%= rvm %> && \
+      export CPPFLAGS='<%= "-D__USE_MINGW_ANSI_STDIO=1" if platform=~/x64-mingw-ucrt/ %>' && \
+      export CFLAGS='-O1 -fno-omit-frame-pointer -fno-fast-math -fstack-protector-strong <%= strip %>' && \
+      export LDFLAGS='-pipe <%= strip %>' && \
+      <%= "export LIBS='-l:libssp.a' &&" if platform =~ /mingw/ %> \
+      <%= "export CC=#{target}-clang &&" if platform =~ /darwin/ %> \
+      export MAKE='make V=1 <%= "-j`nproc`" if parallel %>' && \
+      rake-compiler cross-ruby VERSION=$XRUBIES HOST=<%= target %> && \
+      rm -rf ~/.rake-compiler/builds ~/.rake-compiler/sources \
+    "
 <% end %>
 
 <% if platform=~/linux/ %>
@@ -224,12 +189,14 @@ RUN find /usr/local/rake-compiler/ruby -name rbconfig.rb | while read f ; do sed
 RUN find /usr/local/rake-compiler/ruby -name lib*-ruby*.dll.a | while read f ; do n=`echo $f | sed s/.dll//` ; mv $f $n ; done
 <% end %>
 
+##
+##  Final adjustments
+##
 USER root
 
 # Fix paths in rake-compiler/config.yml
 RUN sed -i -- "s:/root/.rake-compiler:/usr/local/rake-compiler:g" /usr/local/rake-compiler/config.yml
 
-
 <% if platform=~/mingw/ %>
 # Install wrappers for strip commands as a workaround for "Protocol error" in boot2docker.
 COPY build/strip_wrapper /root/
@@ -276,6 +243,12 @@ RUN echo "source /etc/profile.d/rcd-env.sh" >> /etc/rubybashrc
 # Install sudoers configuration
 COPY build/sudoers /etc/sudoers.d/rake-compiler-dock
 
+RUN find /usr/local/rvm -type d -print0 | sudo xargs -0 chmod g+sw
+RUN bash -c " \
+      rvm alias create default 3.1.0 && \
+      rvm use default \
+    "
+
 ENV RUBY_CC_VERSION 3.2.0:3.1.0:3.0.0:2.7.0:2.6.0:2.5.0:2.4.0
 
 CMD bash

History.md

@@ -1,3 +1,14 @@
+1.3.0 / unreleased
+------------------
+
+* Add Ruby-3.2 cross-compilation support.
+* Update RVM installations to latest rubygems.
+* Update to rake-compiler 1.2.1.
+* Reduce pre-installed gems to only rake-compiler and bundler.
+* Install yaml and ffi development headers in the base images, for psych and ffi gem compilation.
+* Ensure autoconf is installed in the base iamges.
+
+
 1.2.2 / 2022-06-27
 ------------------
 

build/mk_i686.rb

@@ -1,3 +1,5 @@
+#! /usr/bin/env ruby
+
 require "fileutils"
 
 Dir["/usr/bin/x86_64-linux-gnu-*"].each do |file|

build/mk_osxcross.sh

@@ -0,0 +1,38 @@
+#! /usr/bin/env bash
+
+set -o errexit
+set -o pipefail
+set -x
+
+git clone -q --depth=1 https://github.com/tpoechtrager/osxcross.git /opt/osxcross
+rm -rf /opt/osxcross/.git
+
+set +x
+cd /opt/osxcross/tarballs
+set -x
+curl -L -o MacOSX11.1.sdk.tar.xz https://github.com/larskanis/MacOSX-SDKs/releases/download/11.1/MacOSX11.1.sdk.tar.xz
+tar -xf MacOSX11.1.sdk.tar.xz -C .
+cp -rf /usr/lib/llvm-10/include/c++ MacOSX11.1.sdk/usr/include/c++
+cp -rf /usr/include/x86_64-linux-gnu/c++/9/bits/ MacOSX11.1.sdk/usr/include/c++/v1/bits
+tar -cJf MacOSX11.1.sdk.tar.xz MacOSX11.1.sdk
+
+set +x
+cd /opt/osxcross
+set -x
+UNATTENDED=1 SDK_VERSION=11.1 OSX_VERSION_MIN=10.13 USE_CLANG_AS=1 ./build.sh
+ln -s /usr/bin/llvm-config-10 /usr/bin/llvm-config
+ENABLE_COMPILER_RT_INSTALL=1 SDK_VERSION=11.1 ./build_compiler_rt.sh
+rm -rf *~ build tarballs/*
+
+echo "export PATH=/opt/osxcross/target/bin:\$PATH" >> /etc/rubybashrc
+echo "export MACOSX_DEPLOYMENT_TARGET=10.13" >> /etc/rubybashrc
+echo "export OSXCROSS_MP_INC=1" >> /etc/rubybashrc
+
+# Add links to build tools without target version kind of:
+#   arm64-apple-darwin-clang   =>   arm64-apple-darwin20.1-clang
+rm -f /opt/osxcross/target/bin/*-apple-darwin-*
+find /opt/osxcross/target/bin/ -name '*-apple-darwin[0-9]*' | sort | while read f ; do d=`echo $f | sed s/darwin[0-9\.]*/darwin/`; echo $f '"$@"' | tee $d && chmod +x $d ; done
+
+# There's no objdump in osxcross but we can use llvm's
+ln -s /usr/lib/llvm-10/bin/llvm-objdump /opt/osxcross/target/bin/x86_64-apple-darwin-objdump
+ln -s /usr/lib/llvm-10/bin/llvm-objdump /opt/osxcross/target/bin/aarch64-apple-darwin-objdump