Multiplatform build support

.github/workflows/ci.yml

@@ -82,7 +82,7 @@ jobs:
       - name: Build docker image
         run: |
           docker buildx create --driver docker-container --use
-          bundle exec rake build:${{ matrix.platform }} RCD_DOCKER_BUILD="docker buildx build --cache-from=type=local,src=tmp/build-cache --cache-to=type=local,dest=tmp/build-cache-new --load"
+          bundle exec rake build:x86:${{ matrix.platform }} RCD_DOCKER_BUILD="docker buildx build --cache-from=type=local,src=tmp/build-cache --cache-to=type=local,dest=tmp/build-cache-new"
           docker images
       - name: Update and prune docker buildx layer cache
         run: |

.github/workflows/publish-images.yml

@@ -57,10 +57,10 @@ jobs:
           ' | tee -a $GITHUB_OUTPUT
       - name: Build docker image
         env:
-          RCD_DOCKER_BUILD: docker buildx build --cache-from=type=local,src=tmp/build-cache --cache-to=type=local,dest=tmp/build-cache-new --load
+          RCD_DOCKER_BUILD: docker buildx build --cache-from=type=local,src=tmp/build-cache --cache-to=type=local,dest=tmp/build-cache-new
         run: |
           docker buildx create --driver docker-container --use
-          bundle exec rake build:${{matrix.platform}}
+          bundle exec rake build:x86:${{matrix.platform}}
           # move build cache and remove outdated layers
           rm -rf tmp/build-cache
           mv tmp/build-cache-new tmp/build-cache

.github/workflows/release-images.yml

@@ -64,10 +64,10 @@ jobs:
           ' | tee -a $GITHUB_OUTPUT
       - name: Build docker image
         env:
-          RCD_DOCKER_BUILD: docker buildx build --cache-from=type=local,src=tmp/build-cache --cache-to=type=local,dest=tmp/build-cache-new --load
+          RCD_DOCKER_BUILD: docker buildx build --cache-from=type=local,src=tmp/build-cache --cache-to=type=local,dest=tmp/build-cache-new
         run: |
           docker buildx create --driver docker-container --use
-          bundle exec rake build:${{matrix.platform}}
+          bundle exec rake build:x86:${{matrix.platform}}
           # move build cache and remove outdated layers
           rm -rf tmp/build-cache
           mv tmp/build-cache-new tmp/build-cache

CONTRIBUTING.md

@@ -64,3 +64,46 @@ docker buildx create --use --driver=docker-container
 bundle exec rake build
 ```
 
+
+### Create builder instance for two architectures
+
+Building with qemu emulation fails currently with a segfault, so that it must be built by a builder instance with at least one remote node for the other architecture.
+Building on native hardware is also much faster (~45 minutes) than on qemu.
+A two-nodes builder requires obviously a ARM and a Intel/AMD device.
+It can be created like this:
+
+```sh
+# Make sure the remote instance can be connected
+$ docker -H ssh://isa info
+
+# Create a new builder with the local instance
+# Disable the garbage collector by the config file
+$ docker buildx create --name isayoga --config build/buildkitd.toml
+
+# Add the remote instance
+$ docker buildx create --name isayoga --config build/buildkitd.toml --append ssh://isa
+
+# They are inactive from the start
+$ docker buildx ls
+NAME/NODE      DRIVER/ENDPOINT                   STATUS     BUILDKIT   PLATFORMS
+isayoga        docker-container
+ \_ isayoga0    \_ unix:///var/run/docker.sock   inactive
+ \_ isayoga1    \_ ssh://isa                     inactive
+default*       docker
+ \_ default     \_ default                       running    v0.13.2    linux/arm64
+
+# Bootstrap the instances
+$ docker buildx inspect --bootstrap --builder isayoga
+
+# Set the new builder as default
+$ docker buildx use isayoga
+
+# Now it should be default and in state "running"
+$ docker buildx ls
+NAME/NODE      DRIVER/ENDPOINT                   STATUS    BUILDKIT   PLATFORMS
+isayoga*       docker-container
+ \_ isayoga0    \_ unix:///var/run/docker.sock   running   v0.18.2    linux/arm64
+ \_ isayoga1    \_ ssh://isa                     running   v0.18.2    linux/amd64, linux/amd64/v2, linux/amd64/v3, linux/386
+default        docker
+ \_ default     \_ default                       running   v0.13.2    linux/arm64
+```

Rakefile

@@ -8,6 +8,27 @@ CLEAN.include("tmp")
 
 RakeCompilerDock::GemHelper.install_tasks
 
+def build_mri_images(platforms, host_platforms, output: )
+  plats = host_platforms.map(&:first).join(",")
+  platforms.each do |platform, _|
+    sdf = "tmp/docker/Dockerfile.mri.#{platform}.#{host_platforms.first[1]}"
+    image_name = RakeCompilerDock::Starter.container_image_name(platform: platform)
+
+    RakeCompilerDock.docker_build(sdf, tag: image_name, platform: plats, output: output)
+
+    if image_name.include?("linux-gnu")
+      RakeCompilerDock.docker_build(sdf, tag: image_name.sub("linux-gnu", "linux"), platform: plats, output: output)
+    end
+  end
+end
+
+def build_jruby_images(host_platforms, output: )
+  image_name = RakeCompilerDock::Starter.container_image_name(rubyvm: "jruby")
+  plats = host_platforms.map(&:first).join(",")
+  sdf = "tmp/docker/Dockerfile.jruby.#{host_platforms.first[1]}"
+  RakeCompilerDock.docker_build(sdf, tag: image_name, platform: plats, output: output)
+end
+
 platforms = [
   # tuple is [platform, target]
   ["aarch64-linux-gnu", "aarch64-linux-gnu"],
@@ -25,47 +46,84 @@ platforms = [
   ["x86_64-linux-musl", "x86_64-unknown-linux-musl"],
 ]
 
+host_platforms = [
+  # tuple is [docker platform, rake task, RUBY_PLATFORM matcher]
+  ["linux/amd64", "x86", /^x86_64|^x64|^amd64/],
+  ["linux/arm64", "arm", /^aarch64|arm64/],
+]
+local_platform = host_platforms.find { |_,_,reg| reg =~ RUBY_PLATFORM } or
+    raise("RUBY_PLATFORM #{RUBY_PLATFORM} is not supported as host")
+
 namespace :build do
 
-  platforms.each do |platform, target|
-    sdf = "Dockerfile.mri.#{platform}"
+  mkdir_p "tmp/docker"
+
+  host_platforms.each do |docker_platform, rake_platform|
+    namespace rake_platform do
 
-    desc "Build image for platform #{platform}"
-    task platform => sdf
-    task sdf do
-      image_name = RakeCompilerDock::Starter.container_image_name(platform: platform)
-      sh(*RakeCompilerDock.docker_build_cmd(platform), "-t", image_name, "-f", "Dockerfile.mri.#{platform}", ".")
-      if image_name.include?("linux-gnu")
-        sh("docker", "tag", image_name, image_name.sub("linux-gnu", "linux"))
+      platforms.each do |platform, target|
+        sdf = "tmp/docker/Dockerfile.mri.#{platform}.#{rake_platform}"
+        df = ERB.new(File.read("Dockerfile.mri.erb"), trim_mode: ">").result(binding)
+        File.write(sdf, df)
+        CLEAN.include(sdf)
+      end
+      sdf = "tmp/docker/Dockerfile.jruby.#{rake_platform}"
+      df = File.read("Dockerfile.jruby")
+      File.write(sdf, df)
+
+      builder = RakeCompilerDock::ParallelDockerBuild.new(platforms.map{|pl, _| "tmp/docker/Dockerfile.mri.#{pl}.#{rake_platform}" } + ["tmp/docker/Dockerfile.jruby.#{rake_platform}"], workdir: "tmp/docker", task_prefix: "common-#{rake_platform}-", platform: docker_platform)
+
+      platforms.each do |platform, target|
+        sdf = "tmp/docker/Dockerfile.mri.#{platform}.#{rake_platform}"
+
+        if docker_platform == local_platform[0]
+          # Load image after build on local platform only
+          desc "Build and load image for platform #{platform} on #{docker_platform}"
+          task platform => sdf do
+            build_mri_images([platform], [local_platform], output: 'load')
+          end
+        else
+          desc "Build image for platform #{platform} on #{docker_platform}"
+          task platform => sdf
+        end
+        multitask :all => platform
       end
-    end
 
-    df = ERB.new(File.read("Dockerfile.mri.erb"), trim_mode: ">").result(binding)
-    File.write(sdf, df)
-    CLEAN.include(sdf)
+      sdf = "tmp/docker/Dockerfile.jruby.#{rake_platform}"
+      if docker_platform == local_platform[0]
+        # Load image after build on local platform only
+        desc "Build and load image for JRuby on #{docker_platform}"
+        task :jruby => sdf do
+          build_jruby_images([local_platform], output: 'load')
+        end
+      else
+        desc "Build image for JRuby on #{docker_platform}"
+        task :jruby => sdf
+      end
+      multitask :all => :jruby
+    end
+    desc "Build all images on #{docker_platform} in parallel"
+    task rake_platform => "#{rake_platform}:all"
   end
 
-  desc "Build image for JRuby"
-  task :jruby => "Dockerfile.jruby"
-  task "Dockerfile.jruby" do
-    image_name = RakeCompilerDock::Starter.container_image_name(rubyvm: "jruby")
-    sh(*RakeCompilerDock.docker_build_cmd("jruby"), "-t", image_name, "-f", "Dockerfile.jruby", ".")
+  all_mri_images = host_platforms.flat_map do |_, rake_platform|
+    platforms.map do |platform, |
+      "#{rake_platform}:#{platform}"
+    end
   end
-
-  RakeCompilerDock::ParallelDockerBuild.new(platforms.map{|pl, _| "Dockerfile.mri.#{pl}" } + ["Dockerfile.jruby"], workdir: "tmp/docker")
-
-  desc "Build images for all MRI platforms in parallel"
+  desc "Build images for all MRI platforms and hosts in parallel"
   if ENV['RCD_USE_BUILDX_CACHE']
-    task :mri => platforms.map(&:first)
+    task :mri => all_mri_images
   else
-    multitask :mri => platforms.map(&:first)
+    multitask :mri => all_mri_images
   end
 
-  desc "Build images for all platforms in parallel"
+  all_images = all_mri_images + host_platforms.map { |_, pl| "#{pl}:jruby" }
+  desc "Build images for all platforms and hosts in parallel"
   if ENV['RCD_USE_BUILDX_CACHE']
-    task :all => platforms.map(&:first) + ["jruby"]
+    task :all => all_images
   else
-    multitask :all => platforms.map(&:first) + ["jruby"]
+    multitask :all => all_images
   end
 end
 
@@ -115,18 +173,9 @@ task :update_lists do
 end
 
 namespace :release do
-  desc "push all docker images"
-  task :images do
-    image_name = RakeCompilerDock::Starter.container_image_name(rubyvm: "jruby")
-    sh("docker", "push", image_name)
-
-    platforms.each do |platform, _|
-      image_name = RakeCompilerDock::Starter.container_image_name(platform: platform)
-      sh("docker", "push", image_name)
-
-      if image_name.include?("linux-gnu")
-        sh("docker", "push", image_name.sub("linux-gnu", "linux"))
-      end
-    end
+  desc "Push all docker images on #{host_platforms.map(&:first).join(",")}"
+  task :images => "build:all" do
+    build_jruby_images(host_platforms, output: 'push')
+    build_mri_images(platforms, host_platforms, output: 'push')
   end
 end

build/buildkitd.toml

@@ -0,0 +1,2 @@
+[worker.oci]
+  gc = false

build/parallel_docker_build.rb

@@ -14,17 +14,29 @@ def docker_build_cmd(platform=nil)
           return nil
         end
       else
-        ENV['RCD_DOCKER_BUILD'] || "docker build"
+        ENV['RCD_DOCKER_BUILD'] || "docker buildx build"
       end
       Shellwords.split(cmd)
     end
+
+    # Run an intermediate dockerfile without tag
+    #
+    # The layers will be reused in subsequent builds, even if they run in parallel.
+    def docker_build(filename, tag: nil, output: false, platform: )
+      cmd = docker_build_cmd
+      return if cmd.nil?
+      tag_args = ["-t", tag] if tag
+      push_args = ["--push"] if output == 'push'
+      push_args = ["--load"] if output == 'load'
+      Class.new.extend(FileUtils).sh(*cmd, "-f", filename, ".", "--platform", platform, *tag_args, *push_args)
+    end
   end
 
   # Run docker builds in parallel, but ensure that common docker layers are reused
   class ParallelDockerBuild
     include Rake::DSL
 
-    def initialize(dockerfiles, workdir: "tmp/docker", inputdir: ".", task_prefix: "common-")
+    def initialize(dockerfiles, workdir: "tmp/docker", inputdir: ".", task_prefix: "common-", platform: "local")
       FileUtils.mkdir_p(workdir)
 
       files = parse_dockerfiles(dockerfiles, inputdir)
@@ -34,6 +46,7 @@ def initialize(dockerfiles, workdir: "tmp/docker", inputdir: ".", task_prefix: "
       # pp vcs
 
       define_common_tasks(vcs, workdir, task_prefix)
+      @platform = platform
     end
 
     # Read given dockerfiles from inputdir and split into a list of commands.
@@ -96,7 +109,7 @@ def define_common_tasks(vcs, workdir, task_prefix, plines=[])
         fn = "#{task_prefix}#{Digest::SHA1.hexdigest(files.join)}"
         File.write(File.join(workdir, fn), (plines + lines).join)
         task fn do
-          docker_build(fn, workdir)
+          RakeCompilerDock.docker_build(File.join(workdir, fn), platform: @platform)
         end
 
         nfn = define_common_tasks(nvcs, workdir, task_prefix, plines + lines)
@@ -109,14 +122,5 @@ def define_common_tasks(vcs, workdir, task_prefix, plines=[])
         fn
       end
     end
-
-    # Run an intermediate dockerfile without tag
-    #
-    # The layers will be reused in subsequent builds, even if they run in parallel.
-    def docker_build(filename, workdir)
-      cmd = RakeCompilerDock.docker_build_cmd
-      return if cmd.nil?
-      sh(*RakeCompilerDock.docker_build_cmd, "-f", File.join(workdir, filename), ".")
-    end
   end
 end

mingw64-ucrt/README.md

@@ -12,47 +12,3 @@ They are built by the following command:
 ```sh
 docker buildx build . -t larskanis/mingw64-ucrt:20.04 --platform linux/arm64,linux/amd64 --push
 ```
-
-
-Create builder instance for two architectures
-------------------
-
-Building with qemu emulation fails currently with a segfault, so that it must be built by a builder instance with at least one remote node for the other architecture.
-Building on native hardware is also much faster (~30 minutes) than on qemu.
-A two-nodes builder requires obviously a ARM and a Intel/AMD device.
-It can be created like this:
-
-```sh
-# Make sure the remote instance can be connected
-$ docker -H ssh://isa info
-
-# Create a new builder with the local instance
-$ docker buildx create --name isayoga
-
-# Add the remote instance
-$ docker buildx create --name isayoga --append ssh://isa
-
-# They are inactive from the start
-$ docker buildx ls
-NAME/NODE      DRIVER/ENDPOINT                   STATUS     BUILDKIT   PLATFORMS
-isayoga        docker-container
- \_ isayoga0    \_ unix:///var/run/docker.sock   inactive
- \_ isayoga1    \_ ssh://isa                     inactive
-default*       docker
- \_ default     \_ default                       running    v0.13.2    linux/arm64
-
-# Bootstrap the instances
-$ docker buildx inspect --bootstrap --builder isayoga
-
-# Set the new builder as default
-$ docker buildx use isayoga
-
-# Now it should be default and in state "running"
-$ docker buildx ls
-NAME/NODE      DRIVER/ENDPOINT                   STATUS    BUILDKIT   PLATFORMS
-isayoga*       docker-container
- \_ isayoga0    \_ unix:///var/run/docker.sock   running   v0.18.2    linux/arm64
- \_ isayoga1    \_ ssh://isa                     running   v0.18.2    linux/amd64, linux/amd64/v2, linux/amd64/v3, linux/386
-default        docker
- \_ default     \_ default                       running   v0.13.2    linux/arm64
-```