Add support for multiple host platforms

This way we can provide images for Intel/AMD based as well as ARM based systems.

Image building is now prefixed by the host platform "arm" or "x86":

```
rake build:arm:arm-linux-musl      # Build image for platform arm-linux-musl on linux/arm64
rake build:arm:arm64-darwin        # Build image for platform arm64-darwin on linux/arm64
rake build:arm:jruby               # Build image for JRuby on linux/arm64
```

and:

```
rake build:x86:arm-linux-musl      # Build image for platform arm-linux-musl on linux/amd64
rake build:x86:arm64-darwin        # Build image for platform arm64-darwin on linux/amd64
rake build:x86:jruby               # Build image for JRuby on linux/amd64
```

Only `docker buildx build` supports multiple platforms, so that it is used now, instead of the classic `docker build`.
However buildx doesn't load the images into the local docker registry.
This is why `rake load:images` is added.
It loads all the images for the local platform and makes it available in `docker images`.

The number of images doubles with this patch, so that I get errors building all in parallel.
Therefore the number of parallel tasks should be limited like:

```
rake -j10 build:all
```

Author: larskanis

Rakefile

@@ -25,47 +25,68 @@ platforms = [
   ["x86_64-linux-musl", "x86_64-unknown-linux-musl"],
 ]
 
+host_platforms = [
+  # tuple is [docker platform, rake task, RUBY_PLATFORM matcher]
+  ["linux/amd64", "x86", /^x86_64|^x64|^amd64/],
+  ["linux/arm64", "arm", /^aarch64|arm64/],
+]
+local_platform = host_platforms.find { |_,_,reg| reg =~ RUBY_PLATFORM } or
+    raise("RUBY_PLATFORM #{RUBY_PLATFORM} is not supported as host")
+
 namespace :build do
 
-  platforms.each do |platform, target|
-    sdf = "Dockerfile.mri.#{platform}"
+  mkdir_p "tmp/docker"
+
+  host_platforms.each do |docker_platform, rake_platform|
+    namespace rake_platform do
 
-    desc "Build image for platform #{platform}"
-    task platform => sdf
-    task sdf do
-      image_name = RakeCompilerDock::Starter.container_image_name(platform: platform)
-      sh(*RakeCompilerDock.docker_build_cmd(platform), "-t", image_name, "-f", "Dockerfile.mri.#{platform}", ".")
-      if image_name.include?("linux-gnu")
-        sh("docker", "tag", image_name, image_name.sub("linux-gnu", "linux"))
+      platforms.each do |platform, target|
+        sdf = "tmp/docker/Dockerfile.mri.#{platform}.#{rake_platform}"
+        df = ERB.new(File.read("Dockerfile.mri.erb"), trim_mode: ">").result(binding)
+        File.write(sdf, df)
+        CLEAN.include(sdf)
       end
-    end
+      sdf = "tmp/docker/Dockerfile.jruby.#{rake_platform}"
+      df = File.read("Dockerfile.jruby")
+      File.write(sdf, df)
 
-    df = ERB.new(File.read("Dockerfile.mri.erb"), trim_mode: ">").result(binding)
-    File.write(sdf, df)
-    CLEAN.include(sdf)
-  end
+      builder = RakeCompilerDock::ParallelDockerBuild.new(platforms.map{|pl, _| "tmp/docker/Dockerfile.mri.#{pl}.#{rake_platform}" } + ["tmp/docker/Dockerfile.jruby.#{rake_platform}"], workdir: "tmp/docker", task_prefix: "common-#{rake_platform}-", platform: docker_platform)
 
-  desc "Build image for JRuby"
-  task :jruby => "Dockerfile.jruby"
-  task "Dockerfile.jruby" do
-    image_name = RakeCompilerDock::Starter.container_image_name(rubyvm: "jruby")
-    sh(*RakeCompilerDock.docker_build_cmd("jruby"), "-t", image_name, "-f", "Dockerfile.jruby", ".")
-  end
+      platforms.each do |platform, target|
+        sdf = "tmp/docker/Dockerfile.mri.#{platform}.#{rake_platform}"
 
-  RakeCompilerDock::ParallelDockerBuild.new(platforms.map{|pl, _| "Dockerfile.mri.#{pl}" } + ["Dockerfile.jruby"], workdir: "tmp/docker")
+        desc "Build image for platform #{platform} on #{docker_platform}"
+        task platform => sdf
+        multitask :all => platform
+      end
+
+      sdf = "tmp/docker/Dockerfile.jruby.#{rake_platform}"
+      desc "Build image for JRuby on #{docker_platform}"
+      task :jruby => sdf
+      multitask :all => :jruby
+    end
+    desc "Build all images on #{docker_platform} in parallel"
+    task rake_platform => "#{rake_platform}:all"
+  end
 
-  desc "Build images for all MRI platforms in parallel"
+  all_mri_images = host_platforms.flat_map do |_, rake_platform|
+    platforms.map do |platform, |
+      "#{rake_platform}:#{platform}"
+    end
+  end
+  desc "Build images for all MRI platforms and hosts in parallel"
   if ENV['RCD_USE_BUILDX_CACHE']
-    task :mri => platforms.map(&:first)
+    task :mri => all_mri_images
   else
-    multitask :mri => platforms.map(&:first)
+    multitask :mri => all_mri_images
   end
 
-  desc "Build images for all platforms in parallel"
+  all_images = all_mri_images + host_platforms.map { |_, pl| "#{pl}:jruby" }
+  desc "Build images for all platforms and hosts in parallel"
   if ENV['RCD_USE_BUILDX_CACHE']
-    task :all => platforms.map(&:first) + ["jruby"]
+    task :all => all_images
   else
-    multitask :all => platforms.map(&:first) + ["jruby"]
+    multitask :all => all_images
   end
 end
 
@@ -114,19 +135,35 @@ task :update_lists do
   end
 end
 
-namespace :release do
-  desc "push all docker images"
-  task :images do
-    image_name = RakeCompilerDock::Starter.container_image_name(rubyvm: "jruby")
-    sh("docker", "push", image_name)
+def build_images(platforms, host_platforms, output: )
+  image_name = RakeCompilerDock::Starter.container_image_name(rubyvm: "jruby")
 
-    platforms.each do |platform, _|
-      image_name = RakeCompilerDock::Starter.container_image_name(platform: platform)
-      sh("docker", "push", image_name)
+  plats = host_platforms.map(&:first).join(",")
+  sdf = "tmp/docker/Dockerfile.jruby.#{host_platforms.first[1]}"
+  RakeCompilerDock.docker_build(sdf, tag: image_name, platform: plats, output: output)
 
-      if image_name.include?("linux-gnu")
-        sh("docker", "push", image_name.sub("linux-gnu", "linux"))
-      end
+  platforms.each do |platform, _|
+    sdf = "tmp/docker/Dockerfile.mri.#{platform}.#{host_platforms.first[1]}"
+    image_name = RakeCompilerDock::Starter.container_image_name(platform: platform)
+
+    RakeCompilerDock.docker_build(sdf, tag: image_name, platform: plats, output: output)
+
+    if image_name.include?("linux-gnu")
+      RakeCompilerDock.docker_build(sdf, tag: image_name.sub("linux-gnu", "linux"), platform: plats, output: output)
     end
   end
 end
+
+namespace :release do
+  desc "Push all docker images on #{host_platforms.map(&:first).join(",")}"
+  task :images => "build:all" do
+    build_images(platforms, host_platforms, output: 'push')
+  end
+end
+
+namespace :load do
+  desc "Build and load into local docker storage"
+  task :images => "build:#{local_platform[1]}" do
+    build_images(platforms, [local_platform], output: 'load')
+  end
+end

build/parallel_docker_build.rb

@@ -14,17 +14,29 @@ def docker_build_cmd(platform=nil)
           return nil
         end
       else
-        ENV['RCD_DOCKER_BUILD'] || "docker build"
+        ENV['RCD_DOCKER_BUILD'] || "docker buildx build"
       end
       Shellwords.split(cmd)
     end
+
+    # Run an intermediate dockerfile without tag
+    #
+    # The layers will be reused in subsequent builds, even if they run in parallel.
+    def docker_build(filename, tag: tag, output: false, platform: )
+      cmd = docker_build_cmd
+      return if cmd.nil?
+      tag_args = ["-t", tag] if tag
+      push_args = ["--push"] if output == 'push'
+      push_args = ["--load"] if output == 'load'
+      Class.new.extend(FileUtils).sh(*cmd, "-f", filename, ".", "--platform", platform, *tag_args, *push_args)
+    end
   end
 
   # Run docker builds in parallel, but ensure that common docker layers are reused
   class ParallelDockerBuild
     include Rake::DSL
 
-    def initialize(dockerfiles, workdir: "tmp/docker", inputdir: ".", task_prefix: "common-")
+    def initialize(dockerfiles, workdir: "tmp/docker", inputdir: ".", task_prefix: "common-", platform: "local")
       FileUtils.mkdir_p(workdir)
 
       files = parse_dockerfiles(dockerfiles, inputdir)
@@ -34,6 +46,7 @@ def initialize(dockerfiles, workdir: "tmp/docker", inputdir: ".", task_prefix: "
       # pp vcs
 
       define_common_tasks(vcs, workdir, task_prefix)
+      @platform = platform
     end
 
     # Read given dockerfiles from inputdir and split into a list of commands.
@@ -96,7 +109,7 @@ def define_common_tasks(vcs, workdir, task_prefix, plines=[])
         fn = "#{task_prefix}#{Digest::SHA1.hexdigest(files.join)}"
         File.write(File.join(workdir, fn), (plines + lines).join)
         task fn do
-          docker_build(fn, workdir)
+          RakeCompilerDock.docker_build(File.join(workdir, fn), platform: @platform)
         end
 
         nfn = define_common_tasks(nvcs, workdir, task_prefix, plines + lines)
@@ -109,14 +122,5 @@ def define_common_tasks(vcs, workdir, task_prefix, plines=[])
         fn
       end
     end
-
-    # Run an intermediate dockerfile without tag
-    #
-    # The layers will be reused in subsequent builds, even if they run in parallel.
-    def docker_build(filename, workdir)
-      cmd = RakeCompilerDock.docker_build_cmd
-      return if cmd.nil?
-      sh(*RakeCompilerDock.docker_build_cmd, "-f", File.join(workdir, filename), ".")
-    end
   end
 end