[Rootfs] Improve API of expand_microarchitectures (#232)

Add a positional argument to select the microarchitectures to expand, and a
keyword argument to select the platforms to expand.  This should provide a
flexible interface to limit the expansion and avoid building for many useless
platforms.

Author: giordano

Project.toml

@@ -1,7 +1,7 @@
 name = "BinaryBuilderBase"
 uuid = "7f725544-6523-48cd-82d1-3fa08ff4056e"
 authors = ["Elliot Saba <[email protected]>"]
-version = "1.7.0"
+version = "1.8.0"
 
 [deps]
 CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193"

src/Rootfs.jl

@@ -675,7 +675,7 @@ end
 choose_shards(::AnyPlatform; kwargs...) = choose_shards(default_host_platform; kwargs...)
 
 """
-    supported_platforms(;exclude::Union{Vector{<:Platform},Function}=x->false,
+    supported_platforms(;exclude::Union{Vector{<:Platform},Function}=Returns(false),
                         experimental::Bool=false)
 
 Return the list of supported platforms as an array of `Platform`s.  These are the platforms we
@@ -690,7 +690,7 @@ or a function that returns true for exclusions i.e.
 supported_platforms(exclude=Sys.islinux)
 ```
 """
-function supported_platforms(;exclude::Union{Vector{<:Platform},Function}=x->false,
+function supported_platforms(;exclude::Union{Vector{<:Platform},Function}=Returns(false),
                              experimental::Bool=false)
     exclude_platforms!(platforms, exclude::Function) = filter(!exclude, platforms)
     exclude_platforms!(platforms, exclude::Vector{<:Platform}) = filter!(!in(exclude), platforms)
@@ -796,11 +796,12 @@ function expand_cxxstring_abis(ps::Vector{<:AbstractPlatform}; kwargs...)
 end
 
 """
-    expand_microarchitectures(p::AbstractPlatform)
+    expand_microarchitectures(p::AbstractPlatform, [microarchitectures::Vector{String}])
 
 Given a `Platform`, returns a vector of `Platforms` with differing `march` attributes
 as specified by the `ARCHITECTURE_FLAGS` mapping.  If the given `Platform` alread has a
-`march` tag specified, only that platform is returned.
+`march` tag specified, only that platform is returned.  If the `microarchitectures`
+argument is given, limit the expansion to the given microarchitectures.
 
 ```jldoctest
 julia> using BinaryBuilderBase
@@ -817,28 +818,30 @@ julia> expand_microarchitectures(Platform("armv7l", "linux"))
  Linux armv7l {call_abi=eabihf, libc=glibc, march=armv7l}
  Linux armv7l {call_abi=eabihf, libc=glibc, march=neonvfpv4}
 
-julia> expand_microarchitectures(Platform("aarch64", "linux"))
-4-element Vector{Platform}:
+julia> expand_microarchitectures(Platform("aarch64", "linux"), ["armv8_0", "a64fx"])
+2-element Vector{Platform}:
  Linux aarch64 {libc=glibc, march=armv8_0}
- Linux aarch64 {libc=glibc, march=armv8_4_crypto_sve}
- Linux aarch64 {libc=glibc, march=armv8_2_crypto}
- Linux aarch64 {libc=glibc, march=armv8_1}
+ Linux aarch64 {libc=glibc, march=a64fx}
 
 julia> expand_microarchitectures(Platform("i686", "windows"))
 2-element Vector{Platform}:
  Windows i686 {march=pentium4}
  Windows i686 {march=prescott}
 ```
 """
-function expand_microarchitectures(platform::AbstractPlatform)
-    # If this already has a `march`, or it's an `AnyPlatform`, just return it.
-    if isa(platform, AnyPlatform) || march(platform) !== nothing
+function expand_microarchitectures(platform::AbstractPlatform,
+                                   microarchitectures::Vector{String}=get_all_march_names(arch(platform)))
+    all_marchs = get_all_march_names(arch(platform))
+
+    # If this already has a `march`, or it's an `AnyPlatform`, or the microarchitectures we
+    # want to expand aren't relative to this platform, just return it.
+    if isa(platform, AnyPlatform) || march(platform) !== nothing ||
+        !any(in(all_marchs), microarchitectures)
         return [platform]
     end
 
-    # Otherwise, return a bunch of Platform objects with appropriately-set `march` tags, but
-    # first filter out some meaningless combinations of microarchitectures.
-    all_marchs = filter(get_all_march_names(arch(platform))) do march
+    # First, filter out some meaningless combinations of microarchitectures.
+    marchs = filter(all_marchs) do march
         if (!Sys.isapple(platform) && march == "apple_m1") ||
             (Sys.isapple(platform) && arch(platform) == "aarch64" && march ∉ ("armv8_0", "apple_m1"))
             # `apple_m1` makes sense only on macOS, and the only aarch64 microarchitectures
@@ -851,39 +854,89 @@ function expand_microarchitectures(platform::AbstractPlatform)
         return true
     end
 
-    return map(all_marchs) do march
-        p = deepcopy(platform)
-        p["march"] = march
-        return p
+    # But if none of the remaining microarchitectures are among those we want to expand,
+    # return the given platform as is.
+    if !any(in(microarchitectures), marchs)
+        return [platform]
     end
+
+    # Otherwise, return a bunch of Platform objects with appropriately-set `march` tags
+    return [(p = deepcopy(platform); p["march"] = march; p) for march in marchs if march in microarchitectures]
 end
 
 """
-    expand_microarchitectures(ps::Vector{<:Platform})
+    expand_microarchitectures(ps::Vector{<:Platform}, [microarchitectures::Vector{String}];
+                              filter=Returns(true))
+
+Expand all platforms in the vector `ps` with the supported microarchitectures.
 
-Expand all platforms in the given vector with the supported microarchitectures.
+If the `microarchitectures` argument is given, limit the expansion to the given
+platforms.  This is useful if you do not want to expand to all available
+microarchitectures.
+
+The expansion is applied only to the platforms matching the `filter` predicate, by
+default all platforms.  This is useful if you want to limit the expansion to some
+platforms, without having to explicitly list its microarchitectures in the second
+argument.
 
 ```jldoctest
 julia> using BinaryBuilderBase
 
 julia> expand_microarchitectures(filter!(p -> Sys.islinux(p) && libc(p) == "glibc", supported_platforms()))
-13-element Vector{Platform}:
+14-element Vector{Platform}:
  Linux i686 {libc=glibc, march=pentium4}
  Linux i686 {libc=glibc, march=prescott}
  Linux x86_64 {libc=glibc, march=x86_64}
  Linux x86_64 {libc=glibc, march=avx}
  Linux x86_64 {libc=glibc, march=avx2}
  Linux x86_64 {libc=glibc, march=avx512}
  Linux aarch64 {libc=glibc, march=armv8_0}
- Linux aarch64 {libc=glibc, march=armv8_4_crypto_sve}
- Linux aarch64 {libc=glibc, march=armv8_2_crypto}
  Linux aarch64 {libc=glibc, march=armv8_1}
+ Linux aarch64 {libc=glibc, march=armv8_2_crypto}
+ Linux aarch64 {libc=glibc, march=a64fx}
+ Linux armv6l {call_abi=eabihf, libc=glibc, march=arm1176jzfs}
  Linux armv7l {call_abi=eabihf, libc=glibc, march=armv7l}
  Linux armv7l {call_abi=eabihf, libc=glibc, march=neonvfpv4}
  Linux powerpc64le {libc=glibc, march=power8}
+
+julia> expand_microarchitectures(filter!(p -> Sys.islinux(p) && libc(p) == "glibc", supported_platforms()), ["x86_64", "avx2"])
+7-element Vector{Platform}:
+ Linux i686 {libc=glibc}
+ Linux x86_64 {libc=glibc, march=x86_64}
+ Linux x86_64 {libc=glibc, march=avx2}
+ Linux aarch64 {libc=glibc}
+ Linux armv6l {call_abi=eabihf, libc=glibc}
+ Linux armv7l {call_abi=eabihf, libc=glibc}
+ Linux powerpc64le {libc=glibc}
+
+julia> expand_microarchitectures(filter!(p -> Sys.islinux(p) && libc(p) == "glibc", supported_platforms()); filter=p->arch(p)=="x86_64")
+9-element Vector{Platform}:
+ Linux i686 {libc=glibc}
+ Linux x86_64 {libc=glibc, march=x86_64}
+ Linux x86_64 {libc=glibc, march=avx}
+ Linux x86_64 {libc=glibc, march=avx2}
+ Linux x86_64 {libc=glibc, march=avx512}
+ Linux aarch64 {libc=glibc}
+ Linux armv6l {call_abi=eabihf, libc=glibc}
+ Linux armv7l {call_abi=eabihf, libc=glibc}
+ Linux powerpc64le {libc=glibc}
 ```
 """
-expand_microarchitectures(ps::Vector{<:AbstractPlatform}) = collect(Iterators.flatten(expand_microarchitectures.(ps)))
+function expand_microarchitectures(ps::Vector{<:AbstractPlatform},
+                                   microarchitectures::Vector{String}=collect(Iterators.flatten(get_all_march_names.(unique!(arch.(ps)))));
+                                   filter=Returns(true),
+                                   )
+    out = map(ps) do p
+        return if filter(p)
+            # If the platform satisfies the predicate, expand its microarchitectures
+            expand_microarchitectures(p, microarchitectures)
+        else
+            # otherwise return it as-is.
+            [p]
+        end
+    end
+    return collect(Iterators.flatten(out))
+end
 
 """
     preferred_libgfortran_version(platform::AbstractPlatform, shard::CompilerShard;

test/rootfs.jl

@@ -57,6 +57,14 @@ using BinaryBuilderBase
         Platform("x86_64", "linux"; libc="glibc", march="avx512", cuda="10.1"),
         Platform("x86_64", "linux"; libc="glibc", march="x86_64", cuda="10.1"),
     ]
+    @test sort(expand_microarchitectures(Platform("x86_64", "linux"), ["x86_64", "avx512"]), by=triplet) == [
+        Platform("x86_64", "linux"; libc="glibc", march="avx512"),
+        Platform("x86_64", "linux"; libc="glibc", march="x86_64"),
+    ]
+    @test expand_microarchitectures(Platform("i686", "linux"), ["x86_64", "avx512"]) == [Platform("i686", "linux")]
+    @test expand_microarchitectures(Platform("aarch64", "linux"; libc="musl"), ["a64fx", "apple_m1"]) == [Platform("aarch64", "linux"; libc="musl")]
+    @test expand_microarchitectures(Platform("aarch64", "macos"), ["a64fx"]) == [Platform("aarch64", "macos")]
+
     @test sort(expand_microarchitectures(filter!(p -> Sys.islinux(p) && libc(p) == "glibc", supported_platforms())), by=triplet) == [
         Platform("aarch64", "linux"; libc="glibc", march="a64fx"),
         Platform("aarch64", "linux"; libc="glibc", march="armv8_0"),
@@ -73,17 +81,18 @@ using BinaryBuilderBase
         Platform("x86_64", "linux"; libc="glibc", march="avx512"),
         Platform("x86_64", "linux"; libc="glibc", march="x86_64"),
     ]
-    @test sort(expand_microarchitectures(filter!(p -> Sys.islinux(p) && arch(p) == "aarch64" && libc(p) == "musl", supported_platforms())), by=triplet) == [
+    @test sort(expand_microarchitectures(filter!(p -> Sys.islinux(p) && libc(p) == "musl", supported_platforms()); filter=p->arch(p) == "aarch64"), by=triplet) == [
         Platform("aarch64", "linux"; libc="musl", march="armv8_0"),
         Platform("aarch64", "linux"; libc="musl", march="armv8_1"),
         Platform("aarch64", "linux"; libc="musl", march="armv8_2_crypto"),
+        Platform("armv6l", "linux"; call_abi="eabihf", libc="musl"),
+        Platform("armv7l", "linux"; call_abi="eabihf", libc="musl"),
+        Platform("i686", "linux"; libc="musl"),
+        Platform("x86_64", "linux"; libc="musl"),
     ]
-    @test sort(expand_microarchitectures(filter!(p -> Sys.isapple(p), supported_platforms())), by=triplet) == [
+    @test sort(expand_microarchitectures(filter!(p -> Sys.isapple(p), supported_platforms()), ["a64fx", "apple_m1", "x86_64", "avx2"]), by=triplet) == [
         Platform("aarch64", "macos"; march="apple_m1"),
-        Platform("aarch64", "macos"; march="armv8_0"),
-        Platform("x86_64", "macos"; march="avx"),
         Platform("x86_64", "macos"; march="avx2"),
-        Platform("x86_64", "macos"; march="avx512"),
         Platform("x86_64", "macos"; march="x86_64"),
     ]
     @test expand_microarchitectures([Platform("x86_64", "windows"; march="avx", cuda="10.1")]) ==