Auto merge of rust-lang#99967 - Mark-Simulacrum:download-llvm-ci, r=jyn514

Download, rather than sccache-cache, LLVM in CI

My hope/expectation is that we can do better than sccache in CI for cached builds -- currently it looks like on macOS those still take upwards of 10-11 minutes, which is a significant amount of time that we could potentially cut.

This enables this mode for all non-dist builders; this should avoid any problems with the artifacts we distribute, while also providing for faster test builders (since they'll make use of PGO'd LLVM on the platforms we do that on, which is hopefully a nice win). It slightly increases the chance of test builders starting to fail only after a PR is merged (if PGO changes runtime behavior), but that should hopefully never happen, so I think this is worthwhile.

Measurements on the PR for apple-1 don't show any noticeable improvement in CI times, but those can be pretty noisy -- I'm inclined to land this since it *should* pretty much always be better and we can reconsider if that ever turns out not to be the case.

Author: bors

.github/workflows/ci.yml

@@ -391,24 +391,28 @@ jobs:
             env:
               RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-gnu --set llvm.allow-old-toolchain"
               SCRIPT: make ci-mingw-subset-1
+              NO_DOWNLOAD_CI_LLVM: 1
               CUSTOM_MINGW: 1
             os: windows-latest-xl
           - name: i686-mingw-2
             env:
               RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-gnu --set llvm.allow-old-toolchain"
               SCRIPT: make ci-mingw-subset-2
+              NO_DOWNLOAD_CI_LLVM: 1
               CUSTOM_MINGW: 1
             os: windows-latest-xl
           - name: x86_64-mingw-1
             env:
               SCRIPT: make ci-mingw-subset-1
               RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-gnu --enable-profiler --set llvm.allow-old-toolchain"
+              NO_DOWNLOAD_CI_LLVM: 1
               CUSTOM_MINGW: 1
             os: windows-latest-xl
           - name: x86_64-mingw-2
             env:
               SCRIPT: make ci-mingw-subset-2
               RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-gnu --enable-profiler --set llvm.allow-old-toolchain"
+              NO_DOWNLOAD_CI_LLVM: 1
               CUSTOM_MINGW: 1
             os: windows-latest-xl
           - name: dist-x86_64-msvc
@@ -433,6 +437,7 @@ jobs:
           - name: dist-i686-mingw
             env:
               RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-gnu --enable-full-tools --enable-profiler --set llvm.allow-old-toolchain"
+              NO_DOWNLOAD_CI_LLVM: 1
               SCRIPT: python x.py dist
               CUSTOM_MINGW: 1
               DIST_REQUIRE_ALL_TOOLS: 1
@@ -441,6 +446,7 @@ jobs:
             env:
               SCRIPT: python x.py dist
               RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler --set llvm.allow-old-toolchain"
+              NO_DOWNLOAD_CI_LLVM: 1
               CUSTOM_MINGW: 1
               DIST_REQUIRE_ALL_TOOLS: 1
             os: windows-latest-xl

src/bootstrap/config.rs

@@ -990,42 +990,7 @@ impl Config {
             config.llvm_from_ci = match llvm.download_ci_llvm {
                 Some(StringOrBool::String(s)) => {
                     assert!(s == "if-available", "unknown option `{}` for download-ci-llvm", s);
-                    // This is currently all tier 1 targets and tier 2 targets with host tools
-                    // (since others may not have CI artifacts)
-                    // https://doc.rust-lang.org/rustc/platform-support.html#tier-1
-                    // FIXME: this is duplicated in bootstrap.py
-                    let supported_platforms = [
-                        // tier 1
-                        "aarch64-unknown-linux-gnu",
-                        "i686-pc-windows-gnu",
-                        "i686-pc-windows-msvc",
-                        "i686-unknown-linux-gnu",
-                        "x86_64-unknown-linux-gnu",
-                        "x86_64-apple-darwin",
-                        "x86_64-pc-windows-gnu",
-                        "x86_64-pc-windows-msvc",
-                        // tier 2 with host tools
-                        "aarch64-apple-darwin",
-                        "aarch64-pc-windows-msvc",
-                        "aarch64-unknown-linux-musl",
-                        "arm-unknown-linux-gnueabi",
-                        "arm-unknown-linux-gnueabihf",
-                        "armv7-unknown-linux-gnueabihf",
-                        "mips-unknown-linux-gnu",
-                        "mips64-unknown-linux-gnuabi64",
-                        "mips64el-unknown-linux-gnuabi64",
-                        "mipsel-unknown-linux-gnu",
-                        "powerpc-unknown-linux-gnu",
-                        "powerpc64-unknown-linux-gnu",
-                        "powerpc64le-unknown-linux-gnu",
-                        "riscv64gc-unknown-linux-gnu",
-                        "s390x-unknown-linux-gnu",
-                        "x86_64-unknown-freebsd",
-                        "x86_64-unknown-illumos",
-                        "x86_64-unknown-linux-musl",
-                        "x86_64-unknown-netbsd",
-                    ];
-                    supported_platforms.contains(&&*config.build.triple)
+                    crate::native::is_ci_llvm_available(&config, llvm_assertions.unwrap_or(false))
                 }
                 Some(StringOrBool::Bool(b)) => b,
                 None => false,
@@ -1176,14 +1141,14 @@ impl Config {
 
         if config.llvm_from_ci {
             let triple = &config.build.triple;
+            let ci_llvm_bin = config.ci_llvm_root().join("bin");
             let mut build_target = config
                 .target_config
                 .entry(config.build)
                 .or_insert_with(|| Target::from_triple(&triple));
 
             check_ci_llvm!(build_target.llvm_config);
             check_ci_llvm!(build_target.llvm_filecheck);
-            let ci_llvm_bin = config.out.join(&*config.build.triple).join("ci-llvm/bin");
             build_target.llvm_config = Some(ci_llvm_bin.join(exe("llvm-config", config.build)));
             build_target.llvm_filecheck = Some(ci_llvm_bin.join(exe("FileCheck", config.build)));
         }

src/bootstrap/native.rs

@@ -114,23 +114,20 @@ pub fn prebuilt_llvm_config(
     Err(Meta { stamp, build_llvm_config, out_dir, root: root.into() })
 }
 
-pub(crate) fn maybe_download_ci_llvm(builder: &Builder<'_>) {
-    let config = &builder.config;
-    if !config.llvm_from_ci {
-        return;
-    }
+/// This retrieves the LLVM sha we *want* to use, according to git history.
+pub(crate) fn detect_llvm_sha(config: &crate::config::Config) -> String {
     let mut rev_list = config.git();
     rev_list.args(&[
         PathBuf::from("rev-list"),
-        format!("--author={}", builder.config.stage0_metadata.config.git_merge_commit_email).into(),
+        format!("--author={}", config.stage0_metadata.config.git_merge_commit_email).into(),
         "-n1".into(),
         "--first-parent".into(),
         "HEAD".into(),
         "--".into(),
-        builder.src.join("src/llvm-project"),
-        builder.src.join("src/bootstrap/download-ci-llvm-stamp"),
+        config.src.join("src/llvm-project"),
+        config.src.join("src/bootstrap/download-ci-llvm-stamp"),
         // the LLVM shared object file is named `LLVM-12-rust-{version}-nightly`
-        builder.src.join("src/version"),
+        config.src.join("src/version"),
     ]);
     let llvm_sha = output(&mut rev_list);
     let llvm_sha = llvm_sha.trim();
@@ -143,8 +140,82 @@ pub(crate) fn maybe_download_ci_llvm(builder: &Builder<'_>) {
         panic!();
     }
 
+    llvm_sha.to_owned()
+}
+
+/// Returns whether the CI-found LLVM is currently usable.
+///
+/// This checks both the build triple platform to confirm we're usable at all,
+/// and then verifies if the current HEAD matches the detected LLVM SHA head,
+/// in which case LLVM is indicated as not available.
+pub(crate) fn is_ci_llvm_available(config: &crate::config::Config, asserts: bool) -> bool {
+    // This is currently all tier 1 targets and tier 2 targets with host tools
+    // (since others may not have CI artifacts)
+    // https://doc.rust-lang.org/rustc/platform-support.html#tier-1
+    let supported_platforms = [
+        // tier 1
+        "aarch64-unknown-linux-gnu",
+        "i686-pc-windows-gnu",
+        "i686-pc-windows-msvc",
+        "i686-unknown-linux-gnu",
+        "x86_64-unknown-linux-gnu",
+        "x86_64-apple-darwin",
+        "x86_64-pc-windows-gnu",
+        "x86_64-pc-windows-msvc",
+        // tier 2 with host tools
+        "aarch64-apple-darwin",
+        "aarch64-pc-windows-msvc",
+        "aarch64-unknown-linux-musl",
+        "arm-unknown-linux-gnueabi",
+        "arm-unknown-linux-gnueabihf",
+        "armv7-unknown-linux-gnueabihf",
+        "mips-unknown-linux-gnu",
+        "mips64-unknown-linux-gnuabi64",
+        "mips64el-unknown-linux-gnuabi64",
+        "mipsel-unknown-linux-gnu",
+        "powerpc-unknown-linux-gnu",
+        "powerpc64-unknown-linux-gnu",
+        "powerpc64le-unknown-linux-gnu",
+        "riscv64gc-unknown-linux-gnu",
+        "s390x-unknown-linux-gnu",
+        "x86_64-unknown-freebsd",
+        "x86_64-unknown-illumos",
+        "x86_64-unknown-linux-musl",
+        "x86_64-unknown-netbsd",
+    ];
+    if !supported_platforms.contains(&&*config.build.triple) {
+        return false;
+    }
+
+    let triple = &*config.build.triple;
+    if (triple == "aarch64-unknown-linux-gnu" || triple.contains("i686")) && asserts {
+        // No alt builder for aarch64-unknown-linux-gnu today.
+        return false;
+    }
+
+    if crate::util::CiEnv::is_ci() {
+        let llvm_sha = detect_llvm_sha(config);
+        let head_sha = output(config.git().arg("rev-parse").arg("HEAD"));
+        let head_sha = head_sha.trim();
+        if llvm_sha == head_sha {
+            eprintln!(
+                "Detected LLVM as non-available: running in CI and modified LLVM in this change"
+            );
+            return false;
+        }
+    }
+
+    true
+}
+
+pub(crate) fn maybe_download_ci_llvm(builder: &Builder<'_>) {
+    let config = &builder.config;
+    if !config.llvm_from_ci {
+        return;
+    }
     let llvm_root = config.ci_llvm_root();
     let llvm_stamp = llvm_root.join(".llvm-stamp");
+    let llvm_sha = detect_llvm_sha(&config);
     let key = format!("{}{}", llvm_sha, config.llvm_assertions);
     if program_out_of_date(&llvm_stamp, &key) && !config.dry_run {
         download_ci_llvm(builder, &llvm_sha);

src/bootstrap/util.rs

@@ -258,6 +258,10 @@ impl CiEnv {
         }
     }
 
+    pub fn is_ci() -> bool {
+        Self::current() != CiEnv::None
+    }
+
     /// If in a CI environment, forces the command to run with colors.
     pub fn force_coloring_in_ci(self, cmd: &mut Command) {
         if self != CiEnv::None {

src/ci/docker/host-x86_64/arm-android/Dockerfile

@@ -29,6 +29,10 @@ ENV PATH=$PATH:/android/sdk/platform-tools
 
 ENV TARGETS=arm-linux-androideabi
 
+# We are intentionally allowing an old toolchain on this builder (and that's
+# incompatible with LLVM downloads today).
+ENV NO_DOWNLOAD_CI_LLVM 1
+
 ENV RUST_CONFIGURE_ARGS --arm-linux-androideabi-ndk=/android/ndk/arm-14 \
     --set llvm.allow-old-toolchain
 

src/ci/docker/host-x86_64/armhf-gnu/Dockerfile

@@ -78,6 +78,8 @@ RUN curl -O https://ci-mirrors.rust-lang.org/rustc/vexpress-v2p-ca15-tc1.dtb
 COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh
 
+COPY static/gitconfig /etc/gitconfig
+
 ENV RUST_CONFIGURE_ARGS --qemu-armhf-rootfs=/tmp/rootfs
 ENV SCRIPT python3 ../x.py --stage 2 test --host='' --target arm-unknown-linux-gnueabihf
 

src/ci/docker/host-x86_64/i686-gnu-nopt/Dockerfile

@@ -27,6 +27,10 @@ RUN mkdir -p /config
 RUN echo "[rust]" > /config/nopt-std-config.toml
 RUN echo "optimize = false" >> /config/nopt-std-config.toml
 
+# We are intentionally allowing an old toolchain on this builder (and that's
+# incompatible with LLVM downloads today).
+ENV NO_DOWNLOAD_CI_LLVM 1
+
 ENV RUST_CONFIGURE_ARGS --build=i686-unknown-linux-gnu --disable-optimize-tests \
     --set llvm.allow-old-toolchain
 ENV SCRIPT python3 ../x.py test --stage 0 --config /config/nopt-std-config.toml library/std \

src/ci/docker/host-x86_64/i686-gnu/Dockerfile

@@ -23,6 +23,9 @@ RUN sh /scripts/sccache.sh
 COPY scripts/cmake.sh /scripts/
 RUN /scripts/cmake.sh
 
+# We are intentionally allowing an old toolchain on this builder (and that's
+# incompatible with LLVM downloads today).
+ENV NO_DOWNLOAD_CI_LLVM 1
 ENV RUST_CONFIGURE_ARGS --build=i686-unknown-linux-gnu \
     --set llvm.allow-old-toolchain
 # Exclude some tests that are unlikely to be platform specific, to speed up

src/ci/docker/host-x86_64/test-various/Dockerfile

@@ -30,6 +30,10 @@ WORKDIR /
 COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh
 
+# We are disabling CI LLVM since this builder needs to build LLD, which is
+# currently unsupported when downloading pre-built LLVM.
+ENV NO_DOWNLOAD_CI_LLVM 1
+
 ENV RUST_CONFIGURE_ARGS \
   --musl-root-x86_64=/usr/local/x86_64-linux-musl \
   --set build.nodejs=/node-v15.14.0-linux-x64/bin/node \

src/ci/docker/host-x86_64/wasm32/Dockerfile

@@ -48,6 +48,8 @@ ENV TARGETS=wasm32-unknown-emscripten
 # Use -O1 optimizations in the link step to reduce time spent optimizing.
 ENV EMCC_CFLAGS=-O1
 
+COPY static/gitconfig /etc/gitconfig
+
 # Emscripten installation is user-specific
 ENV NO_CHANGE_USER=1
 

src/ci/docker/host-x86_64/x86_64-gnu-aux/Dockerfile

@@ -26,6 +26,10 @@ RUN sh /scripts/sccache.sh
 COPY scripts/cmake.sh /scripts/
 RUN /scripts/cmake.sh
 
+# We are intentionally allowing an old toolchain on this builder (and that's
+# incompatible with LLVM downloads today).
+ENV NO_DOWNLOAD_CI_LLVM 1
+
 ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu \
     --set llvm.allow-old-toolchain
 ENV RUST_CHECK_TARGET check-aux

src/ci/docker/host-x86_64/x86_64-gnu-debug/Dockerfile

@@ -31,6 +31,9 @@ RUN sh /scripts/sccache.sh
 ENV RUSTBUILD_FORCE_CLANG_BASED_TESTS 1
 ENV RUN_CHECK_WITH_PARALLEL_QUERIES 1
 
+# llvm.use-linker conflicts with downloading CI LLVM
+ENV NO_DOWNLOAD_CI_LLVM 1
+
 ENV RUST_CONFIGURE_ARGS \
       --build=x86_64-unknown-linux-gnu \
       --enable-debug \

src/ci/docker/host-x86_64/x86_64-gnu-distcheck/Dockerfile

@@ -22,6 +22,10 @@ RUN sh /scripts/sccache.sh
 COPY scripts/cmake.sh /scripts/
 RUN /scripts/cmake.sh
 
+# We are intentionally allowing an old toolchain on this builder (and that's
+# incompatible with LLVM downloads today).
+ENV NO_DOWNLOAD_CI_LLVM 1
+
 ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu --set rust.ignore-git=false \
     --set llvm.allow-old-toolchain
 ENV SCRIPT python3 ../x.py --stage 2 test distcheck

src/ci/docker/host-x86_64/x86_64-gnu-llvm-13-stage1/Dockerfile

@@ -26,7 +26,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh
 
-# using llvm-link-shared due to libffi issues -- see #34486
+# We are disabling CI LLVM since this builder is intentionally using a host
+# LLVM, rather than the typical src/llvm-project LLVM.
+ENV NO_DOWNLOAD_CI_LLVM 1
+
+# Using llvm-link-shared due to libffi issues -- see #34486
 ENV RUST_CONFIGURE_ARGS \
     --build=x86_64-unknown-linux-gnu \
     --llvm-root=/usr/lib/llvm-13 \

src/ci/docker/host-x86_64/x86_64-gnu-llvm-13/Dockerfile

@@ -37,7 +37,11 @@ RUN apt-get update && \
 COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh
 
-# using llvm-link-shared due to libffi issues -- see #34486
+# We are disabling CI LLVM since this builder is intentionally using a host
+# LLVM, rather than the typical src/llvm-project LLVM.
+ENV NO_DOWNLOAD_CI_LLVM 1
+
+# Using llvm-link-shared due to libffi issues -- see #34486
 ENV RUST_CONFIGURE_ARGS \
       --build=x86_64-unknown-linux-gnu \
       --llvm-root=/usr/lib/llvm-13 \

src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile

@@ -80,6 +80,10 @@ COPY host-x86_64/x86_64-gnu-tools/browser-ui-test.version /tmp/
 # the local version of the package is different than the one used by the CI.
 RUN npm install -g browser-ui-test@$(head -n 1 /tmp/browser-ui-test.version) --unsafe-perm=true
 
+# We are intentionally allowing an old toolchain on this builder (and that's
+# incompatible with LLVM downloads today).
+ENV NO_DOWNLOAD_CI_LLVM 1
+
 ENV RUST_CONFIGURE_ARGS \
   --set llvm.allow-old-toolchain \
   --build=x86_64-unknown-linux-gnu \

src/ci/docker/static/gitconfig

@@ -0,0 +1,2 @@
+[safe]
+directory = *