Auto merge of #31884 - alexcrichton:no-bootstrap-from-existing, r=brson

These commits add support to the rustbuild build system to compile non-build-system compilers. For example this will allow creating an ARM compiler on a x86 system. The high level way this works is:

* The only compiler ever run is the build target compiler. No other compiler is assumed to be runnable.
* As a result, all output artifacts come from the build compiler.
* The libs for the stageN cross-compiled compiler will be linked into place after the build target builds them.

This will break the assumption that a compiler never links to anything it didn't itself produce, but it retains the assumption that a compiler only ever links to anything built in the same stage. I believe this means that the stage1 cross-compiled compilers will still be usable.

I tested this by creating an `arm-unknown-linux-gnueabihf` compiler. So far the linking ended up all working OK (including LLVM being cross compiled), but I haven't been able to run it yet (in QEMU for a raspberry pi). I think that's because my system linker is messed up or something like that (some newer option is assumed when it's not actually there).

Overall, though, this means that rustbuild can compile an arm-unknown-linux-gnueabihf compiler. Ideally we could even start shipping nightlies based on this at some point!

Author: bors

src/bootstrap/Cargo.lock

@@ -73,6 +73,7 @@ def download_rust_nightly(self):
 
         if self.rustc().startswith(self.bin_root()) and \
            (not os.path.exists(self.rustc()) or self.rustc_out_of_date()):
+            shutil.rmtree(self.bin_root())
             filename = "rust-std-nightly-" + self.build + ".tar.gz"
             url = "https://static.rust-lang.org/dist/" + self.snap_rustc_date()
             tarball = os.path.join(rustc_cache, filename)

src/bootstrap/bootstrap.py

@@ -58,6 +58,30 @@ pub fn std<'a>(build: &'a Build, stage: u32, target: &str,
     }
 
     build.run(&mut cargo);
+    std_link(build, stage, target, compiler, host);
+}
+
+/// Link all libstd rlibs/dylibs into the sysroot location.
+///
+/// Links those artifacts generated in the given `stage` for `target` produced
+/// by `compiler` into `host`'s sysroot.
+pub fn std_link(build: &Build,
+                stage: u32,
+                target: &str,
+                compiler: &Compiler,
+                host: &str) {
+    let libdir = build.sysroot_libdir(stage, host, target);
+    let out_dir = build.cargo_out(stage, compiler.host, true, target);
+
+    // If we're linking one compiler host's output into another, then we weren't
+    // called from the `std` method above. In that case we clean out what's
+    // already there and then also link compiler-rt into place.
+    if host != compiler.host {
+        let _ = fs::remove_dir_all(&libdir);
+        t!(fs::create_dir_all(&libdir));
+        t!(fs::hard_link(&build.compiler_rt_built.borrow()[target],
+                         libdir.join(staticlib("compiler-rt", target))));
+    }
     add_to_sysroot(&out_dir, &libdir);
 }
 
@@ -99,7 +123,6 @@ pub fn rustc<'a>(build: &'a Build, stage: u32, target: &str,
              host, target);
 
     let out_dir = build.cargo_out(stage, &host, false, target);
-    let rustc = out_dir.join(exe("rustc", target));
     build.clear_if_dirty(&out_dir, &libstd_shim(build, stage, &host, target));
 
     let mut cargo = build.cargo(stage, compiler, false, target, "build");
@@ -131,10 +154,13 @@ pub fn rustc<'a>(build: &'a Build, stage: u32, target: &str,
     if !build.unstable_features {
         cargo.env("CFG_DISABLE_UNSTABLE_FEATURES", "1");
     }
-    if let Some(config) = build.config.target_config.get(target) {
-        if let Some(ref s) = config.llvm_config {
-            cargo.env("LLVM_CONFIG", s);
-        }
+    let target_config = build.config.target_config.get(target);
+    if let Some(ref s) = target_config.and_then(|c| c.llvm_config.as_ref()) {
+        cargo.env("LLVM_CONFIG", s);
+    } else {
+        let llvm_config = build.llvm_out(&build.config.build).join("bin")
+                               .join(exe("llvm-config", target));
+        cargo.env("LLVM_CONFIG", llvm_config);
     }
     if build.config.llvm_static_stdcpp {
         cargo.env("LLVM_STATIC_STDCPP",
@@ -148,12 +174,21 @@ pub fn rustc<'a>(build: &'a Build, stage: u32, target: &str,
     }
     build.run(&mut cargo);
 
-    let sysroot_libdir = build.sysroot_libdir(stage, host, target);
-    add_to_sysroot(&out_dir, &sysroot_libdir);
+    rustc_link(build, stage, target, compiler, compiler.host);
+}
 
-    if host == target {
-        assemble_compiler(build, stage, target, &rustc);
-    }
+/// Link all librustc rlibs/dylibs into the sysroot location.
+///
+/// Links those artifacts generated in the given `stage` for `target` produced
+/// by `compiler` into `host`'s sysroot.
+pub fn rustc_link(build: &Build,
+                  stage: u32,
+                  target: &str,
+                  compiler: &Compiler,
+                  host: &str) {
+    let libdir = build.sysroot_libdir(stage, host, target);
+    let out_dir = build.cargo_out(stage, compiler.host, false, target);
+    add_to_sysroot(&out_dir, &libdir);
 }
 
 /// Cargo's output path for the standard library in a given stage, compiled
@@ -169,39 +204,42 @@ fn compiler_file(compiler: &Path, file: &str) -> String {
 
 /// Prepare a new compiler from the artifacts in `stage`
 ///
-/// This will link the compiler built by `host` during the stage
-/// specified to the sysroot location for `host` to be the official
-/// `stage + 1` compiler for that host. This means that the `rustc` binary
-/// itself will be linked into place along with all supporting dynamic
-/// libraries.
-fn assemble_compiler(build: &Build, stage: u32, host: &str, rustc: &Path) {
+/// This will assemble a compiler in `build/$host/stage$stage`. The compiler
+/// must have been previously produced by the `stage - 1` build.config.build
+/// compiler.
+pub fn assemble_rustc(build: &Build, stage: u32, host: &str) {
+    assert!(stage > 0, "the stage0 compiler isn't assembled, it's downloaded");
+
     // Clear out old files
-    let sysroot = build.sysroot(stage + 1, host);
+    let sysroot = build.sysroot(stage, host);
     let _ = fs::remove_dir_all(&sysroot);
     t!(fs::create_dir_all(&sysroot));
 
     // Link in all dylibs to the libdir
     let sysroot_libdir = sysroot.join(libdir(host));
     t!(fs::create_dir_all(&sysroot_libdir));
-    let src_libdir = build.sysroot_libdir(stage, host, host);
+    let src_libdir = build.sysroot_libdir(stage - 1, &build.config.build, host);
     for f in t!(fs::read_dir(&src_libdir)).map(|f| t!(f)) {
         let filename = f.file_name().into_string().unwrap();
         if is_dylib(&filename) {
             t!(fs::hard_link(&f.path(), sysroot_libdir.join(&filename)));
         }
     }
 
+    let out_dir = build.cargo_out(stage - 1, &build.config.build, false, host);
+
     // Link the compiler binary itself into place
+    let rustc = out_dir.join(exe("rustc", host));
     let bindir = sysroot.join("bin");
     t!(fs::create_dir_all(&bindir));
-    let compiler = build.compiler_path(&Compiler::new(stage + 1, host));
+    let compiler = build.compiler_path(&Compiler::new(stage, host));
     let _ = fs::remove_file(&compiler);
     t!(fs::hard_link(rustc, compiler));
 
     // See if rustdoc exists to link it into place
-    let exe = exe("rustdoc", host);
-    let rustdoc_src = rustc.parent().unwrap().join(&exe);
-    let rustdoc_dst = bindir.join(exe);
+    let rustdoc = exe("rustdoc", host);
+    let rustdoc_src = out_dir.join(&rustdoc);
+    let rustdoc_dst = bindir.join(&rustdoc);
     if fs::metadata(&rustdoc_src).is_ok() {
         let _ = fs::remove_file(&rustdoc_dst);
         t!(fs::hard_link(&rustdoc_src, &rustdoc_dst));

src/bootstrap/build/compile.rs

@@ -39,6 +39,14 @@ mod sanity;
 mod step;
 mod util;
 
+#[cfg(windows)]
+mod job;
+
+#[cfg(not(windows))]
+mod job {
+    pub unsafe fn setup() {}
+}
+
 pub use build::config::Config;
 pub use build::flags::Flags;
 
@@ -114,14 +122,9 @@ impl Build {
     pub fn build(&mut self) {
         use build::step::Source::*;
 
-        // see comments in job.rs for what's going on here
-        #[cfg(windows)]
-        fn setup_job() {
-            mod job;
-            unsafe { job::setup() }
+        unsafe {
+            job::setup();
         }
-        #[cfg(not(windows))] fn setup_job() {}
-        setup_job();
 
         if self.flags.clean {
             return clean::clean(self);
@@ -146,8 +149,19 @@ impl Build {
                 Librustc { stage, compiler } => {
                     compile::rustc(self, stage, target.target, &compiler);
                 }
+                LibstdLink { stage, compiler, host } => {
+                    compile::std_link(self, stage, target.target,
+                                      &compiler, host);
+                }
+                LibrustcLink { stage, compiler, host } => {
+                    compile::rustc_link(self, stage, target.target,
+                                        &compiler, host);
+                }
+                Rustc { stage: 0 } => {
+                    // nothing to do...
+                }
                 Rustc { stage } => {
-                    println!("ok, rustc stage{} in {}", stage, target.target);
+                    compile::assemble_rustc(self, stage, target.target);
                 }
             }
         }
@@ -425,21 +439,7 @@ impl Build {
     }
 
     fn rustc_flags(&self, target: &str) -> Vec<String> {
-        let mut base = match target {
-            "arm-unknown-linux-gnueabihf" => {
-                vec!["-Ctarget-feature=+v6,+vfp2".to_string()]
-            }
-            "mips-unknown-linux-gnu" => {
-                vec!["-Ctarget-cpu=mips32r2".to_string(),
-                     "-Ctarget-feature=+mips32r2".to_string(),
-                     "-Csoft-float".to_string()]
-            }
-            "mipsel-unknown-linux-gnu" => {
-                vec!["-Ctarget-cpu=mips32".to_string(),
-                     "-Ctarget-feature=+mips32".to_string()]
-            }
-            _ => Vec::new(),
-        };
+        let mut base = Vec::new();
         if target != self.config.build && !target.contains("msvc") {
             base.push(format!("-Clinker={}", self.cc(target).display()));
         }

src/bootstrap/build/mod.rs

@@ -115,6 +115,11 @@ pub fn compiler_rt(build: &Build, target: &str) {
     let mode = if build.config.rust_optimize {"Release"} else {"Debug"};
     let (dir, build_target, libname) = if target.contains("linux") {
         let os = if target.contains("android") {"-android"} else {""};
+        let arch = if arch.starts_with("arm") && target.contains("eabihf") {
+            "armhf"
+        } else {
+            arch
+        };
         let target = format!("clang_rt.builtins-{}{}", arch, os);
         ("linux".to_string(), target.clone(), target)
     } else if target.contains("darwin") {
@@ -151,7 +156,10 @@ pub fn compiler_rt(build: &Build, target: &str) {
        .define("COMPILER_RT_DEFAULT_TARGET_TRIPLE", target)
        .define("COMPILER_RT_BUILD_SANITIZERS", "OFF")
        .define("COMPILER_RT_BUILD_EMUTLS", "OFF")
+       // inform about c/c++ compilers, the c++ compiler isn't actually used but
+       // it's needed to get the initial configure to work on all platforms.
        .define("CMAKE_C_COMPILER", build.cc(target))
+       .define("CMAKE_CXX_COMPILER", build.cc(target))
        .build_target(&build_target);
     cfg.build();
 }