Auto merge of #41639 - ishitatsuyuki:unify-tools, r=alexcrichton

Unify tools building

Close #41601

Time saving for up to 10 minutes. Cargo is now only compiled once.

Downsides:

- Out of tree Cargo.lock maintenance
- Cargo.toml `[replace]` version maintenance

Author: bors

.gitmodules

@@ -23,7 +23,7 @@
 	url = https://github.com/rust-lang-nursery/nomicon.git
 [submodule "src/tools/cargo"]
 	path = src/tools/cargo
-	url = https://github.com/rust-lang/cargo
+	url = https://github.com/rust-lang/cargo.git
 [submodule "reference"]
 	path = src/doc/reference
 	url = https://github.com/rust-lang-nursery/reference.git
@@ -32,4 +32,4 @@
 	url = https://github.com/rust-lang/book.git
 [submodule "src/tools/rls"]
 	path = src/tools/rls
-	url = https://github.com/rust-lang-nursery/rls
+	url = https://github.com/rust-lang-nursery/rls.git

src/Cargo.lock

@@ -14,10 +14,6 @@ members = [
   "tools/remote-test-client",
   "tools/remote-test-server",
   "tools/rust-installer",
-]
-
-# These projects have their own Cargo.lock
-exclude = [
   "tools/cargo",
   "tools/rls",
 ]
@@ -38,3 +34,6 @@ debug-assertions = false
 [profile.test]
 debug = false
 debug-assertions = false
+
+[replace]
+"https://github.com/rust-lang/cargo#0.20.0" = { path = "tools/cargo" }

src/Cargo.toml

@@ -38,7 +38,24 @@ use std::path::PathBuf;
 use std::process::{Command, ExitStatus};
 
 fn main() {
-    let args = env::args_os().skip(1).collect::<Vec<_>>();
+    let mut args = env::args_os().skip(1).collect::<Vec<_>>();
+
+    // Append metadata suffix for internal crates. See the corresponding entry
+    // in bootstrap/lib.rs for details.
+    if let Ok(s) = env::var("RUSTC_METADATA_SUFFIX") {
+        for i in 1..args.len() {
+            // Dirty code for borrowing issues
+            let mut new = None;
+            if let Some(current_as_str) = args[i].to_str() {
+                if (&*args[i - 1] == "-C" && current_as_str.starts_with("metadata")) ||
+                   current_as_str.starts_with("-Cmetadata") {
+                    new = Some(format!("{}-{}", current_as_str, s));
+                }
+            }
+            if let Some(new) = new { args[i] = new.into(); }
+        }
+    }
+
     // Detect whether or not we're a build script depending on whether --target
     // is passed (a bit janky...)
     let target = args.windows(2)

src/bootstrap/bin/rustc.rs

@@ -14,6 +14,7 @@
 import datetime
 import hashlib
 import os
+import re
 import shutil
 import subprocess
 import sys
@@ -126,13 +127,13 @@ def unpack(tarball, dst, verbose=False, match=None):
             shutil.move(tp, fp)
     shutil.rmtree(os.path.join(dst, fname))
 
-def run(args, verbose=False, exception=False):
+def run(args, verbose=False, exception=False, cwd=None):
     if verbose:
         print("running: " + ' '.join(args))
     sys.stdout.flush()
     # Use Popen here instead of call() as it apparently allows powershell on
     # Windows to not lock up waiting for input presumably.
-    ret = subprocess.Popen(args)
+    ret = subprocess.Popen(args, cwd=cwd)
     code = ret.wait()
     if code != 0:
         err = "failed to run: " + ' '.join(args)
@@ -297,8 +298,10 @@ def bin_root(self):
 
     def get_toml(self, key):
         for line in self.config_toml.splitlines():
-            if line.startswith(key + ' ='):
-                return self.get_string(line)
+            match = re.match(r'^{}\s*=(.*)$'.format(key), line)
+            if match is not None:
+                value = match.group(1)
+                return self.get_string(value) or value.strip()
         return None
 
     def get_mk(self, key):
@@ -329,6 +332,8 @@ def rustc(self):
 
     def get_string(self, line):
         start = line.find('"')
+        if start == -1:
+            return None
         end = start + 1 + line[start + 1:].find('"')
         return line[start + 1:end]
 
@@ -386,12 +391,21 @@ def build_bootstrap(self):
             args.append("--frozen")
         self.run(args, env)
 
-    def run(self, args, env):
-        proc = subprocess.Popen(args, env=env)
+    def run(self, args, env=None, cwd=None):
+        proc = subprocess.Popen(args, env=env, cwd=cwd)
         ret = proc.wait()
         if ret != 0:
             sys.exit(ret)
 
+    def output(self, args, env=None, cwd=None):
+        proc = subprocess.Popen(args, stdout=subprocess.PIPE, env=env, cwd=cwd)
+        (out, err) = proc.communicate()
+        ret = proc.wait()
+        if ret != 0:
+            print(out)
+            sys.exit(ret)
+        return out
+
     def build_triple(self):
         default_encoding = sys.getdefaultencoding()
         config = self.get_toml('build')
@@ -529,6 +543,54 @@ def build_triple(self):
 
         return "{}-{}".format(cputype, ostype)
 
+    def update_submodules(self):
+        if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
+            self.get_toml('submodules') == "false" or \
+            self.get_mk('CFG_DISABLE_MANAGE_SUBMODULES') == "1":
+            return
+
+        print('Updating submodules')
+        output = self.output(["git", "submodule", "status"], cwd=self.rust_root)
+        submodules = []
+        for line in output.splitlines():
+            # NOTE `git submodule status` output looks like this:
+            #
+            # -5066b7dcab7e700844b0e2ba71b8af9dc627a59b src/liblibc
+            # +b37ef24aa82d2be3a3cc0fe89bf82292f4ca181c src/compiler-rt (remotes/origin/..)
+            #  e058ca661692a8d01f8cf9d35939dfe3105ce968 src/jemalloc (3.6.0-533-ge058ca6)
+            #
+            # The first character can be '-', '+' or ' ' and denotes the
+            # `State` of the submodule Right next to this character is the
+            # SHA-1 of the submodule HEAD And after that comes the path to the
+            # submodule
+            path = line[1:].split(' ')[1]
+            submodules.append([path, line[0]])
+
+        self.run(["git", "submodule", "sync"], cwd=self.rust_root)
+
+        for submod in submodules:
+            path, status = submod
+            if path.endswith(b"llvm") and \
+                (self.get_toml('llvm-config') or self.get_mk('CFG_LLVM_ROOT')):
+                continue
+            if path.endswith(b"jemalloc") and \
+                (self.get_toml('jemalloc') or self.get_mk('CFG_JEMALLOC_ROOT')):
+                continue
+            submod_path = os.path.join(self.rust_root, path)
+
+            if status == ' ':
+                self.run(["git", "reset", "--hard"], cwd=submod_path)
+                self.run(["git", "clean", "-fdx"], cwd=submod_path)
+            elif status == '+':
+                self.run(["git", "submodule", "update", path], cwd=self.rust_root)
+                self.run(["git", "reset", "--hard"], cwd=submod_path)
+                self.run(["git", "clean", "-fdx"], cwd=submod_path)
+            elif status == '-':
+                self.run(["git", "submodule", "init", path], cwd=self.rust_root)
+                self.run(["git", "submodule", "update", path], cwd=self.rust_root)
+            else:
+                raise ValueError('unknown submodule status: ' + status)
+
 def bootstrap():
     parser = argparse.ArgumentParser(description='Build rust')
     parser.add_argument('--config')
@@ -597,6 +659,8 @@ def bootstrap():
     else:
         rb._download_url = 'https://static.rust-lang.org'
 
+    rb.update_submodules()
+
     # Fetch/build the bootstrap
     rb.build = rb.build_triple()
     rb.download_stage0()

src/bootstrap/bootstrap.py

@@ -82,7 +82,7 @@ use std::env;
 use std::ffi::OsString;
 use std::fs::{self, File};
 use std::io::Read;
-use std::path::{Component, PathBuf, Path};
+use std::path::{PathBuf, Path};
 use std::process::Command;
 
 use build_helper::{run_silent, run_suppressed, output, mtime};
@@ -285,129 +285,12 @@ impl Build {
             self.verbose(&format!("auto-detected local-rebuild {}", local_release));
             self.local_rebuild = true;
         }
-        self.verbose("updating submodules");
-        self.update_submodules();
         self.verbose("learning about cargo");
         metadata::build(self);
 
         step::run(self);
     }
 
-    /// Updates all git submodules that we have.
-    ///
-    /// This will detect if any submodules are out of date an run the necessary
-    /// commands to sync them all with upstream.
-    fn update_submodules(&self) {
-        struct Submodule<'a> {
-            path: &'a Path,
-            state: State,
-        }
-
-        enum State {
-            // The submodule may have staged/unstaged changes
-            MaybeDirty,
-            // Or could be initialized but never updated
-            NotInitialized,
-            // The submodule, itself, has extra commits but those changes haven't been commited to
-            // the (outer) git repository
-            OutOfSync,
-        }
-
-        if !self.src_is_git || !self.config.submodules {
-            return
-        }
-        let git = || {
-            let mut cmd = Command::new("git");
-            cmd.current_dir(&self.src);
-            return cmd
-        };
-        let git_submodule = || {
-            let mut cmd = Command::new("git");
-            cmd.current_dir(&self.src).arg("submodule");
-            return cmd
-        };
-
-        // FIXME: this takes a seriously long time to execute on Windows and a
-        //        nontrivial amount of time on Unix, we should have a better way
-        //        of detecting whether we need to run all the submodule commands
-        //        below.
-        let out = output(git_submodule().arg("status"));
-        let mut submodules = vec![];
-        for line in out.lines() {
-            // NOTE `git submodule status` output looks like this:
-            //
-            // -5066b7dcab7e700844b0e2ba71b8af9dc627a59b src/liblibc
-            // +b37ef24aa82d2be3a3cc0fe89bf82292f4ca181c src/compiler-rt (remotes/origin/..)
-            //  e058ca661692a8d01f8cf9d35939dfe3105ce968 src/jemalloc (3.6.0-533-ge058ca6)
-            //
-            // The first character can be '-', '+' or ' ' and denotes the `State` of the submodule
-            // Right next to this character is the SHA-1 of the submodule HEAD
-            // And after that comes the path to the submodule
-            let path = Path::new(line[1..].split(' ').skip(1).next().unwrap());
-            let state = if line.starts_with('-') {
-                State::NotInitialized
-            } else if line.starts_with('+') {
-                State::OutOfSync
-            } else if line.starts_with(' ') {
-                State::MaybeDirty
-            } else {
-                panic!("unexpected git submodule state: {:?}", line.chars().next());
-            };
-
-            submodules.push(Submodule { path: path, state: state })
-        }
-
-        self.run(git_submodule().arg("sync"));
-
-        for submodule in submodules {
-            // If using llvm-root then don't touch the llvm submodule.
-            if submodule.path.components().any(|c| c == Component::Normal("llvm".as_ref())) &&
-                self.config.target_config.get(&self.config.build)
-                    .and_then(|c| c.llvm_config.as_ref()).is_some()
-            {
-                continue
-            }
-
-            if submodule.path.components().any(|c| c == Component::Normal("jemalloc".as_ref())) &&
-                !self.config.use_jemalloc
-            {
-                continue
-            }
-
-            // `submodule.path` is the relative path to a submodule (from the repository root)
-            // `submodule_path` is the path to a submodule from the cwd
-
-            // use `submodule.path` when e.g. executing a submodule specific command from the
-            // repository root
-            // use `submodule_path` when e.g. executing a normal git command for the submodule
-            // (set via `current_dir`)
-            let submodule_path = self.src.join(submodule.path);
-
-            match submodule.state {
-                State::MaybeDirty => {
-                    // drop staged changes
-                    self.run(git().current_dir(&submodule_path)
-                                  .args(&["reset", "--hard"]));
-                    // drops unstaged changes
-                    self.run(git().current_dir(&submodule_path)
-                                  .args(&["clean", "-fdx"]));
-                },
-                State::NotInitialized => {
-                    self.run(git_submodule().arg("init").arg(submodule.path));
-                    self.run(git_submodule().arg("update").arg(submodule.path));
-                },
-                State::OutOfSync => {
-                    // drops submodule commits that weren't reported to the (outer) git repository
-                    self.run(git_submodule().arg("update").arg(submodule.path));
-                    self.run(git().current_dir(&submodule_path)
-                                  .args(&["reset", "--hard"]));
-                    self.run(git().current_dir(&submodule_path)
-                                  .args(&["clean", "-fdx"]));
-                },
-            }
-        }
-    }
-
     /// Clear out `dir` if `input` is newer.
     ///
     /// After this executes, it will also ensure that `dir` exists.
@@ -475,12 +358,30 @@ impl Build {
              .env("RUSTDOC_REAL", self.rustdoc(compiler))
              .env("RUSTC_FLAGS", self.rustc_flags(target).join(" "));
 
-        // Tools don't get debuginfo right now, e.g. cargo and rls don't get
-        // compiled with debuginfo.
         if mode != Mode::Tool {
-             cargo.env("RUSTC_DEBUGINFO", self.config.rust_debuginfo.to_string())
-                  .env("RUSTC_DEBUGINFO_LINES", self.config.rust_debuginfo_lines.to_string())
-                  .env("RUSTC_FORCE_UNSTABLE", "1");
+            // Tools don't get debuginfo right now, e.g. cargo and rls don't
+            // get compiled with debuginfo.
+            cargo.env("RUSTC_DEBUGINFO", self.config.rust_debuginfo.to_string())
+                 .env("RUSTC_DEBUGINFO_LINES", self.config.rust_debuginfo_lines.to_string())
+                 .env("RUSTC_FORCE_UNSTABLE", "1");
+
+            // Currently the compiler depends on crates from crates.io, and
+            // then other crates can depend on the compiler (e.g. proc-macro
+            // crates). Let's say, for example that rustc itself depends on the
+            // bitflags crate. If an external crate then depends on the
+            // bitflags crate as well, we need to make sure they don't
+            // conflict, even if they pick the same verison of bitflags. We'll
+            // want to make sure that e.g. a plugin and rustc each get their
+            // own copy of bitflags.
+
+            // Cargo ensures that this works in general through the -C metadata
+            // flag. This flag will frob the symbols in the binary to make sure
+            // they're different, even though the source code is the exact
+            // same. To solve this problem for the compiler we extend Cargo's
+            // already-passed -C metadata flag with our own. Our rustc.rs
+            // wrapper around the actual rustc will detect -C metadata being
+            // passed and frob it with this extra string we're passing in.
+            cargo.env("RUSTC_METADATA_SUFFIX", "rustc");
         }
 
         // Enable usage of unstable features

src/bootstrap/lib.rs

@@ -32,6 +32,7 @@ static EXCEPTIONS: &'static [&'static str] = &[
     "openssl", // BSD+advertising clause, cargo, mdbook
     "pest", // MPL2, mdbook via handlebars
     "thread-id", // Apache-2.0, mdbook
+    "strings", // this is actually MIT/Apache-2.0 but it's not in the manifest yet
 ];
 
 pub fn check(path: &Path, bad: &mut bool) {