Move submodule initialization to bootstrap.py

Author: ishitatsuyuki

src/bootstrap/bootstrap.py

@@ -14,6 +14,7 @@
 import datetime
 import hashlib
 import os
+import re
 import shutil
 import subprocess
 import sys
@@ -297,8 +298,10 @@ def bin_root(self):
 
     def get_toml(self, key):
         for line in self.config_toml.splitlines():
-            if line.startswith(key + ' ='):
-                return self.get_string(line)
+            match = re.match(r'^{}\s*=(.*)$'.format(key), line)
+            if match is not None:
+                value = match.group(1)
+                return self.get_string(value) or value.strip()
         return None
 
     def get_mk(self, key):
@@ -329,6 +332,8 @@ def rustc(self):
 
     def get_string(self, line):
         start = line.find('"')
+        if start == -1:
+            return None
         end = start + 1 + line[start + 1:].find('"')
         return line[start + 1:end]
 
@@ -386,7 +391,7 @@ def build_bootstrap(self):
             args.append("--frozen")
         self.run(args, env)
 
-    def run(self, args, env):
+    def run(self, args, env=None):
         proc = subprocess.Popen(args, env=env)
         ret = proc.wait()
         if ret != 0:
@@ -529,6 +534,32 @@ def build_triple(self):
 
         return "{}-{}".format(cputype, ostype)
 
+    def update_submodules(self):
+        if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
+            self.get_toml('submodules') == "false" or \
+            self.get_mk('CFG_DISABLE_MANAGE_SUBMODULES') == "1":
+            return
+
+        print('Updating submodules')
+        self.run(["git", "-C", self.rust_root, "submodule", "-q", "sync"])
+        # FIXME: nobody does, but this won't work well with whitespace in
+        # submodule path
+        submodules = [s.split()[1] for s in subprocess.check_output(
+            ["git", "config", "--file", os.path.join(
+                self.rust_root, ".gitmodules"), "--get-regexp", "path"]).splitlines()]
+        for module in submodules:
+            if module.endswith(b"llvm") and \
+                (self.get_toml('llvm-config') or self.get_mk('CFG_LLVM_ROOT')):
+                continue
+            if module.endswith(b"jemalloc") and \
+                (self.get_toml('jemalloc') or self.get_mk('CFG_JEMALLOC_ROOT')):
+                continue
+            self.run(["git", "-C", self.rust_root,
+                      "submodule", "update", "--init", module])
+        self.run(["git", "-C", self.rust_root, "submodule", "-q",
+                  "foreach", "git", "reset", "-q", "--hard"])
+        self.run(["git", "-C", self.rust_root, "submodule",
+                  "-q", "foreach", "git", "clean", "-qdfx"])
 def bootstrap():
     parser = argparse.ArgumentParser(description='Build rust')
     parser.add_argument('--config')
@@ -597,6 +628,8 @@ def bootstrap():
     else:
         rb._download_url = 'https://static.rust-lang.org'
 
+    rb.update_submodules()
+
     # Fetch/build the bootstrap
     rb.build = rb.build_triple()
     rb.download_stage0()

src/bootstrap/lib.rs

@@ -82,7 +82,7 @@ use std::env;
 use std::ffi::OsString;
 use std::fs::{self, File};
 use std::io::Read;
-use std::path::{Component, PathBuf, Path};
+use std::path::{PathBuf, Path};
 use std::process::Command;
 
 use build_helper::{run_silent, run_suppressed, output, mtime};
@@ -285,129 +285,12 @@ impl Build {
             self.verbose(&format!("auto-detected local-rebuild {}", local_release));
             self.local_rebuild = true;
         }
-        self.verbose("updating submodules");
-        self.update_submodules();
         self.verbose("learning about cargo");
         metadata::build(self);
 
         step::run(self);
     }
 
-    /// Updates all git submodules that we have.
-    ///
-    /// This will detect if any submodules are out of date an run the necessary
-    /// commands to sync them all with upstream.
-    fn update_submodules(&self) {
-        struct Submodule<'a> {
-            path: &'a Path,
-            state: State,
-        }
-
-        enum State {
-            // The submodule may have staged/unstaged changes
-            MaybeDirty,
-            // Or could be initialized but never updated
-            NotInitialized,
-            // The submodule, itself, has extra commits but those changes haven't been commited to
-            // the (outer) git repository
-            OutOfSync,
-        }
-
-        if !self.src_is_git || !self.config.submodules {
-            return
-        }
-        let git = || {
-            let mut cmd = Command::new("git");
-            cmd.current_dir(&self.src);
-            return cmd
-        };
-        let git_submodule = || {
-            let mut cmd = Command::new("git");
-            cmd.current_dir(&self.src).arg("submodule");
-            return cmd
-        };
-
-        // FIXME: this takes a seriously long time to execute on Windows and a
-        //        nontrivial amount of time on Unix, we should have a better way
-        //        of detecting whether we need to run all the submodule commands
-        //        below.
-        let out = output(git_submodule().arg("status"));
-        let mut submodules = vec![];
-        for line in out.lines() {
-            // NOTE `git submodule status` output looks like this:
-            //
-            // -5066b7dcab7e700844b0e2ba71b8af9dc627a59b src/liblibc
-            // +b37ef24aa82d2be3a3cc0fe89bf82292f4ca181c src/compiler-rt (remotes/origin/..)
-            //  e058ca661692a8d01f8cf9d35939dfe3105ce968 src/jemalloc (3.6.0-533-ge058ca6)
-            //
-            // The first character can be '-', '+' or ' ' and denotes the `State` of the submodule
-            // Right next to this character is the SHA-1 of the submodule HEAD
-            // And after that comes the path to the submodule
-            let path = Path::new(line[1..].split(' ').skip(1).next().unwrap());
-            let state = if line.starts_with('-') {
-                State::NotInitialized
-            } else if line.starts_with('+') {
-                State::OutOfSync
-            } else if line.starts_with(' ') {
-                State::MaybeDirty
-            } else {
-                panic!("unexpected git submodule state: {:?}", line.chars().next());
-            };
-
-            submodules.push(Submodule { path: path, state: state })
-        }
-
-        self.run(git_submodule().arg("sync"));
-
-        for submodule in submodules {
-            // If using llvm-root then don't touch the llvm submodule.
-            if submodule.path.components().any(|c| c == Component::Normal("llvm".as_ref())) &&
-                self.config.target_config.get(&self.config.build)
-                    .and_then(|c| c.llvm_config.as_ref()).is_some()
-            {
-                continue
-            }
-
-            if submodule.path.components().any(|c| c == Component::Normal("jemalloc".as_ref())) &&
-                !self.config.use_jemalloc
-            {
-                continue
-            }
-
-            // `submodule.path` is the relative path to a submodule (from the repository root)
-            // `submodule_path` is the path to a submodule from the cwd
-
-            // use `submodule.path` when e.g. executing a submodule specific command from the
-            // repository root
-            // use `submodule_path` when e.g. executing a normal git command for the submodule
-            // (set via `current_dir`)
-            let submodule_path = self.src.join(submodule.path);
-
-            match submodule.state {
-                State::MaybeDirty => {
-                    // drop staged changes
-                    self.run(git().current_dir(&submodule_path)
-                                  .args(&["reset", "--hard"]));
-                    // drops unstaged changes
-                    self.run(git().current_dir(&submodule_path)
-                                  .args(&["clean", "-fdx"]));
-                },
-                State::NotInitialized => {
-                    self.run(git_submodule().arg("init").arg(submodule.path));
-                    self.run(git_submodule().arg("update").arg(submodule.path));
-                },
-                State::OutOfSync => {
-                    // drops submodule commits that weren't reported to the (outer) git repository
-                    self.run(git_submodule().arg("update").arg(submodule.path));
-                    self.run(git().current_dir(&submodule_path)
-                                  .args(&["reset", "--hard"]));
-                    self.run(git().current_dir(&submodule_path)
-                                  .args(&["clean", "-fdx"]));
-                },
-            }
-        }
-    }
-
     /// Clear out `dir` if `input` is newer.
     ///
     /// After this executes, it will also ensure that `dir` exists.