Skip to content

Use the beta compiler for building bootstrap tools when download-rustc is set #82739

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 5, 2021
Merged

Use the beta compiler for building bootstrap tools when download-rustc is set #82739

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
151 changes: 83 additions & 68 deletions src/bootstrap/bootstrap.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -383,7 +383,7 @@ def __init__(self):
self.nix_deps_dir = None
self.rustc_commit = None

def download_stage0(self):
def download_toolchain(self, stage0=True, rustc_channel=None):
"""Fetch the build system for Rust, written in Rust

This method will build a cache directory, then it will fetch the
Expand All @@ -393,43 +393,47 @@ def download_stage0(self):
Each downloaded tarball is extracted, after that, the script
will move all the content to the right place.
"""
rustc_channel = self.rustc_channel
if rustc_channel is None:
rustc_channel = self.rustc_channel
rustfmt_channel = self.rustfmt_channel

if self.rustc().startswith(self.bin_root()) and \
(not os.path.exists(self.rustc()) or
self.program_out_of_date(self.rustc_stamp(), self.date + str(self.rustc_commit))):
if os.path.exists(self.bin_root()):
shutil.rmtree(self.bin_root())
download_rustc = self.rustc_commit is not None
bin_root = self.bin_root(stage0)

key = self.date
if not stage0:
key += str(self.rustc_commit)
if self.rustc(stage0).startswith(bin_root) and \
(not os.path.exists(self.rustc(stage0)) or
self.program_out_of_date(self.rustc_stamp(stage0), key)):
if os.path.exists(bin_root):
shutil.rmtree(bin_root)
tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
filename = "rust-std-{}-{}{}".format(
rustc_channel, self.build, tarball_suffix)
pattern = "rust-std-{}".format(self.build)
self._download_component_helper(filename, pattern, tarball_suffix, download_rustc)
self._download_component_helper(filename, pattern, tarball_suffix, stage0)
filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
tarball_suffix)
self._download_component_helper(filename, "rustc", tarball_suffix, download_rustc)
self._download_component_helper(filename, "rustc", tarball_suffix, stage0)
filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
tarball_suffix)
self._download_component_helper(filename, "cargo", tarball_suffix)
if self.rustc_commit is not None:
if not stage0:
filename = "rustc-dev-{}-{}{}".format(rustc_channel, self.build, tarball_suffix)
self._download_component_helper(
filename, "rustc-dev", tarball_suffix, download_rustc
filename, "rustc-dev", tarball_suffix, stage0
)

self.fix_bin_or_dylib("{}/bin/rustc".format(self.bin_root()))
self.fix_bin_or_dylib("{}/bin/rustdoc".format(self.bin_root()))
self.fix_bin_or_dylib("{}/bin/cargo".format(self.bin_root()))
lib_dir = "{}/lib".format(self.bin_root())
self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root))
self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root))
self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))
lib_dir = "{}/lib".format(bin_root)
for lib in os.listdir(lib_dir):
if lib.endswith(".so"):
self.fix_bin_or_dylib(os.path.join(lib_dir, lib), rpath_libz=True)
with output(self.rustc_stamp()) as rust_stamp:
rust_stamp.write(self.date + str(self.rustc_commit))
with output(self.rustc_stamp(stage0)) as rust_stamp:
rust_stamp.write(key)

if self.rustfmt() and self.rustfmt().startswith(self.bin_root()) and (
if self.rustfmt() and self.rustfmt().startswith(bin_root) and (
not os.path.exists(self.rustfmt())
or self.program_out_of_date(self.rustfmt_stamp(), self.rustfmt_channel)
):
Expand All @@ -440,12 +444,13 @@ def download_stage0(self):
self._download_component_helper(
filename, "rustfmt-preview", tarball_suffix, key=date
)
self.fix_bin_or_dylib("{}/bin/rustfmt".format(self.bin_root()))
self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(self.bin_root()))
self.fix_bin_or_dylib("{}/bin/rustfmt".format(bin_root))
self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(bin_root))
with output(self.rustfmt_stamp()) as rustfmt_stamp:
rustfmt_stamp.write(self.rustfmt_channel)

if self.downloading_llvm():
# Avoid downloading LLVM twice (once for stage0 and once for the master rustc)
if self.downloading_llvm() and stage0:
# We want the most recent LLVM submodule update to avoid downloading
# LLVM more often than necessary.
#
Expand Down Expand Up @@ -496,27 +501,26 @@ def downloading_llvm(self):
or (opt == "if-available" and self.build in supported_platforms)

def _download_component_helper(
self, filename, pattern, tarball_suffix, download_rustc=False, key=None
self, filename, pattern, tarball_suffix, stage0=True, key=None
):
if key is None:
if download_rustc:
key = self.rustc_commit
else:
if stage0:
key = self.date
else:
key = self.rustc_commit
cache_dst = os.path.join(self.build_dir, "cache")
rustc_cache = os.path.join(cache_dst, key)
if not os.path.exists(rustc_cache):
os.makedirs(rustc_cache)

if download_rustc:
url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(self.rustc_commit)
else:
if stage0:
url = "{}/dist/{}".format(self._download_url, key)
else:
url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(self.rustc_commit)
tarball = os.path.join(rustc_cache, filename)
if not os.path.exists(tarball):
do_verify = not download_rustc
get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=do_verify)
unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=stage0)
unpack(tarball, tarball_suffix, self.bin_root(stage0), match=pattern, verbose=self.verbose)

def _download_ci_llvm(self, llvm_sha, llvm_assertions):
cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
Expand Down Expand Up @@ -574,10 +578,10 @@ def fix_bin_or_dylib(self, fname, rpath_libz=False):
nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
print(nix_os_msg, fname)

# Only build `stage0/.nix-deps` once.
# Only build `.nix-deps` once.
nix_deps_dir = self.nix_deps_dir
if not nix_deps_dir:
nix_deps_dir = "{}/.nix-deps".format(self.bin_root())
nix_deps_dir = ".nix-deps"
if not os.path.exists(nix_deps_dir):
os.makedirs(nix_deps_dir)

Expand Down Expand Up @@ -635,8 +639,8 @@ def fix_bin_or_dylib(self, fname, rpath_libz=False):
print("warning: failed to call patchelf:", reason)
return

# Return the stage1 compiler to download, if any.
def maybe_download_rustc(self):
# If `download-rustc` is set, download the most recent commit with CI artifacts
def maybe_download_ci_toolchain(self):
# If `download-rustc` is not set, default to rebuilding.
if self.get_toml("download-rustc", section="rust") != "true":
return None
Expand All @@ -656,17 +660,23 @@ def maybe_download_rustc(self):
if status != 0:
print("warning: `download-rustc` is enabled, but there are changes to compiler/")

return commit
if self.verbose:
print("using downloaded stage1 artifacts from CI (commit {})".format(commit))
self.rustc_commit = commit
# FIXME: support downloading artifacts from the beta channel
self.download_toolchain(False, "nightly")

def rustc_stamp(self):
"""Return the path for .rustc-stamp
def rustc_stamp(self, stage0):
"""Return the path for .rustc-stamp at the given stage

>>> rb = RustBuild()
>>> rb.build_dir = "build"
>>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
>>> rb.rustc_stamp(True) == os.path.join("build", "stage0", ".rustc-stamp")
True
>>> rb.rustc_stamp(False) == os.path.join("build", "ci-rustc", ".rustc-stamp")
True
"""
return os.path.join(self.bin_root(), '.rustc-stamp')
return os.path.join(self.bin_root(stage0), '.rustc-stamp')

def rustfmt_stamp(self):
"""Return the path for .rustfmt-stamp
Expand All @@ -676,7 +686,7 @@ def rustfmt_stamp(self):
>>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
True
"""
return os.path.join(self.bin_root(), '.rustfmt-stamp')
return os.path.join(self.bin_root(True), '.rustfmt-stamp')

def llvm_stamp(self):
"""Return the path for .rustfmt-stamp
Expand All @@ -696,21 +706,27 @@ def program_out_of_date(self, stamp_path, key):
with open(stamp_path, 'r') as stamp:
return key != stamp.read()

def bin_root(self):
"""Return the binary root directory
def bin_root(self, stage0):
"""Return the binary root directory for the given stage

>>> rb = RustBuild()
>>> rb.build_dir = "build"
>>> rb.bin_root() == os.path.join("build", "stage0")
>>> rb.bin_root(True) == os.path.join("build", "stage0")
True
>>> rb.bin_root(False) == os.path.join("build", "ci-rustc")
True

When the 'build' property is given should be a nested directory:

>>> rb.build = "devel"
>>> rb.bin_root() == os.path.join("build", "devel", "stage0")
>>> rb.bin_root(True) == os.path.join("build", "devel", "stage0")
True
"""
return os.path.join(self.build_dir, self.build, "stage0")
if stage0:
subdir = "stage0"
else:
subdir = "ci-rustc"
return os.path.join(self.build_dir, self.build, subdir)

def llvm_root(self):
"""Return the CI LLVM root directory
Expand Down Expand Up @@ -773,33 +789,37 @@ def cargo(self):
"""Return config path for cargo"""
return self.program_config('cargo')

def rustc(self):
def rustc(self, stage0):
"""Return config path for rustc"""
return self.program_config('rustc')
return self.program_config('rustc', stage0)

def rustfmt(self):
"""Return config path for rustfmt"""
if not self.rustfmt_channel:
return None
return self.program_config('rustfmt')

def program_config(self, program):
"""Return config path for the given program
def program_config(self, program, stage0=True):
"""Return config path for the given program at the given stage

>>> rb = RustBuild()
>>> rb.config_toml = 'rustc = "rustc"\\n'
>>> rb.program_config('rustc')
'rustc'
>>> rb.config_toml = ''
>>> cargo_path = rb.program_config('cargo')
>>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
>>> cargo_path = rb.program_config('cargo', True)
>>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(True),
... "bin", "cargo")
True
>>> cargo_path = rb.program_config('cargo', False)
>>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(False),
... "bin", "cargo")
True
"""
config = self.get_toml(program)
if config:
return os.path.expanduser(config)
return os.path.join(self.bin_root(), "bin", "{}{}".format(
return os.path.join(self.bin_root(stage0), "bin", "{}{}".format(
program, self.exe_suffix()))

@staticmethod
Expand Down Expand Up @@ -854,14 +874,14 @@ def build_bootstrap(self):
if "CARGO_BUILD_TARGET" in env:
del env["CARGO_BUILD_TARGET"]
env["CARGO_TARGET_DIR"] = build_dir
env["RUSTC"] = self.rustc()
env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
env["RUSTC"] = self.rustc(True)
env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
(os.pathsep + env["LD_LIBRARY_PATH"]) \
if "LD_LIBRARY_PATH" in env else ""
env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
(os.pathsep + env["DYLD_LIBRARY_PATH"]) \
if "DYLD_LIBRARY_PATH" in env else ""
env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
env["LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
(os.pathsep + env["LIBRARY_PATH"]) \
if "LIBRARY_PATH" in env else ""
# preserve existing RUSTFLAGS
Expand All @@ -884,7 +904,7 @@ def build_bootstrap(self):
if self.get_toml("deny-warnings", "rust") != "false":
env["RUSTFLAGS"] += " -Dwarnings"

env["PATH"] = os.path.join(self.bin_root(), "bin") + \
env["PATH"] = os.path.join(self.bin_root(True), "bin") + \
os.pathsep + env["PATH"]
if not os.path.isfile(self.cargo()):
raise Exception("no cargo executable found at `{}`".format(
Expand Down Expand Up @@ -1135,14 +1155,9 @@ def bootstrap(help_triggered):
build.update_submodules()

# Fetch/build the bootstrap
build.rustc_commit = build.maybe_download_rustc()
if build.rustc_commit is not None:
if build.verbose:
commit = build.rustc_commit
print("using downloaded stage1 artifacts from CI (commit {})".format(commit))
# FIXME: support downloading artifacts from the beta channel
build.rustc_channel = "nightly"
build.download_stage0()
build.download_toolchain()
# Download the master compiler if `download-rustc` is set
build.maybe_download_ci_toolchain()
sys.stdout.flush()
build.ensure_vendored()
build.build_bootstrap()
Expand Down
15 changes: 10 additions & 5 deletions src/bootstrap/compile.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,9 @@ impl Step for Std {

// These artifacts were already copied (in `impl Step for Sysroot`).
// Don't recompile them.
if builder.config.download_rustc {
// NOTE: the ABI of the beta compiler is different from the ABI of the downloaded compiler,
// so its artifacts can't be reused.
if builder.config.download_rustc && compiler.stage != 0 {
return;
}

Expand Down Expand Up @@ -513,7 +515,9 @@ impl Step for Rustc {
let compiler = self.compiler;
let target = self.target;

if builder.config.download_rustc {
// NOTE: the ABI of the beta compiler is different from the ABI of the downloaded compiler,
// so its artifacts can't be reused.
if builder.config.download_rustc && compiler.stage != 0 {
// Copy the existing artifacts instead of rebuilding them.
// NOTE: this path is only taken for tools linking to rustc-dev.
builder.ensure(Sysroot { compiler });
Expand Down Expand Up @@ -934,14 +938,15 @@ impl Step for Sysroot {
t!(fs::create_dir_all(&sysroot));

// If we're downloading a compiler from CI, we can use the same compiler for all stages other than 0.
if builder.config.download_rustc {
if builder.config.download_rustc && compiler.stage != 0 {
assert_eq!(
builder.config.build, compiler.host,
"Cross-compiling is not yet supported with `download-rustc`",
);
// Copy the compiler into the correct sysroot.
let stage0_dir = builder.config.out.join(&*builder.config.build.triple).join("stage0");
builder.cp_r(&stage0_dir, &sysroot);
let ci_rustc_dir =
builder.config.out.join(&*builder.config.build.triple).join("ci-rustc");
builder.cp_r(&ci_rustc_dir, &sysroot);
return INTERNER.intern_path(sysroot);
}

Expand Down
Loading