bootstrap.py

#!/usr/bin/env python
#
# Copyright 2015-2016 The Rust Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution and at
# http://rust-lang.org/COPYRIGHT.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.

from __future__ import print_function

import argparse
import contextlib
import hashlib
import os
import shutil
import subprocess
import sys
import tarfile


def get(url, path, verbose=False):
    print("downloading " + url)
    sha_url = url + ".sha256"
    sha_path = path + ".sha256"
    for _url, _path in ((url, path), (sha_url, sha_path)):
        # See http://serverfault.com/questions/301128/how-to-download
        if sys.platform == 'win32':
            run(["PowerShell.exe", "/nologo", "-Command",
                 "(New-Object System.Net.WebClient)"
                 ".DownloadFile('{}', '{}')".format(_url, _path)],
                verbose=verbose)
        else:
            run(["curl", "-o", _path, _url], verbose=verbose)
    print("verifying " + path)
    with open(path, "rb") as f:
        found = hashlib.sha256(f.read()).hexdigest()
    with open(sha_path) as f:
        expected, _ = f.readline().split()
    if found != expected:
        err = ("invalid checksum:\n"
               "    found:    {}\n"
               "    expected: {}").format(found, expected)
        if verbose:
            raise RuntimeError(err)
        sys.exit(err)


def unpack(tarball, dst, verbose=False, match=None):
    print("extracting " + tarball)
    fname = os.path.basename(tarball).replace(".tar.gz", "")
    with contextlib.closing(tarfile.open(tarball)) as tar:
        for p in tar.getnames():
            if "/" not in p:
                continue
            name = p.replace(fname + "/", "", 1)
            if match is not None and not name.startswith(match):
                continue
            name = name[len(match) + 1:]

            fp = os.path.join(dst, name)
            if verbose:
                print("  extracting " + p)
            tar.extract(p, dst)
            tp = os.path.join(dst, p)
            if os.path.isdir(tp) and os.path.exists(fp):
                continue
            shutil.move(tp, fp)
    shutil.rmtree(os.path.join(dst, fname))


def run(args, verbose=False):
    if verbose:
        print("running: " + ' '.join(args))
    sys.stdout.flush()
    # Use Popen here instead of call() as it apparently allows PowerShell on
    # Windows to not lock up waiting for input presumably.
    ret = subprocess.Popen(args)
    code = ret.wait()
    if code != 0:
        err = "failed to run: " + ' '.join(args)
        if verbose:
            raise RuntimeError(err)
        sys.exit(err)


def stage0_data(rust_root):
    nightlies = os.path.join(rust_root, "src/stage0.txt")
    with open(nightlies, 'r') as nightlies:
        data = {}
        for line in nightlies.read().split("\n"):
            if line.startswith("#") or line == '':
                continue
            a, b = line.split(": ", 1)
            data[a] = b
        return data


class RustBuild:
    def __init__(self, build_dir, rust_root, config_toml='', config_mk='',
                 verbose=False):
        self.build_dir = build_dir
        self.config_mk = config_mk
        self.config_toml = config_toml
        self.rust_root = rust_root
        self.verbose = verbose

    def download_stage0(self):
        cache_dst = os.path.join(self.build_dir, "cache")
        rustc_cache = os.path.join(cache_dst, self.stage0_rustc_date())
        cargo_cache = os.path.join(cache_dst, self.stage0_cargo_date())
        if not os.path.exists(rustc_cache):
            os.makedirs(rustc_cache)
        if not os.path.exists(cargo_cache):
            os.makedirs(cargo_cache)

        if self.rustc().startswith(self.bin_root()) and (
                    not os.path.exists(
                        self.rustc()) or self.rustc_out_of_date()):
            if os.path.exists(self.bin_root()):
                shutil.rmtree(self.bin_root())
            channel = self.stage0_rustc_channel()
            filename = "rust-std-" + channel + "-" + self.build + ".tar.gz"
            url = "https://static.rust-lang.org/dist/" + \
                  self.stage0_rustc_date()
            tarball = os.path.join(rustc_cache, filename)
            if not os.path.exists(tarball):
                get(url + "/" + filename, tarball, verbose=self.verbose)
            unpack(tarball, self.bin_root(),
                   match="rust-std-" + self.build,
                   verbose=self.verbose)

            filename = "rustc-" + channel + "-" + self.build + ".tar.gz"
            url = "https://static.rust-lang.org/dist/" + \
                  self.stage0_rustc_date()
            tarball = os.path.join(rustc_cache, filename)
            if not os.path.exists(tarball):
                get(url + "/" + filename, tarball, verbose=self.verbose)
            unpack(tarball, self.bin_root(), match="rustc",
                   verbose=self.verbose)
            with open(self.rustc_stamp(), 'w') as f:
                f.write(self.stage0_rustc_date())

        if self.cargo().startswith(self.bin_root()) and (
                    not os.path.exists(
                        self.cargo()) or self.cargo_out_of_date()):
            channel = self.stage0_cargo_channel()
            filename = "cargo-" + channel + "-" + self.build + ".tar.gz"
            url = "https://static.rust-lang.org/cargo-dist/" + \
                  self.stage0_cargo_date()
            tarball = os.path.join(cargo_cache, filename)
            if not os.path.exists(tarball):
                get(url + "/" + filename, tarball, verbose=self.verbose)
            unpack(tarball, self.bin_root(), match="cargo",
                   verbose=self.verbose)
            with open(self.cargo_stamp(), 'w') as f:
                f.write(self.stage0_cargo_date())

    def stage0_cargo_date(self):
        return self._cargo_date

    def stage0_cargo_channel(self):
        return self._cargo_channel

    def stage0_rustc_date(self):
        return self._rustc_date

    def stage0_rustc_channel(self):
        return self._rustc_channel

    def rustc_stamp(self):
        return os.path.join(self.bin_root(), '.rustc-stamp')

    def cargo_stamp(self):
        return os.path.join(self.bin_root(), '.cargo-stamp')

    def rustc_out_of_date(self):
        if not os.path.exists(self.rustc_stamp()):
            return True
        with open(self.rustc_stamp(), 'r') as f:
            return self.stage0_rustc_date() != f.read()

    def cargo_out_of_date(self):
        if not os.path.exists(self.cargo_stamp()):
            return True
        with open(self.cargo_stamp(), 'r') as f:
            return self.stage0_cargo_date() != f.read()

    def bin_root(self):
        return os.path.join(self.build_dir, self.build, "stage0")

    def get_toml(self, key):
        for line in self.config_toml.splitlines():
            if line.startswith(key + ' ='):
                return self.get_string(line)

    def get_mk(self, key):
        for line in iter(self.config_mk.splitlines()):
            if line.startswith(key):
                return line[line.find(':=') + 2:].strip()

    def cargo(self):
        config = self.get_toml('cargo')

        return config or os.path.join(self.bin_root(),
                                      "bin/cargo" + self.exe_suffix())

    def rustc(self):
        config = self.get_toml('rustc')
        if config:
            return config
        config = self.get_mk('CFG_LOCAL_RUST')
        if config:
            return config + '/bin/rustc' + self.exe_suffix()
        return os.path.join(self.bin_root(), "bin/rustc" + self.exe_suffix())

    def get_string(self, line):
        start = line.find('"')
        end = start + 1 + line[start + 1:].find('"')
        return line[start + 1:end]

    def exe_suffix(self):
        if sys.platform == 'win32':
            return '.exe'
        else:
            return ''

    def build_bootstrap(self):
        env = os.environ.copy()
        env["CARGO_TARGET_DIR"] = os.path.join(self.build_dir, "bootstrap")
        env["RUSTC"] = self.rustc()
        env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib")
        env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib")
        env["PATH"] = os.path.join(self.bin_root(), "bin") + os.pathsep + env[
            "PATH"]
        self.run([self.cargo(), "build", "--manifest-path",
                  os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")],
                 env)

    def run(self, args, env):
        proc = subprocess.Popen(args, env=env)
        ret = proc.wait()
        if ret != 0:
            sys.exit(ret)

    def build_triple(self):
        config = self.get_toml('build')
        if config:
            return config
        config = self.get_mk('CFG_BUILD')
        if config:
            return config
        try:
            ostype = subprocess.check_output(['uname', '-s']).strip()
            cputype = subprocess.check_output(['uname', '-m']).strip()
        except OSError:
            if sys.platform == 'win32':
                return 'x86_64-pc-windows-msvc'
            else:
                err = "uname not found"
                if self.verbose:
                    raise Exception(err)
                sys.exit(err)

        # Darwin's `uname -s` lies and always returns i386. We have to use
        # sysctl instead.
        if ostype == 'Darwin' and cputype == 'i686':
            sysctl = subprocess.check_output(['sysctl', 'hw.optional.x86_64'])
            if sysctl.contains(': 1'):
                cputype = 'x86_64'

        # The goal here is to come up with the same triple as LLVM would,
        # at least for the subset of platforms we're willing to target.
        if ostype == 'Linux':
            ostype = 'unknown-linux-gnu'
        elif ostype == 'FreeBSD':
            ostype = 'unknown-freebsd'
        elif ostype == 'DragonFly':
            ostype = 'unknown-dragonfly'
        elif ostype == 'Bitrig':
            ostype = 'unknown-bitrig'
        elif ostype == 'OpenBSD':
            ostype = 'unknown-openbsd'
        elif ostype == 'NetBSD':
            ostype = 'unknown-netbsd'
        elif ostype == 'Darwin':
            ostype = 'apple-darwin'
        elif ostype.startswith('MINGW'):
            # msys' `uname` does not print gcc configuration, but prints msys
            # configuration. so we cannot believe `uname -m`:
            # msys1 is always i686 and msys2 is always x86_64.
            # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
            # MINGW64 on x86_64.
            ostype = 'pc-windows-gnu'
            cputype = 'i686'
            if os.environ.get('MSYSTEM') == 'MINGW64':
                cputype = 'x86_64'
        elif ostype.startswith('MSYS'):
            ostype = 'pc-windows-gnu'
        elif ostype.startswith('CYGWIN_NT'):
            cputype = 'i686'
            if ostype.endswith('WOW64'):
                cputype = 'x86_64'
            ostype = 'pc-windows-gnu'
        else:
            err = "unknown OS type: " + ostype
            if self.verbose:
                raise ValueError(err)
            sys.exit(err)

        if cputype in {'i386', 'i486', 'i686', 'i786', 'x86'}:
            cputype = 'i686'
        elif cputype in {'xscale', 'arm'}:
            cputype = 'arm'
        elif cputype == 'armv7l':
            cputype = 'arm'
            ostype += 'eabihf'
        elif cputype == 'aarch64':
            cputype = 'aarch64'
        elif cputype in {'powerpc', 'ppc', 'ppc64'}:
            cputype = 'powerpc'
        elif cputype in {'amd64', 'x86_64', 'x86-64', 'x64'}:
            cputype = 'x86_64'
        else:
            err = "unknown cpu type: " + cputype
            if self.verbose:
                raise ValueError(err)
            sys.exit(err)

        return cputype + '-' + ostype


def main():
    parser = argparse.ArgumentParser(description='Build rust')
    parser.add_argument('--config')
    parser.add_argument('-v', '--verbose', action='store_true')

    args = [a for a in sys.argv if a != '-h']
    args, _ = parser.parse_known_args(args)

    rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
    build_dir = os.path.join(os.getcwd(), "build")

    try:
        with open(args.config or 'config.toml') as config:
            config_toml = config.read()
        with open('config.mk') as config_mk_file:
            config_mk = config_mk_file.read()
    except OSError:
        pass

    # Configure initial bootstrap.
    rb = RustBuild(build_dir=build_dir, rust_root=rust_root,
                   config_toml=config_toml, config_mk=config_mk,
                   verbose=args.verbose)

    data = stage0_data(rb.rust_root)
    rb._rustc_channel, rb._rustc_date = data['rustc'].split('-', 1)
    rb._cargo_channel, rb._cargo_date = data['cargo'].split('-', 1)

    # Fetch/build the bootstrap.
    rb.build = rb.build_triple()
    rb.download_stage0()
    sys.stdout.flush()
    rb.build_bootstrap()
    sys.stdout.flush()

    # Run the bootstrap.
    args = [os.path.join(rb.build_dir, "bootstrap/debug/bootstrap")]
    args += ['--src', rb.rust_root, '--build', rb.build]
    args.extend(sys.argv[1:])
    env = os.environ.copy()
    env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
    rb.run(args, env)


if __name__ == '__main__':
    main()