Skip to content

Update Cargo, build curl/OpenSSL statically via features #54919

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 21, 2018
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions config.toml.example
Original file line number Diff line number Diff line change
@@ -182,10 +182,9 @@
# Build the profiler runtime
#profiler = false

# Indicates whether the OpenSSL linked into Cargo will be statically linked or
# not. If static linkage is specified then the build system will download a
# known-good version of OpenSSL, compile it, and link it to Cargo.
#openssl-static = false
# Indicates whether the native libraries linked into Cargo will be statically
# linked or not.
#cargo-native-static = false

# Run the build with low priority, by setting the process group's "nice" value
# to +10 on Unix platforms, and by using a "low priority" job object on Windows.
380 changes: 193 additions & 187 deletions src/Cargo.lock

Large diffs are not rendered by default.

3 changes: 2 additions & 1 deletion src/bootstrap/check.rs
Original file line number Diff line number Diff line change
@@ -224,7 +224,8 @@ impl Step for Rustdoc {
target,
"check",
"src/tools/rustdoc",
SourceType::InTree);
SourceType::InTree,
&[]);

let _folder = builder.fold_output(|| format!("stage{}-rustdoc", compiler.stage));
println!("Checking rustdoc artifacts ({} -> {})", &compiler.host, target);
6 changes: 3 additions & 3 deletions src/bootstrap/config.rs
Original file line number Diff line number Diff line change
@@ -149,7 +149,7 @@ pub struct Config {
pub nodejs: Option<PathBuf>,
pub gdb: Option<PathBuf>,
pub python: Option<PathBuf>,
pub openssl_static: bool,
pub cargo_native_static: bool,
pub configure_args: Vec<String>,

// These are either the stage0 downloaded binaries or the locally installed ones.
@@ -221,7 +221,7 @@ struct Build {
verbose: Option<usize>,
sanitizers: Option<bool>,
profiler: Option<bool>,
openssl_static: Option<bool>,
cargo_native_static: Option<bool>,
configure_args: Option<Vec<String>>,
local_rebuild: Option<bool>,
print_step_timings: Option<bool>,
@@ -474,7 +474,7 @@ impl Config {
set(&mut config.verbose, build.verbose);
set(&mut config.sanitizers, build.sanitizers);
set(&mut config.profiler, build.profiler);
set(&mut config.openssl_static, build.openssl_static);
set(&mut config.cargo_native_static, build.cargo_native_static);
set(&mut config.configure_args, build.configure_args);
set(&mut config.local_rebuild, build.local_rebuild);
set(&mut config.print_step_timings, build.print_step_timings);
2 changes: 1 addition & 1 deletion src/bootstrap/configure.py
Original file line number Diff line number Diff line change
@@ -64,7 +64,7 @@ def v(*args):
o("vendor", "build.vendor", "enable usage of vendored Rust crates")
o("sanitizers", "build.sanitizers", "build the sanitizer runtimes (asan, lsan, msan, tsan)")
o("dist-src", "rust.dist-src", "when building tarballs enables building a source tarball")
o("cargo-openssl-static", "build.openssl-static", "static openssl in cargo")
o("cargo-native-static", "build.cargo-native-static", "static native libraries in cargo")
o("profiler", "build.profiler", "build the profiler runtime")
o("emscripten", None, "compile the emscripten backend as well as LLVM")
o("full-tools", None, "enable all tools")
7 changes: 0 additions & 7 deletions src/bootstrap/dist.rs
Original file line number Diff line number Diff line change
@@ -31,7 +31,6 @@ use channel;
use util::{libdir, is_dylib, exe};
use builder::{Builder, RunConfig, ShouldRun, Step};
use compile;
use native;
use tool::{self, Tool};
use cache::{INTERNER, Interned};
use time;
@@ -990,12 +989,6 @@ impl Step for PlainSourceTarball {
.arg("--debug")
.arg("--vers").arg(CARGO_VENDOR_VERSION)
.arg("cargo-vendor");
if let Some(dir) = builder.openssl_install_dir(builder.config.build) {
builder.ensure(native::Openssl {
target: builder.config.build,
});
cmd.env("OPENSSL_DIR", dir);
}
builder.run(&mut cmd);
}

1 change: 1 addition & 0 deletions src/bootstrap/doc.rs
Original file line number Diff line number Diff line change
@@ -805,6 +805,7 @@ impl Step for Rustdoc {
"doc",
"src/tools/rustdoc",
SourceType::InTree,
&[]
);

cargo.env("RUSTDOCFLAGS", "--document-private-items");
19 changes: 0 additions & 19 deletions src/bootstrap/lib.rs
Original file line number Diff line number Diff line change
@@ -926,25 +926,6 @@ impl Build {
(self.hosts.iter().any(|h| *h == target) || target == self.build)
}

/// Returns the directory that OpenSSL artifacts are compiled into if
/// configured to do so.
fn openssl_dir(&self, target: Interned<String>) -> Option<PathBuf> {
// OpenSSL not used on Windows
if target.contains("windows") {
None
} else if self.config.openssl_static {
Some(self.out.join(&*target).join("openssl"))
} else {
None
}
}

/// Returns the directory that OpenSSL artifacts are installed into if
/// configured as such.
fn openssl_install_dir(&self, target: Interned<String>) -> Option<PathBuf> {
self.openssl_dir(target).map(|p| p.join("install"))
}

/// Given `num` in the form "a.b.c" return a "release string" which
/// describes the release version number.
///
186 changes: 0 additions & 186 deletions src/bootstrap/native.rs
Original file line number Diff line number Diff line change
@@ -531,189 +531,3 @@ impl Step for TestHelpers {
.compile("rust_test_helpers");
}
}

const OPENSSL_VERS: &'static str = "1.0.2n";
const OPENSSL_SHA256: &'static str =
"370babb75f278c39e0c50e8c4e7493bc0f18db6867478341a832a982fd15a8fe";

#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
pub struct Openssl {
pub target: Interned<String>,
}

impl Step for Openssl {
type Output = ();

fn should_run(run: ShouldRun) -> ShouldRun {
run.never()
}

fn run(self, builder: &Builder) {
if builder.config.dry_run {
return;
}
let target = self.target;
let out = match builder.openssl_dir(target) {
Some(dir) => dir,
None => return,
};

let stamp = out.join(".stamp");
let mut contents = String::new();
drop(File::open(&stamp).and_then(|mut f| f.read_to_string(&mut contents)));
if contents == OPENSSL_VERS {
return
}
t!(fs::create_dir_all(&out));

let name = format!("openssl-{}.tar.gz", OPENSSL_VERS);
let tarball = out.join(&name);
if !tarball.exists() {
let tmp = tarball.with_extension("tmp");
// originally from https://www.openssl.org/source/...
let url = format!("https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/{}",
name);
let mut last_error = None;
for _ in 0..3 {
let status = Command::new("curl")
.arg("-o").arg(&tmp)
.arg("-f") // make curl fail if the URL does not return HTTP 200
.arg(&url)
.status()
.expect("failed to spawn curl");

// Retry if download failed.
if !status.success() {
last_error = Some(status.to_string());
continue;
}

// Ensure the hash is correct.
let mut shasum = if target.contains("apple") ||
builder.config.build.contains("netbsd") {
let mut cmd = Command::new("shasum");
cmd.arg("-a").arg("256");
cmd
} else {
Command::new("sha256sum")
};
let output = output(&mut shasum.arg(&tmp));
let found = output.split_whitespace().next().unwrap();

// If the hash is wrong, probably the download is incomplete or S3 served an error
// page. In any case, retry.
if found != OPENSSL_SHA256 {
last_error = Some(format!(
"downloaded openssl sha256 different\n\
expected: {}\n\
found: {}\n",
OPENSSL_SHA256,
found
));
continue;
}

// Everything is fine, so exit the retry loop.
last_error = None;
break;
}
if let Some(error) = last_error {
panic!("failed to download openssl source: {}", error);
}
t!(fs::rename(&tmp, &tarball));
}
let obj = out.join(format!("openssl-{}", OPENSSL_VERS));
let dst = builder.openssl_install_dir(target).unwrap();
drop(fs::remove_dir_all(&obj));
drop(fs::remove_dir_all(&dst));
builder.run(Command::new("tar").arg("zxf").arg(&tarball).current_dir(&out));

let mut configure = Command::new("perl");
configure.arg(obj.join("Configure"));
configure.arg(format!("--prefix={}", dst.display()));
configure.arg("no-dso");
configure.arg("no-ssl2");
configure.arg("no-ssl3");

let os = match &*target {
"aarch64-linux-android" => "linux-aarch64",
"aarch64-unknown-linux-gnu" => "linux-aarch64",
"aarch64-unknown-linux-musl" => "linux-aarch64",
"aarch64-unknown-netbsd" => "BSD-generic64",
"arm-linux-androideabi" => "android",
"arm-unknown-linux-gnueabi" => "linux-armv4",
"arm-unknown-linux-gnueabihf" => "linux-armv4",
"armv6-unknown-netbsd-eabihf" => "BSD-generic32",
"armv7-linux-androideabi" => "android-armv7",
"armv7-unknown-linux-gnueabihf" => "linux-armv4",
"armv7-unknown-netbsd-eabihf" => "BSD-generic32",
"i586-unknown-linux-gnu" => "linux-elf",
"i586-unknown-linux-musl" => "linux-elf",
"i686-apple-darwin" => "darwin-i386-cc",
"i686-linux-android" => "android-x86",
"i686-unknown-freebsd" => "BSD-x86-elf",
"i686-unknown-linux-gnu" => "linux-elf",
"i686-unknown-linux-musl" => "linux-elf",
"i686-unknown-netbsd" => "BSD-x86-elf",
"mips-unknown-linux-gnu" => "linux-mips32",
"mips64-unknown-linux-gnuabi64" => "linux64-mips64",
"mips64el-unknown-linux-gnuabi64" => "linux64-mips64",
"mipsel-unknown-linux-gnu" => "linux-mips32",
"powerpc-unknown-linux-gnu" => "linux-ppc",
"powerpc-unknown-linux-gnuspe" => "linux-ppc",
"powerpc-unknown-netbsd" => "BSD-generic32",
"powerpc64-unknown-linux-gnu" => "linux-ppc64",
"powerpc64le-unknown-linux-gnu" => "linux-ppc64le",
"powerpc64le-unknown-linux-musl" => "linux-ppc64le",
"s390x-unknown-linux-gnu" => "linux64-s390x",
"sparc-unknown-linux-gnu" => "linux-sparcv9",
"sparc64-unknown-linux-gnu" => "linux64-sparcv9",
"sparc64-unknown-netbsd" => "BSD-sparc64",
"x86_64-apple-darwin" => "darwin64-x86_64-cc",
"x86_64-linux-android" => "linux-x86_64",
"x86_64-unknown-freebsd" => "BSD-x86_64",
"x86_64-unknown-dragonfly" => "BSD-x86_64",
"x86_64-unknown-linux-gnu" => "linux-x86_64",
"x86_64-unknown-linux-gnux32" => "linux-x32",
"x86_64-unknown-linux-musl" => "linux-x86_64",
"x86_64-unknown-netbsd" => "BSD-x86_64",
_ => panic!("don't know how to configure OpenSSL for {}", target),
};
configure.arg(os);
configure.env("CC", builder.cc(target));
for flag in builder.cflags(target, GitRepo::Rustc) {
configure.arg(flag);
}
// There is no specific os target for android aarch64 or x86_64,
// so we need to pass some extra cflags
if target == "aarch64-linux-android" || target == "x86_64-linux-android" {
configure.arg("-mandroid");
configure.arg("-fomit-frame-pointer");
}
if target == "sparc64-unknown-netbsd" {
// Need -m64 to get assembly generated correctly for sparc64.
configure.arg("-m64");
if builder.config.build.contains("netbsd") {
// Disable sparc64 asm on NetBSD builders, it uses
// m4(1)'s -B flag, which NetBSD m4 does not support.
configure.arg("no-asm");
}
}
// Make PIE binaries
// Non-PIE linker support was removed in Lollipop
// https://source.android.com/security/enhancements/enhancements50
if target == "i686-linux-android" {
configure.arg("no-asm");
}
configure.current_dir(&obj);
builder.info(&format!("Configuring openssl for {}", target));
builder.run_quiet(&mut configure);
builder.info(&format!("Building openssl for {}", target));
builder.run_quiet(Command::new("make").arg("-j1").current_dir(&obj));
builder.info(&format!("Installing openssl for {}", target));
builder.run_quiet(Command::new("make").arg("install").arg("-j1").current_dir(&obj));

let mut f = t!(File::create(&stamp));
t!(f.write_all(OPENSSL_VERS.as_bytes()));
}
}
18 changes: 12 additions & 6 deletions src/bootstrap/test.rs
Original file line number Diff line number Diff line change
@@ -228,7 +228,8 @@ impl Step for Cargo {
self.host,
"test",
"src/tools/cargo",
SourceType::Submodule);
SourceType::Submodule,
&[]);

if !builder.fail_fast {
cargo.arg("--no-fail-fast");
@@ -290,7 +291,8 @@ impl Step for Rls {
host,
"test",
"src/tools/rls",
SourceType::Submodule);
SourceType::Submodule,
&[]);

// Copy `src/tools/rls/test_data` to a writable drive.
let test_workspace_path = builder.out.join("rls-test-data");
@@ -352,7 +354,8 @@ impl Step for Rustfmt {
host,
"test",
"src/tools/rustfmt",
SourceType::Submodule);
SourceType::Submodule,
&[]);

let dir = testdir(builder, compiler.host);
t!(fs::create_dir_all(&dir));
@@ -407,7 +410,8 @@ impl Step for Miri {
host,
"test",
"src/tools/miri",
SourceType::Submodule);
SourceType::Submodule,
&[]);

// miri tests need to know about the stage sysroot
cargo.env("MIRI_SYSROOT", builder.sysroot(compiler));
@@ -466,7 +470,8 @@ impl Step for Clippy {
host,
"test",
"src/tools/clippy",
SourceType::Submodule);
SourceType::Submodule,
&[]);

// clippy tests need to know about the stage sysroot
cargo.env("SYSROOT", builder.sysroot(compiler));
@@ -1777,7 +1782,8 @@ impl Step for CrateRustdoc {
target,
test_kind.subcommand(),
"src/tools/rustdoc",
SourceType::InTree);
SourceType::InTree,
&[]);
if test_kind.subcommand() == "test" && !builder.fail_fast {
cargo.arg("--no-fail-fast");
}
27 changes: 16 additions & 11 deletions src/bootstrap/tool.rs
Original file line number Diff line number Diff line change
@@ -80,8 +80,8 @@ impl Step for ToolBuild {
"build",
path,
self.source_type,
&self.extra_features,
);
cargo.arg("--features").arg(self.extra_features.join(" "));

let _folder = builder.fold_output(|| format!("stage{}-{}", compiler.stage, tool));
builder.info(&format!("Building stage{} tool {} ({})", compiler.stage, tool, target));
@@ -208,6 +208,7 @@ pub fn prepare_tool_cargo(
command: &'static str,
path: &'static str,
source_type: SourceType,
extra_features: &[String],
) -> Command {
let mut cargo = builder.cargo(compiler, mode, target, command);
let dir = builder.src.join(path);
@@ -221,10 +222,16 @@ pub fn prepare_tool_cargo(
cargo.env("RUSTC_EXTERNAL_TOOL", "1");
}

if let Some(dir) = builder.openssl_install_dir(target) {
cargo.env("OPENSSL_STATIC", "1");
cargo.env("OPENSSL_DIR", dir);
cargo.env("LIBZ_SYS_STATIC", "1");
let mut features = extra_features.iter().cloned().collect::<Vec<_>>();
if builder.build.config.cargo_native_static {
if path.ends_with("cargo") ||
path.ends_with("rls") ||
path.ends_with("clippy") ||
path.ends_with("rustfmt")
{
cargo.env("LIBZ_SYS_STATIC", "1");
features.push("rustc-workspace-hack/all-static".to_string());
}
}

// if tools are using lzma we want to force the build script to build its
@@ -244,6 +251,9 @@ pub fn prepare_tool_cargo(
if let Some(date) = info.commit_date() {
cargo.env("CFG_COMMIT_DATE", date);
}
if features.len() > 0 {
cargo.arg("--features").arg(&features.join(", "));
}
cargo
}

@@ -439,6 +449,7 @@ impl Step for Rustdoc {
"build",
"src/tools/rustdoc",
SourceType::InTree,
&[],
);

// Most tools don't get debuginfo, but rustdoc should.
@@ -495,9 +506,6 @@ impl Step for Cargo {
}

fn run(self, builder: &Builder) -> PathBuf {
builder.ensure(native::Openssl {
target: self.target,
});
// Cargo depends on procedural macros, which requires a full host
// compiler to be available, so we need to depend on that.
builder.ensure(compile::Rustc {
@@ -597,9 +605,6 @@ tool_extended!((self, builder),
if clippy.is_some() {
self.extra_features.push("clippy".to_owned());
}
builder.ensure(native::Openssl {
target: self.target,
});
// RLS depends on procedural macros, which requires a full host
// compiler to be available, so we need to depend on that.
builder.ensure(compile::Rustc {
5 changes: 5 additions & 0 deletions src/ci/docker/dist-i686-linux/Dockerfile
Original file line number Diff line number Diff line change
@@ -83,6 +83,11 @@ RUN ./build-git.sh
COPY dist-x86_64-linux/build-headers.sh /tmp/
RUN ./build-headers.sh

# OpenSSL requires a more recent version of perl
# with so we install newer ones here
COPY dist-x86_64-linux/build-perl.sh /tmp/
RUN ./build-perl.sh

COPY scripts/sccache.sh /scripts/
RUN sh /scripts/sccache.sh

5 changes: 5 additions & 0 deletions src/ci/docker/dist-x86_64-linux/Dockerfile
Original file line number Diff line number Diff line change
@@ -83,6 +83,11 @@ RUN ./build-git.sh
COPY dist-x86_64-linux/build-headers.sh /tmp/
RUN ./build-headers.sh

# OpenSSL requires a more recent version of perl
# with so we install newer ones here
COPY dist-x86_64-linux/build-perl.sh /tmp/
RUN ./build-perl.sh

COPY scripts/sccache.sh /scripts/
RUN sh /scripts/sccache.sh

29 changes: 29 additions & 0 deletions src/ci/docker/dist-x86_64-linux/build-perl.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
#!/usr/bin/env bash
# Copyright 2018 The Rust Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution and at
# http://rust-lang.org/COPYRIGHT.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.

set -ex
source shared.sh

curl https://www.cpan.org/src/5.0/perl-5.28.0.tar.gz | \
tar xzf -

cd perl-5.28.0

# Gotta do some hackery to tell python about our custom OpenSSL build, but other
# than that fairly normal.
CC=gcc \
CFLAGS='-I /rustroot/include' LDFLAGS='-L /rustroot/lib -L /rustroot/lib64' \
hide_output ./configure.gnu
hide_output make -j10
hide_output make install

cd ..
rm -rf perl-5.28.0
2 changes: 1 addition & 1 deletion src/ci/run.sh
Original file line number Diff line number Diff line change
@@ -39,7 +39,7 @@ fi
RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-sccache"
RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --disable-manage-submodules"
RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-locked-deps"
RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-cargo-openssl-static"
RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-cargo-native-static"

if [ "$DIST_SRC" = "" ]; then
RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --disable-dist-src"
2 changes: 1 addition & 1 deletion src/tools/rls
Submodule rls updated from 440a98 to 1c755e
17 changes: 11 additions & 6 deletions src/tools/rustc-workspace-hack/Cargo.toml
Original file line number Diff line number Diff line change
@@ -39,6 +39,7 @@ features = [
"schannel",
"securitybaseapi",
"shellapi",
"sspi",
"synchapi",
"sysinfoapi",
"timezoneapi",
@@ -48,10 +49,14 @@ features = [
"wincrypt",
]

[dependencies.serde_json]
version = "1.0.31"
features = ["raw_value"]
[dependencies]
serde_json = { version = "1.0.31", features = ["raw_value"] }
rand = { version = "0.5.5", features = ["i128_support"] }
curl-sys = { version = "0.4.13", optional = true }

[dependencies.rand]
version = "0.5.5"
features = ["i128_support"]
[target.'cfg(not(windows))'.dependencies]
openssl = { version = "0.10.12", optional = true }


[features]
all-static = ['openssl/vendored', 'curl-sys/static-curl']
2 changes: 1 addition & 1 deletion src/tools/rustfmt