Merge remote-tracking branch 'origin/master' into cargo-miri-test

Author: RalfJung

.travis.yml

@@ -12,49 +12,22 @@ os:
 dist: xenial
 
 before_script:
-# install extra stuff for cross-compilation
+# Linux: install extra stuff for cross-compilation
 - if [[ "$TRAVIS_OS_NAME" == linux ]]; then sudo apt update && sudo apt install gcc-multilib; fi
-# macOS weirdness (https://github.com/travis-ci/travis-ci/issues/6307, https://github.com/travis-ci/travis-ci/issues/10165)
-- if [[ "$TRAVIS_OS_NAME" == osx ]]; then rvm get stable; fi
 # Compute the rust version we use. We do not use "language: rust" to have more control here.
 - |
   if [[ "$TRAVIS_EVENT_TYPE" == cron ]]; then
     RUST_TOOLCHAIN=nightly
   else
     RUST_TOOLCHAIN=$(cat rust-version)
   fi
-- |
-  if [ "$TRAVIS_OS_NAME" == osx ]; then
-    export MIRI_SYSROOT_BASE=~/Library/Caches/miri.miri.miri/
-  else
-    export MIRI_SYSROOT_BASE=~/.cache/miri/
-  fi
 # install Rust
 - curl https://build.travis-ci.org/files/rustup-init.sh -sSf | sh -s -- -y --default-toolchain "$RUST_TOOLCHAIN"
 - export PATH=$HOME/.cargo/bin:$PATH
 - rustc --version
 
 script:
-- set -e
-- |
-  # Build and install miri
-  cargo build --release --all-features --all-targets &&
-  cargo install --all-features --force --path .
-- |
-  # Get ourselves a MIR-full libstd for the host and a foreign architecture
-  cargo miri setup &&
-  if [[ "$TRAVIS_OS_NAME" == osx ]]; then
-    cargo miri setup --target i686-apple-darwin
-  else
-    cargo miri setup --target i686-unknown-linux-gnu
-  fi
-- |
-  # Test miri with full MIR, on the host and other architectures
-  MIRI_SYSROOT=$MIRI_SYSROOT_BASE/HOST cargo test --release --all-features &&
-  MIRI_SYSROOT=$MIRI_SYSROOT_BASE cargo test --release --all-features
-- |
-  # Test cargo integration
-  (cd test-cargo-miri && MIRI_SYSROOT=$MIRI_SYSROOT_BASE/HOST ./run-test.py)
+- ./travis.sh
 
 notifications:
   email:

Cargo.toml

@@ -39,6 +39,10 @@ directories = { version = "1.0", optional = true }
 rustc_version = { version = "0.2.3", optional = true }
 env_logger = "0.5"
 log = "0.4"
+# A noop dependency that changes in the Rust repository, it's a bit of a hack.
+# See the `src/tools/rustc-workspace-hack/README.md` file in `rust-lang/rust`
+# for more information.
+rustc-workspace-hack = "1.0.0"
 
 [build-dependencies]
 vergen = "3"

README.md

@@ -154,9 +154,9 @@ MIRI_LOG=rustc_mir::interpret=debug,miri::stacked_borrows cargo run tests/run-pa
 In addition, you can set `MIRI_BACKTRACE=1` to get a backtrace of where an
 evaluation error was originally created.
 
-### Miri `-Z` flags
+### Miri `-Z` flags and environment variables
 
-Several `-Z` flags are relevant for miri:
+Several `-Z` flags are relevant for Miri:
 
 * `-Zmir-opt-level` controls how many MIR optimizations are performed.  miri
   overrides the default to be `0`; be advised that using any higher level can
@@ -168,6 +168,14 @@ Several `-Z` flags are relevant for miri:
   enforcing the validity invariant, which is enforced by default.  This is
   mostly useful for debugging; it means miri will miss bugs in your program.
 
+Moreover, Miri recognizes some environment variables:
+
+* `MIRI_SYSROOT` (recognized by `miri`, `cargo miri` and the test suite)
+  indicates the sysroot to use.
+* `MIRI_TARGET` (recognized by the test suite) indicates which target
+  architecture to test against.  `miri` and `cargo miri` accept the `--target`
+  flag for the same purpose.
+
 ## Contributing and getting help
 
 Check out the issues on this GitHub repository for some ideas. There's lots that

rust-version

@@ -1 +1 @@
-nightly-2018-12-08
+nightly-2018-12-14

src/bin/cargo-miri.rs

@@ -3,7 +3,7 @@
 extern crate cargo_metadata;
 
 use std::path::{PathBuf, Path};
-use std::io::{self, Write};
+use std::io::{self, Write, BufRead};
 use std::process::Command;
 use std::fs::{self, File};
 
@@ -114,6 +114,36 @@ fn list_targets() -> impl Iterator<Item=cargo_metadata::Target> {
     package.targets.into_iter()
 }
 
+fn xargo_version() -> Option<(u32, u32, u32)> {
+    let out = Command::new("xargo").arg("--version").output().ok()?;
+    if !out.status.success() {
+        return None;
+    }
+    // Parse output. The first line looks like "xargo 0.3.12 (b004f1c 2018-12-13)".
+    let line = out.stderr.lines().nth(0)
+        .expect("malformed `xargo --version` output: not at least one line")
+        .expect("malformed `xargo --version` output: error reading first line");
+    let version = line.split(' ').nth(1)
+        .expect("malformed `xargo --version` output: not at least two words");
+    let mut version_pieces = version.split('.');
+    let major = version_pieces.next()
+        .expect("malformed `xargo --version` output: not a major version piece")
+        .parse()
+        .expect("malformed `xargo --version` output: major version is not an integer");
+    let minor = version_pieces.next()
+        .expect("malformed `xargo --version` output: not a minor version piece")
+        .parse()
+        .expect("malformed `xargo --version` output: minor version is not an integer");
+    let patch = version_pieces.next()
+        .expect("malformed `xargo --version` output: not a patch version piece")
+        .parse()
+        .expect("malformed `xargo --version` output: patch version is not an integer");
+    if !version_pieces.next().is_none() {
+        panic!("malformed `xargo --version` output: more than three pieces in version");
+    }
+    Some((major, minor, patch))
+}
+
 fn ask(question: &str) {
     let mut buf = String::new();
     print!("{} [Y/n] ", question);
@@ -134,14 +164,15 @@ fn setup(ask_user: bool) {
     }
 
     // First, we need xargo
-    if Command::new("xargo").arg("--version").output().is_err()
-    {
+    let xargo = xargo_version();
+    if xargo.map_or(true, |v| v < (0, 3, 13)) {
         if ask_user {
-            ask("It seems you do not have xargo installed. I will run `cargo install xargo`. Proceed?");
+            ask("It seems you do not have a recent enough xargo installed. I will run `cargo install xargo -f`. Proceed?");
         } else {
-            println!("Installing xargo: `cargo install xargo`");
+            println!("Installing xargo: `cargo install xargo -f`");
         }
-        if !Command::new("cargo").args(&["install", "xargo"]).status().unwrap().success() {
+        // FIXME: Go back to using releases, once a 0.3.13 got released.
+        if !Command::new("cargo").args(&["install", "xargo", "-f", "--git", "https://github.com/japaric/xargo"]).status().unwrap().success() {
             show_error(format!("Failed to install xargo"));
         }
     }
@@ -304,8 +335,8 @@ fn main() {
                 _ => {}
             }
         }
-    } else {
-        // This arm is executed when cargo-miri runs `cargo rustc` with the `RUSTC` env var set to itself:
+    } else if let Some("rustc") = std::env::args().nth(1).as_ref().map(AsRef::as_ref) {
+        // This arm is executed when cargo-miri runs `cargo rustc` with the `RUSTC_WRAPPER` env var set to itself:
         // Dependencies get dispatched to rustc, the final test/binary to miri.
 
         let home = option_env!("RUSTUP_HOME").or(option_env!("MULTIRUST_HOME"));
@@ -331,11 +362,11 @@ fn main() {
 
         // this conditional check for the --sysroot flag is there so users can call `cargo-miri` directly
         // without having to pass --sysroot or anything
+        let rustc_args = std::env::args().skip(2);
         let mut args: Vec<String> = if std::env::args().any(|s| s == "--sysroot") {
-            std::env::args().skip(1).collect()
+            rustc_args.collect()
         } else {
-            std::env::args()
-                .skip(1)
+            rustc_args
                 .chain(Some("--sysroot".to_owned()))
                 .chain(Some(sys_root))
                 .collect()
@@ -364,6 +395,8 @@ fn main() {
             Err(ref e) if miri_enabled => panic!("error during miri run: {:?}", e),
             Err(ref e) => panic!("error during rustc call: {:?}", e),
         }
+    } else {
+        show_error(format!("Must be called with either `miri` or `rustc` as first argument."))
     }
 }
 
@@ -388,7 +421,7 @@ where
     let path = std::env::current_exe().expect("current executable path invalid");
     let exit_status = Command::new("cargo")
         .args(&args)
-        .env("RUSTC", path)
+        .env("RUSTC_WRAPPER", path)
         .spawn()
         .expect("could not run cargo")
         .wait()

tests/compiletest.rs

@@ -3,7 +3,7 @@
 #![test_runner(test_runner)]
 
 use std::slice::SliceConcatExt;
-use std::path::{PathBuf, Path};
+use std::path::PathBuf;
 use std::env;
 
 use compiletest_rs as compiletest;
@@ -34,10 +34,11 @@ fn mk_config(mode: &str) -> compiletest::common::ConfigWithTemp {
         config.compile_lib_path = rustc_lib_path();
     }
     config.filter = env::args().nth(1);
+    config.host = get_host();
     config
 }
 
-fn compile_fail(sysroot: &Path, path: &str, target: &str, host: &str, opt: bool) {
+fn compile_fail(path: &str, target: &str, opt: bool) {
     let opt_str = if opt { " with optimizations" } else { "" };
     eprintln!("{}", format!(
         "## Running compile-fail tests in {} against miri for target {}{}",
@@ -47,7 +48,6 @@ fn compile_fail(sysroot: &Path, path: &str, target: &str, host: &str, opt: bool)
     ).green().bold());
 
     let mut flags = Vec::new();
-    flags.push(format!("--sysroot {}", sysroot.display()));
     flags.push("-Dwarnings -Dunused".to_owned()); // overwrite the -Aunused in compiletest-rs
     flags.push("--edition 2018".to_owned());
     if opt {
@@ -60,12 +60,11 @@ fn compile_fail(sysroot: &Path, path: &str, target: &str, host: &str, opt: bool)
     let mut config = mk_config("compile-fail");
     config.src_base = PathBuf::from(path);
     config.target = target.to_owned();
-    config.host = host.to_owned();
     config.target_rustcflags = Some(flags.join(" "));
     compiletest::run_tests(&config);
 }
 
-fn miri_pass(sysroot: &Path, path: &str, target: &str, host: &str, opt: bool) {
+fn miri_pass(path: &str, target: &str, opt: bool) {
     let opt_str = if opt { " with optimizations" } else { "" };
     eprintln!("{}", format!(
         "## Running run-pass tests in {} against miri for target {}{}",
@@ -75,69 +74,33 @@ fn miri_pass(sysroot: &Path, path: &str, target: &str, host: &str, opt: bool) {
     ).green().bold());
 
     let mut flags = Vec::new();
-    flags.push(format!("--sysroot {}", sysroot.display()));
     flags.push("-Dwarnings -Dunused".to_owned()); // overwrite the -Aunused in compiletest-rs
     flags.push("--edition 2018".to_owned());
     if opt {
-        // FIXME: We use opt level 1 because MIR inlining defeats the validation
-        // whitelist.
-        flags.push("-Zmir-opt-level=1".to_owned());
+        flags.push("-Zmir-opt-level=3".to_owned());
     }
 
     let mut config = mk_config("ui");
     config.src_base = PathBuf::from(path);
     config.target = target.to_owned();
-    config.host = host.to_owned();
     config.target_rustcflags = Some(flags.join(" "));
     compiletest::run_tests(&config);
 }
 
-fn is_target_dir<P: Into<PathBuf>>(path: P) -> bool {
-    let mut path = path.into();
-    path.push("lib");
-    path.metadata().map(|m| m.is_dir()).unwrap_or(false)
-}
-
-fn target_has_std<P: Into<PathBuf>>(path: P) -> bool {
-    let mut path = path.into();
-    path.push("lib");
-    std::fs::read_dir(path)
-        .expect("invalid target")
-        .map(|entry| entry.unwrap())
-        .filter(|entry| entry.file_type().unwrap().is_file())
-        .filter_map(|entry| entry.file_name().into_string().ok())
-        .any(|file_name| file_name == "libstd.rlib")
-}
-
-
-fn for_all_targets<F: FnMut(String)>(sysroot: &Path, f: F) {
-    let target_dir = sysroot.join("lib").join("rustlib");
-    let mut targets = std::fs::read_dir(target_dir)
-        .expect("invalid sysroot")
-        .map(|entry| entry.unwrap())
-        .filter(|entry| is_target_dir(entry.path()))
-        .filter(|entry| target_has_std(entry.path()))
-        .map(|entry| entry.file_name().into_string().unwrap())
-        .peekable();
-
-    if targets.peek().is_none() {
-        panic!("No valid targets found");
+/// Make sure the MIRI_SYSROOT env var is set
+fn set_sysroot() {
+    if std::env::var("MIRI_SYSROOT").is_ok() {
+        // Nothing to do
+        return;
     }
-
-    targets.for_each(f);
-}
-
-fn get_sysroot() -> PathBuf {
-    let sysroot = std::env::var("MIRI_SYSROOT").unwrap_or_else(|_| {
-        let sysroot = std::process::Command::new("rustc")
-            .arg("--print")
-            .arg("sysroot")
-            .output()
-            .expect("rustc not found")
-            .stdout;
-        String::from_utf8(sysroot).expect("sysroot is not utf8")
-    });
-    PathBuf::from(sysroot.trim())
+    let sysroot = std::process::Command::new("rustc")
+        .arg("--print")
+        .arg("sysroot")
+        .output()
+        .expect("rustc not found")
+        .stdout;
+    let sysroot = String::from_utf8(sysroot).expect("sysroot is not utf8");
+    std::env::set_var("MIRI_SYSROOT", sysroot.trim());
 }
 
 fn get_host() -> String {
@@ -153,28 +116,20 @@ fn get_host() -> String {
     version_meta.host
 }
 
-fn run_pass_miri(opt: bool) {
-    let sysroot = get_sysroot();
-    let host = get_host();
+fn get_target() -> String {
+    std::env::var("MIRI_TARGET").unwrap_or_else(|_| get_host())
+}
 
-    for_all_targets(&sysroot, |target| {
-        miri_pass(&sysroot, "tests/run-pass", &target, &host, opt);
-    });
+fn run_pass_miri(opt: bool) {
+    miri_pass("tests/run-pass", &get_target(), opt);
 }
 
 fn compile_fail_miri(opt: bool) {
-    let sysroot = get_sysroot();
-    let host = get_host();
-
-    for_all_targets(&sysroot, |target| {
-        compile_fail(&sysroot, "tests/compile-fail", &target, &host, opt);
-    });
+    compile_fail("tests/compile-fail", &get_target(), opt);
 }
 
 fn test_runner(_tests: &[&()]) {
-    // We put everything into a single test to avoid the parallelism `cargo test`
-    // introduces.  We still get parallelism within our tests because `compiletest`
-    // uses `libtest` which runs jobs in parallel.
+    set_sysroot();
 
     run_pass_miri(false);
     run_pass_miri(true);

tests/run-pass/foreign-fn-linkname.rs

@@ -10,7 +10,7 @@
 
 //ignore-windows: Uses POSIX APIs
 
-#![feature(libc)]
+#![feature(rustc_private)]
 #![allow(unused_extern_crates)] // rustc bug https://github.com/rust-lang/rust/issues/56098
 
 extern crate libc;