prepare: report build failure if Cargo.toml is invalid

Author: pietroalbini

local-crates/broken-cargotoml/Cargo.toml

@@ -0,0 +1 @@
+wow! this isn't valid toml? :O

local-crates/broken-cargotoml/src/main.rs

@@ -0,0 +1,3 @@
+fn main() {
+    println!("Hello, world!");
+}

src/errors.rs

@@ -1,3 +1,5 @@
+use results::TestResult;
+
 error_chain! {
     foreign_links {
         IoError(::std::io::Error);
@@ -81,5 +83,10 @@ error_chain! {
         BadConfig {
             description("the config file contains errors")
         }
+
+        OverrideResult(result: TestResult) {
+            description("overridden task result")
+            display("overridden task result to {}", result.to_str())
+        }
     }
 }

src/run.rs

@@ -61,6 +61,7 @@ pub(crate) struct RunCommand {
     quiet: bool,
     enable_timeout: bool,
     local_rustup: bool,
+    hide_output: bool,
 }
 
 impl RunCommand {
@@ -73,6 +74,7 @@ impl RunCommand {
             quiet: false,
             enable_timeout: true,
             local_rustup: false,
+            hide_output: false,
         })
     }
 
@@ -110,6 +112,11 @@ impl RunCommand {
         self
     }
 
+    pub(crate) fn hide_output(mut self, hide_output: bool) -> Self {
+        self.hide_output = hide_output;
+        self
+    }
+
     pub(crate) fn sandboxed(self) -> SandboxedCommand {
         SandboxedCommand::new(self)
     }
@@ -154,7 +161,13 @@ impl RunCommand {
         }
 
         info!("running `{}`", cmdstr);
-        let out = log_command(cmd, capture, self.quiet, self.enable_timeout).map_err(|e| {
+        let out = log_command(
+            cmd,
+            capture,
+            self.quiet,
+            self.enable_timeout,
+            self.hide_output,
+        ).map_err(|e| {
             info!("error running command: {}", e);
             e
         })?;
@@ -255,6 +268,7 @@ fn log_command(
     capture: bool,
     quiet: bool,
     enable_timeout: bool,
+    hide_output: bool,
 ) -> Result<ProcessOutput> {
     let (max_timeout, heartbeat_timeout) = if enable_timeout {
         let max_timeout = Duration::from_secs(MAX_TIMEOUT_SECS);
@@ -290,14 +304,18 @@ fn log_command(
     let stdout = lines(BufReader::new(stdout)).map({
         let logger = logger.clone();
         move |line| {
-            slog_info!(logger, "blam! {}", line);
+            if !hide_output {
+                slog_info!(logger, "blam! {}", line);
+            }
             line
         }
     });
     let stderr = lines(BufReader::new(stderr)).map({
         let logger = logger.clone();
         move |line| {
-            slog_info!(logger, "kablam! {}", line);
+            if !hide_output {
+                slog_info!(logger, "kablam! {}", line);
+            }
             line
         }
     });

src/runner/mod.rs

@@ -79,6 +79,8 @@ fn run_ex_inner<DB: WriteResults + Sync>(
 
                                 let result = if config.is_broken(&task.krate) {
                                     TestResult::BuildFail
+                                } else if let ErrorKind::OverrideResult(res) = e.kind() {
+                                    *res
                                 } else {
                                     TestResult::Error
                                 };

src/runner/prepare.rs

@@ -3,7 +3,7 @@ use crates::Crate;
 use dirs::{EXPERIMENT_DIR, TEST_SOURCE_DIR};
 use errors::*;
 use experiments::Experiment;
-use results::WriteResults;
+use results::{TestResult, WriteResults};
 use run::RunCommand;
 use runner::toml_frobber::TomlFrobber;
 use std::fs;
@@ -92,6 +92,21 @@ pub(super) fn with_frobbed_toml(ex: &Experiment, krate: &Crate, path: &Path) ->
     Ok(())
 }
 
+pub(super) fn validate_manifest(ex: &Experiment, krate: &Crate, tc: &Toolchain) -> Result<()> {
+    info!("validating manifest of {} on toolchain {}", krate, tc);
+    with_work_crate(ex, tc, krate, |path| {
+        RunCommand::new(CARGO.toolchain(tc))
+            .args(&["read-manifest", "--manifest-path", "Cargo.toml"])
+            .cd(path)
+            .hide_output(true)
+            .run()
+            .chain_err(|| format!("invalid syntax in {}'s Cargo.toml", krate))
+            .chain_err(|| ErrorKind::OverrideResult(TestResult::BuildFail))?;
+
+        Ok(())
+    })
+}
+
 fn lockfile_dir(ex_name: &str) -> PathBuf {
     EXPERIMENT_DIR.join(ex_name).join("lockfiles")
 }

src/runner/tasks.rs

@@ -131,6 +131,7 @@ impl Task {
         }
 
         ::runner::prepare::frob_toml(ex, &self.krate)?;
+        ::runner::prepare::validate_manifest(ex, &self.krate, &MAIN_TOOLCHAIN)?;
         ::runner::prepare::capture_lockfile(config, ex, &self.krate, &MAIN_TOOLCHAIN)?;
         ::runner::prepare::fetch_crate_deps(config, ex, &self.krate, &MAIN_TOOLCHAIN)?;
 

tests/minicrater/full/results.expected.json

@@ -30,6 +30,21 @@
       ],
       "url": "https://github.com/rust-lang-nursery/crater/tree/master/local-crates/beta-regression"
     },
+    {
+      "name": "broken-cargotoml (local)",
+      "res": "SameBuildFail",
+      "runs": [
+        {
+          "log": "stable/local/broken-cargotoml",
+          "res": "BuildFail"
+        },
+        {
+          "log": "beta/local/broken-cargotoml",
+          "res": "BuildFail"
+        }
+      ],
+      "url": "https://github.com/rust-lang-nursery/crater/tree/master/local-crates/broken-cargotoml"
+    },
     {
       "name": "build-fail (local)",
       "res": "SameBuildFail",