Auto merge of #5301 - mshal:build-plan, r=matklad

Add --build-plan for 'cargo build'

With 'cargo build --build-plan', cargo does not actually run any
commands, but instead prints out what it would have done in the form of
a JSON data structure.

Fixes #3815

Author: bors

src/bin/cargo/command_prelude.rs

@@ -132,6 +132,10 @@ pub trait AppExt: Sized {
         )
     }
 
+    fn arg_build_plan(self) -> Self {
+        self._arg(opt("build-plan", "Output the build plan in JSON"))
+    }
+
     fn arg_new_opts(self) -> Self {
         self._arg(
             opt(
@@ -275,6 +279,12 @@ pub trait ArgMatchesExt {
         let mut build_config = BuildConfig::new(config, self.jobs()?, &self.target(), mode)?;
         build_config.message_format = message_format;
         build_config.release = self._is_present("release");
+        build_config.build_plan = self._is_present("build-plan");
+        if build_config.build_plan && !config.cli_unstable().unstable_options {
+            Err(format_err!(
+                "`--build-plan` flag is unstable, pass `-Z unstable-options` to enable it"
+            ))?;
+        };
 
         let opts = CompileOptions {
             config,

src/bin/cargo/commands/build.rs

@@ -31,6 +31,7 @@ pub fn cli() -> App {
         .arg(opt("out-dir", "Copy final artifacts to this directory").value_name("PATH"))
         .arg_manifest_path()
         .arg_message_format()
+        .arg_build_plan()
         .after_help(
             "\
 If the --package argument is given, then SPEC is a package id specification

src/cargo/core/compiler/build_config.rs

@@ -14,6 +14,8 @@ pub struct BuildConfig {
     pub mode: CompileMode,
     /// Whether to print std output in json format (for machine reading)
     pub message_format: MessageFormat,
+    /// Output a build plan to stdout instead of actually compiling.
+    pub build_plan: bool,
 }
 
 impl BuildConfig {
@@ -87,6 +89,7 @@ impl BuildConfig {
             release: false,
             mode,
             message_format: MessageFormat::Human,
+            build_plan: false,
         })
     }
 

src/cargo/core/compiler/build_context/mod.rs

@@ -229,6 +229,18 @@ impl<'a, 'cfg> BuildContext<'a, 'cfg> {
         }
         None
     }
+
+    /// Return the list of filenames read by cargo to generate the BuildContext
+    /// (all Cargo.toml, etc).
+    pub fn inputs(&self) -> CargoResult<Vec<PathBuf>> {
+        let mut inputs = Vec::new();
+        for id in self.packages.package_ids() {
+            let pkg = self.get_package(id)?;
+            inputs.push(pkg.manifest_path().to_path_buf());
+        }
+        inputs.sort();
+        Ok(inputs)
+    }
 }
 
 /// Information required to build for a target

src/cargo/core/compiler/build_plan.rs

@@ -0,0 +1,159 @@
+//! A graph-like structure used to represent the rustc commands to build the project and the
+//! interdependencies between them.
+//!
+//! The BuildPlan structure is used to store the dependency graph of a dry run so that it can be
+//! shared with an external build system. Each Invocation in the BuildPlan comprises a single
+//! subprocess and defines the build environment, the outputs produced by the subprocess, and the
+//! dependencies on other Invocations.
+
+use std::collections::BTreeMap;
+
+use core::TargetKind;
+use super::{Context, Kind, Unit};
+use super::context::OutputFile;
+use util::{internal, CargoResult, ProcessBuilder};
+use std::sync::Arc;
+use std::path::PathBuf;
+use serde_json;
+use semver;
+
+#[derive(Debug, Serialize)]
+struct Invocation {
+    package_name: String,
+    package_version: semver::Version,
+    target_kind: TargetKind,
+    kind: Kind,
+    deps: Vec<usize>,
+    outputs: Vec<PathBuf>,
+    links: BTreeMap<PathBuf, PathBuf>,
+    program: String,
+    args: Vec<String>,
+    env: BTreeMap<String, String>,
+    cwd: Option<PathBuf>,
+}
+
+#[derive(Debug)]
+pub struct BuildPlan {
+    invocation_map: BTreeMap<String, usize>,
+    plan: SerializedBuildPlan,
+}
+
+#[derive(Debug, Serialize)]
+struct SerializedBuildPlan {
+    invocations: Vec<Invocation>,
+    inputs: Vec<PathBuf>,
+}
+
+impl Invocation {
+    pub fn new(unit: &Unit, deps: Vec<usize>) -> Invocation {
+        let id = unit.pkg.package_id();
+        Invocation {
+            package_name: id.name().to_string(),
+            package_version: id.version().clone(),
+            kind: unit.kind,
+            target_kind: unit.target.kind().clone(),
+            deps: deps,
+            outputs: Vec::new(),
+            links: BTreeMap::new(),
+            program: String::new(),
+            args: Vec::new(),
+            env: BTreeMap::new(),
+            cwd: None,
+        }
+    }
+
+    pub fn add_output(&mut self, path: &PathBuf, link: &Option<PathBuf>) {
+        self.outputs.push(path.clone());
+        if let Some(ref link) = *link {
+            self.links.insert(link.clone(), path.clone());
+        }
+    }
+
+    pub fn update_cmd(&mut self, cmd: ProcessBuilder) -> CargoResult<()> {
+        self.program = cmd.get_program()
+            .to_str()
+            .ok_or_else(|| format_err!("unicode program string required"))?
+            .to_string();
+        self.cwd = Some(cmd.get_cwd().unwrap().to_path_buf());
+        for arg in cmd.get_args().iter() {
+            self.args.push(
+                arg.to_str()
+                    .ok_or_else(|| format_err!("unicode argument string required"))?
+                    .to_string(),
+            );
+        }
+        for (var, value) in cmd.get_envs() {
+            let value = match value {
+                Some(s) => s,
+                None => continue,
+            };
+            self.env.insert(
+                var.clone(),
+                value
+                    .to_str()
+                    .ok_or_else(|| format_err!("unicode environment value required"))?
+                    .to_string(),
+            );
+        }
+        Ok(())
+    }
+}
+
+impl BuildPlan {
+    pub fn new() -> BuildPlan {
+        BuildPlan {
+            invocation_map: BTreeMap::new(),
+            plan: SerializedBuildPlan::new(),
+        }
+    }
+
+    pub fn add(&mut self, cx: &Context, unit: &Unit) -> CargoResult<()> {
+        let id = self.plan.invocations.len();
+        self.invocation_map.insert(unit.buildkey(), id);
+        let deps = cx.dep_targets(&unit)
+            .iter()
+            .map(|dep| self.invocation_map[&dep.buildkey()])
+            .collect();
+        let invocation = Invocation::new(unit, deps);
+        self.plan.invocations.push(invocation);
+        Ok(())
+    }
+
+    pub fn update(
+        &mut self,
+        invocation_name: String,
+        cmd: ProcessBuilder,
+        outputs: Arc<Vec<OutputFile>>,
+    ) -> CargoResult<()> {
+        let id = self.invocation_map[&invocation_name];
+        let invocation = self.plan
+            .invocations
+            .get_mut(id)
+            .ok_or_else(|| internal(format!("couldn't find invocation for {}", invocation_name)))?;
+
+        invocation.update_cmd(cmd)?;
+        for output in outputs.iter() {
+            invocation.add_output(&output.path, &output.hardlink);
+        }
+
+        Ok(())
+    }
+
+    pub fn set_inputs(&mut self, inputs: Vec<PathBuf>) {
+        self.plan.inputs = inputs;
+    }
+
+    pub fn output_plan(self) {
+        let encoded = serde_json::to_string(&self.plan).unwrap();
+        println!("{}", encoded);
+    }
+}
+
+impl SerializedBuildPlan {
+    pub fn new() -> SerializedBuildPlan {
+        SerializedBuildPlan {
+            invocations: Vec::new(),
+            inputs: Vec::new(),
+        }
+    }
+}

src/cargo/core/compiler/context/mod.rs

@@ -3,26 +3,28 @@ use std::collections::{HashMap, HashSet};
 use std::fmt::Write;
 use std::path::PathBuf;
 use std::sync::Arc;
+use std::cmp::Ordering;
 
 use jobserver::Client;
 
 use core::{Package, PackageId, Resolve, Target};
 use core::profiles::Profile;
 use util::errors::{CargoResult, CargoResultExt};
-use util::{internal, profile, Config};
+use util::{internal, profile, Config, short_hash};
 
 use super::custom_build::{self, BuildDeps, BuildScripts, BuildState};
 use super::fingerprint::Fingerprint;
 use super::job_queue::JobQueue;
 use super::layout::Layout;
 use super::{BuildContext, Compilation, CompileMode, Executor, FileFlavor, Kind};
+use super::build_plan::BuildPlan;
 
 mod unit_dependencies;
 use self::unit_dependencies::build_unit_dependencies;
 
 mod compilation_files;
-pub use self::compilation_files::Metadata;
-use self::compilation_files::{CompilationFiles, OutputFile};
+pub use self::compilation_files::{Metadata, OutputFile};
+use self::compilation_files::CompilationFiles;
 
 /// All information needed to define a Unit.
 ///
@@ -62,6 +64,24 @@ pub struct Unit<'a> {
     pub mode: CompileMode,
 }
 
+impl<'a> Unit<'a> {
+    pub fn buildkey(&self) -> String {
+        format!("{}-{}", self.pkg.name(), short_hash(self))
+	}
+}
+
+impl<'a> Ord for Unit<'a> {
+    fn cmp(&self, other: &Unit) -> Ordering {
+        self.buildkey().cmp(&other.buildkey())
+    }
+}
+
+impl<'a> PartialOrd for Unit<'a> {
+    fn partial_cmp(&self, other: &Unit) -> Option<Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
 pub struct Context<'a, 'cfg: 'a> {
     pub bcx: &'a BuildContext<'a, 'cfg>,
     pub compilation: Compilation<'cfg>,
@@ -121,6 +141,8 @@ impl<'a, 'cfg> Context<'a, 'cfg> {
         exec: &Arc<Executor>,
     ) -> CargoResult<Compilation<'cfg>> {
         let mut queue = JobQueue::new(self.bcx);
+        let mut plan = BuildPlan::new();
+        let build_plan = self.bcx.build_config.build_plan;
         self.prepare_units(export_dir, units)?;
         self.prepare()?;
         custom_build::build_map(&mut self, units)?;
@@ -131,11 +153,16 @@ impl<'a, 'cfg> Context<'a, 'cfg> {
             // part of this, that's all done next as part of the `execute`
             // function which will run everything in order with proper
             // parallelism.
-            super::compile(&mut self, &mut queue, unit, exec)?;
+            super::compile(&mut self, &mut queue, &mut plan, unit, exec)?;
         }
 
         // Now that we've figured out everything that we're going to do, do it!
-        queue.execute(&mut self)?;
+        queue.execute(&mut self, &mut plan)?;
+
+        if build_plan {
+            plan.set_inputs(self.bcx.inputs()?);
+            plan.output_plan();
+        }
 
         for unit in units.iter() {
             for output in self.outputs(unit)?.iter() {
@@ -366,7 +393,9 @@ impl<'a, 'cfg> Context<'a, 'cfg> {
                 return Vec::new();
             }
         }
-        self.unit_dependencies[unit].clone()
+        let mut deps = self.unit_dependencies[unit].clone();
+        deps.sort();
+        deps
     }
 
     pub fn incremental_args(&self, unit: &Unit) -> CargoResult<Vec<String>> {