Seperate doc tests into separate stages

When generating documentation tests, the following stages will happen:

- tests will be gathered from the docs as a list of strings
- tests will be saved to target/doc/tests as individual files
- tests will be compiled into a single binary called "rustdoc-test"
- tests will be run by executing the "rustdoc-test" binary

Doc tests are now actual tests. This allows us to take advantage of the
existing test infrastructure supplied by rustc.

Fixes #32, #54

Author: hjr3

example/src/lib.rs

@@ -11,6 +11,10 @@
 //
 // @matches "included[?id=='nested_modules::example_module'] \
 //          .relationships.parent" []
+/// An example module
+/// ```rust
+/// assert!(true);
+/// ```
 pub mod example_module {
 
     // For the child:

src/error.rs

@@ -44,3 +44,11 @@ pub struct MovedFlag {
     /// A message explaning where the flag moved to
     pub msg: String,
 }
+
+/// Thrown whenever creation of documentation tests fail
+#[derive(Debug, Fail)]
+#[fail(display = "Unable to test documentation: \"{}\"", output)]
+pub struct DocTestErr {
+    /// The output of the Command that failed
+    pub output: String,
+}

src/lib.rs

@@ -36,13 +36,11 @@ use std::env;
 use std::fs::{self, File};
 use std::io::prelude::*;
 use std::io;
-use std::iter;
 use std::path::{Path, PathBuf};
-use std::process::{self, Command, Stdio};
+use std::process::{Command, Stdio};
 
 use cargo::Target;
 use json::Documentation;
-use test::TestResult;
 use ui::Ui;
 
 pub use json::create_documentation;
@@ -228,81 +226,33 @@ pub fn test(config: &Config) -> Result<()> {
         .map_err(|e| failure::Error::from(e.context("could not find generated documentation")))?;
     let docs: Documentation = serde_json::from_reader(doc_json)?;
 
+    // TODO a better way to find crate name?
     let krate = docs.data.as_ref().unwrap();
-    let tests: Vec<_> = iter::once(krate)
-        .chain(docs.included.iter().flat_map(|data| data))
-        .map(|data| (&data.id, test::gather_tests(&data)))
-        .collect();
-
-    // Run the tests.
-    static SUCCESS_MESSAGE: &str = "ok";
-    static FAILURE_MESSAGE: &str = "FAILED";
-
-    let num_tests: usize = tests.iter().map(|&(_, ref tests)| tests.len()).sum();
-    println!("running {} tests", num_tests);
-
-    let mut passed = 0;
-    let mut failures = vec![];
-    for (id, tests) in tests {
-        for (number, test) in tests.iter().enumerate() {
-            // FIXME: Make the name based off the file and line number.
-            let name = format!("{} - {}", id, number);
-            print!("test {} ... ", name);
-            io::stdout().flush()?;
-
-            let message = match test::run_test(config, test)? {
-                TestResult::Success => {
-                    passed += 1;
-                    SUCCESS_MESSAGE
-                }
-                TestResult::Failure(output) => {
-                    failures.push((name, output));
-                    FAILURE_MESSAGE
-                }
-            };
-
-            println!("{}", message);
-        }
-    }
+    let crate_name = krate.id.split("::").next().unwrap();
 
-    if !failures.is_empty() {
-        // Print the output of each failure.
-        for &(ref name, ref output) in &failures {
-            let stdout = String::from_utf8_lossy(&output.stdout);
-            let stdout = stdout.trim();
-
-            if !stdout.is_empty() {
-                println!("\n---- {} stdout ----\n{}", name, stdout);
-            }
-
-            let stderr = String::from_utf8_lossy(&output.stderr);
-            let stderr = stderr.trim();
-
-            if !stderr.is_empty() {
-                println!("\n---- {} stderr ----\n{}", name, stderr);
-            }
-        }
+    let location = config.output_path().join("tests");
+    let tests = {
+        let task = config.ui.start_task("Finding tests");
+        task.report("In Progress");
+        test::find_tests(&docs)
+    };
 
-        // Print a summary of all failures at the bottom.
-        println!("\nfailures:");
-        for &(ref name, _) in &failures {
-            println!("    {}", name);
-        }
+    {
+        let task = config.ui.start_task("Saving tests");
+        task.report("In Progress");
+        test::save_tests(&tests, &location, &crate_name)?;
     }
 
-    println!(
-        "\ntest result: {}. {} passed; {} failed; 0 ignored; 0 measured; 0 filtered out",
-        if failures.is_empty() {
-            SUCCESS_MESSAGE
-        } else {
-            FAILURE_MESSAGE
-        },
-        passed,
-        failures.len()
-    );
-
-    if !failures.is_empty() {
-        process::exit(1);
+    let binary = {
+        let task = config.ui.start_task("Compiling tests");
+        task.report("In Progress");
+        test::compile_tests(&config, &location)?
+    };
+
+    {
+        let task = config.ui.start_task("Executing tests");
+        task.report("In Progress");
+        test::execute_tests(&binary)?;
     }
 
     Ok(())