Auto merge of #10287 - Nilstrieb:docs-semver-check-run-fail, r=ehuss

Add `run-fail` to semver-check for docs

I encountered this missing feature in #10276 and therefore added it here in this separate PR.

If the breaking change does not involve a compilation error but a change in runtime behaviour, you can add `run-fail` to the codeblock. The "before" code must return exit code 0, and the "after" code must be nonzero (like a panic).

Example case that I tested (ignore the trailing dot, it's for github markdown to not hate me)

```
```rust,ignore,run-fail
// MAJOR CHANGE

///////////////////////////////////////////////////////////
// Before
pub fn foo() {}
///////////////////////////////////////////////////////////
// After
pub fn foo() {
    panic!("hey!");
}
///////////////////////////////////////////////////////////
// Example usage that will break.
fn main() {
    updated_crate::foo();
}
```.
```

Author: bors

src/doc/semver-check/src/main.rs

@@ -11,7 +11,7 @@
 use std::error::Error;
 use std::fs;
 use std::path::Path;
-use std::process::Command;
+use std::process::{Command, Output};
 
 fn main() {
     if let Err(e) = doit() {
@@ -24,19 +24,18 @@ const SEPARATOR: &str = "///////////////////////////////////////////////////////
 
 fn doit() -> Result<(), Box<dyn Error>> {
     let filename = std::env::args()
-        .skip(1)
-        .next()
+        .nth(1)
         .unwrap_or_else(|| "../src/reference/semver.md".to_string());
     let contents = fs::read_to_string(filename)?;
     let mut lines = contents.lines().enumerate();
 
     loop {
         // Find a rust block.
-        let block_start = loop {
+        let (block_start, run_program) = loop {
             match lines.next() {
                 Some((lineno, line)) => {
                     if line.trim().starts_with("```rust") && !line.contains("skip") {
-                        break lineno + 1;
+                        break (lineno + 1, line.contains("run-fail"));
                     }
                 }
                 None => return Ok(()),
@@ -83,12 +82,16 @@ fn doit() -> Result<(), Box<dyn Error>> {
         };
         let expect_success = parts[0][0].contains("MINOR");
         println!("Running test from line {}", block_start);
-        if let Err(e) = run_test(
+
+        let result = run_test(
             join(parts[1]),
             join(parts[2]),
             join(parts[3]),
             expect_success,
-        ) {
+            run_program,
+        );
+
+        if let Err(e) = result {
             return Err(format!(
                 "test failed for example starting on line {}: {}",
                 block_start, e
@@ -105,15 +108,23 @@ fn run_test(
     after: String,
     example: String,
     expect_success: bool,
+    run_program: bool,
 ) -> Result<(), Box<dyn Error>> {
     let tempdir = tempfile::TempDir::new()?;
     let before_p = tempdir.path().join("before.rs");
     let after_p = tempdir.path().join("after.rs");
     let example_p = tempdir.path().join("example.rs");
-    compile(before, &before_p, CRATE_NAME, false, true)?;
-    compile(example.clone(), &example_p, "example", true, true)?;
-    compile(after, &after_p, CRATE_NAME, false, true)?;
-    compile(example, &example_p, "example", true, expect_success)?;
+
+    let check_fn = if run_program {
+        run_check
+    } else {
+        compile_check
+    };
+
+    compile_check(before, &before_p, CRATE_NAME, false, true)?;
+    check_fn(example.clone(), &example_p, "example", true, true)?;
+    compile_check(after, &after_p, CRATE_NAME, false, true)?;
+    check_fn(example, &example_p, "example", true, expect_success)?;
     Ok(())
 }
 
@@ -127,34 +138,18 @@ fn check_formatting(path: &Path) -> Result<(), Box<dyn Error>> {
             if !status.success() {
                 return Err(format!("failed to run rustfmt: {}", status).into());
             }
-            return Ok(());
-        }
-        Err(e) => {
-            return Err(format!("failed to run rustfmt: {}", e).into());
+            Ok(())
         }
+        Err(e) => Err(format!("failed to run rustfmt: {}", e).into()),
     }
 }
 
 fn compile(
-    mut contents: String,
+    contents: &str,
     path: &Path,
     crate_name: &str,
     extern_path: bool,
-    expect_success: bool,
-) -> Result<(), Box<dyn Error>> {
-    // If the example has an error message, remove it so that it can be
-    // compared with the actual output, and also to avoid issues with rustfmt
-    // moving it around.
-    let expected_error = match contents.find("// Error:") {
-        Some(index) => {
-            let start = contents[..index].rfind(|ch| ch != ' ').unwrap();
-            let end = contents[index..].find('\n').unwrap();
-            let error = contents[index + 9..index + end].trim().to_string();
-            contents.replace_range(start + 1..index + end, "");
-            Some(error)
-        }
-        None => None,
-    };
+) -> Result<Output, Box<dyn Error>> {
     let crate_type = if contents.contains("fn main()") {
         "bin"
     } else {
@@ -166,7 +161,7 @@ fn compile(
     let out_dir = path.parent().unwrap();
     let mut cmd = Command::new("rustc");
     cmd.args(&[
-        "--edition=2018",
+        "--edition=2021",
         "--crate-type",
         crate_type,
         "--crate-name",
@@ -180,7 +175,32 @@ fn compile(
             .arg(format!("{}={}", CRATE_NAME, epath.display()));
     }
     cmd.arg(path);
-    let output = cmd.output()?;
+    cmd.output().map_err(Into::into)
+}
+
+fn compile_check(
+    mut contents: String,
+    path: &Path,
+    crate_name: &str,
+    extern_path: bool,
+    expect_success: bool,
+) -> Result<(), Box<dyn Error>> {
+    // If the example has an error message, remove it so that it can be
+    // compared with the actual output, and also to avoid issues with rustfmt
+    // moving it around.
+    let expected_error = match contents.find("// Error:") {
+        Some(index) => {
+            let start = contents[..index].rfind(|ch| ch != ' ').unwrap();
+            let end = contents[index..].find('\n').unwrap();
+            let error = contents[index + 9..index + end].trim().to_string();
+            contents.replace_range(start + 1..index + end, "");
+            Some(error)
+        }
+        None => None,
+    };
+
+    let output = compile(&contents, path, crate_name, extern_path)?;
+
     let stderr = std::str::from_utf8(&output.stderr).unwrap();
     match (output.status.success(), expect_success) {
         (true, true) => Ok(()),
@@ -215,3 +235,48 @@ fn compile(
         }
     }
 }
+
+fn run_check(
+    contents: String,
+    path: &Path,
+    crate_name: &str,
+    extern_path: bool,
+    expect_success: bool,
+) -> Result<(), Box<dyn Error>> {
+    let compile_output = compile(&contents, path, crate_name, extern_path)?;
+
+    if !compile_output.status.success() {
+        let stderr = std::str::from_utf8(&compile_output.stderr).unwrap();
+        return Err(format!(
+            "expected success, got error {}\n===== Contents:\n{}\n===== Output:\n{}\n",
+            path.display(),
+            contents,
+            stderr
+        )
+        .into());
+    }
+
+    let binary_path = path.parent().unwrap().join(crate_name);
+
+    let output = Command::new(binary_path).output()?;
+
+    let stderr = std::str::from_utf8(&output.stderr).unwrap();
+
+    match (output.status.success(), expect_success) {
+        (true, false) => Err(format!(
+            "expected panic, got success {}\n===== Contents:\n{}\n===== Output:\n{}\n",
+            path.display(),
+            contents,
+            stderr
+        )
+        .into()),
+        (false, true) => Err(format!(
+            "expected success, got panic {}\n===== Contents:\n{}\n===== Output:\n{}\n",
+            path.display(),
+            contents,
+            stderr,
+        )
+        .into()),
+        (_, _) => Ok(()),
+    }
+}