Add support for manually passing options to examples. (rust-lang#409)

* Add support for manually passing options to examples.

* Ignore files with `.props` extension.

* Use `WalkDir` instead of manually reading sub-directories.

Author: bdalrhm

.github/workflows/copyright.yml

@@ -15,7 +15,7 @@ jobs:
       - name: Get paths for files added
         id: git-diff
         run: |
-          ignore='(.md|expected|ignore|gitignore)$'
+          ignore='(.md|.props|expected|ignore|gitignore)$'
           files=$(git diff --ignore-submodules=all --name-only --diff-filter=A ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} | grep -v -E $ignore | xargs)
           echo "::set-output name=paths::$files"
 

Cargo.lock

@@ -890,6 +890,7 @@ name = "dashboard"
 version = "0.1.0"
 dependencies = [
  "pulldown-cmark 0.8.0",
+ "walkdir",
 ]
 
 [[package]]
@@ -5595,9 +5596,9 @@ dependencies = [
 
 [[package]]
 name = "walkdir"
-version = "2.3.1"
+version = "2.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "777182bc735b6424e1a57516d35ed72cb8019d85c8c9bf536dccb3445c1a2f7d"
+checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56"
 dependencies = [
  "same-file",
  "winapi",

src/tools/dashboard/Cargo.toml

@@ -8,3 +8,4 @@ edition = "2018"
 
 [dependencies]
 pulldown-cmark = { version = "0.8.0", default-features = false }
+walkdir = "2.3.2"

src/tools/dashboard/configs/ref/Appendices/Glossary/263.props

@@ -0,0 +1 @@
+// rmc-flags: --cbmc-args --unwind 4

src/tools/dashboard/configs/ref/Attributes/Limits/45.props

@@ -0,0 +1 @@
+// rmc-codegen-fail

src/tools/dashboard/configs/ref/Linkage/190.props

@@ -0,0 +1 @@
+// rmc-flags: --cbmc-args --unwind 1

src/tools/dashboard/configs/ref/Statements and expressions/Expressions/Loop expressions/133.props

@@ -0,0 +1 @@
+// rmc-flags: --cbmc-args --unwind 4

src/tools/dashboard/configs/ref/Statements and expressions/Expressions/Method call expressions/10.props

@@ -0,0 +1 @@
+// rmc-flags: --cbmc-args --unwind 1

src/tools/dashboard/src/reference.rs

@@ -4,18 +4,23 @@
 //! [The Rust Reference](https://doc.rust-lang.org/nightly/reference),
 //! run them through RMC, and display their results.
 
-use crate::{dashboard, litani::Litani, util};
+use crate::{
+    dashboard,
+    litani::Litani,
+    util::{self, FailStep, TestProps},
+};
 use pulldown_cmark::{Parser, Tag};
 use std::{
-    collections::HashMap,
+    collections::{HashMap, HashSet},
     env,
-    fmt::{Debug, Formatter, Result},
+    fmt::{Debug, Formatter, Result, Write},
     fs::{self, File},
     hash::Hash,
     io::{BufRead, BufReader},
     path::{Path, PathBuf},
     process::{Command, Stdio},
 };
+use walkdir::WalkDir;
 
 /// Parses the chapter/section hierarchy in the markdown file specified by
 /// `summary_path` and returns a mapping from markdown files containing rust
@@ -151,72 +156,98 @@ fn extract(par_from: &Path, par_to: &Path) -> Vec<(Example, PathBuf)> {
     pairs
 }
 
-/// Prepends the text in `path` with the given `text`.
-fn prepend_text(path: &Path, text: &str) {
-    let code = fs::read_to_string(&path).unwrap();
-    let code = format!("{}\n{}", text, code);
-    fs::write(&path, code).unwrap();
+/// Returns a set of paths to the config files for examples in the Rust books.
+fn get_config_paths() -> HashSet<PathBuf> {
+    let config_dir: PathBuf = ["src", "tools", "dashboard", "configs"].iter().collect();
+    let mut config_paths = HashSet::new();
+    for entry in WalkDir::new(config_dir) {
+        let entry = entry.unwrap().into_path();
+        if entry.is_file() {
+            config_paths.insert(entry);
+        }
+    }
+    config_paths
+}
+
+/// Prepends the given `props` to the test file in `props.test`.
+fn prepend_props(props: &TestProps) {
+    let code = fs::read_to_string(&props.path).unwrap();
+    let code = format!("{}{}", props, code);
+    fs::write(&props.path, code).unwrap();
+}
+
+/// Pretty prints the `paths` set.
+fn paths_to_string(paths: HashSet<PathBuf>) -> String {
+    let mut f = String::new();
+    for path in paths {
+        f.write_fmt(format_args!("    {:?}\n", path.to_str().unwrap())).unwrap();
+    }
+    f
 }
 
 /// Pre-processes the examples in `map` before running them with `compiletest`.
 fn preprocess_examples(map: &HashMap<Example, PathBuf>) {
-    // Copy compiler configurations specified in the original markdown code
-    // block.
+    let config_dir: PathBuf = ["src", "tools", "dashboard", "configs"].iter().collect();
+    let test_dir: PathBuf = ["src", "test"].iter().collect();
+    let mut config_paths = get_config_paths();
+    // Copy compiler annotations specified in the original markdown code blocks
+    // and custom configurations under the `config` directory.
     for (from, to) in map.iter() {
+        // Path `to` has the following form:
+        // `src/test/ref/<hierarchy>/<line-num>.rs`
+        // If it has a custom props file, the path to the props file will have
+        // the following form:
+        // `src/tools/dashboard/configs/ref/<hierarchy>/<line-num>.props`
+        // where <hierarchy> and <line-num> are the same for both paths.
+        let mut props_path = config_dir.join(to.strip_prefix(&test_dir).unwrap());
+        props_path.set_extension("props");
+        let mut props = if props_path.exists() {
+            config_paths.remove(&props_path);
+            // Parse the properties in the file. The format follows the same
+            // conventions for the headers in RMC regressions.
+            let mut props = util::parse_test_header(&props_path);
+            // `util::parse_test_header` thinks `props_path` is the path to the
+            // test. That is not the case, `to` is the actual path to the
+            // test/example.
+            props.path = to.clone();
+            props
+        } else {
+            TestProps::new(to.clone(), None, Vec::new(), Vec::new())
+        };
         let file = File::open(&from.path).unwrap();
         // Skip to the first line of the example code block.
         // Line numbers in files start with 1 but `nth(...)` starts with 0.
         // Subtract 1 to account for the difference.
         let line = BufReader::new(file).lines().nth(from.line - 1).unwrap().unwrap();
-        if line.contains("edition2015") {
-            prepend_text(to, "// compile-flags: --edition 2015");
-        } else {
-            prepend_text(to, "// compile-flags: --edition 2018");
+        if !line.contains("edition2015") {
+            props.rustc_args.push(String::from("--edition"));
+            props.rustc_args.push(String::from("2018"));
         }
-        // Most examples with `compile_fail` configuration fail because of
-        // check errors.
-        if line.contains("compile_fail") {
-            prepend_text(to, "// rmc-check-fail");
+        // Most examples with `compile_fail` annotation fail because of check
+        // errors. This heuristic can be overridden by manually specifying the
+        // fail step in the corresponding config file.
+        if props.fail_step.is_none() && line.contains("compile_fail") {
+            props.fail_step = Some(FailStep::Check);
         }
         // RMC should catch run-time errors.
-        if line.contains("should_panic") {
-            prepend_text(to, "// rmc-verify-fail");
+        if props.fail_step.is_none() && line.contains("should_panic") {
+            props.fail_step = Some(FailStep::Verification);
         }
+        // Prepend those properties to test/example file.
+        prepend_props(&props);
     }
-    // For now, we will only manually pre-process the tests that cause infinite loops.
-    // TODO: Add support for manually adding options and assertions (see issue #324).
-    let loop_tests: [PathBuf; 4] = [
-        ["src", "test", "ref", "Appendices", "Glossary", "263.rs"].iter().collect(),
-        ["src", "test", "ref", "Linkage", "190.rs"].iter().collect(),
-        [
-            "src",
-            "test",
-            "ref",
-            "Statements and expressions",
-            "Expressions",
-            "Loop expressions",
-            "133.rs",
-        ]
-        .iter()
-        .collect(),
-        [
-            "src",
-            "test",
-            "ref",
-            "Statements and expressions",
-            "Expressions",
-            "Method call expressions",
-            "10.rs",
-        ]
-        .iter()
-        .collect(),
-    ];
-
-    for test in loop_tests {
-        let code = fs::read_to_string(&test).unwrap();
-        let code = format!("// rmc-flags: --cbmc-args --unwind 1\n{}", code);
-        fs::write(&test, code).unwrap();
+    if !config_paths.is_empty() {
+        panic!(
+            "Error: The examples corresponding to the following config files \
+             were not encountered in the pre-processing step:\n{}This is most \
+             likely because the line numbers of the config files are not in \
+             sync with the line numbers of the corresponding code blocks in \
+             the latest versions of the Rust books. Please update the line \
+             numbers of the config files and rerun the program.",
+            paths_to_string(config_paths)
+        );
     }
+    // TODO: Add support for manually adding assertions (see issue #324).
 }
 
 /// Runs `compiletest` on the `suite` and logs the results to `log_path`.
@@ -310,18 +341,12 @@ fn litani_run_tests() {
     let ref_dir: PathBuf = ["src", "test", "ref"].iter().collect();
     util::add_rmc_and_litani_to_path();
     let mut litani = Litani::init("RMC", &output_prefix, &output_symlink);
-    let mut stack = vec![ref_dir];
     // Run all tests under the `src/test/ref` directory.
-    while !stack.is_empty() {
-        let cur_dir = stack.pop().unwrap();
-        for child in cur_dir.read_dir().unwrap() {
-            let child = child.unwrap().path();
-            if child.is_file() {
-                let test_props = util::parse_test_header(&child);
-                util::add_test_pipeline(&mut litani, &test_props);
-            } else {
-                stack.push(child);
-            }
+    for entry in WalkDir::new(ref_dir) {
+        let entry = entry.unwrap().into_path();
+        if entry.is_file() {
+            let test_props = util::parse_test_header(&entry);
+            util::add_test_pipeline(&mut litani, &test_props);
         }
     }
     litani.run_build();

src/tools/dashboard/src/util.rs

@@ -9,6 +9,7 @@
 use crate::litani::Litani;
 use std::{
     env,
+    fmt::{self, Display, Formatter, Write},
     fs::File,
     io::{BufRead, BufReader},
     path::{Path, PathBuf},
@@ -30,6 +31,17 @@ pub enum FailStep {
     Verification,
 }
 
+impl Display for FailStep {
+    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+        let str = match self {
+            FailStep::Check => "check",
+            FailStep::Codegen => "codegen",
+            FailStep::Verification => "verify",
+        };
+        f.write_str(str)
+    }
+}
+
 /// Data structure representing properties specific to each test.
 pub struct TestProps {
     pub path: PathBuf,
@@ -53,6 +65,29 @@ impl TestProps {
     }
 }
 
+impl Display for TestProps {
+    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+        if let Some(fail_step) = &self.fail_step {
+            f.write_fmt(format_args!("// rmc-{}-fail\n", fail_step))?;
+        }
+        if !self.rustc_args.is_empty() {
+            f.write_str("// compile-flags:")?;
+            for arg in &self.rustc_args {
+                f.write_fmt(format_args!(" {}", arg))?;
+            }
+            f.write_char('\n')?;
+        }
+        if !self.rmc_args.is_empty() {
+            f.write_str("// rmc-flags:")?;
+            for arg in &self.rmc_args {
+                f.write_fmt(format_args!(" {}", arg))?;
+            }
+            f.write_char('\n')?;
+        }
+        Ok(())
+    }
+}
+
 /// Parses strings of the form `rmc-*-fail` and returns the step at which RMC is
 /// expected to panic.
 fn try_parse_fail_step(cur_fail_step: Option<FailStep>, line: &str) -> Option<FailStep> {
@@ -140,7 +175,7 @@ pub fn add_check_job(litani: &mut Litani, test_props: &TestProps) {
         test_props.path.to_str().unwrap(),
         "build",
         exit_status,
-        1,
+        5,
     );
 }
 
@@ -166,15 +201,15 @@ pub fn add_codegen_job(litani: &mut Litani, test_props: &TestProps) {
         test_props.path.to_str().unwrap(),
         "test",
         exit_status,
-        1,
+        5,
     );
 }
 
 // Does verification pass/fail as it is expected to?
 pub fn add_verification_job(litani: &mut Litani, test_props: &TestProps) {
     let exit_status = if test_props.fail_step == Some(FailStep::Verification) { 10 } else { 0 };
     let mut rmc = Command::new("rmc");
-    rmc.args(&test_props.rmc_args).arg(&test_props.path);
+    rmc.arg(&test_props.path).args(&test_props.rmc_args);
     if !test_props.rustc_args.is_empty() {
         rmc.env("RUSTFLAGS", test_props.rustc_args.join(" "));
     }