Break up compiletest runtest.rs into smaller helper modules

Previously compiletest's `runtest.rs` was a massive 4700 lines file that
made reading and navigation very awkward.

This commit intentionally does not neatly reorganize where all the
methods on `TestCx` goes, that is intended for a follow-up PR.

Author: jieyouxu

src/tools/compiletest/src/runtest.rs

@@ -0,0 +1,19 @@
+use super::TestCx;
+
+impl TestCx<'_> {
+    pub(super) fn run_assembly_test(&self) {
+        if self.config.llvm_filecheck.is_none() {
+            self.fatal("missing --llvm-filecheck");
+        }
+
+        let (proc_res, output_path) = self.compile_test_and_save_assembly();
+        if !proc_res.status.success() {
+            self.fatal_proc_rec("compilation failed!", &proc_res);
+        }
+
+        let proc_res = self.verify_with_filecheck(&output_path);
+        if !proc_res.status.success() {
+            self.fatal_proc_rec("verification with 'FileCheck' failed", &proc_res);
+        }
+    }
+}

src/tools/compiletest/src/runtest/assembly.rs

@@ -0,0 +1,22 @@
+use super::{PassMode, TestCx};
+
+impl TestCx<'_> {
+    pub(super) fn run_codegen_test(&self) {
+        if self.config.llvm_filecheck.is_none() {
+            self.fatal("missing --llvm-filecheck");
+        }
+
+        let (proc_res, output_path) = self.compile_test_and_save_ir();
+        if !proc_res.status.success() {
+            self.fatal_proc_rec("compilation failed!", &proc_res);
+        }
+
+        if let Some(PassMode::Build) = self.pass_mode() {
+            return;
+        }
+        let proc_res = self.verify_with_filecheck(&output_path);
+        if !proc_res.status.success() {
+            self.fatal_proc_rec("verification with 'FileCheck' failed", &proc_res);
+        }
+    }
+}

src/tools/compiletest/src/runtest/codegen.rs

@@ -0,0 +1,191 @@
+use std::collections::HashSet;
+
+use super::{Emit, TestCx, WillExecute};
+use crate::errors;
+use crate::util::static_regex;
+
+impl TestCx<'_> {
+    pub(super) fn run_codegen_units_test(&self) {
+        assert!(self.revision.is_none(), "revisions not relevant here");
+
+        let proc_res = self.compile_test(WillExecute::No, Emit::None);
+
+        if !proc_res.status.success() {
+            self.fatal_proc_rec("compilation failed!", &proc_res);
+        }
+
+        self.check_no_compiler_crash(&proc_res, self.props.should_ice);
+
+        const PREFIX: &str = "MONO_ITEM ";
+        const CGU_MARKER: &str = "@@";
+
+        // Some MonoItems can contain {closure@/path/to/checkout/tests/codgen-units/test.rs}
+        // To prevent the current dir from leaking, we just replace the entire path to the test
+        // file with TEST_PATH.
+        let actual: Vec<MonoItem> = proc_res
+            .stdout
+            .lines()
+            .filter(|line| line.starts_with(PREFIX))
+            .map(|line| {
+                line.replace(&self.testpaths.file.display().to_string(), "TEST_PATH").to_string()
+            })
+            .map(|line| str_to_mono_item(&line, true))
+            .collect();
+
+        let expected: Vec<MonoItem> = errors::load_errors(&self.testpaths.file, None)
+            .iter()
+            .map(|e| str_to_mono_item(&e.msg[..], false))
+            .collect();
+
+        let mut missing = Vec::new();
+        let mut wrong_cgus = Vec::new();
+
+        for expected_item in &expected {
+            let actual_item_with_same_name = actual.iter().find(|ti| ti.name == expected_item.name);
+
+            if let Some(actual_item) = actual_item_with_same_name {
+                if !expected_item.codegen_units.is_empty() &&
+                   // Also check for codegen units
+                   expected_item.codegen_units != actual_item.codegen_units
+                {
+                    wrong_cgus.push((expected_item.clone(), actual_item.clone()));
+                }
+            } else {
+                missing.push(expected_item.string.clone());
+            }
+        }
+
+        let unexpected: Vec<_> = actual
+            .iter()
+            .filter(|acgu| !expected.iter().any(|ecgu| acgu.name == ecgu.name))
+            .map(|acgu| acgu.string.clone())
+            .collect();
+
+        if !missing.is_empty() {
+            missing.sort();
+
+            println!("\nThese items should have been contained but were not:\n");
+
+            for item in &missing {
+                println!("{}", item);
+            }
+
+            println!("\n");
+        }
+
+        if !unexpected.is_empty() {
+            let sorted = {
+                let mut sorted = unexpected.clone();
+                sorted.sort();
+                sorted
+            };
+
+            println!("\nThese items were contained but should not have been:\n");
+
+            for item in sorted {
+                println!("{}", item);
+            }
+
+            println!("\n");
+        }
+
+        if !wrong_cgus.is_empty() {
+            wrong_cgus.sort_by_key(|pair| pair.0.name.clone());
+            println!("\nThe following items were assigned to wrong codegen units:\n");
+
+            for &(ref expected_item, ref actual_item) in &wrong_cgus {
+                println!("{}", expected_item.name);
+                println!("  expected: {}", codegen_units_to_str(&expected_item.codegen_units));
+                println!("  actual:   {}", codegen_units_to_str(&actual_item.codegen_units));
+                println!();
+            }
+        }
+
+        if !(missing.is_empty() && unexpected.is_empty() && wrong_cgus.is_empty()) {
+            panic!();
+        }
+
+        #[derive(Clone, Eq, PartialEq)]
+        struct MonoItem {
+            name: String,
+            codegen_units: HashSet<String>,
+            string: String,
+        }
+
+        // [MONO_ITEM] name [@@ (cgu)+]
+        fn str_to_mono_item(s: &str, cgu_has_crate_disambiguator: bool) -> MonoItem {
+            let s = if s.starts_with(PREFIX) { (&s[PREFIX.len()..]).trim() } else { s.trim() };
+
+            let full_string = format!("{}{}", PREFIX, s);
+
+            let parts: Vec<&str> =
+                s.split(CGU_MARKER).map(str::trim).filter(|s| !s.is_empty()).collect();
+
+            let name = parts[0].trim();
+
+            let cgus = if parts.len() > 1 {
+                let cgus_str = parts[1];
+
+                cgus_str
+                    .split(' ')
+                    .map(str::trim)
+                    .filter(|s| !s.is_empty())
+                    .map(|s| {
+                        if cgu_has_crate_disambiguator {
+                            remove_crate_disambiguators_from_set_of_cgu_names(s)
+                        } else {
+                            s.to_string()
+                        }
+                    })
+                    .collect()
+            } else {
+                HashSet::new()
+            };
+
+            MonoItem { name: name.to_owned(), codegen_units: cgus, string: full_string }
+        }
+
+        fn codegen_units_to_str(cgus: &HashSet<String>) -> String {
+            let mut cgus: Vec<_> = cgus.iter().collect();
+            cgus.sort();
+
+            let mut string = String::new();
+            for cgu in cgus {
+                string.push_str(&cgu[..]);
+                string.push(' ');
+            }
+
+            string
+        }
+
+        // Given a cgu-name-prefix of the form <crate-name>.<crate-disambiguator> or
+        // the form <crate-name1>.<crate-disambiguator1>-in-<crate-name2>.<crate-disambiguator2>,
+        // remove all crate-disambiguators.
+        fn remove_crate_disambiguator_from_cgu(cgu: &str) -> String {
+            let Some(captures) =
+                static_regex!(r"^[^\.]+(?P<d1>\.[[:alnum:]]+)(-in-[^\.]+(?P<d2>\.[[:alnum:]]+))?")
+                    .captures(cgu)
+            else {
+                panic!("invalid cgu name encountered: {cgu}");
+            };
+
+            let mut new_name = cgu.to_owned();
+
+            if let Some(d2) = captures.name("d2") {
+                new_name.replace_range(d2.start()..d2.end(), "");
+            }
+
+            let d1 = captures.name("d1").unwrap();
+            new_name.replace_range(d1.start()..d1.end(), "");
+
+            new_name
+        }
+
+        // The name of merged CGUs is constructed as the names of the original
+        // CGUs joined with "--". This function splits such composite CGU names
+        // and handles each component individually.
+        fn remove_crate_disambiguators_from_set_of_cgu_names(cgus: &str) -> String {
+            cgus.split("--").map(remove_crate_disambiguator_from_cgu).collect::<Vec<_>>().join("--")
+        }
+    }
+}

src/tools/compiletest/src/runtest/codegen_units.rs

@@ -18,7 +18,7 @@ impl<'test> TestCx<'test> {
             .unwrap_or_else(|| self.fatal("missing --coverage-dump"))
     }
 
-    pub(crate) fn run_coverage_map_test(&self) {
+    pub(super) fn run_coverage_map_test(&self) {
         let coverage_dump_path = self.coverage_dump_path();
 
         let (proc_res, llvm_ir_path) = self.compile_test_and_save_ir();
@@ -50,7 +50,7 @@ impl<'test> TestCx<'test> {
         }
     }
 
-    pub(crate) fn run_coverage_run_test(&self) {
+    pub(super) fn run_coverage_run_test(&self) {
         let should_run = self.run_if_enabled();
         let proc_res = self.compile_test(should_run, Emit::None);
 

src/tools/compiletest/src/runtest/coverage.rs

@@ -0,0 +1,25 @@
+use super::{TestCx, WillExecute};
+
+impl TestCx<'_> {
+    pub(super) fn run_crash_test(&self) {
+        let pm = self.pass_mode();
+        let proc_res = self.compile_test(WillExecute::No, self.should_emit_metadata(pm));
+
+        if std::env::var("COMPILETEST_VERBOSE_CRASHES").is_ok() {
+            eprintln!("{}", proc_res.status);
+            eprintln!("{}", proc_res.stdout);
+            eprintln!("{}", proc_res.stderr);
+            eprintln!("{}", proc_res.cmdline);
+        }
+
+        // if a test does not crash, consider it an error
+        if proc_res.status.success() || matches!(proc_res.status.code(), Some(1 | 0)) {
+            self.fatal(&format!(
+                "crashtest no longer crashes/triggers ICE, horray! Please give it a meaningful name, \
+            add a doc-comment to the start of the test explaining why it exists and \
+            move it to tests/ui or wherever you see fit. Adding 'Fixes #<issueNr>' to your PR description \
+            ensures that the corresponding ticket is auto-closed upon merge."
+            ));
+        }
+    }
+}

src/tools/compiletest/src/runtest/crash.rs

@@ -0,0 +1,128 @@
+use super::{TestCx, WillExecute};
+use crate::errors;
+
+// FIXME(jieyouxu): `run_rpass_test` got hoisted out of this because apparently valgrind falls back
+// to `run_rpass_test` if valgrind isn't available, which is questionable, but keeping it for
+// refactoring changes to preserve current behavior.
+
+impl TestCx<'_> {
+    pub(super) fn run_incremental_test(&self) {
+        // Basic plan for a test incremental/foo/bar.rs:
+        // - load list of revisions rpass1, cfail2, rpass3
+        //   - each should begin with `cpass`, `rpass`, `cfail`, or `rfail`
+        //   - if `cpass`, expect compilation to succeed, don't execute
+        //   - if `rpass`, expect compilation and execution to succeed
+        //   - if `cfail`, expect compilation to fail
+        //   - if `rfail`, expect compilation to succeed and execution to fail
+        // - create a directory build/foo/bar.incremental
+        // - compile foo/bar.rs with -C incremental=.../foo/bar.incremental and -C rpass1
+        //   - because name of revision starts with "rpass", expect success
+        // - compile foo/bar.rs with -C incremental=.../foo/bar.incremental and -C cfail2
+        //   - because name of revision starts with "cfail", expect an error
+        //   - load expected errors as usual, but filter for those that end in `[rfail2]`
+        // - compile foo/bar.rs with -C incremental=.../foo/bar.incremental and -C rpass3
+        //   - because name of revision starts with "rpass", expect success
+        // - execute build/foo/bar.exe and save output
+        //
+        // FIXME -- use non-incremental mode as an oracle? That doesn't apply
+        // to #[rustc_dirty] and clean tests I guess
+
+        let revision = self.revision.expect("incremental tests require a list of revisions");
+
+        // Incremental workproduct directory should have already been created.
+        let incremental_dir = self.props.incremental_dir.as_ref().unwrap();
+        assert!(incremental_dir.exists(), "init_incremental_test failed to create incremental dir");
+
+        if self.config.verbose {
+            print!("revision={:?} props={:#?}", revision, self.props);
+        }
+
+        if revision.starts_with("cpass") {
+            if self.props.should_ice {
+                self.fatal("can only use should-ice in cfail tests");
+            }
+            self.run_cpass_test();
+        } else if revision.starts_with("rpass") {
+            if self.props.should_ice {
+                self.fatal("can only use should-ice in cfail tests");
+            }
+            self.run_rpass_test();
+        } else if revision.starts_with("rfail") {
+            if self.props.should_ice {
+                self.fatal("can only use should-ice in cfail tests");
+            }
+            self.run_rfail_test();
+        } else if revision.starts_with("cfail") {
+            self.run_cfail_test();
+        } else {
+            self.fatal("revision name must begin with cpass, rpass, rfail, or cfail");
+        }
+    }
+
+    fn run_cpass_test(&self) {
+        let emit_metadata = self.should_emit_metadata(self.pass_mode());
+        let proc_res = self.compile_test(WillExecute::No, emit_metadata);
+
+        if !proc_res.status.success() {
+            self.fatal_proc_rec("compilation failed!", &proc_res);
+        }
+
+        // FIXME(#41968): Move this check to tidy?
+        if !errors::load_errors(&self.testpaths.file, self.revision).is_empty() {
+            self.fatal("compile-pass tests with expected warnings should be moved to ui/");
+        }
+    }
+
+    fn run_cfail_test(&self) {
+        let pm = self.pass_mode();
+        let proc_res = self.compile_test(WillExecute::No, self.should_emit_metadata(pm));
+        self.check_if_test_should_compile(&proc_res, pm);
+        self.check_no_compiler_crash(&proc_res, self.props.should_ice);
+
+        let output_to_check = self.get_output(&proc_res);
+        let expected_errors = errors::load_errors(&self.testpaths.file, self.revision);
+        if !expected_errors.is_empty() {
+            if !self.props.error_patterns.is_empty() || !self.props.regex_error_patterns.is_empty()
+            {
+                self.fatal("both error pattern and expected errors specified");
+            }
+            self.check_expected_errors(expected_errors, &proc_res);
+        } else {
+            self.check_all_error_patterns(&output_to_check, &proc_res, pm);
+        }
+        if self.props.should_ice {
+            match proc_res.status.code() {
+                Some(101) => (),
+                _ => self.fatal("expected ICE"),
+            }
+        }
+
+        self.check_forbid_output(&output_to_check, &proc_res);
+    }
+
+    fn run_rfail_test(&self) {
+        let pm = self.pass_mode();
+        let should_run = self.run_if_enabled();
+        let proc_res = self.compile_test(should_run, self.should_emit_metadata(pm));
+
+        if !proc_res.status.success() {
+            self.fatal_proc_rec("compilation failed!", &proc_res);
+        }
+
+        if let WillExecute::Disabled = should_run {
+            return;
+        }
+
+        let proc_res = self.exec_compiled_test();
+
+        // The value our Makefile configures valgrind to return on failure
+        const VALGRIND_ERR: i32 = 100;
+        if proc_res.status.code() == Some(VALGRIND_ERR) {
+            self.fatal_proc_rec("run-fail test isn't valgrind-clean!", &proc_res);
+        }
+
+        let output_to_check = self.get_output(&proc_res);
+        self.check_correct_failure_status(&proc_res);
+        self.check_all_error_patterns(&output_to_check, &proc_res, pm);
+    }
+}

src/tools/compiletest/src/runtest/debuginfo.rs

@@ -0,0 +1,32 @@
+use std::process::Command;
+
+use super::TestCx;
+
+impl TestCx<'_> {
+    pub(super) fn run_js_doc_test(&self) {
+        if let Some(nodejs) = &self.config.nodejs {
+            let out_dir = self.output_base_dir();
+
+            self.document(&out_dir, &self.testpaths);
+
+            let root = self.config.find_rust_src_root().unwrap();
+            let file_stem =
+                self.testpaths.file.file_stem().and_then(|f| f.to_str()).expect("no file stem");
+            let res = self.run_command_to_procres(
+                Command::new(&nodejs)
+                    .arg(root.join("src/tools/rustdoc-js/tester.js"))
+                    .arg("--doc-folder")
+                    .arg(out_dir)
+                    .arg("--crate-name")
+                    .arg(file_stem.replace("-", "_"))
+                    .arg("--test-file")
+                    .arg(self.testpaths.file.with_extension("js")),
+            );
+            if !res.status.success() {
+                self.fatal_proc_rec("rustdoc-js test failed!", &res);
+            }
+        } else {
+            self.fatal("no nodeJS");
+        }
+    }
+}

src/tools/compiletest/src/runtest/incremental.rs

@@ -0,0 +1,155 @@
+use std::fs;
+use std::path::{Path, PathBuf};
+
+use glob::glob;
+use miropt_test_tools::{files_for_miropt_test, MiroptTest, MiroptTestFile};
+use tracing::debug;
+
+use super::{Emit, TestCx, WillExecute};
+use crate::compute_diff::write_diff;
+
+impl TestCx<'_> {
+    pub(super) fn run_mir_opt_test(&self) {
+        let pm = self.pass_mode();
+        let should_run = self.should_run(pm);
+
+        let mut test_info = files_for_miropt_test(
+            &self.testpaths.file,
+            self.config.get_pointer_width(),
+            self.config.target_cfg().panic.for_miropt_test_tools(),
+        );
+
+        let passes = std::mem::take(&mut test_info.passes);
+
+        let proc_res = self.compile_test_with_passes(should_run, Emit::Mir, passes);
+        if !proc_res.status.success() {
+            self.fatal_proc_rec("compilation failed!", &proc_res);
+        }
+        self.check_mir_dump(test_info);
+
+        if let WillExecute::Yes = should_run {
+            let proc_res = self.exec_compiled_test();
+
+            if !proc_res.status.success() {
+                self.fatal_proc_rec("test run failed!", &proc_res);
+            }
+        }
+    }
+
+    fn check_mir_dump(&self, test_info: MiroptTest) {
+        let test_dir = self.testpaths.file.parent().unwrap();
+        let test_crate =
+            self.testpaths.file.file_stem().unwrap().to_str().unwrap().replace('-', "_");
+
+        let MiroptTest { run_filecheck, suffix, files, passes: _ } = test_info;
+
+        if self.config.bless {
+            for e in
+                glob(&format!("{}/{}.*{}.mir", test_dir.display(), test_crate, suffix)).unwrap()
+            {
+                fs::remove_file(e.unwrap()).unwrap();
+            }
+            for e in
+                glob(&format!("{}/{}.*{}.diff", test_dir.display(), test_crate, suffix)).unwrap()
+            {
+                fs::remove_file(e.unwrap()).unwrap();
+            }
+        }
+
+        for MiroptTestFile { from_file, to_file, expected_file } in files {
+            let dumped_string = if let Some(after) = to_file {
+                self.diff_mir_files(from_file.into(), after.into())
+            } else {
+                let mut output_file = PathBuf::new();
+                output_file.push(self.get_mir_dump_dir());
+                output_file.push(&from_file);
+                debug!(
+                    "comparing the contents of: {} with {}",
+                    output_file.display(),
+                    expected_file.display()
+                );
+                if !output_file.exists() {
+                    panic!(
+                        "Output file `{}` from test does not exist, available files are in `{}`",
+                        output_file.display(),
+                        output_file.parent().unwrap().display()
+                    );
+                }
+                self.check_mir_test_timestamp(&from_file, &output_file);
+                let dumped_string = fs::read_to_string(&output_file).unwrap();
+                self.normalize_output(&dumped_string, &[])
+            };
+
+            if self.config.bless {
+                let _ = fs::remove_file(&expected_file);
+                fs::write(expected_file, dumped_string.as_bytes()).unwrap();
+            } else {
+                if !expected_file.exists() {
+                    panic!("Output file `{}` from test does not exist", expected_file.display());
+                }
+                let expected_string = fs::read_to_string(&expected_file).unwrap();
+                if dumped_string != expected_string {
+                    print!("{}", write_diff(&expected_string, &dumped_string, 3));
+                    panic!(
+                        "Actual MIR output differs from expected MIR output {}",
+                        expected_file.display()
+                    );
+                }
+            }
+        }
+
+        if run_filecheck {
+            let output_path = self.output_base_name().with_extension("mir");
+            let proc_res = self.verify_with_filecheck(&output_path);
+            if !proc_res.status.success() {
+                self.fatal_proc_rec("verification with 'FileCheck' failed", &proc_res);
+            }
+        }
+    }
+
+    fn diff_mir_files(&self, before: PathBuf, after: PathBuf) -> String {
+        let to_full_path = |path: PathBuf| {
+            let full = self.get_mir_dump_dir().join(&path);
+            if !full.exists() {
+                panic!(
+                    "the mir dump file for {} does not exist (requested in {})",
+                    path.display(),
+                    self.testpaths.file.display(),
+                );
+            }
+            full
+        };
+        let before = to_full_path(before);
+        let after = to_full_path(after);
+        debug!("comparing the contents of: {} with {}", before.display(), after.display());
+        let before = fs::read_to_string(before).unwrap();
+        let after = fs::read_to_string(after).unwrap();
+        let before = self.normalize_output(&before, &[]);
+        let after = self.normalize_output(&after, &[]);
+        let mut dumped_string = String::new();
+        for result in diff::lines(&before, &after) {
+            use std::fmt::Write;
+            match result {
+                diff::Result::Left(s) => writeln!(dumped_string, "- {}", s).unwrap(),
+                diff::Result::Right(s) => writeln!(dumped_string, "+ {}", s).unwrap(),
+                diff::Result::Both(s, _) => writeln!(dumped_string, "  {}", s).unwrap(),
+            }
+        }
+        dumped_string
+    }
+
+    fn check_mir_test_timestamp(&self, test_name: &str, output_file: &Path) {
+        let t = |file| fs::metadata(file).unwrap().modified().unwrap();
+        let source_file = &self.testpaths.file;
+        let output_time = t(output_file);
+        let source_time = t(source_file);
+        if source_time > output_time {
+            debug!("source file time: {:?} output file time: {:?}", source_time, output_time);
+            panic!(
+                "test source file `{}` is newer than potentially stale output file `{}`.",
+                source_file.display(),
+                test_name
+            );
+        }
+    }
+}

src/tools/compiletest/src/runtest/js_doc.rs

@@ -0,0 +1,104 @@
+use std::fs;
+
+use super::{ProcRes, ReadFrom, TestCx};
+use crate::util::logv;
+
+impl TestCx<'_> {
+    pub(super) fn run_pretty_test(&self) {
+        if self.props.pp_exact.is_some() {
+            logv(self.config, "testing for exact pretty-printing".to_owned());
+        } else {
+            logv(self.config, "testing for converging pretty-printing".to_owned());
+        }
+
+        let rounds = match self.props.pp_exact {
+            Some(_) => 1,
+            None => 2,
+        };
+
+        let src = fs::read_to_string(&self.testpaths.file).unwrap();
+        let mut srcs = vec![src];
+
+        let mut round = 0;
+        while round < rounds {
+            logv(
+                self.config,
+                format!("pretty-printing round {} revision {:?}", round, self.revision),
+            );
+            let read_from =
+                if round == 0 { ReadFrom::Path } else { ReadFrom::Stdin(srcs[round].to_owned()) };
+
+            let proc_res = self.print_source(read_from, &self.props.pretty_mode);
+            if !proc_res.status.success() {
+                self.fatal_proc_rec(
+                    &format!(
+                        "pretty-printing failed in round {} revision {:?}",
+                        round, self.revision
+                    ),
+                    &proc_res,
+                );
+            }
+
+            let ProcRes { stdout, .. } = proc_res;
+            srcs.push(stdout);
+            round += 1;
+        }
+
+        let mut expected = match self.props.pp_exact {
+            Some(ref file) => {
+                let filepath = self.testpaths.file.parent().unwrap().join(file);
+                fs::read_to_string(&filepath).unwrap()
+            }
+            None => srcs[srcs.len() - 2].clone(),
+        };
+        let mut actual = srcs[srcs.len() - 1].clone();
+
+        if self.props.pp_exact.is_some() {
+            // Now we have to care about line endings
+            let cr = "\r".to_owned();
+            actual = actual.replace(&cr, "");
+            expected = expected.replace(&cr, "");
+        }
+
+        if !self.config.bless {
+            self.compare_source(&expected, &actual);
+        } else if expected != actual {
+            let filepath_buf;
+            let filepath = match &self.props.pp_exact {
+                Some(file) => {
+                    filepath_buf = self.testpaths.file.parent().unwrap().join(file);
+                    &filepath_buf
+                }
+                None => &self.testpaths.file,
+            };
+            fs::write(filepath, &actual).unwrap();
+        }
+
+        // If we're only making sure that the output matches then just stop here
+        if self.props.pretty_compare_only {
+            return;
+        }
+
+        // Finally, let's make sure it actually appears to remain valid code
+        let proc_res = self.typecheck_source(actual);
+        if !proc_res.status.success() {
+            self.fatal_proc_rec("pretty-printed source does not typecheck", &proc_res);
+        }
+
+        if !self.props.pretty_expanded {
+            return;
+        }
+
+        // additionally, run `-Zunpretty=expanded` and try to build it.
+        let proc_res = self.print_source(ReadFrom::Path, "expanded");
+        if !proc_res.status.success() {
+            self.fatal_proc_rec("pretty-printing (expanded) failed", &proc_res);
+        }
+
+        let ProcRes { stdout: expanded_src, .. } = proc_res;
+        let proc_res = self.typecheck_source(expanded_src);
+        if !proc_res.status.success() {
+            self.fatal_proc_rec("pretty-printed source (expanded) does not typecheck", &proc_res);
+        }
+    }
+}

src/tools/compiletest/src/runtest/mir_opt.rs

@@ -0,0 +1,34 @@
+use std::process::Command;
+
+use super::{remove_and_create_dir_all, TestCx};
+
+impl TestCx<'_> {
+    pub(super) fn run_rustdoc_test(&self) {
+        assert!(self.revision.is_none(), "revisions not relevant here");
+
+        let out_dir = self.output_base_dir();
+        remove_and_create_dir_all(&out_dir);
+
+        let proc_res = self.document(&out_dir, &self.testpaths);
+        if !proc_res.status.success() {
+            self.fatal_proc_rec("rustdoc failed!", &proc_res);
+        }
+
+        if self.props.check_test_line_numbers_match {
+            self.check_rustdoc_test_option(proc_res);
+        } else {
+            let root = self.config.find_rust_src_root().unwrap();
+            let mut cmd = Command::new(&self.config.python);
+            cmd.arg(root.join("src/etc/htmldocck.py")).arg(&out_dir).arg(&self.testpaths.file);
+            if self.config.bless {
+                cmd.arg("--bless");
+            }
+            let res = self.run_command_to_procres(&mut cmd);
+            if !res.status.success() {
+                self.fatal_proc_rec_with_ctx("htmldocck failed!", &res, |mut this| {
+                    this.compare_to_default_rustdoc(&out_dir)
+                });
+            }
+        }
+    }
+}

src/tools/compiletest/src/runtest/pretty.rs

@@ -0,0 +1,48 @@
+use std::process::Command;
+
+use super::{remove_and_create_dir_all, TestCx};
+
+impl TestCx<'_> {
+    pub(super) fn run_rustdoc_json_test(&self) {
+        //FIXME: Add bless option.
+
+        assert!(self.revision.is_none(), "revisions not relevant here");
+
+        let out_dir = self.output_base_dir();
+        remove_and_create_dir_all(&out_dir);
+
+        let proc_res = self.document(&out_dir, &self.testpaths);
+        if !proc_res.status.success() {
+            self.fatal_proc_rec("rustdoc failed!", &proc_res);
+        }
+
+        let root = self.config.find_rust_src_root().unwrap();
+        let mut json_out = out_dir.join(self.testpaths.file.file_stem().unwrap());
+        json_out.set_extension("json");
+        let res = self.run_command_to_procres(
+            Command::new(self.config.jsondocck_path.as_ref().unwrap())
+                .arg("--doc-dir")
+                .arg(root.join(&out_dir))
+                .arg("--template")
+                .arg(&self.testpaths.file),
+        );
+
+        if !res.status.success() {
+            self.fatal_proc_rec_with_ctx("jsondocck failed!", &res, |_| {
+                println!("Rustdoc Output:");
+                proc_res.print_info();
+            })
+        }
+
+        let mut json_out = out_dir.join(self.testpaths.file.file_stem().unwrap());
+        json_out.set_extension("json");
+
+        let res = self.run_command_to_procres(
+            Command::new(self.config.jsondoclint_path.as_ref().unwrap()).arg(&json_out),
+        );
+
+        if !res.status.success() {
+            self.fatal_proc_rec("jsondoclint failed!", &res);
+        }
+    }
+}

src/tools/compiletest/src/runtest/run_make.rs

@@ -0,0 +1,233 @@
+use std::collections::HashSet;
+use std::fs::OpenOptions;
+use std::io::Write;
+
+use rustfix::{apply_suggestions, get_suggestions_from_json, Filter};
+use tracing::debug;
+
+use super::{
+    AllowUnused, Emit, ErrorKind, FailMode, LinkToAux, PassMode, TargetLocation, TestCx,
+    TestOutput, Truncated, WillExecute, UI_FIXED,
+};
+use crate::{errors, json};
+
+impl TestCx<'_> {
+    pub(super) fn run_ui_test(&self) {
+        if let Some(FailMode::Build) = self.props.fail_mode {
+            // Make sure a build-fail test cannot fail due to failing analysis (e.g. typeck).
+            let pm = Some(PassMode::Check);
+            let proc_res =
+                self.compile_test_general(WillExecute::No, Emit::Metadata, pm, Vec::new());
+            self.check_if_test_should_compile(&proc_res, pm);
+        }
+
+        let pm = self.pass_mode();
+        let should_run = self.should_run(pm);
+        let emit_metadata = self.should_emit_metadata(pm);
+        let proc_res = self.compile_test(should_run, emit_metadata);
+        self.check_if_test_should_compile(&proc_res, pm);
+        if matches!(proc_res.truncated, Truncated::Yes)
+            && !self.props.dont_check_compiler_stdout
+            && !self.props.dont_check_compiler_stderr
+        {
+            self.fatal_proc_rec(
+                "compiler output got truncated, cannot compare with reference file",
+                &proc_res,
+            );
+        }
+
+        // if the user specified a format in the ui test
+        // print the output to the stderr file, otherwise extract
+        // the rendered error messages from json and print them
+        let explicit = self.props.compile_flags.iter().any(|s| s.contains("--error-format"));
+
+        let expected_fixed = self.load_expected_output(UI_FIXED);
+
+        self.check_and_prune_duplicate_outputs(&proc_res, &[], &[]);
+
+        let mut errors = self.load_compare_outputs(&proc_res, TestOutput::Compile, explicit);
+        let rustfix_input = json::rustfix_diagnostics_only(&proc_res.stderr);
+
+        if self.config.compare_mode.is_some() {
+            // don't test rustfix with nll right now
+        } else if self.config.rustfix_coverage {
+            // Find out which tests have `MachineApplicable` suggestions but are missing
+            // `run-rustfix` or `run-rustfix-only-machine-applicable` headers.
+            //
+            // This will return an empty `Vec` in case the executed test file has a
+            // `compile-flags: --error-format=xxxx` header with a value other than `json`.
+            let suggestions = get_suggestions_from_json(
+                &rustfix_input,
+                &HashSet::new(),
+                Filter::MachineApplicableOnly,
+            )
+            .unwrap_or_default();
+            if !suggestions.is_empty()
+                && !self.props.run_rustfix
+                && !self.props.rustfix_only_machine_applicable
+            {
+                let mut coverage_file_path = self.config.build_base.clone();
+                coverage_file_path.push("rustfix_missing_coverage.txt");
+                debug!("coverage_file_path: {}", coverage_file_path.display());
+
+                let mut file = OpenOptions::new()
+                    .create(true)
+                    .append(true)
+                    .open(coverage_file_path.as_path())
+                    .expect("could not create or open file");
+
+                if let Err(e) = writeln!(file, "{}", self.testpaths.file.display()) {
+                    panic!("couldn't write to {}: {e:?}", coverage_file_path.display());
+                }
+            }
+        } else if self.props.run_rustfix {
+            // Apply suggestions from rustc to the code itself
+            let unfixed_code = self.load_expected_output_from_path(&self.testpaths.file).unwrap();
+            let suggestions = get_suggestions_from_json(
+                &rustfix_input,
+                &HashSet::new(),
+                if self.props.rustfix_only_machine_applicable {
+                    Filter::MachineApplicableOnly
+                } else {
+                    Filter::Everything
+                },
+            )
+            .unwrap();
+            let fixed_code = apply_suggestions(&unfixed_code, &suggestions).unwrap_or_else(|e| {
+                panic!(
+                    "failed to apply suggestions for {:?} with rustfix: {}",
+                    self.testpaths.file, e
+                )
+            });
+
+            errors += self.compare_output("fixed", &fixed_code, &expected_fixed);
+        } else if !expected_fixed.is_empty() {
+            panic!(
+                "the `//@ run-rustfix` directive wasn't found but a `*.fixed` \
+                 file was found"
+            );
+        }
+
+        if errors > 0 {
+            println!("To update references, rerun the tests and pass the `--bless` flag");
+            let relative_path_to_file =
+                self.testpaths.relative_dir.join(self.testpaths.file.file_name().unwrap());
+            println!(
+                "To only update this specific test, also pass `--test-args {}`",
+                relative_path_to_file.display(),
+            );
+            self.fatal_proc_rec(
+                &format!("{} errors occurred comparing output.", errors),
+                &proc_res,
+            );
+        }
+
+        let expected_errors = errors::load_errors(&self.testpaths.file, self.revision);
+
+        if let WillExecute::Yes = should_run {
+            let proc_res = self.exec_compiled_test();
+            let run_output_errors = if self.props.check_run_results {
+                self.load_compare_outputs(&proc_res, TestOutput::Run, explicit)
+            } else {
+                0
+            };
+            if run_output_errors > 0 {
+                self.fatal_proc_rec(
+                    &format!("{} errors occurred comparing run output.", run_output_errors),
+                    &proc_res,
+                );
+            }
+            if self.should_run_successfully(pm) {
+                if !proc_res.status.success() {
+                    self.fatal_proc_rec("test run failed!", &proc_res);
+                }
+            } else if proc_res.status.success() {
+                self.fatal_proc_rec("test run succeeded!", &proc_res);
+            }
+
+            if !self.props.error_patterns.is_empty() || !self.props.regex_error_patterns.is_empty()
+            {
+                // "// error-pattern" comments
+                let output_to_check = self.get_output(&proc_res);
+                self.check_all_error_patterns(&output_to_check, &proc_res, pm);
+            }
+        }
+
+        debug!(
+            "run_ui_test: explicit={:?} config.compare_mode={:?} expected_errors={:?} \
+               proc_res.status={:?} props.error_patterns={:?}",
+            explicit,
+            self.config.compare_mode,
+            expected_errors,
+            proc_res.status,
+            self.props.error_patterns
+        );
+
+        let check_patterns = should_run == WillExecute::No
+            && (!self.props.error_patterns.is_empty()
+                || !self.props.regex_error_patterns.is_empty());
+        if !explicit && self.config.compare_mode.is_none() {
+            let check_annotations = !check_patterns || !expected_errors.is_empty();
+
+            if check_annotations {
+                // "//~ERROR comments"
+                self.check_expected_errors(expected_errors, &proc_res);
+            }
+        } else if explicit && !expected_errors.is_empty() {
+            let msg = format!(
+                "line {}: cannot combine `--error-format` with {} annotations; use `error-pattern` instead",
+                expected_errors[0].line_num,
+                expected_errors[0].kind.unwrap_or(ErrorKind::Error),
+            );
+            self.fatal(&msg);
+        }
+        if check_patterns {
+            // "// error-pattern" comments
+            let output_to_check = self.get_output(&proc_res);
+            self.check_all_error_patterns(&output_to_check, &proc_res, pm);
+        }
+
+        if self.props.run_rustfix && self.config.compare_mode.is_none() {
+            // And finally, compile the fixed code and make sure it both
+            // succeeds and has no diagnostics.
+            let mut rustc = self.make_compile_args(
+                &self.expected_output_path(UI_FIXED),
+                TargetLocation::ThisFile(self.make_exe_name()),
+                emit_metadata,
+                AllowUnused::No,
+                LinkToAux::Yes,
+                Vec::new(),
+            );
+
+            // If a test is revisioned, it's fixed source file can be named "a.foo.fixed", which,
+            // well, "a.foo" isn't a valid crate name. So we explicitly mangle the test name
+            // (including the revision) here to avoid the test writer having to manually specify a
+            // `#![crate_name = "..."]` as a workaround. This is okay since we're only checking if
+            // the fixed code is compilable.
+            if self.revision.is_some() {
+                let crate_name =
+                    self.testpaths.file.file_stem().expect("test must have a file stem");
+                // crate name must be alphanumeric or `_`.
+                let crate_name =
+                    crate_name.to_str().expect("crate name implies file name must be valid UTF-8");
+                // replace `a.foo` -> `a__foo` for crate name purposes.
+                // replace `revision-name-with-dashes` -> `revision_name_with_underscore`
+                let crate_name = crate_name.replace('.', "__");
+                let crate_name = crate_name.replace('-', "_");
+                rustc.arg("--crate-name");
+                rustc.arg(crate_name);
+            }
+
+            let res = self.compose_and_run_compiler(rustc, None, self.testpaths);
+            if !res.status.success() {
+                self.fatal_proc_rec("failed to compile fixed code", &res);
+            }
+            if !res.stderr.is_empty()
+                && !self.props.rustfix_only_machine_applicable
+                && !json::rustfix_diagnostics_only(&res.stderr).is_empty()
+            {
+                self.fatal_proc_rec("fixed code is still producing diagnostics", &res);
+            }
+        }
+    }
+}

src/tools/compiletest/src/runtest/rustdoc.rs

@@ -0,0 +1,34 @@
+use super::{Emit, TestCx, WillExecute};
+
+impl TestCx<'_> {
+    pub(super) fn run_valgrind_test(&self) {
+        assert!(self.revision.is_none(), "revisions not relevant here");
+
+        // FIXME(jieyouxu): does this really make any sense? If a valgrind test isn't testing
+        // valgrind, what is it even testing?
+        if self.config.valgrind_path.is_none() {
+            assert!(!self.config.force_valgrind);
+            return self.run_rpass_test();
+        }
+
+        let should_run = self.run_if_enabled();
+        let mut proc_res = self.compile_test(should_run, Emit::None);
+
+        if !proc_res.status.success() {
+            self.fatal_proc_rec("compilation failed!", &proc_res);
+        }
+
+        if let WillExecute::Disabled = should_run {
+            return;
+        }
+
+        let mut new_config = self.config.clone();
+        new_config.runner = new_config.valgrind_path.clone();
+        let new_cx = TestCx { config: &new_config, ..*self };
+        proc_res = new_cx.exec_compiled_test();
+
+        if !proc_res.status.success() {
+            self.fatal_proc_rec("test run failed!", &proc_res);
+        }
+    }
+}