|
| 1 | +//! Run all tests in a project, similar to `cargo test`, but using the mir interpreter. |
| 2 | +
|
| 3 | +use std::{ |
| 4 | + cell::RefCell, collections::HashMap, fs::read_to_string, panic::AssertUnwindSafe, path::PathBuf, |
| 5 | +}; |
| 6 | + |
| 7 | +use hir::Crate; |
| 8 | +use ide::{AnalysisHost, Change, DiagnosticCode, DiagnosticsConfig}; |
| 9 | +use profile::StopWatch; |
| 10 | +use project_model::{CargoConfig, ProjectWorkspace, RustLibSource, Sysroot}; |
| 11 | + |
| 12 | +use load_cargo::{load_workspace, LoadCargoConfig, ProcMacroServerChoice}; |
| 13 | +use triomphe::Arc; |
| 14 | +use vfs::{AbsPathBuf, FileId}; |
| 15 | +use walkdir::WalkDir; |
| 16 | + |
| 17 | +use crate::cli::{flags, report_metric, Result}; |
| 18 | + |
| 19 | +struct Tester { |
| 20 | + host: AnalysisHost, |
| 21 | + root_file: FileId, |
| 22 | + pass_count: u64, |
| 23 | + ignore_count: u64, |
| 24 | + fail_count: u64, |
| 25 | + stopwatch: StopWatch, |
| 26 | +} |
| 27 | + |
| 28 | +fn string_to_diagnostic_code_leaky(code: &str) -> DiagnosticCode { |
| 29 | + thread_local! { |
| 30 | + static LEAK_STORE: RefCell<HashMap<String, DiagnosticCode>> = RefCell::new(HashMap::new()); |
| 31 | + } |
| 32 | + LEAK_STORE.with_borrow_mut(|s| match s.get(code) { |
| 33 | + Some(c) => *c, |
| 34 | + None => { |
| 35 | + let v = DiagnosticCode::RustcHardError(format!("E{code}").leak()); |
| 36 | + s.insert(code.to_owned(), v); |
| 37 | + v |
| 38 | + } |
| 39 | + }) |
| 40 | +} |
| 41 | + |
| 42 | +fn detect_errors_from_rustc_stderr_file(p: PathBuf) -> HashMap<DiagnosticCode, usize> { |
| 43 | + let text = read_to_string(p).unwrap(); |
| 44 | + let mut result = HashMap::new(); |
| 45 | + { |
| 46 | + let mut text = &*text; |
| 47 | + while let Some(p) = text.find("error[E") { |
| 48 | + text = &text[p + 7..]; |
| 49 | + let code = string_to_diagnostic_code_leaky(&text[..4]); |
| 50 | + *result.entry(code).or_insert(0) += 1; |
| 51 | + } |
| 52 | + } |
| 53 | + result |
| 54 | +} |
| 55 | + |
| 56 | +impl Tester { |
| 57 | + fn new() -> Result<Self> { |
| 58 | + let tmp_file = AbsPathBuf::assert("/tmp/ra-rustc-test.rs".into()); |
| 59 | + std::fs::write(&tmp_file, "")?; |
| 60 | + let mut cargo_config = CargoConfig::default(); |
| 61 | + cargo_config.sysroot = Some(RustLibSource::Discover); |
| 62 | + let workspace = ProjectWorkspace::DetachedFiles { |
| 63 | + files: vec![tmp_file.clone()], |
| 64 | + sysroot: Ok( |
| 65 | + Sysroot::discover(tmp_file.parent().unwrap(), &cargo_config.extra_env).unwrap() |
| 66 | + ), |
| 67 | + rustc_cfg: vec![], |
| 68 | + }; |
| 69 | + let load_cargo_config = LoadCargoConfig { |
| 70 | + load_out_dirs_from_check: false, |
| 71 | + with_proc_macro_server: ProcMacroServerChoice::Sysroot, |
| 72 | + prefill_caches: false, |
| 73 | + }; |
| 74 | + let (host, _vfs, _proc_macro) = |
| 75 | + load_workspace(workspace, &cargo_config.extra_env, &load_cargo_config)?; |
| 76 | + let db = host.raw_database(); |
| 77 | + let krates = Crate::all(db); |
| 78 | + let root_crate = krates.iter().cloned().find(|krate| krate.origin(db).is_local()).unwrap(); |
| 79 | + let root_file = root_crate.root_file(db); |
| 80 | + Ok(Self { |
| 81 | + host, |
| 82 | + root_file, |
| 83 | + pass_count: 0, |
| 84 | + ignore_count: 0, |
| 85 | + fail_count: 0, |
| 86 | + stopwatch: StopWatch::start(), |
| 87 | + }) |
| 88 | + } |
| 89 | + |
| 90 | + fn test(&mut self, p: PathBuf) { |
| 91 | + if p.parent().unwrap().file_name().unwrap() == "auxiliary" { |
| 92 | + // These are not tests |
| 93 | + return; |
| 94 | + } |
| 95 | + if IGNORED_TESTS.iter().any(|ig| p.file_name().is_some_and(|x| x == *ig)) { |
| 96 | + println!("{p:?} IGNORE"); |
| 97 | + self.ignore_count += 1; |
| 98 | + return; |
| 99 | + } |
| 100 | + let stderr_path = p.with_extension("stderr"); |
| 101 | + let expected = if stderr_path.exists() { |
| 102 | + detect_errors_from_rustc_stderr_file(stderr_path) |
| 103 | + } else { |
| 104 | + HashMap::new() |
| 105 | + }; |
| 106 | + let text = read_to_string(&p).unwrap(); |
| 107 | + let mut change = Change::new(); |
| 108 | + // Ignore unstable tests, since they move too fast and we do not intend to support all of them. |
| 109 | + let mut ignore_test = text.contains("#![feature"); |
| 110 | + // Ignore test with extern crates, as this infra don't support them yet. |
| 111 | + ignore_test |= text.contains("// aux-build:") || text.contains("// aux-crate:"); |
| 112 | + // Ignore test with extern modules similarly. |
| 113 | + ignore_test |= text.contains("mod "); |
| 114 | + // These should work, but they don't, and I don't know why, so ignore them. |
| 115 | + ignore_test |= text.contains("extern crate proc_macro"); |
| 116 | + let should_have_no_error = text.contains("// check-pass") |
| 117 | + || text.contains("// build-pass") |
| 118 | + || text.contains("// run-pass"); |
| 119 | + change.change_file(self.root_file, Some(Arc::from(text))); |
| 120 | + self.host.apply_change(change); |
| 121 | + let diagnostic_config = DiagnosticsConfig::test_sample(); |
| 122 | + let diags = self |
| 123 | + .host |
| 124 | + .analysis() |
| 125 | + .diagnostics(&diagnostic_config, ide::AssistResolveStrategy::None, self.root_file) |
| 126 | + .unwrap(); |
| 127 | + let mut actual = HashMap::new(); |
| 128 | + for diag in diags { |
| 129 | + if !matches!(diag.code, DiagnosticCode::RustcHardError(_)) { |
| 130 | + continue; |
| 131 | + } |
| 132 | + if !should_have_no_error && !SUPPORTED_DIAGNOSTICS.contains(&diag.code) { |
| 133 | + continue; |
| 134 | + } |
| 135 | + *actual.entry(diag.code).or_insert(0) += 1; |
| 136 | + } |
| 137 | + // Ignore tests with diagnostics that we don't emit. |
| 138 | + ignore_test |= expected.keys().any(|k| !SUPPORTED_DIAGNOSTICS.contains(k)); |
| 139 | + if ignore_test { |
| 140 | + println!("{p:?} IGNORE"); |
| 141 | + self.ignore_count += 1; |
| 142 | + } else if actual == expected { |
| 143 | + println!("{p:?} PASS"); |
| 144 | + self.pass_count += 1; |
| 145 | + } else { |
| 146 | + println!("{p:?} FAIL"); |
| 147 | + println!("actual (r-a) = {:?}", actual); |
| 148 | + println!("expected (rustc) = {:?}", expected); |
| 149 | + self.fail_count += 1; |
| 150 | + } |
| 151 | + } |
| 152 | + |
| 153 | + fn report(&mut self) { |
| 154 | + println!( |
| 155 | + "Pass count = {}, Fail count = {}, Ignore count = {}", |
| 156 | + self.pass_count, self.fail_count, self.ignore_count |
| 157 | + ); |
| 158 | + println!("Testing time and memory = {}", self.stopwatch.elapsed()); |
| 159 | + report_metric("rustc failed tests", self.fail_count, "#"); |
| 160 | + report_metric("rustc testing time", self.stopwatch.elapsed().time.as_millis() as u64, "ms"); |
| 161 | + } |
| 162 | +} |
| 163 | + |
| 164 | +/// These tests break rust-analyzer (either by panicking or hanging) so we should ignore them. |
| 165 | +const IGNORED_TESTS: &[&str] = &[ |
| 166 | + "trait-with-missing-associated-type-restriction.rs", // #15646 |
| 167 | + "trait-with-missing-associated-type-restriction-fixable.rs", // #15646 |
| 168 | + "resolve-self-in-impl.rs", |
| 169 | + "basic.rs", // ../rust/tests/ui/associated-type-bounds/return-type-notation/basic.rs |
| 170 | + "issue-26056.rs", |
| 171 | + "float-field.rs", |
| 172 | + "invalid_operator_trait.rs", |
| 173 | + "type-alias-impl-trait-assoc-dyn.rs", |
| 174 | + "deeply-nested_closures.rs", // exponential time |
| 175 | + "hang-on-deeply-nested-dyn.rs", // exponential time |
| 176 | + "dyn-rpit-and-let.rs", // unexpected free variable with depth `^1.0` with outer binder ^0 |
| 177 | + "issue-16098.rs", // Huge recursion limit for macros? |
| 178 | + "issue-83471.rs", // crates/hir-ty/src/builder.rs:78:9: assertion failed: self.remaining() > 0 |
| 179 | +]; |
| 180 | + |
| 181 | +const SUPPORTED_DIAGNOSTICS: &[DiagnosticCode] = &[ |
| 182 | + DiagnosticCode::RustcHardError("E0023"), |
| 183 | + DiagnosticCode::RustcHardError("E0046"), |
| 184 | + DiagnosticCode::RustcHardError("E0063"), |
| 185 | + DiagnosticCode::RustcHardError("E0107"), |
| 186 | + DiagnosticCode::RustcHardError("E0117"), |
| 187 | + DiagnosticCode::RustcHardError("E0133"), |
| 188 | + DiagnosticCode::RustcHardError("E0210"), |
| 189 | + DiagnosticCode::RustcHardError("E0268"), |
| 190 | + DiagnosticCode::RustcHardError("E0308"), |
| 191 | + DiagnosticCode::RustcHardError("E0384"), |
| 192 | + DiagnosticCode::RustcHardError("E0407"), |
| 193 | + DiagnosticCode::RustcHardError("E0432"), |
| 194 | + DiagnosticCode::RustcHardError("E0451"), |
| 195 | + DiagnosticCode::RustcHardError("E0507"), |
| 196 | + DiagnosticCode::RustcHardError("E0583"), |
| 197 | + DiagnosticCode::RustcHardError("E0559"), |
| 198 | + DiagnosticCode::RustcHardError("E0616"), |
| 199 | + DiagnosticCode::RustcHardError("E0618"), |
| 200 | + DiagnosticCode::RustcHardError("E0624"), |
| 201 | + DiagnosticCode::RustcHardError("E0774"), |
| 202 | + DiagnosticCode::RustcHardError("E0767"), |
| 203 | + DiagnosticCode::RustcHardError("E0777"), |
| 204 | +]; |
| 205 | + |
| 206 | +impl flags::RustcTests { |
| 207 | + pub fn run(self) -> Result<()> { |
| 208 | + let mut tester = Tester::new()?; |
| 209 | + let walk_dir = WalkDir::new(self.rustc_repo.join("tests/ui")); |
| 210 | + for i in walk_dir { |
| 211 | + let i = i?; |
| 212 | + let p = i.into_path(); |
| 213 | + if let Some(f) = &self.filter { |
| 214 | + if !p.as_os_str().to_string_lossy().contains(f) { |
| 215 | + continue; |
| 216 | + } |
| 217 | + } |
| 218 | + if p.extension().map_or(true, |x| x != "rs") { |
| 219 | + continue; |
| 220 | + } |
| 221 | + if let Err(e) = std::panic::catch_unwind({ |
| 222 | + let tester = AssertUnwindSafe(&mut tester); |
| 223 | + let p = p.clone(); |
| 224 | + move || { |
| 225 | + let tester = tester; |
| 226 | + tester.0.test(p); |
| 227 | + } |
| 228 | + }) { |
| 229 | + println!("panic detected at test {:?}", p); |
| 230 | + std::panic::resume_unwind(e); |
| 231 | + } |
| 232 | + } |
| 233 | + tester.report(); |
| 234 | + Ok(()) |
| 235 | + } |
| 236 | +} |
0 commit comments