diff --git a/Cargo.lock b/Cargo.lock index 7dc3628f4a9f..8ae585749849 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3591,6 +3591,7 @@ dependencies = [ "cranelift-reader", "cranelift-wasm", "libfuzzer-sys", + "once_cell", "proc-macro2", "quote", "rand 0.8.5", diff --git a/cranelift/codegen/src/ir/trapcode.rs b/cranelift/codegen/src/ir/trapcode.rs index 3114114f6dc6..590c82a8b3df 100644 --- a/cranelift/codegen/src/ir/trapcode.rs +++ b/cranelift/codegen/src/ir/trapcode.rs @@ -53,6 +53,25 @@ pub enum TrapCode { User(u16), } +impl TrapCode { + /// Returns a slice of all traps except `TrapCode::User` traps + pub const fn non_user_traps() -> &'static [TrapCode] { + &[ + TrapCode::StackOverflow, + TrapCode::HeapOutOfBounds, + TrapCode::HeapMisaligned, + TrapCode::TableOutOfBounds, + TrapCode::IndirectCallToNull, + TrapCode::BadSignature, + TrapCode::IntegerOverflow, + TrapCode::IntegerDivisionByZero, + TrapCode::BadConversionToInteger, + TrapCode::UnreachableCodeReached, + TrapCode::Interrupt, + ] + } +} + impl Display for TrapCode { fn fmt(&self, f: &mut Formatter) -> fmt::Result { use self::TrapCode::*; @@ -102,24 +121,9 @@ mod tests { use super::*; use alloc::string::ToString; - // Everything but user-defined codes. - const CODES: [TrapCode; 11] = [ - TrapCode::StackOverflow, - TrapCode::HeapOutOfBounds, - TrapCode::HeapMisaligned, - TrapCode::TableOutOfBounds, - TrapCode::IndirectCallToNull, - TrapCode::BadSignature, - TrapCode::IntegerOverflow, - TrapCode::IntegerDivisionByZero, - TrapCode::BadConversionToInteger, - TrapCode::UnreachableCodeReached, - TrapCode::Interrupt, - ]; - #[test] fn display() { - for r in &CODES { + for r in TrapCode::non_user_traps() { let tc = *r; assert_eq!(tc.to_string().parse(), Ok(tc)); } diff --git a/cranelift/interpreter/src/step.rs b/cranelift/interpreter/src/step.rs index 66fe43017d69..8e93fdf1b981 100644 --- a/cranelift/interpreter/src/step.rs +++ b/cranelift/interpreter/src/step.rs @@ -1385,7 +1385,7 @@ impl<'a, V> ControlFlow<'a, V> { } } -#[derive(Error, Debug, PartialEq)] +#[derive(Error, Debug, PartialEq, Eq, Hash)] pub enum CraneliftTrap { #[error("user code: {0}")] User(TrapCode), diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 4fbd6afc1f38..63168f194026 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -9,6 +9,7 @@ cargo-fuzz = true [dependencies] anyhow = { workspace = true } +once_cell = { workspace = true } cranelift-codegen = { workspace = true, features = ["incremental-cache"] } cranelift-reader = { workspace = true } cranelift-wasm = { workspace = true } diff --git a/fuzz/fuzz_targets/cranelift-fuzzgen.rs b/fuzz/fuzz_targets/cranelift-fuzzgen.rs index 559b70d5314d..e063f21ecd64 100644 --- a/fuzz/fuzz_targets/cranelift-fuzzgen.rs +++ b/fuzz/fuzz_targets/cranelift-fuzzgen.rs @@ -1,9 +1,13 @@ #![no_main] use libfuzzer_sys::fuzz_target; +use once_cell::sync::Lazy; +use std::collections::HashMap; +use std::sync::atomic::AtomicU64; +use std::sync::atomic::Ordering; use cranelift_codegen::data_value::DataValue; -use cranelift_codegen::ir::LibCall; +use cranelift_codegen::ir::{LibCall, TrapCode}; use cranelift_codegen::settings; use cranelift_codegen::settings::Configurable; use cranelift_filetests::function_runner::{TestFileCompiler, Trampoline}; @@ -19,6 +23,87 @@ use smallvec::smallvec; const INTERPRETER_FUEL: u64 = 4096; +/// Gather statistics about the fuzzer executions +struct Statistics { + /// Inputs that fuzzgen can build a function with + /// This is also how many compiles we executed + pub valid_inputs: AtomicU64, + + /// Total amount of runs that we tried in the interpreter + /// One fuzzer input can have many runs + pub total_runs: AtomicU64, + /// How many runs were successful? + /// This is also how many runs were run in the backend + pub run_result_success: AtomicU64, + /// How many runs resulted in a timeout? + pub run_result_timeout: AtomicU64, + /// How many runs ended with a trap? + pub run_result_trap: HashMap, +} + +impl Statistics { + pub fn print(&self, valid_inputs: u64) { + // We get valid_inputs as a param since we already loaded it previously. + let total_runs = self.total_runs.load(Ordering::SeqCst); + let run_result_success = self.run_result_success.load(Ordering::SeqCst); + let run_result_timeout = self.run_result_timeout.load(Ordering::SeqCst); + + println!("== FuzzGen Statistics ===================="); + println!("Valid Inputs: {}", valid_inputs); + println!("Total Runs: {}", total_runs); + println!( + "Successful Runs: {} ({:.1}% of Total Runs)", + run_result_success, + (run_result_success as f64 / total_runs as f64) * 100.0 + ); + println!( + "Timed out Runs: {} ({:.1}% of Total Runs)", + run_result_timeout, + (run_result_timeout as f64 / total_runs as f64) * 100.0 + ); + println!("Traps:"); + // Load and filter out empty trap codes. + let mut traps = self + .run_result_trap + .iter() + .map(|(trap, count)| (trap, count.load(Ordering::SeqCst))) + .filter(|(_, count)| *count != 0) + .collect::>(); + + // Sort traps by count in a descending order + traps.sort_by_key(|(_, count)| -(*count as i64)); + + for (trap, count) in traps.into_iter() { + println!( + "\t{}: {} ({:.1}% of Total Runs)", + trap, + count, + (count as f64 / total_runs as f64) * 100.0 + ); + } + } +} + +impl Default for Statistics { + fn default() -> Self { + // Pre-Register all trap codes since we can't modify this hashmap atomically. + let mut run_result_trap = HashMap::new(); + run_result_trap.insert(CraneliftTrap::Debug, AtomicU64::new(0)); + run_result_trap.insert(CraneliftTrap::Resumable, AtomicU64::new(0)); + for trapcode in TrapCode::non_user_traps() { + run_result_trap.insert(CraneliftTrap::User(*trapcode), AtomicU64::new(0)); + } + + Self { + valid_inputs: AtomicU64::new(0), + total_runs: AtomicU64::new(0), + run_result_success: AtomicU64::new(0), + run_result_timeout: AtomicU64::new(0), + run_result_trap, + } + } +} + #[derive(Debug)] enum RunResult { Success(Vec), @@ -79,7 +164,15 @@ fn build_interpreter(testcase: &TestCase) -> Interpreter { interpreter } +static STATISTICS: Lazy = Lazy::new(Statistics::default); + fuzz_target!(|testcase: TestCase| { + // Periodically print statistics + let valid_inputs = STATISTICS.valid_inputs.fetch_add(1, Ordering::SeqCst); + if valid_inputs != 0 && valid_inputs % 10000 == 0 { + STATISTICS.print(valid_inputs); + } + // Native fn let flags = { let mut builder = settings::builder(); @@ -101,13 +194,18 @@ fuzz_target!(|testcase: TestCase| { let trampoline = compiled.get_trampoline(&testcase.func).unwrap(); for args in &testcase.inputs { + STATISTICS.total_runs.fetch_add(1, Ordering::SeqCst); + // We rebuild the interpreter every run so that we don't accidentally carry over any state // between runs, such as fuel remaining. let mut interpreter = build_interpreter(&testcase); let int_res = run_in_interpreter(&mut interpreter, args); match int_res { - RunResult::Success(_) => {} - RunResult::Trap(_) => { + RunResult::Success(_) => { + STATISTICS.run_result_success.fetch_add(1, Ordering::SeqCst); + } + RunResult::Trap(trap) => { + STATISTICS.run_result_trap[&trap].fetch_add(1, Ordering::SeqCst); // If this input traps, skip it and continue trying other inputs // for this function. We've already compiled it anyway. // @@ -120,6 +218,7 @@ fuzz_target!(|testcase: TestCase| { RunResult::Timeout => { // We probably generated an infinite loop, we should drop this entire input. // We could `continue` like we do on traps, but timeouts are *really* expensive. + STATISTICS.run_result_timeout.fetch_add(1, Ordering::SeqCst); return; } RunResult::Error(_) => panic!("interpreter failed: {:?}", int_res),