feat: parse perf file for memmap events instead of relying on /proc/pid/maps

Author: GuillaumeLagrange

Cargo.lock

@@ -11,7 +11,7 @@ path = "src/main.rs"
 
 
 [dependencies]
-anyhow = "1.0.75"
+anyhow = { workspace = true }
 clap = { workspace = true }
 itertools = "0.11.0"
 lazy_static = "1.4.0"
@@ -26,8 +26,8 @@ reqwest = { version = "0.11.22", features = [
 ] }
 reqwest-middleware = "0.2.4"
 reqwest-retry = "0.3.0"
-serde = { version = "1.0.192", features = ["derive"] }
-serde_json = { version = "1.0.108", features = ["preserve_order"] }
+serde = { workspace = true }
+serde_json = { workspace = true }
 url = "2.4.1"
 sha256 = "1.4.0"
 tokio = { version = "1", features = ["macros", "rt"] }
@@ -51,10 +51,11 @@ async-trait = "0.1.82"
 libc = "0.2.171"
 bincode = "1.3.3"
 object = "0.36.7"
-linux-perf-data = "0.11.0"
+# TODO: Make this repo public
+linux-perf-data = { git = "ssh://[email protected]/CodSpeedHQ/linux-perf-data.git", branch = "feat/support-perf-pipe-data-parsing" }
 debugid = "0.8.0"
 memmap2 = "0.9.5"
-nix = { version = "0.29.0", features = ["fs", "time", "user"] }
+nix = { workspace = true, features = ["fs", "time", "user"] }
 futures = "0.3.31"
 runner-shared = { path = "crates/runner-shared" }
 shellexpand = { version = "3.1.1", features = ["tilde"] }
@@ -79,7 +80,11 @@ shell-quote = "0.7.2"
 members = ["crates/exec-harness", "crates/runner-shared"]
 
 [workspace.dependencies]
+anyhow = "1.0.75"
 clap = { version = "4.4.8", features = ["derive", "env", "color"] }
+nix = "0.29.0"
+serde = { version = "1.0.192", features = ["derive"] }
+serde_json = { version = "1.0.108", features = ["preserve_order"] }
 
 [workspace.metadata.release]
 sign-tag = true

Cargo.toml

@@ -11,26 +11,26 @@ use crate::executor::valgrind::helpers::perf_maps::harvest_perf_maps_for_pids;
 use crate::executor::wall_time::perf::debug_info::ProcessDebugInfo;
 use crate::executor::wall_time::perf::jit_dump::harvest_perf_jit_for_pids;
 use crate::executor::wall_time::perf::perf_executable::get_working_perf_executable;
-use crate::executor::wall_time::perf::unwind_data::UnwindDataExt;
 use crate::prelude::*;
 use crate::run::UnwindingMode;
 use anyhow::Context;
 use fifo::{PerfFifo, RunnerFifo};
 use libc::pid_t;
 use nix::sys::time::TimeValLike;
 use nix::time::clock_gettime;
-use perf_map::ProcessSymbols;
+use parse_perf_file::MemmapRecordsOutput;
 use runner_shared::debug_info::ModuleDebugInfo;
 use runner_shared::fifo::Command as FifoCommand;
 use runner_shared::fifo::MarkerType;
 use runner_shared::metadata::PerfMetadata;
-use runner_shared::unwind_data::UnwindData;
 use std::collections::HashSet;
 use std::path::Path;
+use std::path::PathBuf;
 use std::time::Duration;
 use std::{cell::OnceCell, collections::HashMap, process::ExitStatus};
 
 mod jit_dump;
+mod parse_perf_file;
 mod setup;
 
 pub mod debug_info;
@@ -146,8 +146,8 @@ impl PerfRunner {
         ]);
         cmd_builder.wrap_with(perf_wrapper_builder);
 
-        // Copy the perf data to the profile folder
-        let perf_data_file_path = profile_folder.join(PERF_DATA_FILE_NAME);
+        // Output the perf data to the profile folder
+        let perf_data_file_path = get_perf_file_path(profile_folder);
 
         let raw_command = format!(
             "set -o pipefail && {} | cat > {}",
@@ -203,86 +203,12 @@ impl PerfRunner {
         Ok(())
     }
 
-    #[cfg(target_os = "linux")]
-    fn process_memory_mappings(
-        pid: pid_t,
-        symbols_by_pid: &mut HashMap<pid_t, ProcessSymbols>,
-        unwind_data_by_pid: &mut HashMap<pid_t, Vec<UnwindData>>,
-    ) -> anyhow::Result<()> {
-        use procfs::process::MMPermissions;
-
-        let bench_proc =
-            procfs::process::Process::new(pid as _).expect("Failed to find benchmark process");
-        let exe_maps = bench_proc.maps().expect("Failed to read /proc/{pid}/maps");
-
-        if is_codspeed_debug_enabled() {
-            debug!("Process memory mappings for PID {pid}:");
-            for map in exe_maps.iter().sorted_by_key(|m| m.address.0) {
-                let (base_addr, end_addr) = map.address;
-                debug!(
-                    "  {:016x}-{:016x} {:08x} {:?} {:?} ",
-                    base_addr, end_addr, map.offset, map.pathname, map.perms,
-                );
-            }
-        }
-
-        for map in &exe_maps {
-            let page_offset = map.offset;
-            let (base_addr, end_addr) = map.address;
-            let path = match &map.pathname {
-                procfs::process::MMapPath::Path(path) => Some(path.clone()),
-                _ => None,
-            };
-
-            let Some(path) = &path else {
-                if map.perms.contains(MMPermissions::EXECUTE) {
-                    debug!("Found executable mapping without path: {base_addr:x} - {end_addr:x}");
-                }
-                continue;
-            };
-
-            if !map.perms.contains(MMPermissions::EXECUTE) {
-                continue;
-            }
-
-            symbols_by_pid
-                .entry(pid)
-                .or_insert(ProcessSymbols::new(pid))
-                .add_mapping(pid, path, base_addr, end_addr, map.offset);
-            debug!("Added mapping for module {path:?}");
-
-            match UnwindData::new(
-                path.to_string_lossy().as_bytes(),
-                page_offset,
-                base_addr,
-                end_addr,
-                None,
-            ) {
-                Ok(unwind_data) => {
-                    unwind_data_by_pid.entry(pid).or_default().push(unwind_data);
-                    debug!("Added unwind data for {path:?} ({base_addr:x} - {end_addr:x})");
-                }
-                Err(error) => {
-                    debug!(
-                        "Failed to create unwind data for module {}: {}",
-                        path.display(),
-                        error
-                    );
-                }
-            }
-        }
-
-        Ok(())
-    }
-
     async fn handle_fifo(
         mut runner_fifo: RunnerFifo,
         mut perf_fifo: PerfFifo,
     ) -> anyhow::Result<BenchmarkData> {
         let mut bench_order_by_timestamp = Vec::<(u64, String)>::new();
         let mut bench_pids = HashSet::<pid_t>::new();
-        let mut symbols_by_pid = HashMap::<pid_t, ProcessSymbols>::new();
-        let mut unwind_data_by_pid = HashMap::<pid_t, Vec<UnwindData>>::new();
         let mut markers = Vec::<MarkerType>::new();
 
         let mut integration = None;
@@ -319,19 +245,9 @@ impl PerfRunner {
 
             match cmd {
                 FifoCommand::CurrentBenchmark { pid, uri } => {
-                    bench_order_by_timestamp.push((current_time(), uri));
+                    bench_order_by_timestamp.push((current_time(), uri.clone()));
                     bench_pids.insert(pid);
 
-                    #[cfg(target_os = "linux")]
-                    if !symbols_by_pid.contains_key(&pid) && !unwind_data_by_pid.contains_key(&pid)
-                    {
-                        Self::process_memory_mappings(
-                            pid,
-                            &mut symbols_by_pid,
-                            &mut unwind_data_by_pid,
-                        )?;
-                    }
-
                     runner_fifo.send_cmd(FifoCommand::Ack).await?;
                 }
                 FifoCommand::StartBenchmark => {
@@ -400,8 +316,6 @@ impl PerfRunner {
             integration,
             uri_by_ts: bench_order_by_timestamp,
             bench_pids,
-            symbols_by_pid,
-            unwind_data_by_pid,
             markers,
         })
     }
@@ -413,35 +327,45 @@ pub struct BenchmarkData {
 
     uri_by_ts: Vec<(u64, String)>,
     bench_pids: HashSet<pid_t>,
-    symbols_by_pid: HashMap<pid_t, ProcessSymbols>,
-    unwind_data_by_pid: HashMap<pid_t, Vec<UnwindData>>,
     markers: Vec<MarkerType>,
 }
 
 #[derive(Debug)]
 pub enum BenchmarkDataSaveError {
     MissingIntegration,
+    FailedToParsePerfFile,
 }
 
 impl BenchmarkData {
     pub fn save_to<P: AsRef<std::path::Path>>(
         &self,
         path: P,
     ) -> Result<(), BenchmarkDataSaveError> {
-        for proc_sym in self.symbols_by_pid.values() {
+        debug!("Reading perf data from file for mmap extraction");
+        let perf_file_path = get_perf_file_path(&path);
+
+        let MemmapRecordsOutput {
+            symbols_by_pid,
+            unwind_data_by_pid,
+        } = parse_perf_file::parse_for_memmap2(&perf_file_path).map_err(|e| {
+            error!("Failed to parse perf file: {e}");
+            BenchmarkDataSaveError::FailedToParsePerfFile
+        })?;
+
+        for proc_sym in symbols_by_pid.values() {
             proc_sym.save_to(&path).unwrap();
         }
 
         // Collect debug info for each process by looking up file/line for symbols
         let mut debug_info_by_pid = HashMap::<i32, Vec<ModuleDebugInfo>>::new();
-        for (pid, proc_sym) in &self.symbols_by_pid {
+        for (pid, proc_sym) in &symbols_by_pid {
             debug_info_by_pid
                 .entry(*pid)
                 .or_default()
                 .extend(ProcessDebugInfo::new(proc_sym).modules());
         }
 
-        for (pid, modules) in &self.unwind_data_by_pid {
+        for (pid, modules) in &unwind_data_by_pid {
             for module in modules {
                 module.save_to(&path, *pid).unwrap();
             }
@@ -459,7 +383,7 @@ impl BenchmarkData {
 
                 // Check if any of the ignored modules has been loaded in the process
                 for ignore_path in get_objects_path_to_ignore() {
-                    for proc in self.symbols_by_pid.values() {
+                    for proc in symbols_by_pid.values() {
                         if let Some(mapping) = proc.module_mapping(&ignore_path) {
                             let (Some((base_addr, _)), Some((_, end_addr))) = (
                                 mapping.iter().min_by_key(|(base_addr, _)| base_addr),
@@ -474,16 +398,15 @@ impl BenchmarkData {
                 }
 
                 // When python is statically linked, we'll not find it in the ignored modules. Add it manually:
-                let python_modules = self.symbols_by_pid.values().filter_map(|proc| {
+                let python_modules = symbols_by_pid.values().filter_map(|proc| {
                     proc.loaded_modules().find(|path| {
                         path.file_name()
                             .map(|name| name.to_string_lossy().starts_with("python"))
                             .unwrap_or(false)
                     })
                 });
                 for path in python_modules {
-                    if let Some(mapping) = self
-                        .symbols_by_pid
+                    if let Some(mapping) = symbols_by_pid
                         .values()
                         .find_map(|proc| proc.module_mapping(path))
                     {
@@ -507,3 +430,7 @@ impl BenchmarkData {
         Ok(())
     }
 }
+
+fn get_perf_file_path<P: AsRef<Path>>(profile_folder: P) -> PathBuf {
+    profile_folder.as_ref().join(PERF_DATA_FILE_NAME)
+}