Merge pull request #1411 from rust-lang/bytehound

Add bytehound profiler

Author: Kobzol

.github/workflows/ci.yml

@@ -137,6 +137,13 @@ jobs:
           sudo apt-get install -y valgrind
           cargo install --version 0.4.12 cargo-llvm-lines
 
+      - name: Install Bytehound
+        run: |
+          git clone https://github.com/koute/bytehound
+          cd bytehound
+          cargo build --release -p bytehound-preload
+          echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${PWD}/target/release" >> $GITHUB_ENV
+
       - name: Configure environment
         run: |
           sudo apt-get install -y linux-tools-common linux-tools-generic linux-tools-`uname -r`

ci/check-profiling.sh

@@ -118,6 +118,17 @@ RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot=
 test -f results/msout-Test-helloworld-Check-Full
 grep -q "snapshot=0" results/msout-Test-helloworld-Check-Full
 
+# Bytehound.
+RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot=debug \
+    cargo run -p collector --bin collector -- \
+    profile_local bytehound $bindir/rustc \
+        --id Test \
+        --profiles Check \
+        --cargo $bindir/cargo \
+        --include helloworld \
+        --scenarios Full
+test -f results/bhout-Test-helloworld-Check-Full
+
 # eprintln. The output file is empty because a vanilla rustc doesn't print
 # anything to stderr.
 RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot=debug \

collector/README.md

@@ -230,7 +230,7 @@ might be optimized.
 ### Preparation
 
 If you are going to use any of the profilers that rely on line numbers
-(OProfile, Cachegrind, Callgrind, DHAT, or Massif) use the following
+(OProfile, Cachegrind, Callgrind, DHAT, Massif or Bytehound) use the following
 `config.toml` file for your local build.
 ```
 [llvm]
@@ -363,6 +363,14 @@ The mandatory `<PROFILER>` argument must be one of the following.
     [`massif-visualizer`](https://github.com/KDE/massif-visualizer); the latter
     is recommended, though it sometimes fails to read output files that
     `ms_print` can handle.
+- `bytehound`: Profile with
+  [Bytehound](https://github.com/koute/bytehound), a memory profiler. You must add the
+  directory containing `libbytehound.so` to the `LD_LIBRARY_PATH` environment variable
+  when you use this profiler.
+  - **Purpose**. Bytehound is designed to give insight into a program's memory usage.
+  - **Slowdown**. Roughly 2--4x.
+  - **Output**. Raw output is written to files with a `bytehound` prefix. Those
+    files can be viewed with the `bytehound server <filename>` command.
 - `eprintln`: Profile with `eprintln!` statements.
   - **Purpose**. Sometimes it is useful to do ad hoc profiling by inserting
     `eprintln!` statements into rustc, e.g. to count how often particular paths

collector/src/execute.rs

@@ -170,6 +170,7 @@ pub enum Profiler {
     Dhat,
     DhatCopy,
     Massif,
+    Bytehound,
     Eprintln,
     LlvmLines,
     MonoItems,
@@ -187,6 +188,8 @@ impl Profiler {
                 | Profiler::Callgrind
                 | Profiler::Dhat
                 | Profiler::DhatCopy
+                | Profiler::Massif
+                | Profiler::Bytehound
                 | Profiler::Eprintln
                 | Profiler::LlvmLines
                 | Profiler::LlvmIr
@@ -230,6 +233,7 @@ impl PerfTool {
             | ProfileTool(Dhat)
             | ProfileTool(DhatCopy)
             | ProfileTool(Massif)
+            | ProfileTool(Bytehound)
             | ProfileTool(Eprintln)
             | ProfileTool(DepGraph)
             | ProfileTool(MonoItems)
@@ -266,6 +270,7 @@ impl PerfTool {
             | ProfileTool(Dhat)
             | ProfileTool(DhatCopy)
             | ProfileTool(Massif)
+            | ProfileTool(Bytehound)
             | ProfileTool(MonoItems)
             | ProfileTool(LlvmIr)
             | ProfileTool(Eprintln) => true,
@@ -1132,6 +1137,15 @@ impl<'a> Processor for ProfileProcessor<'a> {
                 fs::copy(&tmp_msout_file, &msout_file)?;
             }
 
+            // Bytehound produces (via rustc-fake) a data file called
+            // `bytehound.dat`. We copy it from the temp dir to the output dir, giving
+            // it a new name in the process.
+            Profiler::Bytehound => {
+                let tmp_bytehound_file = filepath(data.cwd.as_ref(), "bytehound.dat");
+                let target_file = filepath(self.output_dir, &out_file("bhout"));
+                fs::copy(tmp_bytehound_file, target_file)?;
+            }
+
             // `eprintln!` statements are redirected (via rustc-fake) to a file
             // called `eprintln`. We copy it from the temp dir to the output
             // dir, giving it a new name in the process.

collector/src/rustc-fake.rs

@@ -325,6 +325,15 @@ fn main() {
                 run_with_determinism_env(cmd);
             }
 
+            "Bytehound" => {
+                let mut cmd = Command::new(tool);
+                cmd.args(args);
+                cmd.env("MEMORY_PROFILER_OUTPUT", "bytehound.dat");
+                cmd.env("LD_PRELOAD", "libbytehound.so");
+
+                run_with_determinism_env(cmd);
+            }
+
             "Eprintln" => {
                 let mut cmd = Command::new(tool);
                 cmd.args(args).stderr(std::process::Stdio::from(