Skip to content

Commit a2902e7

Browse files
erikgrinakererikgrinaker
authored
http-utils: generate heap profiles with jemalloc_pprof (#11075)
## Problem The code to generate symbolized pprof heap profiles and flamegraph SVGs has been upstreamed to the `jemalloc_pprof` crate: * polarsignals/rust-jemalloc-pprof#22 * polarsignals/rust-jemalloc-pprof#23 ## Summary of changes Use `jemalloc_pprof` to generate symbolized pprof heap profiles and flamegraph SVGs. This reintroduces a bunch of internal jemalloc stack frames that we'd previously strip, e.g. each stack now always ends with `prof_backtrace_impl` (where jemalloc takes a stack trace for heap profiling), but that seems ok.
1 parent 435bf45 commit a2902e7

File tree

7 files changed

+21
-302
lines changed

7 files changed

+21
-302
lines changed

‎Cargo.lock

+8-10
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml

+1-3
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,6 @@ anyhow = { version = "1.0", features = ["backtrace"] }
5353
arc-swap = "1.6"
5454
async-compression = { version = "0.4.0", features = ["tokio", "gzip", "zstd"] }
5555
atomic-take = "1.1.0"
56-
backtrace = "0.3.74"
5756
flate2 = "1.0.26"
5857
assert-json-diff = "2"
5958
async-stream = "0.3"
@@ -114,11 +113,10 @@ hyper-util = "0.1"
114113
tokio-tungstenite = "0.21.0"
115114
indexmap = "2"
116115
indoc = "2"
117-
inferno = "0.12.0"
118116
ipnet = "2.10.0"
119117
itertools = "0.10"
120118
itoa = "1.0.11"
121-
jemalloc_pprof = "0.6"
119+
jemalloc_pprof = { version = "0.7", features = ["symbolize", "flamegraph"] }
122120
jsonwebtoken = "9"
123121
lasso = "0.7"
124122
libc = "0.2"

‎libs/http-utils/Cargo.toml

-3
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,8 @@ license.workspace = true
66

77
[dependencies]
88
anyhow.workspace = true
9-
backtrace.workspace = true
109
bytes.workspace = true
11-
inferno.workspace = true
1210
fail.workspace = true
13-
flate2.workspace = true
1411
hyper0.workspace = true
1512
itertools.workspace = true
1613
jemalloc_pprof.workspace = true

‎libs/http-utils/src/endpoint.rs

+12-46
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,15 @@ use std::io::Write as _;
33
use std::str::FromStr;
44
use std::time::Duration;
55

6-
use ::pprof::ProfilerGuardBuilder;
7-
use ::pprof::protos::Message as _;
86
use anyhow::{Context, anyhow};
97
use bytes::{Bytes, BytesMut};
108
use hyper::header::{AUTHORIZATION, CONTENT_DISPOSITION, CONTENT_TYPE, HeaderName};
119
use hyper::http::HeaderValue;
1210
use hyper::{Body, Method, Request, Response};
1311
use metrics::{Encoder, IntCounter, TextEncoder, register_int_counter};
1412
use once_cell::sync::Lazy;
15-
use regex::Regex;
13+
use pprof::ProfilerGuardBuilder;
14+
use pprof::protos::Message as _;
1615
use routerify::ext::RequestExt;
1716
use routerify::{Middleware, RequestInfo, Router, RouterBuilder};
1817
use tokio::sync::{Mutex, Notify, mpsc};
@@ -22,7 +21,6 @@ use tracing::{Instrument, debug, info, info_span, warn};
2221
use utils::auth::{AuthError, Claims, SwappableJwtAuth};
2322

2423
use crate::error::{ApiError, api_error_handler, route_error_handler};
25-
use crate::pprof;
2624
use crate::request::{get_query_param, parse_query_param};
2725

2826
static SERVE_METRICS_COUNT: Lazy<IntCounter> = Lazy::new(|| {
@@ -449,20 +447,6 @@ pub async fn profile_heap_handler(req: Request<Body>) -> Result<Response<Body>,
449447
Some(format) => return Err(ApiError::BadRequest(anyhow!("invalid format {format}"))),
450448
};
451449

452-
// Functions and mappings to strip when symbolizing pprof profiles. If true,
453-
// also remove child frames.
454-
static STRIP_FUNCTIONS: Lazy<Vec<(Regex, bool)>> = Lazy::new(|| {
455-
vec![
456-
(Regex::new("^__rust").unwrap(), false),
457-
(Regex::new("^_start$").unwrap(), false),
458-
(Regex::new("^irallocx_prof").unwrap(), true),
459-
(Regex::new("^prof_alloc_prep").unwrap(), true),
460-
(Regex::new("^std::rt::lang_start").unwrap(), false),
461-
(Regex::new("^std::sys::backtrace::__rust").unwrap(), false),
462-
]
463-
});
464-
const STRIP_MAPPINGS: &[&str] = &["libc", "libgcc", "pthread", "vdso"];
465-
466450
// Obtain profiler handle.
467451
let mut prof_ctl = jemalloc_pprof::PROF_CTL
468452
.as_ref()
@@ -495,45 +479,27 @@ pub async fn profile_heap_handler(req: Request<Body>) -> Result<Response<Body>,
495479
}
496480

497481
Format::Pprof => {
498-
let data = tokio::task::spawn_blocking(move || {
499-
let bytes = prof_ctl.dump_pprof()?;
500-
// Symbolize the profile.
501-
// TODO: consider moving this upstream to jemalloc_pprof and avoiding the
502-
// serialization roundtrip.
503-
let profile = pprof::decode(&bytes)?;
504-
let profile = pprof::symbolize(profile)?;
505-
let profile = pprof::strip_locations(profile, STRIP_MAPPINGS, &STRIP_FUNCTIONS);
506-
pprof::encode(&profile)
507-
})
508-
.await
509-
.map_err(|join_err| ApiError::InternalServerError(join_err.into()))?
510-
.map_err(ApiError::InternalServerError)?;
482+
let data = tokio::task::spawn_blocking(move || prof_ctl.dump_pprof())
483+
.await
484+
.map_err(|join_err| ApiError::InternalServerError(join_err.into()))?
485+
.map_err(ApiError::InternalServerError)?;
511486
Response::builder()
512487
.status(200)
513488
.header(CONTENT_TYPE, "application/octet-stream")
514-
.header(CONTENT_DISPOSITION, "attachment; filename=\"heap.pb\"")
489+
.header(CONTENT_DISPOSITION, "attachment; filename=\"heap.pb.gz\"")
515490
.body(Body::from(data))
516491
.map_err(|err| ApiError::InternalServerError(err.into()))
517492
}
518493

519494
Format::Svg => {
520-
let body = tokio::task::spawn_blocking(move || {
521-
let bytes = prof_ctl.dump_pprof()?;
522-
let profile = pprof::decode(&bytes)?;
523-
let profile = pprof::symbolize(profile)?;
524-
let profile = pprof::strip_locations(profile, STRIP_MAPPINGS, &STRIP_FUNCTIONS);
525-
let mut opts = inferno::flamegraph::Options::default();
526-
opts.title = "Heap inuse".to_string();
527-
opts.count_name = "bytes".to_string();
528-
pprof::flamegraph(profile, &mut opts)
529-
})
530-
.await
531-
.map_err(|join_err| ApiError::InternalServerError(join_err.into()))?
532-
.map_err(ApiError::InternalServerError)?;
495+
let svg = tokio::task::spawn_blocking(move || prof_ctl.dump_flamegraph())
496+
.await
497+
.map_err(|join_err| ApiError::InternalServerError(join_err.into()))?
498+
.map_err(ApiError::InternalServerError)?;
533499
Response::builder()
534500
.status(200)
535501
.header(CONTENT_TYPE, "image/svg+xml")
536-
.body(Body::from(body))
502+
.body(Body::from(svg))
537503
.map_err(|err| ApiError::InternalServerError(err.into()))
538504
}
539505
}

‎libs/http-utils/src/lib.rs

-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ pub mod endpoint;
22
pub mod error;
33
pub mod failpoints;
44
pub mod json;
5-
pub mod pprof;
65
pub mod request;
76

87
extern crate hyper0 as hyper;

0 commit comments

Comments
 (0)