From 1ef622ea497fc4b0dcbf43eb818e01ab5fcda4cb Mon Sep 17 00:00:00 2001 From: Yunfan Yang Date: Mon, 8 Jun 2026 18:20:33 -0400 Subject: [PATCH] Productionize Linux jemalloc heap profiling (#12265) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description Productionizes automatic jemalloc heap profiling on Linux so high-memory events upload a heap profile to Sentry, matching what macOS already does. The key difference from macOS: **the Linux profile is uploaded unsymbolized (raw pprof)**. It carries sample addresses + memory mappings + the GNU build-id, and is symbolized **offline** against the debug-info file (DIF) the release pipeline already uploads to Sentry — the same artifact used to symbolize panics, matched by build-id. This lets the shipped binary stay fully stripped (no symbol-table bloat) while still producing usable, symbolizable profiles. ### What changed - **`app/src/profiling.rs`**: on Linux, dump the gzipped pprof **in-process** via `jemalloc_pprof` (`dump_jemalloc_pprof_bytes`) — no external `pprof` binary, HTTP server, or fixed port (essential for the headless remote-server daemon). The dump is now **raw / unsymbolized**. The external-`pprof` path is kept for macOS, where in-process symbolization stays. - **`app/Cargo.toml`**: build `jemalloc_pprof` **without** the `symbolize` feature, so `dump_pprof()` returns a raw profile instead of symbolizing in-process (in-process symbolization would have required keeping the symbol table in the shipped binary). - **`Cargo.lock`**: bump `jemalloc_pprof` / `pprof_util` to `0.8.2`. This is required: `0.8.2` writes a usable mapping range (`memory_limit = u64::MAX`), whereas `0.8.1` wrote `memory_limit = 0`, leaving pprof unable to bind sample addresses to the binary — so profiles would be unsymbolizable even with the correct DIF. (See this PR: https://github.com/polarsignals/rust-jemalloc-pprof/pull/31) - **`script/linux/bundle`**: enable `jemalloc_pprof,heap_usage_tracking` for the `dev` and `preview` channels, and keep the **normal strip behavior** (`--strip-all` for non-dev builds). The shipped binary stays small; symbols live only in the uploaded DIF. ### Symbolizing a Linux heap profile (offline, via the Sentry DIF) The release pipeline already uploads each build's debug-info file to Sentry (`script/sentry_upload_dif.sh`), keyed by GNU build-id. To analyze a `heap-profile.pb` from an "Excessive memory usage detected" event: ```bash # 0. Use the standalone pprof (the Go-bundled `go tool pprof` misreads the # build-id on large DIFs): go install github.com/google/pprof@latest # 1. Read the main binary's build-id from the raw profile's mappings: BUILD_ID=$(pprof -raw heap-profile.pb \ | sed -n '/^Mappings/,/^Locations/p' \ | grep -E 'warp-(dev|preview)' | head -1 | awk '{print $NF}') # 2. Download the matching DIF from the channel's Sentry project # (a read-only token is sufficient; dev -> warp-client-dev, # preview -> warp-client-preview): ORG=warpdotdev; PROJECT=warp-client-dev ID=$(curl -s "https://us.sentry.io/api/0/projects/$ORG/$PROJECT/files/dsyms/?query=${BUILD_ID:0:20}" \ -H "Authorization: Bearer $SENTRY_AUTH_TOKEN" | jq -r '.[0].id') mkdir -p difs curl -sL "https://us.sentry.io/api/0/projects/$ORG/$PROJECT/files/dsyms/?id=$ID" \ -H "Authorization: Bearer $SENTRY_AUTH_TOKEN" -o "difs/${BUILD_ID}.debug" # 3. Symbolize + analyze (pprof matches the DIF to the profile by build-id): PPROF_BINARY_PATH=./difs pprof -http=: heap-profile.pb # flame graph / top / graph ``` The DIF carries `.symtab` + DWARF, so frames resolve to function names with file:line and inlined frames. (`addr2line -e difs/$BUILD_ID.debug ` works as a lower-level alternative.) ### Notes / scope - No `pprof` binary is bundled on Linux (in-process raw dump). - `stable` Linux builds are unchanged (no profiling features) — enabling for `dev` + `preview` first. ## Linked Issue N/A — infrastructure/observability follow-up. ## Testing - `cargo check` + `cargo clippy -- -D warnings` for the app lib targeting Linux with `heap_usage_tracking` (exercises the new raw-dump branch); macOS host path also checked (validates the `pprof_binary_path` cfg-gate) — passes. - `./script/format` — no changes beyond the edited files. - End-to-end validated on a build sharing this exact code path - [ ] I have manually tested my changes locally with `./script/run` ## Agent Mode - [x] Warp Agent Mode - This PR was created via Warp's AI Agent Mode CHANGELOG-NONE --- 🤖 Generated with Warp Agent Mode Conversation: https://staging.warp.dev/conversation/e6ba6343-1bb8-4cce-a155-6f0b453bad40 Plan: https://staging.warp.dev/drive/notebook/359wXMjgAzXhtOZfnSWUlA --- Cargo.lock | 92 ++++++++++++++++---------------------------- app/Cargo.toml | 9 +++-- app/src/profiling.rs | 90 ++++++++++++++++++++++++++++++------------- script/linux/bundle | 6 ++- 4 files changed, 108 insertions(+), 89 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fd87a8977..e069413c9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -599,7 +599,7 @@ dependencies = [ "objc2-foundation 0.3.2", "parking_lot", "percent-encoding", - "windows-sys 0.59.0", + "windows-sys 0.52.0", "wl-clipboard-rs", "x11rb", ] @@ -2561,7 +2561,7 @@ checksum = "3281776daed25d20f2a7846151561bac63091d41115ba9a5d8ac50891fb749d6" dependencies = [ "candle-core", "candle-nn", - "prost 0.14.3", + "prost", "prost-build", ] @@ -3057,7 +3057,7 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fde0e0ec90c9dfb3b4b1a0891a7dcd0e2bffde2f7efed5fe7c9bb00e5bfb915e" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -4374,7 +4374,7 @@ checksum = "6738d2e996274e499bc7b0d693c858b7720b9cd2543a0643a3087e6cb0a4fa16" dependencies = [ "cfg-if", "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -4679,7 +4679,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -4845,7 +4845,7 @@ name = "field_mask" version = "0.0.0" dependencies = [ "itertools 0.14.0", - "prost 0.14.3", + "prost", "prost-reflect", "prost-types", "thiserror 2.0.17", @@ -6293,7 +6293,7 @@ dependencies = [ "log", "oauth2", "prevent_sleep", - "prost 0.14.3", + "prost", "reqwest", "serde", "serde_json", @@ -6400,7 +6400,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.0", + "socket2 0.5.10", "system-configuration", "tokio", "tower-service", @@ -6755,7 +6755,7 @@ dependencies = [ "ndarray", "ort", "parking_lot", - "prost 0.14.3", + "prost", "rust-embed 8.7.2", "serde", "smol_str", @@ -6942,7 +6942,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi 0.5.2", "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -7109,9 +7109,9 @@ dependencies = [ [[package]] name = "jemalloc_pprof" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74ff642505c7ce8d31c0d43ec0e235c6fd4585d9b8172d8f9dd04d36590200b5" +checksum = "8a0d44c349cfe2654897fadcb9de4f0bfbf48288ec344f700b2bd59f152dd209" dependencies = [ "anyhow", "libc", @@ -7136,7 +7136,7 @@ dependencies = [ "portable-atomic", "portable-atomic-util", "serde_core", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -9388,7 +9388,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.45.0", ] [[package]] @@ -9881,16 +9881,15 @@ dependencies = [ [[package]] name = "pprof_util" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4429d44e5e2c8a69399fc0070379201eed018e3df61e04eb7432811df073c224" +checksum = "eea0cc524de808a6d98d192a3d99fe95617031ad4a52ec0a0f987ef4432e8fe1" dependencies = [ "anyhow", - "backtrace", "flate2", "num", "paste", - "prost 0.13.5", + "prost", ] [[package]] @@ -10107,16 +10106,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "prost" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" -dependencies = [ - "bytes", - "prost-derive 0.13.5", -] - [[package]] name = "prost" version = "0.14.3" @@ -10124,7 +10113,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" dependencies = [ "bytes", - "prost-derive 0.14.3", + "prost-derive", ] [[package]] @@ -10133,32 +10122,19 @@ version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ - "heck 0.5.0", - "itertools 0.14.0", + "heck 0.4.1", + "itertools 0.11.0", "log", "multimap", "petgraph", "prettyplease", - "prost 0.14.3", + "prost", "prost-types", "regex", "syn 2.0.117", "tempfile", ] -[[package]] -name = "prost-derive" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" -dependencies = [ - "anyhow", - "itertools 0.14.0", - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "prost-derive" version = "0.14.3" @@ -10166,7 +10142,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.11.0", "proc-macro2", "quote", "syn 2.0.117", @@ -10178,7 +10154,7 @@ version = "0.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b89455ef41ed200cafc47c76c552ee7792370ac420497e551f16123a9135f76e" dependencies = [ - "prost 0.14.3", + "prost", "prost-reflect-derive", "prost-types", ] @@ -10210,7 +10186,7 @@ version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" dependencies = [ - "prost 0.14.3", + "prost", ] [[package]] @@ -10411,7 +10387,7 @@ dependencies = [ "once_cell", "socket2 0.5.10", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -10858,7 +10834,7 @@ dependencies = [ "futures-lite 1.13.0", "getrandom 0.2.16", "log", - "prost 0.14.3", + "prost", "prost-build", "repo_metadata", "serde", @@ -11301,7 +11277,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -11314,7 +11290,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.11.0", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -11382,7 +11358,7 @@ dependencies = [ "security-framework 3.5.1", "security-framework-sys", "webpki-root-certs 1.0.1", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -13033,7 +13009,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix 1.1.2", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -13293,7 +13269,7 @@ version = "0.3.0" source = "git+https://github.com/warpdotdev/tink-rust?rev=54b9ac9af93b0c08b446a7bc0582836c9403a71b#54b9ac9af93b0c08b446a7bc0582836c9403a71b" dependencies = [ "base64 0.22.1", - "prost 0.14.3", + "prost", "prost-build", "serde", ] @@ -14586,7 +14562,7 @@ dependencies = [ "pin-project", "plist", "pprof", - "prost 0.14.3", + "prost", "prost-build", "prost-types", "qrcode", @@ -15079,7 +15055,7 @@ name = "warp_multi_agent_api" version = "0.0.0" source = "git+https://github.com/warpdotdev/warp-proto-apis.git?rev=c67de64fc4949f693a679552dc88cebc9f7d0180#c67de64fc4949f693a679552dc88cebc9f7d0180" dependencies = [ - "prost 0.14.3", + "prost", "prost-reflect", "prost-reflect-build", "prost-types", @@ -16082,7 +16058,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] diff --git a/app/Cargo.toml b/app/Cargo.toml index 735899ba0..5dfa4fe3a 100644 --- a/app/Cargo.toml +++ b/app/Cargo.toml @@ -123,9 +123,12 @@ image.workspace = true infer = "0.19.0" jaq-json.workspace = true jaq-all.workspace = true -jemalloc_pprof = { version = "0.8.1", optional = true, features = [ - "symbolize", -] } +# Built WITHOUT the `symbolize` feature: `dump_pprof()` then returns a raw +# pprof (sample addresses + mappings + GNU build-id), which is symbolized +# offline against the debug-info file uploaded to Sentry by the release +# process (matched by build-id). This keeps the shipped binary fully +# strippable. +jemalloc_pprof = { version = "0.8.1", optional = true } lsp-types = "0.97.0" indexmap = { version = "2.0.2", features = ["serde"] } input_classifier.workspace = true diff --git a/app/src/profiling.rs b/app/src/profiling.rs index 905be415e..13186ef27 100644 --- a/app/src/profiling.rs +++ b/app/src/profiling.rs @@ -63,9 +63,13 @@ pub fn dump_dhat_heap_profile() { /// Dumps a jemalloc heap profile and sends it to Sentry. /// -/// This function spawns `go tool pprof` to fetch and symbolicate the heap -/// profile from the local HTTP server, then attaches the resulting profile -/// to a Sentry event. +/// On Linux the profile is produced in-process via the `jemalloc_pprof` crate +/// as a raw (unsymbolized) pprof -- sample addresses + mappings + GNU build-id +/// -- and is symbolized offline against the debug-info file uploaded to Sentry +/// by the release process (matched by build-id). On other platforms it spawns +/// the bundled `pprof` binary to fetch and symbolicate the heap profile from +/// the local HTTP server. Either way, the resulting profile is attached to a +/// Sentry event. #[cfg(feature = "heap_usage_tracking")] pub async fn dump_jemalloc_heap_profile(memory_breakdown: serde_json::Value) { use sentry::protocol::{Attachment, AttachmentType}; @@ -113,35 +117,69 @@ pub async fn dump_jemalloc_heap_profile(memory_breakdown: serde_json::Value) { #[cfg(feature = "heap_usage_tracking")] async fn dump_jemalloc_heap_profile_inner() -> anyhow::Result> { - use anyhow::Context as _; + cfg_if::cfg_if! { + if #[cfg(target_os = "linux")] { + // `jemalloc_pprof` only supports Linux. We build it WITHOUT the + // `symbolize` feature, so `dump_pprof()` returns a raw, gzipped + // pprof (sample addresses + mappings + GNU build-id) that is + // symbolized offline against the debug-info file by build-id. Dump + // it directly in-process -- no external `pprof`/Go binary, HTTP + // round-trip, or port dependency required (the latter matter for + // the headless remote server daemon, which has no bundled helpers + // next to it). + dump_jemalloc_pprof_bytes().await + } else { + use anyhow::Context as _; - // Create a temporary file for the profile output. - let temp_dir = tempfile::tempdir().context("Failed to create temporary directory")?; - let profile_path = temp_dir.path().join("heap-profile.pb"); + // Create a temporary file for the profile output. + let temp_dir = tempfile::tempdir().context("Failed to create temporary directory")?; + let profile_path = temp_dir.path().join("heap-profile.pb"); - // Run pprof to fetch and symbolicate the heap profile. - let pprof_path = pprof_binary_path()?; - let output = command::r#async::Command::new(pprof_path) - .args(["--proto", "--symbolize=local", "-output"]) - .arg(&profile_path) - .arg("http://127.0.0.1:9277/debug/pprof/heap") - .output() - .await - .context("Failed to execute pprof")?; + // Run pprof to fetch and symbolicate the heap profile. + let pprof_path = pprof_binary_path()?; + let output = command::r#async::Command::new(pprof_path) + .args(["--proto", "--symbolize=local", "-output"]) + .arg(&profile_path) + .arg("http://127.0.0.1:9277/debug/pprof/heap") + .output() + .await + .context("Failed to execute pprof")?; - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - anyhow::bail!("pprof failed: {stderr}"); + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + anyhow::bail!("pprof failed: {stderr}"); + } + + // Read the profile data from the temporary file. + let profile_data = + std::fs::read(&profile_path).context("Failed to read heap profile from disk")?; + + Ok(profile_data) + } } - - // Read the profile data from the temporary file. - let profile_data = - std::fs::read(&profile_path).context("Failed to read heap profile from disk")?; - - Ok(profile_data) } -#[cfg(feature = "heap_usage_tracking")] +/// Produces a raw (unsymbolized), gzipped pprof heap profile directly from the +/// in-process jemalloc profiler. The profile carries sample addresses, +/// mappings, and the GNU build-id, and is symbolized offline against the +/// matching debug-info file (by build-id). +/// +/// This is the same dump that [`handle_get_heap`] serves over HTTP, but +/// invoked directly so callers don't need to reach the local HTTP server. +/// Requires the `jemalloc_pprof` feature, which is Linux-only. +#[cfg(all(feature = "jemalloc_pprof", target_os = "linux"))] +async fn dump_jemalloc_pprof_bytes() -> anyhow::Result> { + let Some(prof_ctl) = jemalloc_pprof::PROF_CTL.as_ref() else { + anyhow::bail!("heap profiler not initialized"); + }; + let mut prof_ctl = prof_ctl.lock().await; + if !prof_ctl.activated() { + anyhow::bail!("heap profiling not activated"); + } + prof_ctl.dump_pprof() +} + +#[cfg(all(feature = "heap_usage_tracking", not(target_os = "linux")))] fn pprof_binary_path() -> anyhow::Result { cfg_if::cfg_if! { if #[cfg(target_os = "macos")] { diff --git a/script/linux/bundle b/script/linux/bundle index 071083ce5..60f878a9a 100755 --- a/script/linux/bundle +++ b/script/linux/bundle @@ -170,14 +170,16 @@ elif [[ $RELEASE_CHANNEL = "dev" ]]; then BINARY_NAME="warp-dev" APP_NAME="WarpDev" FEATURES="$FEATURES,agent_mode_debug" - # Enable heap profiling using jemalloc through pprof. - FEATURES="$FEATURES,jemalloc_pprof" + # Enable heap usage tracking & profiling using jemalloc through pprof. + FEATURES="$FEATURES,jemalloc_pprof,heap_usage_tracking" export HANDLE_MARKDOWN=1 elif [[ $RELEASE_CHANNEL = "preview" ]]; then WARP_BIN="preview" BINARY_NAME="warp-preview" APP_NAME="WarpPreview" FEATURES="$FEATURES,preview_channel" + # Enable heap usage tracking & profiling using jemalloc through pprof. + FEATURES="$FEATURES,jemalloc_pprof,heap_usage_tracking" elif [[ $RELEASE_CHANNEL = "stable" ]]; then WARP_BIN="stable" BINARY_NAME="warp"