From 7fe9bf64d59fd59fa5445babff10dead8f53a8bb Mon Sep 17 00:00:00 2001 From: Zack Fu Zi Xiang Date: Wed, 14 Feb 2024 12:54:27 +0800 Subject: [PATCH 1/2] perf: add jemalloc heap profile --- Cargo.lock | 124 ++++++++++++++++++++++++++-- Cargo.toml | 4 + heap.pb.gz | Bin 0 -> 3486 bytes src/api/mod.rs | 1 + src/api/pprof.rs | 24 ++++++ src/api/user.rs | 9 ++ src/application.rs | 2 + src/biz/collab/access_control.rs | 2 +- src/biz/collab/storage.rs | 2 +- src/biz/workspace/access_control.rs | 2 +- src/main.rs | 9 ++ 11 files changed, 169 insertions(+), 10 deletions(-) create mode 100644 heap.pb.gz create mode 100644 src/api/pprof.rs diff --git a/Cargo.lock b/Cargo.lock index 87a8d3f1f..bab88b5df 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -532,6 +532,7 @@ dependencies = [ "image", "infra", "itertools 0.11.0", + "jemalloc_pprof", "lazy_static", "mime", "moka", @@ -539,7 +540,7 @@ dependencies = [ "opener", "openssl", "prometheus-client", - "prost", + "prost 0.12.3", "rand 0.8.5", "rcgen", "realtime", @@ -556,6 +557,7 @@ dependencies = [ "sqlx", "tempfile", "thiserror", + "tikv-jemallocator", "token", "tokio", "tokio-stream", @@ -1278,7 +1280,7 @@ dependencies = [ "mime", "mime_guess", "parking_lot 0.12.1", - "prost", + "prost 0.12.3", "realtime-entity", "realtime-protocol", "reqwest", @@ -2751,7 +2753,7 @@ dependencies = [ "gif", "jpeg-decoder", "num-iter", - "num-rational", + "num-rational 0.3.2", "num-traits", "png", "scoped_threadpool", @@ -2852,6 +2854,25 @@ version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" +[[package]] +name = "jemalloc_pprof" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45b38a2cc3eb7b0e332c6368a6fd6a1a603a5be9526f0810f8e0682513538541" +dependencies = [ + "anyhow", + "flate2", + "libc", + "num", + "once_cell", + "paste", + "prost 0.11.9", + "tempfile", + "tikv-jemalloc-ctl", + "tokio", + "tracing", +] + [[package]] name = "jobserver" version = "0.1.27" @@ -3265,6 +3286,20 @@ dependencies = [ "winapi", ] +[[package]] +name = "num" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05180d69e3da0e530ba2a1dae5110317e49e3b7f3d41be227dc5f92e49ee7af" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational 0.4.1", + "num-traits", +] + [[package]] name = "num-bigint" version = "0.4.4" @@ -3293,6 +3328,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "num-complex" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23c6602fda94a57c990fe0df199a035d83576b496aa29f4e634a8ac6004e68a6" +dependencies = [ + "num-traits", +] + [[package]] name = "num-integer" version = "0.1.45" @@ -3325,6 +3369,18 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-rational" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" +dependencies = [ + "autocfg", + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.17" @@ -3851,6 +3907,16 @@ dependencies = [ "syn 2.0.48", ] +[[package]] +name = "prost" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +dependencies = [ + "bytes", + "prost-derive 0.11.9", +] + [[package]] name = "prost" version = "0.12.3" @@ -3858,7 +3924,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "146c289cda302b98a28d40c8b3b90498d6e526dd24ac2ecea73e4e491685b94a" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.12.3", ] [[package]] @@ -3875,7 +3941,7 @@ dependencies = [ "once_cell", "petgraph", "prettyplease", - "prost", + "prost 0.12.3", "prost-types", "regex", "syn 2.0.48", @@ -3883,6 +3949,19 @@ dependencies = [ "which", ] +[[package]] +name = "prost-derive" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +dependencies = [ + "anyhow", + "itertools 0.10.5", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "prost-derive" version = "0.12.3" @@ -3902,7 +3981,7 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "193898f59edcf43c26227dcd4c8427f00d99d61e95dcde58dabd49fa291d470e" dependencies = [ - "prost", + "prost 0.12.3", ] [[package]] @@ -4235,7 +4314,7 @@ dependencies = [ "collab", "collab-entity", "database-entity", - "prost", + "prost 0.12.3", "prost-build", "protoc-bin-vendored", "realtime-protocol", @@ -5555,6 +5634,37 @@ dependencies = [ "weezl", ] +[[package]] +name = "tikv-jemalloc-ctl" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "619bfed27d807b54f7f776b9430d4f8060e66ee138a28632ca898584d462c31c" +dependencies = [ + "libc", + "paste", + "tikv-jemalloc-sys", +] + +[[package]] +name = "tikv-jemalloc-sys" +version = "0.5.4+5.3.0-patched" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9402443cb8fd499b6f327e40565234ff34dbda27460c5b47db0db77443dd85d1" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "tikv-jemallocator" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "965fe0c26be5c56c94e38ba547249074803efd52adfb66de62107d95aab3eaca" +dependencies = [ + "libc", + "tikv-jemalloc-sys", +] + [[package]] name = "time" version = "0.3.31" diff --git a/Cargo.toml b/Cargo.toml index d3b8f4524..b761c946e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -85,6 +85,10 @@ shared-entity = { path = "libs/shared-entity", features = ["cloud"] } workspace-template = { workspace = true } realtime-entity.workspace = true +# Memory profiling +[target.'cfg(not(target_env = "msvc"))'.dependencies] +tikv-jemallocator = { version = "0.5.4", features = ["profiling", "unprefixed_malloc_on_supported_platforms"] } +jemalloc_pprof = "0.1.0" # profiling #puffin = "0.16.0" diff --git a/heap.pb.gz b/heap.pb.gz new file mode 100644 index 0000000000000000000000000000000000000000..ce317ac0671ff0bd2a0bf59299a373fb04d8cd10 GIT binary patch literal 3486 zcmV;P4Po*hiwFP!00000|Gn9HSXIXr0PxOL5=4!Js39axOwxre$lI6Nv`y1yY11Z6 z_q3^b8;YW!xOaP~2)M8a0?PJ4HbIchePaM~dCJu1-Cyu{ z@ue@n{K~XfUwi#4U;WzGJ-_kIZ++Wqy0=29Qfst2y}|gM8Q*>5d*A=Tn?HQ(M?Zf1 zop;~+$xq+^+0Q@t@E0Hb@>jq9&2NAA`;Y(d$4~w=^HY=AVx3|0nKj#Yj@y{dAk%-y zIrj_8xiWH^o1csApE>>J&G!#jurTlo$!1?O&CTla3D34@@sg#>maq8pU;g^HzuQ?5 zTgg_j)hw8;VIeG(g|W449b3;fuy7W^B3TsM$f8*c+r(m79E)e0*%p?-5?K;UW+}|U zQdt_?%F?o^Z z)$ACnVYRG|9cT6I1UtzZ*eQ0Jea0GD6FbA2SqnSM&aqb3#?G_tI*d6I(U$e87>+B@_qT$-3B8c8y(U-RuVIVK>FU;BPOx#roK7*3a&+0XE3) zvLSYl4YT`fKpbY-QqO5$s9wG=F^kwTl%k~=X_FKy#YyqfW@(F*ASFskQnHjHIiyr6P1-7@ zOBqt8lqF?L+oT*RSK2P+N%_(asX!`}ilkzxMEYMhgHJss^NrTt?ufMKPH`WKxgW)k zo?|^;aF$%{4k7oRGoe}hIL%&qmWj|9euFsW9(wwAt_az$P8FfCu8v`G%H6&zI3Gfe z;f5D^XsWZ5tKB}>-Sp@c_rwHo^c<&f;pv}pwU?IXmfv~B{c3In96iT-xH~`1&kDI$ zCe^@^=U^h@%sVE|xZ9&^^UR@zas?Rv{$W&(+o_no`MBdmSa!gXV zdU!aG=jX%4`GG^Zm}M-+Ch#7JbM8m_%j8(;s~DT;S{*O)k6R)(5)~}xa(s%hXI&k8gXO;84-+xR z-o+v|)!B`=StnwS{o?5Pg2IDO!*jgv^7+QDM!CIdBIMybnWt(Jp*@G2v3%n&HpSIm z-z*m}qeYJ8Pk`8K6X1319=<*#p3f<|@3?igM=KdbTZ1);O*mmq;#LY1eB|n`f6Tt{0k;T(O9E1@+3^>rBSPzuVED`$lC4X&<3 zbl*2&9=148-{Q(V{1EkRu61qHrWdq1NUcnVUOymtbG1j%eG16XorqF$t-L>kQgPj? zdK4u~&DFCZ8)F){Z+HZy<$BP45UZr)iXSRM>0!sz_?{_grnUq>tf=tEayFfh#IK3-w2?opluZ1S+^L1@$Mc zQyDb&nNZISTT!3FK3JZOGiHMOii%NYt{Ycrg;*f_5UnLES4kVK7aPp&F_I6od6Mp% z#dT;P2Q?e+J2QgWeO)KH7ac8vmkQS$uKP)+QFCFP_0yi^2P-6(cGG!W*)^p3kiDPw zPk-2@8ir8;FrwU3s0C2Zu&bzruqN(wp#r&f71GYPh->}kf>>FLVZB^AfLg*;Iz;b> zrCb+p)B9l=S8r$*YB^LQfwTg4jm&n`pSf0-oJaiyp8o7~)L*%BTSr zxP}-8%t=i|t_7aHn}l2kd^A8@58PZu+yH#gK@0~5T_8q?L^oTJk-%%UtC3N_jV;vY zjlhvoVl*)F@x?K~<`!ys6L9_-#xfE9i6(=*XI9~g0@KUL-q;F`n40$`oJ30VldTG8u}Sp*D;8$=cZ(-Md!z|afC zoxrT-OW2KFVnWkWkh{f`67E3m0XkwRx))e>fVdBsvF`$MKQN|^cmTLQkuE+c@>Wvb zL%qH(A#mTvbtOV}NyNf&uyc>K2 zSp~eEMbT>UbQjPAdklEAmR4*Hu)2L1Sqn_+9>II-fLn^G&c}g=g1V6Pz>ZxHktcvx zO6lU0z#H2sZ-ZDH$5uwmJSBLEcpA8>ocNhoR}I8QvGuhOn}B`A53%JlVt^Txs9EeA z&BPWlza5Wo@t*}AD5B^&F-OB$$W}2aK@X5^q8c}M;u1YC=C|u0@`Bh3qG|Z;z@`nv zi@>P99^@rpTtheVve?aPdT~S@z|L)#H_5yrcJEsQ$WF1jwbZA}>=F;oa7c#CtD+jI zg&4gC+;@<89oSGq>=t{@ZsHB#*3XDNz{{l_$eX~mJ1E*KcJD^wEnxlq6WEPD!Q1CC zdRuH?gT#KZZ!}&+-Vw8KEEYKc94^|7925_G#!;-*UGeD2qu9U`WhaWN%s)#|rO(%&f zf$Q5s;_&?oJdgi#;O}2x=tbK4f`Ki0)yOrXsnd1H5W&_?WGJvE`aCiWcrl2!&9%U! z2zuDp0dvawk?Vo>`bWqOz-xuKk>S8?+lUb&noIXa0wapAAftd?J+vcj1eQ4{8ZA26 zN6{GIm21RJz~O9aITo0B<|Z-@cr=Tm@uGBv50IOI$JWsRw*VVThzY>DqmSRe!0aa4 z&60pahpDM#;HiAtpi)Gi>nN`SSkO+jN(F}HP;t^ktqO=+MKqu8O$VOXPu0i(hTNp# zXNq4>gitgKc(#|K*}&j#;xn+ z4DX{|v;f$CgIFk*N-wbpc;Gs*7+Bg*ECFVvh#gqGXn!m7Mse+ynK-@ zP~{}+L+{!C^K9NLOqRJSgG%o`%XiW2#pdZ2|9ReT1_ZqA=f8Bh*LxrQ^WSgz`7gG5 zFETBhWn1KJwV4;s@-_to_&{u?1!Ahnwq-WU;zcI2pY6T*KK^g0XDDVUR4;$-tXgMM zC=_^5P5WG2Wrcu(j-y&|iepF5J|ErCRmA_k7=F-hqq!7uvivD*_bS1#?%-HU_FK z>bd%W0Hal{TQb{sg?gD`w$aaPmc`D5}3 z)}%Ko6volID`rJ}juJKKASymbiCTRu8p8~mf`2}N!B}uGO0CwUv{}p+vtDm9D>PcA zS#30!4JM;SY1A9FR)bB?PwPjkuIR4#9A$|r4N5UqaTqOb)mltEZq@OF>ZAX0Kvpk# z{Vp@;ymT5En@Z)WQ~21lX1&5 Scope { + web::scope("/debug/pprof") + + // Usage: + // curl localhost:3000/debug/pprof/heap > heap.pb.gz + // pprof -http=:8080 heap.pb.gz + .service(web::resource("/heap").route(web::get().to(handle_get_heap))) +} + +async fn handle_get_heap() -> Vec { + let mut prof_ctl = match jemalloc_pprof::PROF_CTL.as_ref() { + Some(x) => x.lock().await, + None => return "none: cannot get prof_ctl".into(), + }; + if !prof_ctl.activated() { + return "heap profiling not activated".into(); + } + match prof_ctl.dump_pprof() { + Ok(x) => x, + Err(e) => format!("error: {}", e).into(), + } +} diff --git a/src/api/user.rs b/src/api/user.rs index 7b2699117..4e00a5c5d 100644 --- a/src/api/user.rs +++ b/src/api/user.rs @@ -25,6 +25,7 @@ pub fn user_scope() -> Scope { .service(web::resource("/update").route(web::post().to(update_user_handler))) .service(web::resource("/profile").route(web::get().to(get_user_profile_handler))) .service(web::resource("/workspace").route(web::get().to(get_user_workspace_info_handler))) + .service(web::resource("/test").route(web::get().to(mem_pprof_handler))) // deprecated .service(web::resource("/login").route(web::post().to(login_handler))) @@ -32,6 +33,14 @@ pub fn user_scope() -> Scope { .service(web::resource("/password").route(web::post().to(change_password_handler))) } +async fn mem_pprof_handler() -> Vec { + let mut prof_ctl = jemalloc_pprof::PROF_CTL.as_ref().unwrap().lock().await; + if !prof_ctl.activated() { + panic!("heap profiling not activated"); + } + prof_ctl.dump_pprof().unwrap() +} + #[tracing::instrument(skip(state, path), err)] async fn verify_user_handler( path: web::Path, diff --git a/src/application.rs b/src/application.rs index 5c9abd65c..b555955c4 100644 --- a/src/application.rs +++ b/src/application.rs @@ -1,4 +1,5 @@ use crate::api::metrics::{metrics_scope, AppFlowyCloudMetrics}; +use crate::api::pprof::pprof_scope; use crate::biz::casbin::adapter::PgAdapter; use crate::biz::casbin::MODEL_CONF; use crate::component::auth::HEADER_TOKEN; @@ -136,6 +137,7 @@ pub async fn run( .service(ws_scope()) .service(file_storage_scope()) .service(metrics_scope()) + .service(pprof_scope()) .app_data(Data::new(af_cloud_metric_arc.clone())) .app_data(Data::new(af_realtime_metric_arc.clone())) .app_data(Data::new(registry_arc.clone())) diff --git a/src/biz/collab/access_control.rs b/src/biz/collab/access_control.rs index d0db706d5..dd7ccd0fa 100644 --- a/src/biz/collab/access_control.rs +++ b/src/biz/collab/access_control.rs @@ -36,7 +36,7 @@ where } #[instrument(level = "debug", skip_all, err)] - #[allow(clippy::blocks_in_if_conditions)] + #[allow(clippy::blocks_in_conditions)] async fn check_collab_permission( &self, oid: &str, diff --git a/src/biz/collab/storage.rs b/src/biz/collab/storage.rs index 228ef17d9..00c4d3bc7 100644 --- a/src/biz/collab/storage.rs +++ b/src/biz/collab/storage.rs @@ -182,7 +182,7 @@ where } #[instrument(level = "trace", skip(self, params), oid = %params.oid, err)] - #[allow(clippy::blocks_in_if_conditions)] + #[allow(clippy::blocks_in_conditions)] async fn upsert_collab_with_transaction( &self, workspace_id: &str, diff --git a/src/biz/workspace/access_control.rs b/src/biz/workspace/access_control.rs index beb4d3185..3d72887da 100644 --- a/src/biz/workspace/access_control.rs +++ b/src/biz/workspace/access_control.rs @@ -201,7 +201,7 @@ where } #[instrument(level = "trace", skip_all, err)] - #[allow(clippy::blocks_in_if_conditions)] + #[allow(clippy::blocks_in_conditions)] async fn check_workspace_permission( &self, workspace_id: &Uuid, diff --git a/src/main.rs b/src/main.rs index f204777cb..fd89b9e5a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,6 +3,15 @@ use appflowy_cloud::config::config::get_configuration; use appflowy_cloud::telemetry::init_subscriber; use tracing::info; +// https://github.com/polarsignals/rust-jemalloc-pprof?tab=readme-ov-file#usage +#[cfg(not(target_env = "msvc"))] +#[global_allocator] +static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; + +#[allow(non_upper_case_globals)] +#[export_name = "malloc_conf"] +pub static malloc_conf: &[u8] = b"prof:true,prof_active:true,lg_prof_sample:19\0"; + #[actix_web::main] async fn main() -> anyhow::Result<()> { let level = std::env::var("RUST_LOG").unwrap_or("info".to_string()); From 0745a1bd75999a3a4034b4293238301f653d9d58 Mon Sep 17 00:00:00 2001 From: Zack Fu Zi Xiang Date: Wed, 8 May 2024 18:37:33 +0800 Subject: [PATCH 2/2] chore: merge with main --- Cargo.lock | 11 ++++++----- src/api/user.rs | 8 -------- src/application.rs | 2 ++ 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9591d241f..ecefec0da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -607,6 +607,7 @@ dependencies = [ "sqlx", "tempfile", "thiserror", + "tikv-jemallocator", "tokio", "tokio-stream", "tokio-tungstenite", @@ -685,7 +686,7 @@ dependencies = [ "infra", "log", "parking_lot 0.12.1", - "prost", + "prost 0.12.3", "rand 0.8.5", "redis 0.25.2", "serde", @@ -1482,7 +1483,7 @@ dependencies = [ "gotrue-entity", "mime", "parking_lot 0.12.1", - "prost", + "prost 0.12.3", "reqwest 0.11.27", "scraper", "semver", @@ -1675,7 +1676,7 @@ dependencies = [ "collab-entity", "collab-rt-protocol", "database-entity", - "prost", + "prost 0.12.3", "prost-build", "protoc-bin-vendored", "serde", @@ -6414,7 +6415,7 @@ dependencies = [ "hyper-timeout", "percent-encoding", "pin-project", - "prost", + "prost 0.12.3", "tokio", "tokio-stream", "tower", @@ -6440,7 +6441,7 @@ dependencies = [ name = "tonic-proto" version = "0.1.0" dependencies = [ - "prost", + "prost 0.12.3", "tonic", "tonic-build", "tracing", diff --git a/src/api/user.rs b/src/api/user.rs index e59144f66..4036d521b 100644 --- a/src/api/user.rs +++ b/src/api/user.rs @@ -18,14 +18,6 @@ pub fn user_scope() -> Scope { .service(web::resource("/workspace").route(web::get().to(get_user_workspace_info_handler))) } -async fn mem_pprof_handler() -> Vec { - let mut prof_ctl = jemalloc_pprof::PROF_CTL.as_ref().unwrap().lock().await; - if !prof_ctl.activated() { - panic!("heap profiling not activated"); - } - prof_ctl.dump_pprof().unwrap() -} - #[tracing::instrument(skip(state, path), err)] async fn verify_user_handler( path: web::Path, diff --git a/src/application.rs b/src/application.rs index 19c261042..bdb2b3539 100644 --- a/src/application.rs +++ b/src/application.rs @@ -1,6 +1,7 @@ use crate::api::metrics::metrics_scope; use crate::api::file_storage::file_storage_scope; +use crate::api::pprof::pprof_scope; use crate::api::user::user_scope; use crate::api::workspace::{collab_scope, workspace_scope}; use crate::api::ws::ws_scope; @@ -136,6 +137,7 @@ pub async fn run_actix_server( .service(ws_scope()) .service(file_storage_scope()) .service(metrics_scope()) + .service(pprof_scope()) .app_data(Data::new(state.metrics.registry.clone())) .app_data(Data::new(state.metrics.request_metrics.clone())) .app_data(Data::new(state.metrics.realtime_metrics.clone()))