diff --git a/Cargo.lock b/Cargo.lock index ed6c485b2..55f13a32d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -393,6 +393,55 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" + +[[package]] +name = "anstyle-parse" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" +dependencies = [ + "anstyle", + "windows-sys 0.59.0", +] + [[package]] name = "anyhow" version = "1.0.89" @@ -1354,6 +1403,46 @@ dependencies = [ "inout", ] +[[package]] +name = "clap" +version = "4.5.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "027bb0d98429ae334a8698531da7077bdf906419543a35a55c2cb1b66437d767" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5589e0cba072e0f3d23791efac0fd8627b49c829c196a492e88168e6a669d863" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim 0.11.1", +] + +[[package]] +name = "clap_derive" +version = "4.5.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf4ced95c6f4a675af3da73304b9ac4ed991640c36374e4b46795c49e17cf1ed" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.90", +] + +[[package]] +name = "clap_lex" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" + [[package]] name = "clickhouse" version = "0.11.6" @@ -1483,6 +1572,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" +[[package]] +name = "colorchoice" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" + [[package]] name = "combine" version = "4.6.7" @@ -2549,6 +2644,25 @@ dependencies = [ "syn 2.0.90", ] +[[package]] +name = "env_filter" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0" +dependencies = [ + "log", +] + +[[package]] +name = "env_logger" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcaee3d8e3cfc3fd92428d477bc97fc29ec8716d180c0d74c643bb26166660e0" +dependencies = [ + "env_filter", + "log", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -4032,6 +4146,28 @@ dependencies = [ "cfb", ] +[[package]] +name = "inferno" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "692eda1cc790750b9f5a5e3921ef9c117fd5498b97cfacbc910693e5b29002dc" +dependencies = [ + "ahash 0.8.11", + "clap", + "crossbeam-channel", + "crossbeam-utils 0.8.20", + "dashmap 6.1.0", + "env_logger", + "indexmap 2.5.0", + "itoa 1.0.11", + "log", + "num-format", + "once_cell", + "quick-xml 0.37.2", + "rgb", + "str_stack", +] + [[package]] name = "inotify" version = "0.9.6" @@ -4124,6 +4260,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "iso8601" version = "0.6.1" @@ -4205,23 +4347,20 @@ dependencies = [ ] [[package]] -name = "jemalloc-sys" -version = "0.5.4+5.3.0-patched" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac6c1946e1cea1788cbfde01c993b52a10e2da07f4bac608228d1bed20bfebf2" -dependencies = [ - "cc", - "libc", -] - -[[package]] -name = "jemallocator" -version = "0.5.4" +name = "jemalloc_pprof" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0de374a9f8e63150e6f5e8a60cc14c668226d7a347d8aee1a45766e3c4dd3bc" +checksum = "5622af6d21ff86ed7797ef98e11b8f302da25ec69a7db9f6cde8e2e1c8df9992" dependencies = [ - "jemalloc-sys", + "anyhow", "libc", + "mappings", + "once_cell", + "pprof_util", + "tempfile", + "tikv-jemalloc-ctl", + "tokio 1.42.0", + "tracing", ] [[package]] @@ -4402,7 +4541,7 @@ dependencies = [ "hyper-tls 0.5.0", "image 0.24.9", "itertools 0.12.1", - "jemallocator", + "jemalloc_pprof", "json-patch", "lazy_static", "lettre", @@ -4432,6 +4571,8 @@ dependencies = [ "sqlx", "tar", "thiserror 1.0.64", + "tikv-jemalloc-ctl", + "tikv-jemallocator", "tokio 1.42.0", "tokio-stream", "totp-rs", @@ -4699,6 +4840,19 @@ dependencies = [ "libc", ] +[[package]] +name = "mappings" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e434981a332777c2b3062652d16a55f8e74fa78e6b1882633f0d77399c84fc2a" +dependencies = [ + "anyhow", + "libc", + "once_cell", + "pprof_util", + "tracing", +] + [[package]] name = "markup5ever" version = "0.11.0" @@ -5183,6 +5337,20 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + [[package]] name = "num-bigint" version = "0.4.6" @@ -5210,12 +5378,31 @@ dependencies = [ "zeroize", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +[[package]] +name = "num-format" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3" +dependencies = [ + "arrayvec", + "itoa 1.0.11", +] + [[package]] name = "num-integer" version = "0.1.46" @@ -5236,6 +5423,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -6129,6 +6327,21 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "pprof_util" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa015c78eed2130951e22c58d2095849391e73817ab2e74f71b0b9f63dd8416" +dependencies = [ + "anyhow", + "backtrace", + "flate2", + "inferno", + "num", + "paste", + "prost", +] + [[package]] name = "ppv-lite86" version = "0.2.20" @@ -6408,6 +6621,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "quick-xml" +version = "0.37.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "165859e9e55f79d67b96c5d96f4e88b6f2695a1972849c15a6a3f5c59fc2c003" +dependencies = [ + "memchr", +] + [[package]] name = "quinn" version = "0.11.5" @@ -8319,6 +8541,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "str_stack" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9091b6114800a5f2141aee1d1b9d6ca3592ac062dc5decb3764ec5895a47b4eb" + [[package]] name = "strfmt" version = "0.2.4" @@ -9258,6 +9486,37 @@ dependencies = [ "weezl", ] +[[package]] +name = "tikv-jemalloc-ctl" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f21f216790c8df74ce3ab25b534e0718da5a1916719771d3fec23315c99e468b" +dependencies = [ + "libc", + "paste", + "tikv-jemalloc-sys", +] + +[[package]] +name = "tikv-jemalloc-sys" +version = "0.6.0+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd3c60906412afa9c2b5b5a48ca6a5abe5736aec9eb48ad05037a677e52e4e2d" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "tikv-jemallocator" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cec5ff18518d81584f477e9bfdf957f5bb0979b0bac3af4ca30b5b3ae2d2865" +dependencies = [ + "libc", + "tikv-jemalloc-sys", +] + [[package]] name = "time" version = "0.3.36" @@ -10104,6 +10363,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "0.8.2" diff --git a/apps/labrinth/Cargo.toml b/apps/labrinth/Cargo.toml index 928af6aa3..d7adce63a 100644 --- a/apps/labrinth/Cargo.toml +++ b/apps/labrinth/Cargo.toml @@ -125,7 +125,9 @@ lettre = "0.11.3" derive-new = "0.6.0" rust_iso3166 = "0.1.11" -jemallocator = { version = "0.5.4", optional = true } +tikv-jemallocator = { version = "0.6.0", features = ["profiling", "unprefixed_malloc_on_supported_platforms"] } +tikv-jemalloc-ctl = { version = "0.6.0", features = ["stats"] } +jemalloc_pprof = { version = "0.7.0", features = ["flamegraph"] } async-stripe = { version = "0.39.1", features = ["runtime-tokio-hyper-rustls"] } rusty-money = "0.4.1" @@ -135,6 +137,3 @@ ariadne = { path = "../../packages/ariadne" } [dev-dependencies] actix-http = "3.4.0" - -[features] -jemalloc = ["jemallocator"] diff --git a/apps/labrinth/src/lib.rs b/apps/labrinth/src/lib.rs index f520bd157..8df3ddd49 100644 --- a/apps/labrinth/src/lib.rs +++ b/apps/labrinth/src/lib.rs @@ -352,6 +352,7 @@ pub fn app_config( .app_data(labrinth_config.active_sockets.clone()) .app_data(labrinth_config.automated_moderation_queue.clone()) .app_data(web::Data::new(labrinth_config.stripe_client.clone())) + .configure(routes::debug::config) .configure(routes::v2::config) .configure(routes::v3::config) .configure(routes::internal::config) diff --git a/apps/labrinth/src/main.rs b/apps/labrinth/src/main.rs index fd3739c3d..df39ea10b 100644 --- a/apps/labrinth/src/main.rs +++ b/apps/labrinth/src/main.rs @@ -9,9 +9,14 @@ use std::sync::Arc; use tracing::{error, info}; use tracing_actix_web::TracingLogger; -#[cfg(feature = "jemalloc")] +#[cfg(not(target_env = "msvc"))] #[global_allocator] -static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; +static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; + +#[allow(non_upper_case_globals)] +#[export_name = "malloc_conf"] +pub static malloc_conf: &[u8] = + b"prof:true,prof_active:true,lg_prof_sample:19\0"; #[derive(Clone)] pub struct Pepper { @@ -105,6 +110,8 @@ async fn main() -> std::io::Result<()> { .register_and_set_metrics(&prometheus.registry) .await .expect("Failed to register redis metrics"); + labrinth::routes::debug::jemalloc_mmeory_stats(&prometheus.registry) + .expect("Failed to register jemalloc metrics"); let search_config = search::SearchConfig::new(None); diff --git a/apps/labrinth/src/routes/debug/mod.rs b/apps/labrinth/src/routes/debug/mod.rs new file mode 100644 index 000000000..163909f05 --- /dev/null +++ b/apps/labrinth/src/routes/debug/mod.rs @@ -0,0 +1,87 @@ +use crate::routes::ApiError; +use crate::util::cors::default_cors; +use crate::util::guards::admin_key_guard; +use actix_web::{get, HttpResponse}; +use prometheus::{IntGauge, Registry}; +use std::time::Duration; + +pub fn config(cfg: &mut actix_web::web::ServiceConfig) { + cfg.service( + actix_web::web::scope("/debug") + .wrap(default_cors()) + .service(heap) + .service(flame_graph), + ); +} + +#[get("pprof/heap", guard = "admin_key_guard")] +pub async fn heap() -> Result { + let mut prof_ctl = jemalloc_pprof::PROF_CTL.as_ref().unwrap().lock().await; + require_profiling_activated(&prof_ctl)?; + let pprof = prof_ctl + .dump_pprof() + .map_err(|err| ApiError::InvalidInput(err.to_string()))?; + + Ok(HttpResponse::Ok() + .content_type("application/octet-stream") + .body(pprof)) +} + +#[get("pprof/heap/flamegraph", guard = "admin_key_guard")] +pub async fn flame_graph() -> Result { + let mut prof_ctl = jemalloc_pprof::PROF_CTL.as_ref().unwrap().lock().await; + require_profiling_activated(&prof_ctl)?; + let svg = prof_ctl + .dump_flamegraph() + .map_err(|err| ApiError::InvalidInput(err.to_string()))?; + + Ok(HttpResponse::Ok().content_type("image/svg+xml").body(svg)) +} + +fn require_profiling_activated( + prof_ctl: &jemalloc_pprof::JemallocProfCtl, +) -> Result<(), ApiError> { + if prof_ctl.activated() { + Ok(()) + } else { + Err(ApiError::InvalidInput( + "Profiling is not activated".to_string(), + )) + } +} + +pub fn jemalloc_mmeory_stats( + registry: &Registry, +) -> Result<(), prometheus::Error> { + let allocated_mem = IntGauge::new( + "labrinth_memory_allocated", + "Labrinth allocated memory", + )?; + let resident_mem = + IntGauge::new("labrinth_resident_memory", "Labrinth resident memory")?; + + registry.register(Box::new(allocated_mem.clone()))?; + registry.register(Box::new(resident_mem.clone()))?; + + tokio::spawn(async move { + let e = tikv_jemalloc_ctl::epoch::mib().unwrap(); + let allocated = tikv_jemalloc_ctl::stats::allocated::mib().unwrap(); + let resident = tikv_jemalloc_ctl::stats::resident::mib().unwrap(); + + loop { + e.advance().unwrap(); + + if let Ok(allocated) = allocated.read() { + allocated_mem.set(allocated as i64); + } + + if let Ok(resident) = resident.read() { + resident_mem.set(resident as i64); + } + + tokio::time::sleep(Duration::from_secs(5)).await; + } + }); + + Ok(()) +} diff --git a/apps/labrinth/src/routes/mod.rs b/apps/labrinth/src/routes/mod.rs index c2a990fab..b5afc015f 100644 --- a/apps/labrinth/src/routes/mod.rs +++ b/apps/labrinth/src/routes/mod.rs @@ -12,6 +12,8 @@ pub mod internal; pub mod v2; pub mod v3; +pub mod debug; + pub mod v2_reroute; mod analytics;