From ec3377de23ce2c37fdeceab4ab1f89887aeff472 Mon Sep 17 00:00:00 2001
From: Zack Fu Zi Xiang <speed2exe@live.com.sg>
Date: Wed, 14 Feb 2024 21:31:29 +0800
Subject: [PATCH] feat: cpu profile pyroscope

---
 Cargo.lock                          | 320 ++++++++++++++++++++++++++--
 Cargo.toml                          |   3 +
 src/application.rs                  |   4 +-
 src/biz/collab/access_control.rs    |   2 +-
 src/biz/collab/storage.rs           |   2 +-
 src/biz/workspace/access_control.rs |   2 +-
 src/config/config.rs                |   2 +
 src/main.rs                         |  25 ++-
 8 files changed, 338 insertions(+), 22 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 87a8d3f1f..46b77ad62 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -539,7 +539,9 @@ dependencies = [
  "opener",
  "openssl",
  "prometheus-client",
- "prost",
+ "prost 0.12.3",
+ "pyroscope",
+ "pyroscope_pprofrs",
  "rand 0.8.5",
  "rcgen",
  "realtime",
@@ -725,7 +727,7 @@ version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "edcdbedc2236483ab103a53415653d6b4442ea6141baf1ffa85df29635e88436"
 dependencies = [
- "nix",
+ "nix 0.27.1",
  "rand 0.8.5",
 ]
 
@@ -752,6 +754,17 @@ dependencies = [
  "webpki-roots 0.22.6",
 ]
 
+[[package]]
+name = "atty"
+version = "0.2.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
+dependencies = [
+ "hermit-abi 0.1.19",
+ "libc",
+ "winapi",
+]
+
 [[package]]
 name = "autocfg"
 version = "1.1.0"
@@ -1229,6 +1242,23 @@ dependencies = [
  "inout",
 ]
 
+[[package]]
+name = "clap"
+version = "3.2.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123"
+dependencies = [
+ "atty",
+ "bitflags 1.3.2",
+ "clap_derive",
+ "clap_lex 0.2.4",
+ "indexmap 1.9.3",
+ "once_cell",
+ "strsim",
+ "termcolor",
+ "textwrap",
+]
+
 [[package]]
 name = "clap"
 version = "4.4.18"
@@ -1245,7 +1275,29 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4df4df40ec50c46000231c914968278b1eb05098cf8f1b3a518a95030e71d1c7"
 dependencies = [
  "anstyle",
- "clap_lex",
+ "clap_lex 0.6.0",
+]
+
+[[package]]
+name = "clap_derive"
+version = "3.2.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ae6371b8bdc8b7d3959e9cf7b22d4435ef3e79e138688421ec654acf8c81b008"
+dependencies = [
+ "heck",
+ "proc-macro-error",
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
+[[package]]
+name = "clap_lex"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5"
+dependencies = [
+ "os_str_bytes",
 ]
 
 [[package]]
@@ -1278,7 +1330,7 @@ dependencies = [
  "mime",
  "mime_guess",
  "parking_lot 0.12.1",
- "prost",
+ "prost 0.12.3",
  "realtime-entity",
  "realtime-protocol",
  "reqwest",
@@ -1533,6 +1585,15 @@ version = "0.8.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f"
 
+[[package]]
+name = "cpp_demangle"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7e8227005286ec39567949b33df9896bcadfa6051bccca2488129f108ca23119"
+dependencies = [
+ "cfg-if",
+]
+
 [[package]]
 name = "cpufeatures"
 version = "0.2.12"
@@ -1575,7 +1636,7 @@ dependencies = [
  "anes",
  "cast",
  "ciborium",
- "clap",
+ "clap 4.4.18",
  "criterion-plot",
  "is-terminal",
  "itertools 0.10.5",
@@ -1785,6 +1846,15 @@ dependencies = [
  "validator",
 ]
 
+[[package]]
+name = "debugid"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d"
+dependencies = [
+ "uuid",
+]
+
 [[package]]
 name = "deflate"
 version = "0.8.6"
@@ -2025,6 +2095,18 @@ version = "0.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "27573eac26f4dd11e2b1916c3fe1baa56407c83c71a773a8ba17ec0bca03b6b7"
 
+[[package]]
+name = "findshlibs"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "40b9e59cd0f7e0806cca4be089683ecb6434e602038df21fe6bf6711b2f07f64"
+dependencies = [
+ "cc",
+ "lazy_static",
+ "libc",
+ "winapi",
+]
+
 [[package]]
 name = "finl_unicode"
 version = "1.2.0"
@@ -2365,7 +2447,7 @@ dependencies = [
  "futures-sink",
  "futures-util",
  "http 0.2.11",
- "indexmap",
+ "indexmap 2.1.0",
  "slab",
  "tokio",
  "tokio-util",
@@ -2384,7 +2466,7 @@ dependencies = [
  "futures-sink",
  "futures-util",
  "http 1.0.0",
- "indexmap",
+ "indexmap 2.1.0",
  "slab",
  "tokio",
  "tokio-util",
@@ -2447,6 +2529,15 @@ dependencies = [
  "unicode-segmentation",
 ]
 
+[[package]]
+name = "hermit-abi"
+version = "0.1.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "hermit-abi"
 version = "0.3.4"
@@ -2764,6 +2855,16 @@ version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "206ca75c9c03ba3d4ace2460e57b189f39f43de612c2f85836e65c929701bb2d"
 
+[[package]]
+name = "indexmap"
+version = "1.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
+dependencies = [
+ "autocfg",
+ "hashbrown 0.12.3",
+]
+
 [[package]]
 name = "indexmap"
 version = "2.1.0"
@@ -2814,7 +2915,7 @@ version = "0.4.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0bad00257d07be169d870ab665980b06cdb366d792ad690bf2e76876dc503455"
 dependencies = [
- "hermit-abi",
+ "hermit-abi 0.3.4",
  "rustix",
  "windows-sys 0.52.0",
 ]
@@ -2879,6 +2980,12 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "json"
+version = "0.12.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "078e285eafdfb6c4b434e0d31e8cfcb5115b651496faca5749b88fafd4f23bfd"
+
 [[package]]
 name = "jsonwebtoken"
 version = "8.3.0"
@@ -2929,6 +3036,26 @@ version = "0.2.152"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7"
 
+[[package]]
+name = "libflate"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5ff4ae71b685bbad2f2f391fe74f6b7659a34871c08b210fdc039e43bee07d18"
+dependencies = [
+ "adler32",
+ "crc32fast",
+ "libflate_lz77",
+]
+
+[[package]]
+name = "libflate_lz77"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a52d3a8bfc85f250440e4424db7d857e241a3aebbbe301f3eb606ab15c39acbf"
+dependencies = [
+ "rle-decode-fast",
+]
+
 [[package]]
 name = "libm"
 version = "0.2.8"
@@ -3079,6 +3206,15 @@ version = "2.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149"
 
+[[package]]
+name = "memmap2"
+version = "0.9.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fe751422e4a8caa417e13c3ea66452215d7d63e19e604f4980461212f3ae1322"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "mime"
 version = "0.3.17"
@@ -3180,6 +3316,16 @@ version = "0.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a"
 
+[[package]]
+name = "names"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7bddcd3bf5144b6392de80e04c347cd7fab2508f6df16a85fc496ecd5cec39bc"
+dependencies = [
+ "clap 3.2.25",
+ "rand 0.8.5",
+]
+
 [[package]]
 name = "nanoid"
 version = "0.4.0"
@@ -3213,6 +3359,17 @@ version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54"
 
+[[package]]
+name = "nix"
+version = "0.26.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b"
+dependencies = [
+ "bitflags 1.3.2",
+ "cfg-if",
+ "libc",
+]
+
 [[package]]
 name = "nix"
 version = "0.27.1"
@@ -3341,7 +3498,7 @@ version = "1.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
 dependencies = [
- "hermit-abi",
+ "hermit-abi 0.3.4",
  "libc",
 ]
 
@@ -3456,6 +3613,12 @@ dependencies = [
  "hashbrown 0.12.3",
 ]
 
+[[package]]
+name = "os_str_bytes"
+version = "6.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1"
+
 [[package]]
 name = "overload"
 version = "0.1.1"
@@ -3558,7 +3721,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9"
 dependencies = [
  "fixedbitset",
- "indexmap",
+ "indexmap 2.1.0",
 ]
 
 [[package]]
@@ -3764,6 +3927,26 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
 
+[[package]]
+name = "pprof"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "978385d59daf9269189d052ca8a84c1acfd0715c0599a5d5188d4acc078ca46a"
+dependencies = [
+ "backtrace",
+ "cfg-if",
+ "findshlibs",
+ "libc",
+ "log",
+ "nix 0.26.4",
+ "once_cell",
+ "parking_lot 0.12.1",
+ "smallvec",
+ "symbolic-demangle",
+ "tempfile",
+ "thiserror",
+]
+
 [[package]]
 name = "ppv-lite86"
 version = "0.2.17"
@@ -3851,6 +4034,16 @@ dependencies = [
  "syn 2.0.48",
 ]
 
+[[package]]
+name = "prost"
+version = "0.11.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd"
+dependencies = [
+ "bytes",
+ "prost-derive 0.11.9",
+]
+
 [[package]]
 name = "prost"
 version = "0.12.3"
@@ -3858,7 +4051,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "146c289cda302b98a28d40c8b3b90498d6e526dd24ac2ecea73e4e491685b94a"
 dependencies = [
  "bytes",
- "prost-derive",
+ "prost-derive 0.12.3",
 ]
 
 [[package]]
@@ -3875,7 +4068,7 @@ dependencies = [
  "once_cell",
  "petgraph",
  "prettyplease",
- "prost",
+ "prost 0.12.3",
  "prost-types",
  "regex",
  "syn 2.0.48",
@@ -3883,6 +4076,19 @@ dependencies = [
  "which",
 ]
 
+[[package]]
+name = "prost-derive"
+version = "0.11.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4"
+dependencies = [
+ "anyhow",
+ "itertools 0.10.5",
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
 [[package]]
 name = "prost-derive"
 version = "0.12.3"
@@ -3902,7 +4108,7 @@ version = "0.12.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "193898f59edcf43c26227dcd4c8427f00d99d61e95dcde58dabd49fa291d470e"
 dependencies = [
- "prost",
+ "prost 0.12.3",
 ]
 
 [[package]]
@@ -4002,6 +4208,36 @@ dependencies = [
  "unicase",
 ]
 
+[[package]]
+name = "pyroscope"
+version = "0.5.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac8a53ce01af1087eaeee6ce7c4fbf50ea4040ab1825c0115c4bafa039644ba9"
+dependencies = [
+ "json",
+ "libc",
+ "libflate",
+ "log",
+ "names",
+ "prost 0.11.9",
+ "reqwest",
+ "thiserror",
+ "url",
+ "winapi",
+]
+
+[[package]]
+name = "pyroscope_pprofrs"
+version = "0.2.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "43f010b2a981a7f8449a650f25f309e520b5206ea2d89512dcb146aaa5518ff4"
+dependencies = [
+ "log",
+ "pprof",
+ "pyroscope",
+ "thiserror",
+]
+
 [[package]]
 name = "quanta"
 version = "0.11.1"
@@ -4235,7 +4471,7 @@ dependencies = [
  "collab",
  "collab-entity",
  "database-entity",
- "prost",
+ "prost 0.12.3",
  "prost-build",
  "protoc-bin-vendored",
  "realtime-protocol",
@@ -4510,6 +4746,12 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "rle-decode-fast"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422"
+
 [[package]]
 name = "rsa"
 version = "0.9.6"
@@ -5175,7 +5417,7 @@ dependencies = [
  "futures-util",
  "hashlink",
  "hex",
- "indexmap",
+ "indexmap 2.1.0",
  "log",
  "memchr",
  "once_cell",
@@ -5399,12 +5641,41 @@ dependencies = [
  "unicode-normalization",
 ]
 
+[[package]]
+name = "strsim"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
+
 [[package]]
 name = "subtle"
 version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc"
 
+[[package]]
+name = "symbolic-common"
+version = "12.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1cccfffbc6bb3bb2d3a26cd2077f4d055f6808d266f9d4d158797a4c60510dfe"
+dependencies = [
+ "debugid",
+ "memmap2",
+ "stable_deref_trait",
+ "uuid",
+]
+
+[[package]]
+name = "symbolic-demangle"
+version = "12.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "76a99812da4020a67e76c4eb41f08c87364c14170495ff780f30dd519c221a68"
+dependencies = [
+ "cpp_demangle",
+ "rustc-demangle",
+ "symbolic-common",
+]
+
 [[package]]
 name = "syn"
 version = "1.0.109"
@@ -5514,6 +5785,21 @@ dependencies = [
  "utf-8",
 ]
 
+[[package]]
+name = "termcolor"
+version = "1.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755"
+dependencies = [
+ "winapi-util",
+]
+
+[[package]]
+name = "textwrap"
+version = "0.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d"
+
 [[package]]
 name = "thiserror"
 version = "1.0.56"
@@ -5759,7 +6045,7 @@ version = "0.21.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03"
 dependencies = [
- "indexmap",
+ "indexmap 2.1.0",
  "toml_datetime",
  "winnow",
 ]
@@ -6568,7 +6854,7 @@ dependencies = [
  "collab-entity",
  "collab-folder",
  "getrandom 0.2.12",
- "indexmap",
+ "indexmap 2.1.0",
  "nanoid",
  "serde",
  "serde_json",
diff --git a/Cargo.toml b/Cargo.toml
index d3b8f4524..c4b48a3d6 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -85,6 +85,9 @@ shared-entity = { path = "libs/shared-entity", features = ["cloud"] }
 workspace-template = { workspace = true }
 realtime-entity.workspace = true
 
+# Pyroscope for profiling
+pyroscope = "0.5.7"
+pyroscope_pprofrs = "0.2.7"
 
 # profiling
 #puffin = "0.16.0"
diff --git a/src/application.rs b/src/application.rs
index 5c9abd65c..423b93672 100644
--- a/src/application.rs
+++ b/src/application.rs
@@ -60,7 +60,9 @@ impl Application {
   }
 
   pub async fn run_until_stopped(self) -> Result<(), std::io::Error> {
-    self.server.await
+    let stopped = self.server.await;
+    tracing::error!("AppFlowy Cloud Shutdown: {:?}", stopped);
+    Ok(())
   }
 
   pub fn port(&self) -> u16 {
diff --git a/src/biz/collab/access_control.rs b/src/biz/collab/access_control.rs
index d0db706d5..dd7ccd0fa 100644
--- a/src/biz/collab/access_control.rs
+++ b/src/biz/collab/access_control.rs
@@ -36,7 +36,7 @@ where
   }
 
   #[instrument(level = "debug", skip_all, err)]
-  #[allow(clippy::blocks_in_if_conditions)]
+  #[allow(clippy::blocks_in_conditions)]
   async fn check_collab_permission(
     &self,
     oid: &str,
diff --git a/src/biz/collab/storage.rs b/src/biz/collab/storage.rs
index 228ef17d9..00c4d3bc7 100644
--- a/src/biz/collab/storage.rs
+++ b/src/biz/collab/storage.rs
@@ -182,7 +182,7 @@ where
   }
 
   #[instrument(level = "trace", skip(self, params), oid = %params.oid, err)]
-  #[allow(clippy::blocks_in_if_conditions)]
+  #[allow(clippy::blocks_in_conditions)]
   async fn upsert_collab_with_transaction(
     &self,
     workspace_id: &str,
diff --git a/src/biz/workspace/access_control.rs b/src/biz/workspace/access_control.rs
index beb4d3185..3d72887da 100644
--- a/src/biz/workspace/access_control.rs
+++ b/src/biz/workspace/access_control.rs
@@ -201,7 +201,7 @@ where
   }
 
   #[instrument(level = "trace", skip_all, err)]
-  #[allow(clippy::blocks_in_if_conditions)]
+  #[allow(clippy::blocks_in_conditions)]
   async fn check_workspace_permission(
     &self,
     workspace_id: &Uuid,
diff --git a/src/config/config.rs b/src/config/config.rs
index d8b5d2d0a..e78b47a51 100644
--- a/src/config/config.rs
+++ b/src/config/config.rs
@@ -14,6 +14,7 @@ pub struct Config {
   pub redis_uri: Secret<String>,
   pub s3: S3Setting,
   pub casbin: CasbinSetting,
+  pub pyroscope_url: String,
 }
 
 #[derive(serde::Deserialize, Clone, Debug)]
@@ -132,6 +133,7 @@ pub fn get_configuration() -> Result<Config, anyhow::Error> {
     casbin: CasbinSetting {
       pool_size: get_env_var("APPFLOWY_CASBIN_POOL_SIZE", "8").parse()?,
     },
+    pyroscope_url: get_env_var("APPFLOWY_PYROSCOPE_URL", "http://localhost:4040"),
   };
   Ok(config)
 }
diff --git a/src/main.rs b/src/main.rs
index f204777cb..9a05eac57 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -1,6 +1,9 @@
 use appflowy_cloud::application::{init_state, Application};
-use appflowy_cloud::config::config::get_configuration;
+use appflowy_cloud::config::config::{get_configuration, Config};
 use appflowy_cloud::telemetry::init_subscriber;
+use pyroscope::pyroscope::PyroscopeAgentRunning;
+use pyroscope::PyroscopeAgent;
+use pyroscope_pprofrs::{pprof_backend, PprofConfig};
 use tracing::info;
 
 #[actix_web::main]
@@ -50,6 +53,8 @@ async fn main() -> anyhow::Result<()> {
     dotenvy::dotenv().ok();
   }
 
+  let _pyro_agent_running = init_pyroscope(&conf)?;
+
   let state = init_state(&conf)
     .await
     .map_err(|e| anyhow::anyhow!("Failed to initialize application state: {}", e))?;
@@ -58,3 +63,21 @@ async fn main() -> anyhow::Result<()> {
 
   Ok(())
 }
+
+// https://grafana.com/docs/pyroscope/latest/configure-client/language-sdks/rust/
+fn init_pyroscope(conf: &Config) -> anyhow::Result<PyroscopeAgent<PyroscopeAgentRunning>> {
+  let pyroscope_url = conf.pyroscope_url.as_str();
+  info!("Pyroscope URL: {}", pyroscope_url);
+
+  // Configure profiling backend
+  let pprof_config = PprofConfig::new().sample_rate(100);
+  let backend_impl = pprof_backend(pprof_config);
+
+  // Configure Pyroscope Agent
+  // let agent = PyroscopeAgent::builder(conf.pyroscope_url.as_str(), "appflowy-cloud")
+  let agent = PyroscopeAgent::builder(pyroscope_url, "appflowy-cloud")
+    .backend(backend_impl)
+    .build()?;
+  let pyro_agent_running = agent.start()?;
+  Ok(pyro_agent_running)
+}