diff --git a/Cargo.lock b/Cargo.lock index 77909c23..baaf348a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -60,7 +60,7 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" dependencies = [ - "getrandom 0.2.9", + "getrandom 0.2.11", "once_cell", "version_check", ] @@ -103,8 +103,8 @@ checksum = "cf7d535e1381be3de2c0716c0a1c1e32ad9df1042cddcf7bc18d743569e53319" dependencies = [ "anchor-syn", "anyhow", - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.69", + "quote 1.0.33", "regex", "syn 1.0.109", ] @@ -118,8 +118,8 @@ dependencies = [ "anchor-syn", "anyhow", "bs58 0.4.0", - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.69", + "quote 1.0.33", "rustversion", "syn 1.0.109", ] @@ -131,7 +131,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1be64a48e395fe00b8217287f226078be2cf32dae42fdf8a885b997945c3d28" dependencies = [ "anchor-syn", - "proc-macro2 1.0.58", + "proc-macro2 1.0.69", "syn 1.0.109", ] @@ -142,8 +142,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38ea6713d1938c0da03656ff8a693b17dc0396da66d1ba320557f07e86eca0d4" dependencies = [ "anchor-syn", - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 1.0.109", ] @@ -155,8 +155,8 @@ checksum = "d401f11efb3644285685f8339829a9786d43ed7490bb1699f33c478d04d5a582" dependencies = [ "anchor-syn", "anyhow", - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 1.0.109", ] @@ -169,8 +169,8 @@ dependencies = [ "anchor-syn", "anyhow", "heck 0.3.3", - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 1.0.109", ] @@ -182,8 +182,8 @@ checksum = "6ad769993b5266714e8939e47fbdede90e5c030333c7522d99a4d4748cf26712" dependencies = [ "anchor-syn", "anyhow", - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 1.0.109", ] @@ -195,8 +195,8 @@ checksum = "4e677fae4a016a554acdd0e3b7f178d3acafaa7e7ffac6b8690cf4e171f1c116" dependencies = [ "anchor-syn", "anyhow", - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 1.0.109", ] @@ -225,8 +225,8 @@ checksum = "340beef6809d1c3fcc7ae219153d981e95a8a277ff31985bd7050e32645dc9a8" dependencies = [ "anchor-syn", "anyhow", - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 1.0.109", ] @@ -275,9 +275,9 @@ dependencies = [ "anyhow", "bs58 0.3.1", "heck 0.3.3", - "proc-macro2 1.0.58", + "proc-macro2 1.0.69", "proc-macro2-diagnostics", - "quote 1.0.27", + "quote 1.0.33", "serde", "serde_json", "sha2 0.9.9", @@ -340,7 +340,7 @@ dependencies = [ "num-traits", "rusticata-macros", "thiserror", - "time 0.3.7", + "time 0.3.20", ] [[package]] @@ -349,8 +349,8 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "726535892e8eae7e70657b4c8ea93d26b8553afb1ce617caee529ef96d7dee6c" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 1.0.109", "synstructure", ] @@ -361,8 +361,8 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2777730b2039ac0f95f093556e61b6d26cebed5393ca6f152717777cec3a42ed" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 1.0.109", ] @@ -372,6 +372,17 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" +[[package]] +name = "async-channel" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" +dependencies = [ + "concurrent-queue", + "event-listener", + "futures-core", +] + [[package]] name = "async-compression" version = "0.3.15" @@ -395,15 +406,37 @@ dependencies = [ "event-listener", ] +[[package]] +name = "async-stream" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +dependencies = [ + "proc-macro2 1.0.69", + "quote 1.0.33", + "syn 2.0.39", +] + [[package]] name = "async-trait" version = "0.1.68" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", - "syn 2.0.16", + "proc-macro2 1.0.69", + "quote 1.0.33", + "syn 2.0.39", ] [[package]] @@ -437,6 +470,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "autotools" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef8da1805e028a172334c3b680f93e71126f2327622faef2ec3d893c0a4ad77" +dependencies = [ + "cc", +] + [[package]] name = "aws-creds" version = "0.34.1" @@ -450,7 +492,7 @@ dependencies = [ "rust-ini", "serde", "thiserror", - "time 0.3.7", + "time 0.3.20", "url", ] @@ -463,12 +505,71 @@ dependencies = [ "thiserror", ] +[[package]] +name = "axum" +version = "0.6.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" +dependencies = [ + "async-trait", + "axum-core", + "bitflags 1.3.2", + "bytes", + "futures-util", + "http", + "http-body", + "hyper", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "mime", + "rustversion", + "tower-layer", + "tower-service", +] + [[package]] name = "az" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b7e4c2464d97fe331d41de9d5db0def0a96f4d823b8b32a2efd503578988973" +[[package]] +name = "backoff" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" +dependencies = [ + "futures-core", + "getrandom 0.2.11", + "instant", + "pin-project-lite", + "rand 0.8.5", + "tokio", +] + [[package]] name = "base64" version = "0.12.3" @@ -508,6 +609,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" + [[package]] name = "bitmaps" version = "2.1.0" @@ -575,7 +682,7 @@ dependencies = [ "borsh-derive-internal", "borsh-schema-derive-internal", "proc-macro-crate 0.1.5", - "proc-macro2 1.0.58", + "proc-macro2 1.0.69", "syn 1.0.109", ] @@ -585,8 +692,8 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5449c28a7b352f2d1e592a8a28bf139bc71afb0764a14f3c02500935d8c44065" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 1.0.109", ] @@ -596,8 +703,8 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdbd5696d8bfa21d53d9fe39a714a18538bad11492a42d066dbbc395fb1951c0" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 1.0.109", ] @@ -665,9 +772,9 @@ version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdde5c9cd29ebd706ce1b35600920a33550e402fc998a2e53ad3b42c3c47a192" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", - "syn 2.0.16", + "proc-macro2 1.0.69", + "quote 1.0.33", + "syn 2.0.39", ] [[package]] @@ -715,11 +822,12 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.79" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" dependencies = [ "jobserver", + "libc", ] [[package]] @@ -753,6 +861,28 @@ dependencies = [ "chrono", ] +[[package]] +name = "chrono-tz" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e23185c0e21df6ed832a12e2bda87c7d1def6842881fb634a8511ced741b0d76" +dependencies = [ + "chrono", + "chrono-tz-build", + "phf", +] + +[[package]] +name = "chrono-tz-build" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "433e39f13c9a060046954e0592a8d0a4bcb1040125cbf91cb8ee58964cfb350f" +dependencies = [ + "parse-zoneinfo", + "phf", + "phf_codegen", +] + [[package]] name = "cipher" version = "0.3.0" @@ -780,7 +910,7 @@ checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" dependencies = [ "ansi_term", "atty", - "bitflags", + "bitflags 1.3.2", "strsim 0.8.0", "textwrap 0.11.0", "unicode-width", @@ -794,10 +924,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" dependencies = [ "atty", - "bitflags", + "bitflags 1.3.2", "clap_derive", "clap_lex", - "indexmap", + "indexmap 1.9.3", "once_cell", "strsim 0.10.0", "termcolor", @@ -812,8 +942,8 @@ checksum = "ae6371b8bdc8b7d3959e9cf7b22d4435ef3e79e138688421ec654acf8c81b008" dependencies = [ "heck 0.4.1", "proc-macro-error", - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 1.0.109", ] @@ -839,6 +969,15 @@ dependencies = [ "unreachable", ] +[[package]] +name = "concurrent-queue" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f057a694a54f12365049b0958a1685bb52d567f5593b355fbf685838e873d400" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "console" version = "0.15.7" @@ -993,7 +1132,7 @@ version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096" dependencies = [ - "quote 1.0.27", + "quote 1.0.33", "syn 1.0.109", ] @@ -1160,9 +1299,9 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", - "syn 2.0.16", + "proc-macro2 1.0.69", + "quote 1.0.33", + "syn 2.0.39", ] [[package]] @@ -1194,6 +1333,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0688c2a7f92e427f44895cd63841bff7b29f8d7a1648b9e7e07a4a365b2e1257" +[[package]] +name = "dotenv" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" + [[package]] name = "dyn-clone" version = "1.0.11" @@ -1248,8 +1393,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "079044df30bb07de7d846d41a184c4b00e66ebdac93ee459253474f3a47e50ae" dependencies = [ "enum-ordinalize", - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 1.0.109", ] @@ -1289,8 +1434,8 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8958699f9359f0b04e691a13850d48b7de329138023876d07cbd024c2c820598" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 1.0.109", ] @@ -1302,9 +1447,9 @@ checksum = "e4f76552f53cefc9a7f64987c3701b99d982f7690606fd67de1d09712fbf52f1" dependencies = [ "num-bigint 0.4.3", "num-traits", - "proc-macro2 1.0.58", - "quote 1.0.27", - "syn 2.0.16", + "proc-macro2 1.0.69", + "quote 1.0.33", + "syn 2.0.39", ] [[package]] @@ -1314,8 +1459,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "11f36e95862220b211a6e2aa5eca09b4fa391b13cd52ceb8035a24bf65a79de2" dependencies = [ "once_cell", - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 1.0.109", ] @@ -1332,6 +1477,32 @@ dependencies = [ "termcolor", ] +[[package]] +name = "envconfig" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea81cc7e21f55a9d9b1efb6816904978d0bfbe31a50347cb24b2e75564bcac9b" +dependencies = [ + "envconfig_derive", +] + +[[package]] +name = "envconfig_derive" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dfca278e5f84b45519acaaff758ebfa01f18e96998bc24b8f1b722dd804b9bf" +dependencies = [ + "proc-macro2 1.0.69", + "quote 1.0.33", + "syn 1.0.109", +] + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "errno" version = "0.3.1" @@ -1418,8 +1589,8 @@ dependencies = [ "fixed", "paste", "proc-macro-error", - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 1.0.109", ] @@ -1433,6 +1604,12 @@ dependencies = [ "fixed-macro-impl", ] +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + [[package]] name = "flate2" version = "1.0.26" @@ -1527,9 +1704,9 @@ version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", - "syn 2.0.16", + "proc-macro2 1.0.69", + "quote 1.0.33", + "syn 2.0.39", ] [[package]] @@ -1571,6 +1748,29 @@ dependencies = [ "byteorder", ] +[[package]] +name = "gcp-bigquery-client" +version = "0.16.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb67dedb3fce3a6d25829158e40b22c5e9aa107fdbae0f2fef564931c9161a6a" +dependencies = [ + "async-stream", + "async-trait", + "dyn-clone", + "hyper", + "hyper-rustls 0.24.2", + "log", + "reqwest", + "serde", + "serde_json", + "thiserror", + "time 0.3.20", + "tokio", + "tokio-stream", + "url", + "yup-oauth2", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -1607,9 +1807,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.9" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" +checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" dependencies = [ "cfg-if", "js-sys", @@ -1629,6 +1829,98 @@ dependencies = [ "scroll", ] +[[package]] +name = "google-cloud-auth" +version = "0.9.0" +source = "git+https://github.com/mrgnlabs/google-cloud-rust.git?rev=3f651f2d9fd8cca547bb11490d2575d9bf90f994#3f651f2d9fd8cca547bb11490d2575d9bf90f994" +dependencies = [ + "async-trait", + "base64 0.21.1", + "google-cloud-metadata", + "google-cloud-token", + "home", + "jsonwebtoken", + "reqwest", + "serde", + "serde_json", + "thiserror", + "time 0.3.20", + "tokio", + "tracing", + "urlencoding", +] + +[[package]] +name = "google-cloud-default" +version = "0.1.0" +source = "git+https://github.com/mrgnlabs/google-cloud-rust.git?rev=3f651f2d9fd8cca547bb11490d2575d9bf90f994#3f651f2d9fd8cca547bb11490d2575d9bf90f994" +dependencies = [ + "async-trait", + "google-cloud-auth", + "google-cloud-gax", + "google-cloud-pubsub", +] + +[[package]] +name = "google-cloud-gax" +version = "0.13.0" +source = "git+https://github.com/mrgnlabs/google-cloud-rust.git?rev=3f651f2d9fd8cca547bb11490d2575d9bf90f994#3f651f2d9fd8cca547bb11490d2575d9bf90f994" +dependencies = [ + "google-cloud-token", + "http", + "thiserror", + "tokio", + "tokio-retry", + "tokio-util 0.7.2", + "tonic", + "tower", + "tracing", +] + +[[package]] +name = "google-cloud-googleapis" +version = "0.7.0" +source = "git+https://github.com/mrgnlabs/google-cloud-rust.git?rev=3f651f2d9fd8cca547bb11490d2575d9bf90f994#3f651f2d9fd8cca547bb11490d2575d9bf90f994" +dependencies = [ + "prost", + "prost-types", + "tonic", +] + +[[package]] +name = "google-cloud-metadata" +version = "0.3.2" +source = "git+https://github.com/mrgnlabs/google-cloud-rust.git?rev=3f651f2d9fd8cca547bb11490d2575d9bf90f994#3f651f2d9fd8cca547bb11490d2575d9bf90f994" +dependencies = [ + "reqwest", + "thiserror", + "tokio", +] + +[[package]] +name = "google-cloud-pubsub" +version = "0.13.0" +source = "git+https://github.com/mrgnlabs/google-cloud-rust.git?rev=3f651f2d9fd8cca547bb11490d2575d9bf90f994#3f651f2d9fd8cca547bb11490d2575d9bf90f994" +dependencies = [ + "async-channel", + "async-stream", + "google-cloud-gax", + "google-cloud-googleapis", + "google-cloud-token", + "prost-types", + "thiserror", + "tokio", + "tracing", +] + +[[package]] +name = "google-cloud-token" +version = "0.1.0" +source = "git+https://github.com/mrgnlabs/google-cloud-rust.git?rev=3f651f2d9fd8cca547bb11490d2575d9bf90f994#3f651f2d9fd8cca547bb11490d2575d9bf90f994" +dependencies = [ + "async-trait", +] + [[package]] name = "h2" version = "0.3.19" @@ -1641,7 +1933,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap", + "indexmap 1.9.3", "slab", "tokio", "tokio-util 0.7.2", @@ -1684,6 +1976,12 @@ dependencies = [ "ahash", ] +[[package]] +name = "hashbrown" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" + [[package]] name = "heck" version = "0.3.3" @@ -1768,6 +2066,15 @@ dependencies = [ "hmac 0.8.1", ] +[[package]] +name = "home" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" +dependencies = [ + "windows-sys 0.48.0", +] + [[package]] name = "http" version = "0.2.9" @@ -1840,9 +2147,37 @@ checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" dependencies = [ "http", "hyper", - "rustls", + "rustls 0.20.8", "tokio", - "tokio-rustls", + "tokio-rustls 0.23.4", +] + +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http", + "hyper", + "log", + "rustls 0.21.9", + "rustls-native-certs", + "tokio", + "tokio-rustls 0.24.1", +] + +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper", + "pin-project-lite", + "tokio", + "tokio-io-timeout", ] [[package]] @@ -1923,6 +2258,16 @@ dependencies = [ "hashbrown 0.12.3", ] +[[package]] +name = "indexmap" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" +dependencies = [ + "equivalent", + "hashbrown 0.14.2", +] + [[package]] name = "indicatif" version = "0.16.2" @@ -2003,6 +2348,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "json" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "078e285eafdfb6c4b434e0d31e8cfcb5115b651496faca5749b88fafd4f23bfd" + [[package]] name = "jsonrpc-core" version = "18.0.0" @@ -2018,6 +2369,20 @@ dependencies = [ "serde_json", ] +[[package]] +name = "jsonwebtoken" +version = "8.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6971da4d9c3aa03c3d8f3ff0f4155b534aad021292003895a469716b2a230378" +dependencies = [ + "base64 0.21.1", + "pem", + "ring 0.16.20", + "serde", + "serde_json", + "simple_asn1", +] + [[package]] name = "keccak" version = "0.1.4" @@ -2035,9 +2400,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.144" +version = "0.2.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b00cc1c228a6782d0f076e7b232802e0c5689d41bb5df366f2a6b6621cfdfe1" +checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" [[package]] name = "libloading" @@ -2109,6 +2474,12 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" +[[package]] +name = "linux-raw-sys" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "969488b55f8ac402214f3f5fd243ebb7206cf82de60d3172994707a4bcc2b829" + [[package]] name = "liquidity-incentive-program" version = "0.1.0" @@ -2264,14 +2635,87 @@ dependencies = [ "type-layout", ] +[[package]] +name = "marginfi-v2-indexer" +version = "0.1.0" +dependencies = [ + "anchor-client", + "anyhow", + "backoff", + "base64 0.21.1", + "bincode", + "bs58 0.4.0", + "bytemuck", + "bytes", + "chrono", + "chrono-tz", + "clap 3.2.25", + "concurrent-queue", + "dotenv", + "envconfig", + "fixed", + "fixed-macro", + "futures", + "gcp-bigquery-client", + "google-cloud-auth", + "google-cloud-default", + "google-cloud-gax", + "google-cloud-googleapis", + "google-cloud-pubsub", + "itertools", + "json", + "lazy_static", + "marginfi", + "prost", + "prost-derive", + "protobuf-src", + "pyth-sdk-solana", + "rayon", + "serde", + "serde_json", + "serde_yaml 0.9.27", + "solana-account-decoder", + "solana-client", + "solana-measure", + "solana-metrics", + "solana-sdk", + "solana-transaction-status", + "spl-token", + "thiserror", + "tokio", + "tokio-stream", + "tonic", + "tonic-build", + "tracing", + "tracing-stackdriver", + "tracing-subscriber", + "uuid", + "yup-oauth2", +] + +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + [[package]] name = "maybe-async" version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f1b8c13cb1f814b634a96b2c725449fe7ed464a7b8781de8688be5ffbd3f305" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 1.0.109", ] @@ -2403,11 +2847,17 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a7d5f7076603ebc68de2dc6a650ec331a062a13abaa346975be747bbfa4b789" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 1.0.109", ] +[[package]] +name = "multimap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" + [[package]] name = "native-tls" version = "0.2.11" @@ -2432,7 +2882,7 @@ version = "0.24.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa52e972a9a719cecb6864fb88568781eb706bac2cd1d4f04a648542dbf78069" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cfg-if", "libc", "memoffset 0.6.5", @@ -2457,6 +2907,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + [[package]] name = "num" version = "0.2.1" @@ -2509,8 +2969,8 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "876a53fff98e03a936a674b29568b0e605f06b29372c2489ff4de23f1949743d" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 1.0.109", ] @@ -2582,8 +3042,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcbff9bc912032c62bf65ef1d5aea88983b420f4f839db1e9b0c281a25c9c799" dependencies = [ "proc-macro-crate 1.1.0", - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 1.0.109", ] @@ -2629,7 +3089,7 @@ version = "0.10.52" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "01b8574602df80f7b85fdfc5392fa884a4e3b3f4f35402c070ab34c3d3f78d56" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cfg-if", "foreign-types", "libc", @@ -2644,9 +3104,9 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", - "syn 2.0.16", + "proc-macro2 1.0.69", + "quote 1.0.33", + "syn 2.0.39", ] [[package]] @@ -2720,8 +3180,8 @@ checksum = "5f7d21ccd03305a674437ee1248f3ab5d4b1db095cf1caf49f1713ddf61956b7" dependencies = [ "Inflector", "proc-macro-error", - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 1.0.109", ] @@ -2734,6 +3194,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + [[package]] name = "parking_lot" version = "0.11.2" @@ -2782,6 +3248,15 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "parse-zoneinfo" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c705f256449c60da65e11ff6626e0c16a0a0b96aaa348de61376b249bc340f41" +dependencies = [ + "regex", +] + [[package]] name = "paste" version = "1.0.12" @@ -2830,6 +3305,54 @@ dependencies = [ "num", ] +[[package]] +name = "petgraph" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" +dependencies = [ + "fixedbitset", + "indexmap 2.1.0", +] + +[[package]] +name = "phf" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_codegen" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8d39688d359e6b34654d328e262234662d16cc0f60ec8dcbe5e718709342a5a" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" +dependencies = [ + "phf_shared", + "rand 0.8.5", +] + +[[package]] +name = "phf_shared" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project" version = "1.1.0" @@ -2845,9 +3368,9 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39407670928234ebc5e6e580247dd567ad73a3578460c5990f9503df207e8f07" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", - "syn 2.0.16", + "proc-macro2 1.0.69", + "quote 1.0.33", + "syn 2.0.39", ] [[package]] @@ -2915,6 +3438,16 @@ dependencies = [ "yansi", ] +[[package]] +name = "prettyplease" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" +dependencies = [ + "proc-macro2 1.0.69", + "syn 1.0.109", +] + [[package]] name = "proc-macro-crate" version = "0.1.5" @@ -2941,8 +3474,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ "proc-macro-error-attr", - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 1.0.109", "version_check", ] @@ -2953,8 +3486,8 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.69", + "quote 1.0.33", "version_check", ] @@ -2968,25 +3501,88 @@ dependencies = [ ] [[package]] -name = "proc-macro2" -version = "1.0.58" +name = "proc-macro2" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proc-macro2-diagnostics" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bf29726d67464d49fa6224a1d07936a8c08bb3fba727c7493f6cf1616fdaada" +dependencies = [ + "proc-macro2 1.0.69", + "quote 1.0.33", + "syn 1.0.109", + "version_check", + "yansi", +] + +[[package]] +name = "prost" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" +dependencies = [ + "bytes", + "heck 0.4.1", + "itertools", + "lazy_static", + "log", + "multimap", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 1.0.109", + "tempfile", + "which", +] + +[[package]] +name = "prost-derive" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2 1.0.69", + "quote 1.0.33", + "syn 1.0.109", +] + +[[package]] +name = "prost-types" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa1fb82fc0c281dd9671101b66b771ebbe1eaf967b96ac8740dcba4b70005ca8" +checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" dependencies = [ - "unicode-ident", + "prost", ] [[package]] -name = "proc-macro2-diagnostics" -version = "0.9.1" +name = "protobuf-src" +version = "1.1.0+21.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bf29726d67464d49fa6224a1d07936a8c08bb3fba727c7493f6cf1616fdaada" +checksum = "c7ac8852baeb3cc6fb83b93646fb93c0ffe5d14bf138c945ceb4b9948ee0e3c1" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", - "syn 1.0.109", - "version_check", - "yansi", + "autotools", ] [[package]] @@ -3050,7 +3646,7 @@ dependencies = [ "fxhash", "quinn-proto", "quinn-udp", - "rustls", + "rustls 0.20.8", "thiserror", "tokio", "tracing", @@ -3066,8 +3662,8 @@ dependencies = [ "bytes", "fxhash", "rand 0.8.5", - "ring", - "rustls", + "ring 0.16.20", + "rustls 0.20.8", "rustls-native-certs", "rustls-pemfile 0.2.1", "slab", @@ -3102,11 +3698,11 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.27" +version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f4f29d145265ec1c483c7c654450edde0bfe043d3938d6972630663356d9500" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" dependencies = [ - "proc-macro2 1.0.58", + "proc-macro2 1.0.69", ] [[package]] @@ -3168,7 +3764,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.9", + "getrandom 0.2.11", ] [[package]] @@ -3218,8 +3814,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6413f3de1edee53342e6138e75b56d32e7bc6e332b3bd62d497b1929d4cfbcdd" dependencies = [ "pem", - "ring", - "time 0.3.7", + "ring 0.16.20", + "time 0.3.20", "yasna", ] @@ -3229,7 +3825,7 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -3238,7 +3834,7 @@ version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -3247,7 +3843,7 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" dependencies = [ - "getrandom 0.2.9", + "getrandom 0.2.11", "redox_syscall 0.2.16", "thiserror", ] @@ -3260,9 +3856,24 @@ checksum = "d1a59b5d8e97dee33696bf13c5ba8ab85341c002922fba050069326b9c498974" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-syntax 0.7.2", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax 0.6.29", ] +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + [[package]] name = "regex-syntax" version = "0.7.2" @@ -3285,7 +3896,7 @@ dependencies = [ "http", "http-body", "hyper", - "hyper-rustls", + "hyper-rustls 0.23.2", "hyper-tls", "ipnet", "js-sys", @@ -3295,14 +3906,14 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls", + "rustls 0.20.8", "rustls-pemfile 1.0.2", "serde", "serde_json", "serde_urlencoded", "tokio", "tokio-native-tls", - "tokio-rustls", + "tokio-rustls 0.23.4", "tokio-util 0.7.2", "tower-service", "url", @@ -3323,12 +3934,26 @@ dependencies = [ "cc", "libc", "once_cell", - "spin", - "untrusted", + "spin 0.5.2", + "untrusted 0.7.1", "web-sys", "winapi", ] +[[package]] +name = "ring" +version = "0.17.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb0205304757e5d899b9c2e448b867ffd03ae7f988002e47cd24954391394d0b" +dependencies = [ + "cc", + "getrandom 0.2.11", + "libc", + "spin 0.9.8", + "untrusted 0.9.0", + "windows-sys 0.48.0", +] + [[package]] name = "rpassword" version = "6.0.1" @@ -3378,7 +4003,7 @@ dependencies = [ "serde_derive", "sha2 0.10.6", "thiserror", - "time 0.3.7", + "time 0.3.20", "tokio", "tokio-stream", "url", @@ -3431,11 +4056,24 @@ version = "0.37.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d" dependencies = [ - "bitflags", + "bitflags 1.3.2", "errno", "io-lifetimes", "libc", - "linux-raw-sys", + "linux-raw-sys 0.3.8", + "windows-sys 0.48.0", +] + +[[package]] +name = "rustix" +version = "0.38.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc99bc2d4f1fed22595588a013687477aedf3cdcfb26558c559edb67b4d9b22e" +dependencies = [ + "bitflags 2.4.1", + "errno", + "libc", + "linux-raw-sys 0.4.11", "windows-sys 0.48.0", ] @@ -3446,11 +4084,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f" dependencies = [ "log", - "ring", + "ring 0.16.20", "sct", "webpki", ] +[[package]] +name = "rustls" +version = "0.21.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "629648aced5775d558af50b2b4c7b02983a04b312126d45eeead26e7caa498b9" +dependencies = [ + "log", + "ring 0.17.5", + "rustls-webpki", + "sct", +] + [[package]] name = "rustls-native-certs" version = "0.6.2" @@ -3481,6 +4131,16 @@ dependencies = [ "base64 0.21.1", ] +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring 0.17.5", + "untrusted 0.9.0", +] + [[package]] name = "rustversion" version = "1.0.12" @@ -3546,8 +4206,8 @@ version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "109da1e6b197438deb6db99952990c7f959572794b80ff93707d55a232545e7c" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.69", + "quote 1.0.33", "serde_derive_internals", "syn 1.0.109", ] @@ -3573,8 +4233,8 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bdbda6ac5cd1321e724fa9cee216f3a61885889b896f073b8f82322789c5250e" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 1.0.109", ] @@ -3584,17 +4244,23 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" dependencies = [ - "ring", - "untrusted", + "ring 0.16.20", + "untrusted 0.7.1", ] +[[package]] +name = "seahash" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" + [[package]] name = "security-framework" version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fc758eb7bffce5b308734e9b0c1468893cae9ff70ebf13e7090be8dcbcc83a8" dependencies = [ - "bitflags", + "bitflags 1.3.2", "core-foundation", "core-foundation-sys", "libc", @@ -3619,9 +4285,9 @@ checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" [[package]] name = "serde" -version = "1.0.163" +version = "1.0.192" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2113ab51b87a539ae008b5c6c02dc020ffa39afd2d83cffcb3f4eb2722cebec2" +checksum = "bca2a08484b285dcb282d0f67b26cadc0df8b19f8c12502c13d966bf9482f001" dependencies = [ "serde_derive", ] @@ -3637,13 +4303,13 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.163" +version = "1.0.192" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e" +checksum = "d6c7207fbec9faa48073f3e3074cbe553af6ea512d7c21ba46e434e70ea9fbc1" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", - "syn 2.0.16", + "proc-macro2 1.0.69", + "quote 1.0.33", + "syn 2.0.39", ] [[package]] @@ -3652,8 +4318,8 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85bf8229e7920a9f636479437026331ce11aa132b4dde37d121944a44d6e5f3c" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 1.0.109", ] @@ -3686,12 +4352,25 @@ version = "0.8.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "578a7433b776b56a35785ed5ce9a7e777ac0598aac5a6dd1b4b18a307c7fc71b" dependencies = [ - "indexmap", + "indexmap 1.9.3", "ryu", "serde", "yaml-rust", ] +[[package]] +name = "serde_yaml" +version = "0.9.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cc7a1570e38322cfe4154732e5110f887ea57e22b76f4bfd32b5bdd3368666c" +dependencies = [ + "indexmap 2.1.0", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + [[package]] name = "sha-1" version = "0.10.1" @@ -3788,6 +4467,24 @@ version = "1.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" +[[package]] +name = "simple_asn1" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adc4e5204eb1910f40f9cfa375f6f05b68c3abac4b6fd879c8ff5e7ae8a0a085" +dependencies = [ + "num-bigint 0.4.3", + "num-traits", + "thiserror", + "time 0.3.20", +] + +[[package]] +name = "siphasher" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" + [[package]] name = "sized-chunks" version = "0.6.5" @@ -3990,7 +4687,7 @@ dependencies = [ "lazy_static", "serde", "serde_derive", - "serde_yaml", + "serde_yaml 0.8.26", "solana-clap-utils", "solana-sdk", "url", @@ -4013,7 +4710,7 @@ dependencies = [ "enum_dispatch", "futures", "futures-util", - "indexmap", + "indexmap 1.9.3", "indicatif", "itertools", "jsonrpc-core", @@ -4025,7 +4722,7 @@ dependencies = [ "rand_chacha 0.2.2", "rayon", "reqwest", - "rustls", + "rustls 0.20.8", "semver", "serde", "serde_derive", @@ -4138,8 +4835,8 @@ version = "1.14.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be23cc7a382f54dfe1348edb94610e5cc146b8eb21563cdd04062a403c75ba62" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.69", + "quote 1.0.33", "rustc_version", "syn 1.0.109", ] @@ -4236,7 +4933,7 @@ checksum = "d0937481f080f5dd495fae456c94718a7bacf30fb5fdabb02dcb8a9622e446d5" dependencies = [ "base64 0.13.1", "bincode", - "bitflags", + "bitflags 1.3.2", "blake3", "borsh", "borsh-derive", @@ -4247,7 +4944,7 @@ dependencies = [ "console_error_panic_hook", "console_log", "curve25519-dalek", - "getrandom 0.2.9", + "getrandom 0.2.11", "itertools", "js-sys", "lazy_static", @@ -4428,7 +5125,7 @@ dependencies = [ "assert_matches", "base64 0.13.1", "bincode", - "bitflags", + "bitflags 1.3.2", "borsh", "bs58 0.4.0", "bytemuck", @@ -4477,8 +5174,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33d0acbad862093ea123f3a27364336dcb0c8373522cd6810496a34e932c56c1" dependencies = [ "bs58 0.4.0", - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.69", + "quote 1.0.33", "rustversion", "syn 1.0.109", ] @@ -4530,7 +5227,7 @@ dependencies = [ "crossbeam-channel", "futures-util", "histogram", - "indexmap", + "indexmap 1.9.3", "itertools", "libc", "log", @@ -4541,7 +5238,7 @@ dependencies = [ "quinn", "rand 0.7.3", "rcgen", - "rustls", + "rustls 0.20.8", "solana-metrics", "solana-perf", "solana-sdk", @@ -4686,6 +5383,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + [[package]] name = "spki" version = "0.5.4" @@ -4788,8 +5491,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" dependencies = [ "heck 0.4.1", - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.69", + "quote 1.0.33", "rustversion", "syn 1.0.109", ] @@ -4844,30 +5547,36 @@ version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.69", + "quote 1.0.33", "unicode-ident", ] [[package]] name = "syn" -version = "2.0.16" +version = "2.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6f671d4b5ffdb8eadec19c0ae67fe2639df8684bd7bc4b83d986b8db549cf01" +checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.69", + "quote 1.0.33", "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + [[package]] name = "synstructure" version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 1.0.109", "unicode-xid 0.2.4", ] @@ -4913,8 +5622,8 @@ version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ee42b4e559f17bce0385ebf511a7beb67d5cc33c12c96b7f4e9789919d9c10f" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 1.0.109", ] @@ -4927,7 +5636,7 @@ dependencies = [ "cfg-if", "fastrand", "redox_syscall 0.3.5", - "rustix", + "rustix 0.37.19", "windows-sys 0.45.0", ] @@ -4997,9 +5706,9 @@ version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", - "syn 2.0.16", + "proc-macro2 1.0.69", + "quote 1.0.33", + "syn 2.0.39", ] [[package]] @@ -5025,22 +5734,32 @@ dependencies = [ [[package]] name = "time" -version = "0.3.7" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "004cbc98f30fa233c61a38bc77e96a9106e65c88f2d3bef182ae952027e5753d" +checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" dependencies = [ "itoa", "libc", "num_threads", "serde", + "time-core", "time-macros", ] +[[package]] +name = "time-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" + [[package]] name = "time-macros" -version = "0.2.3" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25eb0ca3468fc0acc11828786797f6ef9aa1555e4a211a60d64cc8e4d1be47d6" +checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36" +dependencies = [ + "time-core", +] [[package]] name = "tiny-bip39" @@ -5096,14 +5815,24 @@ dependencies = [ "winapi", ] +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-macros" version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 1.0.109", ] @@ -5117,17 +5846,38 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-retry" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f" +dependencies = [ + "pin-project", + "rand 0.8.5", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" dependencies = [ - "rustls", + "rustls 0.20.8", "tokio", "webpki", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.9", + "tokio", +] + [[package]] name = "tokio-serde" version = "0.8.0" @@ -5163,9 +5913,9 @@ checksum = "f714dd15bead90401d77e04243611caec13726c2408afd5b31901dfcdcb3b181" dependencies = [ "futures-util", "log", - "rustls", + "rustls 0.20.8", "tokio", - "tokio-rustls", + "tokio-rustls 0.23.4", "tungstenite", "webpki", "webpki-roots", @@ -5209,6 +5959,82 @@ dependencies = [ "serde", ] +[[package]] +name = "tonic" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f219fad3b929bef19b1f86fbc0358d35daed8f2cac972037ac0dc10bbb8d5fb" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64 0.13.1", + "bytes", + "flate2", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost", + "prost-derive", + "rustls-native-certs", + "rustls-pemfile 1.0.2", + "tokio", + "tokio-rustls 0.23.4", + "tokio-stream", + "tokio-util 0.7.2", + "tower", + "tower-layer", + "tower-service", + "tracing", + "tracing-futures", + "webpki-roots", +] + +[[package]] +name = "tonic-build" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bf5e9b9c0f7e0a7c027dcfaba7b2c60816c7049171f679d99ee2ff65d0de8c4" +dependencies = [ + "prettyplease", + "proc-macro2 1.0.69", + "prost-build", + "quote 1.0.33", + "syn 1.0.109", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 1.9.3", + "pin-project", + "pin-project-lite", + "rand 0.8.5", + "slab", + "tokio", + "tokio-util 0.7.2", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + [[package]] name = "tower-service" version = "0.3.2" @@ -5234,9 +6060,9 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", - "syn 2.0.16", + "proc-macro2 1.0.69", + "quote 1.0.33", + "syn 2.0.39", ] [[package]] @@ -5249,6 +6075,27 @@ dependencies = [ "valuable", ] +[[package]] +name = "tracing-futures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" +dependencies = [ + "pin-project", + "tracing", +] + +[[package]] +name = "tracing-log" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f751112709b4e791d8ce53e32c4ed2d353565a795ce84da2285393f41557bdf2" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + [[package]] name = "tracing-opentelemetry" version = "0.17.4" @@ -5262,15 +6109,50 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "tracing-serde" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6b213177105856957181934e4920de57730fc69bf42c37ee5bb664d406d9e1" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-stackdriver" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eff9dd91761e07727176a3dd3a1d64bbb577ea656b7b82fa4be4021832674c49" +dependencies = [ + "Inflector", + "serde", + "serde_json", + "thiserror", + "time 0.3.20", + "tracing-core", + "tracing-subscriber", +] + [[package]] name = "tracing-subscriber" version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex", + "serde", + "serde_json", "sharded-slab", + "smallvec", "thread_local", + "tracing", "tracing-core", + "tracing-log", + "tracing-serde", ] [[package]] @@ -5292,7 +6174,7 @@ dependencies = [ "httparse", "log", "rand 0.8.5", - "rustls", + "rustls 0.20.8", "sha-1", "thiserror", "url", @@ -5317,8 +6199,8 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc4a1cf66ce820973c4b31c5ef47a8e930a53ffbcec191212c33f5a3ad75c6cd" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.69", + "quote 1.0.33", "syn 1.0.109", ] @@ -5392,12 +6274,24 @@ dependencies = [ "void", ] +[[package]] +name = "unsafe-libyaml" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28467d3e1d3c6586d8f25fa243f544f5800fec42d97032474e17222c2b75cfa" + [[package]] name = "untrusted" version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "uriparse" version = "0.6.4" @@ -5419,12 +6313,27 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf-8" version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "uuid" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" +dependencies = [ + "getrandom 0.2.11", +] + [[package]] name = "valuable" version = "0.1.0" @@ -5512,9 +6421,9 @@ dependencies = [ "bumpalo", "log", "once_cell", - "proc-macro2 1.0.58", - "quote 1.0.27", - "syn 2.0.16", + "proc-macro2 1.0.69", + "quote 1.0.33", + "syn 2.0.39", "wasm-bindgen-shared", ] @@ -5536,7 +6445,7 @@ version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14d6b024f1a526bb0234f52840389927257beb670610081360e5a03c5df9c258" dependencies = [ - "quote 1.0.27", + "quote 1.0.33", "wasm-bindgen-macro-support", ] @@ -5546,9 +6455,9 @@ version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e128beba882dd1eb6200e1dc92ae6c5dbaa4311aa7bb211ca035779e5efc39f8" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", - "syn 2.0.16", + "proc-macro2 1.0.69", + "quote 1.0.33", + "syn 2.0.39", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -5588,8 +6497,8 @@ version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" dependencies = [ - "ring", - "untrusted", + "ring 0.16.20", + "untrusted 0.7.1", ] [[package]] @@ -5601,6 +6510,18 @@ dependencies = [ "webpki", ] +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix 0.38.25", +] + [[package]] name = "winapi" version = "0.3.9" @@ -5812,7 +6733,7 @@ dependencies = [ "oid-registry", "rusticata-macros", "thiserror", - "time 0.3.7", + "time 0.3.20", ] [[package]] @@ -5845,7 +6766,34 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" dependencies = [ - "time 0.3.7", + "time 0.3.20", +] + +[[package]] +name = "yup-oauth2" +version = "8.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "364ca376b5c04d9b2be9693054e3e0d2d146b363819d0f9a10c6ee66e4c8406b" +dependencies = [ + "anyhow", + "async-trait", + "base64 0.13.1", + "futures", + "http", + "hyper", + "hyper-rustls 0.24.2", + "itertools", + "log", + "percent-encoding", + "rustls 0.21.9", + "rustls-pemfile 1.0.2", + "seahash", + "serde", + "serde_json", + "time 0.3.20", + "tokio", + "tower-service", + "url", ] [[package]] @@ -5863,9 +6811,9 @@ version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", - "syn 2.0.16", + "proc-macro2 1.0.69", + "quote 1.0.33", + "syn 2.0.39", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 54bef387..689bc928 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,5 @@ [workspace] -members = ["programs/*", "clients/rust/*", "tools/*"] -exclude = ["observability/indexer"] +members = ["programs/*", "clients/rust/*", "tools/*", "observability/indexer"] [workspace.dependencies] solana-client = "1.14.13" @@ -9,6 +8,9 @@ solana-logger = "1.14.13" solana-program = "1.14.13" solana-program-test = "1.14.13" solana-account-decoder = "1.14.13" +solana-measure = "1.14.13" +solana-metrics = "1.14.13" +solana-transaction-status = "1.14.13" anchor-lang = "0.26.0" anchor-spl = "0.26.0" anchor-client = "0.26.0" diff --git a/clients/rust/marginfi-cli/src/processor/mod.rs b/clients/rust/marginfi-cli/src/processor/mod.rs index c3cfa547..52aeac6f 100644 --- a/clients/rust/marginfi-cli/src/processor/mod.rs +++ b/clients/rust/marginfi-cli/src/processor/mod.rs @@ -1582,7 +1582,7 @@ pub fn marginfi_account_create(profile: &Profile, config: &Config) -> Result<()> let tx = Transaction::new_signed_with_payer( &[ix], Some(&signer.pubkey()), - &[&signer, &marginfi_account_key], + &[signer, &marginfi_account_key], recent_blockhash, ); diff --git a/observability/etl/dataflow-etls/.dockerignore b/observability/etl/dataflow-etls/.dockerignore new file mode 100644 index 00000000..7a9b3652 --- /dev/null +++ b/observability/etl/dataflow-etls/.dockerignore @@ -0,0 +1,6 @@ +# Ignore everything except for Python files and the requirements file. +* +!setup.py +!MANIFEST.in +!dataflow_etls/ +!jobs/ diff --git a/observability/etl/dataflow-etls/.gcloudignore b/observability/etl/dataflow-etls/.gcloudignore new file mode 100644 index 00000000..2f71adf0 --- /dev/null +++ b/observability/etl/dataflow-etls/.gcloudignore @@ -0,0 +1,7 @@ +# Ignore everything except for Python files and the requirements file. +* +!setup.py +!MANIFEST.in +!Dockerfile +!dataflow_etls/ +!jobs/ diff --git a/observability/etl/dataflow-etls/.gitignore b/observability/etl/dataflow-etls/.gitignore index b45e2888..0d2e53e5 100644 --- a/observability/etl/dataflow-etls/.gitignore +++ b/observability/etl/dataflow-etls/.gitignore @@ -1,6 +1,7 @@ __pycache__ +.mypy_cache .venv poetry.lock -local_file* +parsed_event_* .idea* beam-temp-* \ No newline at end of file diff --git a/observability/etl/dataflow-etls/.mypy.ini b/observability/etl/dataflow-etls/.mypy.ini index ce9c610b..8e12ff88 100644 --- a/observability/etl/dataflow-etls/.mypy.ini +++ b/observability/etl/dataflow-etls/.mypy.ini @@ -3,7 +3,7 @@ pretty = False show_absolute_path = True show_column_numbers = True show_error_codes = True -files = src/apologies, tests +files = . exclude = scripts/playground.py # This is mostly equivalent to strict=true as of v0.770 diff --git a/observability/etl/dataflow-etls/Dockerfile b/observability/etl/dataflow-etls/Dockerfile index 14057cd4..1f8cfe00 100644 --- a/observability/etl/dataflow-etls/Dockerfile +++ b/observability/etl/dataflow-etls/Dockerfile @@ -1,27 +1,24 @@ -FROM gcr.io/dataflow-templates-base/python3-template-launcher-base +FROM gcr.io/dataflow-templates-base/python39-template-launcher-base +ARG JOB_DIRECTORY ARG WORKDIR=/dataflow/template RUN mkdir -p ${WORKDIR} WORKDIR ${WORKDIR} -COPY requirements.txt . -COPY job.py . -COPY idls . -COPY dataflow_etls . +COPY setup.py . +COPY MANIFEST.in . -ENV FLEX_TEMPLATE_PYTHON_REQUIREMENTS_FILE="/${WORKDIR}/requirements.txt" ENV FLEX_TEMPLATE_PYTHON_PY_FILE="/${WORKDIR}/job.py" +ENV FLEX_TEMPLATE_PYTHON_SETUP_FILE="/${WORKDIR}/setup.py" -# We could get rid of installing libffi-dev and git, or we could leave them. -RUN apt-get update \ - && apt-get install -y libffi-dev git \ - && rm -rf /var/lib/apt/lists/* \ - # Upgrade pip and install the requirements. - && pip install --no-cache-dir --upgrade pip \ - && pip install -U apache-beam==2.44.0 \ - && pip install --no-cache-dir -r $FLEX_TEMPLATE_PYTHON_REQUIREMENTS_FILE \ +# Upgrade pip and install the requirements. +RUN pip install --no-cache-dir --upgrade pip \ + && pip install --no-cache-dir . \ # Download the requirements to speed up launching the Dataflow job. - && pip download --no-cache-dir --dest /tmp/dataflow-etls-requirements-cache -r $FLEX_TEMPLATE_PYTHON_REQUIREMENTS_FILE + && pip download --no-cache-dir --dest /tmp/dataflow-etls-requirements-cache . + +COPY dataflow_etls/ dataflow_etls/ +COPY ${JOB_DIRECTORY}/job.py . # Since we already downloaded all the dependencies, there's no need to rebuild everything. ENV PIP_NO_DEPS=True diff --git a/observability/etl/dataflow-etls/MANIFEST.in b/observability/etl/dataflow-etls/MANIFEST.in new file mode 100644 index 00000000..35ccb062 --- /dev/null +++ b/observability/etl/dataflow-etls/MANIFEST.in @@ -0,0 +1 @@ +include dataflow_etls/idls/**/*.json \ No newline at end of file diff --git a/observability/etl/dataflow-etls/README.md b/observability/etl/dataflow-etls/README.md index 275086aa..bf0938b9 100644 --- a/observability/etl/dataflow-etls/README.md +++ b/observability/etl/dataflow-etls/README.md @@ -15,15 +15,12 @@ python job.py \ - Build image and upload to Artifact Registry: ``` -gcloud builds submit --tag us-east1-docker.pkg.dev/marginfi-dev/main/dataflow/event-parsing-batch:latest . +./scripts/build_job_template +./scripts/upload_job_template ``` - Create/Update template and associate metadata file: ``` -gcloud dataflow flex-template build \ - gs://dataflow_jobs_marginfi_v2/templates/event-parsing-batch.json \ - --image "us-east1-docker.pkg.dev/marginfi-dev/main/dataflow/event-parsing-batch:latest" \ - --sdk-language "PYTHON" \ - --metadata-file "metadata.json" +./scripts/sync_job_template ``` diff --git a/observability/etl/dataflow-etls/dataflow_etls/account_parsing.py b/observability/etl/dataflow-etls/dataflow_etls/account_parsing.py new file mode 100644 index 00000000..8318bd07 --- /dev/null +++ b/observability/etl/dataflow-etls/dataflow_etls/account_parsing.py @@ -0,0 +1,76 @@ +import base64 +from dataclasses import asdict +from datetime import datetime +from typing import List, TypedDict, Dict, Any, Tuple, Generator +from decimal import Decimal + +from anchorpy_core.idl import Idl +from based58 import based58 # type: ignore +from solders.pubkey import Pubkey +import apache_beam as beam # type: ignore +from anchorpy.program.common import NamedInstruction as NamedAccountData + +from dataflow_etls.idl_versions import VersionedProgram, IdlPool, Cluster +from dataflow_etls.orm.accounts import ACCOUNT_UPDATE_TO_RECORD_TYPE, AccountUpdateRecord + +AccountUpdateRaw = TypedDict('AccountUpdateRaw', { + 'id': str, + 'created_at': datetime, + 'timestamp': datetime, + 'owner': str, + 'slot': Decimal, + 'pubkey': str, + 'txn_signature': str, + 'lamports': Decimal, + 'executable': bool, + 'rent_epoch': Decimal, + 'data': str, +}) + + +class OwnerProgramNotSupported(Exception): + pass + + +def parse_account(account_update: AccountUpdateRaw, min_idl_version: int, cluster: Cluster, + idl_pool: IdlPool) -> List[AccountUpdateRecord]: + owner_program_id_str = account_update["owner"] + owner_program_id = Pubkey.from_string(owner_program_id_str) + account_update_slot = int(account_update["slot"]) + + try: + idl_raw, idl_version = idl_pool.get_idl_for_slot(owner_program_id_str, account_update_slot) + except KeyError: + raise OwnerProgramNotSupported(f"Unsupported program {owner_program_id_str}") + + idl = Idl.from_json(idl_raw) + program = VersionedProgram(cluster, idl_version, idl, owner_program_id) + + if idl_version < min_idl_version: + return [] + + account_data_bytes = base64.b64decode(account_update["data"]) + + try: + parsed_account_data: NamedAccountData = program.coder.accounts.parse(account_data_bytes) + except Exception as e: + print(f"failed to parse account data in update {account_update['id']}", e) + return [] + + if parsed_account_data.name not in ACCOUNT_UPDATE_TO_RECORD_TYPE: + print(f"discarding unsupported account type {parsed_account_data.name} in update {account_update['id']}") + return [] + else: + # noinspection PyPep8Naming + AccountUpdateRecordType = ACCOUNT_UPDATE_TO_RECORD_TYPE[parsed_account_data.name] + return [AccountUpdateRecordType(parsed_account_data, account_update, idl_version)] + + +class DispatchEventsDoFn(beam.DoFn): # type: ignore + def process(self, record: AccountUpdateRecord, *args: Tuple[Any], **kwargs: Dict[str, Tuple[Any]]) -> Generator[ + str, None, None]: + yield beam.pvalue.TaggedOutput(record.get_tag(), record) + + +def dictionify_record(record: AccountUpdateRecord) -> Dict[str, Any]: + return asdict(record) diff --git a/observability/etl/dataflow-etls/dataflow_etls/idl_versions.py b/observability/etl/dataflow-etls/dataflow_etls/idl_versions.py index d24da67e..b1cbbd88 100644 --- a/observability/etl/dataflow-etls/dataflow_etls/idl_versions.py +++ b/observability/etl/dataflow-etls/dataflow_etls/idl_versions.py @@ -1,8 +1,9 @@ import glob import os from pathlib import Path -from typing import List, Literal, Tuple, Optional -from anchorpy import Program, Provider +from typing import List, Literal, Tuple, Optional, Dict +from anchorpy import Program, Provider, Wallet +from anchorpy.utils.rpc import AsyncClient from anchorpy_core.idl import Idl from solders.pubkey import Pubkey @@ -20,33 +21,74 @@ def __init__(self, cluster: Cluster, version: int, idl: Idl, program_id: Pubkey, provider: Optional[Provider] = None): self.version = version self.cluster = cluster - super(VersionedProgram, self).__init__(idl, program_id, provider) + super(VersionedProgram, self).__init__(idl, program_id, + provider or Provider(AsyncClient("http://localhost:8899"), + Wallet.dummy())) -class VersionedIdl: - VERSIONS: ClusterIdlBoundaries = {"devnet": { - "A7vUDErNPCTt9qrB6SSM4F6GkxzUe9d8P3cXSmRg4eY4": [(196494976, 0), (196520454, 1), (197246719, 2), (197494521, 3)] - }} +# /!\ Boundaries need to be ordered /!\ +IDL_VERSIONS: ClusterIdlBoundaries = { + "devnet": { + # "A7vUDErNPCTt9qrB6SSM4F6GkxzUe9d8P3cXSmRg4eY4": [(196494976, 0), (196520454, 1), (197246719, 2), (197494521, 3)], + "5Lt5xXZG7bteZferQk9bsiiAS75JqGVPYcTbB8J6vvJK": [], + }, + "mainnet": { + "MFv2hWf31Z9kbCa1snEPYctwafyhdvnV7FZnsebVacA": [], + } +} - @staticmethod - def get_idl_for_slot(cluster: Cluster, program_id: str, slot: int) -> Tuple[Idl, int]: - idl_boundaries = VersionedIdl.VERSIONS[cluster][program_id] - idl_version = None - for boundary_slot, version in idl_boundaries: - # todo: returns latest for upgrade slot, can throw if tx executed in same slot, before upgrade - if boundary_slot > slot: - idl_version = version - break +class ClusterNotSupported(Exception): + pass + - if idl_version is None: +class IdlPool: + idls_per_program: Dict[str, Tuple[List[Tuple[int, Tuple[int, str]]], str, int]] + + def __init__(self, cluster): + idl_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), f"idls/{cluster}") + try: + boundaries_per_program = IDL_VERSIONS[cluster] + except KeyError: + raise ClusterNotSupported(f"Cluster {cluster} is not supported") + + self.idls_per_program = {} + + for program_id in boundaries_per_program: + # Find latest IDL sorted_idls = [int(os.path.basename(path).removesuffix(".json").removeprefix("marginfi-v")) for path in - glob.glob(f"idls/{cluster}/marginfi-v*.json")] + glob.glob(f"{idl_dir}/{program_id}/marginfi-v*.json")] sorted_idls.sort() - idl_version = sorted_idls[-1] + latest_idl_version = sorted_idls[-1] + + path = Path(f"{idl_dir}/{program_id}/marginfi-v{latest_idl_version}.json") + latest_idl_raw = path.read_text() + + self.idls_per_program[program_id] = ([], latest_idl_raw, latest_idl_version) + + # Load all IDLs + boundaries = boundaries_per_program[program_id] + for boundary in boundaries: + version_end_slot = boundary[0] + idl_version = boundary[1] + path = Path(f"{idl_dir}/{program_id}/marginfi-v{idl_version}.json") + idl_raw = path.read_text() + self.idls_per_program[program_id][0].append((version_end_slot, (idl_version, idl_raw))) + + def get_idl_for_slot(self, program_id: str, slot: int) -> Tuple[str, int]: + idl_boundaries, latest_idl, latest_idl_version = self.idls_per_program[program_id] + + idl = None + idl_version = None + for version_end_slot, (current_idl_version, current_idl) in idl_boundaries: + # todo: returns latest for upgrade slot, can throw if tx executed in same slot, before upgrade + if version_end_slot > slot: + idl = current_idl + idl_version = current_idl_version + break - path = Path(f"idls/{cluster}/marginfi-v{idl_version}.json") - raw = path.read_text() - idl = Idl.from_json(raw) + if idl is None: + idl = latest_idl + idl_version = latest_idl_version return idl, idl_version diff --git a/observability/etl/dataflow-etls/dataflow_etls/idls/devnet/5Lt5xXZG7bteZferQk9bsiiAS75JqGVPYcTbB8J6vvJK/marginfi-v0.json b/observability/etl/dataflow-etls/dataflow_etls/idls/devnet/5Lt5xXZG7bteZferQk9bsiiAS75JqGVPYcTbB8J6vvJK/marginfi-v0.json new file mode 100644 index 00000000..8229bc32 --- /dev/null +++ b/observability/etl/dataflow-etls/dataflow_etls/idls/devnet/5Lt5xXZG7bteZferQk9bsiiAS75JqGVPYcTbB8J6vvJK/marginfi-v0.json @@ -0,0 +1,2309 @@ +{ + "version": "0.1.0", + "name": "marginfi", + "instructions": [ + { + "name": "marginfiGroupInitialize", + "accounts": [ + { + "name": "marginfiGroup", + "isMut": true, + "isSigner": true + }, + { + "name": "admin", + "isMut": true, + "isSigner": true + }, + { + "name": "systemProgram", + "isMut": false, + "isSigner": false + } + ], + "args": [] + }, + { + "name": "marginfiGroupConfigure", + "accounts": [ + { + "name": "marginfiGroup", + "isMut": true, + "isSigner": false + }, + { + "name": "admin", + "isMut": false, + "isSigner": true + } + ], + "args": [ + { + "name": "config", + "type": { + "defined": "GroupConfig" + } + } + ] + }, + { + "name": "lendingPoolAddBank", + "accounts": [ + { + "name": "marginfiGroup", + "isMut": false, + "isSigner": false + }, + { + "name": "admin", + "isMut": true, + "isSigner": true + }, + { + "name": "bankMint", + "isMut": false, + "isSigner": false + }, + { + "name": "bank", + "isMut": true, + "isSigner": true + }, + { + "name": "liquidityVaultAuthority", + "isMut": false, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "liquidity_vault_auth" + }, + { + "kind": "account", + "type": "publicKey", + "path": "bank" + } + ] + } + }, + { + "name": "liquidityVault", + "isMut": true, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "liquidity_vault" + }, + { + "kind": "account", + "type": "publicKey", + "path": "bank" + } + ] + } + }, + { + "name": "insuranceVaultAuthority", + "isMut": false, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "insurance_vault_auth" + }, + { + "kind": "account", + "type": "publicKey", + "path": "bank" + } + ] + } + }, + { + "name": "insuranceVault", + "isMut": true, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "insurance_vault" + }, + { + "kind": "account", + "type": "publicKey", + "path": "bank" + } + ] + } + }, + { + "name": "feeVaultAuthority", + "isMut": false, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "fee_vault_auth" + }, + { + "kind": "account", + "type": "publicKey", + "path": "bank" + } + ] + } + }, + { + "name": "feeVault", + "isMut": true, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "fee_vault" + }, + { + "kind": "account", + "type": "publicKey", + "path": "bank" + } + ] + } + }, + { + "name": "rent", + "isMut": false, + "isSigner": false + }, + { + "name": "tokenProgram", + "isMut": false, + "isSigner": false + }, + { + "name": "systemProgram", + "isMut": false, + "isSigner": false + } + ], + "args": [ + { + "name": "bankConfig", + "type": { + "defined": "BankConfig" + } + } + ] + }, + { + "name": "lendingPoolConfigureBank", + "accounts": [ + { + "name": "marginfiGroup", + "isMut": false, + "isSigner": false + }, + { + "name": "admin", + "isMut": false, + "isSigner": true + }, + { + "name": "bank", + "isMut": true, + "isSigner": false + } + ], + "args": [ + { + "name": "bankConfigOpt", + "type": { + "defined": "BankConfigOpt" + } + } + ] + }, + { + "name": "lendingPoolHandleBankruptcy", + "docs": [ + "Handle bad debt of a bankrupt marginfi account for a given bank." + ], + "accounts": [ + { + "name": "marginfiGroup", + "isMut": false, + "isSigner": false + }, + { + "name": "admin", + "isMut": false, + "isSigner": true + }, + { + "name": "bank", + "isMut": true, + "isSigner": false + }, + { + "name": "marginfiAccount", + "isMut": true, + "isSigner": false + }, + { + "name": "liquidityVault", + "isMut": true, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "liquidity_vault" + }, + { + "kind": "account", + "type": "publicKey", + "path": "bank" + } + ] + } + }, + { + "name": "insuranceVault", + "isMut": true, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "insurance_vault" + }, + { + "kind": "account", + "type": "publicKey", + "path": "bank" + } + ] + } + }, + { + "name": "insuranceVaultAuthority", + "isMut": false, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "insurance_vault_auth" + }, + { + "kind": "account", + "type": "publicKey", + "path": "bank" + } + ] + } + }, + { + "name": "tokenProgram", + "isMut": false, + "isSigner": false + } + ], + "args": [] + }, + { + "name": "marginfiAccountInitialize", + "docs": [ + "Initialize a marginfi account for a given group" + ], + "accounts": [ + { + "name": "marginfiGroup", + "isMut": false, + "isSigner": false + }, + { + "name": "marginfiAccount", + "isMut": true, + "isSigner": true + }, + { + "name": "authority", + "isMut": false, + "isSigner": true + }, + { + "name": "feePayer", + "isMut": true, + "isSigner": true + }, + { + "name": "systemProgram", + "isMut": false, + "isSigner": false + } + ], + "args": [] + }, + { + "name": "lendingAccountDeposit", + "accounts": [ + { + "name": "marginfiGroup", + "isMut": false, + "isSigner": false + }, + { + "name": "marginfiAccount", + "isMut": true, + "isSigner": false + }, + { + "name": "signer", + "isMut": false, + "isSigner": true + }, + { + "name": "bank", + "isMut": true, + "isSigner": false + }, + { + "name": "signerTokenAccount", + "isMut": true, + "isSigner": false + }, + { + "name": "bankLiquidityVault", + "isMut": true, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "liquidity_vault" + }, + { + "kind": "account", + "type": "publicKey", + "path": "bank" + } + ] + } + }, + { + "name": "tokenProgram", + "isMut": false, + "isSigner": false + } + ], + "args": [ + { + "name": "amount", + "type": "u64" + } + ] + }, + { + "name": "lendingAccountRepay", + "accounts": [ + { + "name": "marginfiGroup", + "isMut": false, + "isSigner": false + }, + { + "name": "marginfiAccount", + "isMut": true, + "isSigner": false + }, + { + "name": "signer", + "isMut": false, + "isSigner": true + }, + { + "name": "bank", + "isMut": true, + "isSigner": false + }, + { + "name": "signerTokenAccount", + "isMut": true, + "isSigner": false + }, + { + "name": "bankLiquidityVault", + "isMut": true, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "liquidity_vault" + }, + { + "kind": "account", + "type": "publicKey", + "path": "bank" + } + ] + } + }, + { + "name": "tokenProgram", + "isMut": false, + "isSigner": false + } + ], + "args": [ + { + "name": "amount", + "type": "u64" + }, + { + "name": "repayAll", + "type": { + "option": "bool" + } + } + ] + }, + { + "name": "lendingAccountWithdraw", + "accounts": [ + { + "name": "marginfiGroup", + "isMut": false, + "isSigner": false + }, + { + "name": "marginfiAccount", + "isMut": true, + "isSigner": false + }, + { + "name": "signer", + "isMut": false, + "isSigner": true + }, + { + "name": "bank", + "isMut": true, + "isSigner": false + }, + { + "name": "destinationTokenAccount", + "isMut": true, + "isSigner": false + }, + { + "name": "bankLiquidityVaultAuthority", + "isMut": true, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "liquidity_vault_auth" + }, + { + "kind": "account", + "type": "publicKey", + "path": "bank" + } + ] + } + }, + { + "name": "bankLiquidityVault", + "isMut": true, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "liquidity_vault" + }, + { + "kind": "account", + "type": "publicKey", + "path": "bank" + } + ] + } + }, + { + "name": "tokenProgram", + "isMut": false, + "isSigner": false + } + ], + "args": [ + { + "name": "amount", + "type": "u64" + }, + { + "name": "withdrawAll", + "type": { + "option": "bool" + } + } + ] + }, + { + "name": "lendingAccountBorrow", + "accounts": [ + { + "name": "marginfiGroup", + "isMut": false, + "isSigner": false + }, + { + "name": "marginfiAccount", + "isMut": true, + "isSigner": false + }, + { + "name": "signer", + "isMut": false, + "isSigner": true + }, + { + "name": "bank", + "isMut": true, + "isSigner": false + }, + { + "name": "destinationTokenAccount", + "isMut": true, + "isSigner": false + }, + { + "name": "bankLiquidityVaultAuthority", + "isMut": true, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "liquidity_vault_auth" + }, + { + "kind": "account", + "type": "publicKey", + "path": "bank" + } + ] + } + }, + { + "name": "bankLiquidityVault", + "isMut": true, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "liquidity_vault" + }, + { + "kind": "account", + "type": "publicKey", + "path": "bank" + } + ] + } + }, + { + "name": "tokenProgram", + "isMut": false, + "isSigner": false + } + ], + "args": [ + { + "name": "amount", + "type": "u64" + } + ] + }, + { + "name": "lendingAccountLiquidate", + "docs": [ + "Liquidate a lending account balance of an unhealthy marginfi account" + ], + "accounts": [ + { + "name": "marginfiGroup", + "isMut": false, + "isSigner": false + }, + { + "name": "assetBank", + "isMut": true, + "isSigner": false + }, + { + "name": "liabBank", + "isMut": true, + "isSigner": false + }, + { + "name": "liquidatorMarginfiAccount", + "isMut": true, + "isSigner": false + }, + { + "name": "signer", + "isMut": false, + "isSigner": true + }, + { + "name": "liquidateeMarginfiAccount", + "isMut": true, + "isSigner": false + }, + { + "name": "bankLiquidityVaultAuthority", + "isMut": true, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "liquidity_vault_auth" + }, + { + "kind": "account", + "type": "publicKey", + "path": "liab_bank" + } + ] + } + }, + { + "name": "bankLiquidityVault", + "isMut": true, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "liquidity_vault" + }, + { + "kind": "account", + "type": "publicKey", + "path": "liab_bank" + } + ] + } + }, + { + "name": "bankInsuranceVault", + "isMut": true, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "insurance_vault" + }, + { + "kind": "account", + "type": "publicKey", + "path": "liab_bank" + } + ] + } + }, + { + "name": "tokenProgram", + "isMut": false, + "isSigner": false + } + ], + "args": [ + { + "name": "assetAmount", + "type": "u64" + } + ] + }, + { + "name": "lendingPoolAccrueBankInterest", + "accounts": [ + { + "name": "marginfiGroup", + "isMut": false, + "isSigner": false + }, + { + "name": "bank", + "isMut": true, + "isSigner": false + } + ], + "args": [] + }, + { + "name": "lendingPoolCollectBankFees", + "accounts": [ + { + "name": "marginfiGroup", + "isMut": false, + "isSigner": false + }, + { + "name": "bank", + "isMut": true, + "isSigner": false + }, + { + "name": "liquidityVaultAuthority", + "isMut": false, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "liquidity_vault_auth" + }, + { + "kind": "account", + "type": "publicKey", + "path": "bank" + } + ] + } + }, + { + "name": "liquidityVault", + "isMut": true, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "liquidity_vault" + }, + { + "kind": "account", + "type": "publicKey", + "path": "bank" + } + ] + } + }, + { + "name": "insuranceVault", + "isMut": true, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "insurance_vault" + }, + { + "kind": "account", + "type": "publicKey", + "path": "bank" + } + ] + } + }, + { + "name": "feeVault", + "isMut": true, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "fee_vault" + }, + { + "kind": "account", + "type": "publicKey", + "path": "bank" + } + ] + } + }, + { + "name": "tokenProgram", + "isMut": false, + "isSigner": false + } + ], + "args": [] + } + ], + "accounts": [ + { + "name": "MarginfiAccount", + "type": { + "kind": "struct", + "fields": [ + { + "name": "group", + "type": "publicKey" + }, + { + "name": "authority", + "type": "publicKey" + }, + { + "name": "lendingAccount", + "type": { + "defined": "LendingAccount" + } + }, + { + "name": "padding", + "type": { + "array": [ + "u64", + 64 + ] + } + } + ] + } + }, + { + "name": "MarginfiGroup", + "type": { + "kind": "struct", + "fields": [ + { + "name": "admin", + "type": "publicKey" + }, + { + "name": "padding0", + "type": { + "array": [ + "u128", + 32 + ] + } + }, + { + "name": "padding1", + "type": { + "array": [ + "u128", + 32 + ] + } + } + ] + } + }, + { + "name": "Bank", + "type": { + "kind": "struct", + "fields": [ + { + "name": "mint", + "type": "publicKey" + }, + { + "name": "mintDecimals", + "type": "u8" + }, + { + "name": "group", + "type": "publicKey" + }, + { + "name": "ignore1", + "type": { + "array": [ + "u8", + 7 + ] + } + }, + { + "name": "assetShareValue", + "type": { + "defined": "WrappedI80F48" + } + }, + { + "name": "liabilityShareValue", + "type": { + "defined": "WrappedI80F48" + } + }, + { + "name": "liquidityVault", + "type": "publicKey" + }, + { + "name": "liquidityVaultBump", + "type": "u8" + }, + { + "name": "liquidityVaultAuthorityBump", + "type": "u8" + }, + { + "name": "insuranceVault", + "type": "publicKey" + }, + { + "name": "insuranceVaultBump", + "type": "u8" + }, + { + "name": "insuranceVaultAuthorityBump", + "type": "u8" + }, + { + "name": "ignore2", + "type": { + "array": [ + "u8", + 4 + ] + } + }, + { + "name": "collectedInsuranceFeesOutstanding", + "type": { + "defined": "WrappedI80F48" + } + }, + { + "name": "feeVault", + "type": "publicKey" + }, + { + "name": "feeVaultBump", + "type": "u8" + }, + { + "name": "feeVaultAuthorityBump", + "type": "u8" + }, + { + "name": "ignore3", + "type": { + "array": [ + "u8", + 6 + ] + } + }, + { + "name": "collectedGroupFeesOutstanding", + "type": { + "defined": "WrappedI80F48" + } + }, + { + "name": "totalLiabilityShares", + "type": { + "defined": "WrappedI80F48" + } + }, + { + "name": "totalAssetShares", + "type": { + "defined": "WrappedI80F48" + } + }, + { + "name": "lastUpdate", + "type": "i64" + }, + { + "name": "config", + "type": { + "defined": "BankConfig" + } + }, + { + "name": "padding0", + "type": { + "array": [ + "u128", + 32 + ] + } + }, + { + "name": "padding1", + "type": { + "array": [ + "u128", + 32 + ] + } + } + ] + } + } + ], + "types": [ + { + "name": "GroupEventHeader", + "type": { + "kind": "struct", + "fields": [ + { + "name": "signer", + "type": { + "option": "publicKey" + } + }, + { + "name": "marginfiGroup", + "type": "publicKey" + } + ] + } + }, + { + "name": "AccountEventHeader", + "type": { + "kind": "struct", + "fields": [ + { + "name": "signer", + "type": { + "option": "publicKey" + } + }, + { + "name": "marginfiAccount", + "type": "publicKey" + }, + { + "name": "marginfiAccountAuthority", + "type": "publicKey" + }, + { + "name": "marginfiGroup", + "type": "publicKey" + } + ] + } + }, + { + "name": "LiquidationBalances", + "type": { + "kind": "struct", + "fields": [ + { + "name": "liquidateeAssetBalance", + "type": "f64" + }, + { + "name": "liquidateeLiabilityBalance", + "type": "f64" + }, + { + "name": "liquidatorAssetBalance", + "type": "f64" + }, + { + "name": "liquidatorLiabilityBalance", + "type": "f64" + } + ] + } + }, + { + "name": "LendingAccount", + "type": { + "kind": "struct", + "fields": [ + { + "name": "balances", + "type": { + "array": [ + { + "defined": "Balance" + }, + 16 + ] + } + }, + { + "name": "padding", + "type": { + "array": [ + "u64", + 8 + ] + } + } + ] + } + }, + { + "name": "Balance", + "type": { + "kind": "struct", + "fields": [ + { + "name": "active", + "type": "bool" + }, + { + "name": "bankPk", + "type": "publicKey" + }, + { + "name": "ignore1", + "type": { + "array": [ + "u8", + 7 + ] + } + }, + { + "name": "assetShares", + "type": { + "defined": "WrappedI80F48" + } + }, + { + "name": "liabilityShares", + "type": { + "defined": "WrappedI80F48" + } + }, + { + "name": "padding", + "type": { + "array": [ + "u64", + 4 + ] + } + } + ] + } + }, + { + "name": "GroupConfig", + "type": { + "kind": "struct", + "fields": [ + { + "name": "admin", + "type": { + "option": "publicKey" + } + } + ] + } + }, + { + "name": "InterestRateConfig", + "type": { + "kind": "struct", + "fields": [ + { + "name": "optimalUtilizationRate", + "type": { + "defined": "WrappedI80F48" + } + }, + { + "name": "plateauInterestRate", + "type": { + "defined": "WrappedI80F48" + } + }, + { + "name": "maxInterestRate", + "type": { + "defined": "WrappedI80F48" + } + }, + { + "name": "insuranceFeeFixedApr", + "type": { + "defined": "WrappedI80F48" + } + }, + { + "name": "insuranceIrFee", + "type": { + "defined": "WrappedI80F48" + } + }, + { + "name": "protocolFixedFeeApr", + "type": { + "defined": "WrappedI80F48" + } + }, + { + "name": "protocolIrFee", + "type": { + "defined": "WrappedI80F48" + } + }, + { + "name": "padding", + "type": { + "array": [ + "u128", + 8 + ] + } + } + ] + } + }, + { + "name": "InterestRateConfigOpt", + "type": { + "kind": "struct", + "fields": [ + { + "name": "optimalUtilizationRate", + "type": { + "option": { + "defined": "WrappedI80F48" + } + } + }, + { + "name": "plateauInterestRate", + "type": { + "option": { + "defined": "WrappedI80F48" + } + } + }, + { + "name": "maxInterestRate", + "type": { + "option": { + "defined": "WrappedI80F48" + } + } + }, + { + "name": "insuranceFeeFixedApr", + "type": { + "option": { + "defined": "WrappedI80F48" + } + } + }, + { + "name": "insuranceIrFee", + "type": { + "option": { + "defined": "WrappedI80F48" + } + } + }, + { + "name": "protocolFixedFeeApr", + "type": { + "option": { + "defined": "WrappedI80F48" + } + } + }, + { + "name": "protocolIrFee", + "type": { + "option": { + "defined": "WrappedI80F48" + } + } + } + ] + } + }, + { + "name": "BankConfig", + "docs": [ + "TODO: Convert weights to (u64, u64) to avoid precision loss (maybe?)" + ], + "type": { + "kind": "struct", + "fields": [ + { + "name": "assetWeightInit", + "type": { + "defined": "WrappedI80F48" + } + }, + { + "name": "assetWeightMaint", + "type": { + "defined": "WrappedI80F48" + } + }, + { + "name": "liabilityWeightInit", + "type": { + "defined": "WrappedI80F48" + } + }, + { + "name": "liabilityWeightMaint", + "type": { + "defined": "WrappedI80F48" + } + }, + { + "name": "depositLimit", + "type": "u64" + }, + { + "name": "interestRateConfig", + "type": { + "defined": "InterestRateConfig" + } + }, + { + "name": "operationalState", + "type": { + "defined": "BankOperationalState" + } + }, + { + "name": "oracleSetup", + "type": { + "defined": "OracleSetup" + } + }, + { + "name": "oracleKeys", + "type": { + "array": [ + "publicKey", + 5 + ] + } + }, + { + "name": "ignore1", + "type": { + "array": [ + "u8", + 6 + ] + } + }, + { + "name": "borrowLimit", + "type": "u64" + }, + { + "name": "riskTier", + "type": { + "defined": "RiskTier" + } + }, + { + "name": "padding", + "type": { + "array": [ + "u64", + 6 + ] + } + } + ] + } + }, + { + "name": "WrappedI80F48", + "type": { + "kind": "struct", + "fields": [ + { + "name": "value", + "type": "i128" + } + ] + } + }, + { + "name": "BankConfigOpt", + "type": { + "kind": "struct", + "fields": [ + { + "name": "assetWeightInit", + "type": { + "option": { + "defined": "WrappedI80F48" + } + } + }, + { + "name": "assetWeightMaint", + "type": { + "option": { + "defined": "WrappedI80F48" + } + } + }, + { + "name": "liabilityWeightInit", + "type": { + "option": { + "defined": "WrappedI80F48" + } + } + }, + { + "name": "liabilityWeightMaint", + "type": { + "option": { + "defined": "WrappedI80F48" + } + } + }, + { + "name": "depositLimit", + "type": { + "option": "u64" + } + }, + { + "name": "borrowLimit", + "type": { + "option": "u64" + } + }, + { + "name": "ignore1", + "type": { + "array": [ + "u64", + 22 + ] + } + }, + { + "name": "operationalState", + "type": { + "option": { + "defined": "BankOperationalState" + } + } + }, + { + "name": "oracle", + "type": { + "option": { + "defined": "OracleConfig" + } + } + }, + { + "name": "interestRateConfig", + "type": { + "option": { + "defined": "InterestRateConfigOpt" + } + } + }, + { + "name": "riskTier", + "type": { + "option": { + "defined": "RiskTier" + } + } + }, + { + "name": "ignore2", + "type": { + "array": [ + "u64", + 21 + ] + } + } + ] + } + }, + { + "name": "OracleConfig", + "type": { + "kind": "struct", + "fields": [ + { + "name": "setup", + "type": { + "defined": "OracleSetup" + } + }, + { + "name": "keys", + "type": { + "array": [ + "publicKey", + 5 + ] + } + } + ] + } + }, + { + "name": "BalanceIncreaseType", + "type": { + "kind": "enum", + "variants": [ + { + "name": "Any" + }, + { + "name": "RepayOnly" + }, + { + "name": "DepositOnly" + } + ] + } + }, + { + "name": "BalanceDecreaseType", + "type": { + "kind": "enum", + "variants": [ + { + "name": "Any" + }, + { + "name": "WithdrawOnly" + }, + { + "name": "BorrowOnly" + }, + { + "name": "BypassBorrowLimit" + } + ] + } + }, + { + "name": "WeightType", + "type": { + "kind": "enum", + "variants": [ + { + "name": "Initial" + }, + { + "name": "Maintenance" + } + ] + } + }, + { + "name": "BalanceSide", + "type": { + "kind": "enum", + "variants": [ + { + "name": "Assets" + }, + { + "name": "Liabilities" + } + ] + } + }, + { + "name": "RiskRequirementType", + "type": { + "kind": "enum", + "variants": [ + { + "name": "Initial" + }, + { + "name": "Maintenance" + } + ] + } + }, + { + "name": "BankOperationalState", + "type": { + "kind": "enum", + "variants": [ + { + "name": "Paused" + }, + { + "name": "Operational" + }, + { + "name": "ReduceOnly" + } + ] + } + }, + { + "name": "OracleSetup", + "type": { + "kind": "enum", + "variants": [ + { + "name": "None" + }, + { + "name": "Pyth" + } + ] + } + }, + { + "name": "OracleKey", + "type": { + "kind": "enum", + "variants": [ + { + "name": "Pyth", + "fields": [ + "publicKey" + ] + } + ] + } + }, + { + "name": "RiskTier", + "type": { + "kind": "enum", + "variants": [ + { + "name": "Collateral" + }, + { + "name": "Isolated" + } + ] + } + }, + { + "name": "BankVaultType", + "type": { + "kind": "enum", + "variants": [ + { + "name": "Liquidity" + }, + { + "name": "Insurance" + }, + { + "name": "Fee" + } + ] + } + } + ], + "events": [ + { + "name": "MarginfiGroupCreateEvent", + "fields": [ + { + "name": "header", + "type": { + "defined": "GroupEventHeader" + }, + "index": false + } + ] + }, + { + "name": "MarginfiGroupConfigureEvent", + "fields": [ + { + "name": "header", + "type": { + "defined": "GroupEventHeader" + }, + "index": false + }, + { + "name": "config", + "type": { + "defined": "GroupConfig" + }, + "index": false + } + ] + }, + { + "name": "LendingPoolBankCreateEvent", + "fields": [ + { + "name": "header", + "type": { + "defined": "GroupEventHeader" + }, + "index": false + }, + { + "name": "bank", + "type": "publicKey", + "index": false + }, + { + "name": "mint", + "type": "publicKey", + "index": false + } + ] + }, + { + "name": "LendingPoolBankConfigureEvent", + "fields": [ + { + "name": "header", + "type": { + "defined": "GroupEventHeader" + }, + "index": false + }, + { + "name": "bank", + "type": "publicKey", + "index": false + }, + { + "name": "mint", + "type": "publicKey", + "index": false + }, + { + "name": "config", + "type": { + "defined": "BankConfigOpt" + }, + "index": false + } + ] + }, + { + "name": "LendingPoolBankAccrueInterestEvent", + "fields": [ + { + "name": "header", + "type": { + "defined": "GroupEventHeader" + }, + "index": false + }, + { + "name": "bank", + "type": "publicKey", + "index": false + }, + { + "name": "mint", + "type": "publicKey", + "index": false + }, + { + "name": "delta", + "type": "u64", + "index": false + }, + { + "name": "feesCollected", + "type": "f64", + "index": false + }, + { + "name": "insuranceCollected", + "type": "f64", + "index": false + } + ] + }, + { + "name": "LendingPoolBankCollectFeesEvent", + "fields": [ + { + "name": "header", + "type": { + "defined": "GroupEventHeader" + }, + "index": false + }, + { + "name": "bank", + "type": "publicKey", + "index": false + }, + { + "name": "mint", + "type": "publicKey", + "index": false + }, + { + "name": "groupFeesCollected", + "type": "f64", + "index": false + }, + { + "name": "groupFeesOutstanding", + "type": "f64", + "index": false + }, + { + "name": "insuranceFeesCollected", + "type": "f64", + "index": false + }, + { + "name": "insuranceFeesOutstanding", + "type": "f64", + "index": false + } + ] + }, + { + "name": "LendingPoolBankHandleBankruptcyEvent", + "fields": [ + { + "name": "header", + "type": { + "defined": "AccountEventHeader" + }, + "index": false + }, + { + "name": "bank", + "type": "publicKey", + "index": false + }, + { + "name": "mint", + "type": "publicKey", + "index": false + }, + { + "name": "badDebt", + "type": "f64", + "index": false + }, + { + "name": "coveredAmount", + "type": "f64", + "index": false + }, + { + "name": "socializedAmount", + "type": "f64", + "index": false + } + ] + }, + { + "name": "MarginfiAccountCreateEvent", + "fields": [ + { + "name": "header", + "type": { + "defined": "AccountEventHeader" + }, + "index": false + } + ] + }, + { + "name": "LendingAccountDepositEvent", + "fields": [ + { + "name": "header", + "type": { + "defined": "AccountEventHeader" + }, + "index": false + }, + { + "name": "bank", + "type": "publicKey", + "index": false + }, + { + "name": "mint", + "type": "publicKey", + "index": false + }, + { + "name": "amount", + "type": "u64", + "index": false + } + ] + }, + { + "name": "LendingAccountRepayEvent", + "fields": [ + { + "name": "header", + "type": { + "defined": "AccountEventHeader" + }, + "index": false + }, + { + "name": "bank", + "type": "publicKey", + "index": false + }, + { + "name": "mint", + "type": "publicKey", + "index": false + }, + { + "name": "amount", + "type": "u64", + "index": false + }, + { + "name": "closeBalance", + "type": "bool", + "index": false + } + ] + }, + { + "name": "LendingAccountBorrowEvent", + "fields": [ + { + "name": "header", + "type": { + "defined": "AccountEventHeader" + }, + "index": false + }, + { + "name": "bank", + "type": "publicKey", + "index": false + }, + { + "name": "mint", + "type": "publicKey", + "index": false + }, + { + "name": "amount", + "type": "u64", + "index": false + } + ] + }, + { + "name": "LendingAccountWithdrawEvent", + "fields": [ + { + "name": "header", + "type": { + "defined": "AccountEventHeader" + }, + "index": false + }, + { + "name": "bank", + "type": "publicKey", + "index": false + }, + { + "name": "mint", + "type": "publicKey", + "index": false + }, + { + "name": "amount", + "type": "u64", + "index": false + }, + { + "name": "closeBalance", + "type": "bool", + "index": false + } + ] + }, + { + "name": "LendingAccountLiquidateEvent", + "fields": [ + { + "name": "header", + "type": { + "defined": "AccountEventHeader" + }, + "index": false + }, + { + "name": "liquidateeMarginfiAccount", + "type": "publicKey", + "index": false + }, + { + "name": "liquidateeMarginfiAccountAuthority", + "type": "publicKey", + "index": false + }, + { + "name": "assetBank", + "type": "publicKey", + "index": false + }, + { + "name": "assetMint", + "type": "publicKey", + "index": false + }, + { + "name": "liabilityBank", + "type": "publicKey", + "index": false + }, + { + "name": "liabilityMint", + "type": "publicKey", + "index": false + }, + { + "name": "liquidateePreHealth", + "type": "f64", + "index": false + }, + { + "name": "liquidateePostHealth", + "type": "f64", + "index": false + }, + { + "name": "preBalances", + "type": { + "defined": "LiquidationBalances" + }, + "index": false + }, + { + "name": "postBalances", + "type": { + "defined": "LiquidationBalances" + }, + "index": false + } + ] + } + ], + "errors": [ + { + "code": 6000, + "name": "MathError", + "msg": "Math error" + }, + { + "code": 6001, + "name": "BankNotFound", + "msg": "Invalid bank index" + }, + { + "code": 6002, + "name": "LendingAccountBalanceNotFound", + "msg": "Lending account balance not found" + }, + { + "code": 6003, + "name": "BankAssetCapacityExceeded", + "msg": "Bank deposit capacity exceeded" + }, + { + "code": 6004, + "name": "InvalidTransfer", + "msg": "Invalid transfer" + }, + { + "code": 6005, + "name": "MissingPythOrBankAccount", + "msg": "Missing Pyth or Bank account" + }, + { + "code": 6006, + "name": "MissingPythAccount", + "msg": "Missing Pyth account" + }, + { + "code": 6007, + "name": "InvalidOracleAccount", + "msg": "Invalid Pyth account" + }, + { + "code": 6008, + "name": "MissingBankAccount", + "msg": "Missing Bank account" + }, + { + "code": 6009, + "name": "InvalidBankAccount", + "msg": "Invalid Bank account" + }, + { + "code": 6010, + "name": "BadAccountHealth", + "msg": "Bad account health" + }, + { + "code": 6011, + "name": "LendingAccountBalanceSlotsFull", + "msg": "Lending account balance slots are full" + }, + { + "code": 6012, + "name": "BankAlreadyExists", + "msg": "Bank already exists" + }, + { + "code": 6013, + "name": "IllegalLiquidation", + "msg": "Illegal post liquidation state, account is either not unhealthy or liquidation was too big" + }, + { + "code": 6014, + "name": "AccountNotBankrupt", + "msg": "Account is not bankrupt" + }, + { + "code": 6015, + "name": "BalanceNotBadDebt", + "msg": "Account balance is not bad debt" + }, + { + "code": 6016, + "name": "InvalidConfig", + "msg": "Invalid group config" + }, + { + "code": 6017, + "name": "StaleOracle", + "msg": "Stale oracle data" + }, + { + "code": 6018, + "name": "BankPaused", + "msg": "Bank paused" + }, + { + "code": 6019, + "name": "BankReduceOnly", + "msg": "Bank is ReduceOnly mode" + }, + { + "code": 6020, + "name": "BankAccoutNotFound", + "msg": "Bank is missing" + }, + { + "code": 6021, + "name": "OperationDepositOnly", + "msg": "Operation is deposit-only" + }, + { + "code": 6022, + "name": "OperationWithdrawOnly", + "msg": "Operation is withdraw-only" + }, + { + "code": 6023, + "name": "OperationBorrowOnly", + "msg": "Operation is borrow-only" + }, + { + "code": 6024, + "name": "OperationRepayOnly", + "msg": "Operation is repay-only" + }, + { + "code": 6025, + "name": "NoAssetFound", + "msg": "No asset found" + }, + { + "code": 6026, + "name": "NoLiabilityFound", + "msg": "No liability found" + }, + { + "code": 6027, + "name": "InvalidOracleSetup", + "msg": "Invalid oracle setup" + }, + { + "code": 6028, + "name": "IllegalUtilizationRatio", + "msg": "Invalid bank utilization ratio" + }, + { + "code": 6029, + "name": "BankLiabilityCapacityExceeded", + "msg": "Bank borrow cap exceeded" + }, + { + "code": 6030, + "name": "InvalidPrice", + "msg": "Invalid Price" + }, + { + "code": 6031, + "name": "IsolatedAccountIllegalState", + "msg": "Account can have only one liablity when account is under isolated risk" + } + ] +} diff --git a/observability/etl/dataflow-etls/idls/devnet/marginfi-v0.json b/observability/etl/dataflow-etls/dataflow_etls/idls/devnet/A7vUDErNPCTt9qrB6SSM4F6GkxzUe9d8P3cXSmRg4eY4/marginfi-v0.json similarity index 100% rename from observability/etl/dataflow-etls/idls/devnet/marginfi-v0.json rename to observability/etl/dataflow-etls/dataflow_etls/idls/devnet/A7vUDErNPCTt9qrB6SSM4F6GkxzUe9d8P3cXSmRg4eY4/marginfi-v0.json diff --git a/observability/etl/dataflow-etls/idls/devnet/marginfi-v1.json b/observability/etl/dataflow-etls/dataflow_etls/idls/devnet/A7vUDErNPCTt9qrB6SSM4F6GkxzUe9d8P3cXSmRg4eY4/marginfi-v1.json similarity index 100% rename from observability/etl/dataflow-etls/idls/devnet/marginfi-v1.json rename to observability/etl/dataflow-etls/dataflow_etls/idls/devnet/A7vUDErNPCTt9qrB6SSM4F6GkxzUe9d8P3cXSmRg4eY4/marginfi-v1.json diff --git a/observability/etl/dataflow-etls/idls/devnet/marginfi-v2.json b/observability/etl/dataflow-etls/dataflow_etls/idls/devnet/A7vUDErNPCTt9qrB6SSM4F6GkxzUe9d8P3cXSmRg4eY4/marginfi-v2.json similarity index 100% rename from observability/etl/dataflow-etls/idls/devnet/marginfi-v2.json rename to observability/etl/dataflow-etls/dataflow_etls/idls/devnet/A7vUDErNPCTt9qrB6SSM4F6GkxzUe9d8P3cXSmRg4eY4/marginfi-v2.json diff --git a/observability/etl/dataflow-etls/idls/devnet/marginfi-v3.json b/observability/etl/dataflow-etls/dataflow_etls/idls/devnet/A7vUDErNPCTt9qrB6SSM4F6GkxzUe9d8P3cXSmRg4eY4/marginfi-v3.json similarity index 100% rename from observability/etl/dataflow-etls/idls/devnet/marginfi-v3.json rename to observability/etl/dataflow-etls/dataflow_etls/idls/devnet/A7vUDErNPCTt9qrB6SSM4F6GkxzUe9d8P3cXSmRg4eY4/marginfi-v3.json diff --git a/observability/etl/dataflow-etls/idls/devnet/marginfi-v4.json b/observability/etl/dataflow-etls/dataflow_etls/idls/devnet/A7vUDErNPCTt9qrB6SSM4F6GkxzUe9d8P3cXSmRg4eY4/marginfi-v4.json similarity index 100% rename from observability/etl/dataflow-etls/idls/devnet/marginfi-v4.json rename to observability/etl/dataflow-etls/dataflow_etls/idls/devnet/A7vUDErNPCTt9qrB6SSM4F6GkxzUe9d8P3cXSmRg4eY4/marginfi-v4.json diff --git a/observability/etl/dataflow-etls/dataflow_etls/idls/mainnet/MFv2hWf31Z9kbCa1snEPYctwafyhdvnV7FZnsebVacA/marginfi-v0.json b/observability/etl/dataflow-etls/dataflow_etls/idls/mainnet/MFv2hWf31Z9kbCa1snEPYctwafyhdvnV7FZnsebVacA/marginfi-v0.json new file mode 100644 index 00000000..50d97aa6 --- /dev/null +++ b/observability/etl/dataflow-etls/dataflow_etls/idls/mainnet/MFv2hWf31Z9kbCa1snEPYctwafyhdvnV7FZnsebVacA/marginfi-v0.json @@ -0,0 +1,2478 @@ +{ + "version": "0.1.0", + "name": "marginfi", + "instructions": [ + { + "name": "marginfiGroupInitialize", + "accounts": [ + { + "name": "marginfiGroup", + "isMut": true, + "isSigner": true + }, + { + "name": "admin", + "isMut": true, + "isSigner": true + }, + { + "name": "systemProgram", + "isMut": false, + "isSigner": false + } + ], + "args": [] + }, + { + "name": "marginfiGroupConfigure", + "accounts": [ + { + "name": "marginfiGroup", + "isMut": true, + "isSigner": false + }, + { + "name": "admin", + "isMut": false, + "isSigner": true + } + ], + "args": [ + { + "name": "config", + "type": { + "defined": "GroupConfig" + } + } + ] + }, + { + "name": "lendingPoolAddBank", + "accounts": [ + { + "name": "marginfiGroup", + "isMut": false, + "isSigner": false + }, + { + "name": "admin", + "isMut": true, + "isSigner": true + }, + { + "name": "bankMint", + "isMut": false, + "isSigner": false + }, + { + "name": "bank", + "isMut": true, + "isSigner": true + }, + { + "name": "liquidityVaultAuthority", + "isMut": false, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "liquidity_vault_auth" + }, + { + "kind": "account", + "type": "publicKey", + "path": "bank" + } + ] + } + }, + { + "name": "liquidityVault", + "isMut": true, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "liquidity_vault" + }, + { + "kind": "account", + "type": "publicKey", + "path": "bank" + } + ] + } + }, + { + "name": "insuranceVaultAuthority", + "isMut": false, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "insurance_vault_auth" + }, + { + "kind": "account", + "type": "publicKey", + "path": "bank" + } + ] + } + }, + { + "name": "insuranceVault", + "isMut": true, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "insurance_vault" + }, + { + "kind": "account", + "type": "publicKey", + "path": "bank" + } + ] + } + }, + { + "name": "feeVaultAuthority", + "isMut": false, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "fee_vault_auth" + }, + { + "kind": "account", + "type": "publicKey", + "path": "bank" + } + ] + } + }, + { + "name": "feeVault", + "isMut": true, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "fee_vault" + }, + { + "kind": "account", + "type": "publicKey", + "path": "bank" + } + ] + } + }, + { + "name": "rent", + "isMut": false, + "isSigner": false + }, + { + "name": "tokenProgram", + "isMut": false, + "isSigner": false + }, + { + "name": "systemProgram", + "isMut": false, + "isSigner": false + } + ], + "args": [ + { + "name": "bankConfig", + "type": { + "defined": "BankConfigArg" + } + } + ] + }, + { + "name": "lendingPoolConfigureBank", + "accounts": [ + { + "name": "marginfiGroup", + "isMut": false, + "isSigner": false + }, + { + "name": "admin", + "isMut": false, + "isSigner": true + }, + { + "name": "bank", + "isMut": true, + "isSigner": false + } + ], + "args": [ + { + "name": "bankConfigOpt", + "type": { + "defined": "BankConfigOptArg" + } + } + ] + }, + { + "name": "lendingPoolHandleBankruptcy", + "docs": [ + "Handle bad debt of a bankrupt marginfi account for a given bank." + ], + "accounts": [ + { + "name": "marginfiGroup", + "isMut": false, + "isSigner": false + }, + { + "name": "admin", + "isMut": false, + "isSigner": true + }, + { + "name": "bank", + "isMut": true, + "isSigner": false + }, + { + "name": "marginfiAccount", + "isMut": true, + "isSigner": false + }, + { + "name": "liquidityVault", + "isMut": true, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "liquidity_vault" + }, + { + "kind": "account", + "type": "publicKey", + "path": "bank" + } + ] + } + }, + { + "name": "insuranceVault", + "isMut": true, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "insurance_vault" + }, + { + "kind": "account", + "type": "publicKey", + "path": "bank" + } + ] + } + }, + { + "name": "insuranceVaultAuthority", + "isMut": false, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "insurance_vault_auth" + }, + { + "kind": "account", + "type": "publicKey", + "path": "bank" + } + ] + } + }, + { + "name": "tokenProgram", + "isMut": false, + "isSigner": false + } + ], + "args": [] + }, + { + "name": "marginfiAccountInitialize", + "docs": [ + "Initialize a marginfi account for a given group" + ], + "accounts": [ + { + "name": "marginfiGroup", + "isMut": false, + "isSigner": false + }, + { + "name": "marginfiAccount", + "isMut": true, + "isSigner": true + }, + { + "name": "authority", + "isMut": false, + "isSigner": true + }, + { + "name": "feePayer", + "isMut": true, + "isSigner": true + }, + { + "name": "systemProgram", + "isMut": false, + "isSigner": false + } + ], + "args": [] + }, + { + "name": "lendingAccountDeposit", + "accounts": [ + { + "name": "marginfiGroup", + "isMut": false, + "isSigner": false + }, + { + "name": "marginfiAccount", + "isMut": true, + "isSigner": false + }, + { + "name": "signer", + "isMut": false, + "isSigner": true + }, + { + "name": "bank", + "isMut": true, + "isSigner": false + }, + { + "name": "signerTokenAccount", + "isMut": true, + "isSigner": false + }, + { + "name": "bankLiquidityVault", + "isMut": true, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "liquidity_vault" + }, + { + "kind": "account", + "type": "publicKey", + "path": "bank" + } + ] + } + }, + { + "name": "tokenProgram", + "isMut": false, + "isSigner": false + } + ], + "args": [ + { + "name": "amount", + "type": "u64" + } + ] + }, + { + "name": "lendingAccountRepay", + "accounts": [ + { + "name": "marginfiGroup", + "isMut": false, + "isSigner": false + }, + { + "name": "marginfiAccount", + "isMut": true, + "isSigner": false + }, + { + "name": "signer", + "isMut": false, + "isSigner": true + }, + { + "name": "bank", + "isMut": true, + "isSigner": false + }, + { + "name": "signerTokenAccount", + "isMut": true, + "isSigner": false + }, + { + "name": "bankLiquidityVault", + "isMut": true, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "liquidity_vault" + }, + { + "kind": "account", + "type": "publicKey", + "path": "bank" + } + ] + } + }, + { + "name": "tokenProgram", + "isMut": false, + "isSigner": false + } + ], + "args": [ + { + "name": "amount", + "type": "u64" + }, + { + "name": "repayAll", + "type": { + "option": "bool" + } + } + ] + }, + { + "name": "lendingAccountWithdraw", + "accounts": [ + { + "name": "marginfiGroup", + "isMut": false, + "isSigner": false + }, + { + "name": "marginfiAccount", + "isMut": true, + "isSigner": false + }, + { + "name": "signer", + "isMut": false, + "isSigner": true + }, + { + "name": "bank", + "isMut": true, + "isSigner": false + }, + { + "name": "destinationTokenAccount", + "isMut": true, + "isSigner": false + }, + { + "name": "bankLiquidityVaultAuthority", + "isMut": true, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "liquidity_vault_auth" + }, + { + "kind": "account", + "type": "publicKey", + "path": "bank" + } + ] + } + }, + { + "name": "bankLiquidityVault", + "isMut": true, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "liquidity_vault" + }, + { + "kind": "account", + "type": "publicKey", + "path": "bank" + } + ] + } + }, + { + "name": "tokenProgram", + "isMut": false, + "isSigner": false + } + ], + "args": [ + { + "name": "amount", + "type": "u64" + }, + { + "name": "withdrawAll", + "type": { + "option": "bool" + } + } + ] + }, + { + "name": "lendingAccountBorrow", + "accounts": [ + { + "name": "marginfiGroup", + "isMut": false, + "isSigner": false + }, + { + "name": "marginfiAccount", + "isMut": true, + "isSigner": false + }, + { + "name": "signer", + "isMut": false, + "isSigner": true + }, + { + "name": "bank", + "isMut": true, + "isSigner": false + }, + { + "name": "destinationTokenAccount", + "isMut": true, + "isSigner": false + }, + { + "name": "bankLiquidityVaultAuthority", + "isMut": true, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "liquidity_vault_auth" + }, + { + "kind": "account", + "type": "publicKey", + "path": "bank" + } + ] + } + }, + { + "name": "bankLiquidityVault", + "isMut": true, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "liquidity_vault" + }, + { + "kind": "account", + "type": "publicKey", + "path": "bank" + } + ] + } + }, + { + "name": "tokenProgram", + "isMut": false, + "isSigner": false + } + ], + "args": [ + { + "name": "amount", + "type": "u64" + } + ] + }, + { + "name": "lendingAccountLiquidate", + "docs": [ + "Liquidate a lending account balance of an unhealthy marginfi account" + ], + "accounts": [ + { + "name": "marginfiGroup", + "isMut": false, + "isSigner": false + }, + { + "name": "assetBank", + "isMut": true, + "isSigner": false + }, + { + "name": "liabBank", + "isMut": true, + "isSigner": false + }, + { + "name": "liquidatorMarginfiAccount", + "isMut": true, + "isSigner": false + }, + { + "name": "signer", + "isMut": false, + "isSigner": true + }, + { + "name": "liquidateeMarginfiAccount", + "isMut": true, + "isSigner": false + }, + { + "name": "bankLiquidityVaultAuthority", + "isMut": true, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "liquidity_vault_auth" + }, + { + "kind": "account", + "type": "publicKey", + "path": "liab_bank" + } + ] + } + }, + { + "name": "bankLiquidityVault", + "isMut": true, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "liquidity_vault" + }, + { + "kind": "account", + "type": "publicKey", + "path": "liab_bank" + } + ] + } + }, + { + "name": "bankInsuranceVault", + "isMut": true, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "insurance_vault" + }, + { + "kind": "account", + "type": "publicKey", + "path": "liab_bank" + } + ] + } + }, + { + "name": "tokenProgram", + "isMut": false, + "isSigner": false + } + ], + "args": [ + { + "name": "assetAmount", + "type": "u64" + } + ] + }, + { + "name": "lendingPoolAccrueBankInterest", + "accounts": [ + { + "name": "marginfiGroup", + "isMut": false, + "isSigner": false + }, + { + "name": "bank", + "isMut": true, + "isSigner": false + } + ], + "args": [] + }, + { + "name": "lendingPoolCollectBankFees", + "accounts": [ + { + "name": "marginfiGroup", + "isMut": false, + "isSigner": false + }, + { + "name": "bank", + "isMut": true, + "isSigner": false + }, + { + "name": "liquidityVaultAuthority", + "isMut": false, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "liquidity_vault_auth" + }, + { + "kind": "account", + "type": "publicKey", + "path": "bank" + } + ] + } + }, + { + "name": "liquidityVault", + "isMut": true, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "liquidity_vault" + }, + { + "kind": "account", + "type": "publicKey", + "path": "bank" + } + ] + } + }, + { + "name": "insuranceVault", + "isMut": true, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "insurance_vault" + }, + { + "kind": "account", + "type": "publicKey", + "path": "bank" + } + ] + } + }, + { + "name": "feeVault", + "isMut": true, + "isSigner": false, + "pda": { + "seeds": [ + { + "kind": "const", + "type": "string", + "value": "fee_vault" + }, + { + "kind": "account", + "type": "publicKey", + "path": "bank" + } + ] + } + }, + { + "name": "tokenProgram", + "isMut": false, + "isSigner": false + } + ], + "args": [] + } + ], + "accounts": [ + { + "name": "MarginfiAccount", + "type": { + "kind": "struct", + "fields": [ + { + "name": "group", + "type": "publicKey" + }, + { + "name": "authority", + "type": "publicKey" + }, + { + "name": "lendingAccount", + "type": { + "defined": "LendingAccount" + } + }, + { + "name": "padding", + "type": { + "array": [ + "u64", + 64 + ] + } + } + ] + } + }, + { + "name": "MarginfiGroup", + "type": { + "kind": "struct", + "fields": [ + { + "name": "admin", + "type": "publicKey" + }, + { + "name": "padding0", + "type": { + "array": [ + "u128", + 32 + ] + } + }, + { + "name": "padding1", + "type": { + "array": [ + "u128", + 32 + ] + } + } + ] + } + }, + { + "name": "Bank", + "type": { + "kind": "struct", + "fields": [ + { + "name": "mint", + "type": "publicKey" + }, + { + "name": "mintDecimals", + "type": "u8" + }, + { + "name": "group", + "type": "publicKey" + }, + { + "name": "ignore1", + "type": { + "array": [ + "u8", + 7 + ] + } + }, + { + "name": "assetShareValue", + "type": { + "defined": "WrappedI80F48" + } + }, + { + "name": "liabilityShareValue", + "type": { + "defined": "WrappedI80F48" + } + }, + { + "name": "liquidityVault", + "type": "publicKey" + }, + { + "name": "liquidityVaultBump", + "type": "u8" + }, + { + "name": "liquidityVaultAuthorityBump", + "type": "u8" + }, + { + "name": "insuranceVault", + "type": "publicKey" + }, + { + "name": "insuranceVaultBump", + "type": "u8" + }, + { + "name": "insuranceVaultAuthorityBump", + "type": "u8" + }, + { + "name": "ignore2", + "type": { + "array": [ + "u8", + 4 + ] + } + }, + { + "name": "collectedInsuranceFeesOutstanding", + "type": { + "defined": "WrappedI80F48" + } + }, + { + "name": "feeVault", + "type": "publicKey" + }, + { + "name": "feeVaultBump", + "type": "u8" + }, + { + "name": "feeVaultAuthorityBump", + "type": "u8" + }, + { + "name": "ignore3", + "type": { + "array": [ + "u8", + 6 + ] + } + }, + { + "name": "collectedGroupFeesOutstanding", + "type": { + "defined": "WrappedI80F48" + } + }, + { + "name": "totalLiabilityShares", + "type": { + "defined": "WrappedI80F48" + } + }, + { + "name": "totalAssetShares", + "type": { + "defined": "WrappedI80F48" + } + }, + { + "name": "lastUpdate", + "type": "i64" + }, + { + "name": "config", + "type": { + "defined": "BankConfig" + } + }, + { + "name": "padding0", + "type": { + "array": [ + "u128", + 32 + ] + } + }, + { + "name": "padding1", + "type": { + "array": [ + "u128", + 32 + ] + } + } + ] + } + } + ], + "types": [ + { + "name": "GroupEventHeader", + "type": { + "kind": "struct", + "fields": [ + { + "name": "signer", + "type": { + "option": "publicKey" + } + }, + { + "name": "marginfiGroup", + "type": "publicKey" + } + ] + } + }, + { + "name": "AccountEventHeader", + "type": { + "kind": "struct", + "fields": [ + { + "name": "signer", + "type": { + "option": "publicKey" + } + }, + { + "name": "marginfiAccount", + "type": "publicKey" + }, + { + "name": "marginfiAccountAuthority", + "type": "publicKey" + }, + { + "name": "marginfiGroup", + "type": "publicKey" + } + ] + } + }, + { + "name": "LiquidationBalances", + "type": { + "kind": "struct", + "fields": [ + { + "name": "liquidateeAssetBalance", + "type": "f64" + }, + { + "name": "liquidateeLiabilityBalance", + "type": "f64" + }, + { + "name": "liquidatorAssetBalance", + "type": "f64" + }, + { + "name": "liquidatorLiabilityBalance", + "type": "f64" + } + ] + } + }, + { + "name": "LendingAccount", + "type": { + "kind": "struct", + "fields": [ + { + "name": "balances", + "type": { + "array": [ + { + "defined": "Balance" + }, + 16 + ] + } + }, + { + "name": "padding", + "type": { + "array": [ + "u64", + 8 + ] + } + } + ] + } + }, + { + "name": "Balance", + "type": { + "kind": "struct", + "fields": [ + { + "name": "active", + "type": "bool" + }, + { + "name": "bankPk", + "type": "publicKey" + }, + { + "name": "ignore1", + "type": { + "array": [ + "u8", + 7 + ] + } + }, + { + "name": "assetShares", + "type": { + "defined": "WrappedI80F48" + } + }, + { + "name": "liabilityShares", + "type": { + "defined": "WrappedI80F48" + } + }, + { + "name": "padding", + "type": { + "array": [ + "u64", + 4 + ] + } + } + ] + } + }, + { + "name": "GroupConfig", + "type": { + "kind": "struct", + "fields": [ + { + "name": "admin", + "type": { + "option": "publicKey" + } + } + ] + } + }, + { + "name": "InterestRateConfig", + "type": { + "kind": "struct", + "fields": [ + { + "name": "optimalUtilizationRate", + "type": { + "defined": "WrappedI80F48" + } + }, + { + "name": "plateauInterestRate", + "type": { + "defined": "WrappedI80F48" + } + }, + { + "name": "maxInterestRate", + "type": { + "defined": "WrappedI80F48" + } + }, + { + "name": "insuranceFeeFixedApr", + "type": { + "defined": "WrappedI80F48" + } + }, + { + "name": "insuranceIrFee", + "type": { + "defined": "WrappedI80F48" + } + }, + { + "name": "protocolFixedFeeApr", + "type": { + "defined": "WrappedI80F48" + } + }, + { + "name": "protocolIrFee", + "type": { + "defined": "WrappedI80F48" + } + }, + { + "name": "padding", + "type": { + "array": [ + "u128", + 8 + ] + } + } + ] + } + }, + { + "name": "InterestRateConfigOpt", + "type": { + "kind": "struct", + "fields": [ + { + "name": "optimalUtilizationRate", + "type": { + "option": { + "defined": "WrappedI80F48" + } + } + }, + { + "name": "plateauInterestRate", + "type": { + "option": { + "defined": "WrappedI80F48" + } + } + }, + { + "name": "maxInterestRate", + "type": { + "option": { + "defined": "WrappedI80F48" + } + } + }, + { + "name": "insuranceFeeFixedApr", + "type": { + "option": { + "defined": "WrappedI80F48" + } + } + }, + { + "name": "insuranceIrFee", + "type": { + "option": { + "defined": "WrappedI80F48" + } + } + }, + { + "name": "protocolFixedFeeApr", + "type": { + "option": { + "defined": "WrappedI80F48" + } + } + }, + { + "name": "protocolIrFee", + "type": { + "option": { + "defined": "WrappedI80F48" + } + } + } + ] + } + }, + { + "name": "BankConfig", + "docs": [ + "TODO: Convert weights to (u64, u64) to avoid precision loss (maybe?)" + ], + "type": { + "kind": "struct", + "fields": [ + { + "name": "assetWeightInit", + "type": { + "defined": "WrappedI80F48" + } + }, + { + "name": "assetWeightMaint", + "type": { + "defined": "WrappedI80F48" + } + }, + { + "name": "liabilityWeightInit", + "type": { + "defined": "WrappedI80F48" + } + }, + { + "name": "liabilityWeightMaint", + "type": { + "defined": "WrappedI80F48" + } + }, + { + "name": "depositLimit", + "type": "u64" + }, + { + "name": "interestRateConfig", + "type": { + "defined": "InterestRateConfig" + } + }, + { + "name": "operationalState", + "type": { + "defined": "BankOperationalState" + } + }, + { + "name": "oracleSetup", + "type": { + "defined": "OracleSetup" + } + }, + { + "name": "oracleKeys", + "type": { + "array": [ + "publicKey", + 5 + ] + } + }, + { + "name": "ignore1", + "type": { + "array": [ + "u64", + 6 + ] + } + }, + { + "name": "borrowLimit", + "type": "u64" + }, + { + "name": "riskTier", + "type": { + "defined": "RiskTier" + } + }, + { + "name": "padding", + "type": { + "array": [ + "u64", + 6 + ] + } + } + ] + } + }, + { + "name": "BankConfigArg", + "docs": [ + "TODO: Convert weights to (u64, u64) to avoid precision loss (maybe?)" + ], + "type": { + "kind": "struct", + "fields": [ + { + "name": "assetWeightInit", + "type": { + "defined": "WrappedI80F48" + } + }, + { + "name": "assetWeightMaint", + "type": { + "defined": "WrappedI80F48" + } + }, + { + "name": "liabilityWeightInit", + "type": { + "defined": "WrappedI80F48" + } + }, + { + "name": "liabilityWeightMaint", + "type": { + "defined": "WrappedI80F48" + } + }, + { + "name": "depositLimit", + "type": "u64" + }, + { + "name": "interestRateConfig", + "type": { + "defined": "InterestRateConfig" + } + }, + { + "name": "operationalState", + "type": { + "defined": "BankOperationalState" + } + }, + { + "name": "oracleSetup", + "type": { + "defined": "OracleSetup" + } + }, + { + "name": "oracleKeys", + "type": { + "array": [ + "publicKey", + 5 + ] + } + }, + { + "name": "borrowLimit", + "type": "u64" + }, + { + "name": "riskTier", + "type": { + "defined": "RiskTier" + } + }, + { + "name": "padding", + "type": { + "array": [ + "u64", + 6 + ] + } + } + ] + } + }, + { + "name": "WrappedI80F48", + "type": { + "kind": "struct", + "fields": [ + { + "name": "value", + "type": "i128" + } + ] + } + }, + { + "name": "BankConfigOpt", + "type": { + "kind": "struct", + "fields": [ + { + "name": "assetWeightInit", + "type": { + "option": { + "defined": "WrappedI80F48" + } + } + }, + { + "name": "assetWeightMaint", + "type": { + "option": { + "defined": "WrappedI80F48" + } + } + }, + { + "name": "liabilityWeightInit", + "type": { + "option": { + "defined": "WrappedI80F48" + } + } + }, + { + "name": "liabilityWeightMaint", + "type": { + "option": { + "defined": "WrappedI80F48" + } + } + }, + { + "name": "depositLimit", + "type": { + "option": "u64" + } + }, + { + "name": "borrowLimit", + "type": { + "option": "u64" + } + }, + { + "name": "ignore1", + "type": { + "array": [ + "u64", + 176 + ] + } + }, + { + "name": "operationalState", + "type": { + "option": { + "defined": "BankOperationalState" + } + } + }, + { + "name": "oracle", + "type": { + "option": { + "defined": "OracleConfig" + } + } + }, + { + "name": "interestRateConfig", + "type": { + "option": { + "defined": "InterestRateConfigOpt" + } + } + }, + { + "name": "riskTier", + "type": { + "option": { + "defined": "RiskTier" + } + } + }, + { + "name": "ignore2", + "type": { + "array": [ + "u64", + 168 + ] + } + } + ] + } + }, + { + "name": "BankConfigOptArg", + "type": { + "kind": "struct", + "fields": [ + { + "name": "assetWeightInit", + "type": { + "option": { + "defined": "WrappedI80F48" + } + } + }, + { + "name": "assetWeightMaint", + "type": { + "option": { + "defined": "WrappedI80F48" + } + } + }, + { + "name": "liabilityWeightInit", + "type": { + "option": { + "defined": "WrappedI80F48" + } + } + }, + { + "name": "liabilityWeightMaint", + "type": { + "option": { + "defined": "WrappedI80F48" + } + } + }, + { + "name": "depositLimit", + "type": { + "option": "u64" + } + }, + { + "name": "borrowLimit", + "type": { + "option": "u64" + } + }, + { + "name": "operationalState", + "type": { + "option": { + "defined": "BankOperationalState" + } + } + }, + { + "name": "oracle", + "type": { + "option": { + "defined": "OracleConfig" + } + } + }, + { + "name": "interestRateConfig", + "type": { + "option": { + "defined": "InterestRateConfigOpt" + } + } + }, + { + "name": "riskTier", + "type": { + "option": { + "defined": "RiskTier" + } + } + } + ] + } + }, + { + "name": "OracleConfig", + "type": { + "kind": "struct", + "fields": [ + { + "name": "setup", + "type": { + "defined": "OracleSetup" + } + }, + { + "name": "keys", + "type": { + "array": [ + "publicKey", + 5 + ] + } + } + ] + } + }, + { + "name": "BalanceIncreaseType", + "type": { + "kind": "enum", + "variants": [ + { + "name": "Any" + }, + { + "name": "RepayOnly" + }, + { + "name": "DepositOnly" + } + ] + } + }, + { + "name": "BalanceDecreaseType", + "type": { + "kind": "enum", + "variants": [ + { + "name": "Any" + }, + { + "name": "WithdrawOnly" + }, + { + "name": "BorrowOnly" + }, + { + "name": "BypassBorrowLimit" + } + ] + } + }, + { + "name": "WeightType", + "type": { + "kind": "enum", + "variants": [ + { + "name": "Initial" + }, + { + "name": "Maintenance" + } + ] + } + }, + { + "name": "BalanceSide", + "type": { + "kind": "enum", + "variants": [ + { + "name": "Assets" + }, + { + "name": "Liabilities" + } + ] + } + }, + { + "name": "RiskRequirementType", + "type": { + "kind": "enum", + "variants": [ + { + "name": "Initial" + }, + { + "name": "Maintenance" + } + ] + } + }, + { + "name": "BankOperationalState", + "type": { + "kind": "enum", + "variants": [ + { + "name": "Paused" + }, + { + "name": "Operational" + }, + { + "name": "ReduceOnly" + } + ] + } + }, + { + "name": "OracleSetup", + "type": { + "kind": "enum", + "variants": [ + { + "name": "None" + }, + { + "name": "Pyth" + } + ] + } + }, + { + "name": "OracleKey", + "type": { + "kind": "enum", + "variants": [ + { + "name": "Pyth", + "fields": [ + "publicKey" + ] + } + ] + } + }, + { + "name": "RiskTier", + "type": { + "kind": "enum", + "variants": [ + { + "name": "Collateral" + }, + { + "name": "Isolated" + } + ] + } + }, + { + "name": "BankVaultType", + "type": { + "kind": "enum", + "variants": [ + { + "name": "Liquidity" + }, + { + "name": "Insurance" + }, + { + "name": "Fee" + } + ] + } + } + ], + "events": [ + { + "name": "MarginfiGroupCreateEvent", + "fields": [ + { + "name": "header", + "type": { + "defined": "GroupEventHeader" + }, + "index": false + } + ] + }, + { + "name": "MarginfiGroupConfigureEvent", + "fields": [ + { + "name": "header", + "type": { + "defined": "GroupEventHeader" + }, + "index": false + }, + { + "name": "config", + "type": { + "defined": "GroupConfig" + }, + "index": false + } + ] + }, + { + "name": "LendingPoolBankCreateEvent", + "fields": [ + { + "name": "header", + "type": { + "defined": "GroupEventHeader" + }, + "index": false + }, + { + "name": "bank", + "type": "publicKey", + "index": false + }, + { + "name": "mint", + "type": "publicKey", + "index": false + } + ] + }, + { + "name": "LendingPoolBankConfigureEvent", + "fields": [ + { + "name": "header", + "type": { + "defined": "GroupEventHeader" + }, + "index": false + }, + { + "name": "bank", + "type": "publicKey", + "index": false + }, + { + "name": "mint", + "type": "publicKey", + "index": false + }, + { + "name": "config", + "type": { + "defined": "BankConfigOptArg" + }, + "index": false + } + ] + }, + { + "name": "LendingPoolBankAccrueInterestEvent", + "fields": [ + { + "name": "header", + "type": { + "defined": "GroupEventHeader" + }, + "index": false + }, + { + "name": "bank", + "type": "publicKey", + "index": false + }, + { + "name": "mint", + "type": "publicKey", + "index": false + }, + { + "name": "delta", + "type": "u64", + "index": false + }, + { + "name": "feesCollected", + "type": "f64", + "index": false + }, + { + "name": "insuranceCollected", + "type": "f64", + "index": false + } + ] + }, + { + "name": "LendingPoolBankCollectFeesEvent", + "fields": [ + { + "name": "header", + "type": { + "defined": "GroupEventHeader" + }, + "index": false + }, + { + "name": "bank", + "type": "publicKey", + "index": false + }, + { + "name": "mint", + "type": "publicKey", + "index": false + }, + { + "name": "groupFeesCollected", + "type": "f64", + "index": false + }, + { + "name": "groupFeesOutstanding", + "type": "f64", + "index": false + }, + { + "name": "insuranceFeesCollected", + "type": "f64", + "index": false + }, + { + "name": "insuranceFeesOutstanding", + "type": "f64", + "index": false + } + ] + }, + { + "name": "LendingPoolBankHandleBankruptcyEvent", + "fields": [ + { + "name": "header", + "type": { + "defined": "AccountEventHeader" + }, + "index": false + }, + { + "name": "bank", + "type": "publicKey", + "index": false + }, + { + "name": "mint", + "type": "publicKey", + "index": false + }, + { + "name": "badDebt", + "type": "f64", + "index": false + }, + { + "name": "coveredAmount", + "type": "f64", + "index": false + }, + { + "name": "socializedAmount", + "type": "f64", + "index": false + } + ] + }, + { + "name": "MarginfiAccountCreateEvent", + "fields": [ + { + "name": "header", + "type": { + "defined": "AccountEventHeader" + }, + "index": false + } + ] + }, + { + "name": "LendingAccountDepositEvent", + "fields": [ + { + "name": "header", + "type": { + "defined": "AccountEventHeader" + }, + "index": false + }, + { + "name": "bank", + "type": "publicKey", + "index": false + }, + { + "name": "mint", + "type": "publicKey", + "index": false + }, + { + "name": "amount", + "type": "u64", + "index": false + } + ] + }, + { + "name": "LendingAccountRepayEvent", + "fields": [ + { + "name": "header", + "type": { + "defined": "AccountEventHeader" + }, + "index": false + }, + { + "name": "bank", + "type": "publicKey", + "index": false + }, + { + "name": "mint", + "type": "publicKey", + "index": false + }, + { + "name": "amount", + "type": "u64", + "index": false + }, + { + "name": "closeBalance", + "type": "bool", + "index": false + } + ] + }, + { + "name": "LendingAccountBorrowEvent", + "fields": [ + { + "name": "header", + "type": { + "defined": "AccountEventHeader" + }, + "index": false + }, + { + "name": "bank", + "type": "publicKey", + "index": false + }, + { + "name": "mint", + "type": "publicKey", + "index": false + }, + { + "name": "amount", + "type": "u64", + "index": false + } + ] + }, + { + "name": "LendingAccountWithdrawEvent", + "fields": [ + { + "name": "header", + "type": { + "defined": "AccountEventHeader" + }, + "index": false + }, + { + "name": "bank", + "type": "publicKey", + "index": false + }, + { + "name": "mint", + "type": "publicKey", + "index": false + }, + { + "name": "amount", + "type": "u64", + "index": false + }, + { + "name": "closeBalance", + "type": "bool", + "index": false + } + ] + }, + { + "name": "LendingAccountLiquidateEvent", + "fields": [ + { + "name": "header", + "type": { + "defined": "AccountEventHeader" + }, + "index": false + }, + { + "name": "liquidateeMarginfiAccount", + "type": "publicKey", + "index": false + }, + { + "name": "liquidateeMarginfiAccountAuthority", + "type": "publicKey", + "index": false + }, + { + "name": "assetBank", + "type": "publicKey", + "index": false + }, + { + "name": "assetMint", + "type": "publicKey", + "index": false + }, + { + "name": "liabilityBank", + "type": "publicKey", + "index": false + }, + { + "name": "liabilityMint", + "type": "publicKey", + "index": false + }, + { + "name": "liquidateePreHealth", + "type": "f64", + "index": false + }, + { + "name": "liquidateePostHealth", + "type": "f64", + "index": false + }, + { + "name": "preBalances", + "type": { + "defined": "LiquidationBalances" + }, + "index": false + }, + { + "name": "postBalances", + "type": { + "defined": "LiquidationBalances" + }, + "index": false + } + ] + } + ], + "errors": [ + { + "code": 6000, + "name": "MathError", + "msg": "Math error" + }, + { + "code": 6001, + "name": "BankNotFound", + "msg": "Invalid bank index" + }, + { + "code": 6002, + "name": "LendingAccountBalanceNotFound", + "msg": "Lending account balance not found" + }, + { + "code": 6003, + "name": "BankAssetCapacityExceeded", + "msg": "Bank deposit capacity exceeded" + }, + { + "code": 6004, + "name": "InvalidTransfer", + "msg": "Invalid transfer" + }, + { + "code": 6005, + "name": "MissingPythOrBankAccount", + "msg": "Missing Pyth or Bank account" + }, + { + "code": 6006, + "name": "MissingPythAccount", + "msg": "Missing Pyth account" + }, + { + "code": 6007, + "name": "InvalidOracleAccount", + "msg": "Invalid Pyth account" + }, + { + "code": 6008, + "name": "MissingBankAccount", + "msg": "Missing Bank account" + }, + { + "code": 6009, + "name": "InvalidBankAccount", + "msg": "Invalid Bank account" + }, + { + "code": 6010, + "name": "BadAccountHealth", + "msg": "Bad account health" + }, + { + "code": 6011, + "name": "LendingAccountBalanceSlotsFull", + "msg": "Lending account balance slots are full" + }, + { + "code": 6012, + "name": "BankAlreadyExists", + "msg": "Bank already exists" + }, + { + "code": 6013, + "name": "IllegalLiquidation", + "msg": "Illegal post liquidation state, account is either not unhealthy or liquidation was too big" + }, + { + "code": 6014, + "name": "AccountNotBankrupt", + "msg": "Account is not bankrupt" + }, + { + "code": 6015, + "name": "BalanceNotBadDebt", + "msg": "Account balance is not bad debt" + }, + { + "code": 6016, + "name": "InvalidConfig", + "msg": "Invalid group config" + }, + { + "code": 6017, + "name": "StaleOracle", + "msg": "Stale oracle data" + }, + { + "code": 6018, + "name": "BankPaused", + "msg": "Bank paused" + }, + { + "code": 6019, + "name": "BankReduceOnly", + "msg": "Bank is ReduceOnly mode" + }, + { + "code": 6020, + "name": "BankAccoutNotFound", + "msg": "Bank is missing" + }, + { + "code": 6021, + "name": "OperationDepositOnly", + "msg": "Operation is deposit-only" + }, + { + "code": 6022, + "name": "OperationWithdrawOnly", + "msg": "Operation is withdraw-only" + }, + { + "code": 6023, + "name": "OperationBorrowOnly", + "msg": "Operation is borrow-only" + }, + { + "code": 6024, + "name": "OperationRepayOnly", + "msg": "Operation is repay-only" + }, + { + "code": 6025, + "name": "NoAssetFound", + "msg": "No asset found" + }, + { + "code": 6026, + "name": "NoLiabilityFound", + "msg": "No liability found" + }, + { + "code": 6027, + "name": "InvalidOracleSetup", + "msg": "Invalid oracle setup" + }, + { + "code": 6028, + "name": "IllegalUtilizationRatio", + "msg": "Invalid bank utilization ratio" + }, + { + "code": 6029, + "name": "BankLiabilityCapacityExceeded", + "msg": "Bank borrow cap exceeded" + }, + { + "code": 6030, + "name": "InvalidPrice", + "msg": "Invalid Price" + }, + { + "code": 6031, + "name": "IsolatedAccountIllegalState", + "msg": "Account can have only one liablity when account is under isolated risk" + } + ] +} diff --git a/observability/etl/dataflow-etls/dataflow_etls/orm/accounts.py b/observability/etl/dataflow-etls/dataflow_etls/orm/accounts.py new file mode 100644 index 00000000..a4ab8a47 --- /dev/null +++ b/observability/etl/dataflow-etls/dataflow_etls/orm/accounts.py @@ -0,0 +1,245 @@ +import json +import uuid +from dataclasses import dataclass +from typing import Union, Dict, Type, TYPE_CHECKING +from anchorpy.program.common import NamedInstruction as NamedAccountData + +from dataflow_etls.utils import pascal_to_snake_case, time_str, wrapped_i80f48_to_float, enum_to_str + +if TYPE_CHECKING: + from dataflow_etls.account_parsing import AccountUpdateRaw + +# IDL account names +MARGINFI_GROUP_ACCOUNT_NAME = 'MarginfiGroup' +MARGINFI_ACCOUNT_ACCOUNT_NAME = 'MarginfiAccount' +LENDING_POOL_BANK_ACCOUNT_NAME = 'Bank' + + +@dataclass +class AccountUpdateRecordBase: + SCHEMA = ",".join( + [ + "id:STRING", + "created_at:TIMESTAMP", + "idl_version:INTEGER", + "timestamp:TIMESTAMP", + "owner_program:STRING", + "pubkey:STRING", + ] + ) + + id: str + created_at: str + idl_version: int + timestamp: str + owner_program: str + pubkey: str + + def __init__(self, _parsed_data: NamedAccountData, account_update: "AccountUpdateRaw", idl_version: int): + self.id = str(uuid.uuid4()) + self.created_at = time_str() + self.timestamp = time_str(account_update['timestamp']) + self.idl_version = idl_version + self.owner_program = str(account_update['owner']) + self.pubkey = str(account_update['pubkey']) + + @classmethod + def get_tag(cls, snake_case: bool = False) -> str: + if snake_case: + return pascal_to_snake_case(cls.__name__) + else: + return cls.__name__ + + +# Event headers + + +@dataclass +class MarginfiGroupUpdateRecord(AccountUpdateRecordBase): + SCHEMA = AccountUpdateRecordBase.SCHEMA + "," + ",".join( + [ + "admin:STRING", + ] + ) + + admin: str + + def __init__(self, parsed_data: NamedAccountData, account_update: "AccountUpdateRaw", idl_version: int): + super().__init__(parsed_data, account_update, idl_version) + + self.admin = str(parsed_data.data.admin) + + +@dataclass +class MarginfiAccountUpdateRecord(AccountUpdateRecordBase): + SCHEMA = AccountUpdateRecordBase.SCHEMA + "," + ",".join( + [ + "group:STRING", + "authority:STRING", + "active_balances:STRING", + ] + ) + + group: str + authority: str + active_balances: str + + def __init__(self, parsed_data: NamedAccountData, account_update: "AccountUpdateRaw", idl_version: int): + super().__init__(parsed_data, account_update, idl_version) + + self.group = str(parsed_data.data.group) + self.authority = str(parsed_data.data.authority) + self.active_balances = json.dumps([{"bank": str(balance.bank_pk), + "asset_shares": wrapped_i80f48_to_float(balance.asset_shares), + "liability_shares": wrapped_i80f48_to_float(balance.liability_shares) + } for balance in parsed_data.data.lending_account.balances if + balance.active]) + + +@dataclass +class LendingPoolBankUpdateRecord(AccountUpdateRecordBase): + SCHEMA = AccountUpdateRecordBase.SCHEMA + "," + ",".join( + [ + "mint:STRING", + "mint_decimals:INTEGER", + "group:STRING", + "asset_share_value:BIGNUMERIC", + "liability_share_value:BIGNUMERIC", + "liquidity_vault:STRING", + "liquidity_vault_bump:INTEGER", + "liquidity_vault_authority_bump:INTEGER", + "insurance_vault:STRING", + "insurance_vault_bump:INTEGER", + "insurance_vault_authority_bump:INTEGER", + "fee_vault:STRING", + "fee_vault_bump:INTEGER", + "fee_vault_authority_bump:INTEGER", + "collected_insurance_fees_outstanding:BIGNUMERIC", + "collected_group_fees_outstanding:BIGNUMERIC", + "total_liability_shares:BIGNUMERIC", + "total_asset_shares:BIGNUMERIC", + "last_update:BIGNUMERIC", + "config_asset_weight_init:BIGNUMERIC", + "config_asset_weight_maint:BIGNUMERIC", + "config_liability_weight_init:BIGNUMERIC", + "config_liability_weight_maint:BIGNUMERIC", + "config_deposit_limit:BIGNUMERIC", + "config_borrow_limit:BIGNUMERIC", + "config_interest_rate_config_optimal_utilization_rate:BIGNUMERIC", + "config_interest_rate_config_plateau_interest_rate:BIGNUMERIC", + "config_interest_rate_config_max_interest_rate:BIGNUMERIC", + "config_interest_rate_config_insurance_fee_fixed_apr:BIGNUMERIC", + "config_interest_rate_config_insurance_ir_fee:BIGNUMERIC", + "config_interest_rate_config_protocol_fixed_fee_apr:BIGNUMERIC", + "config_interest_rate_config_protocol_ir_fee:BIGNUMERIC", + "config_operational_state:STRING", + "config_oracle_setup:STRING", + "config_oracle_keys:STRING", + "config_risk_tier:STRING", + ] + ) + + mint: str + mint_decimals: int + group: str + asset_share_value: float + liability_share_value: float + liquidity_vault: str + liquidity_vault_bump: int + liquidity_vault_authority_bump: int + insurance_vault: str + insurance_vault_bump: int + insurance_vault_authority_bump: int + fee_vault: str + fee_vault_bump: int + fee_vault_authority_bump: int + collected_insurance_fees_outstanding: float + collected_group_fees_outstanding: float + total_liability_shares: float + total_asset_shares: float + last_update: int + config_asset_weight_init: float + config_asset_weight_maint: float + config_liability_weight_init: float + config_liability_weight_maint: float + config_deposit_limit: int + config_borrow_limit: int + config_interest_rate_config_optimal_utilization_rate: float + config_interest_rate_config_plateau_interest_rate: float + config_interest_rate_config_max_interest_rate: float + config_interest_rate_config_insurance_fee_fixed_apr: float + config_interest_rate_config_insurance_ir_fee: float + config_interest_rate_config_protocol_fixed_fee_apr: float + config_interest_rate_config_protocol_ir_fee: float + config_operational_state: str + config_oracle_setup: str + config_oracle_keys: str + config_risk_tier: str + + def __init__(self, parsed_data: NamedAccountData, account_update: "AccountUpdateRaw", idl_version: int): + super().__init__(parsed_data, account_update, idl_version) + + self.mint = str(parsed_data.data.mint) + self.mint_decimals = int(parsed_data.data.mint_decimals) + self.group = str(parsed_data.data.group) + self.asset_share_value = wrapped_i80f48_to_float(parsed_data.data.asset_share_value) + self.liability_share_value = wrapped_i80f48_to_float(parsed_data.data.liability_share_value) + self.liquidity_vault = str(parsed_data.data.liquidity_vault) + self.liquidity_vault_bump = int(parsed_data.data.liquidity_vault_bump) + self.liquidity_vault_authority_bump = int(parsed_data.data.liquidity_vault_authority_bump) + self.insurance_vault = str(parsed_data.data.insurance_vault) + self.insurance_vault_bump = int(parsed_data.data.insurance_vault_bump) + self.insurance_vault_authority_bump = int(parsed_data.data.insurance_vault_authority_bump) + self.fee_vault = str(parsed_data.data.fee_vault) + self.fee_vault_bump = int(parsed_data.data.fee_vault_bump) + self.fee_vault_authority_bump = int(parsed_data.data.fee_vault_authority_bump) + self.collected_insurance_fees_outstanding = wrapped_i80f48_to_float( + parsed_data.data.collected_insurance_fees_outstanding) + self.collected_group_fees_outstanding = wrapped_i80f48_to_float( + parsed_data.data.collected_group_fees_outstanding) + self.total_liability_shares = wrapped_i80f48_to_float(parsed_data.data.total_liability_shares) + self.total_asset_shares = wrapped_i80f48_to_float(parsed_data.data.total_asset_shares) + self.last_update = int(parsed_data.data.last_update) + + self.config_asset_weight_init = wrapped_i80f48_to_float(parsed_data.data.config.asset_weight_init) + self.config_asset_weight_maint = wrapped_i80f48_to_float(parsed_data.data.config.asset_weight_maint) + self.config_liability_weight_init = wrapped_i80f48_to_float(parsed_data.data.config.liability_weight_init) + self.config_liability_weight_maint = wrapped_i80f48_to_float(parsed_data.data.config.liability_weight_maint) + self.config_deposit_limit = int(parsed_data.data.config.deposit_limit) + self.config_borrow_limit = int(parsed_data.data.config.borrow_limit) + self.config_operational_state = enum_to_str(parsed_data.data.config.operational_state) + self.config_oracle_setup = enum_to_str(parsed_data.data.config.oracle_setup) + self.config_oracle_keys = str([str(pk) for pk in parsed_data.data.config.oracle_keys]) + self.config_risk_tier = enum_to_str(parsed_data.data.config.risk_tier) + + self.config_interest_rate_config_optimal_utilization_rate = wrapped_i80f48_to_float( + parsed_data.data.config.interest_rate_config.optimal_utilization_rate) + self.config_interest_rate_config_plateau_interest_rate = wrapped_i80f48_to_float( + parsed_data.data.config.interest_rate_config.plateau_interest_rate) + self.config_interest_rate_config_max_interest_rate = wrapped_i80f48_to_float( + parsed_data.data.config.interest_rate_config.max_interest_rate) + self.config_interest_rate_config_insurance_fee_fixed_apr = wrapped_i80f48_to_float( + parsed_data.data.config.interest_rate_config.insurance_fee_fixed_apr) + self.config_interest_rate_config_insurance_ir_fee = wrapped_i80f48_to_float( + parsed_data.data.config.interest_rate_config.insurance_ir_fee) + self.config_interest_rate_config_protocol_fixed_fee_apr = wrapped_i80f48_to_float( + parsed_data.data.config.interest_rate_config.protocol_fixed_fee_apr) + self.config_interest_rate_config_protocol_ir_fee = wrapped_i80f48_to_float( + parsed_data.data.config.interest_rate_config.protocol_ir_fee) + + +AccountUpdateRecordTypes = [MarginfiGroupUpdateRecord, + MarginfiAccountUpdateRecord, + LendingPoolBankUpdateRecord] + +AccountUpdateRecord = Union[ + MarginfiGroupUpdateRecord, + MarginfiAccountUpdateRecord, + LendingPoolBankUpdateRecord +] + +ACCOUNT_UPDATE_TO_RECORD_TYPE: Dict[str, Type[AccountUpdateRecord]] = { + f"{MARGINFI_GROUP_ACCOUNT_NAME}": MarginfiGroupUpdateRecord, + f"{MARGINFI_ACCOUNT_ACCOUNT_NAME}": MarginfiAccountUpdateRecord, + f"{LENDING_POOL_BANK_ACCOUNT_NAME}": LendingPoolBankUpdateRecord, +} diff --git a/observability/etl/dataflow-etls/dataflow_etls/orm/events.py b/observability/etl/dataflow-etls/dataflow_etls/orm/events.py index d01a6fc1..115ca4db 100644 --- a/observability/etl/dataflow-etls/dataflow_etls/orm/events.py +++ b/observability/etl/dataflow-etls/dataflow_etls/orm/events.py @@ -1,36 +1,27 @@ -import re import uuid from dataclasses import dataclass -from datetime import datetime, timezone -from typing import Union, Optional, Dict, Type +from typing import Union, Optional, Dict, Type, TYPE_CHECKING from anchorpy import Event, NamedInstruction -from dataflow_etls.transaction_log_parser import InstructionWithLogs +from dataflow_etls.utils import pascal_to_snake_case, wrapped_i80f48_to_float, time_str, map_optional -# IDL event names -MARGINFI_GROUP_CREATE_EVENT = 'MarginfiGroupCreateEvent' -MARGINFI_GROUP_CONFIGURE_EVENT = 'MarginfiGroupConfigureEvent' -LENDING_POOL_BANK_CREATE_EVENT = 'LendingPoolBankCreateEvent' -LENDING_POOL_BANK_CONFIGURE_EVENT = 'LendingPoolBankConfigureEvent' -LENDING_POOL_BANK_ACCRUE_INTEREST_EVENT = 'LendingPoolBankAccrueInterestEvent' -LENDING_POOL_BANK_COLLECT_FEES_EVENT = 'LendingPoolBankCollectFeesEvent' -LENDING_POOL_BANK_HANDLE_BANKRUPTCY_EVENT = 'LendingPoolBankHandleBankruptcyEvent' -MARGINFI_ACCOUNT_CREATE_EVENT = 'MarginfiAccountCreateEvent' -LENDING_ACCOUNT_DEPOSIT_EVENT = 'LendingAccountDepositEvent' -LENDING_ACCOUNT_WITHDRAW_EVENT = 'LendingAccountWithdrawEvent' -LENDING_ACCOUNT_BORROW_EVENT = 'LendingAccountBorrowEvent' -LENDING_ACCOUNT_REPAY_EVENT = 'LendingAccountRepayEvent' -LENDING_ACCOUNT_LIQUIDATE_EVENT = 'LendingAccountLiquidateEvent' - - -def pascal_to_snake_case(string: str) -> str: - return re.sub('(?!^)([A-Z]+)', r'_\1', string).lower() +if TYPE_CHECKING: + from dataflow_etls.transaction_parsing import InstructionWithLogs - -def time_str(dt: Optional[datetime] = None) -> str: - if dt is None: - dt = datetime.now(timezone.utc) - return dt.strftime("%Y-%m-%d %H:%M:%S %Z") +# IDL event names +MARGINFI_GROUP_CREATE_EVENT_NAME = 'MarginfiGroupCreateEvent' +MARGINFI_GROUP_CONFIGURE_EVENT_NAME = 'MarginfiGroupConfigureEvent' +LENDING_POOL_BANK_CREATE_EVENT_NAME = 'LendingPoolBankCreateEvent' +LENDING_POOL_BANK_CONFIGURE_EVENT_NAME = 'LendingPoolBankConfigureEvent' +LENDING_POOL_BANK_ACCRUE_INTEREST_EVENT_NAME = 'LendingPoolBankAccrueInterestEvent' +LENDING_POOL_BANK_COLLECT_FEES_EVENT_NAME = 'LendingPoolBankCollectFeesEvent' +LENDING_POOL_BANK_HANDLE_BANKRUPTCY_EVENT_NAME = 'LendingPoolBankHandleBankruptcyEvent' +MARGINFI_ACCOUNT_CREATE_EVENT_NAME = 'MarginfiAccountCreateEvent' +LENDING_ACCOUNT_DEPOSIT_EVENT_NAME = 'LendingAccountDepositEvent' +LENDING_ACCOUNT_WITHDRAW_EVENT_NAME = 'LendingAccountWithdrawEvent' +LENDING_ACCOUNT_BORROW_EVENT_NAME = 'LendingAccountBorrowEvent' +LENDING_ACCOUNT_REPAY_EVENT_NAME = 'LendingAccountRepayEvent' +LENDING_ACCOUNT_LIQUIDATE_EVENT_NAME = 'LendingAccountLiquidateEvent' @dataclass @@ -56,7 +47,7 @@ class RecordBase: signature: str indexing_address: str - def __init__(self, _event: Event, instruction: InstructionWithLogs, _instruction_args: NamedInstruction): + def __init__(self, _event: Event, instruction: "InstructionWithLogs", _instruction_args: NamedInstruction): self.id = str(uuid.uuid4()) self.created_at = time_str() self.timestamp = time_str(instruction.timestamp) @@ -67,8 +58,11 @@ def __init__(self, _event: Event, instruction: InstructionWithLogs, _instruction self.indexing_address = str(instruction.message.program_id) @classmethod - def get_tag(cls) -> str: - return cls.__name__ + def get_tag(cls, snake_case: bool = False) -> str: + if snake_case: + return pascal_to_snake_case(cls.__name__) + else: + return cls.__name__ # Event headers @@ -89,7 +83,7 @@ class AccountRecordBase(RecordBase): marginfi_account: str marginfi_account_authority: str - def __init__(self, event: Event, instruction: InstructionWithLogs, instruction_args: NamedInstruction): + def __init__(self, event: Event, instruction: "InstructionWithLogs", instruction_args: NamedInstruction): super().__init__(event, instruction, instruction_args) self.signer = str(event.data.header.signer) if event.data.header.signer is not None else None @@ -110,7 +104,7 @@ class GroupRecordBase(RecordBase): signer: Optional[str] marginfi_group: str - def __init__(self, event: Event, instruction: InstructionWithLogs, instruction_args: NamedInstruction): + def __init__(self, event: Event, instruction: "InstructionWithLogs", instruction_args: NamedInstruction): super().__init__(event, instruction, instruction_args) self.signer = str(event.data.header.signer) if event.data.header.signer is not None else None @@ -124,7 +118,7 @@ def __init__(self, event: Event, instruction: InstructionWithLogs, instruction_a class MarginfiGroupCreateRecord(GroupRecordBase): SCHEMA = GroupRecordBase.SCHEMA - def __init__(self, event: Event, instruction: InstructionWithLogs, instruction_args: NamedInstruction): + def __init__(self, event: Event, instruction: "InstructionWithLogs", instruction_args: NamedInstruction): super().__init__(event, instruction, instruction_args) @@ -138,7 +132,7 @@ class MarginfiGroupConfigureRecord(GroupRecordBase): admin: Optional[str] - def __init__(self, event: Event, instruction: InstructionWithLogs, instruction_args: NamedInstruction): + def __init__(self, event: Event, instruction: "InstructionWithLogs", instruction_args: NamedInstruction): super().__init__(event, instruction, instruction_args) self.admin = event.data.config.admin @@ -156,7 +150,7 @@ class LendingPoolBankCreateRecord(GroupRecordBase): bank: str mint: str - def __init__(self, event: Event, instruction: InstructionWithLogs, instruction_args: NamedInstruction): + def __init__(self, event: Event, instruction: "InstructionWithLogs", instruction_args: NamedInstruction): super().__init__(event, instruction, instruction_args) self.bank = str(event.data.bank) @@ -169,76 +163,85 @@ class LendingPoolBankConfigureRecord(GroupRecordBase): [ "bank:STRING", "mint:STRING", - # "asset_weight_init:NUMERIC", - # "asset_weight_maint:NUMERIC", - # "liability_weight_init:NUMERIC", - # "liability_weight_maint:NUMERIC", - # "deposit_limit:BIGNUMERIC", - # "borrow_limit:BIGNUMERIC", - # "operational_state:STRING", - # "oracle_setup:STRING", - # "oracle_keys:STRING", - # "optimal_utilization_rate:NUMERIC", - # "plateau_interest_rate:NUMERIC", - # "max_interest_rate:NUMERIC", - # "insurance_fee_fixed_apr:NUMERIC", - # "insurance_ir_fee:NUMERIC", - # "protocol_fixed_fee_apr:NUMERIC", - # "protocol_ir_fee:NUMERIC", + "asset_weight_init:NUMERIC", + "asset_weight_maint:NUMERIC", + "liability_weight_init:NUMERIC", + "liability_weight_maint:NUMERIC", + "deposit_limit:BIGNUMERIC", + "borrow_limit:BIGNUMERIC", + "operational_state:STRING", + "oracle_setup:STRING", + "oracle_keys:STRING", + "optimal_utilization_rate:NUMERIC", + "plateau_interest_rate:NUMERIC", + "max_interest_rate:NUMERIC", + "insurance_fee_fixed_apr:NUMERIC", + "insurance_ir_fee:NUMERIC", + "protocol_fixed_fee_apr:NUMERIC", + "protocol_ir_fee:NUMERIC", ] ) bank: str mint: str - # todo: config - - # asset_weight_init: Optional[float] - # asset_weight_maint: Optional[float] - # - # liability_weight_init: Optional[float] - # liability_weight_maint: Optional[float] - # - # deposit_limit: Optional[int] - # borrow_limit: Optional[int] - # - # operational_state: Optional[str] - # oracle_setup: Optional[str] - # oracle_keys: Optional[str] - # - # optimal_utilization_rate: Optional[float] - # plateau_interest_rate: Optional[float] - # max_interest_rate: Optional[float] - # - # insurance_fee_fixed_apr: Optional[float] - # insurance_ir_fee: Optional[float] - # protocol_fixed_fee_apr: Optional[float] - # protocol_ir_fee: Optional[float] - - def __init__(self, event: Event, instruction: InstructionWithLogs, instruction_args: NamedInstruction): + asset_weight_init: Optional[float] + asset_weight_maint: Optional[float] + + liability_weight_init: Optional[float] + liability_weight_maint: Optional[float] + + deposit_limit: Optional[int] + borrow_limit: Optional[int] + + operational_state: Optional[str] + oracle_setup: Optional[str] + oracle_keys: Optional[str] + + optimal_utilization_rate: Optional[float] + plateau_interest_rate: Optional[float] + max_interest_rate: Optional[float] + + insurance_fee_fixed_apr: Optional[float] + insurance_ir_fee: Optional[float] + protocol_fixed_fee_apr: Optional[float] + protocol_ir_fee: Optional[float] + + def __init__(self, event: Event, instruction: "InstructionWithLogs", instruction_args: NamedInstruction): super().__init__(event, instruction, instruction_args) self.bank = str(event.data.bank) self.mint = str(event.data.mint) - # self.asset_weight_init = event.data.config.asset_weight_init - # self.asset_weight_maint = event.data.config.asset_weight_maint - # self.liability_weight_init = event.data.config.liability_weight_init - # self.liability_weight_maint = event.data.config.liability_weight_maint - # self.deposit_limit = event.data.config.deposit_limit - # self.borrow_limit = event.data.config.borrow_limit - # - # self.operational_state = str(event.data.config.operational_state) - # self.oracle_setup = str(event.data.config.oracle.setup) if event.data.config.oracle is not None else None - # self.oracle_keys = str(event.data.config.oracle.keys) if event.data.config.oracle is not None else None - # - # self.optimal_utilization_rate = event.data.config.interest_rate_config.optimal_utilization_rate - # self.plateau_interest_rate = event.data.config.interest_rate_config.plateau_interest_rate - # self.max_interest_rate = event.data.config.interest_rate_config.max_interest_rate - # self.insurance_fee_fixed_apr = event.data.config.interest_rate_config.insurance_fee_fixed_apr - # self.insurance_ir_fee = event.data.config.interest_rate_config.insurance_ir_fee - # self.protocol_fixed_fee_apr = event.data.config.interest_rate_config.protocol_fixed_fee_apr - # self.protocol_ir_fee = event.data.config.interest_rate_config.protocol_ir_fee + self.asset_weight_init = map_optional(event.data.config.asset_weight_init, wrapped_i80f48_to_float) + self.asset_weight_maint = map_optional(event.data.config.asset_weight_maint, wrapped_i80f48_to_float) + self.liability_weight_init = map_optional(event.data.config.liability_weight_init, wrapped_i80f48_to_float) + self.liability_weight_maint = map_optional(event.data.config.liability_weight_maint, wrapped_i80f48_to_float) + self.deposit_limit = event.data.config.deposit_limit + self.borrow_limit = event.data.config.borrow_limit + + self.operational_state = map_optional(event.data.config.operational_state, str) + if event.data.config.oracle: + self.oracle_setup = str(event.data.config.oracle.setup) + self.oracle_keys = str([str(pk) for pk in event.data.config.oracle.keys]) + else: + self.oracle_setup = None + self.oracle_keys = None + + self.optimal_utilization_rate = map_optional( + event.data.config.interest_rate_config.optimal_utilization_rate, wrapped_i80f48_to_float) + self.plateau_interest_rate = map_optional( + event.data.config.interest_rate_config.plateau_interest_rate, wrapped_i80f48_to_float) + self.max_interest_rate = map_optional( + event.data.config.interest_rate_config.max_interest_rate, wrapped_i80f48_to_float) + self.insurance_fee_fixed_apr = map_optional( + event.data.config.interest_rate_config.insurance_fee_fixed_apr, wrapped_i80f48_to_float) + self.insurance_ir_fee = map_optional( + event.data.config.interest_rate_config.insurance_ir_fee, wrapped_i80f48_to_float) + self.protocol_fixed_fee_apr = map_optional( + event.data.config.interest_rate_config.protocol_fixed_fee_apr, wrapped_i80f48_to_float) + self.protocol_ir_fee = map_optional( + event.data.config.interest_rate_config.protocol_ir_fee, wrapped_i80f48_to_float) @dataclass @@ -259,7 +262,7 @@ class LendingPoolBankAccrueInterestRecord(GroupRecordBase): fees_collected: float insurance_collected: float - def __init__(self, event: Event, instruction: InstructionWithLogs, instruction_args: NamedInstruction): + def __init__(self, event: Event, instruction: "InstructionWithLogs", instruction_args: NamedInstruction): super().__init__(event, instruction, instruction_args) self.bank = str(event.data.bank) @@ -289,7 +292,7 @@ class LendingPoolBankCollectFeesRecord(GroupRecordBase): insurance_fees_collected: float insurance_fees_outstanding: float - def __init__(self, event: Event, instruction: InstructionWithLogs, instruction_args: NamedInstruction): + def __init__(self, event: Event, instruction: "InstructionWithLogs", instruction_args: NamedInstruction): super().__init__(event, instruction, instruction_args) self.bank = str(event.data.bank) @@ -318,7 +321,7 @@ class LendingPoolBankHandleBankruptcyRecord(GroupRecordBase): covered_amount: float socialized_amount: float - def __init__(self, event: Event, instruction: InstructionWithLogs, instruction_args: NamedInstruction): + def __init__(self, event: Event, instruction: "InstructionWithLogs", instruction_args: NamedInstruction): super().__init__(event, instruction, instruction_args) self.bank = str(event.data.bank) @@ -334,7 +337,7 @@ def __init__(self, event: Event, instruction: InstructionWithLogs, instruction_a class MarginfiAccountCreateRecord(AccountRecordBase): SCHEMA = AccountRecordBase.SCHEMA - def __init__(self, event: Event, instruction: InstructionWithLogs, instruction_args: NamedInstruction): + def __init__(self, event: Event, instruction: "InstructionWithLogs", instruction_args: NamedInstruction): super().__init__(event, instruction, instruction_args) @@ -343,22 +346,28 @@ class LendingAccountChangeLiquidityRecord(AccountRecordBase): SCHEMA = AccountRecordBase.SCHEMA + "," + ",".join( [ "operation:STRING", + "bank:STRING", + "mint:STRING", "amount:BIGNUMERIC", "balance_closed:BOOLEAN" ] ) operation: str + bank: str + mint: str amount: int balance_closed: bool - def __init__(self, event: Event, instruction: InstructionWithLogs, instruction_args: NamedInstruction): + def __init__(self, event: Event, instruction: "InstructionWithLogs", instruction_args: NamedInstruction): super().__init__(event, instruction, instruction_args) self.operation = event.name.removeprefix("LendingAccount").removesuffix("Event").lower() + self.bank = str(event.data.bank) + self.mint = str(event.data.mint) self.amount = event.data.amount self.balance_closed = False - if event.name == LENDING_ACCOUNT_REPAY_EVENT or event.name == LENDING_ACCOUNT_WITHDRAW_EVENT: + if event.name == LENDING_ACCOUNT_REPAY_EVENT_NAME or event.name == LENDING_ACCOUNT_WITHDRAW_EVENT_NAME: self.balance_closed = event.data.close_balance @@ -402,7 +411,7 @@ class LendingAccountLiquidateRecord(AccountRecordBase): liquidator_asset_post_balance: float liquidator_liability_post_balance: float - def __init__(self, event: Event, instruction: InstructionWithLogs, instruction_args: NamedInstruction): + def __init__(self, event: Event, instruction: "InstructionWithLogs", instruction_args: NamedInstruction): super().__init__(event, instruction, instruction_args) self.liquidatee_marginfi_account = str(event.data.liquidatee_marginfi_account) @@ -423,7 +432,18 @@ def __init__(self, event: Event, instruction: InstructionWithLogs, instruction_a self.liquidator_liability_post_balance = event.data.post_balances.liquidator_liability_balance -Record = Union[ +EventRecordTypes = [MarginfiGroupCreateRecord, + MarginfiGroupConfigureRecord, + LendingPoolBankCreateRecord, + LendingPoolBankConfigureRecord, + LendingPoolBankAccrueInterestRecord, + LendingPoolBankCollectFeesRecord, + LendingPoolBankHandleBankruptcyRecord, + MarginfiAccountCreateRecord, + LendingAccountChangeLiquidityRecord, + LendingAccountLiquidateRecord] + +EventRecord = Union[ MarginfiGroupCreateRecord, MarginfiGroupConfigureRecord, LendingPoolBankCreateRecord, @@ -436,18 +456,18 @@ def __init__(self, event: Event, instruction: InstructionWithLogs, instruction_a LendingAccountLiquidateRecord ] -EVENT_TO_RECORD_TYPE: Dict[str, Type[Record]] = { - f"{MARGINFI_GROUP_CREATE_EVENT}": MarginfiGroupCreateRecord, - f"{MARGINFI_GROUP_CONFIGURE_EVENT}": MarginfiGroupConfigureRecord, - f"{LENDING_POOL_BANK_CREATE_EVENT}": LendingPoolBankCreateRecord, - f"{LENDING_POOL_BANK_CONFIGURE_EVENT}": LendingPoolBankConfigureRecord, - f"{LENDING_POOL_BANK_ACCRUE_INTEREST_EVENT}": LendingPoolBankAccrueInterestRecord, - f"{LENDING_POOL_BANK_COLLECT_FEES_EVENT}": LendingPoolBankCollectFeesRecord, - f"{LENDING_POOL_BANK_HANDLE_BANKRUPTCY_EVENT}": LendingPoolBankHandleBankruptcyRecord, - f"{MARGINFI_ACCOUNT_CREATE_EVENT}": MarginfiAccountCreateRecord, - f"{LENDING_ACCOUNT_DEPOSIT_EVENT}": LendingAccountChangeLiquidityRecord, - f"{LENDING_ACCOUNT_WITHDRAW_EVENT}": LendingAccountChangeLiquidityRecord, - f"{LENDING_ACCOUNT_BORROW_EVENT}": LendingAccountChangeLiquidityRecord, - f"{LENDING_ACCOUNT_REPAY_EVENT}": LendingAccountChangeLiquidityRecord, - f"{LENDING_ACCOUNT_LIQUIDATE_EVENT}": LendingAccountLiquidateRecord, +EVENT_TO_RECORD_TYPE: Dict[str, Type[EventRecord]] = { + f"{MARGINFI_GROUP_CREATE_EVENT_NAME}": MarginfiGroupCreateRecord, + f"{MARGINFI_GROUP_CONFIGURE_EVENT_NAME}": MarginfiGroupConfigureRecord, + f"{LENDING_POOL_BANK_CREATE_EVENT_NAME}": LendingPoolBankCreateRecord, + f"{LENDING_POOL_BANK_CONFIGURE_EVENT_NAME}": LendingPoolBankConfigureRecord, + f"{LENDING_POOL_BANK_ACCRUE_INTEREST_EVENT_NAME}": LendingPoolBankAccrueInterestRecord, + f"{LENDING_POOL_BANK_COLLECT_FEES_EVENT_NAME}": LendingPoolBankCollectFeesRecord, + f"{LENDING_POOL_BANK_HANDLE_BANKRUPTCY_EVENT_NAME}": LendingPoolBankHandleBankruptcyRecord, + f"{MARGINFI_ACCOUNT_CREATE_EVENT_NAME}": MarginfiAccountCreateRecord, + f"{LENDING_ACCOUNT_DEPOSIT_EVENT_NAME}": LendingAccountChangeLiquidityRecord, + f"{LENDING_ACCOUNT_WITHDRAW_EVENT_NAME}": LendingAccountChangeLiquidityRecord, + f"{LENDING_ACCOUNT_BORROW_EVENT_NAME}": LendingAccountChangeLiquidityRecord, + f"{LENDING_ACCOUNT_REPAY_EVENT_NAME}": LendingAccountChangeLiquidityRecord, + f"{LENDING_ACCOUNT_LIQUIDATE_EVENT_NAME}": LendingAccountLiquidateRecord, } diff --git a/observability/etl/dataflow-etls/dataflow_etls/transaction_log_parser.py b/observability/etl/dataflow-etls/dataflow_etls/transaction_log_parser.py deleted file mode 100644 index aaac98a0..00000000 --- a/observability/etl/dataflow-etls/dataflow_etls/transaction_log_parser.py +++ /dev/null @@ -1,110 +0,0 @@ -import re -from dataclasses import dataclass -from datetime import datetime -from typing import List, Any, Callable, Optional -from based58 import based58 # type: ignore -from solders.instruction import CompiledInstruction -from solders.pubkey import Pubkey - - -@dataclass -class Instruction: - program_id: Pubkey - accounts: List[Pubkey] - data: bytes - - -@dataclass -class InstructionWithLogs: - timestamp: datetime - idl_version: int - signature: str - message: Instruction - logs: List[str] - inner_instructions: List["InstructionWithLogs"] - logs_truncated: bool - is_cpi: bool - # call_stack: List[Pubkey] - - -INVOKE_MESSAGE = "Program log: " -PROGRAM_LOG = "Program log: " -PROGRAM_DATA = "Program data: " -LOG_TRUNCATED = "Log truncated" - - -def merge_instructions_and_cpis(message_instructions: List[CompiledInstruction], inner_instructions: List[Any]) -> List[ - CompiledInstruction]: - def search(array: List[Any], callback: Callable[[Any], bool]) -> Optional[int]: - for i, elem in enumerate(array): - if callback(elem): - return i - return None - - compiled_instructions: List[CompiledInstruction] = [] - for ix_index, instruction in enumerate(message_instructions): - compiled_instructions.append(instruction) - inner_ixs_index = search(inner_instructions, lambda inner_ixs: bool(inner_ixs["index"] == ix_index)) - if inner_ixs_index is not None: - for ix_raw in inner_instructions[inner_ixs_index]["instructions"]: - compiled_instructions.append( - CompiledInstruction(program_id_index=ix_raw["programIdIndex"], accounts=bytes(ix_raw["accounts"]), - data=based58.b58decode(str.encode(ix_raw["data"])))) - - return compiled_instructions - - -def expand_instructions(account_keys: List[Pubkey], compiled_instructions: List[CompiledInstruction]) -> List[ - Instruction]: - expanded_instructions = [] - for ix in compiled_instructions: - expanded_instruction = Instruction(data=ix.data, - accounts=[account_keys[account_index] for account_index in ix.accounts], - program_id=account_keys[ix.program_id_index]) - expanded_instructions.append(expanded_instruction) - return expanded_instructions - - -def get_latest_ix_ref(instructions: List[InstructionWithLogs], stack_depth: int) -> "InstructionWithLogs": - target_instruction_list = instructions - for i in range(stack_depth - 1): - target_instruction_list = target_instruction_list[-1].inner_instructions - return target_instruction_list[-1] - - -def reconcile_instruction_logs(timestamp: datetime, signature: str, instructions: List[Instruction], logs: List[str], - idl_version: int) -> \ - List[InstructionWithLogs]: - depth = 0 - instructions_consumed = 0 - instructions_with_logs: List[InstructionWithLogs] = [] - - for log in logs: - if log.startswith(LOG_TRUNCATED): - ix = get_latest_ix_ref(instructions_with_logs, depth) - ix.logs_truncated = True - else: - invoke_regex = r"Program (?P\w+) invoke" - matches = re.search(invoke_regex, log) - if matches is not None: - target_instruction_list = instructions_with_logs - for i in range(depth): - target_instruction_list = target_instruction_list[-1].inner_instructions - - message = instructions[instructions_consumed] - target_instruction_list.append( - InstructionWithLogs(timestamp=timestamp, idl_version=idl_version, signature=signature, logs=[log], - message=message, - inner_instructions=[], logs_truncated=False, is_cpi=(depth > 0))) - depth += 1 - instructions_consumed += 1 - else: - if "success" in log or "failed" in log: - ix = get_latest_ix_ref(instructions_with_logs, depth) - ix.logs.append(log) - depth -= 1 - else: - ix = get_latest_ix_ref(instructions_with_logs, depth) - ix.logs.append(log) - - return instructions_with_logs diff --git a/observability/etl/dataflow-etls/dataflow_etls/transaction_parsing.py b/observability/etl/dataflow-etls/dataflow_etls/transaction_parsing.py new file mode 100644 index 00000000..81b9247c --- /dev/null +++ b/observability/etl/dataflow-etls/dataflow_etls/transaction_parsing.py @@ -0,0 +1,240 @@ +import base64 +import json +import re +from dataclasses import dataclass, asdict +from datetime import datetime +from typing import List, Any, Callable, Optional, Dict, Sequence, Tuple, Generator, Union, TypedDict +from decimal import Decimal + +from anchorpy_core.idl import Idl +from solders.message import Message, MessageV0 +from based58 import based58 # type: ignore +from anchorpy import NamedInstruction +from solders.instruction import CompiledInstruction +from solders.pubkey import Pubkey +import apache_beam as beam # type: ignore + +from dataflow_etls.idl_versions import VersionedProgram, IdlPool, Cluster +from dataflow_etls.orm.events import EVENT_TO_RECORD_TYPE, EventRecord + + +@dataclass +class Instruction: + program_id: Pubkey + accounts: List[Pubkey] + data: bytes + + +@dataclass +class InstructionWithLogs: + timestamp: datetime + idl_version: int + signature: str + message: Instruction + logs: List[str] + inner_instructions: List["InstructionWithLogs"] + logs_truncated: bool + is_cpi: bool + # call_stack: List[Pubkey] + + +INVOKE_MESSAGE = "Program log: " +PROGRAM_LOG = "Program log: " +PROGRAM_DATA = "Program data: " +LOG_TRUNCATED = "Log truncated" + +TransactionRaw = TypedDict('TransactionRaw', { + 'id': str, + 'created_at': datetime, + 'timestamp': datetime, + 'signature': str, + 'indexing_address': str, + 'slot': Decimal, + 'signer': str, + 'success': bool, + 'version': str, + 'fee': Decimal, + 'meta': str, + 'message': str, +}) + + +class IndexedProgramNotSupported(Exception): + pass + + +def extract_events_from_tx(tx: TransactionRaw, min_idl_version: int, cluster: Cluster, idl_pool: IdlPool) -> List[ + EventRecord]: + indexed_program_id_str = tx["indexing_address"] + indexed_program_id = Pubkey.from_string(indexed_program_id_str) + tx_slot = int(tx["slot"]) + + try: + idl_raw, idl_version = idl_pool.get_idl_for_slot(indexed_program_id_str, tx_slot) + except KeyError: + raise IndexedProgramNotSupported(f"Unsupported indexed program {indexed_program_id_str}") + + idl = Idl.from_json(idl_raw) + program = VersionedProgram(cluster, idl_version, idl, indexed_program_id) + + if min_idl_version is not None and idl_version < min_idl_version: + return [] + + meta = json.loads(tx["meta"]) + message_bytes = base64.b64decode(tx["message"]) + + tx_version = tx["version"] + message_decoded: Union[Message, MessageV0] + if tx_version == "legacy": + message_decoded = Message.from_bytes(message_bytes) + elif tx_version == "0": + message_decoded = MessageV0.from_bytes(message_bytes[1:]) + else: + return [] + + merged_instructions = merge_instructions_and_cpis(message_decoded.instructions, meta["innerInstructions"]) + expanded_instructions = expand_instructions(message_decoded.account_keys, merged_instructions) + ixs_with_logs = reconcile_instruction_logs(tx["timestamp"], tx["signature"], expanded_instructions, + meta["logMessages"], idl_version) + + records_list = [] + for ix_with_logs in ixs_with_logs: + records_list.extend(extract_events_from_ix(ix_with_logs, program)) + + return records_list + + +def merge_instructions_and_cpis(message_instructions: List[CompiledInstruction], inner_instructions: List[Any]) -> List[ + CompiledInstruction]: + def search(array: List[Any], callback: Callable[[Any], bool]) -> Optional[int]: + for i, elem in enumerate(array): + if callback(elem): + return i + return None + + compiled_instructions: List[CompiledInstruction] = [] + for ix_index, instruction in enumerate(message_instructions): + compiled_instructions.append(instruction) + inner_ixs_index = search(inner_instructions, lambda inner_ixs: bool(inner_ixs["index"] == ix_index)) + if inner_ixs_index is not None: + for ix_raw in inner_instructions[inner_ixs_index]["instructions"]: + compiled_instructions.append( + CompiledInstruction(program_id_index=ix_raw["programIdIndex"], accounts=bytes(ix_raw["accounts"]), + data=based58.b58decode(str.encode(ix_raw["data"])))) + + return compiled_instructions + + +def expand_instructions(account_keys: List[Pubkey], compiled_instructions: List[CompiledInstruction]) -> List[ + Instruction]: + expanded_instructions = [] + for ix in compiled_instructions: + expanded_instruction = Instruction(data=ix.data, + accounts=[account_keys[account_index] for account_index in ix.accounts], + program_id=account_keys[ix.program_id_index]) + expanded_instructions.append(expanded_instruction) + return expanded_instructions + + +def reconcile_instruction_logs(timestamp: datetime, signature: str, instructions: List[Instruction], logs: List[str], + idl_version: int) -> \ + List[InstructionWithLogs]: + depth = 0 + instructions_consumed = 0 + instructions_with_logs: List[InstructionWithLogs] = [] + + for log in logs: + if log.startswith(LOG_TRUNCATED): + ix = get_latest_ix_ref(instructions_with_logs, depth) + ix.logs_truncated = True + else: + invoke_regex = r"Program (?P\w+) invoke" + matches = re.search(invoke_regex, log) + if matches is not None: + target_instruction_list = instructions_with_logs + for i in range(depth): + target_instruction_list = target_instruction_list[-1].inner_instructions + + message = instructions[instructions_consumed] + target_instruction_list.append( + InstructionWithLogs(timestamp=timestamp, idl_version=idl_version, signature=signature, logs=[log], + message=message, + inner_instructions=[], logs_truncated=False, is_cpi=(depth > 0))) + depth += 1 + instructions_consumed += 1 + else: + if "success" in log or "failed" in log: + ix = get_latest_ix_ref(instructions_with_logs, depth) + ix.logs.append(log) + depth -= 1 + else: + ix = get_latest_ix_ref(instructions_with_logs, depth) + ix.logs.append(log) + + return instructions_with_logs + + +def get_latest_ix_ref(instructions: List[InstructionWithLogs], stack_depth: int) -> "InstructionWithLogs": + target_instruction_list = instructions + for i in range(stack_depth - 1): + target_instruction_list = target_instruction_list[-1].inner_instructions + return target_instruction_list[-1] + + +def extract_events_from_ix(ix: InstructionWithLogs, program: VersionedProgram) -> List[EventRecord]: + ix_events: List[EventRecord] = [] + + if ix.message.program_id == program.program_id: + ix_events.extend(create_records_from_ix(ix, program)) + + for inner_ix in ix.inner_instructions: + ix_events.extend(extract_events_from_ix(inner_ix, program)) + + return ix_events + + +def create_records_from_ix(ix: InstructionWithLogs, program: VersionedProgram) -> Sequence[EventRecord]: + records: List[EventRecord] = [] + + try: + parsed_ix: NamedInstruction = program.coder.instruction.parse(ix.message.data) + except Exception as e: + print(f"failed to parse instruction data in tx {ix.signature} ({ix.timestamp})", e) + return records + + for log in ix.logs: + if not log.startswith(PROGRAM_DATA): + continue + + event_encoded = log[len(PROGRAM_DATA):] + try: + event_bytes = base64.b64decode(event_encoded) + except Exception as e: + print(f"error: failed to decode base64 event string in tx {ix.signature}", e) + continue + + try: + event = program.coder.events.parse(event_bytes) + except Exception as e: + print(f"failed to parse event in tx {ix.signature}", e) + continue + + if event is None or event.name not in EVENT_TO_RECORD_TYPE: + print(f"discarding unsupported event in tx {ix.signature}") + print(event) + else: + # noinspection PyPep8Naming + RecordType = EVENT_TO_RECORD_TYPE[event.name] + records.append(RecordType(event, ix, parsed_ix)) + + return records + + +class DispatchEventsDoFn(beam.DoFn): # type: ignore + def process(self, record: EventRecord, *args: Tuple[Any], **kwargs: Dict[str, Tuple[Any]]) -> Generator[ + str, None, None]: + yield beam.pvalue.TaggedOutput(record.get_tag(), record) + + +def dictionify_record(record: EventRecord) -> Dict[str, Any]: + return asdict(record) diff --git a/observability/etl/dataflow-etls/dataflow_etls/utils.py b/observability/etl/dataflow-etls/dataflow_etls/utils.py new file mode 100644 index 00000000..8d44c156 --- /dev/null +++ b/observability/etl/dataflow-etls/dataflow_etls/utils.py @@ -0,0 +1,40 @@ +import re +from datetime import datetime, timezone +from typing import NamedTuple, TypeVar, Optional, Callable, Any + +from decimal import Decimal + + +def pascal_to_snake_case(string: str) -> str: + return re.sub('(?!^)([A-Z]+)', r'_\1', string).lower() + + +WrappedI80F48 = NamedTuple('WrappedI80F48', [('value', int)]) + + +def wrapped_i80f48_to_float(wrapped_i80f48: WrappedI80F48) -> float: + nb_of_fractional_bits = 48 + value = Decimal(wrapped_i80f48.value) + value = value / 2 ** nb_of_fractional_bits + return float(value) + + +def enum_to_str(enum: Any) -> str: + return enum.__class__.__name__.lower() + + +InputType = TypeVar('InputType') +OutputType = TypeVar('OutputType') + + +def map_optional(element: Optional[InputType], fn: Callable[[InputType], OutputType]) -> Optional[OutputType]: + if element is not None: + return fn(element) + else: + return None + + +def time_str(dt: Optional[datetime] = None) -> str: + if dt is None: + dt = datetime.now(timezone.utc) + return dt.strftime("%Y-%m-%d %H:%M:%S %Z") diff --git a/observability/etl/dataflow-etls/job.py b/observability/etl/dataflow-etls/job.py deleted file mode 100644 index 5ed47c20..00000000 --- a/observability/etl/dataflow-etls/job.py +++ /dev/null @@ -1,270 +0,0 @@ -import argparse -import base64 -import json -import logging -from typing import List, Optional, Sequence, Union, Generator, Any, Tuple, Dict -from anchorpy import NamedInstruction -from solders.message import MessageV0, Message -import apache_beam as beam # type: ignore -from apache_beam.options.pipeline_options import PipelineOptions # type: ignore -from solders.pubkey import Pubkey - -from dataflow_etls.orm.events import Record, LendingAccountChangeLiquidityRecord, \ - MarginfiAccountCreateRecord, LendingPoolBankCreateRecord, \ - LendingPoolBankAccrueInterestRecord, LendingPoolBankHandleBankruptcyRecord, \ - LendingAccountLiquidateRecord, EVENT_TO_RECORD_TYPE, LendingPoolBankCollectFeesRecord, \ - LendingPoolBankConfigureRecord, MarginfiGroupConfigureRecord, MarginfiGroupCreateRecord -from dataflow_etls.idl_versions import VersionedIdl, VersionedProgram, Cluster -from dataflow_etls.transaction_log_parser import reconcile_instruction_logs, \ - merge_instructions_and_cpis, expand_instructions, InstructionWithLogs, PROGRAM_DATA - - -class DispatchEventsDoFn(beam.DoFn): # type: ignore - def process(self, record: Record, *args: Tuple[Any], **kwargs: Dict[str, Tuple[Any]]) -> Generator[str, None, None]: - yield beam.pvalue.TaggedOutput(record.get_tag(), record) - - -def create_records_from_ix(ix: InstructionWithLogs, program: VersionedProgram) -> Sequence[Record]: - records: List[Record] = [] - - try: - parsed_ix: NamedInstruction = program.coder.instruction.parse(ix.message.data) - except Exception as e: - print(ix) - print(f"failed to parse instruction data in tx {ix.signature}", e) - return records - - for log in ix.logs: - if not log.startswith(PROGRAM_DATA): - continue - - event_encoded = log[len(PROGRAM_DATA):] - try: - event_bytes = base64.b64decode(event_encoded) - except Exception as e: - print(f"error: failed to decode base64 event string in tx {ix.signature}", e) - continue - - print(f"info decoded with IDL {program.version}") - - try: - event = program.coder.events.parse(event_bytes) - except Exception as e: - print(f"failed to parse event in tx {ix.signature}", e) - continue - - if event is None or event.name not in EVENT_TO_RECORD_TYPE: - print(f"discarding unsupported event in tx {ix.signature}") - print(event) - else: - # noinspection PyPep8Naming - RecordType = EVENT_TO_RECORD_TYPE[event.name] - records.append(RecordType(event, ix, parsed_ix)) - - return records - - -def extract_events_from_ix(ix: InstructionWithLogs, program: VersionedProgram) -> List[Record]: - ix_events: List[Record] = [] - - if ix.message.program_id == program.program_id: - ix_events.extend(create_records_from_ix(ix, program)) - - for inner_ix in ix.inner_instructions: - ix_events.extend(extract_events_from_ix(inner_ix, program)) - - return ix_events - - -def run( - input_table: str, - output_table_namespace: str, - cluster: Cluster, - min_idl_version: int, - start_date: Optional[str] = None, - end_date: Optional[str] = None, - beam_args: Optional[List[str]] = None, -) -> None: - if beam_args is None: - beam_args = [] - - def extract_events_from_tx(tx: Any) -> List[Record]: - indexed_program_id_str = tx["indexing_address"] - indexed_program_id = Pubkey.from_string(indexed_program_id_str) - tx_slot = int(tx["slot"]) - idl, idl_version = VersionedIdl.get_idl_for_slot(cluster, indexed_program_id_str, tx_slot) - program = VersionedProgram(cluster, idl_version, idl, indexed_program_id) - - if min_idl_version is not None and idl_version < min_idl_version: - return [] - - meta = json.loads(tx["meta"]) - message_bytes = base64.b64decode(tx["message"]) - - tx_version = tx["version"] - message_decoded: Union[Message, MessageV0] - if tx_version == "legacy": - message_decoded = Message.from_bytes(message_bytes) - elif tx_version == "0": - message_decoded = MessageV0.from_bytes(message_bytes[1:]) - else: - return [] - - merged_instructions = merge_instructions_and_cpis(message_decoded.instructions, meta["innerInstructions"]) - expanded_instructions = expand_instructions(message_decoded.account_keys, merged_instructions) - ixs_with_logs = reconcile_instruction_logs(tx["timestamp"], tx["signature"], expanded_instructions, - meta["logMessages"], idl_version) - - records_list = [] - for ix_with_logs in ixs_with_logs: - records_list.extend(extract_events_from_ix(ix_with_logs, program)) - - return records_list - - """Build and run the pipeline.""" - pipeline_options = PipelineOptions(beam_args, save_main_session=True) - - if start_date is not None and end_date is not None: - input_query = f'SELECT * FROM `{input_table}` WHERE DATE(timestamp) >= "{start_date}" AND DATE(timestamp) < "{end_date}"' - elif start_date is not None: - input_query = ( - f'SELECT * FROM `{input_table}` WHERE DATE(timestamp) >= "{start_date}"' - ) - elif end_date is not None: - input_query = ( - f'SELECT * FROM `{input_table}` WHERE DATE(timestamp) < "{end_date}"' - ) - else: - input_query = f"SELECT * FROM `{input_table}`" - - with beam.Pipeline(options=pipeline_options) as pipeline: - # Define steps - read_raw_txs = beam.io.ReadFromBigQuery(query=input_query, use_standard_sql=True) - - extract_events = beam.FlatMap(extract_events_from_tx) - - dispatch_events = beam.ParDo(DispatchEventsDoFn()).with_outputs( - MarginfiGroupCreateRecord.get_tag(), - MarginfiGroupConfigureRecord.get_tag(), - LendingPoolBankCreateRecord.get_tag(), - LendingPoolBankConfigureRecord.get_tag(), - LendingPoolBankAccrueInterestRecord.get_tag(), - LendingPoolBankCollectFeesRecord.get_tag(), - LendingPoolBankHandleBankruptcyRecord.get_tag(), - MarginfiAccountCreateRecord.get_tag(), - LendingAccountChangeLiquidityRecord.get_tag(), - LendingAccountLiquidateRecord.get_tag() - ) - - if output_table_namespace == "local_file": # For testing purposes - write_marginfi_group_create_records = beam.io.WriteToText("local_file_marginfi_group_create_records") - write_marginfi_group_configure_records = beam.io.WriteToText("local_file_marginfi_group_configure_records") - write_lending_pool_bank_create_records = beam.io.WriteToText("local_file_lending_pool_bank_create_records") - write_lending_pool_bank_configure_records = beam.io.WriteToText( - "local_file_lending_pool_bank_configure_records") - write_lending_pool_bank_accrue_interest_records = beam.io.WriteToText( - "local_file_lending_pool_bank_accrue_interest_records") - write_lending_pool_bank_collect_fees_records = beam.io.WriteToText( - "local_file_lending_pool_bank_collect_fees_records") - write_lending_pool_bank_handle_bankruptcy_records = beam.io.WriteToText( - "local_file_lending_pool_bank_handle_bankruptcy_records") - write_marginfi_account_create_records = beam.io.WriteToText("local_file_marginfi_account_create_records") - write_lending_account_liquidity_change_records = beam.io.WriteToText( - "local_file_lending_account_liquidity_change_records") - write_lending_account_liquidate_records = beam.io.WriteToText( - "local_file_lending_account_liquidate_records") - else: - print("TODOOOOO") - exit(1) - # write_liquidity_change_events = beam.io.WriteToBigQuery( - # output_table, - # schema=PROCESSED_TRANSACTION_SCHEMA, - # write_disposition=beam.io.BigQueryDisposition.WRITE_APPEND, - # ) - - # Define pipeline - tagged_events = ( - pipeline - | "ReadRawTxs" >> read_raw_txs - | "ExtractEvents" >> extract_events - | "DispatchEvents" >> dispatch_events - ) - - tagged_events[MarginfiGroupCreateRecord.get_tag()] | ( - f"Write{MarginfiGroupCreateRecord.get_tag()}" >> write_marginfi_group_create_records) - tagged_events[MarginfiGroupConfigureRecord.get_tag()] | ( - f"Write{MarginfiGroupConfigureRecord.get_tag()}" >> write_marginfi_group_configure_records) - tagged_events[LendingPoolBankCreateRecord.get_tag()] | ( - f"Write{LendingPoolBankCreateRecord.get_tag()}" >> write_lending_pool_bank_create_records) - tagged_events[LendingPoolBankConfigureRecord.get_tag()] | ( - f"Write{LendingPoolBankConfigureRecord.get_tag()}" >> write_lending_pool_bank_configure_records) - tagged_events[LendingPoolBankAccrueInterestRecord.get_tag()] | ( - f"Write{LendingPoolBankAccrueInterestRecord.get_tag()}" >> write_lending_pool_bank_accrue_interest_records) - tagged_events[LendingPoolBankCollectFeesRecord.get_tag()] | ( - f"Write{LendingPoolBankCollectFeesRecord.get_tag()}" >> write_lending_pool_bank_collect_fees_records) - tagged_events[LendingPoolBankHandleBankruptcyRecord.get_tag()] | ( - f"Write{LendingPoolBankHandleBankruptcyRecord.get_tag()}" >> write_lending_pool_bank_handle_bankruptcy_records) - tagged_events[MarginfiAccountCreateRecord.get_tag()] | ( - f"Write{MarginfiAccountCreateRecord.get_tag()}" >> write_marginfi_account_create_records) - tagged_events[LendingAccountChangeLiquidityRecord.get_tag()] | ( - f"Write{LendingAccountChangeLiquidityRecord.get_tag()}" >> write_lending_account_liquidity_change_records) - tagged_events[LendingAccountLiquidateRecord.get_tag()] | ( - f"Write{LendingAccountLiquidateRecord.get_tag()}" >> write_lending_account_liquidate_records) - - -def main() -> None: - logging.getLogger().setLevel(logging.INFO) - - parser = argparse.ArgumentParser() - parser.add_argument( - "--input_table", - type=str, - required=True, - help="Input BigQuery table specified as: " - "PROJECT.DATASET.TABLE.", - ) - parser.add_argument( - "--output_table_namespace", - type=str, - required=True, - help="Output BigQuery namespace where event tables are located: PROJECT:DATASET.TABLE", - ) - parser.add_argument( - "--cluster", - type=str, - required=False, - default="mainnet", - help="Solana cluster being indexed: mainnet | devnet", - ) - parser.add_argument( - "--min_idl_version", - type=int, - required=False, - default=0, - help="Minimum IDL version to consider: int", - ) - parser.add_argument( - "--start_date", - type=str, - help="Start date to consider (inclusive) as: YYYY-MM-DD", - ) - parser.add_argument( - "--end_date", - type=str, - help="End date to consider (exclusive) as: YYYY-MM-DD", - ) - known_args, remaining_args = parser.parse_known_args() - - run( - input_table=known_args.input_table, - output_table_namespace=known_args.output_table_namespace, - cluster=known_args.cluster, - min_idl_version=known_args.min_idl_version, - start_date=known_args.start_date, - end_date=known_args.end_date, - beam_args=remaining_args, - ) - - -if __name__ == "__main__": - main() diff --git a/observability/etl/dataflow-etls/jobs/marginfi-v2-account-parsing-batch/job.py b/observability/etl/dataflow-etls/jobs/marginfi-v2-account-parsing-batch/job.py new file mode 100644 index 00000000..6f33d274 --- /dev/null +++ b/observability/etl/dataflow-etls/jobs/marginfi-v2-account-parsing-batch/job.py @@ -0,0 +1,141 @@ +import argparse +import logging +from typing import List, Optional, Any, Dict, Union +import apache_beam as beam # type: ignore +from apache_beam.options.pipeline_options import PipelineOptions # type: ignore + +from dataflow_etls.account_parsing import parse_account, OwnerProgramNotSupported, dictionify_record, DispatchEventsDoFn +from dataflow_etls.idl_versions import Cluster, IdlPool +from dataflow_etls.orm.accounts import AccountUpdateRecordTypes, AccountUpdateRecord + + +def run( + input_table: str, + output_table_namespace: str, + cluster: Cluster, + min_idl_version: int, + start_date: Optional[str] = None, + end_date: Optional[str] = None, + beam_args: Optional[List[str]] = None, +) -> None: + if beam_args is None: + beam_args = [] + + idl_pool = IdlPool(cluster) + + def parse_account_internal(tx: Any) -> List[AccountUpdateRecord]: + try: + return parse_account(tx, min_idl_version, cluster, idl_pool) + except OwnerProgramNotSupported: + return [] + + """Build and run the pipeline.""" + pipeline_options = PipelineOptions(beam_args, save_main_session=True) + + if start_date is not None and end_date is not None: + input_query = f'SELECT * FROM `{input_table}` WHERE DATE(timestamp) >= "{start_date}" AND DATE(timestamp) < "{end_date}"' + elif start_date is not None: + input_query = ( + f'SELECT * FROM `{input_table}` WHERE DATE(timestamp) >= "{start_date}"' + ) + elif end_date is not None: + input_query = ( + f'SELECT * FROM `{input_table}` WHERE DATE(timestamp) < "{end_date}"' + ) + else: + input_query = f"SELECT * FROM `{input_table}`" + + with beam.Pipeline(options=pipeline_options) as pipeline: + # Define steps + read_raw_txs = beam.io.ReadFromBigQuery(query=input_query, use_standard_sql=True) + + extract_events = beam.FlatMap(parse_account_internal) + + dispatch_events = beam.ParDo(DispatchEventsDoFn()).with_outputs( + *[rt.get_tag() for rt in AccountUpdateRecordTypes]) + + dictionify_events = beam.Map(dictionify_record) + + writers: Dict[str, Union[beam.io.WriteToText, beam.io.WriteToBigQuery]] = {} + for rt in AccountUpdateRecordTypes: + if output_table_namespace == "local_file": # For testing purposes + writers[rt.get_tag()] = beam.io.WriteToText(f"account_updates_{rt.get_tag(snake_case=True)}") + else: + writers[rt.get_tag()] = beam.io.WriteToBigQuery( + f"{output_table_namespace}_{rt.get_tag(snake_case=True)}", + schema=rt.SCHEMA, + write_disposition=beam.io.BigQueryDisposition.WRITE_APPEND, + create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED, + ) + + # Define pipeline + tagged_events = ( + pipeline + | "ReadRawTxs" >> read_raw_txs + | "ExtractEvents" >> extract_events + | "DispatchEvents" >> dispatch_events + ) + + for rt in AccountUpdateRecordTypes: + (tagged_events[rt.get_tag()] + | f"Dictionify{rt.get_tag()}" >> dictionify_events + | f"Write{rt.get_tag()}" >> writers[rt.get_tag()] + ) + + +def main() -> None: + logging.getLogger().setLevel(logging.INFO) + + parser = argparse.ArgumentParser() + parser.add_argument( + "--input_table", + type=str, + required=True, + help="Input BigQuery table specified as: " + "PROJECT.DATASET.TABLE.", + ) + parser.add_argument( + "--output_table_namespace", + type=str, + required=True, + help="Output BigQuery namespace where parsed account tables are located: PROJECT:DATASET.TABLE", + ) + parser.add_argument( + "--cluster", + type=str, + required=False, + default="mainnet", + help="Solana cluster being indexed: mainnet | devnet", + ) + parser.add_argument( + "--min_idl_version", + type=int, + required=False, + default=0, + help="Minimum IDL version to consider: int", + ) + parser.add_argument( + "--start_date", + type=str, + help="Start date to consider (inclusive) as: YYYY-MM-DD", + ) + parser.add_argument( + "--end_date", + type=str, + help="End date to consider (exclusive) as: YYYY-MM-DD", + ) + known_args, remaining_args = parser.parse_known_args() + + run( + input_table=known_args.input_table, + output_table_namespace=known_args.output_table_namespace, + cluster=known_args.cluster, + min_idl_version=known_args.min_idl_version, + start_date=known_args.start_date, + end_date=known_args.end_date, + beam_args=remaining_args, + ) + + +if __name__ == "__main__": + main() diff --git a/observability/etl/dataflow-etls/jobs/marginfi-v2-account-parsing-batch/metadata.json b/observability/etl/dataflow-etls/jobs/marginfi-v2-account-parsing-batch/metadata.json new file mode 100644 index 00000000..5a82d4b2 --- /dev/null +++ b/observability/etl/dataflow-etls/jobs/marginfi-v2-account-parsing-batch/metadata.json @@ -0,0 +1,50 @@ +{ + "name": "marginfi-v2-account-parsing-batch", + "description": "Parses individual account updates from a BigQuery table and stores them in dedicated BigQuery tables.", + "parameters": [ + { + "name": "input_table", + "label": "BigQuery input table name.", + "helpText": "Name of the input table to consume from.", + "regexes": [ + "([^.]+.)?[^.]+[.].+" + ] + }, + { + "name": "output_table_namespace", + "label": "BigQuery output tables namespace.", + "helpText": "Namespace where the BigQuery output tables are located.", + "regexes": [ + "([^:]+:)?[^.]+[.].+" + ] + }, + { + "name": "cluster", + "label": "Solana cluster.", + "isOptional": true, + "helpText": "Cluster where the account updates are executed (used to pull IDL version depending on tx slot).", + "regexes": ["mainnet|devnet"] + }, + { + "name": "min_idl_version", + "label": "Minimum IDL version.", + "isOptional": true, + "helpText": "Minimum IDL version for which txs will be parsed. Default: 0", + "regexes": [] + }, + { + "name": "start_date", + "label": "Start date.", + "isOptional": true, + "helpText": "Start date to consider (inclusive).", + "regexes": [] + }, + { + "name": "end_date", + "label": "End date.", + "isOptional": true, + "helpText": "End date to consider (exclusive).", + "regexes": [] + } + ] +} diff --git a/observability/etl/dataflow-etls/jobs/marginfi-v2-account-parsing-stream/job.py b/observability/etl/dataflow-etls/jobs/marginfi-v2-account-parsing-stream/job.py new file mode 100644 index 00000000..f1a926bc --- /dev/null +++ b/observability/etl/dataflow-etls/jobs/marginfi-v2-account-parsing-stream/job.py @@ -0,0 +1,146 @@ +import argparse +from dateutil import parser +import json +import logging +from typing import List, Optional, Any, Dict, Union +from decimal import Decimal + +import apache_beam as beam # type: ignore +from apache_beam.options.pipeline_options import PipelineOptions # type: ignore + +from dataflow_etls.account_parsing import parse_account, OwnerProgramNotSupported, dictionify_record, \ + DispatchEventsDoFn, AccountUpdateRaw +from dataflow_etls.idl_versions import Cluster, IdlPool +from dataflow_etls.orm.accounts import AccountUpdateRecordTypes, AccountUpdateRecord + + +def parse_json(message: bytes) -> AccountUpdateRaw: + account_update_raw = json.loads(message.decode("utf-8")) + return AccountUpdateRaw( + id=account_update_raw['id'], + created_at=parser.parse(account_update_raw['created_at']), + timestamp=parser.parse(account_update_raw['timestamp']), + owner=account_update_raw['owner'], + slot=Decimal(account_update_raw['slot']), + pubkey=account_update_raw['pubkey'], + txn_signature=account_update_raw['txn_signature'], + lamports=Decimal(account_update_raw['lamports']), + executable=bool(account_update_raw['executable']), + rent_epoch=Decimal(account_update_raw['rent_epoch']), + data=account_update_raw['data'], + ) + + +def run( + input_topic: str, + input_subscription: str, + output_table_namespace: str, + cluster: Cluster, + min_idl_version: int, + beam_args: Optional[List[str]] = None, +) -> None: + if beam_args is None: + beam_args = [] + + idl_pool = IdlPool(cluster) + + def parse_account_internal(tx: Any) -> List[AccountUpdateRecord]: + try: + return parse_account(tx, min_idl_version, cluster, idl_pool) + except OwnerProgramNotSupported: + return [] + + """Build and run the pipeline.""" + pipeline_options = PipelineOptions(beam_args, save_main_session=True, streaming=True) + + with beam.Pipeline(options=pipeline_options) as pipeline: + # Define steps + read_raw_txs = beam.io.ReadFromPubSub( + topic=input_topic, subscription=input_subscription + ).with_output_types(bytes) + + parse_to_raw_txs = beam.Map(parse_json) + + extract_events = beam.FlatMap(parse_account_internal) + + dispatch_events = beam.ParDo(DispatchEventsDoFn()).with_outputs( + *[rt.get_tag() for rt in AccountUpdateRecordTypes]) + + dictionify_events = beam.Map(dictionify_record) + + writers: Dict[str, Union[beam.io.WriteToText, beam.io.WriteToBigQuery]] = {} + for rt in AccountUpdateRecordTypes: + if output_table_namespace == "local_file": # For testing purposes + writers[rt.get_tag()] = beam.io.WriteToText(f"account_updates_{rt.get_tag(snake_case=True)}") + else: + writers[rt.get_tag()] = beam.io.WriteToBigQuery( + f"{output_table_namespace}_{rt.get_tag(snake_case=True)}", + schema=rt.SCHEMA, + write_disposition=beam.io.BigQueryDisposition.WRITE_APPEND, + create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED, + ) + + # Define pipeline + tagged_events = ( + pipeline + | "ReadRawTxs" >> read_raw_txs + | "ParseTxsToRawTxs" >> parse_to_raw_txs + | "ExtractEvents" >> extract_events + | "DispatchEvents" >> dispatch_events + ) + + for rt in AccountUpdateRecordTypes: + (tagged_events[rt.get_tag()] + | f"Dictionify{rt.get_tag()}" >> dictionify_events + | f"Write{rt.get_tag()}" >> writers[rt.get_tag()] + ) + + +def main() -> None: + logging.getLogger().setLevel(logging.INFO) + + parser = argparse.ArgumentParser() + parser.add_argument( + "--input_topic", + type=str, + help='Input PubSub topic of the form "projects//topics/."', + ) + parser.add_argument( + "--input_subscription", + type=str, + help='Input PubSub subscription of the form "projects//subscriptions/."', + ) + parser.add_argument( + "--output_table_namespace", + type=str, + required=True, + help="Output BigQuery namespace where parsed account tables are located: PROJECT:DATASET.TABLE", + ) + parser.add_argument( + "--cluster", + type=str, + required=False, + default="mainnet", + help="Solana cluster being indexed: mainnet | devnet", + ) + parser.add_argument( + "--min_idl_version", + type=int, + required=False, + default=0, + help="Minimum IDL version to consider: int", + ) + known_args, remaining_args = parser.parse_known_args() + + run( + input_topic=known_args.input_topic, + input_subscription=known_args.input_subscription, + output_table_namespace=known_args.output_table_namespace, + cluster=known_args.cluster, + min_idl_version=known_args.min_idl_version, + beam_args=remaining_args, + ) + + +if __name__ == "__main__": + main() diff --git a/observability/etl/dataflow-etls/jobs/marginfi-v2-account-parsing-stream/metadata.json b/observability/etl/dataflow-etls/jobs/marginfi-v2-account-parsing-stream/metadata.json new file mode 100644 index 00000000..ed2ce4bc --- /dev/null +++ b/observability/etl/dataflow-etls/jobs/marginfi-v2-account-parsing-stream/metadata.json @@ -0,0 +1,46 @@ +{ + "name": "marginfi-v2-account-parsing-stream", + "description": "Parses individual account updates from a Pub/Sub topic/subscription and stores them in dedicated BigQuery tables.", + "parameters": [ + { + "name": "input_topic", + "label": "Input PubSub topic.", + "isOptional": true, + "helpText": "Name of the input PubSub topic to consume from.", + "regexes": [ + "projects/[^/]+/topics/[a-zA-Z][-_.~+%a-zA-Z0-9]{2,}" + ] + }, + { + "name": "input_subscription", + "label": "Input PubSub subscription.", + "isOptional": true, + "helpText": "Name of the input PubSub subscription to consume from.", + "regexes": [ + "projects/[^/]+/subscriptions/[a-zA-Z][-_.~+%a-zA-Z0-9]{2,}" + ] + }, + { + "name": "output_table_namespace", + "label": "BigQuery output tables namespace.", + "helpText": "Namespace where the BigQuery output tables are located.", + "regexes": [ + "([^:]+:)?[^.]+[.].+" + ] + }, + { + "name": "cluster", + "label": "Solana cluster.", + "isOptional": true, + "helpText": "Cluster where the account updates are executed (used to pull IDL version depending on tx slot).", + "regexes": ["mainnet|devnet"] + }, + { + "name": "min_idl_version", + "label": "Minimum IDL version.", + "isOptional": true, + "helpText": "Minimum IDL version for which txs will be parsed. Default: 0", + "regexes": [] + } + ] +} diff --git a/observability/etl/dataflow-etls/jobs/marginfi-v2-event-parsing-batch/job.py b/observability/etl/dataflow-etls/jobs/marginfi-v2-event-parsing-batch/job.py new file mode 100644 index 00000000..4c24d8ee --- /dev/null +++ b/observability/etl/dataflow-etls/jobs/marginfi-v2-event-parsing-batch/job.py @@ -0,0 +1,162 @@ +import argparse +import logging +from typing import List, Optional, Union, Any, Dict +import apache_beam as beam # type: ignore +from apache_beam.options.pipeline_options import PipelineOptions # type: ignore + +from dataflow_etls.orm.events import EventRecordTypes, EventRecord +from dataflow_etls.idl_versions import Cluster, IdlPool +from dataflow_etls.transaction_parsing import dictionify_record, DispatchEventsDoFn, extract_events_from_tx, \ + IndexedProgramNotSupported + + +def run( + input_table: str, + output_table_namespace: str, + cluster: Cluster, + min_idl_version: int, + start_date: Optional[str] = None, + end_date: Optional[str] = None, + start_timestamp: Optional[str] = None, + end_timestamp: Optional[str] = None, + beam_args: Optional[List[str]] = None, +) -> None: + if beam_args is None: + beam_args = [] + + idl_pool = IdlPool(cluster) + + def extract_events_from_tx_internal(tx: Any) -> List[EventRecord]: + try: + return extract_events_from_tx(tx, min_idl_version, cluster, idl_pool) + except IndexedProgramNotSupported: + return [] + + """Build and run the pipeline.""" + pipeline_options = PipelineOptions(beam_args, save_main_session=True) + + if start_date is not None and end_date is not None: + input_query = f'SELECT * FROM `{input_table}` WHERE DATE(timestamp) >= "{start_date}" AND DATE(timestamp) < "{end_date}"' + elif start_timestamp is not None and end_timestamp is not None: + input_query = f'SELECT * FROM `{input_table}` WHERE timestamp >= "{start_timestamp}" AND timestamp < "{end_timestamp}"' + elif start_date is not None: + input_query = ( + f'SELECT * FROM `{input_table}` WHERE DATE(timestamp) >= "{start_date}"' + ) + elif end_date is not None: + input_query = ( + f'SELECT * FROM `{input_table}` WHERE DATE(timestamp) < "{end_date}"' + ) + elif end_timestamp is not None: + input_query = ( + f'SELECT * FROM `{input_table}` WHERE timestamp < "{end_timestamp}"' + ) + print("yes", input_query) + else: + input_query = f"SELECT * FROM `{input_table}`" + + with beam.Pipeline(options=pipeline_options) as pipeline: + # Define steps + read_raw_txs = beam.io.ReadFromBigQuery(query=input_query, use_standard_sql=True) + + extract_events = beam.FlatMap(extract_events_from_tx_internal) + + dispatch_events = beam.ParDo(DispatchEventsDoFn()).with_outputs(*[rt.get_tag() for rt in EventRecordTypes]) + + dictionify_events = beam.Map(dictionify_record) + + writers: Dict[str, Union[beam.io.WriteToText, beam.io.WriteToBigQuery]] = {} + for rt in EventRecordTypes: + if output_table_namespace == "local_file": # For testing purposes + writers[rt.get_tag()] = beam.io.WriteToText(f"events_{rt.get_tag(snake_case=True)}") + else: + writers[rt.get_tag()] = beam.io.WriteToBigQuery( + f"{output_table_namespace}_{rt.get_tag(snake_case=True)}", + schema=rt.SCHEMA, + write_disposition=beam.io.BigQueryDisposition.WRITE_APPEND, + create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED, + ) + + # Define pipeline + tagged_events = ( + pipeline + | "ReadRawTxs" >> read_raw_txs + | "ExtractEvents" >> extract_events + | "DispatchEvents" >> dispatch_events + ) + + for rt in EventRecordTypes: + (tagged_events[rt.get_tag()] + | f"Dictionify{rt.get_tag()}" >> dictionify_events + | f"Write{rt.get_tag()}" >> writers[rt.get_tag()] + ) + + +def main() -> None: + logging.getLogger().setLevel(logging.INFO) + + parser = argparse.ArgumentParser() + parser.add_argument( + "--input_table", + type=str, + required=True, + help="Input BigQuery table specified as: " + "PROJECT.DATASET.TABLE.", + ) + parser.add_argument( + "--output_table_namespace", + type=str, + required=True, + help="Output BigQuery namespace where event tables are located: PROJECT:DATASET.TABLE", + ) + parser.add_argument( + "--cluster", + type=str, + required=False, + default="mainnet", + help="Solana cluster being indexed: mainnet | devnet", + ) + parser.add_argument( + "--min_idl_version", + type=int, + required=False, + default=0, + help="Minimum IDL version to consider: int", + ) + parser.add_argument( + "--start_date", + type=str, + help="Start date to consider (inclusive) as: YYYY-MM-DD", + ) + parser.add_argument( + "--end_date", + type=str, + help="End date to consider (exclusive) as: YYYY-MM-DD", + ) + parser.add_argument( + "--start_timestamp", + type=str, + help="Start timestamp to consider (inclusive) as: YYYY-MM-DD HH:MM:SS", + ) + parser.add_argument( + "--end_timestamp", + type=str, + help="End timestamp to consider (exclusive) as: YYYY-MM-DD HH:MM:SS", + ) + known_args, remaining_args = parser.parse_known_args() + + run( + input_table=known_args.input_table, + output_table_namespace=known_args.output_table_namespace, + cluster=known_args.cluster, + min_idl_version=known_args.min_idl_version, + start_date=known_args.start_date, + end_date=known_args.end_date, + start_timestamp=known_args.start_timestamp, + end_timestamp=known_args.end_timestamp, + beam_args=remaining_args, + ) + + +if __name__ == "__main__": + main() diff --git a/observability/etl/dataflow-etls/metadata.json b/observability/etl/dataflow-etls/jobs/marginfi-v2-event-parsing-batch/metadata.json similarity index 92% rename from observability/etl/dataflow-etls/metadata.json rename to observability/etl/dataflow-etls/jobs/marginfi-v2-event-parsing-batch/metadata.json index ba489e0f..4bac544d 100644 --- a/observability/etl/dataflow-etls/metadata.json +++ b/observability/etl/dataflow-etls/jobs/marginfi-v2-event-parsing-batch/metadata.json @@ -1,6 +1,6 @@ { - "name": "event-parsing-batch", - "description": "Parses individual raw transactions from a BigQuery table and stores them in another BigQuery table.", + "name": "marginfi-v2-event-parsing-batch", + "description": "Parses individual raw transactions from a BigQuery table and stores them in dedicated BigQuery tables.", "parameters": [ { "name": "input_table", diff --git a/observability/etl/dataflow-etls/jobs/marginfi-v2-event-parsing-stream/job.py b/observability/etl/dataflow-etls/jobs/marginfi-v2-event-parsing-stream/job.py new file mode 100644 index 00000000..afbbec20 --- /dev/null +++ b/observability/etl/dataflow-etls/jobs/marginfi-v2-event-parsing-stream/job.py @@ -0,0 +1,156 @@ +import argparse +from dateutil import parser +import json +import logging +from typing import List, Optional, Union, Any, Dict +from decimal import Decimal + +import apache_beam as beam # type: ignore +from apache_beam.options.pipeline_options import PipelineOptions # type: ignore + +from dataflow_etls.orm.events import EventRecord, EventRecordTypes +from dataflow_etls.idl_versions import Cluster, IdlPool +from dataflow_etls.transaction_parsing import extract_events_from_tx, dictionify_record, DispatchEventsDoFn, \ + TransactionRaw, IndexedProgramNotSupported + + +def parse_json(message: bytes) -> TransactionRaw: + tx_raw = json.loads(message.decode("utf-8")) + return TransactionRaw( + id=tx_raw["id"], + created_at=parser.parse(tx_raw['created_at']), + timestamp=parser.parse(tx_raw['timestamp']), + signature=tx_raw['signature'], + indexing_address=tx_raw['indexing_address'], + slot=Decimal(tx_raw['slot']), + signer=tx_raw['signer'], + success=bool(tx_raw['success']), + version=tx_raw['version'], + fee=Decimal(tx_raw['fee']), + meta=tx_raw['meta'], + message=tx_raw['message'], + ) + + +def run( + input_topic: str, + input_subscription: str, + output_table_namespace: str, + cluster: Cluster, + min_idl_version: int, + # window_interval_sec: int = 60, + beam_args: Optional[List[str]] = None, +) -> None: + if beam_args is None: + beam_args = [] + + idl_pool = IdlPool(cluster) + + def extract_events_from_tx_internal(tx: Any) -> List[EventRecord]: + try: + return extract_events_from_tx(tx, min_idl_version, cluster, idl_pool) + except IndexedProgramNotSupported: + return [] + + """Build and run the pipeline.""" + pipeline_options = PipelineOptions(beam_args, save_main_session=True, streaming=True) + + with beam.Pipeline(options=pipeline_options) as pipeline: + # Define steps + read_raw_txs = beam.io.ReadFromPubSub( + topic=input_topic, subscription=input_subscription + ).with_output_types(bytes) + + parse_to_raw_txs = beam.Map(parse_json) + + # group_in_windows = beam.WindowInto(window.FixedWindows(window_interval_sec, 0)) + + extract_events = beam.FlatMap(extract_events_from_tx_internal) + + dispatch_events = beam.ParDo(DispatchEventsDoFn()).with_outputs(*[rt.get_tag() for rt in EventRecordTypes]) + + dictionify_events = beam.Map(dictionify_record) + + writers: Dict[str, Union[beam.io.WriteToText, beam.io.WriteToBigQuery]] = {} + for rt in EventRecordTypes: + if output_table_namespace == "local_file": # For testing purposes + writers[rt.get_tag()] = beam.io.WriteToText(f"events_{rt.get_tag(snake_case=True)}") + else: + writers[rt.get_tag()] = beam.io.WriteToBigQuery( + f"{output_table_namespace}_{rt.get_tag(snake_case=True)}", + schema=rt.SCHEMA, + write_disposition=beam.io.BigQueryDisposition.WRITE_APPEND, + create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED, + ) + + # Define pipeline + tagged_events = ( + pipeline + | "ReadRawTxs" >> read_raw_txs + | "ParseTxsToRawTxs" >> parse_to_raw_txs + | "ExtractEvents" >> extract_events + | "DispatchEvents" >> dispatch_events + ) + + for rt in EventRecordTypes: + (tagged_events[rt.get_tag()] + | f"Dictionify{rt.get_tag()}" >> dictionify_events + | f"Write{rt.get_tag()}" >> writers[rt.get_tag()] + ) + + +def main() -> None: + logging.getLogger().setLevel(logging.INFO) + + parser = argparse.ArgumentParser() + parser.add_argument( + "--input_topic", + type=str, + help='Input PubSub topic of the form "projects//topics/."', + ) + parser.add_argument( + "--input_subscription", + type=str, + help='Input PubSub subscription of the form "projects//subscriptions/."', + ) + parser.add_argument( + "--output_table_namespace", + type=str, + required=True, + help="Output BigQuery namespace where event tables are located: PROJECT:DATASET.TABLE", + ) + parser.add_argument( + "--window_interval_sec", + default=60, + type=int, + help="Window interval in seconds for grouping incoming messages.", + ) + parser.add_argument( + "--cluster", + type=str, + required=False, + default="mainnet", + help="Solana cluster being indexed: mainnet | devnet", + ) + parser.add_argument( + "--min_idl_version", + type=int, + required=False, + default=0, + help="Minimum IDL version to consider: int", + ) + known_args, remaining_args = parser.parse_known_args() + + run( + input_topic=known_args.input_topic, + input_subscription=known_args.input_subscription, + output_table_namespace=known_args.output_table_namespace, + # window_interval_sec=known_args.window_interval_sec, + cluster=known_args.cluster, + min_idl_version=known_args.min_idl_version, + beam_args=remaining_args, + ) + + +if __name__ == "__main__": + main() diff --git a/observability/etl/dataflow-etls/jobs/marginfi-v2-event-parsing-stream/metadata.json b/observability/etl/dataflow-etls/jobs/marginfi-v2-event-parsing-stream/metadata.json new file mode 100644 index 00000000..252e77a6 --- /dev/null +++ b/observability/etl/dataflow-etls/jobs/marginfi-v2-event-parsing-stream/metadata.json @@ -0,0 +1,46 @@ +{ + "name": "marginfi-v2-event-parsing-stream", + "description": "Parses individual raw transactions from a Pub/Sub topic/subscription and stores them in dedicated BigQuery tables.", + "parameters": [ + { + "name": "input_topic", + "label": "Input PubSub topic.", + "isOptional": true, + "helpText": "Name of the input PubSub topic to consume from.", + "regexes": [ + "projects/[^/]+/topics/[a-zA-Z][-_.~+%a-zA-Z0-9]{2,}" + ] + }, + { + "name": "input_subscription", + "label": "Input PubSub subscription.", + "isOptional": true, + "helpText": "Name of the input PubSub subscription to consume from.", + "regexes": [ + "projects/[^/]+/subscriptions/[a-zA-Z][-_.~+%a-zA-Z0-9]{2,}" + ] + }, + { + "name": "output_table_namespace", + "label": "BigQuery output tables namespace.", + "helpText": "Namespace where the BigQuery output tables are located.", + "regexes": [ + "([^:]+:)?[^.]+[.].+" + ] + }, + { + "name": "cluster", + "label": "Solana cluster.", + "isOptional": true, + "helpText": "Cluster where the processed txs are executed (used to pull IDL version depending on tx slot).", + "regexes": ["mainnet|devnet"] + }, + { + "name": "min_idl_version", + "label": "Minimum IDL version.", + "isOptional": true, + "helpText": "Minimum IDL version for which txs will be parsed. Default: 0", + "regexes": [] + } + ] +} diff --git a/observability/etl/dataflow-etls/jobs/marginfi-v2-event-parsing-stream/update_running_pipeline b/observability/etl/dataflow-etls/jobs/marginfi-v2-event-parsing-stream/update_running_pipeline new file mode 100644 index 00000000..3aecd96a --- /dev/null +++ b/observability/etl/dataflow-etls/jobs/marginfi-v2-event-parsing-stream/update_running_pipeline @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -e + +job_id=$1 + +[ -z "$job_id" ] && echo "Missing job_id argument" && exit 1 + +gcloud dataflow flex-template run "$job_id" \ + --template-file-gcs-location gs://dataflow_jobs_marginfi_v2/templates/marginfi-v2-event-parsing-stream.json \ + --region us-central1 \ + --parameters input_topic=projects/marginfi-dev/topics/marginfi-v2-mainnet-transactions-raw,output_table_namespace=marginfi-dev:marginfi_v2.event,cluster=mainnet \ + --update \ No newline at end of file diff --git a/observability/etl/dataflow-etls/pyproject.toml b/observability/etl/dataflow-etls/pyproject.toml index dc2fcb44..a6bfee77 100644 --- a/observability/etl/dataflow-etls/pyproject.toml +++ b/observability/etl/dataflow-etls/pyproject.toml @@ -1,5 +1,5 @@ [tool.poetry] -name = "dataflow-etls" +name = "dataflow_etls" version = "0.1.0" description = "" authors = ["man0s <95379755+losman0s@users.noreply.github.com>"] @@ -8,11 +8,11 @@ authors = ["man0s <95379755+losman0s@users.noreply.github.com>"] job = "dataflow_etls.job:main" [tool.poetry.dependencies] -python = "^3.10" +python = "^3.9" wheel = "^0.38.4" apache-beam = { extras = ["gcp"], version = "2.44.0" } solders = "^0.14.0" -anchorpy = { git = "https://github.com/mrgnlabs/anchorpy", branch = "bump-zstandard"} +anchorpy = "^0.16.0" based58 = "^0.1.1" isort = "^5.12.0" diff --git a/observability/etl/dataflow-etls/requirements.txt b/observability/etl/dataflow-etls/requirements.txt deleted file mode 100644 index e69de29b..00000000 diff --git a/observability/etl/dataflow-etls/scripts/build_job_template b/observability/etl/dataflow-etls/scripts/build_job_template new file mode 100755 index 00000000..f4dd9e90 --- /dev/null +++ b/observability/etl/dataflow-etls/scripts/build_job_template @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -e + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +ROOT_DIR=$(realpath "${SCRIPT_DIR}/../") + +job_dir=$1 +image_version=$2 + +[ -z "$job_dir" ] && echo "Missing job_dir argument" && exit 1 +[ -z "$image_version" ] && echo "Missing image_version argument" && exit 1 + + +job_dir=$(realpath "$job_dir") +job_name=$(basename "$job_dir") +job_dir_rel=$(realpath --relative-to="$ROOT_DIR" "$job_dir") + +cd "$ROOT_DIR" + +local_image_name="$job_name" +gcp_image_name="us-east1-docker.pkg.dev/marginfi-dev/main/dataflow/$local_image_name" + +docker build --platform=linux/amd64 --build-arg JOB_DIRECTORY="$job_dir_rel" --tag "$job_name" "$ROOT_DIR" +docker tag "$local_image_name" "$gcp_image_name:$image_version" diff --git a/observability/etl/dataflow-etls/scripts/create_events.sh b/observability/etl/dataflow-etls/scripts/create_events.sh deleted file mode 100755 index 9f468801..00000000 --- a/observability/etl/dataflow-etls/scripts/create_events.sh +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env bash -set -e - -GROUP=567YJkNDCgG6Q6qDwcNK7p8frSL1HRE5J2tqUEXCXR7v -PROGRAM_ID=A7vUDErNPCTt9qrB6SSM4F6GkxzUe9d8P3cXSmRg4eY4 -NEW_PROFILE_NAME=indexing - -RPC_ENDPOINT=https://devnet.genesysgo.net/ -KEYPAIR_PATH=~/.config/solana/id.json - -[ -z "$PROGRAM_ID" ] && echo "PROGRAM_ID must be specified. Exiting." && exit 1 -[ -z "$NEW_PROFILE_NAME" ] && echo "NEW_PROFILE_NAME must be specified. Exiting." && exit 1 - -#mfi profile create \ -# --cluster devnet \ -# --name "$NEW_PROFILE_NAME" \ -# --keypair-path "$KEYPAIR_PATH" \ -# --rpc-url "$RPC_ENDPOINT" \ -# --program-id "$PROGRAM_ID" -# -#mfi profile set "$NEW_PROFILE_NAME" -# -#mfi group create "$@" -y - -# Add USDC bank -mfi group add-bank \ - --mint F9jRT1xL7PCRepBuey5cQG5vWHFSbnvdWxJWKqtzMDsd \ - --asset-weight-init 0.85 \ - --asset-weight-maint 0.9 \ - --liability-weight-maint 1.1 \ - --liability-weight-init 1.15 \ - --deposit-limit 1000000000000000\ - --borrow-limit 1000000000000000\ - --pyth-oracle 5SSkXsEKQepHHAewytPVwdej4epN1nxgLVM84L4KXgy7 \ - --optimal-utilization-rate 0.9 \ - --plateau-interest-rate 1 \ - --max-interest-rate 10 \ - --insurance-fee-fixed-apr 0.01 \ - --insurance-ir-fee 0.1 \ - --protocol-fixed-fee-apr 0.01 \ - --protocol-ir-fee 0.1 \ - -y \ - "$@" - -## Add SOL bank -#mfi group add-bank \ -# --mint 4Bn9Wn1sgaD5KfMRZjxwKFcrUy6NKdyqLPtzddazYc4x \ -# --asset-weight-init 0.75 \ -# --asset-weight-maint 0.8 \ -# --liability-weight-maint 1.2 \ -# --liability-weight-init 1.25 \ -# --deposit-limit 1000000000000000\ -# --borrow-limit 1000000000000000\ -# --pyth-oracle J83w4HKfqxwcq3BEMMkPFSppX3gqekLyLJBexebFVkix \ -# --optimal-utilization-rate 0.8 \ -# --plateau-interest-rate 1 \ -# --max-interest-rate 20 \ -# --insurance-fee-fixed-apr 0.01 \ -# --insurance-ir-fee 0.1 \ -# --protocol-fixed-fee-apr 0.01 \ -# --protocol-ir-fee 0.1 \ -# -y \ -# "$@" - -SOL_BANK=5dCnHjFUTjuWq2b9hY2J1o94QR1YpiggUC2XKzA1UpYp -USDC_BANK=7A1zpp3Fb7eQmAKy6r3jZnSHhZK9SJEBuUrUqwwMQQfY - -echo "-> Admin configures SOL bank" -mfi bank update "$SOL_BANK" \ - --asset-weight-init 1 \ - --asset-weight-maint 1 \ - --skip-confirmation - -echo "-> Admin configures USDC bank" -mfi bank update "$USDC_BANK" \ - --asset-weight-init 1 \ - --asset-weight-maint 1 \ - --skip-confirmation - -mfi account use 6TZKuwLn8C6cVCxhGhmLtFUTZo4qGPF2a7Qb9F4QJFs1 --skip-confirmation - -echo "-> Random user lends USDC" -mfi account deposit "$USDC_BANK" 0.01 --skip-confirmation - -echo "-> Liquidatee creates new mfi account" -liquidatee_account=$(mfi account create --skip-confirmation) -echo "liquidatee account: $liquidatee_account" - -echo "-> Liquidatee deposits SOL" -mfi account deposit "$SOL_BANK" 0.01 --skip-confirmation - -echo "-> Liquidatee borrows USDC" -mfi account borrow "$USDC_BANK" 0.01 --skip-confirmation - -echo "-> Admin triggers bad health by setting SOL asset weights to 0" -mfi bank update "$SOL_BANK" \ - --asset-weight-init 0 \ - --asset-weight-maint 0 \ - --skip-confirmation - -echo "-> Liquidator creates mfi account" -mfi account create --skip-confirmation - -echo "-> Liquidator deposits USDC to pay off liquidatee's debt" -mfi account deposit "$USDC_BANK" 0.01 --skip-confirmation - -echo "-> Liquidator liquidates liquidatee for half its assets" -mfi account liquidate \ - --liquidatee-marginfi-account="$liquidatee_account" \ - --asset-bank="$SOL_BANK" \ - --liability-bank="$USDC_BANK" \ - --ui-asset-amount=0.0001 \ - --skip-confirmation - -echo "-> Admin handles remainder of bad debt through handle bankruptcy" -mfi group handle-bankruptcy \ - --bank="$USDC_BANK" \ - --marginfi-account="$liquidatee_account" \ - --skip-confirmation diff --git a/observability/etl/dataflow-etls/scripts/generate_sample_events.sh b/observability/etl/dataflow-etls/scripts/generate_sample_events.sh new file mode 100755 index 00000000..d56e201b --- /dev/null +++ b/observability/etl/dataflow-etls/scripts/generate_sample_events.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +set -e + +PROGRAM_ID=5Lt5xXZG7bteZferQk9bsiiAS75JqGVPYcTbB8J6vvJK + +random_id=$(echo $RANDOM | md5sum | head -c 20; echo) +test_profile="test-devnet-$random_id" + +RPC_ENDPOINT=https://devnet.rpcpool.com/ +KEYPAIR_PATH=~/.config/solana/id.json + +echo "-> Creating test profile $test_profile" +mfi profile create \ + --cluster devnet \ + --name "$test_profile" \ + --keypair-path "$KEYPAIR_PATH" \ + --rpc-url "$RPC_ENDPOINT" \ + --program-id "$PROGRAM_ID" + +mfi profile set "$test_profile" + +echo "-> Admin creates marginfi group" +mfi group create -y + +echo "-> Admin updates marginfi group" +mfi group update -y + +echo "-> Admin creates USDC bank" +usdc_bank=$(mfi group add-bank \ + --mint F9jRT1xL7PCRepBuey5cQG5vWHFSbnvdWxJWKqtzMDsd \ + --asset-weight-init 0.85 \ + --asset-weight-maint 0.9 \ + --liability-weight-maint 1.1 \ + --liability-weight-init 1.15 \ + --deposit-limit 1000000000000000\ + --borrow-limit 1000000000000000\ + --pyth-oracle 5SSkXsEKQepHHAewytPVwdej4epN1nxgLVM84L4KXgy7 \ + --optimal-utilization-rate 0.9 \ + --plateau-interest-rate 1 \ + --max-interest-rate 10 \ + --insurance-fee-fixed-apr 0.01 \ + --insurance-ir-fee 0.1 \ + --protocol-fixed-fee-apr 0.01 \ + --protocol-ir-fee 0.1 \ + --risk-tier collateral \ + -y) +echo "USDC bank: $usdc_bank" + +echo "-> Admin creates SOL bank" +sol_bank=$(mfi group add-bank \ + --mint 4Bn9Wn1sgaD5KfMRZjxwKFcrUy6NKdyqLPtzddazYc4x \ + --asset-weight-init 0.75 \ + --asset-weight-maint 0.8 \ + --liability-weight-maint 1.2 \ + --liability-weight-init 1.25 \ + --deposit-limit 1000000000000000\ + --borrow-limit 1000000000000000\ + --pyth-oracle J83w4HKfqxwcq3BEMMkPFSppX3gqekLyLJBexebFVkix \ + --optimal-utilization-rate 0.8 \ + --plateau-interest-rate 1 \ + --max-interest-rate 20 \ + --insurance-fee-fixed-apr 0.01 \ + --insurance-ir-fee 0.1 \ + --protocol-fixed-fee-apr 0.01 \ + --protocol-ir-fee 0.1 \ + --risk-tier collateral \ + -y) +echo "SOL bank: $sol_bank" + +echo "-> Admin configures SOL bank" +mfi bank update "$sol_bank" \ + --asset-weight-init 1 \ + --asset-weight-maint 1 \ + -y + +echo "-> Admin configures USDC bank" +mfi bank update "$usdc_bank" \ + --asset-weight-init 1 \ + --asset-weight-maint 1 \ + -y + +mfi account create -y + +echo "-> Random user lends USDC" +mfi account deposit "$usdc_bank" 0.01 -y + +echo "-> Liquidatee creates new mfi account" +liquidatee_account=$(mfi account create -y) +echo "liquidatee account: $liquidatee_account" + +echo "-> Liquidatee deposits SOL" +mfi account deposit "$sol_bank" 0.01 -y + +echo "-> Liquidatee borrows USDC" +mfi account borrow "$usdc_bank" 0.01 -y + +echo "-> Admin triggers bad health by setting SOL asset weights to 0" +mfi bank update "$sol_bank" \ + --asset-weight-init 0 \ + --asset-weight-maint 0 \ + -y + +echo "-> Liquidator creates mfi account" +mfi account create -y + +echo "-> Liquidator deposits USDC to pay off liquidatee's debt" +mfi account deposit "$usdc_bank" 0.01 -y + +echo "-> Liquidator liquidates liquidatee for half its assets" +mfi account liquidate \ + --liquidatee-marginfi-account="$liquidatee_account" \ + --asset-bank="$sol_bank" \ + --liability-bank="$usdc_bank" \ + --ui-asset-amount=0.0001 \ + -y + +echo "-> Admin handles remainder of bad debt through handle bankruptcy" +mfi bank handle-bankruptcy \ + --bank="$usdc_bank" \ + --marginfi-account="$liquidatee_account" \ + -y + +echo "-> Admin collects fees on USDC bank" +mfi bank collect-fees \ + --bank="$usdc_bank" \ + -y + +echo "-> Admin collects fees on SOL bank" +mfi bank collect-fees \ + --bank="$sol_bank" \ + -y diff --git a/observability/etl/dataflow-etls/scripts/playground.py b/observability/etl/dataflow-etls/scripts/playground.py index f25a354c..35c1fab0 100644 --- a/observability/etl/dataflow-etls/scripts/playground.py +++ b/observability/etl/dataflow-etls/scripts/playground.py @@ -1,6 +1,5 @@ from pathlib import Path from pprint import pprint - import based58 from anchorpy import Program from anchorpy_core.idl import Idl @@ -8,6 +7,18 @@ from solders.pubkey import Pubkey from solders.signature import Signature +from decimal import Decimal + + +def wrapped_i80f48_to_float(wrapped_i80f48_value: int): + nb_of_fractional_bits = 48 + value = Decimal(wrapped_i80f48_value) + value = value / 2 ** nb_of_fractional_bits + return value + + +print(wrapped_i80f48_to_float(-54953626681867088)) + sample_logs = ["Program ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL invoke [1]", "Program log: CreateIdempotent", "Program ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL consumed 7338 of 400000 compute units", "Program ATokenGPvbdGVxr1b2hvZbsiqW5xWH25efTNsLJA8knL success", @@ -31,7 +42,9 @@ sample_message = "gAEABgwF3hNv9VErtBxgns6TItDZs+IZ/Y6TNWGuTe16nnh4yTOAiZ9axygRNRuzZCiQK4q5zvPfaH0CnR4tEUdeu2zY7gIfr9tYuu2+5AyJt/JuaSZFKoWEfKCH3YSULQapSVzpPk3uIRtapASqm9ALTdv++zxMXXSinbwKl99MTnuDkd4jxKWJrU3oI9SPceEA9wtTBJ5T78IpGTiq1XRzIrqD/r/JWnhhuWaJpdf6l+yCA38eN/l9xqMSjBXsYMAOea2MlyWPTiSJ8bs9ECkUjg2DC1oTmdr/EIQEjnvY2+n4WdJAOsgK1DjHp0u5LCmsGj4g7ioWbEEtiXn/B9aQ3U+4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAG3fbh12Whk9nL4UbO63msHLSF7V9bN5E6jPWFfv8AqYd/H/1F6tUC+2l0WY9HS4WATM+2LA2i9Tq46cjeMS2lO8hKNv2sSk0p0XfYfozLqjspIR2sHsbx0eDvsV5tGY4/gcRh2keI2GQribtQSPXEZxYR1wZZ93XFhp/+f3fIFAIGBgABAAcICQEBCggLAgADAQQFCRIkSEoT0tLAwEBCDwAAAAAAAQEA" http_client = Client("https://api.devnet.solana.com") -tx = http_client.get_transaction(Signature.from_string("4WdMP6s8SasqWrWpscurrN1r8ueLZNSdq81nLRJfyhsDU7xMz47t8xYh4hTvTy2C2sZ8Frc3x8urjfkYwKTg7dNT"), max_supported_transaction_version=0) +tx = http_client.get_transaction( + Signature.from_string("4WdMP6s8SasqWrWpscurrN1r8ueLZNSdq81nLRJfyhsDU7xMz47t8xYh4hTvTy2C2sZ8Frc3x8urjfkYwKTg7dNT"), + max_supported_transaction_version=0) message = tx.value.transaction.transaction.message pprint(message) diff --git a/observability/etl/dataflow-etls/scripts/sync_job_template b/observability/etl/dataflow-etls/scripts/sync_job_template new file mode 100755 index 00000000..2cd6126c --- /dev/null +++ b/observability/etl/dataflow-etls/scripts/sync_job_template @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -e + +job_dir=$1 +image_version=$2 + +[ -z "$job_dir" ] && echo "Missing job_dir argument" && exit 1 +[ -z "$image_version" ] && echo "Missing image_version argument" && exit 1 + +job_dir=$(realpath "$job_dir") +job_name=$(basename "$job_dir") + +local_image_name="$job_name" +gcp_image_name="us-east1-docker.pkg.dev/marginfi-dev/main/dataflow/$local_image_name" +gcp_template_gs_path="gs://dataflow_jobs_marginfi_v2/templates/$job_name.json" +metadata_local_path="$job_dir/metadata.json" + +gcloud dataflow flex-template build \ + "$gcp_template_gs_path" \ + --image "$gcp_image_name:$image_version" \ + --sdk-language "PYTHON" \ + --metadata-file "$metadata_local_path" diff --git a/observability/etl/dataflow-etls/scripts/upload_job_template b/observability/etl/dataflow-etls/scripts/upload_job_template new file mode 100755 index 00000000..5475179c --- /dev/null +++ b/observability/etl/dataflow-etls/scripts/upload_job_template @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -e + +job_dir=$1 +image_version=$2 + +[ -z "$job_dir" ] && echo "Missing job_dir argument" && exit 1 +[ -z "$image_version" ] && echo "Missing image_version argument" && exit 1 + +job_dir=$(realpath "$job_dir") +job_name=$(basename "$job_dir") + +local_image_name="$job_name" +gcp_image_name="us-east1-docker.pkg.dev/marginfi-dev/main/dataflow/$local_image_name" + +docker push "$gcp_image_name:$image_version" diff --git a/observability/etl/dataflow-etls/setup.py b/observability/etl/dataflow-etls/setup.py new file mode 100644 index 00000000..5b626d13 --- /dev/null +++ b/observability/etl/dataflow-etls/setup.py @@ -0,0 +1,14 @@ +import setuptools + +setuptools.setup( + name='dataflow_etls', + version='0.1.0', + install_requires=[ + 'anchorpy==0.16.0', + 'apache-beam[gcp]==2.45.0', + "based58==0.1.1", + "solders==0.14.4 ", + ], + packages=setuptools.find_packages(), + include_package_data=True +) diff --git a/observability/indexer/.dockerignore b/observability/indexer/.dockerignore new file mode 100644 index 00000000..82c1af3b --- /dev/null +++ b/observability/indexer/.dockerignore @@ -0,0 +1,6 @@ +# Ignore everything except for Cargo and src files +* +!Cargo.toml +!build.rs +!src/ +!protos/ diff --git a/observability/indexer/CHANGELOG.md b/observability/indexer/CHANGELOG.md new file mode 100644 index 00000000..701c93e0 --- /dev/null +++ b/observability/indexer/CHANGELOG.md @@ -0,0 +1,35 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +## [Unreleased] + +## [0.1.4] - 2023-03-10 + +### Added + +- Interest rates and fees to bank metrics. +- Positions summary to marginfi account metrics. + +## [0.1.3] - 2023-03-08 + +### Fixed + +- Wrong unit for bank deposit/withdraw limit units. + +## [0.1.2] - 2023-03-07 + +### Added + +- Accounts snapshot command. + +### Changed + +- Make crate part of top-level project. +- Switch to custom fork of google-cloud-rust for `tokio` compatibility with solana crates. + +## [0.1.1] - Skipped + +## [0.1.0] - 2023-03-01 + +- Initial release \ No newline at end of file diff --git a/observability/indexer/Cargo.lock b/observability/indexer/Cargo.lock deleted file mode 100644 index e8d56708..00000000 --- a/observability/indexer/Cargo.lock +++ /dev/null @@ -1,4966 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 3 - -[[package]] -name = "Inflector" -version = "0.11.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" -dependencies = [ - "lazy_static", - "regex", -] - -[[package]] -name = "adler" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" - -[[package]] -name = "aead" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b613b8e1e3cf911a086f53f03bf286f52fd7a7258e4fa606f0ef220d39d8877" -dependencies = [ - "generic-array", -] - -[[package]] -name = "aes" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8" -dependencies = [ - "cfg-if", - "cipher 0.3.0", - "cpufeatures", - "opaque-debug", -] - -[[package]] -name = "aes-gcm-siv" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589c637f0e68c877bbd59a4599bbe849cac8e5f3e4b5a3ebae8f528cd218dcdc" -dependencies = [ - "aead", - "aes", - "cipher 0.3.0", - "ctr", - "polyval", - "subtle", - "zeroize", -] - -[[package]] -name = "ahash" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" -dependencies = [ - "getrandom 0.2.8", - "once_cell", - "version_check", -] - -[[package]] -name = "aho-corasick" -version = "0.7.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" -dependencies = [ - "memchr", -] - -[[package]] -name = "alloc-no-stdlib" -version = "2.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" - -[[package]] -name = "alloc-stdlib" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" -dependencies = [ - "alloc-no-stdlib", -] - -[[package]] -name = "android_system_properties" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" -dependencies = [ - "libc", -] - -[[package]] -name = "ansi_term" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" -dependencies = [ - "winapi", -] - -[[package]] -name = "anyhow" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "224afbd727c3d6e4b90103ece64b8d1b67fbb1973b1046c2281eed3f3803f800" - -[[package]] -name = "arrayref" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" - -[[package]] -name = "arrayvec" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" - -[[package]] -name = "asn1-rs" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf6690c370453db30743b373a60ba498fc0d6d83b11f4abfd87a84a075db5dd4" -dependencies = [ - "asn1-rs-derive", - "asn1-rs-impl", - "displaydoc", - "nom", - "num-traits", - "rusticata-macros", - "thiserror", - "time 0.3.17", -] - -[[package]] -name = "asn1-rs-derive" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "726535892e8eae7e70657b4c8ea93d26b8553afb1ce617caee529ef96d7dee6c" -dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", - "syn 1.0.107", - "synstructure", -] - -[[package]] -name = "asn1-rs-impl" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2777730b2039ac0f95f093556e61b6d26cebed5393ca6f152717777cec3a42ed" -dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", - "syn 1.0.107", -] - -[[package]] -name = "assert_matches" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" - -[[package]] -name = "async-channel" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf46fee83e5ccffc220104713af3292ff9bc7c64c7de289f66dae8e38d826833" -dependencies = [ - "concurrent-queue", - "event-listener", - "futures-core", -] - -[[package]] -name = "async-compression" -version = "0.3.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "942c7cd7ae39e91bde4820d74132e9862e62c2f386c3aa90ccf55949f5bad63a" -dependencies = [ - "brotli", - "flate2", - "futures-core", - "memchr", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "async-mutex" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479db852db25d9dbf6204e6cb6253698f175c15726470f78af0d918e99d6156e" -dependencies = [ - "event-listener", -] - -[[package]] -name = "async-stream" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dad5c83079eae9969be7fadefe640a1c566901f05ff91ab221de4b6f68d9507e" -dependencies = [ - "async-stream-impl", - "futures-core", -] - -[[package]] -name = "async-stream-impl" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27" -dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", - "syn 1.0.107", -] - -[[package]] -name = "async-trait" -version = "0.1.64" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd7fce9ba8c3c042128ce72d8b2ddbf3a05747efb67ea0313c635e10bda47a2" -dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", - "syn 1.0.107", -] - -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi 0.1.19", - "libc", - "winapi", -] - -[[package]] -name = "autocfg" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" - -[[package]] -name = "axum" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5694b64066a2459918d8074c2ce0d5a88f409431994c2356617c8ae0c4721fc" -dependencies = [ - "async-trait", - "axum-core", - "bitflags", - "bytes", - "futures-util", - "http", - "http-body", - "hyper", - "itoa", - "matchit", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "rustversion", - "serde", - "sync_wrapper", - "tower", - "tower-http", - "tower-layer", - "tower-service", -] - -[[package]] -name = "axum-core" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cae3e661676ffbacb30f1a824089a8c9150e71017f7e1e38f2aa32009188d34" -dependencies = [ - "async-trait", - "bytes", - "futures-util", - "http", - "http-body", - "mime", - "rustversion", - "tower-layer", - "tower-service", -] - -[[package]] -name = "base64" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" - -[[package]] -name = "base64" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" - -[[package]] -name = "base64" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" - -[[package]] -name = "base64ct" -version = "1.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b645a089122eccb6111b4f81cbc1a49f5900ac4666bb93ac027feaecf15607bf" - -[[package]] -name = "bincode" -version = "1.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" -dependencies = [ - "serde", -] - -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "bitmaps" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "031043d04099746d8db04daf1fa424b2bc8bd69d92b25962dcde24da39ab64a2" -dependencies = [ - "typenum", -] - -[[package]] -name = "blake3" -version = "1.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ae2468a89544a466886840aa467a25b766499f4f04bf7d9fcd10ecee9fccef" -dependencies = [ - "arrayref", - "arrayvec", - "cc", - "cfg-if", - "constant_time_eq", - "digest 0.10.6", -] - -[[package]] -name = "block-buffer" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" -dependencies = [ - "block-padding", - "generic-array", -] - -[[package]] -name = "block-buffer" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e" -dependencies = [ - "generic-array", -] - -[[package]] -name = "block-padding" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d696c370c750c948ada61c69a0ee2cbbb9c50b1019ddb86d9317157a99c2cae" - -[[package]] -name = "borsh" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15bf3650200d8bffa99015595e10f1fbd17de07abbc25bb067da79e769939bfa" -dependencies = [ - "borsh-derive", - "hashbrown 0.11.2", -] - -[[package]] -name = "borsh-derive" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6441c552f230375d18e3cc377677914d2ca2b0d36e52129fe15450a2dce46775" -dependencies = [ - "borsh-derive-internal", - "borsh-schema-derive-internal", - "proc-macro-crate 0.1.5", - "proc-macro2 1.0.51", - "syn 1.0.107", -] - -[[package]] -name = "borsh-derive-internal" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5449c28a7b352f2d1e592a8a28bf139bc71afb0764a14f3c02500935d8c44065" -dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", - "syn 1.0.107", -] - -[[package]] -name = "borsh-schema-derive-internal" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdbd5696d8bfa21d53d9fe39a714a18538bad11492a42d066dbbc395fb1951c0" -dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", - "syn 1.0.107", -] - -[[package]] -name = "brotli" -version = "3.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1a0b1dbcc8ae29329621f8d4f0d835787c1c38bb1401979b49d13b0b305ff68" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", - "brotli-decompressor", -] - -[[package]] -name = "brotli-decompressor" -version = "2.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b6561fd3f895a11e8f72af2cb7d22e08366bebc2b6b57f7744c4bda27034744" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", -] - -[[package]] -name = "bs58" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "771fe0050b883fcc3ea2359b1a96bcfbc090b7116eae7c3c512c7a083fdf23d3" - -[[package]] -name = "bumpalo" -version = "3.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" - -[[package]] -name = "bv" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8834bb1d8ee5dc048ee3124f2c7c1afcc6bc9aed03f11e9dfd8c69470a5db340" -dependencies = [ - "feature-probe", - "serde", -] - -[[package]] -name = "bytemuck" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c041d3eab048880cb0b86b256447da3f18859a163c3b8d8893f4e6368abe6393" -dependencies = [ - "bytemuck_derive", -] - -[[package]] -name = "bytemuck_derive" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aca418a974d83d40a0c1f0c5cba6ff4bc28d8df099109ca459a2118d40b6322" -dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", - "syn 1.0.107", -] - -[[package]] -name = "byteorder" -version = "1.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" - -[[package]] -name = "bytes" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" - -[[package]] -name = "caps" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "190baaad529bcfbde9e1a19022c42781bdb6ff9de25721abdb8fd98c0807730b" -dependencies = [ - "libc", - "thiserror", -] - -[[package]] -name = "cc" -version = "1.0.79" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" -dependencies = [ - "jobserver", -] - -[[package]] -name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - -[[package]] -name = "chrono" -version = "0.4.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f" -dependencies = [ - "iana-time-zone", - "js-sys", - "num-integer", - "num-traits", - "serde", - "time 0.1.45", - "wasm-bindgen", - "winapi", -] - -[[package]] -name = "chrono-tz" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa48fa079165080f11d7753fd0bc175b7d391f276b965fe4b55bfad67856e463" -dependencies = [ - "chrono", - "chrono-tz-build", - "phf", -] - -[[package]] -name = "chrono-tz-build" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9998fb9f7e9b2111641485bf8beb32f92945f97f92a3d061f744cfef335f751" -dependencies = [ - "parse-zoneinfo", - "phf", - "phf_codegen", -] - -[[package]] -name = "cipher" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ee52072ec15386f770805afd189a01c8841be8696bed250fa2f13c4c0d6dfb7" -dependencies = [ - "generic-array", -] - -[[package]] -name = "cipher" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1873270f8f7942c191139cb8a40fd228da6c3fd2fc376d7e92d47aa14aeb59e" -dependencies = [ - "crypto-common", - "inout", -] - -[[package]] -name = "clap" -version = "2.34.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" -dependencies = [ - "ansi_term", - "atty", - "bitflags", - "strsim 0.8.0", - "textwrap 0.11.0", - "unicode-width", - "vec_map", -] - -[[package]] -name = "clap" -version = "3.2.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5" -dependencies = [ - "atty", - "bitflags", - "clap_derive", - "clap_lex", - "indexmap", - "once_cell", - "strsim 0.10.0", - "termcolor", - "textwrap 0.16.0", -] - -[[package]] -name = "clap_derive" -version = "3.2.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea0c8bce528c4be4da13ea6fead8965e95b6073585a2f05204bd8f4119f82a65" -dependencies = [ - "heck", - "proc-macro-error", - "proc-macro2 1.0.51", - "quote 1.0.23", - "syn 1.0.107", -] - -[[package]] -name = "clap_lex" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" -dependencies = [ - "os_str_bytes", -] - -[[package]] -name = "codespan-reporting" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" -dependencies = [ - "termcolor", - "unicode-width", -] - -[[package]] -name = "concurrent-queue" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c278839b831783b70278b14df4d45e1beb1aad306c07bb796637de9a0e323e8e" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "console" -version = "0.15.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d79fbe8970a77e3e34151cc13d3b3e248aa0faaecb9f6091fa07ebefe5ad60" -dependencies = [ - "encode_unicode", - "lazy_static", - "libc", - "unicode-width", - "windows-sys 0.42.0", -] - -[[package]] -name = "console_error_panic_hook" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc" -dependencies = [ - "cfg-if", - "wasm-bindgen", -] - -[[package]] -name = "console_log" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "501a375961cef1a0d44767200e66e4a559283097e91d0730b1d75dfb2f8a1494" -dependencies = [ - "log", - "web-sys", -] - -[[package]] -name = "const-oid" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4c78c047431fee22c1a7bb92e00ad095a02a983affe4d8a72e2a2c62c1b94f3" - -[[package]] -name = "constant_time_eq" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3ad85c1f65dc7b37604eb0e89748faf0b9653065f2a8ef69f96a687ec1e9279" - -[[package]] -name = "core-foundation" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "core-foundation-sys" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" - -[[package]] -name = "cpufeatures" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320" -dependencies = [ - "libc", -] - -[[package]] -name = "crc32fast" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "crossbeam-channel" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" -dependencies = [ - "cfg-if", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-deque" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc" -dependencies = [ - "cfg-if", - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a" -dependencies = [ - "autocfg", - "cfg-if", - "crossbeam-utils", - "memoffset 0.7.1", - "scopeguard", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "crunchy" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" - -[[package]] -name = "crypto-common" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" -dependencies = [ - "generic-array", - "typenum", -] - -[[package]] -name = "crypto-mac" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab" -dependencies = [ - "generic-array", - "subtle", -] - -[[package]] -name = "ctr" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "049bb91fb4aaf0e3c7efa6cd5ef877dbbbd15b39dad06d9948de4ec8a75761ea" -dependencies = [ - "cipher 0.3.0", -] - -[[package]] -name = "curve25519-dalek" -version = "3.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f9d052967f590a76e62eb387bd0bbb1b000182c3cefe5364db6b7211651bc0" -dependencies = [ - "byteorder", - "digest 0.9.0", - "rand_core 0.5.1", - "serde", - "subtle", - "zeroize", -] - -[[package]] -name = "cxx" -version = "1.0.89" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc831ee6a32dd495436e317595e639a587aa9907bef96fe6e6abc290ab6204e9" -dependencies = [ - "cc", - "cxxbridge-flags", - "cxxbridge-macro", - "link-cplusplus", -] - -[[package]] -name = "cxx-build" -version = "1.0.89" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94331d54f1b1a8895cd81049f7eaaaef9d05a7dcb4d1fd08bf3ff0806246789d" -dependencies = [ - "cc", - "codespan-reporting", - "once_cell", - "proc-macro2 1.0.51", - "quote 1.0.23", - "scratch", - "syn 1.0.107", -] - -[[package]] -name = "cxxbridge-flags" -version = "1.0.89" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48dcd35ba14ca9b40d6e4b4b39961f23d835dbb8eed74565ded361d93e1feb8a" - -[[package]] -name = "cxxbridge-macro" -version = "1.0.89" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81bbeb29798b407ccd82a3324ade1a7286e0d29851475990b612670f6f5124d2" -dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", - "syn 1.0.107", -] - -[[package]] -name = "data-encoding" -version = "2.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23d8666cb01533c39dde32bcbab8e227b4ed6679b2c925eba05feabea39508fb" - -[[package]] -name = "der" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6919815d73839e7ad218de758883aae3a257ba6759ce7a9992501efbb53d705c" -dependencies = [ - "const-oid", -] - -[[package]] -name = "der-parser" -version = "8.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d4bc9b0db0a0df9ae64634ac5bdefb7afcb534e182275ca0beadbe486701c1" -dependencies = [ - "asn1-rs", - "displaydoc", - "nom", - "num-bigint 0.4.3", - "num-traits", - "rusticata-macros", -] - -[[package]] -name = "derivation-path" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e5c37193a1db1d8ed868c03ec7b152175f26160a5b740e5e484143877e0adf0" - -[[package]] -name = "dialoguer" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af3c796f3b0b408d9fd581611b47fa850821fcb84aa640b83a3c1a5be2d691f2" -dependencies = [ - "console", - "shell-words", - "tempfile", - "zeroize", -] - -[[package]] -name = "digest" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" -dependencies = [ - "generic-array", -] - -[[package]] -name = "digest" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" -dependencies = [ - "block-buffer 0.10.3", - "crypto-common", - "subtle", -] - -[[package]] -name = "dirs-next" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" -dependencies = [ - "cfg-if", - "dirs-sys-next", -] - -[[package]] -name = "dirs-sys-next" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" -dependencies = [ - "libc", - "redox_users", - "winapi", -] - -[[package]] -name = "displaydoc" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bf95dc3f046b9da4f2d51833c0d3547d8564ef6910f5c1ed130306a75b92886" -dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", - "syn 1.0.107", -] - -[[package]] -name = "dlopen" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71e80ad39f814a9abe68583cd50a2d45c8a67561c3361ab8da240587dda80937" -dependencies = [ - "dlopen_derive", - "lazy_static", - "libc", - "winapi", -] - -[[package]] -name = "dlopen_derive" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f236d9e1b1fbd81cea0f9cbdc8dcc7e8ebcd80e6659cd7cb2ad5f6c05946c581" -dependencies = [ - "libc", - "quote 0.6.13", - "syn 0.15.44", -] - -[[package]] -name = "dotenv" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" - -[[package]] -name = "dyn-clone" -version = "1.0.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9b0705efd4599c15a38151f4721f7bc388306f61084d3bfd50bd07fbca5cb60" - -[[package]] -name = "eager" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abe71d579d1812060163dff96056261deb5bf6729b100fa2e36a68b9649ba3d3" - -[[package]] -name = "ed25519" -version = "1.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91cff35c70bba8a626e3185d8cd48cc11b5437e1a5bcd15b9b5fa3c64b6dfee7" -dependencies = [ - "signature", -] - -[[package]] -name = "ed25519-dalek" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c762bae6dcaf24c4c84667b8579785430908723d5c889f469d76a41d59cc7a9d" -dependencies = [ - "curve25519-dalek", - "ed25519", - "rand 0.7.3", - "serde", - "sha2 0.9.9", - "zeroize", -] - -[[package]] -name = "ed25519-dalek-bip32" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d2be62a4061b872c8c0873ee4fc6f101ce7b889d039f019c5fa2af471a59908" -dependencies = [ - "derivation-path", - "ed25519-dalek", - "hmac 0.12.1", - "sha2 0.10.6", -] - -[[package]] -name = "either" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" - -[[package]] -name = "encode_unicode" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" - -[[package]] -name = "encoding_rs" -version = "0.8.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "enum-iterator" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2953d1df47ac0eb70086ccabf0275aa8da8591a28bd358ee2b52bd9f9e3ff9e9" -dependencies = [ - "enum-iterator-derive", -] - -[[package]] -name = "enum-iterator-derive" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8958699f9359f0b04e691a13850d48b7de329138023876d07cbd024c2c820598" -dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", - "syn 1.0.107", -] - -[[package]] -name = "enum_dispatch" -version = "0.3.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11f36e95862220b211a6e2aa5eca09b4fa391b13cd52ceb8035a24bf65a79de2" -dependencies = [ - "once_cell", - "proc-macro2 1.0.51", - "quote 1.0.23", - "syn 1.0.107", -] - -[[package]] -name = "env_logger" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7" -dependencies = [ - "atty", - "humantime", - "log", - "regex", - "termcolor", -] - -[[package]] -name = "envconfig" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea81cc7e21f55a9d9b1efb6816904978d0bfbe31a50347cb24b2e75564bcac9b" -dependencies = [ - "envconfig_derive", -] - -[[package]] -name = "envconfig_derive" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dfca278e5f84b45519acaaff758ebfa01f18e96998bc24b8f1b722dd804b9bf" -dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", - "syn 1.0.107", -] - -[[package]] -name = "event-listener" -version = "2.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" - -[[package]] -name = "fastrand" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a407cfaa3385c4ae6b23e84623d48c2798d06e3e6a1878f7f59f17b3f86499" -dependencies = [ - "instant", -] - -[[package]] -name = "feature-probe" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "835a3dc7d1ec9e75e2b5fb4ba75396837112d2060b03f7d43bc1897c7f7211da" - -[[package]] -name = "fixedbitset" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" - -[[package]] -name = "flate2" -version = "1.0.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" -dependencies = [ - "crc32fast", - "miniz_oxide", -] - -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - -[[package]] -name = "form_urlencoded" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" -dependencies = [ - "percent-encoding", -] - -[[package]] -name = "futures" -version = "0.3.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13e2792b0ff0340399d58445b88fd9770e3489eff258a4cbc1523418f12abf84" -dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-channel" -version = "0.3.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e5317663a9089767a1ec00a487df42e0ca174b61b4483213ac24448e4664df5" -dependencies = [ - "futures-core", - "futures-sink", -] - -[[package]] -name = "futures-core" -version = "0.3.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec90ff4d0fe1f57d600049061dc6bb68ed03c7d2fbd697274c41805dcb3f8608" - -[[package]] -name = "futures-executor" -version = "0.3.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8de0a35a6ab97ec8869e32a2473f4b1324459e14c29275d14b10cb1fd19b50e" -dependencies = [ - "futures-core", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-io" -version = "0.3.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfb8371b6fb2aeb2d280374607aeabfc99d95c72edfe51692e42d3d7f0d08531" - -[[package]] -name = "futures-macro" -version = "0.3.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95a73af87da33b5acf53acfebdc339fe592ecf5357ac7c0a7734ab9d8c876a70" -dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", - "syn 1.0.107", -] - -[[package]] -name = "futures-sink" -version = "0.3.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f310820bb3e8cfd46c80db4d7fb8353e15dfff853a127158425f31e0be6c8364" - -[[package]] -name = "futures-task" -version = "0.3.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcf79a1bf610b10f42aea489289c5a2c478a786509693b80cd39c44ccd936366" - -[[package]] -name = "futures-util" -version = "0.3.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c1d6de3acfef38d2be4b1f543f553131788603495be83da675e180c8d6b7bd1" -dependencies = [ - "futures-channel", - "futures-core", - "futures-io", - "futures-macro", - "futures-sink", - "futures-task", - "memchr", - "pin-project-lite", - "pin-utils", - "slab", -] - -[[package]] -name = "fxhash" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" -dependencies = [ - "byteorder", -] - -[[package]] -name = "gcp-bigquery-client" -version = "0.16.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "601ca5cb46e9918e053c39b60a0e5f0d1b312e8ed71ab3010dec4300ac9e9149" -dependencies = [ - "async-stream", - "async-trait", - "dyn-clone", - "hyper", - "hyper-rustls", - "log", - "reqwest", - "serde", - "serde_json", - "thiserror", - "time 0.3.17", - "tokio", - "tokio-stream", - "url", - "yup-oauth2", -] - -[[package]] -name = "generic-array" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9" -dependencies = [ - "serde", - "typenum", - "version_check", -] - -[[package]] -name = "gethostname" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1ebd34e35c46e00bb73e81363248d627782724609fe1b6396f553f68fe3862e" -dependencies = [ - "libc", - "winapi", -] - -[[package]] -name = "getrandom" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" -dependencies = [ - "cfg-if", - "js-sys", - "libc", - "wasi 0.9.0+wasi-snapshot-preview1", - "wasm-bindgen", -] - -[[package]] -name = "getrandom" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" -dependencies = [ - "cfg-if", - "js-sys", - "libc", - "wasi 0.11.0+wasi-snapshot-preview1", - "wasm-bindgen", -] - -[[package]] -name = "google-cloud-auth" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d86a8cc190da6ab0cff095fcbfe95a057496428ff2a0a711ca110f2fcbb231" -dependencies = [ - "async-trait", - "base64 0.21.0", - "google-cloud-metadata", - "home", - "jsonwebtoken", - "reqwest", - "serde", - "serde_json", - "thiserror", - "time 0.3.17", - "tokio", - "tracing", - "urlencoding", -] - -[[package]] -name = "google-cloud-gax" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "870d9616855e89b37a824f9e35ca670d6bf4f04fab254b2eb01c7818d26f9d0b" -dependencies = [ - "google-cloud-auth", - "http", - "thiserror", - "tokio", - "tokio-retry", - "tokio-util", - "tonic", - "tower", - "tracing", -] - -[[package]] -name = "google-cloud-googleapis" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bccc14a0bfb89f7b6c955da84992774d94d2f16570e5f7b74c7db11e765f791" -dependencies = [ - "prost", - "prost-types", - "tonic", -] - -[[package]] -name = "google-cloud-metadata" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d2c3b00f0a07a1a9efffc1bdd0603ef853d8a6d4ee9de8d73039cd92fdc8f26" -dependencies = [ - "reqwest", - "thiserror", - "tokio", -] - -[[package]] -name = "google-cloud-pubsub" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb0f24ed765ac6da045ea846ffb2f38ede5aae28645a4fa4025b38c070a82d76" -dependencies = [ - "async-channel", - "async-stream", - "google-cloud-auth", - "google-cloud-gax", - "google-cloud-googleapis", - "prost-types", - "thiserror", - "tokio", - "tracing", -] - -[[package]] -name = "h2" -version = "0.3.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f9f29bc9dda355256b2916cf526ab02ce0aeaaaf2bad60d65ef3f12f11dd0f4" -dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http", - "indexmap", - "slab", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] -name = "hashbrown" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" -dependencies = [ - "ahash", -] - -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" -dependencies = [ - "ahash", -] - -[[package]] -name = "heck" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" - -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - -[[package]] -name = "hermit-abi" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" -dependencies = [ - "libc", -] - -[[package]] -name = "histogram" -version = "0.6.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12cb882ccb290b8646e554b157ab0b71e64e8d5bef775cd66b6531e52d302669" - -[[package]] -name = "hmac" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "126888268dcc288495a26bf004b38c5fdbb31682f992c84ceb046a1f0fe38840" -dependencies = [ - "crypto-mac", - "digest 0.9.0", -] - -[[package]] -name = "hmac" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest 0.10.6", -] - -[[package]] -name = "hmac-drbg" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17ea0a1394df5b6574da6e0c1ade9e78868c9fb0a4e5ef4428e32da4676b85b1" -dependencies = [ - "digest 0.9.0", - "generic-array", - "hmac 0.8.1", -] - -[[package]] -name = "home" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "747309b4b440c06d57b0b25f2aee03ee9b5e5397d288c60e21fc709bb98a7408" -dependencies = [ - "winapi", -] - -[[package]] -name = "http" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - -[[package]] -name = "http-body" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" -dependencies = [ - "bytes", - "http", - "pin-project-lite", -] - -[[package]] -name = "http-range-header" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29" - -[[package]] -name = "httparse" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" - -[[package]] -name = "httpdate" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" - -[[package]] -name = "humantime" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" - -[[package]] -name = "hyper" -version = "0.14.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e011372fa0b68db8350aa7a248930ecc7839bf46d8485577d69f117a75f164c" -dependencies = [ - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "socket2", - "tokio", - "tower-service", - "tracing", - "want", -] - -[[package]] -name = "hyper-rustls" -version = "0.23.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" -dependencies = [ - "http", - "hyper", - "log", - "rustls", - "rustls-native-certs", - "tokio", - "tokio-rustls", -] - -[[package]] -name = "hyper-timeout" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" -dependencies = [ - "hyper", - "pin-project-lite", - "tokio", - "tokio-io-timeout", -] - -[[package]] -name = "hyper-tls" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" -dependencies = [ - "bytes", - "hyper", - "native-tls", - "tokio", - "tokio-native-tls", -] - -[[package]] -name = "iana-time-zone" -version = "0.1.53" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765" -dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "wasm-bindgen", - "winapi", -] - -[[package]] -name = "iana-time-zone-haiku" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" -dependencies = [ - "cxx", - "cxx-build", -] - -[[package]] -name = "idna" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" -dependencies = [ - "unicode-bidi", - "unicode-normalization", -] - -[[package]] -name = "im" -version = "15.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0acd33ff0285af998aaf9b57342af478078f53492322fafc47450e09397e0e9" -dependencies = [ - "bitmaps", - "rand_core 0.6.4", - "rand_xoshiro", - "rayon", - "serde", - "sized-chunks", - "typenum", - "version_check", -] - -[[package]] -name = "indexmap" -version = "1.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" -dependencies = [ - "autocfg", - "hashbrown 0.12.3", -] - -[[package]] -name = "indicatif" -version = "0.16.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d207dc617c7a380ab07ff572a6e52fa202a2a8f355860ac9c38e23f8196be1b" -dependencies = [ - "console", - "lazy_static", - "number_prefix", - "regex", -] - -[[package]] -name = "inout" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" -dependencies = [ - "generic-array", -] - -[[package]] -name = "instant" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "ipnet" -version = "2.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146" - -[[package]] -name = "itertools" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" -dependencies = [ - "either", -] - -[[package]] -name = "itoa" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" - -[[package]] -name = "jobserver" -version = "0.1.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "068b1ee6743e4d11fb9c6a1e6064b3693a1b600e7f5f5988047d98b3dc9fb90b" -dependencies = [ - "libc", -] - -[[package]] -name = "js-sys" -version = "0.3.61" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" -dependencies = [ - "wasm-bindgen", -] - -[[package]] -name = "json" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "078e285eafdfb6c4b434e0d31e8cfcb5115b651496faca5749b88fafd4f23bfd" - -[[package]] -name = "jsonrpc-core" -version = "18.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14f7f76aef2d054868398427f6c54943cf3d1caa9a7ec7d0c38d69df97a965eb" -dependencies = [ - "futures", - "futures-executor", - "futures-util", - "log", - "serde", - "serde_derive", - "serde_json", -] - -[[package]] -name = "jsonwebtoken" -version = "8.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09f4f04699947111ec1733e71778d763555737579e44b85844cae8e1940a1828" -dependencies = [ - "base64 0.13.1", - "pem", - "ring", - "serde", - "serde_json", - "simple_asn1", -] - -[[package]] -name = "keccak" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3afef3b6eff9ce9d8ff9b3601125eec7f0c8cbac7abd14f355d053fa56c98768" -dependencies = [ - "cpufeatures", -] - -[[package]] -name = "lazy_static" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" - -[[package]] -name = "libc" -version = "0.2.139" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" - -[[package]] -name = "libloading" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" -dependencies = [ - "cfg-if", - "winapi", -] - -[[package]] -name = "libsecp256k1" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9d220bc1feda2ac231cb78c3d26f27676b8cf82c96971f7aeef3d0cf2797c73" -dependencies = [ - "arrayref", - "base64 0.12.3", - "digest 0.9.0", - "hmac-drbg", - "libsecp256k1-core", - "libsecp256k1-gen-ecmult", - "libsecp256k1-gen-genmult", - "rand 0.7.3", - "serde", - "sha2 0.9.9", - "typenum", -] - -[[package]] -name = "libsecp256k1-core" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0f6ab710cec28cef759c5f18671a27dae2a5f952cdaaee1d8e2908cb2478a80" -dependencies = [ - "crunchy", - "digest 0.9.0", - "subtle", -] - -[[package]] -name = "libsecp256k1-gen-ecmult" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccab96b584d38fac86a83f07e659f0deafd0253dc096dab5a36d53efe653c5c3" -dependencies = [ - "libsecp256k1-core", -] - -[[package]] -name = "libsecp256k1-gen-genmult" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67abfe149395e3aa1c48a2beb32b068e2334402df8181f818d3aee2b304c4f5d" -dependencies = [ - "libsecp256k1-core", -] - -[[package]] -name = "link-cplusplus" -version = "1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5" -dependencies = [ - "cc", -] - -[[package]] -name = "linked-hash-map" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" - -[[package]] -name = "lock_api" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" -dependencies = [ - "autocfg", - "scopeguard", -] - -[[package]] -name = "log" -version = "0.4.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "marginfi-v2-indexer" -version = "0.1.0" -dependencies = [ - "anyhow", - "base64 0.21.0", - "bincode", - "bs58", - "bytes", - "chrono", - "chrono-tz", - "clap 3.2.23", - "concurrent-queue", - "dotenv", - "env_logger", - "envconfig", - "futures", - "gcp-bigquery-client", - "google-cloud-auth", - "google-cloud-gax", - "google-cloud-googleapis", - "google-cloud-pubsub", - "itertools", - "json", - "lazy_static", - "log", - "prost", - "prost-derive", - "serde", - "serde_json", - "serde_yaml 0.9.17", - "solana-account-decoder", - "solana-client", - "solana-measure", - "solana-metrics", - "solana-sdk", - "solana-transaction-status", - "tokio", - "tonic", - "tonic-build", - "uuid", - "yup-oauth2", -] - -[[package]] -name = "matchit" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" - -[[package]] -name = "memchr" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" - -[[package]] -name = "memmap2" -version = "0.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b182332558b18d807c4ce1ca8ca983b34c3ee32765e47b3f0f69b90355cc1dc" -dependencies = [ - "libc", -] - -[[package]] -name = "memoffset" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" -dependencies = [ - "autocfg", -] - -[[package]] -name = "memoffset" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" -dependencies = [ - "autocfg", -] - -[[package]] -name = "merlin" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58c38e2799fc0978b65dfff8023ec7843e2330bb462f19198840b34b6582397d" -dependencies = [ - "byteorder", - "keccak", - "rand_core 0.6.4", - "zeroize", -] - -[[package]] -name = "mime" -version = "0.3.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" - -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - -[[package]] -name = "miniz_oxide" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" -dependencies = [ - "adler", -] - -[[package]] -name = "mio" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de" -dependencies = [ - "libc", - "log", - "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.42.0", -] - -[[package]] -name = "multimap" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" - -[[package]] -name = "native-tls" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" -dependencies = [ - "lazy_static", - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - -[[package]] -name = "nix" -version = "0.24.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa52e972a9a719cecb6864fb88568781eb706bac2cd1d4f04a648542dbf78069" -dependencies = [ - "bitflags", - "cfg-if", - "libc", - "memoffset 0.6.5", -] - -[[package]] -name = "nom" -version = "7.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" -dependencies = [ - "memchr", - "minimal-lexical", -] - -[[package]] -name = "nom8" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae01545c9c7fc4486ab7debaf2aad7003ac19431791868fb2e8066df97fad2f8" -dependencies = [ - "memchr", -] - -[[package]] -name = "num" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8536030f9fea7127f841b45bb6243b27255787fb4eb83958aa1ef9d2fdc0c36" -dependencies = [ - "num-bigint 0.2.6", - "num-complex", - "num-integer", - "num-iter", - "num-rational", - "num-traits", -] - -[[package]] -name = "num-bigint" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" -dependencies = [ - "autocfg", - "num-integer", - "num-traits", -] - -[[package]] -name = "num-bigint" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" -dependencies = [ - "autocfg", - "num-integer", - "num-traits", -] - -[[package]] -name = "num-complex" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95" -dependencies = [ - "autocfg", - "num-traits", -] - -[[package]] -name = "num-derive" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "876a53fff98e03a936a674b29568b0e605f06b29372c2489ff4de23f1949743d" -dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", - "syn 1.0.107", -] - -[[package]] -name = "num-integer" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" -dependencies = [ - "autocfg", - "num-traits", -] - -[[package]] -name = "num-iter" -version = "0.1.43" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" -dependencies = [ - "autocfg", - "num-integer", - "num-traits", -] - -[[package]] -name = "num-rational" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef" -dependencies = [ - "autocfg", - "num-bigint 0.2.6", - "num-integer", - "num-traits", -] - -[[package]] -name = "num-traits" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" -dependencies = [ - "autocfg", -] - -[[package]] -name = "num_cpus" -version = "1.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" -dependencies = [ - "hermit-abi 0.2.6", - "libc", -] - -[[package]] -name = "num_enum" -version = "0.5.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d829733185c1ca374f17e52b762f24f535ec625d2cc1f070e34c8a9068f341b" -dependencies = [ - "num_enum_derive", -] - -[[package]] -name = "num_enum_derive" -version = "0.5.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2be1598bf1c313dcdd12092e3f1920f463462525a21b7b4e11b4168353d0123e" -dependencies = [ - "proc-macro-crate 1.3.0", - "proc-macro2 1.0.51", - "quote 1.0.23", - "syn 1.0.107", -] - -[[package]] -name = "num_threads" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" -dependencies = [ - "libc", -] - -[[package]] -name = "number_prefix" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" - -[[package]] -name = "oid-registry" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bedf36ffb6ba96c2eb7144ef6270557b52e54b20c0a8e1eb2ff99a6c6959bff" -dependencies = [ - "asn1-rs", -] - -[[package]] -name = "once_cell" -version = "1.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" - -[[package]] -name = "opaque-debug" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" - -[[package]] -name = "openssl" -version = "0.10.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b102428fd03bc5edf97f62620f7298614c45cedf287c271e7ed450bbaf83f2e1" -dependencies = [ - "bitflags", - "cfg-if", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" -dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", - "syn 1.0.107", -] - -[[package]] -name = "openssl-probe" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" - -[[package]] -name = "openssl-sys" -version = "0.9.80" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23bbbf7854cd45b83958ebe919f0e8e516793727652e27fda10a8384cfc790b7" -dependencies = [ - "autocfg", - "cc", - "libc", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "os_str_bytes" -version = "6.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" - -[[package]] -name = "parking_lot" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-sys 0.45.0", -] - -[[package]] -name = "parse-zoneinfo" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c705f256449c60da65e11ff6626e0c16a0a0b96aaa348de61376b249bc340f41" -dependencies = [ - "regex", -] - -[[package]] -name = "pbkdf2" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "216eaa586a190f0a738f2f918511eecfa90f13295abec0e457cdebcceda80cbd" -dependencies = [ - "crypto-mac", -] - -[[package]] -name = "pbkdf2" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" -dependencies = [ - "digest 0.10.6", -] - -[[package]] -name = "pem" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8835c273a76a90455d7344889b0964598e3316e2a79ede8e36f16bdcf2228b8" -dependencies = [ - "base64 0.13.1", -] - -[[package]] -name = "percent-encoding" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" - -[[package]] -name = "percentage" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fd23b938276f14057220b707937bcb42fa76dda7560e57a2da30cb52d557937" -dependencies = [ - "num", -] - -[[package]] -name = "petgraph" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4" -dependencies = [ - "fixedbitset", - "indexmap", -] - -[[package]] -name = "phf" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "928c6535de93548188ef63bb7c4036bd415cd8f36ad25af44b9789b2ee72a48c" -dependencies = [ - "phf_shared", -] - -[[package]] -name = "phf_codegen" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56ac890c5e3ca598bbdeaa99964edb5b0258a583a9eb6ef4e89fc85d9224770" -dependencies = [ - "phf_generator", - "phf_shared", -] - -[[package]] -name = "phf_generator" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1181c94580fa345f50f19d738aaa39c0ed30a600d95cb2d3e23f94266f14fbf" -dependencies = [ - "phf_shared", - "rand 0.8.5", -] - -[[package]] -name = "phf_shared" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1fb5f6f826b772a8d4c0394209441e7d37cbbb967ae9c7e0e8134365c9ee676" -dependencies = [ - "siphasher", -] - -[[package]] -name = "pin-project" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" -dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", - "syn 1.0.107", -] - -[[package]] -name = "pin-project-lite" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" - -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - -[[package]] -name = "pkcs8" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cabda3fb821068a9a4fab19a683eac3af12edf0f34b94a8be53c4972b8149d0" -dependencies = [ - "der", - "spki", - "zeroize", -] - -[[package]] -name = "pkg-config" -version = "0.3.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" - -[[package]] -name = "polyval" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8419d2b623c7c0896ff2d5d96e2cb4ede590fed28fcc34934f4c33c036e620a1" -dependencies = [ - "cfg-if", - "cpufeatures", - "opaque-debug", - "universal-hash", -] - -[[package]] -name = "ppv-lite86" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" - -[[package]] -name = "prettyplease" -version = "0.1.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e97e3215779627f01ee256d2fad52f3d95e8e1c11e9fc6fd08f7cd455d5d5c78" -dependencies = [ - "proc-macro2 1.0.51", - "syn 1.0.107", -] - -[[package]] -name = "proc-macro-crate" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d6ea3c4595b96363c13943497db34af4460fb474a95c43f4446ad341b8c9785" -dependencies = [ - "toml", -] - -[[package]] -name = "proc-macro-crate" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66618389e4ec1c7afe67d51a9bf34ff9236480f8d51e7489b7d5ab0303c13f34" -dependencies = [ - "once_cell", - "toml_edit", -] - -[[package]] -name = "proc-macro-error" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" -dependencies = [ - "proc-macro-error-attr", - "proc-macro2 1.0.51", - "quote 1.0.23", - "syn 1.0.107", - "version_check", -] - -[[package]] -name = "proc-macro-error-attr" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" -dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", - "version_check", -] - -[[package]] -name = "proc-macro2" -version = "0.4.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" -dependencies = [ - "unicode-xid 0.1.0", -] - -[[package]] -name = "proc-macro2" -version = "1.0.51" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "prost" -version = "0.11.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21dc42e00223fc37204bd4aa177e69420c604ca4a183209a8f9de30c6d934698" -dependencies = [ - "bytes", - "prost-derive", -] - -[[package]] -name = "prost-build" -version = "0.11.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3f8ad728fb08fe212df3c05169e940fbb6d9d16a877ddde14644a983ba2012e" -dependencies = [ - "bytes", - "heck", - "itertools", - "lazy_static", - "log", - "multimap", - "petgraph", - "prettyplease", - "prost", - "prost-types", - "regex", - "syn 1.0.107", - "tempfile", - "which", -] - -[[package]] -name = "prost-derive" -version = "0.11.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bda8c0881ea9f722eb9629376db3d0b903b462477c1aafcb0566610ac28ac5d" -dependencies = [ - "anyhow", - "itertools", - "proc-macro2 1.0.51", - "quote 1.0.23", - "syn 1.0.107", -] - -[[package]] -name = "prost-types" -version = "0.11.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e0526209433e96d83d750dd81a99118edbc55739e7e61a46764fd2ad537788" -dependencies = [ - "bytes", - "prost", -] - -[[package]] -name = "qstring" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d464fae65fff2680baf48019211ce37aaec0c78e9264c84a3e484717f965104e" -dependencies = [ - "percent-encoding", -] - -[[package]] -name = "quinn" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b435e71d9bfa0d8889927231970c51fb89c58fa63bffcab117c9c7a41e5ef8f" -dependencies = [ - "bytes", - "futures-channel", - "futures-util", - "fxhash", - "quinn-proto", - "quinn-udp", - "rustls", - "thiserror", - "tokio", - "tracing", - "webpki", -] - -[[package]] -name = "quinn-proto" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fce546b9688f767a57530652488420d419a8b1f44a478b451c3d1ab6d992a55" -dependencies = [ - "bytes", - "fxhash", - "rand 0.8.5", - "ring", - "rustls", - "rustls-native-certs", - "rustls-pemfile 0.2.1", - "slab", - "thiserror", - "tinyvec", - "tracing", - "webpki", -] - -[[package]] -name = "quinn-udp" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b07946277141531aea269befd949ed16b2c85a780ba1043244eda0969e538e54" -dependencies = [ - "futures-util", - "libc", - "quinn-proto", - "socket2", - "tokio", - "tracing", -] - -[[package]] -name = "quote" -version = "0.6.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" -dependencies = [ - "proc-macro2 0.4.30", -] - -[[package]] -name = "quote" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" -dependencies = [ - "proc-macro2 1.0.51", -] - -[[package]] -name = "rand" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" -dependencies = [ - "getrandom 0.1.16", - "libc", - "rand_chacha 0.2.2", - "rand_core 0.5.1", - "rand_hc", -] - -[[package]] -name = "rand" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" -dependencies = [ - "libc", - "rand_chacha 0.3.1", - "rand_core 0.6.4", -] - -[[package]] -name = "rand_chacha" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" -dependencies = [ - "ppv-lite86", - "rand_core 0.5.1", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core 0.6.4", -] - -[[package]] -name = "rand_core" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" -dependencies = [ - "getrandom 0.1.16", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom 0.2.8", -] - -[[package]] -name = "rand_hc" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" -dependencies = [ - "rand_core 0.5.1", -] - -[[package]] -name = "rand_xoshiro" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" -dependencies = [ - "rand_core 0.6.4", -] - -[[package]] -name = "rayon" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "356a0625f1954f730c0201cdab48611198dc6ce21f4acff55089b5a78e6e835b" -dependencies = [ - "crossbeam-channel", - "crossbeam-deque", - "crossbeam-utils", - "num_cpus", -] - -[[package]] -name = "rcgen" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6413f3de1edee53342e6138e75b56d32e7bc6e332b3bd62d497b1929d4cfbcdd" -dependencies = [ - "pem", - "ring", - "time 0.3.17", - "yasna", -] - -[[package]] -name = "redox_syscall" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" -dependencies = [ - "bitflags", -] - -[[package]] -name = "redox_users" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" -dependencies = [ - "getrandom 0.2.8", - "redox_syscall", - "thiserror", -] - -[[package]] -name = "regex" -version = "1.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.6.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" - -[[package]] -name = "remove_dir_all" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" -dependencies = [ - "winapi", -] - -[[package]] -name = "reqwest" -version = "0.11.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21eed90ec8570952d53b772ecf8f206aa1ec9a3d76b2521c56c42973f2d91ee9" -dependencies = [ - "async-compression", - "base64 0.21.0", - "bytes", - "encoding_rs", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "hyper", - "hyper-rustls", - "hyper-tls", - "ipnet", - "js-sys", - "log", - "mime", - "native-tls", - "once_cell", - "percent-encoding", - "pin-project-lite", - "rustls", - "rustls-pemfile 1.0.2", - "serde", - "serde_json", - "serde_urlencoded", - "tokio", - "tokio-native-tls", - "tokio-rustls", - "tokio-util", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", - "webpki-roots", - "winreg", -] - -[[package]] -name = "ring" -version = "0.16.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" -dependencies = [ - "cc", - "libc", - "once_cell", - "spin", - "untrusted", - "web-sys", - "winapi", -] - -[[package]] -name = "rpassword" -version = "6.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bf099a1888612545b683d2661a1940089f6c2e5a8e38979b2159da876bfd956" -dependencies = [ - "libc", - "serde", - "serde_json", - "winapi", -] - -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - -[[package]] -name = "rustc_version" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" -dependencies = [ - "semver", -] - -[[package]] -name = "rusticata-macros" -version = "4.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" -dependencies = [ - "nom", -] - -[[package]] -name = "rustls" -version = "0.20.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f" -dependencies = [ - "log", - "ring", - "sct", - "webpki", -] - -[[package]] -name = "rustls-native-certs" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0167bac7a9f490495f3c33013e7722b53cb087ecbe082fb0c6387c96f634ea50" -dependencies = [ - "openssl-probe", - "rustls-pemfile 1.0.2", - "schannel", - "security-framework", -] - -[[package]] -name = "rustls-pemfile" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5eebeaeb360c87bfb72e84abdb3447159c0eaececf1bef2aecd65a8be949d1c9" -dependencies = [ - "base64 0.13.1", -] - -[[package]] -name = "rustls-pemfile" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" -dependencies = [ - "base64 0.21.0", -] - -[[package]] -name = "rustversion" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5583e89e108996506031660fe09baa5011b9dd0341b89029313006d1fb508d70" - -[[package]] -name = "ryu" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" - -[[package]] -name = "schannel" -version = "0.1.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" -dependencies = [ - "windows-sys 0.42.0", -] - -[[package]] -name = "scopeguard" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" - -[[package]] -name = "scratch" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2" - -[[package]] -name = "sct" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" -dependencies = [ - "ring", - "untrusted", -] - -[[package]] -name = "seahash" -version = "4.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" - -[[package]] -name = "security-framework" -version = "2.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254" -dependencies = [ - "bitflags", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "semver" -version = "1.0.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58bc9567378fc7690d6b2addae4e60ac2eeea07becb2c64b9f218b53865cba2a" - -[[package]] -name = "serde" -version = "1.0.152" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_bytes" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "416bda436f9aab92e02c8e10d49a15ddd339cea90b6e340fe51ed97abb548294" -dependencies = [ - "serde", -] - -[[package]] -name = "serde_derive" -version = "1.0.152" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" -dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", - "syn 1.0.107", -] - -[[package]] -name = "serde_json" -version = "1.0.92" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7434af0dc1cbd59268aa98b4c22c131c0584d2232f6fb166efb993e2832e896a" -dependencies = [ - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "serde_urlencoded" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" -dependencies = [ - "form_urlencoded", - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "serde_yaml" -version = "0.8.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578a7433b776b56a35785ed5ce9a7e777ac0598aac5a6dd1b4b18a307c7fc71b" -dependencies = [ - "indexmap", - "ryu", - "serde", - "yaml-rust", -] - -[[package]] -name = "serde_yaml" -version = "0.9.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fb06d4b6cdaef0e0c51fa881acb721bed3c924cfaa71d9c94a3b771dfdf6567" -dependencies = [ - "indexmap", - "itoa", - "ryu", - "serde", - "unsafe-libyaml", -] - -[[package]] -name = "sha-1" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5058ada175748e33390e40e872bd0fe59a19f265d0158daa551c5a88a76009c" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest 0.10.6", -] - -[[package]] -name = "sha2" -version = "0.9.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" -dependencies = [ - "block-buffer 0.9.0", - "cfg-if", - "cpufeatures", - "digest 0.9.0", - "opaque-debug", -] - -[[package]] -name = "sha2" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest 0.10.6", -] - -[[package]] -name = "sha3" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f81199417d4e5de3f04b1e871023acea7389672c4135918f05aa9cbf2f2fa809" -dependencies = [ - "block-buffer 0.9.0", - "digest 0.9.0", - "keccak", - "opaque-debug", -] - -[[package]] -name = "sha3" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdf0c33fae925bdc080598b84bc15c55e7b9a4a43b3c704da051f977469691c9" -dependencies = [ - "digest 0.10.6", - "keccak", -] - -[[package]] -name = "shell-words" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" - -[[package]] -name = "signal-hook-registry" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" -dependencies = [ - "libc", -] - -[[package]] -name = "signature" -version = "1.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" - -[[package]] -name = "simple_asn1" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adc4e5204eb1910f40f9cfa375f6f05b68c3abac4b6fd879c8ff5e7ae8a0a085" -dependencies = [ - "num-bigint 0.4.3", - "num-traits", - "thiserror", - "time 0.3.17", -] - -[[package]] -name = "siphasher" -version = "0.3.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" - -[[package]] -name = "sized-chunks" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16d69225bde7a69b235da73377861095455d298f2b970996eec25ddbb42b3d1e" -dependencies = [ - "bitmaps", - "typenum", -] - -[[package]] -name = "slab" -version = "0.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" -dependencies = [ - "autocfg", -] - -[[package]] -name = "smallvec" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" - -[[package]] -name = "socket2" -version = "0.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" -dependencies = [ - "libc", - "winapi", -] - -[[package]] -name = "solana-account-decoder" -version = "1.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d852416fe325de2d2af280fd5cb90b11e43796c12d7cab45dc316f82f156590" -dependencies = [ - "Inflector", - "base64 0.13.1", - "bincode", - "bs58", - "bv", - "lazy_static", - "serde", - "serde_derive", - "serde_json", - "solana-config-program", - "solana-sdk", - "solana-vote-program", - "spl-token", - "spl-token-2022", - "thiserror", - "zstd", -] - -[[package]] -name = "solana-clap-utils" -version = "1.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4c3b81f4180d01b699c7d8b7fe7c954afebaed7df80e190cf142a0a977265" -dependencies = [ - "chrono", - "clap 2.34.0", - "rpassword", - "solana-perf", - "solana-remote-wallet", - "solana-sdk", - "thiserror", - "tiny-bip39", - "uriparse", - "url", -] - -[[package]] -name = "solana-cli-config" -version = "1.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e510d15888fe9514532b5c10898bcb0df967a967c1991e08338d71779d6930f7" -dependencies = [ - "dirs-next", - "lazy_static", - "serde", - "serde_derive", - "serde_yaml 0.8.26", - "solana-clap-utils", - "solana-sdk", - "url", -] - -[[package]] -name = "solana-client" -version = "1.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f4fa02b6ea09ce12e15dbd670b2c2ad54b9eba89d61aa75523b07989d05a2cd" -dependencies = [ - "async-mutex", - "async-trait", - "base64 0.13.1", - "bincode", - "bs58", - "bytes", - "clap 2.34.0", - "crossbeam-channel", - "enum_dispatch", - "futures", - "futures-util", - "indexmap", - "indicatif", - "itertools", - "jsonrpc-core", - "lazy_static", - "log", - "quinn", - "quinn-proto", - "rand 0.7.3", - "rand_chacha 0.2.2", - "rayon", - "reqwest", - "rustls", - "semver", - "serde", - "serde_derive", - "serde_json", - "solana-account-decoder", - "solana-clap-utils", - "solana-faucet", - "solana-measure", - "solana-metrics", - "solana-net-utils", - "solana-sdk", - "solana-streamer", - "solana-transaction-status", - "solana-version", - "solana-vote-program", - "spl-token-2022", - "thiserror", - "tokio", - "tokio-stream", - "tokio-tungstenite", - "tungstenite", - "url", -] - -[[package]] -name = "solana-config-program" -version = "1.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f410ad57ae0b1d6fb227511f370ab0849ed810cdc6f282701c57bea97e2d3aa7" -dependencies = [ - "bincode", - "chrono", - "serde", - "serde_derive", - "solana-program-runtime", - "solana-sdk", -] - -[[package]] -name = "solana-faucet" -version = "1.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4783cc072245dca8949d8807cbc0aec19e07f661d24608a63e5e767d14024d4c" -dependencies = [ - "bincode", - "byteorder", - "clap 2.34.0", - "crossbeam-channel", - "log", - "serde", - "serde_derive", - "solana-clap-utils", - "solana-cli-config", - "solana-logger", - "solana-metrics", - "solana-sdk", - "solana-version", - "spl-memo", - "thiserror", - "tokio", -] - -[[package]] -name = "solana-frozen-abi" -version = "1.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "514172234053857e7d8c05f3d938c58b1aa5de4d88d6e433f24378bb47ee53fd" -dependencies = [ - "ahash", - "blake3", - "block-buffer 0.9.0", - "bs58", - "bv", - "byteorder", - "cc", - "either", - "generic-array", - "getrandom 0.1.16", - "hashbrown 0.12.3", - "im", - "lazy_static", - "log", - "memmap2", - "once_cell", - "rand_core 0.6.4", - "rustc_version", - "serde", - "serde_bytes", - "serde_derive", - "serde_json", - "sha2 0.10.6", - "solana-frozen-abi-macro", - "subtle", - "thiserror", -] - -[[package]] -name = "solana-frozen-abi-macro" -version = "1.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4253c6d1af453e4f530d148eca9f1c62df33b51d3e8222eb5fb93837ea9a60f4" -dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", - "rustc_version", - "syn 1.0.107", -] - -[[package]] -name = "solana-logger" -version = "1.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1acbd646ebf820599ad74f85cc37687a9e2a77c988bf20ac9acdaad44e6615d" -dependencies = [ - "env_logger", - "lazy_static", - "log", -] - -[[package]] -name = "solana-measure" -version = "1.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "954d1aabf3a982ffa17aa75e0701b7bb55f2ba9edc9fdb5fa96705b2cb63cb92" -dependencies = [ - "log", - "solana-sdk", -] - -[[package]] -name = "solana-metrics" -version = "1.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc9f30ad6d8f65a3382056783f9f47fbc69bceff5b88aa8d946efaf7ebe60217" -dependencies = [ - "crossbeam-channel", - "gethostname", - "lazy_static", - "log", - "reqwest", - "solana-sdk", -] - -[[package]] -name = "solana-net-utils" -version = "1.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0757489bad3518f2826d691808a076c15abb9fdffaf158a8765405d67a40229" -dependencies = [ - "bincode", - "clap 3.2.23", - "crossbeam-channel", - "log", - "nix", - "rand 0.7.3", - "serde", - "serde_derive", - "socket2", - "solana-logger", - "solana-sdk", - "solana-version", - "tokio", - "url", -] - -[[package]] -name = "solana-perf" -version = "1.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06c2c4d856d380fbb927e91633c3afcd4d9c97c12e0e228316f0b210727628d2" -dependencies = [ - "ahash", - "bincode", - "bv", - "caps", - "curve25519-dalek", - "dlopen", - "dlopen_derive", - "fnv", - "lazy_static", - "libc", - "log", - "nix", - "rand 0.7.3", - "rayon", - "serde", - "solana-metrics", - "solana-rayon-threadlimit", - "solana-sdk", - "solana-vote-program", -] - -[[package]] -name = "solana-program" -version = "1.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f85bd23852cf68f4b534f61ca5d3c9d8431eacf235a309a6d88b76ca3825a65b" -dependencies = [ - "base64 0.13.1", - "bincode", - "bitflags", - "blake3", - "borsh", - "borsh-derive", - "bs58", - "bv", - "bytemuck", - "cc", - "console_error_panic_hook", - "console_log", - "curve25519-dalek", - "getrandom 0.2.8", - "itertools", - "js-sys", - "lazy_static", - "libc", - "libsecp256k1", - "log", - "memoffset 0.6.5", - "num-derive", - "num-traits", - "parking_lot", - "rand 0.7.3", - "rand_chacha 0.2.2", - "rustc_version", - "rustversion", - "serde", - "serde_bytes", - "serde_derive", - "serde_json", - "sha2 0.10.6", - "sha3 0.10.6", - "solana-frozen-abi", - "solana-frozen-abi-macro", - "solana-sdk-macro", - "thiserror", - "tiny-bip39", - "wasm-bindgen", - "zeroize", -] - -[[package]] -name = "solana-program-runtime" -version = "1.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2a216852ea67651be630cec1441a2b03ca812e9856459a9ba320ea842b5a558" -dependencies = [ - "base64 0.13.1", - "bincode", - "eager", - "enum-iterator", - "itertools", - "libc", - "libloading", - "log", - "num-derive", - "num-traits", - "rustc_version", - "serde", - "solana-frozen-abi", - "solana-frozen-abi-macro", - "solana-measure", - "solana-metrics", - "solana-sdk", - "thiserror", -] - -[[package]] -name = "solana-rayon-threadlimit" -version = "1.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ceb0e4955f02f48ce2f07c95e8198e8ccb4e7d8a3dabfee0deea9c72110a856c" -dependencies = [ - "lazy_static", - "num_cpus", -] - -[[package]] -name = "solana-remote-wallet" -version = "1.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d9f272791093ab7929d26da07c1f9026b470f31b72d8ffa223967fc6834a717" -dependencies = [ - "console", - "dialoguer", - "log", - "num-derive", - "num-traits", - "parking_lot", - "qstring", - "semver", - "solana-sdk", - "thiserror", - "uriparse", -] - -[[package]] -name = "solana-sdk" -version = "1.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "981e8db983c2711848bb07aa33f9046e2d23198dbce70c6ad4d08e8e8d1248ae" -dependencies = [ - "assert_matches", - "base64 0.13.1", - "bincode", - "bitflags", - "borsh", - "bs58", - "bytemuck", - "byteorder", - "chrono", - "derivation-path", - "digest 0.10.6", - "ed25519-dalek", - "ed25519-dalek-bip32", - "generic-array", - "hmac 0.12.1", - "itertools", - "js-sys", - "lazy_static", - "libsecp256k1", - "log", - "memmap2", - "num-derive", - "num-traits", - "pbkdf2 0.11.0", - "qstring", - "rand 0.7.3", - "rand_chacha 0.2.2", - "rustc_version", - "rustversion", - "serde", - "serde_bytes", - "serde_derive", - "serde_json", - "sha2 0.10.6", - "sha3 0.10.6", - "solana-frozen-abi", - "solana-frozen-abi-macro", - "solana-logger", - "solana-program", - "solana-sdk-macro", - "thiserror", - "uriparse", - "wasm-bindgen", -] - -[[package]] -name = "solana-sdk-macro" -version = "1.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea3125152e4e4140983544e4700f74fe52fd9a63de5b765ac32a17ceaa63ac8c" -dependencies = [ - "bs58", - "proc-macro2 1.0.51", - "quote 1.0.23", - "rustversion", - "syn 1.0.107", -] - -[[package]] -name = "solana-streamer" -version = "1.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01787658d7ceea8a07162ffc868f88bb5e6ee6e32be16c1be792ea4fe0dcfdc2" -dependencies = [ - "crossbeam-channel", - "futures-util", - "histogram", - "indexmap", - "itertools", - "libc", - "log", - "nix", - "pem", - "percentage", - "pkcs8", - "quinn", - "rand 0.7.3", - "rcgen", - "rustls", - "solana-metrics", - "solana-perf", - "solana-sdk", - "thiserror", - "tokio", - "x509-parser", -] - -[[package]] -name = "solana-transaction-status" -version = "1.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80a8e90dfd78f9616a1d23ca009bc1fa33bd93c50839ce556c27fa53bfe49418" -dependencies = [ - "Inflector", - "base64 0.13.1", - "bincode", - "borsh", - "bs58", - "lazy_static", - "log", - "serde", - "serde_derive", - "serde_json", - "solana-account-decoder", - "solana-measure", - "solana-metrics", - "solana-sdk", - "solana-vote-program", - "spl-associated-token-account", - "spl-memo", - "spl-token", - "spl-token-2022", - "thiserror", -] - -[[package]] -name = "solana-version" -version = "1.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c63817d097273e812b912af9ddccc400b78c50b44fea79141db24f204e72328" -dependencies = [ - "log", - "rustc_version", - "semver", - "serde", - "serde_derive", - "solana-frozen-abi", - "solana-frozen-abi-macro", - "solana-sdk", -] - -[[package]] -name = "solana-vote-program" -version = "1.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9227ea253c0f4eda8c60e102a8084f86b1e66a8a66e235c08011cfa5c1a90c84" -dependencies = [ - "bincode", - "log", - "num-derive", - "num-traits", - "rustc_version", - "serde", - "serde_derive", - "solana-frozen-abi", - "solana-frozen-abi-macro", - "solana-metrics", - "solana-program-runtime", - "solana-sdk", - "thiserror", -] - -[[package]] -name = "solana-zk-token-sdk" -version = "1.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c23c31f925e2b3eecdbc3dea061341a364e64f07bf2e6453e9d85a34dea67ecd" -dependencies = [ - "aes-gcm-siv", - "arrayref", - "base64 0.13.1", - "bincode", - "bytemuck", - "byteorder", - "cipher 0.4.3", - "curve25519-dalek", - "getrandom 0.1.16", - "itertools", - "lazy_static", - "merlin", - "num-derive", - "num-traits", - "rand 0.7.3", - "serde", - "serde_json", - "sha3 0.9.1", - "solana-program", - "solana-sdk", - "subtle", - "thiserror", - "zeroize", -] - -[[package]] -name = "spin" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" - -[[package]] -name = "spki" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d01ac02a6ccf3e07db148d2be087da624fea0221a16152ed01f0496a6b0a27" -dependencies = [ - "base64ct", - "der", -] - -[[package]] -name = "spl-associated-token-account" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16a33ecc83137583902c3e13c02f34151c8b2f2b74120f9c2b3ff841953e083d" -dependencies = [ - "assert_matches", - "borsh", - "num-derive", - "num-traits", - "solana-program", - "spl-token", - "spl-token-2022", - "thiserror", -] - -[[package]] -name = "spl-memo" -version = "3.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd0dc6f70db6bacea7ff25870b016a65ba1d1b6013536f08e4fd79a8f9005325" -dependencies = [ - "solana-program", -] - -[[package]] -name = "spl-token" -version = "3.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e85e168a785e82564160dcb87b2a8e04cee9bfd1f4d488c729d53d6a4bd300d" -dependencies = [ - "arrayref", - "bytemuck", - "num-derive", - "num-traits", - "num_enum", - "solana-program", - "thiserror", -] - -[[package]] -name = "spl-token-2022" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0a97cbf60b91b610c846ccf8eecca96d92a24a19ffbf9fe06cd0c84e76ec45e" -dependencies = [ - "arrayref", - "bytemuck", - "num-derive", - "num-traits", - "num_enum", - "solana-program", - "solana-zk-token-sdk", - "spl-memo", - "spl-token", - "thiserror", -] - -[[package]] -name = "strsim" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" - -[[package]] -name = "strsim" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" - -[[package]] -name = "subtle" -version = "2.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" - -[[package]] -name = "syn" -version = "0.15.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5" -dependencies = [ - "proc-macro2 0.4.30", - "quote 0.6.13", - "unicode-xid 0.1.0", -] - -[[package]] -name = "syn" -version = "1.0.107" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" -dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", - "unicode-ident", -] - -[[package]] -name = "sync_wrapper" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" - -[[package]] -name = "synstructure" -version = "0.12.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" -dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", - "syn 1.0.107", - "unicode-xid 0.2.4", -] - -[[package]] -name = "tempfile" -version = "3.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" -dependencies = [ - "cfg-if", - "fastrand", - "libc", - "redox_syscall", - "remove_dir_all", - "winapi", -] - -[[package]] -name = "termcolor" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" -dependencies = [ - "winapi-util", -] - -[[package]] -name = "textwrap" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" -dependencies = [ - "unicode-width", -] - -[[package]] -name = "textwrap" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" - -[[package]] -name = "thiserror" -version = "1.0.38" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.38" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" -dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", - "syn 1.0.107", -] - -[[package]] -name = "time" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" -dependencies = [ - "libc", - "wasi 0.10.0+wasi-snapshot-preview1", - "winapi", -] - -[[package]] -name = "time" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376" -dependencies = [ - "itoa", - "libc", - "num_threads", - "serde", - "time-core", - "time-macros", -] - -[[package]] -name = "time-core" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" - -[[package]] -name = "time-macros" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d967f99f534ca7e495c575c62638eebc2898a8c84c119b89e250477bc4ba16b2" -dependencies = [ - "time-core", -] - -[[package]] -name = "tiny-bip39" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffc59cb9dfc85bb312c3a78fd6aa8a8582e310b0fa885d5bb877f6dcc601839d" -dependencies = [ - "anyhow", - "hmac 0.8.1", - "once_cell", - "pbkdf2 0.4.0", - "rand 0.7.3", - "rustc-hash", - "sha2 0.9.9", - "thiserror", - "unicode-normalization", - "wasm-bindgen", - "zeroize", -] - -[[package]] -name = "tinyvec" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - -[[package]] -name = "tokio" -version = "1.25.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e00990ebabbe4c14c08aca901caed183ecd5c09562a12c824bb53d3c3fd3af" -dependencies = [ - "autocfg", - "bytes", - "libc", - "memchr", - "mio", - "num_cpus", - "parking_lot", - "pin-project-lite", - "signal-hook-registry", - "socket2", - "tokio-macros", - "windows-sys 0.42.0", -] - -[[package]] -name = "tokio-io-timeout" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" -dependencies = [ - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tokio-macros" -version = "1.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" -dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", - "syn 1.0.107", -] - -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", -] - -[[package]] -name = "tokio-retry" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f" -dependencies = [ - "pin-project", - "rand 0.8.5", - "tokio", -] - -[[package]] -name = "tokio-rustls" -version = "0.23.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" -dependencies = [ - "rustls", - "tokio", - "webpki", -] - -[[package]] -name = "tokio-stream" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d660770404473ccd7bc9f8b28494a811bc18542b915c0855c51e8f419d5223ce" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tokio-tungstenite" -version = "0.17.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f714dd15bead90401d77e04243611caec13726c2408afd5b31901dfcdcb3b181" -dependencies = [ - "futures-util", - "log", - "rustls", - "tokio", - "tokio-rustls", - "tungstenite", - "webpki", - "webpki-roots", -] - -[[package]] -name = "tokio-util" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "pin-project-lite", - "tokio", - "tracing", -] - -[[package]] -name = "toml" -version = "0.5.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" -dependencies = [ - "serde", -] - -[[package]] -name = "toml_datetime" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4553f467ac8e3d374bc9a177a26801e5d0f9b211aa1673fb137a403afd1c9cf5" - -[[package]] -name = "toml_edit" -version = "0.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56c59d8dd7d0dcbc6428bf7aa2f0e823e26e43b3c9aca15bbc9475d23e5fa12b" -dependencies = [ - "indexmap", - "nom8", - "toml_datetime", -] - -[[package]] -name = "tonic" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f219fad3b929bef19b1f86fbc0358d35daed8f2cac972037ac0dc10bbb8d5fb" -dependencies = [ - "async-stream", - "async-trait", - "axum", - "base64 0.13.1", - "bytes", - "flate2", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "hyper", - "hyper-timeout", - "percent-encoding", - "pin-project", - "prost", - "prost-derive", - "rustls-native-certs", - "rustls-pemfile 1.0.2", - "tokio", - "tokio-rustls", - "tokio-stream", - "tokio-util", - "tower", - "tower-layer", - "tower-service", - "tracing", - "tracing-futures", - "webpki-roots", -] - -[[package]] -name = "tonic-build" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bf5e9b9c0f7e0a7c027dcfaba7b2c60816c7049171f679d99ee2ff65d0de8c4" -dependencies = [ - "prettyplease", - "proc-macro2 1.0.51", - "prost-build", - "quote 1.0.23", - "syn 1.0.107", -] - -[[package]] -name = "tower" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" -dependencies = [ - "futures-core", - "futures-util", - "indexmap", - "pin-project", - "pin-project-lite", - "rand 0.8.5", - "slab", - "tokio", - "tokio-util", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tower-http" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f873044bf02dd1e8239e9c1293ea39dad76dc594ec16185d0a1bf31d8dc8d858" -dependencies = [ - "bitflags", - "bytes", - "futures-core", - "futures-util", - "http", - "http-body", - "http-range-header", - "pin-project-lite", - "tower", - "tower-layer", - "tower-service", -] - -[[package]] -name = "tower-layer" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" - -[[package]] -name = "tower-service" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" - -[[package]] -name = "tracing" -version = "0.1.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" -dependencies = [ - "cfg-if", - "log", - "pin-project-lite", - "tracing-attributes", - "tracing-core", -] - -[[package]] -name = "tracing-attributes" -version = "0.1.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" -dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", - "syn 1.0.107", -] - -[[package]] -name = "tracing-core" -version = "0.1.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" -dependencies = [ - "once_cell", -] - -[[package]] -name = "tracing-futures" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" -dependencies = [ - "pin-project", - "tracing", -] - -[[package]] -name = "try-lock" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" - -[[package]] -name = "tungstenite" -version = "0.17.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e27992fd6a8c29ee7eef28fc78349aa244134e10ad447ce3b9f0ac0ed0fa4ce0" -dependencies = [ - "base64 0.13.1", - "byteorder", - "bytes", - "http", - "httparse", - "log", - "rand 0.8.5", - "rustls", - "sha-1", - "thiserror", - "url", - "utf-8", - "webpki", - "webpki-roots", -] - -[[package]] -name = "typenum" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" - -[[package]] -name = "unicode-bidi" -version = "0.3.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54675592c1dbefd78cbd98db9bacd89886e1ca50692a0692baefffdeb92dd58" - -[[package]] -name = "unicode-ident" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" - -[[package]] -name = "unicode-normalization" -version = "0.1.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" -dependencies = [ - "tinyvec", -] - -[[package]] -name = "unicode-width" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" - -[[package]] -name = "unicode-xid" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" - -[[package]] -name = "unicode-xid" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" - -[[package]] -name = "universal-hash" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f214e8f697e925001e66ec2c6e37a4ef93f0f78c2eed7814394e10c62025b05" -dependencies = [ - "generic-array", - "subtle", -] - -[[package]] -name = "unsafe-libyaml" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc7ed8ba44ca06be78ea1ad2c3682a43349126c8818054231ee6f4748012aed2" - -[[package]] -name = "untrusted" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" - -[[package]] -name = "uriparse" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0200d0fc04d809396c2ad43f3c95da3582a2556eba8d453c1087f4120ee352ff" -dependencies = [ - "fnv", - "lazy_static", -] - -[[package]] -name = "url" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" -dependencies = [ - "form_urlencoded", - "idna", - "percent-encoding", -] - -[[package]] -name = "urlencoding" -version = "2.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8db7427f936968176eaa7cdf81b7f98b980b18495ec28f1b5791ac3bfe3eea9" - -[[package]] -name = "utf-8" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" - -[[package]] -name = "uuid" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1674845326ee10d37ca60470760d4288a6f80f304007d92e5c53bab78c9cfd79" -dependencies = [ - "getrandom 0.2.8", -] - -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - -[[package]] -name = "vec_map" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" - -[[package]] -name = "version_check" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" - -[[package]] -name = "want" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" -dependencies = [ - "log", - "try-lock", -] - -[[package]] -name = "wasi" -version = "0.9.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" - -[[package]] -name = "wasi" -version = "0.10.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" - -[[package]] -name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" - -[[package]] -name = "wasm-bindgen" -version = "0.2.84" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" -dependencies = [ - "cfg-if", - "wasm-bindgen-macro", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.84" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" -dependencies = [ - "bumpalo", - "log", - "once_cell", - "proc-macro2 1.0.51", - "quote 1.0.23", - "syn 1.0.107", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-futures" -version = "0.4.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" -dependencies = [ - "cfg-if", - "js-sys", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.84" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" -dependencies = [ - "quote 1.0.23", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.84" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" -dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", - "syn 1.0.107", - "wasm-bindgen-backend", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.84" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" - -[[package]] -name = "web-sys" -version = "0.3.61" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "webpki" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" -dependencies = [ - "ring", - "untrusted", -] - -[[package]] -name = "webpki-roots" -version = "0.22.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" -dependencies = [ - "webpki", -] - -[[package]] -name = "which" -version = "4.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" -dependencies = [ - "either", - "libc", - "once_cell", -] - -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-util" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" -dependencies = [ - "winapi", -] - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "windows-sys" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows-sys" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" -dependencies = [ - "windows-targets", -] - -[[package]] -name = "windows-targets" -version = "0.42.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.42.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" - -[[package]] -name = "windows_i686_gnu" -version = "0.42.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" - -[[package]] -name = "windows_i686_msvc" -version = "0.42.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.42.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.42.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" - -[[package]] -name = "winreg" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" -dependencies = [ - "winapi", -] - -[[package]] -name = "x509-parser" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0ecbeb7b67ce215e40e3cc7f2ff902f94a223acf44995934763467e7b1febc8" -dependencies = [ - "asn1-rs", - "base64 0.13.1", - "data-encoding", - "der-parser", - "lazy_static", - "nom", - "oid-registry", - "rusticata-macros", - "thiserror", - "time 0.3.17", -] - -[[package]] -name = "yaml-rust" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" -dependencies = [ - "linked-hash-map", -] - -[[package]] -name = "yasna" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aed2e7a52e3744ab4d0c05c20aa065258e84c49fd4226f5191b2ed29712710b4" -dependencies = [ - "time 0.3.17", -] - -[[package]] -name = "yup-oauth2" -version = "8.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8cb398cca4dedd0203666d7c3e7a089d14557be759efd57ab75f067949276e7" -dependencies = [ - "anyhow", - "async-trait", - "base64 0.13.1", - "futures", - "http", - "hyper", - "hyper-rustls", - "itertools", - "log", - "percent-encoding", - "rustls", - "rustls-pemfile 1.0.2", - "seahash", - "serde", - "serde_json", - "time 0.3.17", - "tokio", - "tower-service", - "url", -] - -[[package]] -name = "zeroize" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4756f7db3f7b5574938c3eb1c117038b8e07f95ee6718c0efad4ac21508f1efd" -dependencies = [ - "zeroize_derive", -] - -[[package]] -name = "zeroize_derive" -version = "1.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44bf07cb3e50ea2003396695d58bf46bc9887a1f362260446fad6bc4e79bd36c" -dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", - "syn 1.0.107", - "synstructure", -] - -[[package]] -name = "zstd" -version = "0.11.2+zstd.1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" -dependencies = [ - "zstd-safe", -] - -[[package]] -name = "zstd-safe" -version = "5.0.2+zstd.1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" -dependencies = [ - "libc", - "zstd-sys", -] - -[[package]] -name = "zstd-sys" -version = "2.0.6+zstd.1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68a3f9792c0c3dc6c165840a75f47ae1f4da402c2d006881129579f6597e801b" -dependencies = [ - "cc", - "libc", - "pkg-config", -] diff --git a/observability/indexer/Cargo.toml b/observability/indexer/Cargo.toml index 5c355ec5..cc37b40d 100644 --- a/observability/indexer/Cargo.toml +++ b/observability/indexer/Cargo.toml @@ -1,3 +1,5 @@ +cargo-features = ["workspace-inheritance"] + [package] name = "marginfi-v2-indexer" version = "0.1.0" @@ -7,26 +9,41 @@ edition = "2021" name = "mfi-index" path = "src/bin/main.rs" +[features] +mainnet-beta = ["marginfi/mainnet-beta"] + [dependencies] -solana-client = "=1.14.3" -solana-measure = "=1.14.3" -solana-metrics = "=1.14.3" -solana-sdk = "=1.14.3" -solana-transaction-status = "=1.14.3" -solana-account-decoder = "=1.14.3" - -gcp-bigquery-client = "0.16.2" -google-cloud-auth = "0.8.1" -google-cloud-pubsub = "0.12.1" -google-cloud-gax = "0.12.1" -google-cloud-googleapis = "0.7.0" -yup-oauth2 = "8.0.0" +solana-client = { workspace = true } +solana-measure = { workspace = true } +solana-metrics = { workspace = true } +solana-sdk = { workspace = true } +solana-transaction-status = { workspace = true } +solana-account-decoder = { workspace = true } +spl-token = "3.5.0" +gcp-bigquery-client = "0.16.7" +google-cloud-default = { git = " https://github.com/mrgnlabs/google-cloud-rust.git", rev = "3f651f2d9fd8cca547bb11490d2575d9bf90f994", features = ["pubsub"] } +google-cloud-auth = { git = " https://github.com/mrgnlabs/google-cloud-rust.git", rev = "3f651f2d9fd8cca547bb11490d2575d9bf90f994" } +google-cloud-pubsub = { git = " https://github.com/mrgnlabs/google-cloud-rust.git", rev = "3f651f2d9fd8cca547bb11490d2575d9bf90f994" } +google-cloud-gax = { git = " https://github.com/mrgnlabs/google-cloud-rust.git", rev = "3f651f2d9fd8cca547bb11490d2575d9bf90f994" } +google-cloud-googleapis = { git = " https://github.com/mrgnlabs/google-cloud-rust.git", rev = "3f651f2d9fd8cca547bb11490d2575d9bf90f994", features = ["bytes", "pubsub"] } +yup-oauth2 = "8.3.0" + +anchor-client = "0.26.0" +marginfi = { path = "../../programs/marginfi", features = [ + "no-entrypoint", + "client", +] } +pyth-sdk-solana = "0.7.0" + +fixed = "1.12.0" +fixed-macro = "1.2.0" dotenv = "0.15.0" -env_logger = "0.9.3" -log = "0.4.17" +bytemuck = "1.13.1" +tracing = "0.1.36" +tracing-stackdriver = "0.6.1" +tracing-subscriber = { version = "0.3.15", features = ["env-filter", "fmt"] } clap = { version = "3.2.23", features = ["derive"] } -anyhow = "1.0.58" envconfig = "0.10.0" itertools = "0.10.5" json = "0.12.4" @@ -42,15 +59,22 @@ tonic = { version = "0.8.3", features = [ ] } bs58 = "0.4.0" bytes = "1.3.0" -prost = "0.11.3" +thiserror = "1.0" +prost = "0.11.0" prost-derive = "0.11.2" -tokio = { version = "1.21.2", features = ["full"] } +tokio = { version = "1.14.1", features = ["full"] } +tokio-stream = "0.1.12" futures = "0.3.25" lazy_static = "1.4.0" chrono = "0.4.23" base64 = "0.21.0" uuid = { version = "1.2.2", features = ["v4"] } chrono-tz = "0.8.0" +backoff = { version = "0.4.0", features = ["tokio"] } +rayon = "1.6" +anyhow = "1.0.62" [build-dependencies] -tonic-build = "0.8" +anyhow = "1.0.58" +tonic-build = "0.8.2" +protobuf-src = "1.1.0" diff --git a/observability/indexer/Dockerfile b/observability/indexer/Dockerfile new file mode 100644 index 00000000..ad0c5f34 --- /dev/null +++ b/observability/indexer/Dockerfile @@ -0,0 +1,30 @@ +FROM rust:1.72 as builder + +RUN apt-get update -y && apt-get install -y pkg-config build-essential libudev-dev clang cmake protobuf-compiler +RUN rustup component add rustfmt clippy + +WORKDIR /app + +COPY ./Cargo.toml ./Cargo.toml +COPY ./programs ./programs +COPY ./observability/indexer ./observability/indexer +COPY ./test-utils ./test-utils +COPY ./clients/rust ./clients/rust + +ENV CARGO_NET_GIT_FETCH_WITH_CLI=true + +RUN cargo update +RUN cargo build --release + + +FROM debian:stable-slim as runner + +WORKDIR /app + +RUN apt-get update -y && apt-get install -y ca-certificates + +COPY --from=builder /app/target/release/mfi-index ./ + +ENV RUST_LOG=info + +CMD ["/app/mfi-index"] diff --git a/observability/indexer/Dockerfile.dockerignore b/observability/indexer/Dockerfile.dockerignore new file mode 100644 index 00000000..6e35c17e --- /dev/null +++ b/observability/indexer/Dockerfile.dockerignore @@ -0,0 +1,8 @@ +# Ignore everything except for Cargo and src files +**/.env +**/.git +**/target +**/.DS_Store +**/Dockerfile +**/docs +**/*.log diff --git a/observability/indexer/build.rs b/observability/indexer/build.rs index b539d9ce..8a74cf06 100644 --- a/observability/indexer/build.rs +++ b/observability/indexer/build.rs @@ -1,14 +1,16 @@ -fn main() -> Result<(), Box> { +fn main() -> anyhow::Result<()> { + compile_protos()?; + Ok(()) +} + +fn compile_protos() -> anyhow::Result<()> { + std::env::set_var("PROTOC", protobuf_src::protoc()); tonic_build::configure() .protoc_arg("--experimental_allow_proto3_optional") .type_attribute("PubsubTransaction", "#[derive(serde::Serialize)]") .type_attribute("PubsubAccountUpdate", "#[derive(serde::Serialize)]") .compile( - &[ - "protos/geyser.proto", - "protos/solana-storage-v1.10.40.proto", - "protos/gcp_pubsub.proto", - ], + &["protos/geyser.proto", "protos/gcp_pubsub.proto"], &["protos"], ) .unwrap(); diff --git a/observability/indexer/build_image b/observability/indexer/build_image new file mode 100755 index 00000000..837697fe --- /dev/null +++ b/observability/indexer/build_image @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -e + +script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +root_dir=$(git rev-parse --show-toplevel) + +image_tag=$1 + +echo "$root_dir" +echo "$script_dir" + +[ -z "$image_tag" ] && echo "Missing image_tag argument" && exit 1 + +crate_dir_rel=$(realpath --relative-to="$root_dir" "$script_dir") + +echo "$crate_dir_rel" + +cd "$root_dir" + +local_image_name="marginfi-v2-indexer" +gcp_image_name="us-east1-docker.pkg.dev/marginfi-dev/main/$local_image_name" + +docker build -f "$crate_dir_rel/Dockerfile" --platform=linux/amd64 --tag "$gcp_image_name:$image_tag" . diff --git a/observability/indexer/protos/gcp_pubsub.proto b/observability/indexer/protos/gcp_pubsub.proto index 5f62beae..f725265c 100644 --- a/observability/indexer/protos/gcp_pubsub.proto +++ b/observability/indexer/protos/gcp_pubsub.proto @@ -24,8 +24,10 @@ message PubsubAccountUpdate { required string owner = 4; required uint64 slot = 5; required string pubkey = 6; - required uint64 lamports = 7; - required bool executable = 8; - required uint64 rent_epoch = 9; - required string data = 10; + optional string txn_signature = 7; + optional uint64 write_version = 8; + required uint64 lamports = 9; + required bool executable = 10; + required uint64 rent_epoch = 11; + required string data = 12; } diff --git a/observability/indexer/protos/geyser.proto b/observability/indexer/protos/geyser.proto index a8dec4d4..84d22d59 100644 --- a/observability/indexer/protos/geyser.proto +++ b/observability/indexer/protos/geyser.proto @@ -56,7 +56,7 @@ message SubscribeUpdateAccountInfo { uint64 rent_epoch = 5; bytes data = 6; uint64 write_version = 7; - // bytes txn_signature = 8; + optional bytes txn_signature = 8; } message SubscribeUpdateSlot { diff --git a/observability/indexer/protos/solana-storage-v1.10.40.proto b/observability/indexer/protos/solana-storage-v1.10.40.proto index bc90628a..2be3e7a0 100644 --- a/observability/indexer/protos/solana-storage-v1.10.40.proto +++ b/observability/indexer/protos/solana-storage-v1.10.40.proto @@ -3,116 +3,116 @@ syntax = "proto3"; package solana.storage.ConfirmedBlock; message ConfirmedBlock { - string previous_blockhash = 1; - string blockhash = 2; - uint64 parent_slot = 3; - repeated ConfirmedTransaction transactions = 4; - repeated Reward rewards = 5; - UnixTimestamp block_time = 6; - BlockHeight block_height = 7; + string previous_blockhash = 1; + string blockhash = 2; + uint64 parent_slot = 3; + repeated ConfirmedTransaction transactions = 4; + repeated Reward rewards = 5; + UnixTimestamp block_time = 6; + BlockHeight block_height = 7; } message ConfirmedTransaction { - Transaction transaction = 1; - TransactionStatusMeta meta = 2; + Transaction transaction = 1; + TransactionStatusMeta meta = 2; } message Transaction { - repeated bytes signatures = 1; - Message message = 2; + repeated bytes signatures = 1; + Message message = 2; } message Message { - MessageHeader header = 1; - repeated bytes account_keys = 2; - bytes recent_blockhash = 3; - repeated CompiledInstruction instructions = 4; - bool versioned = 5; - repeated MessageAddressTableLookup address_table_lookups = 6; + MessageHeader header = 1; + repeated bytes account_keys = 2; + bytes recent_blockhash = 3; + repeated CompiledInstruction instructions = 4; + bool versioned = 5; + repeated MessageAddressTableLookup address_table_lookups = 6; } message MessageHeader { - uint32 num_required_signatures = 1; - uint32 num_readonly_signed_accounts = 2; - uint32 num_readonly_unsigned_accounts = 3; + uint32 num_required_signatures = 1; + uint32 num_readonly_signed_accounts = 2; + uint32 num_readonly_unsigned_accounts = 3; } message MessageAddressTableLookup { - bytes account_key = 1; - bytes writable_indexes = 2; - bytes readonly_indexes = 3; + bytes account_key = 1; + bytes writable_indexes = 2; + bytes readonly_indexes = 3; } message TransactionStatusMeta { - TransactionError err = 1; - uint64 fee = 2; - repeated uint64 pre_balances = 3; - repeated uint64 post_balances = 4; - repeated InnerInstructions inner_instructions = 5; - bool inner_instructions_none = 10; - repeated string log_messages = 6; - bool log_messages_none = 11; - repeated TokenBalance pre_token_balances = 7; - repeated TokenBalance post_token_balances = 8; - repeated Reward rewards = 9; - repeated bytes loaded_writable_addresses = 12; - repeated bytes loaded_readonly_addresses = 13; + TransactionError err = 1; + uint64 fee = 2; + repeated uint64 pre_balances = 3; + repeated uint64 post_balances = 4; + repeated InnerInstructions inner_instructions = 5; + bool inner_instructions_none = 10; + repeated string log_messages = 6; + bool log_messages_none = 11; + repeated TokenBalance pre_token_balances = 7; + repeated TokenBalance post_token_balances = 8; + repeated Reward rewards = 9; + repeated bytes loaded_writable_addresses = 12; + repeated bytes loaded_readonly_addresses = 13; } message TransactionError { - bytes err = 1; + bytes err = 1; } message InnerInstructions { - uint32 index = 1; - repeated CompiledInstruction instructions = 2; + uint32 index = 1; + repeated CompiledInstruction instructions = 2; } message CompiledInstruction { - uint32 program_id_index = 1; - bytes accounts = 2; - bytes data = 3; + uint32 program_id_index = 1; + bytes accounts = 2; + bytes data = 3; } message TokenBalance { - uint32 account_index = 1; - string mint = 2; - UiTokenAmount ui_token_amount = 3; - string owner = 4; - string program_id = 5; + uint32 account_index = 1; + string mint = 2; + UiTokenAmount ui_token_amount = 3; + string owner = 4; + string program_id = 5; } message UiTokenAmount { - double ui_amount = 1; - uint32 decimals = 2; - string amount = 3; - string ui_amount_string = 4; + double ui_amount = 1; + uint32 decimals = 2; + string amount = 3; + string ui_amount_string = 4; } enum RewardType { - Unspecified = 0; - Fee = 1; - Rent = 2; - Staking = 3; - Voting = 4; + Unspecified = 0; + Fee = 1; + Rent = 2; + Staking = 3; + Voting = 4; } message Reward { - string pubkey = 1; - int64 lamports = 2; - uint64 post_balance = 3; - RewardType reward_type = 4; - string commission = 5; + string pubkey = 1; + int64 lamports = 2; + uint64 post_balance = 3; + RewardType reward_type = 4; + string commission = 5; } message Rewards { - repeated Reward rewards = 1; + repeated Reward rewards = 1; } message UnixTimestamp { - int64 timestamp = 1; + int64 timestamp = 1; } message BlockHeight { - uint64 block_height = 1; + uint64 block_height = 1; } \ No newline at end of file diff --git a/observability/indexer/src/commands/backfill.rs b/observability/indexer/src/commands/backfill.rs index d7facd8f..049ed769 100644 --- a/observability/indexer/src/commands/backfill.rs +++ b/observability/indexer/src/commands/backfill.rs @@ -13,14 +13,13 @@ use base64::{engine::general_purpose, Engine}; use chrono::{NaiveDateTime, Utc}; use envconfig::Envconfig; use futures::future::join_all; -use google_cloud_auth::{credentials::CredentialsFile, Project}; -use google_cloud_gax::project::ProjectOptions; +use google_cloud_default::WithAuthExt; use google_cloud_googleapis::pubsub::v1::PubsubMessage; -use google_cloud_pubsub::client::ClientConfig; +use google_cloud_pubsub::client::{Client, ClientConfig}; use itertools::Itertools; -use log::error; use solana_sdk::{pubkey::Pubkey, signature::Signature, transaction::TransactionVersion}; use std::{str::FromStr, sync::Arc, time::Duration}; +use tracing::error; use uuid::Uuid; #[derive(Envconfig, Debug, Clone)] @@ -72,7 +71,7 @@ pub async fn backfill(config: BackfillConfig) -> Result<()> { }); let transaction_processor = |ctx: Arc| async move { - push_transactions_to_pubsub(ctx, config).await + push_transactions_to_pubsub(ctx, config).await.unwrap() }; tx_crawler.run_async(&transaction_processor).await.unwrap(); @@ -83,18 +82,11 @@ pub async fn backfill(config: BackfillConfig) -> Result<()> { pub async fn push_transactions_to_pubsub( ctx: Arc, config: BackfillConfig, -) { +) -> Result<()> { let topic_name = config.topic_name.as_str(); - let client = google_cloud_pubsub::client::Client::new(ClientConfig { - project_id: Some(config.project_id.clone()), - project: ProjectOptions::Project(Some(Project::FromFile(Box::new( - CredentialsFile::new().await.unwrap(), - )))), - ..Default::default() - }) - .await - .unwrap(); + let client_config = ClientConfig::default().with_auth().await?; + let client = Client::new(client_config).await?; let topic = client.topic(topic_name); topic @@ -168,8 +160,10 @@ pub async fn push_transactions_to_pubsub( message: general_purpose::STANDARD.encode(tx_decoded.message.serialize()), }; + let message_str = serde_json::to_string(&message).unwrap(); + let message_bytes = message_str.as_bytes().to_vec(); messages.push(PubsubMessage { - data: serde_json::to_string(&message).unwrap().as_bytes().to_vec(), + data: message_bytes.into(), ..PubsubMessage::default() }); }); diff --git a/observability/indexer/src/commands/create_table.rs b/observability/indexer/src/commands/create_table.rs index 9d0b83f4..b3e41826 100644 --- a/observability/indexer/src/commands/create_table.rs +++ b/observability/indexer/src/commands/create_table.rs @@ -1,15 +1,24 @@ -use std::str::FromStr; use anyhow::{anyhow, Result}; -use gcp_bigquery_client::{error::BQError, model::{table::Table, time_partitioning::TimePartitioning}}; -use log::{info, warn}; +use gcp_bigquery_client::{ + error::BQError, + model::{table::Table, time_partitioning::TimePartitioning}, +}; +use std::str::FromStr; +use tracing::{info, warn}; use yup_oauth2::parse_service_account_key; -use crate::utils::big_query::{ACCOUNT_SCHEMA, NOT_FOUND_CODE, TRANSACTION_SCHEMA}; +use crate::utils::big_query::{ + ACCOUNT_SCHEMA, METRIC_LENDING_POOL_BANK_SCHEMA, METRIC_MARGINFI_ACCOUNT_SCHEMA, + METRIC_MARGINFI_GROUP_SCHEMA, NOT_FOUND_CODE, TRANSACTION_SCHEMA, +}; #[derive(Debug)] -pub enum TableType { +pub enum TableType { Transaction, Account, + MetricMarginfiGroup, + MetricLendingPoolBank, + MetricMarginfiAccount, } impl FromStr for TableType { @@ -19,9 +28,13 @@ impl FromStr for TableType { match s { "transaction" => Ok(Self::Transaction), "account" => Ok(Self::Account), + "metric_group" => Ok(Self::MetricMarginfiGroup), + "metric_bank" => Ok(Self::MetricLendingPoolBank), + "metric_account" => Ok(Self::MetricMarginfiAccount), _ => Err(anyhow!("Invalid table type")), } - }} + } +} pub async fn create_table( project_id: String, @@ -29,28 +42,28 @@ pub async fn create_table( table_id: String, table_type: TableType, table_friendly_name: Option, - table_description: Option, - ) -> Result<()> { - // Init BigQuery client - let sa_key = parse_service_account_key(std::env::var("GOOGLE_APPLICATION_CREDENTIALS_JSON").unwrap()).unwrap(); - let client = gcp_bigquery_client::Client::from_service_account_key(sa_key, false) + table_description: Option, +) -> Result<()> { + // Init BigQuery client + let sa_key = + parse_service_account_key(std::env::var("GOOGLE_APPLICATION_CREDENTIALS_JSON").unwrap()) + .unwrap(); + let client = gcp_bigquery_client::Client::from_service_account_key(sa_key, false) .await .unwrap(); - let schema = match table_type { + let schema = match table_type { TableType::Transaction => TRANSACTION_SCHEMA.to_owned(), TableType::Account => ACCOUNT_SCHEMA.to_owned(), + TableType::MetricMarginfiGroup => METRIC_MARGINFI_GROUP_SCHEMA.to_owned(), + TableType::MetricLendingPoolBank => METRIC_LENDING_POOL_BANK_SCHEMA.to_owned(), + TableType::MetricMarginfiAccount => METRIC_MARGINFI_ACCOUNT_SCHEMA.to_owned(), }; // Create a new table if needed match client .table() - .get( - &project_id, - &dataset_id, - &table_id, - None, - ) + .get(&project_id, &dataset_id, &table_id, None) .await { Ok(_) => info!("Table {} already exists", table_id), @@ -60,17 +73,13 @@ pub async fn create_table( match client .table() .create( - Table::new( - &project_id, - &dataset_id, - &table_id, - schema, - ) - .friendly_name(&table_friendly_name.unwrap_or_default()) - .description(&table_description.unwrap_or_default()) - .time_partitioning(TimePartitioning::per_day().field("timestamp")), + Table::new(&project_id, &dataset_id, &table_id, schema) + .friendly_name(&table_friendly_name.unwrap_or_default()) + .description(&table_description.unwrap_or_default()) + .time_partitioning(TimePartitioning::per_day().field("timestamp")), ) - .await { + .await + { Ok(_) => info!("Table {} created", table_id), Err(error) => panic!("Error creating table {}: {:#?}", table_id, error), }; diff --git a/observability/indexer/src/commands/geyser_client.rs b/observability/indexer/src/commands/geyser_client.rs deleted file mode 100644 index c8968a68..00000000 --- a/observability/indexer/src/commands/geyser_client.rs +++ /dev/null @@ -1,38 +0,0 @@ -use anyhow::Result; -use tonic::{ - codegen::InterceptedService, - service::Interceptor, - transport::{Channel, ClientTlsConfig, Endpoint}, - Request, Status, -}; -use crate::utils::protos::geyser::geyser_client::GeyserClient; - -pub struct RequestInterceptor { - auth_token: String, -} - -impl Interceptor for RequestInterceptor { - fn call(&mut self, mut request: Request<()>) -> Result, Status> { - request - .metadata_mut() - .insert("x-token", self.auth_token.parse().unwrap()); - Ok(request) - } -} - -pub async fn get_geyser_client( - url: String, - auth_token: String, -) -> Result>> { - let mut endpoint = Endpoint::from_shared(url.clone())?; - - if url.contains("https") { - endpoint = endpoint.tls_config(ClientTlsConfig::new())?; - } - let channel = endpoint.connect().await.unwrap(); - - Ok(GeyserClient::with_interceptor( - channel, - RequestInterceptor { auth_token }, - )) -} diff --git a/observability/indexer/src/commands/index_accounts.rs b/observability/indexer/src/commands/index_accounts.rs index 3bbe0605..026ade7d 100644 --- a/observability/indexer/src/commands/index_accounts.rs +++ b/observability/indexer/src/commands/index_accounts.rs @@ -1,5 +1,5 @@ use crate::{ - commands::geyser_client::get_geyser_client, + utils::geyser_client::get_geyser_client, utils::{ big_query::DATE_FORMAT_STR, protos::{ @@ -16,14 +16,12 @@ use base64::{engine::general_purpose, Engine}; use chrono::{DateTime, Utc}; use envconfig::Envconfig; use futures::{future::join_all, stream, StreamExt}; -use google_cloud_auth::{credentials::CredentialsFile, Project}; -use google_cloud_gax::project::ProjectOptions; +use google_cloud_default::WithAuthExt; use google_cloud_googleapis::pubsub::v1::PubsubMessage; use google_cloud_pubsub::client::{Client, ClientConfig}; use itertools::Itertools; -use log::{debug, error, info, warn}; use solana_measure::measure::Measure; -use solana_sdk::{account::Account, pubkey::Pubkey}; +use solana_sdk::{account::Account, pubkey::Pubkey, signature::Signature}; use std::{ collections::{BTreeMap, BTreeSet, HashMap}, sync::{ @@ -33,6 +31,7 @@ use std::{ time::Duration, }; use tonic::Status; +use tracing::{debug, error, info, warn}; use uuid::Uuid; #[derive(Envconfig, Debug, Clone)] @@ -55,7 +54,7 @@ pub struct IndexAccountsConfig { #[envconfig(from = "INDEX_ACCOUNTS_PUBSUB_TOPIC_NAME")] pub topic_name: String, #[envconfig(from = "GOOGLE_APPLICATION_CREDENTIALS_JSON")] - pub gcp_sa_key: String, + pub gcp_sa_key: Option, } #[derive(Debug, Clone)] @@ -63,6 +62,8 @@ pub struct AccountUpdateData { pub timestamp: DateTime, pub slot: u64, pub address: Pubkey, + pub txn_signature: Option, + pub write_version: Option, pub account_data: Account, } @@ -98,7 +99,7 @@ pub async fn index_accounts(config: IndexAccountsConfig) -> Result<()> { }); let process_account_updates_handle = tokio::spawn({ let context = context.clone(); - async move { push_transactions_to_pubsub(context).await } + async move { push_transactions_to_pubsub(context).await.unwrap() } }); let monitor_handle = tokio::spawn({ let context = context.clone(); @@ -193,7 +194,11 @@ fn process_update(ctx: Arc, update: UpdateOneof) -> Result<()> { UpdateOneof::Account(account_update) => { let update_slot = account_update.slot; if let Some(account_info) = account_update.account { - let address = Pubkey::new(&account_info.pubkey); + let address = &Pubkey::new(&account_info.pubkey); + let txn_signature = account_info + .txn_signature + .clone() + .map(|sig_bytes| Signature::new(&sig_bytes)); let mut account_updates_queue = ctx.account_updates_queue.lock().unwrap(); let slot_account_updates = match account_updates_queue.get_mut(&update_slot) { @@ -204,20 +209,17 @@ fn process_update(ctx: Arc, update: UpdateOneof) -> Result<()> { } }; - let insert_res = slot_account_updates.insert( - address, + slot_account_updates.insert( + *address, AccountUpdateData { - address, + address: *address, timestamp: Utc::now(), slot: update_slot, - account_data: account_info.into(), + txn_signature, + write_version: Some(account_info.write_version), + account_data: account_info.try_into()?, }, ); - - if insert_res.is_some() { - println!("Overwriting existing account update for {}", address); - } - // println!("slot_transactions for {:?} at {}: {}", filters.to_vec(), transaction_update.slot, slot_transactions.len()); } else { anyhow::bail!("Expected `transaction` in `UpdateOneof::Transaction` update"); } @@ -245,18 +247,11 @@ fn process_update(ctx: Arc, update: UpdateOneof) -> Result<()> { Ok(()) } -pub async fn push_transactions_to_pubsub(ctx: Arc) { +pub async fn push_transactions_to_pubsub(ctx: Arc) -> Result<()> { let topic_name = ctx.config.topic_name.as_str(); - let client = Client::new(ClientConfig { - project_id: Some(ctx.config.project_id.clone()), - project: ProjectOptions::Project(Some(Project::FromFile(Box::new( - CredentialsFile::new().await.unwrap(), - )))), - ..Default::default() - }) - .await - .unwrap(); + let client_config = ClientConfig::default().with_auth().await?; + let client = Client::new(client_config).await?; let topic = client.topic(topic_name); topic @@ -276,7 +271,7 @@ pub async fn push_transactions_to_pubsub(ctx: Arc) { if let Some(oldest_slot_with_commitment) = latest_slots_with_commitment.first() { account_updates_per_slot.retain(|slot, account_updates| { if slot < oldest_slot_with_commitment { - warn!( + debug!( "throwing away txs {:?} from slot {}", account_updates .iter() @@ -314,10 +309,6 @@ pub async fn push_transactions_to_pubsub(ctx: Arc) { account_updates_data.iter().for_each(|account_update_data| { ctx.account_updates_counter.fetch_add(1, Ordering::Relaxed); - info!( - "{:?} - {}", - account_update_data.account_data.owner, account_update_data.address - ); let now = Utc::now(); @@ -331,14 +322,18 @@ pub async fn push_transactions_to_pubsub(ctx: Arc) { owner: account_update_data.account_data.owner.to_string(), slot: account_update_data.slot, pubkey: account_update_data.address.to_string(), + txn_signature: account_update_data.txn_signature.map(|sig| sig.to_string()), + write_version: account_update_data.write_version, lamports: account_update_data.account_data.lamports, executable: account_update_data.account_data.executable, rent_epoch: account_update_data.account_data.rent_epoch, data: general_purpose::STANDARD.encode(&account_update_data.account_data.data), }; + let message_str = serde_json::to_string(&message).unwrap(); + let message_bytes = message_str.as_bytes().to_vec(); messages.push(PubsubMessage { - data: serde_json::to_string(&message).unwrap().as_bytes().to_vec(), + data: message_bytes.into(), ..PubsubMessage::default() }); }); @@ -401,8 +396,8 @@ async fn monitor(ctx: Arc) { }; let account_updates_queue_size = ctx.account_updates_queue.lock().unwrap().len(); - info!( - "Time: {:.1}s | Total account udpates: {} | {:.1}s count: {} | {:.1}s rate: {:.1} tx/s | Tx Q size: {} | Stream disconnections: {} | Processing errors: {}\n\tEarliest confirmed slot: {} | Latest confirmed slot: {} | Earliest pending slot: {} | Latest pending slot: {}", + debug!( + "Time: {:.1}s | Total account udpates: {} | {:.1}s count: {} | {:.1}s rate: {:.1} tx/s | Tx Q size: {} | Stream disconnections: {} | Processing errors: {} | Earliest confirmed slot: {} | Latest confirmed slot: {} | Earliest pending slot: {} | Latest pending slot: {}", current_fetch_time, current_fetch_count, current_fetch_time - last_fetch_time, diff --git a/observability/indexer/src/commands/index_transactions.rs b/observability/indexer/src/commands/index_transactions.rs index 9b97c8e3..300bae88 100644 --- a/observability/indexer/src/commands/index_transactions.rs +++ b/observability/indexer/src/commands/index_transactions.rs @@ -1,4 +1,4 @@ -use crate::commands::geyser_client::get_geyser_client; +use crate::utils::geyser_client::get_geyser_client; use crate::utils::{ big_query::DATE_FORMAT_STR, protos::{ @@ -14,12 +14,10 @@ use base64::{engine::general_purpose, Engine}; use chrono::{DateTime, Utc}; use envconfig::Envconfig; use futures::{future::join_all, stream, StreamExt}; -use google_cloud_auth::{credentials::CredentialsFile, Project}; -use google_cloud_gax::project::ProjectOptions; +use google_cloud_default::WithAuthExt; use google_cloud_googleapis::pubsub::v1::PubsubMessage; use google_cloud_pubsub::client::{Client, ClientConfig}; use itertools::Itertools; -use log::{debug, error, info, warn}; use solana_measure::measure::Measure; use solana_sdk::{pubkey::Pubkey, signature::Signature, transaction::TransactionVersion}; use solana_transaction_status::{UiTransactionStatusMeta, VersionedTransactionWithStatusMeta}; @@ -32,6 +30,7 @@ use std::{ time::Duration, }; use tonic::Status; +use tracing::{debug, error, info, warn}; use uuid::Uuid; #[derive(Envconfig, Debug, Clone)] @@ -54,7 +53,7 @@ pub struct IndexTransactionsConfig { #[envconfig(from = "INDEX_TRANSACTIONS_PUBSUB_TOPIC_NAME")] pub topic_name: String, #[envconfig(from = "GOOGLE_APPLICATION_CREDENTIALS_JSON")] - pub gcp_sa_key: String, + pub gcp_sa_key: Option, } #[derive(Debug, Clone)] @@ -98,7 +97,7 @@ pub async fn index_transactions(config: IndexTransactionsConfig) -> Result<()> { }); let process_transactions_handle = tokio::spawn({ let context = context.clone(); - async move { push_transactions_to_pubsub(context).await } + async move { push_transactions_to_pubsub(context).await.unwrap() } }); let monitor_handle = tokio::spawn({ let context = context.clone(); @@ -196,7 +195,8 @@ fn process_update(ctx: Arc, filters: &[String], update: UpdateOneof) -> UpdateOneof::Transaction(transaction_update) => { if let Some(transaction_info) = transaction_update.transaction { let signature = transaction_info.signature.clone(); - let transaction: VersionedTransactionWithStatusMeta = transaction_info.into(); + let transaction: VersionedTransactionWithStatusMeta = + transaction_info.try_into()?; let mut transactions_queue = ctx.transactions_queue.lock().unwrap(); let slot_transactions = match transactions_queue.get_mut(&transaction_update.slot) { @@ -244,18 +244,11 @@ fn process_update(ctx: Arc, filters: &[String], update: UpdateOneof) -> Ok(()) } -pub async fn push_transactions_to_pubsub(ctx: Arc) { +pub async fn push_transactions_to_pubsub(ctx: Arc) -> Result<()> { let topic_name = ctx.config.topic_name.as_str(); - let client = Client::new(ClientConfig { - project_id: Some(ctx.config.project_id.clone()), - project: ProjectOptions::Project(Some(Project::FromFile(Box::new( - CredentialsFile::new().await.unwrap(), - )))), - ..Default::default() - }) - .await - .unwrap(); + let client_config = ClientConfig::default().with_auth().await?; + let client = Client::new(client_config).await.unwrap(); let topic = client.topic(topic_name); topic @@ -275,7 +268,7 @@ pub async fn push_transactions_to_pubsub(ctx: Arc) { if let Some(oldest_slot_with_commitment) = latest_slots_with_commitment.first() { transactions_per_slot.retain(|slot, transactions| { if slot < oldest_slot_with_commitment { - warn!( + debug!( "throwing away txs {:?} from slot {}", transactions .iter() @@ -363,8 +356,10 @@ pub async fn push_transactions_to_pubsub(ctx: Arc) { .encode(transaction_data.transaction.transaction.message.serialize()), }; + let message_str = serde_json::to_string(&message).unwrap(); + let message_bytes = message_str.as_bytes().to_vec(); messages.push(PubsubMessage { - data: serde_json::to_string(&message).unwrap().as_bytes().to_vec(), + data: message_bytes.into(), ..PubsubMessage::default() }); }); @@ -428,8 +423,8 @@ async fn monitor(ctx: Arc) { }; let tx_queue_size = ctx.transactions_queue.lock().unwrap().len(); - info!( - "Time: {:.1}s | Total txs: {} | {:.1}s count: {} | {:.1}s rate: {:.1} tx/s | Tx Q size: {} | Stream disconnections: {} | Processing errors: {}\n\tEarliest confirmed slot: {} | Latest confirmed slot: {} | Earliest pending slot: {} | Latest pending slot: {}", + debug!( + "Time: {:.1}s | Total txs: {} | {:.1}s count: {} | {:.1}s rate: {:.1} tx/s | Tx Q size: {} | Stream disconnections: {} | Processing errors: {} | Earliest confirmed slot: {} | Latest confirmed slot: {} | Earliest pending slot: {} | Latest pending slot: {}", current_fetch_time, current_fetch_count, current_fetch_time - last_fetch_time, diff --git a/observability/indexer/src/commands/mod.rs b/observability/indexer/src/commands/mod.rs index dc295d2f..7c423c76 100644 --- a/observability/indexer/src/commands/mod.rs +++ b/observability/indexer/src/commands/mod.rs @@ -1,5 +1,5 @@ pub mod backfill; pub mod create_table; -pub mod index_transactions; pub mod index_accounts; -pub mod geyser_client; +pub mod index_transactions; +pub mod snapshot_accounts; diff --git a/observability/indexer/src/commands/snapshot_accounts.rs b/observability/indexer/src/commands/snapshot_accounts.rs new file mode 100644 index 00000000..5e2008ca --- /dev/null +++ b/observability/indexer/src/commands/snapshot_accounts.rs @@ -0,0 +1,669 @@ +use crate::utils::geyser_client::get_geyser_client; +use crate::utils::metrics::{LendingPoolBankMetrics, MarginfiAccountMetrics, MarginfiGroupMetrics}; +use crate::utils::protos::SubscribeRequestFilterBlocks; +use crate::utils::snapshot::{AccountRoutingType, BankUpdateRoutingType}; +use crate::utils::{ + protos::geyser::{ + subscribe_update::UpdateOneof, SubscribeRequest, SubscribeRequestFilterAccounts, + SubscribeRequestFilterSlots, SubscribeUpdateSlotStatus, + }, + snapshot::Snapshot, +}; +use anyhow::Result; +use chrono::{DateTime, Utc}; +use envconfig::Envconfig; +use futures::stream::once; +use futures::{future::join_all, pin_mut, StreamExt}; +use gcp_bigquery_client::model::table_data_insert_all_request::TableDataInsertAllRequest; +use itertools::Itertools; +use rayon::prelude::*; +use solana_client::nonblocking::rpc_client::RpcClient; +use solana_measure::measure::Measure; +use solana_sdk::{ + account::Account, + commitment_config::{CommitmentConfig, CommitmentLevel}, + pubkey::Pubkey, + signature::Signature, +}; +use std::sync::atomic::AtomicI64; +use std::{ + collections::{BTreeMap, BTreeSet, HashMap}, + fmt::Debug, + str::FromStr, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }, + time::Duration, +}; +use tokio::sync::Mutex; +use tokio::time::Instant; +use tracing::{debug, error, info, warn}; +use yup_oauth2::parse_service_account_key; + +#[derive(Debug, Clone)] +pub struct PubkeyVec(pub Vec); // Ugh + +impl FromStr for PubkeyVec { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + let targets_raw = json::parse(s).unwrap(); + if !targets_raw.is_array() { + return Err(anyhow::Error::msg(format!( + "Invalid base58 pubkey array: {}", + s + ))); + } + + let mut targets: Vec = vec![]; + for i in 0..targets_raw.len() { + targets.push(Pubkey::from_str(targets_raw[i].as_str().unwrap()).unwrap()); + } + Ok(Self(targets)) + } +} + +#[derive(Envconfig, Debug, Clone)] +pub struct SnapshotAccountsConfig { + #[envconfig(from = "SNAPSHOT_ACCOUNTS_RPC_ENDPOINT")] + pub rpc_endpoint: String, + #[envconfig(from = "SNAPSHOT_ACCOUNTS_RPC_ENDPOINT_GEYSER")] + pub rpc_endpoint_geyser: String, + #[envconfig(from = "SNAPSHOT_ACCOUNTS_RPC_TOKEN")] + pub rpc_token: String, + #[envconfig(from = "SNAPSHOT_ACCOUNTS_SLOTS_BUFFER_SIZE")] + pub slots_buffer_size: u32, + #[envconfig(from = "SNAPSHOT_ACCOUNTS_MONITOR_INTERVAL")] + pub monitor_interval: u64, + #[envconfig(from = "SNAPSHOT_ACCOUNTS_SNAP_INTERVAL")] + pub snap_interval: u64, + #[envconfig(from = "SNAPSHOT_ACCOUNTS_PROGRAM_ID")] + pub program_id: Pubkey, + #[envconfig(from = "SNAPSHOT_ACCOUNTS_ADDITIONAL_ACCOUNTS")] + pub additional_accounts: PubkeyVec, + + #[envconfig(from = "SNAPSHOT_ACCOUNTS_PROJECT_ID")] + pub project_id: String, + #[envconfig(from = "SNAPSHOT_ACCOUNTS_DATASET_ID")] + pub dataset_id: String, + #[envconfig(from = "SNAPSHOT_ACCOUNTS_TABLE_GROUP_METRICS")] + pub table_group: String, + #[envconfig(from = "SNAPSHOT_ACCOUNTS_TABLE_BANK_METRICS")] + pub table_bank: String, + #[envconfig(from = "SNAPSHOT_ACCOUNTS_TABLE_ACCOUNT_METRICS")] + pub table_account: String, + #[envconfig(from = "GOOGLE_APPLICATION_CREDENTIALS_JSON")] + pub gcp_sa_key: Option, +} + +#[derive(Clone, Debug)] +pub struct AccountUpdate { + pub timestamp: DateTime, + pub slot: u64, + pub address: Pubkey, + pub txn_signature: Option, + pub write_version: Option, + pub account_data: Account, +} + +#[derive(Clone)] +pub struct Context { + pub timestamp: Arc, + pub config: Arc, + pub rpc_client: Arc, + pub geyser_subscription_config: Arc>, + account_updates_queue: Arc>>>, + latest_slots_with_commitment: Arc>>, + account_snapshot: Arc>, + stream_disconnection_count: Arc, + update_processing_error_count: Arc, +} + +impl Context { + pub async fn new(config: &SnapshotAccountsConfig) -> Self { + let rpc_client = Arc::new(RpcClient::new_with_commitment( + format!("{}/{}", config.rpc_endpoint, config.rpc_token), + CommitmentConfig { + commitment: CommitmentLevel::Finalized, + }, + )); + Self { + timestamp: Arc::new(AtomicI64::new(0)), + config: Arc::new(config.clone()), + rpc_client: rpc_client.clone(), + geyser_subscription_config: Arc::new(Mutex::new((false, SubscribeRequest::default()))), + account_updates_queue: Arc::new(Mutex::new(BTreeMap::new())), + latest_slots_with_commitment: Arc::new(Mutex::new(BTreeSet::new())), + account_snapshot: Arc::new(Mutex::new(Snapshot::new(config.program_id, rpc_client))), + stream_disconnection_count: Arc::new(AtomicU64::new(0)), + update_processing_error_count: Arc::new(AtomicU64::new(0)), + } + } +} + +async fn compute_geyser_config( + config: &SnapshotAccountsConfig, + non_program_pubkeys: &[Pubkey], +) -> SubscribeRequest { + let mut accounts = config.additional_accounts.0.clone(); + accounts.append(&mut non_program_pubkeys.to_vec()); + accounts.sort(); + accounts.dedup(); + + SubscribeRequest { + accounts: HashMap::from_iter([ + ( + config.program_id.to_string(), + SubscribeRequestFilterAccounts { + owner: vec![config.program_id.to_string()], + account: vec![], + }, + ), + ( + "lol".to_string(), + SubscribeRequestFilterAccounts { + owner: vec![], + account: accounts.iter().map(|x| x.to_string()).collect_vec(), + }, + ), + ]), + slots: HashMap::from_iter([("slots".to_string(), SubscribeRequestFilterSlots {})]), + transactions: HashMap::default(), + blocks: HashMap::from_iter([("blocks".to_string(), SubscribeRequestFilterBlocks {})]), + } +} + +pub async fn snapshot_accounts(config: SnapshotAccountsConfig) -> Result<()> { + let context = Arc::new(Context::new(&config).await); + + info!("Fetching initial snapshot"); + let non_program_accounts = { + let mut snapshot = context.account_snapshot.lock().await; + snapshot.init().await.unwrap(); + println!("Summary: {snapshot}"); + + snapshot + .routing_lookup + .iter() + .filter(|(_, routing_type)| match routing_type { + AccountRoutingType::MarginfiGroup => false, + AccountRoutingType::MarginfiAccount => false, + AccountRoutingType::Bank(_, bank_update_routing_type) => { + !matches!(bank_update_routing_type, BankUpdateRoutingType::State) + } + _ => true, + }) + .map(|(pubkey, _)| *pubkey) + .unique() + .collect_vec() + }; + + let geyser_subscription_config = compute_geyser_config(&config, &non_program_accounts).await; + *context.geyser_subscription_config.lock().await = (false, geyser_subscription_config.clone()); + + let listen_to_updates_handle = tokio::spawn({ + let context = context.clone(); + async move { listen_to_updates(context).await } + }); + + let update_account_map_handle = tokio::spawn({ + let context = context.clone(); + async move { update_account_map(context).await } + }); + let process_account_updates_handle = tokio::spawn({ + let context = context.clone(); + async move { push_transactions_to_bigquery(context).await } + }); + let monitor_handle = tokio::spawn({ + let context = context.clone(); + async move { monitor(context).await } + }); + + join_all([ + listen_to_updates_handle, + process_account_updates_handle, + update_account_map_handle, + monitor_handle, + ]) + .await; + + Ok(()) +} + +async fn listen_to_updates(ctx: Arc) { + loop { + info!("Instantiating geyser client"); + let geyser_client = get_geyser_client( + ctx.config.rpc_endpoint_geyser.to_string(), + ctx.config.rpc_token.to_string(), + ) + .await; + + match geyser_client { + Ok(geyser_client) => { + let geyser_client = Box::pin(geyser_client); + pin_mut!(geyser_client); + let (_, geyser_config) = ctx.geyser_subscription_config.lock().await.clone(); + debug!("Subscribing to geyser with {:?}", geyser_config); + let stream_response = geyser_client + .subscribe(once(async move { geyser_config })) + .await; + + match stream_response { + Ok(stream_response) => { + info!("Subscribed to updates"); + let mut stream = stream_response.into_inner(); + while let Some(received) = stream.next().await { + let mut geyser_sub_config = ctx.geyser_subscription_config.lock().await; + if geyser_sub_config.0 { + warn!("Config update: {:?}", geyser_sub_config.1); + geyser_sub_config.0 = false; + continue; + } + + match received { + Ok(received) => { + if let Some(update) = received.update_oneof { + match process_update(ctx.clone(), update).await { + Ok(_) => {} + Err(err) => { + error!("Error processing update: {}", err); + ctx.update_processing_error_count + .fetch_add(1, Ordering::Relaxed); + } + } + } + } + Err(err) => { + error!("Error pulling next update: {}", err); + tokio::time::sleep(Duration::from_secs(1)).await; + break; + } + } + } + + error!("Stream got disconnected"); + ctx.stream_disconnection_count + .fetch_add(1, Ordering::Relaxed); + } + Err(err) => { + error!("Error establishing geyser sub: {}", err); + tokio::time::sleep(Duration::from_secs(1)).await; + } + } + } + Err(err) => { + error!("Error creating geyser client: {}", err); + tokio::time::sleep(Duration::from_secs(1)).await; + } + } + } +} + +async fn process_update(ctx: Arc, update: UpdateOneof) -> Result<()> { + match update { + UpdateOneof::Account(account_update) => { + let update_slot = account_update.slot; + if let Some(account_info) = account_update.account { + let address = Pubkey::new(&account_info.pubkey); + let txn_signature = account_info + .txn_signature + .clone() + .map(|sig_bytes| Signature::new(&sig_bytes)); + let mut account_updates_queue = ctx.account_updates_queue.lock().await; + + let slot_account_updates = match account_updates_queue.get_mut(&update_slot) { + Some(slot_account_updates) => slot_account_updates, + None => { + account_updates_queue.insert(update_slot, HashMap::default()); + account_updates_queue.get_mut(&update_slot).unwrap() + } + }; + slot_account_updates.insert( + address, + AccountUpdate { + address, + timestamp: Utc::now(), + slot: update_slot, + txn_signature, + write_version: Some(account_info.write_version), + account_data: account_info.try_into()?, + }, + ); + } else { + anyhow::bail!("Expected `transaction` in `UpdateOneof::Transaction` update"); + } + } + UpdateOneof::Slot(slot) => { + if slot.status == SubscribeUpdateSlotStatus::Confirmed as i32 + || slot.status == SubscribeUpdateSlotStatus::Finalized as i32 + { + let mut latest_slots = ctx.latest_slots_with_commitment.lock().await; + let slot_inserted = latest_slots.insert(slot.slot); + if slot_inserted && latest_slots.len() > ctx.config.slots_buffer_size as usize { + let oldest_slot = *latest_slots.first().unwrap(); + latest_slots.remove(&oldest_slot); + } + } + } + UpdateOneof::Block(block_update) => { + if let Some(block_time) = block_update.block_time { + ctx.timestamp.store(block_time.timestamp, Ordering::Relaxed); + } + } + UpdateOneof::Ping(_) => { + debug!("ping"); + } + _ => { + warn!("unknown update"); + } + } + + Ok(()) +} + +pub async fn update_account_map(ctx: Arc) { + loop { + let mut confirmed_account_updates: Vec = vec![]; + { + let mut account_updates_per_slot = ctx.account_updates_queue.lock().await; + let latest_slots_with_commitment = ctx.latest_slots_with_commitment.lock().await; + + // Remove all transactions received in a slot that has not been confirmed in allotted time + if let Some(oldest_slot_with_commitment) = latest_slots_with_commitment.first() { + account_updates_per_slot.retain(|slot, account_updates| { + if slot < oldest_slot_with_commitment { + debug!( + "throwing away txs {:?} from slot {}", + account_updates + .iter() + .map(|(address, _)| address.to_string()) + .collect_vec(), + slot + ); + } + + slot >= oldest_slot_with_commitment + }); + } + + // Add transactions from confirmed slots to the queue of transactions to be indexed + for (slot, slot_account_updates) in account_updates_per_slot.clone().iter() { + if let Some(latest_slot_with_commitment) = latest_slots_with_commitment.last() { + if slot > latest_slot_with_commitment { + break; // Ok because transactions_per_slot is sorted (BtreeMap) + } + } + + if latest_slots_with_commitment.contains(slot) { + confirmed_account_updates.extend(slot_account_updates.values().cloned()); + account_updates_per_slot.remove(slot); + } + } + } + + if confirmed_account_updates.is_empty() { + tokio::time::sleep(Duration::from_millis(10)).await; + continue; + } + + let mut accounts_snapshot = ctx.account_snapshot.lock().await; + for account_update in confirmed_account_updates { + if accounts_snapshot + .routing_lookup + .contains_key(&account_update.address) + { + accounts_snapshot + .udpate_entry(&account_update.address, &account_update.account_data); + } else { + accounts_snapshot + .create_entry(&account_update.address, &account_update.account_data) + .await; + + let non_program_accounts = accounts_snapshot + .routing_lookup + .iter() + .filter(|(_, routing_type)| match routing_type { + AccountRoutingType::MarginfiGroup => false, + AccountRoutingType::MarginfiAccount => false, + AccountRoutingType::Bank(_, bank_update_routing_type) => { + !matches!(bank_update_routing_type, BankUpdateRoutingType::State) + } + _ => true, + }) + .map(|(pubkey, _)| *pubkey) + .unique() + .collect_vec(); + let updated_geyser_config = + compute_geyser_config(&ctx.config, &non_program_accounts).await; + info!("updating geyser sub: {:?}", updated_geyser_config.accounts); + *ctx.geyser_subscription_config.lock().await = (true, updated_geyser_config); + } + } + } +} + +pub async fn push_transactions_to_bigquery(ctx: Arc) { + let bq_client = if let Some(gcp_sa_key) = ctx.config.gcp_sa_key.clone() { + let sa_key = parse_service_account_key(&gcp_sa_key).unwrap(); + gcp_bigquery_client::Client::from_service_account_key(sa_key, false) + .await + .unwrap() + } else { + gcp_bigquery_client::Client::from_application_default_credentials() + .await + .unwrap() + }; + + tokio::time::sleep(Duration::from_secs(5)).await; + while ctx.timestamp.load(Ordering::Relaxed) == 0 { + tokio::time::sleep(Duration::from_millis(100)).await; + } + + info!("Starting to generate snapshots"); + loop { + let start = Instant::now(); + let snapshot = ctx.account_snapshot.lock().await.clone(); + let timestamp = ctx.timestamp.load(Ordering::Relaxed); + + let all_group_metrics = snapshot + .marginfi_groups + .par_iter() + .map(|(marginfi_group_pk, marginfi_group)| { + ( + marginfi_group_pk, + MarginfiGroupMetrics::new( + timestamp, + marginfi_group_pk, + marginfi_group, + &snapshot, + ), + ) + }) + .collect::>(); + let all_bank_metrics = snapshot + .banks + .par_iter() + .map(|(bank_pk, bank_accounts)| { + ( + bank_pk, + LendingPoolBankMetrics::new(timestamp, bank_pk, bank_accounts, &snapshot), + ) + }) + .collect::>(); + let all_marginfi_account_metrics = snapshot + .marginfi_accounts + .par_iter() + .map(|(marginfi_account_pk, marginfi_account)| { + ( + marginfi_account_pk, + MarginfiAccountMetrics::new( + timestamp, + marginfi_account_pk, + marginfi_account, + &snapshot, + ), + ) + }) + .collect::>(); + + let elapsed = Instant::now() - start; + debug!("Time to create metrics: {:?}", elapsed); + + let mut insert_request = TableDataInsertAllRequest::new(); + all_group_metrics + .iter() + .for_each(|(id, metrics_result)| match metrics_result { + Ok(metrics) => insert_request.add_row(None, metrics.to_row()).unwrap(), + Err(err) => warn!("Failed to create metrics for marginfi group {id}: {err}"), + }); + let result = write_to_bq( + &bq_client, + &ctx.config.project_id, + &ctx.config.dataset_id, + &ctx.config.table_group, + timestamp, + insert_request, + ) + .await; + if let Err(error) = result { + warn!( + "Failed to write marginfi group metrics to bigquery: {}", + error + ); + } + + let mut insert_request = TableDataInsertAllRequest::new(); + all_bank_metrics + .iter() + .for_each(|(id, metrics_result)| match metrics_result { + Ok(metrics) => insert_request.add_row(None, metrics.to_row()).unwrap(), + Err(err) => warn!("Failed to create metrics for bank {id}: {err}"), + }); + let result = write_to_bq( + &bq_client, + &ctx.config.project_id, + &ctx.config.dataset_id, + &ctx.config.table_bank, + timestamp, + insert_request, + ) + .await; + if let Err(error) = result { + warn!( + "Failed to write lending pool bank metrics to bigquery: {}", + error + ); + } + + let insert_requests: Vec = all_marginfi_account_metrics + .chunks(7000) + .map(|metrics_results_chunk| { + let mut insert_request: TableDataInsertAllRequest = + TableDataInsertAllRequest::new(); + + metrics_results_chunk.iter().for_each( + |(id, metrics_result)| match metrics_result { + Ok(metrics) => insert_request.add_row(None, metrics.to_row()).unwrap(), + Err(err) => { + warn!("Failed to create metrics for marginfi account {id}: {err}") + } + }, + ); + + insert_request + }) + .collect::>(); + + for insert_request in insert_requests { + let result = write_to_bq( + &bq_client, + &ctx.config.project_id, + &ctx.config.dataset_id, + &ctx.config.table_account, + timestamp, + insert_request, + ) + .await; + if let Err(error) = result { + warn!( + "Failed to write marginfi account metrics to bigquery: {}", + error + ); + } + } + + tokio::time::sleep(Duration::from_secs(ctx.config.snap_interval)).await; + } +} + +pub async fn write_to_bq( + bq_client: &gcp_bigquery_client::Client, + project_id: &str, + dataset_id: &str, + table_id: &str, + timestamp: i64, + insert_request: TableDataInsertAllRequest, +) -> Result<()> { + let result = bq_client + .tabledata() + .insert_all(project_id, dataset_id, table_id, insert_request) + .await; + + let result = match result { + Ok(result) => result, + Err(err) => { + error!("Errors inserting for timestamp {}", timestamp); + error!("details: {:?}", err); + return Ok(()); + } + }; + + if let Some(errors) = result.insert_errors { + error!("Errors inserting for timestamp {}", timestamp); + error!("details:"); + errors.iter().for_each(|error| println!("-{:?}", error)); + } + + Ok(()) +} + +async fn monitor(ctx: Arc) { + let mut main_timing = Measure::start("main"); + + loop { + tokio::time::sleep(Duration::from_secs(ctx.config.monitor_interval)).await; + main_timing.stop(); + let latest_slots = ctx.latest_slots_with_commitment.lock().await.clone(); + let account_updates_queue = ctx.account_updates_queue.lock().await.clone(); + let earliest_block_with_commitment = latest_slots.first().unwrap_or(&0); + let latest_block_with_commitment = latest_slots.last().unwrap_or(&u64::MAX); + let earliest_pending_slot = account_updates_queue + .first_key_value() + .map(|(slot, _)| slot) + .unwrap_or(&0); + let latest_pending_slot = account_updates_queue + .first_key_value() + .map(|(slot, _)| slot) + .unwrap_or(&u64::MAX); + let stream_disconnection_count = ctx.stream_disconnection_count.load(Ordering::Relaxed); + let update_processing_error_count = + ctx.update_processing_error_count.load(Ordering::Relaxed); + let current_fetch_time = main_timing.as_s(); + + let account_updates_queue_size = ctx.account_updates_queue.lock().await.len(); + + info!( + "Time: {:.1}s | Tx Q size: {} | Stream disconnections: {} | Processing errors: {} | Earliest confirmed slot: {} | Latest confirmed slot: {} | Earliest pending slot: {} | Latest pending slot: {}", + current_fetch_time, + account_updates_queue_size, + stream_disconnection_count, + update_processing_error_count, + earliest_block_with_commitment, + latest_block_with_commitment, + earliest_pending_slot, + latest_pending_slot, + ); + } +} diff --git a/observability/indexer/src/common.rs b/observability/indexer/src/common.rs index ca60cf03..4e5330a9 100644 --- a/observability/indexer/src/common.rs +++ b/observability/indexer/src/common.rs @@ -1,6 +1,13 @@ +use backoff::{future::retry, ExponentialBackoffBuilder}; +use fixed::types::I80F48; +use fixed_macro::types::I80F48; +use futures::future::try_join_all; +use marginfi::state::{marginfi_account::MarginfiAccount, marginfi_group::Bank}; +use pyth_sdk_solana::PriceFeed; use serde::{Deserialize, Serialize}; -use solana_sdk::{pubkey::Pubkey, signature::Signature}; -use std::str::FromStr; +use solana_client::{client_error::ClientError, nonblocking::rpc_client::RpcClient}; +use solana_sdk::{instruction::AccountMeta, pubkey::Pubkey, signature::Signature}; +use std::{collections::HashMap, iter::zip, str::FromStr, time::Duration}; #[derive(Serialize, Deserialize, Debug, Clone)] pub struct Target { @@ -32,3 +39,110 @@ pub const DEFAULT_RPC_ENDPOINT: &str = "https://api.mainnet-beta.solana.com"; pub const DEFAULT_SIGNATURE_FETCH_LIMIT: usize = 1_000; pub const DEFAULT_MAX_PENDING_SIGNATURES: usize = 10_000; pub const DEFAULT_MONITOR_INTERVAL: u64 = 5; + +pub const EXP_10_I80F48: [I80F48; 15] = [ + I80F48!(1), + I80F48!(10), + I80F48!(100), + I80F48!(1_000), + I80F48!(10_000), + I80F48!(100_000), + I80F48!(1_000_000), + I80F48!(10_000_000), + I80F48!(100_000_000), + I80F48!(1_000_000_000), + I80F48!(10_000_000_000), + I80F48!(100_000_000_000), + I80F48!(1_000_000_000_000), + I80F48!(10_000_000_000_000), + I80F48!(100_000_000_000_000), +]; + +#[inline(always)] +pub fn pyth_price_to_fixed(price_feed: &PriceFeed) -> anyhow::Result { + let price = I80F48::from_num(price_feed.get_ema_price_unchecked().price); + let exponent = price_feed.get_ema_price_unchecked().expo; + let scaling_factor = EXP_10_I80F48[exponent.unsigned_abs() as usize]; + + let price = if exponent == 0 { + price + } else if exponent < 0 { + price.checked_div(scaling_factor).unwrap() + } else { + price.checked_mul(scaling_factor).unwrap() + }; + + Ok(price) +} + +pub async fn get_multiple_accounts_chunked( + rpc_client: &RpcClient, + keys: &[Pubkey], +) -> Result>, ClientError> { + let zips: Result, ClientError> = + try_join_all(keys.chunks(100).map(|pubkey_chunk| async move { + Ok(zip( + pubkey_chunk, + retry( + ExponentialBackoffBuilder::new() + .with_max_interval(Duration::from_secs(5)) + .build(), + || async { Ok(rpc_client.get_multiple_accounts(pubkey_chunk).await?) }, + ) + .await?, + )) + })) + .await; + + Ok(HashMap::from_iter(zips?.into_iter().flatten().filter_map( + |(key, account)| account.map(|account| (*key, account.data)), + ))) +} + +pub fn load_observation_account_metas( + marginfi_account: &MarginfiAccount, + banks_map: &HashMap, + include_banks: Vec, + exclude_banks: Vec, +) -> Vec { + let mut bank_pks = marginfi_account + .lending_account + .balances + .iter() + .filter_map(|balance| balance.active.then_some(balance.bank_pk)) + .collect::>(); + + for bank_pk in include_banks { + if !bank_pks.contains(&bank_pk) { + bank_pks.push(bank_pk); + } + } + + bank_pks.retain(|bank_pk| !exclude_banks.contains(bank_pk)); + + let mut banks = vec![]; + for bank_pk in bank_pks.clone() { + let bank = banks_map.get(&bank_pk).unwrap(); + banks.push(bank); + } + + let account_metas = banks + .iter() + .zip(bank_pks.iter()) + .flat_map(|(bank, bank_pk)| { + vec![ + AccountMeta { + pubkey: *bank_pk, + is_signer: false, + is_writable: false, + }, + AccountMeta { + pubkey: bank.config.oracle_keys[0], + is_signer: false, + is_writable: false, + }, + ] + }) + .collect::>(); + account_metas +} diff --git a/observability/indexer/src/entrypoint.rs b/observability/indexer/src/entrypoint.rs index 6f52452b..3881b679 100644 --- a/observability/indexer/src/entrypoint.rs +++ b/observability/indexer/src/entrypoint.rs @@ -1,21 +1,27 @@ +use crate::commands::create_table::TableType; +use crate::commands::index_accounts::{index_accounts, IndexAccountsConfig}; +use crate::commands::snapshot_accounts::{snapshot_accounts, SnapshotAccountsConfig}; use crate::commands::{ backfill::{backfill, BackfillConfig}, - create_table::{create_table}, + create_table::create_table, index_transactions::{index_transactions, IndexTransactionsConfig}, }; use anyhow::Result; use clap::Parser; use dotenv::dotenv; use envconfig::Envconfig; -use log::debug; use std::{panic, process}; -use crate::commands::create_table::TableType; -use crate::commands::index_accounts::{index_accounts, IndexAccountsConfig}; +use tracing::debug; +use tracing_subscriber::layer::SubscriberExt; +use tracing_subscriber::EnvFilter; pub const VERSION: &str = env!("CARGO_PKG_VERSION"); #[derive(Debug, Parser)] -pub struct GlobalOptions {} +pub struct GlobalOptions { + #[clap(long)] + pub pretty_log: bool, +} #[derive(Debug, Parser)] #[clap(version = VERSION)] @@ -45,6 +51,7 @@ pub enum Command { Backfill, IndexTransactions, IndexAccounts, + SnapshotAccounts, } #[tokio::main] @@ -56,11 +63,36 @@ pub async fn entry(opts: Opts) -> Result<()> { })); dotenv().ok(); - env_logger::init(); + + let filter = EnvFilter::from_default_env(); + let stackdriver = tracing_stackdriver::layer(); // writes to std::io::Stdout + let subscriber = tracing_subscriber::registry().with(filter); + if opts.global_config.pretty_log { + let subscriber = subscriber.with(tracing_subscriber::fmt::layer().compact()); + tracing::subscriber::set_global_default(subscriber).unwrap(); + } else { + let subscriber = subscriber.with(stackdriver); + tracing::subscriber::set_global_default(subscriber).unwrap(); + }; match opts.command { - Command::CreateTable { project_id, dataset_id, table_type, table_id, table_friendly_name, table_description } => { - create_table(project_id, dataset_id, table_id, table_type, table_friendly_name, table_description).await + Command::CreateTable { + project_id, + dataset_id, + table_type, + table_id, + table_friendly_name, + table_description, + } => { + create_table( + project_id, + dataset_id, + table_id, + table_type, + table_friendly_name, + table_description, + ) + .await } Command::Backfill => { let config = BackfillConfig::init_from_env().unwrap(); @@ -80,5 +112,11 @@ pub async fn entry(opts: Opts) -> Result<()> { index_accounts(config).await } + Command::SnapshotAccounts => { + let config = SnapshotAccountsConfig::init_from_env().unwrap(); + debug!("Config -> {:#?}", &config.clone()); + + snapshot_accounts(config).await + } } } diff --git a/observability/indexer/src/utils/big_query.rs b/observability/indexer/src/utils/big_query.rs index cfbc50b2..1c116b2f 100644 --- a/observability/indexer/src/utils/big_query.rs +++ b/observability/indexer/src/utils/big_query.rs @@ -2,6 +2,7 @@ use gcp_bigquery_client::model::{table_field_schema::TableFieldSchema, table_sch use lazy_static::lazy_static; pub const NOT_FOUND_CODE: i64 = 404; +pub const DATE_FORMAT_STR: &str = "%Y-%m-%d %H:%M:%S"; lazy_static! { pub static ref TRANSACTION_SCHEMA: TableSchema = TableSchema::new(vec![ @@ -18,7 +19,6 @@ lazy_static! { TableFieldSchema::string("meta"), TableFieldSchema::string("message"), ]); - pub static ref ACCOUNT_SCHEMA: TableSchema = TableSchema::new(vec![ TableFieldSchema::string("id"), TableFieldSchema::timestamp("created_at"), @@ -26,6 +26,8 @@ lazy_static! { TableFieldSchema::string("owner"), TableFieldSchema::big_numeric("slot"), TableFieldSchema::string("pubkey"), + TableFieldSchema::string("txn_signature"), + TableFieldSchema::big_numeric("write_version"), TableFieldSchema::big_numeric("lamports"), TableFieldSchema::bool("executable"), TableFieldSchema::big_numeric("rent_epoch"), @@ -33,4 +35,62 @@ lazy_static! { ]); } -pub const DATE_FORMAT_STR: &str = "%Y-%m-%d %H:%M:%S"; +lazy_static! { + pub static ref METRIC_MARGINFI_GROUP_SCHEMA: TableSchema = TableSchema::new(vec![ + TableFieldSchema::string("id"), + TableFieldSchema::timestamp("created_at"), + TableFieldSchema::timestamp("timestamp"), + TableFieldSchema::string("pubkey"), + TableFieldSchema::integer("marginfi_accounts_count"), + TableFieldSchema::integer("banks_count"), + TableFieldSchema::integer("mints_count"), + TableFieldSchema::float("total_assets_in_usd"), + TableFieldSchema::float("total_liabilities_in_usd"), + ]); + pub static ref METRIC_LENDING_POOL_BANK_SCHEMA: TableSchema = TableSchema::new(vec![ + TableFieldSchema::string("id"), + TableFieldSchema::timestamp("created_at"), + TableFieldSchema::timestamp("timestamp"), + TableFieldSchema::string("pubkey"), + TableFieldSchema::string("marginfi_group"), + TableFieldSchema::string("mint"), + TableFieldSchema::float("usd_price"), + TableFieldSchema::string("operational_state"), + TableFieldSchema::float("asset_weight_maintenance"), + TableFieldSchema::float("liability_weight_maintenance"), + TableFieldSchema::float("asset_weight_initial"), + TableFieldSchema::float("liability_weight_initial"), + TableFieldSchema::float("deposit_limit_in_tokens"), + TableFieldSchema::float("borrow_limit_in_tokens"), + TableFieldSchema::float("deposit_limit_in_usd"), + TableFieldSchema::float("borrow_limit_in_usd"), + TableFieldSchema::integer("lenders_count"), + TableFieldSchema::integer("borrowers_count"), + TableFieldSchema::float("deposit_rate"), + TableFieldSchema::float("borrow_rate"), + TableFieldSchema::float("group_fee"), + TableFieldSchema::float("insurance_fee"), + TableFieldSchema::float("total_assets_in_tokens"), + TableFieldSchema::float("total_liabilities_in_tokens"), + TableFieldSchema::float("total_assets_in_usd"), + TableFieldSchema::float("total_liabilities_in_usd"), + TableFieldSchema::float("liquidity_vault_balance"), + TableFieldSchema::float("insurance_vault_balance"), + TableFieldSchema::float("fee_vault_balance"), + ]); + pub static ref METRIC_MARGINFI_ACCOUNT_SCHEMA: TableSchema = TableSchema::new(vec![ + TableFieldSchema::string("id"), + TableFieldSchema::timestamp("created_at"), + TableFieldSchema::timestamp("timestamp"), + TableFieldSchema::string("pubkey"), + TableFieldSchema::string("marginfi_group"), + TableFieldSchema::string("owner"), + TableFieldSchema::float("total_assets_in_usd"), + TableFieldSchema::float("total_liabilities_in_usd"), + TableFieldSchema::float("total_assets_in_usd_maintenance"), + TableFieldSchema::float("total_liabilities_in_usd_maintenance"), + TableFieldSchema::float("total_assets_in_usd_initial"), + TableFieldSchema::float("total_liabilities_in_usd_initial"), + TableFieldSchema::string("positions"), + ]); +} diff --git a/observability/indexer/src/utils/errors.rs b/observability/indexer/src/utils/errors.rs new file mode 100644 index 00000000..a9f0013f --- /dev/null +++ b/observability/indexer/src/utils/errors.rs @@ -0,0 +1,7 @@ +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum GeyserServiceError { + #[error("parsing error in conversion from proto message")] + ProtoMessageConversionFailed, +} diff --git a/observability/indexer/src/utils/geyser_client.rs b/observability/indexer/src/utils/geyser_client.rs new file mode 100644 index 00000000..6d80f164 --- /dev/null +++ b/observability/indexer/src/utils/geyser_client.rs @@ -0,0 +1,153 @@ +use crate::utils::protos::{ + geyser::geyser_client::GeyserClient, SubscribeRequest, SubscribeUpdate, +}; +use anchor_client::anchor_lang::prelude::thiserror::Error; +use anyhow::Result; +use backoff::{future::retry, ExponentialBackoff}; +use futures::stream::once; +use tonic::metadata::{Ascii, MetadataValue}; +use tonic::{ + codegen::InterceptedService, + service::Interceptor, + transport::{Channel, ClientTlsConfig, Endpoint}, + Request, Response, Status, Streaming, +}; +use tracing::{error, info}; + +pub struct RequestInterceptor { + auth_token: String, +} + +impl Interceptor for RequestInterceptor { + fn call(&mut self, mut request: Request<()>) -> Result, Status> { + request + .metadata_mut() + .insert("x-token", self.auth_token.parse().unwrap()); + Ok(request) + } +} + +pub async fn get_geyser_client( + url: String, + auth_token: String, +) -> Result>> { + let mut endpoint = Endpoint::from_shared(url.clone())?; + + if url.contains("https") { + endpoint = endpoint.tls_config(ClientTlsConfig::new())?; + } + let channel = endpoint.connect().await.unwrap(); + + Ok(GeyserClient::with_interceptor( + channel, + RequestInterceptor { auth_token }, + )) +} + +#[derive(Debug, Error)] +pub enum Error { + #[error("XToken: {0}")] + XToken(String), + + #[error("Invalid URI {0}")] + InvalidUri(String), + + #[error("RetrySubscribe")] + RetrySubscribe(anyhow::Error), +} + +#[derive(Debug)] +pub struct RetryChannel { + x_token: Option>, + channel: Channel, +} + +impl RetryChannel { + /// Establish a channel to tonic endpoint + /// The channel does not attempt to connect to the endpoint until first use + pub fn new(endpoint_str: String, x_token_str: String) -> Result { + let endpoint: Endpoint; + + // the client should fail immediately if the x-token is invalid + let x_token: Option> = + match x_token_str.parse::>() { + Ok(metadata) => Some(metadata), + Err(_) => return Err(Error::XToken(x_token_str)), + }; + + let res = Channel::from_shared(endpoint_str.clone()); + match res { + Err(e) => { + error!("{}", e); + return Err(Error::InvalidUri(endpoint_str)); + } + Ok(_endpoint) => { + if _endpoint.uri().scheme_str() == Some("https") { + match _endpoint.tls_config(ClientTlsConfig::new()) { + Err(e) => { + error!("{}", e); + return Err(Error::InvalidUri(endpoint_str)); + } + Ok(e) => endpoint = e, + } + } else { + endpoint = _endpoint; + } + } + } + let channel = endpoint.connect_lazy(); + + Ok(Self { x_token, channel }) + } + + /// Create a new GeyserClient client with Auth interceptor + /// Clients require `&mut self`, due to `Tonic::transport::Channel` limitations, however + /// creating new clients is cheap and thus can be used as a work around for ease of use. + pub fn client(&self) -> RetryClient) -> InterceptedRequestResult + '_> { + let client = + GeyserClient::with_interceptor(self.channel.clone(), move |mut req: Request<()>| { + if let Some(x_token) = self.x_token.clone() { + req.metadata_mut().insert("x-token", x_token); + } + Ok(req) + }); + RetryClient { client } + } + + pub async fn subscribe_retry( + &self, + subscribe_request: &SubscribeRequest, + ) -> Result, anyhow::Error> { + // The default exponential backoff strategy intervals: + // [500ms, 750ms, 1.125s, 1.6875s, 2.53125s, 3.796875s, 5.6953125s, + // 8.5s, 12.8s, 19.2s, 28.8s, 43.2s, 64.8s, 97s, ... ] + retry(ExponentialBackoff::default(), || async { + info!("Retry to connect to the server"); + let mut client = self.client(); + Ok(client.subscribe(subscribe_request).await?) + }) + .await + } +} + +type InterceptedRequestResult = std::result::Result, Status>; + +pub struct RetryClient) -> InterceptedRequestResult> { + client: GeyserClient>, +} + +impl) -> InterceptedRequestResult> RetryClient { + pub async fn subscribe( + &mut self, + subscribe_request: &SubscribeRequest, + ) -> Result, anyhow::Error> { + let subscribe_request = subscribe_request.clone(); + let response: Response> = self + .client + .subscribe(once(async move { subscribe_request })) + .await?; + let stream: Streaming = response.into_inner(); + + Ok(stream) + } +} diff --git a/observability/indexer/src/utils/marginfi_account_dup.rs b/observability/indexer/src/utils/marginfi_account_dup.rs new file mode 100644 index 00000000..2ae8f4f7 --- /dev/null +++ b/observability/indexer/src/utils/marginfi_account_dup.rs @@ -0,0 +1,172 @@ +use fixed::types::I80F48; +use marginfi::{ + constants::TOTAL_ASSET_VALUE_INIT_LIMIT_INACTIVE, + state::{ + marginfi_account::{ + calc_asset_value, Balance, BalanceSide, MarginfiAccount, RiskRequirementType, + WeightType, + }, + marginfi_group::Bank, + price::{OraclePriceFeedAdapter, PriceAdapter}, + }, +}; +use solana_sdk::pubkey::Pubkey; + +pub struct BankAccountWithPriceFeed2 { + bank: Bank, + price_feed: OraclePriceFeedAdapter, + balance: Balance, +} + +impl BankAccountWithPriceFeed2 { + pub fn load( + marginfi_account: &MarginfiAccount, + banks: &std::collections::HashMap, + price_feeds: &std::collections::HashMap, + ) -> anyhow::Result> { + marginfi_account + .lending_account + .balances + .into_iter() + .filter(|balance| balance.active) + .enumerate() + .map(|(_, balance)| { + let bank = banks.get(&balance.bank_pk).cloned().unwrap(); + let price_feed = price_feeds + .get(&bank.config.oracle_keys[0]) + .cloned() + .unwrap(); + Ok(BankAccountWithPriceFeed2 { + bank, + price_feed, + balance, + }) + }) + .collect::>>() + } + + #[inline(always)] + pub fn calc_weighted_assets_and_liabilities_values( + &self, + weight_type: WeightType, + ) -> anyhow::Result<(I80F48, I80F48)> { + let (worst_price, best_price) = self.price_feed.get_price_range()?; + let (mut asset_weight, liability_weight) = self.bank.config.get_weights(weight_type); + let mint_decimals = self.bank.mint_decimals; + + let asset_amount = self + .bank + .get_asset_amount(self.balance.asset_shares.into())?; + let liability_amount = self + .bank + .get_liability_amount(self.balance.liability_shares.into())?; + + if matches!(weight_type, WeightType::Initial) + && self.bank.config.total_asset_value_init_limit + != TOTAL_ASSET_VALUE_INIT_LIMIT_INACTIVE + { + let bank_total_assets_value = calc_asset_value( + self.bank + .get_asset_amount(self.bank.total_asset_shares.into())?, + worst_price, + mint_decimals, + None, + )?; + + let total_asset_value_init_limit = + I80F48::from_num(self.bank.config.total_asset_value_init_limit); + + if bank_total_assets_value > total_asset_value_init_limit { + let discount = total_asset_value_init_limit + .checked_div(bank_total_assets_value) + .unwrap(); + + asset_weight = asset_weight.checked_mul(discount).unwrap(); + } + } + + Ok(( + calc_asset_value(asset_amount, worst_price, mint_decimals, Some(asset_weight))?, + calc_asset_value( + liability_amount, + best_price, + mint_decimals, + Some(liability_weight), + )?, + )) + } + + #[inline] + pub fn is_empty(&self, side: BalanceSide) -> bool { + self.balance.is_empty(side) + } +} + +pub struct RiskEngine2 { + bank_accounts_with_price: Vec, +} + +impl RiskEngine2 { + pub fn load( + marginfi_account: &MarginfiAccount, + banks: &std::collections::HashMap, + price_feeds: &std::collections::HashMap, + ) -> anyhow::Result { + Ok(Self { + bank_accounts_with_price: BankAccountWithPriceFeed2::load( + marginfi_account, + banks, + price_feeds, + )?, + }) + } + + /// Returns the total assets and liabilities of the account in the form of (assets, liabilities) + pub fn get_account_health_components( + &self, + requirement_type: RiskRequirementType, + ) -> anyhow::Result<(I80F48, I80F48)> { + let mut total_assets = I80F48::ZERO; + let mut total_liabilities = I80F48::ZERO; + + for a in &self.bank_accounts_with_price { + let (assets, liabilities) = + a.calc_weighted_assets_and_liabilities_values(requirement_type.to_weight_type())?; + + total_assets = total_assets.checked_add(assets).unwrap(); + total_liabilities = total_liabilities.checked_add(liabilities).unwrap(); + } + + Ok((total_assets, total_liabilities)) + } + + pub fn get_equity_components(&self) -> anyhow::Result<(I80F48, I80F48)> { + self.bank_accounts_with_price + .iter() + .map(|a: &BankAccountWithPriceFeed2| { + a.calc_weighted_assets_and_liabilities_values(WeightType::Equity) + }) + .try_fold( + (I80F48::ZERO, I80F48::ZERO), + |(total_assets, total_liabilities), res| { + let (assets, liabilities) = res?; + let total_assets_sum = total_assets.checked_add(assets).unwrap(); + let total_liabilities_sum = total_liabilities.checked_add(liabilities).unwrap(); + + Ok::<_, anyhow::Error>((total_assets_sum, total_liabilities_sum)) + }, + ) + } + + pub fn get_account_health( + &self, + requirement_type: RiskRequirementType, + ) -> anyhow::Result { + let (total_weighted_assets, total_weighted_liabilities) = + self.get_account_health_components(requirement_type)?; + + Ok(total_weighted_assets + .checked_sub(total_weighted_liabilities) + .unwrap()) + } +} diff --git a/observability/indexer/src/utils/metrics.rs b/observability/indexer/src/utils/metrics.rs new file mode 100644 index 00000000..911a73a9 --- /dev/null +++ b/observability/indexer/src/utils/metrics.rs @@ -0,0 +1,602 @@ +use super::marginfi_account_dup::RiskEngine2; +use crate::utils::big_query::DATE_FORMAT_STR; +use crate::utils::snapshot::{BankAccounts, OracleData, Snapshot}; +use anyhow::anyhow; +use chrono::{NaiveDateTime, Utc}; +use fixed::types::I80F48; +use fixed_macro::types::I80F48; +use itertools::Itertools; +use marginfi::constants::ZERO_AMOUNT_THRESHOLD; +use marginfi::prelude::MarginfiGroup; +use marginfi::state::marginfi_account::{ + calc_asset_value, MarginfiAccount, RiskRequirementType, WeightType, +}; +use marginfi::state::marginfi_group::BankOperationalState; +use marginfi::state::price::OraclePriceFeedAdapter; +use serde::Serialize; +use solana_sdk::pubkey::Pubkey; +use std::collections::HashMap; +use uuid::Uuid; + +#[derive(Debug, Serialize)] +pub struct MarginfiGroupMetricsRow { + pub id: String, + pub created_at: String, + pub timestamp: String, + pub pubkey: String, + pub marginfi_accounts_count: u32, + pub banks_count: u32, + pub mints_count: u32, + pub total_assets_in_usd: f64, + pub total_liabilities_in_usd: f64, +} + +#[derive(Debug)] +pub struct MarginfiGroupMetrics { + pub timestamp: i64, + pub pubkey: Pubkey, + pub marginfi_accounts_count: u32, + pub banks_count: u32, + pub mints_count: u32, + pub total_assets_in_usd: f64, + pub total_liabilities_in_usd: f64, +} + +impl MarginfiGroupMetrics { + pub fn new( + timestamp: i64, + marginfi_group_pk: &Pubkey, + _marginfi_group: &MarginfiGroup, + snapshot: &Snapshot, + ) -> anyhow::Result { + let group_banks_iter = snapshot + .banks + .iter() + .filter(|(_, bank_accounts)| bank_accounts.bank.group.eq(marginfi_group_pk)); + let group_marginfi_accounts_iter = snapshot + .marginfi_accounts + .iter() + .filter(|(_, marginfi_account)| marginfi_account.group.eq(marginfi_group_pk)); + + let ( + total_assets_usd, + total_liabilities_usd, + _total_assets_usd_maint, + _total_liabilities_usd_maint, + ) = group_banks_iter.clone().try_fold( + (0.0, 0.0, 0.0, 0.0), + |mut sums, (bank_pk, bank_accounts)| -> anyhow::Result<(f64, f64, f64, f64)> { + let total_asset_share = bank_accounts.bank.total_asset_shares; + let total_liability_share = bank_accounts.bank.total_liability_shares; + let price_feed_pk = bank_accounts.bank.config.oracle_keys[0]; + let (asset_weight, liability_weight) = bank_accounts + .bank + .config + .get_weights(WeightType::Maintenance); + let price = snapshot + .price_feeds + .get(&price_feed_pk) + .ok_or_else(|| { + anyhow!( + "Price feed {} not found for bank {}", + price_feed_pk, + bank_pk + ) + })? + .get_price(); + + let asset_value_usd = calc_asset_value( + bank_accounts + .bank + .get_asset_amount(total_asset_share.into())?, + price, + bank_accounts.bank.mint_decimals, + None, + )? + .to_num::(); + let asset_value_usd_maint = calc_asset_value( + bank_accounts + .bank + .get_asset_amount(total_asset_share.into())?, + price, + bank_accounts.bank.mint_decimals, + Some(asset_weight), + )? + .to_num::(); + let liability_value_usd = calc_asset_value( + bank_accounts + .bank + .get_liability_amount(total_liability_share.into())?, + price, + bank_accounts.bank.mint_decimals, + None, + )? + .to_num::(); + let liability_value_usd_maint = calc_asset_value( + bank_accounts + .bank + .get_liability_amount(total_liability_share.into())?, + price, + bank_accounts.bank.mint_decimals, + Some(liability_weight), + )? + .to_num::(); + + sums.0 += asset_value_usd; + sums.1 += liability_value_usd; + sums.2 += asset_value_usd_maint; + sums.3 += liability_value_usd_maint; + + Ok(sums) + }, + )?; + + Ok(Self { + timestamp, + pubkey: *marginfi_group_pk, + marginfi_accounts_count: group_marginfi_accounts_iter.count() as u32, + banks_count: group_banks_iter.clone().count() as u32, + mints_count: group_banks_iter + .unique_by(|(_, bank_accounts)| bank_accounts.bank.mint) + .collect_vec() + .len() as u32, + total_assets_in_usd: total_assets_usd, + total_liabilities_in_usd: total_liabilities_usd, + }) + } + + pub fn to_row(&self) -> MarginfiGroupMetricsRow { + MarginfiGroupMetricsRow { + id: Uuid::new_v4().to_string(), + created_at: Utc::now().format(DATE_FORMAT_STR).to_string(), + timestamp: NaiveDateTime::from_timestamp_opt(self.timestamp, 0) + .unwrap() + .format(DATE_FORMAT_STR) + .to_string(), + pubkey: self.pubkey.to_string(), + marginfi_accounts_count: self.marginfi_accounts_count, + banks_count: self.banks_count, + mints_count: self.mints_count, + total_assets_in_usd: self.total_assets_in_usd, + total_liabilities_in_usd: self.total_liabilities_in_usd, + } + } +} + +#[derive(Debug, Serialize)] +pub struct LendingPoolBankMetricsRow { + pub id: String, + pub created_at: String, + pub timestamp: String, + pub pubkey: String, + pub marginfi_group: String, + pub mint: String, + pub usd_price: f64, + pub operational_state: String, + pub asset_weight_maintenance: f64, + pub liability_weight_maintenance: f64, + pub asset_weight_initial: f64, + pub liability_weight_initial: f64, + pub deposit_limit_in_tokens: f64, + pub borrow_limit_in_tokens: f64, + pub deposit_limit_in_usd: f64, + pub borrow_limit_in_usd: f64, + pub lenders_count: u32, + pub borrowers_count: u32, + pub deposit_rate: f64, + pub borrow_rate: f64, + pub group_fee: f64, + pub insurance_fee: f64, + pub total_assets_in_tokens: f64, + pub total_liabilities_in_tokens: f64, + pub total_assets_in_usd: f64, + pub total_liabilities_in_usd: f64, + pub liquidity_vault_balance: f64, + pub insurance_vault_balance: f64, + pub fee_vault_balance: f64, +} + +#[derive(Debug)] +pub struct LendingPoolBankMetrics { + pub timestamp: i64, + pub pubkey: Pubkey, + pub marginfi_group: Pubkey, + pub mint: Pubkey, + pub usd_price: f64, + pub operational_state: BankOperationalState, + pub asset_weight_maintenance: f64, + pub liability_weight_maintenance: f64, + pub asset_weight_initial: f64, + pub liability_weight_initial: f64, + pub deposit_limit_in_tokens: f64, + pub borrow_limit_in_tokens: f64, + pub deposit_limit_in_usd: f64, + pub borrow_limit_in_usd: f64, + pub lenders_count: u32, + pub borrowers_count: u32, + pub deposit_rate: f64, + pub borrow_rate: f64, + pub group_fee: f64, + pub insurance_fee: f64, + pub total_assets_in_tokens: f64, + pub total_liabilities_in_tokens: f64, + pub total_assets_in_usd: f64, + pub total_liabilities_in_usd: f64, + pub liquidity_vault_balance: f64, + pub insurance_vault_balance: f64, + pub fee_vault_balance: f64, +} + +impl LendingPoolBankMetrics { + pub fn new( + timestamp: i64, + bank_pk: &Pubkey, + bank_accounts: &BankAccounts, + snapshot: &Snapshot, + ) -> anyhow::Result { + let total_asset_share = bank_accounts.bank.total_asset_shares; + let total_liability_share = bank_accounts.bank.total_liability_shares; + let (asset_weight_maintenance, liability_weight_maintenance) = bank_accounts + .bank + .config + .get_weights(WeightType::Maintenance); + let (asset_weight_initial, liability_weight_initial) = + bank_accounts.bank.config.get_weights(WeightType::Initial); + let price_feed_pk = bank_accounts.bank.config.oracle_keys[0]; + let price = snapshot + .price_feeds + .get(&price_feed_pk) + .ok_or_else(|| { + anyhow!( + "Price feed {} not found for bank {}", + price_feed_pk, + bank_pk + ) + })? + .get_price(); + + let deposit_limit_usd = calc_asset_value( + bank_accounts.bank.config.deposit_limit.into(), + price, + bank_accounts.bank.mint_decimals, + None, + )? + .to_num::(); + let borrow_limit_usd = calc_asset_value( + bank_accounts.bank.config.borrow_limit.into(), + price, + bank_accounts.bank.mint_decimals, + None, + )? + .to_num::(); + + let asset_amount = bank_accounts + .bank + .get_asset_amount(total_asset_share.into())?; + let asset_value_usd = + calc_asset_value(asset_amount, price, bank_accounts.bank.mint_decimals, None)? + .to_num::(); + let liability_amount = bank_accounts + .bank + .get_liability_amount(total_liability_share.into())?; + let liability_value_usd = calc_asset_value( + liability_amount, + price, + bank_accounts.bank.mint_decimals, + None, + )? + .to_num::(); + + let lenders_count = snapshot + .marginfi_accounts + .iter() + .filter(|(_, account)| { + account.lending_account.balances.iter().any(|a| { + a.active + && I80F48::from(a.asset_shares).gt(&ZERO_AMOUNT_THRESHOLD) + && a.bank_pk.eq(bank_pk) + }) + }) + .count() as u32; + let borrowers_count = snapshot + .marginfi_accounts + .iter() + .filter(|(_, account)| { + account.lending_account.balances.iter().any(|a| { + a.active + && I80F48::from(a.liability_shares).gt(&ZERO_AMOUNT_THRESHOLD) + && a.bank_pk.eq(bank_pk) + }) + }) + .count() as u32; + + let utilization_rate = if asset_amount.is_positive() { + liability_amount + .checked_div(asset_amount) + .ok_or_else(|| anyhow!("Bad math during UR calc"))? + } else { + I80F48::ZERO + }; + let (lending_apr, borrowing_apr, group_fee_apr, insurance_fee_apr) = bank_accounts + .bank + .config + .interest_rate_config + .calc_interest_rate(utilization_rate) + .ok_or_else(|| anyhow!("Bad math during IR calcs"))?; + + Ok(Self { + timestamp, + pubkey: *bank_pk, + marginfi_group: bank_accounts.bank.group, + mint: bank_accounts.bank.mint, + usd_price: price.to_num::(), + operational_state: bank_accounts.bank.config.operational_state, + asset_weight_maintenance: asset_weight_maintenance.to_num::(), + liability_weight_maintenance: liability_weight_maintenance.to_num::(), + asset_weight_initial: asset_weight_initial.to_num::(), + liability_weight_initial: liability_weight_initial.to_num::(), + deposit_limit_in_tokens: bank_accounts.bank.config.deposit_limit as f64 + / (10i64.pow(bank_accounts.bank.mint_decimals as u32) as f64), + borrow_limit_in_tokens: bank_accounts.bank.config.borrow_limit as f64 + / (10i64.pow(bank_accounts.bank.mint_decimals as u32) as f64), + deposit_limit_in_usd: deposit_limit_usd, + borrow_limit_in_usd: borrow_limit_usd, + lenders_count, + borrowers_count, + deposit_rate: lending_apr.to_num::(), + borrow_rate: borrowing_apr.to_num::(), + group_fee: group_fee_apr.to_num::(), + insurance_fee: insurance_fee_apr.to_num::(), + total_assets_in_tokens: asset_amount.to_num::() + / (10i64.pow(bank_accounts.bank.mint_decimals as u32) as f64), + total_liabilities_in_tokens: liability_amount.to_num::() + / (10i64.pow(bank_accounts.bank.mint_decimals as u32) as f64), + total_assets_in_usd: asset_value_usd, + total_liabilities_in_usd: liability_value_usd, + liquidity_vault_balance: (bank_accounts.liquidity_vault_token_account.amount as f64) + / (10i64.pow(bank_accounts.bank.mint_decimals as u32) as f64), + insurance_vault_balance: (bank_accounts.insurance_vault_token_account.amount as f64) + / (10i64.pow(bank_accounts.bank.mint_decimals as u32) as f64), + fee_vault_balance: (bank_accounts.fee_vault_token_account.amount as f64) + / (10i64.pow(bank_accounts.bank.mint_decimals as u32) as f64), + }) + } + + pub fn to_row(&self) -> LendingPoolBankMetricsRow { + LendingPoolBankMetricsRow { + id: Uuid::new_v4().to_string(), + created_at: Utc::now().format(DATE_FORMAT_STR).to_string(), + timestamp: NaiveDateTime::from_timestamp_opt(self.timestamp, 0) + .unwrap() + .format(DATE_FORMAT_STR) + .to_string(), + pubkey: self.pubkey.to_string(), + marginfi_group: self.marginfi_group.to_string(), + mint: self.mint.to_string(), + usd_price: self.usd_price, + operational_state: self.operational_state.to_string(), + asset_weight_maintenance: self.asset_weight_maintenance, + liability_weight_maintenance: self.liability_weight_maintenance, + asset_weight_initial: self.asset_weight_initial, + liability_weight_initial: self.liability_weight_initial, + deposit_limit_in_tokens: self.deposit_limit_in_tokens, + borrow_limit_in_tokens: self.borrow_limit_in_tokens, + deposit_limit_in_usd: self.deposit_limit_in_usd, + borrow_limit_in_usd: self.borrow_limit_in_usd, + lenders_count: self.lenders_count, + borrowers_count: self.borrowers_count, + deposit_rate: self.deposit_rate, + borrow_rate: self.borrow_rate, + group_fee: self.group_fee, + insurance_fee: self.insurance_fee, + total_assets_in_tokens: self.total_assets_in_tokens, + total_liabilities_in_tokens: self.total_liabilities_in_tokens, + total_assets_in_usd: self.total_assets_in_usd, + total_liabilities_in_usd: self.total_liabilities_in_usd, + liquidity_vault_balance: self.liquidity_vault_balance, + insurance_vault_balance: self.insurance_vault_balance, + fee_vault_balance: self.fee_vault_balance, + } + } +} + +#[derive(Debug, Serialize)] +pub struct MarginfiAccountMetricsRow { + pub id: String, + pub created_at: String, + pub timestamp: String, + pub pubkey: String, + pub marginfi_group: String, + pub owner: String, + pub total_assets_in_usd: f64, + pub total_liabilities_in_usd: f64, + pub total_assets_in_usd_maintenance: f64, + pub total_liabilities_in_usd_maintenance: f64, + pub total_assets_in_usd_initial: f64, + pub total_liabilities_in_usd_initial: f64, + pub positions: String, +} + +#[derive(Debug, Serialize)] +pub struct PositionsSummary { + pub bank: String, + pub mint: String, + pub is_asset: bool, + pub amount: f64, + pub usd_value: f64, + pub usd_value_maintenance: f64, + pub usd_value_initial: f64, + pub price: f64, +} + +#[derive(Debug)] +pub struct MarginfiAccountMetrics { + pub timestamp: i64, + pub pubkey: Pubkey, + pub marginfi_group: Pubkey, + pub owner: Pubkey, + pub total_assets_in_usd: f64, + pub total_liabilities_in_usd: f64, + pub total_assets_in_usd_maintenance: f64, + pub total_liabilities_in_usd_maintenance: f64, + pub total_assets_in_usd_initial: f64, + pub total_liabilities_in_usd_initial: f64, + pub positions: Vec, +} + +impl MarginfiAccountMetrics { + pub fn new( + timestamp: i64, + marginfi_account_pk: &Pubkey, + marginfi_account: &MarginfiAccount, + snapshot: &Snapshot, + ) -> anyhow::Result { + let banks = HashMap::from_iter( + snapshot + .banks + .iter() + .map(|(bank_pk, bank_accounts)| (*bank_pk, bank_accounts.clone().bank)), + ); + let price_feeds = + HashMap::from_iter(snapshot.price_feeds.iter().map(|(oracle_pk, oracle_data)| { + match oracle_data { + OracleData::Pyth(price_feed) => ( + *oracle_pk, + OraclePriceFeedAdapter::PythEma(price_feed.clone()), + ), + OracleData::Switchboard(pf) => ( + *oracle_pk, + OraclePriceFeedAdapter::SwitchboardV2(pf.clone()), + ), + } + })); + + let risk_engine = RiskEngine2::load(marginfi_account, &banks, &price_feeds)?; + + let (total_assets_usd, total_liabilities_usd) = risk_engine.get_equity_components()?; + let (total_assets_usd, total_liabilities_usd) = ( + total_assets_usd.to_num::(), + total_liabilities_usd.to_num::(), + ); + let (total_assets_usd_maintenance, total_liabilities_usd_maintenance) = + risk_engine.get_account_health_components(RiskRequirementType::Maintenance)?; + let (total_assets_usd_maintenance, total_liabilities_usd_maintenance) = ( + total_assets_usd_maintenance.to_num::(), + total_liabilities_usd_maintenance.to_num::(), + ); + let (total_assets_usd_initial, total_liabilities_usd_initial) = + risk_engine.get_account_health_components(RiskRequirementType::Initial)?; + let (total_assets_usd_initial, total_liabilities_usd_initial) = ( + total_assets_usd_initial.to_num::(), + total_liabilities_usd_initial.to_num::(), + ); + + let positions = marginfi_account + .lending_account + .balances + .iter() + .filter(|balance| balance.active) + .map(|balance| { + let bank = banks.get(&balance.bank_pk).unwrap(); + let mint = bank.mint; + let (asset_shares, liability_shares): (I80F48, I80F48) = + (balance.asset_shares.into(), balance.liability_shares.into()); + let (asset_weight_maintenance, liability_weight_maintenance) = + bank.config.get_weights(WeightType::Maintenance); + let (asset_weight_initial, liability_weight_initial) = + bank.config.get_weights(WeightType::Initial); + let is_asset = asset_shares.gt(&I80F48!(0.0001)); + + let price_feed_pk = bank.config.oracle_keys[0]; + let price = snapshot + .price_feeds + .get(&price_feed_pk) + .ok_or_else(|| { + anyhow!( + "Price feed {} not found for bank {}", + &price_feed_pk, + &balance.bank_pk + ) + }) + .unwrap() + .get_price(); + + let (amount, weight_maintenance, weight_initial) = if is_asset { + ( + bank.get_asset_amount(asset_shares) + .map_err(|_| anyhow!("Bad math during positions summarizing")) + .unwrap(), + asset_weight_maintenance, + asset_weight_initial, + ) + } else { + ( + bank.get_asset_amount(liability_shares) + .map_err(|_| anyhow!("Bad math during positions summarizing")) + .unwrap(), + liability_weight_maintenance, + liability_weight_initial, + ) + }; + + let usd_value = calc_asset_value(amount, price, bank.mint_decimals, None) + .unwrap() + .to_num::(); + let usd_value_maintenance = + calc_asset_value(amount, price, bank.mint_decimals, Some(weight_maintenance)) + .unwrap() + .to_num::(); + let usd_value_initial = + calc_asset_value(amount, price, bank.mint_decimals, Some(weight_initial)) + .unwrap() + .to_num::(); + + PositionsSummary { + bank: balance.bank_pk.to_string(), + mint: mint.to_string(), + is_asset, + amount: amount.to_num::() / (10i64.pow(bank.mint_decimals as u32) as f64), + usd_value, + usd_value_maintenance, + usd_value_initial, + price: price.to_num::(), + } + }) + .collect_vec(); + + Ok(Self { + timestamp, + pubkey: *marginfi_account_pk, + marginfi_group: marginfi_account.group, + owner: marginfi_account.authority, + total_assets_in_usd: total_assets_usd, + total_liabilities_in_usd: total_liabilities_usd, + total_assets_in_usd_maintenance: total_assets_usd_maintenance, + total_liabilities_in_usd_maintenance: total_liabilities_usd_maintenance, + total_assets_in_usd_initial: total_assets_usd_initial, + total_liabilities_in_usd_initial: total_liabilities_usd_initial, + positions, + }) + } + + pub fn to_row(&self) -> MarginfiAccountMetricsRow { + MarginfiAccountMetricsRow { + id: Uuid::new_v4().to_string(), + created_at: Utc::now().format(DATE_FORMAT_STR).to_string(), + timestamp: NaiveDateTime::from_timestamp_opt(self.timestamp, 0) + .unwrap() + .format(DATE_FORMAT_STR) + .to_string(), + pubkey: self.pubkey.to_string(), + marginfi_group: self.marginfi_group.to_string(), + owner: self.owner.to_string(), + total_assets_in_usd: self.total_assets_in_usd, + total_liabilities_in_usd: self.total_liabilities_in_usd, + total_assets_in_usd_maintenance: self.total_assets_in_usd_maintenance, + total_liabilities_in_usd_maintenance: self.total_liabilities_in_usd_maintenance, + total_assets_in_usd_initial: self.total_assets_in_usd_initial, + total_liabilities_in_usd_initial: self.total_liabilities_in_usd_initial, + positions: serde_json::to_string(&self.positions).unwrap(), + } + } +} diff --git a/observability/indexer/src/utils/mod.rs b/observability/indexer/src/utils/mod.rs index d2341828..88c1d850 100644 --- a/observability/indexer/src/utils/mod.rs +++ b/observability/indexer/src/utils/mod.rs @@ -1,3 +1,8 @@ pub mod big_query; +pub mod errors; +pub mod geyser_client; +pub mod marginfi_account_dup; +pub mod metrics; pub mod protos; +pub mod snapshot; pub mod transactions_crawler; diff --git a/observability/indexer/src/utils/protos.rs b/observability/indexer/src/utils/protos.rs index d79b2ded..0abc79f3 100644 --- a/observability/indexer/src/utils/protos.rs +++ b/observability/indexer/src/utils/protos.rs @@ -18,8 +18,10 @@ pub use geyser::*; pub use solana::storage::confirmed_block::*; mod conversion { + use crate::utils::errors::GeyserServiceError; use itertools::Itertools; use solana_account_decoder::parse_token::UiTokenAmount; + use solana_sdk::account::Account; use solana_sdk::{ hash::Hash, instruction::CompiledInstruction, @@ -32,34 +34,24 @@ mod conversion { signature::Signature, transaction::{TransactionError, VersionedTransaction}, }; - use solana_sdk::account::Account; use solana_transaction_status::{ InnerInstructions, Reward, RewardType, TransactionStatusMeta, TransactionTokenBalance, VersionedTransactionWithStatusMeta, }; - // impl From for CompiledInstruction { - // fn from(instruction_proto: super::CompiledInstruction) -> Self { - // Self { - // instruction: CompiledInstruction { - // program_id_index: instruction_proto.program_id_index as u8, - // accounts: instruction_proto.accounts, - // data: instruction_proto.data, - // }, - // stack_height: None, - // } - // } - // } + impl TryFrom for Account { + type Error = GeyserServiceError; - impl From for Account { - fn from(account_data_proto: super::SubscribeUpdateAccountInfo) -> Self { - Self { + fn try_from( + account_data_proto: super::SubscribeUpdateAccountInfo, + ) -> Result { + Ok(Self { data: account_data_proto.data, owner: Pubkey::new(&account_data_proto.owner), lamports: account_data_proto.lamports, executable: account_data_proto.executable, rent_epoch: account_data_proto.rent_epoch, - } + }) } } @@ -83,73 +75,88 @@ mod conversion { } } - impl From for MessageAddressTableLookup { - fn from(lut_proto: super::MessageAddressTableLookup) -> Self { - Self { - account_key: Pubkey::new(lut_proto.account_key.as_slice()), + impl TryFrom for MessageAddressTableLookup { + type Error = GeyserServiceError; + + fn try_from(lut_proto: super::MessageAddressTableLookup) -> Result { + Ok(Self { + account_key: Pubkey::new(&lut_proto.account_key), writable_indexes: lut_proto.writable_indexes, readonly_indexes: lut_proto.readonly_indexes, - } + }) } } - impl From for legacy::Message { - fn from(message_proto: super::Message) -> Self { + impl TryFrom for legacy::Message { + type Error = GeyserServiceError; + + fn try_from(message_proto: super::Message) -> Result { let message_header_proto = message_proto.header.expect("missing message header"); - Self { + Ok(Self { account_keys: message_proto .account_keys .iter() - .map(|address_bytes| Pubkey::new(address_bytes.as_slice())) - .collect_vec(), + .map(|address_bytes| Pubkey::new(address_bytes)) + .into_iter() + .collect(), header: message_header_proto.into(), recent_blockhash: Hash::new(&message_proto.recent_blockhash), instructions: message_proto.instructions.into_iter().map_into().collect(), - } + }) } } - impl From for v0::Message { - fn from(message_proto: super::Message) -> Self { + impl TryFrom for v0::Message { + type Error = GeyserServiceError; + + fn try_from(message_proto: super::Message) -> Result { let message_header_proto = message_proto.header.expect("missing message header"); - Self { + + Ok(Self { header: message_header_proto.into(), account_keys: message_proto .account_keys .iter() - .map(|address_bytes| Pubkey::new(address_bytes.as_slice())) - .collect_vec(), + .map(|address_bytes| Pubkey::new(address_bytes)) + .into_iter() + .collect(), recent_blockhash: Hash::new(&message_proto.recent_blockhash), instructions: message_proto.instructions.into_iter().map_into().collect(), address_table_lookups: message_proto .address_table_lookups .into_iter() - .map_into() - .collect(), - } + .map(TryFrom::try_from) + .into_iter() + .collect::>()?, + }) } } - impl From for VersionedMessage { - fn from(message_proto: super::Message) -> Self { - match message_proto.versioned { - false => VersionedMessage::Legacy(message_proto.into()), - true => VersionedMessage::V0(message_proto.into()), - } + impl TryFrom for VersionedMessage { + type Error = GeyserServiceError; + + fn try_from(message_proto: super::Message) -> Result { + Ok(match message_proto.versioned { + false => VersionedMessage::Legacy(message_proto.try_into()?), + true => VersionedMessage::V0(message_proto.try_into()?), + }) } } - impl From for VersionedTransaction { - fn from(transaction_proto: super::Transaction) -> Self { + impl TryFrom for VersionedTransaction { + type Error = GeyserServiceError; + + fn try_from(transaction_proto: super::Transaction) -> Result { let message_proto = transaction_proto.message.expect("missing message"); - Self { + + Ok(Self { signatures: transaction_proto .signatures .iter() .map(|sig_bytes| Signature::new(sig_bytes)) .collect_vec(), - message: message_proto.into(), - } + message: message_proto.try_into()?, + }) } } @@ -208,9 +215,24 @@ mod conversion { } } - impl From for TransactionStatusMeta { - fn from(meta_proto: super::TransactionStatusMeta) -> Self { - Self { + impl TryFrom for TransactionStatusMeta { + type Error = GeyserServiceError; + + fn try_from(meta_proto: super::TransactionStatusMeta) -> Result { + let loaded_writable_addresses: Vec = meta_proto + .loaded_writable_addresses + .iter() + .map(|address_bytes| Pubkey::new(address_bytes)) + .into_iter() + .collect(); + let loaded_readable_addresses = meta_proto + .loaded_readonly_addresses + .iter() + .map(|address_bytes| Pubkey::new(address_bytes)) + .into_iter() + .collect(); + + Ok(Self { status: match meta_proto.err { None => Ok(()), Some(err) => Err(bincode::deserialize::(&err.err).unwrap()), @@ -242,35 +264,31 @@ mod conversion { ), rewards: Some(meta_proto.rewards.into_iter().map_into().collect()), loaded_addresses: LoadedAddresses { - writable: meta_proto - .loaded_writable_addresses - .iter() - .map(|address_bytes| Pubkey::new(address_bytes.as_slice())) - .collect_vec(), - readonly: meta_proto - .loaded_readonly_addresses - .iter() - .map(|address_bytes| Pubkey::new(address_bytes.as_slice())) - .collect_vec(), + writable: loaded_writable_addresses, + readonly: loaded_readable_addresses, }, return_data: None, compute_units_consumed: None, - } + }) } } - impl From for VersionedTransactionWithStatusMeta { - fn from(transaction_info: super::SubscribeUpdateTransactionInfo) -> Self { - Self { + impl TryFrom for VersionedTransactionWithStatusMeta { + type Error = GeyserServiceError; + + fn try_from( + transaction_info: super::SubscribeUpdateTransactionInfo, + ) -> Result { + Ok(Self { transaction: transaction_info .transaction .expect("missing transaction") - .into(), + .try_into()?, meta: transaction_info .meta .expect("missing transaction meta") - .into(), - } + .try_into()?, + }) } } } diff --git a/observability/indexer/src/utils/snapshot.rs b/observability/indexer/src/utils/snapshot.rs new file mode 100644 index 00000000..9ea0cf22 --- /dev/null +++ b/observability/indexer/src/utils/snapshot.rs @@ -0,0 +1,286 @@ +use anchor_client::anchor_lang::AccountDeserialize; +use anchor_client::anchor_lang::Discriminator; +use fixed::types::I80F48; +use itertools::Itertools; +use marginfi::{ + prelude::MarginfiGroup, + state::{marginfi_account::MarginfiAccount, marginfi_group::Bank, price::*}, +}; +use solana_account_decoder::UiAccountEncoding; +use solana_client::{ + nonblocking::rpc_client::RpcClient, + rpc_config::{RpcAccountInfoConfig, RpcProgramAccountsConfig}, +}; +use solana_sdk::account_info::IntoAccountInfo; +use solana_sdk::{account::Account, program_pack::Pack, pubkey::Pubkey}; +use spl_token::state::Account as SplAccount; +use std::{ + collections::HashMap, + fmt::{Debug, Display}, + sync::Arc, +}; + +#[derive(Clone, Debug)] +pub struct BankAccounts { + pub bank: Bank, + pub liquidity_vault_token_account: SplAccount, + pub insurance_vault_token_account: SplAccount, + pub fee_vault_token_account: SplAccount, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum AccountRoutingType { + MarginfiGroup, + MarginfiAccount, + Bank(Pubkey, BankUpdateRoutingType), + PriceFeedPyth, + PriceFeedSwitchboard, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum BankUpdateRoutingType { + State, + LiquidityTokenAccount, + InsuranceTokenAccount, + FeeTokenAccount, +} + +#[derive(Clone, Debug)] +pub enum OracleData { + Pyth(PythEmaPriceFeed), + Switchboard(SwitchboardV2PriceFeed), +} + +impl OracleData { + pub fn get_price(&self) -> I80F48 { + match self { + OracleData::Pyth(price_feed) => price_feed.get_price().unwrap(), + OracleData::Switchboard(price_feed) => price_feed.get_price().unwrap(), + } + } +} + +#[derive(Clone)] +pub struct Snapshot { + program_id: Pubkey, + rpc_client: Arc, + pub routing_lookup: HashMap, + + pub marginfi_groups: HashMap, + pub banks: HashMap, + pub marginfi_accounts: HashMap, + pub price_feeds: HashMap, +} + +impl Display for Snapshot { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "Snapshot:\nMarginfi groups: {}\nBanks: {}\nMarginfi accounts: {}\nPriceFeeds: {}\nRouting lookup: {}", + self.marginfi_groups.len(), + self.banks.len(), + self.marginfi_accounts.len(), + self.price_feeds.len(), + self.routing_lookup.len() + ) + } +} + +impl Debug for Snapshot { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "Snapshot:\nBanks: {:?}\nMarginfi accounts: {:?}\nPriceFeeds: {:?}\nRouting lookup: {:?}", + self.banks, self.marginfi_accounts, self.price_feeds, self.routing_lookup + ) + } +} + +impl Snapshot { + pub fn new(program_id: Pubkey, rpc_client: Arc) -> Self { + Self { + program_id, + rpc_client, + routing_lookup: HashMap::new(), + marginfi_groups: HashMap::new(), + banks: HashMap::new(), + marginfi_accounts: HashMap::new(), + price_feeds: HashMap::new(), + } + } + + pub async fn init(&mut self) -> anyhow::Result<()> { + let config = RpcProgramAccountsConfig { + account_config: RpcAccountInfoConfig { + encoding: Some(UiAccountEncoding::Base64), + ..RpcAccountInfoConfig::default() + }, + ..RpcProgramAccountsConfig::default() + }; + + let all_program_accounts = self + .rpc_client + .get_program_accounts_with_config(&self.program_id, config) + .await?; + + for (pubkey, account) in all_program_accounts { + self.create_entry(&pubkey, &account).await; + } + + Ok(()) + } + + // This method assumes that all accounts of interest not owned by the marginfi program are + // inserted in the routing lookup table when a program account is created / received for the + // first time. This is why this only processes program accounts. + pub async fn create_entry(&mut self, account_pubkey: &Pubkey, account: &Account) { + if account.owner == self.program_id { + let discriminator = &account.data[..8]; + if discriminator == Bank::discriminator() { + let bank = Bank::try_deserialize(&mut (&account.data as &[u8])).unwrap(); + self.routing_lookup.insert( + *account_pubkey, + AccountRoutingType::Bank(*account_pubkey, BankUpdateRoutingType::State), + ); + + self.routing_lookup.insert( + bank.liquidity_vault, + AccountRoutingType::Bank( + *account_pubkey, + BankUpdateRoutingType::LiquidityTokenAccount, + ), + ); + self.routing_lookup.insert( + bank.insurance_vault, + AccountRoutingType::Bank( + *account_pubkey, + BankUpdateRoutingType::InsuranceTokenAccount, + ), + ); + self.routing_lookup.insert( + bank.fee_vault, + AccountRoutingType::Bank( + *account_pubkey, + BankUpdateRoutingType::FeeTokenAccount, + ), + ); + + let mut accounts_to_fetch = + vec![bank.liquidity_vault, bank.insurance_vault, bank.fee_vault]; + + match bank.config.oracle_setup { + OracleSetup::None => (), + OracleSetup::PythEma => { + let oracle_address = bank.config.oracle_keys[0]; + self.routing_lookup + .insert(oracle_address, AccountRoutingType::PriceFeedPyth); + accounts_to_fetch.push(oracle_address); + } + OracleSetup::SwitchboardV2 => { + let oracle_address = bank.config.oracle_keys[0]; + self.routing_lookup + .insert(oracle_address, AccountRoutingType::PriceFeedSwitchboard); + accounts_to_fetch.push(oracle_address); + } + } + + self.banks.insert( + *account_pubkey, + BankAccounts { + bank, + liquidity_vault_token_account: SplAccount::default(), + insurance_vault_token_account: SplAccount::default(), + fee_vault_token_account: SplAccount::default(), + }, + ); + + // Fetch all ban-specific accounts + store in current snapshot + let accounts = self + .rpc_client + .get_multiple_accounts(&accounts_to_fetch) + .await + .unwrap() + .into_iter() + .zip(accounts_to_fetch) + .filter_map(|(maybe_account, pubkey)| { + maybe_account.map(|account| (pubkey, account)) + }) + .collect_vec(); + + for (account_pubkey, account) in accounts { + self.udpate_entry(&account_pubkey, &account) + } + } else if discriminator == MarginfiAccount::discriminator() { + let marginfi_account = + MarginfiAccount::try_deserialize(&mut (&account.data as &[u8])).unwrap(); + self.routing_lookup + .insert(*account_pubkey, AccountRoutingType::MarginfiAccount); + self.marginfi_accounts + .insert(*account_pubkey, marginfi_account); + } else if discriminator == MarginfiGroup::discriminator() { + let marginfi_group = + MarginfiGroup::try_deserialize(&mut (&account.data as &[u8])).unwrap(); + self.routing_lookup + .insert(*account_pubkey, AccountRoutingType::MarginfiGroup); + self.marginfi_groups.insert(*account_pubkey, marginfi_group); + } + } + } + + pub fn udpate_entry(&mut self, account_pubkey: &Pubkey, account: &Account) { + let routing_info = self + .routing_lookup + .get(account_pubkey) + .expect("Account not found in routing lookup"); + match routing_info { + AccountRoutingType::PriceFeedPyth => { + let mut account = account.clone(); + let ai = (account_pubkey, &mut account).into_account_info(); + let pf = PythEmaPriceFeed::load_checked(&ai, 0, u64::MAX).unwrap(); + self.price_feeds + .insert(*account_pubkey, OracleData::Pyth(pf)); + } + AccountRoutingType::Bank(bank_pk, BankUpdateRoutingType::LiquidityTokenAccount) => { + self.banks + .get_mut(bank_pk) + .unwrap() + .liquidity_vault_token_account = + SplAccount::unpack_from_slice(&account.data as &[u8]).unwrap(); + } + AccountRoutingType::Bank(bank_pk, BankUpdateRoutingType::InsuranceTokenAccount) => { + self.banks + .get_mut(bank_pk) + .unwrap() + .insurance_vault_token_account = + SplAccount::unpack_from_slice(&account.data as &[u8]).unwrap(); + } + AccountRoutingType::Bank(bank_pk, BankUpdateRoutingType::FeeTokenAccount) => { + self.banks.get_mut(bank_pk).unwrap().fee_vault_token_account = + SplAccount::unpack_from_slice(&account.data as &[u8]).unwrap(); + } + AccountRoutingType::Bank(bank_pk, BankUpdateRoutingType::State) => { + self.banks.get_mut(bank_pk).unwrap().bank = + Bank::try_deserialize(&mut (&account.data as &[u8])).unwrap(); + } + AccountRoutingType::MarginfiAccount => { + self.marginfi_accounts.insert( + *account_pubkey, + MarginfiAccount::try_deserialize(&mut (&account.data as &[u8])).unwrap(), + ); + } + AccountRoutingType::MarginfiGroup => { + self.marginfi_groups.insert( + *account_pubkey, + MarginfiGroup::try_deserialize(&mut (&account.data as &[u8])).unwrap(), + ); + } + AccountRoutingType::PriceFeedSwitchboard => { + let mut account = account.clone(); + let ai = (account_pubkey, &mut account).into_account_info(); + let pf = SwitchboardV2PriceFeed::load_checked(&ai, 0, u64::MAX).unwrap(); + self.price_feeds + .insert(*account_pubkey, OracleData::Switchboard(pf)); + } + } + } +} diff --git a/observability/indexer/src/utils/transactions_crawler.rs b/observability/indexer/src/utils/transactions_crawler.rs index 4dc4d0d3..95a7b1c1 100644 --- a/observability/indexer/src/utils/transactions_crawler.rs +++ b/observability/indexer/src/utils/transactions_crawler.rs @@ -2,7 +2,6 @@ use anyhow::Result; use chrono::{Local, TimeZone}; use concurrent_queue::ConcurrentQueue; use futures::{future::join_all, stream, Future, StreamExt}; -use log::{error, info, warn}; use solana_client::{ nonblocking::rpc_client::RpcClient, rpc_client::GetConfirmedSignaturesForAddress2Config, rpc_config::RpcTransactionConfig, @@ -23,6 +22,7 @@ use std::{ }; use std::{str::FromStr, time::Duration}; use tokio::{join, runtime::Builder}; +use tracing::{error, info, warn}; use crate::common::{ Target, DEFAULT_MAX_PENDING_SIGNATURES, DEFAULT_MONITOR_INTERVAL, DEFAULT_RPC_ENDPOINT, diff --git a/observability/indexer/upload_image b/observability/indexer/upload_image new file mode 100755 index 00000000..35a48088 --- /dev/null +++ b/observability/indexer/upload_image @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -e + +image_tag=$1 + +[ -z "$image_tag" ] && echo "Missing image_tag argument" && exit 1 + +local_image_name="marginfi-v2-indexer" +gcp_image_name="us-east1-docker.pkg.dev/marginfi-dev/main/$local_image_name" + +docker push "$gcp_image_name:$image_tag" diff --git a/programs/marginfi/src/state/marginfi_account.rs b/programs/marginfi/src/state/marginfi_account.rs index 80fd9a1f..fecdec33 100644 --- a/programs/marginfi/src/state/marginfi_account.rs +++ b/programs/marginfi/src/state/marginfi_account.rs @@ -16,7 +16,6 @@ use crate::{ use anchor_lang::prelude::*; use anchor_spl::token::Transfer; use fixed::types::I80F48; - use std::{ cmp::{max, min}, ops::Not, @@ -250,6 +249,7 @@ pub fn calc_asset_value( asset_amount }; + #[cfg(target_os = "solana")] msg!( "weighted_asset_qt: {}, price: {}, expo: {}", weighted_asset_amount, diff --git a/programs/marginfi/src/state/marginfi_group.rs b/programs/marginfi/src/state/marginfi_group.rs index 34ebb565..b08f6945 100644 --- a/programs/marginfi/src/state/marginfi_group.rs +++ b/programs/marginfi/src/state/marginfi_group.rs @@ -20,7 +20,7 @@ use anchor_spl::token::{transfer, Transfer}; use fixed::types::I80F48; use pyth_sdk_solana::{load_price_feed_from_account_info, PriceFeed}; use std::{ - fmt::{Debug, Formatter}, + fmt::{Debug, Display, Formatter}, ops::Not, }; @@ -736,6 +736,17 @@ pub enum BankOperationalState { ReduceOnly, } +#[cfg(feature = "client")] +impl Display for BankOperationalState { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + BankOperationalState::Paused => write!(f, "Paused"), + BankOperationalState::Operational => write!(f, "Operational"), + BankOperationalState::ReduceOnly => write!(f, "ReduceOnly"), + } + } +} + #[repr(u64)] #[derive(Copy, Clone, Debug, AnchorSerialize, AnchorDeserialize, PartialEq, Eq)] pub enum RiskTier { diff --git a/programs/marginfi/src/state/price.rs b/programs/marginfi/src/state/price.rs index 8824c357..1207e0d0 100644 --- a/programs/marginfi/src/state/price.rs +++ b/programs/marginfi/src/state/price.rs @@ -44,6 +44,7 @@ pub trait PriceAdapter { } #[enum_dispatch(PriceAdapter)] +#[cfg_attr(feature = "client", derive(Clone))] pub enum OraclePriceFeedAdapter { PythEma(PythEmaPriceFeed), SwitchboardV2(SwitchboardV2PriceFeed), @@ -116,6 +117,7 @@ impl OraclePriceFeedAdapter { } } +#[cfg_attr(feature = "client", derive(Clone, Debug))] pub struct PythEmaPriceFeed { ema_price: Box, price: Box, @@ -209,6 +211,7 @@ impl PriceAdapter for PythEmaPriceFeed { } } +#[cfg_attr(feature = "client", derive(Clone, Debug))] pub struct SwitchboardV2PriceFeed { aggregator_account: Box, } @@ -317,6 +320,7 @@ impl PriceAdapter for SwitchboardV2PriceFeed { } /// A slimmed down version of the AggregatorAccountData struct copied from the switchboard-v2/src/aggregator.rs +#[cfg_attr(feature = "client", derive(Clone, Debug))] struct LiteAggregatorAccountData { /// Use sliding windoe or round based resolution /// NOTE: This changes result propogation in latest_round_result