From 2b26bef680245afdaafe5ad06fe881ee45d4230f Mon Sep 17 00:00:00 2001 From: siddharth parashar <20308880+shanu516516@users.noreply.github.com> Date: Wed, 14 Feb 2024 13:41:27 +0530 Subject: [PATCH 01/87] get order by uuid fn test --- src/database/models.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/database/models.rs b/src/database/models.rs index 07decf8..6508838 100644 --- a/src/database/models.rs +++ b/src/database/models.rs @@ -1249,6 +1249,15 @@ impl TraderOrder { .order(timestamp.desc()) .first(conn) } + pub fn get_by_uuid(conn: &mut PgConnection, order_id: String) -> QueryResult { + use crate::database::schema::address_customer_id::dsl as addr_dsl; + use crate::database::schema::trader_order::dsl::*; + + trader_order + .filter(uuid.eq(order_id)) + .order(timestamp.desc()) + .first(conn) + } pub fn insert(conn: &mut PgConnection, orders: Vec) -> QueryResult { use crate::database::schema::trader_order::dsl::*; From 7c04ba41209c6ce6e283e947d87b9ae4ff2eaf36 Mon Sep 17 00:00:00 2001 From: siddharth parashar <20308880+shanu516516@users.noreply.github.com> Date: Wed, 14 Feb 2024 14:34:36 +0530 Subject: [PATCH 02/87] RelayerDB in lib --- src/lib.rs | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/rpc.rs | 2 ++ 2 files changed, 64 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index a009e17..5061d99 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -22,3 +22,65 @@ pub mod auth { pub customer_id: i64, } } + +// for openrpc api +use diesel::prelude::PgConnection; +use diesel::r2d2::ConnectionManager; +type ManagedConnection = ConnectionManager; +type ManagedPool = r2d2::Pool; +const MAX_RETRIES: usize = 5; +const RETRY_SLEEP: u64 = 2000; +use crate::error::ApiError; +use log::{debug, error, info, trace}; +use r2d2::PooledConnection; +use std::time::{Duration, Instant}; +pub struct RelayerDB { + pool: ManagedPool, +} +impl RelayerDB { + pub fn from_host(database_url: String) -> Self { + RelayerDB { + pool: { + let manager = ConnectionManager::::new(database_url); + let pool = r2d2::Pool::new(manager).expect("Could not instantiate connection pool"); + pool + }, + } + } + /// Fetch a connection, will retry MAX_RETRIES before giving up. + fn get_conn(&self) -> Result, ApiError> { + let mut retries = MAX_RETRIES; + + Ok(loop { + break match self.pool.get() { + Ok(c) => c, + Err(e) => { + error!("Could not get connection from connection pool! {:?}", e); + std::thread::sleep(Duration::from_millis(RETRY_SLEEP)); + + if retries == 0 { + return Err(ApiError::CommitRetryCountExceeded); + } + + retries -= 1; + + continue; + } + }; + }) + } +} + +#[cfg(test)] +mod test { + + use crate::{database::TraderOrder as TraderOrderDB, RelayerDB}; + #[test] + fn test_check_get_order_by_uuid_from_archiver() { + dotenv::dotenv().expect("Failed loading dotenv"); + let database_url = std::env::var("DATABASE_URL").expect("No database url found!"); + let relayer_db = RelayerDB::from_host(database_url); + let mut pool = relayer_db.get_conn().unwrap(); + TraderOrderDB::get_by_uuid(&mut *pool, "".to_string()); + } +} diff --git a/src/rpc.rs b/src/rpc.rs index 8b15dec..88b1cb0 100644 --- a/src/rpc.rs +++ b/src/rpc.rs @@ -199,6 +199,8 @@ pub fn init_private_methods(database_url: &str) -> RpcModule { #[cfg(test)] mod tests { + use crate::ws::init_methods; + use super::*; use jsonrpsee::{ core::{ From 8c3f53a94ca9e971bb3e28a03557bf5052a01a3b Mon Sep 17 00:00:00 2001 From: siddharth parashar <20308880+shanu516516@users.noreply.github.com> Date: Wed, 14 Feb 2024 14:42:08 +0530 Subject: [PATCH 03/87] pub --- src/database/models.rs | 330 ++++++++++++++++++++--------------------- src/lib.rs | 2 +- src/rpc.rs | 44 +++--- src/ws.rs | 44 +++--- 4 files changed, 210 insertions(+), 210 deletions(-) diff --git a/src/database/models.rs b/src/database/models.rs index 6508838..7bf15f0 100644 --- a/src/database/models.rs +++ b/src/database/models.rs @@ -1830,174 +1830,174 @@ mod tests { use getrandom::getrandom; const DIESEL_TEST_URL: &str = "postgres://relayer:relayer@localhost:5434/test"; - fn make_trader_order(entryprice: f64, execution_price: f64) -> TraderOrder { - let mut bytes = [0u8; 16]; + // fn make_trader_order(entryprice: f64, execution_price: f64) -> TraderOrder { + // let mut bytes = [0u8; 16]; + + // getrandom(&mut bytes).expect("Could not get randomness"); + + // TraderOrder { + // uuid: bytes.encode_hex::(), + // account_id: "my-id".into(), + // position_type: PositionType::LONG, + // order_status: OrderStatus::PENDING, + // order_type: OrderType::MARKET, + // entryprice: BigDecimal::from_f64(entryprice).unwrap(), + // execution_price: BigDecimal::from_f64(execution_price).unwrap(), + // positionsize: BigDecimal::from_f64(0.0).unwrap(), + // leverage: BigDecimal::from_f64(0.0).unwrap(), + // initial_margin: BigDecimal::from_f64(0.0).unwrap(), + // available_margin: BigDecimal::from_f64(0.0).unwrap(), + // timestamp: Utc::now(), + // bankruptcy_price: BigDecimal::from_f64(0.0).unwrap(), + // bankruptcy_value: BigDecimal::from_f64(0.0).unwrap(), + // maintenance_margin: BigDecimal::from_f64(0.0).unwrap(), + // liquidation_price: BigDecimal::from_f64(0.0).unwrap(), + // unrealized_pnl: BigDecimal::from_f64(0.0).unwrap(), + // settlement_price: BigDecimal::from_f64(0.0).unwrap(), + // entry_nonce: 20, + // exit_nonce: 22, + // entry_sequence: 400, + // } + // } + + // fn make_lend_order(balance: f64, payment: f64) -> LendOrder { + // let mut bytes = [0u8; 16]; + + // getrandom(&mut bytes).expect("Could not get randomness"); + + // LendOrder { + // uuid: bytes.encode_hex::(), + // account_id: "lender-id".into(), + // balance: BigDecimal::from_f64(balance).unwrap(), + // order_status: OrderStatus::PENDING, + // order_type: OrderType::MARKET, + // entry_nonce: 40, + // exit_nonce: 600, + // deposit: BigDecimal::from_f64(0.0).unwrap(), + // new_lend_state_amount: BigDecimal::from_f64(0.0).unwrap(), + // timestamp: Utc::now(), + // npoolshare: BigDecimal::from_f64(0.0).unwrap(), + // nwithdraw: BigDecimal::from_f64(0.0).unwrap(), + // payment: BigDecimal::from_f64(payment).unwrap(), + // tlv0: BigDecimal::from_f64(0.0).unwrap(), + // tps0: BigDecimal::from_f64(0.0).unwrap(), + // tlv1: BigDecimal::from_f64(0.0).unwrap(), + // tps1: BigDecimal::from_f64(0.0).unwrap(), + // tlv2: BigDecimal::from_f64(0.0).unwrap(), + // tps2: BigDecimal::from_f64(0.0).unwrap(), + // tlv3: BigDecimal::from_f64(0.0).unwrap(), + // tps3: BigDecimal::from_f64(0.0).unwrap(), + // entry_sequence: 0, + // } + // } + + // #[test] + // fn trader_orders() { + // use crate::database::schema::trader_order::dsl::*; + + // let mut conn = + // PgConnection::establish(DIESEL_TEST_URL).expect("Could not establish test connection!"); + + // conn.test_transaction::<_, diesel::result::Error, _>(|conn| { + // let mut order1 = make_trader_order(1.0, 4.0); + // let mut order2 = make_trader_order(4.0, 400.0); + + // let orders: Vec = vec![order1.clone(), order2.clone()]; + + // let result = diesel::insert_into(trader_order) + // .values(orders) + // .execute(&mut *conn); + + // if let Err(e) = result { + // panic!("insert in database didn't suceed! {:#?}", e); + // } + + // //Test updates/inserts + // let order3 = make_trader_order(989.0, 23.0); + // let order4 = make_trader_order(99.0, 302.0); + + // order1.entryprice = BigDecimal::from_f64(32.0).unwrap(); + // order1.execution_price = BigDecimal::from_f64(89.0).unwrap(); + // order2.entryprice = BigDecimal::from_f64(20.0).unwrap(); + + // TraderOrder::update_or_insert( + // &mut *conn, + // vec![ + // order1.clone(), + // order2.clone(), + // order3.clone(), + // order4.clone(), + // ], + // )?; + + // let o1: TraderOrder = trader_order + // .filter(uuid.eq(order1.uuid)) + // .first(&mut *conn)?; + + // assert_eq!(o1.entryprice, order1.entryprice); + // assert_eq!(o1.execution_price, order1.execution_price); + + // let o2: TraderOrder = trader_order + // .filter(uuid.eq(order2.uuid)) + // .first(&mut *conn)?; + + // assert_eq!(o2.entryprice, order2.entryprice); + // assert_eq!(o2.execution_price, order2.execution_price); + + // Ok(()) + // }); + // } + + // #[test] + // fn lender_orders() { + // use crate::database::schema::lend_order::dsl::*; + + // let mut conn = + // PgConnection::establish(DIESEL_TEST_URL).expect("Could not establish test connection!"); + + // conn.test_transaction::<_, diesel::result::Error, _>(|conn| { + // let mut order1 = make_lend_order(1.0, 4.0); + // let mut order2 = make_lend_order(4.0, 400.0); + + // let orders: Vec = vec![order1.clone(), order2.clone()]; + + // let result = diesel::insert_into(lend_order) + // .values(orders) + // .execute(&mut *conn); + + // if let Err(e) = result { + // panic!("insert in database didn't suceed! {:#?}", e); + // } + + // //Test updates/inserts + // let order3 = make_lend_order(989.0, 23.0); + // let order4 = make_lend_order(99.0, 302.0); + + // order1.balance = BigDecimal::from_f64(32.0).unwrap(); + // order1.payment = BigDecimal::from_f64(89.0).unwrap(); + // order2.balance = BigDecimal::from_f64(20.0).unwrap(); + + // LendOrder::update_or_insert( + // &mut *conn, + // vec![ + // order1.clone(), + // order2.clone(), + // order3.clone(), + // order4.clone(), + // ], + // )?; - getrandom(&mut bytes).expect("Could not get randomness"); + // let o1: LendOrder = lend_order.filter(uuid.eq(order1.uuid)).first(&mut *conn)?; - TraderOrder { - uuid: bytes.encode_hex::(), - account_id: "my-id".into(), - position_type: PositionType::LONG, - order_status: OrderStatus::PENDING, - order_type: OrderType::MARKET, - entryprice: BigDecimal::from_f64(entryprice).unwrap(), - execution_price: BigDecimal::from_f64(execution_price).unwrap(), - positionsize: BigDecimal::from_f64(0.0).unwrap(), - leverage: BigDecimal::from_f64(0.0).unwrap(), - initial_margin: BigDecimal::from_f64(0.0).unwrap(), - available_margin: BigDecimal::from_f64(0.0).unwrap(), - timestamp: Utc::now(), - bankruptcy_price: BigDecimal::from_f64(0.0).unwrap(), - bankruptcy_value: BigDecimal::from_f64(0.0).unwrap(), - maintenance_margin: BigDecimal::from_f64(0.0).unwrap(), - liquidation_price: BigDecimal::from_f64(0.0).unwrap(), - unrealized_pnl: BigDecimal::from_f64(0.0).unwrap(), - settlement_price: BigDecimal::from_f64(0.0).unwrap(), - entry_nonce: 20, - exit_nonce: 22, - entry_sequence: 400, - } - } - - fn make_lend_order(balance: f64, payment: f64) -> LendOrder { - let mut bytes = [0u8; 16]; - - getrandom(&mut bytes).expect("Could not get randomness"); - - LendOrder { - uuid: bytes.encode_hex::(), - account_id: "lender-id".into(), - balance: BigDecimal::from_f64(balance).unwrap(), - order_status: OrderStatus::PENDING, - order_type: OrderType::MARKET, - entry_nonce: 40, - exit_nonce: 600, - deposit: BigDecimal::from_f64(0.0).unwrap(), - new_lend_state_amount: BigDecimal::from_f64(0.0).unwrap(), - timestamp: Utc::now(), - npoolshare: BigDecimal::from_f64(0.0).unwrap(), - nwithdraw: BigDecimal::from_f64(0.0).unwrap(), - payment: BigDecimal::from_f64(payment).unwrap(), - tlv0: BigDecimal::from_f64(0.0).unwrap(), - tps0: BigDecimal::from_f64(0.0).unwrap(), - tlv1: BigDecimal::from_f64(0.0).unwrap(), - tps1: BigDecimal::from_f64(0.0).unwrap(), - tlv2: BigDecimal::from_f64(0.0).unwrap(), - tps2: BigDecimal::from_f64(0.0).unwrap(), - tlv3: BigDecimal::from_f64(0.0).unwrap(), - tps3: BigDecimal::from_f64(0.0).unwrap(), - entry_sequence: 0, - } - } - - #[test] - fn trader_orders() { - use crate::database::schema::trader_order::dsl::*; - - let mut conn = - PgConnection::establish(DIESEL_TEST_URL).expect("Could not establish test connection!"); - - conn.test_transaction::<_, diesel::result::Error, _>(|conn| { - let mut order1 = make_trader_order(1.0, 4.0); - let mut order2 = make_trader_order(4.0, 400.0); - - let orders: Vec = vec![order1.clone(), order2.clone()]; - - let result = diesel::insert_into(trader_order) - .values(orders) - .execute(&mut *conn); - - if let Err(e) = result { - panic!("insert in database didn't suceed! {:#?}", e); - } - - //Test updates/inserts - let order3 = make_trader_order(989.0, 23.0); - let order4 = make_trader_order(99.0, 302.0); - - order1.entryprice = BigDecimal::from_f64(32.0).unwrap(); - order1.execution_price = BigDecimal::from_f64(89.0).unwrap(); - order2.entryprice = BigDecimal::from_f64(20.0).unwrap(); + // assert_eq!(o1.balance, order1.balance); + // assert_eq!(o1.payment, order1.payment); - TraderOrder::update_or_insert( - &mut *conn, - vec![ - order1.clone(), - order2.clone(), - order3.clone(), - order4.clone(), - ], - )?; + // let o2: LendOrder = lend_order.filter(uuid.eq(order2.uuid)).first(&mut *conn)?; - let o1: TraderOrder = trader_order - .filter(uuid.eq(order1.uuid)) - .first(&mut *conn)?; + // assert_eq!(o2.balance, order2.balance); + // assert_eq!(o2.payment, order2.payment); - assert_eq!(o1.entryprice, order1.entryprice); - assert_eq!(o1.execution_price, order1.execution_price); - - let o2: TraderOrder = trader_order - .filter(uuid.eq(order2.uuid)) - .first(&mut *conn)?; - - assert_eq!(o2.entryprice, order2.entryprice); - assert_eq!(o2.execution_price, order2.execution_price); - - Ok(()) - }); - } - - #[test] - fn lender_orders() { - use crate::database::schema::lend_order::dsl::*; - - let mut conn = - PgConnection::establish(DIESEL_TEST_URL).expect("Could not establish test connection!"); - - conn.test_transaction::<_, diesel::result::Error, _>(|conn| { - let mut order1 = make_lend_order(1.0, 4.0); - let mut order2 = make_lend_order(4.0, 400.0); - - let orders: Vec = vec![order1.clone(), order2.clone()]; - - let result = diesel::insert_into(lend_order) - .values(orders) - .execute(&mut *conn); - - if let Err(e) = result { - panic!("insert in database didn't suceed! {:#?}", e); - } - - //Test updates/inserts - let order3 = make_lend_order(989.0, 23.0); - let order4 = make_lend_order(99.0, 302.0); - - order1.balance = BigDecimal::from_f64(32.0).unwrap(); - order1.payment = BigDecimal::from_f64(89.0).unwrap(); - order2.balance = BigDecimal::from_f64(20.0).unwrap(); - - LendOrder::update_or_insert( - &mut *conn, - vec![ - order1.clone(), - order2.clone(), - order3.clone(), - order4.clone(), - ], - )?; - - let o1: LendOrder = lend_order.filter(uuid.eq(order1.uuid)).first(&mut *conn)?; - - assert_eq!(o1.balance, order1.balance); - assert_eq!(o1.payment, order1.payment); - - let o2: LendOrder = lend_order.filter(uuid.eq(order2.uuid)).first(&mut *conn)?; - - assert_eq!(o2.balance, order2.balance); - assert_eq!(o2.payment, order2.payment); - - Ok(()) - }); - } + // Ok(()) + // }); + // } } diff --git a/src/lib.rs b/src/lib.rs index 5061d99..df424ba 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -48,7 +48,7 @@ impl RelayerDB { } } /// Fetch a connection, will retry MAX_RETRIES before giving up. - fn get_conn(&self) -> Result, ApiError> { + pub fn get_conn(&self) -> Result, ApiError> { let mut retries = MAX_RETRIES; Ok(loop { diff --git a/src/rpc.rs b/src/rpc.rs index 88b1cb0..a0293d7 100644 --- a/src/rpc.rs +++ b/src/rpc.rs @@ -211,26 +211,26 @@ mod tests { server::ServerBuilder, }; - #[tokio::test] - async fn test_hello() { - let mut server = ServerBuilder::new() - .build("0.0.0.0:8979") - .await - .expect("Builder failed"); - - let handle = server - .start(init_methods()) - .expect("Server failed to start"); - - let client = HttpClientBuilder::default() - .build("http://127.0.0.1:8979") - .expect("Client builder failed"); - - let response: String = client - .request("hello_method", ObjectParams::new()) - .await - .expect("Client call failed"); - - assert_eq!("Hello, world!".to_string(), response); - } + // #[tokio::test] + // async fn test_hello() { + // let mut server = ServerBuilder::new() + // .build("0.0.0.0:8979") + // .await + // .expect("Builder failed"); + + // let handle = server + // .start(init_methods()) + // .expect("Server failed to start"); + + // let client = HttpClientBuilder::default() + // .build("http://127.0.0.1:8979") + // .expect("Client builder failed"); + + // let response: String = client + // .request("hello_method", ObjectParams::new()) + // .await + // .expect("Client call failed"); + + // assert_eq!("Hello, world!".to_string(), response); + // } } diff --git a/src/ws.rs b/src/ws.rs index a792c0a..72d851b 100644 --- a/src/ws.rs +++ b/src/ws.rs @@ -204,26 +204,26 @@ mod tests { server::ServerBuilder, }; - #[tokio::test] - async fn test_hello() { - let mut server = ServerBuilder::new() - .build("0.0.0.0:8979") - .await - .expect("Builder failed"); - - let handle = server - .start(init_methods()) - .expect("Server failed to start"); - - let client = HttpClientBuilder::default() - .build("http://127.0.0.1:8979") - .expect("Client builder failed"); - - let response: String = client - .request("hello_method", ObjectParams::new()) - .await - .expect("Client call failed"); - - assert_eq!("Hello, world!".to_string(), response); - } + // #[tokio::test] + // async fn test_hello() { + // let mut server = ServerBuilder::new() + // .build("0.0.0.0:8979") + // .await + // .expect("Builder failed"); + + // let handle = server + // .start(init_methods()) + // .expect("Server failed to start"); + + // let client = HttpClientBuilder::default() + // .build("http://127.0.0.1:8979") + // .expect("Client builder failed"); + + // let response: String = client + // .request("hello_method", ObjectParams::new()) + // .await + // .expect("Client call failed"); + + // assert_eq!("Hello, world!".to_string(), response); + // } } From 8875296b4264bcdcb6ae2c6e78db41570276ba7e Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Wed, 14 Feb 2024 10:11:35 +0000 Subject: [PATCH 04/87] git update in cargo toml --- Cargo.toml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 44c2dc3..c057597 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,7 +68,12 @@ tower-http = { version = "0.4", features = [ ] } tracing-subscriber = { version = "0.3.16", features = ["env-filter"] } twilight-relayer-rust = { git = "ssh://git@github.com/twilight-project/twilight-relayer.git", branch = "zkos-0.03.01-zkos-wallet-integration" } -zkos-client-wallet = { git = "ssh://git@github.com/twilight-project/zkos-client-wallet.git", branch = "develop" } -zkos-relayer-wallet = { git = "ssh://git@github.com/twilight-project/zkos-relayer-wallet.git", branch = "develop" } + +[dependencies.zkos-relayer-wallet] +git = "ssh://github.com/twilight-project/zkos-relayer-wallet.git" +branch = "develop" +[dependencies.zkos-client-wallet] +git = "ssh://github.com/twilight-project/zkos-client-wallet.git" +branch = "develop" uuid = { version = "1.3.3", features = ["serde", "v4", "v7"] } verify-keplr-sign = "0.1.0" From b62933b8e464ea2b59bb4c0568db3aa392d141d4 Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Wed, 14 Feb 2024 11:05:38 +0000 Subject: [PATCH 05/87] cargo toml bug --- Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c057597..d16710a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,6 +67,8 @@ tower-http = { version = "0.4", features = [ "validate-request", ] } tracing-subscriber = { version = "0.3.16", features = ["env-filter"] } +uuid = { version = "1.3.3", features = ["serde", "v4", "v7"] } +verify-keplr-sign = "0.1.0" twilight-relayer-rust = { git = "ssh://git@github.com/twilight-project/twilight-relayer.git", branch = "zkos-0.03.01-zkos-wallet-integration" } [dependencies.zkos-relayer-wallet] @@ -75,5 +77,3 @@ branch = "develop" [dependencies.zkos-client-wallet] git = "ssh://github.com/twilight-project/zkos-client-wallet.git" branch = "develop" -uuid = { version = "1.3.3", features = ["serde", "v4", "v7"] } -verify-keplr-sign = "0.1.0" From 3eb4233c3294f7bc9ebf7e233b21640986dfc283 Mon Sep 17 00:00:00 2001 From: TJ Sharp Date: Mon, 11 Mar 2024 20:32:29 -0400 Subject: [PATCH 06/87] Candle fixes --- src/database/models.rs | 15 ++++++++++----- src/rpc/types.rs | 16 +++++++++++++++- src/ws/methods.rs | 13 +++++++++---- 3 files changed, 34 insertions(+), 10 deletions(-) diff --git a/src/database/models.rs b/src/database/models.rs index 96f747b..f9c133f 100644 --- a/src/database/models.rs +++ b/src/database/models.rs @@ -11,7 +11,7 @@ use crate::rpc::{ TradeVolumeArgs, TransactionHashArgs, }; use bigdecimal::{BigDecimal, FromPrimitive, ToPrimitive, Zero}; -use chrono::prelude::*; +use chrono::{DurationRound, prelude::*}; use diesel::pg::Pg; use diesel::prelude::*; use itertools::join; @@ -794,6 +794,8 @@ pub struct BtcUsdPrice { #[derive(Serialize, Deserialize, Debug, Clone, Queryable, QueryableByName)] pub struct CandleData { + #[diesel(sql_type = diesel::sql_types::Timestamptz)] + pub updated_at: DateTime, #[diesel(sql_type = diesel::sql_types::Timestamptz)] pub start: DateTime, #[diesel(sql_type = diesel::sql_types::Timestamptz)] @@ -852,6 +854,8 @@ impl BtcUsdPrice { limit: Option, offset: Option, ) -> QueryResult> { + let start = since.duration_trunc(interval.duration()).unwrap(); + let interval = interval.interval_sql(); let trader_subquery = format!( @@ -872,15 +876,15 @@ impl BtcUsdPrice { ) as sq GROUP BY window_start "#, - since, interval, interval + start, interval, interval ); let ohlc_subquery = format!( r#" SELECT window_ts, - min(timestamp) as start, - max(timestamp) as end, + window_ts as start, + window_ts + {} as end, min(open) as open, min(close) as close, max(price) as high, @@ -898,7 +902,7 @@ impl BtcUsdPrice { WHERE open IS NOT NULL GROUP BY window_ts "#, - since, interval, interval + interval, start, interval, interval ); let query = format!( @@ -909,6 +913,7 @@ impl BtcUsdPrice { {} ) SELECT + now() as updated_at, ohlc.start, ohlc.end, ohlc.open, diff --git a/src/rpc/types.rs b/src/rpc/types.rs index f2e7741..63f7e65 100644 --- a/src/rpc/types.rs +++ b/src/rpc/types.rs @@ -13,7 +13,7 @@ // • Server Time use crate::auth::UserInfo; use crate::database::OrderStatus; -use chrono::prelude::*; +use chrono::{Duration, prelude::*}; use relayerwalletlib::zkoswalletlib::relayer_types::{OrderType, PositionType}; use serde::{Deserialize, Serialize}; @@ -234,6 +234,20 @@ pub enum Interval { } impl Interval { + pub fn duration(&self) -> Duration { + match self { + Interval::ONE_MINUTE => Duration::minutes(1), + Interval::FIVE_MINUTE => Duration::minutes(5), + Interval::FIFTEEN_MINUTE => Duration::minutes(15), + Interval::THIRTY_MINUTE => Duration::minutes(30), + Interval::ONE_HOUR => Duration::hours(1), + Interval::FOUR_HOUR => Duration::hours(4), + Interval::EIGHT_HOUR => Duration::hours(8), + Interval::TWELVE_HOUR => Duration::hours(12), + Interval::ONE_DAY => Duration::days(1), + } + } + pub fn interval_sql(&self) -> String { match self { Interval::ONE_MINUTE => "'1 minute'", diff --git a/src/ws/methods.rs b/src/ws/methods.rs index ff73017..19dc341 100644 --- a/src/ws/methods.rs +++ b/src/ws/methods.rs @@ -67,14 +67,19 @@ pub(super) fn candle_update( let _: JoinHandle> = tokio::task::spawn(async move { loop { let mut conn = ctx.pool.get()?; - let since = Utc::now() - chrono::Duration::minutes(5); + let since = Utc::now() - chrono::Duration::milliseconds(250); let candles = BtcUsdPrice::candles(&mut conn, interval.clone(), since, None, None)?; let result = serde_json::to_value(&candles)?; - if let Err(e) = sink.send(&result) { - error!("Error sending candle updates: {:?}", e); + + if candles.len() > 0 { + if let Err(e) = sink.send(&result) { + error!("Error sending candle updates: {:?}", e); + } + sleep(Duration::from_millis(250)).await; + } else { + sleep(Duration::from_millis(100)).await; } - sleep(Duration::from_secs(5)).await; } Ok(()) }); From d6196fece7afe2172e1a895fe0982e54a521cfaf Mon Sep 17 00:00:00 2001 From: TJ Sharp Date: Wed, 13 Mar 2024 18:53:51 -0400 Subject: [PATCH 07/87] Update filter --- src/database/models.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/database/models.rs b/src/database/models.rs index f9c133f..1a9ce09 100644 --- a/src/database/models.rs +++ b/src/database/models.rs @@ -872,7 +872,7 @@ impl BtcUsdPrice { coalesce(c.positionsize, 0) as positionsize FROM generate_series('{}', now(), {}) t(timestamp) LEFT JOIN trader_order c - ON c.timestamp BETWEEN t.timestamp AND t.timestamp + {} + ON c.timestamp >= t.timestamp AND c.timestamp < t.timestamp + {} ) as sq GROUP BY window_start "#, @@ -897,7 +897,7 @@ impl BtcUsdPrice { first_value(price) OVER (PARTITION BY t.timestamp ORDER BY c.timestamp desc) AS close FROM generate_series('{}', now(), {}) t(timestamp) LEFT JOIN btc_usd_price c - ON c.timestamp BETWEEN t.timestamp AND t.timestamp + {} + ON c.timestamp >= t.timestamp AND c.timestamp < t.timestamp + {} ) as w WHERE open IS NOT NULL GROUP BY window_ts From 8d3652e34426373f138247cf4f1a6c0a970f7f72 Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Tue, 19 Mar 2024 16:00:17 +0530 Subject: [PATCH 08/87] delay in ws --- src/ws/methods.rs | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/src/ws/methods.rs b/src/ws/methods.rs index 3efeb24..9335002 100644 --- a/src/ws/methods.rs +++ b/src/ws/methods.rs @@ -63,19 +63,6 @@ pub(super) fn candle_update( sink.accept()?; let CandleSubscription { interval } = params.parse()?; - // let time = match interval { - // Interval::ONE_MINUTE => chrono::Duration::minutes(1), - // Interval::FIVE_MINUTE => chrono::Duration::minutes(5), - // Interval::FIFTEEN_MINUTE => chrono::Duration::minutes(15), - // Interval::THIRTY_MINUTE => chrono::Duration::minutes(30), - // Interval::ONE_HOUR => chrono::Duration::minutes(60), - // Interval::FOUR_HOUR => chrono::Duration::hours(4), - // Interval::EIGHT_HOUR => chrono::Duration::hours(8), - // Interval::TWELVE_HOUR => chrono::Duration::hours(12), - // Interval::ONE_DAY => chrono::Duration::hours(24), - // _ => chrono::Duration::minutes(1), - // }; - // let mut last_candle_vec: Vec = Vec::new(); let _: JoinHandle> = tokio::task::spawn(async move { loop { let mut conn = ctx.pool.get()?; @@ -90,7 +77,7 @@ pub(super) fn candle_update( } sleep(Duration::from_millis(250)).await; } else { - sleep(Duration::from_millis(100)).await; + sleep(Duration::from_millis(200)).await; } } Ok(()) From 081a98e7cf2b3a43630e84a404a4785c45908128 Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Tue, 19 Mar 2024 15:24:33 +0000 Subject: [PATCH 09/87] 24hr change data interval added --- src/database/models.rs | 15 ++++++++++++--- src/rpc/types.rs | 3 +++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/database/models.rs b/src/database/models.rs index 0b64fa7..21102bf 100644 --- a/src/database/models.rs +++ b/src/database/models.rs @@ -11,7 +11,7 @@ use crate::rpc::{ TradeVolumeArgs, TransactionHashArgs, }; use bigdecimal::{BigDecimal, FromPrimitive, ToPrimitive, Zero}; -use chrono::{DurationRound, prelude::*}; +use chrono::{prelude::*, DurationRound}; use diesel::pg::Pg; use diesel::prelude::*; use itertools::join; @@ -854,8 +854,17 @@ impl BtcUsdPrice { limit: Option, offset: Option, ) -> QueryResult> { - let start = since.duration_trunc(interval.duration()).unwrap(); - + let start: DateTime; + // temp for 24 hour candle change + // need to create new api for 24hour candle change data + match interval { + Interval::ONE_DAY_CHNAGE => { + start = since + chrono::Duration::seconds(5); + } + _ => { + start = since.duration_trunc(interval.duration()).unwrap(); + } + } let interval = interval.interval_sql(); let trader_subquery = format!( diff --git a/src/rpc/types.rs b/src/rpc/types.rs index 63f7e65..ad76b0c 100644 --- a/src/rpc/types.rs +++ b/src/rpc/types.rs @@ -231,6 +231,7 @@ pub enum Interval { EIGHT_HOUR, TWELVE_HOUR, ONE_DAY, + ONE_DAY_CHNAGE, } impl Interval { @@ -245,6 +246,7 @@ impl Interval { Interval::EIGHT_HOUR => Duration::hours(8), Interval::TWELVE_HOUR => Duration::hours(12), Interval::ONE_DAY => Duration::days(1), + Interval::ONE_DAY_CHNAGE => Duration::days(1), } } @@ -259,6 +261,7 @@ impl Interval { Interval::EIGHT_HOUR => "'8 hours'", Interval::TWELVE_HOUR => "'12 hours'", Interval::ONE_DAY => "'1 day'", + Interval::ONE_DAY_CHNAGE => "'1 day'", } .into() } From 3b2b4eab7b0f3043b4a6fa32dd7d4a444a462625 Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Wed, 20 Mar 2024 11:07:04 +0000 Subject: [PATCH 10/87] typo error --- src/database/models.rs | 2 +- src/rpc/types.rs | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/database/models.rs b/src/database/models.rs index 21102bf..65550ad 100644 --- a/src/database/models.rs +++ b/src/database/models.rs @@ -858,7 +858,7 @@ impl BtcUsdPrice { // temp for 24 hour candle change // need to create new api for 24hour candle change data match interval { - Interval::ONE_DAY_CHNAGE => { + Interval::ONE_DAY_CHANGE => { start = since + chrono::Duration::seconds(5); } _ => { diff --git a/src/rpc/types.rs b/src/rpc/types.rs index ad76b0c..f880845 100644 --- a/src/rpc/types.rs +++ b/src/rpc/types.rs @@ -13,7 +13,7 @@ // • Server Time use crate::auth::UserInfo; use crate::database::OrderStatus; -use chrono::{Duration, prelude::*}; +use chrono::{prelude::*, Duration}; use relayerwalletlib::zkoswalletlib::relayer_types::{OrderType, PositionType}; use serde::{Deserialize, Serialize}; @@ -231,7 +231,7 @@ pub enum Interval { EIGHT_HOUR, TWELVE_HOUR, ONE_DAY, - ONE_DAY_CHNAGE, + ONE_DAY_CHANGE, } impl Interval { @@ -246,7 +246,7 @@ impl Interval { Interval::EIGHT_HOUR => Duration::hours(8), Interval::TWELVE_HOUR => Duration::hours(12), Interval::ONE_DAY => Duration::days(1), - Interval::ONE_DAY_CHNAGE => Duration::days(1), + Interval::ONE_DAY_CHANGE => Duration::days(1), } } @@ -261,7 +261,7 @@ impl Interval { Interval::EIGHT_HOUR => "'8 hours'", Interval::TWELVE_HOUR => "'12 hours'", Interval::ONE_DAY => "'1 day'", - Interval::ONE_DAY_CHNAGE => "'1 day'", + Interval::ONE_DAY_CHANGE => "'1 day'", } .into() } From 14762d31c263545f2928f27dffc4f1fb3de0a3ad Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Thu, 21 Mar 2024 06:46:58 +0000 Subject: [PATCH 11/87] websocket 24hour change correction --- src/ws/methods.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/ws/methods.rs b/src/ws/methods.rs index 9335002..2866203 100644 --- a/src/ws/methods.rs +++ b/src/ws/methods.rs @@ -66,7 +66,10 @@ pub(super) fn candle_update( let _: JoinHandle> = tokio::task::spawn(async move { loop { let mut conn = ctx.pool.get()?; - let since = Utc::now() - chrono::Duration::milliseconds(250); + let since: DateTime = match interval { + Interval::ONE_DAY_CHANGE => Utc::now() - chrono::Duration::hours(24), + _ => Utc::now() - chrono::Duration::milliseconds(250), + }; let candles = BtcUsdPrice::candles(&mut conn, interval.clone(), since, None, None)?; let result = serde_json::to_value(&candles)?; From ac937fe95871f4b0d5146dc25cbfab038ec6a76d Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Thu, 21 Mar 2024 07:12:19 +0000 Subject: [PATCH 12/87] 24hour change candle update every 1 sec --- src/ws/methods.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/ws/methods.rs b/src/ws/methods.rs index 2866203..28a8602 100644 --- a/src/ws/methods.rs +++ b/src/ws/methods.rs @@ -78,9 +78,16 @@ pub(super) fn candle_update( if let Err(e) = sink.send(&result) { error!("Error sending candle updates: {:?}", e); } - sleep(Duration::from_millis(250)).await; + match interval { + Interval::ONE_DAY_CHANGE => { + sleep(Duration::from_millis(1000)).await; + } + _ => { + sleep(Duration::from_millis(250)).await; + } + }; } else { - sleep(Duration::from_millis(200)).await; + sleep(Duration::from_millis(250)).await; } } Ok(()) From 5848a6dd820d6714f42a063cca281d82d58165a3 Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Fri, 22 Mar 2024 17:20:51 +0530 Subject: [PATCH 13/87] order query --- Cargo.toml | 6 +++--- src/database/models.rs | 10 ++++++++++ src/rpc.rs | 5 +++++ src/rpc/public_methods.rs | 38 +++++++++++++++++++++++++++++++++++++- 4 files changed, 55 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index d16710a..177c66c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -69,11 +69,11 @@ tower-http = { version = "0.4", features = [ tracing-subscriber = { version = "0.3.16", features = ["env-filter"] } uuid = { version = "1.3.3", features = ["serde", "v4", "v7"] } verify-keplr-sign = "0.1.0" -twilight-relayer-rust = { git = "ssh://git@github.com/twilight-project/twilight-relayer.git", branch = "zkos-0.03.01-zkos-wallet-integration" } +twilight-relayer-rust = { git = "ssh://git@github.com/twilight-project/twilight-relayer.git", branch = "client-side-trader-order-create" } [dependencies.zkos-relayer-wallet] git = "ssh://github.com/twilight-project/zkos-relayer-wallet.git" -branch = "develop" +branch = "clienttradeorder" [dependencies.zkos-client-wallet] git = "ssh://github.com/twilight-project/zkos-client-wallet.git" -branch = "develop" +branch = "clienttradeorder" diff --git a/src/database/models.rs b/src/database/models.rs index 7bf15f0..f3a0c5d 100644 --- a/src/database/models.rs +++ b/src/database/models.rs @@ -1249,6 +1249,16 @@ impl TraderOrder { .order(timestamp.desc()) .first(conn) } + pub fn get_by_signature( + conn: &mut PgConnection, + account_id: String, + ) -> QueryResult { + use crate::database::schema::trader_order::dsl::*; + trader_order + .filter(account_id.eq(account_id)) + .order(timestamp.desc()) + .first(conn) + } pub fn get_by_uuid(conn: &mut PgConnection, order_id: String) -> QueryResult { use crate::database::schema::address_customer_id::dsl as addr_dsl; use crate::database::schema::trader_order::dsl::*; diff --git a/src/rpc.rs b/src/rpc.rs index a0293d7..7acbc6c 100644 --- a/src/rpc.rs +++ b/src/rpc.rs @@ -99,6 +99,11 @@ pub fn init_public_methods(database_url: &str) -> RpcModule { "transaction_hashes", Box::new(public_methods::transaction_hashes), ); + register_method( + &mut module, + "trader_order_info", + Box::new(public_methods::trader_order_info), + ); module } diff --git a/src/rpc/public_methods.rs b/src/rpc/public_methods.rs index c045cab..89076c4 100644 --- a/src/rpc/public_methods.rs +++ b/src/rpc/public_methods.rs @@ -2,7 +2,8 @@ use super::*; use crate::database::*; use chrono::prelude::*; use jsonrpsee::{core::error::Error, server::logger::Params}; - +use relayerwalletlib::verify_client_message::verify_query_order; +use twilight_relayer_rust::relayer; pub(super) fn btc_usd_price( _: Params<'_>, ctx: &RelayerContext, @@ -150,3 +151,38 @@ pub(super) fn transaction_hashes( pub(super) fn server_time(_: Params<'_>, _: &RelayerContext) -> Result { Ok(serde_json::to_value(Utc::now()).expect("Failed to get timestamp")) } + +pub(super) fn trader_order_info( + params: Params<'_>, + ctx: &RelayerContext, +) -> Result { + let args: RpcArgs = params.parse()?; + // let (order) = args.unpack(); + let (customer_id, order) = args.unpack(); + let Order { data } = order; + + let Ok(bytes) = hex::decode(&data) else { + return Ok(format!("Invalid hex data").into()); + }; + + let Ok(tx) = bincode::deserialize::(&bytes) else { + return Ok(format!("Invalid bincode").into()); + }; + + if let Err(_) = verify_query_order( + tx.msg.clone(), + &bincode::serialize(&tx.query_trader_order).unwrap(), + ) { + return Ok(format!("Invalid order params").into()); + } + + match ctx.pool.get() { + Ok(mut conn) => { + match TraderOrder::get_by_signature(&mut conn, tx.query_trader_order.account_id) { + Ok(o) => Ok(serde_json::to_value(o).expect("Error converting response")), + Err(e) => Err(Error::Custom(format!("Error fetching order info: {:?}", e))), + } + } + Err(e) => Err(Error::Custom(format!("Database error: {:?}", e))), + } +} From 5f872ef1c1d5b31279e68cb7da7cd53d5aa40b86 Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Fri, 22 Mar 2024 17:41:49 +0530 Subject: [PATCH 14/87] order query for trader order --- src/bin/api.rs | 2 +- src/bin/archiver.rs | 4 +-- src/bin/auth.rs | 30 +++++++++--------- src/database/models.rs | 18 +++++------ src/database/sql_types.rs | 2 -- src/lib.rs | 62 -------------------------------------- src/rpc.rs | 4 +-- src/rpc/private_methods.rs | 18 +++++------ src/rpc/public_methods.rs | 2 +- src/rpc/types.rs | 1 - src/ws.rs | 24 +++++++-------- src/ws/methods.rs | 12 +++----- 12 files changed, 55 insertions(+), 124 deletions(-) diff --git a/src/bin/api.rs b/src/bin/api.rs index 8706baf..27e0745 100644 --- a/src/bin/api.rs +++ b/src/bin/api.rs @@ -1,11 +1,11 @@ use jsonrpsee::server::ServerBuilder; use log::info; +use relayerarchiverlib::{rpc, ws}; use std::{net::SocketAddr, time::Duration}; use structopt::StructOpt; use tokio::time::sleep; use tower::ServiceBuilder; use tower_http::cors::{Any, CorsLayer}; -use twilight_relayerAPI::{rpc, ws}; #[derive(Debug, StructOpt)] #[structopt(name = "Relayer API", about = "Twilight Relayer API server")] diff --git a/src/bin/archiver.rs b/src/bin/archiver.rs index cb50f4d..dafc92b 100644 --- a/src/bin/archiver.rs +++ b/src/bin/archiver.rs @@ -1,7 +1,7 @@ use crossbeam_channel::unbounded; use log::warn; -use twilight_relayerAPI::kafka; -use twilight_relayerAPI::DatabaseArchiver; +use relayerarchiverlib::kafka; +use relayerarchiverlib::DatabaseArchiver; const SNAPSHOT_TOPIC: &str = "CoreEventLogTopic"; const ARCHIVER_GROUP: &str = "Archiver"; diff --git a/src/bin/auth.rs b/src/bin/auth.rs index ef0c168..5b9b67a 100644 --- a/src/bin/auth.rs +++ b/src/bin/auth.rs @@ -4,14 +4,14 @@ use hmac::{Hmac, Mac}; use http::{Request, StatusCode}; use hyper::{body::to_bytes, server::Server, Body, Response}; use log::debug; -use serde::{Deserialize, Serialize}; -use sha2::Sha256; -use std::net::SocketAddr; -use tower::{make::Shared, ServiceBuilder}; -use twilight_relayerAPI::{ +use relayerarchiverlib::{ auth::{AuthInfo, UserInfo}, database::{AddressCustomerId, CustomerApiKeyLinking}, }; +use serde::Deserialize; +use sha2::Sha256; +use std::net::SocketAddr; +use tower::{make::Shared, ServiceBuilder}; use verify_keplr_sign::{verify_arbitrary, Signature}; type HS = Hmac; @@ -48,7 +48,7 @@ async fn verify_signature(request: Request) -> VerifyResult { data, } = match serde_json::from_slice(&request) { Ok(auth) => auth, - Err(e) => return VerifyResult::InvalidJson, + Err(_e) => return VerifyResult::InvalidJson, }; let is_ok = verify_arbitrary( @@ -69,7 +69,7 @@ async fn register_handler(account_address: String) -> Result, htt let database_url = std::env::var("DATABASE_URL").expect("No database url set!"); let mut conn = match PgConnection::establish(&database_url) { Ok(c) => c, - Err(e) => { + Err(_e) => { return Response::builder() .status(StatusCode::INTERNAL_SERVER_ERROR) .body("Internal db error".into()); @@ -83,7 +83,7 @@ async fn register_handler(account_address: String) -> Result, htt "User already registered, call /regenerate if you need a new API key".into(), ); } - Err(e) => { + Err(_e) => { return Response::builder() .status(StatusCode::INTERNAL_SERVER_ERROR) .body("Internal db error".into()); @@ -92,7 +92,7 @@ async fn register_handler(account_address: String) -> Result, htt let (api_key, api_secret) = match CustomerApiKeyLinking::create(&mut conn, customer_id) { Ok(link) => (link.api_key, link.api_salt_key), - Err(e) => { + Err(_e) => { return Response::builder() .status(StatusCode::INTERNAL_SERVER_ERROR) .body("Internal db error".into()); @@ -114,7 +114,7 @@ async fn regenerate_handler(address: String) -> Result, http::Err let database_url = std::env::var("DATABASE_URL").expect("No database url set!"); let mut conn = match PgConnection::establish(&database_url) { Ok(c) => c, - Err(e) => { + Err(_e) => { return Response::builder() .status(StatusCode::INTERNAL_SERVER_ERROR) .body("Internal db error".into()); @@ -128,7 +128,7 @@ async fn regenerate_handler(address: String) -> Result, http::Err .status(StatusCode::NOT_FOUND) .body("User not found, please call /register".into()); } - Err(e) => { + Err(_e) => { return Response::builder() .status(StatusCode::INTERNAL_SERVER_ERROR) .body("Internal db error".into()); @@ -137,7 +137,7 @@ async fn regenerate_handler(address: String) -> Result, http::Err let (api_key, api_secret) = match CustomerApiKeyLinking::regenerate(&mut conn, customer_id) { Ok(link) => (link.api_key, link.api_salt_key), - Err(e) => { + Err(_e) => { return Response::builder() .status(StatusCode::INTERNAL_SERVER_ERROR) .body("Internal db error".into()); @@ -159,7 +159,7 @@ async fn check_signature(request: Request) -> Result, http: let database_url = std::env::var("DATABASE_URL").expect("No database url set!"); let mut conn = match PgConnection::establish(&database_url) { Ok(c) => c, - Err(e) => { + Err(_e) => { return Response::builder() .status(StatusCode::INTERNAL_SERVER_ERROR) .body("Internal db error".into()); @@ -174,12 +174,12 @@ async fn check_signature(request: Request) -> Result, http: api_key, sig, body, - datetime, + datetime: _, } = serde_json::from_slice(&request).expect("f"); let key = match CustomerApiKeyLinking::get_key(&mut conn, api_key) { Ok(k) => k, - Err(e) => { + Err(_e) => { return Response::builder() .status(StatusCode::UNAUTHORIZED) .body("No customer with that key".into()); diff --git a/src/database/models.rs b/src/database/models.rs index 1911b3a..301f35f 100644 --- a/src/database/models.rs +++ b/src/database/models.rs @@ -12,7 +12,7 @@ use crate::rpc::{ }; use bigdecimal::{BigDecimal, FromPrimitive, ToPrimitive, Zero}; use chrono::{prelude::*, DurationRound}; -use diesel::pg::Pg; +// use diesel::pg::Pg; use diesel::prelude::*; use itertools::join; use serde::{Deserialize, Serialize}; @@ -1264,16 +1264,16 @@ impl TraderOrder { } pub fn get_by_signature( conn: &mut PgConnection, - account_id: String, + accountid: String, ) -> QueryResult { use crate::database::schema::trader_order::dsl::*; trader_order - .filter(account_id.eq(account_id)) + .filter(account_id.eq(accountid)) .order(timestamp.desc()) .first(conn) } pub fn get_by_uuid(conn: &mut PgConnection, order_id: String) -> QueryResult { - use crate::database::schema::address_customer_id::dsl as addr_dsl; + // use crate::database::schema::address_customer_id::dsl as addr_dsl; use crate::database::schema::trader_order::dsl::*; trader_order @@ -1434,7 +1434,7 @@ impl TraderOrder { args: TradeVolumeArgs, ) -> QueryResult { use crate::database::schema::address_customer_id::dsl as acct_dsl; - use crate::database::schema::trader_order::dsl::*; + // use crate::database::schema::trader_order::dsl::*; let accounts: Vec = acct_dsl::address_customer_id .filter(acct_dsl::customer_id.eq(customer_id)) @@ -1460,8 +1460,8 @@ impl TraderOrder { } pub fn order_book(conn: &mut PgConnection) -> QueryResult { - use crate::database::schema::trader_order::dsl::*; - use diesel::dsl::{max, sum}; + // use crate::database::schema::trader_order::dsl::*; + // use diesel::dsl::{max, sum}; let query = r#" WITH orders AS ( @@ -1532,7 +1532,7 @@ impl TraderOrder { pub fn open_orders(conn: &mut PgConnection, customer_id: i64) -> QueryResult> { use crate::database::schema::address_customer_id::dsl as acct_dsl; - use crate::database::schema::trader_order::dsl::*; + // use crate::database::schema::trader_order::dsl::*; let account: Vec = acct_dsl::address_customer_id .filter(acct_dsl::customer_id.eq(customer_id)) @@ -1561,7 +1561,7 @@ impl TraderOrder { } pub fn list_past_24hrs(conn: &mut PgConnection) -> QueryResult> { - use crate::database::schema::trader_order::dsl::*; + // use crate::database::schema::trader_order::dsl::*; let query = r#"SELECT trader_order.position_type as side, diff --git a/src/database/sql_types.rs b/src/database/sql_types.rs index 2fed71b..55fa42f 100644 --- a/src/database/sql_types.rs +++ b/src/database/sql_types.rs @@ -7,7 +7,6 @@ use crate::database::schema::sql_types::{ }; use diesel::*; use diesel::{ - backend::Backend, deserialize::FromSql, pg::Pg, serialize::{self, IsNull, Output, ToSql}, @@ -15,7 +14,6 @@ use diesel::{ use relayerwalletlib::zkoswalletlib::relayer_types; use serde::{Deserialize, Serialize}; use std::io::Write; -use twilight_relayer_rust::relayer; #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] pub enum TXType { diff --git a/src/lib.rs b/src/lib.rs index df424ba..a009e17 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -22,65 +22,3 @@ pub mod auth { pub customer_id: i64, } } - -// for openrpc api -use diesel::prelude::PgConnection; -use diesel::r2d2::ConnectionManager; -type ManagedConnection = ConnectionManager; -type ManagedPool = r2d2::Pool; -const MAX_RETRIES: usize = 5; -const RETRY_SLEEP: u64 = 2000; -use crate::error::ApiError; -use log::{debug, error, info, trace}; -use r2d2::PooledConnection; -use std::time::{Duration, Instant}; -pub struct RelayerDB { - pool: ManagedPool, -} -impl RelayerDB { - pub fn from_host(database_url: String) -> Self { - RelayerDB { - pool: { - let manager = ConnectionManager::::new(database_url); - let pool = r2d2::Pool::new(manager).expect("Could not instantiate connection pool"); - pool - }, - } - } - /// Fetch a connection, will retry MAX_RETRIES before giving up. - pub fn get_conn(&self) -> Result, ApiError> { - let mut retries = MAX_RETRIES; - - Ok(loop { - break match self.pool.get() { - Ok(c) => c, - Err(e) => { - error!("Could not get connection from connection pool! {:?}", e); - std::thread::sleep(Duration::from_millis(RETRY_SLEEP)); - - if retries == 0 { - return Err(ApiError::CommitRetryCountExceeded); - } - - retries -= 1; - - continue; - } - }; - }) - } -} - -#[cfg(test)] -mod test { - - use crate::{database::TraderOrder as TraderOrderDB, RelayerDB}; - #[test] - fn test_check_get_order_by_uuid_from_archiver() { - dotenv::dotenv().expect("Failed loading dotenv"); - let database_url = std::env::var("DATABASE_URL").expect("No database url found!"); - let relayer_db = RelayerDB::from_host(database_url); - let mut pool = relayer_db.get_conn().unwrap(); - TraderOrderDB::get_by_uuid(&mut *pool, "".to_string()); - } -} diff --git a/src/rpc.rs b/src/rpc.rs index 7acbc6c..d89f13e 100644 --- a/src/rpc.rs +++ b/src/rpc.rs @@ -1,7 +1,7 @@ use diesel::prelude::PgConnection; use diesel::r2d2::ConnectionManager; use jsonrpsee::{core::error::Error, server::logger::Params, RpcModule}; -use kafka::producer::{Producer, Record, RequiredAcks}; +use kafka::producer::{Producer, RequiredAcks}; use serde::Serialize; use std::sync::{Arc, Mutex}; use tokio::time::Duration; @@ -113,7 +113,7 @@ pub fn init_private_methods(database_url: &str) -> RpcModule { let broker_host = std::env::var("BROKER").expect("missing environment variable BROKER"); let broker = vec![broker_host.to_owned()]; - let mut kafka = Producer::from_hosts(broker) + let kafka = Producer::from_hosts(broker) .with_ack_timeout(Duration::from_secs(1)) .with_required_acks(RequiredAcks::One) .create() diff --git a/src/rpc/private_methods.rs b/src/rpc/private_methods.rs index 26648dd..9e3f1c0 100644 --- a/src/rpc/private_methods.rs +++ b/src/rpc/private_methods.rs @@ -1,9 +1,7 @@ use super::*; -use crate::{auth::AuthInfo, database::*}; -use chrono::prelude::*; +use crate::database::*; use jsonrpsee::{core::error::Error, server::logger::Params}; use kafka::producer::Record; -use log::info; use relayerwalletlib::verify_client_message::{ verify_query_order, verify_settle_requests, verify_trade_lend_order, }; @@ -403,12 +401,12 @@ pub(super) fn lend_order_info( } } -pub(super) fn last_day_apy( - _: Params<'_>, - ctx: &RelayerContext, -) -> Result { - todo!("APY") -} +// pub(super) fn last_day_apy( +// _: Params<'_>, +// ctx: &RelayerContext, +// ) -> Result { +// todo!("APY") +// } pub(super) fn unrealized_pnl( params: Params<'_>, @@ -431,7 +429,7 @@ pub(super) fn open_orders( ctx: &RelayerContext, ) -> Result { let args: RpcArgs<()> = params.parse()?; - let (id, params) = args.unpack(); + let (id, _params) = args.unpack(); match ctx.pool.get() { Ok(mut conn) => match TraderOrder::open_orders(&mut conn, id) { diff --git a/src/rpc/public_methods.rs b/src/rpc/public_methods.rs index 89076c4..ef2c4d8 100644 --- a/src/rpc/public_methods.rs +++ b/src/rpc/public_methods.rs @@ -158,7 +158,7 @@ pub(super) fn trader_order_info( ) -> Result { let args: RpcArgs = params.parse()?; // let (order) = args.unpack(); - let (customer_id, order) = args.unpack(); + let (_customer_id, order) = args.unpack(); let Order { data } = order; let Ok(bytes) = hex::decode(&data) else { diff --git a/src/rpc/types.rs b/src/rpc/types.rs index f880845..e3f187c 100644 --- a/src/rpc/types.rs +++ b/src/rpc/types.rs @@ -14,7 +14,6 @@ use crate::auth::UserInfo; use crate::database::OrderStatus; use chrono::{prelude::*, Duration}; -use relayerwalletlib::zkoswalletlib::relayer_types::{OrderType, PositionType}; use serde::{Deserialize, Serialize}; #[derive(Debug, Serialize, Deserialize)] diff --git a/src/ws.rs b/src/ws.rs index 72d851b..fc75908 100644 --- a/src/ws.rs +++ b/src/ws.rs @@ -1,6 +1,6 @@ use crate::database::{NewOrderBookOrder, TraderOrder}; use crate::kafka::start_consumer; -use bigdecimal::ToPrimitive; +// use bigdecimal::ToPrimitive; use chrono::prelude::*; use crossbeam_channel::{unbounded, Sender as CrossbeamSender}; use diesel::prelude::PgConnection; @@ -8,7 +8,7 @@ use diesel::r2d2::ConnectionManager; use jsonrpsee::RpcModule; use log::{error, info, trace}; use relayerwalletlib::zkoswalletlib::relayer_types::{OrderStatus, OrderType}; -use serde::{Deserialize, Serialize}; +// use serde::{Deserialize, Serialize}; use std::time::{Duration, Instant}; use tokio::{ sync::broadcast::{channel, Sender}, @@ -78,7 +78,7 @@ impl WsContext { match to.order_status { OrderStatus::PENDING | OrderStatus::FILLED => { - recent_trades2.send(to); + let _ = recent_trades2.send(to); } _ => {} } @@ -194,15 +194,15 @@ pub fn init_methods(database_url: &str) -> RpcModule { #[cfg(test)] mod tests { - use super::*; - use jsonrpsee::{ - core::{ - client::ClientT, - params::{ArrayParams, ObjectParams}, - }, - http_client::HttpClientBuilder, - server::ServerBuilder, - }; + // use super::*; + // use jsonrpsee::{ + // core::{ + // client::ClientT, + // params::{ArrayParams, ObjectParams}, + // }, + // http_client::HttpClientBuilder, + // server::ServerBuilder, + // }; // #[tokio::test] // async fn test_hello() { diff --git a/src/ws/methods.rs b/src/ws/methods.rs index 28a8602..537a566 100644 --- a/src/ws/methods.rs +++ b/src/ws/methods.rs @@ -1,5 +1,6 @@ +#[allow(unreachable_code)] use crate::{ - database::{BtcUsdPrice, CandleData, TraderOrder}, + database::{BtcUsdPrice, TraderOrder}, error::ApiError, rpc::{CandleSubscription, Interval}, }; @@ -10,10 +11,7 @@ use jsonrpsee::{ }; use log::{error, info, warn}; use serde::Serialize; -use std::{ - sync::Arc, - time::{Duration, Instant}, -}; +use std::{sync::Arc, time::Duration}; use tokio::{ sync::broadcast::{error::TryRecvError, Receiver}, task::JoinHandle, @@ -107,7 +105,7 @@ pub(super) fn spawn_order_book( let _: JoinHandle> = tokio::task::spawn(async move { loop { match rx.try_recv() { - Ok(mesg) => { + Ok(_mesg) => { let mut conn = ctx.pool.get()?; let orders = TraderOrder::order_book(&mut conn)?; let result = serde_json::to_value(&orders)?; @@ -138,7 +136,7 @@ pub(super) fn spawn_order_book( pub(super) fn heartbeat( _params: Params<'_>, mut sink: SubscriptionSink, - ctx: Arc, + _ctx: Arc, ) -> SubscriptionResult { sink.accept()?; From de5489a957321f90f136b47bdd563a17ff7001f8 Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Fri, 22 Mar 2024 14:52:37 +0000 Subject: [PATCH 15/87] trader order query api --- src/rpc/public_methods.rs | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/src/rpc/public_methods.rs b/src/rpc/public_methods.rs index ef2c4d8..c9dfdf8 100644 --- a/src/rpc/public_methods.rs +++ b/src/rpc/public_methods.rs @@ -156,26 +156,20 @@ pub(super) fn trader_order_info( params: Params<'_>, ctx: &RelayerContext, ) -> Result { - let args: RpcArgs = params.parse()?; - // let (order) = args.unpack(); - let (_customer_id, order) = args.unpack(); - let Order { data } = order; - + let Order { data } = params.parse()?; let Ok(bytes) = hex::decode(&data) else { return Ok(format!("Invalid hex data").into()); }; - + // println!("bytes:{:?}", bytes); let Ok(tx) = bincode::deserialize::(&bytes) else { return Ok(format!("Invalid bincode").into()); }; - - if let Err(_) = verify_query_order( + if let Err(arg) = verify_query_order( tx.msg.clone(), &bincode::serialize(&tx.query_trader_order).unwrap(), ) { - return Ok(format!("Invalid order params").into()); + return Ok(format!("Invalid order params:{:?}", arg).into()); } - match ctx.pool.get() { Ok(mut conn) => { match TraderOrder::get_by_signature(&mut conn, tx.query_trader_order.account_id) { From 98541d197210c1aad3e11d1e19f860136438c5ad Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Fri, 22 Mar 2024 19:55:50 +0000 Subject: [PATCH 16/87] query lend order from public api --- src/database/models.rs | 8 ++++++++ src/rpc.rs | 5 +++++ src/rpc/public_methods.rs | 28 ++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+) diff --git a/src/database/models.rs b/src/database/models.rs index 301f35f..e84941a 100644 --- a/src/database/models.rs +++ b/src/database/models.rs @@ -1692,6 +1692,14 @@ impl LendOrder { .order(timestamp.desc()) .first(conn) } + pub fn get_by_signature(conn: &mut PgConnection, accountid: String) -> QueryResult { + use crate::database::schema::lend_order::dsl::*; + + lend_order + .filter(account_id.eq(accountid)) + .order(timestamp.desc()) + .first(conn) + } pub fn insert(conn: &mut PgConnection, orders: Vec) -> QueryResult { use crate::database::schema::lend_order::dsl::*; diff --git a/src/rpc.rs b/src/rpc.rs index d89f13e..3dccb78 100644 --- a/src/rpc.rs +++ b/src/rpc.rs @@ -104,6 +104,11 @@ pub fn init_public_methods(database_url: &str) -> RpcModule { "trader_order_info", Box::new(public_methods::trader_order_info), ); + register_method( + &mut module, + "lend_order_info", + Box::new(public_methods::lend_order_info), + ); module } diff --git a/src/rpc/public_methods.rs b/src/rpc/public_methods.rs index c9dfdf8..eb6a2cd 100644 --- a/src/rpc/public_methods.rs +++ b/src/rpc/public_methods.rs @@ -180,3 +180,31 @@ pub(super) fn trader_order_info( Err(e) => Err(Error::Custom(format!("Database error: {:?}", e))), } } +pub(super) fn lend_order_info( + params: Params<'_>, + ctx: &RelayerContext, +) -> Result { + let Order { data } = params.parse()?; + let Ok(bytes) = hex::decode(&data) else { + return Ok(format!("Invalid hex data").into()); + }; + // println!("bytes:{:?}", bytes); + let Ok(tx) = bincode::deserialize::(&bytes) else { + return Ok(format!("Invalid bincode").into()); + }; + if let Err(arg) = verify_query_order( + tx.msg.clone(), + &bincode::serialize(&tx.query_lend_order).unwrap(), + ) { + return Ok(format!("Invalid order params:{:?}", arg).into()); + } + match ctx.pool.get() { + Ok(mut conn) => { + match LendOrder::get_by_signature(&mut conn, tx.query_lend_order.account_id) { + Ok(o) => Ok(serde_json::to_value(o).expect("Error converting response")), + Err(e) => Err(Error::Custom(format!("Error fetching order info: {:?}", e))), + } + } + Err(e) => Err(Error::Custom(format!("Database error: {:?}", e))), + } +} From 75474759c78cd018248c7478af033872ec250d22 Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Tue, 26 Mar 2024 13:15:09 +0530 Subject: [PATCH 17/87] rpc open api shifted to public api module --- src/database/models.rs | 17 ++- src/database/sql_types.rs | 10 +- src/rpc.rs | 32 ++++- src/rpc/private_methods.rs | 50 ++++--- src/rpc/public_methods.rs | 268 ++++++++++++++++++++++++++++++++++++- src/rpc/types.rs | 2 +- src/ws/methods.rs | 6 +- 7 files changed, 348 insertions(+), 37 deletions(-) diff --git a/src/database/models.rs b/src/database/models.rs index e84941a..4dca157 100644 --- a/src/database/models.rs +++ b/src/database/models.rs @@ -1,3 +1,4 @@ +#![allow(warnings)] use crate::database::{ schema::{ address_customer_id, btc_usd_price, current_nonce, customer_account, @@ -186,7 +187,7 @@ impl AddressCustomerId { .filter(address.eq(addr)) .first::(conn) { - Ok(o) => return Ok(None), + Ok(_o) => return Ok(None), Err(diesel::result::Error::NotFound) => { let mut account = NewCustomerAccount::default(); account.customer_registration_id = Uuid::new_v4().to_string(); @@ -1303,7 +1304,7 @@ impl TraderOrder { .load(conn)?; let accounts: Vec<_> = accounts.into_iter().map(|a| a.address).collect(); - let price = BtcUsdPrice::get(conn)?; + let _price = BtcUsdPrice::get(conn)?; let closed = vec![ OrderStatus::PENDING, OrderStatus::CANCELLED, @@ -1485,7 +1486,7 @@ impl TraderOrder { let shorts: Vec = diesel::sql_query(query).get_results(conn)?; - let mut ask: Vec<_> = shorts + let ask: Vec<_> = shorts .into_iter() .map(|order| Ask { id: order.uuid, @@ -1516,7 +1517,7 @@ impl TraderOrder { let longs: Vec = diesel::sql_query(query).get_results(conn)?; - let mut bid = longs + let bid = longs .into_iter() .map(|order| Bid { id: order.uuid, @@ -1700,6 +1701,14 @@ impl LendOrder { .order(timestamp.desc()) .first(conn) } + pub fn get_by_uuid(conn: &mut PgConnection, order_id: String) -> QueryResult { + use crate::database::schema::lend_order::dsl::*; + + lend_order + .filter(uuid.eq(order_id)) + .order(timestamp.desc()) + .first(conn) + } pub fn insert(conn: &mut PgConnection, orders: Vec) -> QueryResult { use crate::database::schema::lend_order::dsl::*; diff --git a/src/database/sql_types.rs b/src/database/sql_types.rs index 55fa42f..3fef715 100644 --- a/src/database/sql_types.rs +++ b/src/database/sql_types.rs @@ -1,5 +1,5 @@ #![allow(non_camel_case_types)] - +#![allow(warnings)] use crate::database::schema::sql_types::{ LendPoolCommandType as LendPoolCommandTypeSql, OrderStatus as OrderStatusSql, OrderType as OrderTypeSql, PositionSizeCommand as PositionSizeCommandSql, @@ -108,10 +108,10 @@ impl OrderStatus { use OrderStatus::*; match self { - SETTLED => true, - LENDED => true, - LIQUIDATE => true, - CANCELLED => true, + SETTLED => false, + LENDED => false, + LIQUIDATE => false, + CANCELLED => false, PENDING => false, FILLED => true, } diff --git a/src/rpc.rs b/src/rpc.rs index 3dccb78..6fc8dce 100644 --- a/src/rpc.rs +++ b/src/rpc.rs @@ -41,7 +41,7 @@ pub fn init_public_methods(database_url: &str) -> RpcModule { let broker_host = std::env::var("BROKER").expect("missing environment variable BROKER"); let broker = vec![broker_host.to_owned()]; - let mut kafka = Producer::from_hosts(broker) + let kafka = Producer::from_hosts(broker) .with_ack_timeout(Duration::from_secs(1)) .with_required_acks(RequiredAcks::One) .create() @@ -109,6 +109,31 @@ pub fn init_public_methods(database_url: &str) -> RpcModule { "lend_order_info", Box::new(public_methods::lend_order_info), ); + register_method( + &mut module, + "submit_trade_order", + Box::new(public_methods::submit_trade_order), + ); + register_method( + &mut module, + "submit_lend_order", + Box::new(public_methods::submit_lend_order), + ); + register_method( + &mut module, + "settle_trade_order", + Box::new(public_methods::settle_trade_order), + ); + register_method( + &mut module, + "settle_lend_order", + Box::new(public_methods::settle_lend_order), + ); + register_method( + &mut module, + "cancel_trader_order", + Box::new(public_methods::cancel_trader_order), + ); module } @@ -147,6 +172,11 @@ pub fn init_private_methods(database_url: &str) -> RpcModule { "settle_trade_order", Box::new(private_methods::settle_trade_order), ); + register_method( + &mut module, + "cancel_trader_order", + Box::new(private_methods::cancel_trader_order), + ); register_method( &mut module, "submit_bulk_order", diff --git a/src/rpc/private_methods.rs b/src/rpc/private_methods.rs index 9e3f1c0..b740174 100644 --- a/src/rpc/private_methods.rs +++ b/src/rpc/private_methods.rs @@ -3,9 +3,11 @@ use crate::database::*; use jsonrpsee::{core::error::Error, server::logger::Params}; use kafka::producer::Record; use relayerwalletlib::verify_client_message::{ - verify_query_order, verify_settle_requests, verify_trade_lend_order, + verify_client_create_trader_order, verify_query_order, verify_settle_requests, + verify_trade_lend_order, }; use twilight_relayer_rust::relayer; +use zkoswalletlib::relayer_rpcclient::method::RequestResponse; pub(super) fn submit_lend_order( params: Params<'_>, @@ -98,7 +100,7 @@ pub(super) fn settle_lend_order( return Ok(format!("Order not found").into()); }; - if ord.order_status.is_closed() { + if !ord.order_status.is_closed() { return Ok(format!("Order closed").into()); } @@ -138,22 +140,29 @@ pub(super) fn submit_trade_order( return Ok(format!("Invalid hex data").into()); }; - let Ok(tx) = bincode::deserialize::(&bytes) else { + let Ok(tx) = bincode::deserialize::(&bytes) else { return Ok(format!("Invalid bincode").into()); }; - if let Err(_) = verify_trade_lend_order(&tx.input) { + if let Err(_) = verify_client_create_trader_order(&tx.tx) { return Ok(format!("Invalid order params").into()); } + let Ok(transaction_ser) = bincode::serialize(&tx.tx) else { + return Ok(format!("Invalid bincode").into()); + }; + let mut order = tx.create_trader_order.clone(); let meta = relayer::Meta::default(); - - order.account_id = tx.input.input.as_owner_address().cloned().unwrap(); - let margin = order.initial_margin / 10000.0; - order.initial_margin = margin; - order.available_margin = margin; - + let response_id = order.account_id.clone(); + order.available_margin = order.initial_margin; + + let Ok(response) = serde_json::to_value(&RequestResponse::new( + "Order request submitted successfully".to_string(), + response_id, + )) else { + return Ok(format!("Invalid response").into()); + }; let Ok(mut conn) = ctx.pool.get() else { return Ok(format!("Database connection error").into()); }; @@ -162,11 +171,8 @@ pub(super) fn submit_trade_order( return Ok(format!("Failed to update customer id!").into()); } - let order = relayer::RpcCommand::CreateTraderOrder( - order.clone(), - meta, - tx.input.encode_as_hex_string(), - ); + let order = + relayer::RpcCommand::CreateTraderOrder(order.clone(), meta, hex::encode(transaction_ser)); let Ok(serialized) = serde_json::to_vec(&order) else { return Ok(format!("Could not serialize order").into()); }; @@ -175,7 +181,7 @@ pub(super) fn submit_trade_order( if let Err(e) = ctx.kafka.lock().expect("Lock poisoned!").send(&record) { Ok(format!("Could not send order {:?}", e).into()) } else { - Ok("OK".into()) + Ok(response) } } @@ -211,7 +217,7 @@ pub(super) fn settle_trade_order( return Ok(format!("Order not found").into()); }; - if ord.order_status.is_closed() { + if !ord.order_status.is_closed() { return Ok(format!("Order closed").into()); } @@ -237,7 +243,7 @@ pub(super) fn settle_trade_order( } } -pub(super) fn cancel_order( +pub(super) fn cancel_trader_order( params: Params<'_>, ctx: &RelayerContext, ) -> Result { @@ -297,12 +303,12 @@ pub(super) fn cancel_order( pub(super) fn submit_bulk_order( params: Params<'_>, - ctx: &RelayerContext, + _ctx: &RelayerContext, ) -> Result { - let topic = std::env::var("RPC_CLIENT_REQUEST").expect("No client topic!"); + let _topic = std::env::var("RPC_CLIENT_REQUEST").expect("No client topic!"); let args: RpcArgs> = params.parse()?; - let (account_id, orders) = args.unpack(); - let account_id = format!("{:016x}", account_id); + let (account_id, _orders) = args.unpack(); + let _account_id = format!("{:016x}", account_id); // TODO: bulk orders with ZkOS?? Ok("OK".into()) diff --git a/src/rpc/public_methods.rs b/src/rpc/public_methods.rs index eb6a2cd..f29ae4d 100644 --- a/src/rpc/public_methods.rs +++ b/src/rpc/public_methods.rs @@ -2,8 +2,13 @@ use super::*; use crate::database::*; use chrono::prelude::*; use jsonrpsee::{core::error::Error, server::logger::Params}; -use relayerwalletlib::verify_client_message::verify_query_order; +use kafka::producer::Record; +use relayerwalletlib::verify_client_message::{ + verify_client_create_trader_order, verify_query_order, verify_settle_requests, + verify_trade_lend_order, +}; use twilight_relayer_rust::relayer; +use zkoswalletlib::relayer_rpcclient::method::RequestResponse; pub(super) fn btc_usd_price( _: Params<'_>, ctx: &RelayerContext, @@ -180,6 +185,7 @@ pub(super) fn trader_order_info( Err(e) => Err(Error::Custom(format!("Database error: {:?}", e))), } } + pub(super) fn lend_order_info( params: Params<'_>, ctx: &RelayerContext, @@ -208,3 +214,263 @@ pub(super) fn lend_order_info( Err(e) => Err(Error::Custom(format!("Database error: {:?}", e))), } } + +pub(super) fn submit_trade_order( + params: Params<'_>, + ctx: &RelayerContext, +) -> Result { + let topic = std::env::var("RPC_CLIENT_REQUEST").expect("No client topic!"); + let Order { data } = params.parse()?; + + let Ok(bytes) = hex::decode(&data) else { + return Ok(format!("Invalid hex data").into()); + }; + + let Ok(tx) = bincode::deserialize::(&bytes) else { + return Ok(format!("Invalid bincode").into()); + }; + + if let Err(_) = verify_client_create_trader_order(&tx.tx) { + return Ok(format!("Invalid order params").into()); + } + + let Ok(transaction_ser) = bincode::serialize(&tx.tx) else { + return Ok(format!("Invalid bincode").into()); + }; + + let mut order = tx.create_trader_order.clone(); + let meta = relayer::Meta::default(); + let response_id = order.account_id.clone(); + order.available_margin = order.initial_margin; + + let Ok(response) = serde_json::to_value(&RequestResponse::new( + "Order request submitted successfully".to_string(), + response_id, + )) else { + return Ok(format!("Invalid response").into()); + }; + + let order = + relayer::RpcCommand::CreateTraderOrder(order.clone(), meta, hex::encode(transaction_ser)); + let Ok(serialized) = serde_json::to_vec(&order) else { + return Ok(format!("Could not serialize order").into()); + }; + + let record = Record::from_key_value(&topic, "CreateTraderOrder", serialized); + if let Err(e) = ctx.kafka.lock().expect("Lock poisoned!").send(&record) { + Ok(format!("Could not send order {:?}", e).into()) + } else { + Ok(response) + } +} + +pub(super) fn submit_lend_order( + params: Params<'_>, + ctx: &RelayerContext, +) -> Result { + let topic = std::env::var("RPC_CLIENT_REQUEST").expect("No client topic!"); + let Order { data } = params.parse()?; + + let Ok(bytes) = hex::decode(&data) else { + return Ok(format!("Invalid hex data").into()); + }; + + let Ok(tx) = bincode::deserialize::(&bytes) else { + return Ok(format!("Invalid bincode").into()); + }; + + if let Err(_) = verify_trade_lend_order(&tx.input) { + return Ok(format!("Invalid order params").into()); + } + + let mut order = tx.create_lend_order.clone(); + let response_id = order.account_id.clone(); + let meta = relayer::Meta::default(); + order.balance = order.deposit; + let Ok(response) = serde_json::to_value(&RequestResponse::new( + "Order request submitted successfully".to_string(), + response_id, + )) else { + return Ok(format!("Invalid response").into()); + }; + let order = + relayer::RpcCommand::CreateLendOrder(order.clone(), meta, tx.input.encode_as_hex_string()); + let Ok(serialized) = serde_json::to_vec(&order) else { + return Ok(format!("Could not serialize order").into()); + }; + + let record = Record::from_key_value(&topic, "CreateLendOrder", serialized); + if let Err(e) = ctx.kafka.lock().expect("Lock poisoned!").send(&record) { + Ok(format!("Could not send order {:?}", e).into()) + } else { + Ok(response) + } +} + +pub(super) fn settle_trade_order( + params: Params<'_>, + ctx: &RelayerContext, +) -> Result { + let topic = std::env::var("RPC_CLIENT_REQUEST").expect("No client topic!"); + let Order { data } = params.parse()?; + + let Ok(bytes) = hex::decode(&data) else { + return Ok(format!("Invalid hex data").into()); + }; + + let Ok(tx) = bincode::deserialize::(&bytes) else { + return Ok(format!("Invalid bincode").into()); + }; + + if let Err(_) = verify_settle_requests(&tx.msg) { + return Ok(format!("Invalid order params").into()); + } + + let order = tx.execute_trader_order.clone(); + let response_id = order.account_id.clone(); + let Ok(response) = serde_json::to_value(&RequestResponse::new( + "Order request submitted successfully".to_string(), + response_id, + )) else { + return Ok(format!("Invalid response").into()); + }; + let Ok(mut conn) = ctx.pool.get() else { + return Ok(format!("Database connection error").into()); + }; + + let Ok(ord) = TraderOrder::get_by_uuid(&mut conn, order.uuid.to_string()) else { + return Ok(format!("Order not found").into()); + }; + + if !ord.order_status.is_closed() { + return Ok(format!("Order closed").into()); + } + + let meta = relayer::Meta::default(); + + let order = + relayer::RpcCommand::ExecuteTraderOrder(order.clone(), meta, tx.msg.encode_as_hex_string()); + let Ok(serialized) = serde_json::to_vec(&order) else { + return Ok(format!("Could not serialize order").into()); + }; + + let record = Record::from_key_value(&topic, "ExecuteTraderOrder", serialized); + if let Err(e) = ctx.kafka.lock().expect("Lock poisoned!").send(&record) { + Ok(format!("Could not send order {:?}", e).into()) + } else { + Ok(response) + } +} + +pub(super) fn settle_lend_order( + params: Params<'_>, + ctx: &RelayerContext, +) -> Result { + let topic = std::env::var("RPC_CLIENT_REQUEST").expect("No client topic!"); + let Order { data } = params.parse()?; + + let Ok(bytes) = hex::decode(&data) else { + return Ok(format!("Invalid hex data").into()); + }; + + let Ok(tx) = bincode::deserialize::(&bytes) else { + return Ok(format!("Invalid bincode").into()); + }; + + if let Err(_) = verify_settle_requests(&tx.msg) { + return Ok(format!("Invalid order params").into()); + } + + let order = tx.execute_lend_order.clone(); + let response_id = order.account_id.clone(); + let Ok(response) = serde_json::to_value(&RequestResponse::new( + "Order request submitted successfully".to_string(), + response_id, + )) else { + return Ok(format!("Invalid response").into()); + }; + let Ok(mut conn) = ctx.pool.get() else { + return Ok(format!("Database connection error").into()); + }; + + let Ok(ord) = LendOrder::get_by_uuid(&mut conn, order.uuid.to_string()) else { + return Ok(format!("Order not found").into()); + }; + + if !ord.order_status.is_closed() { + return Ok(format!("Order closed").into()); + } + + let meta = relayer::Meta::default(); + + let order = + relayer::RpcCommand::ExecuteLendOrder(order.clone(), meta, tx.msg.encode_as_hex_string()); + let Ok(serialized) = serde_json::to_vec(&order) else { + return Ok(format!("Could not serialize order").into()); + }; + + let record = Record::from_key_value(&topic, "ExecuteLendOrder", serialized); + if let Err(e) = ctx.kafka.lock().expect("Lock poisoned!").send(&record) { + Ok(format!("Could not send order {:?}", e).into()) + } else { + Ok(response) + } +} + +pub(super) fn cancel_trader_order( + params: Params<'_>, + ctx: &RelayerContext, +) -> Result { + let topic = std::env::var("RPC_CLIENT_REQUEST").expect("No client topic!"); + let Order { data } = params.parse()?; + + let Ok(bytes) = hex::decode(&data) else { + return Ok(format!("Invalid hex data").into()); + }; + + let Ok(tx) = bincode::deserialize::(&bytes) else { + return Ok(format!("Invalid bincode").into()); + }; + + if let Err(_) = verify_query_order( + tx.msg.convert_cancel_to_query(), + &bincode::serialize(&tx.cancel_trader_order).unwrap(), + ) { + return Ok(format!("Invalid order params").into()); + } + + let order = tx.cancel_trader_order.clone(); + let response_id = order.account_id.clone(); + let Ok(response) = serde_json::to_value(&RequestResponse::new( + "Order request submitted successfully".to_string(), + response_id, + )) else { + return Ok(format!("Invalid response").into()); + }; + let Ok(mut conn) = ctx.pool.get() else { + return Ok(format!("Database connection error").into()); + }; + + let Ok(ord) = TraderOrder::get_by_uuid(&mut conn, order.uuid.to_string()) else { + return Ok(format!("Order not found").into()); + }; + + if !ord.order_status.is_cancelable() { + return Ok(format!("Order not cancelable").into()); + } + + let meta = relayer::Meta::default(); + + let order = + relayer::RpcCommand::CancelTraderOrder(order.clone(), meta, tx.msg.encode_as_hex_string()); + let Ok(serialized) = serde_json::to_vec(&order) else { + return Ok(format!("Could not serialize order").into()); + }; + + let record = Record::from_key_value(&topic, "CancelTraderOrder", serialized); + if let Err(e) = ctx.kafka.lock().expect("Lock poisoned!").send(&record) { + Ok(format!("Could not send order {:?}", e).into()) + } else { + Ok(response) + } +} diff --git a/src/rpc/types.rs b/src/rpc/types.rs index e3f187c..fbad9f1 100644 --- a/src/rpc/types.rs +++ b/src/rpc/types.rs @@ -1,5 +1,5 @@ #![allow(non_camel_case_types)] - +#![allow(warnings)] // • Live Price Data // • Historical Price Data // • Funding Rate diff --git a/src/ws/methods.rs b/src/ws/methods.rs index 537a566..6d30c5c 100644 --- a/src/ws/methods.rs +++ b/src/ws/methods.rs @@ -1,4 +1,4 @@ -#[allow(unreachable_code)] +#![allow(warnings)] use crate::{ database::{BtcUsdPrice, TraderOrder}, error::ApiError, @@ -88,7 +88,7 @@ pub(super) fn candle_update( sleep(Duration::from_millis(250)).await; } } - Ok(()) + // Ok(()) }); Ok(()) @@ -148,7 +148,7 @@ pub(super) fn heartbeat( } sleep(Duration::from_secs(5)).await; } - Ok(()) + // Ok(()) }); Ok(()) From 4b2b13e2ae22f4cd9140d5167af3757fbd7f2f97 Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Tue, 26 Mar 2024 14:33:55 +0530 Subject: [PATCH 18/87] request id added in order submit and settle rpc --- src/rpc/private_methods.rs | 123 +++++++++++++++++++++++-------------- src/rpc/public_methods.rs | 96 ++++++++++++++++++----------- 2 files changed, 136 insertions(+), 83 deletions(-) diff --git a/src/rpc/private_methods.rs b/src/rpc/private_methods.rs index b740174..ec4dcb0 100644 --- a/src/rpc/private_methods.rs +++ b/src/rpc/private_methods.rs @@ -33,12 +33,15 @@ pub(super) fn submit_lend_order( let mut order = tx.create_lend_order.clone(); let meta = relayer::Meta::default(); - - order.account_id = tx.input.input.as_owner_address().cloned().unwrap(); - let deposit = order.deposit / 10000.0; - let balance = order.balance / 10000.0; - order.deposit = deposit; - order.balance = balance; + let public_key = order.account_id.clone(); + order.balance = order.deposit; + let response = RequestResponse::new( + "Order request submitted successfully".to_string(), + public_key, + ); + let Ok(response_value) = serde_json::to_value(&response) else { + return Ok(format!("Invalid response").into()); + }; let Ok(mut conn) = ctx.pool.get() else { return Ok(format!("Database connection error").into()); @@ -48,8 +51,12 @@ pub(super) fn submit_lend_order( return Ok(format!("Failed to update customer id!").into()); } - let order = - relayer::RpcCommand::CreateLendOrder(order.clone(), meta, tx.input.encode_as_hex_string()); + let order = relayer::RpcCommand::CreateLendOrder( + order.clone(), + meta, + tx.input.encode_as_hex_string(), + response.get_id(), + ); let Ok(serialized) = serde_json::to_vec(&order) else { return Ok(format!("Could not serialize order").into()); }; @@ -58,7 +65,7 @@ pub(super) fn submit_lend_order( if let Err(e) = ctx.kafka.lock().expect("Lock poisoned!").send(&record) { Ok(format!("Could not send order {:?}", e).into()) } else { - Ok("OK".into()) + Ok(response_value) } } @@ -84,8 +91,15 @@ pub(super) fn settle_lend_order( return Ok(format!("Invalid order params").into()); } - let mut order = tx.execute_lend_order.clone(); - + let order = tx.execute_lend_order.clone(); + let public_key = order.account_id.clone(); + let response = RequestResponse::new( + "Order request submitted successfully".to_string(), + public_key, + ); + let Ok(response_value) = serde_json::to_value(&response) else { + return Ok(format!("Invalid response").into()); + }; let Ok(mut conn) = ctx.pool.get() else { return Ok(format!("Database connection error").into()); }; @@ -106,14 +120,12 @@ pub(super) fn settle_lend_order( let meta = relayer::Meta::default(); - let Some(account_id) = tx.msg.output.as_output_data().get_owner_address() else { - return Ok(format!("Missing owner address").into()); - }; - - order.account_id = account_id.to_string(); - - let order = - relayer::RpcCommand::ExecuteLendOrder(order.clone(), meta, tx.msg.encode_as_hex_string()); + let order = relayer::RpcCommand::ExecuteLendOrder( + order.clone(), + meta, + tx.msg.encode_as_hex_string(), + response.get_id(), + ); let Ok(serialized) = serde_json::to_vec(&order) else { return Ok(format!("Could not serialize order").into()); }; @@ -122,7 +134,7 @@ pub(super) fn settle_lend_order( if let Err(e) = ctx.kafka.lock().expect("Lock poisoned!").send(&record) { Ok(format!("Could not send order {:?}", e).into()) } else { - Ok("OK".into()) + Ok(response_value) } } @@ -154,13 +166,13 @@ pub(super) fn submit_trade_order( let mut order = tx.create_trader_order.clone(); let meta = relayer::Meta::default(); - let response_id = order.account_id.clone(); + let public_key = order.account_id.clone(); order.available_margin = order.initial_margin; - - let Ok(response) = serde_json::to_value(&RequestResponse::new( + let response = RequestResponse::new( "Order request submitted successfully".to_string(), - response_id, - )) else { + public_key, + ); + let Ok(response_value) = serde_json::to_value(&response) else { return Ok(format!("Invalid response").into()); }; let Ok(mut conn) = ctx.pool.get() else { @@ -171,8 +183,12 @@ pub(super) fn submit_trade_order( return Ok(format!("Failed to update customer id!").into()); } - let order = - relayer::RpcCommand::CreateTraderOrder(order.clone(), meta, hex::encode(transaction_ser)); + let order = relayer::RpcCommand::CreateTraderOrder( + order.clone(), + meta, + hex::encode(transaction_ser), + response.get_id(), + ); let Ok(serialized) = serde_json::to_vec(&order) else { return Ok(format!("Could not serialize order").into()); }; @@ -181,7 +197,7 @@ pub(super) fn submit_trade_order( if let Err(e) = ctx.kafka.lock().expect("Lock poisoned!").send(&record) { Ok(format!("Could not send order {:?}", e).into()) } else { - Ok(response) + Ok(response_value) } } @@ -207,8 +223,15 @@ pub(super) fn settle_trade_order( return Ok(format!("Invalid order params").into()); } - let mut order = tx.execute_trader_order.clone(); - + let order = tx.execute_trader_order.clone(); + let public_key = order.account_id.clone(); + let response = RequestResponse::new( + "Order request submitted successfully".to_string(), + public_key, + ); + let Ok(response_value) = serde_json::to_value(&response) else { + return Ok(format!("Invalid response").into()); + }; let Ok(mut conn) = ctx.pool.get() else { return Ok(format!("Database connection error").into()); }; @@ -223,14 +246,12 @@ pub(super) fn settle_trade_order( let meta = relayer::Meta::default(); - let Some(account_id) = tx.msg.output.as_output_data().get_owner_address() else { - return Ok(format!("Missing owner address").into()); - }; - - order.account_id = account_id.to_string(); - - let order = - relayer::RpcCommand::ExecuteTraderOrder(order.clone(), meta, tx.msg.encode_as_hex_string()); + let order = relayer::RpcCommand::ExecuteTraderOrder( + order.clone(), + meta, + tx.msg.encode_as_hex_string(), + response.get_id(), + ); let Ok(serialized) = serde_json::to_vec(&order) else { return Ok(format!("Could not serialize order").into()); }; @@ -239,7 +260,7 @@ pub(super) fn settle_trade_order( if let Err(e) = ctx.kafka.lock().expect("Lock poisoned!").send(&record) { Ok(format!("Could not send order {:?}", e).into()) } else { - Ok("OK".into()) + Ok(response_value) } } @@ -268,8 +289,15 @@ pub(super) fn cancel_trader_order( return Ok(format!("Invalid order params").into()); } - let mut order = tx.cancel_trader_order.clone(); - + let order = tx.cancel_trader_order.clone(); + let public_key = order.account_id.clone(); + let response = RequestResponse::new( + "Order request submitted successfully".to_string(), + public_key, + ); + let Ok(response_value) = serde_json::to_value(&response) else { + return Ok(format!("Invalid response").into()); + }; let Ok(mut conn) = ctx.pool.get() else { return Ok(format!("Database connection error").into()); }; @@ -284,11 +312,12 @@ pub(super) fn cancel_trader_order( let meta = relayer::Meta::default(); - let account_id = tx.msg.public_key.clone(); - order.account_id = account_id; - - let order = - relayer::RpcCommand::CancelTraderOrder(order.clone(), meta, tx.msg.encode_as_hex_string()); + let order = relayer::RpcCommand::CancelTraderOrder( + order.clone(), + meta, + tx.msg.encode_as_hex_string(), + response.get_id(), + ); let Ok(serialized) = serde_json::to_vec(&order) else { return Ok(format!("Could not serialize order").into()); }; @@ -297,7 +326,7 @@ pub(super) fn cancel_trader_order( if let Err(e) = ctx.kafka.lock().expect("Lock poisoned!").send(&record) { Ok(format!("Could not send order {:?}", e).into()) } else { - Ok("OK".into()) + Ok(response_value) } } diff --git a/src/rpc/public_methods.rs b/src/rpc/public_methods.rs index f29ae4d..52c7a36 100644 --- a/src/rpc/public_methods.rs +++ b/src/rpc/public_methods.rs @@ -240,18 +240,22 @@ pub(super) fn submit_trade_order( let mut order = tx.create_trader_order.clone(); let meta = relayer::Meta::default(); - let response_id = order.account_id.clone(); + let public_key = order.account_id.clone(); order.available_margin = order.initial_margin; - - let Ok(response) = serde_json::to_value(&RequestResponse::new( + let response = RequestResponse::new( "Order request submitted successfully".to_string(), - response_id, - )) else { + public_key, + ); + let Ok(response_value) = serde_json::to_value(&response) else { return Ok(format!("Invalid response").into()); }; - let order = - relayer::RpcCommand::CreateTraderOrder(order.clone(), meta, hex::encode(transaction_ser)); + let order = relayer::RpcCommand::CreateTraderOrder( + order.clone(), + meta, + hex::encode(transaction_ser), + response.get_id(), + ); let Ok(serialized) = serde_json::to_vec(&order) else { return Ok(format!("Could not serialize order").into()); }; @@ -260,7 +264,7 @@ pub(super) fn submit_trade_order( if let Err(e) = ctx.kafka.lock().expect("Lock poisoned!").send(&record) { Ok(format!("Could not send order {:?}", e).into()) } else { - Ok(response) + Ok(response_value) } } @@ -284,17 +288,22 @@ pub(super) fn submit_lend_order( } let mut order = tx.create_lend_order.clone(); - let response_id = order.account_id.clone(); + let public_key = order.account_id.clone(); let meta = relayer::Meta::default(); order.balance = order.deposit; - let Ok(response) = serde_json::to_value(&RequestResponse::new( + let response = RequestResponse::new( "Order request submitted successfully".to_string(), - response_id, - )) else { + public_key, + ); + let Ok(response_value) = serde_json::to_value(&response) else { return Ok(format!("Invalid response").into()); }; - let order = - relayer::RpcCommand::CreateLendOrder(order.clone(), meta, tx.input.encode_as_hex_string()); + let order = relayer::RpcCommand::CreateLendOrder( + order.clone(), + meta, + tx.input.encode_as_hex_string(), + response.get_id(), + ); let Ok(serialized) = serde_json::to_vec(&order) else { return Ok(format!("Could not serialize order").into()); }; @@ -303,7 +312,7 @@ pub(super) fn submit_lend_order( if let Err(e) = ctx.kafka.lock().expect("Lock poisoned!").send(&record) { Ok(format!("Could not send order {:?}", e).into()) } else { - Ok(response) + Ok(response_value) } } @@ -327,11 +336,12 @@ pub(super) fn settle_trade_order( } let order = tx.execute_trader_order.clone(); - let response_id = order.account_id.clone(); - let Ok(response) = serde_json::to_value(&RequestResponse::new( + let public_key = order.account_id.clone(); + let response = RequestResponse::new( "Order request submitted successfully".to_string(), - response_id, - )) else { + public_key, + ); + let Ok(response_value) = serde_json::to_value(&response) else { return Ok(format!("Invalid response").into()); }; let Ok(mut conn) = ctx.pool.get() else { @@ -348,8 +358,12 @@ pub(super) fn settle_trade_order( let meta = relayer::Meta::default(); - let order = - relayer::RpcCommand::ExecuteTraderOrder(order.clone(), meta, tx.msg.encode_as_hex_string()); + let order = relayer::RpcCommand::ExecuteTraderOrder( + order.clone(), + meta, + tx.msg.encode_as_hex_string(), + response.get_id(), + ); let Ok(serialized) = serde_json::to_vec(&order) else { return Ok(format!("Could not serialize order").into()); }; @@ -358,7 +372,7 @@ pub(super) fn settle_trade_order( if let Err(e) = ctx.kafka.lock().expect("Lock poisoned!").send(&record) { Ok(format!("Could not send order {:?}", e).into()) } else { - Ok(response) + Ok(response_value) } } @@ -382,11 +396,12 @@ pub(super) fn settle_lend_order( } let order = tx.execute_lend_order.clone(); - let response_id = order.account_id.clone(); - let Ok(response) = serde_json::to_value(&RequestResponse::new( + let public_key = order.account_id.clone(); + let response = RequestResponse::new( "Order request submitted successfully".to_string(), - response_id, - )) else { + public_key, + ); + let Ok(response_value) = serde_json::to_value(&response) else { return Ok(format!("Invalid response").into()); }; let Ok(mut conn) = ctx.pool.get() else { @@ -403,8 +418,12 @@ pub(super) fn settle_lend_order( let meta = relayer::Meta::default(); - let order = - relayer::RpcCommand::ExecuteLendOrder(order.clone(), meta, tx.msg.encode_as_hex_string()); + let order = relayer::RpcCommand::ExecuteLendOrder( + order.clone(), + meta, + tx.msg.encode_as_hex_string(), + response.get_id(), + ); let Ok(serialized) = serde_json::to_vec(&order) else { return Ok(format!("Could not serialize order").into()); }; @@ -413,7 +432,7 @@ pub(super) fn settle_lend_order( if let Err(e) = ctx.kafka.lock().expect("Lock poisoned!").send(&record) { Ok(format!("Could not send order {:?}", e).into()) } else { - Ok(response) + Ok(response_value) } } @@ -440,11 +459,12 @@ pub(super) fn cancel_trader_order( } let order = tx.cancel_trader_order.clone(); - let response_id = order.account_id.clone(); - let Ok(response) = serde_json::to_value(&RequestResponse::new( + let public_key = order.account_id.clone(); + let response = RequestResponse::new( "Order request submitted successfully".to_string(), - response_id, - )) else { + public_key, + ); + let Ok(response_value) = serde_json::to_value(&response) else { return Ok(format!("Invalid response").into()); }; let Ok(mut conn) = ctx.pool.get() else { @@ -461,8 +481,12 @@ pub(super) fn cancel_trader_order( let meta = relayer::Meta::default(); - let order = - relayer::RpcCommand::CancelTraderOrder(order.clone(), meta, tx.msg.encode_as_hex_string()); + let order = relayer::RpcCommand::CancelTraderOrder( + order.clone(), + meta, + tx.msg.encode_as_hex_string(), + response.get_id(), + ); let Ok(serialized) = serde_json::to_vec(&order) else { return Ok(format!("Could not serialize order").into()); }; @@ -471,6 +495,6 @@ pub(super) fn cancel_trader_order( if let Err(e) = ctx.kafka.lock().expect("Lock poisoned!").send(&record) { Ok(format!("Could not send order {:?}", e).into()) } else { - Ok(response) + Ok(response_value) } } From 3f17cbf443a809f11982e007b298af52fa12e50d Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Tue, 26 Mar 2024 15:02:37 +0530 Subject: [PATCH 19/87] order status sql types update --- src/database/sql_types.rs | 44 ++++++++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/src/database/sql_types.rs b/src/database/sql_types.rs index 3fef715..2393091 100644 --- a/src/database/sql_types.rs +++ b/src/database/sql_types.rs @@ -88,6 +88,14 @@ pub enum OrderStatus { CANCELLED, PENDING, FILLED, + DuplicateOrder, + UtxoError, + Error, + NoResponseFromChain, + BincodeError, + HexCodeError, + SerializationError, + RequestSubmitted, } impl OrderStatus { @@ -95,12 +103,8 @@ impl OrderStatus { use OrderStatus::*; match self { - SETTLED => false, - LENDED => false, - LIQUIDATE => false, - CANCELLED => false, PENDING => true, - FILLED => false, + _ => false, } } @@ -108,12 +112,8 @@ impl OrderStatus { use OrderStatus::*; match self { - SETTLED => false, - LENDED => false, - LIQUIDATE => false, - CANCELLED => false, - PENDING => false, FILLED => true, + _ => false, } } } @@ -248,6 +248,14 @@ impl ToSql for OrderStatus { OrderStatus::CANCELLED => out.write_all(b"CANCELLED")?, OrderStatus::PENDING => out.write_all(b"PENDING")?, OrderStatus::FILLED => out.write_all(b"FILLED")?, + OrderStatus::DuplicateOrder => out.write_all(b"DuplicateOrder")?, + OrderStatus::UtxoError => out.write_all(b"UtxoError")?, + OrderStatus::Error => out.write_all(b"Error")?, + OrderStatus::NoResponseFromChain => out.write_all(b"NoResponseFromChain")?, + OrderStatus::BincodeError => out.write_all(b"BincodeError")?, + OrderStatus::HexCodeError => out.write_all(b"HexCodeError")?, + OrderStatus::SerializationError => out.write_all(b"SerializationError")?, + OrderStatus::RequestSubmitted => out.write_all(b"RequestSubmitted")?, } Ok(IsNull::No) } @@ -282,6 +290,14 @@ impl FromSql for OrderStatus { b"CANCELLED" => Ok(OrderStatus::CANCELLED), b"PENDING" => Ok(OrderStatus::PENDING), b"FILLED" => Ok(OrderStatus::FILLED), + b"DuplicateOrder" => Ok(OrderStatus::DuplicateOrder), + b"UtxoError" => Ok(OrderStatus::UtxoError), + b"Error" => Ok(OrderStatus::Error), + b"NoResponseFromChain" => Ok(OrderStatus::NoResponseFromChain), + b"BincodeError" => Ok(OrderStatus::BincodeError), + b"HexCodeError" => Ok(OrderStatus::HexCodeError), + b"SerializationError" => Ok(OrderStatus::SerializationError), + b"RequestSubmitted" => Ok(OrderStatus::RequestSubmitted), _ => panic!("Invalid enum type in database!"), } } @@ -340,6 +356,14 @@ impl From for OrderStatus { relayer_types::OrderStatus::CANCELLED => OrderStatus::CANCELLED, relayer_types::OrderStatus::PENDING => OrderStatus::PENDING, relayer_types::OrderStatus::FILLED => OrderStatus::FILLED, + relayer_types::OrderStatus::DuplicateOrder => OrderStatus::DuplicateOrder, + relayer_types::OrderStatus::UtxoError => OrderStatus::UtxoError, + relayer_types::OrderStatus::Error => OrderStatus::Error, + relayer_types::OrderStatus::NoResponseFromChain => OrderStatus::NoResponseFromChain, + relayer_types::OrderStatus::BincodeError => OrderStatus::BincodeError, + relayer_types::OrderStatus::HexCodeError => OrderStatus::HexCodeError, + relayer_types::OrderStatus::SerializationError => OrderStatus::SerializationError, + relayer_types::OrderStatus::RequestSubmitted => OrderStatus::RequestSubmitted, } } } From 34022cae672406a6ec3a65f4c3e92d6c88250e2e Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Wed, 27 Mar 2024 10:00:40 +0000 Subject: [PATCH 20/87] temp migration --- .../down.sql | 5 ++++ .../up.sql | 4 ++++ .../2024-03-27-095235_order_status/down.sql | 13 +++++++++++ .../2024-03-27-095235_order_status/up.sql | 13 +++++++++++ src/archiver.rs | 23 +++++++++++++++++++ src/database/models.rs | 14 +++++++++++ src/database/schema.rs | 1 + src/database/sql_types.rs | 8 +++++++ src/rpc/types.rs | 4 ++++ src/ws.rs | 1 + 10 files changed, 86 insertions(+) create mode 100644 migrations/2024-03-27-065655_alter_transaction_hash/down.sql create mode 100644 migrations/2024-03-27-065655_alter_transaction_hash/up.sql create mode 100644 migrations/2024-03-27-095235_order_status/down.sql create mode 100644 migrations/2024-03-27-095235_order_status/up.sql diff --git a/migrations/2024-03-27-065655_alter_transaction_hash/down.sql b/migrations/2024-03-27-065655_alter_transaction_hash/down.sql new file mode 100644 index 0000000..80e0342 --- /dev/null +++ b/migrations/2024-03-27-065655_alter_transaction_hash/down.sql @@ -0,0 +1,5 @@ +-- This file should undo anything in `up.sql` + +ALTER TABLE "transaction_hash" DROP COLUMN "request_id"; + + diff --git a/migrations/2024-03-27-065655_alter_transaction_hash/up.sql b/migrations/2024-03-27-065655_alter_transaction_hash/up.sql new file mode 100644 index 0000000..8537789 --- /dev/null +++ b/migrations/2024-03-27-065655_alter_transaction_hash/up.sql @@ -0,0 +1,4 @@ +-- Your SQL goes here +ALTER TABLE "transaction_hash" ADD COLUMN "request_id" VARCHAR NOT NULL; + + diff --git a/migrations/2024-03-27-095235_order_status/down.sql b/migrations/2024-03-27-095235_order_status/down.sql new file mode 100644 index 0000000..17b04aa --- /dev/null +++ b/migrations/2024-03-27-095235_order_status/down.sql @@ -0,0 +1,13 @@ +-- This file should undo anything in `up.sql` + + +ALTER TYPE order_status DROP VALUE 'DuplicateOrder'; +ALTER TYPE order_status DROP VALUE 'UtxoError'; +ALTER TYPE order_status DROP VALUE 'Error'; +ALTER TYPE order_status DROP VALUE 'NoResponseFromChain'; +ALTER TYPE order_status DROP VALUE 'BincodeError'; +ALTER TYPE order_status DROP VALUE 'HexCodeError'; +ALTER TYPE order_status DROP VALUE 'SerializationError'; +ALTER TYPE order_status DROP VALUE 'RequestSubmitted'; +ALTER TYPE order_status DROP VALUE 'OrderNotFound'; +ALTER TYPE order_status DROP VALUE 'RejectedFromChain'; \ No newline at end of file diff --git a/migrations/2024-03-27-095235_order_status/up.sql b/migrations/2024-03-27-095235_order_status/up.sql new file mode 100644 index 0000000..d9705b4 --- /dev/null +++ b/migrations/2024-03-27-095235_order_status/up.sql @@ -0,0 +1,13 @@ +-- Your SQL goes here + + +ALTER TYPE order_status ADD VALUE 'DuplicateOrder'; +ALTER TYPE order_status ADD VALUE 'UtxoError'; +ALTER TYPE order_status ADD VALUE 'Error'; +ALTER TYPE order_status ADD VALUE 'NoResponseFromChain'; +ALTER TYPE order_status ADD VALUE 'BincodeError'; +ALTER TYPE order_status ADD VALUE 'HexCodeError'; +ALTER TYPE order_status ADD VALUE 'SerializationError'; +ALTER TYPE order_status ADD VALUE 'RequestSubmitted'; +ALTER TYPE order_status ADD VALUE 'OrderNotFound'; +ALTER TYPE order_status ADD VALUE 'RejectedFromChain'; diff --git a/src/archiver.rs b/src/archiver.rs index deda284..df5ba9d 100644 --- a/src/archiver.rs +++ b/src/archiver.rs @@ -368,6 +368,7 @@ impl DatabaseArchiver { order_status, datetime, output, + request_id, ) => { let hash = NewTxHash { order_id: uuid.to_string(), @@ -377,9 +378,31 @@ impl DatabaseArchiver { order_status: order_status.into(), datetime, output, + request_id, }; self.tx_hash(hash)?; } + Event::TxHashUpdate( + _uuid, + _account_id, + _tx_hash, + _order_type, + _order_status, + _datetime, + _output, + ) => { + // let hash = UpdateTxHash { + // order_id: uuid.to_string(), + // account_id, + // tx_hash, + // order_type: order_type.into(), + // order_status: order_status.into(), + // datetime, + // output, + // }; + // self.tx_hash(hash)?; + info!("Update TxHash!!"); + } } Ok(()) diff --git a/src/database/models.rs b/src/database/models.rs index 4dca157..f855ee4 100644 --- a/src/database/models.rs +++ b/src/database/models.rs @@ -33,6 +33,7 @@ pub struct TxHash { pub order_status: OrderStatus, pub datetime: String, pub output: Option, + pub request_id: String, } #[derive(Serialize, Deserialize, Debug, Clone, Insertable, Queryable)] @@ -45,6 +46,7 @@ pub struct NewTxHash { pub order_status: OrderStatus, pub datetime: String, pub output: Option, + pub request_id: String, } impl TxHash { @@ -73,6 +75,18 @@ impl TxHash { transaction_hash.filter(account_id.eq(acct_id)).load(conn) } } + TransactionHashArgs::RequestId { + id: reqt_id, + status, + } => { + if let Some(status) = status { + transaction_hash + .filter(request_id.eq(reqt_id).and(order_status.eq(status))) + .load(conn) + } else { + transaction_hash.filter(request_id.eq(reqt_id)).load(conn) + } + } } } diff --git a/src/database/schema.rs b/src/database/schema.rs index 3747a6f..7316f57 100644 --- a/src/database/schema.rs +++ b/src/database/schema.rs @@ -232,6 +232,7 @@ diesel::table! { order_status -> OrderStatus, datetime -> Varchar, output -> Nullable, + request_id -> Varchar, } } diff --git a/src/database/sql_types.rs b/src/database/sql_types.rs index 2393091..398ae42 100644 --- a/src/database/sql_types.rs +++ b/src/database/sql_types.rs @@ -96,6 +96,8 @@ pub enum OrderStatus { HexCodeError, SerializationError, RequestSubmitted, + OrderNotFound, + RejectedFromChain, } impl OrderStatus { @@ -256,6 +258,8 @@ impl ToSql for OrderStatus { OrderStatus::HexCodeError => out.write_all(b"HexCodeError")?, OrderStatus::SerializationError => out.write_all(b"SerializationError")?, OrderStatus::RequestSubmitted => out.write_all(b"RequestSubmitted")?, + OrderStatus::OrderNotFound => out.write_all(b"OrderNotFound")?, + OrderStatus::RejectedFromChain => out.write_all(b"RejectedFromChain")?, } Ok(IsNull::No) } @@ -298,6 +302,8 @@ impl FromSql for OrderStatus { b"HexCodeError" => Ok(OrderStatus::HexCodeError), b"SerializationError" => Ok(OrderStatus::SerializationError), b"RequestSubmitted" => Ok(OrderStatus::RequestSubmitted), + b"OrderNotFound" => Ok(OrderStatus::OrderNotFound), + b"RejectedFromChain" => Ok(OrderStatus::RejectedFromChain), _ => panic!("Invalid enum type in database!"), } } @@ -364,6 +370,8 @@ impl From for OrderStatus { relayer_types::OrderStatus::HexCodeError => OrderStatus::HexCodeError, relayer_types::OrderStatus::SerializationError => OrderStatus::SerializationError, relayer_types::OrderStatus::RequestSubmitted => OrderStatus::RequestSubmitted, + relayer_types::OrderStatus::OrderNotFound => OrderStatus::OrderNotFound, + relayer_types::OrderStatus::RejectedFromChain => OrderStatus::RejectedFromChain, } } } diff --git a/src/rpc/types.rs b/src/rpc/types.rs index fbad9f1..ad9dc2f 100644 --- a/src/rpc/types.rs +++ b/src/rpc/types.rs @@ -60,6 +60,10 @@ pub enum TransactionHashArgs { id: String, status: Option, }, + RequestId { + id: String, + status: Option, + }, } #[derive(Debug, Serialize, Deserialize)] diff --git a/src/ws.rs b/src/ws.rs index fc75908..3487c03 100644 --- a/src/ws.rs +++ b/src/ws.rs @@ -105,6 +105,7 @@ impl WsContext { ) => {} Event::Stop(_stop) => {} Event::TxHash(..) => {} + Event::TxHashUpdate(..) => {} } } if let Err(e) = notify.send(completion) { From c6b009f00201e964536808511c7302619fbecec3 Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Wed, 27 Mar 2024 14:22:54 +0000 Subject: [PATCH 21/87] Txhash update added --- .env | 1 - .../up.sql | 2 +- src/archiver.rs | 38 +++++++++---------- src/database/models.rs | 4 +- src/database/schema.rs | 2 +- 5 files changed, 23 insertions(+), 24 deletions(-) diff --git a/.env b/.env index 36637ef..70640c3 100644 --- a/.env +++ b/.env @@ -14,7 +14,6 @@ POSTGRESQL_URL=postgresql://postgres:my_password@localhost:5432/my_database QUESTDB_URL=postgresql://quest:my_password@localhost:8812/qdb QUESTDB_INFLUX_URL=127.0.0.1:9009 BROKER=localhost:9092 -DATABASE_URL=postgres://relayer:relayer@localhost:5433/relayer # bitmex socket connection url BITMEX_BTC_SOCKET_ORDERBOOK_URL=wss://ws.bitmex.com/realtime?subscribe=orderBookL2_25:XBTUSD diff --git a/migrations/2024-03-27-065655_alter_transaction_hash/up.sql b/migrations/2024-03-27-065655_alter_transaction_hash/up.sql index 8537789..c2fbf68 100644 --- a/migrations/2024-03-27-065655_alter_transaction_hash/up.sql +++ b/migrations/2024-03-27-065655_alter_transaction_hash/up.sql @@ -1,4 +1,4 @@ -- Your SQL goes here -ALTER TABLE "transaction_hash" ADD COLUMN "request_id" VARCHAR NOT NULL; +ALTER TABLE "transaction_hash" ADD COLUMN "request_id" VARCHAR; diff --git a/src/archiver.rs b/src/archiver.rs index df5ba9d..7d39f65 100644 --- a/src/archiver.rs +++ b/src/archiver.rs @@ -378,30 +378,30 @@ impl DatabaseArchiver { order_status: order_status.into(), datetime, output, - request_id, + request_id: Some(request_id), }; self.tx_hash(hash)?; } Event::TxHashUpdate( - _uuid, - _account_id, - _tx_hash, - _order_type, - _order_status, - _datetime, - _output, + uuid, + account_id, + tx_hash, + order_type, + order_status, + datetime, + output, ) => { - // let hash = UpdateTxHash { - // order_id: uuid.to_string(), - // account_id, - // tx_hash, - // order_type: order_type.into(), - // order_status: order_status.into(), - // datetime, - // output, - // }; - // self.tx_hash(hash)?; - info!("Update TxHash!!"); + let hash = NewTxHash { + order_id: uuid.to_string(), + account_id, + tx_hash, + order_type: order_type.into(), + order_status: order_status.into(), + datetime, + output, + request_id: None, + }; + self.tx_hash(hash)?; } } diff --git a/src/database/models.rs b/src/database/models.rs index f855ee4..0b018e7 100644 --- a/src/database/models.rs +++ b/src/database/models.rs @@ -33,7 +33,7 @@ pub struct TxHash { pub order_status: OrderStatus, pub datetime: String, pub output: Option, - pub request_id: String, + pub request_id: Option, } #[derive(Serialize, Deserialize, Debug, Clone, Insertable, Queryable)] @@ -46,7 +46,7 @@ pub struct NewTxHash { pub order_status: OrderStatus, pub datetime: String, pub output: Option, - pub request_id: String, + pub request_id: Option, } impl TxHash { diff --git a/src/database/schema.rs b/src/database/schema.rs index 7316f57..27b9bd7 100644 --- a/src/database/schema.rs +++ b/src/database/schema.rs @@ -232,7 +232,7 @@ diesel::table! { order_status -> OrderStatus, datetime -> Varchar, output -> Nullable, - request_id -> Varchar, + request_id -> Nullable, } } From 1a5ac6bfba666ae6772ded23fb37b199b632352a Mon Sep 17 00:00:00 2001 From: TJ Sharp Date: Thu, 28 Mar 2024 18:27:45 -0400 Subject: [PATCH 22/87] Add aggregation tables --- Cargo.toml | 4 +- Dockerfile | 3 +- .../down.sql | 14 ++ .../up.sql | 176 ++++++++++++++++++ src/archiver.rs | 1 + src/database/models.rs | 132 ++++++------- src/database/schema.rs | 45 +++++ 7 files changed, 306 insertions(+), 69 deletions(-) create mode 100644 migrations/2024-03-26-213323_add_aggregation_tables/down.sql create mode 100644 migrations/2024-03-26-213323_add_aggregation_tables/up.sql diff --git a/Cargo.toml b/Cargo.toml index 177c66c..578d767 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,8 +72,8 @@ verify-keplr-sign = "0.1.0" twilight-relayer-rust = { git = "ssh://git@github.com/twilight-project/twilight-relayer.git", branch = "client-side-trader-order-create" } [dependencies.zkos-relayer-wallet] -git = "ssh://github.com/twilight-project/zkos-relayer-wallet.git" +git = "ssh://git@github.com/twilight-project/zkos-relayer-wallet.git" branch = "clienttradeorder" [dependencies.zkos-client-wallet] -git = "ssh://github.com/twilight-project/zkos-client-wallet.git" +git = "ssh://git@github.com/twilight-project/zkos-client-wallet.git" branch = "clienttradeorder" diff --git a/Dockerfile b/Dockerfile index bcad935..6f4a194 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.72.0-slim-buster as builder +FROM rust:1.77.0-slim-buster as builder ARG SSH_KEY RUN apt-get update && apt-get install -y openssh-client git libssl-dev build-essential libpq-dev pkg-config @@ -13,6 +13,7 @@ COPY . ./twilight-relayerAPI RUN --mount=type=cache,target=/usr/local/cargo/registry \ --mount=type=cache,target=${PWD}/target \ cd ./twilight-relayerAPI && \ + git config --global url."ssh://git@github.com/".insteadOf ssh://github.com/ && \ cargo --config "net.git-fetch-with-cli = true" b --release --bins FROM debian:10-slim diff --git a/migrations/2024-03-26-213323_add_aggregation_tables/down.sql b/migrations/2024-03-26-213323_add_aggregation_tables/down.sql new file mode 100644 index 0000000..1009034 --- /dev/null +++ b/migrations/2024-03-26-213323_add_aggregation_tables/down.sql @@ -0,0 +1,14 @@ +-- This file should undo anything in `up.sql` +DROP INDEX trader_order_time; +DROP INDEX price_time; + +DROP TABLE candles_1min; +DROP TABLE candles_1hour; +DROP TABLE candles_1day; + +DROP FUNCTION get_ohlc_interval; +DROP FUNCTION get_volume_interval; +DROP FUNCTION get_candles_interval; +DROP FUNCTION update_candles_1min; +DROP FUNCTION update_candles_1hour; +DROP FUNCTION update_candles_1day; diff --git a/migrations/2024-03-26-213323_add_aggregation_tables/up.sql b/migrations/2024-03-26-213323_add_aggregation_tables/up.sql new file mode 100644 index 0000000..3b6d64c --- /dev/null +++ b/migrations/2024-03-26-213323_add_aggregation_tables/up.sql @@ -0,0 +1,176 @@ +-- Your SQL goes here + +CREATE INDEX trader_order_time ON trader_order(timestamp); +CREATE INDEX price_time ON btc_usd_price(timestamp); + +CREATE TABLE candles_1min ( + start_time timestamptz primary key, + end_time timestamptz not null, + low numeric not null, + high numeric not null, + open numeric not null, + close numeric not null, + trades integer not null, + btc_volume numeric not null, + usd_volume numeric not null +); + +CREATE TABLE candles_1hour ( + start_time timestamptz primary key, + end_time timestamptz not null, + low numeric not null, + high numeric not null, + open numeric not null, + close numeric not null, + trades integer not null, + btc_volume numeric not null, + usd_volume numeric not null +); + +CREATE TABLE candles_1day ( + start_time timestamptz primary key, + end_time timestamptz not null, + low numeric not null, + high numeric not null, + open numeric not null, + close numeric not null, + trades integer not null, + btc_volume numeric not null, + usd_volume numeric not null +); + +CREATE FUNCTION get_ohlc_interval(intvl interval, trunc_by text, since timestamptz) +RETURNS TABLE(start_time timestamptz, end_time timestamptz, high numeric, low numeric, open numeric, close numeric) +AS $$ SELECT DISTINCT + bucket as start_time, + bucket + intvl - interval '1 microsecond' as end_time, + max(price) over w as high, + min(price) over w as low, + first_value(price) over w as close, + last_value(price) over w as open +from ( + SELECT date_trunc(trunc_by, timestamp) as bucket,* + FROM btc_usd_price + WHERE timestamp BETWEEN since AND now() +) as t +WINDOW w as (partition by bucket order by timestamp asc ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) +$$ +LANGUAGE SQL; + + +CREATE FUNCTION get_volume_interval(intvl interval, trunc_by text, since timestamptz) +RETURNS TABLE(start_time timestamptz, end_time timestamptz, usd_volume numeric, btc_volume numeric, trades integer) +AS $$ SELECT DISTINCT + bucket as start_time, + bucket + intvl - interval '1 microsecond' as end_time, + sum(entryprice) over w as usd_volume, + sum(positionsize) over w as btc_volume, + count(*) over w as trades +FROM ( + select date_trunc(trunc_by, timestamp) as bucket,* + from trader_order + WHERE timestamp BETWEEN since AND now() +) as t +WINDOW w as (partition by bucket ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) +$$ +LANGUAGE SQL; + + +CREATE FUNCTION get_candles_interval(intvl interval, trunc_by text, since timestamptz) +RETURNS TABLE( + start_time timestamptz, + end_time timestamptz, + usd_volume numeric, + btc_volume numeric, + trades integer, + open numeric, + high numeric, + low numeric, + close numeric +) +AS $$ +WITH t1 AS ( + SELECT * FROM get_ohlc_interval(intvl, trunc_by, since) +), t2 AS ( + SELECT * FROM get_volume_interval(intvl, trunc_by, since) +) SELECT + coalesce(t1.start_time, t2.start_time) as start_time, + coalesce(t1.end_time, t2.end_time) as start_time, + coalesce(t2.usd_volume, 0) as usd_volume, + coalesce(t2.btc_volume, 0) as btc_volume, + coalesce(t2.trades, 0) as trades, + coalesce(t1.open, 0) as open, + coalesce(t1.high, 0) as high, + coalesce(t1.low, 0) as low, + coalesce(t1.close, 0) as close +FROM + t1 FULL OUTER JOIN t2 + ON t1.start_time = t2.start_time +$$ +LANGUAGE SQL; + +CREATE FUNCTION update_candles_1min(since timestamptz) +RETURNS void +AS $$ INSERT INTO candles_1min ( + start_time, end_time, usd_volume, btc_volume, trades, open, high, low, close +) +SELECT * FROM get_candles_interval('1 minute', 'minute', since) + ON CONFLICT(start_time) + DO UPDATE SET + start_time = excluded.start_time, + end_time = excluded.end_time, + usd_volume = excluded.usd_volume, + btc_volume = excluded.btc_volume, + trades = excluded.trades, + open = excluded.open, + high = excluded.high, + low = excluded.low, + close = excluded.close + ; + +$$ +LANGUAGE SQL; + +CREATE FUNCTION update_candles_1hour(since timestamptz) +RETURNS void +AS $$ INSERT INTO candles_1hour ( + start_time, end_time, usd_volume, btc_volume, trades, open, high, low, close +) +SELECT * FROM get_candles_interval('1 hour', 'hour', since) + ON CONFLICT(start_time) + DO UPDATE SET + start_time = excluded.start_time, + end_time = excluded.end_time, + usd_volume = excluded.usd_volume, + btc_volume = excluded.btc_volume, + trades = excluded.trades, + open = excluded.open, + high = excluded.high, + low = excluded.low, + close = excluded.close + ; + +$$ +LANGUAGE SQL; + +CREATE FUNCTION update_candles_1day(since timestamptz) +RETURNS void +AS $$ INSERT INTO candles_1day ( + start_time, end_time, usd_volume, btc_volume, trades, open, high, low, close +) +SELECT * FROM get_candles_interval('1 day', 'day', since) + ON CONFLICT(start_time) + DO UPDATE SET + start_time = excluded.start_time, + end_time = excluded.end_time, + usd_volume = excluded.usd_volume, + btc_volume = excluded.btc_volume, + trades = excluded.trades, + open = excluded.open, + high = excluded.high, + low = excluded.low, + close = excluded.close + ; + +$$ +LANGUAGE SQL; diff --git a/src/archiver.rs b/src/archiver.rs index deda284..1eec739 100644 --- a/src/archiver.rs +++ b/src/archiver.rs @@ -346,6 +346,7 @@ impl DatabaseArchiver { .expect("Bad datetime format") .into(); CurrentPriceUpdate::insert(&mut *self.get_conn()?, current_price, ts)?; + BtcUsdPrice::update_candles(&mut *self.get_conn()?)?; } Event::PoolUpdate(lend_pool_command, lend_pool, ..) => { self.lend_pool_updates(lend_pool)?; diff --git a/src/database/models.rs b/src/database/models.rs index e84941a..fdbafbd 100644 --- a/src/database/models.rs +++ b/src/database/models.rs @@ -819,6 +819,14 @@ pub struct CandleData { } impl BtcUsdPrice { + pub fn update_candles(conn: &mut PgConnection) -> QueryResult<()> { + diesel::sql_query("SELECT * from update_candles_1min(now() - interval '5 minute')").execute(conn)?; + diesel::sql_query("SELECT * from update_candles_1hour(now() - interval '5 minute')").execute(conn)?; + diesel::sql_query("SELECT * from update_candles_1day(now() - interval '5 minute')").execute(conn)?; + + Ok(()) + } + pub fn get(conn: &mut PgConnection) -> QueryResult { use crate::database::schema::btc_usd_price::dsl::*; @@ -855,88 +863,80 @@ impl BtcUsdPrice { offset: Option, ) -> QueryResult> { let start: DateTime; + let table: String; // temp for 24 hour candle change // need to create new api for 24hour candle change data match interval { Interval::ONE_DAY_CHANGE => { start = since + chrono::Duration::seconds(5); + table = "candles_1hour".into() + } + Interval::ONE_MINUTE + | Interval::FIVE_MINUTE + | Interval::FIFTEEN_MINUTE + | Interval::THIRTY_MINUTE => { + start = since.duration_trunc(interval.duration()).unwrap(); + table = "candles_1min".into() + } + Interval::ONE_HOUR + | Interval::FOUR_HOUR + | Interval::EIGHT_HOUR + | Interval::TWELVE_HOUR => { + start = since.duration_trunc(interval.duration()).unwrap(); + table = "candles_1hour".into() } - _ => { + Interval::ONE_DAY => { start = since.duration_trunc(interval.duration()).unwrap(); + table = "candles_1day".into() } } let interval = interval.interval_sql(); - let trader_subquery = format!( - r#" - SELECT - window_start, - sum(entryprice) as usd_volume, - sum(positionsize) as btc_volume, - count(*) as trades - FROM ( - SELECT - t.timestamp as window_start, - coalesce(c.entryprice, 0) as entryprice, - coalesce(c.positionsize, 0) as positionsize - FROM generate_series('{}', now(), {}) t(timestamp) - LEFT JOIN trader_order c - ON c.timestamp >= t.timestamp AND c.timestamp < t.timestamp + {} - ) as sq - GROUP BY window_start - "#, - start, interval, interval - ); - - let ohlc_subquery = format!( - r#" - SELECT - window_ts, - window_ts as start, - window_ts + {} as end, - min(open) as open, - min(close) as close, - max(price) as high, - min(price) as low - FROM ( - SELECT - t.timestamp as window_ts, - c.*, - first_value(price) OVER (PARTITION BY t.timestamp ORDER BY c.timestamp asc) AS open, - first_value(price) OVER (PARTITION BY t.timestamp ORDER BY c.timestamp desc) AS close - FROM generate_series('{}', now(), {}) t(timestamp) - LEFT JOIN btc_usd_price c - ON c.timestamp >= t.timestamp AND c.timestamp < t.timestamp + {} - ) as w - WHERE open IS NOT NULL - GROUP BY window_ts - "#, - interval, start, interval, interval + let subquery = format!(r#" + with t as ( + select * from + generate_series('{}', now(), {}) timestamp + ), c as ( + select * from {} + where start_time between '{}' and now() + ) + select + t.timestamp as bucket, + c.start_time as start, + c.open, + c.close, + c.high, + c.low, + c.btc_volume, + c.trades, + c.usd_volume + from t + inner join c + on c.start_time >= t.timestamp AND c.start_time < t.timestamp + interval {} order by c.start_time + "#, + start, interval, table, start, interval ); - let query = format!( - r#" - WITH volumes AS ( - {} - ), ohlc AS ( - {} - ) - SELECT + let query = format!(r#" + select distinct now() as updated_at, - ohlc.start, - ohlc.end, - ohlc.open, - ohlc.close, - ohlc.high, - ohlc.low, {} as resolution, - volumes.btc_volume as btc_volume, - volumes.trades as trades, - volumes.usd_volume as usd_volume - FROM volumes - JOIN ohlc ON volumes.window_start = ohlc.window_ts + bucket as start, + bucket + interval {} as end, + max(high) over w as high, + min(low) over w as low, + last_value(close) over w as close, + first_value(open) over w as open, + sum(btc_volume) over w as btc_volume, + sum(usd_volume) over w as usd_volume, + sum(trades) over w as trades + from ( + {} + ) as s + WINDOW w as (partition by bucket order by start asc ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) + order by start asc "#, - trader_subquery, ohlc_subquery, interval + interval, interval, subquery, ); let query = if let Some(limit) = limit { diff --git a/src/database/schema.rs b/src/database/schema.rs index 3747a6f..fb7d19a 100644 --- a/src/database/schema.rs +++ b/src/database/schema.rs @@ -42,6 +42,48 @@ diesel::table! { } } +diesel::table! { + candles_1day (start_time) { + start_time -> Timestamptz, + end_time -> Timestamptz, + low -> Numeric, + high -> Numeric, + open -> Numeric, + close -> Numeric, + trades -> Int4, + btc_volume -> Numeric, + usd_volume -> Numeric, + } +} + +diesel::table! { + candles_1hour (start_time) { + start_time -> Timestamptz, + end_time -> Timestamptz, + low -> Numeric, + high -> Numeric, + open -> Numeric, + close -> Numeric, + trades -> Int4, + btc_volume -> Numeric, + usd_volume -> Numeric, + } +} + +diesel::table! { + candles_1min (start_time) { + start_time -> Timestamptz, + end_time -> Timestamptz, + low -> Numeric, + high -> Numeric, + open -> Numeric, + close -> Numeric, + trades -> Int4, + btc_volume -> Numeric, + usd_volume -> Numeric, + } +} + diesel::table! { current_nonce (id) { id -> Int8, @@ -242,6 +284,9 @@ diesel::joinable!(customer_order_linking -> customer_account (customer_account_i diesel::allow_tables_to_appear_in_same_query!( address_customer_id, btc_usd_price, + candles_1day, + candles_1hour, + candles_1min, current_nonce, customer_account, customer_apikey_linking, From 4265680395b5d898e024a90c6958f07ac3f74dfc Mon Sep 17 00:00:00 2001 From: TJ Sharp Date: Sun, 7 Apr 2024 12:20:37 -0400 Subject: [PATCH 23/87] date_trunc for candle update --- src/bin/test_feed.rs | 41 +++++++++++++++++++++++++++++++++++++++++ src/database/models.rs | 6 +++--- 2 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 src/bin/test_feed.rs diff --git a/src/bin/test_feed.rs b/src/bin/test_feed.rs new file mode 100644 index 0000000..f366af6 --- /dev/null +++ b/src/bin/test_feed.rs @@ -0,0 +1,41 @@ +use chrono::prelude::*; +use kafka::producer::{Producer, Record, RequiredAcks}; +use rand::Rng; +use relayerarchiverlib::{rpc, ws}; +use std::{net::SocketAddr, time::Duration}; +use tokio::time::sleep; +use twilight_relayer_rust::db::Event; + + +#[tokio::main] +async fn main() { + dotenv::dotenv().expect("dotenv file not found!"); + let mut rng = rand::thread_rng(); + let mut current_price: f64 = rng.gen_range(60000.0..80000.0); + + let broker_host = std::env::var("BROKER").expect("missing environment variable BROKER"); + let topic = std::env::var("TRADERORDER_EVENT_LOG").expect("No topic!!!!"); + + let broker = vec![broker_host.to_owned()]; + let mut kafka = Producer::from_hosts(broker) + .with_ack_timeout(Duration::from_secs(1)) + .with_required_acks(RequiredAcks::One) + .create() + .unwrap(); + + + loop { + current_price += rng.gen_range(-50.0..50.0); + let system_time = Utc::now().to_rfc3339(); + + println!("New price: {}, {}", current_price, system_time); + let event = Event::CurrentPriceUpdate(current_price, system_time); + let serialized = serde_json::to_vec(&event).expect("Phuque"); + let record = Record::from_key_value(&topic, "CurrentPriceUpdate", serialized); + + kafka.send(&record).expect("NOOOOO!!"); + + sleep(Duration::from_secs(1)).await; + } + +} diff --git a/src/database/models.rs b/src/database/models.rs index fdbafbd..d16bbfd 100644 --- a/src/database/models.rs +++ b/src/database/models.rs @@ -820,9 +820,9 @@ pub struct CandleData { impl BtcUsdPrice { pub fn update_candles(conn: &mut PgConnection) -> QueryResult<()> { - diesel::sql_query("SELECT * from update_candles_1min(now() - interval '5 minute')").execute(conn)?; - diesel::sql_query("SELECT * from update_candles_1hour(now() - interval '5 minute')").execute(conn)?; - diesel::sql_query("SELECT * from update_candles_1day(now() - interval '5 minute')").execute(conn)?; + diesel::sql_query("SELECT * from update_candles_1min(date_trunc('minute', now() - interval '5 minute'))").execute(conn)?; + diesel::sql_query("SELECT * from update_candles_1hour(date_trunc('hour', now() - interval '5 minute'))").execute(conn)?; + diesel::sql_query("SELECT * from update_candles_1day(date_trunc('day', now() - interval '5 minute'))").execute(conn)?; Ok(()) } From c8c794ad10bfd3164ab9e6151ced18839e260a03 Mon Sep 17 00:00:00 2001 From: TJ Sharp Date: Wed, 1 May 2024 20:36:47 -0400 Subject: [PATCH 24/87] Increase connection limit --- src/ws.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/ws.rs b/src/ws.rs index fc75908..6b941df 100644 --- a/src/ws.rs +++ b/src/ws.rs @@ -140,7 +140,10 @@ impl WsContext { pub fn init_methods(database_url: &str) -> RpcModule { let manager = ConnectionManager::::new(database_url); - let pool = r2d2::Pool::new(manager).expect("Could not instantiate connection pool"); + let pool = r2d2::Pool::builder() + .max_size(50) + .build(manager) + .expect("Could not instantiate connection pool"); let mut module = RpcModule::new(WsContext::with_pool(pool)); From bb7d2298c83795dc167a6a80db2c6058c144881d Mon Sep 17 00:00:00 2001 From: TJ Sharp Date: Sat, 4 May 2024 15:00:24 -0400 Subject: [PATCH 25/87] Catchup for candle updates --- .../up.sql | 12 +++++----- src/archiver.rs | 10 +++++--- src/database/models.rs | 23 ++++++++++++++----- src/kafka.rs | 14 +++++++++-- src/ws.rs | 2 +- 5 files changed, 43 insertions(+), 18 deletions(-) diff --git a/migrations/2024-03-26-213323_add_aggregation_tables/up.sql b/migrations/2024-03-26-213323_add_aggregation_tables/up.sql index 3b6d64c..fb6ade4 100644 --- a/migrations/2024-03-26-213323_add_aggregation_tables/up.sql +++ b/migrations/2024-03-26-213323_add_aggregation_tables/up.sql @@ -109,12 +109,12 @@ FROM $$ LANGUAGE SQL; -CREATE FUNCTION update_candles_1min(since timestamptz) +CREATE FUNCTION update_candles_1min() RETURNS void AS $$ INSERT INTO candles_1min ( start_time, end_time, usd_volume, btc_volume, trades, open, high, low, close ) -SELECT * FROM get_candles_interval('1 minute', 'minute', since) +SELECT * FROM get_candles_interval('1 minute', 'minute', (SELECT coalesce(max(start_time) - interval '10 minute', now() - interval '1 week') FROM candles_1min)) ON CONFLICT(start_time) DO UPDATE SET start_time = excluded.start_time, @@ -131,12 +131,12 @@ SELECT * FROM get_candles_interval('1 minute', 'minute', since) $$ LANGUAGE SQL; -CREATE FUNCTION update_candles_1hour(since timestamptz) +CREATE FUNCTION update_candles_1hour() RETURNS void AS $$ INSERT INTO candles_1hour ( start_time, end_time, usd_volume, btc_volume, trades, open, high, low, close ) -SELECT * FROM get_candles_interval('1 hour', 'hour', since) +SELECT * FROM get_candles_interval('1 hour', 'hour', (SELECT coalesce(max(start_time) - interval '10 minute', now() - interval '1 week') FROM candles_1hour)) ON CONFLICT(start_time) DO UPDATE SET start_time = excluded.start_time, @@ -153,12 +153,12 @@ SELECT * FROM get_candles_interval('1 hour', 'hour', since) $$ LANGUAGE SQL; -CREATE FUNCTION update_candles_1day(since timestamptz) +CREATE FUNCTION update_candles_1day() RETURNS void AS $$ INSERT INTO candles_1day ( start_time, end_time, usd_volume, btc_volume, trades, open, high, low, close ) -SELECT * FROM get_candles_interval('1 day', 'day', since) +SELECT * FROM get_candles_interval('1 day', 'day', (SELECT coalesce(max(start_time) - interval '10 minute', now() - interval '1 week') FROM candles_1day)) ON CONFLICT(start_time) DO UPDATE SET start_time = excluded.start_time, diff --git a/src/archiver.rs b/src/archiver.rs index 1eec739..5f7a052 100644 --- a/src/archiver.rs +++ b/src/archiver.rs @@ -346,7 +346,6 @@ impl DatabaseArchiver { .expect("Bad datetime format") .into(); CurrentPriceUpdate::insert(&mut *self.get_conn()?, current_price, ts)?; - BtcUsdPrice::update_candles(&mut *self.get_conn()?)?; } Event::PoolUpdate(lend_pool_command, lend_pool, ..) => { self.lend_pool_updates(lend_pool)?; @@ -387,15 +386,20 @@ impl DatabaseArchiver { } /// Worker task that loops indefinitely, batching commits to postgres backend. - pub fn run(mut self, rx: Receiver<(Completion, Vec)>) -> Result<(), ApiError> { + pub fn run(mut self, rx: Receiver<(Completion, Vec, bool)>) -> Result<(), ApiError> { let mut deadline = Instant::now() + Duration::from_millis(BATCH_INTERVAL); loop { match rx.recv_deadline(deadline) { - Ok((completion, msgs)) => { + Ok((completion, msgs, catchup)) => { for msg in msgs { self.process_msg(msg)?; } + + if !catchup { + BtcUsdPrice::update_candles(&mut *self.get_conn()?)?; + } + self.commit_orders()?; self.completions .send(completion) diff --git a/src/database/models.rs b/src/database/models.rs index d16bbfd..49b151b 100644 --- a/src/database/models.rs +++ b/src/database/models.rs @@ -820,9 +820,18 @@ pub struct CandleData { impl BtcUsdPrice { pub fn update_candles(conn: &mut PgConnection) -> QueryResult<()> { - diesel::sql_query("SELECT * from update_candles_1min(date_trunc('minute', now() - interval '5 minute'))").execute(conn)?; - diesel::sql_query("SELECT * from update_candles_1hour(date_trunc('hour', now() - interval '5 minute'))").execute(conn)?; - diesel::sql_query("SELECT * from update_candles_1day(date_trunc('day', now() - interval '5 minute'))").execute(conn)?; + diesel::sql_query( + "SELECT * from update_candles_1min()", + ) + .execute(conn)?; + diesel::sql_query( + "SELECT * from update_candles_1hour()", + ) + .execute(conn)?; + diesel::sql_query( + "SELECT * from update_candles_1day()", + ) + .execute(conn)?; Ok(()) } @@ -892,7 +901,8 @@ impl BtcUsdPrice { } let interval = interval.interval_sql(); - let subquery = format!(r#" + let subquery = format!( + r#" with t as ( select * from generate_series('{}', now(), {}) timestamp @@ -917,7 +927,8 @@ impl BtcUsdPrice { start, interval, table, start, interval ); - let query = format!(r#" + let query = format!( + r#" select distinct now() as updated_at, {} as resolution, @@ -936,7 +947,7 @@ impl BtcUsdPrice { WINDOW w as (partition by bucket order by start asc ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) order by start asc "#, - interval, interval, subquery, + interval, interval, subquery, ); let query = if let Some(limit) = limit { diff --git a/src/kafka.rs b/src/kafka.rs index 0ea7305..ca22342 100644 --- a/src/kafka.rs +++ b/src/kafka.rs @@ -4,12 +4,14 @@ use log::{error, info}; use std::thread::{self, JoinHandle}; use twilight_relayer_rust::db::Event; +// > 500 offset behind, we'll batch update. +const CATCHUP_INTERVAL: i64 = 500; pub type Completion = (i32, i64); pub fn start_consumer( group: String, topic: String, - tx: Sender<(Completion, Vec)>, + tx: Sender<(Completion, Vec, bool)>, ) -> (Sender, JoinHandle<()>) { let (tx_consumed, rx_consumed) = unbounded::(); @@ -27,10 +29,17 @@ pub fn start_consumer( .create() .unwrap(); + con.client_mut().load_metadata_all().unwrap(); + let mut connection_status = true; while connection_status { let sender_clone = tx.clone(); let mss = con.poll().unwrap(); + let latest = con + .client_mut() + .fetch_topic_offsets(&topic, kafka::client::FetchOffset::Latest) + .unwrap(); + let latest = latest[0].offset; if !mss.is_empty() { for ms in mss.iter() { let mut max_offset = 0i64; @@ -53,8 +62,9 @@ pub fn start_consumer( .collect(); let token = (ms.partition(), max_offset); + let catchup = latest - max_offset > CATCHUP_INTERVAL; - match sender_clone.send((token, events)) { + match sender_clone.send((token, events, catchup)) { Ok(_) => {} Err(_arg) => { connection_status = false; diff --git a/src/ws.rs b/src/ws.rs index 6b941df..4631a6e 100644 --- a/src/ws.rs +++ b/src/ws.rs @@ -61,7 +61,7 @@ impl WsContext { let mut deadline = Instant::now() + Duration::from_millis(WS_UPDATE_INTERVAL); loop { match rx.recv_deadline(deadline) { - Ok((completion, msgs)) => { + Ok((completion, msgs, _)) => { for msg in msgs { match msg { Event::TraderOrder(to, ..) From 997026ec28128d81a8b090be310a53d0c417f1e4 Mon Sep 17 00:00:00 2001 From: thomas SHARP Date: Mon, 13 May 2024 07:38:49 -0400 Subject: [PATCH 26/87] Spawn query task for candles --- src/rpc/types.rs | 2 +- src/ws.rs | 10 +++++- src/ws/methods.rs | 79 +++++++++++++++++++++++++++++++++-------------- 3 files changed, 65 insertions(+), 26 deletions(-) diff --git a/src/rpc/types.rs b/src/rpc/types.rs index e3f187c..49b7738 100644 --- a/src/rpc/types.rs +++ b/src/rpc/types.rs @@ -219,7 +219,7 @@ pub struct CandleSubscription { pub interval: Interval, } -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Copy, Eq, Hash, PartialEq, Clone, Debug, Serialize, Deserialize)] pub enum Interval { ONE_MINUTE, FIVE_MINUTE, diff --git a/src/ws.rs b/src/ws.rs index 4631a6e..2bc1de1 100644 --- a/src/ws.rs +++ b/src/ws.rs @@ -1,5 +1,6 @@ use crate::database::{NewOrderBookOrder, TraderOrder}; use crate::kafka::start_consumer; +use crate::rpc::Interval; // use bigdecimal::ToPrimitive; use chrono::prelude::*; use crossbeam_channel::{unbounded, Sender as CrossbeamSender}; @@ -9,7 +10,11 @@ use jsonrpsee::RpcModule; use log::{error, info, trace}; use relayerwalletlib::zkoswalletlib::relayer_types::{OrderStatus, OrderType}; // use serde::{Deserialize, Serialize}; -use std::time::{Duration, Instant}; +use std::{ + collections::HashMap, + sync::RwLock, + time::{Duration, Instant}, +}; use tokio::{ sync::broadcast::{channel, Sender}, task::JoinHandle, @@ -31,6 +36,7 @@ pub struct WsContext { price_feed: Sender<(f64, DateTime)>, order_book: Sender, recent_trades: Sender, + pub candles: RwLock>>, pub pool: ManagedPool, _completions: CrossbeamSender, _watcher: JoinHandle<()>, @@ -105,6 +111,7 @@ impl WsContext { ) => {} Event::Stop(_stop) => {} Event::TxHash(..) => {} + Event::TxHashUpdate(..) => {} } } if let Err(e) = notify.send(completion) { @@ -130,6 +137,7 @@ impl WsContext { price_feed, order_book, recent_trades, + candles: Default::default(), pool, _completions: completions, _watcher, diff --git a/src/ws/methods.rs b/src/ws/methods.rs index 537a566..5c2d9b1 100644 --- a/src/ws/methods.rs +++ b/src/ws/methods.rs @@ -13,7 +13,7 @@ use log::{error, info, warn}; use serde::Serialize; use std::{sync::Arc, time::Duration}; use tokio::{ - sync::broadcast::{error::TryRecvError, Receiver}, + sync::broadcast::{channel, error::TryRecvError, Receiver}, task::JoinHandle, time::sleep, }; @@ -61,31 +61,62 @@ pub(super) fn candle_update( sink.accept()?; let CandleSubscription { interval } = params.parse()?; + + let spawn = match ctx.candles.read() { + Ok(r) => !r.contains_key(&interval), + Err(e) => { + sink.send(&format!("RwLock poisoned!")); + return Ok(()); + } + }; + + if spawn { + info!("SPAWNING new subscriber for {:?}", interval); + let Ok(mut l) = ctx.candles.write() else { sink.send(&"Write Lock poisoned!"); return Ok(()); }; + let (tx, _) = channel(10); + l.insert(interval, tx.clone()); + + let c = ctx.clone(); + let _: JoinHandle> = tokio::task::spawn(async move { + loop { + let mut conn = c.pool.get()?; + let since: DateTime = match interval { + Interval::ONE_DAY_CHANGE => Utc::now() - chrono::Duration::hours(24), + _ => Utc::now() - chrono::Duration::milliseconds(250), + }; + let candles = BtcUsdPrice::candles(&mut conn, interval, since, None, None)?; + + if candles.len() > 0 { + let result = serde_json::to_value(&candles)?; + if let Err(e) = tx.send(result) { + error!("Error sending candle updates: {:?}", e); + } + match interval { + Interval::ONE_DAY_CHANGE => { + sleep(Duration::from_millis(1000)).await; + } + _ => { + sleep(Duration::from_millis(250)).await; + } + }; + } else { + sleep(Duration::from_millis(250)).await; + } + + } + Ok(()) + }); + } + + let Ok(l) = ctx.candles.read() else { sink.send(&"Failed to acquire rx candles channel"); return Ok(()); }; + + let mut rx = l.get(&interval).unwrap().subscribe(); let _: JoinHandle> = tokio::task::spawn(async move { loop { - let mut conn = ctx.pool.get()?; - let since: DateTime = match interval { - Interval::ONE_DAY_CHANGE => Utc::now() - chrono::Duration::hours(24), - _ => Utc::now() - chrono::Duration::milliseconds(250), - }; - let candles = BtcUsdPrice::candles(&mut conn, interval.clone(), since, None, None)?; - - let result = serde_json::to_value(&candles)?; - - if candles.len() > 0 { - if let Err(e) = sink.send(&result) { - error!("Error sending candle updates: {:?}", e); - } - match interval { - Interval::ONE_DAY_CHANGE => { - sleep(Duration::from_millis(1000)).await; - } - _ => { - sleep(Duration::from_millis(250)).await; - } - }; - } else { - sleep(Duration::from_millis(250)).await; + let Ok(msg) = rx.recv().await else { error!("Recv channel broken!"); break; }; + + if let Err(e) = sink.send(&msg) { + error!("Error sending candle updates: {:?}", e); } } Ok(()) From ab35d323ce2f4dc8c2d22c803ddb22e49a1e5c40 Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Sat, 18 May 2024 17:59:55 +0000 Subject: [PATCH 27/87] delete test feed --- src/bin/test_feed.rs | 41 ----------------------------------------- 1 file changed, 41 deletions(-) delete mode 100644 src/bin/test_feed.rs diff --git a/src/bin/test_feed.rs b/src/bin/test_feed.rs deleted file mode 100644 index f366af6..0000000 --- a/src/bin/test_feed.rs +++ /dev/null @@ -1,41 +0,0 @@ -use chrono::prelude::*; -use kafka::producer::{Producer, Record, RequiredAcks}; -use rand::Rng; -use relayerarchiverlib::{rpc, ws}; -use std::{net::SocketAddr, time::Duration}; -use tokio::time::sleep; -use twilight_relayer_rust::db::Event; - - -#[tokio::main] -async fn main() { - dotenv::dotenv().expect("dotenv file not found!"); - let mut rng = rand::thread_rng(); - let mut current_price: f64 = rng.gen_range(60000.0..80000.0); - - let broker_host = std::env::var("BROKER").expect("missing environment variable BROKER"); - let topic = std::env::var("TRADERORDER_EVENT_LOG").expect("No topic!!!!"); - - let broker = vec![broker_host.to_owned()]; - let mut kafka = Producer::from_hosts(broker) - .with_ack_timeout(Duration::from_secs(1)) - .with_required_acks(RequiredAcks::One) - .create() - .unwrap(); - - - loop { - current_price += rng.gen_range(-50.0..50.0); - let system_time = Utc::now().to_rfc3339(); - - println!("New price: {}, {}", current_price, system_time); - let event = Event::CurrentPriceUpdate(current_price, system_time); - let serialized = serde_json::to_vec(&event).expect("Phuque"); - let record = Record::from_key_value(&topic, "CurrentPriceUpdate", serialized); - - kafka.send(&record).expect("NOOOOO!!"); - - sleep(Duration::from_secs(1)).await; - } - -} From 26930d9518698ad4bbfbd8d6a43902cbc9f9543b Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Sat, 18 May 2024 18:39:29 +0000 Subject: [PATCH 28/87] branch update --- Cargo.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 578d767..4b1a92c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -69,11 +69,11 @@ tower-http = { version = "0.4", features = [ tracing-subscriber = { version = "0.3.16", features = ["env-filter"] } uuid = { version = "1.3.3", features = ["serde", "v4", "v7"] } verify-keplr-sign = "0.1.0" -twilight-relayer-rust = { git = "ssh://git@github.com/twilight-project/twilight-relayer.git", branch = "client-side-trader-order-create" } +twilight-relayer-rust = { git = "ssh://git@github.com/twilight-project/twilight-relayer.git", branch = "develop-rpc" } [dependencies.zkos-relayer-wallet] git = "ssh://git@github.com/twilight-project/zkos-relayer-wallet.git" -branch = "clienttradeorder" +branch = "develop-rpc" [dependencies.zkos-client-wallet] git = "ssh://git@github.com/twilight-project/zkos-client-wallet.git" -branch = "clienttradeorder" +branch = "develop-rpc" From c4aec67c3f874e0a9e71d43df506317ae3b9bda5 Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Sat, 18 May 2024 20:06:34 +0000 Subject: [PATCH 29/87] orderbook update --- Cargo.toml | 2 +- src/database/models.rs | 19 +++++-------------- src/ws/methods.rs | 18 +++++++++++++----- 3 files changed, 19 insertions(+), 20 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 4b1a92c..37ac212 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -69,7 +69,7 @@ tower-http = { version = "0.4", features = [ tracing-subscriber = { version = "0.3.16", features = ["env-filter"] } uuid = { version = "1.3.3", features = ["serde", "v4", "v7"] } verify-keplr-sign = "0.1.0" -twilight-relayer-rust = { git = "ssh://git@github.com/twilight-project/twilight-relayer.git", branch = "develop-rpc" } +twilight-relayer-rust = { git = "ssh://git@github.com/twilight-project/twilight-relayer.git", branch = "halt-operation-rpc" } [dependencies.zkos-relayer-wallet] git = "ssh://git@github.com/twilight-project/zkos-relayer-wallet.git" diff --git a/src/database/models.rs b/src/database/models.rs index 2372370..d092fc3 100644 --- a/src/database/models.rs +++ b/src/database/models.rs @@ -835,18 +835,9 @@ pub struct CandleData { impl BtcUsdPrice { pub fn update_candles(conn: &mut PgConnection) -> QueryResult<()> { - diesel::sql_query( - "SELECT * from update_candles_1min()", - ) - .execute(conn)?; - diesel::sql_query( - "SELECT * from update_candles_1hour()", - ) - .execute(conn)?; - diesel::sql_query( - "SELECT * from update_candles_1day()", - ) - .execute(conn)?; + diesel::sql_query("SELECT * from update_candles_1min()").execute(conn)?; + diesel::sql_query("SELECT * from update_candles_1hour()").execute(conn)?; + diesel::sql_query("SELECT * from update_candles_1day()").execute(conn)?; Ok(()) } @@ -1493,8 +1484,8 @@ impl TraderOrder { WITH orders AS ( SELECT * FROM trader_order WHERE id IN ( - SELECT MAX(id) FROM trader_order - WHERE order_type = 'LIMIT' + SELECT MAX(id) FROM trader_order + WHERE order_type = 'LIMIT' AND position_type = 'SHORT' GROUP BY uuid ) AND order_status <> 'FILLED' diff --git a/src/ws/methods.rs b/src/ws/methods.rs index c6bf76e..387416a 100644 --- a/src/ws/methods.rs +++ b/src/ws/methods.rs @@ -72,7 +72,10 @@ pub(super) fn candle_update( if spawn { info!("SPAWNING new subscriber for {:?}", interval); - let Ok(mut l) = ctx.candles.write() else { sink.send(&"Write Lock poisoned!"); return Ok(()); }; + let Ok(mut l) = ctx.candles.write() else { + sink.send(&"Write Lock poisoned!"); + return Ok(()); + }; let (tx, _) = channel(10); l.insert(interval, tx.clone()); @@ -102,18 +105,23 @@ pub(super) fn candle_update( } else { sleep(Duration::from_millis(250)).await; } - } Ok(()) }); } - let Ok(l) = ctx.candles.read() else { sink.send(&"Failed to acquire rx candles channel"); return Ok(()); }; + let Ok(l) = ctx.candles.read() else { + sink.send(&"Failed to acquire rx candles channel"); + return Ok(()); + }; let mut rx = l.get(&interval).unwrap().subscribe(); - let _: JoinHandle> = tokio::task::spawn(async move { + let _result = tokio::task::spawn(async move { loop { - let Ok(msg) = rx.recv().await else { error!("Recv channel broken!"); break; }; + let Ok(msg) = rx.recv().await else { + error!("Recv channel broken!"); + break; + }; if let Err(e) = sink.send(&msg) { error!("Error sending candle updates: {:?}", e); From 0a52bea75b7f8cfec9d0fa7b877c6d24920e8644 Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Tue, 21 May 2024 20:10:43 +0530 Subject: [PATCH 30/87] liquidation event update --- src/bin/api.rs | 2 +- src/database/models.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bin/api.rs b/src/bin/api.rs index 68bc9b7..f08042e 100644 --- a/src/bin/api.rs +++ b/src/bin/api.rs @@ -32,7 +32,7 @@ struct Opt { )] ws_listen_addr: SocketAddr, } - +// #[tokio::main(flavor = "multi_thread", worker_threads = 10)] #[tokio::main] async fn main() { let opts = Opt::from_args(); diff --git a/src/database/models.rs b/src/database/models.rs index d092fc3..7b24bcd 100644 --- a/src/database/models.rs +++ b/src/database/models.rs @@ -517,7 +517,7 @@ fn lend_pool_to_batch( let uuid = order.uuid.to_string(); vec![(LendPoolCommandType::ADD_FUNDING_DATA, uuid, pay).into()] } - relayer_db::LendPoolCommand::AddTraderOrderLiquidation(_, order, p) => { + relayer_db::LendPoolCommand::AddTraderOrderLiquidation(_, order, p, _) => { let pay = Some(BigDecimal::from_f64(p).expect("Invalid floating point number")); let uuid = order.uuid.to_string(); vec![(LendPoolCommandType::ADD_TRADER_ORDER_LIQUIDATION, uuid, pay).into()] From 41827be00c75801509f927d283db013d1e4288cf Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Wed, 22 May 2024 10:38:57 +0000 Subject: [PATCH 31/87] orderbook correction --- src/database/models.rs | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/src/database/models.rs b/src/database/models.rs index d092fc3..dbe6353 100644 --- a/src/database/models.rs +++ b/src/database/models.rs @@ -1481,23 +1481,23 @@ impl TraderOrder { // use diesel::dsl::{max, sum}; let query = r#" - WITH orders AS ( - SELECT * FROM trader_order - WHERE id IN ( - SELECT MAX(id) FROM trader_order - WHERE order_type = 'LIMIT' AND position_type = 'SHORT' - GROUP BY uuid - ) - AND order_status <> 'FILLED' + WITH orders AS ( + SELECT * FROM trader_order + WHERE id IN ( + SELECT MAX(id) FROM trader_order + WHERE order_type = 'LIMIT' AND position_type = 'SHORT' + GROUP BY uuid ) - SELECT - MAX(uuid) AS uuid, - entryprice, - SUM(positionsize) AS positionsize - FROM orders - GROUP BY entryprice - ORDER BY positionsize DESC - LIMIT 10; + AND order_status <> 'FILLED' AND order_status <> 'CANCELLED' AND order_status <> 'LIQUIDATE' + ) + SELECT + MAX(uuid) AS uuid, + entryprice, + SUM(positionsize) AS positionsize + FROM orders + GROUP BY entryprice + ORDER BY entryprice ASC + LIMIT 15; "#; let shorts: Vec = diesel::sql_query(query).get_results(conn)?; @@ -1519,7 +1519,7 @@ impl TraderOrder { WHERE order_type = 'LIMIT' AND position_type = 'LONG' GROUP BY uuid ) - AND order_status <> 'FILLED' + AND order_status <> 'FILLED' AND order_status <> 'CANCELLED' AND order_status <> 'LIQUIDATE' ) SELECT MAX(uuid) AS uuid, @@ -1527,8 +1527,8 @@ impl TraderOrder { SUM(positionsize) AS positionsize FROM orders GROUP BY entryprice - ORDER BY positionsize DESC - LIMIT 10; + ORDER BY entryprice DESC + LIMIT 15; "#; let longs: Vec = diesel::sql_query(query).get_results(conn)?; From 5b043b991d77f997342dccb4e404822c7d9c0656 Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Wed, 22 May 2024 12:17:13 +0000 Subject: [PATCH 32/87] missing orders added in recent order script --- src/database/models.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/database/models.rs b/src/database/models.rs index 53ca264..5cfde05 100644 --- a/src/database/models.rs +++ b/src/database/models.rs @@ -1588,7 +1588,7 @@ impl TraderOrder { FROM trader_order INNER JOIN ( SELECT uuid,min(timestamp) AS timestamp - FROM trader_order GROUP BY uuid + FROM trader_order WHERE trader_order.order_status <> 'PENDING' GROUP BY uuid ) as t ON trader_order.uuid = t.uuid AND trader_order.timestamp = t.timestamp WHERE t.timestamp > now() - INTERVAL '1 day' @@ -1603,7 +1603,7 @@ impl TraderOrder { trader_order.timestamp as timestamp FROM trader_order INNER JOIN ( - SELECT uuid,min(timestamp) AS timestamp + SELECT uuid,max(timestamp) AS timestamp FROM trader_order GROUP BY uuid ) as t ON trader_order.uuid = t.uuid AND trader_order.timestamp = t.timestamp From f0187f3163f54271ddc36d2e3f99e2fc3223fe74 Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Wed, 22 May 2024 22:52:00 +0000 Subject: [PATCH 33/87] correcting bid and ask for recent order --- src/database/models.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/database/models.rs b/src/database/models.rs index 5cfde05..94b0375 100644 --- a/src/database/models.rs +++ b/src/database/models.rs @@ -1597,7 +1597,10 @@ impl TraderOrder { UNION ALL SELECT - trader_order.position_type as side, + (CASE WHEN trader_order.position_type = 'LONG' THEN position_type('SHORT') + ELSE position_type('LONG') + END) as side , + trader_order.settlement_price as price, trader_order.positionsize as positionsize, trader_order.timestamp as timestamp From ec4d75ed171699426dc04ea0ea1eea45672ac521 Mon Sep 17 00:00:00 2001 From: thomas SHARP Date: Mon, 27 May 2024 20:11:13 -0400 Subject: [PATCH 34/87] Fix aggregation start time --- migrations/2024-03-26-213323_add_aggregation_tables/up.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/migrations/2024-03-26-213323_add_aggregation_tables/up.sql b/migrations/2024-03-26-213323_add_aggregation_tables/up.sql index fb6ade4..b3f596f 100644 --- a/migrations/2024-03-26-213323_add_aggregation_tables/up.sql +++ b/migrations/2024-03-26-213323_add_aggregation_tables/up.sql @@ -114,7 +114,7 @@ RETURNS void AS $$ INSERT INTO candles_1min ( start_time, end_time, usd_volume, btc_volume, trades, open, high, low, close ) -SELECT * FROM get_candles_interval('1 minute', 'minute', (SELECT coalesce(max(start_time) - interval '10 minute', now() - interval '1 week') FROM candles_1min)) +SELECT * FROM get_candles_interval('1 minute', 'minute', (SELECT coalesce(max(start_time) - interval '10 minute', '1970-01-01 00:00:00.0000+00'::timestamptz) FROM candles_1min)) ON CONFLICT(start_time) DO UPDATE SET start_time = excluded.start_time, @@ -136,7 +136,7 @@ RETURNS void AS $$ INSERT INTO candles_1hour ( start_time, end_time, usd_volume, btc_volume, trades, open, high, low, close ) -SELECT * FROM get_candles_interval('1 hour', 'hour', (SELECT coalesce(max(start_time) - interval '10 minute', now() - interval '1 week') FROM candles_1hour)) +SELECT * FROM get_candles_interval('1 hour', 'hour', (SELECT coalesce(max(start_time) - interval '10 minute', '1970-01-01 00:00:00.0000+00'::timestamptz) FROM candles_1hour)) ON CONFLICT(start_time) DO UPDATE SET start_time = excluded.start_time, @@ -158,7 +158,7 @@ RETURNS void AS $$ INSERT INTO candles_1day ( start_time, end_time, usd_volume, btc_volume, trades, open, high, low, close ) -SELECT * FROM get_candles_interval('1 day', 'day', (SELECT coalesce(max(start_time) - interval '10 minute', now() - interval '1 week') FROM candles_1day)) +SELECT * FROM get_candles_interval('1 day', 'day', (SELECT coalesce(max(start_time) - interval '10 minute', '1970-01-01 00:00:00.0000+00'::timestamptz) FROM candles_1day)) ON CONFLICT(start_time) DO UPDATE SET start_time = excluded.start_time, From 710d102945b7611fc52fdc50e5abd4c565d44032 Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Thu, 30 May 2024 15:54:26 +0000 Subject: [PATCH 35/87] cargo update for zkos-rust --- Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 37ac212..68274ee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -73,7 +73,7 @@ twilight-relayer-rust = { git = "ssh://git@github.com/twilight-project/twilight- [dependencies.zkos-relayer-wallet] git = "ssh://git@github.com/twilight-project/zkos-relayer-wallet.git" -branch = "develop-rpc" +branch = "develop-rust" [dependencies.zkos-client-wallet] git = "ssh://git@github.com/twilight-project/zkos-client-wallet.git" -branch = "develop-rpc" +branch = "develop-rust" From 578b142fbd467dbb3b429bb9cb7894cfb436c7a1 Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Thu, 6 Jun 2024 08:39:58 +0000 Subject: [PATCH 36/87] temperary update --- src/kafka.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/kafka.rs b/src/kafka.rs index ca22342..cdd20d4 100644 --- a/src/kafka.rs +++ b/src/kafka.rs @@ -54,7 +54,9 @@ pub fn start_consumer( let message: Event = match serde_json::from_str(&msg_data) { Ok(event) => event, Err(e) => { - panic!("Invalid message! {:?} {}", e, msg_data); + println!("Invalid message! {:?} {}\n", e, msg_data); + // continue; + Event::Stop(e.to_string()) } }; message From f94973ad6cad42e1b4ae5246b03ca837d4dbf369 Mon Sep 17 00:00:00 2001 From: thomas SHARP Date: Mon, 10 Jun 2024 09:16:44 -0400 Subject: [PATCH 37/87] Update orderbook with sorted set commands --- src/database/models.rs | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/src/database/models.rs b/src/database/models.rs index 94b0375..1426929 100644 --- a/src/database/models.rs +++ b/src/database/models.rs @@ -1489,12 +1489,29 @@ impl TraderOrder { GROUP BY uuid ) AND order_status <> 'FILLED' AND order_status <> 'CANCELLED' AND order_status <> 'LIQUIDATE' + ), commands AS ( + SELECT MAX(id) as id,uuid FROM sorted_set_command + WHERE uuid IN ( SELECT uuid FROM orders ) + GROUP BY uuid + ), updates AS ( + SELECT * FROM sorted_set_command + WHERE id IN ( SELECT id FROM commands ) + ), updated AS( + SELECT + orders.uuid as uuid, + COALESCE(amount, entryprice) as entryprice, + positionsize as positionsize, + updates.command as command + FROM orders + LEFT JOIN updates + ON updates.uuid = orders.uuid ) SELECT MAX(uuid) AS uuid, entryprice, SUM(positionsize) AS positionsize - FROM orders + FROM updated + WHERE command IS NULL OR command <> 'REMOVE_CLOSE_LIMIT_PRICE' GROUP BY entryprice ORDER BY entryprice ASC LIMIT 15; @@ -1520,12 +1537,29 @@ impl TraderOrder { GROUP BY uuid ) AND order_status <> 'FILLED' AND order_status <> 'CANCELLED' AND order_status <> 'LIQUIDATE' + ), commands AS ( + SELECT MAX(id) as id,uuid FROM sorted_set_command + WHERE uuid IN ( SELECT uuid FROM orders ) + GROUP BY uuid + ), updates AS ( + SELECT * FROM sorted_set_command + WHERE id IN ( SELECT id FROM commands ) + ), updated AS( + SELECT + orders.uuid as uuid, + COALESCE(amount, entryprice) as entryprice, + positionsize as positionsize, + updates.command as command + FROM orders + LEFT JOIN updates + ON updates.uuid = orders.uuid ) SELECT MAX(uuid) AS uuid, entryprice, SUM(positionsize) AS positionsize - FROM orders + FROM updated + WHERE command IS NULL OR command <> 'REMOVE_CLOSE_LIMIT_PRICE' GROUP BY entryprice ORDER BY entryprice DESC LIMIT 15; From 46483bad42bad69ded2fbf1b076d2fb5bad3740f Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Thu, 13 Jun 2024 20:40:24 +0000 Subject: [PATCH 38/87] optimized script for recent order api --- src/database/models.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/database/models.rs b/src/database/models.rs index 1426929..bb61b23 100644 --- a/src/database/models.rs +++ b/src/database/models.rs @@ -1614,7 +1614,7 @@ impl TraderOrder { pub fn list_past_24hrs(conn: &mut PgConnection) -> QueryResult> { // use crate::database::schema::trader_order::dsl::*; - let query = r#"SELECT + let query = r#"Select * from (SELECT trader_order.position_type as side, trader_order.entryprice as price, trader_order.positionsize as positionsize, @@ -1622,11 +1622,11 @@ impl TraderOrder { FROM trader_order INNER JOIN ( SELECT uuid,min(timestamp) AS timestamp - FROM trader_order WHERE trader_order.order_status <> 'PENDING' GROUP BY uuid + FROM trader_order WHERE trader_order.order_status = 'FILLED' GROUP BY uuid order by timestamp desc limit 100 ) as t ON trader_order.uuid = t.uuid AND trader_order.timestamp = t.timestamp WHERE t.timestamp > now() - INTERVAL '1 day' - AND trader_order.order_status = 'FILLED' + AND trader_order.order_status = 'FILLED' UNION ALL @@ -1641,11 +1641,11 @@ impl TraderOrder { FROM trader_order INNER JOIN ( SELECT uuid,max(timestamp) AS timestamp - FROM trader_order GROUP BY uuid + FROM trader_order GROUP BY uuid order by timestamp desc limit 100 ) as t ON trader_order.uuid = t.uuid AND trader_order.timestamp = t.timestamp WHERE t.timestamp > now() - INTERVAL '1 day' - AND trader_order.order_status IN ('SETTLED', 'LIQUIDATE') + AND trader_order.order_status IN ('SETTLED', 'LIQUIDATE') order by timestamp desc limit 100 ) as recent_order order by timestamp desc limit 50 "#; diesel::sql_query(query).load(conn) From 4879fb03af418d3c49f7288fa8c56d6f15e1db4e Mon Sep 17 00:00:00 2001 From: thomas SHARP Date: Mon, 17 Jun 2024 07:24:39 -0400 Subject: [PATCH 39/87] Update query --- src/database/models.rs | 64 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/src/database/models.rs b/src/database/models.rs index bb61b23..64c9cc3 100644 --- a/src/database/models.rs +++ b/src/database/models.rs @@ -1505,7 +1505,39 @@ impl TraderOrder { FROM orders LEFT JOIN updates ON updates.uuid = orders.uuid + ), sorted_set AS ( + SELECT * + FROM sorted_set_command + WHERE id IN ( + SELECT MAX(id) FROM sorted_set_command + WHERE uuid IS NOT NULL + GROUP BY uuid + ) + AND command IN ('ADD_CLOSE_LIMIT_PRICE', 'UPDATE_CLOSE_LIMIT_PRICE') + AND position_type ='LONG' ORDER BY id DESC ) + SELECT + trader_o.uuid AS uuid, + sort.amount AS entryprice, + trader_o.positionsize as positionsize + FROM ( + SELECT * + FROM trader_order + WHERE id IN ( + SELECT MAX(id) FROM trader_order + WHERE position_type = 'SHORT' AND uuid IN ( + SELECT uuid + FROM sorted_Set + ) + GROUP BY uuid + ) + AND order_status = 'FILLED' + ) AS trader_o + LEFT OUTER JOIN ( + SELECT amount,uuid FROM sorted_Set + ) AS sort + ON trader_o.uuid = sort.uuid + UNION ALL SELECT MAX(uuid) AS uuid, entryprice, @@ -1517,6 +1549,7 @@ impl TraderOrder { LIMIT 15; "#; + let shorts: Vec = diesel::sql_query(query).get_results(conn)?; let ask: Vec<_> = shorts @@ -1553,7 +1586,38 @@ impl TraderOrder { FROM orders LEFT JOIN updates ON updates.uuid = orders.uuid + ), sorted_set AS ( + SELECT * + FROM sorted_set_command + WHERE id IN ( + SELECT MAX(id) FROM sorted_set_command + WHERE uuid IS NOT NULL + GROUP BY uuid + ) + AND command IN ('ADD_CLOSE_LIMIT_PRICE', 'UPDATE_CLOSE_LIMIT_PRICE') + AND position_type ='LONG' + ORDER BY id DESC ) + SELECT + trader_o.uuid AS uuid, + sort.amount AS entryprice, + trader_o.positionsize AS positionsize + FROM ( + SELECT * + FROM trader_order + WHERE id IN ( + SELECT MAX(id) FROM trader_order + WHERE position_type = 'LONG' + AND uuid IN ( SELECT uuid FROM sorted_set) + GROUP BY uuid + ) + AND trader_order.order_status = 'FILLED' + ) AS trader_o + LEFT OUTER JOIN ( + SELECT amount,uuid FROM sorted_set + ) AS sort + ON trader_o.uuid = sort.uuid + UNION ALL SELECT MAX(uuid) AS uuid, entryprice, From 2bcc7849d2affba070c9ea9ecb26e40fd5aa6d5a Mon Sep 17 00:00:00 2001 From: thomas SHARP Date: Mon, 17 Jun 2024 07:43:25 -0400 Subject: [PATCH 40/87] Increase batch size --- src/archiver.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/archiver.rs b/src/archiver.rs index 8ea3053..2cce94b 100644 --- a/src/archiver.rs +++ b/src/archiver.rs @@ -12,7 +12,7 @@ use twilight_relayer_rust::{ }; const BATCH_INTERVAL: u64 = 100; -const BATCH_SIZE: usize = 500; +const BATCH_SIZE: usize = 5_000; const MAX_RETRIES: usize = 5; const RETRY_SLEEP: u64 = 2000; From 7a5590ed1e1d82a98d92335bc27e58bbbe86bee5 Mon Sep 17 00:00:00 2001 From: thomas SHARP Date: Mon, 17 Jun 2024 07:56:00 -0400 Subject: [PATCH 41/87] commit interval --- src/archiver.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/archiver.rs b/src/archiver.rs index 2cce94b..a8f403e 100644 --- a/src/archiver.rs +++ b/src/archiver.rs @@ -423,7 +423,6 @@ impl DatabaseArchiver { BtcUsdPrice::update_candles(&mut *self.get_conn()?)?; } - self.commit_orders()?; self.completions .send(completion) .map_err(|e| ApiError::CrossbeamChannel(format!("{:?}", e)))?; From 50a22ffac94adde0d42e6dd1ae9068c1e6663d2a Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Wed, 19 Jun 2024 20:34:21 +0000 Subject: [PATCH 42/87] batch size update --- src/archiver.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/archiver.rs b/src/archiver.rs index a8f403e..06aafbc 100644 --- a/src/archiver.rs +++ b/src/archiver.rs @@ -12,7 +12,7 @@ use twilight_relayer_rust::{ }; const BATCH_INTERVAL: u64 = 100; -const BATCH_SIZE: usize = 5_000; +const BATCH_SIZE: usize = 2_000; const MAX_RETRIES: usize = 5; const RETRY_SLEEP: u64 = 2000; @@ -413,6 +413,7 @@ impl DatabaseArchiver { let mut deadline = Instant::now() + Duration::from_millis(BATCH_INTERVAL); loop { + println!("Deadline: {:?}\n", deadline); match rx.recv_deadline(deadline) { Ok((completion, msgs, catchup)) => { for msg in msgs { From 7f705f8dd5d9e51c424fcfad359e507b80792738 Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Thu, 20 Jun 2024 13:11:35 +0000 Subject: [PATCH 43/87] BtcUsdPrice::update_candles removed from archiver --- .../down.sql | 1 + .../up.sql | 45 +++++++++++++++++++ src/archiver.rs | 6 +-- 3 files changed, 49 insertions(+), 3 deletions(-) create mode 100644 migrations/2024-06-20-130635_candle_data_trigger/down.sql create mode 100644 migrations/2024-06-20-130635_candle_data_trigger/up.sql diff --git a/migrations/2024-06-20-130635_candle_data_trigger/down.sql b/migrations/2024-06-20-130635_candle_data_trigger/down.sql new file mode 100644 index 0000000..d9a93fe --- /dev/null +++ b/migrations/2024-06-20-130635_candle_data_trigger/down.sql @@ -0,0 +1 @@ +-- This file should undo anything in `up.sql` diff --git a/migrations/2024-06-20-130635_candle_data_trigger/up.sql b/migrations/2024-06-20-130635_candle_data_trigger/up.sql new file mode 100644 index 0000000..b9a9fc3 --- /dev/null +++ b/migrations/2024-06-20-130635_candle_data_trigger/up.sql @@ -0,0 +1,45 @@ +-- Your SQL goes here + +-- FUNCTION: public.create_price_triger_after_insert() + +-- DROP FUNCTION IF EXISTS public.create_price_triger_after_insert(); + +CREATE OR REPLACE FUNCTION public.create_price_triger_after_insert_for_candle_data_generation() + RETURNS trigger + LANGUAGE 'plpgsql' + COST 100 + VOLATILE NOT LEAKPROOF +AS $BODY$ +BEGIN + + CASE + WHEN (NOW() <( New."timestamp" + INTERVAL '250 ms')) THEN + PERFORM update_candles_1min(); + PERFORM update_candles_1hour(); + PERFORM update_candles_1day(); + ELSE + NULL; + END CASE; + + RETURN New; +END; +$BODY$; + +ALTER FUNCTION public.create_price_triger_after_insert_for_candle_data_generation() + OWNER TO relayer; + + +-- Trigger: after_insert_price_trigger + +-- DROP TRIGGER IF EXISTS after_insert_price_trigger ON public.btc_usd_price; + +CREATE OR REPLACE TRIGGER after_insert_price_trigger_for_candle_data_generation + AFTER INSERT + ON public.btc_usd_price + FOR EACH ROW + EXECUTE FUNCTION public.create_price_triger_after_insert_for_candle_data_generation(); + + + + + \ No newline at end of file diff --git a/src/archiver.rs b/src/archiver.rs index 06aafbc..0f67af3 100644 --- a/src/archiver.rs +++ b/src/archiver.rs @@ -420,9 +420,9 @@ impl DatabaseArchiver { self.process_msg(msg)?; } - if !catchup { - BtcUsdPrice::update_candles(&mut *self.get_conn()?)?; - } + // if !catchup { + // BtcUsdPrice::update_candles(&mut *self.get_conn()?)?; + // } self.completions .send(completion) From 43f3dc47650038d76e8ac651c11d42d38e354175 Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Thu, 20 Jun 2024 14:14:48 +0000 Subject: [PATCH 44/87] trigger added for null request id --- .../up.sql | 43 ++++++++++++++++++- src/archiver.rs | 3 +- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/migrations/2024-06-20-130635_candle_data_trigger/up.sql b/migrations/2024-06-20-130635_candle_data_trigger/up.sql index b9a9fc3..852ca77 100644 --- a/migrations/2024-06-20-130635_candle_data_trigger/up.sql +++ b/migrations/2024-06-20-130635_candle_data_trigger/up.sql @@ -41,5 +41,46 @@ CREATE OR REPLACE TRIGGER after_insert_price_trigger_for_candle_data_generation +-- FUNCTION: public.update_request_id_after_pending_order_fill() + +-- DROP FUNCTION IF EXISTS public.update_request_id_after_pending_order_fill(); + +CREATE OR REPLACE FUNCTION public.update_request_id_after_pending_order_fill() + RETURNS trigger + LANGUAGE 'plpgsql' + COST 100 + VOLATILE NOT LEAKPROOF +AS $BODY$ +BEGIN + + CASE + WHEN (New.request_id is NULL and New.order_status = 'FILLED') THEN + UPDATE public.transaction_hash + SET request_id=(Select request_id from public.transaction_hash where order_id = New.order_id and order_status = 'PENDING' limit 1) + WHERE id = New.id and order_status = 'FILLED'; + + WHEN (New.request_id is NULL and New.order_status = 'LIQUIDATE') THEN + UPDATE public.transaction_hash + SET request_id=(Select request_id from public.transaction_hash where order_id = New.order_id and order_status = 'FILLED' limit 1) + WHERE id = New.id and order_status = 'LIQUIDATE'; + ELSE + NULL; + END CASE; + + RETURN New; +END; +$BODY$; + +ALTER FUNCTION public.update_request_id_after_pending_order_fill() + OWNER TO relayer; - \ No newline at end of file + +-- Trigger: after_insert_update_request_id_after_pending_order_fill + +-- DROP TRIGGER IF EXISTS after_insert_update_request_id_after_pending_order_fill ON public.transaction_hash; + +CREATE OR REPLACE TRIGGER after_insert_update_request_id_after_pending_order_fill + AFTER INSERT + ON public.transaction_hash + FOR EACH ROW + EXECUTE FUNCTION public.update_request_id_after_pending_order_fill(); \ No newline at end of file diff --git a/src/archiver.rs b/src/archiver.rs index 0f67af3..3fc07d8 100644 --- a/src/archiver.rs +++ b/src/archiver.rs @@ -413,9 +413,8 @@ impl DatabaseArchiver { let mut deadline = Instant::now() + Duration::from_millis(BATCH_INTERVAL); loop { - println!("Deadline: {:?}\n", deadline); match rx.recv_deadline(deadline) { - Ok((completion, msgs, catchup)) => { + Ok((completion, msgs, _catchup)) => { for msg in msgs { self.process_msg(msg)?; } From 5d3d6ea013e1c21cb2953179479ce3445aa0c68d Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Thu, 20 Jun 2024 14:43:01 +0000 Subject: [PATCH 45/87] minor update on trigger --- migrations/2024-06-20-130635_candle_data_trigger/up.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migrations/2024-06-20-130635_candle_data_trigger/up.sql b/migrations/2024-06-20-130635_candle_data_trigger/up.sql index 852ca77..129796f 100644 --- a/migrations/2024-06-20-130635_candle_data_trigger/up.sql +++ b/migrations/2024-06-20-130635_candle_data_trigger/up.sql @@ -13,7 +13,7 @@ AS $BODY$ BEGIN CASE - WHEN (NOW() <( New."timestamp" + INTERVAL '250 ms')) THEN + WHEN (NOW() <( New."timestamp" + INTERVAL '500 ms')) THEN PERFORM update_candles_1min(); PERFORM update_candles_1hour(); PERFORM update_candles_1day(); From f679853955c5ebaeab273142ea632864440cc486 Mon Sep 17 00:00:00 2001 From: thomas SHARP Date: Sun, 23 Jun 2024 11:21:20 -0400 Subject: [PATCH 46/87] Remove catchup logic --- src/archiver.rs | 6 +----- src/kafka.rs | 5 +---- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/src/archiver.rs b/src/archiver.rs index 3fc07d8..a622438 100644 --- a/src/archiver.rs +++ b/src/archiver.rs @@ -414,15 +414,11 @@ impl DatabaseArchiver { loop { match rx.recv_deadline(deadline) { - Ok((completion, msgs, _catchup)) => { + Ok((completion, msgs)) => { for msg in msgs { self.process_msg(msg)?; } - // if !catchup { - // BtcUsdPrice::update_candles(&mut *self.get_conn()?)?; - // } - self.completions .send(completion) .map_err(|e| ApiError::CrossbeamChannel(format!("{:?}", e)))?; diff --git a/src/kafka.rs b/src/kafka.rs index cdd20d4..76ffed3 100644 --- a/src/kafka.rs +++ b/src/kafka.rs @@ -4,8 +4,6 @@ use log::{error, info}; use std::thread::{self, JoinHandle}; use twilight_relayer_rust::db::Event; -// > 500 offset behind, we'll batch update. -const CATCHUP_INTERVAL: i64 = 500; pub type Completion = (i32, i64); pub fn start_consumer( @@ -64,9 +62,8 @@ pub fn start_consumer( .collect(); let token = (ms.partition(), max_offset); - let catchup = latest - max_offset > CATCHUP_INTERVAL; - match sender_clone.send((token, events, catchup)) { + match sender_clone.send((token, events)) { Ok(_) => {} Err(_arg) => { connection_status = false; From 19d37117bafdb0d5e0f37e24059534ab44ee6e9c Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Mon, 24 Jun 2024 14:00:39 +0000 Subject: [PATCH 47/87] recent order query optimize --- src/database/models.rs | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/database/models.rs b/src/database/models.rs index 64c9cc3..fcbb197 100644 --- a/src/database/models.rs +++ b/src/database/models.rs @@ -1549,7 +1549,6 @@ impl TraderOrder { LIMIT 15; "#; - let shorts: Vec = diesel::sql_query(query).get_results(conn)?; let ask: Vec<_> = shorts @@ -1686,14 +1685,14 @@ impl TraderOrder { FROM trader_order INNER JOIN ( SELECT uuid,min(timestamp) AS timestamp - FROM trader_order WHERE trader_order.order_status = 'FILLED' GROUP BY uuid order by timestamp desc limit 100 + FROM trader_order WHERE trader_order.order_status = 'FILLED' and timestamp > now() - INTERVAL '1 day' GROUP BY uuid order by timestamp desc limit 50 + ) as t ON trader_order.uuid = t.uuid AND trader_order.timestamp = t.timestamp - WHERE t.timestamp > now() - INTERVAL '1 day' - AND trader_order.order_status = 'FILLED' UNION ALL + SELECT (CASE WHEN trader_order.position_type = 'LONG' THEN position_type('SHORT') ELSE position_type('LONG') @@ -1702,14 +1701,19 @@ impl TraderOrder { trader_order.settlement_price as price, trader_order.positionsize as positionsize, trader_order.timestamp as timestamp + FROM trader_order INNER JOIN ( SELECT uuid,max(timestamp) AS timestamp - FROM trader_order GROUP BY uuid order by timestamp desc limit 100 + FROM trader_order + where order_status IN ('SETTLED', 'LIQUIDATE') + AND timestamp > now() - INTERVAL '1 day' + GROUP BY uuid order by timestamp desc limit 50 + ) as t ON trader_order.uuid = t.uuid AND trader_order.timestamp = t.timestamp - WHERE t.timestamp > now() - INTERVAL '1 day' - AND trader_order.order_status IN ('SETTLED', 'LIQUIDATE') order by timestamp desc limit 100 ) as recent_order order by timestamp desc limit 50 + + order by timestamp desc ) as recent_order order by timestamp desc limit 50 "#; diesel::sql_query(query).load(conn) From de0d44797d2f3dd9bea6ee519a7aea113c32ba26 Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Mon, 24 Jun 2024 14:00:47 +0000 Subject: [PATCH 48/87] repeated index removed --- .../2024-03-26-213323_add_aggregation_tables/down.sql | 5 +++-- migrations/2024-03-26-213323_add_aggregation_tables/up.sql | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/migrations/2024-03-26-213323_add_aggregation_tables/down.sql b/migrations/2024-03-26-213323_add_aggregation_tables/down.sql index 1009034..e70010b 100644 --- a/migrations/2024-03-26-213323_add_aggregation_tables/down.sql +++ b/migrations/2024-03-26-213323_add_aggregation_tables/down.sql @@ -1,6 +1,7 @@ -- This file should undo anything in `up.sql` -DROP INDEX trader_order_time; -DROP INDEX price_time; +-- DROP INDEX trader_order_time; +-- DROP INDEX price_time; +DROP INDEX trader_order_uuid; DROP TABLE candles_1min; DROP TABLE candles_1hour; diff --git a/migrations/2024-03-26-213323_add_aggregation_tables/up.sql b/migrations/2024-03-26-213323_add_aggregation_tables/up.sql index b3f596f..c6baca6 100644 --- a/migrations/2024-03-26-213323_add_aggregation_tables/up.sql +++ b/migrations/2024-03-26-213323_add_aggregation_tables/up.sql @@ -1,7 +1,9 @@ -- Your SQL goes here -CREATE INDEX trader_order_time ON trader_order(timestamp); -CREATE INDEX price_time ON btc_usd_price(timestamp); +-- CREATE INDEX trader_order_time ON trader_order(timestamp); +-- CREATE INDEX price_time ON btc_usd_price(timestamp); + +CREATE INDEX trader_order_uuid ON trader_order(uuid); CREATE TABLE candles_1min ( start_time timestamptz primary key, From e684bf837d8010c08d06d1b477deb9543990c375 Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Tue, 25 Jun 2024 08:48:14 +0000 Subject: [PATCH 49/87] minor bug fixed --- migrations/2024-03-26-213323_add_aggregation_tables/down.sql | 2 ++ migrations/2024-03-26-213323_add_aggregation_tables/up.sql | 2 ++ src/archiver.rs | 2 +- src/bin/api.rs | 4 ++-- src/kafka.rs | 2 +- src/ws.rs | 2 +- 6 files changed, 9 insertions(+), 5 deletions(-) diff --git a/migrations/2024-03-26-213323_add_aggregation_tables/down.sql b/migrations/2024-03-26-213323_add_aggregation_tables/down.sql index e70010b..9354dd6 100644 --- a/migrations/2024-03-26-213323_add_aggregation_tables/down.sql +++ b/migrations/2024-03-26-213323_add_aggregation_tables/down.sql @@ -2,6 +2,8 @@ -- DROP INDEX trader_order_time; -- DROP INDEX price_time; DROP INDEX trader_order_uuid; +DROP INDEX transaction_hash_account_id; +DROP INDEX transaction_hash_request_id; DROP TABLE candles_1min; DROP TABLE candles_1hour; diff --git a/migrations/2024-03-26-213323_add_aggregation_tables/up.sql b/migrations/2024-03-26-213323_add_aggregation_tables/up.sql index c6baca6..d87c7a9 100644 --- a/migrations/2024-03-26-213323_add_aggregation_tables/up.sql +++ b/migrations/2024-03-26-213323_add_aggregation_tables/up.sql @@ -4,6 +4,8 @@ -- CREATE INDEX price_time ON btc_usd_price(timestamp); CREATE INDEX trader_order_uuid ON trader_order(uuid); +CREATE INDEX transaction_hash_account_id ON transaction_hash(account_id); +CREATE INDEX transaction_hash_request_id ON transaction_hash(request_id); CREATE TABLE candles_1min ( start_time timestamptz primary key, diff --git a/src/archiver.rs b/src/archiver.rs index a622438..fcf9026 100644 --- a/src/archiver.rs +++ b/src/archiver.rs @@ -409,7 +409,7 @@ impl DatabaseArchiver { } /// Worker task that loops indefinitely, batching commits to postgres backend. - pub fn run(mut self, rx: Receiver<(Completion, Vec, bool)>) -> Result<(), ApiError> { + pub fn run(mut self, rx: Receiver<(Completion, Vec)>) -> Result<(), ApiError> { let mut deadline = Instant::now() + Duration::from_millis(BATCH_INTERVAL); loop { diff --git a/src/bin/api.rs b/src/bin/api.rs index f08042e..4cb96e7 100644 --- a/src/bin/api.rs +++ b/src/bin/api.rs @@ -32,8 +32,8 @@ struct Opt { )] ws_listen_addr: SocketAddr, } -// #[tokio::main(flavor = "multi_thread", worker_threads = 10)] -#[tokio::main] +#[tokio::main(flavor = "multi_thread", worker_threads = 40)] +// #[tokio::main] async fn main() { let opts = Opt::from_args(); dotenv::dotenv().expect("dotenv file not found!"); diff --git a/src/kafka.rs b/src/kafka.rs index 76ffed3..7692d0c 100644 --- a/src/kafka.rs +++ b/src/kafka.rs @@ -9,7 +9,7 @@ pub type Completion = (i32, i64); pub fn start_consumer( group: String, topic: String, - tx: Sender<(Completion, Vec, bool)>, + tx: Sender<(Completion, Vec)>, ) -> (Sender, JoinHandle<()>) { let (tx_consumed, rx_consumed) = unbounded::(); diff --git a/src/ws.rs b/src/ws.rs index 2bc1de1..aff5cbb 100644 --- a/src/ws.rs +++ b/src/ws.rs @@ -67,7 +67,7 @@ impl WsContext { let mut deadline = Instant::now() + Duration::from_millis(WS_UPDATE_INTERVAL); loop { match rx.recv_deadline(deadline) { - Ok((completion, msgs, _)) => { + Ok((completion, msgs)) => { for msg in msgs { match msg { Event::TraderOrder(to, ..) From ff215cbfc8b1fdc02241cac9a6748ce34a8e827d Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Wed, 26 Jun 2024 18:11:32 +0000 Subject: [PATCH 50/87] index bug removed --- migrations/2024-03-26-213323_add_aggregation_tables/down.sql | 3 +-- migrations/2024-03-26-213323_add_aggregation_tables/up.sql | 3 +-- migrations/2024-03-27-095235_order_status/down.sql | 4 +++- migrations/2024-03-27-095235_order_status/up.sql | 2 ++ 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/migrations/2024-03-26-213323_add_aggregation_tables/down.sql b/migrations/2024-03-26-213323_add_aggregation_tables/down.sql index 9354dd6..fc66c07 100644 --- a/migrations/2024-03-26-213323_add_aggregation_tables/down.sql +++ b/migrations/2024-03-26-213323_add_aggregation_tables/down.sql @@ -2,8 +2,7 @@ -- DROP INDEX trader_order_time; -- DROP INDEX price_time; DROP INDEX trader_order_uuid; -DROP INDEX transaction_hash_account_id; -DROP INDEX transaction_hash_request_id; + DROP TABLE candles_1min; DROP TABLE candles_1hour; diff --git a/migrations/2024-03-26-213323_add_aggregation_tables/up.sql b/migrations/2024-03-26-213323_add_aggregation_tables/up.sql index d87c7a9..b6433e0 100644 --- a/migrations/2024-03-26-213323_add_aggregation_tables/up.sql +++ b/migrations/2024-03-26-213323_add_aggregation_tables/up.sql @@ -4,8 +4,7 @@ -- CREATE INDEX price_time ON btc_usd_price(timestamp); CREATE INDEX trader_order_uuid ON trader_order(uuid); -CREATE INDEX transaction_hash_account_id ON transaction_hash(account_id); -CREATE INDEX transaction_hash_request_id ON transaction_hash(request_id); + CREATE TABLE candles_1min ( start_time timestamptz primary key, diff --git a/migrations/2024-03-27-095235_order_status/down.sql b/migrations/2024-03-27-095235_order_status/down.sql index 17b04aa..142d298 100644 --- a/migrations/2024-03-27-095235_order_status/down.sql +++ b/migrations/2024-03-27-095235_order_status/down.sql @@ -10,4 +10,6 @@ ALTER TYPE order_status DROP VALUE 'HexCodeError'; ALTER TYPE order_status DROP VALUE 'SerializationError'; ALTER TYPE order_status DROP VALUE 'RequestSubmitted'; ALTER TYPE order_status DROP VALUE 'OrderNotFound'; -ALTER TYPE order_status DROP VALUE 'RejectedFromChain'; \ No newline at end of file +ALTER TYPE order_status DROP VALUE 'RejectedFromChain'; +DROP INDEX transaction_hash_account_id; +DROP INDEX transaction_hash_request_id; \ No newline at end of file diff --git a/migrations/2024-03-27-095235_order_status/up.sql b/migrations/2024-03-27-095235_order_status/up.sql index d9705b4..10aa1bb 100644 --- a/migrations/2024-03-27-095235_order_status/up.sql +++ b/migrations/2024-03-27-095235_order_status/up.sql @@ -11,3 +11,5 @@ ALTER TYPE order_status ADD VALUE 'SerializationError'; ALTER TYPE order_status ADD VALUE 'RequestSubmitted'; ALTER TYPE order_status ADD VALUE 'OrderNotFound'; ALTER TYPE order_status ADD VALUE 'RejectedFromChain'; +CREATE INDEX transaction_hash_account_id ON transaction_hash(account_id); +CREATE INDEX transaction_hash_request_id ON transaction_hash(request_id); \ No newline at end of file From 8c281e44ccd108e436fa7932136dbd531c4bf65d Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Thu, 27 Jun 2024 12:53:23 +0000 Subject: [PATCH 51/87] orderbook data correction --- src/database/models.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/database/models.rs b/src/database/models.rs index fcbb197..c5381ae 100644 --- a/src/database/models.rs +++ b/src/database/models.rs @@ -1525,7 +1525,7 @@ impl TraderOrder { FROM trader_order WHERE id IN ( SELECT MAX(id) FROM trader_order - WHERE position_type = 'SHORT' AND uuid IN ( + WHERE position_type = 'LONG' AND uuid IN ( SELECT uuid FROM sorted_Set ) @@ -1594,7 +1594,7 @@ impl TraderOrder { GROUP BY uuid ) AND command IN ('ADD_CLOSE_LIMIT_PRICE', 'UPDATE_CLOSE_LIMIT_PRICE') - AND position_type ='LONG' + AND position_type ='SHORT' ORDER BY id DESC ) SELECT @@ -1606,7 +1606,7 @@ impl TraderOrder { FROM trader_order WHERE id IN ( SELECT MAX(id) FROM trader_order - WHERE position_type = 'LONG' + WHERE position_type = 'SHORT' AND uuid IN ( SELECT uuid FROM sorted_set) GROUP BY uuid ) From c28f448e8e0ec9d84c7e22310f87f18f3ff0c4a9 Mon Sep 17 00:00:00 2001 From: thomas SHARP Date: Mon, 15 Jul 2024 09:46:27 -0400 Subject: [PATCH 52/87] Add redis caching --- Cargo.toml | 1 + src/archiver.rs | 187 +++++++++++++++++++++++++++++++++++++- src/bin/api.rs | 1 + src/bin/archiver.rs | 3 +- src/database/models.rs | 5 +- src/database/sql_types.rs | 33 +++++++ src/error.rs | 2 + src/kafka.rs | 5 - 8 files changed, 228 insertions(+), 9 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 68274ee..d67d7bd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -70,6 +70,7 @@ tracing-subscriber = { version = "0.3.16", features = ["env-filter"] } uuid = { version = "1.3.3", features = ["serde", "v4", "v7"] } verify-keplr-sign = "0.1.0" twilight-relayer-rust = { git = "ssh://git@github.com/twilight-project/twilight-relayer.git", branch = "halt-operation-rpc" } +redis = { version = "0.25.4", features = ["r2d2"] } [dependencies.zkos-relayer-wallet] git = "ssh://git@github.com/twilight-project/zkos-relayer-wallet.git" diff --git a/src/archiver.rs b/src/archiver.rs index fcf9026..3d89a3c 100644 --- a/src/archiver.rs +++ b/src/archiver.rs @@ -5,7 +5,12 @@ use diesel::prelude::PgConnection; use diesel::r2d2::ConnectionManager; use log::{debug, error, info, trace}; use r2d2::PooledConnection; -use std::time::{Duration, Instant}; +use redis::Client; +use serde_json::json; +use std::{ + collections::HashMap, + time::{Duration, Instant}, +}; use twilight_relayer_rust::{ db::{self as relayer_db, Event}, relayer, @@ -15,12 +20,55 @@ const BATCH_INTERVAL: u64 = 100; const BATCH_SIZE: usize = 2_000; const MAX_RETRIES: usize = 5; const RETRY_SLEEP: u64 = 2000; +const PIPELINE_CHUNK: usize = 512; type ManagedConnection = ConnectionManager; type ManagedPool = r2d2::Pool; +const UPDATE_FN: &str = r#" + -- args: + local id = ARGV[1] + local status = ARGV[2] + local side = ARGV[3] + local price = tonumber(ARGV[4]) + local size = tonumber(ARGV[5]) + local time = tonumber(ARGV[6]) + + if status == "FILLED" or status == "SETTLED" or status == "LIQUIDATE" then + redis.call('HDEL', 'orders', id) + + local table = { order_id = id, side = side, price = price, positionsize = size, timestamp = time } + + local order_json = cjson.encode(table) + redis.call('ZADD', 'recent_orders', price, order_json) + + local result = tonumber(redis.pcall('ZRANGEBYSCORE', side, price, price)[1]) + local new_size = result - size + + redis.call('ZREM', side, result) + redis.call('ZADD', side, price, new_size) + + return + end + + -- just opened a new order + if status == "PENDING" then + redis.call('HSET', 'orders', id, price) + + local result = tonumber(redis.pcall('ZRANGEBYSCORE', side, price, price)[1]) + local new_size = result + size + + redis.call('ZREM', side, result) + redis.call('ZADD', side, price, new_size) + end + + -- TODO: clean out expired > 24h... +"#; + pub struct DatabaseArchiver { + redis: Client, pool: ManagedPool, + script_sha: String, trader_orders: Vec, lend_orders: Vec, position_size: Vec, @@ -34,9 +82,14 @@ pub struct DatabaseArchiver { impl DatabaseArchiver { /// Start an archiver, provided a postgres connection string. - pub fn from_host(database_url: String, completions: Sender) -> DatabaseArchiver { + pub fn from_host( + database_url: &str, + redis_url: &str, + completions: Sender, + ) -> DatabaseArchiver { let manager = ConnectionManager::::new(database_url); let pool = r2d2::Pool::new(manager).expect("Could not instantiate connection pool"); + let redis = Client::open(redis_url).expect("Could not establish redis connection"); let mut conn = pool.get().expect("Could not get pooled connection!"); @@ -51,8 +104,19 @@ impl DatabaseArchiver { let lend_pool_commands = Vec::with_capacity(BATCH_SIZE); let nonce = Nonce::get(&mut conn).expect("Failed to query for current nonce"); + Self::load_cache(pool.clone(), redis.clone()); + + let mut redis_conn = redis.get_connection().expect("Redis connection"); + let script_sha: String = redis::cmd("SCRIPT") + .arg("LOAD") + .arg(UPDATE_FN) + .query(&mut redis_conn) + .expect("Script load failed"); + DatabaseArchiver { + redis, pool, + script_sha, trader_orders, lend_orders, position_size, @@ -65,6 +129,123 @@ impl DatabaseArchiver { } } + fn load_cache(pool: ManagedPool, redis: Client) { + let mut conn = pool.get().expect("Could not get pooled connection!"); + + let recent_orders = + TraderOrder::list_past_24hrs(&mut conn).expect("Failed to load recent orders"); + let order_book = TraderOrder::order_book(&mut conn).expect("Failed to load order book"); + + for chunk in recent_orders.chunks(PIPELINE_CHUNK) { + let mut redis_conn = redis + .get_connection() + .expect("Failed to acquire redis connection"); + let mut pipe = redis::pipe(); + + for item in chunk { + let timestamp = item.timestamp.timestamp_millis(); + + let order = json!({ + "order_id": item.order_id.clone(), + "side": item.side.to_string(), + "price": item.price, + "positionsize": item.positionsize, + "timestamp": item.timestamp.timestamp_millis(), + }); + + let order = serde_json::to_string(&order).expect("Invalid JSON"); + + let mut cmd = redis::cmd("ZADD"); + cmd.arg("recent_orders").arg(timestamp).arg(order); + pipe.add_command(cmd); + } + + pipe.atomic().execute(&mut redis_conn); + } + + let OrderBook { bid, ask } = order_book; + + let mut redis_conn = redis + .get_connection() + .expect("Failed to acquire redis connection"); + let mut bids = HashMap::new(); + for bid in bid.iter() { + let key = (bid.price * 100.0) as i64; + bids.entry(key) + .and_modify(|size| *size += bid.positionsize) + .or_insert(bid.positionsize); + + redis::cmd("HSET") + .arg("orders") + .arg("id") + .arg(bid.id.clone()) + .arg("price") + .arg(bid.price) + .execute(&mut redis_conn); + } + + for (price, size) in bids.into_iter() { + redis::cmd("ZADD") + .arg("bid") + .arg(price) + .arg(size) + .execute(&mut redis_conn); + } + + let mut asks = HashMap::new(); + for ask in ask.iter() { + let key = (ask.price * 100.0) as i64; + asks.entry(key) + .and_modify(|size| *size += ask.positionsize) + .or_insert(ask.positionsize); + + redis::cmd("HSET") + .arg("orders") + .arg("id") + .arg(ask.id.clone()) + .arg("price") + .arg(ask.price) + .execute(&mut redis_conn); + } + + for (price, size) in asks.into_iter() { + redis::cmd("ZADD") + .arg("ask") + .arg(price) + .arg(size) + .execute(&mut redis_conn); + } + } + + fn update_sorted_set(&self, cmd: &relayer::SortedSetCommand) -> Result<(), ApiError> { + Ok(()) + } + + fn update_order_cache(&self, order: &InsertTraderOrder) -> Result<(), ApiError> { + let mut pipe = redis::pipe(); + + let side = match order.position_type { + PositionType::LONG => "bid", + PositionType::SHORT => "ask", + }; + + if order.order_status == OrderStatus::FILLED { println!("J"); } + + let mut cmd = redis::cmd("EVALSHA"); + cmd.arg(&self.script_sha) + .arg(order.uuid.clone()) + .arg(order.order_status.as_str()) + .arg(side) + .arg(order.entryprice.to_string()) + .arg(order.positionsize.to_string()) + .arg(order.timestamp.timestamp_millis()); + + pipe.add_command(cmd); + let mut redis_conn = self.redis.get_connection()?; + pipe.atomic().execute(&mut redis_conn); + Ok(()) + } + /// Fetch a connection, will retry MAX_RETRIES before giving up. fn get_conn(&self) -> Result, ApiError> { let mut retries = MAX_RETRIES; @@ -95,6 +276,7 @@ impl DatabaseArchiver { sorted_set_update: relayer::SortedSetCommand, ) -> Result<(), ApiError> { debug!("Appending sorted set update"); + let _ = self.update_sorted_set(&sorted_set_update); self.sorted_set.push(sorted_set_update); if self.sorted_set.len() == self.sorted_set.capacity() { @@ -179,6 +361,7 @@ impl DatabaseArchiver { /// queue. fn trader_order(&mut self, order: InsertTraderOrder) -> Result<(), ApiError> { debug!("Appending trader order"); + let _ = self.update_order_cache(&order); self.trader_orders.push(order); if self.trader_orders.len() == self.trader_orders.capacity() { diff --git a/src/bin/api.rs b/src/bin/api.rs index 4cb96e7..919c0b0 100644 --- a/src/bin/api.rs +++ b/src/bin/api.rs @@ -45,6 +45,7 @@ async fn main() { .init(); let database_url = std::env::var("DATABASE_URL").expect("No database url found!"); + let redis_url = std::env::var("ORDERBOOK_REDIS").expect("No redis url found!"); info!("Database backend: {}", database_url); let cors = CorsLayer::new() diff --git a/src/bin/archiver.rs b/src/bin/archiver.rs index dafc92b..abfdca3 100644 --- a/src/bin/archiver.rs +++ b/src/bin/archiver.rs @@ -18,12 +18,13 @@ fn main() { } let database_url = std::env::var("DATABASE_URL").expect("No database url found!"); + let redis_url = std::env::var("ORDERBOOK_REDIS").expect("No redis url found!"); let (tx, rx) = unbounded(); let (completions, _handle) = kafka::start_consumer(ARCHIVER_GROUP.into(), SNAPSHOT_TOPIC.into(), tx); - let database_worker = DatabaseArchiver::from_host(database_url, completions); + let database_worker = DatabaseArchiver::from_host(&database_url, &redis_url, completions); database_worker .run(rx) diff --git a/src/database/models.rs b/src/database/models.rs index c5381ae..3ae0fa5 100644 --- a/src/database/models.rs +++ b/src/database/models.rs @@ -1150,6 +1150,8 @@ pub struct TraderOrder { #[derive(Serialize, Deserialize, Debug, Clone, QueryableByName, Queryable)] pub struct RecentOrder { + #[diesel(sql_type = diesel::sql_types::Text)] + pub order_id: String, #[diesel(sql_type = crate::database::schema::sql_types::PositionType)] pub side: PositionType, #[diesel(sql_type = diesel::sql_types::Numeric)] @@ -1514,7 +1516,7 @@ impl TraderOrder { GROUP BY uuid ) AND command IN ('ADD_CLOSE_LIMIT_PRICE', 'UPDATE_CLOSE_LIMIT_PRICE') - AND position_type ='LONG' ORDER BY id DESC + AND position_type ='SHORT' ORDER BY id DESC ) SELECT trader_o.uuid AS uuid, @@ -1678,6 +1680,7 @@ impl TraderOrder { // use crate::database::schema::trader_order::dsl::*; let query = r#"Select * from (SELECT + trader_order.uuid as order_id, trader_order.position_type as side, trader_order.entryprice as price, trader_order.positionsize as positionsize, diff --git a/src/database/sql_types.rs b/src/database/sql_types.rs index 398ae42..bb27a62 100644 --- a/src/database/sql_types.rs +++ b/src/database/sql_types.rs @@ -5,6 +5,7 @@ use crate::database::schema::sql_types::{ OrderType as OrderTypeSql, PositionSizeCommand as PositionSizeCommandSql, PositionType as PositionTypeSql, SortedSetCommandType as SortedSetCommandTypeSql, }; +use core::fmt::Display; use diesel::*; use diesel::{ deserialize::FromSql, @@ -101,6 +102,29 @@ pub enum OrderStatus { } impl OrderStatus { + pub fn as_str(&self) -> &'static str { + use OrderStatus::*; + + match self { + SETTLED => "SETTLED", + LENDED => "LENDED", + LIQUIDATE => "LIQUIDATE", + CANCELLED => "CANCELLED", + PENDING => "PENDING", + FILLED => "FILLED", + DuplicateOrder => "DuplicateOrder", + UtxoError => "UtxoError", + Error => "Error", + NoResponseFromChain => "NoResponseFromChain", + BincodeError => "BincodeError", + HexCodeError => "HexCodeError", + SerializationError => "SerializationError", + RequestSubmitted => "RequestSubmitted", + OrderNotFound => "OrderNotFound", + RejectedFromChain => "RejectedFromChain", + } + } + pub fn is_cancelable(&self) -> bool { use OrderStatus::*; @@ -353,6 +377,15 @@ impl FromSql for PositionType { } } +impl Display for PositionType { + fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { + match self { + PositionType::LONG => write!(fmt, "LONG"), + PositionType::SHORT => write!(fmt, "SHORT"), + } + } +} + impl From for OrderStatus { fn from(status: relayer_types::OrderStatus) -> OrderStatus { match status { diff --git a/src/error.rs b/src/error.rs index 79b6d72..8ec4659 100644 --- a/src/error.rs +++ b/src/error.rs @@ -12,4 +12,6 @@ pub enum ApiError { JsonError(#[from] serde_json::Error), #[error("Connection pool error {0:?}")] R2d2(#[from] r2d2::Error), + #[error("Redis error {0:?}")] + Redis(#[from] redis::RedisError), } diff --git a/src/kafka.rs b/src/kafka.rs index 7692d0c..9b41b11 100644 --- a/src/kafka.rs +++ b/src/kafka.rs @@ -33,11 +33,6 @@ pub fn start_consumer( while connection_status { let sender_clone = tx.clone(); let mss = con.poll().unwrap(); - let latest = con - .client_mut() - .fetch_topic_offsets(&topic, kafka::client::FetchOffset::Latest) - .unwrap(); - let latest = latest[0].offset; if !mss.is_empty() { for ms in mss.iter() { let mut max_offset = 0i64; From 1612a96b8c7da256191153a6e663f5073d9a3b43 Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Wed, 17 Jul 2024 07:08:56 +0000 Subject: [PATCH 53/87] event type added --- src/archiver.rs | 1 + src/ws.rs | 1 + 2 files changed, 2 insertions(+) diff --git a/src/archiver.rs b/src/archiver.rs index fcf9026..a5d07e7 100644 --- a/src/archiver.rs +++ b/src/archiver.rs @@ -403,6 +403,7 @@ impl DatabaseArchiver { }; self.tx_hash(hash)?; } + Event::AdvanceStateQueue(_, _) => {} } Ok(()) diff --git a/src/ws.rs b/src/ws.rs index aff5cbb..5b72ff5 100644 --- a/src/ws.rs +++ b/src/ws.rs @@ -112,6 +112,7 @@ impl WsContext { Event::Stop(_stop) => {} Event::TxHash(..) => {} Event::TxHashUpdate(..) => {} + Event::AdvanceStateQueue(..) => {} } } if let Err(e) = notify.send(completion) { From 33ce2ba02db0f84026c185c493cad6be4047163e Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Wed, 17 Jul 2024 08:10:23 +0000 Subject: [PATCH 54/87] query bug fixed --- src/database/models.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/database/models.rs b/src/database/models.rs index 3ae0fa5..763372d 100644 --- a/src/database/models.rs +++ b/src/database/models.rs @@ -1697,6 +1697,7 @@ impl TraderOrder { SELECT + trader_order.uuid as order_id, (CASE WHEN trader_order.position_type = 'LONG' THEN position_type('SHORT') ELSE position_type('LONG') END) as side , From 542afef1f846f928fbfabc43e17ac42812911bd9 Mon Sep 17 00:00:00 2001 From: thomas SHARP Date: Wed, 24 Jul 2024 19:11:53 -0400 Subject: [PATCH 55/87] Redis updates --- src/archiver.rs | 5 +-- src/bin/api.rs | 7 ++-- src/database/models.rs | 17 +++++---- src/rpc.rs | 23 +++++++++--- src/rpc/public_methods.rs | 30 ++++++++-------- src/rpc/util.rs | 73 +++++++++++++++++++++++++++++++++++++++ src/ws.rs | 11 +++--- src/ws/methods.rs | 7 ++-- 8 files changed, 132 insertions(+), 41 deletions(-) create mode 100644 src/rpc/util.rs diff --git a/src/archiver.rs b/src/archiver.rs index b72d87a..f78f3ac 100644 --- a/src/archiver.rs +++ b/src/archiver.rs @@ -150,7 +150,7 @@ impl DatabaseArchiver { "side": item.side.to_string(), "price": item.price, "positionsize": item.positionsize, - "timestamp": item.timestamp.timestamp_millis(), + "timestamp": item.timestamp.to_rfc3339(), }); let order = serde_json::to_string(&order).expect("Invalid JSON"); @@ -229,8 +229,6 @@ impl DatabaseArchiver { PositionType::SHORT => "ask", }; - if order.order_status == OrderStatus::FILLED { println!("J"); } - let mut cmd = redis::cmd("EVALSHA"); cmd.arg(&self.script_sha) .arg(order.uuid.clone()) @@ -586,7 +584,6 @@ impl DatabaseArchiver { }; self.tx_hash(hash)?; } - Event::AdvanceStateQueue(_, _) => {} } Ok(()) diff --git a/src/bin/api.rs b/src/bin/api.rs index 919c0b0..a2107a8 100644 --- a/src/bin/api.rs +++ b/src/bin/api.rs @@ -1,5 +1,6 @@ use jsonrpsee::server::ServerBuilder; use log::info; +use redis::Client; use relayerarchiverlib::{rpc, ws}; use std::{net::SocketAddr, time::Duration}; use structopt::StructOpt; @@ -67,7 +68,7 @@ async fn main() { .await .expect("Failed to build public API server"); - let methods = rpc::init_public_methods(&database_url); + let methods = rpc::init_public_methods(&database_url, &redis_url); let _pub_handle = public_server .start(methods) .expect("Failed to start API server"); @@ -81,7 +82,7 @@ async fn main() { .await .expect("Failed to build private API server"); - let methods = rpc::init_private_methods(&database_url); + let methods = rpc::init_private_methods(&database_url, &redis_url); let _priv_handle = private_server .start(methods) .expect("Failed to start API server"); @@ -93,7 +94,7 @@ async fn main() { .await .expect("Failed to build websocket server"); - let ws_methods = ws::init_methods(&database_url); + let ws_methods = ws::init_methods(&database_url, &redis_url); let _ws_handle = ws_server .start(ws_methods) .expect("Failed to start websocket server"); diff --git a/src/database/models.rs b/src/database/models.rs index 763372d..7041533 100644 --- a/src/database/models.rs +++ b/src/database/models.rs @@ -1679,7 +1679,7 @@ impl TraderOrder { pub fn list_past_24hrs(conn: &mut PgConnection) -> QueryResult> { // use crate::database::schema::trader_order::dsl::*; - let query = r#"Select * from (SELECT + let query = r#"SELECT * FROM (SELECT trader_order.uuid as order_id, trader_order.position_type as side, trader_order.entryprice as price, @@ -1694,14 +1694,14 @@ impl TraderOrder { ON trader_order.uuid = t.uuid AND trader_order.timestamp = t.timestamp UNION ALL - SELECT - trader_order.uuid as order_id, - (CASE WHEN trader_order.position_type = 'LONG' THEN position_type('SHORT') - ELSE position_type('LONG') - END) as side , - + trader_order.uuid as order_id, + ( + CASE WHEN trader_order.position_type = 'LONG' THEN position_type('SHORT') + ELSE position_type('LONG') + END + ) as side, trader_order.settlement_price as price, trader_order.positionsize as positionsize, trader_order.timestamp as timestamp @@ -1716,8 +1716,7 @@ impl TraderOrder { ) as t ON trader_order.uuid = t.uuid AND trader_order.timestamp = t.timestamp - - order by timestamp desc ) as recent_order order by timestamp desc limit 50 + order by timestamp desc ) as recent_order order by timestamp desc limit 50 "#; diesel::sql_query(query).load(conn) diff --git a/src/rpc.rs b/src/rpc.rs index 6fc8dce..447f327 100644 --- a/src/rpc.rs +++ b/src/rpc.rs @@ -2,6 +2,7 @@ use diesel::prelude::PgConnection; use diesel::r2d2::ConnectionManager; use jsonrpsee::{core::error::Error, server::logger::Params, RpcModule}; use kafka::producer::{Producer, RequiredAcks}; +use redis::Client; use serde::Serialize; use std::sync::{Arc, Mutex}; use tokio::time::Duration; @@ -9,10 +10,13 @@ use tokio::time::Duration; mod private_methods; mod public_methods; mod types; +mod util; + pub use types::{ CandleSubscription, Candles, HistoricalFundingArgs, HistoricalPriceArgs, Interval, Order, OrderHistoryArgs, OrderId, PnlArgs, RpcArgs, TradeVolumeArgs, TransactionHashArgs, }; +pub use util::{order_book, recent_orders}; type ManagedConnection = ConnectionManager; type ManagedPool = r2d2::Pool; @@ -22,6 +26,7 @@ type HandlerType = pub struct RelayerContext { pub pool: ManagedPool, + pub client: Client, pub kafka: Arc>, } @@ -35,9 +40,10 @@ fn register_method( } } -pub fn init_public_methods(database_url: &str) -> RpcModule { +pub fn init_public_methods(database_url: &str, redis_url: &str) -> RpcModule { let manager = ConnectionManager::::new(database_url); let pool = r2d2::Pool::new(manager).expect("Could not instantiate connection pool"); + let client = Client::open(redis_url).expect("Could not establish redis connection"); let broker_host = std::env::var("BROKER").expect("missing environment variable BROKER"); let broker = vec![broker_host.to_owned()]; @@ -48,7 +54,11 @@ pub fn init_public_methods(database_url: &str) -> RpcModule { .unwrap(); let kafka = Arc::new(Mutex::new(kafka)); - let mut module = RpcModule::new(RelayerContext { pool, kafka }); + let mut module = RpcModule::new(RelayerContext { + client, + pool, + kafka, + }); register_method( &mut module, "btc_usd_price", @@ -137,9 +147,10 @@ pub fn init_public_methods(database_url: &str) -> RpcModule { module } -pub fn init_private_methods(database_url: &str) -> RpcModule { +pub fn init_private_methods(database_url: &str, redis_url: &str) -> RpcModule { let manager = ConnectionManager::::new(database_url); let pool = r2d2::Pool::new(manager).expect("Could not instantiate connection pool"); + let client = Client::open(redis_url).expect("Could not establish redis connection"); let broker_host = std::env::var("BROKER").expect("missing environment variable BROKER"); let broker = vec![broker_host.to_owned()]; @@ -150,7 +161,11 @@ pub fn init_private_methods(database_url: &str) -> RpcModule { .unwrap(); let kafka = Arc::new(Mutex::new(kafka)); - let mut module = RpcModule::new(RelayerContext { pool, kafka }); + let mut module = RpcModule::new(RelayerContext { + client, + pool, + kafka, + }); register_method( &mut module, diff --git a/src/rpc/public_methods.rs b/src/rpc/public_methods.rs index 52c7a36..6c2f4e8 100644 --- a/src/rpc/public_methods.rs +++ b/src/rpc/public_methods.rs @@ -1,6 +1,7 @@ use super::*; use crate::database::*; use chrono::prelude::*; +use itertools::Itertools; use jsonrpsee::{core::error::Error, server::logger::Params}; use kafka::producer::Record; use relayerwalletlib::verify_client_message::{ @@ -9,6 +10,7 @@ use relayerwalletlib::verify_client_message::{ }; use twilight_relayer_rust::relayer; use zkoswalletlib::relayer_rpcclient::method::RequestResponse; + pub(super) fn btc_usd_price( _: Params<'_>, ctx: &RelayerContext, @@ -97,26 +99,26 @@ pub(super) fn open_limit_orders( _: Params<'_>, ctx: &RelayerContext, ) -> Result { - match ctx.pool.get() { - Ok(mut conn) => match TraderOrder::order_book(&mut conn) { - Ok(o) => Ok(serde_json::to_value(o).expect("Error converting response")), - Err(e) => Err(Error::Custom(format!("Error fetching order info: {:?}", e))), - }, - Err(e) => Err(Error::Custom(format!("Database error: {:?}", e))), - } + let Ok(mut conn) = ctx.client.get_connection() else { + return Ok("Redis connection error.".into()); + }; + + let book = order_book(&mut conn); + + Ok(serde_json::to_value(book).expect("Failed to serialize order book")) } pub(super) fn recent_trade_orders( _: Params<'_>, ctx: &RelayerContext, ) -> Result { - match ctx.pool.get() { - Ok(mut conn) => match TraderOrder::list_past_24hrs(&mut conn) { - Ok(o) => Ok(serde_json::to_value(o).expect("Error converting response")), - Err(e) => Err(Error::Custom(format!("Error fetching order info: {:?}", e))), - }, - Err(e) => Err(Error::Custom(format!("Database error: {:?}", e))), - } + let Ok(mut conn) = ctx.client.get_connection() else { + return Ok("Redis connection error.".into()); + }; + + let orders = recent_orders(&mut conn); + + Ok(serde_json::to_value(&orders).expect("Failed to serialize recent orders")) } pub(super) fn position_size( diff --git a/src/rpc/util.rs b/src/rpc/util.rs new file mode 100644 index 0000000..9868106 --- /dev/null +++ b/src/rpc/util.rs @@ -0,0 +1,73 @@ +use crate::database::{Ask, Bid, OrderBook, RecentOrder}; +use chrono::{TimeDelta, Utc}; +use itertools::Itertools; + +const BOOK_LIMIT: usize = 10; + +pub fn order_book(conn: &mut redis::Connection) -> OrderBook { + let asks: redis::Iter = redis::cmd("ZSCAN") + .arg("ask") + .cursor_arg(0) + .clone() + .iter(conn) + .unwrap(); + + let ask: Vec<_> = asks + .chunks(2) + .into_iter() + .take(BOOK_LIMIT) + .map(|mut chunk| { + let price = chunk.next().unwrap(); + let positionsize = chunk.next().unwrap(); + + Ask { + id: "".into(), + positionsize, + price, + } + }) + .collect(); + + let bids: redis::Iter = redis::cmd("ZSCAN") + .arg("bid") + .cursor_arg(0) + .clone() + .iter(conn) + .unwrap(); + + let bids: Vec<_> = bids.collect(); + let bid: Vec<_> = bids + .chunks(2) + .rev() + .into_iter() + .take(BOOK_LIMIT) + .map(|chunk| { + let price = chunk[0]; + let positionsize = chunk[1]; + + Bid { + id: "".into(), + positionsize, + price, + } + }) + .collect(); + + OrderBook { ask, bid } +} + +pub fn recent_orders(conn: &mut redis::Connection) -> Vec { + let max = Utc::now(); + let min = max - TimeDelta::days(1); + + let orders: Vec = redis::cmd("ZRANGEBYSCORE") + .arg("recent_orders") + .arg(min.timestamp_millis()) + .arg(max.timestamp_millis()) + .query(conn) + .unwrap(); + + orders.into_iter().map(|order| { + serde_json::from_str(&order).expect("Invalid recent order!") + }).collect() +} diff --git a/src/ws.rs b/src/ws.rs index 5b72ff5..6e42d04 100644 --- a/src/ws.rs +++ b/src/ws.rs @@ -8,6 +8,7 @@ use diesel::prelude::PgConnection; use diesel::r2d2::ConnectionManager; use jsonrpsee::RpcModule; use log::{error, info, trace}; +use redis::Client; use relayerwalletlib::zkoswalletlib::relayer_types::{OrderStatus, OrderType}; // use serde::{Deserialize, Serialize}; use std::{ @@ -33,6 +34,7 @@ type ManagedConnection = ConnectionManager; type ManagedPool = r2d2::Pool; pub struct WsContext { + client: Client, price_feed: Sender<(f64, DateTime)>, order_book: Sender, recent_trades: Sender, @@ -44,7 +46,7 @@ pub struct WsContext { } impl WsContext { - pub fn with_pool(pool: ManagedPool) -> WsContext { + pub fn with_pool(pool: ManagedPool, client: Client) -> WsContext { let (price_feed, _) = channel::<(f64, DateTime)>(BROADCAST_CHANNEL_CAPACITY); let (order_book, _) = channel::(BROADCAST_CHANNEL_CAPACITY); let (recent_trades, _) = channel::(BROADCAST_CHANNEL_CAPACITY); @@ -112,7 +114,6 @@ impl WsContext { Event::Stop(_stop) => {} Event::TxHash(..) => {} Event::TxHashUpdate(..) => {} - Event::AdvanceStateQueue(..) => {} } } if let Err(e) = notify.send(completion) { @@ -135,6 +136,7 @@ impl WsContext { }); WsContext { + client, price_feed, order_book, recent_trades, @@ -147,14 +149,15 @@ impl WsContext { } } -pub fn init_methods(database_url: &str) -> RpcModule { +pub fn init_methods(database_url: &str, redis_url: &str) -> RpcModule { let manager = ConnectionManager::::new(database_url); let pool = r2d2::Pool::builder() .max_size(50) .build(manager) .expect("Could not instantiate connection pool"); + let client = Client::open(redis_url).expect("Could not establish redis connection"); - let mut module = RpcModule::new(WsContext::with_pool(pool)); + let mut module = RpcModule::new(WsContext::with_pool(pool, client)); module .register_subscription( diff --git a/src/ws/methods.rs b/src/ws/methods.rs index 387416a..53c6c97 100644 --- a/src/ws/methods.rs +++ b/src/ws/methods.rs @@ -1,8 +1,8 @@ #![allow(warnings)] use crate::{ - database::{BtcUsdPrice, TraderOrder}, + database::{Ask, Bid, BtcUsdPrice, OrderBook, TraderOrder}, error::ApiError, - rpc::{CandleSubscription, Interval}, + rpc::{order_book, CandleSubscription, Interval}, }; use chrono::prelude::*; use jsonrpsee::{ @@ -146,7 +146,8 @@ pub(super) fn spawn_order_book( match rx.try_recv() { Ok(_mesg) => { let mut conn = ctx.pool.get()?; - let orders = TraderOrder::order_book(&mut conn)?; + let mut redis_conn = ctx.client.get_connection().expect("REDIS connection."); + let orders = order_book(&mut redis_conn); let result = serde_json::to_value(&orders)?; if let Err(e) = sink.send(&result) { From 156a6b53985bcba088ee59f1cf1c09fae7a6e0d1 Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Fri, 26 Jul 2024 20:01:05 +0000 Subject: [PATCH 56/87] funding updates seperated from trader order table --- .../2024-03-27-095235_order_status/down.sql | 1 + .../2024-03-27-095235_order_status/up.sql | 1 + .../down.sql | 10 + .../up.sql | 51 +++++ src/archiver.rs | 38 +++- src/database/models.rs | 180 +++++++++++++++++- src/database/schema.rs | 34 ++++ src/database/sql_types.rs | 4 + 8 files changed, 317 insertions(+), 2 deletions(-) create mode 100644 migrations/2024-07-26-163222_funding_updates_table/down.sql create mode 100644 migrations/2024-07-26-163222_funding_updates_table/up.sql diff --git a/migrations/2024-03-27-095235_order_status/down.sql b/migrations/2024-03-27-095235_order_status/down.sql index 142d298..dd6fc44 100644 --- a/migrations/2024-03-27-095235_order_status/down.sql +++ b/migrations/2024-03-27-095235_order_status/down.sql @@ -11,5 +11,6 @@ ALTER TYPE order_status DROP VALUE 'SerializationError'; ALTER TYPE order_status DROP VALUE 'RequestSubmitted'; ALTER TYPE order_status DROP VALUE 'OrderNotFound'; ALTER TYPE order_status DROP VALUE 'RejectedFromChain'; +ALTER TYPE order_status DROP VALUE 'FilledUpdated'; DROP INDEX transaction_hash_account_id; DROP INDEX transaction_hash_request_id; \ No newline at end of file diff --git a/migrations/2024-03-27-095235_order_status/up.sql b/migrations/2024-03-27-095235_order_status/up.sql index 10aa1bb..d86d3b5 100644 --- a/migrations/2024-03-27-095235_order_status/up.sql +++ b/migrations/2024-03-27-095235_order_status/up.sql @@ -11,5 +11,6 @@ ALTER TYPE order_status ADD VALUE 'SerializationError'; ALTER TYPE order_status ADD VALUE 'RequestSubmitted'; ALTER TYPE order_status ADD VALUE 'OrderNotFound'; ALTER TYPE order_status ADD VALUE 'RejectedFromChain'; +ALTER TYPE order_status ADD VALUE 'FilledUpdated'; CREATE INDEX transaction_hash_account_id ON transaction_hash(account_id); CREATE INDEX transaction_hash_request_id ON transaction_hash(request_id); \ No newline at end of file diff --git a/migrations/2024-07-26-163222_funding_updates_table/down.sql b/migrations/2024-07-26-163222_funding_updates_table/down.sql new file mode 100644 index 0000000..1eccb8e --- /dev/null +++ b/migrations/2024-07-26-163222_funding_updates_table/down.sql @@ -0,0 +1,10 @@ +-- This file should undo anything in `up.sql` +DROP TABLE IF EXISTS public.trader_order_funding_updated; + +-- Index: timestamp_trader_order_funding_updated + +DROP INDEX IF EXISTS public.timestamp_trader_order_funding_updated; + +-- Index: trader_order_funding_updated_uuid + +DROP INDEX IF EXISTS public.trader_order_funding_updated_uuid; \ No newline at end of file diff --git a/migrations/2024-07-26-163222_funding_updates_table/up.sql b/migrations/2024-07-26-163222_funding_updates_table/up.sql new file mode 100644 index 0000000..5c72c0a --- /dev/null +++ b/migrations/2024-07-26-163222_funding_updates_table/up.sql @@ -0,0 +1,51 @@ +-- Your SQL goes here + +-- DROP TABLE IF EXISTS public.trader_order_funding_updated; + +CREATE TABLE IF NOT EXISTS public.trader_order_funding_updated +( + id bigint NOT NULL DEFAULT nextval('trader_order_id_seq'::regclass), + uuid character varying(64) COLLATE pg_catalog."default" NOT NULL, + account_id character varying COLLATE pg_catalog."default" NOT NULL, + position_type position_type NOT NULL, + order_status order_status NOT NULL, + order_type order_type NOT NULL, + entryprice numeric NOT NULL, + execution_price numeric NOT NULL, + positionsize numeric NOT NULL, + leverage numeric NOT NULL, + initial_margin numeric NOT NULL, + available_margin numeric NOT NULL, + "timestamp" timestamp with time zone NOT NULL, + bankruptcy_price numeric NOT NULL, + bankruptcy_value numeric NOT NULL, + maintenance_margin numeric NOT NULL, + liquidation_price numeric NOT NULL, + unrealized_pnl numeric NOT NULL, + settlement_price numeric NOT NULL, + entry_nonce bigint NOT NULL, + exit_nonce bigint NOT NULL, + entry_sequence bigint NOT NULL, + CONSTRAINT trader_order_funding_updated_pkey PRIMARY KEY (id) +) + +TABLESPACE pg_default; + +ALTER TABLE IF EXISTS public.trader_order_funding_updated + OWNER to relayer; +-- Index: timestamp_trader_order_funding_updated + +-- DROP INDEX IF EXISTS public.timestamp_trader_order_funding_updated; + +CREATE INDEX IF NOT EXISTS timestamp_trader_order_funding_updated + ON public.trader_order_funding_updated USING btree + ("timestamp" ASC NULLS LAST) + TABLESPACE pg_default; +-- Index: trader_order_funding_updated_uuid + +-- DROP INDEX IF EXISTS public.trader_order_funding_updated_uuid; + +CREATE INDEX IF NOT EXISTS trader_order_funding_updated_uuid + ON public.trader_order_funding_updated USING btree + (uuid COLLATE pg_catalog."default" ASC NULLS LAST) + TABLESPACE pg_default; \ No newline at end of file diff --git a/src/archiver.rs b/src/archiver.rs index a5d07e7..a8a4f56 100644 --- a/src/archiver.rs +++ b/src/archiver.rs @@ -22,6 +22,7 @@ type ManagedPool = r2d2::Pool; pub struct DatabaseArchiver { pool: ManagedPool, trader_orders: Vec, + trader_order_funding_updated: Vec, lend_orders: Vec, position_size: Vec, tx_hashes: Vec, @@ -43,6 +44,7 @@ impl DatabaseArchiver { migrations::run_migrations(&mut *conn).expect("Failed to run database migrations!"); let trader_orders = Vec::with_capacity(BATCH_SIZE); + let trader_order_funding_updated = Vec::with_capacity(BATCH_SIZE); let lend_orders = Vec::with_capacity(BATCH_SIZE); let position_size = Vec::with_capacity(BATCH_SIZE); let tx_hashes = Vec::with_capacity(BATCH_SIZE); @@ -54,6 +56,7 @@ impl DatabaseArchiver { DatabaseArchiver { pool, trader_orders, + trader_order_funding_updated, lend_orders, position_size, tx_hashes, @@ -202,6 +205,36 @@ impl DatabaseArchiver { Ok(()) } + /// Add a trader order funidng update to the next update batch, if the queue is full, commit and clear the + /// queue. + fn trader_order_funding_update( + &mut self, + order: InsertTraderOrderFundingUpdates, + ) -> Result<(), ApiError> { + debug!("Appending trader order"); + self.trader_order_funding_updated.push(order); + + if self.trader_order_funding_updated.len() == self.trader_order_funding_updated.capacity() { + self.commit_trader_order_funding_updated()?; + } + + Ok(()) + } + + /// Commit a batch of trader orders funidng update to the database. If we're failing to update the database, we + /// should exit. + fn commit_trader_order_funding_updated(&mut self) -> Result<(), ApiError> { + debug!("Committing trader orders"); + + let mut conn = self.get_conn()?; + + let mut orders = Vec::with_capacity(self.trader_order_funding_updated.capacity()); + std::mem::swap(&mut orders, &mut self.trader_order_funding_updated); + + TraderOrderFundingUpdates::insert(&mut conn, orders)?; + + Ok(()) + } /// Add a lend order to the next update batch, if the queue is full, commit and clear the /// queue. @@ -292,6 +325,9 @@ impl DatabaseArchiver { if self.trader_orders.len() > 0 { self.commit_trader_orders()?; } + if self.trader_order_funding_updated.len() > 0 { + self.commit_trader_order_funding_updated()?; + } if self.lend_orders.len() > 0 { self.commit_lend_orders()?; @@ -329,7 +365,7 @@ impl DatabaseArchiver { self.trader_order(trader_order.into())?; } Event::TraderOrderFundingUpdate(trader_order, _cmd) => { - self.trader_order(trader_order.into())?; + self.trader_order_funding_update(trader_order.into())?; } Event::TraderOrderLiquidation(trader_order, _cmd, _seq) => { self.trader_order(trader_order.into())?; diff --git a/src/database/models.rs b/src/database/models.rs index c5381ae..ea04e21 100644 --- a/src/database/models.rs +++ b/src/database/models.rs @@ -3,7 +3,8 @@ use crate::database::{ schema::{ address_customer_id, btc_usd_price, current_nonce, customer_account, customer_apikey_linking, customer_order_linking, funding_rate, lend_order, lend_pool, - lend_pool_command, position_size_log, sorted_set_command, trader_order, transaction_hash, + lend_pool_command, position_size_log, sorted_set_command, trader_order, + trader_order_funding_updated, transaction_hash, }, sql_types::*, }; @@ -1148,6 +1149,35 @@ pub struct TraderOrder { pub entry_sequence: i64, } +#[derive( + Serialize, Deserialize, Debug, Clone, QueryableByName, Queryable, Insertable, AsChangeset, +)] +#[diesel(table_name = trader_order_funding_updated)] +pub struct TraderOrderFundingUpdates { + pub id: i64, + pub uuid: String, + pub account_id: String, + pub position_type: PositionType, + pub order_status: OrderStatus, + pub order_type: OrderType, + pub entryprice: BigDecimal, + pub execution_price: BigDecimal, + pub positionsize: BigDecimal, + pub leverage: BigDecimal, + pub initial_margin: BigDecimal, + pub available_margin: BigDecimal, + pub timestamp: DateTime, + pub bankruptcy_price: BigDecimal, + pub bankruptcy_value: BigDecimal, + pub maintenance_margin: BigDecimal, + pub liquidation_price: BigDecimal, + pub unrealized_pnl: BigDecimal, + pub settlement_price: BigDecimal, + pub entry_nonce: i64, + pub exit_nonce: i64, + pub entry_sequence: i64, +} + #[derive(Serialize, Deserialize, Debug, Clone, QueryableByName, Queryable)] pub struct RecentOrder { #[diesel(sql_type = crate::database::schema::sql_types::PositionType)] @@ -1192,6 +1222,32 @@ pub struct InsertTraderOrder { pub entry_sequence: i64, } +#[derive(Serialize, Deserialize, Debug, Clone, Queryable, Insertable, AsChangeset)] +#[diesel(table_name = trader_order_funding_updated)] +pub struct InsertTraderOrderFundingUpdates { + pub uuid: String, + pub account_id: String, + pub position_type: PositionType, + pub order_status: OrderStatus, + pub order_type: OrderType, + pub entryprice: BigDecimal, + pub execution_price: BigDecimal, + pub positionsize: BigDecimal, + pub leverage: BigDecimal, + pub initial_margin: BigDecimal, + pub available_margin: BigDecimal, + pub timestamp: DateTime, + pub bankruptcy_price: BigDecimal, + pub bankruptcy_value: BigDecimal, + pub maintenance_margin: BigDecimal, + pub liquidation_price: BigDecimal, + pub unrealized_pnl: BigDecimal, + pub settlement_price: BigDecimal, + pub entry_nonce: i64, + pub exit_nonce: i64, + pub entry_sequence: i64, +} + #[derive( Serialize, Deserialize, Debug, Clone, Queryable, QueryableByName, Insertable, AsChangeset, )] @@ -1720,6 +1776,19 @@ impl TraderOrder { } } +impl TraderOrderFundingUpdates { + pub fn insert( + conn: &mut PgConnection, + orders: Vec, + ) -> QueryResult { + use crate::database::schema::trader_order_funding_updated::dsl::*; + + let query = diesel::insert_into(trader_order_funding_updated).values(&orders); + + query.execute(conn) + } +} + #[derive(Serialize, Deserialize, Debug, Clone)] pub struct OrderBook { pub bid: Vec, @@ -1949,6 +2018,115 @@ impl From for InsertTraderOrder { } } } +impl From for TraderOrderFundingUpdates { + fn from(src: relayer::TraderOrder) -> TraderOrderFundingUpdates { + let relayer::TraderOrder { + uuid, + account_id, + position_type, + order_status, + order_type, + entryprice, + execution_price, + positionsize, + leverage, + initial_margin, + available_margin, + timestamp, + bankruptcy_price, + bankruptcy_value, + maintenance_margin, + liquidation_price, + unrealized_pnl, + settlement_price, + entry_nonce, + exit_nonce, + entry_sequence, + } = src; + + TraderOrderFundingUpdates { + id: 0, + uuid: uuid.to_string(), + account_id, + position_type: position_type.into(), + order_status: order_status.into(), + order_type: order_type.into(), + // TODO: maybe a TryFrom impl instead... + entryprice: BigDecimal::from_f64(entryprice).unwrap().round(2), + execution_price: BigDecimal::from_f64(execution_price).unwrap().round(2), + positionsize: BigDecimal::from_f64(positionsize).unwrap(), + leverage: BigDecimal::from_f64(leverage).unwrap(), + initial_margin: BigDecimal::from_f64(initial_margin).unwrap(), + available_margin: BigDecimal::from_f64(available_margin).unwrap().round(4), + timestamp: DateTime::parse_from_rfc3339(×tamp) + .expect("Bad datetime format") + .into(), + bankruptcy_price: BigDecimal::from_f64(bankruptcy_price).unwrap().round(2), + bankruptcy_value: BigDecimal::from_f64(bankruptcy_value).unwrap().round(4), + maintenance_margin: BigDecimal::from_f64(maintenance_margin).unwrap().round(4), + liquidation_price: BigDecimal::from_f64(liquidation_price).unwrap().round(2), + unrealized_pnl: BigDecimal::from_f64(unrealized_pnl).unwrap().round(2), + settlement_price: BigDecimal::from_f64(settlement_price).unwrap().round(2), + entry_nonce: entry_nonce as i64, + exit_nonce: exit_nonce as i64, + entry_sequence: entry_sequence as i64, + } + } +} +impl From for InsertTraderOrderFundingUpdates { + fn from(src: relayer::TraderOrder) -> InsertTraderOrderFundingUpdates { + let relayer::TraderOrder { + uuid, + account_id, + position_type, + order_status, + order_type, + entryprice, + execution_price, + positionsize, + leverage, + initial_margin, + available_margin, + timestamp, + bankruptcy_price, + bankruptcy_value, + maintenance_margin, + liquidation_price, + unrealized_pnl, + settlement_price, + entry_nonce, + exit_nonce, + entry_sequence, + } = src; + + InsertTraderOrderFundingUpdates { + uuid: uuid.to_string(), + account_id, + position_type: position_type.into(), + order_status: order_status.into(), + order_type: order_type.into(), + // TODO: maybe a TryFrom impl instead... + entryprice: BigDecimal::from_f64(entryprice).unwrap().round(2), + execution_price: BigDecimal::from_f64(execution_price).unwrap().round(2), + positionsize: BigDecimal::from_f64(positionsize).unwrap(), + leverage: BigDecimal::from_f64(leverage).unwrap(), + initial_margin: BigDecimal::from_f64(initial_margin).unwrap(), + available_margin: BigDecimal::from_f64(available_margin).unwrap().round(4), + timestamp: DateTime::parse_from_rfc3339(×tamp) + .expect("Bad datetime format") + .into(), + bankruptcy_price: BigDecimal::from_f64(bankruptcy_price).unwrap().round(2), + bankruptcy_value: BigDecimal::from_f64(bankruptcy_value).unwrap().round(4), + maintenance_margin: BigDecimal::from_f64(maintenance_margin).unwrap().round(4), + liquidation_price: BigDecimal::from_f64(liquidation_price).unwrap().round(2), + unrealized_pnl: BigDecimal::from_f64(unrealized_pnl).unwrap().round(2), + settlement_price: BigDecimal::from_f64(settlement_price).unwrap().round(2), + entry_nonce: entry_nonce as i64, + exit_nonce: exit_nonce as i64, + entry_sequence: entry_sequence as i64, + } + } +} impl From for InsertLendOrder { fn from(src: relayer::LendOrder) -> InsertLendOrder { diff --git a/src/database/schema.rs b/src/database/schema.rs index c31d159..2b0986f 100644 --- a/src/database/schema.rs +++ b/src/database/schema.rs @@ -260,6 +260,39 @@ diesel::table! { } } +diesel::table! { + use diesel::sql_types::*; + use super::sql_types::PositionType; + use super::sql_types::OrderStatus; + use super::sql_types::OrderType; + + trader_order_funding_updated (id) { + id -> Int8, + #[max_length = 64] + uuid -> Varchar, + account_id -> Varchar, + position_type -> PositionType, + order_status -> OrderStatus, + order_type -> OrderType, + entryprice -> Numeric, + execution_price -> Numeric, + positionsize -> Numeric, + leverage -> Numeric, + initial_margin -> Numeric, + available_margin -> Numeric, + timestamp -> Timestamptz, + bankruptcy_price -> Numeric, + bankruptcy_value -> Numeric, + maintenance_margin -> Numeric, + liquidation_price -> Numeric, + unrealized_pnl -> Numeric, + settlement_price -> Numeric, + entry_nonce -> Int8, + exit_nonce -> Int8, + entry_sequence -> Int8, + } +} + diesel::table! { use diesel::sql_types::*; use super::sql_types::OrderType; @@ -299,5 +332,6 @@ diesel::allow_tables_to_appear_in_same_query!( position_size_log, sorted_set_command, trader_order, + trader_order_funding_updated, transaction_hash, ); diff --git a/src/database/sql_types.rs b/src/database/sql_types.rs index 398ae42..3524367 100644 --- a/src/database/sql_types.rs +++ b/src/database/sql_types.rs @@ -98,6 +98,7 @@ pub enum OrderStatus { RequestSubmitted, OrderNotFound, RejectedFromChain, + FilledUpdated, } impl OrderStatus { @@ -260,6 +261,7 @@ impl ToSql for OrderStatus { OrderStatus::RequestSubmitted => out.write_all(b"RequestSubmitted")?, OrderStatus::OrderNotFound => out.write_all(b"OrderNotFound")?, OrderStatus::RejectedFromChain => out.write_all(b"RejectedFromChain")?, + OrderStatus::FilledUpdated => out.write_all(b"FilledUpdated")?, } Ok(IsNull::No) } @@ -304,6 +306,7 @@ impl FromSql for OrderStatus { b"RequestSubmitted" => Ok(OrderStatus::RequestSubmitted), b"OrderNotFound" => Ok(OrderStatus::OrderNotFound), b"RejectedFromChain" => Ok(OrderStatus::RejectedFromChain), + b"FilledUpdated" => Ok(OrderStatus::FilledUpdated), _ => panic!("Invalid enum type in database!"), } } @@ -372,6 +375,7 @@ impl From for OrderStatus { relayer_types::OrderStatus::RequestSubmitted => OrderStatus::RequestSubmitted, relayer_types::OrderStatus::OrderNotFound => OrderStatus::OrderNotFound, relayer_types::OrderStatus::RejectedFromChain => OrderStatus::RejectedFromChain, + relayer_types::OrderStatus::FilledUpdated => OrderStatus::FilledUpdated, } } } From d0c8da594485f6e7a0f5b935f9258243a918dd7d Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Mon, 29 Jul 2024 13:52:54 +0000 Subject: [PATCH 57/87] minor change --- src/archiver.rs | 1 + src/database/sql_types.rs | 1 + src/ws.rs | 1 + 3 files changed, 3 insertions(+) diff --git a/src/archiver.rs b/src/archiver.rs index 2af9375..707042a 100644 --- a/src/archiver.rs +++ b/src/archiver.rs @@ -620,6 +620,7 @@ impl DatabaseArchiver { }; self.tx_hash(hash)?; } + Event::AdvanceStateQueue(_, _) => {} } Ok(()) diff --git a/src/database/sql_types.rs b/src/database/sql_types.rs index 10de8d0..82f2227 100644 --- a/src/database/sql_types.rs +++ b/src/database/sql_types.rs @@ -123,6 +123,7 @@ impl OrderStatus { RequestSubmitted => "RequestSubmitted", OrderNotFound => "OrderNotFound", RejectedFromChain => "RejectedFromChain", + FilledUpdated => "FilledUpdated", } } diff --git a/src/ws.rs b/src/ws.rs index 6e42d04..3413717 100644 --- a/src/ws.rs +++ b/src/ws.rs @@ -114,6 +114,7 @@ impl WsContext { Event::Stop(_stop) => {} Event::TxHash(..) => {} Event::TxHashUpdate(..) => {} + Event::AdvanceStateQueue(..) => {} } } if let Err(e) = notify.send(completion) { From 72da538ced4131b115140ac7a4d68341a4c8bfe1 Mon Sep 17 00:00:00 2001 From: thomas SHARP Date: Mon, 29 Jul 2024 09:00:47 -0400 Subject: [PATCH 58/87] Cache updates --- src/archiver.rs | 59 ++++++++++++++++++--------------------- src/bin/api.rs | 1 - src/database/models.rs | 14 ++++++++++ src/rpc/public_methods.rs | 1 - 4 files changed, 41 insertions(+), 34 deletions(-) diff --git a/src/archiver.rs b/src/archiver.rs index 707042a..5faa39a 100644 --- a/src/archiver.rs +++ b/src/archiver.rs @@ -1,3 +1,4 @@ +use bigdecimal::ToPrimitive; use crate::{database::*, error::ApiError, kafka::Completion, migrations}; use chrono::prelude::*; use crossbeam_channel::{Receiver, Sender}; @@ -26,21 +27,22 @@ type ManagedConnection = ConnectionManager; type ManagedPool = r2d2::Pool; const UPDATE_FN: &str = r#" - -- args: + -- args: local id = ARGV[1] local status = ARGV[2] local side = ARGV[3] local price = tonumber(ARGV[4]) local size = tonumber(ARGV[5]) - local time = tonumber(ARGV[6]) + local timestamp = ARGV[6] + local time = tonumber(ARGV[7]) if status == "FILLED" or status == "SETTLED" or status == "LIQUIDATE" then redis.call('HDEL', 'orders', id) - local table = { order_id = id, side = side, price = price, positionsize = size, timestamp = time } + local table = { order_id = id, side = side, price = price, positionsize = size, timestamp = timestamp } local order_json = cjson.encode(table) - redis.call('ZADD', 'recent_orders', price, order_json) + redis.call('ZADD', 'recent_orders', time, order_json) local result = tonumber(redis.pcall('ZRANGEBYSCORE', side, price, price)[1]) local new_size = result - size @@ -137,7 +139,7 @@ impl DatabaseArchiver { let recent_orders = TraderOrder::list_past_24hrs(&mut conn).expect("Failed to load recent orders"); - let order_book = TraderOrder::order_book(&mut conn).expect("Failed to load order book"); + let order_book_orders = TraderOrder::order_book_orders(&mut conn).expect("Failed to load order book"); for chunk in recent_orders.chunks(PIPELINE_CHUNK) { let mut redis_conn = redis @@ -166,24 +168,32 @@ impl DatabaseArchiver { pipe.atomic().execute(&mut redis_conn); } - let OrderBook { bid, ask } = order_book; - let mut redis_conn = redis .get_connection() .expect("Failed to acquire redis connection"); let mut bids = HashMap::new(); - for bid in bid.iter() { - let key = (bid.price * 100.0) as i64; - bids.entry(key) - .and_modify(|size| *size += bid.positionsize) - .or_insert(bid.positionsize); + let mut asks = HashMap::new(); + + for order in order_book_orders.iter() { + let key = (order.entryprice.to_f64().unwrap() * 100.0) as i64; + let positionsize = order.positionsize.to_f64().unwrap(); + + if order.position_type == PositionType::SHORT { + asks.entry(key) + .and_modify(|size| *size += positionsize) + .or_insert(positionsize); + } else { + bids.entry(key) + .and_modify(|size| *size += positionsize) + .or_insert(positionsize); + } redis::cmd("HSET") .arg("orders") .arg("id") - .arg(bid.id.clone()) + .arg(order.uuid.clone()) .arg("price") - .arg(bid.price) + .arg(order.entryprice.to_f64()) .execute(&mut redis_conn); } @@ -191,23 +201,7 @@ impl DatabaseArchiver { redis::cmd("ZADD") .arg("bid") .arg(price) - .arg(size) - .execute(&mut redis_conn); - } - - let mut asks = HashMap::new(); - for ask in ask.iter() { - let key = (ask.price * 100.0) as i64; - asks.entry(key) - .and_modify(|size| *size += ask.positionsize) - .or_insert(ask.positionsize); - - redis::cmd("HSET") - .arg("orders") - .arg("id") - .arg(ask.id.clone()) - .arg("price") - .arg(ask.price) + .arg(size.to_f64()) .execute(&mut redis_conn); } @@ -215,7 +209,7 @@ impl DatabaseArchiver { redis::cmd("ZADD") .arg("ask") .arg(price) - .arg(size) + .arg(size.to_f64()) .execute(&mut redis_conn); } } @@ -239,6 +233,7 @@ impl DatabaseArchiver { .arg(side) .arg(order.entryprice.to_string()) .arg(order.positionsize.to_string()) + .arg(order.timestamp.to_rfc3339()) .arg(order.timestamp.timestamp_millis()); pipe.add_command(cmd); diff --git a/src/bin/api.rs b/src/bin/api.rs index a2107a8..e9d2610 100644 --- a/src/bin/api.rs +++ b/src/bin/api.rs @@ -1,6 +1,5 @@ use jsonrpsee::server::ServerBuilder; use log::info; -use redis::Client; use relayerarchiverlib::{rpc, ws}; use std::{net::SocketAddr, time::Duration}; use structopt::StructOpt; diff --git a/src/database/models.rs b/src/database/models.rs index c986ae8..95594f3 100644 --- a/src/database/models.rs +++ b/src/database/models.rs @@ -1534,6 +1534,20 @@ impl TraderOrder { Ok(tv.volume.to_f64().unwrap()) } + pub fn order_book_orders(conn: &mut PgConnection) -> QueryResult> { + let query = r#" + SELECT * FROM trader_order + WHERE id IN ( + SELECT MAX(id) FROM trader_order + WHERE order_type = 'LIMIT' + GROUP BY uuid + ) + AND order_status NOT IN ('FILLED', 'CANCELLED', 'LIQUIDATE') + "#; + + diesel::sql_query(query).get_results(conn) + } + pub fn order_book(conn: &mut PgConnection) -> QueryResult { // use crate::database::schema::trader_order::dsl::*; // use diesel::dsl::{max, sum}; diff --git a/src/rpc/public_methods.rs b/src/rpc/public_methods.rs index 6c2f4e8..5149631 100644 --- a/src/rpc/public_methods.rs +++ b/src/rpc/public_methods.rs @@ -1,7 +1,6 @@ use super::*; use crate::database::*; use chrono::prelude::*; -use itertools::Itertools; use jsonrpsee::{core::error::Error, server::logger::Params}; use kafka::producer::Record; use relayerwalletlib::verify_client_message::{ From 6f5103e143f1f511af06ded41c88192de613f48d Mon Sep 17 00:00:00 2001 From: thomas SHARP Date: Mon, 5 Aug 2024 09:12:46 -0400 Subject: [PATCH 59/87] Update types --- src/archiver.rs | 23 +++++++++++++---------- src/database/sql_types.rs | 1 - src/ws.rs | 1 - 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/archiver.rs b/src/archiver.rs index 5faa39a..e392651 100644 --- a/src/archiver.rs +++ b/src/archiver.rs @@ -1,6 +1,7 @@ use bigdecimal::ToPrimitive; use crate::{database::*, error::ApiError, kafka::Completion, migrations}; use chrono::prelude::*; +use chrono::TimeDelta; use crossbeam_channel::{Receiver, Sender}; use diesel::prelude::PgConnection; use diesel::r2d2::ConnectionManager; @@ -27,7 +28,7 @@ type ManagedConnection = ConnectionManager; type ManagedPool = r2d2::Pool; const UPDATE_FN: &str = r#" - -- args: + -- args: local id = ARGV[1] local status = ARGV[2] local side = ARGV[3] @@ -35,6 +36,7 @@ const UPDATE_FN: &str = r#" local size = tonumber(ARGV[5]) local timestamp = ARGV[6] local time = tonumber(ARGV[7]) + local exp_time = tonumber(ARGV[8]) if status == "FILLED" or status == "SETTLED" or status == "LIQUIDATE" then redis.call('HDEL', 'orders', id) @@ -65,6 +67,7 @@ const UPDATE_FN: &str = r#" end -- TODO: clean out expired > 24h... + redis.call('ZREMRANGEBYSCORE', 'recent_orders', 0, exp_time) "#; pub struct DatabaseArchiver { @@ -153,8 +156,8 @@ impl DatabaseArchiver { let order = json!({ "order_id": item.order_id.clone(), "side": item.side.to_string(), - "price": item.price, - "positionsize": item.positionsize, + "price": item.price.to_f64().unwrap(), + "positionsize": item.positionsize.to_f64().unwrap(), "timestamp": item.timestamp.to_rfc3339(), }); @@ -193,7 +196,7 @@ impl DatabaseArchiver { .arg("id") .arg(order.uuid.clone()) .arg("price") - .arg(order.entryprice.to_f64()) + .arg(order.entryprice.to_f64().unwrap()) .execute(&mut redis_conn); } @@ -201,7 +204,7 @@ impl DatabaseArchiver { redis::cmd("ZADD") .arg("bid") .arg(price) - .arg(size.to_f64()) + .arg(size.to_f64().unwrap()) .execute(&mut redis_conn); } @@ -209,7 +212,7 @@ impl DatabaseArchiver { redis::cmd("ZADD") .arg("ask") .arg(price) - .arg(size.to_f64()) + .arg(size.to_f64().unwrap()) .execute(&mut redis_conn); } } @@ -231,10 +234,11 @@ impl DatabaseArchiver { .arg(order.uuid.clone()) .arg(order.order_status.as_str()) .arg(side) - .arg(order.entryprice.to_string()) - .arg(order.positionsize.to_string()) + .arg(order.entryprice.to_f64().unwrap()) + .arg(order.positionsize.to_f64().unwrap()) .arg(order.timestamp.to_rfc3339()) - .arg(order.timestamp.timestamp_millis()); + .arg(order.timestamp.timestamp_millis()) + .arg((Utc::now() - TimeDelta::days(1)).timestamp_millis()); pipe.add_command(cmd); let mut redis_conn = self.redis.get_connection()?; @@ -615,7 +619,6 @@ impl DatabaseArchiver { }; self.tx_hash(hash)?; } - Event::AdvanceStateQueue(_, _) => {} } Ok(()) diff --git a/src/database/sql_types.rs b/src/database/sql_types.rs index 82f2227..4c05775 100644 --- a/src/database/sql_types.rs +++ b/src/database/sql_types.rs @@ -409,7 +409,6 @@ impl From for OrderStatus { relayer_types::OrderStatus::RequestSubmitted => OrderStatus::RequestSubmitted, relayer_types::OrderStatus::OrderNotFound => OrderStatus::OrderNotFound, relayer_types::OrderStatus::RejectedFromChain => OrderStatus::RejectedFromChain, - relayer_types::OrderStatus::FilledUpdated => OrderStatus::FilledUpdated, } } } diff --git a/src/ws.rs b/src/ws.rs index 3413717..6e42d04 100644 --- a/src/ws.rs +++ b/src/ws.rs @@ -114,7 +114,6 @@ impl WsContext { Event::Stop(_stop) => {} Event::TxHash(..) => {} Event::TxHashUpdate(..) => {} - Event::AdvanceStateQueue(..) => {} } } if let Err(e) = notify.send(completion) { From bd6b98992079474c8ed280988de742cb5a29595d Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Tue, 6 Aug 2024 14:32:18 +0000 Subject: [PATCH 60/87] lua script update --- .env | 5 ++++- src/archiver.rs | 31 +++++++++++++++++++++---------- src/bin/archiver.rs | 2 +- src/database/sql_types.rs | 1 + src/ws.rs | 1 + 5 files changed, 28 insertions(+), 12 deletions(-) diff --git a/.env b/.env index 70640c3..06db19e 100644 --- a/.env +++ b/.env @@ -8,8 +8,9 @@ # QUESTDB_INFLUX_URL = questdb:9009 # When relayer is inside host machine -DATABASE_URL=postgres://relayer:relayer@localhost:5433/relayer +DATABASE_URL=postgres://relayer:relayer@localhost:5436/relayer REDIS_HOSTNAME=redis://default:foobared@localhost/0 +ORDERBOOK_REDIS=redis://default:foobared@localhost/0 POSTGRESQL_URL=postgresql://postgres:my_password@localhost:5432/my_database QUESTDB_URL=postgresql://quest:my_password@localhost:8812/qdb QUESTDB_INFLUX_URL=127.0.0.1:9009 @@ -71,3 +72,5 @@ LENDORDER_EVENT_LOG = CoreEventLogTopic LENDPOOL_EVENT_LOG = CoreEventLogTopic CORE_EVENT_LOG = CoreEventLogTopic SNAPSHOT_LOG = SnapShotLogTopic + +RUST_BACKTRACE=full \ No newline at end of file diff --git a/src/archiver.rs b/src/archiver.rs index e392651..63b9b2f 100644 --- a/src/archiver.rs +++ b/src/archiver.rs @@ -1,5 +1,5 @@ -use bigdecimal::ToPrimitive; use crate::{database::*, error::ApiError, kafka::Completion, migrations}; +use bigdecimal::ToPrimitive; use chrono::prelude::*; use chrono::TimeDelta; use crossbeam_channel::{Receiver, Sender}; @@ -37,6 +37,7 @@ const UPDATE_FN: &str = r#" local timestamp = ARGV[6] local time = tonumber(ARGV[7]) local exp_time = tonumber(ARGV[8]) + redis.call('ECHO', 'id: ' .. id) if status == "FILLED" or status == "SETTLED" or status == "LIQUIDATE" then redis.call('HDEL', 'orders', id) @@ -46,7 +47,12 @@ const UPDATE_FN: &str = r#" local order_json = cjson.encode(table) redis.call('ZADD', 'recent_orders', time, order_json) - local result = tonumber(redis.pcall('ZRANGEBYSCORE', side, price, price)[1]) + local result = tonumber(redis.pcall('ZRANGEBYSCORE', side, price, price)[1]) or 0 + + if result == 0 then + return + end + local new_size = result - size redis.call('ZREM', side, result) @@ -59,13 +65,14 @@ const UPDATE_FN: &str = r#" if status == "PENDING" then redis.call('HSET', 'orders', id, price) - local result = tonumber(redis.pcall('ZRANGEBYSCORE', side, price, price)[1]) + local result = tonumber(redis.pcall('ZRANGEBYSCORE', side, price, price)[1]) or 0 local new_size = result + size - redis.call('ZREM', side, result) + if result ~= 0 then + redis.call('ZREM', side, result) + end redis.call('ZADD', side, price, new_size) end - -- TODO: clean out expired > 24h... redis.call('ZREMRANGEBYSCORE', 'recent_orders', 0, exp_time) "#; @@ -142,7 +149,8 @@ impl DatabaseArchiver { let recent_orders = TraderOrder::list_past_24hrs(&mut conn).expect("Failed to load recent orders"); - let order_book_orders = TraderOrder::order_book_orders(&mut conn).expect("Failed to load order book"); + let order_book_orders = + TraderOrder::order_book_orders(&mut conn).expect("Failed to load order book"); for chunk in recent_orders.chunks(PIPELINE_CHUNK) { let mut redis_conn = redis @@ -231,18 +239,20 @@ impl DatabaseArchiver { let mut cmd = redis::cmd("EVALSHA"); cmd.arg(&self.script_sha) + .arg(0) .arg(order.uuid.clone()) .arg(order.order_status.as_str()) .arg(side) - .arg(order.entryprice.to_f64().unwrap()) - .arg(order.positionsize.to_f64().unwrap()) + .arg(order.entryprice.to_f64().unwrap() as i64) + .arg(order.positionsize.to_f64().unwrap() as i64) .arg(order.timestamp.to_rfc3339()) - .arg(order.timestamp.timestamp_millis()) - .arg((Utc::now() - TimeDelta::days(1)).timestamp_millis()); + .arg(order.timestamp.timestamp_millis() as i64) + .arg((Utc::now() - TimeDelta::days(1)).timestamp_millis() as i64); pipe.add_command(cmd); let mut redis_conn = self.redis.get_connection()?; pipe.atomic().execute(&mut redis_conn); + Ok(()) } @@ -619,6 +629,7 @@ impl DatabaseArchiver { }; self.tx_hash(hash)?; } + Event::AdvanceStateQueue(_, _) => {} } Ok(()) diff --git a/src/bin/archiver.rs b/src/bin/archiver.rs index abfdca3..4c69173 100644 --- a/src/bin/archiver.rs +++ b/src/bin/archiver.rs @@ -4,7 +4,7 @@ use relayerarchiverlib::kafka; use relayerarchiverlib::DatabaseArchiver; const SNAPSHOT_TOPIC: &str = "CoreEventLogTopic"; -const ARCHIVER_GROUP: &str = "Archiver"; +const ARCHIVER_GROUP: &str = "Archiver_Redis"; fn main() { tracing_subscriber::fmt::Subscriber::builder() diff --git a/src/database/sql_types.rs b/src/database/sql_types.rs index 4c05775..82f2227 100644 --- a/src/database/sql_types.rs +++ b/src/database/sql_types.rs @@ -409,6 +409,7 @@ impl From for OrderStatus { relayer_types::OrderStatus::RequestSubmitted => OrderStatus::RequestSubmitted, relayer_types::OrderStatus::OrderNotFound => OrderStatus::OrderNotFound, relayer_types::OrderStatus::RejectedFromChain => OrderStatus::RejectedFromChain, + relayer_types::OrderStatus::FilledUpdated => OrderStatus::FilledUpdated, } } } diff --git a/src/ws.rs b/src/ws.rs index 6e42d04..3413717 100644 --- a/src/ws.rs +++ b/src/ws.rs @@ -114,6 +114,7 @@ impl WsContext { Event::Stop(_stop) => {} Event::TxHash(..) => {} Event::TxHashUpdate(..) => {} + Event::AdvanceStateQueue(..) => {} } } if let Err(e) = notify.send(completion) { From da25799ea35be16d09c4b9d60d842b6f514f6333 Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Fri, 9 Aug 2024 12:30:28 +0000 Subject: [PATCH 61/87] minor bug --- src/database/sql_types.rs | 2 ++ src/rpc/util.rs | 13 +++++++------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/database/sql_types.rs b/src/database/sql_types.rs index 82f2227..23c8b45 100644 --- a/src/database/sql_types.rs +++ b/src/database/sql_types.rs @@ -76,7 +76,9 @@ impl diesel::query_builder::QueryId for OrderTypeSql { #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, FromSqlRow, AsExpression)] #[diesel(sql_type = PositionTypeSql)] pub enum PositionType { + #[serde(alias = "bid", alias = "Long", alias = "Bid")] LONG, + #[serde(alias = "ask", alias = "Short", alias = "Ask")] SHORT, } diff --git a/src/rpc/util.rs b/src/rpc/util.rs index 9868106..4b9ca38 100644 --- a/src/rpc/util.rs +++ b/src/rpc/util.rs @@ -17,8 +17,8 @@ pub fn order_book(conn: &mut redis::Connection) -> OrderBook { .into_iter() .take(BOOK_LIMIT) .map(|mut chunk| { - let price = chunk.next().unwrap(); let positionsize = chunk.next().unwrap(); + let price = chunk.next().unwrap(); Ask { id: "".into(), @@ -42,8 +42,8 @@ pub fn order_book(conn: &mut redis::Connection) -> OrderBook { .into_iter() .take(BOOK_LIMIT) .map(|chunk| { - let price = chunk[0]; - let positionsize = chunk[1]; + let positionsize = chunk[0]; + let price = chunk[1]; Bid { id: "".into(), @@ -67,7 +67,8 @@ pub fn recent_orders(conn: &mut redis::Connection) -> Vec { .query(conn) .unwrap(); - orders.into_iter().map(|order| { - serde_json::from_str(&order).expect("Invalid recent order!") - }).collect() + orders + .into_iter() + .map(|order| serde_json::from_str(&order).expect("Invalid recent order!")) + .collect() } From aae7f773f22d7240ec1e3d25013e95f499966cd7 Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Mon, 12 Aug 2024 11:16:55 +0000 Subject: [PATCH 62/87] minor update correcting side for order settle --- src/archiver.rs | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/archiver.rs b/src/archiver.rs index 63b9b2f..9c6a0f7 100644 --- a/src/archiver.rs +++ b/src/archiver.rs @@ -231,11 +231,20 @@ impl DatabaseArchiver { fn update_order_cache(&self, order: &InsertTraderOrder) -> Result<(), ApiError> { let mut pipe = redis::pipe(); - - let side = match order.position_type { - PositionType::LONG => "bid", - PositionType::SHORT => "ask", - }; + let side; + if order.order_status == OrderStatus::SETTLED + || order.order_status == OrderStatus::LIQUIDATE + { + side = match order.position_type { + PositionType::LONG => "ask", + PositionType::SHORT => "bid", + } + } else { + side = match order.position_type { + PositionType::LONG => "bid", + PositionType::SHORT => "ask", + }; + } let mut cmd = redis::cmd("EVALSHA"); cmd.arg(&self.script_sha) @@ -371,6 +380,7 @@ impl DatabaseArchiver { /// queue. fn trader_order(&mut self, order: InsertTraderOrder) -> Result<(), ApiError> { debug!("Appending trader order"); + let _ = self.update_order_cache(&order); self.trader_orders.push(order); From 12cc3036133494879f26919f01889e9ce2ff507e Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Mon, 12 Aug 2024 15:11:38 +0000 Subject: [PATCH 63/87] duplicate order in api fixed --- .env | 2 +- src/archiver.rs | 53 +++++++++++++++++++++++++++++++++++++++---------- src/rpc/util.rs | 4 ++-- 3 files changed, 46 insertions(+), 13 deletions(-) diff --git a/.env b/.env index 06db19e..1e812e0 100644 --- a/.env +++ b/.env @@ -8,7 +8,7 @@ # QUESTDB_INFLUX_URL = questdb:9009 # When relayer is inside host machine -DATABASE_URL=postgres://relayer:relayer@localhost:5436/relayer +DATABASE_URL=postgres://relayer:relayer@localhost:5433/relayer REDIS_HOSTNAME=redis://default:foobared@localhost/0 ORDERBOOK_REDIS=redis://default:foobared@localhost/0 POSTGRESQL_URL=postgresql://postgres:my_password@localhost:5432/my_database diff --git a/src/archiver.rs b/src/archiver.rs index 9c6a0f7..7450207 100644 --- a/src/archiver.rs +++ b/src/archiver.rs @@ -28,15 +28,16 @@ type ManagedConnection = ConnectionManager; type ManagedPool = r2d2::Pool; const UPDATE_FN: &str = r#" - -- args: + -- args: local id = ARGV[1] local status = ARGV[2] local side = ARGV[3] local price = tonumber(ARGV[4]) - local size = tonumber(ARGV[5]) - local timestamp = ARGV[6] - local time = tonumber(ARGV[7]) - local exp_time = tonumber(ARGV[8]) + local price_cents = tonumber(ARGV[5]) + local size = tonumber(ARGV[6]) + local timestamp = ARGV[7] + local time = tonumber(ARGV[8]) + local exp_time = tonumber(ARGV[9]) redis.call('ECHO', 'id: ' .. id) if status == "FILLED" or status == "SETTLED" or status == "LIQUIDATE" then @@ -47,7 +48,7 @@ const UPDATE_FN: &str = r#" local order_json = cjson.encode(table) redis.call('ZADD', 'recent_orders', time, order_json) - local result = tonumber(redis.pcall('ZRANGEBYSCORE', side, price, price)[1]) or 0 + local result = tonumber(redis.pcall('ZRANGEBYSCORE', side, price_cents, price_cents)[1]) or 0 if result == 0 then return @@ -56,7 +57,7 @@ const UPDATE_FN: &str = r#" local new_size = result - size redis.call('ZREM', side, result) - redis.call('ZADD', side, price, new_size) + redis.call('ZADD', side, price_cents, new_size) return end @@ -65,13 +66,13 @@ const UPDATE_FN: &str = r#" if status == "PENDING" then redis.call('HSET', 'orders', id, price) - local result = tonumber(redis.pcall('ZRANGEBYSCORE', side, price, price)[1]) or 0 + local result = tonumber(redis.pcall('ZRANGEBYSCORE', side, price_cents, price_cents)[1]) or 0 local new_size = result + size if result ~= 0 then redis.call('ZREM', side, result) end - redis.call('ZADD', side, price, new_size) + redis.call('ZADD', side, price_cents, new_size) end -- TODO: clean out expired > 24h... redis.call('ZREMRANGEBYSCORE', 'recent_orders', 0, exp_time) @@ -151,7 +152,38 @@ impl DatabaseArchiver { TraderOrder::list_past_24hrs(&mut conn).expect("Failed to load recent orders"); let order_book_orders = TraderOrder::order_book_orders(&mut conn).expect("Failed to load order book"); + { + let mut pipe = redis::pipe(); + + let mut redis_conn = redis + .get_connection() + .expect("Failed to acquire redis connection"); + + let mut cmd = redis::cmd("DEL"); + cmd.arg("recent_orders"); + + pipe.add_command(cmd); + // cmd.execute(&mut redis_conn); + let mut cmd = redis::cmd("DEL"); + cmd.arg("orders"); + + pipe.add_command(cmd); + // cmd.execute(&mut redis_conn); + + let mut cmd = redis::cmd("DEL"); + cmd.arg("bid"); + + pipe.add_command(cmd); + // cmd.execute(&mut redis_conn); + + let mut cmd = redis::cmd("DEL"); + cmd.arg("ask"); + // cmd.execute(&mut redis_conn); + pipe.add_command(cmd); + + pipe.atomic().execute(&mut redis_conn); + } for chunk in recent_orders.chunks(PIPELINE_CHUNK) { let mut redis_conn = redis .get_connection() @@ -252,7 +284,8 @@ impl DatabaseArchiver { .arg(order.uuid.clone()) .arg(order.order_status.as_str()) .arg(side) - .arg(order.entryprice.to_f64().unwrap() as i64) + .arg((order.entryprice.to_f64().unwrap()) as i64) + .arg((order.entryprice.to_f64().unwrap() * 100.0) as i64) .arg(order.positionsize.to_f64().unwrap() as i64) .arg(order.timestamp.to_rfc3339()) .arg(order.timestamp.timestamp_millis() as i64) diff --git a/src/rpc/util.rs b/src/rpc/util.rs index 4b9ca38..dad26c6 100644 --- a/src/rpc/util.rs +++ b/src/rpc/util.rs @@ -18,7 +18,7 @@ pub fn order_book(conn: &mut redis::Connection) -> OrderBook { .take(BOOK_LIMIT) .map(|mut chunk| { let positionsize = chunk.next().unwrap(); - let price = chunk.next().unwrap(); + let price = chunk.next().unwrap() / 100.0; Ask { id: "".into(), @@ -43,7 +43,7 @@ pub fn order_book(conn: &mut redis::Connection) -> OrderBook { .take(BOOK_LIMIT) .map(|chunk| { let positionsize = chunk[0]; - let price = chunk[1]; + let price = chunk[1] / 100.0; Bid { id: "".into(), From 841edf37466b03e354edf3eae3a2a38fe41ba374 Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Tue, 20 Aug 2024 14:38:42 +0000 Subject: [PATCH 64/87] minor bug in orderbook --- src/archiver.rs | 12 ++++++++---- src/rpc/util.rs | 3 +++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/archiver.rs b/src/archiver.rs index 7450207..db64c20 100644 --- a/src/archiver.rs +++ b/src/archiver.rs @@ -41,14 +41,14 @@ const UPDATE_FN: &str = r#" redis.call('ECHO', 'id: ' .. id) if status == "FILLED" or status == "SETTLED" or status == "LIQUIDATE" then - redis.call('HDEL', 'orders', id) + local old_price = redis.call('HDEL', 'orders', id) local table = { order_id = id, side = side, price = price, positionsize = size, timestamp = timestamp } local order_json = cjson.encode(table) redis.call('ZADD', 'recent_orders', time, order_json) - local result = tonumber(redis.pcall('ZRANGEBYSCORE', side, price_cents, price_cents)[1]) or 0 + local result = tonumber(redis.pcall('ZRANGEBYSCORE', side, old_price, old_price)[1]) or 0 if result == 0 then return @@ -57,14 +57,18 @@ const UPDATE_FN: &str = r#" local new_size = result - size redis.call('ZREM', side, result) - redis.call('ZADD', side, price_cents, new_size) + + if new_size > 0 + then + redis.call('ZADD', side, old_price, new_size) + end return end -- just opened a new order if status == "PENDING" then - redis.call('HSET', 'orders', id, price) + redis.call('HSET', 'orders', id, price_cents) local result = tonumber(redis.pcall('ZRANGEBYSCORE', side, price_cents, price_cents)[1]) or 0 local new_size = result + size diff --git a/src/rpc/util.rs b/src/rpc/util.rs index dad26c6..acf55bd 100644 --- a/src/rpc/util.rs +++ b/src/rpc/util.rs @@ -3,6 +3,7 @@ use chrono::{TimeDelta, Utc}; use itertools::Itertools; const BOOK_LIMIT: usize = 10; +const RECENT_ORDER_LIMIT: usize = 25; pub fn order_book(conn: &mut redis::Connection) -> OrderBook { let asks: redis::Iter = redis::cmd("ZSCAN") @@ -69,6 +70,8 @@ pub fn recent_orders(conn: &mut redis::Connection) -> Vec { orders .into_iter() + .rev() + .take(RECENT_ORDER_LIMIT) .map(|order| serde_json::from_str(&order).expect("Invalid recent order!")) .collect() } From 2026f89583843d6ab3f98323f84bf7c54f46b911 Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Thu, 19 Jun 2025 11:29:51 +0000 Subject: [PATCH 65/87] price correction for recent order at settlement or liq. in redis --- src/archiver.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/archiver.rs b/src/archiver.rs index db64c20..cd5d8a6 100644 --- a/src/archiver.rs +++ b/src/archiver.rs @@ -268,14 +268,17 @@ impl DatabaseArchiver { fn update_order_cache(&self, order: &InsertTraderOrder) -> Result<(), ApiError> { let mut pipe = redis::pipe(); let side; + let price; if order.order_status == OrderStatus::SETTLED || order.order_status == OrderStatus::LIQUIDATE { + price = order.settlement_price.to_f64().unwrap(); side = match order.position_type { PositionType::LONG => "ask", PositionType::SHORT => "bid", } } else { + price = order.entryprice.to_f64().unwrap(); side = match order.position_type { PositionType::LONG => "bid", PositionType::SHORT => "ask", @@ -288,8 +291,8 @@ impl DatabaseArchiver { .arg(order.uuid.clone()) .arg(order.order_status.as_str()) .arg(side) - .arg((order.entryprice.to_f64().unwrap()) as i64) - .arg((order.entryprice.to_f64().unwrap() * 100.0) as i64) + .arg((price) as i64) + .arg((price * 100.0) as i64) .arg(order.positionsize.to_f64().unwrap() as i64) .arg(order.timestamp.to_rfc3339()) .arg(order.timestamp.timestamp_millis() as i64) From 77a04dcf2acf127676828dc221f61505e019e7d1 Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Fri, 20 Jun 2025 09:58:59 +0000 Subject: [PATCH 66/87] public method pool_share_value added --- src/archiver.rs | 2 +- src/database/models.rs | 5 +++++ src/rpc.rs | 5 +++++ src/rpc/public_methods.rs | 19 +++++++++++++++++++ 4 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/archiver.rs b/src/archiver.rs index cd5d8a6..d3f8f26 100644 --- a/src/archiver.rs +++ b/src/archiver.rs @@ -261,7 +261,7 @@ impl DatabaseArchiver { } } - fn update_sorted_set(&self, cmd: &relayer::SortedSetCommand) -> Result<(), ApiError> { + fn update_sorted_set(&self, _cmd: &relayer::SortedSetCommand) -> Result<(), ApiError> { Ok(()) } diff --git a/src/database/models.rs b/src/database/models.rs index 95594f3..24880f0 100644 --- a/src/database/models.rs +++ b/src/database/models.rs @@ -418,6 +418,11 @@ impl LendPool { lend_pool.order_by(nonce.desc()).first(conn) } + pub fn get_pool_share_value(&self) -> f64 { + let tps = self.total_pool_share.to_f64().unwrap_or(1.0); + let tlv = self.total_locked_value.to_f64().unwrap_or(0.0); + tlv / tps * 100.0 + } pub fn insert( conn: &mut PgConnection, diff --git a/src/rpc.rs b/src/rpc.rs index 447f327..6eac06d 100644 --- a/src/rpc.rs +++ b/src/rpc.rs @@ -144,6 +144,11 @@ pub fn init_public_methods(database_url: &str, redis_url: &str) -> RpcModule, + ctx: &RelayerContext, +) -> Result { + match ctx.pool.get() { + Ok(mut conn) => match LendPool::get(&mut conn) { + Ok(o) => { + let value = o.get_pool_share_value(); + Ok(serde_json::to_value(value).expect("Error converting response")) + } + Err(e) => Err(Error::Custom(format!( + "Error fetching lend pool info: {:?}", + e + ))), + }, + Err(e) => Err(Error::Custom(format!("Database error: {:?}", e))), + } +} From 53a1d9c13f8439d102768068c9ad190a7202f106 Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Sat, 21 Jun 2025 20:12:38 +0000 Subject: [PATCH 67/87] redis check --- src/archiver.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/archiver.rs b/src/archiver.rs index d3f8f26..e5fe3c8 100644 --- a/src/archiver.rs +++ b/src/archiver.rs @@ -41,7 +41,8 @@ const UPDATE_FN: &str = r#" redis.call('ECHO', 'id: ' .. id) if status == "FILLED" or status == "SETTLED" or status == "LIQUIDATE" then - local old_price = redis.call('HDEL', 'orders', id) + local old_price = tonumber(redis.call('HGET', 'orders', id)) + redis.call('HDEL', 'orders', id) local table = { order_id = id, side = side, price = price, positionsize = size, timestamp = timestamp } @@ -49,7 +50,7 @@ const UPDATE_FN: &str = r#" redis.call('ZADD', 'recent_orders', time, order_json) local result = tonumber(redis.pcall('ZRANGEBYSCORE', side, old_price, old_price)[1]) or 0 - + redis.call('ECHO', 'result: ' .. result) if result == 0 then return end From 2cef085c660abaeb4a40673e5bed8fb5057e6646 Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Sat, 21 Jun 2025 20:46:04 +0000 Subject: [PATCH 68/87] redis orderbook insert update --- src/archiver.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/archiver.rs b/src/archiver.rs index e5fe3c8..a097968 100644 --- a/src/archiver.rs +++ b/src/archiver.rs @@ -224,7 +224,7 @@ impl DatabaseArchiver { for order in order_book_orders.iter() { let key = (order.entryprice.to_f64().unwrap() * 100.0) as i64; - let positionsize = order.positionsize.to_f64().unwrap(); + let positionsize = order.positionsize.to_f64().unwrap() as i64; if order.position_type == PositionType::SHORT { asks.entry(key) @@ -238,10 +238,10 @@ impl DatabaseArchiver { redis::cmd("HSET") .arg("orders") - .arg("id") + // .arg("id") .arg(order.uuid.clone()) - .arg("price") - .arg(order.entryprice.to_f64().unwrap()) + // .arg("price") + .arg((order.entryprice.to_f64().unwrap() * 100.0) as i64) .execute(&mut redis_conn); } @@ -249,7 +249,7 @@ impl DatabaseArchiver { redis::cmd("ZADD") .arg("bid") .arg(price) - .arg(size.to_f64().unwrap()) + .arg(size.to_i64().unwrap()) .execute(&mut redis_conn); } @@ -257,7 +257,7 @@ impl DatabaseArchiver { redis::cmd("ZADD") .arg("ask") .arg(price) - .arg(size.to_f64().unwrap()) + .arg(size.to_i64().unwrap()) .execute(&mut redis_conn); } } From 35cc94d714742ce536c57dc85141e8b0b3d22da7 Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Sat, 21 Jun 2025 21:03:21 +0000 Subject: [PATCH 69/87] old side remove correction --- src/archiver.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/archiver.rs b/src/archiver.rs index a097968..b3a88b0 100644 --- a/src/archiver.rs +++ b/src/archiver.rs @@ -55,7 +55,7 @@ const UPDATE_FN: &str = r#" return end - local new_size = result - size + local new_size = result - (size*old_price/price_cents) redis.call('ZREM', side, result) From 5f0219261513ae9ace6097501a0ebed2d1406d65 Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Sat, 21 Jun 2025 21:20:59 +0000 Subject: [PATCH 70/87] echo removed --- src/archiver.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/archiver.rs b/src/archiver.rs index b3a88b0..f771ea6 100644 --- a/src/archiver.rs +++ b/src/archiver.rs @@ -50,7 +50,6 @@ const UPDATE_FN: &str = r#" redis.call('ZADD', 'recent_orders', time, order_json) local result = tonumber(redis.pcall('ZRANGEBYSCORE', side, old_price, old_price)[1]) or 0 - redis.call('ECHO', 'result: ' .. result) if result == 0 then return end From 629391406af5cf8645b7519d9e0b737ebdb35dcf Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Sun, 22 Jun 2025 22:29:59 +0000 Subject: [PATCH 71/87] TraderOrderLimitUpdate --- Cargo.toml | 2 +- src/archiver.rs | 22 ++++++++++++++++++++-- src/database/models.rs | 16 ++++++++++++++++ src/ws.rs | 19 +++++++++++++++++++ 4 files changed, 56 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index d67d7bd..e665384 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -69,7 +69,7 @@ tower-http = { version = "0.4", features = [ tracing-subscriber = { version = "0.3.16", features = ["env-filter"] } uuid = { version = "1.3.3", features = ["serde", "v4", "v7"] } verify-keplr-sign = "0.1.0" -twilight-relayer-rust = { git = "ssh://git@github.com/twilight-project/twilight-relayer.git", branch = "halt-operation-rpc" } +twilight-relayer-rust = { git = "ssh://git@github.com/twilight-project/twilight-relayer.git", branch = "halt-operation-rpc-with-price-update" } redis = { version = "0.25.4", features = ["r2d2"] } [dependencies.zkos-relayer-wallet] diff --git a/src/archiver.rs b/src/archiver.rs index f771ea6..bff5731 100644 --- a/src/archiver.rs +++ b/src/archiver.rs @@ -48,7 +48,7 @@ const UPDATE_FN: &str = r#" local order_json = cjson.encode(table) redis.call('ZADD', 'recent_orders', time, order_json) - +-- check if the order is limit order local result = tonumber(redis.pcall('ZRANGEBYSCORE', side, old_price, old_price)[1]) or 0 if result == 0 then return @@ -62,7 +62,7 @@ const UPDATE_FN: &str = r#" then redis.call('ZADD', side, old_price, new_size) end - +-- ------------------------------ return end @@ -430,6 +430,20 @@ impl DatabaseArchiver { Ok(()) } + /// Add a trader order to the next update batch, if the queue is full, commit and clear the + /// queue. for trader order limit updates on settlement + fn trader_order_limit_update(&mut self, order: InsertTraderOrder) -> Result<(), ApiError> { + // debug!("Appending trader order"); + + // let _ = self.update_order_cache(&order); + // self.trader_orders.push(order); + + // if self.trader_orders.len() == self.trader_orders.capacity() { + // self.commit_trader_orders()?; + // } + + Ok(()) + } /// Commit a batch of trader orders to the database. If we're failing to update the database, we /// should exit. @@ -607,6 +621,10 @@ impl DatabaseArchiver { Event::TraderOrderFundingUpdate(trader_order, _cmd) => { self.trader_order_funding_update(trader_order.into())?; } + // added for limit order update for settlement order + Event::TraderOrderLimitUpdate(trader_order, _cmd, _seq) => { + self.trader_order_limit_update(trader_order.into())?; + } Event::TraderOrderLiquidation(trader_order, _cmd, _seq) => { self.trader_order(trader_order.into())?; } diff --git a/src/database/models.rs b/src/database/models.rs index 24880f0..cf5f159 100644 --- a/src/database/models.rs +++ b/src/database/models.rs @@ -1295,6 +1295,22 @@ impl NewOrderBookOrder { } } } + // added for limit order update for settlement order + pub fn new_limit(to: TraderOrder, price: f64, positionsize: f64) -> Self { + if to.position_type == PositionType::SHORT { + Self::Bid { + id: to.uuid, + positionsize: positionsize, + price: price, + } + } else { + Self::Ask { + id: to.uuid, + positionsize: positionsize, + price: price, + } + } + } } pub fn unrealizedpnl( diff --git a/src/ws.rs b/src/ws.rs index 3413717..9d91855 100644 --- a/src/ws.rs +++ b/src/ws.rs @@ -91,6 +91,25 @@ impl WsContext { _ => {} } } + // added for limit order update for settlement order + Event::TraderOrderLimitUpdate(to, cmd, _seq) => { + let settlement_price = match cmd { + twilight_relayer_rust::relayer::RpcCommand::ExecuteTraderOrder(execute_trader_order, _meta, _zkos_hex_string, _request_id) => { + execute_trader_order.execution_price + } + _ => 0.0, // Default value for other command types + }; + let positionsize = + to.positionsize * settlement_price / to.entryprice; + let order = NewOrderBookOrder::new_limit( + TraderOrder::from(to.clone()), + settlement_price, + positionsize, + ); + if let Err(e) = order_book2.send(order) { + info!("No order book subscribers present {:?}", e); + } + } Event::LendOrder(_lend_order, _cmd, _seq) => {} Event::FundingRateUpdate( _funding_rate, From 7a6f768454100afe1de6a5f28556f199728286da Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Sun, 22 Jun 2025 22:49:41 +0000 Subject: [PATCH 72/87] orderbook insert correction at restart --- src/archiver.rs | 4 ++-- src/database/models.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/archiver.rs b/src/archiver.rs index bff5731..859715c 100644 --- a/src/archiver.rs +++ b/src/archiver.rs @@ -48,7 +48,7 @@ const UPDATE_FN: &str = r#" local order_json = cjson.encode(table) redis.call('ZADD', 'recent_orders', time, order_json) --- check if the order is limit order + -- check if the order is limit order local result = tonumber(redis.pcall('ZRANGEBYSCORE', side, old_price, old_price)[1]) or 0 if result == 0 then return @@ -62,7 +62,7 @@ const UPDATE_FN: &str = r#" then redis.call('ZADD', side, old_price, new_size) end --- ------------------------------ + -- ------------------------------ return end diff --git a/src/database/models.rs b/src/database/models.rs index cf5f159..4f2680b 100644 --- a/src/database/models.rs +++ b/src/database/models.rs @@ -1563,7 +1563,7 @@ impl TraderOrder { WHERE order_type = 'LIMIT' GROUP BY uuid ) - AND order_status NOT IN ('FILLED', 'CANCELLED', 'LIQUIDATE') + AND order_status NOT IN ('FILLED', 'CANCELLED', 'LIQUIDATE', 'SETTLED') "#; diesel::sql_query(query).get_results(conn) From 727625b04a0f7659d1bb2bf846cfcba969383040 Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Mon, 23 Jun 2025 19:59:03 +0000 Subject: [PATCH 73/87] redis script update on settle order by limit price --- src/archiver.rs | 108 +++++++++++++++++++++++++++++++++++------ src/database/models.rs | 6 +-- src/ws.rs | 6 +-- 3 files changed, 97 insertions(+), 23 deletions(-) diff --git a/src/archiver.rs b/src/archiver.rs index 859715c..e7d6e03 100644 --- a/src/archiver.rs +++ b/src/archiver.rs @@ -28,7 +28,7 @@ type ManagedConnection = ConnectionManager; type ManagedPool = r2d2::Pool; const UPDATE_FN: &str = r#" - -- args: + -- args: local id = ARGV[1] local status = ARGV[2] local side = ARGV[3] @@ -38,17 +38,21 @@ const UPDATE_FN: &str = r#" local timestamp = ARGV[7] local time = tonumber(ARGV[8]) local exp_time = tonumber(ARGV[9]) + -- OPEN_LIMIT or CLOSE_LIMIT or OPEN_MARKET or CLOSE_MARKET + local execution_type = ARGV[10] redis.call('ECHO', 'id: ' .. id) - if status == "FILLED" or status == "SETTLED" or status == "LIQUIDATE" then + if (status == "FILLED" or status == "SETTLED" or status == "LIQUIDATE") and execution_type ~= "CLOSE_LIMIT" then local old_price = tonumber(redis.call('HGET', 'orders', id)) redis.call('HDEL', 'orders', id) + local table = { order_id = id, side = side, price = price, positionsize = size, timestamp = timestamp } local order_json = cjson.encode(table) redis.call('ZADD', 'recent_orders', time, order_json) - -- check if the order is limit order + + local result = tonumber(redis.pcall('ZRANGEBYSCORE', side, old_price, old_price)[1]) or 0 if result == 0 then return @@ -57,17 +61,36 @@ const UPDATE_FN: &str = r#" local new_size = result - (size*old_price/price_cents) redis.call('ZREM', side, result) - + if new_size > 0 then redis.call('ZADD', side, old_price, new_size) end - -- ------------------------------ return end + -- settle order on limit -- just opened a new order - if status == "PENDING" then + if status == "PENDING" or (status == "FILLED" and execution_type == "CLOSE_LIMIT") then + -- if the limit order is already exist then remove the old limit price and position size + local is_exist =redis.call('HEXISTS', 'orders', id) + if is_exist == 1 then + local old_price = tonumber(redis.call('HGET', 'orders', id)) + redis.call('HDEL', 'orders', id) + local old_position_size = tonumber(redis.pcall('ZRANGEBYSCORE', side, old_price, old_price)[1]) or 0 + if old_position_size > 0 then + + local new_size = old_position_size - size + + redis.call('ZREM', side, old_position_size) + + if new_size > 0 + then + redis.call('ZADD', side, old_price, new_size) + end + end + end + -- add the new limit price and position size redis.call('HSET', 'orders', id, price_cents) local result = tonumber(redis.pcall('ZRANGEBYSCORE', side, price_cents, price_cents)[1]) or 0 @@ -269,15 +292,23 @@ impl DatabaseArchiver { let mut pipe = redis::pipe(); let side; let price; + let execution_type; if order.order_status == OrderStatus::SETTLED || order.order_status == OrderStatus::LIQUIDATE { + execution_type = "CLOSE_MARKET"; price = order.settlement_price.to_f64().unwrap(); side = match order.position_type { PositionType::LONG => "ask", PositionType::SHORT => "bid", } } else { + execution_type = match order.order_type { + OrderType::LIMIT => "OPEN_LIMIT", + OrderType::MARKET => "OPEN_MARKET", + OrderType::DARK => "OPEN_MARKET", + _ => "", //lend not applicable + }; price = order.entryprice.to_f64().unwrap(); side = match order.position_type { PositionType::LONG => "bid", @@ -296,7 +327,8 @@ impl DatabaseArchiver { .arg(order.positionsize.to_f64().unwrap() as i64) .arg(order.timestamp.to_rfc3339()) .arg(order.timestamp.timestamp_millis() as i64) - .arg((Utc::now() - TimeDelta::days(1)).timestamp_millis() as i64); + .arg((Utc::now() - TimeDelta::days(1)).timestamp_millis() as i64) + .arg(execution_type); pipe.add_command(cmd); let mut redis_conn = self.redis.get_connection()?; @@ -432,15 +464,50 @@ impl DatabaseArchiver { } /// Add a trader order to the next update batch, if the queue is full, commit and clear the /// queue. for trader order limit updates on settlement - fn trader_order_limit_update(&mut self, order: InsertTraderOrder) -> Result<(), ApiError> { - // debug!("Appending trader order"); + fn trader_order_limit_update_redis( + &mut self, + order: InsertTraderOrder, + settlement_price: f64, + ) -> Result<(), ApiError> { + let mut pipe = redis::pipe(); + let side; + let price; + let execution_type; + if order.order_status == OrderStatus::SETTLED + || order.order_status == OrderStatus::LIQUIDATE + { + execution_type = "CLOSE_MARKET"; + price = order.settlement_price.to_f64().unwrap(); + side = match order.position_type { + PositionType::LONG => "ask", + PositionType::SHORT => "bid", + } + } else { + execution_type = "CLOSE_LIMIT"; + price = settlement_price; + side = match order.position_type { + PositionType::LONG => "ask", + PositionType::SHORT => "bid", + }; + } - // let _ = self.update_order_cache(&order); - // self.trader_orders.push(order); + let mut cmd = redis::cmd("EVALSHA"); + cmd.arg(&self.script_sha) + .arg(0) + .arg(order.uuid.clone()) + .arg(order.order_status.as_str()) + .arg(side) + .arg((price) as i64) + .arg((price * 100.0) as i64) + .arg(order.positionsize.to_f64().unwrap() as i64) + .arg(order.timestamp.to_rfc3339()) + .arg(order.timestamp.timestamp_millis() as i64) + .arg((Utc::now() - TimeDelta::days(1)).timestamp_millis() as i64) + .arg(execution_type); - // if self.trader_orders.len() == self.trader_orders.capacity() { - // self.commit_trader_orders()?; - // } + pipe.add_command(cmd); + let mut redis_conn = self.redis.get_connection()?; + pipe.atomic().execute(&mut redis_conn); Ok(()) } @@ -622,8 +689,17 @@ impl DatabaseArchiver { self.trader_order_funding_update(trader_order.into())?; } // added for limit order update for settlement order - Event::TraderOrderLimitUpdate(trader_order, _cmd, _seq) => { - self.trader_order_limit_update(trader_order.into())?; + Event::TraderOrderLimitUpdate(trader_order, cmd, _seq) => { + let settlement_price = match cmd { + twilight_relayer_rust::relayer::RpcCommand::ExecuteTraderOrder( + execute_trader_order, + _meta, + _zkos_hex_string, + _request_id, + ) => execute_trader_order.execution_price, + _ => 0.0, // Default value for other command types + }; + self.trader_order_limit_update_redis(trader_order.into(), settlement_price)?; } Event::TraderOrderLiquidation(trader_order, _cmd, _seq) => { self.trader_order(trader_order.into())?; diff --git a/src/database/models.rs b/src/database/models.rs index 4f2680b..9eb8cf9 100644 --- a/src/database/models.rs +++ b/src/database/models.rs @@ -1296,17 +1296,17 @@ impl NewOrderBookOrder { } } // added for limit order update for settlement order - pub fn new_limit(to: TraderOrder, price: f64, positionsize: f64) -> Self { + pub fn new_close_limit(to: TraderOrder, price: f64) -> Self { if to.position_type == PositionType::SHORT { Self::Bid { id: to.uuid, - positionsize: positionsize, + positionsize: to.positionsize.to_f64().unwrap(), price: price, } } else { Self::Ask { id: to.uuid, - positionsize: positionsize, + positionsize: to.positionsize.to_f64().unwrap(), price: price, } } diff --git a/src/ws.rs b/src/ws.rs index 9d91855..3cd3138 100644 --- a/src/ws.rs +++ b/src/ws.rs @@ -99,12 +99,10 @@ impl WsContext { } _ => 0.0, // Default value for other command types }; - let positionsize = - to.positionsize * settlement_price / to.entryprice; - let order = NewOrderBookOrder::new_limit( + + let order = NewOrderBookOrder::new_close_limit( TraderOrder::from(to.clone()), settlement_price, - positionsize, ); if let Err(e) = order_book2.send(order) { info!("No order book subscribers present {:?}", e); From a61de64cdf1d8631db2a3410ba2e0f474b35a0db Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Mon, 23 Jun 2025 21:54:56 +0000 Subject: [PATCH 74/87] orderbook view added in postgresql and changed query orderbook from trader_order to orderview --- .env | 2 +- .../down.sql | 7 ++ .../up.sql | 90 +++++++++++++++++++ src/database/models.rs | 17 ++-- src/database/schema.rs | 29 ++++++ 5 files changed, 137 insertions(+), 8 deletions(-) create mode 100644 migrations/2025-06-23-213723_create_orderbook_view/down.sql create mode 100644 migrations/2025-06-23-213723_create_orderbook_view/up.sql diff --git a/.env b/.env index 1e812e0..16565e5 100644 --- a/.env +++ b/.env @@ -8,7 +8,7 @@ # QUESTDB_INFLUX_URL = questdb:9009 # When relayer is inside host machine -DATABASE_URL=postgres://relayer:relayer@localhost:5433/relayer +DATABASE_URL=postgres://relayer:relayer1@localhost:5435/relayer REDIS_HOSTNAME=redis://default:foobared@localhost/0 ORDERBOOK_REDIS=redis://default:foobared@localhost/0 POSTGRESQL_URL=postgresql://postgres:my_password@localhost:5432/my_database diff --git a/migrations/2025-06-23-213723_create_orderbook_view/down.sql b/migrations/2025-06-23-213723_create_orderbook_view/down.sql new file mode 100644 index 0000000..1643ecc --- /dev/null +++ b/migrations/2025-06-23-213723_create_orderbook_view/down.sql @@ -0,0 +1,7 @@ + +DROP VIEW IF EXISTS orderbook; + + +DROP INDEX IF EXISTS sorted_set_command_cmd_uuid; +DROP INDEX IF EXISTS trader_order_open_limit_idx; +DROP INDEX IF EXISTS trader_order_uuid_id_desc; diff --git a/migrations/2025-06-23-213723_create_orderbook_view/up.sql b/migrations/2025-06-23-213723_create_orderbook_view/up.sql new file mode 100644 index 0000000..c1292d2 --- /dev/null +++ b/migrations/2025-06-23-213723_create_orderbook_view/up.sql @@ -0,0 +1,90 @@ + +CREATE OR REPLACE VIEW orderbook AS +WITH ranked AS ( + SELECT t.*, + row_number() OVER ( + PARTITION BY t.uuid + ORDER BY t.id DESC + ) AS rn + FROM trader_order t +) + + +SELECT r.id, + r.uuid, + r.account_id, + r.position_type, + r.order_status, + r.order_type, + CAST(r.entryprice AS numeric(20,2)) AS entryprice, + r.execution_price, + r.positionsize, + r.leverage, + r.initial_margin, + r.available_margin, + r."timestamp", + r.bankruptcy_price, + r.bankruptcy_value, + r.maintenance_margin, + r.liquidation_price, + r.unrealized_pnl, + r.settlement_price, + r.entry_nonce, + r.exit_nonce, + r.entry_sequence +FROM ranked r +WHERE r.rn = 1 + AND r.order_type = 'LIMIT' + AND r.order_status NOT IN ('FILLED','CANCELLED','LIQUIDATE','SETTLED') + +UNION ALL + + +SELECT r.id, + r.uuid, + r.account_id, + CASE + WHEN r.position_type = 'LONG' THEN 'SHORT' + WHEN r.position_type = 'SHORT' THEN 'LONG' + ELSE r.position_type + END AS position_type, + r.order_status, + r.order_type, + CAST(sc.amount AS numeric(20,2)) AS entryprice, + r.execution_price, + r.positionsize, + r.leverage, + r.initial_margin, + r.available_margin, + r."timestamp", + r.bankruptcy_price, + r.bankruptcy_value, + r.maintenance_margin, + r.liquidation_price, + r.unrealized_pnl, + r.settlement_price, + r.entry_nonce, + r.exit_nonce, + r.entry_sequence +FROM ranked r +JOIN sorted_set_command sc + ON sc.uuid = r.uuid + AND sc.command = 'ADD_CLOSE_LIMIT_PRICE' +WHERE r.rn = 1 + AND r.order_status = 'FILLED'; + +-- /* 2️⃣ Performance indexes ------------------------------------------------- */ + +-- /* Latest row look-ups for both open & filled branches */ +CREATE INDEX IF NOT EXISTS trader_order_uuid_id_desc + ON trader_order (uuid, id DESC); + +-- /* Selective index for open LIMIT orders */ +CREATE INDEX IF NOT EXISTS trader_order_open_limit_idx + ON trader_order (uuid, id DESC) + WHERE order_type = 'LIMIT' + AND order_status NOT IN ('FILLED','CANCELLED','LIQUIDATE','SETTLED'); + +-- /* Join filter on sorted_set_command */ +CREATE INDEX IF NOT EXISTS sorted_set_command_cmd_uuid + ON sorted_set_command (command, uuid); diff --git a/src/database/models.rs b/src/database/models.rs index 9eb8cf9..33e0ae4 100644 --- a/src/database/models.rs +++ b/src/database/models.rs @@ -1556,14 +1556,17 @@ impl TraderOrder { } pub fn order_book_orders(conn: &mut PgConnection) -> QueryResult> { + // let query = r#" + // SELECT * FROM trader_order + // WHERE id IN ( + // SELECT MAX(id) FROM trader_order + // WHERE order_type = 'LIMIT' + // GROUP BY uuid + // ) + // AND order_status NOT IN ('FILLED', 'CANCELLED', 'LIQUIDATE', 'SETTLED') + // "#; let query = r#" - SELECT * FROM trader_order - WHERE id IN ( - SELECT MAX(id) FROM trader_order - WHERE order_type = 'LIMIT' - GROUP BY uuid - ) - AND order_status NOT IN ('FILLED', 'CANCELLED', 'LIQUIDATE', 'SETTLED') + SELECT * FROM orderbook "#; diesel::sql_query(query).get_results(conn) diff --git a/src/database/schema.rs b/src/database/schema.rs index 2b0986f..7ef954a 100644 --- a/src/database/schema.rs +++ b/src/database/schema.rs @@ -335,3 +335,32 @@ diesel::allow_tables_to_appear_in_same_query!( trader_order_funding_updated, transaction_hash, ); + +// // /* ---- View: orderbook -------------------------------------------- */ +// diesel::table! { +// // Read-only view. No primary key. +// orderbook (id) { +// id -> Int8, +// uuid -> Uuid, +// account_id -> Text, +// position_type -> Varchar, +// order_status -> Varchar, +// order_type -> Varchar, +// entryprice -> Numeric, +// execution_price -> Numeric, +// positionsize -> Numeric, +// leverage -> Numeric, +// initial_margin -> Numeric, +// available_margin -> Numeric, +// timestamp -> Timestamp, +// bankruptcy_price -> Numeric, +// bankruptcy_value -> Numeric, +// maintenance_margin -> Numeric, +// liquidation_price -> Numeric, +// unrealized_pnl -> Numeric, +// settlement_price -> Nullable, +// entry_nonce -> Nullable, +// exit_nonce -> Nullable, +// entry_sequence -> Nullable, +// } +// } From 61f34a87dc136a5f99eacd9c1f7dba61292af667 Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Tue, 24 Jun 2025 20:23:27 +0000 Subject: [PATCH 75/87] websocket update Recent order struct correction --- redis_script.lua | 77 +++++++++++++++++++++++++++++++++++++++++++++++ src/ws.rs | 30 ++++++++++++++---- src/ws/methods.rs | 2 +- 3 files changed, 102 insertions(+), 7 deletions(-) create mode 100644 redis_script.lua diff --git a/redis_script.lua b/redis_script.lua new file mode 100644 index 0000000..7342f20 --- /dev/null +++ b/redis_script.lua @@ -0,0 +1,77 @@ + -- args: + local id = ARGV[1] + local status = ARGV[2] + local side = ARGV[3] + local price = tonumber(ARGV[4]) + local price_cents = tonumber(ARGV[5]) + local size = tonumber(ARGV[6]) + local timestamp = ARGV[7] + local time = tonumber(ARGV[8]) + local exp_time = tonumber(ARGV[9]) + -- OPEN_LIMIT or CLOSE_LIMIT or OPEN_MARKET or CLOSE_MARKET + local execution_type = ARGV[10] + redis.call('ECHO', 'id: ' .. id) + + if (status == "FILLED" or status == "SETTLED" or status == "LIQUIDATE") and execution_type ~= "CLOSE_LIMIT" then + local old_price = tonumber(redis.call('HGET', 'orders', id)) + redis.call('HDEL', 'orders', id) + + + local table = { order_id = id, side = side, price = price, positionsize = size, timestamp = timestamp } + + local order_json = cjson.encode(table) + redis.call('ZADD', 'recent_orders', time, order_json) + + + -- check if the order is limit order + local result = tonumber(redis.pcall('ZRANGEBYSCORE', side, old_price, old_price)[1]) or 0 + if result == 0 then + return + end + + local new_size = result - (size*old_price/price_cents) + + redis.call('ZREM', side, result) + + if new_size > 0 + then + redis.call('ZADD', side, old_price, new_size) + end + -- ------------------------------ + return + end + + -- settle order on limit + -- just opened a new order + if status == "PENDING" or (status == "FILLED" and execution_type == "CLOSE_LIMIT") then + -- if the limit order is already exist then remove the old limit price and position size + local is_exist =redis.call('HEXISTS', 'orders', id) + if is_exist == 1 then + local old_price = tonumber(redis.call('HGET', 'orders', id)) + redis.call('HDEL', 'orders', id) + local old_position_size = tonumber(redis.pcall('ZRANGEBYSCORE', side, old_price, old_price)[1]) or 0 + if old_position_size > 0 then + + local new_size = old_position_size - size + + redis.call('ZREM', side, old_position_size) + + if new_size > 0 + then + redis.call('ZADD', side, old_price, new_size) + end + end + end + -- add the new limit price and position size + redis.call('HSET', 'orders', id, price_cents) + + local result = tonumber(redis.pcall('ZRANGEBYSCORE', side, price_cents, price_cents)[1]) or 0 + local new_size = result + size + + if result ~= 0 then + redis.call('ZREM', side, result) + end + redis.call('ZADD', side, price_cents, new_size) + end + -- TODO: clean out expired > 24h... + redis.call('ZREMRANGEBYSCORE', 'recent_orders', 0, exp_time) \ No newline at end of file diff --git a/src/ws.rs b/src/ws.rs index 3cd3138..820c613 100644 --- a/src/ws.rs +++ b/src/ws.rs @@ -10,7 +10,7 @@ use jsonrpsee::RpcModule; use log::{error, info, trace}; use redis::Client; use relayerwalletlib::zkoswalletlib::relayer_types::{OrderStatus, OrderType}; -// use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Serialize}; use std::{ collections::HashMap, sync::RwLock, @@ -20,7 +20,8 @@ use tokio::{ sync::broadcast::{channel, Sender}, task::JoinHandle, }; -use twilight_relayer_rust::{db::Event, relayer}; +use twilight_relayer_rust::db::Event; +use twilight_relayer_rust::relayer::PositionType; mod methods; @@ -33,11 +34,19 @@ const BROADCAST_CHANNEL_CAPACITY: usize = 20; type ManagedConnection = ConnectionManager; type ManagedPool = r2d2::Pool; +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +struct RecentOrder { + order_id: String, + side: PositionType, + price: f64, + positionsize: f64, + timestamp: String, +} pub struct WsContext { client: Client, price_feed: Sender<(f64, DateTime)>, order_book: Sender, - recent_trades: Sender, + recent_trades: Sender, pub candles: RwLock>>, pub pool: ManagedPool, _completions: CrossbeamSender, @@ -49,7 +58,7 @@ impl WsContext { pub fn with_pool(pool: ManagedPool, client: Client) -> WsContext { let (price_feed, _) = channel::<(f64, DateTime)>(BROADCAST_CHANNEL_CAPACITY); let (order_book, _) = channel::(BROADCAST_CHANNEL_CAPACITY); - let (recent_trades, _) = channel::(BROADCAST_CHANNEL_CAPACITY); + let (recent_trades, _) = channel::(BROADCAST_CHANNEL_CAPACITY); let price_feed2 = price_feed.clone(); let order_book2 = order_book.clone(); @@ -85,8 +94,17 @@ impl WsContext { } match to.order_status { - OrderStatus::PENDING | OrderStatus::FILLED => { - let _ = recent_trades2.send(to); + OrderStatus::SETTLED + | OrderStatus::FILLED + | OrderStatus::LIQUIDATE => { + let recent_order = RecentOrder { + order_id: to.uuid.to_string(), + side: to.position_type.into(), + price: to.execution_price.into(), + positionsize: to.positionsize.into(), + timestamp: to.timestamp, + }; + let _ = recent_trades2.send(recent_order); } _ => {} } diff --git a/src/ws/methods.rs b/src/ws/methods.rs index 53c6c97..9109fc2 100644 --- a/src/ws/methods.rs +++ b/src/ws/methods.rs @@ -151,7 +151,7 @@ pub(super) fn spawn_order_book( let result = serde_json::to_value(&orders)?; if let Err(e) = sink.send(&result) { - error!("Error sending candle updates: {:?}", e); + error!("Error sending orderbook updates: {:?}", e); } sleep(Duration::from_secs(5)).await; } From bbbefb6bd655ebdcf6586badc04700b9af1a61d9 Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Tue, 24 Jun 2025 21:05:13 +0000 Subject: [PATCH 76/87] new orderbook entry correction --- src/database/models.rs | 31 +++++++++++++++++++++++++++++++ src/ws/methods.rs | 6 +++--- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/src/database/models.rs b/src/database/models.rs index 33e0ae4..7feec39 100644 --- a/src/database/models.rs +++ b/src/database/models.rs @@ -1835,6 +1835,37 @@ pub struct OrderBook { pub bid: Vec, pub ask: Vec, } +impl OrderBook { + pub fn new(bid: Vec, ask: Vec) -> OrderBook { + OrderBook { bid, ask } + } + pub fn add_order(&mut self, order: NewOrderBookOrder) { + match order { + NewOrderBookOrder::Bid { + id, + positionsize, + price, + } => { + self.bid.push(Bid { + id, + positionsize, + price, + }); + } + NewOrderBookOrder::Ask { + id, + positionsize, + price, + } => { + self.ask.push(Ask { + id, + positionsize, + price, + }); + } + } + } +} #[derive(Serialize, Deserialize, Debug, Clone)] pub struct Ask { diff --git a/src/ws/methods.rs b/src/ws/methods.rs index 9109fc2..a5c6b75 100644 --- a/src/ws/methods.rs +++ b/src/ws/methods.rs @@ -144,11 +144,11 @@ pub(super) fn spawn_order_book( let _: JoinHandle> = tokio::task::spawn(async move { loop { match rx.try_recv() { - Ok(_mesg) => { + Ok(mesg) => { let mut conn = ctx.pool.get()?; let mut redis_conn = ctx.client.get_connection().expect("REDIS connection."); - let orders = order_book(&mut redis_conn); - let result = serde_json::to_value(&orders)?; + let mut orders = order_book(&mut redis_conn); + let result = serde_json::to_value(&orders.add_order(mesg))?; if let Err(e) = sink.send(&result) { error!("Error sending orderbook updates: {:?}", e); From ddccf641f428ea6c67490e487fba8a9d8d8414f4 Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Tue, 24 Jun 2025 21:39:39 +0000 Subject: [PATCH 77/87] minor correction in order book --- src/ws/methods.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ws/methods.rs b/src/ws/methods.rs index a5c6b75..50981a6 100644 --- a/src/ws/methods.rs +++ b/src/ws/methods.rs @@ -148,7 +148,8 @@ pub(super) fn spawn_order_book( let mut conn = ctx.pool.get()?; let mut redis_conn = ctx.client.get_connection().expect("REDIS connection."); let mut orders = order_book(&mut redis_conn); - let result = serde_json::to_value(&orders.add_order(mesg))?; + orders.add_order(mesg); + let result = serde_json::to_value(&orders)?; if let Err(e) = sink.send(&result) { error!("Error sending orderbook updates: {:?}", e); From ba7801a58155c40afb3c0e1d2ba47a1b19211b3b Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Tue, 24 Jun 2025 22:04:22 +0000 Subject: [PATCH 78/87] minor correction orderbook new data only --- src/database/models.rs | 12 ++++++------ src/ws/methods.rs | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/database/models.rs b/src/database/models.rs index 7feec39..99c8797 100644 --- a/src/database/models.rs +++ b/src/database/models.rs @@ -1283,13 +1283,13 @@ impl NewOrderBookOrder { pub fn new(to: TraderOrder) -> Self { if to.position_type == PositionType::LONG { Self::Bid { - id: to.uuid, + id: "".to_string(), positionsize: to.positionsize.to_f64().unwrap(), price: to.entryprice.to_f64().unwrap(), } } else { Self::Ask { - id: to.uuid, + id: "".to_string(), positionsize: to.positionsize.to_f64().unwrap(), price: to.entryprice.to_f64().unwrap(), } @@ -1299,13 +1299,13 @@ impl NewOrderBookOrder { pub fn new_close_limit(to: TraderOrder, price: f64) -> Self { if to.position_type == PositionType::SHORT { Self::Bid { - id: to.uuid, + id: "".to_string(), positionsize: to.positionsize.to_f64().unwrap(), price: price, } } else { Self::Ask { - id: to.uuid, + id: "".to_string(), positionsize: to.positionsize.to_f64().unwrap(), price: price, } @@ -1847,7 +1847,7 @@ impl OrderBook { price, } => { self.bid.push(Bid { - id, + id: "".to_string(), positionsize, price, }); @@ -1858,7 +1858,7 @@ impl OrderBook { price, } => { self.ask.push(Ask { - id, + id: "".to_string(), positionsize, price, }); diff --git a/src/ws/methods.rs b/src/ws/methods.rs index 50981a6..ddb2cc4 100644 --- a/src/ws/methods.rs +++ b/src/ws/methods.rs @@ -147,9 +147,9 @@ pub(super) fn spawn_order_book( Ok(mesg) => { let mut conn = ctx.pool.get()?; let mut redis_conn = ctx.client.get_connection().expect("REDIS connection."); - let mut orders = order_book(&mut redis_conn); - orders.add_order(mesg); - let result = serde_json::to_value(&orders)?; + // let mut orders = order_book(&mut redis_conn); + // orders.add_order(mesg); + let result = serde_json::to_value(&mesg)?; if let Err(e) = sink.send(&result) { error!("Error sending orderbook updates: {:?}", e); From cf74e45a41d8a801dad9834e2993ab417729001d Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Tue, 24 Jun 2025 22:21:52 +0000 Subject: [PATCH 79/87] all lowercase in NewOrderBookOrder --- src/database/models.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/database/models.rs b/src/database/models.rs index 99c8797..0fbe365 100644 --- a/src/database/models.rs +++ b/src/database/models.rs @@ -1265,7 +1265,8 @@ pub struct OrderBookOrder { positionsize: BigDecimal, } -#[derive(Serialize, Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone, Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "lowercase")] pub enum NewOrderBookOrder { Bid { id: String, From 0a4d31c4edf54d4b4b2e55e6f066a6367d01057a Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Tue, 24 Jun 2025 22:32:01 +0000 Subject: [PATCH 80/87] bug fix --- src/database/models.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/database/models.rs b/src/database/models.rs index 0fbe365..153720e 100644 --- a/src/database/models.rs +++ b/src/database/models.rs @@ -1265,7 +1265,7 @@ pub struct OrderBookOrder { positionsize: BigDecimal, } -#[derive(Serialize, Deserialize, Debug, Clone, Serialize, Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone)] #[serde(rename_all = "lowercase")] pub enum NewOrderBookOrder { Bid { From 54bf8d7e85888add4b9361d883bab4584e26381f Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Tue, 24 Jun 2025 22:46:23 +0000 Subject: [PATCH 81/87] migration update --- .../down.sql | 1 + .../up.sql | 44 ++++++++++++++----- 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/migrations/2025-06-23-213723_create_orderbook_view/down.sql b/migrations/2025-06-23-213723_create_orderbook_view/down.sql index 1643ecc..3dc2d36 100644 --- a/migrations/2025-06-23-213723_create_orderbook_view/down.sql +++ b/migrations/2025-06-23-213723_create_orderbook_view/down.sql @@ -5,3 +5,4 @@ DROP VIEW IF EXISTS orderbook; DROP INDEX IF EXISTS sorted_set_command_cmd_uuid; DROP INDEX IF EXISTS trader_order_open_limit_idx; DROP INDEX IF EXISTS trader_order_uuid_id_desc; +DROP INDEX IF EXISTS sorted_set_command_uuid_id_desc_close_limit; diff --git a/migrations/2025-06-23-213723_create_orderbook_view/up.sql b/migrations/2025-06-23-213723_create_orderbook_view/up.sql index c1292d2..da261e1 100644 --- a/migrations/2025-06-23-213723_create_orderbook_view/up.sql +++ b/migrations/2025-06-23-213723_create_orderbook_view/up.sql @@ -1,4 +1,4 @@ - +-- Orderbook view CREATE OR REPLACE VIEW orderbook AS WITH ranked AS ( SELECT t.*, @@ -7,16 +7,27 @@ WITH ranked AS ( ORDER BY t.id DESC ) AS rn FROM trader_order t -) +), +sc_latest AS ( + SELECT uuid, + MAX(id) AS max_sc_id + FROM sorted_set_command + WHERE command IN ( + 'ADD_CLOSE_LIMIT_PRICE'::sorted_set_command_type, + 'UPDATE_CLOSE_LIMIT_PRICE'::sorted_set_command_type + ) + GROUP BY uuid +) +-- /* === branch A: newest OPEN-LIMIT order per uuid ================= */ SELECT r.id, r.uuid, r.account_id, r.position_type, r.order_status, r.order_type, - CAST(r.entryprice AS numeric(20,2)) AS entryprice, + CAST(r.entryprice AS numeric) AS entryprice, r.execution_price, r.positionsize, r.leverage, @@ -39,7 +50,7 @@ WHERE r.rn = 1 UNION ALL - +-- /* === branch B: newest FILLED order + latest ADD/UPDATE cmd ====== */ SELECT r.id, r.uuid, r.account_id, @@ -50,7 +61,7 @@ SELECT r.id, END AS position_type, r.order_status, r.order_type, - CAST(sc.amount AS numeric(20,2)) AS entryprice, + CAST(sc.amount AS numeric) AS entryprice, r.execution_price, r.positionsize, r.leverage, @@ -66,25 +77,34 @@ SELECT r.id, r.entry_nonce, r.exit_nonce, r.entry_sequence -FROM ranked r -JOIN sorted_set_command sc - ON sc.uuid = r.uuid - AND sc.command = 'ADD_CLOSE_LIMIT_PRICE' +FROM ranked r +JOIN sc_latest ls ON ls.uuid = r.uuid +JOIN sorted_set_command sc ON sc.id = ls.max_sc_id WHERE r.rn = 1 AND r.order_status = 'FILLED'; --- /* 2️⃣ Performance indexes ------------------------------------------------- */ +-- /* ================================================================ */ +-- /* 2️⃣ Performance indexes */ +-- /* ================================================================ */ -- /* Latest row look-ups for both open & filled branches */ CREATE INDEX IF NOT EXISTS trader_order_uuid_id_desc ON trader_order (uuid, id DESC); --- /* Selective index for open LIMIT orders */ +-- /* Selective index for open-LIMIT orders */ CREATE INDEX IF NOT EXISTS trader_order_open_limit_idx ON trader_order (uuid, id DESC) WHERE order_type = 'LIMIT' AND order_status NOT IN ('FILLED','CANCELLED','LIQUIDATE','SETTLED'); --- /* Join filter on sorted_set_command */ +-- /* Join filter on any close-limit command (ADD or UPDATE) */ CREATE INDEX IF NOT EXISTS sorted_set_command_cmd_uuid ON sorted_set_command (command, uuid); + +-- /* Optional: index that matches sc_latest’s GROUP BY + MAX pattern */ +CREATE INDEX IF NOT EXISTS sorted_set_command_uuid_id_desc_close_limit + ON sorted_set_command (uuid, id DESC) + WHERE command IN ( + 'ADD_CLOSE_LIMIT_PRICE'::sorted_set_command_type, + 'UPDATE_CLOSE_LIMIT_PRICE'::sorted_set_command_type + ); From c8f1a916575db2bf44d4045c1b28e25edeaa4452 Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Tue, 24 Jun 2025 23:11:39 +0000 Subject: [PATCH 82/87] Recend order wrong entry price --- src/ws.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ws.rs b/src/ws.rs index 820c613..7caeec4 100644 --- a/src/ws.rs +++ b/src/ws.rs @@ -100,7 +100,7 @@ impl WsContext { let recent_order = RecentOrder { order_id: to.uuid.to_string(), side: to.position_type.into(), - price: to.execution_price.into(), + price: to.entryprice.into(), positionsize: to.positionsize.into(), timestamp: to.timestamp, }; From 41a28c807edb9546069e1ecb7c056bdb31551d3c Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Wed, 25 Jun 2025 10:08:13 +0000 Subject: [PATCH 83/87] position size update for settle order in redis script --- redis_script.lua | 10 ++++++---- src/archiver.rs | 8 ++++++-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/redis_script.lua b/redis_script.lua index 7342f20..6109d62 100644 --- a/redis_script.lua +++ b/redis_script.lua @@ -23,13 +23,16 @@ redis.call('ZADD', 'recent_orders', time, order_json) - -- check if the order is limit order local result = tonumber(redis.pcall('ZRANGEBYSCORE', side, old_price, old_price)[1]) or 0 if result == 0 then return end - - local new_size = result - (size*old_price/price_cents) + local new_size = 0 + if (status == "SETTLED" or status == "LIQUIDATE") then + new_size = result - size + else + new_size = result - (size*old_price/price_cents) + end redis.call('ZREM', side, result) @@ -37,7 +40,6 @@ then redis.call('ZADD', side, old_price, new_size) end - -- ------------------------------ return end diff --git a/src/archiver.rs b/src/archiver.rs index e7d6e03..23c9864 100644 --- a/src/archiver.rs +++ b/src/archiver.rs @@ -57,8 +57,12 @@ const UPDATE_FN: &str = r#" if result == 0 then return end - - local new_size = result - (size*old_price/price_cents) + local new_size = 0 + if (status == "SETTLED" or status == "LIQUIDATE") then + new_size = result - size + else + new_size = result - (size*old_price/price_cents) + end redis.call('ZREM', side, result) From 6365b3ed5c595204fd11be605f883d38a5eca40b Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Thu, 26 Jun 2025 13:52:29 +0000 Subject: [PATCH 84/87] remove .env files --- .compose.env | 75 --------------------------------------------------- .env | 76 ---------------------------------------------------- 2 files changed, 151 deletions(-) delete mode 100644 .compose.env delete mode 100644 .env diff --git a/.compose.env b/.compose.env deleted file mode 100644 index 2d81271..0000000 --- a/.compose.env +++ /dev/null @@ -1,75 +0,0 @@ -# When relayer is inside docker - -NGINX_AUTH_SECRET="FNiPGdfBuvqToL7PHu4EHzu7ehSr8+L5yV2kkxSutSo" -RUST_LOG=info -DATABASE_URL=postgres://relayer:relayer@database:5432/relayer -REDIS_HOSTNAME=redis://default:foobared@redis-server/0 -POSTGRESQL_URL = postgresql://postgres:my_password@postgresql-master:5432/my_database -BROKER=kafka:29092 -QUESTDB_URL = postgresql://quest:my_password@questdb:8812/qdb -QUESTDB_INFLUX_URL = questdb:9009 - -# When relayer is inside host machine -# REDIS_HOSTNAME=redis://default:foobared@localhost/0 -# POSTGRESQL_URL = postgresql://postgres:my_password@localhost:5432/my_database -# QUESTDB_URL = postgresql://quest:my_password@localhost:8812/qdb -# QUESTDB_INFLUX_URL = 127.0.0.1:9009 -# BROKER = localhost:9092 - -# bitmex socket connection url - -BITMEX_BTC_SOCKET_ORDERBOOK_URL=wss://ws.bitmex.com/realtime?subscribe=orderBookL2_25:XBTUSD -BITMEX_BTC_SOCKET_INSTRUMENT_URL=wss://ws.bitmex.com/realtime?subscribe=instrument - -# Binance BTC socket URL - -# https://binance-docs.github.io/apidocs/spot/en/#individual-symbol-mini-ticker-stream - -# A single connection to stream.binance.com is only valid for 24 hours; expect to be disconnected at the 24 hour mark - -# The websocket server will send a ping frame every 3 minutes. If the websocket server does not receive a pong frame back from the connection within a 10 minute period, the connection will be disconnected. Unsolicited pong frames are allowed. - -BINANCE_BTC_SOCKET=wss://stream.binance.com:9443/ws/btcusdt@miniTicker - -# Binance BTC API URL - -BINANCE_BTC_API=https://api.binance.com/api/v3/ticker/price?symbol=BTCUSDT - -# coincap API 2.0 BTC Price Socket URL - -# https://docs.coincap.io/#37dcec0b-1f7b-4d98-b152-0217a6798058 - -COINCAP_BTC_SOCKET=wss://ws.coincap.io/prices?assets=bitcoin - - -RelayerVersion=1.000 -SnapshotVersion=1.001 - -# RPC loadbalance mode -# RPC_QUEUE_MODE = KAFKA -# RPC_QUEUE_MODE = AERON -RPC_QUEUE_MODE = DIRECT - -#RPC Thread Count -RPC_SERVER_THREAD = 15 -RPC_SERVER_SOCKETADDR = 0.0.0.0:3032 - - -KAFKA_STATUS = Enabled - -# kafka topics -# Topics should be already created before running the application -# kafkalib::kafka_topic::kafka_new_topic("BinanceMiniTickerPayload"); -# kafkalib::kafka_topic::kafka_new_topic("CLIENT-REQUEST"); -# kafkalib::kafka_topic::kafka_new_topic("TraderOrderEventLog1"); -# kafkalib::kafka_topic::kafka_new_topic("LendOrderEventLog1"); -# kafkalib::kafka_topic::kafka_new_topic("LendPoolEventLog1"); - -PRICE_LOG = BinanceMiniTickerPayload -RPC_CLIENT_REQUEST=CLIENT-REQUEST - -TRADERORDER_EVENT_LOG = CoreEventLogTopic -LENDORDER_EVENT_LOG = CoreEventLogTopic -LENDPOOL_EVENT_LOG = CoreEventLogTopic -CORE_EVENT_LOG = CoreEventLogTopic -SNAPSHOT_LOG = SnapShotLogTopic diff --git a/.env b/.env deleted file mode 100644 index 16565e5..0000000 --- a/.env +++ /dev/null @@ -1,76 +0,0 @@ -# When relayer is inside docker - -# DATABASE_URL=postgres://relayer:relayer@localhost:5433/relayer -# REDIS_HOSTNAME=redis://default:foobared@redis-server/0 -# POSTGRESQL_URL = postgresql://postgres:my_password@postgresql-master:5432/my_database -# BROKER=localhost:9092 -# QUESTDB_URL = postgresql://quest:my_password@questdb:8812/qdb -# QUESTDB_INFLUX_URL = questdb:9009 - -# When relayer is inside host machine -DATABASE_URL=postgres://relayer:relayer1@localhost:5435/relayer -REDIS_HOSTNAME=redis://default:foobared@localhost/0 -ORDERBOOK_REDIS=redis://default:foobared@localhost/0 -POSTGRESQL_URL=postgresql://postgres:my_password@localhost:5432/my_database -QUESTDB_URL=postgresql://quest:my_password@localhost:8812/qdb -QUESTDB_INFLUX_URL=127.0.0.1:9009 -BROKER=localhost:9092 -# bitmex socket connection url - -BITMEX_BTC_SOCKET_ORDERBOOK_URL=wss://ws.bitmex.com/realtime?subscribe=orderBookL2_25:XBTUSD -BITMEX_BTC_SOCKET_INSTRUMENT_URL=wss://ws.bitmex.com/realtime?subscribe=instrument - -# Binance BTC socket URL - -# https://binance-docs.github.io/apidocs/spot/en/#individual-symbol-mini-ticker-stream - -# A single connection to stream.binance.com is only valid for 24 hours; expect to be disconnected at the 24 hour mark - -# The websocket server will send a ping frame every 3 minutes. If the websocket server does not receive a pong frame back from the connection within a 10 minute period, the connection will be disconnected. Unsolicited pong frames are allowed. - -BINANCE_BTC_SOCKET=wss://stream.binance.com:9443/ws/btcusdt@miniTicker - -# Binance BTC API URL - -BINANCE_BTC_API=https://api.binance.com/api/v3/ticker/price?symbol=BTCUSDT - -# coincap API 2.0 BTC Price Socket URL - -# https://docs.coincap.io/#37dcec0b-1f7b-4d98-b152-0217a6798058 - -COINCAP_BTC_SOCKET=wss://ws.coincap.io/prices?assets=bitcoin - - -RelayerVersion=1.000 -SnapshotVersion=1.001 - -# RPC loadbalance mode -# RPC_QUEUE_MODE = KAFKA -# RPC_QUEUE_MODE = AERON -RPC_QUEUE_MODE = DIRECT - -#RPC Thread Count -RPC_SERVER_THREAD = 15 -RPC_SERVER_SOCKETADDR = 0.0.0.0:3032 - - -KAFKA_STATUS = Enabled - -# kafka topics -# Topics should be already created before running the application -# kafkalib::kafka_topic::kafka_new_topic("BinanceMiniTickerPayload"); -# kafkalib::kafka_topic::kafka_new_topic("CLIENT-REQUEST"); -# kafkalib::kafka_topic::kafka_new_topic("TraderOrderEventLog1"); -# kafkalib::kafka_topic::kafka_new_topic("LendOrderEventLog1"); -# kafkalib::kafka_topic::kafka_new_topic("LendPoolEventLog1"); - -PRICE_LOG = BinanceMiniTickerPayload -RPC_CLIENT_REQUEST = CLIENT-REQUEST - -TRADERORDER_EVENT_LOG = CoreEventLogTopic -LENDORDER_EVENT_LOG = CoreEventLogTopic -LENDPOOL_EVENT_LOG = CoreEventLogTopic -CORE_EVENT_LOG = CoreEventLogTopic -SNAPSHOT_LOG = SnapShotLogTopic - -RUST_BACKTRACE=full \ No newline at end of file From 5a2b26ac3e524eda5b1d7e2176b3842c5e29194a Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Thu, 26 Jun 2025 14:20:18 +0000 Subject: [PATCH 85/87] .env.example added --- .env.example | 20 ++++++++++++++++++++ .gitignore | 3 ++- 2 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 .env.example diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..410613e --- /dev/null +++ b/.env.example @@ -0,0 +1,20 @@ +# Authentication and Logging Configuration +NGINX_AUTH_SECRET="FNiPGdfBuvqToL7PHu4EHzu7ehSr8+L5yV2kkxSutSo" # Secret key for NGINX authentication +RUST_LOG=info # Logging level for Rust applications + +# Database Connection URLs +DATABASE_URL= # Main PostgreSQL database connection +REDIS_HOSTNAME= # Redis cache connection +BROKER= # Kafka message broker connection + +# Event Logging Topics +PRICE_LOG=BinanceMiniTickerPayload # Topic for price updates +RPC_CLIENT_REQUEST=CLIENT-REQUEST # Topic for RPC client requests +RPC_CLIENT_FAILED_REQUEST=CLIENT-FAILED-REQUEST # Topic for failed RPC requests + +# Event Log Topics for Different Components +TRADERORDER_EVENT_LOG=CoreEventLogTopic # Topic for trader order events +LENDORDER_EVENT_LOG=CoreEventLogTopic # Topic for lending order events +LENDPOOL_EVENT_LOG=CoreEventLogTopic # Topic for lending pool events +CORE_EVENT_LOG=CoreEventLogTopic # Topic for core system events +SNAPSHOT_LOG=SnapShotLogTopic # Topic for snapshot events \ No newline at end of file diff --git a/.gitignore b/.gitignore index b8045f3..27c6cb4 100644 --- a/.gitignore +++ b/.gitignore @@ -28,4 +28,5 @@ node_modules #/target /Cargo.lock -.vscode \ No newline at end of file +.vscode +.env From 77c00e26db356b29554420093f7531fa3b3f97bd Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Thu, 26 Jun 2025 14:31:07 +0000 Subject: [PATCH 86/87] branch update to develop --- Cargo.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e665384..3e32910 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -69,12 +69,12 @@ tower-http = { version = "0.4", features = [ tracing-subscriber = { version = "0.3.16", features = ["env-filter"] } uuid = { version = "1.3.3", features = ["serde", "v4", "v7"] } verify-keplr-sign = "0.1.0" -twilight-relayer-rust = { git = "ssh://git@github.com/twilight-project/twilight-relayer.git", branch = "halt-operation-rpc-with-price-update" } +twilight-relayer-rust = { git = "ssh://git@github.com/twilight-project/twilight-relayer.git", branch = "develop" } redis = { version = "0.25.4", features = ["r2d2"] } [dependencies.zkos-relayer-wallet] git = "ssh://git@github.com/twilight-project/zkos-relayer-wallet.git" -branch = "develop-rust" +branch = "develop" [dependencies.zkos-client-wallet] git = "ssh://git@github.com/twilight-project/zkos-client-wallet.git" -branch = "develop-rust" +branch = "develop" From b0f06def21531461caf9ddbbcc0188599f74a700 Mon Sep 17 00:00:00 2001 From: Siddharth Parashar <20308880+shanu516516@users.noreply.github.com> Date: Thu, 26 Jun 2025 15:55:12 +0000 Subject: [PATCH 87/87] docker file update --- Dockerfile | 19 +++++++++++-------- README.md | 7 +++++++ 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6f4a194..b77f982 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,22 @@ -FROM rust:1.77.0-slim-buster as builder +FROM rust:1.87 as builder ARG SSH_KEY RUN apt-get update && apt-get install -y openssh-client git libssl-dev build-essential libpq-dev pkg-config RUN mkdir /root/.ssh -RUN echo "${SSH_KEY}" > /root/.ssh/id_rsa && \ +RUN echo "${SSH_KEY}" > /root/.ssh/id_ed25519 && \ touch /root/.ssh/known_hosts && \ ssh-keyscan github.com >> /root/.ssh/known_hosts && \ - chmod 0600 /root/.ssh/id_rsa - + chmod 0600 /root/.ssh/id_ed25519 COPY . ./twilight-relayerAPI +WORKDIR /twilight-relayerAPI + +RUN git config --global url."ssh://git@github.com/".insteadOf https://github.com/ +RUN git config --global url."ssh://git@github.com/".insteadOf ssh://github.com/ -RUN --mount=type=cache,target=/usr/local/cargo/registry \ - --mount=type=cache,target=${PWD}/target \ - cd ./twilight-relayerAPI && \ +RUN eval `ssh-agent -s` && ssh-add /root/.ssh/id_ed25519 && \ + git config --global url."ssh://git@github.com/".insteadOf https://github.com/ && \ git config --global url."ssh://git@github.com/".insteadOf ssh://github.com/ && \ + git config --global url."ssh://git@github.com/twilight-project/zkos-relayer-wallet.git".insteadOf https://github.com/twilight-project/zkos-relayer-wallet.git && \ cargo --config "net.git-fetch-with-cli = true" b --release --bins FROM debian:10-slim @@ -24,7 +27,7 @@ COPY --from=builder ./twilight-relayerAPI/target/release/api ./ COPY --from=builder ./twilight-relayerAPI/target/release/archiver ./ COPY --from=builder ./twilight-relayerAPI/target/release/auth ./ COPY ./scripts/run.sh ./ -COPY ./.compose.env .env +COPY ./.env ./.env ENTRYPOINT ["/app/run.sh"] diff --git a/README.md b/README.md index 1471432..b3581e5 100644 --- a/README.md +++ b/README.md @@ -3,12 +3,14 @@ For subscriptions and rpcs, see [here](./docs/API.md) ## install diesel_cli (Only necessary for development) + ```command sudo apt install libpq-dev cargo install diesel_cli --no-default-features --features postgres ``` ## running with docker-compose + Build the container, you will need an ssh-key with read-only access to the twilight-relayer repo for this: ```console @@ -34,7 +36,12 @@ Check the .env file for the DATABASE_URL environment variable, be sure it's set `cargo r --release --bin api` +## Run the auth server + +`cargo r --release --bin auth` + ## Testing + Tests are using uuid features that require additional compiler flags: ```command