Merge develop into main: Add Redis Caching, Enhanced Order Book, and Major API Refactor

.env.example

@@ -0,0 +1,20 @@
+# Authentication and Logging Configuration
+NGINX_AUTH_SECRET="FNiPGdfBuvqToL7PHu4EHzu7ehSr8+L5yV2kkxSutSo"  # Secret key for NGINX authentication
+RUST_LOG=info  # Logging level for Rust applications
+
+# Database Connection URLs
+DATABASE_URL=<PostgreSQL_database_connection_string>  # Main PostgreSQL database connection
+REDIS_HOSTNAME=<Redis_cache_connection_string>  # Redis cache connection
+BROKER=<Kafka_message_broker_connection>  # Kafka message broker connection
+
+# Event Logging Topics
+PRICE_LOG=BinanceMiniTickerPayload  # Topic for price updates
+RPC_CLIENT_REQUEST=CLIENT-REQUEST  # Topic for RPC client requests
+RPC_CLIENT_FAILED_REQUEST=CLIENT-FAILED-REQUEST  # Topic for failed RPC requests
+
+# Event Log Topics for Different Components
+TRADERORDER_EVENT_LOG=CoreEventLogTopic  # Topic for trader order events
+LENDORDER_EVENT_LOG=CoreEventLogTopic  # Topic for lending order events
+LENDPOOL_EVENT_LOG=CoreEventLogTopic  # Topic for lending pool events
+CORE_EVENT_LOG=CoreEventLogTopic  # Topic for core system events
+SNAPSHOT_LOG=SnapShotLogTopic  # Topic for snapshot events

.gitignore

@@ -28,4 +28,5 @@ node_modules
 
 #/target
 /Cargo.lock
-.vscode
+.vscode
+.env

Cargo.toml

@@ -11,9 +11,9 @@ name = "archiver"
 [[bin]]
 name = "api"
 
-# [lib]
-# name = "relayerarchiverlib"
-# path = "src/lib.rs"
+[lib]
+name = "relayerarchiverlib"
+path = "src/lib.rs"
 
 
 [dependencies]
@@ -67,14 +67,14 @@ tower-http = { version = "0.4", features = [
     "validate-request",
 ] }
 tracing-subscriber = { version = "0.3.16", features = ["env-filter"] }
-twilight-relayer-rust = { git = "ssh://git@github.com/twilight-project/twilight-relayer.git", branch = "output-archiver-0.03.031" }
-
 uuid = { version = "1.3.3", features = ["serde", "v4", "v7"] }
 verify-keplr-sign = "0.1.0"
+twilight-relayer-rust = { git = "ssh://git@github.com/twilight-project/twilight-relayer.git", branch = "develop" }
+redis = { version = "0.25.4", features = ["r2d2"] }
 
 [dependencies.zkos-relayer-wallet]
-git = "ssh://github.com/twilight-project/zkos-relayer-wallet.git"
+git = "ssh://git@github.com/twilight-project/zkos-relayer-wallet.git"
 branch = "develop"
 [dependencies.zkos-client-wallet]
-git = "ssh://github.com/twilight-project/zkos-client-wallet.git"
+git = "ssh://git@github.com/twilight-project/zkos-client-wallet.git"
 branch = "develop"

Dockerfile

@@ -1,18 +1,22 @@
-FROM rust:1.72.0-slim-buster as builder
+FROM rust:1.87 as builder
 
 ARG SSH_KEY
 RUN apt-get update && apt-get install -y openssh-client git libssl-dev build-essential libpq-dev pkg-config
 RUN mkdir /root/.ssh
-RUN echo "${SSH_KEY}" > /root/.ssh/id_rsa && \
+RUN echo "${SSH_KEY}" > /root/.ssh/id_ed25519 && \
     touch /root/.ssh/known_hosts && \
     ssh-keyscan github.com >> /root/.ssh/known_hosts && \
-    chmod 0600 /root/.ssh/id_rsa
-
+    chmod 0600 /root/.ssh/id_ed25519
 COPY . ./twilight-relayerAPI
+WORKDIR /twilight-relayerAPI
+
+RUN git config --global url."ssh://git@github.com/".insteadOf https://github.com/
+RUN git config --global url."ssh://git@github.com/".insteadOf ssh://github.com/
 
-RUN --mount=type=cache,target=/usr/local/cargo/registry \
-    --mount=type=cache,target=${PWD}/target \
-    cd ./twilight-relayerAPI && \
+RUN eval `ssh-agent -s` && ssh-add /root/.ssh/id_ed25519 && \
+    git config --global url."ssh://git@github.com/".insteadOf https://github.com/ && \
+    git config --global url."ssh://git@github.com/".insteadOf ssh://github.com/ && \
+    git config --global url."ssh://git@github.com/twilight-project/zkos-relayer-wallet.git".insteadOf https://github.com/twilight-project/zkos-relayer-wallet.git && \
     cargo --config "net.git-fetch-with-cli = true" b --release --bins
 
 FROM debian:10-slim
@@ -23,7 +27,7 @@ COPY --from=builder ./twilight-relayerAPI/target/release/api ./
 COPY --from=builder ./twilight-relayerAPI/target/release/archiver ./
 COPY --from=builder ./twilight-relayerAPI/target/release/auth ./
 COPY ./scripts/run.sh ./
-COPY ./.compose.env .env
+COPY ./.env ./.env
 
 
 ENTRYPOINT ["/app/run.sh"]

README.md

@@ -3,12 +3,14 @@
 For subscriptions and rpcs, see [here](./docs/API.md)
 
 ## install diesel_cli (Only necessary for development)
+
 ```command
 sudo apt install libpq-dev
 cargo install diesel_cli --no-default-features --features postgres
 ```
 
 ## running with docker-compose
+
 Build the container, you will need an ssh-key with read-only access to the twilight-relayer repo for this:
 
 ```console
@@ -34,7 +36,12 @@ Check the .env file for the DATABASE_URL environment variable, be sure it's set
 
 `cargo r --release --bin api`
 
+## Run the auth server
+
+`cargo r --release --bin auth`
+
 ## Testing
+
 Tests are using uuid features that require additional compiler flags:
 
 ```command

migrations/2024-03-26-213323_add_aggregation_tables/down.sql

@@ -0,0 +1,16 @@
+-- This file should undo anything in `up.sql`
+-- DROP INDEX trader_order_time;
+-- DROP INDEX price_time;
+DROP INDEX trader_order_uuid;
+
+
+DROP TABLE candles_1min;
+DROP TABLE candles_1hour;
+DROP TABLE candles_1day;
+
+DROP FUNCTION get_ohlc_interval;
+DROP FUNCTION get_volume_interval;
+DROP FUNCTION get_candles_interval;
+DROP FUNCTION update_candles_1min;
+DROP FUNCTION update_candles_1hour;
+DROP FUNCTION update_candles_1day;