diff --git a/Cargo.lock b/Cargo.lock index fdce29426..6aff08232 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -265,9 +265,9 @@ dependencies = [ [[package]] name = "actix-web-prom" -version = "0.8.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76743e67d4e7efa9fc2ac7123de0dd7b2ca592668e19334f1d81a3b077afc6ac" +checksum = "56a34f1825c3ae06567a9d632466809bbf34963c86002e8921b64f32d48d289d" dependencies = [ "actix-web", "futures-core", diff --git a/apps/labrinth/.env b/apps/labrinth/.env index 640a8964f..3a73d5053 100644 --- a/apps/labrinth/.env +++ b/apps/labrinth/.env @@ -95,6 +95,7 @@ SENDY_API_KEY=none ANALYTICS_ALLOWED_ORIGINS='["http://127.0.0.1:3000", "http://localhost:3000", "https://modrinth.com", "https://www.modrinth.com", "*"]' +CLICKHOUSE_REPLICATED=false CLICKHOUSE_URL=http://localhost:8123 CLICKHOUSE_USER=default CLICKHOUSE_PASSWORD= diff --git a/apps/labrinth/Cargo.toml b/apps/labrinth/Cargo.toml index e4915fec2..856cd8f2a 100644 --- a/apps/labrinth/Cargo.toml +++ b/apps/labrinth/Cargo.toml @@ -17,7 +17,7 @@ actix-multipart = "0.6.1" actix-cors = "0.7.0" actix-ws = "0.3.0" actix-files = "0.6.5" -actix-web-prom = { version = "0.8.0", features = ["process"] } +actix-web-prom = { version = "0.9.0", features = ["process"] } governor = "0.6.3" tokio = { version = "1.35.1", features = ["sync"] } diff --git a/apps/labrinth/src/clickhouse/mod.rs b/apps/labrinth/src/clickhouse/mod.rs index f74daa6a0..c896a1266 100644 --- a/apps/labrinth/src/clickhouse/mod.rs +++ b/apps/labrinth/src/clickhouse/mod.rs @@ -35,10 +35,24 @@ pub async fn init_client_with_database( .execute() .await?; + let clickhouse_replicated = + dotenvy::var("CLICKHOUSE_REPLICATED").unwrap() == "true"; + let cluster_line = if clickhouse_replicated { + "ON cluster '{cluster}'" + } else { + "" + }; + + let engine = if clickhouse_replicated { + "ReplicatedMergeTree('/clickhouse/{installation}/{cluster}/tables/{shard}/{database}/{table}', '{replica}')" + } else { + "MergeTree()" + }; + client .query(&format!( " - CREATE TABLE IF NOT EXISTS {database}.views + CREATE TABLE IF NOT EXISTS {database}.views {cluster_line} ( recorded DateTime64(4), domain String, @@ -53,8 +67,9 @@ pub async fn init_client_with_database( user_agent String, headers Array(Tuple(String, String)) ) - ENGINE = MergeTree() + ENGINE = {engine} PRIMARY KEY (project_id, recorded, ip) + SETTINGS index_granularity = 8192 " )) .execute() @@ -63,7 +78,7 @@ pub async fn init_client_with_database( client .query(&format!( " - CREATE TABLE IF NOT EXISTS {database}.downloads + CREATE TABLE IF NOT EXISTS {database}.downloads {cluster_line} ( recorded DateTime64(4), domain String, @@ -78,8 +93,9 @@ pub async fn init_client_with_database( user_agent String, headers Array(Tuple(String, String)) ) - ENGINE = MergeTree() + ENGINE = {engine} PRIMARY KEY (project_id, recorded, ip) + SETTINGS index_granularity = 8192 " )) .execute() @@ -88,7 +104,7 @@ pub async fn init_client_with_database( client .query(&format!( " - CREATE TABLE IF NOT EXISTS {database}.playtime + CREATE TABLE IF NOT EXISTS {database}.playtime {cluster_line} ( recorded DateTime64(4), seconds UInt64, @@ -101,8 +117,9 @@ pub async fn init_client_with_database( game_version String, parent UInt64 ) - ENGINE = MergeTree() + ENGINE = {engine} PRIMARY KEY (project_id, recorded, user_id) + SETTINGS index_granularity = 8192 " )) .execute() diff --git a/apps/labrinth/src/lib.rs b/apps/labrinth/src/lib.rs index 2e4b6cf02..5d946fbdc 100644 --- a/apps/labrinth/src/lib.rs +++ b/apps/labrinth/src/lib.rs @@ -473,6 +473,7 @@ pub fn check_env_vars() -> bool { failed |= true; } + failed |= check_var::("CLICKHOUSE_REPLICATED"); failed |= check_var::("CLICKHOUSE_URL"); failed |= check_var::("CLICKHOUSE_USER"); failed |= check_var::("CLICKHOUSE_PASSWORD"); diff --git a/apps/labrinth/src/main.rs b/apps/labrinth/src/main.rs index 6be1aebcf..9310a0886 100644 --- a/apps/labrinth/src/main.rs +++ b/apps/labrinth/src/main.rs @@ -92,7 +92,10 @@ async fn main() -> std::io::Result<()> { let prometheus = PrometheusMetricsBuilder::new("labrinth") .endpoint("/metrics") + .exclude_regex(r"^/api/v1/.*$") + .exclude_regex(r"^/maven/.*$") .exclude("/_internal/launcher_socket") + .mask_unmatched_patterns("UNKNOWN") .build() .expect("Failed to create prometheus metrics middleware");