refactor(ci): introduce meta backend agnostic e2e tests (#19568)

Signed-off-by: Bugen Zhao <[email protected]>

ci/scripts/run-e2e-test.sh

@@ -48,13 +48,6 @@ cluster_start() {
     # Give it a while to make sure the single-node is ready.
     sleep 10
   else
-    # Initialize backends.
-    if [[ $mode == *"mysql-backend" ]]; then
-      mysql -h mysql -P 3306 -u root -p123456 -e "DROP DATABASE IF EXISTS metadata; CREATE DATABASE metadata;"
-    elif [[ $mode == *"pg-backend" ]]; then
-      PGPASSWORD=postgres psql -h db -p 5432 -U postgres -c "DROP DATABASE IF EXISTS metadata;" -c "CREATE DATABASE metadata;"
-    fi
-
     risedev ci-start "$mode"
   fi
 }

ci/workflows/main-cron.yml

@@ -4,6 +4,37 @@ auto-retry: &auto-retry
     - signal_reason: agent_stop
       limit: 3
 
+sql-backend: &sql-backend
+  matrix:
+    setup:
+      backend: [""]
+      endpoint: [""]
+    adjustments:
+      - with:
+          backend: ""
+          endpoint: ""
+        skip: true # hack
+      - with:
+          backend: "sqlite"
+          # sqlite3 /tmp/rwmeta.db
+          endpoint: "sqlite:///tmp/rwmeta.db?mode=rwc"
+      - with:
+          backend: "postgres"
+          # PGPASSWORD=postgres psql -h db -p 5432 -U postgres -d rwmeta
+          endpoint: "postgres://postgres:postgres@db:5432/rwmeta"
+      - with:
+          backend: "mysql"
+          # mysql -h mysql -P 3306 -u root -p123456 -D rwmeta
+          endpoint: "mysql://root:123456@mysql:3306/rwmeta"
+  env:
+    RISEDEV_SQL_ENDPOINT: "{{matrix.endpoint}}"
+
+docker-compose-common: &docker-compose-common
+    config: ci/docker-compose.yml
+    mount-buildkite-agent: true
+    propagate-environment: true
+    run: rw-build-env
+
 steps:
   - label: "build"
     command: "ci/scripts/build.sh -p ci-release"
@@ -62,8 +93,9 @@ steps:
     timeout_in_minutes: 10
     retry: *auto-retry
 
-  - label: "end-to-end test (release)"
+  - label: "end-to-end test (release, {{matrix.backend}} backend)"
     key: "e2e-test-release"
+    <<: *sql-backend
     command: "ci/scripts/cron-e2e-test.sh -p ci-release -m ci-3streaming-2serving-3fe"
     if: |
       !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
@@ -75,11 +107,10 @@ steps:
       - "docslt"
     plugins:
       - docker-compose#v5.1.0:
-          run: rw-build-env
-          config: ci/docker-compose.yml
-          mount-buildkite-agent: true
+          <<: *docker-compose-common
+          run: pg-mysql-backend-test-env
       - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 25
+    timeout_in_minutes: 30
     retry: *auto-retry
 
   - label: "slow end-to-end test (release)"
@@ -716,55 +747,15 @@ steps:
       - "build"
       - "build-other"
       - "docslt"
-    plugins:
-      - docker-compose#v4.9.0:
-          run: rw-build-env
-          config: ci/docker-compose.yml
-          mount-buildkite-agent: true
-      - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 25
-    retry: *auto-retry
-
-  - label: "end-to-end test (postgres backend)"
-    key: e2e-test-postgres-backend
-    command: "ci/scripts/e2e-test.sh -p ci-release -m ci-3streaming-2serving-3fe-pg-backend"
-    if: |
-      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
-      || build.pull_request.labels includes "ci/run-e2e-test-other-backends"
-      || build.env("CI_STEPS") =~ /(^|,)e2e-test-other-backends?(,|$$)/
-    depends_on:
-      - "build"
-      - "build-other"
-      - "docslt"
     plugins:
       - docker-compose#v5.1.0:
-          run: pg-mysql-backend-test-env
+          run: rw-build-env
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
       - ./ci/plugins/upload-failure-logs
     timeout_in_minutes: 25
     retry: *auto-retry
 
-  - label: "end-to-end test (mysql backend)"
-    key: e2e-test-mysql-backend
-    command: "ci/scripts/e2e-test.sh -p ci-release -m ci-3streaming-2serving-3fe-mysql-backend"
-    if: |
-      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
-      || build.pull_request.labels includes "ci/run-e2e-test-other-backends"
-      || build.env("CI_STEPS") =~ /(^|,)e2e-test-other-backends?(,|$$)/
-    depends_on:
-      - "build"
-      - "build-other"
-      - "docslt"
-    plugins:
-      - docker-compose#v5.1.0:
-          run: pg-mysql-backend-test-env
-          config: ci/docker-compose.yml
-          mount-buildkite-agent: true
-      - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 30
-    retry: *auto-retry
-
   - label: "end-to-end test for opendal (parallel)"
     key: "e2e-test-opendal-parallel"
     command: "ci/scripts/e2e-test-parallel-for-opendal.sh -p ci-release"

ci/workflows/pull-request.yml

@@ -13,12 +13,39 @@ cargo-cache: &cargo-cache
   backend: s3
   s3:
     bucket: rw-ci-cache-bucket
-    args: '--no-progress'
+    args: "--no-progress"
   paths:
     - ".cargo/registry/index"
     - ".cargo/registry/cache"
     - ".cargo/git"
 
+other-sql-backend: &other-sql-backend
+  matrix:
+    setup:
+      label: [""]
+      endpoint: [""]
+    adjustments:
+      - with:
+          label: ""
+          endpoint: ""
+        skip: true # hack
+      - with:
+          label: "postgres"
+          # PGPASSWORD=postgres psql -h db -p 5432 -U postgres -d rwmeta
+          endpoint: "postgres://postgres:postgres@db:5432/rwmeta"
+      - with:
+          label: "mysql"
+          # mysql -h mysql -P 3306 -u root -p123456 -D rwmeta
+          endpoint: "mysql://root:123456@mysql:3306/rwmeta"
+  env:
+    RISEDEV_SQL_ENDPOINT: "{{matrix.endpoint}}"
+
+docker-compose-common: &docker-compose-common
+    config: ci/docker-compose.yml
+    mount-buildkite-agent: true
+    propagate-environment: true
+    run: rw-build-env
+
 steps:
   - label: "check ci image rebuild"
     plugins:
@@ -789,34 +816,18 @@ steps:
     timeout_in_minutes: 30
     retry: *auto-retry
 
-  - label: "end-to-end test (postgres backend)"
-    command: "ci/scripts/e2e-test.sh -p ci-dev -m ci-3streaming-2serving-3fe-pg-backend"
-    if: build.pull_request.labels includes "ci/run-e2e-test-other-backends" || build.env("CI_STEPS") =~ /(^|,)e2e-test-other-backends?(,|$$)/
-    depends_on:
-      - "build"
-      - "build-other"
-      - "docslt"
-    plugins:
-      - docker-compose#v5.1.0:
-          run: pg-mysql-backend-test-env
-          config: ci/docker-compose.yml
-          mount-buildkite-agent: true
-      - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 30
-    retry: *auto-retry
-
-  - label: "end-to-end test (mysql backend)"
-    command: "ci/scripts/e2e-test.sh -p ci-dev -m ci-3streaming-2serving-3fe-mysql-backend"
+  - label: "end-to-end test ({{matrix.label}} backend)"
+    <<: *other-sql-backend
+    command: "ci/scripts/e2e-test.sh -p ci-dev -m ci-3streaming-2serving-3fe"
     if: build.pull_request.labels includes "ci/run-e2e-test-other-backends" || build.env("CI_STEPS") =~ /(^|,)e2e-test-other-backends?(,|$$)/
     depends_on:
       - "build"
       - "build-other"
       - "docslt"
     plugins:
       - docker-compose#v5.1.0:
+          <<: *docker-compose-common
           run: pg-mysql-backend-test-env
-          config: ci/docker-compose.yml
-          mount-buildkite-agent: true
       - ./ci/plugins/upload-failure-logs
     timeout_in_minutes: 32
     retry: *auto-retry

risedev.yml

@@ -777,125 +777,8 @@ profile:
     config-path: src/config/ci.toml
     steps:
       - use: minio
-      - use: sqlite
-      - use: meta-node
-        meta-backend: sqlite
-      - use: compute-node
-        port: 5687
-        exporter-port: 1222
-        enable-tiered-cache: true
-        role: streaming
-        parallelism: 4
-      - use: compute-node
-        port: 5688
-        exporter-port: 1223
-        enable-tiered-cache: true
-        role: streaming
-        parallelism: 4
-      - use: compute-node
-        port: 5689
-        exporter-port: 1224
-        enable-tiered-cache: true
-        role: streaming
-        parallelism: 4
-      - use: compute-node
-        port: 5685
-        exporter-port: 1225
-        enable-tiered-cache: true
-        role: serving
-        parallelism: 4
-      - use: compute-node
-        port: 5686
-        exporter-port: 1226
-        enable-tiered-cache: true
-        role: serving
-        parallelism: 8
-      - use: frontend
-        port: 4565
-        exporter-port: 2222
-        health-check-port: 6786
-      - use: frontend
-        port: 4566
-        exporter-port: 2223
-        health-check-port: 6787
-      - use: frontend
-        port: 4567
-        exporter-port: 2224
-        health-check-port: 6788
-      - use: compactor
-
-  ci-3streaming-2serving-3fe-pg-backend:
-    config-path: src/config/ci.toml
-    steps:
-      - use: minio
-      - use: postgres
-        port: 5432
-        address: db
-        database: metadata
-        user: postgres
-        password: postgres
-        user-managed: true
-        application: metastore
       - use: meta-node
-        meta-backend: postgres
-      - use: compute-node
-        port: 5687
-        exporter-port: 1222
-        enable-tiered-cache: true
-        role: streaming
-        parallelism: 4
-      - use: compute-node
-        port: 5688
-        exporter-port: 1223
-        enable-tiered-cache: true
-        role: streaming
-        parallelism: 4
-      - use: compute-node
-        port: 5689
-        exporter-port: 1224
-        enable-tiered-cache: true
-        role: streaming
-        parallelism: 4
-      - use: compute-node
-        port: 5685
-        exporter-port: 1225
-        enable-tiered-cache: true
-        role: serving
-        parallelism: 4
-      - use: compute-node
-        port: 5686
-        exporter-port: 1226
-        enable-tiered-cache: true
-        role: serving
-        parallelism: 8
-      - use: frontend
-        port: 4565
-        exporter-port: 2222
-        health-check-port: 6786
-      - use: frontend
-        port: 4566
-        exporter-port: 2223
-        health-check-port: 6787
-      - use: frontend
-        port: 4567
-        exporter-port: 2224
-        health-check-port: 6788
-      - use: compactor
-
-  ci-3streaming-2serving-3fe-mysql-backend:
-    config-path: src/config/ci.toml
-    steps:
-      - use: minio
-      - use: mysql
-        port: 3306
-        address: mysql
-        database: metadata
-        user: root
-        password: 123456
-        user-managed: true
-        application: metastore
-      - use: meta-node
-        meta-backend: mysql
+        meta-backend: env
       - use: compute-node
         port: 5687
         exporter-port: 1222

src/risedevtool/src/bin/risedev-dev.rs

@@ -384,7 +384,7 @@ fn main() -> Result<()> {
     tracing_subscriber::fmt()
         .with_env_filter(
             EnvFilter::builder()
-                .with_default_directive(LevelFilter::WARN.into())
+                .with_default_directive(LevelFilter::INFO.into())
                 .with_env_var("RISEDEV_RUST_LOG")
                 .from_env_lossy(),
         )

src/risedevtool/src/task/meta_node_service.rs

@@ -42,6 +42,10 @@ use crate::{
 fn sql_endpoint_from_env() -> String {
     static SQL_ENDPOINT: LazyLock<String> = LazyLock::new(|| {
         if let Ok(endpoint) = env::var("RISEDEV_SQL_ENDPOINT") {
+            tracing::info!(
+                "sql endpoint from env RISEDEV_SQL_ENDPOINT resolved to `{}`",
+                endpoint
+            );
             endpoint
         } else {
             let temp_path = NamedTempFile::with_suffix(".db").unwrap().into_temp_path();