feat(anubis-judge0): add nginx lb between anubis and judge0 replicas

Pantheonix · Mar 2, 2024 · 13fbc85 · 13fbc85
1 parent 096f6b7
commit 13fbc85
Show file tree

Hide file tree

Showing 6 changed files with 119 additions and 56 deletions.
diff --git a/.github/workflows/eval-lb.yaml b/.github/workflows/eval-lb.yaml
@@ -0,0 +1,55 @@
+name: Eval Nginx Load Balancer - Build Docker image and publish to GitHub Packages
+
+on:
+  push:
+    branches:
+      - develop
+
+    paths:
+      - "anubis-eval/eval-lb/**"
+      - ".github/workflows/eval-lb.yaml"
+
+  pull_request:
+    branches:
+      - develop
+
+    paths:
+      - "anubis-eval/eval-lb/**"
+      - ".github/workflows/eval-lb.yaml"
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: asgard-eval-lb
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        with:
+          images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ env.IMAGE_NAME }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+        with:
+          context: anubis-eval/eval-lb
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
diff --git a/anubis-eval/.env.template b/anubis-eval/.env.template
@@ -6,15 +6,16 @@ ROCKET_DATABASES={anubis-submissions={url="postgres://postgres:2002@anubis-psql-
 CONFIG_JWT_SECRET_KEY=z7F+ut_aphaxeja0&ba*p9spew!4fe0rAFRO5HestitIKOv5nistlz3b=+edu1aP
 CONFIG_DAPR_HTTP_PORT=3503
 CONFIG_DAPR_EVAL_METADATA_ENDPOINT=http://dapr-app-id:[email protected]:3503/api/enki/problem/{problem_id}/eval-metadata
-CONFIG_DAPR_JUDGE_ENDPOINT=http://dapr-app-id:[email protected]:3503/submissions/batch
-CONFIG_DAPR_GET_SUBMISSION_BATCH_ENDPOINT=http://dapr-app-id:[email protected]:3503/submissions/batch?tokens={tokens}
+CONFIG_DAPR_JUDGE_SUBMISSION_BATCH_ENDPOINT=http://judge0-lb:4000/submissions/batch
+CONFIG_DAPR_JUDGE_SUBMISSION_ENDPOINT=http://judge0-lb:4000/submissions
+CONFIG_DAPR_GET_SUBMISSION_BATCH_ENDPOINT=http://judge0-lb:4000/submissions/batch?tokens={tokens}
+CONFIG_DAPR_GET_SUBMISSION_ENDPOINT=http://judge0-lb:4000/submissions/{token}
 CONFIG_DAPR_STATE_STORE_POST_ENDPOINT=http://127.0.0.1:3503/v1.0/state/statestore
 CONFIG_DAPR_STATE_STORE_GET_ENDPOINT=http://127.0.0.1:3503/v1.0/state/statestore/{key}
 CONFIG_EVAL_CRON_SCHEDULE='1/5 * * * * *'
 CONFIG_DEFAULT_NO_SUBMISSIONS_PER_PAGE=10
-CONFIG_DEFAULT_CACHE_TTL_SECONDS=120
 CONFIG_EVAL_BATCH_SIZE=4
-CONFIG_ALLOWED_ORIGINS="http://localhost:10000;https://pantheonix.live;https://pantheonix-midgard.web.app;https://pantheonix-midgard.firebaseapp.com"
+CONFIG_ALLOWED_ORIGINS="https://localhost:10000;http://localhost:10000;https://pantheonix.live;https://pantheonix-midgard.web.app;https://pantheonix-midgard.firebaseapp.com"
 
 #DATABASE_URL=postgres://postgres:2002@localhost:5435/anubis-submissions
 DATABASE_URL=postgres://postgres:2002@anubis-psql-db:5432/anubis-submissions

diff --git a/anubis-eval/eval-lb/Dockerfile b/anubis-eval/eval-lb/Dockerfile
@@ -0,0 +1,4 @@
+FROM nginx:1.24-alpine
+
+COPY ./nginx.conf /etc/nginx/nginx.conf
+
diff --git a/anubis-eval/eval-lb/nginx.conf b/anubis-eval/eval-lb/nginx.conf
@@ -0,0 +1,13 @@
+user nginx;
+events {
+    worker_connections 1000;
+}
+http {
+    client_max_body_size 20M;
+    server {
+        listen 4000;
+        location / {
+            proxy_pass http://server:2358;
+        }
+    }
+}
diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -87,6 +87,7 @@ services:
       ]
     volumes_from:
       - asgard-dapr-config
+    restart: unless-stopped
     depends_on:
       odin-api-gateway:
         condition: service_started
@@ -109,7 +110,7 @@ services:
       context: quetzalcoatl-auth
       dockerfile: ../quetzalcoatl-auth/Dockerfile
     ports:
-      - "5210:5210"
+      - "5210"
     env_file:
       - envs/.env.quetzalcoatl
     restart: unless-stopped
@@ -144,6 +145,7 @@ services:
       ]
     volumes_from:
       - asgard-dapr-config
+    restart: unless-stopped
     depends_on:
       quetzalcoatl-auth-api:
         condition: service_started
@@ -182,7 +184,7 @@ services:
       context: enki-problems
       dockerfile: ../enki-problems/Dockerfile
     ports:
-      - "5211:5211"
+      - "5211"
     env_file:
       - envs/.env.enki
     restart: unless-stopped
@@ -223,6 +225,7 @@ services:
       ]
     volumes_from:
       - asgard-dapr-config
+    restart: unless-stopped
     depends_on:
       enki-problems-api:
         condition: service_started
@@ -244,7 +247,7 @@ services:
     volumes:
       - enki-db:/data/db
     ports:
-      - 30001:30001
+      - "30001:30001"
     healthcheck:
       # for docker-compose
       test:
@@ -272,7 +275,7 @@ services:
       context: hermes-tests
       dockerfile: ../hermes-tests/Dockerfile
     ports:
-      - "5212:5212"
+      - "5212"
     env_file:
       - envs/.env.hermes
     restart: unless-stopped
@@ -302,6 +305,7 @@ services:
       ]
     volumes_from:
       - asgard-dapr-config
+    restart: unless-stopped
     depends_on:
       hermes-tests-api:
         condition: service_started
@@ -324,14 +328,12 @@ services:
       context: anubis-eval
       dockerfile: ../anubis-eval/Dockerfile
     ports:
-      - "5213:5213"
+      - "5213"
     env_file:
       - envs/.env.anubis
     restart: unless-stopped
     depends_on:
-      server:
-        condition: service_started
-      judge0-server-dapr:
+      judge0-lb:
         condition: service_started
       anubis-psql-db:
         condition: service_started
@@ -363,6 +365,7 @@ services:
       ]
     volumes_from:
       - asgard-dapr-config
+    restart: unless-stopped
     depends_on:
       anubis-eval-api:
         condition: service_started
@@ -379,6 +382,7 @@ services:
   anubis-psql-db:
     container_name: anubis-psql-db
     image: postgres:14.1
+    command: postgres -c 'max_connections=250'
     volumes:
       - anubis-db:/var/lib/postgresql/data/
     restart: unless-stopped
@@ -391,12 +395,30 @@ services:
       - asgard
       - pantheonix
 
+  ############ JUDGE0 ############
+
+  judge0-lb:
+    container_name: judge0-lb
+    build:
+      context: anubis-eval/eval-lb
+      dockerfile: ../eval-lb/Dockerfile
+    ports:
+      - "4000"
+    restart: unless-stopped
+    depends_on:
+      server:
+        condition: service_started
+    profiles:
+      - judge0
+      - asgard
+      - pantheonix
+
   server:
     image: judge0/judge0:1.13.0
     volumes:
       - ./envs/.env.judge0:/judge0.conf:ro
     ports:
-      - "2358:2358"
+      - "2358"
     privileged: true
     <<: *default-logging
     restart: unless-stopped
@@ -408,43 +430,7 @@ services:
       db:
         condition: service_started
     profiles:
-      - anubis
-      - asgard
-      - pantheonix
-
-  judge0-server-dapr:
-    container_name: judge0-server-dapr
-    image: daprio/daprd:1.11.2
-    command:
-      [
-        "./daprd",
-        "-app-id",
-        "judge0-server",
-        "-app-port",
-        "2358",
-        "-placement-host-address",
-        "asgard-dapr-placement:40000",
-        "-dapr-http-port",
-        "3504",
-        "-dapr-grpc-port",
-        "50004",
-        "-components-path",
-        "/components",
-        "-config",
-        "/config/config.yaml",
-      ]
-    volumes_from:
-      - asgard-dapr-config
-    depends_on:
-      server:
-        condition: service_started
-      asgard-dapr-placement:
-        condition: service_started
-      asgard-dapr-config:
-        condition: service_started
-    network_mode: "service:server"
-    profiles:
-      - anubis
+      - judge0
       - asgard
       - pantheonix
 
@@ -457,7 +443,7 @@ services:
     <<: *default-logging
     restart: unless-stopped
     profiles:
-      - anubis
+      - judge0
       - asgard
       - pantheonix
 
@@ -469,7 +455,7 @@ services:
     <<: *default-logging
     restart: unless-stopped
     profiles:
-      - anubis
+      - judge0
       - asgard
       - pantheonix
 
@@ -487,7 +473,7 @@ services:
     <<: *default-logging
     restart: unless-stopped
     profiles:
-      - anubis
+      - judge0
       - asgard
       - pantheonix
 
@@ -499,6 +485,7 @@ services:
     command: ["./placement", "-port", "40000", "-log-level", "debug"]
     ports:
       - "40000"
+    restart: unless-stopped
     depends_on:
       asgard-zipkin:
         condition: service_started
@@ -516,6 +503,7 @@ services:
     image: redis:alpine
     ports:
       - "6379"
+    restart: unless-stopped
     volumes:
       - dapr-redis-data:/data
     profiles:
@@ -527,8 +515,9 @@ services:
     container_name: asgard-rabbitmq
     image: rabbitmq:3-management-alpine
     ports:
-      - "5672:5672"
+      - "5672"
       - "15672:15672"
+    restart: unless-stopped
     volumes:
       - rabbitmq-data:/var/lib/rabbitmq
       - rabbitmq-logs:/var/log/rabbitmq
@@ -542,6 +531,7 @@ services:
     image: openzipkin/zipkin:latest
     ports:
       - "19411:9411"
+    restart: unless-stopped
     profiles:
       - dapr
       - asgard

diff --git a/enki-problems/.env.template b/enki-problems/.env.template
@@ -4,7 +4,7 @@ ASPNETCORE_URLS=http://+:5211
 ConnectionStrings__Default="mongodb://enki-mongo-db:30001/EnkiProblems/?replicaSet=enki-rs"
 Redis__Configuration="asgard-redis:6379,abortConnect=false,connectTimeout=30000,responseTimeout=30000"
 AuthServer__SecurityKey="z7F+ut_aphaxeja0&ba*p9spew!4fe0rAFRO5HestitIKOv5nistlz3b=+edu1aP"
-Dapr__HermesAddress="http://localhost:50001"
+Dapr__GrpcEndpoint="http://localhost:50001"
 Dapr__HermesAppId="hermes-tests"
 AllowedHosts="*"
-AllowedOrigins="http://localhost:10000;https://pantheonix.live;https://pantheonix-midgard.web.app;https://pantheonix-midgard.firebaseapp.com"
+AllowedOrigins="https://localhost:10000;http://localhost:10000;https://pantheonix.live;https://pantheonix-midgard.web.app;https://pantheonix-midgard.firebaseapp.com"
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,4 @@
		FROM nginx:1.24-alpine

		COPY ./nginx.conf /etc/nginx/nginx.conf