From c3c54ed05d779cd1e88395af9c3c0501d0c315e7 Mon Sep 17 00:00:00 2001
From: Nicky Gerritsen <nickygerritsen@me.com>
Date: Sun, 21 Aug 2022 20:16:54 +0200
Subject: [PATCH 1/2] Allow to set up load balancing when installing DOMjudge.
---
provision-contest/ansible/domserver.yml | 12 ++---
.../ansible/group_vars/all/all.yml.example | 4 ++
.../ansible/group_vars/all/secret.yml.example | 4 ++
.../ansible/roles/domserver/tasks/main.yml | 20 ++++++---
.../domserver/templates/dbpasswords.secret.j2 | 4 ++
.../templates/nginx-domjudge-inner.j2 | 5 +++
.../templates/nginx-domjudge.conf.j2 | 45 +++++++++++++++++++
.../ansible/roles/mysql_server/tasks/main.yml | 20 +++++++++
8 files changed, 103 insertions(+), 11 deletions(-)
diff --git a/provision-contest/ansible/domserver.yml b/provision-contest/ansible/domserver.yml
index 3c1ff1fb..f467dd5f 100644
--- a/provision-contest/ansible/domserver.yml
+++ b/provision-contest/ansible/domserver.yml
@@ -37,18 +37,18 @@
tags: ssh
- role: mysql_server
tags: mysql_server
- - role: domjudge_checkout
- tags: domjudge_checkout
- - role: domjudge_build
- tags: domjudge_build
- - role: domserver
- tags: domserver
- role: mysql_replication
tags: mysql_replication
when: REPLICATION_PASSWORD is defined
- role: keepalived
tags: keepalived
when: KEEPALIVED_PRIORITY is defined
+ - role: domjudge_checkout
+ tags: domjudge_checkout
+ - role: domjudge_build
+ tags: domjudge_build
+ - role: domserver
+ tags: domserver
- role: prometheus_target_web
tags: prometheus_target_web
vars:
diff --git a/provision-contest/ansible/group_vars/all/all.yml.example b/provision-contest/ansible/group_vars/all/all.yml.example
index b2e08b3d..fcea1e81 100644
--- a/provision-contest/ansible/group_vars/all/all.yml.example
+++ b/provision-contest/ansible/group_vars/all/all.yml.example
@@ -52,6 +52,10 @@ HOSTS:
domjudge-laptop: 10.3.3.200
pc2: 10.3.3.241
+# Set this to true to enable load balancing instead of (automatic) failover
+# DBA_PASSWORD needs to be set for this in secret.yml
+DOMSERVER_LOADBALANCING: false
+
TIMEZONE: "Asia/Dhaka"
PHP_FPM_MAX_CHILDREN: 400
diff --git a/provision-contest/ansible/group_vars/all/secret.yml.example b/provision-contest/ansible/group_vars/all/secret.yml.example
index 2b1a0932..bd06e2bd 100644
--- a/provision-contest/ansible/group_vars/all/secret.yml.example
+++ b/provision-contest/ansible/group_vars/all/secret.yml.example
@@ -8,6 +8,10 @@
# Set this to enable master-master replication between two domservers.
#REPLICATION_PASSWORD: {some-strong-replication-password}
+# Database administrator password. Needed when DOMSERVER_LOADBALANCING is true
+# Note: this user will have access from the whole SERVER_IP_PREFIX range
+#DBA_PASSWORD: some-dba-password
+
# Database user password.
DB_PASSWORD: {some-strong-database-password}
diff --git a/provision-contest/ansible/roles/domserver/tasks/main.yml b/provision-contest/ansible/roles/domserver/tasks/main.yml
index 57415436..8e17d7cb 100644
--- a/provision-contest/ansible/roles/domserver/tasks/main.yml
+++ b/provision-contest/ansible/roles/domserver/tasks/main.yml
@@ -19,17 +19,27 @@
owner: domjudge
notify: Fix permissions on domjudge inplace-install
+- name: set the DBA credentials
+ set_fact:
+ dba_credentials: |
+ {% if DBA_PASSWORD is defined %}
+ -u domjudge_dba -p {{ DBA_PASSWORD }}
+ {% else %}
+ -s -u root
+ {% endif %}
+
# When using replication, the DB will be dropped and recreated on the slave later.
-- name: Check if the database is configured
- command: "{{ DJ_DIR }}/bin/dj_setup_database -s -u root status"
+- name: check if the database is configured
+ command: "{{ DJ_DIR }}/bin/dj_setup_database {{ dba_credentials }} status"
register: db_status
ignore_errors: true
changed_when: false
check_mode: no
+ when: not DOMSERVER_LOADBALANCING or groups['domserver'][0] == inventory_hostname
-- name: Make sure the database is configured
- command: "{{ DJ_DIR }}/bin/dj_setup_database -s -u root bare-install"
- when: "'failed' in db_status.stdout"
+- name: make sure the database is configured
+ command: "{{ DJ_DIR }}/bin/dj_setup_database {{ dba_credentials }} bare-install"
+ when: "(not DOMSERVER_LOADBALANCING or groups['domserver'][0] == inventory_hostname) and 'failed' in db_status.stdout"
- name: Install required packages
apt:
diff --git a/provision-contest/ansible/roles/domserver/templates/dbpasswords.secret.j2 b/provision-contest/ansible/roles/domserver/templates/dbpasswords.secret.j2
index 15706d84..711b48a1 100644
--- a/provision-contest/ansible/roles/domserver/templates/dbpasswords.secret.j2
+++ b/provision-contest/ansible/roles/domserver/templates/dbpasswords.secret.j2
@@ -1,3 +1,7 @@
# {{ansible_managed}}
# Format: 'unused:<db_host>:<db_name>:<user>:<password>:<db_port>'
+{% if DOMSERVER_LOADBALANCING %}
+unused:{{DOMSERVER_IP}}:domjudge:domjudge:{{DB_PASSWORD}}:3306
+{% else %}
unused:localhost:domjudge:domjudge:{{DB_PASSWORD}}:3306
+{% endif %}
diff --git a/provision-contest/ansible/roles/domserver/templates/nginx-domjudge-inner.j2 b/provision-contest/ansible/roles/domserver/templates/nginx-domjudge-inner.j2
index 7e0d1904..5db3567b 100644
--- a/provision-contest/ansible/roles/domserver/templates/nginx-domjudge-inner.j2
+++ b/provision-contest/ansible/roles/domserver/templates/nginx-domjudge-inner.j2
@@ -11,6 +11,11 @@ set $domjudgeRoot {{ DJ_DIR }}/webapp/public;
set $prefix '';
location / {
+{% if DOMSERVER_LOADBALANCING %}
+ if ($access_allowed = false) {
+ return 403;
+ }
+{% endif %}
root $domjudgeRoot;
try_files $uri @domjudgeFront;
diff --git a/provision-contest/ansible/roles/domserver/templates/nginx-domjudge.conf.j2 b/provision-contest/ansible/roles/domserver/templates/nginx-domjudge.conf.j2
index 3e005e7b..16f923e0 100644
--- a/provision-contest/ansible/roles/domserver/templates/nginx-domjudge.conf.j2
+++ b/provision-contest/ansible/roles/domserver/templates/nginx-domjudge.conf.j2
@@ -7,6 +7,36 @@ upstream domjudge {
server unix:/var/run/php-fpm-domjudge.sock; # if using with etc/domjudge-fpm.conf
}
+{% if DOMSERVER_LOADBALANCING %}
+upstream domjudge-loadbalanced {
+ least_conn;
+{% for host in groups['domserver'] %}
+ server {{ hostvars[host].ansible_host }}:81;
+{% endfor %}
+}
+
+server {
+ listen 81;
+ listen [::]:81;
+ server_name _default_;
+
+ add_header Strict-Transport-Security max-age=31556952;
+ include /etc/nginx/snippets/domjudge-inner;
+
+ set_real_ip_from {{ SERVER_IP_PREFIX }}.0/24;
+ real_ip_header X-Forwarded-For;
+ real_ip_recursive on;
+}
+
+map $realip_remote_addr $access_allowed {
+ default false;
+{% for host in groups['domserver'] %}
+ {{ hostvars[host].ansible_host }} true;
+{% endfor %}
+}
+
+{% endif %}
+
server {
listen 80 default;
server_name _default_;
@@ -24,5 +54,20 @@ server {
add_header Strict-Transport-Security max-age=31556952;
send_timeout 36000s;
+{% if DOMSERVER_LOADBALANCING %}
+ location / {
+ proxy_pass http://domjudge-loadbalanced;
+ proxy_http_version 1.1;
+ proxy_set_header Upgrade $http_upgrade;
+ proxy_set_header Connection "upgrade";
+ proxy_set_header X-Forwarded-Proto $scheme;
+ proxy_set_header Host $http_host;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_request_buffering off;
+ proxy_buffering off;
+ }
+{% else %}
include /etc/nginx/snippets/domjudge-inner;
+{% endif %}
}
diff --git a/provision-contest/ansible/roles/mysql_server/tasks/main.yml b/provision-contest/ansible/roles/mysql_server/tasks/main.yml
index 0e9a6a9a..2c54e770 100644
--- a/provision-contest/ansible/roles/mysql_server/tasks/main.yml
+++ b/provision-contest/ansible/roles/mysql_server/tasks/main.yml
@@ -72,3 +72,23 @@
loop:
- load-db
- dump-db
+
+- name: create mysql user for for DOMjudge database administration
+ mysql_user:
+ name: domjudge_dba
+ host: '{{ SERVER_IP_PREFIX }}.%'
+ password: "{{ DBA_PASSWORD }}"
+ append_privs: true
+ priv: 'domjudge.*:ALL,GRANT/*.*:CREATE USER,RELOAD'
+ state: present
+ when: DBA_PASSWORD is defined
+
+- name: create mysql user for for DOMjudge when we are doing loadbalancing
+ mysql_user:
+ name: domjudge
+ host: '{{ SERVER_IP_PREFIX }}.%'
+ password: "{{ DB_PASSWORD }}"
+ append_privs: true
+ priv: 'domjudge.*:SELECT,INSERT,UPDATE,DELETE'
+ state: present
+ when: DOMSERVER_LOADBALANCING
From e7216371a69bcf8f553ef3e33860ce22b33933f6 Mon Sep 17 00:00:00 2001
From: Nicky Gerritsen <nickygerritsen@me.com>
Date: Mon, 22 Aug 2022 09:47:37 +0200
Subject: [PATCH 2/2] Make sure admin machines still work, add MySQL users only
for needed hosts and use SSL for LB communication.
---
.../ansible/roles/domserver/tasks/main.yml | 6 ++---
.../domserver/templates/dbpasswords.secret.j2 | 2 +-
.../templates/nginx-domjudge-inner.j2 | 2 +-
.../templates/nginx-domjudge.conf.j2 | 22 ++++++++++++-------
.../ansible/roles/mysql_server/tasks/main.yml | 10 +++++----
5 files changed, 25 insertions(+), 17 deletions(-)
diff --git a/provision-contest/ansible/roles/domserver/tasks/main.yml b/provision-contest/ansible/roles/domserver/tasks/main.yml
index 8e17d7cb..6add053a 100644
--- a/provision-contest/ansible/roles/domserver/tasks/main.yml
+++ b/provision-contest/ansible/roles/domserver/tasks/main.yml
@@ -22,7 +22,7 @@
- name: set the DBA credentials
set_fact:
dba_credentials: |
- {% if DBA_PASSWORD is defined %}
+ {% if host_type == 'domserver' and DBA_PASSWORD is defined %}
-u domjudge_dba -p {{ DBA_PASSWORD }}
{% else %}
-s -u root
@@ -35,11 +35,11 @@
ignore_errors: true
changed_when: false
check_mode: no
- when: not DOMSERVER_LOADBALANCING or groups['domserver'][0] == inventory_hostname
+ when: not DOMSERVER_LOADBALANCING or groups['domserver'][0] == inventory_hostname or host_type != 'domserver'
- name: make sure the database is configured
command: "{{ DJ_DIR }}/bin/dj_setup_database {{ dba_credentials }} bare-install"
- when: "(not DOMSERVER_LOADBALANCING or groups['domserver'][0] == inventory_hostname) and 'failed' in db_status.stdout"
+ when: "(not DOMSERVER_LOADBALANCING or groups['domserver'][0] == inventory_hostname or host_type != 'domserver') and 'failed' in db_status.stdout"
- name: Install required packages
apt:
diff --git a/provision-contest/ansible/roles/domserver/templates/dbpasswords.secret.j2 b/provision-contest/ansible/roles/domserver/templates/dbpasswords.secret.j2
index 711b48a1..0b8b5865 100644
--- a/provision-contest/ansible/roles/domserver/templates/dbpasswords.secret.j2
+++ b/provision-contest/ansible/roles/domserver/templates/dbpasswords.secret.j2
@@ -1,6 +1,6 @@
# {{ansible_managed}}
# Format: 'unused:<db_host>:<db_name>:<user>:<password>:<db_port>'
-{% if DOMSERVER_LOADBALANCING %}
+{% if host_type == 'domserver' and DOMSERVER_LOADBALANCING %}
unused:{{DOMSERVER_IP}}:domjudge:domjudge:{{DB_PASSWORD}}:3306
{% else %}
unused:localhost:domjudge:domjudge:{{DB_PASSWORD}}:3306
diff --git a/provision-contest/ansible/roles/domserver/templates/nginx-domjudge-inner.j2 b/provision-contest/ansible/roles/domserver/templates/nginx-domjudge-inner.j2
index 5db3567b..1756013f 100644
--- a/provision-contest/ansible/roles/domserver/templates/nginx-domjudge-inner.j2
+++ b/provision-contest/ansible/roles/domserver/templates/nginx-domjudge-inner.j2
@@ -11,7 +11,7 @@ set $domjudgeRoot {{ DJ_DIR }}/webapp/public;
set $prefix '';
location / {
-{% if DOMSERVER_LOADBALANCING %}
+{% if host_type == 'domserver' and DOMSERVER_LOADBALANCING %}
if ($access_allowed = false) {
return 403;
}
diff --git a/provision-contest/ansible/roles/domserver/templates/nginx-domjudge.conf.j2 b/provision-contest/ansible/roles/domserver/templates/nginx-domjudge.conf.j2
index 16f923e0..76577501 100644
--- a/provision-contest/ansible/roles/domserver/templates/nginx-domjudge.conf.j2
+++ b/provision-contest/ansible/roles/domserver/templates/nginx-domjudge.conf.j2
@@ -7,19 +7,25 @@ upstream domjudge {
server unix:/var/run/php-fpm-domjudge.sock; # if using with etc/domjudge-fpm.conf
}
-{% if DOMSERVER_LOADBALANCING %}
+{% if host_type == 'domserver' and DOMSERVER_LOADBALANCING %}
upstream domjudge-loadbalanced {
least_conn;
+ keepalive 100;
{% for host in groups['domserver'] %}
- server {{ hostvars[host].ansible_host }}:81;
+ server {{ hostvars[host].ansible_host }}:444;
{% endfor %}
}
server {
- listen 81;
- listen [::]:81;
+ listen 444 ssl http2;
+ listen [::]:444 ssl http2;
server_name _default_;
+ ssl_certificate {{DOMSERVER_SSL_CERT}};
+ ssl_certificate_key {{DOMSERVER_SSL_KEY}};
+ ssl_session_timeout 5m;
+ ssl_prefer_server_ciphers on;
+
add_header Strict-Transport-Security max-age=31556952;
include /etc/nginx/snippets/domjudge-inner;
@@ -30,6 +36,7 @@ server {
map $realip_remote_addr $access_allowed {
default false;
+ {{ DOMSERVER_IP }} true;
{% for host in groups['domserver'] %}
{{ hostvars[host].ansible_host }} true;
{% endfor %}
@@ -54,12 +61,11 @@ server {
add_header Strict-Transport-Security max-age=31556952;
send_timeout 36000s;
-{% if DOMSERVER_LOADBALANCING %}
+{% if host_type == 'domserver' and DOMSERVER_LOADBALANCING %}
location / {
- proxy_pass http://domjudge-loadbalanced;
+ proxy_pass https://domjudge-loadbalanced;
proxy_http_version 1.1;
- proxy_set_header Upgrade $http_upgrade;
- proxy_set_header Connection "upgrade";
+ proxy_set_header Connection "";
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
diff --git a/provision-contest/ansible/roles/mysql_server/tasks/main.yml b/provision-contest/ansible/roles/mysql_server/tasks/main.yml
index 2c54e770..a2477398 100644
--- a/provision-contest/ansible/roles/mysql_server/tasks/main.yml
+++ b/provision-contest/ansible/roles/mysql_server/tasks/main.yml
@@ -76,19 +76,21 @@
- name: create mysql user for for DOMjudge database administration
mysql_user:
name: domjudge_dba
- host: '{{ SERVER_IP_PREFIX }}.%'
+ host: '{{ item }}'
password: "{{ DBA_PASSWORD }}"
append_privs: true
priv: 'domjudge.*:ALL,GRANT/*.*:CREATE USER,RELOAD'
state: present
- when: DBA_PASSWORD is defined
+ when: host_type == 'domserver' and DBA_PASSWORD is defined
+ loop: "{{ groups['domserver'] | map('extract', hostvars, 'ansible_host') + [DOMSERVER_IP] }}"
- name: create mysql user for for DOMjudge when we are doing loadbalancing
mysql_user:
name: domjudge
- host: '{{ SERVER_IP_PREFIX }}.%'
+ host: '{{ item }}'
password: "{{ DB_PASSWORD }}"
append_privs: true
priv: 'domjudge.*:SELECT,INSERT,UPDATE,DELETE'
state: present
- when: DOMSERVER_LOADBALANCING
+ when: host_type == 'domserver' and DOMSERVER_LOADBALANCING
+ loop: "{{ groups['domserver'] | map('extract', hostvars, 'ansible_host') + [DOMSERVER_IP] }}"