fix: verify backend is UP in HAProxy before removing old container

Author: NetworkCats

deploy/ansible/roles/app/tasks/main.yml

@@ -179,9 +179,21 @@
         docker kill -s HUP rustyip-haproxy
       changed_when: true
 
-    - name: Wait for HAProxy to settle
-      ansible.builtin.pause:
-        seconds: 5
+    - name: Wait for new backend to be UP in HAProxy
+      ansible.builtin.shell: |
+        for i in $(seq 1 30); do
+          csv=$(docker exec rustyip-haproxy wget -qO- \
+            "http://127.0.0.1:9000/stats;csv;norefresh" 2>/dev/null)
+          status=$(echo "$csv" | awk -F, '$1 == "app" && $2 == "{{ deploy_target }}" {print $18}')
+          if [ "$status" = "UP" ]; then
+            echo "backend {{ deploy_target }} is UP in HAProxy"
+            exit 0
+          fi
+          sleep 2
+        done
+        echo "timeout waiting for backend {{ deploy_target }} to become UP"
+        exit 1
+      changed_when: false
 
     - name: Stop old active container
       ansible.builtin.shell: |

deploy/ansible/roles/app/templates/haproxy.cfg.j2

@@ -65,13 +65,21 @@ frontend https_in
 backend global_rate_tracking
     stick-table type string len 6 size 1 expire 10m store http_req_rate(1s)
 
+frontend stats_internal
+    bind 127.0.0.1:9000
+    mode http
+    stats enable
+    stats uri /stats
+    stats refresh 2s
+
 resolvers docker_dns
     nameserver dns1 127.0.0.11:53
     resolve_retries 3
     timeout resolve 1s
     timeout retry 1s
     hold valid 10s
-    hold nx 5s
+    hold nx 30s
+    hold obsolete 30s
 
 backend app
     balance roundrobin