From 52d0d0d4f0a8661c50c6ff248cf6ffe6cde8dcb5 Mon Sep 17 00:00:00 2001
From: Thomas <substantialimpulse@pm.me>
Date: Wed, 3 Jan 2024 14:08:01 +0100
Subject: [PATCH] add node_exporter as container for prometheus

---
 roles/monitoring/defaults/main.yml            |  2 ++
 roles/monitoring/tasks/main.yml               | 14 +++++++--
 .../templates/docker-compose.yml.j2           | 29 ++++++++++++++++++-
 .../prometheus.yml.j2}                        |  6 ++--
 4 files changed, 45 insertions(+), 6 deletions(-)
 rename roles/monitoring/{files/prometheus.yml => templates/prometheus.yml.j2} (86%)

diff --git a/roles/monitoring/defaults/main.yml b/roles/monitoring/defaults/main.yml
index ab3e39e..a42e84d 100644
--- a/roles/monitoring/defaults/main.yml
+++ b/roles/monitoring/defaults/main.yml
@@ -1,5 +1,6 @@
 ---
 monitoring_domain: monitoring.{{ app_domain }}
+monitoring_network: monitoring
 grafana_install_location: "/srv/grafana"
 grafana_data_location: "{{ grafana_install_location }}/data"
 
@@ -8,3 +9,4 @@ prometheus_data_location: "{{ prometheus_install_location }}/data"
 prometheus_config_location: "{{ prometheus_install_location }}/prometheus.yml"
 
 grafana_port: "3000"
+prometheus_port: "9090"
diff --git a/roles/monitoring/tasks/main.yml b/roles/monitoring/tasks/main.yml
index 6d96e2a..14a4828 100644
--- a/roles/monitoring/tasks/main.yml
+++ b/roles/monitoring/tasks/main.yml
@@ -22,6 +22,16 @@
     - "{{ prometheus_data_location }}"
   become: true
 
+- name: Create networks
+  community.docker.docker_network:
+    name: "{{ item }}"
+    driver: "overlay"
+    driver_options:
+      com.docker.network.bridge.name: "{{ item }}"
+  become: true
+  loop:
+    - "{{ monitoring_network }}"
+
 - name: Create docker-compose.yml
   ansible.builtin.template:
     src: docker-compose.yml.j2
@@ -36,8 +46,8 @@
     - wait for grafana
 
 - name: Copy prometheus config
-  ansible.builtin.copy:
-    src: prometheus.yml
+  ansible.builtin.template:
+    src: prometheus.yml.j2
     dest: "{{ prometheus_config_location }}"
     owner: root
     group: root
diff --git a/roles/monitoring/templates/docker-compose.yml.j2 b/roles/monitoring/templates/docker-compose.yml.j2
index 6583ee7..a8a44cb 100644
--- a/roles/monitoring/templates/docker-compose.yml.j2
+++ b/roles/monitoring/templates/docker-compose.yml.j2
@@ -7,6 +7,7 @@ services:
     networks:
       {{ proxy_network }}:
       {{ db_network }}:
+      {{ monitoring_network }}:
     volumes:
      - "{{grafana_data_location}}:/var/lib/grafana"
     environment:
@@ -38,7 +39,7 @@ services:
   prometheus:
     image: prom/prometheus
     networks:
-      {{ db_network }}:
+      {{ monitoring_network }}:
     volumes:
      - "{{prometheus_data_location}}:/prometheus"
      - "{{prometheus_config_location}}:/etc/prometheus/prometheus.yml"
@@ -56,6 +57,30 @@ services:
         delay: 5s
         max_attempts: 3
         window: 120s
+  node_exporter:
+    image: quay.io/prometheus/node-exporter:latest
+    command:
+      - '--path.rootfs=/host'
+    network_mode: host
+    pid: host
+    restart: always
+    volumes:
+      - '/:/host:ro,rslave'
+    networks:
+      {{ monitoring_network }}:
+    deploy:
+      update_config:
+        order: start-first
+        failure_action: rollback
+        delay: 10s
+      rollback_config:
+        parallelism: 0
+        order: start-first
+      restart_policy:
+        condition: any
+        delay: 5s
+        max_attempts: 3
+        window: 120s
 
  
 networks:
@@ -63,3 +88,5 @@ networks:
     external: true
   {{ db_network }}:
     external: true
+  {{ monitoring_network }}:
+    external: true
diff --git a/roles/monitoring/files/prometheus.yml b/roles/monitoring/templates/prometheus.yml.j2
similarity index 86%
rename from roles/monitoring/files/prometheus.yml
rename to roles/monitoring/templates/prometheus.yml.j2
index 427a4df..7afd12d 100644
--- a/roles/monitoring/files/prometheus.yml
+++ b/roles/monitoring/templates/prometheus.yml.j2
@@ -1,4 +1,4 @@
-# MANAGED BY ANSIBLE; DO NOT CHANGE HERE
+{{ ansible_managed | comment }}
 
 global:
   scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
@@ -19,10 +19,10 @@ scrape_configs:
   - job_name: "prometheus"
 
     static_configs:
-      - targets: ["localhost:9090"]
+      - targets: ["localhost:{{prometheus_port}}"]
 
   - job_name: node
     # If prometheus-node-exporter is installed, grab stats about the local
     # machine by default.
     static_configs:
-      - targets: ["localhost:9100"]
+      - targets: ["node-exporter:9100"]