add node_exporter as container for prometheus

roles/monitoring/defaults/main.yml

@@ -1,5 +1,6 @@
 ---
 monitoring_domain: monitoring.{{ app_domain }}
+monitoring_network: monitoring
 grafana_install_location: "/srv/grafana"
 grafana_data_location: "{{ grafana_install_location }}/data"
 
@@ -8,3 +9,4 @@ prometheus_data_location: "{{ prometheus_install_location }}/data"
 prometheus_config_location: "{{ prometheus_install_location }}/prometheus.yml"
 
 grafana_port: "3000"
+prometheus_port: "9090"

roles/monitoring/tasks/main.yml

@@ -22,6 +22,16 @@
     - "{{ prometheus_data_location }}"
   become: true
 
+- name: Create networks
+  community.docker.docker_network:
+    name: "{{ item }}"
+    driver: "overlay"
+    driver_options:
+      com.docker.network.bridge.name: "{{ item }}"
+  become: true
+  loop:
+    - "{{ monitoring_network }}"
+
 - name: Create docker-compose.yml
   ansible.builtin.template:
     src: docker-compose.yml.j2
@@ -36,8 +46,8 @@
     - wait for grafana
 
 - name: Copy prometheus config
-  ansible.builtin.copy:
-    src: prometheus.yml
+  ansible.builtin.template:
+    src: prometheus.yml.j2
     dest: "{{ prometheus_config_location }}"
     owner: root
     group: root

roles/monitoring/templates/docker-compose.yml.j2

@@ -7,6 +7,7 @@ services:
     networks:
       {{ proxy_network }}:
       {{ db_network }}:
+      {{ monitoring_network }}:
     volumes:
      - "{{grafana_data_location}}:/var/lib/grafana"
     environment:
@@ -38,7 +39,7 @@ services:
   prometheus:
     image: prom/prometheus
     networks:
-      {{ db_network }}:
+      {{ monitoring_network }}:
     volumes:
      - "{{prometheus_data_location}}:/prometheus"
      - "{{prometheus_config_location}}:/etc/prometheus/prometheus.yml"
@@ -56,10 +57,36 @@ services:
         delay: 5s
         max_attempts: 3
         window: 120s
+  node_exporter:
+    image: quay.io/prometheus/node-exporter:latest
+    command:
+      - '--path.rootfs=/host'
+    network_mode: host
+    pid: host
+    restart: always
+    volumes:
+      - '/:/host:ro,rslave'
+    networks:
+      {{ monitoring_network }}:
+    deploy:
+      update_config:
+        order: start-first
+        failure_action: rollback
+        delay: 10s
+      rollback_config:
+        parallelism: 0
+        order: start-first
+      restart_policy:
+        condition: any
+        delay: 5s
+        max_attempts: 3
+        window: 120s
 
 
 networks:
   {{ proxy_network }}:
     external: true
   {{ db_network }}:
     external: true
+  {{ monitoring_network }}:
+    external: true

roles/monitoring/files/prometheus.yml → roles/monitoring/templates/prometheus.yml.j2

@@ -1,4 +1,4 @@
-# MANAGED BY ANSIBLE; DO NOT CHANGE HERE
+{{ ansible_managed | comment }}
 
 global:
   scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
@@ -19,10 +19,10 @@ scrape_configs:
   - job_name: "prometheus"
 
     static_configs:
-      - targets: ["localhost:9090"]
+      - targets: ["localhost:{{prometheus_port}}"]
 
   - job_name: node
     # If prometheus-node-exporter is installed, grab stats about the local
     # machine by default.
     static_configs:
-      - targets: ["localhost:9100"]
+      - targets: ["node-exporter:9100"]