feat: monitoring

captcha/docker-compose.yml

@@ -35,6 +35,36 @@ services:
       options:
         max-size: 10M
         max-file: 3
+
+  postgres-exporter:
+    hostname: captcha.postgres-exporter
+    image: quay.io/prometheuscommunity/postgres-exporter
+    platform: linux/amd64
+    environment:
+      DATA_SOURCE_NAME: "${POSTGRES_URL}"
+    networks:
+      - monitoring
+      - internal
+    deploy:
+      mode: replicated
+      replicas: 1
+      restart_policy:
+        condition: unless-stopped
+        delay: 30s
+        window: 120s
+      resources:
+        limits:
+          memory: 200MB
+          cpus: '1'
+        reservations:
+          memory: 50MB
+          cpus: '0.10'
+    logging:
+      driver: local
+      options:
+        max-size: 1M
+        max-file: 3
+
   application:
     image: ghcr.io/teknologi-umum/captcha:edge
     environment:
@@ -127,3 +157,5 @@ networks:
     ipam:
       config:
         - subnet: 172.16.22.0/28
+  monitoring:
+    external: true

monitoring/.env.example

@@ -0,0 +1,2 @@
+GF_SECURITY_ADMIN_USER=
+GF_SECURITY_ADMIN_PASSWORD=

monitoring/docker-compose.yml

@@ -0,0 +1,88 @@
+services:
+  prometheus:
+    image: prom/prometheus:latest
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml
+      - prometheus-data:/prometheus
+    deploy:
+      mode: replicated
+      replicas: 1
+      restart_policy:
+        condition: on-failure
+        delay: 30s
+        max_attempts: 10
+        window: 120s
+      resources:
+        limits:
+          memory: 250MB
+          cpus: '1'
+        reservations:
+          memory: 50MB
+          cpus: '0.10'
+    healthcheck:
+      test: "wget -O - -T 5 -S http://localhost:9090/api/v1/status/runtimeinfo"
+      interval: 30s
+      timeout: 10s
+      retries: 5
+    networks:
+      - monitoring
+
+  grafana:
+    image: grafana/grafana:latest
+    user: '472'
+    deploy:
+      mode: replicated
+      replicas: 1
+      restart_policy:
+        condition: on-failure
+        delay: 30s
+        max_attempts: 10
+        window: 120s
+      resources:
+        limits:
+          memory: 250MB
+          cpus: '1'
+        reservations:
+          memory: 50MB
+          cpus: '0.10'
+    environment:
+      GF_INSTALL_PLUGINS: 'grafana-clock-panel,grafana-simple-json-datasource'
+      GF_SECURITY_ADMIN_USER:
+      GF_SECURITY_ADMIN_PASSWORD:
+      GF_USERS_ALLOW_SIGN_UP: false
+    healthcheck:
+      test: "wget -O - -T 5 -S http://localhost:3000/api/health"
+      interval: 30s
+      timeout: 10s
+      retries: 5
+    volumes:
+      - grafana-data:/var/lib/grafana
+      - ./grafana-datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.grafana.entrypoints=web,websecure"
+      - "traefik.http.routers.grafana.rule=Host(`grafana.teknologiumum.com`)"
+      - "traefik.http.routers.grafana.tls.certresolver=tlsresolver"
+      - "traefik.http.routers.grafana.middlewares=grafana-rate,grafana-redirectscheme"
+      - "traefik.http.services.grafana.loadbalancer.server.port=3000"
+      - "traefik.http.services.grafana.loadbalancer.server.scheme=http"
+      - "traefik.http.services.grafana.loadbalancer.healthcheck.interval=30s"
+      - "traefik.http.services.grafana.loadbalancer.healthcheck.path=/api/health"
+      - "traefik.http.middlewares.grafana-rate.ratelimit.average=100"
+      - "traefik.http.middlewares.grafana-rate.ratelimit.burst=20"
+      - "traefik.http.middlewares.grafana-rate.ratelimit.period=1s"
+      - "traefik.http.middlewares.grafana-redirectscheme.redirectscheme.scheme=https"
+      - "traefik.http.middlewares.grafana-redirectscheme.redirectscheme.permanent=true"
+    depends_on:
+      prometheus:
+        condition: service_healthy
+    networks:
+      - monitoring
+
+volumes:
+  prometheus-data:
+  grafana-data:
+
+networks:
+  monitoring:
+    external: true

monitoring/grafana-datasource.yml

@@ -0,0 +1,50 @@
+# config file version
+apiVersion: 1
+
+# list of datasources that should be deleted from the database
+deleteDatasources:
+  - name: Prometheus
+    orgId: 1
+
+# list of datasources to insert/update depending
+# whats available in the database
+datasources:
+  # <string, required> name of the datasource. Required
+  - name: Prometheus
+    # <string, required> datasource type. Required
+    type: prometheus
+    # <string, required> access mode. direct or proxy. Required
+    access: proxy
+    # <int> org id. will default to orgId 1 if not specified
+    orgId: 1
+    # <string> url
+    url: http://prometheus:9090
+    # <string> database password, if used
+    # password:
+    # <string> database user, if used
+    # user:
+    # <string> database name, if used
+    # database:
+    # <bool> enable/disable basic auth
+    basicAuth: false
+    # <string> basic auth username, if used
+    # basicAuthUser:
+    # <string> basic auth password, if used
+    # basicAuthPassword: 
+    # <bool> enable/disable with credentials headers
+    withCredentials:
+    # <bool> mark as default datasource. Max one per org
+    isDefault: true
+    # <map> fields that will be converted to json and stored in json_data
+    jsonData:
+      graphiteVersion: "1.1"
+      tlsAuth: false
+      tlsAuthWithCACert: false
+    # <string> json object of data that will be encrypted.
+    # secureJsonData:
+    #   tlsCACert: "..."
+    #   tlsClientCert: "..."
+    #   tlsClientKey: "..."
+    version: 1
+    # <bool> allow users to edit datasources from the UI.
+    editable: false

monitoring/prometheus.yml

@@ -0,0 +1,35 @@
+global:
+  # How frequently to scrape targets by default.
+  scrape_interval: 30s
+
+  # How long until a scrape request times out.
+  scrape_timeout: 15s
+
+  # How frequently to evaluate rules.
+  evaluation_interval: 10m
+
+# A list of scrape configurations.
+scrape_configs:
+  - job_name: Traefik
+    static_configs:
+      - labels:
+          application: Traefik
+        targets:
+          - traefik:8082
+
+  - job_name: Uptime Kuma
+    static_configs:
+      - labels:
+          application: Uptime Kuma
+        targets:
+          - 192.168.193.71:43241
+    basic_auth:
+      username: <your user>
+      password: <your password>
+
+  - job_name: Captcha Postgres
+    static_configs:
+      - labels:
+          application: PostgreSQL
+        targets:
+          - captcha.postgres-exporter:9187

traefik/docker-compose.yml

@@ -19,6 +19,9 @@ services:
       - "--entrypoints.websecure.http3"
       - "--global.sendanonymoususage=true"
       - "--log.level=INFO"
+      - "--metrics.prometheus=true"
+      - "--entryPoints.metrics.address=:8082"
+      - "--metrics.prometheus.entryPoint=metrics"
     environment:
       - TZ=UTC
     ports:
@@ -79,6 +82,7 @@ services:
       - projects
       - conference
       - gold
+      - monitoring
 
 networks:
   pesto:
@@ -120,3 +124,5 @@ networks:
     external: true
   gold:
     external: true
+  monitoring:
+    external: true