05-thanos-ruler.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: thanos-ruler-rules
  namespace: monitoring
data:
  alert_down_services.rules.yaml: |
    groups:
    - name: metamonitoring
      rules:
      - alert: PrometheusReplicaDown
        annotations:
          message: Prometheus replica in cluster {{$labels.cluster}} has disappeared from Prometheus target discovery.
          runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-prometheusdown
        expr: |
          label_replace(
            absent(sum(up{job="prometheus", cluster="eu1", instance=~".*:9090"}) by (job, cluster) == 2),
            "cluster", "eu1", "",""
          )
          or
          label_replace(
            absent(sum(up{job="prometheus", cluster="us1", instance=~".*:9090"}) by (job, cluster) == 2),
            "cluster", "us1", "",""
          )
        for: 1s # for demo purposes, typically 5m is a fine number.
        labels:
          severity: critical
---
apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
  labels:
    app: thanos-ruler
  name: thanos-ruler
spec:
  replicas: 1
  selector:
    matchLabels:
      app: thanos-ruler
  serviceName: thanos-ruler
  template:
    metadata:
      labels:
        app: thanos-ruler
        thanos-store-api: "true"
    spec:
      containers:
        - name: thanos
          image: improbable/thanos:v0.10.0
          args:
            - rule
            - --log.level=debug
            - --data-dir=/data
            - --eval-interval=15s
            - --cluster.disable
            - --rule-file=/etc/thanos-ruler/*.rules.yaml
            - --alertmanagers.url=http://%%ALERTMANAGER_URL%%
            - --query=thanos-querier.default.svc:9090
            - --objstore.config=$(OBJSTORE_CONFIG)
            # We don't want to override underlying metric's cluster label.
            - --label=ruler_cluster="%%CLUSTER%%"
            - --label=replica="$(POD_NAME)"
          env:
            - name: POD_NAME
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
            - name: OBJSTORE_CONFIG
              valueFrom:
                secretKeyRef:
                  key: thanos.yaml
                  name: thanos-objectstorage
          ports:
            - name: http
              containerPort: 10902
            - name: grpc
              containerPort: 10901
          resources:
            limits:
              cpu: 500m
              memory: 500Mi
            requests:
              cpu: 500m
              memory: 500Mi
          volumeMounts:
            - mountPath: /etc/thanos-ruler
              name: config
      volumes:
        - configMap:
            name: thanos-ruler-rules
          name: config
---
apiVersion: v1
kind: Service
metadata:
  labels:
    app: thanos-ruler
  name: thanos-ruler
spec:
  ports:
    - port: 9090
      protocol: TCP
      targetPort: http
      name: http
  selector:
    statefulset.kubernetes.io/pod-name: thanos-ruler-0
  type: NodePort