Signed-off-by: Kushal Shukla <[email protected]>

1. Added rules in prombench As a Config Map.
2. removed some queries from load-generator and added on rules
3. rename fakewebserver file

Author: kushalShukla-web

prombench/manifests/prombench/benchmark/2_fake-webserver.yaml renamed to prombench/manifests/prombench/benchmark/2a_fake-webserver.yaml

@@ -0,0 +1,25 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: prometheus-rules
+  namespace: prombench-{{ .PR_NUMBER }}
+data:
+  rules.yml: |
+    groups:
+      - name: recording_rules
+        interval: 15s
+        rules:
+          - record: instance:codelab_api_requests:rate5m
+            expr: sum by(instance) (rate(codelab_api_requests_total{method=~"GET|POST"}[5m]))
+
+          - record: instance:node_cpu_usage:rate5m
+            expr: sum by(instance) (rate(node_cpu_seconds_total{mode!="idle"}[5m]))
+
+          - record: instance:codelab_api_request_duration:quantile_99
+            expr: histogram_quantile(0.99, sum by(instance, le) (rate(codelab_api_request_duration_seconds_bucket{method="POST"}[5m])))
+            
+          - record: instance:codelab_api_request_duration_seconds_count:filtered
+            expr: codelab_api_request_duration_seconds_count{method="POST"} unless codelab_api_request_duration_seconds_count{status="500"}
+
+          - record: instance:codelab_api_request_duration_seconds_count:top10
+            expr: topk(10, sum(codelab_api_request_duration_seconds_count) by (method, job))

prombench/manifests/prombench/benchmark/2b_prometheus-rules.yaml

@@ -7,7 +7,11 @@ data:
   prometheus.yml: |
     global:
       scrape_interval: 15s
-
+      evaluation_interval: 15s
+    
+    rule_files:
+      - "/etc/prometheus/rules/rules.yml"
+    
     scrape_configs:
     - job_name: kubelets
       scheme: https
@@ -805,6 +809,8 @@ spec:
           mountPath: /prometheus
         - name: prometheus-executable
           mountPath: /usr/bin
+        - name: prometheus-rules
+          mountPath: /etc/prometheus/rules
         ports:
         - name: prom-web
           containerPort: 9090
@@ -818,6 +824,9 @@ spec:
           path: /mnt/disks/ssd0
       - name: prometheus-executable
         emptyDir: {}
+      - name: prometheus-rules
+        configMap:
+          name: prometheus-rules
       terminationGracePeriodSeconds: 300
       nodeSelector:
         node-name: prometheus-{{ .PR_NUMBER }}

prombench/manifests/prombench/benchmark/3_prometheus-test-pr_deployment.yaml

@@ -7,7 +7,11 @@ data:
   prometheus.yml: |
     global:
       scrape_interval: 15s
-
+      evaluation_interval: 15s
+    
+    rule_files:
+    - "/etc/prometheus/rules/rules.yml"
+    
     scrape_configs:
     - job_name: kubelets
       scheme: https
@@ -777,6 +781,8 @@ spec:
           mountPath: /etc/prometheus
         - name: instance-ssd
           mountPath: /prometheus
+        - name: prometheus-rules
+          mountPath: /etc/prometheus/rules
         ports:
         - name: prom-web
           containerPort: 9090
@@ -788,6 +794,9 @@ spec:
         hostPath:
           # /mnt is where GKE keeps it's SSD.
           path: /mnt/disks/ssd0
+      - name: prometheus-rules
+        configMap:
+          name: prometheus-rules
       terminationGracePeriodSeconds: 300
       nodeSelector:
         node-name: prometheus-{{ .PR_NUMBER }}

prombench/manifests/prombench/benchmark/3_prometheus-test-release_deployment.yaml

@@ -27,7 +27,6 @@ data:
         - expr: sum by(instance) (rate(node_cpu_seconds_total{mode!="idle"}[5m]))
         - expr: sum by(instance) (rate(node_cpu_seconds_total[5m]))
         - expr: sum by(instance) (rate(codelab_api_requests_total[5m]))
-        - expr: sum by(instance) (rate(codelab_api_requests_total{method=~"GET|POST"}[5m]))
       - name: aggr_range
         interval: 30s
         type: range
@@ -36,7 +35,6 @@ data:
         step: 15s
         queries:
         - expr: sum by(image) (container_memory_rss)
-        - expr: sum by(instance) (rate(node_cpu_seconds_total{mode!="idle"}[5m]))
         - expr: sum by(instance) (rate(node_cpu_seconds_total[5m]))
         - expr: sum by(instance) (rate(codelab_api_requests_total[5m]))
         - expr: sum by(instance) (rate(codelab_api_requests_total{method=~"GET|POST"}[5m]))
@@ -53,20 +51,18 @@ data:
         queries:
         - expr: rate(go_memstats_frees_total[5m]) * 60
         - expr: rate(codelab_api_requests_total{method=~"GET|POST"}[5m]) - rate(codelab_api_request_duration_seconds_sum{method=~"GET|POST"}[5m])
-        - expr: rate(go_gc_duration_seconds_sum{job=~"fake-webservers-[0-9]+"}[5m]) / rate(go_gc_duration_seconds_count{job=~"fake-webservers-[0-9]+"}[5m])
+        - expr: rate(go_gc_duration_seconds_sum{job=~"fake-webservers-[0-9]+"}[5m]) / rate(go_gc_duration_seconds_count{job=~"fake-webservers-[0-9]+"}[5m]) 
         - expr: sum by (instance, job) (rate(codelab_api_request_errors_total[5m])) / sum by (instance, job) (rate(go_memstats_mallocs_total[10m]))
       - name: logic_operator
         interval: 30s
         type: instant
         queries:
         - expr: codelab_api_request_duration_seconds_sum{status="200"} or codelab_api_request_duration_seconds_sum{status="500"}
-        - expr: codelab_api_request_duration_seconds_count{method="POST"} unless codelab_api_request_duration_seconds_count{status="500"}
       - name: topk
         interval: 10s
         type: instant
         queries:
         - expr: topk(20, sum(rate(go_gc_duration_seconds_count[5m])) by (instance, job))
-        - expr: topk(10, sum(codelab_api_request_duration_seconds_count) by (method,job))
 ---
 apiVersion: apps/v1
 kind: Deployment