added grafana for quotas

Author: hifly81

README.adoc

@@ -2586,6 +2586,8 @@ scripts/tear-down-oauth.sh
 This scenario will show how to set network bandwidth quotas on principal (user).
 The scenario requires a listener with SASL_PLAIN authentication.
 
+Bootstrap will initialize a Prometheus server on port 9090 and a Grafana UI with Kafka Quota dashboard on port 3000.
+
 Start scenario:
 
 [source,bash]
@@ -2617,6 +2619,12 @@ Run a consumer perf test and see how throttling applies:
 docker exec broker kafka-consumer-perf-test --topic test --messages 1000000 --consumer.config /tmp/alice.properties --broker-list broker:9092
 ----
 
+You can visualize metrics using the Grafana dashboard for Kafka Quotas fetched from https://github.com/confluentinc/jmx-monitoring-stacks repository
+
+Open Grafana UI at http://localhost:3000/d/cwWEgYqMz/kafka-quotas
+
+image::images/quotas.png[Kafka Quotas Grafana]
+
 
 Teardown:
 

images/quotas.png

@@ -0,0 +1,30 @@
+global:
+  # How frequently to scrape targets by default.
+  # Default 15s
+  scrape_interval: 60s
+  # How frequently to evaluate rules.
+  # Default 15s
+  evaluation_interval: 15s
+  # How long until a scrape request times out.
+  # Default to 10s.
+  # Required because cp-demo is using cpu throttling, so let's leave enough time to fetch the metrics in particular for the first time as it needs to compile all rexps
+  scrape_timeout: 30s
+
+
+scrape_configs:
+  - job_name: "prometheus"
+    static_configs:
+      - targets: ["localhost:9090"]
+
+
+  - job_name: "kafka-broker"
+    static_configs:
+      - targets:
+          - "broker:1234"
+        labels:
+          env: "dev"
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: hostname
+        regex: '([^:]+)(:[0-9]+)?'
+        replacement: '${1}'

quotas/config/prometheus/prometheus.yml

@@ -9,8 +9,12 @@ services:
     container_name: broker
     ports:
       - "9092:9092"
+      - "9101:9101"
+      - "1234:1234"
     volumes:
       - ./config/alice.properties:/tmp/alice.properties
+      - ./config/agent/jmx_prometheus_javaagent-0.18.0.jar:/tmp/jmx_prometheus_javaagent-0.18.0.jar
+      - ./config/agent/kafka_broker.yml:/tmp/kafka_broker.yml
     environment:
       KAFKA_NODE_ID: 1
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,AUTH:SASL_PLAINTEXT
@@ -35,4 +39,27 @@ services:
         password="admin-secret" \
         user_admin="admin-secret" \
         user_alice="alice-secret";
-      CLUSTER_ID: 'QTnB2tAgTWa1ec5wYon2jg'
+      CLUSTER_ID: 'QTnB2tAgTWa1ec5wYon2jg'
+      KAFKA_JMX_PORT: 9101
+      KAFKA_JMX_HOSTNAME: "localhost"
+      EXTRA_ARGS: -javaagent:/tmp/jmx_prometheus_javaagent-0.18.0.jar=1234:/tmp/kafka_broker.yml
+
+  prometheus:
+    image: prom/prometheus:v2.29.2
+    container_name: prometheus
+    ports:
+      - "9090:9090"
+    volumes:
+      - ./config/prometheus/:/etc/prometheus
+
+  grafana:
+    image: grafana/grafana:8.1.3
+    container_name: grafana
+    environment:
+      - "GF_SECURITY_ADMIN_USER=admin"
+      - "GF_SECURITY_ADMIN_PASSWORD=password"
+      - "GF_USERS_ALLOW_SIGN_UP=false"
+    ports:
+      - "3000:3000"
+    volumes:
+      - ./config/grafana/provisioning/:/etc/grafana/provisioning

quotas/docker-compose.yml

@@ -2,5 +2,20 @@
 
 sh scripts/tear-down.sh
 
+echo "download jmx exporter files..."
+mkdir -p quotas/config/agent
+
+wget -P quotas/config/agent https://raw.githubusercontent.com/confluentinc/jmx-monitoring-stacks/7.2-post/shared-assets/jmx-exporter/kafka_broker.yml
+wget -P quotas/config/agent https://github.com/confluentinc/jmx-monitoring-stacks/raw/7.2-post/shared-assets/jmx-exporter/jmx_prometheus_javaagent-0.18.0.jar
+
+echo "download grafana files..."
+mkdir -p quotas/config/grafana/provisioning/dashboards
+mkdir -p quotas/config/grafana/provisioning/datasources
+
+wget -P quotas/config/grafana/provisioning/datasources https://raw.githubusercontent.com/confluentinc/jmx-monitoring-stacks/7.2-post/jmxexporter-prometheus-grafana/assets/grafana/provisioning/datasources/datasource.yml
+wget -P quotas/config/grafana/provisioning/dashboards https://raw.githubusercontent.com/confluentinc/jmx-monitoring-stacks/7.2-post/jmxexporter-prometheus-grafana/assets/grafana/provisioning/dashboards/dashboard.yml
+wget -P quotas/config/grafana/provisioning/dashboards https://raw.githubusercontent.com/confluentinc/jmx-monitoring-stacks/7.2-post/jmxexporter-prometheus-grafana/assets/grafana/provisioning/dashboards/kafka-quotas.json
+
+
 echo "Starting Kafka cluster..."
 docker-compose -f quotas/docker-compose.yml --env-file .env up -d

scripts/bootstrap-quotas.sh

@@ -2,3 +2,6 @@
 
 echo "Stopping docker containers..."
 docker-compose -f quotas/docker-compose.yml --env-file .env down --volumes
+
+rm -Rf quotas/config/agent
+rm -Rf quotas/config/grafana