kafka: Add topic partition size metric

kafka/changelog.d/23021.added

@@ -0,0 +1 @@
+Add kafka.log.partition.size metric for topic partition disk usage.

kafka/datadog_checks/kafka/data/metrics.yaml

@@ -915,3 +915,21 @@ jmx_metrics:
         - tag_name: kafka_cluster_id
           bean_name: kafka.server:type=KafkaServer,name=ClusterId
           attribute: Value
+
+    #
+    # Topic partition size
+    #
+    - include:
+        domain: 'kafka.log'
+        bean_regex: 'kafka\.log:type=Log,name=Size,topic=(.*),partition=(.*)'
+        attribute:
+          Value:
+            metric_type: gauge
+            alias: kafka.log.partition.size
+        tags:
+          topic: $1
+          partition: $2
+      dynamic_tags:
+        - tag_name: kafka_cluster_id
+          bean_name: kafka.server:type=KafkaServer,name=ClusterId
+          attribute: Value

kafka/metadata.csv

@@ -1,5 +1,6 @@
 metric_name,metric_type,interval,unit_name,per_unit_name,description,orientation,integration,short_name,curated_metric
 kafka.broker.start_time,gauge,10,millisecond,,The start time of the Kafka broker in milliseconds since epoch.,0,kafka,broker start time,
+kafka.log.partition.size,gauge,10,byte,,The size in bytes of a topic partition log on disk.,0,kafka,partition size,
 kafka.consumer.bytes_consumed,gauge,10,byte,second,The average number of bytes consumed per second for a specific topic.,1,kafka,con bytes consumed,
 kafka.consumer.bytes_in,gauge,10,byte,second,Consumer bytes in rate.,1,kafka,con bytes in,
 kafka.consumer.delayed_requests,gauge,10,request,,Number of delayed consumer requests.,-1,kafka,con delayed req,

kafka/tests/common.py

@@ -71,4 +71,6 @@
     "kafka.server.socket.connection_count",
     # Broker
     "kafka.broker.start_time",
+    # Topic partition size
+    "kafka.log.partition.size",
 ]

kafka/tests/compose/docker-compose.yml

@@ -23,10 +23,21 @@ services:
       KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
       KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
-      KAFKA_CREATE_TOPICS: "marvel:2:1,dc:2:1"
       KAFKA_JMX_PORT: 9999
       KAFKA_JMX_HOSTNAME: localhost
     depends_on:
       zookeeper:
         condition: service_healthy
+  kafka-init:
+    image: confluentinc/cp-kafka:${CONFLUENT_VERSION}
+    depends_on:
+      - kafka
+    entrypoint: ["/bin/bash", "-c"]
+    command:
+      - |
+        echo "Waiting for Kafka to be ready..."
+        cub kafka-ready -b localhost:9092 1 60 2>/dev/null || sleep 10
+        kafka-topics --create --if-not-exists --topic marvel --partitions 2 --replication-factor 1 --bootstrap-server kafka:9092
+        kafka-topics --create --if-not-exists --topic dc --partitions 2 --replication-factor 1 --bootstrap-server kafka:9092
+        echo "Topics created."