Dev

VERSION

@@ -1 +1 @@
-v2.2.1
+v2.2.1-dev

docker/clickhouse/config.d/retention.xml

@@ -0,0 +1,16 @@
+<clickhouse>
+    <!--
+      MergeTree TTL deletion is performed during background merges.  The
+      upstream default can defer a TTL merge for four hours; run eligible TTL
+      merges every 15 minutes so monitoring data is reclaimed promptly.
+    -->
+    <merge_tree>
+        <merge_with_ttl_timeout>900</merge_with_ttl_timeout>
+    </merge_tree>
+
+    <!-- Keep ClickHouse's own diagnostic logs from growing without bound. -->
+    <logger>
+        <size>100M</size>
+        <count>3</count>
+    </logger>
+</clickhouse>

docker/docker-compose/base/docker-compose.clickhouse.yml

@@ -4,6 +4,7 @@ services:
     container_name: clickhouse-server
     volumes:
       - ../../create_tables:/docker-entrypoint-initdb.d
+      - ../../clickhouse/config.d/retention.xml:/etc/clickhouse-server/config.d/retention.xml:ro
       - ch_data:/var/lib/clickhouse/
       - ch_logs:/var/log/clickhouse-server/
     environment:

docker/docker-compose/base/docker-compose.kafka.yml

@@ -9,10 +9,19 @@ x-kafka-broker-env: &kafka-broker-env
   KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
   KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2
   KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
-  KAFKA_LOG_RETENTION_HOURS: 4
-  KAFKA_LOG_RETENTION_BYTES: 10737418240
-  KAFKA_LOG_SEGMENT_BYTES: 1073741824
-  KAFKA_LOG_CLEANUP_POLICY: delete
+  # Retention is evaluated only for closed log segments.  Keep the segment
+  # smaller and roll it frequently so that the four-hour retention period is
+  # effective even for low-volume partitions.
+  #
+  # log.retention.bytes applies to *each partition replica*, not to the broker.
+  # With 12 partitions and replication factor 3, a 10 GiB value can consume
+  # tens of GiB across the three broker volumes.
+  KAFKA_LOG_RETENTION_HOURS: "${KAFKA_LOG_RETENTION_HOURS:-4}"
+  KAFKA_LOG_RETENTION_BYTES: "${KAFKA_LOG_RETENTION_BYTES:-16777216}"
+  KAFKA_LOG_SEGMENT_BYTES: "${KAFKA_LOG_SEGMENT_BYTES:-4194304}"
+  KAFKA_LOG_SEGMENT_MS: "${KAFKA_LOG_SEGMENT_MS:-900000}"
+  KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: "${KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS:-300000}"
+  KAFKA_LOG_CLEANUP_POLICY: "${KAFKA_LOG_CLEANUP_POLICY:-delete}"
 
 services:
   kafka1:

docker/docker_swarm/docker-compose.swarm.yml

@@ -11,9 +11,14 @@ x-kafka-broker-env: &kafka-broker-env
   KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: "${KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR:-3}"
   KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: "${KAFKA_TRANSACTION_STATE_LOG_MIN_ISR:-2}"
   KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "${KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR:-3}"
+  # Retention applies only to closed segments and retention.bytes is per
+  # partition replica.  Frequent, small segments make the retention window
+  # effective and keep the replicated pipeline topics bounded on disk.
   KAFKA_LOG_RETENTION_HOURS: "${KAFKA_LOG_RETENTION_HOURS:-4}"
-  KAFKA_LOG_RETENTION_BYTES: "${KAFKA_LOG_RETENTION_BYTES:-10737418240}"
-  KAFKA_LOG_SEGMENT_BYTES: "${KAFKA_LOG_SEGMENT_BYTES:-1073741824}"
+  KAFKA_LOG_RETENTION_BYTES: "${KAFKA_LOG_RETENTION_BYTES:-16777216}"
+  KAFKA_LOG_SEGMENT_BYTES: "${KAFKA_LOG_SEGMENT_BYTES:-4194304}"
+  KAFKA_LOG_SEGMENT_MS: "${KAFKA_LOG_SEGMENT_MS:-900000}"
+  KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: "${KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS:-300000}"
   KAFKA_LOG_CLEANUP_POLICY: "${KAFKA_LOG_CLEANUP_POLICY:-delete}"
 
 x-hamstring-kafka-wait-env: &hamstring-kafka-wait-env
@@ -113,6 +118,7 @@ services:
         mode: ingress
     volumes:
       - "${HAMSTRING_ROOT:-../..}/docker/create_tables:/docker-entrypoint-initdb.d"
+      - "${HAMSTRING_ROOT:-../..}/docker/clickhouse/config.d/retention.xml:/etc/clickhouse-server/config.d/retention.xml:ro"
       - ch_data:/var/lib/clickhouse/
       - ch_logs:/var/log/clickhouse-server/
     healthcheck:

docs/monitoring.rst

@@ -33,6 +33,32 @@ All modules send their monitoring-relevant information to Kafka, from which it i
 default configuration defined in ``docker-compose.yml``, `Grafana` automatically loads the dashboard views and fills
 them with the data in `ClickHouse`. The dashboard views can then be observed on ``localhost:3000`` (by default).
 
+Storage retention
+-----------------
+
+HAMSTRING treats Kafka as transient transport. By default each Kafka partition replica has a four-hour / 16 MiB
+retention target, whichever limit is reached first. Kafka can delete only *closed* log segments, so the Compose files
+also roll a segment every 15 minutes (or after 4 MiB); this makes time-based retention work for low-volume topics.
+All Kafka values can be overridden at startup, for example:
+
+.. code-block:: console
+
+   KAFKA_LOG_RETENTION_HOURS=12 KAFKA_LOG_RETENTION_BYTES=67108864 \\
+     HOST_IP=127.0.0.1 docker compose -f docker/docker-compose.yml --profile prod up
+
+``KAFKA_LOG_RETENTION_BYTES`` is a limit **per partition replica**, rather than a per-broker or per-volume limit.
+Increasing the partition count, replication factor, or this value increases the possible disk usage accordingly.
+
+ClickHouse monitoring tables have a one-day TTL (alerts have a 60-day TTL). TTL deletion happens in background
+merges; the supplied ClickHouse configuration checks for TTL work every 15 minutes. Its server logs are also rotated
+at 100 MiB with three retained files.
+
+The new retention settings apply after restarting Kafka and ClickHouse. They reclaim data as the brokers close new
+segments and ClickHouse performs TTL merges; they do not remove a stopped stack's volume files immediately. If this is
+a disposable local environment and its historical data is not needed, stop the stack and remove its volumes with
+``docker compose -f docker/docker-compose.yml down -v`` before starting it again. Do not remove production volumes as
+a retention operation.
+
 `Datatest` mode
 ---------------