From 4542706892ba7ffc2951572765f1129d0aefd4b6 Mon Sep 17 00:00:00 2001 From: maldwg Date: Fri, 3 Jul 2026 11:53:09 +0200 Subject: [PATCH] Fix kafka grafana and clickhouse upgrade pitfalls --- .../base/docker-compose.clickhouse.yml | 3 + .../base/docker-compose.grafana.yml | 2 + .../dev/docker-compose.monitoring.yml | 2 + .../prod/docker-compose.monitoring.yml | 2 + docker/docker_swarm/docker-compose.swarm.yml | 40 ++++------- .../dashboards/latencies.json | 68 +++++++++---------- docker/grafana-provisioning/datasources.yaml | 4 +- src/monitoring/clickhouse_batch_sender.py | 12 +++- src/monitoring/monitoring_agent.py | 2 +- 9 files changed, 71 insertions(+), 64 deletions(-) diff --git a/docker/docker-compose/base/docker-compose.clickhouse.yml b/docker/docker-compose/base/docker-compose.clickhouse.yml index 58985d8f..fda33561 100644 --- a/docker/docker-compose/base/docker-compose.clickhouse.yml +++ b/docker/docker-compose/base/docker-compose.clickhouse.yml @@ -6,6 +6,9 @@ services: - ../../create_tables:/docker-entrypoint-initdb.d - ch_data:/var/lib/clickhouse/ - ch_logs:/var/log/clickhouse-server/ + environment: + - CLICKHOUSE_USER=${CLICKHOUSE_USER:-default} + - CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD:-hamstring} networks: - hamstring restart: "unless-stopped" diff --git a/docker/docker-compose/base/docker-compose.grafana.yml b/docker/docker-compose/base/docker-compose.grafana.yml index a101eba4..422c00ed 100644 --- a/docker/docker-compose/base/docker-compose.grafana.yml +++ b/docker/docker-compose/base/docker-compose.grafana.yml @@ -15,6 +15,8 @@ services: - GF_SECURITY_ADMIN_USER=admin - GF_SECURITY_ADMIN_PASSWORD=admin - GF_INSTALL_PLUGINS=grafana-clickhouse-datasource + - CLICKHOUSE_USER=${CLICKHOUSE_USER:-default} + - CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD:-hamstring} healthcheck: test: [ "CMD-SHELL", "nc -z localhost 3000" ] interval: 10s diff --git a/docker/docker-compose/dev/docker-compose.monitoring.yml b/docker/docker-compose/dev/docker-compose.monitoring.yml index fed026b9..103e433f 100644 --- a/docker/docker-compose/dev/docker-compose.monitoring.yml +++ b/docker/docker-compose/dev/docker-compose.monitoring.yml @@ -21,3 +21,5 @@ services: - hamstring environment: - GROUP_ID=monitoring_agent + - CLICKHOUSE_USER=${CLICKHOUSE_USER:-default} + - CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD:-hamstring} diff --git a/docker/docker-compose/prod/docker-compose.monitoring.yml b/docker/docker-compose/prod/docker-compose.monitoring.yml index 8d38f2c8..77c46167 100644 --- a/docker/docker-compose/prod/docker-compose.monitoring.yml +++ b/docker/docker-compose/prod/docker-compose.monitoring.yml @@ -19,3 +19,5 @@ services: - hamstring environment: - GROUP_ID=monitoring_agent + - CLICKHOUSE_USER=${CLICKHOUSE_USER:-default} + - CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD:-hamstring} diff --git a/docker/docker_swarm/docker-compose.swarm.yml b/docker/docker_swarm/docker-compose.swarm.yml index 3c164450..dd354518 100644 --- a/docker/docker_swarm/docker-compose.swarm.yml +++ b/docker/docker_swarm/docker-compose.swarm.yml @@ -19,8 +19,7 @@ x-kafka-broker-env: &kafka-broker-env services: kafka1: image: "${KAFKA_IMAGE:-confluentinc/cp-kafka:8.2.2}" - networks: - - hamstring + networks: [hamstring] ports: - target: 8097 published: "${KAFKA1_EXTERNAL_PORT:-8097}" @@ -33,19 +32,13 @@ services: KAFKA_LISTENERS: INTERNAL://0.0.0.0:19092,EXTERNAL://0.0.0.0:8097,CONTROLLER://0.0.0.0:29093 volumes: - kafka-broker-data1:/var/lib/kafka/data - healthcheck: - test: ["CMD-SHELL", "kafka-broker-api-versions --bootstrap-server localhost:19092 >/dev/null 2>&1"] - interval: 30s - timeout: 10s - retries: 10 deploy: restart_policy: condition: any kafka2: image: "${KAFKA_IMAGE:-confluentinc/cp-kafka:8.2.2}" - networks: - - hamstring + networks: [hamstring] ports: - target: 8098 published: "${KAFKA2_EXTERNAL_PORT:-8098}" @@ -59,19 +52,13 @@ services: KAFKA_LOG4J_LOGGERS: "${KAFKA_LOG4J_LOGGERS:-kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO}" volumes: - kafka-broker-data2:/var/lib/kafka/data - healthcheck: - test: ["CMD-SHELL", "kafka-broker-api-versions --bootstrap-server localhost:19093 >/dev/null 2>&1"] - interval: 30s - timeout: 10s - retries: 10 deploy: restart_policy: condition: any kafka3: image: "${KAFKA_IMAGE:-confluentinc/cp-kafka:8.2.2}" - networks: - - hamstring + networks: [hamstring] ports: - target: 8099 published: "${KAFKA3_EXTERNAL_PORT:-8099}" @@ -85,11 +72,6 @@ services: KAFKA_LOG4J_LOGGERS: "${KAFKA_LOG4J_LOGGERS:-kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO}" volumes: - kafka-broker-data3:/var/lib/kafka/data - healthcheck: - test: ["CMD-SHELL", "kafka-broker-api-versions --bootstrap-server localhost:19094 >/dev/null 2>&1"] - interval: 30s - timeout: 10s - retries: 10 deploy: restart_policy: condition: any @@ -98,6 +80,9 @@ services: image: "${CLICKHOUSE_IMAGE:-clickhouse/clickhouse-server:26.5-alpine}" networks: - hamstring + environment: + CLICKHOUSE_USER: "${CLICKHOUSE_USER:-default}" + CLICKHOUSE_PASSWORD: "${CLICKHOUSE_PASSWORD:-hamstring}" ports: - target: 8123 published: "${CLICKHOUSE_HTTP_PORT:-8123}" @@ -121,7 +106,7 @@ services: condition: any grafana: - image: "${GRAFANA_IMAGE:-grafana/grafana:13.1.0-debian}" + image: "${GRAFANA_IMAGE:-grafana/grafana:13.0.3-slim}" networks: - hamstring ports: @@ -137,10 +122,13 @@ services: GF_SECURITY_ADMIN_USER: "${GRAFANA_ADMIN_USER:-admin}" GF_SECURITY_ADMIN_PASSWORD: "${GRAFANA_ADMIN_PASSWORD:-admin}" GF_INSTALL_PLUGINS: "${GRAFANA_INSTALL_PLUGINS:-grafana-clickhouse-datasource}" + CLICKHOUSE_USER: "${CLICKHOUSE_USER:-default}" + CLICKHOUSE_PASSWORD: "${CLICKHOUSE_PASSWORD:-hamstring}" healthcheck: test: ["CMD-SHELL", "nc -z localhost 3000"] interval: 10s - timeout: 5s + timeout: 30s + start_period: 120s retries: 3 deploy: restart_policy: @@ -188,6 +176,8 @@ services: environment: GROUP_ID: "${MONITORING_AGENT_GROUP_ID:-monitoring_agent}" HAMSTRING_CONFIG_CHECKSUM: "${HAMSTRING_CONFIG_CHECKSUM:-d9bc812b76c2b58c633b412a6c07f981f3c4d787}" + CLICKHOUSE_USER: "${CLICKHOUSE_USER:-default}" + CLICKHOUSE_PASSWORD: "${CLICKHOUSE_PASSWORD:-hamstring}" deploy: restart_policy: condition: any @@ -329,10 +319,6 @@ networks: hamstring: driver: overlay attachable: true - ipam: - driver: default - config: - - subnet: "${HAMSTRING_SUBNET:-172.27.0.0/16}" volumes: ch_data: diff --git a/docker/grafana-provisioning/dashboards/latencies.json b/docker/grafana-provisioning/dashboards/latencies.json index 3e686ff8..206efa9d 100644 --- a/docker/grafana-provisioning/dashboards/latencies.json +++ b/docker/grafana-provisioning/dashboards/latencies.json @@ -87,7 +87,7 @@ "overrides": [] }, "gridPos": { - "h": 3, + "h": 5, "w": 19, "x": 0, "y": 0 @@ -150,7 +150,7 @@ "uid": "grafana" }, "gridPos": { - "h": 7, + "h": 10, "w": 5, "x": 19, "y": 0 @@ -227,7 +227,7 @@ "h": 7, "w": 14, "x": 0, - "y": 3 + "y": 5 }, "id": 79, "options": { @@ -340,7 +340,7 @@ "h": 7, "w": 5, "x": 14, - "y": 3 + "y": 5 }, "id": 43, "options": { @@ -470,7 +470,7 @@ "h": 4, "w": 11, "x": 0, - "y": 10 + "y": 12 }, "id": 78, "options": { @@ -591,7 +591,7 @@ "h": 4, "w": 8, "x": 11, - "y": 10 + "y": 12 }, "id": 80, "options": { @@ -698,7 +698,7 @@ "h": 3, "w": 19, "x": 0, - "y": 14 + "y": 16 }, "id": 81, "options": { @@ -788,7 +788,7 @@ "h": 3, "w": 19, "x": 0, - "y": 17 + "y": 19 }, "id": 82, "options": { @@ -843,7 +843,7 @@ "h": 1, "w": 24, "x": 0, - "y": 20 + "y": 22 }, "id": 33, "panels": [], @@ -969,7 +969,7 @@ "h": 6, "w": 6, "x": 0, - "y": 21 + "y": 23 }, "id": 22, "options": { @@ -1069,7 +1069,7 @@ "h": 3, "w": 5, "x": 6, - "y": 21 + "y": 23 }, "id": 29, "options": { @@ -1236,7 +1236,7 @@ "h": 6, "w": 6, "x": 12, - "y": 21 + "y": 23 }, "id": 25, "options": { @@ -1336,7 +1336,7 @@ "h": 3, "w": 5, "x": 18, - "y": 21 + "y": 23 }, "id": 35, "options": { @@ -1416,7 +1416,7 @@ "h": 3, "w": 5, "x": 6, - "y": 24 + "y": 26 }, "id": 28, "options": { @@ -1495,7 +1495,7 @@ "h": 3, "w": 5, "x": 18, - "y": 24 + "y": 26 }, "id": 36, "options": { @@ -1662,7 +1662,7 @@ "h": 6, "w": 6, "x": 0, - "y": 27 + "y": 29 }, "id": 23, "options": { @@ -1762,7 +1762,7 @@ "h": 3, "w": 5, "x": 6, - "y": 27 + "y": 29 }, "id": 30, "options": { @@ -1929,7 +1929,7 @@ "h": 6, "w": 6, "x": 12, - "y": 27 + "y": 29 }, "id": 26, "options": { @@ -2029,7 +2029,7 @@ "h": 3, "w": 5, "x": 18, - "y": 27 + "y": 29 }, "id": 37, "options": { @@ -2109,7 +2109,7 @@ "h": 3, "w": 5, "x": 6, - "y": 30 + "y": 32 }, "id": 31, "options": { @@ -2188,7 +2188,7 @@ "h": 3, "w": 5, "x": 18, - "y": 30 + "y": 32 }, "id": 38, "options": { @@ -2354,7 +2354,7 @@ "h": 6, "w": 6, "x": 0, - "y": 33 + "y": 35 }, "id": 24, "options": { @@ -2454,7 +2454,7 @@ "h": 3, "w": 5, "x": 6, - "y": 33 + "y": 35 }, "id": 32, "options": { @@ -2621,7 +2621,7 @@ "h": 6, "w": 6, "x": 12, - "y": 33 + "y": 35 }, "id": 27, "options": { @@ -2721,7 +2721,7 @@ "h": 3, "w": 5, "x": 18, - "y": 33 + "y": 35 }, "id": 39, "options": { @@ -2801,7 +2801,7 @@ "h": 3, "w": 5, "x": 6, - "y": 36 + "y": 38 }, "id": 34, "options": { @@ -2880,7 +2880,7 @@ "h": 3, "w": 5, "x": 18, - "y": 36 + "y": 38 }, "id": 40, "options": { @@ -2934,7 +2934,7 @@ "h": 1, "w": 24, "x": 0, - "y": 39 + "y": 41 }, "id": 71, "panels": [], @@ -3059,7 +3059,7 @@ "h": 6, "w": 8, "x": 0, - "y": 40 + "y": 42 }, "id": 74, "options": { @@ -3246,7 +3246,7 @@ "h": 6, "w": 8, "x": 8, - "y": 40 + "y": 42 }, "id": 75, "options": { @@ -3431,7 +3431,7 @@ "h": 6, "w": 8, "x": 16, - "y": 40 + "y": 42 }, "id": 76, "options": { @@ -3544,7 +3544,7 @@ "h": 3, "w": 8, "x": 0, - "y": 46 + "y": 48 }, "id": 57, "options": { @@ -3637,7 +3637,7 @@ "h": 3, "w": 8, "x": 8, - "y": 46 + "y": 48 }, "id": 48, "options": { @@ -3730,7 +3730,7 @@ "h": 3, "w": 8, "x": 16, - "y": 46 + "y": 48 }, "id": 49, "options": { diff --git a/docker/grafana-provisioning/datasources.yaml b/docker/grafana-provisioning/datasources.yaml index 57c157fa..0c990048 100644 --- a/docker/grafana-provisioning/datasources.yaml +++ b/docker/grafana-provisioning/datasources.yaml @@ -9,8 +9,10 @@ datasources: protocol: native - username: default + username: ${CLICKHOUSE_USER} tlsSkipVerify: false + secureJsonData: + password: ${CLICKHOUSE_PASSWORD} - name: prometheus type: prometheus diff --git a/src/monitoring/clickhouse_batch_sender.py b/src/monitoring/clickhouse_batch_sender.py index 8bb4ae54..c4c21c46 100644 --- a/src/monitoring/clickhouse_batch_sender.py +++ b/src/monitoring/clickhouse_batch_sender.py @@ -19,12 +19,22 @@ CLICKHOUSE_HOSTNAME = CONFIG["environment"]["monitoring"]["clickhouse_server"][ "hostname" ] +CLICKHOUSE_USERNAME = os.getenv("CLICKHOUSE_USER", "default") +CLICKHOUSE_PASSWORD = os.getenv("CLICKHOUSE_PASSWORD", "hamstring") BATCH_SIZE = CONFIG["pipeline"]["monitoring"]["clickhouse_connector"]["batch_size"] BATCH_TIMEOUT = CONFIG["pipeline"]["monitoring"]["clickhouse_connector"][ "batch_timeout" ] +def create_clickhouse_client(): + return clickhouse_connect.get_client( + host=CLICKHOUSE_HOSTNAME, + username=CLICKHOUSE_USERNAME, + password=CLICKHOUSE_PASSWORD, + ) + + @dataclass class Table: """Defines the table name and allowed column fields with types. @@ -227,7 +237,7 @@ def __init__(self): def _connect_client(self): return retry_forever( - lambda: clickhouse_connect.get_client(host=CLICKHOUSE_HOSTNAME), + create_clickhouse_client, "ClickHouse client connection", ) diff --git a/src/monitoring/monitoring_agent.py b/src/monitoring/monitoring_agent.py index f69ec9cc..059fd938 100644 --- a/src/monitoring/monitoring_agent.py +++ b/src/monitoring/monitoring_agent.py @@ -51,7 +51,7 @@ def _iter_statements(sql_content: str): sql_content = _load_contents(file_path) with retry_forever( - lambda: clickhouse_connect.get_client(host=CLICKHOUSE_HOSTNAME), + create_clickhouse_client, "ClickHouse table preparation connection", retry_config=RETRY_CONFIG, ) as client: