diff --git a/.github/workflows/build_test_linux.yml b/.github/workflows/build_test_linux.yml index cb462c97..13e54f89 100644 --- a/.github/workflows/build_test_linux.yml +++ b/.github/workflows/build_test_linux.yml @@ -5,8 +5,6 @@ on: - main - dev pull_request: - branches: - - main jobs: build_test_linux: name: ${{ matrix.python-version }} on ${{ matrix.os }} @@ -15,8 +13,10 @@ jobs: fail-fast: false matrix: os: [ "ubuntu-latest" ] - python-version: [ "3.11", - "3.10", + python-version: [ + "3.12", + "3.13", + "3.14" ] steps: diff --git a/.github/workflows/build_test_macos.yml b/.github/workflows/build_test_macos.yml index c8863956..408ea62e 100644 --- a/.github/workflows/build_test_macos.yml +++ b/.github/workflows/build_test_macos.yml @@ -5,8 +5,6 @@ on: - main - dev pull_request: - branches: - - main jobs: build_test_macos: name: ${{ matrix.python-version }} on ${{ matrix.os }} @@ -15,8 +13,9 @@ jobs: fail-fast: false matrix: os: [ "macos-14" ] - python-version: [ "3.11", - "3.10", + python-version: [ "3.12", + "3.13", + "3.14" ] steps: diff --git a/.github/workflows/formatting.yml b/.github/workflows/formatting.yml index ea4f6182..530be8c5 100644 --- a/.github/workflows/formatting.yml +++ b/.github/workflows/formatting.yml @@ -6,8 +6,6 @@ on: - main - dev pull_request: - branches: - - main jobs: build: @@ -16,10 +14,10 @@ jobs: steps: - uses: actions/checkout@v4.2.0 - - name: Setup py-3.11 + - name: Setup py-3.14 uses: actions/setup-python@v4 with: - python-version: '3.11' + python-version: '3.14' cache: 'pip' cache-dependency-path: | requirements/requirements.*.txt diff --git a/.github/workflows/publish_dev.yml b/.github/workflows/publish_dev.yml index 81114a36..daa2e4c7 100644 --- a/.github/workflows/publish_dev.yml +++ b/.github/workflows/publish_dev.yml @@ -124,7 +124,7 @@ jobs: ref: ${{ github.event.pull_request.base.ref }} - name: Setup Docker buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Log into registry ${{ env.REGISTRY }} uses: docker/login-action@v3 @@ -143,10 +143,10 @@ jobs: type=raw,value=latest-dev - name: Build and push Docker image - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: . push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - file: ./docker/dockerfiles/Dockerfile.${{ matrix.container }} \ No newline at end of file + file: ./docker/dockerfiles/Dockerfile.${{ matrix.container }} diff --git a/.github/workflows/publish_main.yml b/.github/workflows/publish_main.yml index a2bb0234..c4dd1bc9 100644 --- a/.github/workflows/publish_main.yml +++ b/.github/workflows/publish_main.yml @@ -149,7 +149,7 @@ jobs: echo "major_minor=v${MAJOR}.${MINOR}" >> "$GITHUB_OUTPUT" - name: Setup Docker buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Log into registry ${{ env.REGISTRY }} uses: docker/login-action@v3 @@ -170,7 +170,7 @@ jobs: type=raw,value=latest - name: Build and push Docker image - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: . push: true diff --git a/.github/workflows/test_coverage.yml b/.github/workflows/test_coverage.yml index c9adffe6..b8a3d651 100644 --- a/.github/workflows/test_coverage.yml +++ b/.github/workflows/test_coverage.yml @@ -17,7 +17,7 @@ jobs: fail-fast: false matrix: os: [ "ubuntu-latest" ] - python-version: [ "3.11" ] + python-version: [ "3.14" ] permissions: issues: write pull-requests: write diff --git a/README.md b/README.md index 79667bce..b8650b60 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,10 @@ ## Getting Started +### Requirements + +HAMSTRING currently supports Python 3.12 through 3.14. + #### Run **HAMSTRING** using Docker Compose: ```sh HOST_IP=127.0.0.1 docker compose -f docker/docker-compose.yml --profile prod up diff --git a/config.yaml b/config.yaml index 1e1a3431..012d7e46 100644 --- a/config.yaml +++ b/config.yaml @@ -46,33 +46,33 @@ pipeline: max_workers: 1 instances: dga_collector: - max_workers: 2 + max_workers: 1 domainator_collector: - max_workers: 2 + max_workers: 1 log_filtering.prefilter: executor: thread - max_workers: 2 + max_workers: 1 instances: dga_filter: - max_workers: 2 + max_workers: 1 no_filter: - max_workers: 2 + max_workers: 1 data_inspection.inspector: executor: thread max_workers: 2 instances: dga_inspector: - max_workers: 2 + max_workers: 1 no_inspector: - max_workers: 2 + max_workers: 1 data_analysis.detector: executor: thread - max_workers: 2 + max_workers: 1 instances: RF-dga_detector: - max_workers: 2 + max_workers: 1 domainator: - max_workers: 3 + max_workers: 2 domainator_attributor: max_workers: 1 domainator_attributor_behaviour: @@ -83,12 +83,10 @@ pipeline: max_workers: 1 pipeline.alerter: executor: thread - max_workers: 2 + max_workers: 1 instances: generic: - max_workers: 2 - attributor: - max_workers: 2 + max_workers: 1 monitoring.agent: executor: thread max_workers: 2 @@ -197,7 +195,7 @@ pipeline: use_scaler: false checksum: a4aac4c585f1e614c3cf0d737e80b960c5de6e87b253f7cdd07125d9ce486476 base_url: https://heibox.uni-heidelberg.de/d/0d5cbcbe16cd46a58021/ - threshold: 0.05 + threshold: 0.5 consume_from: inspector inspector_name: "domainator_inspector" next_detectors: @@ -218,7 +216,7 @@ pipeline: use_scaler: false checksum: d8f302edc166ecc80985838a30b5dff16ccc83480ea3c2480652f49c8f6b5e9b base_url: https://heibox.uni-heidelberg.de/d/0d5cbcbe16cd46a58021/ - threshold: 0.05 + threshold: 0.5 consume_from: detector detector_name: "domainator" next_detectors: "" @@ -236,7 +234,7 @@ pipeline: use_scaler: false checksum: 9a0970b4160b22f4c3c5ac99760f0ace5500dd25c5a195ff13254ad3c11d5dcd base_url: https://heibox.uni-heidelberg.de/d/0d5cbcbe16cd46a58021/ - threshold: 0.05 + threshold: 0.5 consume_from: detector detector_name: "domainator" next_detectors: "" @@ -254,7 +252,7 @@ pipeline: use_scaler: false checksum: 360bd26881beabce7e7581963240915de807c48b5e4a3501a657139f2ecb8a8b base_url: https://heibox.uni-heidelberg.de/d/0d5cbcbe16cd46a58021/ - threshold: 0.05 + threshold: 0.5 consume_from: detector detector_name: "domainator" next_detectors: "" diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 3d409010..203e8b15 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,33 +1,19 @@ services: - zookeeper: - extends: - file: "docker-compose/base/docker-compose.kafka.yml" - service: zookeeper - kafka1: extends: file: "docker-compose/base/docker-compose.kafka.yml" service: kafka1 - depends_on: - zookeeper: - condition: service_healthy kafka2: extends: file: "docker-compose/base/docker-compose.kafka.yml" service: kafka2 - depends_on: - zookeeper: - condition: service_healthy kafka3: extends: file: "docker-compose/base/docker-compose.kafka.yml" service: kafka3 - depends_on: - zookeeper: - condition: service_healthy clickhouse-server: extends: file: "docker-compose/base/docker-compose.clickhouse.yml" @@ -305,8 +291,6 @@ networks: volumes: ch_data: ch_logs: - kafka-data1: - kafka-data2: - kafka-data3: - zk-data: - zk-txn-logs: + kafka-broker-data1: + kafka-broker-data2: + kafka-broker-data3: diff --git a/docker/docker-compose/base/docker-compose.clickhouse.yml b/docker/docker-compose/base/docker-compose.clickhouse.yml index 6c9edd7b..58985d8f 100644 --- a/docker/docker-compose/base/docker-compose.clickhouse.yml +++ b/docker/docker-compose/base/docker-compose.clickhouse.yml @@ -1,6 +1,6 @@ services: clickhouse-server: - image: clickhouse/clickhouse-server:24.3.12.75-alpine + image: clickhouse/clickhouse-server:26.5-alpine container_name: clickhouse-server volumes: - ../../create_tables:/docker-entrypoint-initdb.d diff --git a/docker/docker-compose/base/docker-compose.grafana.yml b/docker/docker-compose/base/docker-compose.grafana.yml index 9301166d..a101eba4 100644 --- a/docker/docker-compose/base/docker-compose.grafana.yml +++ b/docker/docker-compose/base/docker-compose.grafana.yml @@ -1,6 +1,6 @@ services: grafana: - image: grafana/grafana:11.2.2-security-01 + image: grafana/grafana:13.0.3-slim container_name: grafana networks: - hamstring diff --git a/docker/docker-compose/base/docker-compose.kafka-exporter.yml b/docker/docker-compose/base/docker-compose.kafka-exporter.yml index 804fe2ef..90e093f7 100644 --- a/docker/docker-compose/base/docker-compose.kafka-exporter.yml +++ b/docker/docker-compose/base/docker-compose.kafka-exporter.yml @@ -1,6 +1,6 @@ services: kafka-exporter: - image: danielqsj/kafka-exporter + image: danielqsj/kafka-exporter:latest container_name: kafka-exporter ports: - "9308:9308" diff --git a/docker/docker-compose/base/docker-compose.kafka.yml b/docker/docker-compose/base/docker-compose.kafka.yml index 5b0bd59a..49bd65f1 100644 --- a/docker/docker-compose/base/docker-compose.kafka.yml +++ b/docker/docker-compose/base/docker-compose.kafka.yml @@ -1,133 +1,90 @@ -services: - zookeeper: - image: confluentinc/cp-zookeeper:7.9.3 - container_name: zookeeper - restart: "unless-stopped" - networks: - - hamstring - environment: - ZOOKEEPER_CLIENT_PORT: 2181 - ZOOKEEPER_TICK_TIME: 2000 - ZOOKEEPER_SERVER_ID: 1 - volumes: - - zk-data:/var/lib/zookeeper/data - - zk-txn-logs:/var/lib/zookeeper/log - healthcheck: - test: ["CMD-SHELL", "curl -s localhost:8080/commands | grep ruok"] - interval: 10s - timeout: 5s - retries: 3 +x-kafka-broker-env: &kafka-broker-env + CLUSTER_ID: "${KAFKA_CLUSTER_ID:-MkU3OEVBNTcwNTJENDM2Qk}" + KAFKA_PROCESS_ROLES: broker,controller + KAFKA_CONTROLLER_QUORUM_VOTERS: "${KAFKA_CONTROLLER_QUORUM_VOTERS:-1@kafka1:29093,2@kafka2:29093,3@kafka3:29093}" + KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 + KAFKA_LOG_RETENTION_HOURS: 4 + KAFKA_LOG_RETENTION_BYTES: 10737418240 + KAFKA_LOG_SEGMENT_BYTES: 1073741824 + KAFKA_LOG_CLEANUP_POLICY: delete +services: kafka1: - image: confluentinc/cp-kafka:7.9.3 + image: "${KAFKA_IMAGE:-confluentinc/cp-kafka:8.2.2}" container_name: kafka1 restart: "unless-stopped" networks: - hamstring ports: - "8097:8097" - depends_on: - zookeeper: - condition: service_healthy healthcheck: - test: ["CMD-SHELL", "nc -z localhost 19092"] + test: ["CMD-SHELL", "kafka-broker-api-versions --bootstrap-server localhost:19092 >/dev/null 2>&1"] interval: 30s timeout: 10s retries: 10 environment: - KAFKA_BROKER_ID: 1 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT - KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:19092,EXTERNAL://${HOST_IP}:8097 - KAFKA_LISTENERS: INTERNAL://0.0.0.0:19092,EXTERNAL://0.0.0.0:8097 - KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL - KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3 - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2 - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 - KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer - KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" - KAFKA_LOG_RETENTION_HOURS: 4 - KAFKA_LOG_RETENTION_BYTES: 10737418240 - KAFKA_LOG_SEGMENT_BYTES: 1073741824 - KAFKA_LOG_CLEANUP_POLICY: delete + <<: *kafka-broker-env + KAFKA_NODE_ID: 1 + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:19092,EXTERNAL://${HOST_IP:-localhost}:8097 + KAFKA_LISTENERS: INTERNAL://0.0.0.0:19092,EXTERNAL://0.0.0.0:8097,CONTROLLER://0.0.0.0:29093 volumes: - - kafka-data1:/var/lib/kafka/data + - kafka-broker-data1:/var/lib/kafka/data kafka2: - image: confluentinc/cp-kafka:7.9.3 + image: "${KAFKA_IMAGE:-confluentinc/cp-kafka:8.2.2}" container_name: kafka2 restart: "unless-stopped" networks: - hamstring ports: - "8098:8098" - depends_on: - zookeeper: - condition: service_healthy healthcheck: - test: ["CMD-SHELL", "nc -z localhost 19093"] + test: ["CMD-SHELL", "kafka-broker-api-versions --bootstrap-server localhost:19093 >/dev/null 2>&1"] interval: 30s timeout: 10s retries: 10 environment: - KAFKA_BROKER_ID: 2 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT - KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka2:19093,EXTERNAL://${HOST_IP}:8098 - KAFKA_LISTENERS: INTERNAL://0.0.0.0:19093,EXTERNAL://0.0.0.0:8098 - KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL - KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" + <<: *kafka-broker-env + KAFKA_NODE_ID: 2 + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka2:19093,EXTERNAL://${HOST_IP:-localhost}:8098 + KAFKA_LISTENERS: INTERNAL://0.0.0.0:19093,EXTERNAL://0.0.0.0:8098,CONTROLLER://0.0.0.0:29093 KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3 - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2 - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 - KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer - KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" - KAFKA_LOG_RETENTION_HOURS: 4 - KAFKA_LOG_RETENTION_BYTES: 10737418240 - KAFKA_LOG_SEGMENT_BYTES: 1073741824 - KAFKA_LOG_CLEANUP_POLICY: delete volumes: - - kafka-data2:/var/lib/kafka/data + - kafka-broker-data2:/var/lib/kafka/data kafka3: - image: confluentinc/cp-kafka:7.9.3 + image: "${KAFKA_IMAGE:-confluentinc/cp-kafka:8.2.2}" container_name: kafka3 restart: "unless-stopped" networks: - hamstring ports: - "8099:8099" - depends_on: - zookeeper: - condition: service_healthy healthcheck: - test: ["CMD-SHELL", "nc -z localhost 19094"] + test: ["CMD-SHELL", "kafka-broker-api-versions --bootstrap-server localhost:19094 >/dev/null 2>&1"] interval: 30s timeout: 10s retries: 10 environment: - KAFKA_BROKER_ID: 3 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT - KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka3:19094,EXTERNAL://${HOST_IP}:8099 - KAFKA_LISTENERS: INTERNAL://0.0.0.0:19094,EXTERNAL://0.0.0.0:8099 - KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL - KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" + <<: *kafka-broker-env + KAFKA_NODE_ID: 3 + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka3:19094,EXTERNAL://${HOST_IP:-localhost}:8099 + KAFKA_LISTENERS: INTERNAL://0.0.0.0:19094,EXTERNAL://0.0.0.0:8099,CONTROLLER://0.0.0.0:29093 KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3 - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2 - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 - KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer - KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" - KAFKA_LOG_RETENTION_HOURS: 4 - KAFKA_LOG_RETENTION_BYTES: 10737418240 - KAFKA_LOG_SEGMENT_BYTES: 1073741824 - KAFKA_LOG_CLEANUP_POLICY: delete volumes: - - kafka-data3:/var/lib/kafka/data + - kafka-broker-data3:/var/lib/kafka/data networks: hamstring: external: true + +volumes: + kafka-broker-data1: + kafka-broker-data2: + kafka-broker-data3: diff --git a/docker/docker-compose/dev/docker-compose.yml b/docker/docker-compose/dev/docker-compose.yml index 22c06420..56e6badd 100644 --- a/docker/docker-compose/dev/docker-compose.yml +++ b/docker/docker-compose/dev/docker-compose.yml @@ -1,33 +1,19 @@ services: - zookeeper: - extends: - file: "docker-compose/base/docker-compose.kafka.yml" - service: zookeeper - kafka1: extends: file: "docker-compose/base/docker-compose.kafka.yml" service: kafka1 - depends_on: - zookeeper: - condition: service_healthy kafka2: extends: file: "docker-compose/base/docker-compose.kafka.yml" service: kafka2 - depends_on: - zookeeper: - condition: service_healthy kafka3: extends: file: "docker-compose/base/docker-compose.kafka.yml" service: kafka3 - depends_on: - zookeeper: - condition: service_healthy logserver: @@ -145,8 +131,6 @@ networks: volumes: ch_data: ch_logs: - kafka-data1: - kafka-data2: - kafka-data3: - zk-data: - zk-txn-logs: + kafka-broker-data1: + kafka-broker-data2: + kafka-broker-data3: diff --git a/docker/docker-compose/prod/docker-compose.yml b/docker/docker-compose/prod/docker-compose.yml index 21d2110a..879101eb 100644 --- a/docker/docker-compose/prod/docker-compose.yml +++ b/docker/docker-compose/prod/docker-compose.yml @@ -1,33 +1,19 @@ services: - zookeeper: - extends: - file: "docker-compose/base/docker-compose.kafka.yml" - service: zookeeper - kafka1: extends: file: "docker-compose/base/docker-compose.kafka.yml" service: kafka1 - depends_on: - zookeeper: - condition: service_healthy kafka2: extends: file: "docker-compose/base/docker-compose.kafka.yml" service: kafka2 - depends_on: - zookeeper: - condition: service_healthy kafka3: extends: file: "docker-compose/base/docker-compose.kafka.yml" service: kafka3 - depends_on: - zookeeper: - condition: service_healthy logserver: @@ -145,8 +131,6 @@ networks: volumes: ch_data: ch_logs: - kafka-data1: - kafka-data2: - kafka-data3: - zk-data: - zk-txn-logs: + kafka-broker-data1: + kafka-broker-data2: + kafka-broker-data3: diff --git a/docker/docker_swarm/docker-compose.swarm.yml b/docker/docker_swarm/docker-compose.swarm.yml index 00d59a34..3c164450 100644 --- a/docker/docker_swarm/docker-compose.swarm.yml +++ b/docker/docker_swarm/docker-compose.swarm.yml @@ -1,28 +1,24 @@ # Compose file to deploy directly with docker stack deploy. --- -services: - zookeeper: - image: "${ZOOKEEPER_IMAGE:-confluentinc/cp-zookeeper:7.9.3}" - networks: - - hamstring - environment: - ZOOKEEPER_CLIENT_PORT: "2181" - ZOOKEEPER_TICK_TIME: "2000" - ZOOKEEPER_SERVER_ID: "1" - volumes: - - zk-data:/var/lib/zookeeper/data - - zk-txn-logs:/var/lib/zookeeper/log - healthcheck: - test: ["CMD-SHELL", "curl -s localhost:8080/commands | grep ruok"] - interval: 10s - timeout: 5s - retries: 3 - deploy: - restart_policy: - condition: any +x-kafka-broker-env: &kafka-broker-env + CLUSTER_ID: "${KAFKA_CLUSTER_ID:-MkU3OEVBNTcwNTJENDM2Qk}" + KAFKA_PROCESS_ROLES: broker,controller + KAFKA_CONTROLLER_QUORUM_VOTERS: "${KAFKA_CONTROLLER_QUORUM_VOTERS:-1@kafka1:29093,2@kafka2:29093,3@kafka3:29093}" + KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "${KAFKA_AUTO_CREATE_TOPICS_ENABLE:-false}" + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: "${KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR:-3}" + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: "${KAFKA_TRANSACTION_STATE_LOG_MIN_ISR:-2}" + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "${KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR:-3}" + KAFKA_LOG_RETENTION_HOURS: "${KAFKA_LOG_RETENTION_HOURS:-4}" + KAFKA_LOG_RETENTION_BYTES: "${KAFKA_LOG_RETENTION_BYTES:-10737418240}" + KAFKA_LOG_SEGMENT_BYTES: "${KAFKA_LOG_SEGMENT_BYTES:-1073741824}" + KAFKA_LOG_CLEANUP_POLICY: "${KAFKA_LOG_CLEANUP_POLICY:-delete}" +services: kafka1: - image: "${KAFKA_IMAGE:-confluentinc/cp-kafka:7.9.3}" + image: "${KAFKA_IMAGE:-confluentinc/cp-kafka:8.2.2}" networks: - hamstring ports: @@ -31,26 +27,14 @@ services: protocol: tcp mode: ingress environment: - KAFKA_BROKER_ID: "1" - KAFKA_ZOOKEEPER_CONNECT: "${KAFKA_ZOOKEEPER_CONNECT:-zookeeper:2181}" - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT + <<: *kafka-broker-env + KAFKA_NODE_ID: "1" KAFKA_ADVERTISED_LISTENERS: "INTERNAL://kafka1:19092,EXTERNAL://${KAFKA_EXTERNAL_HOST:-localhost}:${KAFKA1_EXTERNAL_PORT:-8097}" - KAFKA_LISTENERS: INTERNAL://0.0.0.0:19092,EXTERNAL://0.0.0.0:8097 - KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL - KAFKA_AUTO_CREATE_TOPICS_ENABLE: "${KAFKA_AUTO_CREATE_TOPICS_ENABLE:-false}" - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: "${KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR:-3}" - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: "${KAFKA_TRANSACTION_STATE_LOG_MIN_ISR:-2}" - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "${KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR:-3}" - KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer - KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "${KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND:-true}" - KAFKA_LOG_RETENTION_HOURS: "${KAFKA_LOG_RETENTION_HOURS:-4}" - KAFKA_LOG_RETENTION_BYTES: "${KAFKA_LOG_RETENTION_BYTES:-10737418240}" - KAFKA_LOG_SEGMENT_BYTES: "${KAFKA_LOG_SEGMENT_BYTES:-1073741824}" - KAFKA_LOG_CLEANUP_POLICY: "${KAFKA_LOG_CLEANUP_POLICY:-delete}" + KAFKA_LISTENERS: INTERNAL://0.0.0.0:19092,EXTERNAL://0.0.0.0:8097,CONTROLLER://0.0.0.0:29093 volumes: - - kafka-data1:/var/lib/kafka/data + - kafka-broker-data1:/var/lib/kafka/data healthcheck: - test: ["CMD-SHELL", "nc -z localhost 19092"] + test: ["CMD-SHELL", "kafka-broker-api-versions --bootstrap-server localhost:19092 >/dev/null 2>&1"] interval: 30s timeout: 10s retries: 10 @@ -59,7 +43,7 @@ services: condition: any kafka2: - image: "${KAFKA_IMAGE:-confluentinc/cp-kafka:7.9.3}" + image: "${KAFKA_IMAGE:-confluentinc/cp-kafka:8.2.2}" networks: - hamstring ports: @@ -68,27 +52,15 @@ services: protocol: tcp mode: ingress environment: - KAFKA_BROKER_ID: "2" - KAFKA_ZOOKEEPER_CONNECT: "${KAFKA_ZOOKEEPER_CONNECT:-zookeeper:2181}" - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT + <<: *kafka-broker-env + KAFKA_NODE_ID: "2" KAFKA_ADVERTISED_LISTENERS: "INTERNAL://kafka2:19093,EXTERNAL://${KAFKA_EXTERNAL_HOST:-localhost}:${KAFKA2_EXTERNAL_PORT:-8098}" - KAFKA_LISTENERS: INTERNAL://0.0.0.0:19093,EXTERNAL://0.0.0.0:8098 - KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL - KAFKA_AUTO_CREATE_TOPICS_ENABLE: "${KAFKA_AUTO_CREATE_TOPICS_ENABLE:-false}" + KAFKA_LISTENERS: INTERNAL://0.0.0.0:19093,EXTERNAL://0.0.0.0:8098,CONTROLLER://0.0.0.0:29093 KAFKA_LOG4J_LOGGERS: "${KAFKA_LOG4J_LOGGERS:-kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO}" - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: "${KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR:-3}" - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: "${KAFKA_TRANSACTION_STATE_LOG_MIN_ISR:-2}" - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "${KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR:-3}" - KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer - KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "${KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND:-true}" - KAFKA_LOG_RETENTION_HOURS: "${KAFKA_LOG_RETENTION_HOURS:-4}" - KAFKA_LOG_RETENTION_BYTES: "${KAFKA_LOG_RETENTION_BYTES:-10737418240}" - KAFKA_LOG_SEGMENT_BYTES: "${KAFKA_LOG_SEGMENT_BYTES:-1073741824}" - KAFKA_LOG_CLEANUP_POLICY: "${KAFKA_LOG_CLEANUP_POLICY:-delete}" volumes: - - kafka-data2:/var/lib/kafka/data + - kafka-broker-data2:/var/lib/kafka/data healthcheck: - test: ["CMD-SHELL", "nc -z localhost 19093"] + test: ["CMD-SHELL", "kafka-broker-api-versions --bootstrap-server localhost:19093 >/dev/null 2>&1"] interval: 30s timeout: 10s retries: 10 @@ -97,7 +69,7 @@ services: condition: any kafka3: - image: "${KAFKA_IMAGE:-confluentinc/cp-kafka:7.9.3}" + image: "${KAFKA_IMAGE:-confluentinc/cp-kafka:8.2.2}" networks: - hamstring ports: @@ -106,27 +78,15 @@ services: protocol: tcp mode: ingress environment: - KAFKA_BROKER_ID: "3" - KAFKA_ZOOKEEPER_CONNECT: "${KAFKA_ZOOKEEPER_CONNECT:-zookeeper:2181}" - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT + <<: *kafka-broker-env + KAFKA_NODE_ID: "3" KAFKA_ADVERTISED_LISTENERS: "INTERNAL://kafka3:19094,EXTERNAL://${KAFKA_EXTERNAL_HOST:-localhost}:${KAFKA3_EXTERNAL_PORT:-8099}" - KAFKA_LISTENERS: INTERNAL://0.0.0.0:19094,EXTERNAL://0.0.0.0:8099 - KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL - KAFKA_AUTO_CREATE_TOPICS_ENABLE: "${KAFKA_AUTO_CREATE_TOPICS_ENABLE:-false}" + KAFKA_LISTENERS: INTERNAL://0.0.0.0:19094,EXTERNAL://0.0.0.0:8099,CONTROLLER://0.0.0.0:29093 KAFKA_LOG4J_LOGGERS: "${KAFKA_LOG4J_LOGGERS:-kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO}" - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: "${KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR:-3}" - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: "${KAFKA_TRANSACTION_STATE_LOG_MIN_ISR:-2}" - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "${KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR:-3}" - KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer - KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "${KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND:-true}" - KAFKA_LOG_RETENTION_HOURS: "${KAFKA_LOG_RETENTION_HOURS:-4}" - KAFKA_LOG_RETENTION_BYTES: "${KAFKA_LOG_RETENTION_BYTES:-10737418240}" - KAFKA_LOG_SEGMENT_BYTES: "${KAFKA_LOG_SEGMENT_BYTES:-1073741824}" - KAFKA_LOG_CLEANUP_POLICY: "${KAFKA_LOG_CLEANUP_POLICY:-delete}" volumes: - - kafka-data3:/var/lib/kafka/data + - kafka-broker-data3:/var/lib/kafka/data healthcheck: - test: ["CMD-SHELL", "nc -z localhost 19094"] + test: ["CMD-SHELL", "kafka-broker-api-versions --bootstrap-server localhost:19094 >/dev/null 2>&1"] interval: 30s timeout: 10s retries: 10 @@ -135,7 +95,7 @@ services: condition: any clickhouse-server: - image: "${CLICKHOUSE_IMAGE:-clickhouse/clickhouse-server:24.3.12.75-alpine}" + image: "${CLICKHOUSE_IMAGE:-clickhouse/clickhouse-server:26.5-alpine}" networks: - hamstring ports: @@ -161,7 +121,7 @@ services: condition: any grafana: - image: "${GRAFANA_IMAGE:-grafana/grafana:11.2.2-security-01}" + image: "${GRAFANA_IMAGE:-grafana/grafana:13.1.0-debian}" networks: - hamstring ports: @@ -377,10 +337,8 @@ networks: volumes: ch_data: ch_logs: - kafka-data1: - kafka-data2: - kafka-data3: - zk-data: - zk-txn-logs: + kafka-broker-data1: + kafka-broker-data2: + kafka-broker-data3: # code: language=yaml diff --git a/docker/dockerfiles/Dockerfile.alerter b/docker/dockerfiles/Dockerfile.alerter index 9338c7e5..891aac85 100644 --- a/docker/dockerfiles/Dockerfile.alerter +++ b/docker/dockerfiles/Dockerfile.alerter @@ -2,7 +2,7 @@ # * Install python3-venv for the built-in Python3 venv module (not installed by default) # * Install gcc libpython3-dev to compile C Python modules # * In the virtualenv: Update pip setuputils and wheel to support building new packages -FROM debian:12-slim AS build +FROM debian:13-slim AS build RUN apt-get update && \ apt-get install --no-install-suggests --no-install-recommends --yes python3-venv gcc g++ build-essential libpython3-dev && \ python3 -m venv /venv && \ @@ -14,7 +14,7 @@ COPY requirements/requirements.alerter.txt /requirements.alerter.txt RUN /venv/bin/pip install --disable-pip-version-check -r /requirements.alerter.txt # Copy the virtualenv into a distroless image -FROM gcr.io/distroless/python3-debian12 +FROM gcr.io/distroless/python3-debian13 COPY --from=build-venv /venv /venv COPY src/base /app/src/base COPY src/alerter /app/src/alerter diff --git a/docker/dockerfiles/Dockerfile.detector b/docker/dockerfiles/Dockerfile.detector index 65c8b84b..3b3cef9d 100644 --- a/docker/dockerfiles/Dockerfile.detector +++ b/docker/dockerfiles/Dockerfile.detector @@ -2,7 +2,7 @@ # * Install python3-venv for the built-in Python3 venv module (not installed by default) # * Install gcc libpython3-dev to compile C Python modules # * In the virtualenv: Update pip setuputils and wheel to support building new packages -FROM debian:12-slim AS build +FROM debian:13-slim AS build RUN apt-get update && \ apt-get install --no-install-suggests --no-install-recommends --yes python3-venv gcc g++ build-essential libpython3-dev && \ python3 -m venv /venv && \ @@ -14,7 +14,7 @@ COPY requirements/requirements.detector.txt /requirements.detector.txt RUN /venv/bin/pip install --disable-pip-version-check -r /requirements.detector.txt # Copy the virtualenv into a distroless image -FROM gcr.io/distroless/python3-debian12 +FROM gcr.io/distroless/python3-debian13 COPY --from=build-venv /venv /venv COPY src/base /app/src/base COPY src/detector /app/src/detector diff --git a/docker/dockerfiles/Dockerfile.inspector b/docker/dockerfiles/Dockerfile.inspector index 98cec5f8..6fc5a48f 100644 --- a/docker/dockerfiles/Dockerfile.inspector +++ b/docker/dockerfiles/Dockerfile.inspector @@ -2,7 +2,7 @@ # * Install python3-venv for the built-in Python3 venv module (not installed by default) # * Install gcc libpython3-dev to compile C Python modules # * In the virtualenv: Update pip setuputils and wheel to support building new packages -FROM debian:12-slim AS build +FROM debian:13-slim AS build RUN apt-get update && \ apt-get install --no-install-suggests --no-install-recommends --yes python3-venv gcc libpython3-dev && \ python3 -m venv /venv && \ @@ -14,7 +14,7 @@ COPY requirements/requirements.inspector.txt /requirements.inspector.txt RUN /venv/bin/pip install --disable-pip-version-check -r /requirements.inspector.txt # Copy the virtualenv into a distroless image -FROM gcr.io/distroless/python3-debian12 +FROM gcr.io/distroless/python3-debian13 COPY --from=build-venv /venv /venv COPY src/base /app/src/base COPY src/inspector /app/src/inspector diff --git a/docker/dockerfiles/Dockerfile.logcollector b/docker/dockerfiles/Dockerfile.logcollector index f5410ad1..ff2c9b89 100644 --- a/docker/dockerfiles/Dockerfile.logcollector +++ b/docker/dockerfiles/Dockerfile.logcollector @@ -2,7 +2,7 @@ # * Install python3-venv for the built-in Python3 venv module (not installed by default) # * Install gcc libpython3-dev to compile C Python modules # * In the virtualenv: Update pip setuputils and wheel to support building new packages -FROM debian:12-slim AS build +FROM debian:13-slim AS build RUN apt-get update && \ apt-get install --no-install-suggests --no-install-recommends --yes python3-venv gcc libpython3-dev && \ python3 -m venv /venv && \ @@ -14,7 +14,7 @@ COPY requirements/requirements.logcollector.txt /requirements.logcollector.txt RUN /venv/bin/pip install --disable-pip-version-check -r /requirements.logcollector.txt # Copy the virtualenv into a distroless image -FROM gcr.io/distroless/python3-debian12 +FROM gcr.io/distroless/python3-debian13 COPY --from=build-venv /venv /venv COPY src/base /app/src/base COPY src/logcollector /app/src/logcollector diff --git a/docker/dockerfiles/Dockerfile.logserver b/docker/dockerfiles/Dockerfile.logserver index 3437b86d..cb2ac556 100644 --- a/docker/dockerfiles/Dockerfile.logserver +++ b/docker/dockerfiles/Dockerfile.logserver @@ -2,7 +2,7 @@ # * Install python3-venv for the built-in Python3 venv module (not installed by default) # * Install gcc libpython3-dev to compile C Python modules # * In the virtualenv: Update pip setuputils and wheel to support building new packages -FROM debian:12-slim AS build +FROM debian:13-slim AS build RUN apt-get update && \ apt-get install --no-install-suggests --no-install-recommends --yes python3-venv gcc libpython3-dev && \ python3 -m venv /venv && \ @@ -14,7 +14,7 @@ COPY requirements/requirements.logserver.txt /requirements.logserver.txt RUN /venv/bin/pip install --disable-pip-version-check -r /requirements.logserver.txt # Copy the virtualenv into a distroless image -FROM gcr.io/distroless/python3-debian12 +FROM gcr.io/distroless/python3-debian13 COPY --from=build-venv /venv /venv COPY src/base /app/src/base COPY src/logserver /app/src/logserver diff --git a/docker/dockerfiles/Dockerfile.monitoring b/docker/dockerfiles/Dockerfile.monitoring index 05d3b4ec..402505d3 100644 --- a/docker/dockerfiles/Dockerfile.monitoring +++ b/docker/dockerfiles/Dockerfile.monitoring @@ -2,7 +2,7 @@ # * Install python3-venv for the built-in Python3 venv module (not installed by default) # * Install gcc libpython3-dev to compile C Python modules # * In the virtualenv: Update pip setuputils and wheel to support building new packages -FROM debian:12-slim AS build +FROM debian:13-slim AS build RUN apt-get update && \ apt-get install --no-install-suggests --no-install-recommends --yes python3-venv gcc libpython3-dev && \ python3 -m venv /venv && \ @@ -14,7 +14,7 @@ COPY requirements/requirements.monitoring.txt /requirements.monitoring.txt RUN /venv/bin/pip install --disable-pip-version-check -r /requirements.monitoring.txt # Copy the virtualenv into a distroless image -FROM gcr.io/distroless/python3-debian12 +FROM gcr.io/distroless/python3-debian13 COPY --from=build-venv /venv /venv COPY src/base /app/src/base COPY src/monitoring /app/src/monitoring diff --git a/docker/dockerfiles/Dockerfile.prefilter b/docker/dockerfiles/Dockerfile.prefilter index fefbc468..cb279ee4 100644 --- a/docker/dockerfiles/Dockerfile.prefilter +++ b/docker/dockerfiles/Dockerfile.prefilter @@ -2,7 +2,7 @@ # * Install python3-venv for the built-in Python3 venv module (not installed by default) # * Install gcc libpython3-dev to compile C Python modules # * In the virtualenv: Update pip setuputils and wheel to support building new packages -FROM debian:12-slim AS build +FROM debian:13-slim AS build RUN apt-get update && \ apt-get install --no-install-suggests --no-install-recommends --yes python3-venv gcc libpython3-dev && \ python3 -m venv /venv && \ @@ -14,7 +14,7 @@ COPY requirements/requirements.prefilter.txt /requirements.prefilter.txt RUN /venv/bin/pip install --disable-pip-version-check -r /requirements.prefilter.txt # Copy the virtualenv into a distroless image -FROM gcr.io/distroless/python3-debian12 +FROM gcr.io/distroless/python3-debian13 COPY --from=build-venv /venv /venv COPY src/base /app/src/base COPY src/prefilter /app/src/prefilter diff --git a/requirements/requirements.alerter.txt b/requirements/requirements.alerter.txt index 8e8937fc..385c98be 100644 --- a/requirements/requirements.alerter.txt +++ b/requirements/requirements.alerter.txt @@ -1,5 +1,5 @@ -PyYAML~=6.0.1 -colorlog~=6.8.2 -confluent-kafka~=2.4.0 -marshmallow_dataclass~=8.7.1 -clickhouse_connect~=0.8.3 +PyYAML +colorlog +confluent-kafka +marshmallow_dataclass +clickhouse_connect diff --git a/requirements/requirements.detector.txt b/requirements/requirements.detector.txt index 14457ab6..75e22c77 100644 --- a/requirements/requirements.detector.txt +++ b/requirements/requirements.detector.txt @@ -1,11 +1,11 @@ xgboost -scikit-learn~=1.5.2 +scikit-learn pandas requests -colorlog~=6.8.2 -PyYAML~=6.0.1 -confluent-kafka~=2.4.0 -marshmallow_dataclass~=8.7.1 -clickhouse_connect~=0.8.3 +colorlog +PyYAML +confluent-kafka +marshmallow_dataclass +clickhouse_connect pylcs Levenshtein diff --git a/requirements/requirements.dev.txt b/requirements/requirements.dev.txt index 31206bee..416634f5 100644 --- a/requirements/requirements.dev.txt +++ b/requirements/requirements.dev.txt @@ -1 +1 @@ -pre-commit~=3.7.1 +pre-commit diff --git a/requirements/requirements.inspector.txt b/requirements/requirements.inspector.txt index 2ee4c1a5..8f98f229 100644 --- a/requirements/requirements.inspector.txt +++ b/requirements/requirements.inspector.txt @@ -1,10 +1,10 @@ -PyYAML~=6.0.1 -confluent-kafka~=2.4.0 -colorlog~=6.8.2 -scipy~=1.12.0 -streamad~=0.3.1 -numpy~=1.26.4 -marshmallow_dataclass~=8.7.1 -clickhouse_connect~=0.8.3 -scipy==1.12.0 -setuptools==81.0.0 +PyYAML +confluent-kafka +colorlog +scipy +streamad +numpy +marshmallow_dataclass +clickhouse_connect +scipy +setuptools<81 diff --git a/requirements/requirements.logcollector.txt b/requirements/requirements.logcollector.txt index 8e8937fc..385c98be 100644 --- a/requirements/requirements.logcollector.txt +++ b/requirements/requirements.logcollector.txt @@ -1,5 +1,5 @@ -PyYAML~=6.0.1 -colorlog~=6.8.2 -confluent-kafka~=2.4.0 -marshmallow_dataclass~=8.7.1 -clickhouse_connect~=0.8.3 +PyYAML +colorlog +confluent-kafka +marshmallow_dataclass +clickhouse_connect diff --git a/requirements/requirements.logserver.txt b/requirements/requirements.logserver.txt index a241460d..28bf4449 100644 --- a/requirements/requirements.logserver.txt +++ b/requirements/requirements.logserver.txt @@ -1,6 +1,6 @@ -PyYAML~=6.0.1 -colorlog~=6.8.2 -confluent-kafka~=2.4.0 -marshmallow_dataclass~=8.7.1 -aiofiles~=24.1.0 -clickhouse_connect~=0.8.3 +PyYAML +colorlog +confluent-kafka +marshmallow_dataclass +aiofiles +clickhouse_connect diff --git a/requirements/requirements.monitoring.txt b/requirements/requirements.monitoring.txt index a2e9d98f..451c9e42 100644 --- a/requirements/requirements.monitoring.txt +++ b/requirements/requirements.monitoring.txt @@ -1,5 +1,5 @@ -clickhouse_connect~=0.8.3 -confluent-kafka~=2.4.0 -marshmallow_dataclass~=8.7.1 -colorlog~=6.8.2 -PyYAML~=6.0.1 +clickhouse_connect +confluent-kafka +marshmallow_dataclass +colorlog +PyYAML diff --git a/requirements/requirements.prefilter.txt b/requirements/requirements.prefilter.txt index 8e8937fc..385c98be 100644 --- a/requirements/requirements.prefilter.txt +++ b/requirements/requirements.prefilter.txt @@ -1,5 +1,5 @@ -PyYAML~=6.0.1 -colorlog~=6.8.2 -confluent-kafka~=2.4.0 -marshmallow_dataclass~=8.7.1 -clickhouse_connect~=0.8.3 +PyYAML +colorlog +confluent-kafka +marshmallow_dataclass +clickhouse_connect diff --git a/requirements/requirements.train.txt b/requirements/requirements.train.txt index 82e15a1f..f4e23952 100644 --- a/requirements/requirements.train.txt +++ b/requirements/requirements.train.txt @@ -1,7 +1,7 @@ numpy xgboost -scikit-learn~=1.5.2 -scipy~=1.12.0 +scikit-learn +scipy torch pyarrow polars diff --git a/tests/kafka/test_exactly_once_kafka_produce_handler.py b/tests/kafka/test_exactly_once_kafka_produce_handler.py index 02ddf47b..3a30988c 100644 --- a/tests/kafka/test_exactly_once_kafka_produce_handler.py +++ b/tests/kafka/test_exactly_once_kafka_produce_handler.py @@ -47,6 +47,7 @@ def test_init(self, mock_producer, mock_uuid): mock_producer_instance.init_transactions.assert_called_once() @patch("src.base.retry.time.sleep", return_value=None) + @patch("src.base.kafka_handler.HOSTNAME", "default_tid") @patch("src.base.kafka_handler.uuid") @patch("src.base.kafka_handler.logger") @patch(