From c43185b550727873bb81e669477856f4a168053e Mon Sep 17 00:00:00 2001 From: yasin Date: Sat, 25 Jun 2022 19:09:50 +0430 Subject: [PATCH 1/4] fix: change advertised listeners Signed-off-by: yasin --- README.md | 2 ++ zk-multiple-kafka-multiple.yml | 6 +++--- zk-single-kafka-multiple.yml | 6 +++--- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index ca99eb7..a02e911 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,7 @@ If you want to have three brokers and experiment with kafka replication / fault- Run with: ``` +export DOCKER_HOST_IP="DOCKER HOST IP" docker-compose -f zk-single-kafka-multiple.yml up docker-compose -f zk-single-kafka-multiple.yml down ``` @@ -100,6 +101,7 @@ If you want to have three zookeeper nodes and three kafka brokers to experiment Run with: ``` +export DOCKER_HOST_IP="DOCKER HOST IP" docker-compose -f zk-multiple-kafka-multiple.yml up docker-compose -f zk-multiple-kafka-multiple.yml down ``` diff --git a/zk-multiple-kafka-multiple.yml b/zk-multiple-kafka-multiple.yml index af993dc..ec1f3d1 100644 --- a/zk-multiple-kafka-multiple.yml +++ b/zk-multiple-kafka-multiple.yml @@ -43,7 +43,7 @@ services: ports: - "9092:9092" environment: - KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092 + KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP}:9092 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183" @@ -63,7 +63,7 @@ services: ports: - "9093:9093" environment: - KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka2:19093,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9093 + KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka2:19093,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP}:9093 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183" @@ -83,7 +83,7 @@ services: ports: - "9094:9094" environment: - KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka3:19094,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9094 + KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka3:19094,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP}:9094 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183" diff --git a/zk-single-kafka-multiple.yml b/zk-single-kafka-multiple.yml index 9e49667..7025167 100644 --- a/zk-single-kafka-multiple.yml +++ b/zk-single-kafka-multiple.yml @@ -20,7 +20,7 @@ services: ports: - "9092:9092" environment: - KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092 + KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP}:9092 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181" @@ -38,7 +38,7 @@ services: ports: - "9093:9093" environment: - KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka2:19093,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9093 + KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka2:19093,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP}:9093 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181" @@ -57,7 +57,7 @@ services: ports: - "9094:9094" environment: - KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka3:19094,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9094 + KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka3:19094,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP}:9094 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181" From 3926ed10d5b15df1491d56321009e0938992ddd7 Mon Sep 17 00:00:00 2001 From: yasinlachiny Date: Sun, 5 Mar 2023 16:25:36 +0100 Subject: [PATCH 2/4] fix: resolve conflicts --- zk-multiple-kafka-multiple.yml | 35 ++++++++++++++++++---------------- zk-single-kafka-multiple.yml | 31 ++++++++++++++++-------------- 2 files changed, 36 insertions(+), 30 deletions(-) diff --git a/zk-multiple-kafka-multiple.yml b/zk-multiple-kafka-multiple.yml index ec1f3d1..42f6854 100644 --- a/zk-multiple-kafka-multiple.yml +++ b/zk-multiple-kafka-multiple.yml @@ -2,7 +2,7 @@ version: '2.1' services: zoo1: - image: confluentinc/cp-zookeeper:7.1.1 + image: confluentinc/cp-zookeeper:7.3.2 hostname: zoo1 container_name: zoo1 ports: @@ -13,7 +13,7 @@ services: ZOOKEEPER_SERVERS: zoo1:2888:3888;zoo2:2888:3888;zoo3:2888:3888 zoo2: - image: confluentinc/cp-zookeeper:7.1.1 + image: confluentinc/cp-zookeeper:7.3.2 hostname: zoo2 container_name: zoo2 ports: @@ -24,7 +24,7 @@ services: ZOOKEEPER_SERVERS: zoo1:2888:3888;zoo2:2888:3888;zoo3:2888:3888 zoo3: - image: confluentinc/cp-zookeeper:7.1.1 + image: confluentinc/cp-zookeeper:7.3.2 hostname: zoo3 container_name: zoo3 ports: @@ -37,15 +37,16 @@ services: kafka1: - image: confluentinc/cp-kafka:7.1.1 + image: confluentinc/cp-kafka:7.3.2 hostname: kafka1 container_name: kafka1 ports: - "9092:9092" + - "29092:29092" environment: - KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP}:9092 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT - KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:19092,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092,DOCKER://host.docker.internal:29092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183" KAFKA_BROKER_ID: 1 KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" @@ -57,15 +58,16 @@ services: - zoo3 kafka2: - image: confluentinc/cp-kafka:7.1.1 + image: confluentinc/cp-kafka:7.3.2 hostname: kafka2 container_name: kafka2 ports: - "9093:9093" + - "29093:29093" environment: - KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka2:19093,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP}:9093 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT - KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka2:19093,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9093,DOCKER://host.docker.internal:29093 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183" KAFKA_BROKER_ID: 2 KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" @@ -77,15 +79,16 @@ services: - zoo3 kafka3: - image: confluentinc/cp-kafka:7.1.1 + image: confluentinc/cp-kafka:7.3.2 hostname: kafka3 container_name: kafka3 ports: - "9094:9094" + - "29094:29094" environment: - KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka3:19094,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP}:9094 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT - KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka3:19094,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9094,DOCKER://host.docker.internal:29094 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183" KAFKA_BROKER_ID: 3 KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" @@ -94,4 +97,4 @@ services: depends_on: - zoo1 - zoo2 - - zoo3 + - zoo3 \ No newline at end of file diff --git a/zk-single-kafka-multiple.yml b/zk-single-kafka-multiple.yml index 7025167..01e26e5 100644 --- a/zk-single-kafka-multiple.yml +++ b/zk-single-kafka-multiple.yml @@ -2,7 +2,7 @@ version: '2.1' services: zoo1: - image: confluentinc/cp-zookeeper:7.1.1 + image: confluentinc/cp-zookeeper:7.3.2 hostname: zoo1 container_name: zoo1 ports: @@ -14,15 +14,16 @@ services: kafka1: - image: confluentinc/cp-kafka:7.1.1 + image: confluentinc/cp-kafka:7.3.2 hostname: kafka1 container_name: kafka1 ports: - "9092:9092" + - "29092:29092" environment: - KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP}:9092 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT - KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:19092,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092,DOCKER://host.docker.internal:29092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181" KAFKA_BROKER_ID: 1 KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" @@ -32,15 +33,16 @@ services: - zoo1 kafka2: - image: confluentinc/cp-kafka:7.1.1 + image: confluentinc/cp-kafka:7.3.2 hostname: kafka2 container_name: kafka2 ports: - "9093:9093" + - "29093:29093" environment: - KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka2:19093,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP}:9093 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT - KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka2:19093,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9093,DOCKER://host.docker.internal:29093 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181" KAFKA_BROKER_ID: 2 KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" @@ -51,19 +53,20 @@ services: kafka3: - image: confluentinc/cp-kafka:7.1.1 + image: confluentinc/cp-kafka:7.3.2 hostname: kafka3 container_name: kafka3 ports: - "9094:9094" + - "29094:29094" environment: - KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka3:19094,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP}:9094 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT - KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka3:19094,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9094,DOCKER://host.docker.internal:29094 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181" KAFKA_BROKER_ID: 3 KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" depends_on: - - zoo1 + - zoo1 \ No newline at end of file From c198edebb5969c5c2e16c51477fa9ccbca7e2844 Mon Sep 17 00:00:00 2001 From: yasinlachiny Date: Sun, 5 Mar 2023 16:43:50 +0100 Subject: [PATCH 3/4] fix: resolve conflicts --- README.md | 95 ++++++++++--------- full-stack.yml | 51 ++++++---- jmx-exporter.yml | 70 ++++++++++++++ platform-config.yaml | 25 +++++ ...ultiple-kafka-multiple-schema-registry.yml | 35 +++---- zk-multiple-kafka-single.yml | 15 +-- zk-single-kafka-single.yml | 13 +-- 7 files changed, 213 insertions(+), 91 deletions(-) create mode 100644 jmx-exporter.yml create mode 100644 platform-config.yaml diff --git a/README.md b/README.md index a02e911..3aa6cc7 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ [![Actions Status](https://github.com/conduktor/kafka-stack-docker-compose/workflows/CI/badge.svg)](https://github.com/conduktor/kafka-stack-docker-compose/actions) -# An open-source project by [![Conduktor.io](https://www.conduktor.io/uploads/conduktor.svg)](https://conduktor.io/) +# An open-source project by [![Conduktor.io](https://www.conduktor.io/images/logo.svg)](https://conduktor.io/) This project is sponsored by [Conduktor.io](https://www.conduktor.io/), a graphical desktop user interface for Apache Kafka. Once you have started your cluster, you can use Conduktor to easily manage it. -Just connect against `localhost:9092` if using Docker, or `192.168.99.100` if using Docker Toolbox +Just connect against `localhost:9092`. If you are on Mac or Windows and want to connect from another container, use `host.docker.internal:29092` # kafka-stack-docker-compose @@ -15,38 +15,60 @@ This replicates as well as possible real deployment configurations, where you ha ## Stack version - - Zookeeper version: 3.6.3 (Confluent 7.1.1) - - Kafka version: 3.1.0 (Confluent 7.1.1) - - Kafka Schema Registry: Confluent 7.1.1 - - Kafka Rest Proxy: Confluent 7.1.1 - - Kafka Connect: Confluent 7.1.1 - - ksqlDB Server: Confluent 7.1.1 + - Conduktor Platform: latest + - Zookeeper version: 3.6.3 (Confluent 7.3.2) + - Kafka version: 3.3.0 (Confluent 7.3.2) + - Kafka Schema Registry: Confluent 7.3.2 + - Kafka Rest Proxy: Confluent 7.3.2 + - Kafka Connect: Confluent 7.3.2 + - ksqlDB Server: Confluent 7.3.2 - Zoonavigator: 1.1.1 -For a UI tool to access your local Kafka cluster, use the free version of [Conduktor](https://www.conduktor.io/download) +For a UI tool to access your local Kafka cluster, use the free version of [Conduktor](https://www.conduktor.io/get-started) # Requirements -## Docker - -Please export your environment before starting the stack: -``` -export DOCKER_HOST_IP=127.0.0.1 -``` -(that's the default value and you actually don't need to do a thing) +Kafka will be exposed on `127.0.0.1` or `DOCKER_HOST_IP` if set in the environment. +(You probably don't need to set it if you're not using Docker-Toolbox) ## Docker-Toolbox -If you are using Docker for Mac <= 1.11, or Docker Toolbox for Windows -(your docker machine IP is usually `192.168.99.100`) - -Please export your environment before starting the stack: +Docker toolbox is [deprecated](https://github.com/docker-archive/toolbox) and not maintained anymore for several years. +We can't guarantee this stack will work with Docker Toolbox, but if you want to try anyway, please export your environment before starting the stack: ``` export DOCKER_HOST_IP=192.168.99.100 ``` +(your docker machine IP is usually `192.168.99.100`) + +## Apple M1 support +Confluent platform supports Apple M1 (ARM64) since version `7.2.0`! Basically, this stack will work out of the box. + +If you want to downgrade confluent platform version, there are two ways: + +1. Add `platform: linux/amd64`. It will work as docker is able to emulate AMD64 instructions. +2. Previous versions have been [built](https://github.com/arm64-compat/confluent-platform) for ARM64 by the community. If you want to use it, just change the image in the corresponding yml. Since it is a not an official image, use it at your own risks. + +## Full stack -## Mac M1 issues +To ease you journey with kafka just connect to [localhost:8080](http://localhost:8080/) -Currently, the Docker Images are not working with M1 Mac. This is because they haven't been built by Confluent for that platform. See [confluentinc/common-docker/#117](https://github.com/confluentinc/common-docker/issues/117) for more details +login: `admin@admin.io` +password: `admin` + + - Conduktor-platform: `$DOCKER_HOST_IP:8080` + - Single Zookeeper: `$DOCKER_HOST_IP:2181` + - Single Kafka: `$DOCKER_HOST_IP:9092` + - Kafka Schema Registry: `$DOCKER_HOST_IP:8081` + - Kafka Rest Proxy: `$DOCKER_HOST_IP:8082` + - Kafka Connect: `$DOCKER_HOST_IP:8083` + - KSQL Server: `$DOCKER_HOST_IP:8088` +- (experimental) JMX port at `$DOCKER_HOST_IP:9001` + + Run with: + ``` + docker-compose -f full-stack.yml up + docker-compose -f full-stack.yml down + ``` +** Note: if you find that you can not connect to [localhost:8080](http://localhost:8080/) please run `docker-compose -f full-stack.yml build` to rebuild the port mappings. ## Single Zookeeper / Single Kafka @@ -106,25 +128,6 @@ docker-compose -f zk-multiple-kafka-multiple.yml up docker-compose -f zk-multiple-kafka-multiple.yml down ``` -## Full stack - -Need a UI? We recommend using [Conduktor](https://conduktor.io) as your tool to bring a unified UI to all these components - - - Single Zookeeper: `$DOCKER_HOST_IP:2181` - - Single Kafka: `$DOCKER_HOST_IP:9092` - - Kafka Schema Registry: `$DOCKER_HOST_IP:8081` - - Kafka Rest Proxy: `$DOCKER_HOST_IP:8082` - - Kafka Connect: `$DOCKER_HOST_IP:8083` - - KSQL Server: `$DOCKER_HOST_IP:8088` - - Zoonavigator Web: `$DOCKER_HOST_IP:8004` -- (experimental) JMX port at `$DOCKER_HOST_IP:9999` - - Run with: - ``` - docker-compose -f full-stack.yml up - docker-compose -f full-stack.yml down - ``` - # FAQ ## Kafka @@ -157,12 +160,12 @@ A: yes. Say you want to change `zoo1` port to `12181` (only relevant lines are s A: yes. Say you want to change `kafka1` port to `12345` (only relevant lines are shown). Note only `LISTENER_DOCKER_EXTERNAL` changes: ``` kafka1: - image: confluentinc/cp-kafka:7.1.1 + image: confluentinc/cp-kafka:7.2.1 hostname: kafka1 ports: - "12345:12345" environment: - KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:12345 + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:19092,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:12345,DOCKER://host.docker.internal:29092 ``` **Q: Kafka is using a lot of disk space for testing. Can I reduce it?** @@ -171,7 +174,7 @@ A: yes. This is for testing only!!! Reduce the KAFKA_LOG_SEGMENT_BYTES to 16MB a ``` kafka1: - image: confluentinc/cp-kafka:7.1.1 + image: confluentinc/cp-kafka:7.2.1 ... environment: ... @@ -187,11 +190,11 @@ For example, if the IP of your machine is `50.10.2.3`, follow the sample mapping ``` kafka1: - image: confluentinc/cp-kafka:7.1.1 + image: confluentinc/cp-kafka:7.2.1 ... environment: ... - KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka2:19093,LISTENER_DOCKER_EXTERNAL://50.10.2.3:9093 + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka2:19093,EXTERNAL://50.10.2.3:9093,DOCKER://host.docker.internal:29093 ``` **Q: How do I add connectors to kafka connect?** diff --git a/full-stack.yml b/full-stack.yml index 28b24ad..ba39bcb 100644 --- a/full-stack.yml +++ b/full-stack.yml @@ -2,7 +2,7 @@ version: '2.1' services: zoo1: - image: confluentinc/cp-zookeeper:7.1.1 + image: confluentinc/cp-zookeeper:7.3.2 hostname: zoo1 container_name: zoo1 ports: @@ -13,23 +13,24 @@ services: ZOOKEEPER_SERVERS: zoo1:2888:3888 kafka1: - image: confluentinc/cp-kafka:7.1.1 + image: confluentinc/cp-kafka:7.3.2 hostname: kafka1 container_name: kafka1 ports: - "9092:9092" + - "29092:29092" - "9999:9999" environment: - KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT - KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:19092,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092,DOCKER://host.docker.internal:29092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181" KAFKA_BROKER_ID: 1 KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 - KAFKA_JMX_PORT: 9999 + KAFKA_JMX_PORT: 9001 KAFKA_JMX_HOSTNAME: ${DOCKER_HOST_IP:-127.0.0.1} KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" @@ -37,7 +38,7 @@ services: - zoo1 kafka-schema-registry: - image: confluentinc/cp-schema-registry:7.1.1 + image: confluentinc/cp-schema-registry:7.3.2 hostname: kafka-schema-registry container_name: kafka-schema-registry ports: @@ -52,7 +53,7 @@ services: kafka-rest-proxy: - image: confluentinc/cp-kafka-rest:7.1.1 + image: confluentinc/cp-kafka-rest:7.3.2 hostname: kafka-rest-proxy container_name: kafka-rest-proxy ports: @@ -70,7 +71,7 @@ services: kafka-connect: - image: confluentinc/cp-kafka-connect:7.1.1 + image: confluentinc/cp-kafka-connect:7.3.2 hostname: kafka-connect container_name: kafka-connect ports: @@ -112,7 +113,7 @@ services: ksqldb-server: - image: confluentinc/cp-ksqldb-server:7.1.1 + image: confluentinc/cp-ksqldb-server:7.3.2 hostname: ksqldb-server container_name: ksqldb-server ports: @@ -125,11 +126,29 @@ services: - zoo1 - kafka1 - zoonavigator: - image: elkozmon/zoonavigator:1.1.1 - container_name: zoonavigator + conduktor-platform: + image: conduktor/conduktor-platform:latest ports: - - "8004:8000" + - 8080:8080 + volumes: + - conduktor_data:/var/conduktor + - type: bind + source: "./platform-config.yaml" + target: /opt/conduktor/platform-config.yaml + read_only: true + depends_on: + - kafka1 + - kafka-schema-registry + - kafka-connect environment: - HTTP_PORT: 8000 - AUTO_CONNECT_CONNECTION_STRING: zoo1:2181 + CDK_IN_CONF_FILE: /opt/conduktor/platform-config.yaml + KAFKA_BOOTSTRAP_SERVER: PLAINTEXT://kafka1:19092 + SR_SERVER: "http://kafka-schema-registry:8081" + ORGANISATION_NAME: "default" + ADMIN_EMAIL: "admin@admin.io" + ADMIN_PSW: "admin" + + +volumes: + conduktor_data: {} + diff --git a/jmx-exporter.yml b/jmx-exporter.yml new file mode 100644 index 0000000..274652a --- /dev/null +++ b/jmx-exporter.yml @@ -0,0 +1,70 @@ +lowercaseOutputName: true +lowercaseOutputLabelNames: true +rules: + # Special cases and very specific rules + - pattern : kafka.server<>Value + name: kafka_server_$1_$2 + type: GAUGE + labels: + clientId: "$3" + topic: "$4" + partition: "$5" + - pattern : kafka.server<>Value + name: kafka_server_$1_$2 + type: GAUGE + labels: + clientId: "$3" + broker: "$4:$5" + + - pattern : kafka.server<>OneMinuteRate + name: kafka_server_kafkarequesthandlerpool_requesthandleravgidlepercent_total + type: GAUGE + + - pattern : kafka.server<>connections + name: kafka_server_socketservermetrics_connections + type: GAUGE + labels: + client_software_name: "$1" + client_software_version: "$2" + listener: "$3" + network_processor: "$4" + + - pattern : 'kafka.server<>(.+):' + name: kafka_server_socketservermetrics_$3 + type: GAUGE + labels: + listener: "$1" + network_processor: "$2" + + # Count and Value + - pattern: kafka.(.+)<>(Count|Value) + name: kafka_$1_$2_$3 + labels: + "$4": "$5" + "$6": "$7" + - pattern: kafka.(.+)<>(Count|Value) + name: kafka_$1_$2_$3 + labels: + "$4": "$5" + - pattern: kafka.(.+)<>(Count|Value) + name: kafka_$1_$2_$3 + + # Percentile + - pattern: kafka.(.+)<>(\d+)thPercentile + name: kafka_$1_$2_$3 + type: GAUGE + labels: + "$4": "$5" + "$6": "$7" + quantile: "0.$8" + - pattern: kafka.(.+)<>(\d+)thPercentile + name: kafka_$1_$2_$3 + type: GAUGE + labels: + "$4": "$5" + quantile: "0.$6" + - pattern: kafka.(.+)<>(\d+)thPercentile + name: kafka_$1_$2_$3 + type: GAUGE + labels: + quantile: "0.$4" diff --git a/platform-config.yaml b/platform-config.yaml new file mode 100644 index 0000000..513a3e6 --- /dev/null +++ b/platform-config.yaml @@ -0,0 +1,25 @@ +organization: + name: "${ORGANISATION_NAME}" + +clusters: + - id: default + name: My Local Kafka Cluster + color: "#0013E7" + ignoreUntrustedCertificate: false + bootstrapServers: "$KAFKA_BOOTSTRAP_SERVER" + properties: + schemaRegistry: + url: "$SR_SERVER" + ignoreUntrustedCertificate: false + properties: + labels: {} + kafkaConnects: + - url: http://kafka-connect:8083 + name: full stack kafka connect + +auth: + demo-users: + - email: "${ADMIN_EMAIL}" + password: "${ADMIN_PSW}" + groups: + - ADMIN \ No newline at end of file diff --git a/zk-multiple-kafka-multiple-schema-registry.yml b/zk-multiple-kafka-multiple-schema-registry.yml index df62a77..980a812 100644 --- a/zk-multiple-kafka-multiple-schema-registry.yml +++ b/zk-multiple-kafka-multiple-schema-registry.yml @@ -2,7 +2,7 @@ version: '2.1' services: zoo1: - image: confluentinc/cp-zookeeper:7.1.1 + image: confluentinc/cp-zookeeper:7.3.2 hostname: zoo1 container_name: zoo1 ports: @@ -13,7 +13,7 @@ services: ZOOKEEPER_SERVERS: zoo1:2888:3888;zoo2:2888:3888;zoo3:2888:3888 zoo2: - image: confluentinc/cp-zookeeper:7.1.1 + image: confluentinc/cp-zookeeper:7.3.2 hostname: zoo2 container_name: zoo2 ports: @@ -24,7 +24,7 @@ services: ZOOKEEPER_SERVERS: zoo1:2888:3888;zoo2:2888:3888;zoo3:2888:3888 zoo3: - image: confluentinc/cp-zookeeper:7.1.1 + image: confluentinc/cp-zookeeper:7.3.2 hostname: zoo3 container_name: zoo3 ports: @@ -36,15 +36,16 @@ services: kafka1: - image: confluentinc/cp-kafka:7.1.1 + image: confluentinc/cp-kafka:7.3.2 hostname: kafka1 container_name: kafka1 ports: - "9092:9092" + - "29092:29092" environment: - KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT - KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:19092,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092,DOCKER://host.docker.internal:29092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183" KAFKA_BROKER_ID: 1 KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" @@ -56,15 +57,16 @@ services: - zoo3 kafka2: - image: confluentinc/cp-kafka:7.1.1 + image: confluentinc/cp-kafka:7.3.2 hostname: kafka2 container_name: kafka2 ports: - "9093:9093" + - "29093:29093" environment: - KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka2:19093,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9093 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT - KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka2:19093,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9093,DOCKER://host.docker.internal:29093 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183" KAFKA_BROKER_ID: 2 KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" @@ -76,15 +78,16 @@ services: - zoo3 kafka3: - image: confluentinc/cp-kafka:7.1.1 + image: confluentinc/cp-kafka:7.3.2 hostname: kafka3 container_name: kafka3 ports: - "9094:9094" + - "29094:29094" environment: - KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka3:19094,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9094 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT - KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka3:19094,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9094,DOCKER://host.docker.internal:29094 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183" KAFKA_BROKER_ID: 3 KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" @@ -96,7 +99,7 @@ services: - zoo3 kafka-schema-registry: - image: confluentinc/cp-schema-registry:7.1.1 + image: confluentinc/cp-schema-registry:7.3.2 hostname: kafka-schema-registry container_name: kafka-schema-registry depends_on: diff --git a/zk-multiple-kafka-single.yml b/zk-multiple-kafka-single.yml index f5d492e..3aa804f 100644 --- a/zk-multiple-kafka-single.yml +++ b/zk-multiple-kafka-single.yml @@ -2,7 +2,7 @@ version: '2.1' services: zoo1: - image: confluentinc/cp-zookeeper:7.1.1 + image: confluentinc/cp-zookeeper:7.3.2 hostname: zoo1 container_name: zoo1 ports: @@ -13,7 +13,7 @@ services: ZOOKEEPER_SERVERS: zoo1:2888:3888;zoo2:2888:3888;zoo3:2888:3888 zoo2: - image: confluentinc/cp-zookeeper:7.1.1 + image: confluentinc/cp-zookeeper:7.3.2 hostname: zoo2 container_name: zoo2 ports: @@ -24,7 +24,7 @@ services: ZOOKEEPER_SERVERS: zoo1:2888:3888;zoo2:2888:3888;zoo3:2888:3888 zoo3: - image: confluentinc/cp-zookeeper:7.1.1 + image: confluentinc/cp-zookeeper:7.3.2 hostname: zoo3 container_name: zoo3 ports: @@ -36,16 +36,17 @@ services: kafka1: - image: confluentinc/cp-kafka:7.1.1 + image: confluentinc/cp-kafka:7.3.2 hostname: kafka1 container_name: kafka1 ports: - "9092:9092" + - "29092:29092" - "9999:9999" environment: - KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT - KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:19092,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092,DOCKER://host.docker.internal:29092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183" KAFKA_BROKER_ID: 1 KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" diff --git a/zk-single-kafka-single.yml b/zk-single-kafka-single.yml index ab67625..4a2edb5 100644 --- a/zk-single-kafka-single.yml +++ b/zk-single-kafka-single.yml @@ -2,7 +2,7 @@ version: '2.1' services: zoo1: - image: confluentinc/cp-zookeeper:7.1.1 + image: confluentinc/cp-zookeeper:7.3.2 hostname: zoo1 container_name: zoo1 ports: @@ -13,16 +13,17 @@ services: ZOOKEEPER_SERVERS: zoo1:2888:3888 kafka1: - image: confluentinc/cp-kafka:7.1.1 + image: confluentinc/cp-kafka:7.3.2 hostname: kafka1 container_name: kafka1 ports: - "9092:9092" + - "29092:29092" - "9999:9999" environment: - KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT - KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:19092,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092,DOCKER://host.docker.internal:29092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181" KAFKA_BROKER_ID: 1 KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" @@ -34,4 +35,4 @@ services: KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" depends_on: - - zoo1 \ No newline at end of file + - zoo1 From c7fd01c583a9a13aec03c39c2370b84c60873818 Mon Sep 17 00:00:00 2001 From: yasinlachiny Date: Sun, 5 Mar 2023 16:48:10 +0100 Subject: [PATCH 4/4] fix: remove localhost --- zk-multiple-kafka-multiple.yml | 6 +++--- zk-single-kafka-multiple.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/zk-multiple-kafka-multiple.yml b/zk-multiple-kafka-multiple.yml index 42f6854..faacf0f 100644 --- a/zk-multiple-kafka-multiple.yml +++ b/zk-multiple-kafka-multiple.yml @@ -44,7 +44,7 @@ services: - "9092:9092" - "29092:29092" environment: - KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:19092,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092,DOCKER://host.docker.internal:29092 + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:19092,EXTERNAL://${DOCKER_HOST_IP}:9092,DOCKER://host.docker.internal:29092 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183" @@ -65,7 +65,7 @@ services: - "9093:9093" - "29093:29093" environment: - KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka2:19093,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9093,DOCKER://host.docker.internal:29093 + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka2:19093,EXTERNAL://${DOCKER_HOST_IP}:9093,DOCKER://host.docker.internal:29093 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183" @@ -86,7 +86,7 @@ services: - "9094:9094" - "29094:29094" environment: - KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka3:19094,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9094,DOCKER://host.docker.internal:29094 + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka3:19094,EXTERNAL://${DOCKER_HOST_IP}:9094,DOCKER://host.docker.internal:29094 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183" diff --git a/zk-single-kafka-multiple.yml b/zk-single-kafka-multiple.yml index 01e26e5..a292f06 100644 --- a/zk-single-kafka-multiple.yml +++ b/zk-single-kafka-multiple.yml @@ -21,7 +21,7 @@ services: - "9092:9092" - "29092:29092" environment: - KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:19092,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092,DOCKER://host.docker.internal:29092 + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:19092,EXTERNAL://${DOCKER_HOST_IP}:9092,DOCKER://host.docker.internal:29092 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181" @@ -40,7 +40,7 @@ services: - "9093:9093" - "29093:29093" environment: - KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka2:19093,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9093,DOCKER://host.docker.internal:29093 + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka2:19093,EXTERNAL://${DOCKER_HOST_IP}:9093,DOCKER://host.docker.internal:29093 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181" @@ -60,7 +60,7 @@ services: - "9094:9094" - "29094:29094" environment: - KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka3:19094,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9094,DOCKER://host.docker.internal:29094 + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka3:19094,EXTERNAL://${DOCKER_HOST_IP}:9094,DOCKER://host.docker.internal:29094 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"