diff --git a/hadoop-ozone/dist/src/main/compose/ozone-ha/kafka-config b/hadoop-ozone/dist/src/main/compose/ozone-ha/kafka-config new file mode 100644 index 000000000000..e4f91ee9e78d --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone-ha/kafka-config @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +OZONE-SITE.XML_ozone.om.plugin.destination.kafka=enabled +OZONE-SITE.XML_ozone.om.plugin.destination.kafka.classname=org.apache.hadoop.ozone.om.eventlistener.OMEventListenerKafkaPublisher +OZONE-SITE.XML_ozone.om.plugin.kafka.topic=ozone.om.events +OZONE-SITE.XML_ozone.om.plugin.kafka.bootstrap.servers=kafka-3:29092,kafka-1:29092,kafka-2:29092 +OZONE-SITE.XML_ozone.om.plugin.kafka.key.serializer=org.apache.kafka.common.serialization.StringSerializer +OZONE-SITE.XML_ozone.om.plugin.kafka.value.serializer=org.apache.kafka.common.serialization.StringSerializer diff --git a/hadoop-ozone/dist/src/main/compose/ozone-ha/kafka.yaml b/hadoop-ozone/dist/src/main/compose/ozone-ha/kafka.yaml new file mode 100644 index 000000000000..98bbdf989995 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone-ha/kafka.yaml @@ -0,0 +1,85 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +x-om-kafka-config: + &om-kafka-config + env_file: + - docker-config + - kafka-config + +services: + om1: + <<: *om-kafka-config + om2: + <<: *om-kafka-config + om3: + <<: *om-kafka-config + + zookeeper: + image: confluentinc/cp-zookeeper:7.6.0 + hostname: zookeeper + container_name: zookeeper + ports: + - "2181:2181" + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_TICK_TIME: 2000 + + kafka-1: + image: confluentinc/cp-kafka:7.6.0 + hostname: kafka-1 + container_name: kafka-1 + ports: + - "9092:9092" + environment: + KAFKA_BROKER_ID: 1 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-1:29092,PLAINTEXT_HOST://localhost:9092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT + KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:29092,PLAINTEXT_HOST://0.0.0.0:9092 + KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 + + kafka-2: + image: confluentinc/cp-kafka:7.6.0 + hostname: kafka-2 + container_name: kafka-2 + ports: + - "9093:9093" + environment: + KAFKA_BROKER_ID: 2 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-2:29092,PLAINTEXT_HOST://localhost:9093 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT + KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:29092,PLAINTEXT_HOST://0.0.0.0:9093 + KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 + + kafka-3: + image: confluentinc/cp-kafka:7.6.0 + hostname: kafka-3 + container_name: kafka-3 + ports: + - "9094:9094" + environment: + KAFKA_BROKER_ID: 3 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-3:29092,PLAINTEXT_HOST://localhost:9094 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT + KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:29092,PLAINTEXT_HOST://0.0.0.0:9094 + KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 +