update CKafka pkgs (#8)

* update CKafka pkgs

* bugfix for docker

* add docker-compose

README.md

@@ -5,7 +5,7 @@
 [![Join the Gitter Chat](https://img.shields.io/gitter/room/nwjs/nw.js.svg)](https://gitter.im/QPod/)
 [![Docker Pulls](https://img.shields.io/docker/pulls/qpod/qpod.svg)](https://hub.docker.com/r/qpod/qpod)
 [![Docker Starts](https://img.shields.io/docker/stars/qpod/qpod.svg)](https://hub.docker.com/r/qpod/qpod)
-[![Recent Code Update](https://img.shields.io/github/last-commit/QPod/media-lab.svg)](https://github.com/QPod/data-lab/stargazers)
+[![Recent Code Update](https://img.shields.io/github/last-commit/QPod/data-lab.svg)](https://github.com/QPod/data-lab/stargazers)
 
 Please generously STAR★ our project or donate to us!  [![GitHub Starts](https://img.shields.io/github/stars/QPod/data-lab.svg?label=Stars&style=social)](https://github.com/QPod/data-lab/stargazers)
 [![Donate-PayPal](https://img.shields.io/badge/Donate-PayPal-blue.svg)](https://paypal.me/haobibo)
@@ -14,7 +14,7 @@ Please generously STAR★ our project or donate to us!  [![GitHub Starts](https:
 
 [Wiki & Document](https://github.com/QPod/docker-images/wiki) | [中文使用指引(含中国地区镜像)](https://github.com/QPod/docker-images/wiki/QPod%E4%B8%AD%E6%96%87%E6%8C%87%E5%BC%95)
 
-## Building blocks for multimedia media projects
+## Building blocks for data lake and pipelines projects
 
 Building blocks for the following big data project use cases are supported in this project:
 - Flink

docker_kafka/Dockerfile

@@ -3,12 +3,24 @@ ARG BASE_NAMESPACE
 ARG BASE_IMG="jdk11"
 FROM ${BASE_NAMESPACE:+$BASE_NAMESPACE/}${BASE_IMG}
 
+ARG KAFKA_VERSION="7.3.3"
+
 LABEL maintainer="[email protected]"
 
+# ref: https://github.com/confluentinc/kafka-images/blob/master/kafka/Dockerfile.ubi8
+
 COPY work /opt/utils/
 
-ENV KAFKA_HOME=/opt/kafka
+ENV KAFKA_HOME=/opt/kafka \
+    KAFKA_VERSION="${KAFKA_VERSION}" \
+    COMPONENT=kafka
 
 RUN source /opt/utils/script-confluent-kafka.sh \
  && echo "Install confluent-kafka:" && setup_confluent_kafka \
+ && echo "Setup confluent kafka to use KRaft" && setup_confluent_kafka_kraft \
+ && pip install -U confluent-kafka \
  && echo "Clean up" && list_installed_packages && install__clean
+
+EXPOSE 9092
+VOLUME ["/var/lib/${COMPONENT}/data", "/etc/${COMPONENT}/secrets"]
+CMD ["/etc/confluent/docker/run"]

docker_kafka/docker-compose.yml

@@ -0,0 +1,32 @@
+---
+version: '2'
+services:
+
+  broker:
+    image: qpod/kafka:latest
+    hostname: broker
+    container_name: broker
+    ports:
+      - "9092:9092"
+      - "9101:9101"
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT'
+      KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092'
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
+      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+      KAFKA_JMX_PORT: 9101
+      KAFKA_JMX_HOSTNAME: localhost
+      KAFKA_PROCESS_ROLES: 'broker,controller'
+      KAFKA_NODE_ID: 1
+      KAFKA_CONTROLLER_QUORUM_VOTERS: '1@broker:29093'
+      KAFKA_LISTENERS: 'PLAINTEXT://broker:29092,CONTROLLER://broker:29093,PLAINTEXT_HOST://0.0.0.0:9092'
+      KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT'
+      KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
+      KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
+    volumes:
+      - ./kafka-broker/data:/var/lib/kafka-broker/data
+      - ./kafka-broker/secrets:/etc/kafka-broker/secrets
+    command: "bash -c '/etc/confluent/docker/run'"

docker_kafka/work/script-confluent-kafka.sh

@@ -2,20 +2,29 @@ source /opt/utils/script-utils.sh
 
 setup_confluent_kafka() {
   # ref: https://docs.confluent.io/platform/current/installation/installing_cp/zip-tar.html#get-the-software
-  local VER_C_KAFKA_MINOR=${VERSION_CONFLUENT_KAFKA:-"7.2.2"}
-  local VER_C_KAFKA_MAJOR=${VERSION_CONFLUENT_KAFKA:-"7.2"}
-  install_tar_gz "http://packages.confluent.io/archive/${VER_C_KAFKA_MAJOR}/confluent-${VER_C_KAFKA_MINOR}.tar.gz"
-  mv /opt/confluent-* /opt/kafka
+  local VER_C_KAFKA_MINOR=${KAFKA_VERSION:-"7.3.3"}
+  local VER_C_KAFKA_MAJOR=${VER_C_KAFKA_MINOR%.*}
+  local URL_C_KAFKA="http://packages.confluent.io/archive/${VER_C_KAFKA_MAJOR}/confluent-${VER_C_KAFKA_MINOR}.tar.gz"
+  echo "Installing Confluent Kafka from: $URL_C_KAFKA}"
+  install_tar_gz "${URL_C_KAFKA}"
+  install_zip https://github.com/confluentinc/kafka-images/archive/refs/heads/master.zip
+
   export KAFKA_HOME=/opt/kafka
+  mv /opt/confluent-* /opt/kafka
+  mv /opt/kafka-images* ${KAFKA_HOME}/docker
+  mv "${KAFKA_HOME}/docker/kafka/include/etc/confluent/docker/"  "${KAFKA_HOME}/etc/"
+  ln -sf ${KAFKA_HOME}/etc /etc/confluent
   ls -alh ${KAFKA_HOME}/*
+
+  echo "Setting up kafka dirs:" && mkdir -pv /var/lib/kafka/data /etc/kafka/secrets
 }
 
 setup_confluent_kafka_kraft() {
-  # ref: https://github.com/confluentinc/cp-all-in-one/blob/7.2.1-post/cp-all-in-one-kraft/update_run.sh
+  # ref: https://github.com/confluentinc/cp-all-in-one/blob/7.3.3-post/cp-all-in-one-kraft/update_run.sh
   # Docker workaround: Remove check for KAFKA_ZOOKEEPER_CONNECT parameter
-  sed -i '/KAFKA_ZOOKEEPER_CONNECT/d' ${KAFKA_HOME}/etc/confluent/docker/configure
+  sed -i '/KAFKA_ZOOKEEPER_CONNECT/d' /etc/confluent/docker/configure
   # Docker workaround: Ignore cub zk-ready
-  sed -i 's/cub zk-ready/echo ignore zk-ready/' ${KAFKA_HOME}/etc/confluent/docker/ensure
+  sed -i 's/cub zk-ready/echo ignore zk-ready/' /etc/confluent/docker/ensure
   # KRaft required step: Format the storage directory with a new cluster ID
-  echo "kafka-storage format --ignore-formatted -t $(kafka-storage random-uuid) -c ${KAFKA_HOME}/etc/kafka/kafka.properties" >>${KAFKA_HOME}/etc/confluent/docker/ensure
+  echo "kafka-storage format --ignore-formatted -t $(kafka-storage random-uuid) -c /etc/kafka/kafka.properties" >> /etc/confluent/docker/ensure
 }