docker-compose.yml

version: "3.8"
volumes:
  ingester-volume:
services:
  zookeeper:
    image: confluentinc/cp-zookeeper:7.4.3
    container_name: zookeeper
    ports:
    - "2181:2181"
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
  kafka:
    image: confluentinc/cp-kafka:7.4.3
    container_name: kafka
    ports:
      - "29092:29092"       
      - "9092:9092"
    depends_on:
      - zookeeper
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092   # req on 29092(plaintext), on 9092(plaintext_host)
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: true
  kafka-connect:
    image: confluentinc/cp-kafka-connect:7.4.3
    container_name: kafka-connect
    ports:
      - '8083:8083'
    depends_on:
      kafka:
        condition: service_started
      elastic:
        condition: service_healthy
    volumes:
      - "./connect-plugins:/connect-plugins"
    healthcheck:
      test:     
        [
          "CMD-SHELL",
          "curl localhost:8083/ | grep -q commit"
        ]
      interval: 10s
      timeout: 10s
      retries: 24  
    environment:
      CONNECT_BOOTSTRAP_SERVERS: kafka:29092
      CONNECT_REST_PORT: 8083
      CONNECT_GROUP_ID: 'connect'
      CONNECT_CONFIG_STORAGE_TOPIC: connect-config
      CONNECT_OFFSET_STORAGE_TOPIC: connect-offsets
      CONNECT_STATUS_STORAGE_TOPIC: connect-status
      CONNECT_REPLICATION_FACTOR: 1
      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_KEY_CONVERTER: 'org.apache.kafka.connect.storage.StringConverter'
      CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: 'false'
      CONNECT_VALUE_CONVERTER: 'org.apache.kafka.connect.json.JsonConverter'
      CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: 'false'
      CONNECT_INTERNAL_KEY_CONVERTER: 'org.apache.kafka.connect.storage.StringConverter'
      CONNECT_INTERNAL_VALUE_CONVERTER: 'org.apache.kafka.connect.json.JsonConverter'
      CONNECT_PRODUCER_INTERCEPTOR_CLASSES: 'io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor'
      CONNECT_CONSUMER_INTERCEPTOR_CLASSES: 'io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor'
      CONNECT_REST_ADVERTISED_HOST_NAME: 'connect'
      CONNECT_ZOOKEEPER_CONNECT: zookeeper:2181
      CONNECT_PLUGIN_PATH: /connect-plugins
      CONNECT_LOG4J_ROOT_LOGLEVEL: INFO
      CONNECT_LOG4J_LOGGERS: org.reflections=ERROR
      CLASSPATH: /usr/share/java/monitoring-interceptors/monitoring-interceptors-3.3.0.jar

  elastic:
    image: elasticsearch:8.11.1
    container_name: elastic
    environment:
      - xpack.security.enabled=false
      - "discovery.type=single-node"
    volumes:
      - ingester-volume:/usr/share/elasticsearch/data
    ports:
      - "9200:9200"
    healthcheck:
      test:     
        [
          "CMD-SHELL",
          "curl -s -X GET http://localhost:9200/_cluster/health?pretty | grep status | grep -q '\\(green\\|yellow\\)'"
        ]
      interval: 10s
      timeout: 10s
      retries: 24
  query-interface:
    build:
      context: ./
      dockerfile: query_interface/dockerfile
    ports:
      - "5000:5000"
    depends_on:
      elastic:
        condition: service_healthy
  ingester:
    build:
      context: ./
      dockerfile: ingester/dockerfile
    ports:
      - "3000:3000"
    depends_on:
      kafka-connect:
        condition: service_healthy