docker-compose.yml

networks:
  provider_network:
    external:
      name: provider_network
services:
  kafka:
    container_name: kafka
    depends_on:
    - zookeeper
    environment:
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka:9092,EXTERNAL://nedeljko:9094
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_LISTENERS: INTERNAL://kafka:9092,EXTERNAL://:9094
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      KAFKA_LOG4J_ROOT_LOGLEVEL: WARN
      KAFKA_TOOLS_LOG4J_LOGLEVEL: ERROR
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      TZ: Europe/Zurich
    image: wurstmeister/kafka
    networks:
    - provider_network
    ports:
    - 9092:9092
  mongo_db:
    container_name: mongo_db
    environment:
      TZ: Europe/Zurich
    image: mongo:3.6.18
    networks:
    - provider_network
    volumes:
    - ../data/mongodir/:/data/db
  provider:
    build: provider
    container_name: provider
    depends_on:
    - sql_db
    environment:
      DNS_NAME: http://localhost
      KAFKA_HOST: kafka:9092
      MONGO_HOST: mongo_db
      SPARK_BLOCKMANAGER_PORT: 13666
      SPARK_DRIVER_HOST: provider
      SPARK_DRIVER_PORT: 5005
      SPARK_HOST: spark-master
      SQL_DBNAME: sample
      SQL_HOST: mysql+pymysql://mysql:mysql@sql_db/
      TZ: Europe/Zurich
    networks:
    - provider_network
    ports:
    - 80:5000
    - 5005:5005
    - 50051:50051
    - 13666:13666
    volumes:
    - ../:/local/data
  spark-master:
    container_name: spark-master
    environment:
      PACKAGES: org.apache.spark:spark-sql-kafka-0-10_2.11:2.4.0
      SPARK_CONF_DIR: /conf
      TZ: Europe/Zurich
    expose:
    - 7001
    - 7002
    - 7003
    - 7004
    - 7005
    - 7006
    - 7077
    - 6066
    image: nedeljkoradulovic88/spark
    networks:
    - provider_network
    ports:
    - 4040:4040
    - 6066:6066
    - 7077:7077
    - 8080:8080
  sql_db:
    command:
    - mysqld
    - --default-authentication-plugin=mysql_native_password
    container_name: sql_db
    depends_on:
    - spark-master
    environment:
      MYSQL_DATABASE: sample
      MYSQL_PASSWORD: mysql
      MYSQL_ROOT_PASSWORD: yHnIXC1Skr9k2
      MYSQL_USER: mysql
      TZ: Europe/Zurich
    image: mysql:5.7.21
    networks:
    - provider_network
    ports:
    - 3306:3306
    volumes:
    - ../data/sqldir/:/var/lib/mysql
  worker1:
    command: bin/spark-class org.apache.spark.deploy.worker.Worker spark://spark-master:7077
    container_name: worker1
    environment:
      PACKAGES: org.apache.spark:spark-sql-kafka-0-10_2.11:2.4.0
      SPARK_CONF_DIR: /conf
      SPARK_WORKER_CORES: 2
      SPARK_WORKER_MEMORY: 2g
      SPARK_WORKER_PORT: 8881
      SPARK_WORKER_WEBUI_PORT: 8082
      TZ: Europe/Zurich
    expose:
    - 7012
    - 7013
    - 7014
    - 7015
    - 7016
    - 8881
    hostname: worker1
    image: nedeljkoradulovic88/spark
    links:
    - spark-master
    networks:
    - provider_network
    ports:
    - 8082:8082
    volumes:
    - ../spark/:/tmp
  zookeeper:
    container_name: zookeeper
    environment:
      TZ: Europe/Zurich
    image: wurstmeister/zookeeper
    networks:
    - provider_network
    ports:
    - 2181:2181
version: '3'