docker-compose.yml

version: "3"

services:
  namenode:
    image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8
    container_name: namenode
    hostname: namenode
    restart: always
    ports:
      - 9870:9870
      - 9000:9000
    volumes:
      - hadoop_namenode:/hadoop/dfs/name
    env_file:
      - ./hadoop.env
    networks:
      - my_other_network
    environment:
      - CLUSTER_NAME=test
      - HDFS_CONF_dfs_blocksize=67108864
    platform: linux/amd64
    deploy:
      resources:
        limits:
          memory: 2G

  datanode:
    image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
    container_name: datanode
    hostname: datanode
    restart: always
    ports:
      - 9864:9864
    volumes:
      - hadoop_datanode:/hadoop/dfs/data
    environment:
      SERVICE_PRECONDITION: "namenode:9870"
    env_file:
      - ./hadoop.env
    networks:
      - my_other_network
    platform: linux/amd64
    deploy:
      resources:
        limits:
          memory: 4G

  spark-master:
    build:
      context: .
      dockerfile: Dockerfile
    hostname: spark-master
    container_name: spark-master
    command: bin/spark-class org.apache.spark.deploy.master.Master
    ports:
      - 8080:8080
      - 7077:7077
    environment:
      - SPARK_DRIVER_MEMORY=2G
      - SPARK_EXECUTOR_MEMORY=2G

    depends_on:
      - namenode
      - datanode
    networks:
      - my_other_network
    deploy:
      resources:
        limits:
          memory: 7G

  spark-worker-1:
    build:
      context: .
      dockerfile: Dockerfile
    hostname: spark-worker-1
    container_name: spark-worker-1
    command: bin/spark-class org.apache.spark.deploy.worker.Worker spark://spark-master:7077
    depends_on:
      - spark-master
    ports:
      - "0.0.0.0:8081:8081"
    networks:
      - my_other_network
    environment:
      SPARK_MODE: worker
      SPARK_WORKER_CORES: 2
      SPARK_WORKER_MEMORY: 2g
      SPARK_MASTER_URL: spark://spark-master:7077

volumes:
  hadoop_namenode:
  hadoop_datanode:
  hadoop_historyserver:


networks:
  my_other_network:
    driver: bridge
    name: my_other_network