docker-compose.yml

version: '2'
services:
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.0
    container_name: elasticsearch
    environment:
      - cluster.name=elasticsearch-cluster
      - bootstrap.memory_lock=false
      - ES_JAVA_OPTS=-Xms1g -Xmx1g
      - discovery.type=single-node
      - network.host=0.0.0.0
      - http.port=9200
    ulimits:
      memlock:
        soft: -1
        hard: -1
    ports:
      - 9200:9200
      - 9300:9300
    networks:
      - devnet
  kibana:
    image: docker.elastic.co/kibana/kibana-oss:7.10.0
    container_name: kibana
    ports:
      - 5601:5601
    networks:
      - devnet
    depends_on:
      - elasticsearch
  tube:
    image: "quay.io/cdis/tube:master"
    command: bash -c "python run_config.py && sleep 50 && python run_etl.py"
    networks:
      - devnet
    environment:
      - DICTIONARY_URL=https://s3.amazonaws.com/dictionary-artifacts/ndhdictionary/master/schema.json
      - ES_URL=elasticsearch
      - ES_INDEX_NAME=etl
      - HADOOP_URL=hdfs://spark:9000
      - HADOOP_HOST=spark
      - LOG_LEVEL=WARN
    volumes:
      - ./configs/creds.json:/usr/share/gen3/tube/creds.json
      - ./configs/etlMapping.yaml:/usr/share/gen3/tube/etlMapping.yaml
      - ./configs/settings.yaml:/usr/share/gen3/tube/settings.yaml
      - ./configs/analyzers.yaml:/usr/share/gen3/tube/analyzers.yaml
      - ./configs/user.yaml:/usr/share/gen3/tube/user.yaml
    extra_hosts:
      - "host.docker.internal:host-gateway"
    depends_on:
      - spark
  spark:
    image: "quay.io/cdis/gen3-spark:master"
    command: bash -c "python run_config.py && hdfs namenode -format && hdfs --daemon start namenode && hdfs --daemon start datanode && yarn --daemon start resourcemanager && yarn --daemon start nodemanager && hdfs dfsadmin -safemode leave &&  hdfs dfs -mkdir /result && while true; do sleep 5; done"
    expose:
      - 22
      - 8030
      - 8031
      - 8032
      - 9000
    networks:
      - devnet
    environment:
      - HADOOP_URL=hdfs://0.0.0.0:9000
      - HADOOP_HOST=0.0.0.0
networks:
  devnet: