docker-compose.yml

version: '3'

x-spark-common: &spark-common
  image: bitnami/spark:3.2.4
  volumes:
    - ./project-core:/opt/bitnami/spark/jobs
  command: bin/spark-class org.apache.spark.deploy.worker.Worker ${SPARK_MASTER_URL}
  depends_on:
    - spark-master
  env_file:
    - .env
  environment:
    SPARK_MODE: worker
    SPARK_WORKER_CORES: ${SPARK_WORKER_CORES}
    SPARK_WORKER_MEMORY: ${SPARK_WORKER_MEMORY}
    SPARK_MASTER_URL: ${SPARK_MASTER_URL}
    MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
    MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
  networks:
    - app-network

services:
  zookeeper:
    image: confluentinc/cp-zookeeper:7.4.0
    hostname: zookeeper
    container_name: zookeeper
    ports:
      - "2181:2181"
    env_file:
      - .env
    healthcheck:
      test: [ 'CMD', 'bash', '-c', "echo 'ruok' | nc localhost ${ZOOKEEPER_CLIENT_PORT}" ]
      interval: 10s
      timeout: 5s
      retries: 5
    networks:
      - app-network

  broker:
    image: confluentinc/cp-server:7.4.0
    hostname: broker
    container_name: broker
    depends_on:
      zookeeper:
        condition: service_healthy
    ports:
      - "9092:9092"
      - "9101:9101"
    environment:
      KAFKA_BROKER_ID: ${KAFKA_BROKER_ID}
      KAFKA_ZOOKEEPER_CONNECT: ${KAFKA_ZOOKEEPER_CONNECT}
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: ${KAFKA_LISTENER_SECURITY_PROTOCOL_MAP}
      KAFKA_ADVERTISED_LISTENERS: ${KAFKA_ADVERTISED_LISTENERS}
      KAFKA_METRIC_REPORTERS: ${KAFKA_METRIC_REPORTERS}
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: ${KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR}
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: ${KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS}
      KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: ${KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR}
      KAFKA_CONFLUENT_BALANCER_TOPIC_REPLICATION_FACTOR: ${KAFKA_CONFLUENT_BALANCER_TOPIC_REPLICATION_FACTOR}
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: ${KAFKA_TRANSACTION_STATE_LOG_MIN_ISR}
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: ${KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR}
      KAFKA_JMX_PORT: ${KAFKA_JMX_PORT}
      KAFKA_JMX_HOSTNAME: ${KAFKA_JMX_HOSTNAME}
      KAFKA_CONFLUENT_SCHEMA_REGISTRY_URL: ${KAFKA_CONFLUENT_SCHEMA_REGISTRY_URL}
      CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: ${CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS}
      CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: ${CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS}
      CONFLUENT_METRICS_ENABLE: ${CONFLUENT_METRICS_ENABLE}
      CONFLUENT_SUPPORT_CUSTOMER_ID: ${CONFLUENT_SUPPORT_CUSTOMER_ID}
    healthcheck:
      test: [ 'CMD', 'bash', '-c', "nc -z localhost 9092" ]
      interval: 10s
      timeout: 5s
      retries: 5
    networks:
      - app-network
    restart: on-failure

  init-kafka:
    build: ./init-kafka
    networks:
      - app-network
    depends_on:
      broker:
        condition: service_healthy

  spark-master:
    image: bitnami/spark:3.2.4
    volumes:
      - ./project-core:/opt/bitnami/spark/jobs
    env_file:
      - .env
    environment:
      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
    command: bin/spark-class org.apache.spark.deploy.master.Master
    ports:
      - "9090:8080"
      - "7077:7077"
    networks:
      - app-network

  twitter-streamer-1:
    build: ./twitter-streamer
    volumes:
      - ./project-core/src:/app/src
    env_file:
      - .env
    networks:
      - app-network
    depends_on:
      - broker

  reddit-streamer-1:
    build: ./reddit-streamer
    volumes:
      - ./project-core/src:/app/src
    env_file:
      - .env
    networks:
      - app-network
    depends_on:
      - broker

  fastapi-ml-endpoint:
    build: ./fastapi-ml-endpoint
    ports:
      - "8000:8000"
    restart: always
    networks:
      - app-network

  spark-worker-1:
    <<: *spark-common

  spark-worker-2:
    <<: *spark-common

  flask-backend:
    build: ./flask-dashboard # Now it builds from the Dockerfile
    container_name: flask-backend
    volumes:
      - ./flask-dashboard:/app # Mounts the app folder
    ports:
      - "5000:5000"
    networks:
      - app-network

  minio:
    image: minio/minio:latest
    container_name: minio
    ports:
      - "9000:9000" # web console/API
      - "9001:9001" # web admin console
    env_file:
      - .env
    command: server /data --console-address ":9001"
    volumes:
      - ./minio/data:/data
    networks:
      - app-network
    healthcheck:
      test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ]
      interval: 30s
      timeout: 10s
      retries: 3

  minio-init:
    image: minio/mc:latest
    container_name: minio-init
    depends_on:
      minio:
        condition: service_healthy # Ensures MinIO is ready
    entrypoint: >
      /bin/sh -c " echo 'Waiting for MinIO...'; mc alias set myminio http://minio:9000 admin password; mc mb myminio/reddit-data; mc mb myminio/twitter-data; mc mb myminio/mlflow; mc mb myminio/processed-data; mc mb myminio/zenml; echo 'Buckets created!'; exit 0;"
    networks:
      - app-network

  redis:
    image: redis:latest
    ports:
      - "6379:6379"
    volumes:
      - ./redis:/data
    command: [ "redis-server", "--appendonly", "yes" ]
    restart: always
    networks:
      - app-network

  postgres:
    image: postgres:14
    env_file:
      - .env
    ports:
      - "5432:5432"
    volumes:
      - ./postgres:/var/lib/postgresql/data
    networks:
      - app-network
    command: >
      bash -c "
        docker-entrypoint.sh postgres &
        until pg_isready -U $$POSTGRES_USER -h localhost; do sleep 2; done &&
        psql -U $$POSTGRES_USER -tc \"SELECT 1 FROM pg_database WHERE datname = 'mlflow'\" | grep -q 1 || echo \"CREATE DATABASE mlflow;\" | psql -U $$POSTGRES_USER &&
        psql -U $$POSTGRES_USER -tc \"SELECT 1 FROM pg_database WHERE datname = 'airflow'\" | grep -q 1 || echo \"CREATE DATABASE airflow;\" | psql -U $$POSTGRES_USER &&
        psql -U $$POSTGRES_USER -tc \"SELECT 1 FROM pg_database WHERE datname = 'zenml'\" | grep -q 1 || echo \"CREATE DATABASE zenml;\" | psql -U $$POSTGRES_USER &&
        wait"
    restart: always
    healthcheck:
      test: [ "CMD-SHELL", "pg_isready -U $POSTGRES_USER -h localhost" ]
      interval: 10s
      timeout: 5s
      retries: 5
  mlflow:
    image: mlflow:latest
    build: ./mlflow
    env_file:
      - .env
    depends_on:
      postgres:
        condition: service_healthy
      minio:
        condition: service_healthy
    ports:
      - "5001:5001"
    networks:
      - app-network
    command: mlflow server --host 0.0.0.0 --port 5001 --backend-store-uri postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/mlflow --default-artifact-root s3://mlflow/
    restart: always
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:5001/api/2.0/mlflow/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 10s

  kafka-minio:
    build: ./kafka-minio
    volumes:
      - ./project-core/src:/app/src
    env_file:
      - .env
    networks:
      - app-network
    depends_on:
      broker:
        condition: service_healthy
      minio:
        condition: service_healthy

  etl-data-aggregator:
    build: ./etl-data-aggregator
    volumes:
      - ./project-core/src:/app/src
    env_file:
      - .env
    networks:
      - app-network
    depends_on:
      minio:
        condition: service_healthy
  zenml:
    image: zenmldocker/zenml-server:latest
    ports:
      - "8082:8080"
    env_file:
      - .env
    networks:
      - app-network
    depends_on:
      postgres:
        condition: service_healthy
      minio:
        condition: service_healthy
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8080/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 10s

networks:
  app-network: