docker-compose.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

# Basic Airflow cluster configuration for CeleryExecutor with Redis and PostgreSQL.
#
# WARNING: This configuration is for local development. Do not use it in a production deployment.
#
# This configuration supports basic configuration using environment variables or an .env file
# The following variables are supported:
#
# AIRFLOW_IMAGE_NAME           - Docker image name used to run Airflow.
#                                Default: apache/airflow:|version|
# AIRFLOW_UID                  - User ID in Airflow containers
#                                Default: 50000
# AIRFLOW_GID                  - Group ID in Airflow containers
#                                Default: 50000
#
# Those configurations are useful mostly in case of standalone testing/running Airflow in test/try-out mode
#
# _AIRFLOW_WWW_USER_USERNAME   - Username for the administrator account (if requested).
#                                Default: airflow
# _AIRFLOW_WWW_USER_PASSWORD   - Password for the administrator account (if requested).
#                                Default: airflow
# _PIP_ADDITIONAL_REQUIREMENTS - Additional PIP requirements to add when starting all containers.
#                                Default: ''
#
# Feel free to modify this file to suit your needs.
---
version: "3"
x-airflow-common: &airflow-common
  # In order to add custom dependencies or upgrade provider packages you can use your extended image.
  # Comment the image line, place your Dockerfile in the directory where you placed the docker-compose.yaml
  # and uncomment the "build" line below, Then run `docker-compose build` to build the images.
  # image: apache/airflow:latest-python3.8
  build: .
  env_file: .env
  environment: &airflow-common-env
    AIRFLOW__CORE__EXECUTOR: LocalExecutor # Use LocalExecutor instead of CeleryExecutor to reduce number of containers and thus reduce memory usage
    AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres-airflow/airflow # hostname must be same as service name
    # AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://airflow:airflow@postgres-airflow/airflow
    # AIRFLOW__CELERY__BROKER_URL: redis://:@redis:6379/0
    AIRFLOW__CORE__FERNET_KEY: ""
    AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "false"
    AIRFLOW__CORE__LOAD_EXAMPLES: "false"
    AIRFLOW__API__AUTH_BACKEND: "airflow.api.auth.backend.basic_auth"
    _PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS:-}
    AIRFLOW_CONN_POSTGRES_DEFAULT: postgres://airflow:airflow@postgres-airflow:5432/airflow
  volumes:
    - ./dags:/opt/airflow/dags
    - ./logs:/opt/airflow/logs
    - ./plugins:/opt/airflow/plugins
    - ./filesystem:/opt/airflow/filesystem
    - ./great_expectations:/opt/airflow/great_expectations
  user: "${AIRFLOW_UID:-50000}:${AIRFLOW_GID:-0}"
  depends_on: &airflow-common-depends-on
    # redis:
    #   condition: service_healthy
    postgres-airflow:
      condition: service_healthy
    postgres-source:
      condition: service_healthy
    postgres-dest:
      condition: service_healthy
    postgres-store:
      condition: service_healthy

services:
  postgres-airflow: # since the defaults are specified, can just connect with `psql -U airflow`
    image: postgres:latest
    environment:
      POSTGRES_USER: airflow
      POSTGRES_PASSWORD: airflow
      POSTGRES_DB: airflow
    healthcheck:
      test: ["CMD", "pg_isready", "-U", "airflow"]
      interval: 5s
      retries: 5
    restart: always

  # redis:
  #   image: redis:latest
  #   expose:
  #     - 6379
  #   healthcheck:
  #     test: ["CMD", "redis-cli", "ping"]
  #     interval: 5s
  #     timeout: 30s
  #     retries: 50
  #   restart: always

  airflow-webserver:
    <<: *airflow-common
    command: webserver
    ports:
      - 8080:8080
    healthcheck:
      test: ["CMD", "curl", "--fail", "http://localhost:8080/health"]
      interval: 10s
      timeout: 10s
      retries: 5
    restart: always
    depends_on:
      <<: *airflow-common-depends-on
      airflow-init:
        condition: service_completed_successfully

  airflow-scheduler:
    <<: *airflow-common
    command: scheduler
    healthcheck:
      test:
        [
          "CMD-SHELL",
          'airflow jobs check --job-type SchedulerJob --hostname "$${HOSTNAME}"',
        ]
      interval: 10s
      timeout: 10s
      retries: 5
    restart: always
    depends_on:
      <<: *airflow-common-depends-on
      airflow-init:
        condition: service_completed_successfully

  # airflow-worker:
  #   <<: *airflow-common
  #   command: celery worker
  #   healthcheck:
  #     test:
  #       - "CMD-SHELL"
  #       - 'celery --app airflow.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}"'
  #     interval: 10s
  #     timeout: 10s
  #     retries: 5
  #   restart: always
  #   depends_on:
  #     <<: *airflow-common-depends-on
  #     airflow-init:
  #       condition: service_completed_successfully

  airflow-init:
    <<: *airflow-common
    command: version
    environment:
      <<: *airflow-common-env
      _AIRFLOW_DB_UPGRADE: "true"
      _AIRFLOW_WWW_USER_CREATE: "true"
      _AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME:-airflow}
      _AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD:-airflow}

  airflow-cli:
    <<: *airflow-common
    profiles:
      - debug
    environment:
      <<: *airflow-common-env
      CONNECTION_CHECK_MAX_COUNT: "0"
    # Workaround for entrypoint issue. See: https://github.com/apache/airflow/issues/16252
    command:
      - bash
      - -c
      - airflow

  # flower:
  #   <<: *airflow-common
  #   command: celery flower
  #   ports:
  #     - 5555:5555
  #   healthcheck:
  #     test: ["CMD", "curl", "--fail", "http://localhost:5555/"]
  #     interval: 10s
  #     timeout: 10s
  #     retries: 5
  #   restart: always
  #   depends_on:
  #     <<: *airflow-common-depends-on
  #     airflow-init:
  #       condition: service_completed_successfully

  # URI connection is `psql postgres://username:password@hostname:5432/database_name`
  postgres-source: # this is hostname; connect using `psql -h hostname -U username -d database_name` (can specify switches in CLI or connect with URI string instead)
    image: postgres:latest
    environment:
      POSTGRES_USER: sourcedb1
      POSTGRES_PASSWORD: sourcedb1
      POSTGRES_DB: sourcedb
    logging:
      options:
        max-size: 10m
        max-file: "3"
    ports:
      - "5433:5432"
    volumes:
      - ./source-data:/source-data
      - ./filesystem:/filesystem
      - ./database-setup/sourcedb.sql:/docker-entrypoint-initdb.d/sourcedb.sql
    healthcheck:
      test: ["CMD", "pg_isready", "-U", "sourcedb1"]
      interval: 5s
      retries: 5
    restart: always

  # connect using `psql -h postgres-dest -U destdb1 -d destdb`
  # alternatively `psql postgres://destdb1:destdb1@postgres-dest:5432/destdb`
  postgres-dest:
    image: postgres:latest
    environment:
      POSTGRES_USER: destdb1
      POSTGRES_PASSWORD: destdb1
      POSTGRES_DB: destdb
    logging:
      options:
        max-size: 10m
        max-file: "3"
    ports:
      - "5434:5432"
    volumes:
      - ./dest-data:/dest-data
      - ./filesystem:/filesystem
      - ./database-setup/destinationdb.sql:/docker-entrypoint-initdb.d/destinationdb.sql
    healthcheck:
      test: ["CMD", "pg_isready", "-U", "destdb1"]
      interval: 5s
      retries: 5
    restart: always

  postgres-store:
    image: postgres:latest
    environment:
      POSTGRES_USER: storedb1
      POSTGRES_PASSWORD: storedb1
      POSTGRES_DB: storedb
    logging:
      options:
        max-size: 10m
        max-file: "3"
    ports:
      - "5435:5432"
    volumes:
      - ./great_expectations:/great_expectations
      - ./database-setup/storedb.sql:/docker-entrypoint-initdb.d/storedb.sql
    healthcheck:
      test: ["CMD", "pg_isready", "-U", "storedb1"]
      interval: 5s
      retries: 5
    restart: always