From c45dbee0d85ad6e63853711253693986847c7c52 Mon Sep 17 00:00:00 2001 From: Thomas Gerber Date: Fri, 19 May 2023 17:45:17 -0700 Subject: [PATCH] Docker Extension 0.1.4 for Airbyte v40 (#297) Updated the docker composes with the latest Airbyte v0.40.9 env vars --- docker-extension/.env | 177 --------------------- docker-extension/Makefile | 2 +- docker-extension/docker-compose_amd64.yaml | 57 +++++-- docker-extension/docker-compose_arm64.yaml | 58 ++++--- 4 files changed, 82 insertions(+), 212 deletions(-) delete mode 100644 docker-extension/.env diff --git a/docker-extension/.env b/docker-extension/.env deleted file mode 100644 index 7fc7d30..0000000 --- a/docker-extension/.env +++ /dev/null @@ -1,177 +0,0 @@ -# This file needs to be both in the root of the repository -# and in kube/base/faros/config -# until there is support for removing load restrictions in remote directories -# see: https://github.com/kubernetes-sigs/kustomize/issues/4052 -# and: https://github.com/faros-ai/faros-community-edition/issues/193 -COMPOSE_PROFILES="airbyte,default,faros-db" - -############################## Faros ########################################## -# The db_host, db_port, db_user and db_pass are identical to those used by Airbyte. -# This is because we use a single db service for the Faros Community Edition deployment and host all required databases -# on the same host. That service is faros-db, which extends the airbyte db service. -# These can be overriden as needed. -# Note that hosts (faros host, faros config host, airbyte host) MUST correspond a service running postgres. -FAROS_CONFIG_DB_HOST=faros-db -FAROS_CONFIG_DB_PORT=5432 -FAROS_CONFIG_DB_USER=docker -FAROS_CONFIG_DB_PASSWORD=docker -FAROS_DB_NAME=faros -FAROS_DB_HOST=faros-db -FAROS_DB_PORT=5432 -FAROS_DB_USER=docker -FAROS_DB_PASSWORD=docker -FAROS_INIT_LOG_LEVEL=info -FAROS_AIRBYTE_FORCE_SETUP=false -FAROS_INIT_IMAGE=farosai/faros-ce-init:latest - -############################## Airbyte ######################################## -VERSION=0.40.9 -# Airbyte Internal Job Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db -DATABASE_USER=docker -DATABASE_PASSWORD=docker -DATABASE_HOST=faros-db -DATABASE_PORT=5432 -DATABASE_DB=airbyte -# translate manually DATABASE_URL=jdbc:postgresql://${DATABASE_HOST}:${DATABASE_PORT/${DATABASE_DB} (do not include the username or password here) -DATABASE_URL=jdbc:postgresql://faros-db:5432/airbyte -# Host must correspond to the service name running webapp -AIRBYTE_URL=http://airbyte-webapp:80 -AIRBYTE_DESTINATION_HASURA_URL=http://localhost:8080 - -############################## Hasura ######################################### -HASURA_DB_NAME=hasura -HASURA_PORT=8080 -HASURA_GRAPHQL_ADMIN_SECRET=admin -HASURA_URL=http://hasura:8080 -HASURA_VERSION=v2.1.1 - -############################## Metabase ####################################### -METABASE_IMAGE="metabase/metabase" -METABASE_VERSION=v0.45.2.1 -METABASE_DB_NAME=metabase -METABASE_PORT=3000 -METABASE_USER=admin@admin.com -METABASE_PASSWORD=admin -METABASE_URL=http://metabase:3000 -METABASE_USE_SSL=false -# The db_host used in Metabase to connect to the Faros database. -METABASE_FAROS_DB_HOST=faros-db - -############################## n8n ############################################ -N8N_DB_NAME=n8n -N8N_PORT=5678 -N8N_VERSION=0.164.1 -N8N_DOCKER_MOUNT=n8n_data - -############################## AirByte default config ######################### -# This file only contains Docker relevant variables. -# -# Variables with defaults have been omitted to avoid duplication of defaults. -# The only exception to the non-default rule are env vars related to scaling. -# -# See https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/java/io/airbyte/config/Configs.java -# for the latest environment variables. -# -# # Contributors - please organise this env file according to the above linked file. - -# Source: https://github.com/airbytehq/airbyte/blob/v0.40.9/.env - -### SHARED ### - -# When using the airbyte-db via default docker image -CONFIG_ROOT=/data -DATA_DOCKER_MOUNT=airbyte_data -DB_DOCKER_MOUNT=airbyte_db - -# Workspace storage for running jobs (logs, etc) -WORKSPACE_ROOT=/tmp/workspace -WORKSPACE_DOCKER_MOUNT=airbyte_workspace - -# Local mount to access local files from filesystem -# todo (cgardens) - when we are mount raw directories instead of named volumes, *_DOCKER_MOUNT must -# be the same as *_ROOT. -# Issue: https://github.com/airbytehq/airbyte/issues/578 -LOCAL_ROOT=/tmp/airbyte_local -LOCAL_DOCKER_MOUNT=/tmp/airbyte_local -# todo (cgardens) - hack to handle behavior change in docker compose. *_PARENT directories MUST -# already exist on the host filesystem and MUST be parents of *_ROOT. -# Issue: https://github.com/airbytehq/airbyte/issues/577 -HACK_LOCAL_ROOT_PARENT=/tmp - - -### DATABASE ### -JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=0.29.15.001 - -# Airbyte Internal Config Database, defaults to Job Database if empty. Explicitly left empty to mute docker compose warnings. -CONFIG_DATABASE_USER= -CONFIG_DATABASE_PASSWORD= -CONFIG_DATABASE_URL= -CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=0.35.15.001 - -### AIRBYTE SERVICES ### -TEMPORAL_HOST=airbyte-temporal:7233 -INTERNAL_API_HOST=airbyte-server:8001 -WEBAPP_URL=http://localhost:8000/ -# Although not present as an env var, required for webapp configuration. -API_URL=/api/v1/ - - -### JOBS ### -# Relevant to scaling. -SYNC_JOB_MAX_ATTEMPTS=1 -SYNC_JOB_MAX_TIMEOUT_DAYS=1 -JOB_MAIN_CONTAINER_CPU_REQUEST= -JOB_MAIN_CONTAINER_CPU_LIMIT= -JOB_MAIN_CONTAINER_MEMORY_REQUEST= -JOB_MAIN_CONTAINER_MEMORY_LIMIT= - -NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT= -NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST= -NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT= -NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST= - -### LOGGING/MONITORING/TRACKING ### -TRACKING_STRATEGY=logging -JOB_ERROR_REPORTING_STRATEGY=logging -# Although not present as an env var, expected by Log4J configuration. -LOG_LEVEL=INFO - - -### APPLICATIONS ### -# Worker # -WORKERS_MICRONAUT_ENVIRONMENTS=control -# Relevant to scaling. -MAX_SYNC_WORKERS=5 -MAX_SPEC_WORKERS=5 -MAX_CHECK_WORKERS=5 -MAX_DISCOVER_WORKERS=5 -# Temporal Activity configuration -ACTIVITY_MAX_ATTEMPT= -ACTIVITY_INITIAL_DELAY_BETWEEN_ATTEMPTS_SECONDS= -ACTIVITY_MAX_DELAY_BETWEEN_ATTEMPTS_SECONDS= -WORKFLOW_FAILURE_RESTART_DELAY_SECONDS= - - -### FEATURE FLAGS ### -AUTO_DISABLE_FAILING_CONNECTIONS=false -EXPOSE_SECRETS_IN_EXPORT=false -FORCE_MIGRATE_SECRET_STORE=false - -### MONITORING FLAGS ### -# Accepted values are datadog and otel (open telemetry) -METRIC_CLIENT= -# Useful only when metric client is set to be otel. Must start with http:// or https://. -OTEL_COLLECTOR_ENDPOINT="http://host.docker.internal:4317" - -USE_STREAM_CAPABLE_STATE=true - -# extra settings to limit warnings during CE setup -RUN_DATABASE_MIGRATION_ON_STARTUP=true -SECRET_PERSISTENCE=NONE -WORKER_ENVIRONMENT=docker -NEW_SCHEDULER= -DEPLOYMENT_MODE= -JOB_ERROR_REPORTING_STRATEGY= -JOB_ERROR_REPORTING_SENTRY_DSN= -LOG_CONNECTOR_MESSAGES= -TEMPORAL_HISTORY_RETENTION_IN_DAYS= diff --git a/docker-extension/Makefile b/docker-extension/Makefile index 244fe91..742cce9 100644 --- a/docker-extension/Makefile +++ b/docker-extension/Makefile @@ -18,7 +18,7 @@ update-extension: build-extension ## Update the extension prepare-buildx: ## Create buildx builder for multi-arch build, if not exists docker buildx inspect $(BUILDER) || docker buildx create --name=$(BUILDER) --driver=docker-container --driver-opt=network=host -push-extension: prepare-buildx ## Build & Upload extension image to hub. Do not push if tag already exists: make push-extension tag=0.1 +push-extension: prepare-buildx ## Build & Upload extension image to hub. Do not push if tag already exists: make push-extension TAG=0.1 docker pull $(IMAGE):$(TAG) && echo "Failure: Tag already exists" || docker buildx build --push --builder=$(BUILDER) --platform=linux/amd64,linux/arm64 --build-arg TAG=$(TAG) --tag=$(IMAGE):$(TAG) . help: ## Show this help diff --git a/docker-extension/docker-compose_amd64.yaml b/docker-extension/docker-compose_amd64.yaml index 44445d1..c91fb67 100644 --- a/docker-extension/docker-compose_amd64.yaml +++ b/docker-extension/docker-compose_amd64.yaml @@ -51,10 +51,10 @@ services: retries: 5 # Airbyte - # Source: https://github.com/airbytehq/airbyte/blob/v0.39.37-alpha/docker-compose.yaml + # Source: https://github.com/airbytehq/airbyte/blob/v0.40.9/docker-compose.yaml #https://github.com/compose-spec/compose-spec/blob/master/spec.md#using-extensions-as-fragments init: - image: airbyte/init:0.39.37-alpha + image: airbyte/init:0.40.9 logging: *default-logging container_name: init restart: "no" @@ -67,15 +67,15 @@ services: volumes: - tmp:/local_parent bootloader: - image: airbyte/bootloader:0.39.37-alpha + image: airbyte/bootloader:0.40.9 logging: *default-logging container_name: airbyte-bootloader restart: "no" depends_on: - faros-db - entrypoint: /bin/sh -c "airbyte-bootloader-0.39.37-alpha/bin/airbyte-bootloader && tail -f /dev/null" + entrypoint: /bin/sh -c "airbyte-bootloader-0.40.9/bin/airbyte-bootloader && tail -f /dev/null" environment: - - AIRBYTE_VERSION=0.39.37-alpha + - AIRBYTE_VERSION=0.40.9 - CONFIG_DATABASE_PASSWORD= - CONFIG_DATABASE_URL= - CONFIG_DATABASE_USER= @@ -85,7 +85,7 @@ services: - LOG_LEVEL=INFO - RUN_DATABASE_MIGRATION_ON_STARTUP=true faros-db: - image: airbyte/db:0.39.37-alpha + image: airbyte/db:0.40.9 logging: *default-logging container_name: airbyte-db restart: unless-stopped @@ -103,14 +103,14 @@ services: ports: - 5432:5432 worker: - image: airbyte/worker:0.39.37-alpha + image: airbyte/worker:0.40.9 logging: *default-logging container_name: airbyte-worker restart: unless-stopped depends_on: - faros-db environment: - - AIRBYTE_VERSION=0.39.37-alpha + - AIRBYTE_VERSION=0.40.9 - AUTO_DISABLE_FAILING_CONNECTIONS=false - CONFIG_DATABASE_PASSWORD= - CONFIG_DATABASE_URL= @@ -121,6 +121,7 @@ services: - DATABASE_URL=jdbc:postgresql://faros-db:5432/airbyte - DATABASE_USER=docker - DEPLOYMENT_MODE= + - INTERNAL_API_HOST=airbyte-server:8001 - JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=0.29.15.001 - JOB_MAIN_CONTAINER_CPU_LIMIT - JOB_MAIN_CONTAINER_CPU_REQUEST @@ -129,10 +130,15 @@ services: - LOCAL_DOCKER_MOUNT=/tmp/airbyte_local - LOCAL_ROOT=/tmp/airbyte_local - LOG_LEVEL=INFO + - LOG_CONNECTOR_MESSAGES= - MAX_CHECK_WORKERS=5 - MAX_DISCOVER_WORKERS=5 - MAX_SPEC_WORKERS=5 - MAX_SYNC_WORKERS=5 + - NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT= + - NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST= + - NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT= + - NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST= - SECRET_PERSISTENCE=NONE - SYNC_JOB_MAX_ATTEMPTS=1 - SYNC_JOB_MAX_TIMEOUT_DAYS=1 @@ -150,13 +156,16 @@ services: - ACTIVITY_INITIAL_DELAY_BETWEEN_ATTEMPTS_SECONDS= - ACTIVITY_MAX_DELAY_BETWEEN_ATTEMPTS_SECONDS= - WORKFLOW_FAILURE_RESTART_DELAY_SECONDS= - - USE_STREAM_CAPABLE_STATE=false + - USE_STREAM_CAPABLE_STATE=true + - MICRONAUT_ENVIRONMENTS=control volumes: - /var/run/docker.sock:/var/run/docker.sock - workspace:/tmp/workspace - tmp:/tmp/airbyte_local + ports: + - 9000:9000 server: - image: airbyte/server:0.39.37-alpha + image: airbyte/server:0.40.9 logging: *default-logging container_name: airbyte-server depends_on: @@ -164,7 +173,7 @@ services: restart: unless-stopped environment: - AIRBYTE_ROLE= - - AIRBYTE_VERSION=0.39.37-alpha + - AIRBYTE_VERSION=0.40.9 - CONFIG_DATABASE_PASSWORD= - CONFIG_DATABASE_URL= - CONFIG_DATABASE_USER= @@ -183,6 +192,8 @@ services: - SECRET_PERSISTENCE=NONE - TEMPORAL_HOST=airbyte-temporal:7233 - TRACKING_STRATEGY=logging + - JOB_ERROR_REPORTING_STRATEGY= + - JOB_ERROR_REPORTING_SENTRY_DSN= - WEBAPP_URL=http://localhost:8000/ - WORKER_ENVIRONMENT=docker - WORKSPACE_ROOT=/tmp/workspace @@ -193,7 +204,7 @@ services: - data:/data - tmp:/tmp/airbyte_local webapp: - image: airbyte/webapp:0.39.37-alpha + image: airbyte/webapp:0.40.9 logging: *default-logging container_name: airbyte-webapp restart: unless-stopped @@ -201,11 +212,9 @@ services: - 8000:80 environment: - AIRBYTE_ROLE= - - AIRBYTE_VERSION=0.39.37-alpha + - AIRBYTE_VERSION=0.40.9 - API_URL=/api/v1/ - - FULLSTORY= - INTERNAL_API_HOST=airbyte-server:8001 - - IS_DEMO= - OPENREPLAY= - PAPERCUPS_STORYTIME=disabled - TRACKING_STRATEGY=logging @@ -215,7 +224,7 @@ services: - worker - airbyte-temporal airbyte-temporal: - image: airbyte/temporal:0.39.37-alpha + image: airbyte/temporal:0.40.9 logging: *default-logging container_name: airbyte-temporal restart: unless-stopped @@ -231,6 +240,22 @@ services: - POSTGRES_USER=docker volumes: - tmp:/etc/temporal/config/dynamicconfig + airbyte-cron: + image: airbyte/cron:0.40.9 + logging: *default-logging + container_name: airbyte-cron + restart: unless-stopped + environment: + - DB=postgresql + - DB_PORT=5432 + - LOG_LEVEL=INFO + - POSTGRES_PWD=docker + - POSTGRES_SEEDS=faros-db + - POSTGRES_USER=docker + - TEMPORAL_HISTORY_RETENTION_IN_DAYS= + - WORKSPACE_ROOT=/tmp/workspace + volumes: + - workspace:/tmp/workspace # Other services hasura: diff --git a/docker-extension/docker-compose_arm64.yaml b/docker-extension/docker-compose_arm64.yaml index 4dfb0c6..260d24a 100644 --- a/docker-extension/docker-compose_arm64.yaml +++ b/docker-extension/docker-compose_arm64.yaml @@ -50,11 +50,8 @@ services: start_period: 10s retries: 5 - # Airbyte - # Source: https://github.com/airbytehq/airbyte/blob/v0.39.37-alpha/docker-compose.yaml - #https://github.com/compose-spec/compose-spec/blob/master/spec.md#using-extensions-as-fragments init: - image: airbyte/init:0.39.37-alpha + image: airbyte/init:0.40.9 logging: *default-logging container_name: init restart: "no" @@ -67,15 +64,15 @@ services: volumes: - tmp:/local_parent bootloader: - image: airbyte/bootloader:0.39.37-alpha + image: airbyte/bootloader:0.40.9 logging: *default-logging container_name: airbyte-bootloader restart: "no" depends_on: - faros-db - entrypoint: /bin/sh -c "airbyte-bootloader-0.39.37-alpha/bin/airbyte-bootloader && tail -f /dev/null" + entrypoint: /bin/sh -c "airbyte-bootloader-0.40.9/bin/airbyte-bootloader && tail -f /dev/null" environment: - - AIRBYTE_VERSION=0.39.37-alpha + - AIRBYTE_VERSION=0.40.9 - CONFIG_DATABASE_PASSWORD= - CONFIG_DATABASE_URL= - CONFIG_DATABASE_USER= @@ -85,7 +82,7 @@ services: - LOG_LEVEL=INFO - RUN_DATABASE_MIGRATION_ON_STARTUP=true faros-db: - image: airbyte/db:0.39.37-alpha + image: airbyte/db:0.40.9 logging: *default-logging container_name: airbyte-db restart: unless-stopped @@ -103,14 +100,14 @@ services: ports: - 5432:5432 worker: - image: airbyte/worker:0.39.37-alpha + image: airbyte/worker:0.40.9 logging: *default-logging container_name: airbyte-worker restart: unless-stopped depends_on: - faros-db environment: - - AIRBYTE_VERSION=0.39.37-alpha + - AIRBYTE_VERSION=0.40.9 - AUTO_DISABLE_FAILING_CONNECTIONS=false - CONFIG_DATABASE_PASSWORD= - CONFIG_DATABASE_URL= @@ -121,6 +118,7 @@ services: - DATABASE_URL=jdbc:postgresql://faros-db:5432/airbyte - DATABASE_USER=docker - DEPLOYMENT_MODE= + - INTERNAL_API_HOST=airbyte-server:8001 - JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=0.29.15.001 - JOB_MAIN_CONTAINER_CPU_LIMIT - JOB_MAIN_CONTAINER_CPU_REQUEST @@ -129,10 +127,15 @@ services: - LOCAL_DOCKER_MOUNT=/tmp/airbyte_local - LOCAL_ROOT=/tmp/airbyte_local - LOG_LEVEL=INFO + - LOG_CONNECTOR_MESSAGES= - MAX_CHECK_WORKERS=5 - MAX_DISCOVER_WORKERS=5 - MAX_SPEC_WORKERS=5 - MAX_SYNC_WORKERS=5 + - NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT= + - NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST= + - NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT= + - NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST= - SECRET_PERSISTENCE=NONE - SYNC_JOB_MAX_ATTEMPTS=1 - SYNC_JOB_MAX_TIMEOUT_DAYS=1 @@ -150,13 +153,16 @@ services: - ACTIVITY_INITIAL_DELAY_BETWEEN_ATTEMPTS_SECONDS= - ACTIVITY_MAX_DELAY_BETWEEN_ATTEMPTS_SECONDS= - WORKFLOW_FAILURE_RESTART_DELAY_SECONDS= - - USE_STREAM_CAPABLE_STATE=false + - USE_STREAM_CAPABLE_STATE=true + - MICRONAUT_ENVIRONMENTS=control volumes: - /var/run/docker.sock:/var/run/docker.sock - workspace:/tmp/workspace - tmp:/tmp/airbyte_local + ports: + - 9000:9000 server: - image: airbyte/server:0.39.37-alpha + image: airbyte/server:0.40.9 logging: *default-logging container_name: airbyte-server depends_on: @@ -164,7 +170,7 @@ services: restart: unless-stopped environment: - AIRBYTE_ROLE= - - AIRBYTE_VERSION=0.39.37-alpha + - AIRBYTE_VERSION=0.40.9 - CONFIG_DATABASE_PASSWORD= - CONFIG_DATABASE_URL= - CONFIG_DATABASE_USER= @@ -183,6 +189,8 @@ services: - SECRET_PERSISTENCE=NONE - TEMPORAL_HOST=airbyte-temporal:7233 - TRACKING_STRATEGY=logging + - JOB_ERROR_REPORTING_STRATEGY= + - JOB_ERROR_REPORTING_SENTRY_DSN= - WEBAPP_URL=http://localhost:8000/ - WORKER_ENVIRONMENT=docker - WORKSPACE_ROOT=/tmp/workspace @@ -193,7 +201,7 @@ services: - data:/data - tmp:/tmp/airbyte_local webapp: - image: airbyte/webapp:0.39.37-alpha + image: airbyte/webapp:0.40.9 logging: *default-logging container_name: airbyte-webapp restart: unless-stopped @@ -201,11 +209,9 @@ services: - 8000:80 environment: - AIRBYTE_ROLE= - - AIRBYTE_VERSION=0.39.37-alpha + - AIRBYTE_VERSION=0.40.9 - API_URL=/api/v1/ - - FULLSTORY= - INTERNAL_API_HOST=airbyte-server:8001 - - IS_DEMO= - OPENREPLAY= - PAPERCUPS_STORYTIME=disabled - TRACKING_STRATEGY=logging @@ -215,7 +221,7 @@ services: - worker - airbyte-temporal airbyte-temporal: - image: airbyte/temporal:0.39.37-alpha + image: airbyte/temporal:0.40.9 logging: *default-logging container_name: airbyte-temporal restart: unless-stopped @@ -231,6 +237,22 @@ services: - POSTGRES_USER=docker volumes: - tmp:/etc/temporal/config/dynamicconfig + airbyte-cron: + image: airbyte/cron:0.40.9 + logging: *default-logging + container_name: airbyte-cron + restart: unless-stopped + environment: + - DB=postgresql + - DB_PORT=5432 + - LOG_LEVEL=INFO + - POSTGRES_PWD=docker + - POSTGRES_SEEDS=faros-db + - POSTGRES_USER=docker + - TEMPORAL_HISTORY_RETENTION_IN_DAYS= + - WORKSPACE_ROOT=/tmp/workspace + volumes: + - workspace:/tmp/workspace # Other services hasura: