diff --git a/docker-compose-with-temporal.yml b/docker-compose-with-temporal.yml index 4128be0..fe9830e 100644 --- a/docker-compose-with-temporal.yml +++ b/docker-compose-with-temporal.yml @@ -10,6 +10,7 @@ services: environment: - DEPLOYMENT_TEMPLATE_TYPE=docker-compose - SERVICE_TYPE=MAIN_BACKEND,DB_CONNECTOR,DB_SSH_CONNECTOR + - RTEL_SERVICE_NAME=main-backend - DBCONNECTOR_POSTGRES_POOL_MAX_SIZE=100 - DBCONNECTOR_QUERY_TIMEOUT_MS=120000 - WORKFLOW_BACKEND_HOST=http://workflows-backend:3000 @@ -48,6 +49,7 @@ services: environment: - DEPLOYMENT_TEMPLATE_TYPE=docker-compose - SERVICE_TYPE=JOBS_RUNNER + - RTEL_SERVICE_NAME=jobs-runner networks: - backend-network depends_on: @@ -69,6 +71,7 @@ services: environment: - DEPLOYMENT_TEMPLATE_TYPE=docker-compose - SERVICE_TYPE=WORKFLOW_TEMPORAL_WORKER + - RTEL_SERVICE_NAME=workflow-worker - DISABLE_DATABASE_MIGRATIONS=true - WORKFLOW_BACKEND_HOST=http://workflows-backend:3000 - WORKFLOW_TEMPORAL_CLUSTER_FRONTEND_HOST=temporal @@ -88,6 +91,7 @@ services: environment: - DEPLOYMENT_TEMPLATE_TYPE=docker-compose - SERVICE_TYPE=WORKFLOW_BACKEND,DB_CONNECTOR,DB_SSH_CONNECTOR + - RTEL_SERVICE_NAME=workflow-backend - WORKFLOW_BACKEND_HOST=http://workflows-backend:3000 - WORKFLOW_TEMPORAL_CLUSTER_FRONTEND_HOST=temporal - WORKFLOW_TEMPORAL_CLUSTER_FRONTEND_PORT=7233 @@ -119,6 +123,7 @@ services: env_file: ./docker.env environment: - DEPLOYMENT_TEMPLATE_TYPE=docker-compose + - RTEL_SERVICE_NAME=code-executor - NODE_OPTIONS=--max_old_space_size=1024 networks: - code-executor-network @@ -130,6 +135,47 @@ services: privileged: true restart: on-failure + telemetry: + image: "telemetry:ra-dev-0" # TODO: change to public image + restart: on-failure + env_file: docker.env + ports: + - '9125:9125/udp' + command: + - bash + - -c + - | + if [[ "$${RTEL_ENABLED:-false}" == "false" ]]; then + echo 1>&2 "RTEL_ENABLED is not true, going to sleep." + sleep inf + else + exec retool-telemetry + fi + environment: + RTEL_DEPLOYMENT_MODE: 'docker-compose' + DEPLOYMENT_TEMPLATE_TYPE: 'docker-compose' + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - /var/lib/docker/:/var/lib/docker:ro + - /:/rootfs:ro + - /sys:/sys:ro + + # Uncomment this to use custom [vector](https://vector.dev) config, i.e. + # to add extra telemetry sinks to your own destination. + - ${PWD}/vector-custom.yaml:/etc/vector-custom/vector-custom.yaml:ro + + # Uncomment this to use custom + # [grafana-agent](https://grafana.com/docs/agent/latest/flow/) river + # config, i.e. to add extra telemetry sources to your own destination. + # - ${PWD}/grafana-agent-custom.river:/etc/grafana-agent-custom/grafana-agent-custom.river:ro + devices: + - /dev/kmsg:/dev/kmsg + cap_add: + - SYS_ADMIN + networks: + - backend-network + - code-executor-network + # Retool's storage database. See these docs to migrate to an externally hosted database: https://docs.retool.com/docs/configuring-retools-storage-database postgres: image: "postgres:11.13" diff --git a/docker-compose.yml b/docker-compose.yml index 347a589..138f1f7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,6 +6,7 @@ services: dockerfile: Dockerfile env_file: ./docker.env environment: + - RTEL_SERVICE_NAME=main-backend - DEPLOYMENT_TEMPLATE_TYPE=docker-compose - SERVICE_TYPE=MAIN_BACKEND,DB_CONNECTOR_SERVICE,DB_SSH_CONNECTOR_SERVICE - DBCONNECTOR_POSTGRES_POOL_MAX_SIZE=100 @@ -52,6 +53,7 @@ services: environment: - DEPLOYMENT_TEMPLATE_TYPE=docker-compose - SERVICE_TYPE=JOBS_RUNNER + - RTEL_SERVICE_NAME=jobs-runner networks: - backend-network depends_on: @@ -71,6 +73,7 @@ services: environment: - DEPLOYMENT_TEMPLATE_TYPE=docker-compose - SERVICE_TYPE=WORKFLOW_TEMPORAL_WORKER + - RTEL_SERVICE_NAME=workflow-worker - NODE_OPTIONS=--max_old_space_size=1024 - DISABLE_DATABASE_MIGRATIONS=true - WORKFLOW_BACKEND_HOST=http://workflows-backend:3000 @@ -97,6 +100,7 @@ services: environment: - DEPLOYMENT_TEMPLATE_TYPE=docker-compose - SERVICE_TYPE=WORKFLOW_BACKEND,DB_CONNECTOR,DB_SSH_CONNECTOR + - RTEL_SERVICE_NAME=workflow-backend - WORKFLOW_BACKEND_HOST=http://workflows-backend:3000 - CODE_EXECUTOR_INGRESS_DOMAIN=http://code-executor:3004 - DBCONNECTOR_POSTGRES_POOL_MAX_SIZE=100 @@ -120,6 +124,47 @@ services: - ./retool:/usr/local/retool-git-repo - ${BOOTSTRAP_SOURCE:-./retool}:/usr/local/retool-repo + telemetry: + image: "telemetry:ra-dev-0" # TODO: update to public image + restart: on-failure + env_file: docker.env + ports: + - '9125:9125/udp' + command: + - bash + - -c + - | + if [[ "$${RTEL_ENABLED:-false}" == "false" ]]; then + echo 1>&2 "RTEL_ENABLED is not true, going to sleep." + sleep inf + else + exec retool-telemetry + fi + environment: + RTEL_DEPLOYMENT_MODE: 'docker-compose' + DEPLOYMENT_TEMPLATE_TYPE: 'docker-compose' + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - /var/lib/docker/:/var/lib/docker:ro + - /:/rootfs:ro + - /sys:/sys:ro + + # Uncomment this to use custom [vector](https://vector.dev) config, i.e. + # to add extra telemetry sinks to your own destination. + - ${PWD}/vector-custom.yaml:/etc/vector-custom/vector-custom.yaml:ro + + # Uncomment this to use custom + # [grafana-agent](https://grafana.com/docs/agent/latest/flow/) river + # config, i.e. to add extra telemetry sources to your own destination. + # - ${PWD}/grafana-agent-custom.river:/etc/grafana-agent-custom/grafana-agent-custom.river:ro + devices: + - /dev/kmsg:/dev/kmsg + cap_add: + - SYS_ADMIN + networks: + - backend-network + - code-executor-network + code-executor: build: context: ./ @@ -129,6 +174,7 @@ services: environment: - DEPLOYMENT_TEMPLATE_TYPE=docker-compose - NODE_OPTIONS=--max_old_space_size=1024 + - RTEL_SERVICE_NAME=code-executor networks: - code-executor-network # code-executor uses nsjail to sandbox code execution. nsjail requires diff --git a/docker.env.template b/docker.env.template index 85bd21e..1b57454 100644 --- a/docker.env.template +++ b/docker.env.template @@ -28,3 +28,9 @@ POSTGRES_PASSWORD=randomstring ## If you wish for Retool to be hosted on a server with a public IP address, then you can use these configs to run the nginx container # HOSTNAME=https://retool.company.com # DOMAINS=http://localhost -> http://api:3000 + +## If you wish to enable telemetry collection and forwarding to Retool for help troubleshooting, uncomment these. +# RTEL_ENABLED=true +# RTEL_SEND_TO_RETOOL=true +# STATSD_HOST=telemetry +# STATSD_PORT="9125" diff --git a/docker_setup b/docker_setup index 4a5aab0..9824150 100755 --- a/docker_setup +++ b/docker_setup @@ -1,5 +1,19 @@ #!/bin/bash +if [ -f ./docker.env ]; then + echo "Found existing docker.env file..." + echo "exiting to avoid overwriting existing the configuration file..." + exit 0 +fi +touch docker.env + +if [ -f ./retooldb.env ]; then + echo "Found existing retooldb.env file..." + echo "exiting to avoid overwriting existing the configuration file..." + exit 0 +fi +touch retooldb.env + postgresPassword=$(cat /dev/urandom | base64 | head -c 64) retooldbPostgresPassword=$(cat /dev/urandom | base64 | head -c 64) jwtSecret=$(cat /dev/urandom | base64 | head -c 256) @@ -21,20 +35,6 @@ if [ -z "$hostname" ]; then hostname=$publicIpAddress fi -if [ -f ./docker.env ]; then - echo "Found existing docker.env file..." - echo "exiting to avoid overwriting existing the configuration file..." - exit 0 -fi -touch docker.env - -if [ -f ./retooldb.env ]; then - echo "Found existing retooldb.env file..." - echo "exiting to avoid overwriting existing the configuration file..." - exit 0 -fi -touch retooldb.env - echo '## For a complete list of all environment variables, see docs.retool.com/docs/environment-variables' >> docker.env echo '' >> docker.env @@ -42,7 +42,7 @@ echo '## Set node environment to production' >> docker.env echo 'NODE_ENV=production' >> docker.env echo '' >> docker.env echo '## Set the JWT secret for the API server' >> docker.env -echo "JWT_SECRET=${jwtSecret}" >> docker.env +echo "JWT_SECRET=\"${jwtSecret}\"" >> docker.env echo '' >> docker.env echo '## Set and generate postgres credentials' >> docker.env @@ -50,7 +50,7 @@ echo 'POSTGRES_DB=hammerhead_production' >> docker.env echo 'POSTGRES_USER=retool_internal_user' >> docker.env echo 'POSTGRES_HOST=postgres' >> docker.env echo 'POSTGRES_PORT=5432' >> docker.env -echo "POSTGRES_PASSWORD=${postgresPassword}" >> docker.env +echo "POSTGRES_PASSWORD=\"${postgresPassword}\"" >> docker.env echo '' >> docker.env echo '## Set and generate retooldb postgres credentials' >> docker.env @@ -58,14 +58,14 @@ echo 'RETOOLDB_POSTGRES_DB=postgres' >> docker.env echo 'RETOOLDB_POSTGRES_USER=root' >> docker.env echo 'RETOOLDB_POSTGRES_HOST=retooldb-postgres' >> docker.env echo 'RETOOLDB_POSTGRES_PORT=5432' >> docker.env -echo "RETOOLDB_POSTGRES_PASSWORD=${retooldbPostgresPassword}" >> docker.env +echo "RETOOLDB_POSTGRES_PASSWORD=\"${retooldbPostgresPassword}\"" >> docker.env echo '' >> docker.env echo '## Set and generate retooldb postgres credentials' >> retooldb.env echo 'POSTGRES_HOST=retooldb-postgres' >> retooldb.env echo 'POSTGRES_DB=postgres' >> retooldb.env echo 'POSTGRES_USER=root' >> retooldb.env -echo "POSTGRES_PASSWORD=${retooldbPostgresPassword}" >> retooldb.env +echo "POSTGRES_PASSWORD=\"${retooldbPostgresPassword}\"" >> retooldb.env echo 'POSTGRES_PORT=5432' >> retooldb.env echo '' >> retooldb.env @@ -80,7 +80,7 @@ echo '' >> docker.env echo '## Set key to encrypt and decrypt database passwords, etc.' >> docker.env echo '## This random string value should be stored privately, and should not be changed over the liftetime of the deployment' >> docker.env -echo "ENCRYPTION_KEY=${encryptionKey}" >> docker.env +echo "ENCRYPTION_KEY=\"${encryptionKey}\"" >> docker.env echo '' >> docker.env echo "## Google SSO configuration" >> docker.env diff --git a/get-docker-compose.sh b/get-docker-compose.sh index b993712..fbbb2a1 100755 --- a/get-docker-compose.sh +++ b/get-docker-compose.sh @@ -4,5 +4,5 @@ if command -v docker-compose &> /dev/null ; then exit 0 fi -sudo -E curl -L https://github.com/docker/compose/releases/download/1.29.0/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose +sudo -E curl -L https://github.com/docker/compose/releases/download/v2.24.7/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose sudo chmod +x /usr/local/bin/docker-compose diff --git a/grafana-agent-custom.river b/grafana-agent-custom.river new file mode 100644 index 0000000..4297226 --- /dev/null +++ b/grafana-agent-custom.river @@ -0,0 +1,5 @@ +// Use this file for custom +// [grafana-agent](https://grafana.com/docs/agent/latest/flow/) river config, +// i.e. to add extra telemetry sources. +// +// Note: must enable the corresponding telemetry volume mount in your compose file. diff --git a/vector-custom.yaml b/vector-custom.yaml new file mode 100644 index 0000000..e6f1c08 --- /dev/null +++ b/vector-custom.yaml @@ -0,0 +1,5 @@ +# Use this file for custom [vector](https://vector.dev) config, i.e. to add +# extra telemetry sinks to your own destination. +# +# Note: must enable the corresponding telemetry volume mount in your compose +# file.