From d51b8c28f375b510ab28dcbcf3e3c1200994490f Mon Sep 17 00:00:00 2001 From: Ryan Artecona Date: Wed, 13 Mar 2024 17:30:40 -0700 Subject: [PATCH 1/2] Add telemetry support to docker-compose deployments --- docker-compose-with-temporal.yml | 46 +++++++++++++++++++++++++++++++ docker-compose.yml | 47 ++++++++++++++++++++++++++++++++ docker.env.template | 6 ++++ docker_setup | 38 +++++++++++++------------- get-docker-compose.sh | 2 +- grafana-agent-custom.river | 5 ++++ vector-custom.yaml | 5 ++++ 7 files changed, 129 insertions(+), 20 deletions(-) create mode 100644 grafana-agent-custom.river create mode 100644 vector-custom.yaml diff --git a/docker-compose-with-temporal.yml b/docker-compose-with-temporal.yml index 4128be03..fe9830ed 100644 --- a/docker-compose-with-temporal.yml +++ b/docker-compose-with-temporal.yml @@ -10,6 +10,7 @@ services: environment: - DEPLOYMENT_TEMPLATE_TYPE=docker-compose - SERVICE_TYPE=MAIN_BACKEND,DB_CONNECTOR,DB_SSH_CONNECTOR + - RTEL_SERVICE_NAME=main-backend - DBCONNECTOR_POSTGRES_POOL_MAX_SIZE=100 - DBCONNECTOR_QUERY_TIMEOUT_MS=120000 - WORKFLOW_BACKEND_HOST=http://workflows-backend:3000 @@ -48,6 +49,7 @@ services: environment: - DEPLOYMENT_TEMPLATE_TYPE=docker-compose - SERVICE_TYPE=JOBS_RUNNER + - RTEL_SERVICE_NAME=jobs-runner networks: - backend-network depends_on: @@ -69,6 +71,7 @@ services: environment: - DEPLOYMENT_TEMPLATE_TYPE=docker-compose - SERVICE_TYPE=WORKFLOW_TEMPORAL_WORKER + - RTEL_SERVICE_NAME=workflow-worker - DISABLE_DATABASE_MIGRATIONS=true - WORKFLOW_BACKEND_HOST=http://workflows-backend:3000 - WORKFLOW_TEMPORAL_CLUSTER_FRONTEND_HOST=temporal @@ -88,6 +91,7 @@ services: environment: - DEPLOYMENT_TEMPLATE_TYPE=docker-compose - SERVICE_TYPE=WORKFLOW_BACKEND,DB_CONNECTOR,DB_SSH_CONNECTOR + - RTEL_SERVICE_NAME=workflow-backend - WORKFLOW_BACKEND_HOST=http://workflows-backend:3000 - WORKFLOW_TEMPORAL_CLUSTER_FRONTEND_HOST=temporal - WORKFLOW_TEMPORAL_CLUSTER_FRONTEND_PORT=7233 @@ -119,6 +123,7 @@ services: env_file: ./docker.env environment: - DEPLOYMENT_TEMPLATE_TYPE=docker-compose + - RTEL_SERVICE_NAME=code-executor - NODE_OPTIONS=--max_old_space_size=1024 networks: - code-executor-network @@ -130,6 +135,47 @@ services: privileged: true restart: on-failure + telemetry: + image: "telemetry:ra-dev-0" # TODO: change to public image + restart: on-failure + env_file: docker.env + ports: + - '9125:9125/udp' + command: + - bash + - -c + - | + if [[ "$${RTEL_ENABLED:-false}" == "false" ]]; then + echo 1>&2 "RTEL_ENABLED is not true, going to sleep." + sleep inf + else + exec retool-telemetry + fi + environment: + RTEL_DEPLOYMENT_MODE: 'docker-compose' + DEPLOYMENT_TEMPLATE_TYPE: 'docker-compose' + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - /var/lib/docker/:/var/lib/docker:ro + - /:/rootfs:ro + - /sys:/sys:ro + + # Uncomment this to use custom [vector](https://vector.dev) config, i.e. + # to add extra telemetry sinks to your own destination. + - ${PWD}/vector-custom.yaml:/etc/vector-custom/vector-custom.yaml:ro + + # Uncomment this to use custom + # [grafana-agent](https://grafana.com/docs/agent/latest/flow/) river + # config, i.e. to add extra telemetry sources to your own destination. + # - ${PWD}/grafana-agent-custom.river:/etc/grafana-agent-custom/grafana-agent-custom.river:ro + devices: + - /dev/kmsg:/dev/kmsg + cap_add: + - SYS_ADMIN + networks: + - backend-network + - code-executor-network + # Retool's storage database. See these docs to migrate to an externally hosted database: https://docs.retool.com/docs/configuring-retools-storage-database postgres: image: "postgres:11.13" diff --git a/docker-compose.yml b/docker-compose.yml index 347a5894..b6a3cbca 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,6 +6,7 @@ services: dockerfile: Dockerfile env_file: ./docker.env environment: + - RTEL_SERVICE_NAME=main-backend - DEPLOYMENT_TEMPLATE_TYPE=docker-compose - SERVICE_TYPE=MAIN_BACKEND,DB_CONNECTOR_SERVICE,DB_SSH_CONNECTOR_SERVICE - DBCONNECTOR_POSTGRES_POOL_MAX_SIZE=100 @@ -52,6 +53,7 @@ services: environment: - DEPLOYMENT_TEMPLATE_TYPE=docker-compose - SERVICE_TYPE=JOBS_RUNNER + - RTEL_SERVICE_NAME=jobs-runner networks: - backend-network depends_on: @@ -71,6 +73,7 @@ services: environment: - DEPLOYMENT_TEMPLATE_TYPE=docker-compose - SERVICE_TYPE=WORKFLOW_TEMPORAL_WORKER + - RTEL_SERVICE_NAME=workflow-worker - NODE_OPTIONS=--max_old_space_size=1024 - DISABLE_DATABASE_MIGRATIONS=true - WORKFLOW_BACKEND_HOST=http://workflows-backend:3000 @@ -97,6 +100,7 @@ services: environment: - DEPLOYMENT_TEMPLATE_TYPE=docker-compose - SERVICE_TYPE=WORKFLOW_BACKEND,DB_CONNECTOR,DB_SSH_CONNECTOR + - RTEL_SERVICE_NAME=workflow-backend - WORKFLOW_BACKEND_HOST=http://workflows-backend:3000 - CODE_EXECUTOR_INGRESS_DOMAIN=http://code-executor:3004 - DBCONNECTOR_POSTGRES_POOL_MAX_SIZE=100 @@ -120,6 +124,48 @@ services: - ./retool:/usr/local/retool-git-repo - ${BOOTSTRAP_SOURCE:-./retool}:/usr/local/retool-repo + telemetry: + # TODO: update to `latest` when released as stable + image: "tryretool/telemetry:3.39.0-edge" + restart: on-failure + env_file: docker.env + ports: + - '9125:9125/udp' + command: + - bash + - -c + - | + if [[ "$${RTEL_ENABLED:-false}" == "false" ]]; then + echo 1>&2 "RTEL_ENABLED is not true, going to sleep." + sleep inf + else + exec retool-telemetry + fi + environment: + RTEL_DEPLOYMENT_MODE: 'docker-compose' + DEPLOYMENT_TEMPLATE_TYPE: 'docker-compose' + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - /var/lib/docker/:/var/lib/docker:ro + - /:/rootfs:ro + - /sys:/sys:ro + + # Uncomment this to use custom [vector](https://vector.dev) config, i.e. + # to add extra telemetry sinks to your own destination. + - ${PWD}/vector-custom.yaml:/etc/vector-custom/vector-custom.yaml:ro + + # Uncomment this to use custom + # [grafana-agent](https://grafana.com/docs/agent/latest/flow/) river + # config, i.e. to add extra telemetry sources to your own destination. + # - ${PWD}/grafana-agent-custom.river:/etc/grafana-agent-custom/grafana-agent-custom.river:ro + devices: + - /dev/kmsg:/dev/kmsg + cap_add: + - SYS_ADMIN + networks: + - backend-network + - code-executor-network + code-executor: build: context: ./ @@ -129,6 +175,7 @@ services: environment: - DEPLOYMENT_TEMPLATE_TYPE=docker-compose - NODE_OPTIONS=--max_old_space_size=1024 + - RTEL_SERVICE_NAME=code-executor networks: - code-executor-network # code-executor uses nsjail to sandbox code execution. nsjail requires diff --git a/docker.env.template b/docker.env.template index 85bd21ee..1b57454a 100644 --- a/docker.env.template +++ b/docker.env.template @@ -28,3 +28,9 @@ POSTGRES_PASSWORD=randomstring ## If you wish for Retool to be hosted on a server with a public IP address, then you can use these configs to run the nginx container # HOSTNAME=https://retool.company.com # DOMAINS=http://localhost -> http://api:3000 + +## If you wish to enable telemetry collection and forwarding to Retool for help troubleshooting, uncomment these. +# RTEL_ENABLED=true +# RTEL_SEND_TO_RETOOL=true +# STATSD_HOST=telemetry +# STATSD_PORT="9125" diff --git a/docker_setup b/docker_setup index 4a5aab01..9824150d 100755 --- a/docker_setup +++ b/docker_setup @@ -1,5 +1,19 @@ #!/bin/bash +if [ -f ./docker.env ]; then + echo "Found existing docker.env file..." + echo "exiting to avoid overwriting existing the configuration file..." + exit 0 +fi +touch docker.env + +if [ -f ./retooldb.env ]; then + echo "Found existing retooldb.env file..." + echo "exiting to avoid overwriting existing the configuration file..." + exit 0 +fi +touch retooldb.env + postgresPassword=$(cat /dev/urandom | base64 | head -c 64) retooldbPostgresPassword=$(cat /dev/urandom | base64 | head -c 64) jwtSecret=$(cat /dev/urandom | base64 | head -c 256) @@ -21,20 +35,6 @@ if [ -z "$hostname" ]; then hostname=$publicIpAddress fi -if [ -f ./docker.env ]; then - echo "Found existing docker.env file..." - echo "exiting to avoid overwriting existing the configuration file..." - exit 0 -fi -touch docker.env - -if [ -f ./retooldb.env ]; then - echo "Found existing retooldb.env file..." - echo "exiting to avoid overwriting existing the configuration file..." - exit 0 -fi -touch retooldb.env - echo '## For a complete list of all environment variables, see docs.retool.com/docs/environment-variables' >> docker.env echo '' >> docker.env @@ -42,7 +42,7 @@ echo '## Set node environment to production' >> docker.env echo 'NODE_ENV=production' >> docker.env echo '' >> docker.env echo '## Set the JWT secret for the API server' >> docker.env -echo "JWT_SECRET=${jwtSecret}" >> docker.env +echo "JWT_SECRET=\"${jwtSecret}\"" >> docker.env echo '' >> docker.env echo '## Set and generate postgres credentials' >> docker.env @@ -50,7 +50,7 @@ echo 'POSTGRES_DB=hammerhead_production' >> docker.env echo 'POSTGRES_USER=retool_internal_user' >> docker.env echo 'POSTGRES_HOST=postgres' >> docker.env echo 'POSTGRES_PORT=5432' >> docker.env -echo "POSTGRES_PASSWORD=${postgresPassword}" >> docker.env +echo "POSTGRES_PASSWORD=\"${postgresPassword}\"" >> docker.env echo '' >> docker.env echo '## Set and generate retooldb postgres credentials' >> docker.env @@ -58,14 +58,14 @@ echo 'RETOOLDB_POSTGRES_DB=postgres' >> docker.env echo 'RETOOLDB_POSTGRES_USER=root' >> docker.env echo 'RETOOLDB_POSTGRES_HOST=retooldb-postgres' >> docker.env echo 'RETOOLDB_POSTGRES_PORT=5432' >> docker.env -echo "RETOOLDB_POSTGRES_PASSWORD=${retooldbPostgresPassword}" >> docker.env +echo "RETOOLDB_POSTGRES_PASSWORD=\"${retooldbPostgresPassword}\"" >> docker.env echo '' >> docker.env echo '## Set and generate retooldb postgres credentials' >> retooldb.env echo 'POSTGRES_HOST=retooldb-postgres' >> retooldb.env echo 'POSTGRES_DB=postgres' >> retooldb.env echo 'POSTGRES_USER=root' >> retooldb.env -echo "POSTGRES_PASSWORD=${retooldbPostgresPassword}" >> retooldb.env +echo "POSTGRES_PASSWORD=\"${retooldbPostgresPassword}\"" >> retooldb.env echo 'POSTGRES_PORT=5432' >> retooldb.env echo '' >> retooldb.env @@ -80,7 +80,7 @@ echo '' >> docker.env echo '## Set key to encrypt and decrypt database passwords, etc.' >> docker.env echo '## This random string value should be stored privately, and should not be changed over the liftetime of the deployment' >> docker.env -echo "ENCRYPTION_KEY=${encryptionKey}" >> docker.env +echo "ENCRYPTION_KEY=\"${encryptionKey}\"" >> docker.env echo '' >> docker.env echo "## Google SSO configuration" >> docker.env diff --git a/get-docker-compose.sh b/get-docker-compose.sh index b9937121..fbbb2a12 100755 --- a/get-docker-compose.sh +++ b/get-docker-compose.sh @@ -4,5 +4,5 @@ if command -v docker-compose &> /dev/null ; then exit 0 fi -sudo -E curl -L https://github.com/docker/compose/releases/download/1.29.0/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose +sudo -E curl -L https://github.com/docker/compose/releases/download/v2.24.7/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose sudo chmod +x /usr/local/bin/docker-compose diff --git a/grafana-agent-custom.river b/grafana-agent-custom.river new file mode 100644 index 00000000..42972263 --- /dev/null +++ b/grafana-agent-custom.river @@ -0,0 +1,5 @@ +// Use this file for custom +// [grafana-agent](https://grafana.com/docs/agent/latest/flow/) river config, +// i.e. to add extra telemetry sources. +// +// Note: must enable the corresponding telemetry volume mount in your compose file. diff --git a/vector-custom.yaml b/vector-custom.yaml new file mode 100644 index 00000000..e6f1c08c --- /dev/null +++ b/vector-custom.yaml @@ -0,0 +1,5 @@ +# Use this file for custom [vector](https://vector.dev) config, i.e. to add +# extra telemetry sinks to your own destination. +# +# Note: must enable the corresponding telemetry volume mount in your compose +# file. From 4f1994b9f11b684d963840d068675f1389b2c33f Mon Sep 17 00:00:00 2001 From: Ryan Artecona Date: Tue, 2 Apr 2024 17:16:51 -0700 Subject: [PATCH 2/2] Feedback, improvements --- docker-compose-with-temporal.yml | 13 ++----------- docker-compose.yml | 12 +----------- docker_setup | 7 +++++++ 3 files changed, 10 insertions(+), 22 deletions(-) diff --git a/docker-compose-with-temporal.yml b/docker-compose-with-temporal.yml index fe9830ed..6d884e67 100644 --- a/docker-compose-with-temporal.yml +++ b/docker-compose-with-temporal.yml @@ -136,21 +136,12 @@ services: restart: on-failure telemetry: - image: "telemetry:ra-dev-0" # TODO: change to public image + # TODO: update to `latest` when released as stable + image: "tryretool/telemetry:3.40.0-edge" restart: on-failure env_file: docker.env ports: - '9125:9125/udp' - command: - - bash - - -c - - | - if [[ "$${RTEL_ENABLED:-false}" == "false" ]]; then - echo 1>&2 "RTEL_ENABLED is not true, going to sleep." - sleep inf - else - exec retool-telemetry - fi environment: RTEL_DEPLOYMENT_MODE: 'docker-compose' DEPLOYMENT_TEMPLATE_TYPE: 'docker-compose' diff --git a/docker-compose.yml b/docker-compose.yml index b6a3cbca..02e10c9e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -126,21 +126,11 @@ services: telemetry: # TODO: update to `latest` when released as stable - image: "tryretool/telemetry:3.39.0-edge" + image: "tryretool/telemetry:3.40.0-edge" restart: on-failure env_file: docker.env ports: - '9125:9125/udp' - command: - - bash - - -c - - | - if [[ "$${RTEL_ENABLED:-false}" == "false" ]]; then - echo 1>&2 "RTEL_ENABLED is not true, going to sleep." - sleep inf - else - exec retool-telemetry - fi environment: RTEL_DEPLOYMENT_MODE: 'docker-compose' DEPLOYMENT_TEMPLATE_TYPE: 'docker-compose' diff --git a/docker_setup b/docker_setup index 9824150d..83b37f44 100755 --- a/docker_setup +++ b/docker_setup @@ -93,5 +93,12 @@ echo '' >> docker.env echo '## Uncomment this line if HTTPS is not set up' >> docker.env echo '# COOKIE_INSECURE=true' >> docker.env +echo '' >> docker.env + +echo '## If you wish to enable telemetry collection and forwarding to Retool for help troubleshooting, uncomment these.' >> docker.env +echo '# RTEL_ENABLED=true' >> docker.env +echo '# RTEL_SEND_TO_RETOOL=true' >> docker.env +echo '# STATSD_HOST=telemetry' >> docker.env +echo '# STATSD_PORT="9125"' >> docker.env echo "Cool! Now add your license key in docker.env then run docker-compose up to launch Retool."