Support S3 blob storage (#701)

Author: alxndrsn

.env.template

@@ -39,3 +39,9 @@ HTTPS_PORT=443
 # SENTRY_ORG_SUBDOMAIN=
 # SENTRY_KEY=
 # SENTRY_PROJECT=
+
+# Optional: configure S3-compatible storage for binary files
+# S3_SERVER=
+# S3_ACCESS_KEY=
+# S3_SECRET_KEY=
+# S3_BUCKET_NAME=

docker-compose.yml

@@ -70,6 +70,10 @@ services:
       - SENTRY_ORG_SUBDOMAIN=${SENTRY_ORG_SUBDOMAIN:-o130137}
       - SENTRY_KEY=${SENTRY_KEY:-3cf75f54983e473da6bd07daddf0d2ee}
       - SENTRY_PROJECT=${SENTRY_PROJECT:-1298632}
+      - S3_SERVER=${S3_SERVER:-}
+      - S3_ACCESS_KEY=${S3_ACCESS_KEY:-}
+      - S3_SECRET_KEY=${S3_SECRET_KEY:-}
+      - S3_BUCKET_NAME=${S3_BUCKET_NAME:-}
     command: [ "wait-for-it", "${DB_HOST:-postgres14}:5432", "--", "./start-odk.sh" ]
     restart: always
     logging:

files/service/config.json.template

@@ -44,6 +44,13 @@
         "orgSubdomain": "${SENTRY_ORG_SUBDOMAIN}",
         "key": "${SENTRY_KEY}",
         "project": "${SENTRY_PROJECT}"
+      },
+      "s3blobStore": {
+        "server": "${S3_SERVER}",
+        "accessKey": "${S3_ACCESS_KEY}",
+        "secretKey": "${S3_SECRET_KEY}",
+        "bucketName": "${S3_BUCKET_NAME}",
+        "requestTimeout": 60000
       }
     }
   }

files/service/crontab

@@ -1,3 +1,4 @@
+0 5 * * * root /usr/odk/upload-blobs.sh
 0 4 * * * root /usr/odk/purge-forms.sh
 0 3 * * * root /usr/odk/run-analytics.sh
 0 2 * * * root /usr/odk/process-backlog.sh

files/service/scripts/start-odk.sh

@@ -4,7 +4,7 @@ echo "generating local service configuration.."
 
 ENKETO_API_KEY=$(cat /etc/secrets/enketo-api-key) \
 BASE_URL=$( [ "${HTTPS_PORT}" = 443 ] && echo https://"${DOMAIN}" || echo https://"${DOMAIN}":"${HTTPS_PORT}" ) \
-envsubst '$DOMAIN $BASE_URL $SYSADMIN_EMAIL $ENKETO_API_KEY $DB_HOST $DB_USER $DB_PASSWORD $DB_NAME $DB_SSL $EMAIL_FROM $EMAIL_HOST $EMAIL_PORT $EMAIL_SECURE $EMAIL_IGNORE_TLS $EMAIL_USER $EMAIL_PASSWORD $OIDC_ENABLED $OIDC_ISSUER_URL $OIDC_CLIENT_ID $OIDC_CLIENT_SECRET $SENTRY_ORG_SUBDOMAIN $SENTRY_KEY $SENTRY_PROJECT' \
+envsubst '$DOMAIN $BASE_URL $SYSADMIN_EMAIL $ENKETO_API_KEY $DB_HOST $DB_USER $DB_PASSWORD $DB_NAME $DB_SSL $EMAIL_FROM $EMAIL_HOST $EMAIL_PORT $EMAIL_SECURE $EMAIL_IGNORE_TLS $EMAIL_USER $EMAIL_PASSWORD $OIDC_ENABLED $OIDC_ISSUER_URL $OIDC_CLIENT_ID $OIDC_CLIENT_SECRET $SENTRY_ORG_SUBDOMAIN $SENTRY_KEY $SENTRY_PROJECT $S3_SERVER $S3_ACCESS_KEY $S3_SECRET_KEY $S3_BUCKET_NAME' \
     < /usr/share/odk/config.json.template \
     > /usr/odk/config/local.json
 

files/service/scripts/upload-blobs.sh

@@ -0,0 +1,4 @@
+#!/bin/sh -eu
+
+cd /usr/odk
+/usr/local/bin/node lib/bin/s3.js upload-pending >/proc/1/fd/1 2>/proc/1/fd/2