backend: allow configuring S3_REGION

config/s3-dev-blank-region.json

@@ -0,0 +1,17 @@
+{
+  "default": {
+    "server": {
+      "port": 8384
+    },
+    "external": {
+      "s3blobStore": {
+        "region": "",
+        "server": "http://localhost:9000",
+        "accessKey": "odk-central-dev",
+        "secretKey": "topSecret123",
+        "bucketName": "odk-central-bucket",
+        "requestTimeout": 60000
+      }
+    }
+  }
+}

config/s3-dev-with-region.json

@@ -0,0 +1,17 @@
+{
+  "default": {
+    "server": {
+      "port": 8385
+    },
+    "external": {
+      "s3blobStore": {
+        "region": "eu-west-1",
+        "server": "http://localhost:9000",
+        "accessKey": "odk-central-dev",
+        "secretKey": "topSecret123",
+        "bucketName": "odk-central-bucket",
+        "requestTimeout": 60000
+      }
+    }
+  }
+}

lib/bin/s3-create-bucket.js

@@ -9,15 +9,15 @@
 
 const Minio = require('minio');
 
-const { server, bucketName, accessKey, secretKey } = require('config').get('default.external.s3blobStore');
+const { server, region, bucketName, accessKey, secretKey } = require('config').get('default.external.s3blobStore');
 
 const minioClient = (() => {
   const url = new URL(server);
   const useSSL = url.protocol === 'https:';
   const endPoint = (url.hostname + url.pathname).replace(/\/$/, '');
   const port = parseInt(url.port, 10);
 
-  return new Minio.Client({ endPoint, port, useSSL, accessKey, secretKey });
+  return new Minio.Client({ endPoint, port, useSSL, accessKey, secretKey, region });
 })();
 
 const log = (...args) => console.log(__filename, ...args);
@@ -29,10 +29,11 @@ minioClient.bucketExists(bucketName)
       return;
     }
 
-    log('Creating bucket:', bucketName);
-    return minioClient.makeBucket(bucketName)
-      .then(() => log('Bucket created OK.'));
+    log('Creating bucket:', bucketName, 'in', region ?? 'default region');
+    if (region) return minioClient.makeBucket(bucketName, region);
+    else        return minioClient.makeBucket(bucketName); // eslint-disable-line no-multi-spaces
   })
+  .then(() => log('Bucket created OK.'))
   .catch(err => {
     log('ERROR CREATING MINIO BUCKET:', err);
     process.exit(1);

lib/external/s3.js

@@ -12,7 +12,7 @@ const disabled = { enabled: false };
 const init = (config) => {
   if (!config) return disabled;
 
-  const { server, accessKey, secretKey, bucketName, requestTimeout, objectPrefix } = config;
+  const { server, accessKey, secretKey, bucketName, region, requestTimeout, objectPrefix } = config;
   if (!(server && accessKey && secretKey && bucketName)) return disabled;
 
   const http = require('node:http');
@@ -83,7 +83,7 @@ const init = (config) => {
       return req;
     };
 
-    const clientConfig = { endPoint, port, useSSL, accessKey, secretKey, transport: { request } };
+    const clientConfig = { endPoint, port, useSSL, accessKey, secretKey, region, transport: { request } };
 
     return new Minio.Client(clientConfig);
   })();

test/e2e/s3/run-tests.sh

@@ -1,7 +1,6 @@
 #!/bin/bash -eu
 set -o pipefail
 
-serverUrl="http://localhost:8383"
 userEmail="[email protected]"
 userPassword="secret1234"
 
@@ -17,11 +16,6 @@ cleanup() {
 }
 trap cleanup EXIT SIGINT SIGTERM SIGHUP
 
-if curl -s -o /dev/null $serverUrl; then
-  log "!!! Error: server already running at: $serverUrl"
-  exit 1
-fi
-
 make base
 
 if [[ "${CI-}" = '' ]]; then
@@ -52,20 +46,48 @@ EOF
   read -rp ''
 fi
 
-NODE_CONFIG_ENV=s3-dev node lib/bin/s3-create-bucket.js
-NODE_CONFIG_ENV=s3-dev make run &
-serverPid=$!
+run_suite() {
+  suite="$1"
+  configEnv="$2"
 
-log 'Waiting for backend to start...'
-timeout 30 bash -c "while ! curl -s -o /dev/null $serverUrl; do sleep 1; done"
-log 'Backend started!'
+  log "Running suite '$suite' with config '$configEnv'..."
 
-cd test/e2e/s3
-npx mocha test.js
+  case "$suite" in
+    smoke) testOptions=(--fgrep @smoke-test) ;;
+    all)   testOptions=() ;;
+    *) log "Unrecongised test suite: $suite"; exit 1 ;;
+  esac
 
-if ! curl -s -o /dev/null "$serverUrl"; then
-  log '!!! Backend died.'
-  exit 1
-fi
+  NODE_CONFIG_ENV="$configEnv" node lib/bin/s3-create-bucket.js
+
+  serverPort="$(NODE_CONFIG_ENV="$configEnv" node -e 'console.log(require("config").default.server.port)')"
+  serverUrl="http://localhost:$serverPort"
+  if curl -s -o /dev/null $serverUrl; then
+    log "!!! Error: server already running at: $serverUrl"
+    exit 1
+  fi
+
+  NODE_CONFIG_ENV="$configEnv" make run &
+  serverPid=$!
+
+  log 'Waiting for backend to start...'
+  timeout 30 bash -c "while ! curl -s -o /dev/null $serverUrl; do sleep 1; done"
+  log 'Backend started!'
+
+  cd test/e2e/s3
+  NODE_CONFIG_ENV="$configEnv" NODE_CONFIG_DIR=../../../config npx mocha "${testOptions[@]}" test.js
+  cd -
+
+  if ! curl -s -o /dev/null "$serverUrl"; then
+    log '!!! Backend died.'
+    exit 1
+  fi
+
+  log "Suite '$suite' with config '$configEnv' completed OK."
+}
+
+run_suite smoke s3-dev-with-region
+run_suite smoke s3-dev-blank-region
+run_suite all   s3-dev
 
 log "Tests completed OK."

test/e2e/s3/test-forms/0.xml

@@ -0,0 +1,56 @@
+<?xml version="1.0"?>
+<h:html xmlns="http://www.w3.org/2002/xforms" xmlns:h="http://www.w3.org/1999/xhtml" xmlns:ev="http://www.w3.org/2001/xml-events" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:jr="http://openrosa.org/javarosa" xmlns:orx="http://openrosa.org/xforms" xmlns:odk="http://www.opendatakit.org/xforms">
+  <h:head>
+    <h:title>Blob Test 0</h:title>
+    <model odk:xforms-version="1.0.0">
+      <itext>
+        <translation lang="default" default="true()">
+          <text id="image-big-1-bin">
+            <value>Big Bin 1</value>
+            <value form="image">jr://images/big-1.bin</value>
+          </text>
+        </translation>
+      </itext>
+      <instance>
+        <data id="blob_test_1" version="20240506130956">
+          <grid_multi_columns_no_buttons/>
+          <draw_image/>
+          <file/>
+          <meta>
+            <instanceID/>
+          </meta>
+        </data>
+      </instance>
+      <instance id="image">
+        <root>
+          <item>
+            <itextId>image-big-1-bin</itextId>
+            <name>a</name>
+          </item>
+        </root>
+      </instance>
+      <bind nodeset="/data/grid_multi_columns_no_buttons" type="string"/>
+      <bind nodeset="/data/draw_image" type="binary" orx:max-pixels="200"/>
+      <bind nodeset="/data/file" type="binary"/>
+      <bind nodeset="/data/meta/instanceID" type="string" readonly="true()" jr:preload="uid"/>
+    </model>
+  </h:head>
+  <h:body>
+    <select appearance="columns-pack no-buttons" ref="/data/grid_multi_columns_no_buttons">
+      <label>Select multiple with packed columns and no buttons</label>
+      <hint>select_multiple type with columns-pack no-buttons appearance, 4 image choices</hint>
+      <itemset nodeset="instance('image')/root/item">
+        <value ref="name"/>
+        <label ref="jr:itext(itextId)"/>
+      </itemset>
+    </select>
+    <upload mediatype="image/*" appearance="draw" ref="/data/draw_image">
+      <label>Draw</label>
+      <hint>image type with draw appearance</hint>
+    </upload>
+    <upload mediatype="application/*" ref="/data/file">
+      <label>File</label>
+      <hint>file type with no appearance &lt;br/&gt; WARNING: any kind of file could be uploaded including files that contain viruses or other malware. Be sure to take proper precautions when downloading files from server.</hint>
+    </upload>
+  </h:body>
+</h:html>

test/e2e/s3/test.js

@@ -17,12 +17,13 @@ const fs = require('node:fs');
 const { randomBytes } = require('node:crypto');
 const _ = require('lodash');
 const should = require('should');
+const tmp = require('tmp');
 
 const SUITE_NAME = 'test/e2e/s3';
 const log = require('../util/logger')(SUITE_NAME);
 const { apiClient, mimetypeFor, Redirect } = require('../util/api');
 
-const serverUrl = 'http://localhost:8383';
+const serverUrl = `http://localhost:${require('config').default.server.port}`;
 const userEmail = '[email protected]';
 const userPassword = 'secret1234';
 
@@ -65,7 +66,7 @@ describe('s3 support', () => {
       bigFileSizeMb = 100,
     } = opts; // eslint-disable-line object-curly-newline
 
-    attDir = `./test-forms/${testNumber}-attachments`;
+    attDir = opts.attDir || `./test-forms/${testNumber}-attachments`;
 
     // given
     fs.mkdirSync(attDir, { recursive:true });
@@ -85,6 +86,24 @@ describe('s3 support', () => {
     await assertNoneRedirect(actualAttachments);
   }
 
+  it('should shift submission attachment to s3 @smoke-test', async function() {
+    this.timeout(TIMEOUT);
+
+    // given
+    // randomise attDir to ensure re-runs do not use the same generated files
+    await setup(0, { bigFiles: 1, bigFileSizeMb: 0.01, attDir: tmp.dirSync().name });
+    await assertNewStatuses({ pending: 1 });
+
+    // when
+    await cli('upload-pending');
+
+    // then
+    await assertNewStatuses({ uploaded: 1 });
+    // and
+    await assertAllRedirect(actualAttachments);
+    await assertAllDownloadsMatchOriginal(actualAttachments);
+  });
+
   it('should shift submission attachments to s3', async function() {
     this.timeout(TIMEOUT);
 
@@ -403,7 +422,7 @@ function cli(cmd) {
 
   cmd = `exec node lib/bin/s3 ${cmd}`; // eslint-disable-line no-param-reassign
   log.debug('cli()', 'calling:', cmd);
-  const env = { ..._.pick(process.env, 'PATH'), NODE_CONFIG_ENV:'s3-dev' };
+  const env = _.omit(process.env, 'NODE_CONFIG_DIR');
 
   const promise = new Promise((resolve, reject) => {
     const child = exec(cmd, { env, cwd:'../../..' }, (err, stdout, stderr) => {