fix(http): process file uploads correctly (#3232)

examples/file-uploads/README.md → examples/http-file-uploads/README.md

@@ -1,9 +1,7 @@
-# HTTP file uploads with Artillery Pro
+# HTTP file uploads
 
 This example shows you how to perform HTTP file uploads from an Artillery test script.
 
-The file upload functionality is a part of Artillery Pro, which [needs to be installed](https://artillery.io/docs/guides/getting-started/installing-artillery-pro.html) before running the tests in this directory.
-
 ## Running the HTTP server
 
 This example includes an Express.js application running an HTTP server.

examples/file-uploads/file-uploads.yml → examples/http-file-uploads/file-uploads.yml

@@ -3,9 +3,7 @@ config:
   phases:
     - duration: 10min
       arrivalRate: 25
-  # Enables the file upload plugin from Artillery Pro.
-  plugins:
-    http-file-uploads: {}
+
   # To randomize the files to upload during the test scenario,
   # set up variables with the names of the files to use. These
   # files are placed in the `/files` directory.

packages/artillery/lib/cmds/run.js

@@ -498,10 +498,12 @@ async function prepareTestExecutionPlan(inputFiles, flags, args) {
 
   const script3 = await addOverrides(script2, flags);
   const script4 = await addVariables(script3, flags);
+  // The resolveConfigTemplates function expects the config and script path to be passed explicitly because it is used in Fargate as well where the two arguments will not be available on the script
   const script5 = await resolveConfigTemplates(
     script4,
     flags,
-    script4._configPath
+    script4._configPath,
+    script4._scriptPath
   );
 
   if (!script5.config.target) {

packages/artillery/lib/platform/aws-ecs/legacy/bom.js

@@ -23,6 +23,7 @@ function _convertToPosixPath(p) {
   return p.split(path.sep).join(path.posix.sep);
 }
 
+// NOTE: absoluteScriptPath here is actually the absolute path to the config file
 function createBOM(absoluteScriptPath, extraFiles, opts, callback) {
   A.waterfall(
     [
@@ -34,7 +35,8 @@ function createBOM(absoluteScriptPath, extraFiles, opts, callback) {
           opts: {
             scriptData,
             absoluteScriptPath,
-            flags: opts.flags
+            flags: opts.flags,
+            scenarioPath: opts.scenarioPath // Absolute path to the file that holds scenarios
           },
           localFilePaths: [absoluteScriptPath],
           npmModules: []
@@ -157,7 +159,8 @@ function applyScriptChanges(context, next) {
   resolveConfigTemplates(
     context.opts.scriptData,
     context.opts.flags,
-    context.opts.absoluteScriptPath
+    context.opts.absoluteScriptPath,
+    context.opts.scenarioPath
   ).then((resolvedConfig) => {
     context.opts.scriptData = resolvedConfig;
     return next(null, context);

packages/artillery/lib/platform/aws-ecs/legacy/create-test.js

@@ -83,7 +83,11 @@ function prepareManifest(context, callback) {
   createBOM(
     fileToAnalyse,
     extraFiles,
-    { packageJsonPath: context.packageJsonPath, flags: context.flags},
+    {
+      packageJsonPath: context.packageJsonPath,
+      flags: context.flags,
+      scenarioPath: context.scriptPath
+    },
     (err, bom) => {
       debug(err);
       debug(bom);

packages/artillery/lib/platform/aws-lambda/dependencies.js

@@ -13,6 +13,7 @@ const _createLambdaBom = async (
   let createBomOpts = {};
   let entryPoint = absoluteScriptPath;
   let extraFiles = [];
+  createBomOpts.scenarioPath = absoluteScriptPath;
   if (absoluteConfigPath) {
     entryPoint = absoluteConfigPath;
     extraFiles.push(absoluteScriptPath);

packages/artillery/lib/util.js

@@ -98,11 +98,12 @@ function addDefaultPlugins(script) {
   return finalScript;
 }
 
-async function resolveConfigTemplates(script, flags, configPath) {
+async function resolveConfigTemplates(script, flags, configPath, scriptPath) {
   const cliVariables = flags.variables ? JSON.parse(flags.variables) : {};
 
   script.config = engineUtil.template(script.config, {
     vars: {
+      $scenarioFile: scriptPath,
       $dirname: path.dirname(configPath),
       $testId: global.artillery.testRunId,
       $processEnvironment: process.env,
@@ -198,7 +199,7 @@ async function checkConfig(script, scriptPath, flags) {
     );
     payloadSpec.path = resolvedPathToPayload;
   });
-
+  script._scriptPath = absoluteScriptPath;
   return script;
 }
 

packages/artillery/package.json

@@ -44,7 +44,7 @@
   },
   "scripts": {
     "test:unit": "tap --timeout=420 test/unit/*.test.js",
-    "test:acceptance": "tap --timeout=420 test/cli/*.test.js && bash test/lib/run.sh && tap --timeout=420 test/publish-metrics/**/*.test.js",
+    "test:acceptance": "tap --timeout=420 test/cli/*.test.js && bash test/lib/run.sh && tap --timeout=420 test/publish-metrics/**/*.test.js && tap --timeout=420 test/integration/**/*.test.js",
     "test": " npm run test:unit && npm run test:acceptance",
     "test:windows": "npm run test:unit && tap --timeout=420 test/cli/*.test.js",
     "test:aws": "tap --timeout=4200 test/cloud-e2e/**/*.test.js",

packages/artillery/test/integration/core/fixtures/http-file-upload-processor.js

@@ -0,0 +1,11 @@
+'use strict';
+
+function getResponse(req, res, context, ee, next) {
+  // We log the response body here so we can access it from the output
+  console.log('RESPONSE BODY: ', res.body, ' RESPONSE BODY END');
+  next();
+}
+
+module.exports = {
+  getResponse
+};

packages/artillery/test/integration/core/fixtures/http-file-upload.yml

@@ -0,0 +1,21 @@
+config:
+  phases:
+    - duration: 1
+      arrivalRate: 1
+  processor: "../fixtures/http-file-upload-processor.js"
+
+  variables:
+    filename:
+      - "artillery-installation.pdf"
+scenarios:
+  - name: "Hello"
+    flow:
+    - post:
+        url: "/upload"
+        afterResponse: "getResponse"
+        formData:
+          name: "Artillery"
+          logo:
+            fromFile: "./files/artillery-logo.jpg"
+          guide:
+            fromFile: "./files/{{ filename }}"

packages/artillery/test/integration/core/http-file-upload.test.js

@@ -0,0 +1,126 @@
+'use strict';
+
+const { createTestServer } = require('../../targets/http-file-upload-server');
+const { test, beforeEach, afterEach } = require('tap');
+const fs = require('fs');
+const crypto = require('crypto');
+const { $ } = require('zx');
+
+let server;
+let port;
+
+beforeEach(async () => {
+  server = await createTestServer();
+  port = server.info.port;
+});
+
+afterEach((t) => {
+  server.stop();
+});
+
+async function calculateFileHash(filePath) {
+  return new Promise((resolve, reject) => {
+    const hash = crypto.createHash('sha256');
+    const stream = fs.createReadStream(filePath);
+
+    stream.on('data', (data) => hash.update(data));
+    stream.on('end', () => resolve(hash.digest('hex')));
+    stream.on('error', reject);
+  });
+}
+
+test('HTTP engine successfully handles file uploads', async (t) => {
+  const expectedFiles = [
+    {
+      fieldName: 'guide',
+      fileName: 'artillery-installation.pdf',
+      contentType: 'application/pdf'
+    },
+    {
+      fieldName: 'logo',
+      fileName: 'artillery-logo.jpg',
+      contentType: 'image/jpeg'
+    }
+  ];
+
+  const expectedOtherFields = {
+    name: 'Artillery'
+  };
+
+  const override = {
+    config: {
+      target: `http://127.0.0.1:${port}`
+    }
+  };
+
+  /// Run the test
+  let output;
+  try {
+    output =
+      await $`artillery run ${__dirname}/fixtures/http-file-upload.yml --overrides ${JSON.stringify(
+        override
+      )}`;
+  } catch (err) {
+    console.error('There has been an error in test run execution: ', err);
+    t.fail(err);
+  }
+  // We log the response body from the processor so we can parse it from output
+  const match = output.stdout.match(/RESPONSE BODY: (.*) RESPONSE BODY END/s);
+  let data;
+  if (match) {
+    try {
+      data = JSON.parse(match[1].trim());
+    } catch (err) {
+      console.error('Error parsing response body: ', err);
+    }
+  } else {
+    console.error('Response body not found in output');
+  }
+
+  const files = data?.files;
+  const fields = data?.fields;
+  t.ok(
+    data?.files && data?.fields,
+    'Should successfully upload a combination of file and non-file form fields'
+  );
+  t.equal(data.status, 'success', 'Should have a success status');
+  t.equal(
+    files.length,
+    expectedFiles.length,
+    `${expectedFiles.length} files should be uploaded`
+  );
+  t.match(fields, expectedOtherFields, 'Should have the expected other fields');
+
+  for (const expectedFile of expectedFiles) {
+    const uploadedFile = files.find(
+      (f) => f.fieldName === expectedFile.fieldName
+    );
+
+    if (!uploadedFile) {
+      t.fail(
+        `Could not find uploaded file with fieldName ${expectedFile.fieldName}`
+      );
+      continue;
+    }
+
+    const expectedHash = await calculateFileHash(
+      `${__dirname}/fixtures/files/${expectedFile.fileName}`
+    );
+
+    t.equal(
+      uploadedFile.originalFilename,
+      expectedFile.fileName,
+      `Should have uploaded the ${expectedFile.fileName} file under the correct field`
+    );
+    t.equal(
+      uploadedFile.fileHash,
+      expectedHash,
+      'Uploaded file should match the sent file'
+    );
+    t.equal(
+      uploadedFile.headers['content-type'],
+      expectedFile.contentType,
+      'Should have uploaded file with correct content type'
+    );
+  }
+});

packages/artillery/test/targets/http-file-upload-server.js

@@ -0,0 +1,85 @@
+const Hapi = require('@hapi/hapi');
+const path = require('path');
+const crypto = require('crypto');
+
+const createTestServer = async (port) => {
+  const server = Hapi.server({
+    port: port || 0,
+    host: '127.0.0.1'
+  });
+
+  server.route({
+    method: 'GET',
+    path: '/',
+    handler: (request, h) => {
+      return {
+        status: 'success',
+        message: 'Hello!'
+      };
+    }
+  });
+
+  server.route({
+    method: 'POST',
+    path: '/upload',
+    options: {
+      payload: {
+        maxBytes: 10485760, // 10 MB
+        output: 'stream',
+        parse: true,
+        multipart: {
+          output: 'stream'
+        }
+      }
+    },
+    handler: async (request, h) => {
+      const data = request.payload;
+      const files = [];
+      const fields = {};
+
+      for (const key in data) {
+        if (!data[key].hapi || !data[key]._data) {
+          // Handle non-file fields
+          fields[key] = data[key];
+          continue;
+        }
+
+        // Handle file fields
+        const file = data[key];
+        const filename = path.basename(file.hapi.filename);
+
+        // calculate a hash of the file so it can be compared in tests
+        const hash = crypto.createHash('sha256');
+        await new Promise((resolve, reject) => {
+          file.on('end', () => resolve());
+          file.on('error', (err) => reject(err));
+          file.on('data', (chunk) => {
+            hash.update(chunk);
+          });
+        });
+
+        files.push({
+          fieldName: key,
+          originalFilename: filename,
+          fileHash: hash.digest('hex'),
+          headers: file.hapi.headers
+        });
+      }
+
+      return {
+        status: 'success',
+        message: 'Files and fields uploaded successfully',
+        files,
+        fields
+      };
+    }
+  });
+
+  await server.start();
+  console.log(`File upload server listening on ${server.info.uri}`);
+  return server;
+};
+
+module.exports = {
+  createTestServer
+};