Feature: Added a new default reporter to publish newman run to postman

lib/reporters/postman/helpers/constants.js

@@ -0,0 +1,65 @@
+/**
+ * An exhaustive set of constants used across various functions
+ */
+module.exports = {
+    /**
+     * Used as a source in the collection run object
+     */
+    NEWMAN_STRING: 'newman',
+
+    /**
+     * The status of the newman run in process
+     */
+    NEWMAN_RUN_STATUS_FINISHED: 'finished',
+
+    /**
+     * The success result of a particular test
+     */
+    NEWMAN_TEST_STATUS_PASS: 'pass',
+
+    /**
+     * The failure result of a particular test
+     */
+    NEWMAN_TEST_STATUS_FAIL: 'fail',
+
+    /**
+     * The skipped status of a particular test
+     */
+    NEWMAN_TEST_STATUS_SKIPPED: 'skipped',
+
+    /**
+     * Use this as a fallback collection name when creating collection run object
+     */
+    FALLBACK_COLLECTION_RUN_NAME: 'Collection Run',
+
+    /**
+     * The base URL for postman API
+     */
+    POSTMAN_API_BASE_URL: 'https://api.getpostman.com',
+
+    /**
+     * The API path used to upload newman run data
+     */
+    POSTMAN_API_UPLOAD_PATH: '/newman-runs',
+
+    /**
+     * Used as a fall back error message for the upload API call
+     */
+    RESPONSE_FALLBACK_ERROR_MESSAGE: 'Something went wrong while uploading newman run data to Postman',
+
+    /**
+     * Regex pattern to extract the collection id from the postman api collection url
+     */
+    COLLECTION_UID_FROM_URL_EXTRACTION_PATTERN: /https?:\/\/api\.getpostman.*\.com\/(?:collections)\/([A-Za-z0-9-]+)/,
+
+    /**
+     * Regex pattern to extract the environment id from the postman api environment url
+     */
+    ENVIRONMENT_UID_FROM_URL_EXTRACTION_PATTERN: /https?:\/\/api\.getpostman.*\.com\/(?:environments)\/([A-Za-z0-9-]+)/,
+
+    /**
+     * Regex pattern to extract the api key from the postman api collection url
+     */
+    API_KEY_FROM_URL_EXTRACTION_PATTERN:
+        /https:\/\/api.getpostman.com\/([a-z]+)s\/([a-z0-9-]+)\?apikey=([a-z0-9A-Z-]+)/
+};

lib/reporters/postman/helpers/run-utils.js

@@ -0,0 +1,249 @@
+const _ = require('lodash'),
+    uuid = require('uuid'),
+    {
+        NEWMAN_STRING,
+        FALLBACK_COLLECTION_RUN_NAME,
+        NEWMAN_RUN_STATUS_FINISHED,
+        NEWMAN_TEST_STATUS_PASS,
+        NEWMAN_TEST_STATUS_FAIL,
+        NEWMAN_TEST_STATUS_SKIPPED
+    } = require('./constants');
+
+/**
+ * Returns a request object that contains url, method, headers and body data
+ *
+ * Example request object: {
+ *  url: 'https://postman-echo.com/get?user=abc&pass=123,
+ *  method: 'get',
+ *  headers: {
+ *    'Authorization': 'Basic as1ews',
+ *    'Accept': 'application/json'
+ *  },
+ *  body: {
+ *    mode: 'raw',
+ *    raw: 'this is a raw body'
+ *  }
+ * }
+ *
+ * @private
+ * @param {Object} request - a postman-collection SDK's request object
+ * @returns {Object}
+ */
+function _buildRequestObject (request) {
+    if (!request) {
+        return {};
+    }
+
+    return {
+        url: _.invoke(request, 'url.toString', ''),
+        method: _.get(request, 'method', ''),
+        headers: request.getHeaders({ enabled: false }), // only get the headers that were actually sent in the request
+        body: _.get(_.invoke(request, 'toJSON'), 'body')
+    };
+}
+
+/**
+ * Returns a response object that contains response name, code, time, size, headers and body
+ *
+ * Example Response object: {
+ *  code: 200
+ *  name: 'OK'
+ *  time: 213
+ *  size: 43534
+ *  headers: [{key: 'content-type', value: 'application/json'}, {key: 'Connection', value: 'keep-alive'}].
+ *  body: 'who's thereee!'
+ * }
+ *
+ * @private
+ * @param {Object} response - a postman-collection SDK's response object
+ * @returns {Object}
+ */
+function _buildResponseObject (response) {
+    if (!response) {
+        return {};
+    }
+
+    const headersArray = _.get(response, 'headers.members', []),
+        headers = _.map(headersArray, (header) => {
+            return _.pick(header, ['key', 'value']);
+        });
+
+    return {
+        code: response.code,
+        name: response.status,
+        time: response.responseTime,
+        size: response.responseSize,
+        headers: headers,
+        body: response.stream ? new TextDecoder('utf-8').decode(new Uint8Array(response.stream)) : null
+    };
+}
+
+/**
+ * Returns an array of assertions, with each assertion containing name, error and status (pass/fail)
+ * Example assertions array: [
+ *  {
+ *    name: 'Status code should be 200',
+ *    error: null,
+ *    status: 'pass'
+ *  },
+ *  {
+ *    name: 'Status code should be 404',
+ *    error: 'AssertionError: expected response to have status code 404 but got 200',
+ *    status: 'fail'
+ *  }
+ * ]
+ *
+ * @private
+ * @param {Array} assertions - A list of all the assertions performed during the newman run
+ * @returns {Array}
+ */
+function _buildTestObject (assertions) {
+    const tests = [];
+
+    assertions && assertions.forEach((assert) => {
+        let status;
+
+        if (assert.skipped) {
+            status = NEWMAN_TEST_STATUS_SKIPPED;
+        }
+        else if (assert.error) {
+            status = NEWMAN_TEST_STATUS_FAIL;
+        }
+        else {
+            status = NEWMAN_TEST_STATUS_PASS;
+        }
+
+        tests.push({
+            name: assert.assertion,
+            error: assert.error ? _.pick(assert.error, ['name', 'message', 'stack']) : null,
+            status: status
+        });
+    });
+
+    return tests;
+}
+
+/**
+ * Calculates the number of skipped tests for the run
+ *
+ * @private
+ * @param {Object} runSummary - newman run summary data
+ * @returns {Number}
+ */
+function _extractSkippedTestCountFromRun (runSummary) {
+    let skippedTestCount = 0;
+
+    _.forEach(_.get(runSummary, 'run.executions', []), (execution) => {
+        _.forEach(_.get(execution, 'assertions', []), (assertion) => {
+            if (_.get(assertion, 'skipped')) {
+                skippedTestCount++;
+            }
+        });
+    });
+
+    return skippedTestCount;
+}
+
+/**
+ * Converts a newman execution array to an iterations array.
+ * An execution is a flat array, which contains the requests run in order over multiple iterations.
+ * This function converts this flat array into an array of arrays with a single element representing a single iteration.
+ * Hence each iteration is an array, which contains all the requests that were run in that particular iteration
+ * A request object contains request data, response data, the test assertion results, etc.
+ *
+ * Example element of a execution array
+ * {
+ *  cursor: {} // details about the pagination
+ *  item: {} // current request meta data
+ *  request: {} // the request data like url, method, headers, etc.
+ *  response: {} // the response data received for this request
+ *  assertions: [] // an array of all the test results
+ * }
+ *
+ * @private
+ * @param {Array} executions - An array of newman run executions data
+ * @param {Number} iterationCount - The number of iterations newman ran for
+ * @returns {Array}
+ */
+function _executionToIterationConverter (executions, iterationCount) {
+    const iterations = [],
+        validIterationCount = _.isSafeInteger(iterationCount) && iterationCount > 0;
+
+    if (!validIterationCount) {
+        executions = [executions]; //  Assuming only one iteration of the newman run was performed
+    }
+    else {
+    // Note: The second parameter of _.chunk is the size of each chunk and not the number of chunks.
+    // The number of chunks is equal to the number of iterations, hence the below calculation.
+        executions = _.chunk(executions, (executions.length / iterationCount)); // Group the requests iterations wise
+    }
+
+    _.forEach(executions, (iter) => {
+        const iteration = [];
+
+        // eslint-disable-next-line lodash/prefer-map
+        _.forEach(iter, (req) => {
+            iteration.push({
+                id: req.item.id,
+                name: req.item.name || '',
+                request: _buildRequestObject(req.request),
+                response: _buildResponseObject(req.response),
+                error: req.requestError || null,
+                tests: _buildTestObject(req.assertions)
+            });
+        });
+
+        iterations.push(iteration);
+    });
+
+    return iterations;
+}
+
+/**
+ * Converts a newman run summary object to a collection run object.
+ *
+ * @param {Object} collectionRunOptions - newman run options
+ * @param {Object} runSummary - newman run summary data
+ * @returns {Object}
+ */
+function buildCollectionRunObject (collectionRunOptions, runSummary) {
+    if (!collectionRunOptions || !runSummary) {
+        throw new Error('Cannot build Collection run object without collectionRunOptions or runSummary');
+    }
+
+    let failedTestCount = _.get(runSummary, 'run.stats.assertions.failed', 0),
+        skippedTestCount = _extractSkippedTestCountFromRun(runSummary),
+        totalTestCount = _.get(runSummary, 'run.stats.assertions.total', 0),
+        executions = _.get(runSummary, 'run.executions'),
+        iterationCount = _.get(runSummary, 'run.stats.iterations.total', 1), // default no of iterations is 1
+        totalRequests = _.get(runSummary, 'run.stats.requests.total', 0),
+        collectionRunObj = {
+            id: uuid.v4(),
+            collection: _.get(collectionRunOptions, 'collection.id'),
+            environment: _.get(collectionRunOptions, 'environment.id'),
+            folder: _.get(collectionRunOptions, 'folder.id'),
+            name: _.get(collectionRunOptions, 'collection.name', FALLBACK_COLLECTION_RUN_NAME),
+            status: NEWMAN_RUN_STATUS_FINISHED,
+            source: NEWMAN_STRING,
+            delay: collectionRunOptions.delayRequest || 0,
+            currentIteration: iterationCount,
+            failedTestCount: failedTestCount,
+            skippedTestCount: skippedTestCount,
+            passedTestCount: (totalTestCount - (failedTestCount + skippedTestCount)),
+            totalTestCount: totalTestCount,
+            iterations: _executionToIterationConverter(executions, iterationCount),
+            // total time of all responses
+            totalTime: _.get(runSummary, 'run.timings.responseAverage', 0) * totalRequests,
+            totalRequests: totalRequests,
+            startedAt: _.get(runSummary, 'run.timings.started'),
+            createdAt: _.get(runSummary, 'run.timings.completed') // time when run was completed and ingested into DB
+        };
+
+    collectionRunObj = _.omitBy(collectionRunObj, _.isNil);
+
+    return collectionRunObj;
+}
+
+module.exports = {
+    buildCollectionRunObject
+};

lib/reporters/postman/helpers/upload-run.js

@@ -0,0 +1,80 @@
+const _ = require('lodash'),
+    print = require('../../../print'),
+    request = require('postman-request'),
+    {
+        POSTMAN_API_BASE_URL,
+        POSTMAN_API_UPLOAD_PATH,
+        RESPONSE_FALLBACK_ERROR_MESSAGE
+    } = require('./constants'),
+    { buildCollectionRunObject } = require('./run-utils');
+
+/**
+ * 1. Converts the newman run summary into a collection run object.
+ * 2. Makes an API call to postman API to upload the collection run data to postman.
+ *
+ * @param {String} postmanApiKey - Postman API Key used for authentication
+ * @param {Object} collectionRunOptions - newman run options.
+ * @param {String} collectionRunOptions.verbose -
+ *  If set, it shows detailed information of collection run and each request sent.
+ * @param {Object} runSummary - newman run summary data.
+ * @param {Function} callback - The callback function whose invocation marks the end of the uploadRun routine.
+ * @returns {Promise}
+ */
+function uploadRun (postmanApiKey, collectionRunOptions, runSummary, callback) {
+    let collectionRunObj, runOverviewObj, requestConfig;
+
+    if (!runSummary) {
+        return callback(new Error('runSummary is a required parameter to upload run data'));
+    }
+
+    try {
+        // convert the newman run summary data to collection run object
+        collectionRunObj = buildCollectionRunObject(collectionRunOptions, runSummary);
+    }
+    catch (error) {
+        return callback(new Error('Failed to serialize the run for upload. Please try again.'));
+    }
+
+    requestConfig = {
+        url: POSTMAN_API_BASE_URL + POSTMAN_API_UPLOAD_PATH,
+        body: JSON.stringify({
+            collectionRun: collectionRunObj,
+            runOverview: runOverviewObj
+        }),
+        headers: {
+            'content-type': 'application/json',
+            accept: 'application/vnd.postman.v2+json',
+            'x-api-key': postmanApiKey
+        }
+    };
+
+    return request.post(requestConfig, (error, response, body) => {
+        if (error) {
+            return callback(new Error(_.get(error, 'message', RESPONSE_FALLBACK_ERROR_MESSAGE)));
+        }
+
+        // logging the response body in case verbose option is enabled
+        if (collectionRunOptions.verbose) {
+            print.lf('Response received from postman run publish API');
+            print.lf(body);
+        }
+
+        // case 1: upload successful
+        if (_.inRange(response.statusCode, 200, 300)) {
+            return callback(null, JSON.parse(body));
+        }
+
+        // case 2: upload unsuccessful due to some client side error e.g. api key invalid
+        if (_.inRange(response.statusCode, 400, 500)) {
+            return callback(new Error(_.get(JSON.parse(body),
+                'processorErrorBody.message', RESPONSE_FALLBACK_ERROR_MESSAGE)));
+        }
+
+        // case 3: Unexpected response received from server (5xx)
+        return callback(new Error(RESPONSE_FALLBACK_ERROR_MESSAGE));
+    });
+}
+
+module.exports = {
+    uploadRun
+};