Feature/SuppressErroredRequests

README.md

@@ -293,6 +293,7 @@ return of the `newman.run` function is a run instance, which emits run events th
 | options.insecure          | Disables SSL verification checks and allows self-signed SSL certificates.<br /><br />_Optional_<br />Type: `boolean`, Default value: `false` |
 | options.bail              | A switch to specify whether or not to gracefully stop a collection run (after completing the current test script) on encountering the first error. Takes additional modifiers as arguments to specify whether to end the run with an error for invalid name or path.<br /><br/>Available modifiers: `folder` and `failure`.<br />eg. `bail : ['folder']`<br /><br />_Optional_<br />Type: `boolean\|object`, Default value: `false` |
 | options.suppressExitCode  | If present, allows overriding the default exit code from the current collection run, useful for bypassing collection result failures. Takes no arguments.<br /><br />_Optional_<br />Type: `boolean`, Default value: `false` |
+| options.suppressRequestErrors  | If present, ignores request errors of the provided hostnames from the current collection run, useful for bypassing collection result errors.<br /><br />Example: 123.random.z,123.random.x<br /><br />_Optional_<br />Type: `array`, Default value: `[]` |
 | options.reporters         | Specify one reporter name as `string` or provide more than one reporter name as an `array`.<br /><br />Available reporters: `cli`, `json`, `junit`, `progress` and `emojitrain`.<br /><br />_Optional_<br />Type: `string\|array` |
 | options.reporter          | Specify options for the reporter(s) declared in `options.reporters`. <br /> e.g. `reporter : { junit : { export : './xmlResults.xml' } }` <br /> e.g. `reporter : { html : { export : './htmlResults.html', template: './customTemplate.hbs' } }` <br /><br />_Optional_<br />Type: `object` |
 | options.color             | Enable or Disable colored CLI output.<br/><br/>Available options: `on`, `off` and `auto`<br /><br />_Optional_<br />Type: `string`, Default value: `auto` |

bin/newman.js

@@ -43,6 +43,8 @@ program
         ' and whether to end the run with an error based on the optional modifier', util.cast.csvParse)
     .option('--ignore-redirects', 'Prevents Newman from automatically following 3XX redirect responses')
     .option('-x , --suppress-exit-code', 'Specify whether or not to override the default exit code for the current run')
+    .option('--suppress-request-errors [hostnames]', 'Specify the hostnames to be ignored on request errors',
+        util.cast.csvParse, [])
     .option('--silent', 'Prevents Newman from showing output to CLI')
     .option('--disable-unicode', 'Forces Unicode compliant symbols to be replaced by their plain text equivalents')
     .option('--color <value>', 'Enable/Disable colored output (auto|on|off)', util.cast.colorOptions, 'auto')

lib/run/summary.js

@@ -38,6 +38,13 @@ RunSummary = function RunSummary (emitter, options) {
          */
         globals: _.get(options, 'globals'),
 
+        /**
+         * Hostnames ignored during the run failures
+         *
+         * @type {VariableScope}
+         */
+        suppressRequestErrors: _.get(options, 'suppressRequestErrors'),
+
         /**
          * Holds information related to the run.
          */
@@ -370,6 +377,9 @@ _.assign(RunSummary, {
             // push failures sent from "before" events
             emitter.on(event, function (err, o) {
                 if (!err) { return; }
+                else if (_.isEqual('Error', err.name) && _.includes(summary.suppressRequestErrors, err.hostname)) {
+                    return;
+                }
 
                 var item = o && o.item,
                     source = event;

test/cli/suppress-request-errors.test.js

@@ -0,0 +1,26 @@
+describe('newman run --suppress-request-errors', function () {
+    it('should accept the --suppress-request-errors parameter', function (done) {
+        // eslint-disable-next-line max-len
+        exec('node ./bin/newman.js run test/fixtures/run/single-get-request.json --suppress-request-errors "123.random.z,123.random.x"', function (code) {
+            expect(code, 'should have exit code of 0').to.equal(0);
+            done();
+        });
+    });
+
+    // eslint-disable-next-line max-len
+    it('should exit non-zero if --suppress-request-errors parameter is absent on a failing collection', function (done) {
+        exec('node ./bin/newman.js run test/fixtures/run/failed-request.json', function (code) {
+            expect(code, 'should have non-zero exit code').to.be.greaterThan(0);
+            done();
+        });
+    });
+
+    // eslint-disable-next-line max-len
+    it('should exit with code zero if --suppress-request-errors parameter is present on a failing collection', function (done) {
+        // eslint-disable-next-line max-len
+        exec('node ./bin/newman.js run test/fixtures/run/failed-request.json --suppress-request-errors "123.random.z,123.random.x"', function (code) {
+            expect(code, 'should have exit code of 0').to.equal(0);
+            done();
+        });
+    });
+});

test/library/suppress-request-errors.test.js

@@ -0,0 +1,29 @@
+describe('newman.run suppressRequestErrors', function () {
+    it('should accept the suppressRequestErrors parameter', function (done) {
+        newman.run({
+            collection: 'test/fixtures/run/single-get-request.json',
+            suppressRequestErrors: '123.random.z,123.random.x'
+        }, done);
+    });
+
+    it('should fail if suppressRequestErrors parameter is absent on a failing collection', function (done) {
+        newman.run({
+            collection: 'test/fixtures/run/failed-request.json'
+        }, function (err, summary) {
+            expect(err).to.be.null;
+            expect(summary.run.failures, 'should have 1 failure').to.have.lengthOf(1);
+            done();
+        });
+    });
+
+    it('should not fail if suppressRequestErrors parameter is present on a failing collection', function (done) {
+        newman.run({
+            collection: 'test/fixtures/run/failed-request.json',
+            suppressRequestErrors: '123.random.z,123.random.x'
+        }, function (err, summary) {
+            expect(err).to.be.null;
+            expect(summary.run.failures, 'should have 0 failure').to.have.lengthOf(0);
+            done();
+        });
+    });
+});

test/unit/cli.test.js

@@ -45,6 +45,7 @@ describe('cli parser', function () {
                     expect(opts).to.eql({
                         collection: 'collection.json',
                         reporters: ['cli'],
+                        suppressRequestErrors: [],
                         delayRequest: 0,
                         globalVar: [],
                         envVar: [],

test/unit/run-summary.test.js

@@ -33,7 +33,8 @@ describe('run summary', function () {
     it('should have the relevant top-level data structures', function () {
         var summary = new Summary(new EventEmitter());
 
-        expect(_.keys(summary).sort()).to.eql(['collection', 'environment', 'globals', 'run'].sort());
+        expect(_.keys(summary).sort()).to.eql(['collection', 'environment', 'globals', 'run',
+            'suppressRequestErrors'].sort());
     });
 
     it('should have run related properties', function () {