From 4cd21798c80af3db363dbe11f885e725ff9f45f6 Mon Sep 17 00:00:00 2001 From: Vijayarajan Ravindran Date: Thu, 9 Feb 2023 14:22:13 +0530 Subject: [PATCH 1/3] Suppressing errors on request domains. Feature request: https://github.com/postmanlabs/newman/issues/1918 Introduced a new command line argument named supressRequestErrors to newman cli. The new argument will receive a array of domain names to be ignored of their errors (DNS lookup errors). The Run Summary instance will suppress the errors if the errored url / domain is supplied by the user in this argument. --- bin/newman.js | 2 ++ lib/run/summary.js | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/bin/newman.js b/bin/newman.js index 31affd9a6..93db6d059 100755 --- a/bin/newman.js +++ b/bin/newman.js @@ -43,6 +43,8 @@ program ' and whether to end the run with an error based on the optional modifier', util.cast.csvParse) .option('--ignore-redirects', 'Prevents Newman from automatically following 3XX redirect responses') .option('-x , --suppress-exit-code', 'Specify whether or not to override the default exit code for the current run') + .option('--suppress-request-errors [hostnames]', 'Specify the hostnames to be ignored on request errors', + util.cast.csvParse, []) .option('--silent', 'Prevents Newman from showing output to CLI') .option('--disable-unicode', 'Forces Unicode compliant symbols to be replaced by their plain text equivalents') .option('--color ', 'Enable/Disable colored output (auto|on|off)', util.cast.colorOptions, 'auto') diff --git a/lib/run/summary.js b/lib/run/summary.js index 0779fffee..40976be5d 100644 --- a/lib/run/summary.js +++ b/lib/run/summary.js @@ -38,6 +38,13 @@ RunSummary = function RunSummary (emitter, options) { */ globals: _.get(options, 'globals'), + /** + * Hostnames ignored during the run failures + * + * @type {VariableScope} + */ + suppressRequestErrors: _.get(options, 'suppressRequestErrors'), + /** * Holds information related to the run. */ @@ -370,6 +377,9 @@ _.assign(RunSummary, { // push failures sent from "before" events emitter.on(event, function (err, o) { if (!err) { return; } + else if (_.isEqual('Error', err.name) && _.includes(summary.suppressRequestErrors, err.hostname)) { + return; + } var item = o && o.item, source = event; From 8be3a92ec0e3facfe32eaf839b0be0e63fcab539 Mon Sep 17 00:00:00 2001 From: Vijayarajan Ravindran Date: Thu, 9 Feb 2023 14:33:04 +0530 Subject: [PATCH 2/3] Adding / Updating Test cases to support the new argument. --- test/cli/suppress-request-errors.test.js | 26 ++++++++++++++++++ test/library/suppress-request-errors.test.js | 29 ++++++++++++++++++++ test/unit/cli.test.js | 1 + test/unit/run-summary.test.js | 3 +- 4 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 test/cli/suppress-request-errors.test.js create mode 100644 test/library/suppress-request-errors.test.js diff --git a/test/cli/suppress-request-errors.test.js b/test/cli/suppress-request-errors.test.js new file mode 100644 index 000000000..e2e0f861b --- /dev/null +++ b/test/cli/suppress-request-errors.test.js @@ -0,0 +1,26 @@ +describe('newman run --suppress-request-errors', function () { + it('should accept the --suppress-request-errors parameter', function (done) { + // eslint-disable-next-line max-len + exec('node ./bin/newman.js run test/fixtures/run/single-get-request.json --suppress-request-errors "123.random.z,123.random.x"', function (code) { + expect(code, 'should have exit code of 0').to.equal(0); + done(); + }); + }); + + // eslint-disable-next-line max-len + it('should exit non-zero if --suppress-request-errors parameter is absent on a failing collection', function (done) { + exec('node ./bin/newman.js run test/fixtures/run/failed-request.json', function (code) { + expect(code, 'should have non-zero exit code').to.be.greaterThan(0); + done(); + }); + }); + + // eslint-disable-next-line max-len + it('should exit with code zero if --suppress-request-errors parameter is present on a failing collection', function (done) { + // eslint-disable-next-line max-len + exec('node ./bin/newman.js run test/fixtures/run/failed-request.json --suppress-request-errors "123.random.z,123.random.x"', function (code) { + expect(code, 'should have exit code of 0').to.equal(0); + done(); + }); + }); +}); diff --git a/test/library/suppress-request-errors.test.js b/test/library/suppress-request-errors.test.js new file mode 100644 index 000000000..ab5a35264 --- /dev/null +++ b/test/library/suppress-request-errors.test.js @@ -0,0 +1,29 @@ +describe('newman.run suppressRequestErrors', function () { + it('should accept the suppressRequestErrors parameter', function (done) { + newman.run({ + collection: 'test/fixtures/run/single-get-request.json', + suppressRequestErrors: '123.random.z,123.random.x' + }, done); + }); + + it('should fail if suppressRequestErrors parameter is absent on a failing collection', function (done) { + newman.run({ + collection: 'test/fixtures/run/failed-request.json' + }, function (err, summary) { + expect(err).to.be.null; + expect(summary.run.failures, 'should have 1 failure').to.have.lengthOf(1); + done(); + }); + }); + + it('should not fail if suppressRequestErrors parameter is present on a failing collection', function (done) { + newman.run({ + collection: 'test/fixtures/run/failed-request.json', + suppressRequestErrors: '123.random.z,123.random.x' + }, function (err, summary) { + expect(err).to.be.null; + expect(summary.run.failures, 'should have 0 failure').to.have.lengthOf(0); + done(); + }); + }); +}); diff --git a/test/unit/cli.test.js b/test/unit/cli.test.js index b523e2b22..e55fa7bbb 100644 --- a/test/unit/cli.test.js +++ b/test/unit/cli.test.js @@ -45,6 +45,7 @@ describe('cli parser', function () { expect(opts).to.eql({ collection: 'collection.json', reporters: ['cli'], + suppressRequestErrors: [], delayRequest: 0, globalVar: [], envVar: [], diff --git a/test/unit/run-summary.test.js b/test/unit/run-summary.test.js index d6b99e91b..6da1fe16b 100644 --- a/test/unit/run-summary.test.js +++ b/test/unit/run-summary.test.js @@ -33,7 +33,8 @@ describe('run summary', function () { it('should have the relevant top-level data structures', function () { var summary = new Summary(new EventEmitter()); - expect(_.keys(summary).sort()).to.eql(['collection', 'environment', 'globals', 'run'].sort()); + expect(_.keys(summary).sort()).to.eql(['collection', 'environment', 'globals', 'run', + 'suppressRequestErrors'].sort()); }); it('should have run related properties', function () { From 3254da4551ac05c6adb0c9cff39f1908044cc68a Mon Sep 17 00:00:00 2001 From: Vijayarajan Ravindran Date: Thu, 9 Feb 2023 14:33:33 +0530 Subject: [PATCH 3/3] Updating Readme to describe the new command line argument. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 9a3a16c1c..ec51c4b6e 100644 --- a/README.md +++ b/README.md @@ -293,6 +293,7 @@ return of the `newman.run` function is a run instance, which emits run events th | options.insecure | Disables SSL verification checks and allows self-signed SSL certificates.

_Optional_
Type: `boolean`, Default value: `false` | | options.bail | A switch to specify whether or not to gracefully stop a collection run (after completing the current test script) on encountering the first error. Takes additional modifiers as arguments to specify whether to end the run with an error for invalid name or path.

Available modifiers: `folder` and `failure`.
eg. `bail : ['folder']`

_Optional_
Type: `boolean\|object`, Default value: `false` | | options.suppressExitCode | If present, allows overriding the default exit code from the current collection run, useful for bypassing collection result failures. Takes no arguments.

_Optional_
Type: `boolean`, Default value: `false` | +| options.suppressRequestErrors | If present, ignores request errors of the provided hostnames from the current collection run, useful for bypassing collection result errors.

Example: 123.random.z,123.random.x

_Optional_
Type: `array`, Default value: `[]` | | options.reporters | Specify one reporter name as `string` or provide more than one reporter name as an `array`.

Available reporters: `cli`, `json`, `junit`, `progress` and `emojitrain`.

_Optional_
Type: `string\|array` | | options.reporter | Specify options for the reporter(s) declared in `options.reporters`.
e.g. `reporter : { junit : { export : './xmlResults.xml' } }`
e.g. `reporter : { html : { export : './htmlResults.html', template: './customTemplate.hbs' } }`

_Optional_
Type: `object` | | options.color | Enable or Disable colored CLI output.

Available options: `on`, `off` and `auto`

_Optional_
Type: `string`, Default value: `auto` |