From 6df6748ac97d8a82acbc192e81b596e8eba1a068 Mon Sep 17 00:00:00 2001 From: Ardalan Naghshineh Date: Mon, 20 Jul 2020 13:07:08 -0600 Subject: [PATCH 1/6] node_modules should be ignored by git --- .gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..40b878d --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +node_modules/ \ No newline at end of file From 515a7e4d360fee0120c936bd5b0bd67d459318fd Mon Sep 17 00:00:00 2001 From: Ardalan Naghshineh Date: Mon, 20 Jul 2020 13:07:57 -0600 Subject: [PATCH 2/6] Errors should not be caught silently --- src/core/proxy.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/core/proxy.js b/src/core/proxy.js index 3bde32d..de61955 100644 --- a/src/core/proxy.js +++ b/src/core/proxy.js @@ -35,7 +35,10 @@ const requestHandler = async (request, proxy, overrides = {}) => { headers: response.headers, body: response.body }); - } catch(error) {await request.abort()} + } catch (error) { + console.error(error) + await request.abort(); + } }; // For reassigning proxy of page From c1990e9ced58442b4fa23b5fc0c3a17cc31dd3db Mon Sep 17 00:00:00 2001 From: Ardalan Naghshineh Date: Mon, 20 Jul 2020 13:12:02 -0600 Subject: [PATCH 3/6] Consumer should be able to enable `ignoreInvalidCookies` --- README.md | 2 +- src/core/proxy.js | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c872c7a..2fb67b7 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ npm i puppeteer-page-proxy - `proxy` <[string](https://developer.mozilla.org/en-US/docs/Glossary/String)|[object](https://developer.mozilla.org/en-US/docs/Glossary/Object)> Proxy to use in the current page. * Begins with a protocol (e.g. http://, https://, socks://) * In the case of [proxy per request](https://github.com/Cuadrix/puppeteer-page-proxy#proxy-per-request), this can be an object with optional properties for overriding requests:\ -`url`, `method`, `postData`, `headers`\ +`url`, `method`, `postData`, `headers`, `ignoreInvalidCookies`\ See [httpRequest.continue](https://github.com/puppeteer/puppeteer/blob/main/docs/api.md#httprequestcontinueoverrides) for more info about the above properties. #### PageProxy.lookup(page[, lookupService, isJSON, timeout]) diff --git a/src/core/proxy.js b/src/core/proxy.js index de61955..7bbc801 100644 --- a/src/core/proxy.js +++ b/src/core/proxy.js @@ -19,7 +19,11 @@ const requestHandler = async (request, proxy, overrides = {}) => { agent: setAgent(proxy), responseType: "buffer", maxRedirects: 15, - throwHttpErrors: false + throwHttpErrors: false, + ignoreInvalidCookies: + typeof overrides.ignoreInvalidCookies === "undefined" + ? false + : overrides.ignoreInvalidCookies }; try { const response = await got(overrides.url || request.url(), options); @@ -94,4 +98,4 @@ const useProxy = async (target, data) => { } }; -module.exports = useProxy; \ No newline at end of file +module.exports = useProxy; From a3048ef401e2a3a162f6f863db402c23c2fd9d7a Mon Sep 17 00:00:00 2001 From: Ardalan Naghshineh Date: Mon, 20 Jul 2020 15:30:09 -0600 Subject: [PATCH 4/6] Throw errors instead of just logging them to console so that the consumer can catch them and control what happens --- src/core/proxy.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/proxy.js b/src/core/proxy.js index 7bbc801..d448beb 100644 --- a/src/core/proxy.js +++ b/src/core/proxy.js @@ -40,8 +40,8 @@ const requestHandler = async (request, proxy, overrides = {}) => { body: response.body }); } catch (error) { - console.error(error) await request.abort(); + throw error; } }; From 92f79a56ee04a74af8eca0e80f97230ddf6d3404 Mon Sep 17 00:00:00 2001 From: Ardalan Naghshineh Date: Mon, 20 Jul 2020 15:30:48 -0600 Subject: [PATCH 5/6] Fixed the infinite redirects bug on Instagram. Location headers should be handled by Puppeteer and not `got` --- src/core/proxy.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/proxy.js b/src/core/proxy.js index d448beb..1459df4 100644 --- a/src/core/proxy.js +++ b/src/core/proxy.js @@ -23,7 +23,8 @@ const requestHandler = async (request, proxy, overrides = {}) => { ignoreInvalidCookies: typeof overrides.ignoreInvalidCookies === "undefined" ? false - : overrides.ignoreInvalidCookies + : overrides.ignoreInvalidCookies, + followRedirect: false }; try { const response = await got(overrides.url || request.url(), options); From e00f4aa62c656220cf07cd310224f426496160d4 Mon Sep 17 00:00:00 2001 From: Ardalan Naghshineh Date: Mon, 20 Jul 2020 20:59:37 -0600 Subject: [PATCH 6/6] PR review changes --- README.md | 2 +- src/core/proxy.js | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 2fb67b7..c872c7a 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ npm i puppeteer-page-proxy - `proxy` <[string](https://developer.mozilla.org/en-US/docs/Glossary/String)|[object](https://developer.mozilla.org/en-US/docs/Glossary/Object)> Proxy to use in the current page. * Begins with a protocol (e.g. http://, https://, socks://) * In the case of [proxy per request](https://github.com/Cuadrix/puppeteer-page-proxy#proxy-per-request), this can be an object with optional properties for overriding requests:\ -`url`, `method`, `postData`, `headers`, `ignoreInvalidCookies`\ +`url`, `method`, `postData`, `headers`\ See [httpRequest.continue](https://github.com/puppeteer/puppeteer/blob/main/docs/api.md#httprequestcontinueoverrides) for more info about the above properties. #### PageProxy.lookup(page[, lookupService, isJSON, timeout]) diff --git a/src/core/proxy.js b/src/core/proxy.js index 1459df4..0445aa1 100644 --- a/src/core/proxy.js +++ b/src/core/proxy.js @@ -20,10 +20,7 @@ const requestHandler = async (request, proxy, overrides = {}) => { responseType: "buffer", maxRedirects: 15, throwHttpErrors: false, - ignoreInvalidCookies: - typeof overrides.ignoreInvalidCookies === "undefined" - ? false - : overrides.ignoreInvalidCookies, + ignoreInvalidCookies: true, followRedirect: false }; try { @@ -42,7 +39,6 @@ const requestHandler = async (request, proxy, overrides = {}) => { }); } catch (error) { await request.abort(); - throw error; } };