From 39843458be8152eb46172fefdf24fe12692c1ea4 Mon Sep 17 00:00:00 2001 From: jbigman Date: Fri, 11 Aug 2023 17:59:27 +0200 Subject: [PATCH] Reverted changes on processFullDetailApps and added tests --- README.md | 2 +- lib/utils/processPages.js | 36 +++++------- test/lib.search.js | 113 +++++++++++++++++++++++++++++++------- 3 files changed, 109 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 919e8177..e6c8ef71 100644 --- a/README.md +++ b/README.md @@ -621,7 +621,7 @@ Once that limit is reached, further requests will be held until the second passe import gplay from "google-play-scraper"; // the following method will perform batches of 10 requests per second -gplay.search({term: 'panda', throttle: 10}).then(console.log); +gplay.search({term: 'panda', throttle: {limit: 10, interval: 1000}}).then(console.log); ``` By default, no throttling is applied. diff --git a/lib/utils/processPages.js b/lib/utils/processPages.js index db35420e..ed78ad12 100644 --- a/lib/utils/processPages.js +++ b/lib/utils/processPages.js @@ -28,24 +28,18 @@ async function processPages (html, opts, savedApps, mappings) { } async function processFullDetailApps (apps, opts) { - const getAppDetails = (app) => appDetails({ - appId: app.appId, - lang: opts.lang, - country: opts.country, - cache: opts.cache, - throttle: opts.throttle, - requestOptions: opts.requestOptions - }); - - if (opts.throttle !== undefined) { - const throttle = pThrottle({ - limit: opts.throttle.limit ? opts.throttle.limit : 1, - interval: opts.throttle.interval ? opts.throttle.interval : 1000 - }); - return Promise.all(apps.map(await throttle(getAppDetails))); - } else { - return Promise.all(apps.map(getAppDetails)); - } + const promises = apps.map(app => ( + appDetails({ + appId: app.appId, + lang: opts.lang, + country: opts.country, + cache: opts.cache, + throttle: opts.throttle, + requestOptions: opts.requestOptions + }) + )); + + return Promise.all(promises); } const REQUEST_MAPPINGS = { @@ -89,9 +83,9 @@ function checkFinished (opts, savedApps, nextToken) { } function getBodyForRequests ({ - numberOfApps = 100, - withToken = '%token%' -}) { + numberOfApps = 100, + withToken = '%token%' + }) { const body = `f.req=%5B%5B%5B%22qnKhOb%22%2C%22%5B%5Bnull%2C%5B%5B10%2C%5B10%2C${numberOfApps}%5D%5D%2Ctrue%2Cnull%2C%5B96%2C27%2C4%2C8%2C57%2C30%2C110%2C79%2C11%2C16%2C49%2C1%2C3%2C9%2C12%2C104%2C55%2C56%2C51%2C10%2C34%2C77%5D%5D%2Cnull%2C%5C%22${withToken}%5C%22%5D%5D%22%2Cnull%2C%22generic%22%5D%5D%5D`; return body; diff --git a/test/lib.search.js b/test/lib.search.js index 94d05ae5..b40f8c90 100644 --- a/test/lib.search.js +++ b/test/lib.search.js @@ -23,17 +23,26 @@ describe('Search method', () => { // preregister tend to have some fields missing, increasing chances of failure // by searching "preregister" we have more chances of getting some in the results it('should search for pre register', () => - gplay.search({ term: 'preregister', num: 10 }) + gplay.search({ + term: 'preregister', + num: 10 + }) .then((apps) => apps.map(assertValidApp))); it('should fetch multiple pages of distinct results', () => - gplay.search({ term: 'p', num: 55 }) + gplay.search({ + term: 'p', + num: 55 + }) .then((apps) => { assert.equal(apps.length, 55, 'should return as many apps as requested'); })); it('should fetch multiple pages of when not starting from cluster of subsections', () => - gplay.search({ term: 'p', num: 65 }) + gplay.search({ + term: 'p', + num: 65 + }) .then((apps) => { assert.equal(apps.length, 65, 'should return as many apps as requested'); })); @@ -41,17 +50,27 @@ describe('Search method', () => { describe('country and language specific', () => { describe('without more results section', () => { it('should fetch a valid application list for eu country', () => { - return gplay.search({ term: 'Panda vs Zombies', country: 'GH' }) + return gplay.search({ + term: 'Panda vs Zombies', + country: 'GH' + }) .then((apps) => apps.map(assertValidApp)); }); it('should fetch a valid application list for non eu country', () => { - return gplay.search({ term: 'Facebook', country: 'GE' }) + return gplay.search({ + term: 'Facebook', + country: 'GE' + }) .then((apps) => apps.map(assertValidApp)); }); it('should fetch a valid application list for eu country with specific language', () => { - return gplay.search({ term: 'Panda vs Zombies', country: 'BE', lang: 'it' }) + return gplay.search({ + term: 'Panda vs Zombies', + country: 'BE', + lang: 'it' + }) .then((apps) => apps.map(assertValidApp)); }); }); @@ -67,7 +86,11 @@ describe('Search method', () => { }); it('should return few netflix apps from german store with german language', () => { - return gplay.search({ term: 'netflix', lang: 'de', country: 'DE' }) + return gplay.search({ + term: 'netflix', + lang: 'de', + country: 'DE' + }) .then((apps) => { assert.equal(apps[0].appId, 'com.netflix.mediaclient'); // Don't check specific ids, as results may vary @@ -94,12 +117,20 @@ describe('Search method', () => { }); it('should return empty set when no results found in eu country store', () => { - return gplay.search({ term: 'ASyyDASDyyASDASD', country: 'DE', lang: 'SP' }) + return gplay.search({ + term: 'ASyyDASDyyASDASD', + country: 'DE', + lang: 'SP' + }) .then(assert.isEmpty); }); it('should return empty set when no results found in us store with other language', () => { - return gplay.search({ term: 'ASyyDASDyyASDASD', country: 'US', lang: 'FR' }) + return gplay.search({ + term: 'ASyyDASDyyASDASD', + country: 'US', + lang: 'FR' + }) .then(assert.isEmpty); }); }); @@ -114,7 +145,10 @@ describe('Search method', () => { }); it('should return apps from suggested search in european country', () => { - return gplay.search({ term: 'runing tracker', country: 'GR' }) + return gplay.search({ + term: 'runing tracker', + country: 'GR' + }) .then((apps) => { apps.map(assertValidApp); assertIdsInArray(apps, 'com.runtastic.android', 'running.tracker.gps.map'); @@ -124,18 +158,18 @@ describe('Search method', () => { describe('search with full details', () => { it('should search for "runing app" with fullDetail', () => { - const options = { - term: 'preregister', - num: 10, - fullDetail: true - }; - gplay.search(options).then((apps) => - apps.map(assertValidApp) - ); - } + const options = { + term: 'preregister', + num: 10, + fullDetail: true + }; + gplay.search(options).then((apps) => + apps.map(assertValidApp) + ); + } ); - it('should search for "runing app" with fullDetail and throttling interval', async () => { + it('should search for "runing app" with fullDetail and throttling limit + interval', async () => { const dateBefore = new Date(); const throttle = { limit: 1, @@ -143,6 +177,7 @@ describe('Search method', () => { }; return gplay.search({ term: 'runing app', + num: 5, fullDetail: true, throttle }) @@ -153,5 +188,43 @@ describe('Search method', () => { assert.isAbove(interval, apps.length * throttle.interval); }); }).timeout(10000); + + it('should search for "runing app" with fullDetail and throttling limit', async () => { + const dateBefore = new Date(); + const throttle = { + limit: 1, + }; + return gplay.search({ + term: 'runing app', + num: 5, + fullDetail: true, + throttle + }) + .then((apps) => { + apps.map(assertValidApp); + const dateAfter = new Date(); + const interval = dateAfter - dateBefore; + assert.isBelow(interval, 5000); + }); + }).timeout(10000); + + it('should search for "runing app" with fullDetail and throttling interval', async () => { + const dateBefore = new Date(); + const throttle = { + interval: 500 + }; + return gplay.search({ + term: 'runing app', + num: 5, + fullDetail: true, + throttle + }) + .then((apps) => { + apps.map(assertValidApp); + const dateAfter = new Date(); + const interval = dateAfter - dateBefore; + assert.isBelow(interval, 5000); + }); + }).timeout(10000); }); });