From a489b3d468399c9920215c6bfbe96a5af893151f Mon Sep 17 00:00:00 2001 From: Dan Matthews Date: Mon, 22 Oct 2018 10:20:38 +0200 Subject: [PATCH] refactor around promises so it can be used in other apps --- README.md | 5 +- app.js | 32 +++++- package-lock.json | 53 ++++++++++ package.json | 20 ++++ src/crawler.js | 249 +++++++++++++++++++++++++++------------------- 5 files changed, 252 insertions(+), 107 deletions(-) create mode 100644 package-lock.json diff --git a/README.md b/README.md index 147d3c9..acafff7 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ # Crawler -Network scanning tool for Ark (v2) +Network scanning tool for ARK. +This currently works for ARK v1 and will be updated to work with V2's API structure shortly. ## Installation @@ -9,4 +10,4 @@ Network scanning tool for Ark (v2) ## Usage -`node app.js http://:` +`npm start http://:` diff --git a/app.js b/app.js index f49c909..855335d 100644 --- a/app.js +++ b/app.js @@ -1,12 +1,42 @@ const Crawler = require('./src/crawler') +const { URL } = require('url') +const { forEach, keys } = require('lodash') const crawler = new Crawler() const args = process.argv.slice(2) +const report = (crawler) => { + let blockStats = {} + + forEach(crawler.heights, (item) => { + if (blockStats[item.height]) { + blockStats[item.height].count += 1 + blockStats[item.height].ids[item.id] += 1 + } else { + blockStats[item.height] = {} + blockStats[item.height].count = 1 + blockStats[item.height].ids = {} + blockStats[item.height].ids[item.id] = 1 + } + }) + + console.log(`===========================================`) + console.log(`Total nodes visited: ${keys(crawler.nodes).length}`) + console.log(`Total nodes online: ${crawler.heights.length}`) + console.log(`------------------------------------------`) + console.log(`Block stats:`) + console.log(blockStats) + console.log(`------------------------------------------`) + console.log(`Finished scanning in ${new Date() - crawler.startTime}ms`) + process.exit(0) +} + let node = {ip: '167.99.243.111', port: 4003} + if (args.length === 1) { const url = new URL(args[0]) node.ip = url.hostname node.port = url.port } -crawler.run(node) + +crawler.run(node).then(report) diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..f1564ec --- /dev/null +++ b/package-lock.json @@ -0,0 +1,53 @@ +{ + "name": "crawler", + "version": "0.0.1", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "axios": { + "version": "0.18.0", + "resolved": "http://registry.npmjs.org/axios/-/axios-0.18.0.tgz", + "integrity": "sha1-MtU+SFHv3AoRmTts0AB4nXDAUQI=", + "requires": { + "follow-redirects": "^1.3.0", + "is-buffer": "^1.1.5" + } + }, + "debug": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", + "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", + "requires": { + "ms": "2.0.0" + } + }, + "delay": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/delay/-/delay-4.1.0.tgz", + "integrity": "sha512-8Hea6/aOu3bPdDBQhSRUEUzF0QwuWmSPuIK+sxNdvcJtSfzb6HXrTd9DFJBCJcV9o83fFECqTgllqdnmUfq9+w==" + }, + "follow-redirects": { + "version": "1.5.9", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.9.tgz", + "integrity": "sha512-Bh65EZI/RU8nx0wbYF9shkFZlqLP+6WT/5FnA3cE/djNSuKNHJEinGGZgu/cQEkeeb2GdFOgenAmn8qaqYke2w==", + "requires": { + "debug": "=3.1.0" + } + }, + "is-buffer": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==" + }, + "lodash": { + "version": "4.17.11", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", + "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==" + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + } + } +} diff --git a/package.json b/package.json index 3be178d..6425066 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,26 @@ { + "name": "crawler", + "version": "0.0.1", + "description": "loops through the ARK network to get the height of connected peers", + "main": "app.js", + "scripts": { + "start": "node app.js" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/roks0n/crawler.git" + }, + "engines": { + "node": "^8.11.2" + }, + "keywords": [], + "author": "A(rk)-Team", + "license": "Apache-2.0", + "homepage": "https://github.com/roks0n/crawler#readme", + "devDependencies": {}, "dependencies": { "axios": "^0.18.0", "delay": "^4.0.1", + "lodash": "^4.17.11" } } diff --git a/src/crawler.js b/src/crawler.js index 2773673..5e08b09 100644 --- a/src/crawler.js +++ b/src/crawler.js @@ -1,145 +1,186 @@ const axios = require('axios') const delay = require('delay') +const { map, values } = require('lodash') -module.exports = class Crawler { - - constructor (timeout=3000) { - this.queued = [] - this.visited = [] - this.scanned = [] +class Crawler { + /** + * Initializes the internal request reactor. + * @method constructor + */ + constructor (timeout = 5000) { + this._counter = 0 + this._queue = [] this.timeout = timeout + setTimeout(() => { this.start() }, 100) } - async run (node) { - this.startTime = null - this.running = 0 + /** + * Runs a height check on the entire network connected to the + * initial node. + * @method run + * @param {object} node {ip: [address], port: [4001]} + * @return {Promise} + */ + async run (node) { + this.heights = [] + this.nodes = {} this.startTime = new Date() - this.queued.push(node) - this.processQueue() - this._checkWhenFinished() + + await this.fetchNetwork(node) + await this.scanNodes() + console.log('done') + + return this; } /** - * [_checkWhenFinished recursive loop, checking when there are no more running processes] - * @return {[void]} + * Enqueues a function to be run by the internal reactor. + * @method queue + * @param {Function} fn Any function to be processed synchronously. + * @return {void} */ - async _checkWhenFinished () { - if (this.running === 0) { - this._report() - } else { - await delay(500) - this._checkWhenFinished() - } + queue (fn) { + this._queue.push(fn) } /** - * [processQueue processes the queue] - * @return {[void]} + * Primary runner for the internal reactor. + * @method start + * @return {void} */ - async processQueue () { - while (this.queued.length > 0) { - const node = this.queued.shift() - this._getNode(node) + start () { + if (this._counter < 500 && this._queue.length > 0) { + this._counter += 1 + setTimeout(() => { this._counter -= 1 }, 1000) + this._queue.shift()() + process.stdout.write('.') + this.start() + } else { + setTimeout(() => { this.start() }, 100) } } /** - * [_report runs a report of the block heights and block ids] - * @return {[void]} + * Walks the peer list starting with `node` + * @method fetchNetwork + * @param {object} node {ip: [address], port: [4001]} + * @return {Promise} */ - async _report () { - let blockStats = {} - for (const item of this.scanned) { - if (!blockStats[item.height]) { - blockStats[item.height] = {} - blockStats[item.height].count = 1 - blockStats[item.height].ids = {} - blockStats[item.height].ids[item.id] = 1 + fetchNetwork (node) { + return new Promise((resolve) => { + if (node.ip === '127.0.0.1') { + // Ignore localhost + resolve() } else { - blockStats[item.height].count += 1 - blockStats[item.height].ids[item.id] += 1 + this.nodes[node.ip] = 'queued' + this.queue(() => { + axios + .get(`http://${node.ip}:${node.port}/api/peers`, {timeout: this.timeout}) + .then((response) => { this.peerResponseSuccess(node, response).then(resolve) }) + .catch((err) => { this.peerResponseError(node, err).then(resolve) }) + }) } - } + }) + } - console.log(`===========================================`) - console.log(`Total nodes visited: ${this.visited.length}`) - console.log(`Total nodes online: ${this.scanned.length}`) - console.log(`------------------------------------------`) - console.log(`Block stats:`) - console.log(blockStats) - console.log(`------------------------------------------`) - console.log(`Finished scanning in ${new Date() - this.startTime}ms`) + /** + * Logs when a connection error occurs attempting to get the peer list of `node` + * @method peerResponseError + * @param {object} node {ip: [address], port: [4001]} + * @param {error} err The error object + * @return {Promise} + */ + async peerResponseError (node, err) { + if (!this.nodes[node.ip]) { + this.nodes[node.ip] = 'error' + console.error(`There was a problem getting peer list from http://${node.ip}:${node.port}`) + } + return true } /** - * [_getNode gets data from a node] - * @param {[Object]} node - * @return {[void]} + * Handles a successful(ish) peer request + * @method peerResponseSuccess + * @param {object} peer {ip: [address], port: [4001]} + * @param {axios response} response response object from axios + * @return {Promise} */ - async _getNode (node) { - this.running += 2 - this.visited.push(node.ip) - this._getPeerList(node) - .then(peers => { - for (const peer of peers) { - if (this.visited.includes(peer.ip)) { - continue - } - this.queued.push(peer) - } - if (this.queued.length) { - this.processQueue() + peerResponseSuccess (peer, response) { + if (response.status === 200 && response.data && response.data.peers) { + this.nodes[peer.ip] = peer + + const promises = map(response.data.peers, (newPeer) => { + if (this.nodes[newPeer.ip]) { + return Promise.resolve() + } else { + return this.fetchNetwork(newPeer) } - this.running-- - }) - .catch(err => { - console.error(`There was a problem getting peer list from http://${node.ip}:4003`) - this.running-- }) - this._getHeight(node) - .then(data => { - this.scanned.push(data) - this.running-- - }) - .catch(err => { - console.error(`There was a problem getting peer height from http://${node.ip}:4003`) - this.running-- - }) + return Promise.all(promises) + } else { + return this.peerResponseError(node, response) + } } /** - * [_getPeerList gets a list of peers the node is connected to] - * @param {[Object]} node - * @return {[Array]} array of node objects + * Scans all available peer nodes for current height + * @method scanNodes + * @return {Promise} */ - async _getPeerList (node) { - try { - const result = await axios.get( - `http://${node.ip}:4003/api/peers`, - {timeout: this.timeout} - ) - return result.data.peers - } catch (err) { - throw err - } + scanNodes () { + const promises = map(this.nodes, (peer, ip) => { + if (peer === 'error') { + return Promise.resolve() + } else if (peer === 'queued') { + console.log(`${ip} still queued`) + return Promise.resolve() + } else { + return new Promise((resolve) => { + this.queue(() => { + axios + .get(`http://${peer.ip}:${peer.port}/api/blocks/getHeight`, {timeout: this.timeout}) + .then((response) => { this.heightResponseSuccess(peer, response).then(resolve) }) + .catch((err) => { this.heightResponseError(peer, err).then(resolve) }) + }) + }) + } + }) + + return Promise.all(promises) } /** - * [_getHeight get height of the node] - * @param {[Object]} node - * @return {[Object]} object containing height and id of the block at the given height + * Logs when an error occurs fetching a node's current height + * @method heightResponseError + * @param {object} node {ip: [address], port: [4001]} + * @param {error} err the error object + * @return {Promise} */ - async _getHeight (node) { - try { - const result = await axios.get( - `http://${node.ip}:4003/api/blocks/getHeight`, - {timeout: this.timeout} - ) - return {id: result.data.id, height: result.data.height} - } catch (err) { - throw err - } + async heightResponseError (node, err) { + console.error(`There was a problem getting the current height of http://${node.ip}:${node.port}`) + console.error(err) + return true } + /** + * Handles a successful height request + * @method heightResponseSuccess + * @param {object} node {ip: [address], port: [4001]} + * @param {axios response} response response object from axios + * @return {Promise} + */ + heightResponseSuccess (node, response) { + if (response.status === 200 && response.data && response.data.height) { + this.heights.push({ + height: response.data.height, + id: response.data.id + }) + return Promise.resolve() + } else { + return this.heightResponseError(node, response) + } + } } + +module.exports = Crawler;