From c3449974eca87e0efb5931fa70f48778bc019146 Mon Sep 17 00:00:00 2001 From: roks0n Date: Sun, 12 May 2019 22:56:23 +0200 Subject: [PATCH] refactor to support p2p sockets, bump to version 1.1.0 --- app.js | 9 +- package-lock.json | 123 +++++++++++++++------ package.json | 17 +-- src/crawler.js | 265 +++++++++++++++++----------------------------- src/peer.js | 34 ++++++ 5 files changed, 240 insertions(+), 208 deletions(-) create mode 100644 src/peer.js diff --git a/app.js b/app.js index 6aff90f..490903c 100644 --- a/app.js +++ b/app.js @@ -9,7 +9,9 @@ const report = (crawler) => { const blockStats = {} const versionStats = {} - for (const item of crawler.heights) { + const nodes = Object.values(crawler.nodes) + + for (const item of nodes) { if (blockStats[item.height]) { blockStats[item.height].count += 1 blockStats[item.height].ids[item.id] += 1 @@ -17,6 +19,7 @@ const report = (crawler) => { blockStats[item.height] = {} blockStats[item.height].count = 1 blockStats[item.height].height = item.height + // todo block ids blockStats[item.height].ids = {} blockStats[item.height].ids[item.id] = 1 } @@ -31,7 +34,7 @@ const report = (crawler) => { } } - const allDelays = crawler.heights.filter(item => item.delay).map(item => item.delay) + const allDelays = nodes.filter(item => item.latency).map(item => item.latency) const averageDelay = allDelays.reduce((a, b) => a + b, 0) / allDelays.length const maxDelay = Math.max(...allDelays) const minDelay = Math.min(...allDelays) @@ -78,4 +81,4 @@ if (args.length === 1) { node.port = url.port } -crawler.run(node).then(report) +crawler.run(node).then(report).catch(err => console.error(err)) diff --git a/package-lock.json b/package-lock.json index 2990092..758e8bd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -94,14 +94,10 @@ "integrity": "sha512-+Ryf6g3BKoRc7jfp7ad8tM4TtMiaWvbF/1/sQcZPkkS7ag3D5nMBCe2UfOTONtAkaG0tO0ij3C5Lwmf1EiyjHg==", "dev": true }, - "axios": { - "version": "0.18.0", - "resolved": "http://registry.npmjs.org/axios/-/axios-0.18.0.tgz", - "integrity": "sha1-MtU+SFHv3AoRmTts0AB4nXDAUQI=", - "requires": { - "follow-redirects": "^1.3.0", - "is-buffer": "^1.1.5" - } + "async-limiter": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.0.tgz", + "integrity": "sha512-jp/uFnooOiO+L211eZOoSyzpOITMXx1rBITauYykG3BRYPu8h0UcxsPNB04RR5vo4Tyz3+ay17tR6JVf9qzYWg==" }, "balanced-match": { "version": "1.0.0", @@ -109,6 +105,11 @@ "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", "dev": true }, + "base64-js": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.0.tgz", + "integrity": "sha512-ccav/yGvoa80BQDljCxsmmQ3Xvx60/UpBIij5QN21W3wBi/hhIC9OoO+KLpu9IJTS9j4DRVJ3aDDF9cMSoa2lw==" + }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -119,6 +120,15 @@ "concat-map": "0.0.1" } }, + "buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.2.1.tgz", + "integrity": "sha512-c+Ko0loDaFfuPWiL02ls9Xd3GO3cPVmUobQ6t3rXNUk304u6hGq+8N/kFi+QEIKhzK3uwolVhLzszmfLmMLnqg==", + "requires": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4" + } + }, "callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -157,6 +167,11 @@ "integrity": "sha1-/xnt6Kml5XkyQUewwR8PvLq+1jk=", "dev": true }, + "clone": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/clone/-/clone-2.1.1.tgz", + "integrity": "sha1-0hfR6WERjjrJpLi7oyhVU79kfNs=" + }, "color-convert": { "version": "1.9.3", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", @@ -172,6 +187,11 @@ "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", "dev": true }, + "component-emitter": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.2.1.tgz", + "integrity": "sha1-E3kY1teCg/ffemt8WmPhQOaUJeY=" + }, "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -197,14 +217,6 @@ "which": "^1.2.9" } }, - "debug": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", - "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", - "requires": { - "ms": "2.0.0" - } - }, "deep-is": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", @@ -616,14 +628,6 @@ "integrity": "sha512-R+H8IZclI8AAkSBRQJLVOsxwAoHd6WC40b4QTNWIjzAa6BXOBfQcM587MXDTVPeYaopFNWHUFLx7eNmHDSxMWg==", "dev": true }, - "follow-redirects": { - "version": "1.5.9", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.9.tgz", - "integrity": "sha512-Bh65EZI/RU8nx0wbYF9shkFZlqLP+6WT/5FnA3cE/djNSuKNHJEinGGZgu/cQEkeeb2GdFOgenAmn8qaqYke2w==", - "requires": { - "debug": "=3.1.0" - } - }, "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -704,6 +708,11 @@ "safer-buffer": ">= 2.1.2 < 3" } }, + "ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + }, "ignore": { "version": "4.0.6", "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz", @@ -786,11 +795,6 @@ "integrity": "sha1-d8mYQFJ6qOyxqLppe4BkWnqSap0=", "dev": true }, - "is-buffer": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", - "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==" - }, "is-callable": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.1.4.tgz", @@ -883,6 +887,11 @@ "type-check": "~0.3.2" } }, + "linked-list": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/linked-list/-/linked-list-0.1.0.tgz", + "integrity": "sha1-eYsP+X0bkqT9CEgPVa6k6dSdN78=" + }, "load-json-file": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-2.0.0.tgz", @@ -943,7 +952,8 @@ "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "dev": true }, "mute-stream": { "version": "0.0.7", @@ -1133,6 +1143,11 @@ "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", "dev": true }, + "querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" + }, "read-pkg": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-2.0.0.tgz", @@ -1218,6 +1233,24 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", "dev": true }, + "sc-channel": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/sc-channel/-/sc-channel-1.2.0.tgz", + "integrity": "sha512-M3gdq8PlKg0zWJSisWqAsMmTVxYRTpVRqw4CWAdKBgAfVKumFcTjoCV0hYu7lgUXccCtCD8Wk9VkkE+IXCxmZA==", + "requires": { + "component-emitter": "1.2.1" + } + }, + "sc-errors": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/sc-errors/-/sc-errors-1.4.1.tgz", + "integrity": "sha512-dBn92iIonpChTxYLgKkIT/PCApvmYT6EPIbRvbQKTgY6tbEbIy8XVUv4pGyKwEK4nCmvX4TKXcN0iXC6tNW6rQ==" + }, + "sc-formatter": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/sc-formatter/-/sc-formatter-3.0.2.tgz", + "integrity": "sha512-9PbqYBpCq+OoEeRQ3QfFIGE6qwjjBcd2j7UjgDlhnZbtSnuGgHdcRklPKYGuYFH82V/dwd+AIpu8XvA1zqTd+A==" + }, "semver": { "version": "5.7.0", "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz", @@ -1256,6 +1289,23 @@ "is-fullwidth-code-point": "^2.0.0" } }, + "socketcluster-client": { + "version": "14.2.2", + "resolved": "https://registry.npmjs.org/socketcluster-client/-/socketcluster-client-14.2.2.tgz", + "integrity": "sha512-vofmFcTaHaIf+MqAR0OZS7e30X4jxbDPJl+taCe8kLGJ5rVOrKeuU0sGyHyHyqW87AIR6jqc4KODl4WQJ4SsAA==", + "requires": { + "buffer": "^5.2.1", + "clone": "2.1.1", + "component-emitter": "1.2.1", + "linked-list": "0.1.0", + "querystring": "0.2.0", + "sc-channel": "^1.2.0", + "sc-errors": "^1.4.1", + "sc-formatter": "^3.0.1", + "uuid": "3.2.1", + "ws": "5.1.1" + } + }, "spdx-correct": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.1.0.tgz", @@ -1419,6 +1469,11 @@ "punycode": "^2.1.0" } }, + "uuid": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.2.1.tgz", + "integrity": "sha512-jZnMwlb9Iku/O3smGWvZhauCf6cvvpKi4BKRiliS3cxnI+Gz9j5MEpTz2UFuXiKPJocb7gnsLHwiS05ige5BEA==" + }, "validate-npm-package-license": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz", @@ -1458,6 +1513,14 @@ "requires": { "mkdirp": "^0.5.1" } + }, + "ws": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-5.1.1.tgz", + "integrity": "sha512-bOusvpCb09TOBLbpMKszd45WKC2KPtxiyiHanv+H2DE3Az+1db5a/L7sVJZVDPUC1Br8f0SKRr1KjLpD1U/IAw==", + "requires": { + "async-limiter": "~1.0.0" + } } } } diff --git a/package.json b/package.json index 9f318a7..7e6112f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "Crawler", - "version": "1.0.0", + "version": "1.1.0", "description": "Crawler scans the ARK network to get information about the peers in the network.", "main": "app.js", "scripts": { @@ -14,9 +14,14 @@ "engines": { "node": "^8.11.2" }, - "keywords": [], - "author": "A(rk)-Team", - "license": "Apache-2.0", + "keywords": [ + "ark acosystem", + "network", + "scanner", + "blockchain" + ], + "author": "roks0n", + "license": "MIT", "homepage": "https://github.com/deadlock-delegate/crawler#readme", "devDependencies": { "eslint": "^5.16.0", @@ -27,7 +32,7 @@ "eslint-plugin-standard": "^4.0.0" }, "dependencies": { - "axios": "^0.18.0", - "lodash": "^4.17.11" + "lodash": "^4.17.11", + "socketcluster-client": "^14.2.2" } } diff --git a/src/crawler.js b/src/crawler.js index 1363890..e4f7f8f 100644 --- a/src/crawler.js +++ b/src/crawler.js @@ -1,5 +1,8 @@ -const axios = require('axios') const { map } = require('lodash') +const Peers = require('./peer') + +const VISITED = 1 +const NOT_VISITED = 0 class Crawler { /** @@ -7,30 +10,40 @@ class Crawler { * @method constructor */ constructor (timeout = 3000) { - this._counter = 0 - this._queue = [] this.timeout = timeout this.headers = {} - setTimeout(() => { this.start() }, 100) + this.socket = undefined + this.request = { + headers: { + nethash: 'no-nethash', + version: 'no-version', + port: 1337 + } + } } /** - * Runs a height check on the entire network connected to the - * initial node. + * Runs a height check on the entire network connected to the initial peer. * @method run - * @param {object} node {ip: [address], port: [4001]} + * @param {object} peer {ip: [address], port: [4001]} * @return {Promise} */ - async run (node) { + async run (peer) { this.heights = [] this.nodes = {} + this.samplePeers = {} this.startTime = new Date() + this.peers = new Peers() + + this.peers.add(peer.ip, peer.port) try { - await this.setNetworkConfig(node) - await this.fetchPeers(node) - await this.fetchHeights() - console.log('. done') + console.log(`... discovering network peers`) + await this.discoverPeers(peer) + console.log(`... scanning network`) + await this.scanNetwork() + console.log(`... disconnecting from all peers`) + this.peers.disconnect() } catch (err) { console.error(err) } @@ -38,174 +51,88 @@ class Crawler { return this } - /** - * Enqueues a function to be run by the internal reactor. - * @method queue - * @param {Function} fn Any function to be processed synchronously. - * @return {void} - */ - queue (fn) { - this._queue.push(fn) - } - - /** - * Primary runner for the internal reactor. - * @method start - * @return {void} - */ - start () { - if (this._counter < 500 && this._queue.length > 0) { - this._counter += 1 - setTimeout(() => { this._counter -= 1 }, 1000) - this._queue.shift()() - process.stdout.write('.') - this.start() - } else { - setTimeout(() => { this.start() }, 100) - } - } - - async setNetworkConfig (node) { - const response = await axios.get( - `http://${node.ip}:${node.port}/config`, - { timeout: this.timeout } - ) - this.headers = { - nethash: response.data.data.network.nethash, - version: response.data.data.network.version, - 'API-Version': 2, - port: 4001 - } - } + async discoverPeers (peer) { + return new Promise(async (resolve, reject) => { + const connection = this.peers.get(peer.ip) + if (!connection) { + reject(new Error(`No connection exists for ${peer.ip}:${peer.port}`)) + } + connection.emit( + 'p2p.peer.getPeers', + this.request, + (err, response) => { + if (err) { + reject(new Error(err)) + return + } + + if (peer.ip in this.samplePeers) { + this.samplePeers[peer.ip] = VISITED + } + + response.data.map((peer) => { + if (!(peer.ip in this.nodes)) { + this.nodes[peer.ip] = peer + } + + if (!this.peers.get(peer.ip)) { + this.peers.add(peer.ip, peer.port) + } + }) - /** - * Walks the peer list starting with `node` - * @method fetchPeers - * @param {object} node {ip: [address], port: [4001]} - * @return {Promise} - */ - fetchPeers (node) { - return new Promise((resolve) => { - if (node.ip === '127.0.0.1') { - // Ignore localhost - resolve() - } else { - this.nodes[node.ip] = 'queued' - this.queue(() => { - axios - .get(`http://${node.ip}:${node.port}/peer/list`, { - timeout: this.timeout, - headers: this.headers + if (this.samplePeers[peer.ip] === VISITED) { + resolve() + return + } + + // note: this is not very efficient on large arrays + const samplePeers = response.data + .map(x => ({ x, r: Math.random() })) + .sort((a, b) => a.r - b.r) + .map(a => a.x) + .slice(0, 10) + .filter(a => a.ip !== peer.ip) + .map((peer) => { + this.samplePeers[peer.ip] = NOT_VISITED + return this.discoverPeers(peer) }) - .then((response) => { this.peerResponseSuccess(node, response).then(resolve) }) - .catch((err) => { this.peerResponseError(node, err).then(resolve) }) - }) - } - }) - } - /** - * Logs when a connection error occurs attempting to get the peer list of `node` - * @method peerResponseError - * @param {object} node {ip: [address], port: [4001]} - * @param {error} err The error object - * @return {Promise} - */ - async peerResponseError (node, err) { - this.nodes[node.ip] = 'error' - console.error(`\nThere was a problem getting peer list from http://${node.ip}:${node.port}`) - return true + Promise.all(samplePeers).then(resolve) + } + ) + }) } - /** - * Handles a successful(ish) peer request - * @method peerResponseSuccess - * @param {object} peer {ip: [address], port: [4001]} - * @param {axios response} response response object from axios - * @return {Promise} - */ - peerResponseSuccess (peer, response) { - if (response.status === 200 && response.data && response.data.peers) { - this.nodes[peer.ip] = peer - - const promises = map(response.data.peers, (newPeer) => { - if (this.nodes[newPeer.ip]) { - return Promise.resolve() - } else { - return this.fetchPeers(newPeer) + scanNetwork () { + const promises = map(this.nodes, (peer) => { + return new Promise((resolve, reject) => { + const connection = this.peers.get(peer.ip) + if (!connection) { + return resolve() } - }) - - return Promise.all(promises) - } else { - return this.peerResponseError(peer, response) - } - } - /** - * Scans all available peer nodes for current height - * @method fetchHeights - * @return {Promise} - */ - fetchHeights () { - const promises = map(this.nodes, (peer, ip) => { - if (peer === 'error') { - return Promise.resolve() - } else if (peer === 'queued') { - console.log(`${ip} still queued`) - return Promise.resolve() - } else { - return new Promise((resolve) => { - this.queue(() => { - axios - .get(`http://${peer.ip}:${peer.port}/peer/height`, { - timeout: this.timeout, - headers: this.headers - }) - .then((response) => { this.heightResponseSuccess(peer, response).then(resolve) }) - .catch((err) => { this.heightResponseError(peer, err).then(resolve) }) - }) - }) - } + connection.emit( + 'p2p.peer.getStatus', + this.request, + (err, response) => { + if (err) { + console.log(err) + return reject(new Error(err)) + } + this.heights.push({ + height: response.data.header.height, + id: response.data.header.id + }) + this.nodes[peer.ip].height = response.data.header.height + this.nodes[peer.ip].id = response.data.header.id + return resolve() + } + ) + }) }) return Promise.all(promises) } - - /** - * Logs when an error occurs fetching a node's current height - * @method heightResponseError - * @param {object} node {ip: [address], port: [4001]} - * @param {error} err the error object - * @return {Promise} - */ - async heightResponseError (node, err) { - console.error(`\nThere was a problem getting the current height of http://${node.ip}:${node.port}`) - console.error(err) - return true - } - - /** - * Handles a successful height request - * @method heightResponseSuccess - * @param {object} node {ip: [address], port: [4001]} - * @param {axios response} response response object from axios - * @return {Promise} - */ - heightResponseSuccess (node, response) { - if (response.status === 200 && response.data && response.data.height) { - this.heights.push({ - height: response.data.height, - id: response.data.id, - version: node.version, - os: node.os, - delay: node.delay - }) - return Promise.resolve() - } else { - return this.heightResponseError(node, response) - } - } } module.exports = Crawler diff --git a/src/peer.js b/src/peer.js new file mode 100644 index 0000000..a5731fe --- /dev/null +++ b/src/peer.js @@ -0,0 +1,34 @@ +const SocketClient = require('socketcluster-client') + +class Peers { + constructor () { + this.connections = new Map() + } + + add (ip, port) { + let connection = this.connections.get(ip) + if (connection) { + return connection + } + connection = SocketClient.create({ hostname: ip, port }) + this.connections.set(ip, connection) + return connection + } + + get (ip) { + return this.connections.get(ip) + } + + map () { + return this.connections + } + + disconnect () { + for (const [ip, connection] of this.connections.entries()) { + connection.destroy() + this.connections.delete(ip) + } + } +} + +module.exports = Peers