From 9e6c8a53b64d5bd86bdbdd4bad465786388fb18e Mon Sep 17 00:00:00 2001
From: roks0n <haluzan.rok@gmail.com>
Date: Mon, 15 Oct 2018 22:35:27 +0200
Subject: [PATCH] first

---
 .gitignore     |   4 ++
 README.md      |  12 ++++
 app.js         |  12 ++++
 package.json   |   6 ++
 src/crawler.js | 145 +++++++++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 179 insertions(+)
 create mode 100644 README.md
 create mode 100644 app.js
 create mode 100644 package.json
 create mode 100644 src/crawler.js

diff --git a/.gitignore b/.gitignore
index ad46b30..3b36221 100644
--- a/.gitignore
+++ b/.gitignore
@@ -59,3 +59,7 @@ typings/
 
 # next.js build output
 .next
+.idea
+*.log
+node_modules
+.DS_Store
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..147d3c9
--- /dev/null
+++ b/README.md
@@ -0,0 +1,12 @@
+# Crawler
+
+Network scanning tool for Ark (v2)
+
+
+## Installation
+
+`npm install`
+
+## Usage
+
+`node app.js http://<ip>:<port>`
diff --git a/app.js b/app.js
new file mode 100644
index 0000000..f49c909
--- /dev/null
+++ b/app.js
@@ -0,0 +1,12 @@
+const Crawler = require('./src/crawler')
+
+const crawler = new Crawler()
+const args = process.argv.slice(2)
+
+let node = {ip: '167.99.243.111', port: 4003}
+if (args.length === 1) {
+    const url = new URL(args[0])
+    node.ip = url.hostname
+    node.port = url.port
+}
+crawler.run(node)
diff --git a/package.json b/package.json
new file mode 100644
index 0000000..3be178d
--- /dev/null
+++ b/package.json
@@ -0,0 +1,6 @@
+{
+  "dependencies": {
+    "axios": "^0.18.0",
+    "delay": "^4.0.1",
+  }
+}
diff --git a/src/crawler.js b/src/crawler.js
new file mode 100644
index 0000000..2773673
--- /dev/null
+++ b/src/crawler.js
@@ -0,0 +1,145 @@
+const axios = require('axios')
+const delay = require('delay')
+
+module.exports = class Crawler {
+
+  constructor (timeout=3000) {
+    this.queued = []
+    this.visited = []
+    this.scanned = []
+    this.timeout = timeout
+  }
+
+   async run (node) {
+    this.startTime = null
+    this.running = 0
+    this.startTime = new Date()
+    this.queued.push(node)
+    this.processQueue()
+    this._checkWhenFinished()
+  }
+
+  /**
+   * [_checkWhenFinished recursive loop, checking when there are no more running processes]
+   * @return {[void]}
+   */
+  async _checkWhenFinished () {
+    if (this.running === 0) {
+      this._report()
+    } else {
+      await delay(500)
+      this._checkWhenFinished()
+    }
+  }
+
+  /**
+   * [processQueue processes the queue]
+   * @return {[void]}
+   */
+  async processQueue () {
+    while (this.queued.length > 0) {
+      const node = this.queued.shift()
+      this._getNode(node)
+    }
+  }
+
+  /**
+   * [_report runs a report of the block heights and block ids]
+   * @return {[void]}
+   */
+  async _report () {
+    let blockStats = {}
+    for (const item of this.scanned) {
+      if (!blockStats[item.height]) {
+        blockStats[item.height] = {}
+        blockStats[item.height].count = 1
+        blockStats[item.height].ids = {}
+        blockStats[item.height].ids[item.id] = 1
+      } else {
+        blockStats[item.height].count += 1
+        blockStats[item.height].ids[item.id] += 1
+      }
+    }
+
+    console.log(`===========================================`)
+    console.log(`Total nodes visited: ${this.visited.length}`)
+    console.log(`Total nodes online: ${this.scanned.length}`)
+    console.log(`------------------------------------------`)
+    console.log(`Block stats:`)
+    console.log(blockStats)
+    console.log(`------------------------------------------`)
+    console.log(`Finished scanning in ${new Date() - this.startTime}ms`)
+  }
+
+  /**
+   * [_getNode gets data from a node]
+   * @param  {[Object]} node
+   * @return {[void]}
+   */
+  async _getNode (node) {
+    this.running += 2
+    this.visited.push(node.ip)
+    this._getPeerList(node)
+      .then(peers => {
+        for (const peer of peers) {
+          if (this.visited.includes(peer.ip)) {
+            continue
+          }
+          this.queued.push(peer)
+        }
+        if (this.queued.length) {
+          this.processQueue()
+        }
+        this.running--
+      })
+      .catch(err => {
+        console.error(`There was a problem getting peer list from http://${node.ip}:4003`)
+        this.running--
+      })
+
+    this._getHeight(node)
+      .then(data => {
+        this.scanned.push(data)
+        this.running--
+      })
+      .catch(err => {
+        console.error(`There was a problem getting peer height from http://${node.ip}:4003`)
+        this.running--
+      })
+  }
+
+  /**
+   * [_getPeerList gets a list of peers the node is connected to]
+   * @param  {[Object]} node
+   * @return {[Array]} array of node objects
+   */
+  async _getPeerList (node) {
+    try {
+      const result = await axios.get(
+        `http://${node.ip}:4003/api/peers`,
+        {timeout: this.timeout}
+      )
+      return result.data.peers
+    } catch (err) {
+      throw err
+    }
+  }
+
+  /**
+   * [_getHeight get height of the node]
+   * @param  {[Object]} node
+   * @return {[Object]} object containing height and id of the block at the given height
+   */
+  async _getHeight (node) {
+    try {
+      const result = await axios.get(
+        `http://${node.ip}:4003/api/blocks/getHeight`,
+        {timeout: this.timeout}
+      )
+      return {id: result.data.id, height: result.data.height}
+    } catch (err) {
+      throw err
+    }
+  }
+
+}