Initialize crawling with a set of seed nodes

- A downloaded peers.json can be used to initialize traversal - All collected information is stored in an output file
Tuditi · Apr 10, 2020 · 080b327 · 080b327
1 parent 4b51c9c
commit 080b327
Show file tree

Hide file tree

Showing 11 changed files with 446 additions and 76 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,7 +2,9 @@
 
 ## [2.0.0] - 2020-04-07
 
-- Support for Hydra peers
+- Support for Hydra peers and network definition files.
+- Storing some extra information about network nodes in a
+  timestamped output file.
 
 ## [1.3.0] - 2020-02-04
 

diff --git a/README.md b/README.md
@@ -10,10 +10,25 @@ you sponsor them for their hard work.
 
 ## Usage
 
-`node . http://<ip>:<port>`
+- [Hydra mainnet](http://hydra.iop.global/)
+
+  `node . hyd-mainnet.json`
 
-For port use the p2p port, which is 4701 for [Hydra mainnet](http://hydra.iop.global/) or
-4702 for [Hydra devnet](http://dev.hydra.iop.global/).
+- [Hydra devnet](http://dev.hydra.iop.global/)
+
+  `node . hyd-devnet.json`
+
+- [ARK mainnet](https://explorer.ark.io/)
+
+  `node . ark-mainnet.json`
+
+- [ARK devnet](https://dev.explorer.ark.io/)
+
+  `node . ark-devnet.json`
+
+- If you think the seed nodes for the networks has changed since we downloaded them, just run the download script using `curl` on your system to refresh them from github
+
+  `./download.sh`
 
 ## Credits
 
@@ -24,5 +39,5 @@ For port use the p2p port, which is 4701 for [Hydra mainnet](http://hydra.iop.gl
 
 ## License
 
-ARK Delegate: [MIT](LICENSE) © roks0n
-Hydra patches: [MIT](LICENSE) © Decentralized Society Foundation, Panama
+- ARK Delegate: [MIT](LICENSE) © roks0n
+- Hydra patches: [MIT](LICENSE) © Decentralized Society Foundation, Panama
diff --git a/app.js b/app.js
@@ -1,9 +1,9 @@
-const Crawler = require('./src/crawler')
-const { URL } = require('url')
-const { orderBy } = require('lodash/collection')
+const { orderBy } = require('lodash')
+const fs = require('fs')
+const path = require('path')
+const moment = require('moment')
 
-const crawler = new Crawler()
-const args = process.argv.slice(2)
+const Crawler = require('./src/crawler')
 
 const report = (crawler) => {
   const blockStats = {}
@@ -16,8 +16,6 @@ const report = (crawler) => {
       continue
     }
 
-    console.log(JSON.stringify(node, undefined, 2))
-
     if (blockStats[node.height]) {
       blockStats[node.height].count += 1
       blockStats[node.height].ids[node.id] += 1
@@ -77,14 +75,33 @@ const report = (crawler) => {
   console.log('------------------------------------------')
   console.log(`Finished scanning in ${new Date() - crawler.startTime}ms`)
 
-  process.exit(0)
+  return crawler
 }
 
-const node = { ip: undefined, port: undefined }
-if (args.length === 1) {
-  const url = new URL(args[0])
-  node.ip = url.hostname
-  node.port = url.port
+const main = async () => {
+  try {
+    const crawler = new Crawler()
+    const args = process.argv.slice(2)
+    const outputFilename = `${path.basename(args[0], '.json')}-${moment().format('YYYYMMDD-HHmmss')}.json`
+
+    const inputStr = fs.readFileSync(args[0], { encoding: 'utf-8' })
+    let input = JSON.parse(inputStr)
+    if ('list' in input) {
+      input = input.list
+    }
+
+    for (const node of input) {
+      crawler.add(node)
+    }
+
+    await crawler.run()
+    await report(crawler)
+
+    const outputStr = JSON.stringify(Object.values(crawler.nodes), undefined, 2)
+    fs.writeFileSync(outputFilename, outputStr, { encoding: 'utf-8' })
+  } catch (err) {
+    console.error(err)
+  }
 }
 
-crawler.run(node).then(report).catch(err => console.error(err))
+main().then(() => {})
diff --git a/ark-devnet.json b/ark-devnet.json
@@ -0,0 +1,16 @@
+[{
+  "ip": "167.114.29.51",
+  "port": 4002
+}, {
+  "ip": "167.114.29.52",
+  "port": 4002
+}, {
+  "ip": "167.114.29.53",
+  "port": 4002
+}, {
+  "ip": "167.114.29.54",
+  "port": 4002
+}, {
+  "ip": "167.114.29.55",
+  "port": 4002
+}]
diff --git a/ark-mainnet.json b/ark-mainnet.json
@@ -0,0 +1,193 @@
+[{
+  "ip": "5.196.105.32",
+  "port": 4001
+}, {
+  "ip": "5.196.105.33",
+  "port": 4001
+}, {
+  "ip": "5.196.105.34",
+  "port": 4001
+}, {
+  "ip": "5.196.105.35",
+  "port": 4001
+}, {
+  "ip": "5.196.105.36",
+  "port": 4001
+}, {
+  "ip": "5.196.105.37",
+  "port": 4001
+}, {
+  "ip": "5.196.105.38",
+  "port": 4001
+}, {
+  "ip": "5.196.105.39",
+  "port": 4001
+}, {
+  "ip": "178.32.65.136",
+  "port": 4001
+}, {
+  "ip": "178.32.65.137",
+  "port": 4001
+}, {
+  "ip": "178.32.65.138",
+  "port": 4001
+}, {
+  "ip": "178.32.65.139",
+  "port": 4001
+}, {
+  "ip": "178.32.65.140",
+  "port": 4001
+}, {
+  "ip": "178.32.65.141",
+  "port": 4001
+}, {
+  "ip": "178.32.65.142",
+  "port": 4001
+}, {
+  "ip": "178.32.65.143",
+  "port": 4001
+}, {
+  "ip": "5.196.105.40",
+  "port": 4001
+}, {
+  "ip": "5.196.105.41",
+  "port": 4001
+}, {
+  "ip": "5.196.105.42",
+  "port": 4001
+}, {
+  "ip": "5.196.105.43",
+  "port": 4001
+}, {
+  "ip": "5.196.105.44",
+  "port": 4001
+}, {
+  "ip": "5.196.105.45",
+  "port": 4001
+}, {
+  "ip": "5.196.105.46",
+  "port": 4001
+}, {
+  "ip": "5.196.105.47",
+  "port": 4001
+}, {
+  "ip": "54.38.120.32",
+  "port": 4001
+}, {
+  "ip": "54.38.120.33",
+  "port": 4001
+}, {
+  "ip": "54.38.120.34",
+  "port": 4001
+}, {
+  "ip": "54.38.120.35",
+  "port": 4001
+}, {
+  "ip": "54.38.120.36",
+  "port": 4001
+}, {
+  "ip": "54.38.120.37",
+  "port": 4001
+}, {
+  "ip": "54.38.120.38",
+  "port": 4001
+}, {
+  "ip": "54.38.120.39",
+  "port": 4001
+}, {
+  "ip": "151.80.125.32",
+  "port": 4001
+}, {
+  "ip": "151.80.125.33",
+  "port": 4001
+}, {
+  "ip": "151.80.125.34",
+  "port": 4001
+}, {
+  "ip": "151.80.125.35",
+  "port": 4001
+}, {
+  "ip": "151.80.125.36",
+  "port": 4001
+}, {
+  "ip": "151.80.125.37",
+  "port": 4001
+}, {
+  "ip": "151.80.125.38",
+  "port": 4001
+}, {
+  "ip": "151.80.125.39",
+  "port": 4001
+}, {
+  "ip": "213.32.41.104",
+  "port": 4001
+}, {
+  "ip": "213.32.41.105",
+  "port": 4001
+}, {
+  "ip": "213.32.41.106",
+  "port": 4001
+}, {
+  "ip": "213.32.41.107",
+  "port": 4001
+}, {
+  "ip": "213.32.41.108",
+  "port": 4001
+}, {
+  "ip": "213.32.41.109",
+  "port": 4001
+}, {
+  "ip": "213.32.41.110",
+  "port": 4001
+}, {
+  "ip": "213.32.41.111",
+  "port": 4001
+}, {
+  "ip": "5.135.22.92",
+  "port": 4001
+}, {
+  "ip": "5.135.22.93",
+  "port": 4001
+}, {
+  "ip": "5.135.22.94",
+  "port": 4001
+}, {
+  "ip": "5.135.22.95",
+  "port": 4001
+}, {
+  "ip": "5.135.52.96",
+  "port": 4001
+}, {
+  "ip": "5.135.52.97",
+  "port": 4001
+}, {
+  "ip": "5.135.52.98",
+  "port": 4001
+}, {
+  "ip": "5.135.52.99",
+  "port": 4001
+}, {
+  "ip": "51.255.105.52",
+  "port": 4001
+}, {
+  "ip": "51.255.105.53",
+  "port": 4001
+}, {
+  "ip": "51.255.105.54",
+  "port": 4001
+}, {
+  "ip": "51.255.105.55",
+  "port": 4001
+}, {
+  "ip": "46.105.160.104",
+  "port": 4001
+}, {
+  "ip": "46.105.160.105",
+  "port": 4001
+}, {
+  "ip": "46.105.160.106",
+  "port": 4001
+}, {
+  "ip": "46.105.160.107",
+  "port": 4001
+}]
diff --git a/download.sh b/download.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+curl -L https://github.com/ArkEcosystem/peers/raw/master/mainnet.json -o ark-mainnet.json
+curl -L https://github.com/ArkEcosystem/peers/raw/master/devnet.json -o ark-devnet.json
+curl -L https://github.com/Internet-of-People/hydra-peers/raw/master/mainnet.json -o hyd-mainnet.json
+curl -L https://github.com/Internet-of-People/hydra-peers/raw/master/devnet.json -o hyd-devnet.json
diff --git a/hyd-devnet.json b/hyd-devnet.json
@@ -0,0 +1,30 @@
+{
+    "list": [
+      {
+        "ip"  : "35.204.124.143",
+        "port": 4702
+      },
+      {
+        "ip"  : "34.68.118.161",
+        "port": 4702
+      },
+      {
+        "ip"  : "35.228.196.114",
+        "port": 4702
+      },
+      {
+        "ip"  : "34.87.3.205",
+        "port": 4702
+      },
+      {
+        "ip"  : "35.185.32.241",
+        "port": 4702
+      },
+      {
+        "ip"  : "35.240.62.119",
+        "port": 4702
+      }
+    ],
+    "sources": []
+  }
+