WIP: sync with bitcoin 0.12 REST

app/controllers/addresses.js

@@ -131,7 +131,7 @@ var logtime = function(str, reset) {
 var cache = {};
 exports.multitxs = function(req, res, next) {
   if (!checkSync(req, res)) return;
-  //logtime('Start', 1);
+  logtime('Start', 1);
 
   function processTxs(txs, from, to, cb) {
     var nbTxs = txs.length;
@@ -204,7 +204,7 @@ exports.multitxs = function(req, res, next) {
   var addrStrs = req.param('addrs');
 
   if (cache[addrStrs] && from > 0) {
-    //logtime('Cache hit');
+    logtime('Cache hit');
     txs =cache[addrStrs]; 
     return processTxs(txs, from, to, function(err, transactions) {
       //logtime('After process Txs');
@@ -229,6 +229,7 @@ exports.multitxs = function(req, res, next) {
       });
     }, function(err) { // finished callback
       if (err) return common.handleErrors(err, res);
+console.log('[addresses.js.234:txs:]',txs); //TODO
 
       var MAX = 9999999999;
       txs = _.uniq(_.flatten(txs), 'txid');
@@ -244,7 +245,7 @@ exports.multitxs = function(req, res, next) {
         cache[addrStrs] = txs; 
         // 5 min. just to purge memory. Cache is overwritten in from=0 requests.
         setTimeout(function(){
-          console.log('Deleting cache');
+          console.log('Deleting cache:', addrStrs.substr(0,20));
           delete cache[addrStrs];
         }, 5 * 60 * 1000); 
       }

config/config.js

@@ -75,6 +75,7 @@ var bitcoindConf = {
   port: process.env.BITCOIND_PORT || b_port,
   p2pPort: process.env.BITCOIND_P2P_PORT || p2p_port,
   p2pHost: process.env.BITCOIND_P2P_HOST || process.env.BITCOIND_HOST || '127.0.0.1',
+  restUrl: process.env.BITCOIND_REST_URL ||'http://127.0.0.1:18332/rest',
   dataDir: dataDir,
   // DO NOT CHANGE THIS!
   disableAgent: true

insight.js

@@ -105,7 +105,7 @@ if (!config.disableP2pSync) {
 
 // historic_sync process
 var historicSync = new HistoricSync({
-  shouldBroadcastSync: true
+  shouldBroadcastSync: true,
 });
 peerSync.historicSync = historicSync;
 

lib/BlockDb.js

@@ -146,6 +146,7 @@ BlockDb.prototype.add = function(b, height, cb) {
   });
 
   var dbScript = this._addBlockScript(b, height);
+
   dbScript = dbScript.concat(this._addTxsScript(txs, b.hash, height));
   this.txDb.addMany(b.tx, function(err) {
     if (err) return cb(err);

lib/HistoricSync.js

@@ -3,6 +3,7 @@
 var imports = require('soop').imports();
 var util = require('util');
 var async = require('async');
+var request = require('request');
 
 var bitcore = require('bitcore');
 var networks = bitcore.networks;
@@ -43,15 +44,13 @@ function HistoricSync(opts) {
   this.rpc = new RpcClient(config.bitcoind);
 
   this.getBitcoinCoreVersion(function(bitcoinVersion) {
-    if (bitcoinVersion > 100000 && !config.forceRPCsync) {
+    if (bitcoinVersion < 120000) {
       info('-------------------------------------------------------');
-      info('- Bitcoin Core version >0.10 only works with RPC sync -');
-      info('- Set the env variable INSIGHT_FORCE_RPC_SYNC = 1     -');
+      info('- Bitcoin Core version <0.12 not supported            -');
       info('-------------------------------------------------------');
       process.exit(1);
     } else {
       info('Bitcoin Core version ', bitcoinVersion);
-      info('Using RPC sync ');
     }
   });
 
@@ -130,14 +129,37 @@ HistoricSync.prototype.updatePercentage = function() {
   if (this.syncPercentage > 100) this.syncPercentage = 100;
 };
 
-HistoricSync.prototype.getBlockFromRPC = function(cb) {
+
+HistoricSync.prototype.getBlockFromREST = function(cb) {
   var self = this;
+  if (!self.currentRESTHash) return cb();
+
+  var blockInfo;
+  var url = config.bitcoind.restUrl + '/block/' + self.currentRESTHash + '.json';
+  console.log('GET ', url); //TODO
+  request(url, function(err, ret, body) {
+    if (err || ret.statusCode != 200) {
+      console.log('FAILED', err, ret ? ret.statusCode : ''); //TODO
+      return setTimeout(function() {
+        self.getBlockFromREST(cb);
+      }, 2000);
+    };
+
+    var json = JSON.parse(body);
+    self.currentRESTHash = json.nextblockhash;
+    return cb(null, json);
+  });
+};
+
 
+HistoricSync.prototype.getBlockFromRPC = function(cb) {
+  var self = this;
   if (!self.currentRpcHash) return cb();
 
   var blockInfo;
   self.rpc.getBlock(self.currentRpcHash, function(err, ret) {
     if (err) return cb(err);
+
     if (ret) {
       blockInfo = ret.result;
       // this is to match block retreived from file
@@ -283,75 +305,37 @@ HistoricSync.prototype.updateStartBlock = function(opts, next) {
   }
 };
 
-HistoricSync.prototype.prepareFileSync = function(opts, next) {
-  var self = this;
-
-  if (config.forceRPCsync) return next();
-
-  if (opts.forceRPC || !config.bitcoind.dataDir ||
-    self.height > self.blockChainHeight * PERCENTAGE_TO_START_FROM_RPC) return next();
-
-
-  try {
-    self.blockExtractor = new BlockExtractor(config.bitcoind.dataDir, config.network);
-  } catch (e) {
-    info(e.message + '. Disabling file sync.');
-    return next();
-  }
-
-  self.getFn = self.getBlockFromFile;
-  self.allowReorgs = true;
-  self.sync.bDb.getLastFileIndex(function(err, idx) {
-
-    if (opts.forceStartFile)
-      self.blockExtractor.currentFileIndex = opts.forceStartFile;
-    else if (idx) self.blockExtractor.currentFileIndex = idx;
-
-    var h = self.genesis;
-
-    info('Seeking file to:' + self.startBlock);
-    //forward till startBlock
-    async.whilst(
-      function() {
-        return h !== self.startBlock;
-      },
-      function(w_cb) {
-        self.getBlockFromFile(function(err, b) {
-          if (!b) return w_cb('Could not find block ' + self.startBlock);
-          h = b.hash;
-          setImmediate(function() {
-            return w_cb(err);
-          });
-        });
-      }, function(err) {
-        console.log('\tFOUND Starting Block!');
-
-        // TODO SET HEIGHT
-        return next(err);
-      });
-  });
-};
-
 //NOP
 HistoricSync.prototype.prepareRpcSync = function(opts, next) {
   var self = this;
 
-  if (self.blockExtractor) return next();
+  if (self.restSync) return next();
   self.getFn = self.getBlockFromRPC;
   self.allowReorgs = true;
   self.currentRpcHash = self.startBlock;
   return next();
 };
 
+HistoricSync.prototype.prepareRESTSync = function(opts, next) {
+  var self = this;
+  self.getFn = self.getBlockFromREST;
+  self.allowReorgs = true;
+  self.restSync = true;
+  self.currentRESTHash = self.startBlock;
+
+  return next();
+};
+
+
+
 HistoricSync.prototype.showSyncStartMessage = function() {
   var self = this;
 
   info('Got ' + self.height +
     ' blocks in current DB, out of ' + self.blockChainHeight + ' block at bitcoind');
 
-  if (self.blockExtractor) {
-    info('bitcoind dataDir configured...importing blocks from .dat files');
-    info('First file index: ' + self.blockExtractor.currentFileIndex);
+  if (self.restSync) {
+    info('syncing from REST');
   } else {
     info('syncing from RPC (slow)');
   }
@@ -368,7 +352,7 @@ HistoricSync.prototype.setupSyncStatus = function() {
   if (step < 10) step = 10;
 
   self.step = step;
-  self.type = self.blockExtractor ? 'from .dat Files' : 'from RPC calls';
+  self.type = self.restSync ? 'from REST' : 'from RPC calls';
   self.status = 'syncing';
   self.startTs = Date.now();
   self.endTs = null;
@@ -407,7 +391,7 @@ HistoricSync.prototype.prepareToSync = function(opts, next) {
         self.updateStartBlock(opts, s_c);
       },
       function(s_c) {
-        self.prepareFileSync(opts, s_c);
+        self.prepareRESTSync(opts, s_c);
       },
       function(s_c) {
         self.prepareRpcSync(opts, s_c);
@@ -444,6 +428,7 @@ HistoricSync.prototype.start = function(opts, next) {
           if (err) return w_cb(self.setError(err));
 
           if (blockInfo && blockInfo.hash && (!opts.stopAt || opts.stopAt !== blockInfo.hash)) {
+
             self.sync.storeTipBlock(blockInfo, self.allowReorgs, function(err, height) {
               if (err) return w_cb(self.setError(err));
               if (height >= 0) self.height = height;

lib/PeerSync.js

@@ -69,9 +69,9 @@ PeerSync.prototype.handleTx = function(info) {
 
   this.sync.storeTx(tx, function(err, relatedAddrs) {
     if (err) {
-      self.log('[p2p_sync] Error in handle TX: ' + JSON.stringify(err));
+      return self.log('[p2p_sync] Error in handle TX: ' + JSON.stringify(err));
     }
-    else if (self.shouldBroadcast) {
+    if (self.shouldBroadcast) {
       sockets.broadcastTx(tx);
       self._broadcastAddr(tx.txid, relatedAddrs);
     }