diff --git a/app/controllers/addresses.js b/app/controllers/addresses.js index ca2fc3a4e..3c6965215 100644 --- a/app/controllers/addresses.js +++ b/app/controllers/addresses.js @@ -131,7 +131,7 @@ var logtime = function(str, reset) { var cache = {}; exports.multitxs = function(req, res, next) { if (!checkSync(req, res)) return; - //logtime('Start', 1); + logtime('Start', 1); function processTxs(txs, from, to, cb) { var nbTxs = txs.length; @@ -204,7 +204,7 @@ exports.multitxs = function(req, res, next) { var addrStrs = req.param('addrs'); if (cache[addrStrs] && from > 0) { - //logtime('Cache hit'); + logtime('Cache hit'); txs =cache[addrStrs]; return processTxs(txs, from, to, function(err, transactions) { //logtime('After process Txs'); @@ -229,6 +229,7 @@ exports.multitxs = function(req, res, next) { }); }, function(err) { // finished callback if (err) return common.handleErrors(err, res); +console.log('[addresses.js.234:txs:]',txs); //TODO var MAX = 9999999999; txs = _.uniq(_.flatten(txs), 'txid'); @@ -244,7 +245,7 @@ exports.multitxs = function(req, res, next) { cache[addrStrs] = txs; // 5 min. just to purge memory. Cache is overwritten in from=0 requests. setTimeout(function(){ - console.log('Deleting cache'); + console.log('Deleting cache:', addrStrs.substr(0,20)); delete cache[addrStrs]; }, 5 * 60 * 1000); } diff --git a/config/config.js b/config/config.js index 96402b409..0ad4ab671 100644 --- a/config/config.js +++ b/config/config.js @@ -75,6 +75,7 @@ var bitcoindConf = { port: process.env.BITCOIND_PORT || b_port, p2pPort: process.env.BITCOIND_P2P_PORT || p2p_port, p2pHost: process.env.BITCOIND_P2P_HOST || process.env.BITCOIND_HOST || '127.0.0.1', + restUrl: process.env.BITCOIND_REST_URL ||'http://127.0.0.1:18332/rest', dataDir: dataDir, // DO NOT CHANGE THIS! disableAgent: true diff --git a/insight.js b/insight.js index 52d673957..ab30652e2 100755 --- a/insight.js +++ b/insight.js @@ -105,7 +105,7 @@ if (!config.disableP2pSync) { // historic_sync process var historicSync = new HistoricSync({ - shouldBroadcastSync: true + shouldBroadcastSync: true, }); peerSync.historicSync = historicSync; diff --git a/lib/BlockDb.js b/lib/BlockDb.js index 41c5b86ea..f02fff66c 100644 --- a/lib/BlockDb.js +++ b/lib/BlockDb.js @@ -146,6 +146,7 @@ BlockDb.prototype.add = function(b, height, cb) { }); var dbScript = this._addBlockScript(b, height); + dbScript = dbScript.concat(this._addTxsScript(txs, b.hash, height)); this.txDb.addMany(b.tx, function(err) { if (err) return cb(err); diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index 505ce4c0b..403874b75 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -3,6 +3,7 @@ var imports = require('soop').imports(); var util = require('util'); var async = require('async'); +var request = require('request'); var bitcore = require('bitcore'); var networks = bitcore.networks; @@ -43,15 +44,13 @@ function HistoricSync(opts) { this.rpc = new RpcClient(config.bitcoind); this.getBitcoinCoreVersion(function(bitcoinVersion) { - if (bitcoinVersion > 100000 && !config.forceRPCsync) { + if (bitcoinVersion < 120000) { info('-------------------------------------------------------'); - info('- Bitcoin Core version >0.10 only works with RPC sync -'); - info('- Set the env variable INSIGHT_FORCE_RPC_SYNC = 1 -'); + info('- Bitcoin Core version <0.12 not supported -'); info('-------------------------------------------------------'); process.exit(1); } else { info('Bitcoin Core version ', bitcoinVersion); - info('Using RPC sync '); } }); @@ -130,14 +129,37 @@ HistoricSync.prototype.updatePercentage = function() { if (this.syncPercentage > 100) this.syncPercentage = 100; }; -HistoricSync.prototype.getBlockFromRPC = function(cb) { + +HistoricSync.prototype.getBlockFromREST = function(cb) { var self = this; + if (!self.currentRESTHash) return cb(); + + var blockInfo; + var url = config.bitcoind.restUrl + '/block/' + self.currentRESTHash + '.json'; + console.log('GET ', url); //TODO + request(url, function(err, ret, body) { + if (err || ret.statusCode != 200) { + console.log('FAILED', err, ret ? ret.statusCode : ''); //TODO + return setTimeout(function() { + self.getBlockFromREST(cb); + }, 2000); + }; + + var json = JSON.parse(body); + self.currentRESTHash = json.nextblockhash; + return cb(null, json); + }); +}; + +HistoricSync.prototype.getBlockFromRPC = function(cb) { + var self = this; if (!self.currentRpcHash) return cb(); var blockInfo; self.rpc.getBlock(self.currentRpcHash, function(err, ret) { if (err) return cb(err); + if (ret) { blockInfo = ret.result; // this is to match block retreived from file @@ -283,75 +305,37 @@ HistoricSync.prototype.updateStartBlock = function(opts, next) { } }; -HistoricSync.prototype.prepareFileSync = function(opts, next) { - var self = this; - - if (config.forceRPCsync) return next(); - - if (opts.forceRPC || !config.bitcoind.dataDir || - self.height > self.blockChainHeight * PERCENTAGE_TO_START_FROM_RPC) return next(); - - - try { - self.blockExtractor = new BlockExtractor(config.bitcoind.dataDir, config.network); - } catch (e) { - info(e.message + '. Disabling file sync.'); - return next(); - } - - self.getFn = self.getBlockFromFile; - self.allowReorgs = true; - self.sync.bDb.getLastFileIndex(function(err, idx) { - - if (opts.forceStartFile) - self.blockExtractor.currentFileIndex = opts.forceStartFile; - else if (idx) self.blockExtractor.currentFileIndex = idx; - - var h = self.genesis; - - info('Seeking file to:' + self.startBlock); - //forward till startBlock - async.whilst( - function() { - return h !== self.startBlock; - }, - function(w_cb) { - self.getBlockFromFile(function(err, b) { - if (!b) return w_cb('Could not find block ' + self.startBlock); - h = b.hash; - setImmediate(function() { - return w_cb(err); - }); - }); - }, function(err) { - console.log('\tFOUND Starting Block!'); - - // TODO SET HEIGHT - return next(err); - }); - }); -}; - //NOP HistoricSync.prototype.prepareRpcSync = function(opts, next) { var self = this; - if (self.blockExtractor) return next(); + if (self.restSync) return next(); self.getFn = self.getBlockFromRPC; self.allowReorgs = true; self.currentRpcHash = self.startBlock; return next(); }; +HistoricSync.prototype.prepareRESTSync = function(opts, next) { + var self = this; + self.getFn = self.getBlockFromREST; + self.allowReorgs = true; + self.restSync = true; + self.currentRESTHash = self.startBlock; + + return next(); +}; + + + HistoricSync.prototype.showSyncStartMessage = function() { var self = this; info('Got ' + self.height + ' blocks in current DB, out of ' + self.blockChainHeight + ' block at bitcoind'); - if (self.blockExtractor) { - info('bitcoind dataDir configured...importing blocks from .dat files'); - info('First file index: ' + self.blockExtractor.currentFileIndex); + if (self.restSync) { + info('syncing from REST'); } else { info('syncing from RPC (slow)'); } @@ -368,7 +352,7 @@ HistoricSync.prototype.setupSyncStatus = function() { if (step < 10) step = 10; self.step = step; - self.type = self.blockExtractor ? 'from .dat Files' : 'from RPC calls'; + self.type = self.restSync ? 'from REST' : 'from RPC calls'; self.status = 'syncing'; self.startTs = Date.now(); self.endTs = null; @@ -407,7 +391,7 @@ HistoricSync.prototype.prepareToSync = function(opts, next) { self.updateStartBlock(opts, s_c); }, function(s_c) { - self.prepareFileSync(opts, s_c); + self.prepareRESTSync(opts, s_c); }, function(s_c) { self.prepareRpcSync(opts, s_c); @@ -444,6 +428,7 @@ HistoricSync.prototype.start = function(opts, next) { if (err) return w_cb(self.setError(err)); if (blockInfo && blockInfo.hash && (!opts.stopAt || opts.stopAt !== blockInfo.hash)) { + self.sync.storeTipBlock(blockInfo, self.allowReorgs, function(err, height) { if (err) return w_cb(self.setError(err)); if (height >= 0) self.height = height; diff --git a/lib/PeerSync.js b/lib/PeerSync.js index d3e8d7c53..bdd8e2cd1 100644 --- a/lib/PeerSync.js +++ b/lib/PeerSync.js @@ -69,9 +69,9 @@ PeerSync.prototype.handleTx = function(info) { this.sync.storeTx(tx, function(err, relatedAddrs) { if (err) { - self.log('[p2p_sync] Error in handle TX: ' + JSON.stringify(err)); + return self.log('[p2p_sync] Error in handle TX: ' + JSON.stringify(err)); } - else if (self.shouldBroadcast) { + if (self.shouldBroadcast) { sockets.broadcastTx(tx); self._broadcastAddr(tx.txid, relatedAddrs); } diff --git a/lib/Sync.js b/lib/Sync.js index 36b0fa7df..e5abcd9a0 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -1,15 +1,17 @@ 'use strict'; -var imports = require('soop').imports(); +var imports = require('soop').imports(); -var config = imports.config || require('../config/config'); -var bitcore = require('bitcore'); -var networks = bitcore.networks; -var async = require('async'); +var config = imports.config || require('../config/config'); +var bitcore = require('bitcore'); +var networks = bitcore.networks; +var async = require('async'); +var _ = require('lodash'); +var Address = require('../app/models/Address'); var logger = require('./logger').logger; -var d = logger.log; -var info = logger.info; +var d = logger.log; +var info = logger.info; @@ -19,7 +21,7 @@ function Sync(opts) { this.id = syncId++; this.opts = opts || {}; this.bDb = require('./BlockDb').default(); - this.txDb = require('./TransactionDb').default(); + this.txDb = require('./TransactionDb').default(); this.network = config.network === 'testnet' ? networks.testnet : networks.livenet; this.cachedLastHash = null; } @@ -46,35 +48,35 @@ Sync.prototype.destroy = function(next) { }; /* - * Arrives a NEW block, which is the new TIP - * - * Case 0) Simple case - * A-B-C-D-E(TIP)-NEW - * - * Case 1) - * A-B-C-D-E(TIP) - * \ - * NEW - * - * 1) Declare D-E orphans (and possible invalidate TXs on them) - * - * Case 2) - * A-B-C-D-E(TIP) - * \ - * F-G-NEW - * 1) Set F-G as connected (mark TXs as valid) - * 2) Set new heights in F-G-NEW - * 3) Declare D-E orphans (and possible invalidate TXs on them) - * - * - * Case 3) - * - * A-B-C-D-E(TIP) ... NEW - * - * NEW is ignored (if allowReorgs is false) - * - * - */ + * Arrives a NEW block, which is the new TIP + * + * Case 0) Simple case + * A-B-C-D-E(TIP)-NEW + * + * Case 1) + * A-B-C-D-E(TIP) + * \ + * NEW + * + * 1) Declare D-E orphans (and possible invalidate TXs on them) + * + * Case 2) + * A-B-C-D-E(TIP) + * \ + * F-G-NEW + * 1) Set F-G as connected (mark TXs as valid) + * 2) Set new heights in F-G-NEW + * 3) Declare D-E orphans (and possible invalidate TXs on them) + * + * + * Case 3) + * + * A-B-C-D-E(TIP) ... NEW + * + * NEW is ignored (if allowReorgs is false) + * + * + */ Sync.prototype.storeTipBlock = function(b, allowReorgs, cb) { @@ -85,17 +87,18 @@ Sync.prototype.storeTipBlock = function(b, allowReorgs, cb) { if (!b) return cb(); var self = this; - if ( self.storingBlock ) { + if (self.storingBlock) { logger.debug('Storing a block already. Delaying storeTipBlock with:' + - b.hash); - return setTimeout( function() { + b.hash); + return setTimeout(function() { logger.debug('Retrying storeTipBlock with: ' + b.hash); - self.storeTipBlock(b,allowReorgs,cb); + self.storeTipBlock(b, allowReorgs, cb); }, 1000); } - self.storingBlock=1; - var oldTip, oldNext, oldHeight, needReorg = false, height = -1; + self.storingBlock = 1; + var oldTip, oldNext, oldHeight, needReorg = false, + height = -1; var newPrev = b.previousblockhash; async.series([ @@ -108,6 +111,7 @@ Sync.prototype.storeTipBlock = function(b, allowReorgs, cb) { // }); // }, function(c) { +console.log('[Sync.js.115:newPrev:]',newPrev); //TODO if (!allowReorgs || newPrev === self.cachedLastHash) return c(); self.bDb.has(newPrev, function(err, val) { // Genesis? no problem @@ -137,7 +141,7 @@ Sync.prototype.storeTipBlock = function(b, allowReorgs, cb) { return c(); }); }, - function(c) { + function(c) { if (!allowReorgs) return c(); if (needReorg) { info('NEW TIP: %s NEED REORG (old tip: %s #%d)', b.hash, oldTip, oldHeight); @@ -147,14 +151,13 @@ Sync.prototype.storeTipBlock = function(b, allowReorgs, cb) { height = h; return c(); }); - } - else { + } else { height = oldHeight + 1; return c(); } }, function(c) { - self.cachedLastHash = b.hash; // just for speed up. + self.cachedLastHash = b.hash; // just for speed up. self.bDb.add(b, height, c); }, function(c) { @@ -174,7 +177,7 @@ Sync.prototype.storeTipBlock = function(b, allowReorgs, cb) { if (err && err.toString().match(/WARN/)) { err = null; } - self.storingBlock=0; + self.storingBlock = 0; return cb(err, height); }); }; @@ -192,7 +195,7 @@ Sync.prototype.processReorg = function(oldTip, oldNext, newPrev, oldHeight, cb) // Case 3 + allowReorgs = true return c(new Error('Could not found block:' + newPrev)); } - if (height<0) return c(); + if (height < 0) return c(); newHeight = height + 1; info('Reorg Case 1) OldNext: %s NewHeight: %d', oldNext, newHeight); @@ -270,7 +273,7 @@ Sync.prototype.setBranchConnectedBackwards = function(fromHash, cb) { }); }, function() { - return hashInterator && yHeight<=0; + return hashInterator && yHeight <= 0; }, function() { info('\tFound yBlock: %s #%d', hashInterator, yHeight); @@ -286,7 +289,7 @@ Sync.prototype.setBranchConnectedBackwards = function(fromHash, cb) { }, function(err) { return cb(err, hashInterator, lastHash, heightIter); - }); + }); }); }; diff --git a/lib/TransactionDb.js b/lib/TransactionDb.js index 4c2c17c99..cac87ef48 100644 --- a/lib/TransactionDb.js +++ b/lib/TransactionDb.js @@ -681,6 +681,7 @@ TransactionDb.prototype._addManyFromObjs = function(txs, next) { TransactionDb.prototype._addManyFromHashes = function(txs, next) { var self = this; var dbScript = []; + async.eachLimit(txs, CONCURRENCY, function(tx, each_cb) { if (tx === genesisTXID) return each_cb(); diff --git a/package.json b/package.json index 0b06e9880..64d4dbc24 100644 --- a/package.json +++ b/package.json @@ -3,25 +3,32 @@ "description": "An open-source bitcoin blockchain API. The Insight API provides you with a convenient, powerful and simple way to query and broadcast data on the bitcoin network and build your own services with it.", "version": "0.2.18", "repository": "git://github.com/bitpay/insight-api.git", - "contributors": [{ - "name": "Matias Alejo Garcia", - "email": "ematiu@gmail.com" - }, { - "name": "Manuel Araoz", - "email": "manuelaraoz@gmail.com" - }, { - "name": "Mario Colque", - "email": "colquemario@gmail.com" - }, { - "name": "Gustavo Cortez", - "email": "cmgustavo83@gmail.com" - }, { - "name": "Juan Ignacio Sosa Lopez", - "email": "bechilandia@gmail.com" - }, { - "name": "Ivan Socolsky", - "email": "jungans@gmail.com" - }], + "contributors": [ + { + "name": "Matias Alejo Garcia", + "email": "ematiu@gmail.com" + }, + { + "name": "Manuel Araoz", + "email": "manuelaraoz@gmail.com" + }, + { + "name": "Mario Colque", + "email": "colquemario@gmail.com" + }, + { + "name": "Gustavo Cortez", + "email": "cmgustavo83@gmail.com" + }, + { + "name": "Juan Ignacio Sosa Lopez", + "email": "bechilandia@gmail.com" + }, + { + "name": "Ivan Socolsky", + "email": "jungans@gmail.com" + } + ], "bugs": { "url": "https://github.com/bitpay/insight-api/issues" }, @@ -47,7 +54,6 @@ "async": "*", "base58-native": "0.1.2", "bignum": "*", - "morgan": "*", "bitcore": "git://github.com/bitpay/bitcore.git#51c53b16ced6bcaf4b78329955d6814579fe4ee9", "bufferput": "git://github.com/bitpay/node-bufferput.git", "buffertools": "*", @@ -62,8 +68,9 @@ "microtime": "^0.6.0", "mkdirp": "^0.5.0", "moment": "~2.5.0", + "morgan": "*", "preconditions": "^1.0.7", - "request": "^2.48.0", + "request": "^2.69.0", "socket.io": "1.0.6", "socket.io-client": "1.0.6", "soop": "=0.1.5", diff --git a/test/test.CredentialStore.js b/test/test.CredentialStore.js deleted file mode 100644 index 608453d20..000000000 --- a/test/test.CredentialStore.js +++ /dev/null @@ -1,101 +0,0 @@ -'use strict'; - -var chai = require('chai'); -var assert = require('assert'); -var sinon = require('sinon'); -var should = chai.should; -var expect = chai.expect; - -describe('credentialstore test', function() { - - var globalConfig = require('../config/config'); - var leveldb_stub = sinon.stub(); - leveldb_stub.post = sinon.stub(); - leveldb_stub.get = sinon.stub(); - var plugin = require('../plugins/credentialstore'); - var express_mock = null; - var request = null; - var response = null; - - beforeEach(function() { - - express_mock = sinon.stub(); - express_mock.post = sinon.stub(); - express_mock.get = sinon.stub(); - - plugin.init(express_mock, {db: leveldb_stub}); - - request = sinon.stub(); - request.on = sinon.stub(); - request.param = sinon.stub(); - response = sinon.stub(); - response.send = sinon.stub(); - response.status = sinon.stub(); - response.json = sinon.stub(); - response.end = sinon.stub(); - }); - - it('initializes correctly', function() { - assert(plugin.db === leveldb_stub); - assert(express_mock.post.calledWith( - globalConfig.apiPrefix + '/credentials', plugin.post - )); - assert(express_mock.get.calledWith( - globalConfig.apiPrefix + '/credentials/:username', plugin.get - )); - }); - - it('writes a message correctly', function() { - - var data = 'username=1&secret=2&record=3'; - request.on.onFirstCall().callsArgWith(1, data); - request.on.onFirstCall().returnsThis(); - request.on.onSecondCall().callsArg(1); - leveldb_stub.put = sinon.stub(); - - leveldb_stub.put.onFirstCall().callsArg(2); - response.json.returnsThis(); - - plugin.post(request, response); - - assert(leveldb_stub.put.firstCall.args[0] === 'credentials-store-12'); - assert(leveldb_stub.put.firstCall.args[1] === '3'); - assert(response.json.calledWith({success: true})); - }); - - it('retrieves a message correctly', function() { - - request.param.onFirstCall().returns('username'); - request.param.onSecondCall().returns('secret'); - - var returnValue = '!@#$%'; - leveldb_stub.get.onFirstCall().callsArgWith(1, null, returnValue); - response.send.returnsThis(); - - plugin.get(request, response); - - assert(leveldb_stub.get.firstCall.args[0] === 'credentials-store-usernamesecret'); - assert(response.send.calledWith(returnValue)); - assert(response.end.calledOnce); - }); - - it('fails with messages that are too long', function() { - - response.writeHead = sinon.stub(); - request.connection = {}; - request.connection.destroy = sinon.stub(); - var data = 'blob'; - for (var i = 0; i < 2048; i++) { - data += '----'; - } - request.on.onFirstCall().callsArgWith(1, data); - request.on.onFirstCall().returnsThis(); - response.writeHead.returnsThis(); - - plugin.post(request, response); - - assert(response.writeHead.calledWith(413, {'Content-Type': 'text/plain'})); - assert(response.end.calledOnce); - assert(request.connection.destroy.calledOnce); - }); -}); diff --git a/test/test.PublicProfile.js b/test/test.PublicProfile.js deleted file mode 100644 index 5444b2e7a..000000000 --- a/test/test.PublicProfile.js +++ /dev/null @@ -1,113 +0,0 @@ -'use strict'; - -var chai = require('chai'); -var assert = require('assert'); -var sinon = require('sinon'); -var should = chai.should; -var expect = chai.expect; -var bitauth = require('bitauth'); - -describe('public profile test', function() { - - var globalConfig = require('../config/config'); - var leveldb_stub = sinon.stub(); - leveldb_stub.put = sinon.stub(); - leveldb_stub.get = sinon.stub(); - var plugin = require('../plugins/publicInfo/publicInfo.js'); - var express_mock = null; - var request = null; - var response = null; - - beforeEach(function() { - - express_mock = sinon.stub(); - express_mock.post = sinon.stub(); - express_mock.get = sinon.stub(); - - plugin.init(express_mock, {db: leveldb_stub}); - - request = sinon.stub(); - request.on = sinon.stub(); - request.param = sinon.stub(); - response = sinon.stub(); - response.send = sinon.stub(); - response.status = sinon.stub(); - response.json = sinon.stub(); - response.end = sinon.stub(); - }); - - it('initializes correctly', function() { - assert(plugin.db === leveldb_stub); - assert(express_mock.post.calledWith( - globalConfig.apiPrefix + '/public', plugin.post - )); - assert(express_mock.get.calledWith( - globalConfig.apiPrefix + '/public/:sin', plugin.get - )); - }); - - it('writes a message correctly', function(done) { - - var privateKey = bitauth.generateSin(); - var protocol = 'https'; - var dataToSign = protocol + '://hosturlSTUFF'; - var signature = bitauth.sign(dataToSign, privateKey.priv); - request.get = function() { return 'host'; }; - request.protocol = protocol; - request.url = 'url'; - request.headers = { - 'x-identity': privateKey.pub, - 'x-signature': signature - }; - request.on.onFirstCall().callsArgWith(1, 'STUFF'); - request.on.onFirstCall().returnsThis(); - request.on.onSecondCall().callsArg(1); - - leveldb_stub.put.onFirstCall().callsArg(2); - response.status.returns(response); - response.json.returns(response); - - request.testCallback = function() { - assert(leveldb_stub.put.firstCall.args[0] === 'public-info-' + privateKey.sin); - assert(leveldb_stub.put.firstCall.args[1] === 'STUFF'); - assert(response.json.calledOnce); - assert(response.end.calledOnce); - done(); - }; - - plugin.post(request, response); - }); - - it('fails if the signature is invalid', function() { - var data = 'uecord3'; - request.get = function() { return ''; }; - request.headers = {}; - request.on.onFirstCall().callsArgWith(1, data); - request.on.onFirstCall().returnsThis(); - request.on.onSecondCall().callsArg(1); - leveldb_stub.put = sinon.stub(); - - leveldb_stub.put.onFirstCall().callsArg(2); - response.json.returnsThis(); - response.status.returnsThis(); - - plugin.post(request, response); - - assert(response.end.calledOnce); - }); - - it('retrieves a message correctly', function() { - - request.param.onFirstCall().returns('SIN'); - - var returnValue = '!@#$%'; - leveldb_stub.get.onFirstCall().callsArgWith(1, null, returnValue); - response.send.returnsThis(); - - plugin.get(request, response); - - assert(leveldb_stub.get.firstCall.args[0] === 'public-info-SIN'); - assert(response.send.calledWith(returnValue)); - assert(response.end.calledOnce); - }); -});