Graceful degradation for pruned and txindex-less nodes

.env-sample

@@ -44,6 +44,9 @@
 # used if BTCEXP_ADDRESS_API=electrumx
 #BTCEXP_ELECTRUMX_SERVERS=tls://electrumx.server.com:50002,tcp://127.0.0.1:50001,...
 
+# Use the Electrumx server as an external txindex. This is only available in Electrs.
+#BTCEXP_ELECTRUM_TXINDEX=true
+
 # Set number of concurrent RPC requests. Should be lower than your node's "rpcworkqueue" value.
 # Note that Bitcoin Core's default rpcworkqueue=16.
 # Default: 10
@@ -110,6 +113,9 @@
 # Default: 1024
 #BTCEXP_OLD_SPACE_MAX_SIZE=2048
 
+# The number of recent blocks to search for transactions when txindex is disabled
+#BTCEXP_NOTXINDEX_SEARCH_DEPTH=3
+
 # Show tools list in a sub-nav at top of screen
 # Default: true
 BTCEXP_UI_SHOW_TOOLS_SUBHEADER=true

app.js

@@ -193,29 +193,29 @@ function verifyRpcConnection() {
 	if (!global.activeBlockchain) {
 		debugLog(`Verifying RPC connection...`);
 
-		coreApi.getNetworkInfo().then(function(getnetworkinfo) {
-			coreApi.getBlockchainInfo().then(function(getblockchaininfo) {
-				global.activeBlockchain = getblockchaininfo.chain;
+		Promise.all([
+			coreApi.getNetworkInfo(),
+			coreApi.getBlockchainInfo(),
+			coreApi.getIndexInfo(),
+		]).then(([ getnetworkinfo, getblockchaininfo, getindexinfo ]) => {
+			global.activeBlockchain = getblockchaininfo.chain;
 
-				// we've verified rpc connection, no need to keep trying
-				clearInterval(global.verifyRpcConnectionIntervalId);
+			// we've verified rpc connection, no need to keep trying
+			clearInterval(global.verifyRpcConnectionIntervalId);
 
-				onRpcConnectionVerified(getnetworkinfo, getblockchaininfo);
-
-			}).catch(function(err) {
-				utils.logError("329u0wsdgewg6ed", err);
-			});
+			onRpcConnectionVerified(getnetworkinfo, getblockchaininfo, getindexinfo);
 		}).catch(function(err) {
 			utils.logError("32ugegdfsde", err);
 		});
 	}
 }
 
-function onRpcConnectionVerified(getnetworkinfo, getblockchaininfo) {
+function onRpcConnectionVerified(getnetworkinfo, getblockchaininfo, getindexinfo) {
 	// localservicenames introduced in 0.19
 	var services = getnetworkinfo.localservicesnames ? ("[" + getnetworkinfo.localservicesnames.join(", ") + "]") : getnetworkinfo.localservices;
 
 	global.getnetworkinfo = getnetworkinfo;
+	global.getindexinfo = getindexinfo;
 
 	var bitcoinCoreVersionRegex = /^.*\/Satoshi\:(.*)\/.*$/;
 

app/api/coreApi.js

@@ -228,6 +228,10 @@ function getMempoolInfo() {
 	return tryCacheThenRpcApi(miscCache, "getMempoolInfo", 5 * ONE_SEC, rpcApi.getMempoolInfo);
 }
 
+function getIndexInfo() {
+	return tryCacheThenRpcApi(miscCache, "getIndexInfo", 10 * ONE_SEC, rpcApi.getIndexInfo);
+}
+
 function getMempoolTxids() {
 	// no caching, that would be dumb
 	return rpcApi.getMempoolTxids();
@@ -544,6 +548,12 @@ function getBlockByHeight(blockHeight) {
 	});
 }
 
+function getBlockHashByHeight(blockHeight) {
+	return tryCacheThenRpcApi(blockCache, "getBlockHashByHeight-" + blockHeight, ONE_HR, function() {
+		return rpcApi.getBlockHashByHeight(blockHeight);
+	});
+}
+
 function getBlocksByHeight(blockHeights) {
 	return new Promise(function(resolve, reject) {
 		var promises = [];
@@ -626,9 +636,9 @@ function getBlocksByHash(blockHashes) {
 	});
 }
 
-function getRawTransaction(txid) {
+function getRawTransaction(txid, blockhash) {
 	var rpcApiFunction = function() {
-		return rpcApi.getRawTransaction(txid);
+		return rpcApi.getRawTransaction(txid, blockhash);
 	};
 
 	return tryCacheThenRpcApi(txCache, "getRawTransaction-" + txid, ONE_HR, rpcApiFunction, shouldCacheTransaction);
@@ -720,11 +730,11 @@ function getAddress(address) {
 	});
 }
 
-function getRawTransactions(txids) {
+function getRawTransactions(txids, blockhash) {
 	return new Promise(function(resolve, reject) {
 		var promises = [];
 		for (var i = 0; i < txids.length; i++) {
-			promises.push(getRawTransaction(txids[i]));
+			promises.push(getRawTransaction(txids[i], blockhash));
 		}
 
 		Promise.all(promises).then(function(results) {
@@ -828,9 +838,15 @@ function summarizeBlockAnalysisData(blockHeight, tx, inputs) {
 	return txSummary;
 }
 
-function getRawTransactionsWithInputs(txids, maxInputs=-1) {
+function getRawTransactionsWithInputs(txids, maxInputs=-1, blockhash) {
+	// Get just the transactions without their prevouts when txindex is disabled
+	if (global.getindexinfo && !global.getindexinfo.txindex) {
+		return getRawTransactions(txids, blockhash)
+			.then(transactions => ({ transactions, txInputsByTransaction: {} }))
+	}
+
 	return new Promise(function(resolve, reject) {
-		getRawTransactions(txids).then(function(transactions) {
+		getRawTransactions(txids, blockhash).then(function(transactions) {
 			var maxInputsTracked = config.site.txMaxInput;
 
 			if (maxInputs <= 0) {
@@ -885,10 +901,8 @@ function getRawTransactionsWithInputs(txids, maxInputs=-1) {
 				});
 
 				resolve({ transactions:transactions, txInputsByTransaction:txInputsByTransaction });
-			});
-		}).catch(function(err) {
-			reject(err);
-		});
+			}).catch(reject);
+		}).catch(reject);
 	});
 }
 
@@ -905,7 +919,7 @@ function getBlockByHashWithTransactions(blockHash, txLimit, txOffset) {
 				txids.push(block.tx[i]);
 			}
 
-			getRawTransactionsWithInputs(txids, config.site.txMaxInput).then(function(txsResult) {
+			getRawTransactionsWithInputs(txids, config.site.txMaxInput, blockHash).then(function(txsResult) {
 				if (txsResult.transactions && txsResult.transactions.length > 0) {
 					block.coinbaseTx = txsResult.transactions[0];
 					block.totalFees = utils.getBlockTotalFeesFromCoinbaseTxAndBlockHeight(block.coinbaseTx, block.height);
@@ -918,13 +932,18 @@ function getBlockByHashWithTransactions(blockHash, txLimit, txOffset) {
 				}
 
 				resolve({ getblock:block, transactions:txsResult.transactions, txInputsByTransaction:txsResult.txInputsByTransaction });
+			}).catch(function(err) {
+				// likely due to pruning or no txindex, report the error but continue with an empty transaction list
+				resolve({ getblock:block, transactions:[], txInputsByTransaction:{} });
 			});
-		}).catch(function(err) {
-			reject(err);
-		});
+		}).catch(reject);
 	});
 }
 
+function getTxOut(txid, vout) {
+	return rpcApi.getTxOut(txid, vout)
+}
+
 function getHelp() {
 	return new Promise(function(resolve, reject) {
 		tryCacheThenRpcApi(miscCache, "getHelp", ONE_DAY, rpcApi.getHelp).then(function(helpContent) {
@@ -1056,7 +1075,9 @@ module.exports = {
 	getMempoolInfo: getMempoolInfo,
 	getMempoolTxids: getMempoolTxids,
 	getMiningInfo: getMiningInfo,
+	getIndexInfo: getIndexInfo,
 	getBlockByHeight: getBlockByHeight,
+	getBlockHashByHeight: getBlockHashByHeight,
 	getBlocksByHeight: getBlocksByHeight,
 	getBlockByHash: getBlockByHash,
 	getBlocksByHash: getBlocksByHash,
@@ -1084,5 +1105,6 @@ module.exports = {
 	getBlocksStatsByHeight: getBlocksStatsByHeight,
 	buildBlockAnalysisData: buildBlockAnalysisData,
 	getBlockHeaderByHeight: getBlockHeaderByHeight,
-	getBlockHeadersByHeight: getBlockHeadersByHeight
-};
+	getBlockHeadersByHeight: getBlockHeadersByHeight,
+	getTxOut: getTxOut
+};

app/api/electrumAddressApi.js

@@ -323,6 +323,25 @@ function getAddressBalance(addrScripthash) {
 	});
 }
 
+// Lookup the confirming block hash of a given txid. This only works with Electrs.
+// https://github.com/romanz/electrs/commit/a0a3d4f9392e21f9e92fdc274c88fed6d0634794
+function lookupTxBlockHash(txid) {
+	if (electrumClients.length == 0) {
+		return Promise.reject({ error: "No ElectrumX Connection", userText: noConnectionsErrorText });
+	}
+
+	return runOnAllServers(function(electrumClient) {
+		return electrumClient.request('blockchain.transaction.get_confirmed_blockhash', [txid]);
+	}).then(function(results) {
+		var blockhash = results[0].result;
+		if (results.slice(1).every(({ result }) => result == blockhash)) {
+			return blockhash;
+		} else {
+			return Promise.reject({conflictedResults:results});
+		}
+	});
+}
+
 function logStats(cmd, dt, success) {
 	if (!global.electrumStats.rpc[cmd]) {
 		global.electrumStats.rpc[cmd] = {count:0, time:0, successes:0, failures:0};
@@ -341,6 +360,7 @@ function logStats(cmd, dt, success) {
 
 module.exports = {
 	connectToServers: connectToServers,
-	getAddressDetails: getAddressDetails
+	getAddressDetails: getAddressDetails,
+	lookupTxBlockHash: lookupTxBlockHash,
 };
 

app/api/rpcApi.js

@@ -50,6 +50,10 @@ function getMiningInfo() {
 	return getRpcData("getmininginfo");
 }
 
+function getIndexInfo() {
+	return getRpcData("getindexinfo");
+}
+
 function getUptimeSeconds() {
 	return getRpcData("uptime");
 }
@@ -189,34 +193,38 @@ function getBlockHeaderByHeight(blockHeight) {
 	});
 }
 
+function getBlockHashByHeight(blockHeight) {
+	return getRpcDataWithParams({method:"getblockhash", parameters:[blockHeight]});
+}
+
 function getBlockByHash(blockHash) {
 	debugLog("getBlockByHash: %s", blockHash);
 
-	return new Promise(function(resolve, reject) {
-		getRpcDataWithParams({method:"getblock", parameters:[blockHash]}).then(function(block) {
-			getRawTransaction(block.tx[0]).then(function(tx) {
+	return getRpcDataWithParams({method:"getblock", parameters:[blockHash]})
+		.then(function(block) {
+			return getRawTransaction(block.tx[0], blockHash).then(function(tx) {
 				block.coinbaseTx = tx;
 				block.totalFees = utils.getBlockTotalFeesFromCoinbaseTxAndBlockHeight(tx, block.height);
-				block.subsidy = coinConfig.blockRewardFunction(block.height, global.activeBlockchain);
 				block.miner = utils.getMinerFromCoinbaseTx(tx);
-
-				resolve(block);
-
-			}).catch(function(err) {
-				reject(err);
-			});
+				return block;
+			})
 		}).catch(function(err) {
-			reject(err);
-		});
-	});
+				// the block is pruned, use `getblockheader` instead
+				debugLog('getblock failed, falling back to getblockheader', blockHash, err);
+				return getRpcDataWithParams({method:"getblockheader", parameters:[blockHash]})
+					.then(function(block) { block.tx = []; return block });
+		}).then(function(block) {
+				block.subsidy = coinConfig.blockRewardFunction(block.height, global.activeBlockchain);
+				return block;
+		})
 }
 
 function getAddress(address) {
 	return getRpcDataWithParams({method:"validateaddress", parameters:[address]});
 }
 
-function getRawTransaction(txid) {
-	debugLog("getRawTransaction: %s", txid);
+function getRawTransaction(txid, blockhash) {
+	debugLog("getRawTransaction: %s %s", txid, blockhash);
 
 	return new Promise(function(resolve, reject) {
 		if (coins[config.coin].genesisCoinbaseTransactionIdsByNetwork[global.activeBlockchain] && txid == coins[config.coin].genesisCoinbaseTransactionIdsByNetwork[global.activeBlockchain]) {
@@ -238,7 +246,8 @@ function getRawTransaction(txid) {
 			});
 
 		} else {
-			getRpcDataWithParams({method:"getrawtransaction", parameters:[txid, 1]}).then(function(result) {
+			var extra_params = blockhash ? [ blockhash ] : [];
+			getRpcDataWithParams({method:"getrawtransaction", parameters:[txid, 1, ...extra_params]}).then(function(result) {
 				if (result == null || result.code && result.code < 0) {
 					reject(result);
 
@@ -248,12 +257,59 @@ function getRawTransaction(txid) {
 				resolve(result);
 
 			}).catch(function(err) {
-				reject(err);
+				if (global.getindexinfo && !global.getindexinfo.txindex && !blockhash) {
+					noTxIndexTransactionLookup(txid).then(resolve, reject);
+				} else {
+					reject(err);
+				}
 			});
 		}
 	});
 }
 
+async function noTxIndexTransactionLookup(txid) {
+	// Try looking up with an external Electrum server, using 'get_confirmed_blockhash'.
+	// This is only available in Electrs and requires enabling BTCEXP_ELECTRUM_TXINDEX.
+	if (config.addressApi == "electrumx" && config.electrumTxIndex) {
+		try {
+			var blockhash = await electrumAddressApi.lookupTxBlockHash(txid);
+			return await getRawTransaction(txid, blockhash);
+		} catch (err) {
+			debugLog(`Electrs blockhash lookup failed for ${txid}:`, err);
+		}
+	}
+
+	// Try looking up in wallet transactions
+	for (var wallet of await listWallets()) {
+		try { return await getWalletTransaction(wallet, txid); }
+		catch (_) {}
+	}
+
+	// Try looking up in recent blocks
+	var tip_height = await getRpcDataWithParams({method:"getblockcount", parameters:[]});
+	for (var height=tip_height; height>Math.max(tip_height - config.noTxIndexSearchDepth, 0); height--) {
+		var blockhash = await getRpcDataWithParams({method:"getblockhash", parameters:[height]});
+		try { return await getRawTransaction(txid, blockhash); }
+		catch (_) {}
+	}
+
+	throw new Error(`The requested tx ${txid} cannot be found in wallet transactions, mempool transactions and recently confirmed transactions`)
+}
+
+function listWallets() {
+	return getRpcDataWithParams({method:"listwallets", parameters:[]})
+}
+
+async function getWalletTransaction(wallet, txid) {
+	global.rpcClient.wallet = wallet;
+	try {
+		return await getRpcDataWithParams({method:"gettransaction", parameters:[ txid, true, true ]})
+			.then(wtx => ({ ...wtx, ...wtx.decoded, decoded: null }))
+	} finally {
+		global.rpcClient.wallet = null;
+	}
+}
+
 function getUtxo(txid, outputIndex) {
 	debugLog("getUtxo: %s (%d)", txid, outputIndex);
 
@@ -331,6 +387,10 @@ function getMempoolTxDetails(txid, includeAncDec=true) {
 	});
 }
 
+function getTxOut(txid, vout) {
+	return getRpcDataWithParams({method:"gettxout", parameters:[txid, vout]});
+}
+
 function getHelp() {
 	return getRpcData("help");
 }
@@ -483,6 +543,7 @@ module.exports = {
 	getMempoolInfo: getMempoolInfo,
 	getMempoolTxids: getMempoolTxids,
 	getMiningInfo: getMiningInfo,
+	getIndexInfo: getIndexInfo,
 	getBlockByHeight: getBlockByHeight,
 	getBlockByHash: getBlockByHash,
 	getRawTransaction: getRawTransaction,
@@ -502,6 +563,8 @@ module.exports = {
 	getBlockStatsByHeight: getBlockStatsByHeight,
 	getBlockHeaderByHash: getBlockHeaderByHash,
 	getBlockHeaderByHeight: getBlockHeaderByHeight,
+	getBlockHashByHeight: getBlockHashByHeight,
+	getTxOut: getTxOut,
 
 	minRpcVersions: minRpcVersions
 };