From 49f70ca28ae9c0345612782123bd9aedc1f3871d Mon Sep 17 00:00:00 2001 From: softsimon Date: Sun, 18 Oct 2020 21:47:47 +0700 Subject: [PATCH] Slow down retries on Electrs errors in the main mempool loop. --- backend/src/api/bitcoin/electrs-api.ts | 16 +-- backend/src/api/blocks.ts | 129 ++++++++++--------- backend/src/api/mempool.ts | 164 ++++++++++++------------- backend/src/index.ts | 17 ++- 4 files changed, 161 insertions(+), 165 deletions(-) diff --git a/backend/src/api/bitcoin/electrs-api.ts b/backend/src/api/bitcoin/electrs-api.ts index 87ae5cc7c..87d07ffa2 100644 --- a/backend/src/api/bitcoin/electrs-api.ts +++ b/backend/src/api/bitcoin/electrs-api.ts @@ -11,7 +11,7 @@ class ElectrsApi { return new Promise((resolve, reject) => { request(config.ELECTRS_API_URL + '/mempool', { json: true, timeout: 10000 }, (err, res, response) => { if (err) { - reject(err); + reject('getMempoolInfo error: ' + err.message || err); } else if (res.statusCode !== 200) { reject(response); } else { @@ -32,7 +32,7 @@ class ElectrsApi { return new Promise((resolve, reject) => { request(config.ELECTRS_API_URL + '/mempool/txids', { json: true, timeout: 10000, forever: true }, (err, res, response) => { if (err) { - reject(err); + reject('getRawMempool error: ' + err.message || err); } else if (res.statusCode !== 200) { reject(response); } else { @@ -50,7 +50,7 @@ class ElectrsApi { return new Promise((resolve, reject) => { request(config.ELECTRS_API_URL + '/tx/' + txId, { json: true, timeout: 10000, forever: true }, (err, res, response) => { if (err) { - reject(err); + reject('getRawTransaction error: ' + err.message || err); } else if (res.statusCode !== 200) { reject(response); } else { @@ -68,7 +68,7 @@ class ElectrsApi { return new Promise((resolve, reject) => { request(config.ELECTRS_API_URL + '/blocks/tip/height', { json: true, timeout: 10000 }, (err, res, response) => { if (err) { - reject(err); + reject('getBlockHeightTip error: ' + err.message || err); } else if (res.statusCode !== 200) { reject(response); } else { @@ -82,7 +82,7 @@ class ElectrsApi { return new Promise((resolve, reject) => { request(config.ELECTRS_API_URL + '/block/' + hash + '/txids', { json: true, timeout: 10000 }, (err, res, response) => { if (err) { - reject(err); + reject('getTxIdsForBlock error: ' + err.message || err); } else if (res.statusCode !== 200) { reject(response); } else { @@ -100,7 +100,7 @@ class ElectrsApi { return new Promise((resolve, reject) => { request(config.ELECTRS_API_URL + '/block-height/' + height, { json: true, timeout: 10000 }, (err, res, response) => { if (err) { - reject(err); + reject('getBlockHash error: ' + err.message || err); } else if (res.statusCode !== 200) { reject(response); } else { @@ -114,7 +114,7 @@ class ElectrsApi { return new Promise((resolve, reject) => { request(config.ELECTRS_API_URL + '/blocks/' + height, { json: true, timeout: 10000 }, (err, res, response) => { if (err) { - reject(err); + reject('getBlocksFromHeight error: ' + err.message || err); } else if (res.statusCode !== 200) { reject(response); } else { @@ -128,7 +128,7 @@ class ElectrsApi { return new Promise((resolve, reject) => { request(config.ELECTRS_API_URL + '/block/' + hash, { json: true, timeout: 10000 }, (err, res, response) => { if (err) { - reject(err); + reject('getBlock error: ' + err.message || err); } else if (res.statusCode !== 200) { reject(response); } else { diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index cd34cbdde..b12b472d9 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -25,84 +25,79 @@ class Blocks { this.newBlockCallbacks.push(fn); } - public async updateBlocks() { - try { - const blockHeightTip = await bitcoinApi.getBlockHeightTip(); + public async $updateBlocks() { + const blockHeightTip = await bitcoinApi.getBlockHeightTip(); - if (this.blocks.length === 0) { - this.currentBlockHeight = blockHeightTip - config.INITIAL_BLOCK_AMOUNT; + if (this.blocks.length === 0) { + this.currentBlockHeight = blockHeightTip - config.INITIAL_BLOCK_AMOUNT; + } else { + this.currentBlockHeight = this.blocks[this.blocks.length - 1].height; + } + + if (blockHeightTip - this.currentBlockHeight > config.INITIAL_BLOCK_AMOUNT * 2) { + logger.info(`${blockHeightTip - this.currentBlockHeight} blocks since tip. Fast forwarding to the ${config.INITIAL_BLOCK_AMOUNT} recent blocks`); + this.currentBlockHeight = blockHeightTip - config.INITIAL_BLOCK_AMOUNT; + } + + if (!this.lastDifficultyAdjustmentTime) { + const heightDiff = blockHeightTip % 2016; + const blockHash = await bitcoinApi.getBlockHash(blockHeightTip - heightDiff); + const block = await bitcoinApi.getBlock(blockHash); + this.lastDifficultyAdjustmentTime = block.timestamp; + } + + while (this.currentBlockHeight < blockHeightTip) { + if (this.currentBlockHeight === 0) { + this.currentBlockHeight = blockHeightTip; } else { - this.currentBlockHeight = this.blocks[this.blocks.length - 1].height; + this.currentBlockHeight++; + logger.debug(`New block found (#${this.currentBlockHeight})!`); } - if (blockHeightTip - this.currentBlockHeight > config.INITIAL_BLOCK_AMOUNT * 2) { - logger.info(`${blockHeightTip - this.currentBlockHeight} blocks since tip. Fast forwarding to the ${config.INITIAL_BLOCK_AMOUNT} recent blocks`); - this.currentBlockHeight = blockHeightTip - config.INITIAL_BLOCK_AMOUNT; + const blockHash = await bitcoinApi.getBlockHash(this.currentBlockHeight); + const block = await bitcoinApi.getBlock(blockHash); + const txIds = await bitcoinApi.getTxIdsForBlock(blockHash); + + const mempool = memPool.getMempool(); + let found = 0; + let notFound = 0; + + const transactions: TransactionExtended[] = []; + + for (let i = 0; i < txIds.length; i++) { + if (mempool[txIds[i]]) { + transactions.push(mempool[txIds[i]]); + found++; + } else { + logger.debug(`Fetching block tx ${i} of ${txIds.length}`); + const tx = await memPool.getTransactionExtended(txIds[i]); + if (tx) { + transactions.push(tx); + } + notFound++; + } } - if (!this.lastDifficultyAdjustmentTime) { - const heightDiff = blockHeightTip % 2016; - const blockHash = await bitcoinApi.getBlockHash(blockHeightTip - heightDiff); - const block = await bitcoinApi.getBlock(blockHash); + logger.debug(`${found} of ${txIds.length} found in mempool. ${notFound} not found.`); + + block.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0); + block.coinbaseTx = this.stripCoinbaseTransaction(transactions[0]); + transactions.sort((a, b) => b.feePerVsize - a.feePerVsize); + block.medianFee = transactions.length > 1 ? Common.median(transactions.map((tx) => tx.feePerVsize)) : 0; + block.feeRange = transactions.length > 1 ? Common.getFeesInRange(transactions.slice(0, transactions.length - 1), 8) : [0, 0]; + + if (block.height % 2016 === 0) { this.lastDifficultyAdjustmentTime = block.timestamp; } - while (this.currentBlockHeight < blockHeightTip) { - if (this.currentBlockHeight === 0) { - this.currentBlockHeight = blockHeightTip; - } else { - this.currentBlockHeight++; - logger.debug(`New block found (#${this.currentBlockHeight})!`); - } - - const blockHash = await bitcoinApi.getBlockHash(this.currentBlockHeight); - const block = await bitcoinApi.getBlock(blockHash); - const txIds = await bitcoinApi.getTxIdsForBlock(blockHash); - - const mempool = memPool.getMempool(); - let found = 0; - let notFound = 0; - - const transactions: TransactionExtended[] = []; - - for (let i = 0; i < txIds.length; i++) { - if (mempool[txIds[i]]) { - transactions.push(mempool[txIds[i]]); - found++; - } else { - logger.debug(`Fetching block tx ${i} of ${txIds.length}`); - const tx = await memPool.getTransactionExtended(txIds[i]); - if (tx) { - transactions.push(tx); - } - notFound++; - } - } - - logger.debug(`${found} of ${txIds.length} found in mempool. ${notFound} not found.`); - - block.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0); - block.coinbaseTx = this.stripCoinbaseTransaction(transactions[0]); - transactions.sort((a, b) => b.feePerVsize - a.feePerVsize); - block.medianFee = transactions.length > 1 ? Common.median(transactions.map((tx) => tx.feePerVsize)) : 0; - block.feeRange = transactions.length > 1 ? Common.getFeesInRange(transactions.slice(0, transactions.length - 1), 8) : [0, 0]; - - if (block.height % 2016 === 0) { - this.lastDifficultyAdjustmentTime = block.timestamp; - } - - this.blocks.push(block); - if (this.blocks.length > config.KEEP_BLOCK_AMOUNT) { - this.blocks.shift(); - } - - if (this.newBlockCallbacks.length) { - this.newBlockCallbacks.forEach((cb) => cb(block, txIds, transactions)); - } + this.blocks.push(block); + if (this.blocks.length > config.KEEP_BLOCK_AMOUNT) { + this.blocks.shift(); } - } catch (err) { - logger.err('updateBlocks error' + err); + if (this.newBlockCallbacks.length) { + this.newBlockCallbacks.forEach((cb) => cb(block, txIds, transactions)); + } } } diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index 915fdb783..7a6c77e10 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -47,12 +47,8 @@ class Mempool { } } - public async updateMemPoolInfo() { - try { - this.mempoolInfo = await bitcoinApi.getMempoolInfo(); - } catch (e) { - logger.err('Error getMempoolInfo ' + e.message || e); - } + public async $updateMemPoolInfo() { + this.mempoolInfo = await bitcoinApi.getMempoolInfo(); } public getMempoolInfo(): MempoolInfo | undefined { @@ -93,100 +89,96 @@ class Mempool { } } - public async updateMempool() { + public async $updateMempool() { logger.debug('Updating mempool'); const start = new Date().getTime(); let hasChange: boolean = false; const currentMempoolSize = Object.keys(this.mempoolCache).length; let txCount = 0; - try { - const transactions = await bitcoinApi.getRawMempool(); - const diff = transactions.length - currentMempoolSize; - const newTransactions: TransactionExtended[] = []; + const transactions = await bitcoinApi.getRawMempool(); + const diff = transactions.length - currentMempoolSize; + const newTransactions: TransactionExtended[] = []; - for (const txid of transactions) { - if (!this.mempoolCache[txid]) { - const transaction = await this.getTransactionExtended(txid); - if (transaction) { - this.mempoolCache[txid] = transaction; - txCount++; - if (this.inSync) { - this.txPerSecondArray.push(new Date().getTime()); - this.vBytesPerSecondArray.push({ - unixTime: new Date().getTime(), - vSize: transaction.vsize, - }); - } - hasChange = true; - if (diff > 0) { - logger.debug('Fetched transaction ' + txCount + ' / ' + diff); - } else { - logger.debug('Fetched transaction ' + txCount); - } - newTransactions.push(transaction); - } else { - logger.debug('Error finding transaction in mempool.'); + for (const txid of transactions) { + if (!this.mempoolCache[txid]) { + const transaction = await this.getTransactionExtended(txid); + if (transaction) { + this.mempoolCache[txid] = transaction; + txCount++; + if (this.inSync) { + this.txPerSecondArray.push(new Date().getTime()); + this.vBytesPerSecondArray.push({ + unixTime: new Date().getTime(), + vSize: transaction.vsize, + }); } - } - - if ((new Date().getTime()) - start > config.MEMPOOL_REFRESH_RATE_MS * 10) { - break; - } - } - - // Prevent mempool from clear on bitcoind restart by delaying the deletion - if ((config.NETWORK === 'mainnet' || !config.NETWORK) - && this.mempoolProtection === 0 && transactions.length / currentMempoolSize <= 0.80) { - this.mempoolProtection = 1; - this.inSync = false; - logger.warn(`Mempool clear protection triggered because transactions.length: ${transactions.length} and currentMempoolSize: ${currentMempoolSize}.`); - setTimeout(() => { - this.mempoolProtection = 2; - logger.warn('Mempool clear protection resumed.'); - }, 1000 * 60 * 2); - } - - let newMempool = {}; - const deletedTransactions: TransactionExtended[] = []; - - if (this.mempoolProtection !== 1) { - this.mempoolProtection = 0; - // Index object for faster search - const transactionsObject = {}; - transactions.forEach((txId) => transactionsObject[txId] = true); - - // Replace mempool to separate deleted transactions - for (const tx in this.mempoolCache) { - if (transactionsObject[tx]) { - newMempool[tx] = this.mempoolCache[tx]; + hasChange = true; + if (diff > 0) { + logger.debug('Fetched transaction ' + txCount + ' / ' + diff); } else { - deletedTransactions.push(this.mempoolCache[tx]); + logger.debug('Fetched transaction ' + txCount); } + newTransactions.push(transaction); + } else { + logger.debug('Error finding transaction in mempool.'); } - } else { - newMempool = this.mempoolCache; } - const newTransactionsStripped = newTransactions.map((tx) => Common.stripTransaction(tx)); - this.latestTransactions = newTransactionsStripped.concat(this.latestTransactions).slice(0, 6); - - if (!this.inSync && transactions.length === Object.keys(newMempool).length) { - this.inSync = true; - logger.info('The mempool is now in sync!'); + if ((new Date().getTime()) - start > config.MEMPOOL_REFRESH_RATE_MS * 10) { + break; } - - if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) { - this.mempoolCache = newMempool; - this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions); - } - - const end = new Date().getTime(); - const time = end - start; - logger.debug(`New mempool size: ${Object.keys(newMempool).length} Change: ${diff}`); - logger.debug('Mempool updated in ' + time / 1000 + ' seconds'); - } catch (err) { - logger.err('getRawMempool error. ' + err.message || err); } + + // Prevent mempool from clear on bitcoind restart by delaying the deletion + if ((config.NETWORK === 'mainnet' || !config.NETWORK) + && this.mempoolProtection === 0 && transactions.length / currentMempoolSize <= 0.80) { + this.mempoolProtection = 1; + this.inSync = false; + logger.warn(`Mempool clear protection triggered because transactions.length: ${transactions.length} and currentMempoolSize: ${currentMempoolSize}.`); + setTimeout(() => { + this.mempoolProtection = 2; + logger.warn('Mempool clear protection resumed.'); + }, 1000 * 60 * 2); + } + + let newMempool = {}; + const deletedTransactions: TransactionExtended[] = []; + + if (this.mempoolProtection !== 1) { + this.mempoolProtection = 0; + // Index object for faster search + const transactionsObject = {}; + transactions.forEach((txId) => transactionsObject[txId] = true); + + // Replace mempool to separate deleted transactions + for (const tx in this.mempoolCache) { + if (transactionsObject[tx]) { + newMempool[tx] = this.mempoolCache[tx]; + } else { + deletedTransactions.push(this.mempoolCache[tx]); + } + } + } else { + newMempool = this.mempoolCache; + } + + const newTransactionsStripped = newTransactions.map((tx) => Common.stripTransaction(tx)); + this.latestTransactions = newTransactionsStripped.concat(this.latestTransactions).slice(0, 6); + + if (!this.inSync && transactions.length === Object.keys(newMempool).length) { + this.inSync = true; + logger.info('The mempool is now in sync!'); + } + + if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) { + this.mempoolCache = newMempool; + this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions); + } + + const end = new Date().getTime(); + const time = end - start; + logger.debug(`New mempool size: ${Object.keys(newMempool).length} Change: ${diff}`); + logger.debug('Mempool updated in ' + time / 1000 + ' seconds'); } private updateTxPerSecond() { diff --git a/backend/src/index.ts b/backend/src/index.ts index ffa8944da..c66a80a01 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -27,6 +27,7 @@ class Server { private wss: WebSocket.Server | undefined; private server: https.Server | http.Server | undefined; private app: Express; + private retryOnElectrsErrorAfterSeconds = 5; constructor() { this.app = express(); @@ -114,10 +115,18 @@ class Server { } async runMempoolIntervalFunctions() { - await memPool.updateMemPoolInfo(); - await blocks.updateBlocks(); - await memPool.updateMempool(); - setTimeout(this.runMempoolIntervalFunctions.bind(this), config.ELECTRS_POLL_RATE_MS); + try { + await memPool.$updateMemPoolInfo(); + await blocks.$updateBlocks(); + await memPool.$updateMempool(); + setTimeout(this.runMempoolIntervalFunctions.bind(this), config.ELECTRS_POLL_RATE_MS); + this.retryOnElectrsErrorAfterSeconds = 5; + } catch (e) { + this.retryOnElectrsErrorAfterSeconds *= 2; + this.retryOnElectrsErrorAfterSeconds = Math.min(this.retryOnElectrsErrorAfterSeconds, 3600); + logger.warn(`runMempoolIntervalFunctions error: ${(e.message || e)}. Retrying in ${this.retryOnElectrsErrorAfterSeconds} sec.`); + setTimeout(this.runMempoolIntervalFunctions.bind(this), 1000 * this.retryOnElectrsErrorAfterSeconds); + } } setUpWebsocketHandling() {