From 3c0fa71a104c44ed9c8e2633d9316d2f390f62a7 Mon Sep 17 00:00:00 2001 From: softsimon Date: Mon, 28 Dec 2020 20:17:32 +0700 Subject: [PATCH] Updates for general transaction and block fetching. --- backend/src/api/bitcoin/bitcoin-api.ts | 27 +++++++++++++++++- backend/src/api/bitcoin/electrum-api.ts | 7 ++++- backend/src/api/blocks.ts | 37 +++++++++++-------------- backend/src/api/common.ts | 8 ++++++ backend/src/api/transaction-utils.ts | 36 ++++++++---------------- backend/src/routes.ts | 18 ++++++------ 6 files changed, 77 insertions(+), 56 deletions(-) diff --git a/backend/src/api/bitcoin/bitcoin-api.ts b/backend/src/api/bitcoin/bitcoin-api.ts index c5452a533..3b3c704c9 100644 --- a/backend/src/api/bitcoin/bitcoin-api.ts +++ b/backend/src/api/bitcoin/bitcoin-api.ts @@ -6,6 +6,7 @@ import { IEsploraApi } from './esplora-api.interface'; import blocks from '../blocks'; import bitcoinBaseApi from './bitcoin-base.api'; import mempool from '../mempool'; +import { TransactionExtended } from '../../mempool.interfaces'; class BitcoinApi implements AbstractBitcoinApi { private rawMempoolCache: IBitcoinApi.RawMempool | null = null; @@ -32,6 +33,11 @@ class BitcoinApi implements AbstractBitcoinApi { } $getRawTransaction(txId: string, skipConversion = false, addPrevout = false): Promise { + // If the transaction is in the mempool we also already fetched the fee, just prevouts are missing + const txInMempool = mempool.getMempool()[txId]; + if (txInMempool && addPrevout) { + return this.$addPrevouts(txInMempool); + } return this.bitcoindClient.getRawTransaction(txId, true) .then((transaction: IBitcoinApi.Transaction) => { if (skipConversion) { @@ -55,7 +61,12 @@ class BitcoinApi implements AbstractBitcoinApi { return this.bitcoindClient.getBlockHash(height); } - $getBlock(hash: string): Promise { + async $getBlock(hash: string): Promise { + const foundBlock = blocks.getBlocks().find((block) => block.id === hash); + if (foundBlock) { + return foundBlock; + } + return this.bitcoindClient.getBlock(hash) .then((block: IBitcoinApi.Block) => this.convertBlock(block)); } @@ -163,6 +174,9 @@ class BitcoinApi implements AbstractBitcoinApi { } private async $appendMempoolFeeData(transaction: IEsploraApi.Transaction): Promise { + if (transaction.fee) { + return transaction; + } let mempoolEntry: IBitcoinApi.MempoolEntry; if (!mempool.isInSync() && !this.rawMempoolCache) { this.rawMempoolCache = await bitcoinBaseApi.$getRawMempoolVerbose(); @@ -176,6 +190,17 @@ class BitcoinApi implements AbstractBitcoinApi { return transaction; } + protected async $addPrevouts(transaction: TransactionExtended): Promise { + for (const vin of transaction.vin) { + if (vin.prevout) { + continue; + } + const innerTx = await this.$getRawTransaction(vin.txid, false); + vin.prevout = innerTx.vout[vin.vout]; + } + return transaction; + } + private async $calculateFeeFromInputs(transaction: IEsploraApi.Transaction, addPrevout: boolean): Promise { if (transaction.vin[0].is_coinbase) { transaction.fee = 0; diff --git a/backend/src/api/bitcoin/electrum-api.ts b/backend/src/api/bitcoin/electrum-api.ts index 697db749e..68c8b1b41 100644 --- a/backend/src/api/bitcoin/electrum-api.ts +++ b/backend/src/api/bitcoin/electrum-api.ts @@ -8,6 +8,7 @@ import * as sha256 from 'crypto-js/sha256'; import * as hexEnc from 'crypto-js/enc-hex'; import BitcoinApi from './bitcoin-api'; import bitcoinBaseApi from './bitcoin-base.api'; +import mempool from '../mempool'; class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi { private electrumClient: any; @@ -27,6 +28,10 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi { } async $getRawTransaction(txId: string, skipConversion = false, addPrevout = false): Promise { + const txInMempool = mempool.getMempool()[txId]; + if (txInMempool && addPrevout) { + return this.$addPrevouts(txInMempool); + } const transaction: IBitcoinApi.Transaction = await this.electrumClient.blockchain_transaction_get(txId, true); if (!transaction) { throw new Error('Unable to get transaction: ' + txId); @@ -93,7 +98,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi { const history = await this.$getScriptHashHistory(addressInfo.scriptPubKey); const transactions: IEsploraApi.Transaction[] = []; for (const h of history) { - const tx = await this.$getRawTransaction(h.tx_hash); + const tx = await this.$getRawTransaction(h.tx_hash, false, true); if (tx) { transactions.push(tx); } diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 80c77b7c5..1c5774640 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -64,19 +64,28 @@ class Blocks { const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash); const mempool = memPool.getMempool(); - let found = 0; + let transactionsFound = 0; for (let i = 0; i < txIds.length; i++) { // When using bitcoind, just fetch the coinbase tx for now if (config.MEMPOOL.BACKEND !== 'none' && i === 0) { - const tx = await transactionUtils.$getTransactionExtended(txIds[i]); - if (tx) { - transactions.push(tx); + let txFound = false; + let findCoinbaseTxTries = 0; + // It takes Electrum Server a few seconds to index the transaction after a block is found + while (findCoinbaseTxTries < 5 && !txFound) { + const tx = await transactionUtils.$getTransactionExtended(txIds[i]); + if (tx) { + txFound = true; + transactions.push(tx); + } else { + await Common.sleep(1000); + findCoinbaseTxTries++; + } } } if (mempool[txIds[i]]) { transactions.push(mempool[txIds[i]]); - found++; + transactionsFound++; } else if (config.MEMPOOL.BACKEND === 'esplora') { logger.debug(`Fetching block tx ${i} of ${txIds.length}`); const tx = await transactionUtils.$getTransactionExtended(txIds[i]); @@ -86,11 +95,11 @@ class Blocks { } } - logger.debug(`${found} of ${txIds.length} found in mempool. ${txIds.length - found} not found.`); + logger.debug(`${transactionsFound} of ${txIds.length} found in mempool. ${txIds.length - transactionsFound} not found.`); const blockExtended: BlockExtended = Object.assign({}, block); blockExtended.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0); - blockExtended.coinbaseTx = this.stripCoinbaseTransaction(transactions[0]); + blockExtended.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]); transactions.sort((a, b) => b.feePerVsize - a.feePerVsize); blockExtended.medianFee = transactions.length > 1 ? Common.median(transactions.map((tx) => tx.feePerVsize)) : 0; blockExtended.feeRange = transactions.length > 1 ? Common.getFeesInRange(transactions.slice(0, transactions.length - 1), 8) : [0, 0]; @@ -118,20 +127,6 @@ class Blocks { public getCurrentBlockHeight(): number { return this.currentBlockHeight; } - - private stripCoinbaseTransaction(tx: TransactionExtended): TransactionMinerInfo { - return { - vin: [{ - scriptsig: tx.vin[0].scriptsig || tx.vin[0]['coinbase'] - }], - vout: tx.vout - .map((vout) => ({ - scriptpubkey_address: vout.scriptpubkey_address, - value: vout.value - })) - .filter((vout) => vout.value) - }; - } } export default new Blocks(); diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index 0bb90c6df..3d6fb7161 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -56,4 +56,12 @@ export class Common { value: tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0), }; } + + static sleep(ms: number): Promise { + return new Promise((resolve) => { + setTimeout(() => { + resolve(); + }, ms); + }); + } } diff --git a/backend/src/api/transaction-utils.ts b/backend/src/api/transaction-utils.ts index 9d0ff990c..d4ca8124b 100644 --- a/backend/src/api/transaction-utils.ts +++ b/backend/src/api/transaction-utils.ts @@ -6,27 +6,6 @@ import { IEsploraApi } from './bitcoin/esplora-api.interface'; class TransactionUtils { constructor() { } - public async $addPrevoutsToTransaction(transaction: TransactionExtended): Promise { - if (transaction.vin[0].is_coinbase) { - return transaction; - } - for (const vin of transaction.vin) { - const innerTx = await bitcoinApi.$getRawTransaction(vin.txid); - vin.prevout = innerTx.vout[vin.vout]; - } - return transaction; - } - - public extendTransaction(transaction: IEsploraApi.Transaction): TransactionExtended { - transaction['vsize'] = Math.round(transaction.weight / 4); - transaction['feePerVsize'] = Math.max(1, (transaction.fee || 0) / (transaction.weight / 4)); - if (!transaction.status.confirmed) { - transaction['firstSeen'] = Math.round((new Date().getTime() / 1000)); - } - // @ts-ignore - return transaction; - } - public stripCoinbaseTransaction(tx: TransactionExtended): TransactionMinerInfo { return { vin: [{ @@ -41,10 +20,10 @@ class TransactionUtils { }; } - public async $getTransactionExtended(txId: string, inMempool = false, addPrevouts = false): Promise { + public async $getTransactionExtended(txId: string, forceBitcoind = false, addPrevouts = false): Promise { try { let transaction: IEsploraApi.Transaction; - if (inMempool) { + if (forceBitcoind) { transaction = await bitcoinApi.$getRawTransactionBitcoind(txId, false, addPrevouts); } else { transaction = await bitcoinApi.$getRawTransaction(txId, false, addPrevouts); @@ -52,11 +31,20 @@ class TransactionUtils { return this.extendTransaction(transaction); } catch (e) { logger.debug('getTransactionExtended error: ' + (e.message || e)); - console.log(e); return null; } } + private extendTransaction(transaction: IEsploraApi.Transaction): TransactionExtended { + const transactionExtended: TransactionExtended = Object.assign({ + vsize: Math.round(transaction.weight / 4), + feePerVsize: Math.max(1, (transaction.fee || 0) / (transaction.weight / 4)), + }, transaction); + if (!transaction.status.confirmed) { + transactionExtended.firstSeen = Math.round((new Date().getTime() / 1000)); + } + return transactionExtended; + } } export default new TransactionUtils(); diff --git a/backend/src/routes.ts b/backend/src/routes.ts index a130e6025..ef39a596d 100644 --- a/backend/src/routes.ts +++ b/backend/src/routes.ts @@ -528,13 +528,7 @@ class Routes { public async getTransaction(req: Request, res: Response) { try { - let transaction: TransactionExtended | null; - const txInMempool = mempool.getMempool()[req.params.txId]; - if (txInMempool) { - transaction = txInMempool; - } else { - transaction = await transactionUtils.$getTransactionExtended(req.params.txId, false, true); - } + const transaction = await transactionUtils.$getTransactionExtended(req.params.txId, false, true); if (transaction) { res.json(transaction); @@ -563,8 +557,9 @@ class Routes { try { const txIds = await bitcoinApi.$getTxIdsForBlock(req.params.hash); const transactions: TransactionExtended[] = []; + const startingIndex = Math.max(0, parseInt(req.params.index, 10)); - for (let i = 0; i < Math.min(15, txIds.length); i++) { + for (let i = startingIndex; i < Math.min(startingIndex + 10, txIds.length); i++) { const transaction = await transactionUtils.$getTransactionExtended(txIds[i], false, true); if (transaction) { transactions.push(transaction); @@ -577,7 +572,12 @@ class Routes { } public async getBlockHeight(req: Request, res: Response) { - res.status(404).send('Not implemented'); + try { + const blockHash = await bitcoinApi.$getBlockHash(parseInt(req.params.height, 10)); + res.send(blockHash); + } catch (e) { + res.status(500).send(e.message); + } } public async getAddress(req: Request, res: Response) {