diff --git a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts index f610ed883..7f4a5e53a 100644 --- a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts +++ b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts @@ -8,6 +8,7 @@ export interface AbstractBitcoinApi { $getBlockHeightTip(): Promise; $getBlockHashTip(): Promise; $getTxIdsForBlock(hash: string): Promise; + $getTxsForBlock(hash: string): Promise; $getBlockHash(height: number): Promise; $getBlockHeader(hash: string): Promise; $getBlock(hash: string): Promise; diff --git a/backend/src/api/bitcoin/bitcoin-api.ts b/backend/src/api/bitcoin/bitcoin-api.ts index 3ccea01ef..a1cf767d9 100644 --- a/backend/src/api/bitcoin/bitcoin-api.ts +++ b/backend/src/api/bitcoin/bitcoin-api.ts @@ -81,6 +81,10 @@ class BitcoinApi implements AbstractBitcoinApi { .then((rpcBlock: IBitcoinApi.Block) => rpcBlock.tx); } + $getTxsForBlock(hash: string): Promise { + throw new Error('Method getTxsForBlock not supported by the Bitcoin RPC API.'); + } + $getRawBlock(hash: string): Promise { return this.bitcoindClient.getBlock(hash, 0) .then((raw: string) => Buffer.from(raw, "hex")); diff --git a/backend/src/api/bitcoin/esplora-api.ts b/backend/src/api/bitcoin/esplora-api.ts index 73a44a845..ff10751e0 100644 --- a/backend/src/api/bitcoin/esplora-api.ts +++ b/backend/src/api/bitcoin/esplora-api.ts @@ -89,6 +89,10 @@ class ElectrsApi implements AbstractBitcoinApi { return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/block/' + hash + '/txids'); } + $getTxsForBlock(hash: string): Promise { + return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/block/' + hash + '/txs'); + } + $getBlockHash(height: number): Promise { return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/block-height/' + height); } diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 7cd37f637..4dbf4305e 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -70,6 +70,9 @@ class Blocks { * @param blockHash * @param blockHeight * @param onlyCoinbase - Set to true if you only need the coinbase transaction + * @param txIds - optional ordered list of transaction ids if already known + * @param quiet - don't print non-essential logs + * @param addMempoolData - calculate sigops etc * @returns Promise */ private async $getTransactionsExtended( @@ -80,62 +83,77 @@ class Blocks { quiet: boolean = false, addMempoolData: boolean = false, ): Promise { - const transactions: TransactionExtended[] = []; + const isEsplora = config.MEMPOOL.BACKEND === 'esplora'; + const transactionMap: { [txid: string]: TransactionExtended } = {}; + if (!txIds) { txIds = await bitcoinApi.$getTxIdsForBlock(blockHash); } const mempool = memPool.getMempool(); - let transactionsFound = 0; - let transactionsFetched = 0; + let foundInMempool = 0; + let totalFound = 0; - for (let i = 0; i < txIds.length; i++) { - if (mempool[txIds[i]]) { - // We update blocks before the mempool (index.ts), therefore we can - // optimize here by directly fetching txs in the "outdated" mempool - transactions.push(mempool[txIds[i]]); - transactionsFound++; - } else if (config.MEMPOOL.BACKEND === 'esplora' || !memPool.hasPriority() || i === 0) { - // Otherwise we fetch the tx data through backend services (esplora, electrum, core rpc...) - if (!quiet && (i % (Math.round((txIds.length) / 10)) === 0 || i + 1 === txIds.length)) { // Avoid log spam - logger.debug(`Indexing tx ${i + 1} of ${txIds.length} in block #${blockHeight}`); - } - try { - const tx = await transactionUtils.$getTransactionExtended(txIds[i], false, false, false, addMempoolData); - transactions.push(tx); - transactionsFetched++; - } catch (e) { - try { - if (config.MEMPOOL.BACKEND === 'esplora') { - // Try again with core - const tx = await transactionUtils.$getTransactionExtended(txIds[i], false, false, true, addMempoolData); - transactions.push(tx); - transactionsFetched++; - } else { - throw e; - } - } catch (e) { - if (i === 0) { - const msg = `Cannot fetch coinbase tx ${txIds[i]}. Reason: ` + (e instanceof Error ? e.message : e); - logger.err(msg); - throw new Error(msg); - } else { - logger.err(`Cannot fetch tx ${txIds[i]}. Reason: ` + (e instanceof Error ? e.message : e)); - } - } + // Copy existing transactions from the mempool + if (!onlyCoinbase) { + for (const txid of txIds) { + if (mempool[txid]) { + transactionMap[txid] = mempool[txid]; + foundInMempool++; + totalFound++; } } + } - if (onlyCoinbase === true) { - break; // Fetch the first transaction and exit + // Skip expensive lookups while mempool has priority + if (onlyCoinbase) { + try { + const coinbase = await transactionUtils.$getTransactionExtended(txIds[0], false, false, false, addMempoolData); + return [coinbase]; + } catch (e) { + const msg = `Cannot fetch coinbase tx ${txIds[0]}. Reason: ` + (e instanceof Error ? e.message : e); + logger.err(msg); + throw new Error(msg); + } + } + + // Fetch remaining txs in bulk + if (isEsplora && (txIds.length - totalFound > 500)) { + try { + const rawTransactions = await bitcoinApi.$getTxsForBlock(blockHash); + for (const tx of rawTransactions) { + if (!transactionMap[tx.txid]) { + transactionMap[tx.txid] = addMempoolData ? transactionUtils.extendMempoolTransaction(tx) : transactionUtils.extendTransaction(tx); + totalFound++; + } + } + } catch (e) { + logger.err(`Cannot fetch bulk txs for block ${blockHash}. Reason: ` + (e instanceof Error ? e.message : e)); + } + } + + // Fetch remaining txs individually + for (const txid of txIds.filter(txid => !transactionMap[txid])) { + if (!transactionMap[txid]) { + if (!quiet && (totalFound % (Math.round((txIds.length) / 10)) === 0 || totalFound + 1 === txIds.length)) { // Avoid log spam + logger.debug(`Indexing tx ${totalFound + 1} of ${txIds.length} in block #${blockHeight}`); + } + try { + const tx = await transactionUtils.$getTransactionExtended(txid, false, false, false, addMempoolData); + transactionMap[txid] = tx; + totalFound++; + } catch (e) { + logger.err(`Cannot fetch tx ${txid}. Reason: ` + (e instanceof Error ? e.message : e)); + } } } if (!quiet) { - logger.debug(`${transactionsFound} of ${txIds.length} found in mempool. ${transactionsFetched} fetched through backend service.`); + logger.debug(`${foundInMempool} of ${txIds.length} found in mempool. ${totalFound - foundInMempool} fetched through backend service.`); } - return transactions; + // Return list of transactions, preserving block order + return txIds.map(txid => transactionMap[txid]).filter(tx => tx != null); } /** diff --git a/backend/src/api/transaction-utils.ts b/backend/src/api/transaction-utils.ts index a48c9f259..0b10afdfb 100644 --- a/backend/src/api/transaction-utils.ts +++ b/backend/src/api/transaction-utils.ts @@ -53,7 +53,7 @@ class TransactionUtils { return (await this.$getTransactionExtended(txId, addPrevouts, lazyPrevouts, forceCore, true)) as MempoolTransactionExtended; } - private extendTransaction(transaction: IEsploraApi.Transaction): TransactionExtended { + public extendTransaction(transaction: IEsploraApi.Transaction): TransactionExtended { // @ts-ignore if (transaction.vsize) { // @ts-ignore