diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 88f85ba6e..20c015f44 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -419,8 +419,8 @@ class Blocks { let newlyIndexed = 0; let totalIndexed = indexedBlockSummariesHashesArray.length; let indexedThisRun = 0; - let timer = new Date().getTime() / 1000; - const startedAt = new Date().getTime() / 1000; + let timer = Date.now() / 1000; + const startedAt = Date.now() / 1000; for (const block of indexedBlocks) { if (indexedBlockSummariesHashes[block.hash] === true) { @@ -428,17 +428,24 @@ class Blocks { } // Logging - const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer)); + const elapsedSeconds = (Date.now() / 1000) - timer; if (elapsedSeconds > 5) { - const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); - const blockPerSeconds = Math.max(1, indexedThisRun / elapsedSeconds); + const runningFor = (Date.now() / 1000) - startedAt; + const blockPerSeconds = indexedThisRun / elapsedSeconds; const progress = Math.round(totalIndexed / indexedBlocks.length * 10000) / 100; - logger.debug(`Indexing block summary for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexedBlocks.length} (${progress}%) | elapsed: ${runningFor} seconds`, logger.tags.mining); - timer = new Date().getTime() / 1000; + logger.debug(`Indexing block summary for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexedBlocks.length} (${progress}%) | elapsed: ${runningFor.toFixed(2)} seconds`, logger.tags.mining); + timer = Date.now() / 1000; indexedThisRun = 0; } - await this.$getStrippedBlockTransactions(block.hash, true, true); // This will index the block summary + + if (config.MEMPOOL.BACKEND === 'esplora') { + const txs = (await bitcoinApi.$getTxsForBlock(block.hash)).map(tx => transactionUtils.extendTransaction(tx)); + const cpfpSummary = await this.$indexCPFP(block.hash, block.height, txs); + await this.$getStrippedBlockTransactions(block.hash, true, true, cpfpSummary, block.height); // This will index the block summary + } else { + await this.$getStrippedBlockTransactions(block.hash, true, true); // This will index the block summary + } // Logging indexedThisRun++; @@ -477,18 +484,18 @@ class Blocks { // Logging let count = 0; let countThisRun = 0; - let timer = new Date().getTime() / 1000; - const startedAt = new Date().getTime() / 1000; + let timer = Date.now() / 1000; + const startedAt = Date.now() / 1000; for (const height of unindexedBlockHeights) { // Logging const hash = await bitcoinApi.$getBlockHash(height); - const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer); + const elapsedSeconds = (Date.now() / 1000) - timer; if (elapsedSeconds > 5) { - const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); - const blockPerSeconds = (countThisRun / elapsedSeconds); + const runningFor = (Date.now() / 1000) - startedAt; + const blockPerSeconds = countThisRun / elapsedSeconds; const progress = Math.round(count / unindexedBlockHeights.length * 10000) / 100; - logger.debug(`Indexing cpfp clusters for #${height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${count}/${unindexedBlockHeights.length} (${progress}%) | elapsed: ${runningFor} seconds`); - timer = new Date().getTime() / 1000; + logger.debug(`Indexing cpfp clusters for #${height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${count}/${unindexedBlockHeights.length} (${progress}%) | elapsed: ${runningFor.toFixed(2)} seconds`); + timer = Date.now() / 1000; countThisRun = 0; } @@ -567,8 +574,8 @@ class Blocks { let totalIndexed = await blocksRepository.$blockCountBetweenHeight(currentBlockHeight, lastBlockToIndex); let indexedThisRun = 0; let newlyIndexed = 0; - const startedAt = new Date().getTime() / 1000; - let timer = new Date().getTime() / 1000; + const startedAt = Date.now() / 1000; + let timer = Date.now() / 1000; while (currentBlockHeight >= lastBlockToIndex) { const endBlock = Math.max(0, lastBlockToIndex, currentBlockHeight - chunkSize + 1); @@ -588,13 +595,13 @@ class Blocks { } ++indexedThisRun; ++totalIndexed; - const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer); + const elapsedSeconds = (Date.now() / 1000) - timer; if (elapsedSeconds > 5 || blockHeight === lastBlockToIndex) { - const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); - const blockPerSeconds = Math.max(1, indexedThisRun / elapsedSeconds); + const runningFor = (Date.now() / 1000) - startedAt; + const blockPerSeconds = indexedThisRun / elapsedSeconds; const progress = Math.round(totalIndexed / indexingBlockAmount * 10000) / 100; - logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${runningFor} seconds`, logger.tags.mining); - timer = new Date().getTime() / 1000; + logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexingBlockAmount} (${progress.toFixed(2)}%) | elapsed: ${runningFor.toFixed(2)} seconds`, logger.tags.mining); + timer = Date.now() / 1000; indexedThisRun = 0; loadingIndicators.setProgress('block-indexing', progress, false); } @@ -942,10 +949,15 @@ class Blocks { }), }; } else { - // Call Core RPC - const block = await bitcoinClient.getBlock(hash, 2); - summary = this.summarizeBlock(block); - height = block.height; + if (config.MEMPOOL.BACKEND === 'esplora') { + const txs = (await bitcoinApi.$getTxsForBlock(hash)).map(tx => transactionUtils.extendTransaction(tx)); + summary = this.summarizeBlockTransactions(hash, txs); + } else { + // Call Core RPC + const block = await bitcoinClient.getBlock(hash, 2); + summary = this.summarizeBlock(block); + height = block.height; + } } if (height == null) { const block = await bitcoinApi.$getBlock(hash); @@ -1068,8 +1080,17 @@ class Blocks { if (Common.blocksSummariesIndexingEnabled() && cleanBlock.fee_amt_percentiles === null) { cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash); if (cleanBlock.fee_amt_percentiles === null) { - const block = await bitcoinClient.getBlock(cleanBlock.hash, 2); - const summary = this.summarizeBlock(block); + + let summary; + if (config.MEMPOOL.BACKEND === 'esplora') { + const txs = (await bitcoinApi.$getTxsForBlock(cleanBlock.hash)).map(tx => transactionUtils.extendTransaction(tx)); + summary = this.summarizeBlockTransactions(cleanBlock.hash, txs); + } else { + // Call Core RPC + const block = await bitcoinClient.getBlock(cleanBlock.hash, 2); + summary = this.summarizeBlock(block); + } + await BlocksSummariesRepository.$saveTransactions(cleanBlock.height, cleanBlock.hash, summary.transactions); cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash); } @@ -1129,19 +1150,29 @@ class Blocks { return this.currentBlockHeight; } - public async $indexCPFP(hash: string, height: number): Promise { - const block = await bitcoinClient.getBlock(hash, 2); - const transactions = block.tx.map(tx => { - tx.fee *= 100_000_000; - return tx; - }); + public async $indexCPFP(hash: string, height: number, txs?: TransactionExtended[]): Promise { + let transactions = txs; + if (!transactions) { + if (config.MEMPOOL.BACKEND === 'esplora') { + transactions = (await bitcoinApi.$getTxsForBlock(hash)).map(tx => transactionUtils.extendTransaction(tx)); + } + if (!transactions) { + const block = await bitcoinClient.getBlock(hash, 2); + transactions = block.tx.map(tx => { + tx.fee *= 100_000_000; + return tx; + }); + } + } - const summary = Common.calculateCpfp(height, transactions); + const summary = Common.calculateCpfp(height, transactions as TransactionExtended[]); await this.$saveCpfp(hash, height, summary); const effectiveFeeStats = Common.calcEffectiveFeeStatistics(summary.transactions); await blocksRepository.$saveEffectiveFeeStats(hash, effectiveFeeStats); + + return summary; } public async $saveCpfp(hash: string, height: number, cpfpSummary: CpfpSummary): Promise { diff --git a/backend/src/index.ts b/backend/src/index.ts index 51d407f6f..222537d0b 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -195,7 +195,8 @@ class Server { // rerun immediately if we skipped the mempool update, otherwise wait POLL_RATE_MS const elapsed = Date.now() - start; - const remainingTime = Math.max(0, config.MEMPOOL.POLL_RATE_MS - elapsed) + const pollRate = config.MEMPOOL.POLL_RATE_MS * (indexer.indexerRunning ? 10 : 1); + const remainingTime = Math.max(0, pollRate - elapsed); setTimeout(this.runMainUpdateLoop.bind(this), numHandledBlocks > 0 ? 0 : remainingTime); this.backendRetryCount = 0; } catch (e: any) { diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index ca54fc1b7..0953f9b84 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -1,3 +1,4 @@ +import bitcoinApi from '../api/bitcoin/bitcoin-api-factory'; import { BlockExtended, BlockExtension, BlockPrice, EffectiveFeeStats } from '../mempool.interfaces'; import DB from '../database'; import logger from '../logger'; @@ -12,6 +13,7 @@ import config from '../config'; import chainTips from '../api/chain-tips'; import blocks from '../api/blocks'; import BlocksAuditsRepository from './BlocksAuditsRepository'; +import transactionUtils from '../api/transaction-utils'; interface DatabaseBlock { id: string; @@ -1036,8 +1038,17 @@ class BlocksRepository { { extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id); if (extras.feePercentiles === null) { - const block = await bitcoinClient.getBlock(dbBlk.id, 2); - const summary = blocks.summarizeBlock(block); + + let summary; + if (config.MEMPOOL.BACKEND === 'esplora') { + const txs = (await bitcoinApi.$getTxsForBlock(dbBlk.id)).map(tx => transactionUtils.extendTransaction(tx)); + summary = blocks.summarizeBlockTransactions(dbBlk.id, txs); + } else { + // Call Core RPC + const block = await bitcoinClient.getBlock(dbBlk.id, 2); + summary = blocks.summarizeBlock(block); + } + await BlocksSummariesRepository.$saveTransactions(dbBlk.height, dbBlk.id, summary.transactions); extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id); }