From 363fc1b00bd336618008bfc9fa62cc5a6594f3a2 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 31 Jul 2023 15:03:54 +0900 Subject: [PATCH 1/4] Get blocks from electrs again --- backend/src/api/bitcoin/bitcoin.routes.ts | 4 ++-- backend/src/api/blocks.ts | 12 ++++++------ backend/src/api/mempool.ts | 2 +- backend/src/api/mining/mining.ts | 8 ++++---- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/backend/src/api/bitcoin/bitcoin.routes.ts b/backend/src/api/bitcoin/bitcoin.routes.ts index ffdb2e629..b24c83932 100644 --- a/backend/src/api/bitcoin/bitcoin.routes.ts +++ b/backend/src/api/bitcoin/bitcoin.routes.ts @@ -6,7 +6,7 @@ import websocketHandler from '../websocket-handler'; import mempool from '../mempool'; import feeApi from '../fee-api'; import mempoolBlocks from '../mempool-blocks'; -import bitcoinApi, { bitcoinCoreApi } from './bitcoin-api-factory'; +import bitcoinApi from './bitcoin-api-factory'; import { Common } from '../common'; import backendInfo from '../backend-info'; import transactionUtils from '../transaction-utils'; @@ -483,7 +483,7 @@ class BitcoinRoutes { returnBlocks.push(localBlock); nextHash = localBlock.previousblockhash; } else { - const block = await bitcoinCoreApi.$getBlock(nextHash); + const block = await bitcoinApi.$getBlock(nextHash); returnBlocks.push(block); nextHash = block.previousblockhash; } diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index f86bc53e9..88f85ba6e 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -599,7 +599,7 @@ class Blocks { loadingIndicators.setProgress('block-indexing', progress, false); } const blockHash = await bitcoinApi.$getBlockHash(blockHeight); - const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash); + const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash); const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, null, true); const blockExtended = await this.$getBlockExtended(block, transactions); @@ -656,7 +656,7 @@ class Blocks { const heightDiff = blockHeightTip % 2016; const blockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff); this.updateTimerProgress(timer, 'got block hash for initial difficulty adjustment'); - const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash); + const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash); this.updateTimerProgress(timer, 'got block for initial difficulty adjustment'); this.lastDifficultyAdjustmentTime = block.timestamp; this.currentDifficulty = block.difficulty; @@ -664,7 +664,7 @@ class Blocks { if (blockHeightTip >= 2016) { const previousPeriodBlockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff - 2016); this.updateTimerProgress(timer, 'got previous block hash for initial difficulty adjustment'); - const previousPeriodBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(previousPeriodBlockHash); + const previousPeriodBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(previousPeriodBlockHash); this.updateTimerProgress(timer, 'got previous block for initial difficulty adjustment'); this.previousDifficultyRetarget = (block.difficulty - previousPeriodBlock.difficulty) / previousPeriodBlock.difficulty * 100; logger.debug(`Initial difficulty adjustment data set.`); @@ -862,7 +862,7 @@ class Blocks { } const blockHash = await bitcoinApi.$getBlockHash(height); - const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash); + const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash); const transactions = await this.$getTransactionsExtended(blockHash, block.height, true); const blockExtended = await this.$getBlockExtended(block, transactions); @@ -874,7 +874,7 @@ class Blocks { } public async $indexStaleBlock(hash: string): Promise { - const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(hash); + const block: IEsploraApi.Block = await bitcoinApi.$getBlock(hash); const transactions = await this.$getTransactionsExtended(hash, block.height, true); const blockExtended = await this.$getBlockExtended(block, transactions); @@ -899,7 +899,7 @@ class Blocks { } // Bitcoin network, add our custom data on top - const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(hash); + const block: IEsploraApi.Block = await bitcoinApi.$getBlock(hash); if (block.stale) { return await this.$indexStaleBlock(hash); } else { diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index d5214de5d..b9b648012 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -1,5 +1,5 @@ import config from '../config'; -import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory'; +import bitcoinApi from './bitcoin/bitcoin-api-factory'; import { MempoolTransactionExtended, TransactionExtended, VbytesPerSecond } from '../mempool.interfaces'; import logger from '../logger'; import { Common } from './common'; diff --git a/backend/src/api/mining/mining.ts b/backend/src/api/mining/mining.ts index e190492b8..beda0bc48 100644 --- a/backend/src/api/mining/mining.ts +++ b/backend/src/api/mining/mining.ts @@ -11,7 +11,7 @@ import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjust import config from '../../config'; import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository'; import PricesRepository from '../../repositories/PricesRepository'; -import { bitcoinCoreApi } from '../bitcoin/bitcoin-api-factory'; +import bitcoinApi from '../bitcoin/bitcoin-api-factory'; import { IEsploraApi } from '../bitcoin/esplora-api.interface'; import database from '../../database'; @@ -201,7 +201,7 @@ class Mining { try { const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp; - const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0)); + const genesisBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(await bitcoinApi.$getBlockHash(0)); const genesisTimestamp = genesisBlock.timestamp * 1000; const indexedTimestamp = await HashratesRepository.$getWeeklyHashrateTimestamps(); @@ -312,7 +312,7 @@ class Mining { const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp; try { - const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0)); + const genesisBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(await bitcoinApi.$getBlockHash(0)); const genesisTimestamp = genesisBlock.timestamp * 1000; const indexedTimestamp = (await HashratesRepository.$getRawNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp); const lastMidnight = this.getDateMidnight(new Date()); @@ -421,7 +421,7 @@ class Mining { } const blocks: any = await BlocksRepository.$getBlocksDifficulty(); - const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0)); + const genesisBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(await bitcoinApi.$getBlockHash(0)); let currentDifficulty = genesisBlock.difficulty; let totalIndexed = 0; From bafc0bd9cfb579ae0e2d428164dc4c8ebb408039 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 31 Jul 2023 16:03:18 +0900 Subject: [PATCH 2/4] fix indexing log prints --- backend/src/api/blocks.ts | 42 +++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 88f85ba6e..2e792bdba 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -419,8 +419,8 @@ class Blocks { let newlyIndexed = 0; let totalIndexed = indexedBlockSummariesHashesArray.length; let indexedThisRun = 0; - let timer = new Date().getTime() / 1000; - const startedAt = new Date().getTime() / 1000; + let timer = Date.now() / 1000; + const startedAt = Date.now() / 1000; for (const block of indexedBlocks) { if (indexedBlockSummariesHashes[block.hash] === true) { @@ -428,13 +428,13 @@ class Blocks { } // Logging - const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer)); + const elapsedSeconds = (Date.now() / 1000) - timer; if (elapsedSeconds > 5) { - const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); - const blockPerSeconds = Math.max(1, indexedThisRun / elapsedSeconds); + const runningFor = (Date.now() / 1000) - startedAt; + const blockPerSeconds = indexedThisRun / elapsedSeconds; const progress = Math.round(totalIndexed / indexedBlocks.length * 10000) / 100; - logger.debug(`Indexing block summary for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexedBlocks.length} (${progress}%) | elapsed: ${runningFor} seconds`, logger.tags.mining); - timer = new Date().getTime() / 1000; + logger.debug(`Indexing block summary for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexedBlocks.length} (${progress}%) | elapsed: ${runningFor.toFixed(2)} seconds`, logger.tags.mining); + timer = Date.now() / 1000; indexedThisRun = 0; } @@ -477,18 +477,18 @@ class Blocks { // Logging let count = 0; let countThisRun = 0; - let timer = new Date().getTime() / 1000; - const startedAt = new Date().getTime() / 1000; + let timer = Date.now() / 1000; + const startedAt = Date.now() / 1000; for (const height of unindexedBlockHeights) { // Logging const hash = await bitcoinApi.$getBlockHash(height); - const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer); + const elapsedSeconds = (Date.now() / 1000) - timer; if (elapsedSeconds > 5) { - const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); - const blockPerSeconds = (countThisRun / elapsedSeconds); + const runningFor = (Date.now() / 1000) - startedAt; + const blockPerSeconds = countThisRun / elapsedSeconds; const progress = Math.round(count / unindexedBlockHeights.length * 10000) / 100; - logger.debug(`Indexing cpfp clusters for #${height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${count}/${unindexedBlockHeights.length} (${progress}%) | elapsed: ${runningFor} seconds`); - timer = new Date().getTime() / 1000; + logger.debug(`Indexing cpfp clusters for #${height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${count}/${unindexedBlockHeights.length} (${progress}%) | elapsed: ${runningFor.toFixed(2)} seconds`); + timer = Date.now() / 1000; countThisRun = 0; } @@ -567,8 +567,8 @@ class Blocks { let totalIndexed = await blocksRepository.$blockCountBetweenHeight(currentBlockHeight, lastBlockToIndex); let indexedThisRun = 0; let newlyIndexed = 0; - const startedAt = new Date().getTime() / 1000; - let timer = new Date().getTime() / 1000; + const startedAt = Date.now() / 1000; + let timer = Date.now() / 1000; while (currentBlockHeight >= lastBlockToIndex) { const endBlock = Math.max(0, lastBlockToIndex, currentBlockHeight - chunkSize + 1); @@ -588,13 +588,13 @@ class Blocks { } ++indexedThisRun; ++totalIndexed; - const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer); + const elapsedSeconds = (Date.now() / 1000) - timer; if (elapsedSeconds > 5 || blockHeight === lastBlockToIndex) { - const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); - const blockPerSeconds = Math.max(1, indexedThisRun / elapsedSeconds); + const runningFor = (Date.now() / 1000) - startedAt; + const blockPerSeconds = indexedThisRun / elapsedSeconds; const progress = Math.round(totalIndexed / indexingBlockAmount * 10000) / 100; - logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${runningFor} seconds`, logger.tags.mining); - timer = new Date().getTime() / 1000; + logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexingBlockAmount} (${progress.toFixed(2)}%) | elapsed: ${runningFor.toFixed(2)} seconds`, logger.tags.mining); + timer = Date.now() / 1000; indexedThisRun = 0; loadingIndicators.setProgress('block-indexing', progress, false); } From 6b7d8d95f7beab28d8f5b6457c431bfbcc791517 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 31 Jul 2023 16:41:36 +0900 Subject: [PATCH 3/4] reduce mempool poll rate while indexing --- backend/src/index.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/src/index.ts b/backend/src/index.ts index 51d407f6f..222537d0b 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -195,7 +195,8 @@ class Server { // rerun immediately if we skipped the mempool update, otherwise wait POLL_RATE_MS const elapsed = Date.now() - start; - const remainingTime = Math.max(0, config.MEMPOOL.POLL_RATE_MS - elapsed) + const pollRate = config.MEMPOOL.POLL_RATE_MS * (indexer.indexerRunning ? 10 : 1); + const remainingTime = Math.max(0, pollRate - elapsed); setTimeout(this.runMainUpdateLoop.bind(this), numHandledBlocks > 0 ? 0 : remainingTime); this.backendRetryCount = 0; } catch (e: any) { From 0d25ef0b5b8a05a83fde361addaae6c2ad41110f Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 31 Jul 2023 18:02:18 +0900 Subject: [PATCH 4/4] Get block txs from esplora, index CPFP together with summaries --- backend/src/api/blocks.ts | 59 +++++++++++++++----- backend/src/repositories/BlocksRepository.ts | 15 ++++- 2 files changed, 58 insertions(+), 16 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 2e792bdba..20c015f44 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -438,7 +438,14 @@ class Blocks { indexedThisRun = 0; } - await this.$getStrippedBlockTransactions(block.hash, true, true); // This will index the block summary + + if (config.MEMPOOL.BACKEND === 'esplora') { + const txs = (await bitcoinApi.$getTxsForBlock(block.hash)).map(tx => transactionUtils.extendTransaction(tx)); + const cpfpSummary = await this.$indexCPFP(block.hash, block.height, txs); + await this.$getStrippedBlockTransactions(block.hash, true, true, cpfpSummary, block.height); // This will index the block summary + } else { + await this.$getStrippedBlockTransactions(block.hash, true, true); // This will index the block summary + } // Logging indexedThisRun++; @@ -942,10 +949,15 @@ class Blocks { }), }; } else { - // Call Core RPC - const block = await bitcoinClient.getBlock(hash, 2); - summary = this.summarizeBlock(block); - height = block.height; + if (config.MEMPOOL.BACKEND === 'esplora') { + const txs = (await bitcoinApi.$getTxsForBlock(hash)).map(tx => transactionUtils.extendTransaction(tx)); + summary = this.summarizeBlockTransactions(hash, txs); + } else { + // Call Core RPC + const block = await bitcoinClient.getBlock(hash, 2); + summary = this.summarizeBlock(block); + height = block.height; + } } if (height == null) { const block = await bitcoinApi.$getBlock(hash); @@ -1068,8 +1080,17 @@ class Blocks { if (Common.blocksSummariesIndexingEnabled() && cleanBlock.fee_amt_percentiles === null) { cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash); if (cleanBlock.fee_amt_percentiles === null) { - const block = await bitcoinClient.getBlock(cleanBlock.hash, 2); - const summary = this.summarizeBlock(block); + + let summary; + if (config.MEMPOOL.BACKEND === 'esplora') { + const txs = (await bitcoinApi.$getTxsForBlock(cleanBlock.hash)).map(tx => transactionUtils.extendTransaction(tx)); + summary = this.summarizeBlockTransactions(cleanBlock.hash, txs); + } else { + // Call Core RPC + const block = await bitcoinClient.getBlock(cleanBlock.hash, 2); + summary = this.summarizeBlock(block); + } + await BlocksSummariesRepository.$saveTransactions(cleanBlock.height, cleanBlock.hash, summary.transactions); cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash); } @@ -1129,19 +1150,29 @@ class Blocks { return this.currentBlockHeight; } - public async $indexCPFP(hash: string, height: number): Promise { - const block = await bitcoinClient.getBlock(hash, 2); - const transactions = block.tx.map(tx => { - tx.fee *= 100_000_000; - return tx; - }); + public async $indexCPFP(hash: string, height: number, txs?: TransactionExtended[]): Promise { + let transactions = txs; + if (!transactions) { + if (config.MEMPOOL.BACKEND === 'esplora') { + transactions = (await bitcoinApi.$getTxsForBlock(hash)).map(tx => transactionUtils.extendTransaction(tx)); + } + if (!transactions) { + const block = await bitcoinClient.getBlock(hash, 2); + transactions = block.tx.map(tx => { + tx.fee *= 100_000_000; + return tx; + }); + } + } - const summary = Common.calculateCpfp(height, transactions); + const summary = Common.calculateCpfp(height, transactions as TransactionExtended[]); await this.$saveCpfp(hash, height, summary); const effectiveFeeStats = Common.calcEffectiveFeeStatistics(summary.transactions); await blocksRepository.$saveEffectiveFeeStats(hash, effectiveFeeStats); + + return summary; } public async $saveCpfp(hash: string, height: number, cpfpSummary: CpfpSummary): Promise { diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 078b85a03..5f20f5abd 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -1,3 +1,4 @@ +import bitcoinApi from '../api/bitcoin/bitcoin-api-factory'; import { BlockExtended, BlockExtension, BlockPrice, EffectiveFeeStats } from '../mempool.interfaces'; import DB from '../database'; import logger from '../logger'; @@ -12,6 +13,7 @@ import config from '../config'; import chainTips from '../api/chain-tips'; import blocks from '../api/blocks'; import BlocksAuditsRepository from './BlocksAuditsRepository'; +import transactionUtils from '../api/transaction-utils'; interface DatabaseBlock { id: string; @@ -1036,8 +1038,17 @@ class BlocksRepository { { extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id); if (extras.feePercentiles === null) { - const block = await bitcoinClient.getBlock(dbBlk.id, 2); - const summary = blocks.summarizeBlock(block); + + let summary; + if (config.MEMPOOL.BACKEND === 'esplora') { + const txs = (await bitcoinApi.$getTxsForBlock(dbBlk.id)).map(tx => transactionUtils.extendTransaction(tx)); + summary = blocks.summarizeBlockTransactions(dbBlk.id, txs); + } else { + // Call Core RPC + const block = await bitcoinClient.getBlock(dbBlk.id, 2); + summary = blocks.summarizeBlock(block); + } + await BlocksSummariesRepository.$saveTransactions(dbBlk.height, dbBlk.id, summary.transactions); extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id); }