diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 198f2a204..d2487414f 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -10,6 +10,7 @@ import bitcoinClient from './bitcoin/bitcoin-client'; import { IEsploraApi } from './bitcoin/esplora-api.interface'; import poolsRepository from '../repositories/PoolsRepository'; import blocksRepository from '../repositories/BlocksRepository'; +import loadingIndicators from './loading-indicators'; class Blocks { private blocks: BlockExtended[] = []; @@ -41,7 +42,12 @@ class Blocks { * @param onlyCoinbase - Set to true if you only need the coinbase transaction * @returns Promise */ - private async $getTransactionsExtended(blockHash: string, blockHeight: number, onlyCoinbase: boolean): Promise { + private async $getTransactionsExtended( + blockHash: string, + blockHeight: number, + onlyCoinbase: boolean, + quiet: boolean = false, + ): Promise { const transactions: TransactionExtended[] = []; const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash); @@ -57,7 +63,7 @@ class Blocks { transactionsFound++; } else if (config.MEMPOOL.BACKEND === 'esplora' || memPool.isInSync() || i === 0) { // Otherwise we fetch the tx data through backend services (esplora, electrum, core rpc...) - if (i % (Math.round((txIds.length) / 10)) === 0 || i + 1 === txIds.length) { // Avoid log spam + if (!quiet && (i % (Math.round((txIds.length) / 10)) === 0 || i + 1 === txIds.length)) { // Avoid log spam logger.debug(`Indexing tx ${i + 1} of ${txIds.length} in block #${blockHeight}`); } try { @@ -83,7 +89,9 @@ class Blocks { } }); - logger.debug(`${transactionsFound} of ${txIds.length} found in mempool. ${transactionsFetched} fetched through backend service.`); + if (!quiet) { + logger.debug(`${transactionsFound} of ${txIds.length} found in mempool. ${transactionsFetched} fetched through backend service.`); + } return transactions; } @@ -94,13 +102,10 @@ class Blocks { * @param transactions * @returns BlockExtended */ - private getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): BlockExtended { - const blockExtended: BlockExtended = Object.assign({}, block); - - blockExtended.extras = { - reward: transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0), - coinbaseTx: transactionUtils.stripCoinbaseTransaction(transactions[0]), - }; + private async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise { + const blockExtended: BlockExtended = Object.assign({extras: {}}, block); + blockExtended.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0); + blockExtended.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]); const transactionsTmp = [...transactions]; transactionsTmp.shift(); @@ -111,6 +116,22 @@ class Blocks { blockExtended.extras.feeRange = transactionsTmp.length > 0 ? Common.getFeesInRange(transactionsTmp, 8) : [0, 0]; + const indexingAvailable = + ['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) && + config.DATABASE.ENABLED === true; + if (indexingAvailable) { + let pool: PoolTag; + if (blockExtended.extras?.coinbaseTx !== undefined) { + pool = await this.$findBlockMiner(blockExtended.extras?.coinbaseTx); + } else { + pool = await poolsRepository.$getUnknownPool(); + } + blockExtended.extras.pool = { + id: pool.id, + name: pool.name + }; + } + return blockExtended; } @@ -153,19 +174,21 @@ class Blocks { */ public async $generateBlockDatabase() { if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false || // Bitcoin only - config.MEMPOOL.INDEXING_BLOCKS_AMOUNT === 0 || // Indexing must be enabled + config.MEMPOOL.INDEXING_BLOCKS_AMOUNT === 0 || // Indexing of older blocks must be enabled !memPool.isInSync() || // We sync the mempool first - this.blockIndexingStarted === true // Indexing must not already be in progress + this.blockIndexingStarted === true || // Indexing must not already be in progress + config.DATABASE.ENABLED === false ) { return; } const blockchainInfo = await bitcoinClient.getBlockchainInfo(); - if (blockchainInfo.blocks !== blockchainInfo.headers) { + if (blockchainInfo.blocks !== blockchainInfo.headers) { // Wait for node to sync return; } this.blockIndexingStarted = true; + const startedAt = new Date().getTime() / 1000; try { let currentBlockHeight = blockchainInfo.blocks; @@ -180,6 +203,7 @@ class Blocks { logger.info(`Indexing blocks from #${currentBlockHeight} to #${lastBlockToIndex}`); const chunkSize = 10000; + let totaIndexed = 0; while (currentBlockHeight >= lastBlockToIndex) { const endBlock = Math.max(0, lastBlockToIndex, currentBlockHeight - chunkSize + 1); @@ -198,21 +222,17 @@ class Blocks { break; } try { - logger.debug(`Indexing block #${blockHeight}`); + if (totaIndexed % 100 === 0 || blockHeight === lastBlockToIndex) { + const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); + const blockPerSeconds = Math.round(totaIndexed / elapsedSeconds); + logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds} blocks/sec | total: ${totaIndexed} | elapsed: ${elapsedSeconds} seconds`); + } const blockHash = await bitcoinApi.$getBlockHash(blockHeight); const block = await bitcoinApi.$getBlock(blockHash); - const transactions = await this.$getTransactionsExtended(blockHash, block.height, true); - const blockExtended = this.getBlockExtended(block, transactions); - - let miner: PoolTag; - if (blockExtended?.extras?.coinbaseTx) { - miner = await this.$findBlockMiner(blockExtended.extras.coinbaseTx); - } else { - miner = await poolsRepository.$getUnknownPool(); - } - - const coinbase: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(transactions[0].txid, true); - await blocksRepository.$saveBlockInDatabase(blockExtended, blockHash, coinbase.hex, miner); + const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, true); + const blockExtended = await this.$getBlockExtended(block, transactions); + await blocksRepository.$saveBlockInDatabase(blockExtended); + ++totaIndexed; } catch (e) { logger.err(`Something went wrong while indexing blocks.` + e); } @@ -271,17 +291,13 @@ class Blocks { const block = await bitcoinApi.$getBlock(blockHash); const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash); const transactions = await this.$getTransactionsExtended(blockHash, block.height, false); - const blockExtended: BlockExtended = this.getBlockExtended(block, transactions); - const coinbase: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(transactions[0].txid, true); + const blockExtended: BlockExtended = await this.$getBlockExtended(block, transactions); - if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === true) { - let miner: PoolTag; - if (blockExtended?.extras?.coinbaseTx) { - miner = await this.$findBlockMiner(blockExtended.extras.coinbaseTx); - } else { - miner = await poolsRepository.$getUnknownPool(); - } - await blocksRepository.$saveBlockInDatabase(blockExtended, blockHash, coinbase.hex, miner); + const indexingAvailable = + ['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) && + config.DATABASE.ENABLED === true; + if (indexingAvailable) { + await blocksRepository.$saveBlockInDatabase(blockExtended); } if (block.height % 2016 === 0) { @@ -304,6 +320,96 @@ class Blocks { } } + /** + * Index a block if it's missing from the database. Returns the block after indexing + */ + public async $indexBlock(height: number): Promise { + const dbBlock = await blocksRepository.$getBlockByHeight(height); + if (dbBlock != null) { + return this.prepareBlock(dbBlock); + } + + const blockHash = await bitcoinApi.$getBlockHash(height); + const block = await bitcoinApi.$getBlock(blockHash); + const transactions = await this.$getTransactionsExtended(blockHash, block.height, true); + const blockExtended = await this.$getBlockExtended(block, transactions); + + await blocksRepository.$saveBlockInDatabase(blockExtended); + + return blockExtended; + } + + public async $getBlocksExtras(fromHeight: number): Promise { + const indexingAvailable = + ['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) && + config.DATABASE.ENABLED === true; + + try { + loadingIndicators.setProgress('blocks', 0); + + let currentHeight = fromHeight ? fromHeight : this.getCurrentBlockHeight(); + const returnBlocks: BlockExtended[] = []; + + if (currentHeight < 0) { + return returnBlocks; + } + + // Check if block height exist in local cache to skip the hash lookup + const blockByHeight = this.getBlocks().find((b) => b.height === currentHeight); + let startFromHash: string | null = null; + if (blockByHeight) { + startFromHash = blockByHeight.id; + } else { + startFromHash = await bitcoinApi.$getBlockHash(currentHeight); + } + + let nextHash = startFromHash; + for (let i = 0; i < 10 && currentHeight >= 0; i++) { + let block = this.getBlocks().find((b) => b.height === currentHeight); + if (!block && indexingAvailable) { + block = this.prepareBlock(await this.$indexBlock(currentHeight)); + } else if (!block) { + block = this.prepareBlock(await bitcoinApi.$getBlock(nextHash)); + } + returnBlocks.push(block); + nextHash = block.previousblockhash; + loadingIndicators.setProgress('blocks', i / 10 * 100); + currentHeight--; + } + + return returnBlocks; + } catch (e) { + loadingIndicators.setProgress('blocks', 100); + throw e; + } + } + + private prepareBlock(block: any): BlockExtended { + return { + id: block.id ?? block.hash, // hash for indexed block + timestamp: block?.timestamp ?? block?.blockTimestamp, // blockTimestamp for indexed block + height: block?.height, + version: block?.version, + bits: block?.bits, + nonce: block?.nonce, + difficulty: block?.difficulty, + merkle_root: block?.merkle_root, + tx_count: block?.tx_count, + size: block?.size, + weight: block?.weight, + previousblockhash: block?.previousblockhash, + extras: { + medianFee: block?.medianFee, + feeRange: block?.feeRange ?? [], // TODO + reward: block?.reward, + pool: block?.extras?.pool ?? (block?.pool_id ? { + id: block?.pool_id, + name: block?.pool_name, + } : undefined), + } + }; + } + public getLastDifficultyAdjustmentTime(): number { return this.lastDifficultyAdjustmentTime; } diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 24ecc03cf..b8557ff65 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -6,7 +6,7 @@ import logger from '../logger'; const sleep = (ms: number) => new Promise(res => setTimeout(res, ms)); class DatabaseMigration { - private static currentVersion = 4; + private static currentVersion = 5; private queryTimeout = 120000; private statisticsAddedIndexed = false; @@ -229,6 +229,10 @@ class DatabaseMigration { } } + if (version < 5 && (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === true)) { + queries.push('ALTER TABLE blocks ADD `reward` double unsigned NOT NULL DEFAULT "0"'); + } + return queries; } diff --git a/backend/src/index.ts b/backend/src/index.ts index 557c269dd..07808c98a 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -290,6 +290,10 @@ class Server { ; } + this.app + .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-extras', routes.getBlocksExtras) + .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-extras/:height', routes.getBlocksExtras); + if (config.MEMPOOL.BACKEND !== 'esplora') { this.app .get(config.MEMPOOL.API_URL_PREFIX + 'mempool', routes.getMempool) diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index 7dfcd3956..2d5092145 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -1,7 +1,7 @@ import { IEsploraApi } from './api/bitcoin/esplora-api.interface'; export interface PoolTag { - id: number | null, // mysql row id + id: number, // mysql row id name: string, link: string, regexes: string, // JSON array @@ -83,10 +83,14 @@ export interface BlockExtension { reward?: number; coinbaseTx?: TransactionMinerInfo; matchRate?: number; + pool?: { + id: number; + name: string; + } } export interface BlockExtended extends IEsploraApi.Block { - extras?: BlockExtension; + extras: BlockExtension; } export interface TransactionMinerInfo { diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 654376402..18023760f 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -11,38 +11,36 @@ class BlocksRepository { /** * Save indexed block data in the database */ - public async $saveBlockInDatabase( - block: BlockExtended, - blockHash: string, - coinbaseHex: string | undefined, - poolTag: PoolTag - ) { + public async $saveBlockInDatabase(block: BlockExtended) { const connection = await DB.pool.getConnection(); try { const query = `INSERT INTO blocks( height, hash, blockTimestamp, size, weight, tx_count, coinbase_raw, difficulty, - pool_id, fees, fee_span, median_fee + pool_id, fees, fee_span, median_fee, + reward ) VALUE ( ?, ?, FROM_UNIXTIME(?), ?, ?, ?, ?, ?, - ?, ?, ?, ? + ?, ?, ?, ?, + ? )`; const params: any[] = [ block.height, - blockHash, + block.id, block.timestamp, block.size, block.weight, block.tx_count, - coinbaseHex ? coinbaseHex : '', + '', block.difficulty, - poolTag.id, + block.extras?.pool?.id, // Should always be set to something 0, '[]', - block.extras ? block.extras.medianFee : 0, + block.extras.medianFee ?? 0, + block.extras?.reward ?? 0, ]; await connection.query(query, params); @@ -136,6 +134,26 @@ class BlocksRepository { return rows[0].blockTimestamp; } + + /** + * Get one block by height + */ + public async $getBlockByHeight(height: number): Promise { + const connection = await DB.pool.getConnection(); + const [rows]: any[] = await connection.query(` + SELECT *, UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp, pools.id as pool_id, pools.name as pool_name, pools.link as pool_link, pools.addresses as pool_addresses, pools.regexes as pool_regexes + FROM blocks + JOIN pools ON blocks.pool_id = pools.id + WHERE height = ${height}; + `); + connection.release(); + + if (rows.length <= 0) { + return null; + } + + return rows[0]; + } } export default new BlocksRepository(); \ No newline at end of file diff --git a/backend/src/repositories/PoolsRepository.ts b/backend/src/repositories/PoolsRepository.ts index d1fb0da9a..b89725452 100644 --- a/backend/src/repositories/PoolsRepository.ts +++ b/backend/src/repositories/PoolsRepository.ts @@ -7,7 +7,7 @@ class PoolsRepository { */ public async $getPools(): Promise { const connection = await DB.pool.getConnection(); - const [rows] = await connection.query('SELECT * FROM pools;'); + const [rows] = await connection.query('SELECT id, name, addresses, regexes FROM pools;'); connection.release(); return rows; } @@ -17,7 +17,7 @@ class PoolsRepository { */ public async $getUnknownPool(): Promise { const connection = await DB.pool.getConnection(); - const [rows] = await connection.query('SELECT * FROM pools where name = "Unknown"'); + const [rows] = await connection.query('SELECT id, name FROM pools where name = "Unknown"'); connection.release(); return rows[0]; } diff --git a/backend/src/routes.ts b/backend/src/routes.ts index 8ae2f9609..e06177ddd 100644 --- a/backend/src/routes.ts +++ b/backend/src/routes.ts @@ -564,6 +564,14 @@ class Routes { } } + public async getBlocksExtras(req: Request, res: Response) { + try { + res.json(await blocks.$getBlocksExtras(parseInt(req.params.height, 10))) + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + public async getBlocks(req: Request, res: Response) { try { loadingIndicators.setProgress('blocks', 0); diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index 3a0a0ec0b..6b2319a59 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -15,7 +15,8 @@ "PRICE_FEED_UPDATE_INTERVAL": __MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__, "USE_SECOND_NODE_FOR_MINFEE": __MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__, "EXTERNAL_ASSETS": __MEMPOOL_EXTERNAL_ASSETS__, - "STDOUT_LOG_MIN_PRIORITY": "__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__" + "STDOUT_LOG_MIN_PRIORITY": "__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__", + "INDEXING_BLOCKS_AMOUNT": __MEMPOOL_INDEXING_BLOCKS_AMOUNT__ }, "CORE_RPC": { "HOST": "__CORE_RPC_HOST__",