From fac49d0b9850e2938efd1aacb364de7cb52e4095 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Tue, 8 Feb 2022 15:47:43 +0900 Subject: [PATCH] Added /api/v1/blocksExtras endpoint --- backend/src/api/blocks.ts | 160 +++++++++++++++---- backend/src/api/database-migration.ts | 6 +- backend/src/index.ts | 4 + backend/src/mempool.interfaces.ts | 4 +- backend/src/repositories/BlocksRepository.ts | 42 +++-- backend/src/routes.ts | 8 + 6 files changed, 179 insertions(+), 45 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 198f2a204..c113f4efa 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -10,6 +10,7 @@ import bitcoinClient from './bitcoin/bitcoin-client'; import { IEsploraApi } from './bitcoin/esplora-api.interface'; import poolsRepository from '../repositories/PoolsRepository'; import blocksRepository from '../repositories/BlocksRepository'; +import loadingIndicators from './loading-indicators'; class Blocks { private blocks: BlockExtended[] = []; @@ -94,13 +95,10 @@ class Blocks { * @param transactions * @returns BlockExtended */ - private getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): BlockExtended { - const blockExtended: BlockExtended = Object.assign({}, block); - - blockExtended.extras = { - reward: transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0), - coinbaseTx: transactionUtils.stripCoinbaseTransaction(transactions[0]), - }; + private async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise { + const blockExtended: BlockExtended = Object.assign({extras: {}}, block); + blockExtended.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0); + blockExtended.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]); const transactionsTmp = [...transactions]; transactionsTmp.shift(); @@ -111,6 +109,24 @@ class Blocks { blockExtended.extras.feeRange = transactionsTmp.length > 0 ? Common.getFeesInRange(transactionsTmp, 8) : [0, 0]; + const indexingAvailable = + ['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) && + config.DATABASE.ENABLED === true; + if (indexingAvailable) { + let pool: PoolTag; + if (blockExtended.extras?.coinbaseTx !== undefined) { + pool = await this.$findBlockMiner(blockExtended.extras?.coinbaseTx); + } else { + pool = await poolsRepository.$getUnknownPool(); + } + blockExtended.extras.pool = pool; + + if (transactions.length > 0) { + const coinbase: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(transactions[0].txid, true); + blockExtended.extras.coinbaseHex = coinbase.hex; + } + } + return blockExtended; } @@ -153,15 +169,16 @@ class Blocks { */ public async $generateBlockDatabase() { if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false || // Bitcoin only - config.MEMPOOL.INDEXING_BLOCKS_AMOUNT === 0 || // Indexing must be enabled + config.MEMPOOL.INDEXING_BLOCKS_AMOUNT === 0 || // Indexing of older blocks must be enabled !memPool.isInSync() || // We sync the mempool first - this.blockIndexingStarted === true // Indexing must not already be in progress + this.blockIndexingStarted === true || // Indexing must not already be in progress + config.DATABASE.ENABLED === false ) { return; } const blockchainInfo = await bitcoinClient.getBlockchainInfo(); - if (blockchainInfo.blocks !== blockchainInfo.headers) { + if (blockchainInfo.blocks !== blockchainInfo.headers) { // Wait for node to sync return; } @@ -202,17 +219,8 @@ class Blocks { const blockHash = await bitcoinApi.$getBlockHash(blockHeight); const block = await bitcoinApi.$getBlock(blockHash); const transactions = await this.$getTransactionsExtended(blockHash, block.height, true); - const blockExtended = this.getBlockExtended(block, transactions); - - let miner: PoolTag; - if (blockExtended?.extras?.coinbaseTx) { - miner = await this.$findBlockMiner(blockExtended.extras.coinbaseTx); - } else { - miner = await poolsRepository.$getUnknownPool(); - } - - const coinbase: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(transactions[0].txid, true); - await blocksRepository.$saveBlockInDatabase(blockExtended, blockHash, coinbase.hex, miner); + const blockExtended = await this.$getBlockExtended(block, transactions); + await blocksRepository.$saveBlockInDatabase(blockExtended); } catch (e) { logger.err(`Something went wrong while indexing blocks.` + e); } @@ -271,17 +279,13 @@ class Blocks { const block = await bitcoinApi.$getBlock(blockHash); const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash); const transactions = await this.$getTransactionsExtended(blockHash, block.height, false); - const blockExtended: BlockExtended = this.getBlockExtended(block, transactions); - const coinbase: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(transactions[0].txid, true); + const blockExtended: BlockExtended = await this.$getBlockExtended(block, transactions); - if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === true) { - let miner: PoolTag; - if (blockExtended?.extras?.coinbaseTx) { - miner = await this.$findBlockMiner(blockExtended.extras.coinbaseTx); - } else { - miner = await poolsRepository.$getUnknownPool(); - } - await blocksRepository.$saveBlockInDatabase(blockExtended, blockHash, coinbase.hex, miner); + const indexingAvailable = + ['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) && + config.DATABASE.ENABLED === true; + if (indexingAvailable) { + await blocksRepository.$saveBlockInDatabase(blockExtended); } if (block.height % 2016 === 0) { @@ -304,6 +308,100 @@ class Blocks { } } + /** + * Index a block if it's missing from the database. Returns the block after indexing + */ + public async $indexBlock(height: number): Promise { + const dbBlock = await blocksRepository.$getBlockByHeight(height); + if (dbBlock != null) { + return this.prepareBlock(dbBlock); + } + + const blockHash = await bitcoinApi.$getBlockHash(height); + const block = await bitcoinApi.$getBlock(blockHash); + const transactions = await this.$getTransactionsExtended(blockHash, block.height, true); + const blockExtended = await this.$getBlockExtended(block, transactions); + + await blocksRepository.$saveBlockInDatabase(blockExtended); + + return blockExtended; + } + + public async $getBlocksExtras(fromHeight: number): Promise { + const indexingAvailable = + ['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) && + config.DATABASE.ENABLED === true; + + try { + loadingIndicators.setProgress('blocks', 0); + + let currentHeight = fromHeight ? fromHeight : this.getCurrentBlockHeight(); + const returnBlocks: BlockExtended[] = []; + + if (currentHeight < 0) { + return returnBlocks; + } + + // Check if block height exist in local cache to skip the hash lookup + const blockByHeight = this.getBlocks().find((b) => b.height === currentHeight); + let startFromHash: string | null = null; + if (blockByHeight) { + startFromHash = blockByHeight.id; + } else { + startFromHash = await bitcoinApi.$getBlockHash(currentHeight); + } + + let nextHash = startFromHash; + for (let i = 0; i < 10 && currentHeight >= 0; i++) { + let block = this.getBlocks().find((b) => b.height === currentHeight); + if (!block && indexingAvailable) { + block = this.prepareBlock(await this.$indexBlock(currentHeight)); + } else if (!block) { + block = this.prepareBlock(await bitcoinApi.$getBlock(nextHash)); + } + returnBlocks.push(block); + nextHash = block.previousblockhash; + loadingIndicators.setProgress('blocks', i / 10 * 100); + currentHeight--; + } + + return returnBlocks; + } catch (e) { + loadingIndicators.setProgress('blocks', 100); + throw e; + } + } + + private prepareBlock(block: any): BlockExtended { + return { + id: block.id ?? block.hash, // hash for indexed block + timestamp: block?.timestamp ?? block?.blockTimestamp, // blockTimestamp for indexed block + height: block?.height, + version: block?.version, + bits: block?.bits, + nonce: block?.nonce, + difficulty: block?.difficulty, + merkle_root: block?.merkle_root, + tx_count: block?.tx_count, + size: block?.size, + weight: block?.weight, + previousblockhash: block?.previousblockhash, + extras: { + medianFee: block?.medianFee, + feeRange: block?.feeRange ?? [], // TODO + reward: block?.reward, + coinbaseHex: block?.extras?.coinbaseHex ?? block?.coinbase_raw, // coinbase_raw for indexed block + pool: block?.extras?.pool ?? (block?.pool_id ? { + id: block?.pool_id, + name: block?.pool_name, + link: block?.pool_link, + regexes: block?.pool_regexes, + addresses: block?.pool_addresses, + } : undefined), + } + }; + } + public getLastDifficultyAdjustmentTime(): number { return this.lastDifficultyAdjustmentTime; } diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 24ecc03cf..b8557ff65 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -6,7 +6,7 @@ import logger from '../logger'; const sleep = (ms: number) => new Promise(res => setTimeout(res, ms)); class DatabaseMigration { - private static currentVersion = 4; + private static currentVersion = 5; private queryTimeout = 120000; private statisticsAddedIndexed = false; @@ -229,6 +229,10 @@ class DatabaseMigration { } } + if (version < 5 && (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === true)) { + queries.push('ALTER TABLE blocks ADD `reward` double unsigned NOT NULL DEFAULT "0"'); + } + return queries; } diff --git a/backend/src/index.ts b/backend/src/index.ts index 557c269dd..07808c98a 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -290,6 +290,10 @@ class Server { ; } + this.app + .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-extras', routes.getBlocksExtras) + .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-extras/:height', routes.getBlocksExtras); + if (config.MEMPOOL.BACKEND !== 'esplora') { this.app .get(config.MEMPOOL.API_URL_PREFIX + 'mempool', routes.getMempool) diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index 7dfcd3956..5559c4588 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -83,10 +83,12 @@ export interface BlockExtension { reward?: number; coinbaseTx?: TransactionMinerInfo; matchRate?: number; + coinbaseHex?: string; + pool?: PoolTag; } export interface BlockExtended extends IEsploraApi.Block { - extras?: BlockExtension; + extras: BlockExtension; } export interface TransactionMinerInfo { diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 654376402..758ae1197 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -11,38 +11,36 @@ class BlocksRepository { /** * Save indexed block data in the database */ - public async $saveBlockInDatabase( - block: BlockExtended, - blockHash: string, - coinbaseHex: string | undefined, - poolTag: PoolTag - ) { + public async $saveBlockInDatabase(block: BlockExtended) { const connection = await DB.pool.getConnection(); try { const query = `INSERT INTO blocks( height, hash, blockTimestamp, size, weight, tx_count, coinbase_raw, difficulty, - pool_id, fees, fee_span, median_fee + pool_id, fees, fee_span, median_fee, + reward ) VALUE ( ?, ?, FROM_UNIXTIME(?), ?, ?, ?, ?, ?, - ?, ?, ?, ? + ?, ?, ?, ?, + ? )`; const params: any[] = [ block.height, - blockHash, + block.id, block.timestamp, block.size, block.weight, block.tx_count, - coinbaseHex ? coinbaseHex : '', + block.extras?.coinbaseHex ?? '', block.difficulty, - poolTag.id, + block.extras?.pool?.id, // Should always be set to something 0, '[]', - block.extras ? block.extras.medianFee : 0, + block.extras.medianFee ?? 0, + block.extras?.reward ?? 0, ]; await connection.query(query, params); @@ -136,6 +134,26 @@ class BlocksRepository { return rows[0].blockTimestamp; } + + /** + * Get one block by height + */ + public async $getBlockByHeight(height: number): Promise { + const connection = await DB.pool.getConnection(); + const [rows]: any[] = await connection.query(` + SELECT *, UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp, pools.id as pool_id, pools.name as pool_name, pools.link as pool_link, pools.addresses as pool_addresses, pools.regexes as pool_regexes + FROM blocks + JOIN pools ON blocks.pool_id = pools.id + WHERE height = ${height}; + `); + connection.release(); + + if (rows.length <= 0) { + return null; + } + + return rows[0]; + } } export default new BlocksRepository(); \ No newline at end of file diff --git a/backend/src/routes.ts b/backend/src/routes.ts index 8ae2f9609..e06177ddd 100644 --- a/backend/src/routes.ts +++ b/backend/src/routes.ts @@ -564,6 +564,14 @@ class Routes { } } + public async getBlocksExtras(req: Request, res: Response) { + try { + res.json(await blocks.$getBlocksExtras(parseInt(req.params.height, 10))) + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + public async getBlocks(req: Request, res: Response) { try { loadingIndicators.setProgress('blocks', 0);