From 647f12ffaab15069cc59a32a2198d30c0eeb0f58 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Mon, 24 Jan 2022 17:43:11 +0900 Subject: [PATCH] Added MEMPOOL.INDEXING_BLOCKS_AMOUNT in the config (default 432 blocks) --- backend/src/api/blocks.ts | 22 +++++++++++++------- backend/src/config.ts | 2 ++ backend/src/repositories/BlocksRepository.ts | 4 ++++ docker/backend/start.sh | 2 ++ frontend/mempool-frontend-config.sample.json | 2 +- 5 files changed, 23 insertions(+), 9 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 7569001e2..49c080896 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -146,31 +146,35 @@ class Blocks { * Index all blocks metadata for the mining dashboard */ public async $generateBlockDatabase() { - if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { + if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false || + config.MEMPOOL.INDEXING_BLOCKS_AMOUNT <= 0) { return; } try { let currentBlockHeight = await bitcoinClient.getBlockCount(); - const indexedBlockCount = await blocksRepository.$blockCount(); + const lastBlockToIndex = currentBlockHeight - config.MEMPOOL.INDEXING_BLOCKS_AMOUNT + 1; - logger.info(`Starting block indexing. Current tip at block #${currentBlockHeight}`); - logger.info(`Need to index ${currentBlockHeight - indexedBlockCount} blocks. Working on it!`); + logger.info(`Indexing blocks from #${currentBlockHeight} to #${lastBlockToIndex}`); const chunkSize = 10000; - while (currentBlockHeight >= 0) { - const endBlock = Math.max(0, currentBlockHeight - chunkSize + 1); + while (currentBlockHeight >= lastBlockToIndex) { + const endBlock = Math.max(0, lastBlockToIndex, currentBlockHeight - chunkSize + 1); + const missingBlockHeights: number[] = await blocksRepository.$getMissingBlocksBetweenHeights( currentBlockHeight, endBlock); if (missingBlockHeights.length <= 0) { - logger.debug(`No missing blocks between #${currentBlockHeight} to #${endBlock}, moving on`); + logger.debug(`No missing blocks between #${currentBlockHeight} to #${endBlock}`); currentBlockHeight -= chunkSize; continue; } - logger.info(`Indexing ${chunkSize} blocks from #${currentBlockHeight} to #${endBlock}`); + logger.debug(`Indexing ${missingBlockHeights.length} blocks from #${currentBlockHeight} to #${endBlock}`); for (const blockHeight of missingBlockHeights) { + if (blockHeight < lastBlockToIndex) { + break; + } try { logger.debug(`Indexing block #${blockHeight}`); const blockHash = await bitcoinApi.$getBlockHash(blockHeight); @@ -187,8 +191,10 @@ class Blocks { currentBlockHeight -= chunkSize; } + logger.info('Block indexing completed'); } catch (e) { logger.err('An error occured in $generateBlockDatabase(). Skipping block indexing. ' + e); + console.log(e); } } diff --git a/backend/src/config.ts b/backend/src/config.ts index 3cc928327..7cd789934 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -14,6 +14,7 @@ interface IConfig { BLOCK_WEIGHT_UNITS: number; INITIAL_BLOCKS_AMOUNT: number; MEMPOOL_BLOCKS_AMOUNT: number; + INDEXING_BLOCKS_AMOUNT: number; PRICE_FEED_UPDATE_INTERVAL: number; USE_SECOND_NODE_FOR_MINFEE: boolean; EXTERNAL_ASSETS: string[]; @@ -77,6 +78,7 @@ const defaults: IConfig = { 'BLOCK_WEIGHT_UNITS': 4000000, 'INITIAL_BLOCKS_AMOUNT': 8, 'MEMPOOL_BLOCKS_AMOUNT': 8, + 'INDEXING_BLOCKS_AMOUNT': 432, // ~3 days at 10 minutes / block. Set to 0 to disable indexing 'PRICE_FEED_UPDATE_INTERVAL': 3600, 'USE_SECOND_NODE_FOR_MINFEE': false, 'EXTERNAL_ASSETS': [ diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 8d36d8410..413d4a226 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -48,6 +48,10 @@ class BlocksRepository { * Get all block height that have not been indexed between [startHeight, endHeight] */ public async $getMissingBlocksBetweenHeights(startHeight: number, endHeight: number): Promise { + if (startHeight < endHeight) { + return []; + } + const connection = await DB.pool.getConnection(); const [rows] : any[] = await connection.query(` SELECT height diff --git a/docker/backend/start.sh b/docker/backend/start.sh index f982a18e5..f9c784641 100644 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -13,6 +13,7 @@ __MEMPOOL_RECOMMENDED_FEE_PERCENTILE__=${MEMPOOL_RECOMMENDED_FEE_PERCENTILE:=50} __MEMPOOL_BLOCK_WEIGHT_UNITS__=${MEMPOOL_BLOCK_WEIGHT_UNITS:=4000000} __MEMPOOL_INITIAL_BLOCKS_AMOUNT__=${MEMPOOL_INITIAL_BLOCKS_AMOUNT:=8} __MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__=${MEMPOOL_MEMPOOL_BLOCKS_AMOUNT:=8} +__MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=432} __MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__=${MEMPOOL_PRICE_FEED_UPDATE_INTERVAL:=3600} __MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__=${MEMPOOL_USE_SECOND_NODE_FOR_MINFEE:=false} __MEMPOOL_EXTERNAL_ASSETS__=${MEMPOOL_EXTERNAL_ASSETS:=[]} @@ -74,6 +75,7 @@ sed -i "s/__MEMPOOL_RECOMMENDED_FEE_PERCENTILE__/${__MEMPOOL_RECOMMENDED_FEE_PER sed -i "s/__MEMPOOL_BLOCK_WEIGHT_UNITS__/${__MEMPOOL_BLOCK_WEIGHT_UNITS__}/g" mempool-config.json sed -i "s/__MEMPOOL_INITIAL_BLOCKS_AMOUNT__/${__MEMPOOL_INITIAL_BLOCKS_AMOUNT__}/g" mempool-config.json sed -i "s/__MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__/${__MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__}/g" mempool-config.json +sed -i "s/__MEMPOOL_INDEXING_BLOCKS_AMOUNT__/${__MEMPOOL_INDEXING_BLOCKS_AMOUNT__}/g" mempool-config.json sed -i "s/__MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__/${__MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__}/g" mempool-config.json sed -i "s/__MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__/${__MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__}/g" mempool-config.json sed -i "s/__MEMPOOL_EXTERNAL_ASSETS__/${__MEMPOOL_EXTERNAL_ASSETS__}/g" mempool-config.json diff --git a/frontend/mempool-frontend-config.sample.json b/frontend/mempool-frontend-config.sample.json index 0715cb0bd..5cb296537 100644 --- a/frontend/mempool-frontend-config.sample.json +++ b/frontend/mempool-frontend-config.sample.json @@ -10,8 +10,8 @@ "NGINX_PROTOCOL": "http", "NGINX_HOSTNAME": "127.0.0.1", "NGINX_PORT": "80", - "BLOCK_WEIGHT_UNITS": 4000000, "MEMPOOL_BLOCKS_AMOUNT": 8, + "INDEXING_BLOCKS_AMOUNT": 432, "BASE_MODULE": "mempool", "MEMPOOL_WEBSITE_URL": "https://mempool.space", "LIQUID_WEBSITE_URL": "https://liquid.network",