Added MEMPOOL.INDEXING_BLOCKS_AMOUNT in the config (default 432 blocks)

backend/src/api/blocks.ts

@@ -146,31 +146,35 @@ class Blocks {
    * Index all blocks metadata for the mining dashboard
    */
   public async $generateBlockDatabase() {
-    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
+    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false ||
+      config.MEMPOOL.INDEXING_BLOCKS_AMOUNT <= 0) {
       return;
     }
 
     try {
       let currentBlockHeight = await bitcoinClient.getBlockCount();
-      const indexedBlockCount = await blocksRepository.$blockCount();
+      const lastBlockToIndex = currentBlockHeight - config.MEMPOOL.INDEXING_BLOCKS_AMOUNT + 1;
 
-      logger.info(`Starting block indexing. Current tip at block #${currentBlockHeight}`);
+      logger.info(`Indexing blocks from #${currentBlockHeight} to #${lastBlockToIndex}`);
-      logger.info(`Need to index ${currentBlockHeight - indexedBlockCount} blocks. Working on it!`);
 
       const chunkSize = 10000;
-      while (currentBlockHeight >= 0) {
+      while (currentBlockHeight >= lastBlockToIndex) {
-        const endBlock = Math.max(0, currentBlockHeight - chunkSize + 1);
+        const endBlock = Math.max(0, lastBlockToIndex, currentBlockHeight - chunkSize + 1);
+
         const missingBlockHeights: number[] = await blocksRepository.$getMissingBlocksBetweenHeights(
           currentBlockHeight, endBlock);
         if (missingBlockHeights.length <= 0) {
-          logger.debug(`No missing blocks between #${currentBlockHeight} to #${endBlock}, moving on`);
+          logger.debug(`No missing blocks between #${currentBlockHeight} to #${endBlock}`);
           currentBlockHeight -= chunkSize;
           continue;
         }
 
-        logger.info(`Indexing ${chunkSize} blocks from #${currentBlockHeight} to #${endBlock}`);
+        logger.debug(`Indexing ${missingBlockHeights.length} blocks from #${currentBlockHeight} to #${endBlock}`);
 
         for (const blockHeight of missingBlockHeights) {
+          if (blockHeight < lastBlockToIndex) {
+            break;
+          }
           try {
             logger.debug(`Indexing block #${blockHeight}`);
             const blockHash = await bitcoinApi.$getBlockHash(blockHeight);
@@ -187,8 +191,10 @@ class Blocks {
 
         currentBlockHeight -= chunkSize;
       }
+      logger.info('Block indexing completed');
     } catch (e) {
       logger.err('An error occured in $generateBlockDatabase(). Skipping block indexing. ' + e);
+      console.log(e);
     }
   }
 

backend/src/config.ts

@@ -14,6 +14,7 @@ interface IConfig {
     BLOCK_WEIGHT_UNITS: number;
     INITIAL_BLOCKS_AMOUNT: number;
     MEMPOOL_BLOCKS_AMOUNT: number;
+    INDEXING_BLOCKS_AMOUNT: number;
     PRICE_FEED_UPDATE_INTERVAL: number;
     USE_SECOND_NODE_FOR_MINFEE: boolean;
     EXTERNAL_ASSETS: string[];
@@ -77,6 +78,7 @@ const defaults: IConfig = {
     'BLOCK_WEIGHT_UNITS': 4000000,
     'INITIAL_BLOCKS_AMOUNT': 8,
     'MEMPOOL_BLOCKS_AMOUNT': 8,
+    'INDEXING_BLOCKS_AMOUNT': 432, // ~3 days at 10 minutes / block. Set to 0 to disable indexing
     'PRICE_FEED_UPDATE_INTERVAL': 3600,
     'USE_SECOND_NODE_FOR_MINFEE': false,
     'EXTERNAL_ASSETS': [

backend/src/repositories/BlocksRepository.ts

@@ -48,6 +48,10 @@ class BlocksRepository {
    * Get all block height that have not been indexed between [startHeight, endHeight]
    */
   public async $getMissingBlocksBetweenHeights(startHeight: number, endHeight: number): Promise<number[]> {
+    if (startHeight < endHeight) {
+      return [];
+    }
+
     const connection = await DB.pool.getConnection();
     const [rows] : any[] = await connection.query(`
       SELECT height

docker/backend/start.sh

@@ -13,6 +13,7 @@ __MEMPOOL_RECOMMENDED_FEE_PERCENTILE__=${MEMPOOL_RECOMMENDED_FEE_PERCENTILE:=50}
 __MEMPOOL_BLOCK_WEIGHT_UNITS__=${MEMPOOL_BLOCK_WEIGHT_UNITS:=4000000}
 __MEMPOOL_INITIAL_BLOCKS_AMOUNT__=${MEMPOOL_INITIAL_BLOCKS_AMOUNT:=8}
 __MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__=${MEMPOOL_MEMPOOL_BLOCKS_AMOUNT:=8}
+__MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=432}
 __MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__=${MEMPOOL_PRICE_FEED_UPDATE_INTERVAL:=3600}
 __MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__=${MEMPOOL_USE_SECOND_NODE_FOR_MINFEE:=false}
 __MEMPOOL_EXTERNAL_ASSETS__=${MEMPOOL_EXTERNAL_ASSETS:=[]}
@@ -74,6 +75,7 @@ sed -i "s/__MEMPOOL_RECOMMENDED_FEE_PERCENTILE__/${__MEMPOOL_RECOMMENDED_FEE_PER
 sed -i "s/__MEMPOOL_BLOCK_WEIGHT_UNITS__/${__MEMPOOL_BLOCK_WEIGHT_UNITS__}/g" mempool-config.json
 sed -i "s/__MEMPOOL_INITIAL_BLOCKS_AMOUNT__/${__MEMPOOL_INITIAL_BLOCKS_AMOUNT__}/g" mempool-config.json
 sed -i "s/__MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__/${__MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__}/g" mempool-config.json
+sed -i "s/__MEMPOOL_INDEXING_BLOCKS_AMOUNT__/${__MEMPOOL_INDEXING_BLOCKS_AMOUNT__}/g" mempool-config.json
 sed -i "s/__MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__/${__MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__}/g" mempool-config.json
 sed -i "s/__MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__/${__MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__}/g" mempool-config.json
 sed -i "s/__MEMPOOL_EXTERNAL_ASSETS__/${__MEMPOOL_EXTERNAL_ASSETS__}/g" mempool-config.json

frontend/mempool-frontend-config.sample.json

@@ -10,8 +10,8 @@
   "NGINX_PROTOCOL": "http",
   "NGINX_HOSTNAME": "127.0.0.1",
   "NGINX_PORT": "80",
-  "BLOCK_WEIGHT_UNITS": 4000000,
   "MEMPOOL_BLOCKS_AMOUNT": 8,
+  "INDEXING_BLOCKS_AMOUNT": 432,
   "BASE_MODULE": "mempool",
   "MEMPOOL_WEBSITE_URL": "https://mempool.space",
   "LIQUID_WEBSITE_URL": "https://liquid.network",