Index more data using getblockstats core RPC

backend/src/api/blocks.ts

@@ -108,23 +108,14 @@ class Blocks {
     blockExtended.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
     blockExtended.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
 
-    const transactionsTmp = [...transactions];
-    transactionsTmp.shift();
-    transactionsTmp.sort((a, b) => b.effectiveFeePerVsize - a.effectiveFeePerVsize);
-    blockExtended.extras.medianFee = transactionsTmp.length > 0 ?
-      Common.median(transactionsTmp.map((tx) => tx.effectiveFeePerVsize)) : 0;
-
     const stats = await bitcoinClient.getBlockStats(block.id);
-    blockExtended.extras.feeRange = stats.feerate_percentiles;
+    const coinbaseRaw: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(transactions[0].txid, true);
+    blockExtended.extras.coinbaseRaw = coinbaseRaw.hex;
+    blockExtended.extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles
+    blockExtended.extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat();
     blockExtended.extras.totalFees = stats.totalfee;
     blockExtended.extras.avgFee = stats.avgfee;
     blockExtended.extras.avgFeeRate = stats.avgfeerate;
-    blockExtended.extras.maxFee = stats.maxfee;
-    blockExtended.extras.maxFeeRate = stats.maxfeerate;
-    blockExtended.extras.minFee = stats.minfee;
-    blockExtended.extras.minFeeRate = stats.minfeerate;
-    blockExtended.extras.subsidy = stats.subsidy;
-    blockExtended.extras.medianFeeValue = stats.medianfee;
 
     if (Common.indexingEnabled()) {
       let pool: PoolTag;
@@ -190,7 +181,6 @@ class Blocks {
     }
 
     this.blockIndexingStarted = true;
-    const startedAt = new Date().getTime() / 1000;
 
     try {
       let currentBlockHeight = blockchainInfo.blocks;
@@ -207,6 +197,9 @@ class Blocks {
       const chunkSize = 10000;
       let totaIndexed = await blocksRepository.$blockCount(null, null);
       let indexedThisRun = 0;
+      const startedAt = new Date().getTime() / 1000;
+      let timer = new Date().getTime() / 1000;
+
       while (currentBlockHeight >= lastBlockToIndex) {
         const endBlock = Math.max(0, lastBlockToIndex, currentBlockHeight - chunkSize + 1);
 
@@ -225,13 +218,16 @@ class Blocks {
             break;
           }
           ++indexedThisRun;
-          if (++totaIndexed % 100 === 0 || blockHeight === lastBlockToIndex) {
+          const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
-            const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
+          if (elapsedSeconds > 5 || blockHeight === lastBlockToIndex) {
+            const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
             const blockPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds));
             const progress = Math.round(totaIndexed / indexingBlockAmount * 100);
             const timeLeft = Math.round((indexingBlockAmount - totaIndexed) / blockPerSeconds);
-            logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds} blocks/sec | total: ${totaIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${elapsedSeconds} seconds | left: ~${timeLeft} seconds`);
+            logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds} blocks/sec | total: ${totaIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${runningFor} seconds | left: ~${timeLeft} seconds`);
-          }
+            timer = new Date().getTime() / 1000;
+            indexedThisRun = 0;
+            }
           const blockHash = await bitcoinApi.$getBlockHash(blockHeight);
           const block = await bitcoinApi.$getBlock(blockHash);
           const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, true);

backend/src/api/database-migration.ts

@@ -148,15 +148,12 @@ class DatabaseMigration {
         logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.`);
         await this.$executeQuery(connection, 'TRUNCATE blocks;'); // Need to re-index
         await this.$executeQuery(connection, `ALTER TABLE blocks
-          ADD avg_fee int unsigned NULL,
+          ADD avg_fee INT UNSIGNED NULL,
-          ADD avg_fee_rate int unsigned NULL,
+          ADD avg_fee_rate INT UNSIGNED NULL
-          ADD max_fee int unsigned NULL,
-          ADD max_fee_rate int unsigned NULL,
-          ADD min_fee int unsigned NULL,
-          ADD min_fee_rate int unsigned NULL,
-          ADD median_fee_value int unsigned NULL,
-          ADD subsidy float unsigned NULL;
         `);
+        await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `reward` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
+        await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `median_fee` INT UNSIGNED NOT NULL DEFAULT "0"');
+        await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `fees` INT UNSIGNED NOT NULL DEFAULT "0"');
       }
 
       connection.release();

backend/src/mempool.interfaces.ts

@@ -90,12 +90,7 @@ export interface BlockExtension {
   };
   avgFee?: number;
   avgFeeRate?: number;
-  maxFee?: number;
+  coinbaseRaw?: string;
-  maxFeeRate?: number;
-  minFee?: number;
-  minFeeRate?: number;
-  subsidy?: number;
-  medianFeeValue?: number; // The actual median fee amount from getblockstats RPC
 }
 
 export interface BlockExtended extends IEsploraApi.Block {

backend/src/repositories/BlocksRepository.ts

@@ -16,17 +16,13 @@ class BlocksRepository {
         weight,           tx_count,            coinbase_raw,   difficulty,
         pool_id,          fees,                fee_span,       median_fee,
         reward,           version,             bits,           nonce,
-        merkle_root,      previous_block_hash, avg_fee,        avg_fee_rate,
+        merkle_root,      previous_block_hash, avg_fee,        avg_fee_rate
-        max_fee,          max_fee_rate,        min_fee,        min_fee_rate,
-        median_fee_value, subsidy
       ) VALUE (
         ?, ?, FROM_UNIXTIME(?), ?,
         ?, ?, ?, ?,
         ?, ?, ?, ?,
         ?, ?, ?, ?,
-        ?, ?, ?, ?,
+        ?, ?, ?, ?
-        ?, ?, ?, ?,
-        ?, ?
       )`;
 
       const params: any[] = [
@@ -36,7 +32,7 @@ class BlocksRepository {
         block.size,
         block.weight,
         block.tx_count,
-        '',
+        block.extras.coinbaseRaw,
         block.difficulty,
         block.extras.pool?.id, // Should always be set to something
         block.extras.totalFees,
@@ -50,12 +46,6 @@ class BlocksRepository {
         block.previousblockhash,
         block.extras.avgFee,
         block.extras.avgFeeRate,
-        block.extras.maxFee,
-        block.extras.maxFeeRate,
-        block.extras.minFee,
-        block.extras.minFeeRate,
-        block.extras.medianFeeValue,
-        block.extras.subsidy,
       ];
 
       await connection.query(query, params);