Index more data using getblockstats core RPC

backend/src/api/blocks.ts

@@ -111,14 +111,20 @@ class Blocks {
     const transactionsTmp = [...transactions];
     transactionsTmp.shift();
     transactionsTmp.sort((a, b) => b.effectiveFeePerVsize - a.effectiveFeePerVsize);
-
     blockExtended.extras.medianFee = transactionsTmp.length > 0 ?
       Common.median(transactionsTmp.map((tx) => tx.effectiveFeePerVsize)) : 0;
-    blockExtended.extras.feeRange = transactionsTmp.length > 0 ?
+
-      Common.getFeesInRange(transactionsTmp, 8) : [0, 0];
+    const stats = await bitcoinClient.getBlockStats(block.id);
-    blockExtended.extras.totalFees = transactionsTmp.reduce((acc, tx) => {
+    blockExtended.extras.feeRange = stats.feerate_percentiles;
-      return acc + tx.fee;
+    blockExtended.extras.totalFees = stats.totalfee;
-    }, 0)
+    blockExtended.extras.avgFee = stats.avgfee;
+    blockExtended.extras.avgFeeRate = stats.avgfeerate;
+    blockExtended.extras.maxFee = stats.maxfee;
+    blockExtended.extras.maxFeeRate = stats.maxfeerate;
+    blockExtended.extras.minFee = stats.minfee;
+    blockExtended.extras.minFeeRate = stats.minfeerate;
+    blockExtended.extras.subsidy = stats.subsidy;
+    blockExtended.extras.medianFeeValue = stats.medianfee;
 
     if (Common.indexingEnabled()) {
       let pool: PoolTag;

backend/src/api/database-migration.ts

@@ -6,7 +6,7 @@ import logger from '../logger';
 const sleep = (ms: number) => new Promise(res => setTimeout(res, ms));
 
 class DatabaseMigration {
-  private static currentVersion = 10;
+  private static currentVersion = 11;
   private queryTimeout = 120000;
   private statisticsAddedIndexed = false;
 
@@ -92,13 +92,13 @@ class DatabaseMigration {
         await this.$executeQuery(connection, this.getCreateBlocksTableQuery(), await this.$checkIfTableExists('blocks'));
       }
       if (databaseSchemaVersion < 5 && isBitcoin === true) {
-        logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.'`);
+        logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.`);
         await this.$executeQuery(connection, 'TRUNCATE blocks;'); // Need to re-index
         await this.$executeQuery(connection, 'ALTER TABLE blocks ADD `reward` double unsigned NOT NULL DEFAULT "0"');
       }
 
       if (databaseSchemaVersion < 6 && isBitcoin === true) {
-        logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.'`);
+        logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.`);
         await this.$executeQuery(connection, 'TRUNCATE blocks;');  // Need to re-index
         // Cleanup original blocks fields type
         await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `height` integer unsigned NOT NULL DEFAULT "0"');
@@ -125,7 +125,7 @@ class DatabaseMigration {
       }
 
       if (databaseSchemaVersion < 8 && isBitcoin === true) {
-        logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.'`);
+        logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.`);
         await this.$executeQuery(connection, 'TRUNCATE hashrates;'); // Need to re-index
         await this.$executeQuery(connection, 'ALTER TABLE `hashrates` DROP INDEX `PRIMARY`');
         await this.$executeQuery(connection, 'ALTER TABLE `hashrates` ADD `id` int NOT NULL AUTO_INCREMENT PRIMARY KEY FIRST');
@@ -134,7 +134,7 @@ class DatabaseMigration {
       }
 
       if (databaseSchemaVersion < 9 && isBitcoin === true) {
-        logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.'`);
+        logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.`);
         await this.$executeQuery(connection, 'TRUNCATE hashrates;'); // Need to re-index
         await this.$executeQuery(connection, 'ALTER TABLE `state` CHANGE `name` `name` varchar(100)');
         await this.$executeQuery(connection, 'ALTER TABLE `hashrates` ADD UNIQUE `hashrate_timestamp_pool_id` (`hashrate_timestamp`, `pool_id`)');
@@ -144,6 +144,21 @@ class DatabaseMigration {
         await this.$executeQuery(connection, 'ALTER TABLE `blocks` ADD INDEX `blockTimestamp` (`blockTimestamp`)');
       }
 
+      if (databaseSchemaVersion < 11 && isBitcoin === true) {
+        logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.`);
+        await this.$executeQuery(connection, 'TRUNCATE blocks;'); // Need to re-index
+        await this.$executeQuery(connection, `ALTER TABLE blocks
+          ADD avg_fee int unsigned NULL,
+          ADD avg_fee_rate int unsigned NULL,
+          ADD max_fee int unsigned NULL,
+          ADD max_fee_rate int unsigned NULL,
+          ADD min_fee int unsigned NULL,
+          ADD min_fee_rate int unsigned NULL,
+          ADD median_fee_value int unsigned NULL,
+          ADD subsidy float unsigned NULL;
+        `);
+      }
+
       connection.release();
     } catch (e) {
       connection.release();

backend/src/mempool.interfaces.ts

@@ -79,7 +79,7 @@ export interface TransactionStripped {
 
 export interface BlockExtension {
   totalFees?: number;
-  medianFee?: number;
+  medianFee?: number; // Actually the median fee rate that we compute ourself
   feeRange?: number[];
   reward?: number;
   coinbaseTx?: TransactionMinerInfo;
@@ -87,7 +87,15 @@ export interface BlockExtension {
   pool?: {
     id: number;
     name: string;
-  }
+  };
+  avgFee?: number;
+  avgFeeRate?: number;
+  maxFee?: number;
+  maxFeeRate?: number;
+  minFee?: number;
+  minFeeRate?: number;
+  subsidy?: number;
+  medianFeeValue?: number; // The actual median fee amount from getblockstats RPC
 }
 
 export interface BlockExtended extends IEsploraApi.Block {

backend/src/repositories/BlocksRepository.ts

@@ -12,17 +12,21 @@ class BlocksRepository {
 
     try {
       const query = `INSERT INTO blocks(
-        height,  hash,     blockTimestamp, size,
+        height,           hash,                blockTimestamp, size,
-        weight,  tx_count, coinbase_raw,   difficulty,
+        weight,           tx_count,            coinbase_raw,   difficulty,
-        pool_id, fees,     fee_span,       median_fee,
+        pool_id,          fees,                fee_span,       median_fee,
-        reward,  version,  bits,           nonce,
+        reward,           version,             bits,           nonce,
-        merkle_root,       previous_block_hash
+        merkle_root,      previous_block_hash, avg_fee,        avg_fee_rate,
+        max_fee,          max_fee_rate,        min_fee,        min_fee_rate,
+        median_fee_value, subsidy
       ) VALUE (
         ?, ?, FROM_UNIXTIME(?), ?,
         ?, ?, ?, ?,
         ?, ?, ?, ?,
         ?, ?, ?, ?,
-        ?,    ?
+        ?, ?, ?, ?,
+        ?, ?, ?, ?,
+        ?, ?
       )`;
 
       const params: any[] = [
@@ -35,18 +39,25 @@ class BlocksRepository {
         '',
         block.difficulty,
         block.extras.pool?.id, // Should always be set to something
-        0,
+        block.extras.totalFees,
-        '[]',
+        JSON.stringify(block.extras.feeRange),
-        block.extras.medianFee ?? 0,
+        block.extras.medianFee,
-        block.extras.reward ?? 0,
+        block.extras.reward,
         block.version,
         block.bits,
         block.nonce,
         block.merkle_root,
-        block.previousblockhash
+        block.previousblockhash,
+        block.extras.avgFee,
+        block.extras.avgFeeRate,
+        block.extras.maxFee,
+        block.extras.maxFeeRate,
+        block.extras.minFee,
+        block.extras.minFeeRate,
+        block.extras.medianFeeValue,
+        block.extras.subsidy,
       ];
 
-      // logger.debug(query);
       await connection.query(query, params);
       connection.release();
     } catch (e: any) {
@@ -272,7 +283,7 @@ class BlocksRepository {
   /**
    * Get one block by height
    */
-   public async $getBlockByHeight(height: number): Promise<object | null> {
+  public async $getBlockByHeight(height: number): Promise<object | null> {
     const connection = await DB.pool.getConnection();
     try {
       const [rows]: any[] = await connection.query(`
@@ -298,7 +309,7 @@ class BlocksRepository {
   /**
    * Return blocks difficulty
    */
-   public async $getBlocksDifficulty(interval: string | null): Promise<object[]> {
+  public async $getBlocksDifficulty(interval: string | null): Promise<object[]> {
     interval = Common.getSqlInterval(interval);
 
     const connection = await DB.pool.getConnection();