Add separate effective fee stats fields

backend/src/api/blocks.ts

@@ -244,7 +244,7 @@ class Blocks {
    */
   private async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise<BlockExtended> {
     const coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
-    
+
     const blk: Partial<BlockExtended> = Object.assign({}, block);
     const extras: Partial<BlockExtension> = {};
 
@@ -268,15 +268,17 @@ class Blocks {
       extras.segwitTotalWeight = 0;
     } else {
       const stats: IBitcoinApi.BlockStats = await bitcoinClient.getBlockStats(block.id);
-      let feeStats = {
+      const feeStats = {
         medianFee: stats.feerate_percentiles[2], // 50th percentiles
         feeRange: [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat(),
       };
-      if (transactions?.length > 1) {
-        feeStats = Common.calcEffectiveFeeStatistics(transactions);
-      }
       extras.medianFee = feeStats.medianFee;
       extras.feeRange = feeStats.feeRange;
+      if (transactions?.length > 1) {
+        const effectiveFeeStats = Common.calcEffectiveFeeStatistics(transactions);
+        extras.effectiveMedianFee = effectiveFeeStats.effective_median;
+        extras.effectiveFeeRange = effectiveFeeStats.effective_range;
+      }
       extras.totalFees = stats.totalfee;
       extras.avgFee = stats.avgfee;
       extras.avgFeeRate = stats.avgfeerate;
@@ -296,7 +298,7 @@ class Blocks {
         extras.medianFeeAmt = extras.feePercentiles[3];
       }
     }
-  
+
     extras.virtualSize = block.weight / 4.0;
     if (coinbaseTx?.vout.length > 0) {
       extras.coinbaseAddress = coinbaseTx.vout[0].scriptpubkey_address ?? null;
@@ -1316,6 +1318,8 @@ class Blocks {
         avg_fee_rate: block.extras.avgFeeRate ?? null,
         median_fee_rate: block.extras.medianFee ?? null,
         fee_rate_percentiles: block.extras.feeRange ?? null,
+        effective_median_fee_rate: block.extras.effectiveMedianFee ?? null,
+        effective_fee_rate_percentiles: block.extras.effectiveFeeRange ?? null,
         total_inputs: block.extras.totalInputs ?? null,
         total_input_amt: block.extras.totalInputAmt ?? null,
         total_outputs: block.extras.totalOutputs ?? null,
@@ -1378,6 +1382,17 @@ class Blocks {
         'perc_90': cleanBlock.fee_rate_percentiles[5],
         'max': cleanBlock.fee_rate_percentiles[6],
       };
+      if (cleanBlock.effective_fee_rate_percentiles) {
+        cleanBlock.effective_fee_rate_percentiles = {
+          'min': cleanBlock.effective_fee_rate_percentiles[0],
+          'perc_10': cleanBlock.effective_fee_rate_percentiles[1],
+          'perc_25': cleanBlock.effective_fee_rate_percentiles[2],
+          'perc_50': cleanBlock.effective_fee_rate_percentiles[3],
+          'perc_75': cleanBlock.effective_fee_rate_percentiles[4],
+          'perc_90': cleanBlock.effective_fee_rate_percentiles[5],
+          'max': cleanBlock.effective_fee_rate_percentiles[6],
+        };
+      }
 
       // Re-org can happen after indexing so we need to always get the
       // latest state from core

backend/src/api/common.ts

@@ -1,6 +1,6 @@
 import * as bitcoinjs from 'bitcoinjs-lib';
 import { Request } from 'express';
-import { EffectiveFeeStats, MempoolBlockWithTransactions, TransactionExtended, MempoolTransactionExtended, TransactionStripped, WorkingEffectiveFeeStats, TransactionClassified, TransactionFlags } from '../mempool.interfaces';
+import { EffectiveFeeStats, MempoolBlockWithTransactions, TransactionExtended, MempoolTransactionExtended, TransactionStripped, WorkingEffectiveFeeStats, TransactionClassified, TransactionFlags, FeeStats } from '../mempool.interfaces';
 import config from '../config';
 import { NodeSocket } from '../repositories/NodesSocketsRepository';
 import { isIP } from 'net';
@@ -856,6 +856,15 @@ export class Common {
     }
   }
 
+  static calcFeeStatistics(transactions: { txid: string, feePerVsize: number }[]): FeeStats {
+    // skip the coinbase, then sort the remaining fee rates
+    const sortedRates = transactions.slice(1).map(tx => tx.feePerVsize).sort((a, b) => a - b);
+    return {
+      median: Math.round(Common.getNthPercentile(50, sortedRates)),
+      range: [0, 10, 25, 50, 75, 90, 100].map(n => Math.round(Common.getNthPercentile(n, sortedRates))),
+    };
+  }
+
   static calcEffectiveFeeStatistics(transactions: { weight: number, fee: number, effectiveFeePerVsize?: number, txid: string, acceleration?: boolean }[]): EffectiveFeeStats {
     const sortedTxs = transactions.map(tx => { return { txid: tx.txid, weight: tx.weight, rate: tx.effectiveFeePerVsize || ((tx.fee || 0) / (tx.weight / 4)) }; }).sort((a, b) => a.rate - b.rate);
 
@@ -898,8 +907,8 @@ export class Common {
     );
 
     return {
-      medianFee: medianFeeRate,
+      effective_median: medianFeeRate,
-      feeRange: [
+      effective_range: [
         minFee,
         [10,25,50,75,90].map(n => Common.getNthPercentile(n, sortedTxs).rate),
         maxFee,
@@ -1150,16 +1159,16 @@ export class OnlineFeeStatsCalculator {
     }
     return {
       minFee: this.feeRange[0].min,
-      medianFee: this.feeRange[Math.floor(this.feeRange.length / 2)].avg,
+      effective_median: this.feeRange[Math.floor(this.feeRange.length / 2)].avg,
       maxFee: this.feeRange[this.feeRange.length - 1].max,
-      feeRange: this.feeRange.map(f => f.avg),
+      effective_range: this.feeRange.map(f => f.avg),
     };
   }
 
   getFeeStats(): EffectiveFeeStats {
     const stats = this.getRawFeeStats();
-    stats.feeRange[0] = stats.minFee;
+    stats.effective_range[0] = stats.minFee;
-    stats.feeRange[stats.feeRange.length - 1] = stats.maxFee;
+    stats.effective_range[stats.effective_range.length - 1] = stats.maxFee;
     return stats;
   }
 }

backend/src/api/database-migration.ts

@@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
 import { RowDataPacket } from 'mysql2';
 
 class DatabaseMigration {
-  private static currentVersion = 94;
+  private static currentVersion = 95;
   private queryTimeout = 3600_000;
   private statisticsAddedIndexed = false;
   private uniqueLogs: string[] = [];
@@ -1118,6 +1118,16 @@ class DatabaseMigration {
       }
       await this.updateToSchemaVersion(94);
     }
+
+    if (databaseSchemaVersion < 95) {
+      // Version 95
+      await this.$executeQuery(`
+        ALTER TABLE \`blocks\`
+          ADD \`effective_median_fee\` BIGINT UNSIGNED NOT NULL DEFAULT 0,
+          ADD \`effective_fee_span\` JSON DEFAULT NULL;
+      `);
+      await this.updateToSchemaVersion(95);
+    }
   }
 
   /**

backend/src/api/fee-api.ts

@@ -63,7 +63,8 @@ class FeeApi {
   }
 
   private optimizeMedianFee(pBlock: MempoolBlock, nextBlock: MempoolBlock | undefined, previousFee?: number): number {
-    const useFee = previousFee ? (pBlock.medianFee + previousFee) / 2 : pBlock.medianFee;
+    const medianFee = pBlock.effectiveMedianFee ?? pBlock.medianFee;
+    const useFee = previousFee ? (medianFee + previousFee) / 2 : medianFee;
     if (pBlock.blockVSize <= 500000) {
       return this.defaultFee;
     }

backend/src/api/mempool-blocks.ts

@@ -1,6 +1,6 @@
 import { GbtGenerator, GbtResult, ThreadTransaction as RustThreadTransaction, ThreadAcceleration as RustThreadAcceleration } from 'rust-gbt';
 import logger from '../logger';
-import { MempoolBlock, MempoolTransactionExtended, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats, TransactionClassified, TransactionCompressed, MempoolDeltaChange, GbtCandidates, PoolTag } from '../mempool.interfaces';
+import { MempoolBlock, MempoolTransactionExtended, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, FeeStats, TransactionClassified, TransactionCompressed, MempoolDeltaChange, GbtCandidates, PoolTag, EffectiveFeeStats } from '../mempool.interfaces';
 import { Common, OnlineFeeStatsCalculator } from './common';
 import config from '../config';
 import { Worker } from 'worker_threads';
@@ -33,6 +33,8 @@ class MempoolBlocks {
         totalFees: block.totalFees,
         medianFee: block.medianFee,
         feeRange: block.feeRange,
+        effectiveMedianFee: block.effectiveMedianFee,
+        effectiveFeeRange: block.effectiveFeeRange,
       };
     });
   }
@@ -527,7 +529,7 @@ class MempoolBlocks {
         totalSize,
         totalWeight,
         totalFees,
-        (hasBlockStack && blockIndex === lastBlockIndex && feeStatsCalculator) ? feeStatsCalculator.getRawFeeStats() : undefined,
+        (hasBlockStack && blockIndex === lastBlockIndex && feeStatsCalculator) ? feeStatsCalculator.getFeeStats() : undefined,
       );
     };
 
@@ -541,17 +543,20 @@ class MempoolBlocks {
     return mempoolBlocks;
   }
 
-  private dataToMempoolBlocks(transactionIds: string[], transactions: MempoolTransactionExtended[], totalSize: number, totalWeight: number, totalFees: number, feeStats?: EffectiveFeeStats ): MempoolBlockWithTransactions {
+  private dataToMempoolBlocks(transactionIds: string[], transactions: MempoolTransactionExtended[], totalSize: number, totalWeight: number, totalFees: number, effectiveFeeStats?: EffectiveFeeStats ): MempoolBlockWithTransactions {
-    if (!feeStats) {
+    const feeStats = Common.calcFeeStatistics(transactions);
-      feeStats = Common.calcEffectiveFeeStatistics(transactions);
+    if (!effectiveFeeStats) {
+      effectiveFeeStats = Common.calcEffectiveFeeStatistics(transactions);
     }
     return {
       blockSize: totalSize,
       blockVSize: (totalWeight / 4), // fractional vsize to avoid rounding errors
       nTx: transactionIds.length,
       totalFees: totalFees,
-      medianFee: feeStats.medianFee, // Common.percentile(transactions.map((tx) => tx.effectiveFeePerVsize), config.MEMPOOL.RECOMMENDED_FEE_PERCENTILE),
+      medianFee: feeStats.median,
-      feeRange: feeStats.feeRange, //Common.getFeesInRange(transactions, rangeLength),
+      feeRange: feeStats.range,
+      effectiveMedianFee: effectiveFeeStats.effective_median,
+      effectiveFeeRange: effectiveFeeStats.effective_range,
       transactionIds: transactionIds,
       transactions: transactions.map((tx) => Common.classifyTransaction(tx)),
     };

backend/src/mempool.interfaces.ts

@@ -73,6 +73,8 @@ export interface MempoolBlock {
   medianFee: number;
   totalFees: number;
   feeRange: number[];
+  effectiveMedianFee?: number;
+  effectiveFeeRange?: number[];
 }
 
 export interface MempoolBlockWithTransactions extends MempoolBlock {
@@ -288,8 +290,10 @@ export const TransactionFlags = {
 
 export interface BlockExtension {
   totalFees: number;
-  medianFee: number; // median fee rate
+  medianFee: number; // core median fee rate
-  feeRange: number[]; // fee rate percentiles
+  feeRange: number[]; // core fee rate percentiles
+  effectiveMedianFee?: number; // effective median fee rate
+  effectiveFeeRange?: number[]; // effective fee rate percentiles
   reward: number;
   matchRate: number | null;
   expectedFees: number | null;
@@ -369,9 +373,18 @@ export interface MempoolStats {
   tx_count: number;
 }
 
+// Core fee stats
+// measured in individual sats/vbyte
+export interface FeeStats {
+  median: number; // median core fee rate
+  range: number[]; // 0th, 10th, 25th, 50th, 75th, 90th, 100th percentiles
+}
+
+// Mempool effective fee stats
+// measured in effective sats/vbyte
 export interface EffectiveFeeStats {
-  medianFee: number; // median effective fee rate
+  effective_median: number; // median effective fee rate by weight
-  feeRange: number[]; // 2nd, 10th, 25th, 50th, 75th, 90th, 98th percentiles
+  effective_range: number[]; // 2nd, 10th, 25th, 50th, 75th, 90th, 98th percentiles
 }
 
 export interface WorkingEffectiveFeeStats extends EffectiveFeeStats {

backend/src/repositories/AccelerationRepository.ts

@@ -315,12 +315,12 @@ class AccelerationRepository {
             Infinity
           );
           const feeStats = Common.calcEffectiveFeeStatistics(template);
-          boostRate = feeStats.medianFee;
+          boostRate = feeStats.effective_median;
         }
         const accelerationSummaries = accelerations.map(acc => ({
           ...acc,
           pools: acc.pools,
-        }))
+        }));
         for (const acc of accelerations) {
           if (blockTxs[acc.txid] && acc.pools.includes(block.extras.pool.id)) {
             const tx = blockTxs[acc.txid];

backend/src/repositories/BlocksRepository.ts

@@ -33,6 +33,8 @@ interface DatabaseBlock {
   totalFees: number;
   medianFee: number;
   feeRange: string;
+  effectiveMedianFee?: number;
+  effectiveFeeRange?: string;
   reward: number;
   poolId: number;
   poolName: string;
@@ -77,6 +79,8 @@ const BLOCK_DB_FIELDS = `
   blocks.fees AS totalFees,
   blocks.median_fee AS medianFee,
   blocks.fee_span AS feeRange,
+  blocks.effective_median_fee AS effectiveMedianFee,
+  blocks.effective_fee_span AS effectiveFeeRange,
   blocks.reward,
   pools.unique_id AS poolId,
   pools.name AS poolName,
@@ -108,7 +112,7 @@ class BlocksRepository {
   /**
    * Save indexed block data in the database
    */
-  public async $saveBlockInDatabase(block: BlockExtended) {
+  public async $saveBlockInDatabase(block: BlockExtended): Promise<void> {
     const truncatedCoinbaseSignature = block?.extras?.coinbaseSignature?.substring(0, 500);
     const truncatedCoinbaseSignatureAscii = block?.extras?.coinbaseSignatureAscii?.substring(0, 500);
 
@@ -117,6 +121,7 @@ class BlocksRepository {
         height,             hash,                blockTimestamp,    size,
         weight,             tx_count,            coinbase_raw,      difficulty,
         pool_id,            fees,                fee_span,          median_fee,
+        effective_fee_span, effective_median_fee,
         reward,             version,             bits,              nonce,
         merkle_root,        previous_block_hash, avg_fee,           avg_fee_rate,
         median_timestamp,   header,              coinbase_address,  coinbase_addresses,
@@ -128,6 +133,7 @@ class BlocksRepository {
         ?, ?, FROM_UNIXTIME(?), ?,
         ?, ?, ?, ?,
         ?, ?, ?, ?,
+        ?, ?,
         ?, ?, ?, ?,
         ?, ?, ?, ?,
         FROM_UNIXTIME(?), ?, ?, ?,
@@ -155,6 +161,8 @@ class BlocksRepository {
         block.extras.totalFees,
         JSON.stringify(block.extras.feeRange),
         block.extras.medianFee,
+        block.extras.effectiveFeeRange ? JSON.stringify(block.extras.effectiveFeeRange) : null,
+        block.extras.effectiveMedianFee,
         block.extras.reward,
         block.version,
         block.bits,
@@ -968,16 +976,16 @@ class BlocksRepository {
 
   /**
    * Save indexed effective fee statistics
-   * 
+   *
-   * @param id 
+   * @param id
-   * @param feeStats 
+   * @param feeStats
    */
   public async $saveEffectiveFeeStats(id: string, feeStats: EffectiveFeeStats): Promise<void> {
     try {
       await DB.query(`
-        UPDATE blocks SET median_fee = ?, fee_span = ?
+        UPDATE blocks SET effective_median_fee = ?, effective_fee_span = ?
         WHERE hash = ?`,
-        [feeStats.medianFee, JSON.stringify(feeStats.feeRange), id]
+        [feeStats.effective_median, JSON.stringify(feeStats.effective_range), id]
       );
     } catch (e) {
       logger.err(`Cannot update block fee stats. Reason: ` + (e instanceof Error ? e.message : e));
@@ -1065,11 +1073,13 @@ class BlocksRepository {
     blk.weight = dbBlk.weight;
     blk.previousblockhash = dbBlk.previousblockhash;
     blk.mediantime = dbBlk.mediantime;
-    
+
     // BlockExtension
     extras.totalFees = dbBlk.totalFees;
     extras.medianFee = dbBlk.medianFee;
     extras.feeRange = JSON.parse(dbBlk.feeRange);
+    extras.effectiveMedianFee = dbBlk.effectiveMedianFee;
+    extras.effectiveFeeRange = dbBlk.effectiveFeeRange ? JSON.parse(dbBlk.effectiveFeeRange) : null;
     extras.reward = dbBlk.reward;
     extras.pool = {
       id: dbBlk.poolId,