online calculation of stack-of-n-blocks fee statistics

backend/src/api/common.ts

@@ -1,4 +1,4 @@
-import { Ancestor, CpfpInfo, CpfpSummary, EffectiveFeeStats, MempoolBlockWithTransactions, TransactionExtended, TransactionStripped } from '../mempool.interfaces';
+import { Ancestor, CpfpInfo, CpfpSummary, EffectiveFeeStats, MempoolBlockWithTransactions, TransactionExtended, TransactionStripped, WorkingEffectiveFeeStats } from '../mempool.interfaces';
 import config from '../config';
 import { NodeSocket } from '../repositories/NodesSocketsRepository';
 import { isIP } from 'net';
@@ -442,3 +442,119 @@ export class Common {
     return sortedDistribution[Math.floor((sortedDistribution.length - 1) * (n / 100))];
   }
 }
+
+/**
+ * Class to calculate average fee rates of a list of transactions
+ * at certain weight percentiles, in a single pass
+ * 
+ * init with:
+ *   maxWeight - the total weight to measure percentiles relative to (e.g. 4MW for a single block)
+ *   percentileBandWidth - how many weight units to average over for each percentile (as a % of maxWeight)
+ *   percentiles - an array of weight percentiles to compute, in %
+ * 
+ * then call .processNext(tx) for each transaction, in descending order
+ * 
+ * retrieve the final results with .getFeeStats()
+ */
+export class OnlineFeeStatsCalculator {
+  private maxWeight: number;
+  private percentiles = [10,25,50,75,90];
+
+  private bandWidthPercent = 2;
+  private bandWidth: number = 0;
+  private bandIndex = 0;
+  private leftBound = 0;
+  private rightBound = 0;
+  private inBand = false;
+  private totalBandFee = 0;
+  private totalBandWeight = 0;
+  private minBandRate = Infinity;
+  private maxBandRate = 0;
+
+  private feeRange: { avg: number, min: number, max: number }[] = [];
+  private totalWeight: number = 0;
+
+  constructor (maxWeight: number, percentileBandWidth?: number, percentiles?: number[]) {
+    this.maxWeight = maxWeight;
+    if (percentiles && percentiles.length) {
+      this.percentiles = percentiles;
+    }
+    if (percentileBandWidth != null) {
+      this.bandWidthPercent = percentileBandWidth;
+    }
+    this.bandWidth = this.maxWeight * (this.bandWidthPercent / 100);
+    // add min/max percentiles aligned to the ends of the range
+    this.percentiles.unshift(this.bandWidthPercent / 2);
+    this.percentiles.push(100 - (this.bandWidthPercent / 2));
+    this.setNextBounds();
+  }
+
+  processNext(tx: { weight: number, fee: number, effectiveFeePerVsize?: number, feePerVsize?: number, rate?: number, txid: string }): void {
+    let left = this.totalWeight;
+    const right = this.totalWeight + tx.weight;
+    if (!this.inBand && right <= this.leftBound) {
+      this.totalWeight += tx.weight;
+      return;
+    }
+
+    while (left < right) {
+      if (right > this.leftBound) {
+        this.inBand = true;
+        const txRate = (tx.rate || tx.effectiveFeePerVsize || tx.feePerVsize || 0);
+        const weight = Math.min(right, this.rightBound) - Math.max(left, this.leftBound);
+        this.totalBandFee += (txRate * weight);
+        this.totalBandWeight += weight;
+        this.maxBandRate = Math.max(this.maxBandRate, txRate);
+        this.minBandRate = Math.min(this.minBandRate, txRate);
+      }
+      left = Math.min(right, this.rightBound);
+
+      if (left >= this.rightBound) {
+        this.inBand = false;
+        const avgBandFeeRate = this.totalBandWeight ? (this.totalBandFee / this.totalBandWeight) : 0;
+        this.feeRange.unshift({ avg: avgBandFeeRate, min: this.minBandRate, max: this.maxBandRate });
+        this.bandIndex++;
+        this.setNextBounds();
+        this.totalBandFee = 0;
+        this.totalBandWeight = 0;
+        this.minBandRate = Infinity;
+        this.maxBandRate = 0;
+      }
+    }
+    this.totalWeight += tx.weight;
+  }
+
+  private setNextBounds(): void {
+    const nextPercentile = this.percentiles[this.bandIndex];
+    if (nextPercentile != null) {
+      this.leftBound = ((nextPercentile / 100) * this.maxWeight) - (this.bandWidth / 2);
+      this.rightBound = this.leftBound + this.bandWidth;
+    } else {
+      this.leftBound = Infinity;
+      this.rightBound = Infinity;
+    }
+  }
+
+  getRawFeeStats(): WorkingEffectiveFeeStats {
+    if (this.totalBandWeight > 0) {
+      const avgBandFeeRate = this.totalBandWeight ? (this.totalBandFee / this.totalBandWeight) : 0;
+      this.feeRange.unshift({ avg: avgBandFeeRate, min: this.minBandRate, max: this.maxBandRate });
+    }
+    while (this.feeRange.length < this.percentiles.length) {
+      this.feeRange.unshift({ avg: 0, min: 0, max: 0 });
+    }
+    return {
+      minFee: this.feeRange[0].min,
+      medianFee: this.feeRange[Math.floor(this.feeRange.length / 2)].avg,
+      maxFee: this.feeRange[this.feeRange.length - 1].max,
+      feeRange: this.feeRange.map(f => f.avg),
+    };
+  }
+
+  getFeeStats(): EffectiveFeeStats {
+    const stats = this.getRawFeeStats();
+    stats.feeRange[0] = stats.minFee;
+    stats.feeRange[stats.feeRange.length - 1] = stats.maxFee;
+    return stats;
+  }
+}

backend/src/api/mempool-blocks.ts

@@ -1,6 +1,6 @@
 import logger from '../logger';
-import { MempoolBlock, TransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction } from '../mempool.interfaces';
+import { MempoolBlock, TransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats } from '../mempool.interfaces';
-import { Common } from './common';
+import { Common, OnlineFeeStatsCalculator } from './common';
 import config from '../config';
 import { Worker } from 'worker_threads';
 import path from 'path';
@@ -104,6 +104,8 @@ class MempoolBlocks {
 
   private calculateMempoolBlocks(transactionsSorted: TransactionExtended[]): MempoolBlockWithTransactions[] {
     const mempoolBlocks: MempoolBlockWithTransactions[] = [];
+    let feeStatsCalculator: OnlineFeeStatsCalculator = new OnlineFeeStatsCalculator(config.MEMPOOL.BLOCK_WEIGHT_UNITS);
+    let onlineStats = false;
     let blockSize = 0;
     let blockWeight = 0;
     let blockVsize = 0;
@@ -111,7 +113,7 @@ class MempoolBlocks {
     const sizeLimit = (config.MEMPOOL.BLOCK_WEIGHT_UNITS / 4) * 1.2;
     let transactionIds: string[] = [];
     let transactions: TransactionExtended[] = [];
-    transactionsSorted.forEach((tx) => {
+    transactionsSorted.forEach((tx, index) => {
       if (blockWeight + tx.weight <= config.MEMPOOL.BLOCK_WEIGHT_UNITS
         || mempoolBlocks.length === config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT - 1) {
         tx.position = {
@@ -126,6 +128,9 @@ class MempoolBlocks {
           transactions.push(tx);
         }
         transactionIds.push(tx.txid);
+        if (onlineStats) {
+          feeStatsCalculator.processNext(tx);
+        }
       } else {
         mempoolBlocks.push(this.dataToMempoolBlocks(transactionIds, transactions, blockSize, blockWeight, blockFees));
         blockVsize = 0;
@@ -133,6 +138,16 @@ class MempoolBlocks {
           block: mempoolBlocks.length,
           vsize: blockVsize + (tx.vsize / 2),
         };
+
+        if (mempoolBlocks.length === config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT - 1) {
+          const stackWeight = transactionsSorted.slice(index).reduce((total, tx) => total + (tx.weight || 0), 0);
+          if (stackWeight > config.MEMPOOL.BLOCK_WEIGHT_UNITS) {
+            onlineStats = true;
+            feeStatsCalculator = new OnlineFeeStatsCalculator(stackWeight, 0.5);
+            feeStatsCalculator.processNext(tx);
+          }
+        }
+
         blockVsize += tx.vsize;
         blockWeight = tx.weight;
         blockSize = tx.size;
@@ -142,7 +157,8 @@ class MempoolBlocks {
       }
     });
     if (transactions.length) {
-      mempoolBlocks.push(this.dataToMempoolBlocks(transactionIds, transactions, blockSize, blockWeight, blockFees));
+      const feeStats = onlineStats ? feeStatsCalculator.getRawFeeStats() : undefined;
+      mempoolBlocks.push(this.dataToMempoolBlocks(transactionIds, transactions, blockSize, blockWeight, blockFees, feeStats));
     }
 
     return mempoolBlocks;
@@ -310,7 +326,16 @@ class MempoolBlocks {
       }
     }
 
-    const readyBlocks: { transactionIds, transactions, totalSize, totalWeight, totalFees }[] = [];
+    let hasBlockStack = blocks.length >= 8;
+    let stackWeight;
+    let feeStatsCalculator: OnlineFeeStatsCalculator | void;
+    if (hasBlockStack) {
+      stackWeight = blocks[blocks.length - 1].reduce((total, tx) => total + (mempool[tx]?.weight || 0), 0);
+      hasBlockStack = stackWeight > config.MEMPOOL.BLOCK_WEIGHT_UNITS;
+      feeStatsCalculator = new OnlineFeeStatsCalculator(stackWeight, 0.5);
+    }
+
+    const readyBlocks: { transactionIds, transactions, totalSize, totalWeight, totalFees, feeStats }[] = [];
     const sizeLimit = (config.MEMPOOL.BLOCK_WEIGHT_UNITS / 4) * 1.2;
     // update this thread's mempool with the results
     for (let blockIndex = 0; blockIndex < blocks.length; blockIndex++) {
@@ -333,6 +358,11 @@ class MempoolBlocks {
           };
           mempoolTx.cpfpChecked = true;
 
+          // online calculation of stack-of-blocks fee stats
+          if (hasBlockStack && blockIndex === blocks.length - 1 && feeStatsCalculator) {
+            feeStatsCalculator.processNext(mempoolTx);
+          }
+
           totalSize += mempoolTx.size;
           totalVsize += mempoolTx.vsize;
           totalWeight += mempoolTx.weight;
@@ -348,7 +378,8 @@ class MempoolBlocks {
         transactions,
         totalSize,
         totalWeight,
-        totalFees
+        totalFees,
+        feeStats: (hasBlockStack && blockIndex === blocks.length - 1 && feeStatsCalculator) ? feeStatsCalculator.getRawFeeStats() : undefined,
       });
     }
 
@@ -382,7 +413,9 @@ class MempoolBlocks {
       }
     }
 
-    const mempoolBlocks = readyBlocks.map(b => this.dataToMempoolBlocks(b.transactionIds, b.transactions, b.totalSize, b.totalWeight, b.totalFees));
+    const mempoolBlocks = readyBlocks.map((b, index) => {
+      return this.dataToMempoolBlocks(b.transactionIds, b.transactions, b.totalSize, b.totalWeight, b.totalFees, b.feeStats);
+    });
 
     if (saveResults) {
       const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, mempoolBlocks);
@@ -393,8 +426,10 @@ class MempoolBlocks {
     return mempoolBlocks;
   }
 
-  private dataToMempoolBlocks(transactionIds: string[], transactions: TransactionExtended[], totalSize: number, totalWeight: number, totalFees: number): MempoolBlockWithTransactions {
+  private dataToMempoolBlocks(transactionIds: string[], transactions: TransactionExtended[], totalSize: number, totalWeight: number, totalFees: number, feeStats?: EffectiveFeeStats ): MempoolBlockWithTransactions {
-    const feeStats = Common.calcEffectiveFeeStatistics(transactions);
+    if (!feeStats) {
+      feeStats = Common.calcEffectiveFeeStatistics(transactions);
+    }
     return {
       blockSize: totalSize,
       blockVSize: (totalWeight / 4), // fractional vsize to avoid rounding errors

backend/src/mempool.interfaces.ts

@@ -237,6 +237,11 @@ export interface EffectiveFeeStats {
   feeRange: number[]; // 2nd, 10th, 25th, 50th, 75th, 90th, 98th percentiles
 }
 
+export interface WorkingEffectiveFeeStats extends EffectiveFeeStats {
+  minFee: number;
+  maxFee: number;
+}
+
 export interface CpfpSummary {
   transactions: TransactionExtended[];
   clusters: { root: string, height: number, txs: Ancestor[], effectiveFeePerVsize: number }[];