diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index b044e2866..fc952d6a8 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -1,4 +1,4 @@ -import { Ancestor, CpfpInfo, CpfpSummary, EffectiveFeeStats, MempoolBlockWithTransactions, TransactionExtended, TransactionStripped } from '../mempool.interfaces'; +import { Ancestor, CpfpInfo, CpfpSummary, EffectiveFeeStats, MempoolBlockWithTransactions, TransactionExtended, TransactionStripped, WorkingEffectiveFeeStats } from '../mempool.interfaces'; import config from '../config'; import { NodeSocket } from '../repositories/NodesSocketsRepository'; import { isIP } from 'net'; @@ -442,3 +442,119 @@ export class Common { return sortedDistribution[Math.floor((sortedDistribution.length - 1) * (n / 100))]; } } + +/** + * Class to calculate average fee rates of a list of transactions + * at certain weight percentiles, in a single pass + * + * init with: + * maxWeight - the total weight to measure percentiles relative to (e.g. 4MW for a single block) + * percentileBandWidth - how many weight units to average over for each percentile (as a % of maxWeight) + * percentiles - an array of weight percentiles to compute, in % + * + * then call .processNext(tx) for each transaction, in descending order + * + * retrieve the final results with .getFeeStats() + */ +export class OnlineFeeStatsCalculator { + private maxWeight: number; + private percentiles = [10,25,50,75,90]; + + private bandWidthPercent = 2; + private bandWidth: number = 0; + private bandIndex = 0; + private leftBound = 0; + private rightBound = 0; + private inBand = false; + private totalBandFee = 0; + private totalBandWeight = 0; + private minBandRate = Infinity; + private maxBandRate = 0; + + private feeRange: { avg: number, min: number, max: number }[] = []; + private totalWeight: number = 0; + + constructor (maxWeight: number, percentileBandWidth?: number, percentiles?: number[]) { + this.maxWeight = maxWeight; + if (percentiles && percentiles.length) { + this.percentiles = percentiles; + } + if (percentileBandWidth != null) { + this.bandWidthPercent = percentileBandWidth; + } + this.bandWidth = this.maxWeight * (this.bandWidthPercent / 100); + // add min/max percentiles aligned to the ends of the range + this.percentiles.unshift(this.bandWidthPercent / 2); + this.percentiles.push(100 - (this.bandWidthPercent / 2)); + this.setNextBounds(); + } + + processNext(tx: { weight: number, fee: number, effectiveFeePerVsize?: number, feePerVsize?: number, rate?: number, txid: string }): void { + let left = this.totalWeight; + const right = this.totalWeight + tx.weight; + if (!this.inBand && right <= this.leftBound) { + this.totalWeight += tx.weight; + return; + } + + while (left < right) { + if (right > this.leftBound) { + this.inBand = true; + const txRate = (tx.rate || tx.effectiveFeePerVsize || tx.feePerVsize || 0); + const weight = Math.min(right, this.rightBound) - Math.max(left, this.leftBound); + this.totalBandFee += (txRate * weight); + this.totalBandWeight += weight; + this.maxBandRate = Math.max(this.maxBandRate, txRate); + this.minBandRate = Math.min(this.minBandRate, txRate); + } + left = Math.min(right, this.rightBound); + + if (left >= this.rightBound) { + this.inBand = false; + const avgBandFeeRate = this.totalBandWeight ? (this.totalBandFee / this.totalBandWeight) : 0; + this.feeRange.unshift({ avg: avgBandFeeRate, min: this.minBandRate, max: this.maxBandRate }); + this.bandIndex++; + this.setNextBounds(); + this.totalBandFee = 0; + this.totalBandWeight = 0; + this.minBandRate = Infinity; + this.maxBandRate = 0; + } + } + this.totalWeight += tx.weight; + } + + private setNextBounds(): void { + const nextPercentile = this.percentiles[this.bandIndex]; + if (nextPercentile != null) { + this.leftBound = ((nextPercentile / 100) * this.maxWeight) - (this.bandWidth / 2); + this.rightBound = this.leftBound + this.bandWidth; + } else { + this.leftBound = Infinity; + this.rightBound = Infinity; + } + } + + getRawFeeStats(): WorkingEffectiveFeeStats { + if (this.totalBandWeight > 0) { + const avgBandFeeRate = this.totalBandWeight ? (this.totalBandFee / this.totalBandWeight) : 0; + this.feeRange.unshift({ avg: avgBandFeeRate, min: this.minBandRate, max: this.maxBandRate }); + } + while (this.feeRange.length < this.percentiles.length) { + this.feeRange.unshift({ avg: 0, min: 0, max: 0 }); + } + return { + minFee: this.feeRange[0].min, + medianFee: this.feeRange[Math.floor(this.feeRange.length / 2)].avg, + maxFee: this.feeRange[this.feeRange.length - 1].max, + feeRange: this.feeRange.map(f => f.avg), + }; + } + + getFeeStats(): EffectiveFeeStats { + const stats = this.getRawFeeStats(); + stats.feeRange[0] = stats.minFee; + stats.feeRange[stats.feeRange.length - 1] = stats.maxFee; + return stats; + } +} diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 62717ed7e..803b7e56e 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -1,6 +1,6 @@ import logger from '../logger'; -import { MempoolBlock, TransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction } from '../mempool.interfaces'; -import { Common } from './common'; +import { MempoolBlock, TransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats } from '../mempool.interfaces'; +import { Common, OnlineFeeStatsCalculator } from './common'; import config from '../config'; import { Worker } from 'worker_threads'; import path from 'path'; @@ -104,6 +104,8 @@ class MempoolBlocks { private calculateMempoolBlocks(transactionsSorted: TransactionExtended[]): MempoolBlockWithTransactions[] { const mempoolBlocks: MempoolBlockWithTransactions[] = []; + let feeStatsCalculator: OnlineFeeStatsCalculator = new OnlineFeeStatsCalculator(config.MEMPOOL.BLOCK_WEIGHT_UNITS); + let onlineStats = false; let blockSize = 0; let blockWeight = 0; let blockVsize = 0; @@ -111,7 +113,7 @@ class MempoolBlocks { const sizeLimit = (config.MEMPOOL.BLOCK_WEIGHT_UNITS / 4) * 1.2; let transactionIds: string[] = []; let transactions: TransactionExtended[] = []; - transactionsSorted.forEach((tx) => { + transactionsSorted.forEach((tx, index) => { if (blockWeight + tx.weight <= config.MEMPOOL.BLOCK_WEIGHT_UNITS || mempoolBlocks.length === config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT - 1) { tx.position = { @@ -126,6 +128,9 @@ class MempoolBlocks { transactions.push(tx); } transactionIds.push(tx.txid); + if (onlineStats) { + feeStatsCalculator.processNext(tx); + } } else { mempoolBlocks.push(this.dataToMempoolBlocks(transactionIds, transactions, blockSize, blockWeight, blockFees)); blockVsize = 0; @@ -133,6 +138,16 @@ class MempoolBlocks { block: mempoolBlocks.length, vsize: blockVsize + (tx.vsize / 2), }; + + if (mempoolBlocks.length === config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT - 1) { + const stackWeight = transactionsSorted.slice(index).reduce((total, tx) => total + (tx.weight || 0), 0); + if (stackWeight > config.MEMPOOL.BLOCK_WEIGHT_UNITS) { + onlineStats = true; + feeStatsCalculator = new OnlineFeeStatsCalculator(stackWeight, 0.5); + feeStatsCalculator.processNext(tx); + } + } + blockVsize += tx.vsize; blockWeight = tx.weight; blockSize = tx.size; @@ -142,7 +157,8 @@ class MempoolBlocks { } }); if (transactions.length) { - mempoolBlocks.push(this.dataToMempoolBlocks(transactionIds, transactions, blockSize, blockWeight, blockFees)); + const feeStats = onlineStats ? feeStatsCalculator.getRawFeeStats() : undefined; + mempoolBlocks.push(this.dataToMempoolBlocks(transactionIds, transactions, blockSize, blockWeight, blockFees, feeStats)); } return mempoolBlocks; @@ -310,7 +326,16 @@ class MempoolBlocks { } } - const readyBlocks: { transactionIds, transactions, totalSize, totalWeight, totalFees }[] = []; + let hasBlockStack = blocks.length >= 8; + let stackWeight; + let feeStatsCalculator: OnlineFeeStatsCalculator | void; + if (hasBlockStack) { + stackWeight = blocks[blocks.length - 1].reduce((total, tx) => total + (mempool[tx]?.weight || 0), 0); + hasBlockStack = stackWeight > config.MEMPOOL.BLOCK_WEIGHT_UNITS; + feeStatsCalculator = new OnlineFeeStatsCalculator(stackWeight, 0.5); + } + + const readyBlocks: { transactionIds, transactions, totalSize, totalWeight, totalFees, feeStats }[] = []; const sizeLimit = (config.MEMPOOL.BLOCK_WEIGHT_UNITS / 4) * 1.2; // update this thread's mempool with the results for (let blockIndex = 0; blockIndex < blocks.length; blockIndex++) { @@ -333,6 +358,11 @@ class MempoolBlocks { }; mempoolTx.cpfpChecked = true; + // online calculation of stack-of-blocks fee stats + if (hasBlockStack && blockIndex === blocks.length - 1 && feeStatsCalculator) { + feeStatsCalculator.processNext(mempoolTx); + } + totalSize += mempoolTx.size; totalVsize += mempoolTx.vsize; totalWeight += mempoolTx.weight; @@ -348,7 +378,8 @@ class MempoolBlocks { transactions, totalSize, totalWeight, - totalFees + totalFees, + feeStats: (hasBlockStack && blockIndex === blocks.length - 1 && feeStatsCalculator) ? feeStatsCalculator.getRawFeeStats() : undefined, }); } @@ -382,7 +413,9 @@ class MempoolBlocks { } } - const mempoolBlocks = readyBlocks.map(b => this.dataToMempoolBlocks(b.transactionIds, b.transactions, b.totalSize, b.totalWeight, b.totalFees)); + const mempoolBlocks = readyBlocks.map((b, index) => { + return this.dataToMempoolBlocks(b.transactionIds, b.transactions, b.totalSize, b.totalWeight, b.totalFees, b.feeStats); + }); if (saveResults) { const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, mempoolBlocks); @@ -393,8 +426,10 @@ class MempoolBlocks { return mempoolBlocks; } - private dataToMempoolBlocks(transactionIds: string[], transactions: TransactionExtended[], totalSize: number, totalWeight: number, totalFees: number): MempoolBlockWithTransactions { - const feeStats = Common.calcEffectiveFeeStatistics(transactions); + private dataToMempoolBlocks(transactionIds: string[], transactions: TransactionExtended[], totalSize: number, totalWeight: number, totalFees: number, feeStats?: EffectiveFeeStats ): MempoolBlockWithTransactions { + if (!feeStats) { + feeStats = Common.calcEffectiveFeeStatistics(transactions); + } return { blockSize: totalSize, blockVSize: (totalWeight / 4), // fractional vsize to avoid rounding errors diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index ab4c4cd25..7204c174e 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -237,6 +237,11 @@ export interface EffectiveFeeStats { feeRange: number[]; // 2nd, 10th, 25th, 50th, 75th, 90th, 98th percentiles } +export interface WorkingEffectiveFeeStats extends EffectiveFeeStats { + minFee: number; + maxFee: number; +} + export interface CpfpSummary { transactions: TransactionExtended[]; clusters: { root: string, height: number, txs: Ancestor[], effectiveFeePerVsize: number }[];