Merge pull request #3743 from mempool/mononaut/full-stack-fee-stats
stack-of-n-blocks fee statistics
This commit is contained in:
commit
adc395fc3d
@ -1,4 +1,4 @@
|
|||||||
import { Ancestor, CpfpInfo, CpfpSummary, EffectiveFeeStats, MempoolBlockWithTransactions, TransactionExtended, TransactionStripped } from '../mempool.interfaces';
|
import { Ancestor, CpfpInfo, CpfpSummary, EffectiveFeeStats, MempoolBlockWithTransactions, TransactionExtended, TransactionStripped, WorkingEffectiveFeeStats } from '../mempool.interfaces';
|
||||||
import config from '../config';
|
import config from '../config';
|
||||||
import { NodeSocket } from '../repositories/NodesSocketsRepository';
|
import { NodeSocket } from '../repositories/NodesSocketsRepository';
|
||||||
import { isIP } from 'net';
|
import { isIP } from 'net';
|
||||||
@ -442,3 +442,119 @@ export class Common {
|
|||||||
return sortedDistribution[Math.floor((sortedDistribution.length - 1) * (n / 100))];
|
return sortedDistribution[Math.floor((sortedDistribution.length - 1) * (n / 100))];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Class to calculate average fee rates of a list of transactions
|
||||||
|
* at certain weight percentiles, in a single pass
|
||||||
|
*
|
||||||
|
* init with:
|
||||||
|
* maxWeight - the total weight to measure percentiles relative to (e.g. 4MW for a single block)
|
||||||
|
* percentileBandWidth - how many weight units to average over for each percentile (as a % of maxWeight)
|
||||||
|
* percentiles - an array of weight percentiles to compute, in %
|
||||||
|
*
|
||||||
|
* then call .processNext(tx) for each transaction, in descending order
|
||||||
|
*
|
||||||
|
* retrieve the final results with .getFeeStats()
|
||||||
|
*/
|
||||||
|
export class OnlineFeeStatsCalculator {
|
||||||
|
private maxWeight: number;
|
||||||
|
private percentiles = [10,25,50,75,90];
|
||||||
|
|
||||||
|
private bandWidthPercent = 2;
|
||||||
|
private bandWidth: number = 0;
|
||||||
|
private bandIndex = 0;
|
||||||
|
private leftBound = 0;
|
||||||
|
private rightBound = 0;
|
||||||
|
private inBand = false;
|
||||||
|
private totalBandFee = 0;
|
||||||
|
private totalBandWeight = 0;
|
||||||
|
private minBandRate = Infinity;
|
||||||
|
private maxBandRate = 0;
|
||||||
|
|
||||||
|
private feeRange: { avg: number, min: number, max: number }[] = [];
|
||||||
|
private totalWeight: number = 0;
|
||||||
|
|
||||||
|
constructor (maxWeight: number, percentileBandWidth?: number, percentiles?: number[]) {
|
||||||
|
this.maxWeight = maxWeight;
|
||||||
|
if (percentiles && percentiles.length) {
|
||||||
|
this.percentiles = percentiles;
|
||||||
|
}
|
||||||
|
if (percentileBandWidth != null) {
|
||||||
|
this.bandWidthPercent = percentileBandWidth;
|
||||||
|
}
|
||||||
|
this.bandWidth = this.maxWeight * (this.bandWidthPercent / 100);
|
||||||
|
// add min/max percentiles aligned to the ends of the range
|
||||||
|
this.percentiles.unshift(this.bandWidthPercent / 2);
|
||||||
|
this.percentiles.push(100 - (this.bandWidthPercent / 2));
|
||||||
|
this.setNextBounds();
|
||||||
|
}
|
||||||
|
|
||||||
|
processNext(tx: { weight: number, fee: number, effectiveFeePerVsize?: number, feePerVsize?: number, rate?: number, txid: string }): void {
|
||||||
|
let left = this.totalWeight;
|
||||||
|
const right = this.totalWeight + tx.weight;
|
||||||
|
if (!this.inBand && right <= this.leftBound) {
|
||||||
|
this.totalWeight += tx.weight;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
while (left < right) {
|
||||||
|
if (right > this.leftBound) {
|
||||||
|
this.inBand = true;
|
||||||
|
const txRate = (tx.rate || tx.effectiveFeePerVsize || tx.feePerVsize || 0);
|
||||||
|
const weight = Math.min(right, this.rightBound) - Math.max(left, this.leftBound);
|
||||||
|
this.totalBandFee += (txRate * weight);
|
||||||
|
this.totalBandWeight += weight;
|
||||||
|
this.maxBandRate = Math.max(this.maxBandRate, txRate);
|
||||||
|
this.minBandRate = Math.min(this.minBandRate, txRate);
|
||||||
|
}
|
||||||
|
left = Math.min(right, this.rightBound);
|
||||||
|
|
||||||
|
if (left >= this.rightBound) {
|
||||||
|
this.inBand = false;
|
||||||
|
const avgBandFeeRate = this.totalBandWeight ? (this.totalBandFee / this.totalBandWeight) : 0;
|
||||||
|
this.feeRange.unshift({ avg: avgBandFeeRate, min: this.minBandRate, max: this.maxBandRate });
|
||||||
|
this.bandIndex++;
|
||||||
|
this.setNextBounds();
|
||||||
|
this.totalBandFee = 0;
|
||||||
|
this.totalBandWeight = 0;
|
||||||
|
this.minBandRate = Infinity;
|
||||||
|
this.maxBandRate = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
this.totalWeight += tx.weight;
|
||||||
|
}
|
||||||
|
|
||||||
|
private setNextBounds(): void {
|
||||||
|
const nextPercentile = this.percentiles[this.bandIndex];
|
||||||
|
if (nextPercentile != null) {
|
||||||
|
this.leftBound = ((nextPercentile / 100) * this.maxWeight) - (this.bandWidth / 2);
|
||||||
|
this.rightBound = this.leftBound + this.bandWidth;
|
||||||
|
} else {
|
||||||
|
this.leftBound = Infinity;
|
||||||
|
this.rightBound = Infinity;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
getRawFeeStats(): WorkingEffectiveFeeStats {
|
||||||
|
if (this.totalBandWeight > 0) {
|
||||||
|
const avgBandFeeRate = this.totalBandWeight ? (this.totalBandFee / this.totalBandWeight) : 0;
|
||||||
|
this.feeRange.unshift({ avg: avgBandFeeRate, min: this.minBandRate, max: this.maxBandRate });
|
||||||
|
}
|
||||||
|
while (this.feeRange.length < this.percentiles.length) {
|
||||||
|
this.feeRange.unshift({ avg: 0, min: 0, max: 0 });
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
minFee: this.feeRange[0].min,
|
||||||
|
medianFee: this.feeRange[Math.floor(this.feeRange.length / 2)].avg,
|
||||||
|
maxFee: this.feeRange[this.feeRange.length - 1].max,
|
||||||
|
feeRange: this.feeRange.map(f => f.avg),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
getFeeStats(): EffectiveFeeStats {
|
||||||
|
const stats = this.getRawFeeStats();
|
||||||
|
stats.feeRange[0] = stats.minFee;
|
||||||
|
stats.feeRange[stats.feeRange.length - 1] = stats.maxFee;
|
||||||
|
return stats;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
import logger from '../logger';
|
import logger from '../logger';
|
||||||
import { MempoolBlock, TransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction } from '../mempool.interfaces';
|
import { MempoolBlock, TransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats } from '../mempool.interfaces';
|
||||||
import { Common } from './common';
|
import { Common, OnlineFeeStatsCalculator } from './common';
|
||||||
import config from '../config';
|
import config from '../config';
|
||||||
import { Worker } from 'worker_threads';
|
import { Worker } from 'worker_threads';
|
||||||
import path from 'path';
|
import path from 'path';
|
||||||
@ -104,6 +104,8 @@ class MempoolBlocks {
|
|||||||
|
|
||||||
private calculateMempoolBlocks(transactionsSorted: TransactionExtended[]): MempoolBlockWithTransactions[] {
|
private calculateMempoolBlocks(transactionsSorted: TransactionExtended[]): MempoolBlockWithTransactions[] {
|
||||||
const mempoolBlocks: MempoolBlockWithTransactions[] = [];
|
const mempoolBlocks: MempoolBlockWithTransactions[] = [];
|
||||||
|
let feeStatsCalculator: OnlineFeeStatsCalculator = new OnlineFeeStatsCalculator(config.MEMPOOL.BLOCK_WEIGHT_UNITS);
|
||||||
|
let onlineStats = false;
|
||||||
let blockSize = 0;
|
let blockSize = 0;
|
||||||
let blockWeight = 0;
|
let blockWeight = 0;
|
||||||
let blockVsize = 0;
|
let blockVsize = 0;
|
||||||
@ -111,7 +113,7 @@ class MempoolBlocks {
|
|||||||
const sizeLimit = (config.MEMPOOL.BLOCK_WEIGHT_UNITS / 4) * 1.2;
|
const sizeLimit = (config.MEMPOOL.BLOCK_WEIGHT_UNITS / 4) * 1.2;
|
||||||
let transactionIds: string[] = [];
|
let transactionIds: string[] = [];
|
||||||
let transactions: TransactionExtended[] = [];
|
let transactions: TransactionExtended[] = [];
|
||||||
transactionsSorted.forEach((tx) => {
|
transactionsSorted.forEach((tx, index) => {
|
||||||
if (blockWeight + tx.weight <= config.MEMPOOL.BLOCK_WEIGHT_UNITS
|
if (blockWeight + tx.weight <= config.MEMPOOL.BLOCK_WEIGHT_UNITS
|
||||||
|| mempoolBlocks.length === config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT - 1) {
|
|| mempoolBlocks.length === config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT - 1) {
|
||||||
tx.position = {
|
tx.position = {
|
||||||
@ -126,6 +128,9 @@ class MempoolBlocks {
|
|||||||
transactions.push(tx);
|
transactions.push(tx);
|
||||||
}
|
}
|
||||||
transactionIds.push(tx.txid);
|
transactionIds.push(tx.txid);
|
||||||
|
if (onlineStats) {
|
||||||
|
feeStatsCalculator.processNext(tx);
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
mempoolBlocks.push(this.dataToMempoolBlocks(transactionIds, transactions, blockSize, blockWeight, blockFees));
|
mempoolBlocks.push(this.dataToMempoolBlocks(transactionIds, transactions, blockSize, blockWeight, blockFees));
|
||||||
blockVsize = 0;
|
blockVsize = 0;
|
||||||
@ -133,6 +138,16 @@ class MempoolBlocks {
|
|||||||
block: mempoolBlocks.length,
|
block: mempoolBlocks.length,
|
||||||
vsize: blockVsize + (tx.vsize / 2),
|
vsize: blockVsize + (tx.vsize / 2),
|
||||||
};
|
};
|
||||||
|
|
||||||
|
if (mempoolBlocks.length === config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT - 1) {
|
||||||
|
const stackWeight = transactionsSorted.slice(index).reduce((total, tx) => total + (tx.weight || 0), 0);
|
||||||
|
if (stackWeight > config.MEMPOOL.BLOCK_WEIGHT_UNITS) {
|
||||||
|
onlineStats = true;
|
||||||
|
feeStatsCalculator = new OnlineFeeStatsCalculator(stackWeight, 0.5);
|
||||||
|
feeStatsCalculator.processNext(tx);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
blockVsize += tx.vsize;
|
blockVsize += tx.vsize;
|
||||||
blockWeight = tx.weight;
|
blockWeight = tx.weight;
|
||||||
blockSize = tx.size;
|
blockSize = tx.size;
|
||||||
@ -142,7 +157,8 @@ class MempoolBlocks {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
if (transactions.length) {
|
if (transactions.length) {
|
||||||
mempoolBlocks.push(this.dataToMempoolBlocks(transactionIds, transactions, blockSize, blockWeight, blockFees));
|
const feeStats = onlineStats ? feeStatsCalculator.getRawFeeStats() : undefined;
|
||||||
|
mempoolBlocks.push(this.dataToMempoolBlocks(transactionIds, transactions, blockSize, blockWeight, blockFees, feeStats));
|
||||||
}
|
}
|
||||||
|
|
||||||
return mempoolBlocks;
|
return mempoolBlocks;
|
||||||
@ -310,7 +326,16 @@ class MempoolBlocks {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const readyBlocks: { transactionIds, transactions, totalSize, totalWeight, totalFees }[] = [];
|
let hasBlockStack = blocks.length >= 8;
|
||||||
|
let stackWeight;
|
||||||
|
let feeStatsCalculator: OnlineFeeStatsCalculator | void;
|
||||||
|
if (hasBlockStack) {
|
||||||
|
stackWeight = blocks[blocks.length - 1].reduce((total, tx) => total + (mempool[tx]?.weight || 0), 0);
|
||||||
|
hasBlockStack = stackWeight > config.MEMPOOL.BLOCK_WEIGHT_UNITS;
|
||||||
|
feeStatsCalculator = new OnlineFeeStatsCalculator(stackWeight, 0.5);
|
||||||
|
}
|
||||||
|
|
||||||
|
const readyBlocks: { transactionIds, transactions, totalSize, totalWeight, totalFees, feeStats }[] = [];
|
||||||
const sizeLimit = (config.MEMPOOL.BLOCK_WEIGHT_UNITS / 4) * 1.2;
|
const sizeLimit = (config.MEMPOOL.BLOCK_WEIGHT_UNITS / 4) * 1.2;
|
||||||
// update this thread's mempool with the results
|
// update this thread's mempool with the results
|
||||||
for (let blockIndex = 0; blockIndex < blocks.length; blockIndex++) {
|
for (let blockIndex = 0; blockIndex < blocks.length; blockIndex++) {
|
||||||
@ -333,6 +358,11 @@ class MempoolBlocks {
|
|||||||
};
|
};
|
||||||
mempoolTx.cpfpChecked = true;
|
mempoolTx.cpfpChecked = true;
|
||||||
|
|
||||||
|
// online calculation of stack-of-blocks fee stats
|
||||||
|
if (hasBlockStack && blockIndex === blocks.length - 1 && feeStatsCalculator) {
|
||||||
|
feeStatsCalculator.processNext(mempoolTx);
|
||||||
|
}
|
||||||
|
|
||||||
totalSize += mempoolTx.size;
|
totalSize += mempoolTx.size;
|
||||||
totalVsize += mempoolTx.vsize;
|
totalVsize += mempoolTx.vsize;
|
||||||
totalWeight += mempoolTx.weight;
|
totalWeight += mempoolTx.weight;
|
||||||
@ -348,7 +378,8 @@ class MempoolBlocks {
|
|||||||
transactions,
|
transactions,
|
||||||
totalSize,
|
totalSize,
|
||||||
totalWeight,
|
totalWeight,
|
||||||
totalFees
|
totalFees,
|
||||||
|
feeStats: (hasBlockStack && blockIndex === blocks.length - 1 && feeStatsCalculator) ? feeStatsCalculator.getRawFeeStats() : undefined,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -382,7 +413,9 @@ class MempoolBlocks {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const mempoolBlocks = readyBlocks.map(b => this.dataToMempoolBlocks(b.transactionIds, b.transactions, b.totalSize, b.totalWeight, b.totalFees));
|
const mempoolBlocks = readyBlocks.map((b, index) => {
|
||||||
|
return this.dataToMempoolBlocks(b.transactionIds, b.transactions, b.totalSize, b.totalWeight, b.totalFees, b.feeStats);
|
||||||
|
});
|
||||||
|
|
||||||
if (saveResults) {
|
if (saveResults) {
|
||||||
const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, mempoolBlocks);
|
const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, mempoolBlocks);
|
||||||
@ -393,8 +426,10 @@ class MempoolBlocks {
|
|||||||
return mempoolBlocks;
|
return mempoolBlocks;
|
||||||
}
|
}
|
||||||
|
|
||||||
private dataToMempoolBlocks(transactionIds: string[], transactions: TransactionExtended[], totalSize: number, totalWeight: number, totalFees: number): MempoolBlockWithTransactions {
|
private dataToMempoolBlocks(transactionIds: string[], transactions: TransactionExtended[], totalSize: number, totalWeight: number, totalFees: number, feeStats?: EffectiveFeeStats ): MempoolBlockWithTransactions {
|
||||||
const feeStats = Common.calcEffectiveFeeStatistics(transactions);
|
if (!feeStats) {
|
||||||
|
feeStats = Common.calcEffectiveFeeStatistics(transactions);
|
||||||
|
}
|
||||||
return {
|
return {
|
||||||
blockSize: totalSize,
|
blockSize: totalSize,
|
||||||
blockVSize: (totalWeight / 4), // fractional vsize to avoid rounding errors
|
blockVSize: (totalWeight / 4), // fractional vsize to avoid rounding errors
|
||||||
|
@ -237,6 +237,11 @@ export interface EffectiveFeeStats {
|
|||||||
feeRange: number[]; // 2nd, 10th, 25th, 50th, 75th, 90th, 98th percentiles
|
feeRange: number[]; // 2nd, 10th, 25th, 50th, 75th, 90th, 98th percentiles
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface WorkingEffectiveFeeStats extends EffectiveFeeStats {
|
||||||
|
minFee: number;
|
||||||
|
maxFee: number;
|
||||||
|
}
|
||||||
|
|
||||||
export interface CpfpSummary {
|
export interface CpfpSummary {
|
||||||
transactions: TransactionExtended[];
|
transactions: TransactionExtended[];
|
||||||
clusters: { root: string, height: number, txs: Ancestor[], effectiveFeePerVsize: number }[];
|
clusters: { root: string, height: number, txs: Ancestor[], effectiveFeePerVsize: number }[];
|
||||||
|
Loading…
x
Reference in New Issue
Block a user