machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #3743 from mempool/mononaut/full-stack-fee-stats

Browse Source 
stack-of-n-blocks fee statistics
This commit is contained in:
softsimon 2023-05-11 19:48:02 -05:00 committed by GitHub
commit adc395fc3d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 166 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

118
backend/src/api/common.ts
View File

@ -1,4 +1,4 @@
import { Ancestor, CpfpInfo, CpfpSummary, EffectiveFeeStats, MempoolBlockWithTransactions, TransactionExtended, TransactionStripped } from '../mempool.interfaces';
import { Ancestor, CpfpInfo, CpfpSummary, EffectiveFeeStats, MempoolBlockWithTransactions, TransactionExtended, TransactionStripped, WorkingEffectiveFeeStats } from '../mempool.interfaces';
import config from '../config';
import { NodeSocket } from '../repositories/NodesSocketsRepository';
import { isIP } from 'net';
@ -442,3 +442,119 @@ export class Common {
return sortedDistribution[Math.floor((sortedDistribution.length - 1) * (n / 100))];
}
}
/**
* Class to calculate average fee rates of a list of transactions
* at certain weight percentiles, in a single pass
*
* init with:
* maxWeight - the total weight to measure percentiles relative to (e.g. 4MW for a single block)
* percentileBandWidth - how many weight units to average over for each percentile (as a % of maxWeight)
* percentiles - an array of weight percentiles to compute, in %
*
* then call .processNext(tx) for each transaction, in descending order
*
* retrieve the final results with .getFeeStats()
*/
export class OnlineFeeStatsCalculator {
private maxWeight: number;
private percentiles = [10,25,50,75,90];
private bandWidthPercent = 2;
private bandWidth: number = 0;
private bandIndex = 0;
private leftBound = 0;
private rightBound = 0;
private inBand = false;
private totalBandFee = 0;
private totalBandWeight = 0;
private minBandRate = Infinity;
private maxBandRate = 0;
private feeRange: { avg: number, min: number, max: number }[] = [];
private totalWeight: number = 0;
constructor (maxWeight: number, percentileBandWidth?: number, percentiles?: number[]) {
this.maxWeight = maxWeight;
if (percentiles && percentiles.length) {
this.percentiles = percentiles;
}
if (percentileBandWidth != null) {
this.bandWidthPercent = percentileBandWidth;
}
this.bandWidth = this.maxWeight * (this.bandWidthPercent / 100);
// add min/max percentiles aligned to the ends of the range
this.percentiles.unshift(this.bandWidthPercent / 2);
this.percentiles.push(100 - (this.bandWidthPercent / 2));
this.setNextBounds();
}
processNext(tx: { weight: number, fee: number, effectiveFeePerVsize?: number, feePerVsize?: number, rate?: number, txid: string }): void {
let left = this.totalWeight;
const right = this.totalWeight + tx.weight;
if (!this.inBand && right <= this.leftBound) {
this.totalWeight += tx.weight;
return;
}
while (left < right) {
if (right > this.leftBound) {
this.inBand = true;
const txRate = (tx.rate || tx.effectiveFeePerVsize || tx.feePerVsize || 0);
const weight = Math.min(right, this.rightBound) - Math.max(left, this.leftBound);
this.totalBandFee += (txRate * weight);
this.totalBandWeight += weight;
this.maxBandRate = Math.max(this.maxBandRate, txRate);
this.minBandRate = Math.min(this.minBandRate, txRate);
}
left = Math.min(right, this.rightBound);
if (left >= this.rightBound) {
this.inBand = false;
const avgBandFeeRate = this.totalBandWeight ? (this.totalBandFee / this.totalBandWeight) : 0;
this.feeRange.unshift({ avg: avgBandFeeRate, min: this.minBandRate, max: this.maxBandRate });
this.bandIndex++;
this.setNextBounds();
this.totalBandFee = 0;
this.totalBandWeight = 0;
this.minBandRate = Infinity;
this.maxBandRate = 0;
}
}
this.totalWeight += tx.weight;
}
private setNextBounds(): void {
const nextPercentile = this.percentiles[this.bandIndex];
if (nextPercentile != null) {
this.leftBound = ((nextPercentile / 100) * this.maxWeight) - (this.bandWidth / 2);
this.rightBound = this.leftBound + this.bandWidth;
} else {
this.leftBound = Infinity;
this.rightBound = Infinity;
}
}
getRawFeeStats(): WorkingEffectiveFeeStats {
if (this.totalBandWeight > 0) {
const avgBandFeeRate = this.totalBandWeight ? (this.totalBandFee / this.totalBandWeight) : 0;
this.feeRange.unshift({ avg: avgBandFeeRate, min: this.minBandRate, max: this.maxBandRate });
}
while (this.feeRange.length < this.percentiles.length) {
this.feeRange.unshift({ avg: 0, min: 0, max: 0 });
}
return {
minFee: this.feeRange[0].min,
medianFee: this.feeRange[Math.floor(this.feeRange.length / 2)].avg,
maxFee: this.feeRange[this.feeRange.length - 1].max,
feeRange: this.feeRange.map(f => f.avg),
};
}
getFeeStats(): EffectiveFeeStats {
const stats = this.getRawFeeStats();
stats.feeRange[0] = stats.minFee;
stats.feeRange[stats.feeRange.length - 1] = stats.maxFee;
return stats;
}
}

53
backend/src/api/mempool-blocks.ts
View File

@ -1,6 +1,6 @@
import logger from '../logger';
import { MempoolBlock, TransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction } from '../mempool.interfaces';
import { Common } from './common';
import { MempoolBlock, TransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats } from '../mempool.interfaces';
import { Common, OnlineFeeStatsCalculator } from './common';
import config from '../config';
import { Worker } from 'worker_threads';
import path from 'path';
@ -104,6 +104,8 @@ class MempoolBlocks {
private calculateMempoolBlocks(transactionsSorted: TransactionExtended[]): MempoolBlockWithTransactions[] {
const mempoolBlocks: MempoolBlockWithTransactions[] = [];
let feeStatsCalculator: OnlineFeeStatsCalculator = new OnlineFeeStatsCalculator(config.MEMPOOL.BLOCK_WEIGHT_UNITS);
let onlineStats = false;
let blockSize = 0;
let blockWeight = 0;
let blockVsize = 0;
@ -111,7 +113,7 @@ class MempoolBlocks {
const sizeLimit = (config.MEMPOOL.BLOCK_WEIGHT_UNITS / 4) * 1.2;
let transactionIds: string[] = [];
let transactions: TransactionExtended[] = [];
transactionsSorted.forEach((tx) => {
transactionsSorted.forEach((tx, index) => {
if (blockWeight + tx.weight <= config.MEMPOOL.BLOCK_WEIGHT_UNITS
|| mempoolBlocks.length === config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT - 1) {
tx.position = {
@ -126,6 +128,9 @@ class MempoolBlocks {
transactions.push(tx);
}
transactionIds.push(tx.txid);
if (onlineStats) {
feeStatsCalculator.processNext(tx);
}
} else {
mempoolBlocks.push(this.dataToMempoolBlocks(transactionIds, transactions, blockSize, blockWeight, blockFees));
blockVsize = 0;
@ -133,6 +138,16 @@ class MempoolBlocks {
block: mempoolBlocks.length,
vsize: blockVsize + (tx.vsize / 2),
};
if (mempoolBlocks.length === config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT - 1) {
const stackWeight = transactionsSorted.slice(index).reduce((total, tx) => total + (tx.weight || 0), 0);
if (stackWeight > config.MEMPOOL.BLOCK_WEIGHT_UNITS) {
onlineStats = true;
feeStatsCalculator = new OnlineFeeStatsCalculator(stackWeight, 0.5);
feeStatsCalculator.processNext(tx);
}
}
blockVsize += tx.vsize;
blockWeight = tx.weight;
blockSize = tx.size;
@ -142,7 +157,8 @@ class MempoolBlocks {
}
});
if (transactions.length) {
mempoolBlocks.push(this.dataToMempoolBlocks(transactionIds, transactions, blockSize, blockWeight, blockFees));
const feeStats = onlineStats ? feeStatsCalculator.getRawFeeStats() : undefined;
mempoolBlocks.push(this.dataToMempoolBlocks(transactionIds, transactions, blockSize, blockWeight, blockFees, feeStats));
}
return mempoolBlocks;
@ -310,7 +326,16 @@ class MempoolBlocks {
}
}
const readyBlocks: { transactionIds, transactions, totalSize, totalWeight, totalFees }[] = [];
let hasBlockStack = blocks.length >= 8;
let stackWeight;
let feeStatsCalculator: OnlineFeeStatsCalculator | void;
if (hasBlockStack) {
stackWeight = blocks[blocks.length - 1].reduce((total, tx) => total + (mempool[tx]?.weight || 0), 0);
hasBlockStack = stackWeight > config.MEMPOOL.BLOCK_WEIGHT_UNITS;
feeStatsCalculator = new OnlineFeeStatsCalculator(stackWeight, 0.5);
}
const readyBlocks: { transactionIds, transactions, totalSize, totalWeight, totalFees, feeStats }[] = [];
const sizeLimit = (config.MEMPOOL.BLOCK_WEIGHT_UNITS / 4) * 1.2;
// update this thread's mempool with the results
for (let blockIndex = 0; blockIndex < blocks.length; blockIndex++) {
@ -333,6 +358,11 @@ class MempoolBlocks {
};
mempoolTx.cpfpChecked = true;
// online calculation of stack-of-blocks fee stats
if (hasBlockStack && blockIndex === blocks.length - 1 && feeStatsCalculator) {
feeStatsCalculator.processNext(mempoolTx);
}
totalSize += mempoolTx.size;
totalVsize += mempoolTx.vsize;
totalWeight += mempoolTx.weight;
@ -348,7 +378,8 @@ class MempoolBlocks {
transactions,
totalSize,
totalWeight,
totalFees
totalFees,
feeStats: (hasBlockStack && blockIndex === blocks.length - 1 && feeStatsCalculator) ? feeStatsCalculator.getRawFeeStats() : undefined,
});
}
@ -382,7 +413,9 @@ class MempoolBlocks {
}
}
const mempoolBlocks = readyBlocks.map(b => this.dataToMempoolBlocks(b.transactionIds, b.transactions, b.totalSize, b.totalWeight, b.totalFees));
const mempoolBlocks = readyBlocks.map((b, index) => {
return this.dataToMempoolBlocks(b.transactionIds, b.transactions, b.totalSize, b.totalWeight, b.totalFees, b.feeStats);
});
if (saveResults) {
const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, mempoolBlocks);
@ -393,8 +426,10 @@ class MempoolBlocks {
return mempoolBlocks;
}
private dataToMempoolBlocks(transactionIds: string[], transactions: TransactionExtended[], totalSize: number, totalWeight: number, totalFees: number): MempoolBlockWithTransactions {
const feeStats = Common.calcEffectiveFeeStatistics(transactions);
private dataToMempoolBlocks(transactionIds: string[], transactions: TransactionExtended[], totalSize: number, totalWeight: number, totalFees: number, feeStats?: EffectiveFeeStats ): MempoolBlockWithTransactions {
if (!feeStats) {
feeStats = Common.calcEffectiveFeeStatistics(transactions);
}
return {
blockSize: totalSize,
blockVSize: (totalWeight / 4), // fractional vsize to avoid rounding errors

5
backend/src/mempool.interfaces.ts
View File

@ -237,6 +237,11 @@ export interface EffectiveFeeStats {
feeRange: number[]; // 2nd, 10th, 25th, 50th, 75th, 90th, 98th percentiles
}
export interface WorkingEffectiveFeeStats extends EffectiveFeeStats {
minFee: number;
maxFee: number;
}
export interface CpfpSummary {
transactions: TransactionExtended[];
clusters: { root: string, height: number, txs: Ancestor[], effectiveFeePerVsize: number }[];