diff --git a/backend/src/api/bitcoin/bitcoin.routes.ts b/backend/src/api/bitcoin/bitcoin.routes.ts index 93b614006..c65363315 100644 --- a/backend/src/api/bitcoin/bitcoin.routes.ts +++ b/backend/src/api/bitcoin/bitcoin.routes.ts @@ -19,6 +19,7 @@ import bitcoinClient from './bitcoin-client'; import difficultyAdjustment from '../difficulty-adjustment'; import transactionRepository from '../../repositories/TransactionRepository'; import rbfCache from '../rbf-cache'; +import { calculateCpfp } from '../cpfp'; class BitcoinRoutes { public initRoutes(app: Application) { @@ -217,7 +218,7 @@ class BitcoinRoutes { return; } - const cpfpInfo = Common.setRelativesAndGetCpfpInfo(tx, mempool.getMempool()); + const cpfpInfo = calculateCpfp(tx, mempool.getMempool()); res.json(cpfpInfo); return; diff --git a/backend/src/api/cpfp.ts b/backend/src/api/cpfp.ts new file mode 100644 index 000000000..cf54771bb --- /dev/null +++ b/backend/src/api/cpfp.ts @@ -0,0 +1,282 @@ +import { CpfpInfo, MempoolTransactionExtended } from '../mempool.interfaces'; + +const CPFP_UPDATE_INTERVAL = 60_000; // update CPFP info at most once per 60s per transaction + +interface GraphTx extends MempoolTransactionExtended { + depends: string[]; + spentby: string[]; + ancestorMap: Map; + fees: { + base: number; + ancestor: number; + }; + ancestorcount: number; + ancestorsize: number; + ancestorRate: number; + individualRate: number; + score: number; +} + +/** + * Takes a mempool transaction and a copy of the current mempool, and calculates the CPFP data for + * that transaction (and all others in the same cluster) + */ +export function calculateCpfp(tx: MempoolTransactionExtended, mempool: { [txid: string]: MempoolTransactionExtended }): CpfpInfo { + if (tx.cpfpUpdated && Date.now() < (tx.cpfpUpdated + CPFP_UPDATE_INTERVAL)) { + tx.cpfpDirty = false; + return { + ancestors: tx.ancestors || [], + bestDescendant: tx.bestDescendant || null, + descendants: tx.descendants || [], + effectiveFeePerVsize: tx.effectiveFeePerVsize || tx.adjustedFeePerVsize || tx.feePerVsize, + sigops: tx.sigops, + adjustedVsize: tx.adjustedVsize, + acceleration: tx.acceleration + }; + } + + const ancestorMap = new Map(); + const graphTx = mempoolToGraphTx(tx); + ancestorMap.set(tx.txid, graphTx); + + const allRelatives = expandRelativesGraph(mempool, ancestorMap); + const relativesMap = initializeRelatives(allRelatives); + const cluster = calculateCpfpCluster(tx.txid, relativesMap); + + let totalVsize = 0; + let totalFee = 0; + for (const tx of cluster.values()) { + totalVsize += tx.adjustedVsize; + totalFee += tx.fee; + } + const effectiveFeePerVsize = totalFee / totalVsize; + for (const tx of cluster.values()) { + mempool[tx.txid].effectiveFeePerVsize = effectiveFeePerVsize; + mempool[tx.txid].ancestors = Array.from(tx.ancestorMap.values()).map(tx => ({ txid: tx.txid, weight: tx.weight, fee: tx.fee })); + mempool[tx.txid].descendants = Array.from(cluster.values()).filter(entry => entry.txid !== tx.txid && !tx.ancestorMap.has(entry.txid)).map(tx => ({ txid: tx.txid, weight: tx.weight, fee: tx.fee })); + mempool[tx.txid].bestDescendant = null; + mempool[tx.txid].cpfpChecked = true; + mempool[tx.txid].cpfpDirty = true; + mempool[tx.txid].cpfpUpdated = Date.now(); + } + + tx = mempool[tx.txid]; + + return { + ancestors: tx.ancestors || [], + bestDescendant: tx.bestDescendant || null, + descendants: tx.descendants || [], + effectiveFeePerVsize: tx.effectiveFeePerVsize || tx.adjustedFeePerVsize || tx.feePerVsize, + sigops: tx.sigops, + adjustedVsize: tx.adjustedVsize, + acceleration: tx.acceleration + }; +} + +function mempoolToGraphTx(tx: MempoolTransactionExtended): GraphTx { + return { + ...tx, + depends: [], + spentby: [], + ancestorMap: new Map(), + fees: { + base: tx.fee, + ancestor: tx.fee, + }, + ancestorcount: 1, + ancestorsize: tx.adjustedVsize, + ancestorRate: 0, + individualRate: 0, + score: 0, + }; +} + +/** + * Takes a map of transaction ancestors, and expands it into a full graph of up to 50 in-mempool relatives + */ +function expandRelativesGraph(mempool: { [txid: string]: MempoolTransactionExtended }, ancestors: Map): Map { + const relatives: Map = new Map(); + const stack: GraphTx[] = Array.from(ancestors.values()); + while (stack.length > 0) { + if (relatives.size > 50) { + return relatives; + } + + const nextTx = stack.pop(); + if (!nextTx) { + continue; + } + relatives.set(nextTx.txid, nextTx); + + for (const relativeTxid of [...nextTx.depends, ...nextTx.spentby]) { + if (relatives.has(relativeTxid)) { + // already processed this tx + continue; + } + let mempoolTx = ancestors.get(relativeTxid); + if (!mempoolTx && mempool[relativeTxid]) { + mempoolTx = mempoolToGraphTx(mempool[relativeTxid]); + } + if (mempoolTx) { + stack.push(mempoolTx); + } + } + } + + return relatives; +} + + /** + * Efficiently sets a Map of in-mempool ancestors for each member of an expanded relative graph + * by running setAncestors on each leaf, and caching intermediate results. + * then initializes ancestor data for each transaction + * + * @param all + */ + function initializeRelatives(mempoolTxs: Map): Map { + const visited: Map> = new Map(); + const leaves: GraphTx[] = Array.from(mempoolTxs.values()).filter(entry => entry.spentby.length === 0); + for (const leaf of leaves) { + setAncestors(leaf, mempoolTxs, visited); + } + mempoolTxs.forEach(entry => { + entry.ancestorMap?.forEach(ancestor => { + entry.ancestorcount++; + entry.ancestorsize += ancestor.adjustedVsize; + entry.fees.ancestor += ancestor.fees.base; + }); + setAncestorScores(entry); + }); + return mempoolTxs; +} + +/** + * Given a root transaction and a list of in-mempool ancestors, + * Calculate the CPFP cluster + * + * @param tx + * @param ancestors + */ +function calculateCpfpCluster(txid: string, graph: Map): Map { + const tx = graph.get(txid); + if (!tx) { + return new Map([]); + } + + // Initialize individual & ancestor fee rates + graph.forEach(entry => setAncestorScores(entry)); + + // Sort by descending ancestor score + let sortedRelatives = Array.from(graph.values()).sort(mempoolComparator); + + // Iterate until we reach a cluster that includes our target tx + let maxIterations = 50; + let best = sortedRelatives.shift(); + let bestCluster = new Map(best?.ancestorMap?.entries() || []); + while (sortedRelatives.length && best && (best.txid !== tx.txid && !best.ancestorMap.has(tx.txid)) && maxIterations > 0) { + maxIterations--; + if (bestCluster && bestCluster.has(tx.txid)) { + break; + } else { + // Remove this cluster (it doesn't include our target tx) + // and update scores, ancestor totals and dependencies for the survivors + removeAncestors(bestCluster, graph); + + // re-sort + sortedRelatives = Array.from(graph.values()).sort(mempoolComparator); + + // Grab the next highest scoring entry + best = sortedRelatives.shift(); + if (best) { + bestCluster = new Map(best?.ancestorMap?.entries() || []); + bestCluster.set(best?.txid, best); + } + } + } + + return bestCluster; +} + + /** + * Remove a cluster of transactions from an in-mempool dependency graph + * and update the survivors' scores and ancestors + * + * @param cluster + * @param ancestors + */ + function removeAncestors(cluster: Map, all: Map): void { + // remove + cluster.forEach(tx => { + all.delete(tx.txid); + }); + + // update survivors + all.forEach(tx => { + cluster.forEach(remove => { + if (tx.ancestorMap?.has(remove.txid)) { + // remove as dependency + tx.ancestorMap.delete(remove.txid); + tx.depends = tx.depends.filter(parent => parent !== remove.txid); + // update ancestor sizes and fees + tx.ancestorsize -= remove.adjustedVsize; + tx.fees.ancestor -= remove.fees.base; + } + }); + // recalculate fee rates + setAncestorScores(tx); + }); +} + +/** + * Recursively traverses an in-mempool dependency graph, and sets a Map of in-mempool ancestors + * for each transaction. + * + * @param tx + * @param all + */ +function setAncestors(tx: GraphTx, all: Map, visited: Map>, depth: number = 0): Map { + // sanity check for infinite recursion / too many ancestors (should never happen) + if (depth > 50) { + return tx.ancestorMap; + } + + // initialize the ancestor map for this tx + tx.ancestorMap = new Map(); + tx.depends.forEach(parentId => { + const parent = all.get(parentId); + if (parent) { + // add the parent + tx.ancestorMap?.set(parentId, parent); + // check for a cached copy of this parent's ancestors + let ancestors = visited.get(parent.txid); + if (!ancestors) { + // recursively fetch the parent's ancestors + ancestors = setAncestors(parent, all, visited, depth + 1); + } + // and add to this tx's map + ancestors.forEach((ancestor, ancestorId) => { + tx.ancestorMap?.set(ancestorId, ancestor); + }); + } + }); + visited.set(tx.txid, tx.ancestorMap); + + return tx.ancestorMap; +} + +/** + * Take a mempool transaction, and set the fee rates and ancestor score + * + * @param tx + */ +function setAncestorScores(tx: GraphTx): GraphTx { + tx.individualRate = (tx.fees.base * 100_000_000) / tx.adjustedVsize; + tx.ancestorRate = (tx.fees.ancestor * 100_000_000) / tx.ancestorsize; + tx.score = Math.min(tx.individualRate, tx.ancestorRate); + return tx; +} + +// Sort by descending score +function mempoolComparator(a: GraphTx, b: GraphTx): number { + return b.score - a.score; +} \ No newline at end of file diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 09c5d8e79..af071321e 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -335,7 +335,6 @@ class MempoolBlocks { } private resetRustGbt(): void { - console.log('reset rust gbt'); this.rustInitialized = false; this.rustGbtGenerator = new GbtGenerator(); } @@ -418,8 +417,6 @@ class MempoolBlocks { } const removedUids = removed.map(tx => this.getUid(tx)).filter(uid => (uid !== null && uid !== undefined)) as number[]; - console.log(`removing ${removed.length} (${removedUids.length} with uids)`); - const accelerations = useAccelerations ? mempool.getAccelerations() : {}; const acceleratedList = accelerationPool ? Object.values(accelerations).filter(acc => newMempool[acc.txid] && acc.pools.includes(accelerationPool)) : Object.values(accelerations).filter(acc => newMempool[acc.txid]); const convertedAccelerations = acceleratedList.map(acc => { @@ -429,8 +426,6 @@ class MempoolBlocks { }; }); - console.log(`${acceleratedList.length} accelerations`); - // run the block construction algorithm in a separate thread, and wait for a result try { const { blocks, blockWeights, rates, clusters, overflow } = this.convertNapiResultTxids( @@ -458,7 +453,7 @@ class MempoolBlocks { } } - private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], candidates: GbtCandidates | undefined, accelerations, accelerationPool, saveResults): MempoolBlockWithTransactions[] { + private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], candidates: GbtCandidates | undefined, accelerations, accelerationPool, saveResults): MempoolBlockWithTransactions[] { for (const [txid, rate] of rates) { if (txid in mempool) { mempool[txid].cpfpDirty = (rate !== mempool[txid].effectiveFeePerVsize); diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index 540e0828b..b08848b26 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -18,7 +18,6 @@ class Mempool { private mempoolCacheDelta: number = -1; private mempoolCache: { [txId: string]: MempoolTransactionExtended } = {}; private mempoolCandidates: { [txid: string ]: boolean } = {}; - private minFeeMempool: { [txId: string]: boolean } = {}; private spendMap = new Map(); private mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0, total_fee: 0, maxmempool: 300000000, mempoolminfee: Common.isLiquid() ? 0.00000100 : 0.00001000, minrelaytxfee: Common.isLiquid() ? 0.00000100 : 0.00001000 }; @@ -449,12 +448,10 @@ class Mempool { public async getNextCandidates(minFeeTransactions: string[], blockHeight: number): Promise { if (this.limitGBT) { const newCandidateTxMap = {}; - this.minFeeMempool = {}; for (const txid of minFeeTransactions) { if (this.mempoolCache[txid]) { newCandidateTxMap[txid] = true; } - this.minFeeMempool[txid] = true; } const removed: MempoolTransactionExtended[] = []; const added: MempoolTransactionExtended[] = []; @@ -466,16 +463,22 @@ class Mempool { } else { for (const txid of Object.keys(this.mempoolCandidates)) { if (!newCandidateTxMap[txid]) { - const tx = this.mempoolCache[txid]; - removed.push(tx); + removed.push(this.mempoolCache[txid]); + if (this.mempoolCache[txid]) { + this.mempoolCache[txid].effectiveFeePerVsize = this.mempoolCache[txid].adjustedFeePerVsize; + this.mempoolCache[txid].ancestors = []; + this.mempoolCache[txid].descendants = []; + this.mempoolCache[txid].bestDescendant = null; + this.mempoolCache[txid].cpfpChecked = false; + this.mempoolCache[txid].cpfpUpdated = undefined; + } } } } for (const txid of Object.keys(newCandidateTxMap)) { if (!this.mempoolCandidates[txid]) { - const tx = this.mempoolCache[txid]; - added.push(tx); + added.push(this.mempoolCache[txid]); } } diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index ac79d828a..f34f7e764 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -33,6 +33,7 @@ interface AddressTransactions { removed: MempoolTransactionExtended[], } import bitcoinSecondClient from './bitcoin/bitcoin-second-client'; +import { calculateCpfp } from './cpfp'; // valid 'want' subscriptions const wantable = [ @@ -702,6 +703,9 @@ class WebsocketHandler { accelerated: mempoolTx.acceleration || undefined, } }; + if (!mempoolTx.cpfpChecked) { + calculateCpfp(mempoolTx, newMempool); + } if (mempoolTx.cpfpDirty) { positionData['cpfp'] = { ancestors: mempoolTx.ancestors, diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index 64c10d322..a12351fe3 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -107,6 +107,7 @@ export interface MempoolTransactionExtended extends TransactionExtended { inputs?: number[]; lastBoosted?: number; cpfpDirty?: boolean; + cpfpUpdated?: number; } export interface AuditTransaction { @@ -187,6 +188,9 @@ export interface CpfpInfo { bestDescendant?: BestDescendant | null; descendants?: Ancestor[]; effectiveFeePerVsize?: number; + sigops?: number; + adjustedVsize?: number, + acceleration?: boolean, } export interface TransactionStripped {