diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 781973055..79afbae13 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -28,9 +28,8 @@ "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json", "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master", "AUDIT": false, - "ADVANCED_GBT_AUDIT": false, - "ADVANCED_GBT_MEMPOOL": false, "RUST_GBT": false, + "LIMIT_GBT": false, "CPFP_INDEXING": false, "DISK_CACHE_BLOCK_INTERVAL": 6, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index 5cbff22a3..41e9cce4e 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -28,9 +28,8 @@ "POOLS_JSON_TREE_URL": "__MEMPOOL_POOLS_JSON_TREE_URL__", "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__", "AUDIT": true, - "ADVANCED_GBT_AUDIT": true, - "ADVANCED_GBT_MEMPOOL": true, "RUST_GBT": false, + "LIMIT_GBT": false, "CPFP_INDEXING": true, "MAX_BLOCKS_BULK_QUERY": 999, "DISK_CACHE_BLOCK_INTERVAL": 999, diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index e261e2adc..75661951e 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -41,9 +41,8 @@ describe('Mempool Backend Config', () => { POOLS_JSON_TREE_URL: 'https://api.github.com/repos/mempool/mining-pools/git/trees/master', POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json', AUDIT: false, - ADVANCED_GBT_AUDIT: false, - ADVANCED_GBT_MEMPOOL: false, RUST_GBT: false, + LIMIT_GBT: false, CPFP_INDEXING: false, MAX_BLOCKS_BULK_QUERY: 0, DISK_CACHE_BLOCK_INTERVAL: 6, diff --git a/backend/src/api/bitcoin/bitcoin.routes.ts b/backend/src/api/bitcoin/bitcoin.routes.ts index 93b614006..c65363315 100644 --- a/backend/src/api/bitcoin/bitcoin.routes.ts +++ b/backend/src/api/bitcoin/bitcoin.routes.ts @@ -19,6 +19,7 @@ import bitcoinClient from './bitcoin-client'; import difficultyAdjustment from '../difficulty-adjustment'; import transactionRepository from '../../repositories/TransactionRepository'; import rbfCache from '../rbf-cache'; +import { calculateCpfp } from '../cpfp'; class BitcoinRoutes { public initRoutes(app: Application) { @@ -217,7 +218,7 @@ class BitcoinRoutes { return; } - const cpfpInfo = Common.setRelativesAndGetCpfpInfo(tx, mempool.getMempool()); + const cpfpInfo = calculateCpfp(tx, mempool.getMempool()); res.json(cpfpInfo); return; diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index 3021d947a..d8cf0d73f 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -574,69 +574,6 @@ export class Common { } } - static setRelativesAndGetCpfpInfo(tx: MempoolTransactionExtended, memPool: { [txid: string]: MempoolTransactionExtended }): CpfpInfo { - const parents = this.findAllParents(tx, memPool); - const lowerFeeParents = parents.filter((parent) => parent.adjustedFeePerVsize < tx.effectiveFeePerVsize); - - let totalWeight = (tx.adjustedVsize * 4) + lowerFeeParents.reduce((prev, val) => prev + (val.adjustedVsize * 4), 0); - let totalFees = tx.fee + lowerFeeParents.reduce((prev, val) => prev + val.fee, 0); - - tx.ancestors = parents - .map((t) => { - return { - txid: t.txid, - weight: (t.adjustedVsize * 4), - fee: t.fee, - }; - }); - - // Add high (high fee) decendant weight and fees - if (tx.bestDescendant) { - totalWeight += tx.bestDescendant.weight; - totalFees += tx.bestDescendant.fee; - } - - tx.effectiveFeePerVsize = Math.max(0, totalFees / (totalWeight / 4)); - tx.cpfpChecked = true; - - return { - ancestors: tx.ancestors, - bestDescendant: tx.bestDescendant || null, - }; - } - - - private static findAllParents(tx: MempoolTransactionExtended, memPool: { [txid: string]: MempoolTransactionExtended }): MempoolTransactionExtended[] { - let parents: MempoolTransactionExtended[] = []; - tx.vin.forEach((parent) => { - if (parents.find((p) => p.txid === parent.txid)) { - return; - } - - const parentTx = memPool[parent.txid]; - if (parentTx) { - if (tx.bestDescendant && tx.bestDescendant.fee / (tx.bestDescendant.weight / 4) > parentTx.adjustedFeePerVsize) { - if (parentTx.bestDescendant && parentTx.bestDescendant.fee < tx.fee + tx.bestDescendant.fee) { - parentTx.bestDescendant = { - weight: (tx.adjustedVsize * 4) + tx.bestDescendant.weight, - fee: tx.fee + tx.bestDescendant.fee, - txid: tx.txid, - }; - } - } else if (tx.adjustedFeePerVsize > parentTx.adjustedFeePerVsize) { - parentTx.bestDescendant = { - weight: (tx.adjustedVsize * 4), - fee: tx.fee, - txid: tx.txid - }; - } - parents.push(parentTx); - parents = parents.concat(this.findAllParents(parentTx, memPool)); - } - }); - return parents; - } - // calculates the ratio of matched transactions to projected transactions by weight static getSimilarity(projectedBlock: MempoolBlockWithTransactions, transactions: TransactionExtended[]): number { let matchedWeight = 0; diff --git a/backend/src/api/cpfp.ts b/backend/src/api/cpfp.ts new file mode 100644 index 000000000..604c1b3c9 --- /dev/null +++ b/backend/src/api/cpfp.ts @@ -0,0 +1,286 @@ +import { CpfpInfo, MempoolTransactionExtended } from '../mempool.interfaces'; +import memPool from './mempool'; + +const CPFP_UPDATE_INTERVAL = 60_000; // update CPFP info at most once per 60s per transaction +const MAX_GRAPH_SIZE = 50; // the maximum number of in-mempool relatives to consider + +interface GraphTx extends MempoolTransactionExtended { + depends: string[]; + spentby: string[]; + ancestorMap: Map; + fees: { + base: number; + ancestor: number; + }; + ancestorcount: number; + ancestorsize: number; + ancestorRate: number; + individualRate: number; + score: number; +} + +/** + * Takes a mempool transaction and a copy of the current mempool, and calculates the CPFP data for + * that transaction (and all others in the same cluster) + */ +export function calculateCpfp(tx: MempoolTransactionExtended, mempool: { [txid: string]: MempoolTransactionExtended }): CpfpInfo { + if (tx.cpfpUpdated && Date.now() < (tx.cpfpUpdated + CPFP_UPDATE_INTERVAL)) { + tx.cpfpDirty = false; + return { + ancestors: tx.ancestors || [], + bestDescendant: tx.bestDescendant || null, + descendants: tx.descendants || [], + effectiveFeePerVsize: tx.effectiveFeePerVsize || tx.adjustedFeePerVsize || tx.feePerVsize, + sigops: tx.sigops, + adjustedVsize: tx.adjustedVsize, + acceleration: tx.acceleration + }; + } + + const ancestorMap = new Map(); + const graphTx = mempoolToGraphTx(tx); + ancestorMap.set(tx.txid, graphTx); + + const allRelatives = expandRelativesGraph(mempool, ancestorMap); + const relativesMap = initializeRelatives(allRelatives); + const cluster = calculateCpfpCluster(tx.txid, relativesMap); + + let totalVsize = 0; + let totalFee = 0; + for (const tx of cluster.values()) { + totalVsize += tx.adjustedVsize; + totalFee += tx.fee; + } + const effectiveFeePerVsize = totalFee / totalVsize; + for (const tx of cluster.values()) { + mempool[tx.txid].effectiveFeePerVsize = effectiveFeePerVsize; + mempool[tx.txid].ancestors = Array.from(tx.ancestorMap.values()).map(tx => ({ txid: tx.txid, weight: tx.weight, fee: tx.fee })); + mempool[tx.txid].descendants = Array.from(cluster.values()).filter(entry => entry.txid !== tx.txid && !tx.ancestorMap.has(entry.txid)).map(tx => ({ txid: tx.txid, weight: tx.weight, fee: tx.fee })); + mempool[tx.txid].bestDescendant = null; + mempool[tx.txid].cpfpChecked = true; + mempool[tx.txid].cpfpDirty = true; + mempool[tx.txid].cpfpUpdated = Date.now(); + } + + tx = mempool[tx.txid]; + + return { + ancestors: tx.ancestors || [], + bestDescendant: tx.bestDescendant || null, + descendants: tx.descendants || [], + effectiveFeePerVsize: tx.effectiveFeePerVsize || tx.adjustedFeePerVsize || tx.feePerVsize, + sigops: tx.sigops, + adjustedVsize: tx.adjustedVsize, + acceleration: tx.acceleration + }; +} + +function mempoolToGraphTx(tx: MempoolTransactionExtended): GraphTx { + return { + ...tx, + depends: tx.vin.map(v => v.txid), + spentby: tx.vout.map((v, i) => memPool.getFromSpendMap(tx.txid, i)).map(tx => tx?.txid).filter(txid => txid != null) as string[], + ancestorMap: new Map(), + fees: { + base: tx.fee, + ancestor: tx.fee, + }, + ancestorcount: 1, + ancestorsize: tx.adjustedVsize, + ancestorRate: 0, + individualRate: 0, + score: 0, + }; +} + +/** + * Takes a map of transaction ancestors, and expands it into a full graph of up to MAX_GRAPH_SIZE in-mempool relatives + */ +function expandRelativesGraph(mempool: { [txid: string]: MempoolTransactionExtended }, ancestors: Map): Map { + const relatives: Map = new Map(); + const stack: GraphTx[] = Array.from(ancestors.values()); + while (stack.length > 0) { + if (relatives.size > MAX_GRAPH_SIZE) { + return relatives; + } + + const nextTx = stack.pop(); + if (!nextTx) { + continue; + } + relatives.set(nextTx.txid, nextTx); + + for (const relativeTxid of [...nextTx.depends, ...nextTx.spentby]) { + if (relatives.has(relativeTxid)) { + // already processed this tx + continue; + } + let mempoolTx = ancestors.get(relativeTxid); + if (!mempoolTx && mempool[relativeTxid]) { + mempoolTx = mempoolToGraphTx(mempool[relativeTxid]); + } + if (mempoolTx) { + stack.push(mempoolTx); + } + } + } + + return relatives; +} + + /** + * Efficiently sets a Map of in-mempool ancestors for each member of an expanded relative graph + * by running setAncestors on each leaf, and caching intermediate results. + * then initializes ancestor data for each transaction + * + * @param all + */ + function initializeRelatives(mempoolTxs: Map): Map { + const visited: Map> = new Map(); + const leaves: GraphTx[] = Array.from(mempoolTxs.values()).filter(entry => entry.spentby.length === 0); + for (const leaf of leaves) { + setAncestors(leaf, mempoolTxs, visited); + } + mempoolTxs.forEach(entry => { + entry.ancestorMap?.forEach(ancestor => { + entry.ancestorcount++; + entry.ancestorsize += ancestor.adjustedVsize; + entry.fees.ancestor += ancestor.fees.base; + }); + setAncestorScores(entry); + }); + return mempoolTxs; +} + +/** + * Given a root transaction and a list of in-mempool ancestors, + * Calculate the CPFP cluster + * + * @param tx + * @param ancestors + */ +function calculateCpfpCluster(txid: string, graph: Map): Map { + const tx = graph.get(txid); + if (!tx) { + return new Map([]); + } + + // Initialize individual & ancestor fee rates + graph.forEach(entry => setAncestorScores(entry)); + + // Sort by descending ancestor score + let sortedRelatives = Array.from(graph.values()).sort(mempoolComparator); + + // Iterate until we reach a cluster that includes our target tx + let maxIterations = MAX_GRAPH_SIZE; + let best = sortedRelatives.shift(); + let bestCluster = new Map(best?.ancestorMap?.entries() || []); + while (sortedRelatives.length && best && (best.txid !== tx.txid && !best.ancestorMap.has(tx.txid)) && maxIterations > 0) { + maxIterations--; + if ((best && best.txid === tx.txid) || (bestCluster && bestCluster.has(tx.txid))) { + break; + } else { + // Remove this cluster (it doesn't include our target tx) + // and update scores, ancestor totals and dependencies for the survivors + removeAncestors(bestCluster, graph); + + // re-sort + sortedRelatives = Array.from(graph.values()).sort(mempoolComparator); + + // Grab the next highest scoring entry + best = sortedRelatives.shift(); + if (best) { + bestCluster = new Map(best?.ancestorMap?.entries() || []); + bestCluster.set(best?.txid, best); + } + } + } + + bestCluster.set(tx.txid, tx); + + return bestCluster; +} + + /** + * Remove a cluster of transactions from an in-mempool dependency graph + * and update the survivors' scores and ancestors + * + * @param cluster + * @param ancestors + */ + function removeAncestors(cluster: Map, all: Map): void { + // remove + cluster.forEach(tx => { + all.delete(tx.txid); + }); + + // update survivors + all.forEach(tx => { + cluster.forEach(remove => { + if (tx.ancestorMap?.has(remove.txid)) { + // remove as dependency + tx.ancestorMap.delete(remove.txid); + tx.depends = tx.depends.filter(parent => parent !== remove.txid); + // update ancestor sizes and fees + tx.ancestorsize -= remove.adjustedVsize; + tx.fees.ancestor -= remove.fees.base; + } + }); + // recalculate fee rates + setAncestorScores(tx); + }); +} + +/** + * Recursively traverses an in-mempool dependency graph, and sets a Map of in-mempool ancestors + * for each transaction. + * + * @param tx + * @param all + */ +function setAncestors(tx: GraphTx, all: Map, visited: Map>, depth: number = 0): Map { + // sanity check for infinite recursion / too many ancestors (should never happen) + if (depth > MAX_GRAPH_SIZE) { + return tx.ancestorMap; + } + + // initialize the ancestor map for this tx + tx.ancestorMap = new Map(); + tx.depends.forEach(parentId => { + const parent = all.get(parentId); + if (parent) { + // add the parent + tx.ancestorMap?.set(parentId, parent); + // check for a cached copy of this parent's ancestors + let ancestors = visited.get(parent.txid); + if (!ancestors) { + // recursively fetch the parent's ancestors + ancestors = setAncestors(parent, all, visited, depth + 1); + } + // and add to this tx's map + ancestors.forEach((ancestor, ancestorId) => { + tx.ancestorMap?.set(ancestorId, ancestor); + }); + } + }); + visited.set(tx.txid, tx.ancestorMap); + + return tx.ancestorMap; +} + +/** + * Take a mempool transaction, and set the fee rates and ancestor score + * + * @param tx + */ +function setAncestorScores(tx: GraphTx): GraphTx { + tx.individualRate = (tx.fees.base * 100_000_000) / tx.adjustedVsize; + tx.ancestorRate = (tx.fees.ancestor * 100_000_000) / tx.ancestorsize; + tx.score = Math.min(tx.individualRate, tx.ancestorRate); + return tx; +} + +// Sort by descending score +function mempoolComparator(a: GraphTx, b: GraphTx): number { + return b.score - a.score; +} \ No newline at end of file diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index b9da7d4e8..1132663f7 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -1,6 +1,6 @@ import { GbtGenerator, GbtResult, ThreadTransaction as RustThreadTransaction, ThreadAcceleration as RustThreadAcceleration } from 'rust-gbt'; import logger from '../logger'; -import { MempoolBlock, MempoolTransactionExtended, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats, PoolTag, TransactionClassified, TransactionCompressed, MempoolDeltaChange } from '../mempool.interfaces'; +import { MempoolBlock, MempoolTransactionExtended, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats, TransactionClassified, TransactionCompressed, MempoolDeltaChange, GbtCandidates } from '../mempool.interfaces'; import { Common, OnlineFeeStatsCalculator } from './common'; import config from '../config'; import { Worker } from 'worker_threads'; @@ -18,6 +18,7 @@ class MempoolBlocks { private nextUid: number = 1; private uidMap: Map = new Map(); // map short numerical uids to full txids + private txidMap: Map = new Map(); // map full txids back to short numerical uids public getMempoolBlocks(): MempoolBlock[] { return this.mempoolBlocks.map((block) => { @@ -40,132 +41,6 @@ class MempoolBlocks { return this.mempoolBlockDeltas; } - public updateMempoolBlocks(memPool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false): MempoolBlockWithTransactions[] { - const latestMempool = memPool; - const memPoolArray: MempoolTransactionExtended[] = []; - for (const i in latestMempool) { - memPoolArray.push(latestMempool[i]); - } - const start = new Date().getTime(); - - // Clear bestDescendants & ancestors - memPoolArray.forEach((tx) => { - tx.bestDescendant = null; - tx.ancestors = []; - tx.cpfpChecked = false; - if (!tx.effectiveFeePerVsize) { - tx.effectiveFeePerVsize = tx.adjustedFeePerVsize; - } - }); - - // First sort - memPoolArray.sort((a, b) => { - if (a.adjustedFeePerVsize === b.adjustedFeePerVsize) { - // tie-break by lexicographic txid order for stability - return a.txid < b.txid ? -1 : 1; - } else { - return b.adjustedFeePerVsize - a.adjustedFeePerVsize; - } - }); - - // Loop through and traverse all ancestors and sum up all the sizes + fees - // Pass down size + fee to all unconfirmed children - let sizes = 0; - memPoolArray.forEach((tx) => { - sizes += tx.weight; - if (sizes > 4000000 * 8) { - return; - } - Common.setRelativesAndGetCpfpInfo(tx, memPool); - }); - - // Final sort, by effective fee - memPoolArray.sort((a, b) => { - if (a.effectiveFeePerVsize === b.effectiveFeePerVsize) { - // tie-break by lexicographic txid order for stability - return a.txid < b.txid ? -1 : 1; - } else { - return b.effectiveFeePerVsize - a.effectiveFeePerVsize; - } - }); - - const end = new Date().getTime(); - const time = end - start; - logger.debug('Mempool blocks calculated in ' + time / 1000 + ' seconds'); - - const blocks = this.calculateMempoolBlocks(memPoolArray); - - if (saveResults) { - const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, blocks); - this.mempoolBlocks = blocks; - this.mempoolBlockDeltas = deltas; - } - - return blocks; - } - - private calculateMempoolBlocks(transactionsSorted: MempoolTransactionExtended[]): MempoolBlockWithTransactions[] { - const mempoolBlocks: MempoolBlockWithTransactions[] = []; - let feeStatsCalculator: OnlineFeeStatsCalculator = new OnlineFeeStatsCalculator(config.MEMPOOL.BLOCK_WEIGHT_UNITS); - let onlineStats = false; - let blockSize = 0; - let blockWeight = 0; - let blockVsize = 0; - let blockFees = 0; - const sizeLimit = (config.MEMPOOL.BLOCK_WEIGHT_UNITS / 4) * 1.2; - let transactionIds: string[] = []; - let transactions: MempoolTransactionExtended[] = []; - transactionsSorted.forEach((tx, index) => { - if (blockWeight + tx.weight <= config.MEMPOOL.BLOCK_WEIGHT_UNITS - || mempoolBlocks.length === config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT - 1) { - tx.position = { - block: mempoolBlocks.length, - vsize: blockVsize + (tx.vsize / 2), - }; - blockWeight += tx.weight; - blockVsize += tx.vsize; - blockSize += tx.size; - blockFees += tx.fee; - if (blockVsize <= sizeLimit) { - transactions.push(tx); - } - transactionIds.push(tx.txid); - if (onlineStats) { - feeStatsCalculator.processNext(tx); - } - } else { - mempoolBlocks.push(this.dataToMempoolBlocks(transactionIds, transactions, blockSize, blockWeight, blockFees)); - blockVsize = 0; - tx.position = { - block: mempoolBlocks.length, - vsize: blockVsize + (tx.vsize / 2), - }; - - if (mempoolBlocks.length === config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT - 1) { - const stackWeight = transactionsSorted.slice(index).reduce((total, tx) => total + (tx.weight || 0), 0); - if (stackWeight > config.MEMPOOL.BLOCK_WEIGHT_UNITS) { - onlineStats = true; - feeStatsCalculator = new OnlineFeeStatsCalculator(stackWeight, 0.5, [10, 20, 30, 40, 50, 60, 70, 80, 90]); - feeStatsCalculator.processNext(tx); - } - } - - blockVsize += tx.vsize; - blockWeight = tx.weight; - blockSize = tx.size; - blockFees = tx.fee; - transactionIds = [tx.txid]; - transactions = [tx]; - } - }); - if (transactions.length) { - const feeStats = onlineStats ? feeStatsCalculator.getRawFeeStats() : undefined; - mempoolBlocks.push(this.dataToMempoolBlocks(transactionIds, transactions, blockSize, blockWeight, blockFees, feeStats)); - } - - return mempoolBlocks; - } - private calculateMempoolDeltas(prevBlocks: MempoolBlockWithTransactions[], mempoolBlocks: MempoolBlockWithTransactions[]): MempoolBlockDelta[] { const mempoolBlockDeltas: MempoolBlockDelta[] = []; for (let i = 0; i < Math.max(mempoolBlocks.length, prevBlocks.length); i++) { @@ -207,7 +82,7 @@ class MempoolBlocks { return mempoolBlockDeltas; } - public async $makeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise { + public async $makeBlockTemplates(transactions: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, candidates: GbtCandidates | undefined, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise { const start = Date.now(); // reset mempool short ids @@ -215,7 +90,8 @@ class MempoolBlocks { this.resetUids(); } // set missing short ids - for (const tx of Object.values(newMempool)) { + for (const txid of transactions) { + const tx = newMempool[txid]; this.setUid(tx, !saveResults); } @@ -224,7 +100,8 @@ class MempoolBlocks { // prepare a stripped down version of the mempool with only the minimum necessary data // to reduce the overhead of passing this data to the worker thread const strippedMempool: Map = new Map(); - Object.values(newMempool).forEach(entry => { + for (const txid of transactions) { + const entry = newMempool[txid]; if (entry.uid !== null && entry.uid !== undefined) { const stripped = { uid: entry.uid, @@ -237,7 +114,7 @@ class MempoolBlocks { }; strippedMempool.set(entry.uid, stripped); } - }); + } // (re)initialize tx selection worker thread if (!this.txSelectionWorker) { @@ -268,7 +145,7 @@ class MempoolBlocks { // clean up thread error listener this.txSelectionWorker?.removeListener('error', threadErrorListener); - const processed = this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), accelerations, accelerationPool, saveResults); + const processed = this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), candidates, accelerations, accelerationPool, saveResults); logger.debug(`makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); @@ -279,10 +156,10 @@ class MempoolBlocks { return this.mempoolBlocks; } - public async $updateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], accelerationDelta: string[] = [], saveResults: boolean = false, useAccelerations: boolean = false): Promise { + public async $updateBlockTemplates(transactions: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], candidates: GbtCandidates | undefined, accelerationDelta: string[] = [], saveResults: boolean = false, useAccelerations: boolean = false): Promise { if (!this.txSelectionWorker) { // need to reset the worker - await this.$makeBlockTemplates(newMempool, saveResults, useAccelerations); + await this.$makeBlockTemplates(transactions, newMempool, candidates, saveResults, useAccelerations); return; } @@ -292,9 +169,9 @@ class MempoolBlocks { const addedAndChanged: MempoolTransactionExtended[] = useAccelerations ? accelerationDelta.map(txid => newMempool[txid]).filter(tx => tx != null).concat(added) : added; for (const tx of addedAndChanged) { - this.setUid(tx, true); + this.setUid(tx, false); } - const removedUids = removed.map(tx => this.getUid(tx)).filter(uid => uid != null) as number[]; + const removedTxs = removed.filter(tx => tx.uid != null) as MempoolTransactionExtended[]; // prepare a stripped down version of the mempool with only the minimum necessary data // to reduce the overhead of passing this data to the worker thread @@ -320,15 +197,15 @@ class MempoolBlocks { }); this.txSelectionWorker?.once('error', reject); }); - this.txSelectionWorker.postMessage({ type: 'update', added: addedStripped, removed: removedUids }); + this.txSelectionWorker.postMessage({ type: 'update', added: addedStripped, removed: removedTxs.map(tx => tx.uid) as number[] }); const { blocks, rates, clusters } = this.convertResultTxids(await workerResultPromise); - this.removeUids(removedUids); + this.removeUids(removedTxs); // clean up thread error listener this.txSelectionWorker?.removeListener('error', threadErrorListener); - this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), accelerations, null, saveResults); + this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), candidates, accelerations, null, saveResults); logger.debug(`updateBlockTemplates completed in ${(Date.now() - start) / 1000} seconds`); } catch (e) { logger.err('updateBlockTemplates failed. ' + (e instanceof Error ? e.message : e)); @@ -340,25 +217,28 @@ class MempoolBlocks { this.rustGbtGenerator = new GbtGenerator(); } - public async $rustMakeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise { + public async $rustMakeBlockTemplates(txids: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, candidates: GbtCandidates | undefined, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise { const start = Date.now(); // reset mempool short ids if (saveResults) { this.resetUids(); } + + const transactions = txids.map(txid => newMempool[txid]).filter(tx => tx != null); // set missing short ids - for (const tx of Object.values(newMempool)) { + for (const tx of transactions) { this.setUid(tx, !saveResults); } // set short ids for transaction inputs - for (const tx of Object.values(newMempool)) { + for (const tx of transactions) { tx.inputs = tx.vin.map(v => this.getUid(newMempool[v.txid])).filter(uid => (uid !== null && uid !== undefined)) as number[]; } const accelerations = useAccelerations ? mempool.getAccelerations() : {}; const acceleratedList = accelerationPool ? Object.values(accelerations).filter(acc => newMempool[acc.txid] && acc.pools.includes(accelerationPool)) : Object.values(accelerations).filter(acc => newMempool[acc.txid]); const convertedAccelerations = acceleratedList.map(acc => { + this.setUid(newMempool[acc.txid], true); return { uid: this.getUid(newMempool[acc.txid]), delta: acc.feeDelta, @@ -369,15 +249,15 @@ class MempoolBlocks { const rustGbt = saveResults ? this.rustGbtGenerator : new GbtGenerator(); try { const { blocks, blockWeights, rates, clusters, overflow } = this.convertNapiResultTxids( - await rustGbt.make(Object.values(newMempool) as RustThreadTransaction[], convertedAccelerations as RustThreadAcceleration[], this.nextUid), + await rustGbt.make(transactions as RustThreadTransaction[], convertedAccelerations as RustThreadAcceleration[], this.nextUid), ); if (saveResults) { this.rustInitialized = true; } - const mempoolSize = Object.keys(newMempool).length; + const expectedSize = transactions.length; const resultMempoolSize = blocks.reduce((total, block) => total + block.length, 0) + overflow.length; - logger.debug(`RUST updateBlockTemplates returned ${resultMempoolSize} txs out of ${mempoolSize} in the mempool, ${overflow.length} were unmineable`); - const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, accelerations, accelerationPool, saveResults); + logger.debug(`RUST updateBlockTemplates returned ${resultMempoolSize} txs out of ${expectedSize} in the mempool, ${overflow.length} were unmineable`); + const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, candidates, accelerations, accelerationPool, saveResults); logger.debug(`RUST makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); return processed; } catch (e) { @@ -389,36 +269,37 @@ class MempoolBlocks { return this.mempoolBlocks; } - public async $oneOffRustBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, useAccelerations: boolean, accelerationPool?: number): Promise { - return this.$rustMakeBlockTemplates(newMempool, false, useAccelerations, accelerationPool); + public async $oneOffRustBlockTemplates(transactions: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, candidates: GbtCandidates | undefined, useAccelerations: boolean, accelerationPool?: number): Promise { + return this.$rustMakeBlockTemplates(transactions, newMempool, candidates, false, useAccelerations, accelerationPool); } - public async $rustUpdateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, mempoolSize: number, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], useAccelerations: boolean, accelerationPool?: number): Promise { + public async $rustUpdateBlockTemplates(transactions: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], candidates: GbtCandidates | undefined, useAccelerations: boolean, accelerationPool?: number): Promise { // GBT optimization requires that uids never get too sparse // as a sanity check, we should also explicitly prevent uint32 uid overflow - if (this.nextUid + added.length >= Math.min(Math.max(262144, 2 * mempoolSize), MAX_UINT32)) { + if (this.nextUid + added.length >= Math.min(Math.max(262144, 2 * transactions.length), MAX_UINT32)) { this.resetRustGbt(); } if (!this.rustInitialized) { // need to reset the worker - return this.$rustMakeBlockTemplates(newMempool, true, useAccelerations, accelerationPool); + return this.$rustMakeBlockTemplates(transactions, newMempool, candidates, true, useAccelerations, accelerationPool); } const start = Date.now(); // set missing short ids for (const tx of added) { - this.setUid(tx, true); + this.setUid(tx, false); } // set short ids for transaction inputs for (const tx of added) { tx.inputs = tx.vin.map(v => this.getUid(newMempool[v.txid])).filter(uid => (uid !== null && uid !== undefined)) as number[]; } - const removedUids = removed.map(tx => this.getUid(tx)).filter(uid => (uid !== null && uid !== undefined)) as number[]; + const removedTxs = removed.filter(tx => tx.uid != null) as MempoolTransactionExtended[]; const accelerations = useAccelerations ? mempool.getAccelerations() : {}; const acceleratedList = accelerationPool ? Object.values(accelerations).filter(acc => newMempool[acc.txid] && acc.pools.includes(accelerationPool)) : Object.values(accelerations).filter(acc => newMempool[acc.txid]); const convertedAccelerations = acceleratedList.map(acc => { + this.setUid(newMempool[acc.txid], true); return { uid: this.getUid(newMempool[acc.txid]), delta: acc.feeDelta, @@ -430,18 +311,18 @@ class MempoolBlocks { const { blocks, blockWeights, rates, clusters, overflow } = this.convertNapiResultTxids( await this.rustGbtGenerator.update( added as RustThreadTransaction[], - removedUids, + removedTxs.map(tx => tx.uid) as number[], convertedAccelerations as RustThreadAcceleration[], this.nextUid, ), ); const resultMempoolSize = blocks.reduce((total, block) => total + block.length, 0) + overflow.length; - logger.debug(`RUST updateBlockTemplates returned ${resultMempoolSize} txs out of ${mempoolSize} in the mempool, ${overflow.length} were unmineable`); - if (mempoolSize !== resultMempoolSize) { - throw new Error('GBT returned wrong number of transactions , cache is probably out of sync'); + logger.debug(`RUST updateBlockTemplates returned ${resultMempoolSize} txs out of ${transactions.length} candidates, ${overflow.length} were unmineable`); + if (transactions.length !== resultMempoolSize) { + throw new Error(`GBT returned wrong number of transactions ${transactions.length} vs ${resultMempoolSize}, cache is probably out of sync`); } else { - const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, accelerations, accelerationPool, true); - this.removeUids(removedUids); + const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, candidates, accelerations, accelerationPool, true); + this.removeUids(removedTxs); logger.debug(`RUST updateBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); return processed; } @@ -452,7 +333,7 @@ class MempoolBlocks { } } - private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], accelerations, accelerationPool, saveResults): MempoolBlockWithTransactions[] { + private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], candidates: GbtCandidates | undefined, accelerations, accelerationPool, saveResults): MempoolBlockWithTransactions[] { for (const [txid, rate] of rates) { if (txid in mempool) { mempool[txid].cpfpDirty = (rate !== mempool[txid].effectiveFeePerVsize); @@ -486,6 +367,9 @@ class MempoolBlocks { if (txid === memberTxid) { matched = true; } else { + if (!mempool[txid]) { + console.log('txid missing from mempool! ', txid, candidates?.txs[txid]); + } const relative = { txid: txid, fee: mempool[txid].fee, @@ -518,6 +402,16 @@ class MempoolBlocks { let totalWeight = 0; let totalFees = 0; const transactions: MempoolTransactionExtended[] = []; + + // backfill purged transactions + if (candidates?.txs && blockIndex === blocks.length - 1) { + for (const txid of Object.keys(mempool)) { + if (!candidates.txs[txid]) { + block.push(txid); + } + } + } + for (const txid of block) { if (txid) { mempoolTx = mempool[txid]; @@ -526,16 +420,6 @@ class MempoolBlocks { block: blockIndex, vsize: totalVsize + (mempoolTx.vsize / 2), }; - if (!mempoolTx.cpfpChecked) { - if (mempoolTx.ancestors?.length) { - mempoolTx.ancestors = []; - } - if (mempoolTx.descendants?.length) { - mempoolTx.descendants = []; - } - mempoolTx.bestDescendant = null; - mempoolTx.cpfpChecked = true; - } const acceleration = accelerations[txid]; if (isAccelerated[txid] || (acceleration && (!accelerationPool || acceleration.pools.includes(accelerationPool)))) { @@ -610,30 +494,38 @@ class MempoolBlocks { private resetUids(): void { this.uidMap.clear(); + this.txidMap.clear(); this.nextUid = 1; } private setUid(tx: MempoolTransactionExtended, skipSet = false): number { - if (tx.uid === null || tx.uid === undefined || !skipSet) { + if (!this.txidMap.has(tx.txid) || !skipSet) { const uid = this.nextUid; this.nextUid++; this.uidMap.set(uid, tx.txid); + this.txidMap.set(tx.txid, uid); tx.uid = uid; return uid; } else { + tx.uid = this.txidMap.get(tx.txid) as number; return tx.uid; } } private getUid(tx: MempoolTransactionExtended): number | void { - if (tx?.uid !== null && tx?.uid !== undefined && this.uidMap.has(tx.uid)) { - return tx.uid; + if (tx) { + return this.txidMap.get(tx.txid); } } - private removeUids(uids: number[]): void { - for (const uid of uids) { - this.uidMap.delete(uid); + private removeUids(txs: MempoolTransactionExtended[]): void { + for (const tx of txs) { + const uid = this.txidMap.get(tx.txid); + if (uid != null) { + this.uidMap.delete(uid); + this.txidMap.delete(tx.txid); + } + tx.uid = undefined; } } diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index 4c2f248ac..176bedddb 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -1,6 +1,6 @@ import config from '../config'; import bitcoinApi from './bitcoin/bitcoin-api-factory'; -import { MempoolTransactionExtended, TransactionExtended, VbytesPerSecond } from '../mempool.interfaces'; +import { MempoolTransactionExtended, TransactionExtended, VbytesPerSecond, GbtCandidates } from '../mempool.interfaces'; import logger from '../logger'; import { Common } from './common'; import transactionUtils from './transaction-utils'; @@ -11,18 +11,20 @@ import bitcoinSecondClient from './bitcoin/bitcoin-second-client'; import rbfCache from './rbf-cache'; import { Acceleration } from './services/acceleration'; import redisCache from './redis-cache'; +import blocks from './blocks'; class Mempool { private inSync: boolean = false; private mempoolCacheDelta: number = -1; private mempoolCache: { [txId: string]: MempoolTransactionExtended } = {}; + private mempoolCandidates: { [txid: string ]: boolean } = {}; private spendMap = new Map(); private mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0, total_fee: 0, maxmempool: 300000000, mempoolminfee: Common.isLiquid() ? 0.00000100 : 0.00001000, minrelaytxfee: Common.isLiquid() ? 0.00000100 : 0.00001000 }; private mempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => void) | undefined; private $asyncMempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, mempoolSize: number, newTransactions: MempoolTransactionExtended[], - deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => Promise) | undefined; + deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[], candidates?: GbtCandidates) => Promise) | undefined; private accelerations: { [txId: string]: Acceleration } = {}; @@ -40,6 +42,8 @@ class Mempool { private missingTxCount = 0; private mainLoopTimeout: number = 120000; + public limitGBT = config.MEMPOOL.USE_SECOND_NODE_FOR_MINFEE && config.MEMPOOL.LIMIT_GBT; + constructor() { setInterval(this.updateTxPerSecond.bind(this), 1000); } @@ -74,7 +78,8 @@ class Mempool { } public setAsyncMempoolChangedCallback(fn: (newMempool: { [txId: string]: MempoolTransactionExtended; }, mempoolSize: number, - newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => Promise): void { + newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[], + candidates?: GbtCandidates) => Promise): void { this.$asyncMempoolChangedCallback = fn; } @@ -86,6 +91,10 @@ class Mempool { return this.spendMap; } + public getFromSpendMap(txid, index): MempoolTransactionExtended | void { + return this.spendMap.get(`${txid}:${index}`); + } + public async $setMempool(mempoolData: { [txId: string]: MempoolTransactionExtended }) { this.mempoolCache = mempoolData; let count = 0; @@ -108,6 +117,9 @@ class Mempool { await redisCache.$addTransaction(this.mempoolCache[txid]); } this.mempoolCache[txid].flags = Common.getTransactionFlags(this.mempoolCache[txid]); + this.mempoolCache[txid].cpfpChecked = false; + this.mempoolCache[txid].cpfpDirty = true; + this.mempoolCache[txid].cpfpUpdated = undefined; } if (config.MEMPOOL.CACHE_ENABLED && config.REDIS.ENABLED) { await redisCache.$flushTransactions(); @@ -117,7 +129,7 @@ class Mempool { this.mempoolChangedCallback(this.mempoolCache, [], [], []); } if (this.$asyncMempoolChangedCallback) { - await this.$asyncMempoolChangedCallback(this.mempoolCache, count, [], [], []); + await this.$asyncMempoolChangedCallback(this.mempoolCache, count, [], [], [], this.limitGBT ? { txs: {}, added: [], removed: [] } : undefined); } this.addToSpendMap(Object.values(this.mempoolCache)); } @@ -160,6 +172,10 @@ class Mempool { return newTransactions; } + public getMempoolCandidates(): { [txid: string]: boolean } { + return this.mempoolCandidates; + } + public async $updateMemPoolInfo() { this.mempoolInfo = await this.$getMempoolInfo(); } @@ -189,7 +205,7 @@ class Mempool { return txTimes; } - public async $updateMempool(transactions: string[], accelerations: Acceleration[] | null, pollRate: number): Promise { + public async $updateMempool(transactions: string[], accelerations: Acceleration[] | null, minFeeMempool: string[], minFeeTip: number, pollRate: number): Promise { logger.debug(`Updating mempool...`); // warn if this run stalls the main loop for more than 2 minutes @@ -330,6 +346,8 @@ class Mempool { } } + const candidates = await this.getNextCandidates(minFeeMempool, minFeeTip, deletedTransactions); + const newMempoolSize = currentMempoolSize + newTransactions.length - deletedTransactions.length; const newTransactionsStripped = newTransactions.map((tx) => Common.stripTransaction(tx)); this.latestTransactions = newTransactionsStripped.concat(this.latestTransactions).slice(0, 6); @@ -341,12 +359,14 @@ class Mempool { this.mempoolCacheDelta = Math.abs(transactions.length - newMempoolSize); + const candidatesChanged = candidates?.added?.length || candidates?.removed?.length; + if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) { this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions, accelerationDelta); } - if (this.$asyncMempoolChangedCallback && (hasChange || deletedTransactions.length)) { + if (this.$asyncMempoolChangedCallback && (hasChange || deletedTransactions.length || candidatesChanged)) { this.updateTimerProgress(timer, 'running async mempool callback'); - await this.$asyncMempoolChangedCallback(this.mempoolCache, newMempoolSize, newTransactions, deletedTransactions, accelerationDelta); + await this.$asyncMempoolChangedCallback(this.mempoolCache, newMempoolSize, newTransactions, deletedTransactions, accelerationDelta, candidates); this.updateTimerProgress(timer, 'completed async mempool callback'); } @@ -432,6 +452,64 @@ class Mempool { } } + public async getNextCandidates(minFeeTransactions: string[], blockHeight: number, deletedTransactions: MempoolTransactionExtended[]): Promise { + if (this.limitGBT) { + const deletedTxsMap = {}; + for (const tx of deletedTransactions) { + deletedTxsMap[tx.txid] = tx; + } + const newCandidateTxMap = {}; + for (const txid of minFeeTransactions) { + if (this.mempoolCache[txid]) { + newCandidateTxMap[txid] = true; + } + } + const accelerations = this.getAccelerations(); + for (const txid of Object.keys(accelerations)) { + if (this.mempoolCache[txid]) { + newCandidateTxMap[txid] = true; + } + } + const removed: MempoolTransactionExtended[] = []; + const added: MempoolTransactionExtended[] = []; + // don't prematurely remove txs included in a new block + if (blockHeight > blocks.getCurrentBlockHeight()) { + for (const txid of Object.keys(this.mempoolCandidates)) { + newCandidateTxMap[txid] = true; + } + } else { + for (const txid of Object.keys(this.mempoolCandidates)) { + if (!newCandidateTxMap[txid]) { + if (this.mempoolCache[txid]) { + removed.push(this.mempoolCache[txid]); + this.mempoolCache[txid].effectiveFeePerVsize = this.mempoolCache[txid].adjustedFeePerVsize; + this.mempoolCache[txid].ancestors = []; + this.mempoolCache[txid].descendants = []; + this.mempoolCache[txid].bestDescendant = null; + this.mempoolCache[txid].cpfpChecked = false; + this.mempoolCache[txid].cpfpUpdated = undefined; + } else if (deletedTxsMap[txid]) { + removed.push(deletedTxsMap[txid]); + } + } + } + } + + for (const txid of Object.keys(newCandidateTxMap)) { + if (!this.mempoolCandidates[txid]) { + added.push(this.mempoolCache[txid]); + } + } + + this.mempoolCandidates = newCandidateTxMap; + return { + txs: this.mempoolCandidates, + added, + removed + }; + } + } + private startTimer() { const state: any = { start: Date.now(), diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 2884f72eb..63e7f383c 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -2,7 +2,7 @@ import logger from '../logger'; import * as WebSocket from 'ws'; import { BlockExtended, TransactionExtended, MempoolTransactionExtended, WebsocketResponse, - OptimizedStatistic, ILoadingIndicators + OptimizedStatistic, ILoadingIndicators, GbtCandidates, } from '../mempool.interfaces'; import blocks from './blocks'; import memPool from './mempool'; @@ -18,7 +18,6 @@ import feeApi from './fee-api'; import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository'; import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository'; import Audit from './audit'; -import { deepClone } from '../utils/clone'; import priceUpdater from '../tasks/price-updater'; import { ApiPrice } from '../repositories/PricesRepository'; import accelerationApi from './services/acceleration'; @@ -32,6 +31,8 @@ interface AddressTransactions { confirmed: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], } +import bitcoinSecondClient from './bitcoin/bitcoin-second-client'; +import { calculateCpfp } from './cpfp'; // valid 'want' subscriptions const wantable = [ @@ -436,21 +437,26 @@ class WebsocketHandler { } async $handleMempoolChange(newMempool: { [txid: string]: MempoolTransactionExtended }, mempoolSize: number, - newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]): Promise { + newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[], + candidates?: GbtCandidates): Promise { if (!this.wss) { throw new Error('WebSocket.Server is not set'); } this.printLogs(); - if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) { - if (config.MEMPOOL.RUST_GBT) { - await mempoolBlocks.$rustUpdateBlockTemplates(newMempool, mempoolSize, newTransactions, deletedTransactions, config.MEMPOOL_SERVICES.ACCELERATIONS); - } else { - await mempoolBlocks.$updateBlockTemplates(newMempool, newTransactions, deletedTransactions, accelerationDelta, true, config.MEMPOOL_SERVICES.ACCELERATIONS); - } + const transactionIds = (memPool.limitGBT && candidates) ? Object.keys(candidates?.txs || {}) : Object.keys(newMempool); + let added = newTransactions; + let removed = deletedTransactions; + if (memPool.limitGBT) { + added = candidates?.added || []; + removed = candidates?.removed || []; + } + + if (config.MEMPOOL.RUST_GBT) { + await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, newMempool, added, removed, candidates, config.MEMPOOL_SERVICES.ACCELERATIONS); } else { - mempoolBlocks.updateMempoolBlocks(newMempool, true); + await mempoolBlocks.$updateBlockTemplates(transactionIds, newMempool, added, removed, candidates, accelerationDelta, true, config.MEMPOOL_SERVICES.ACCELERATIONS); } const mBlocks = mempoolBlocks.getMempoolBlocks(); @@ -689,6 +695,9 @@ class WebsocketHandler { accelerated: mempoolTx.acceleration || undefined, } }; + if (!mempoolTx.cpfpChecked) { + calculateCpfp(mempoolTx, newMempool); + } if (mempoolTx.cpfpDirty) { positionData['cpfp'] = { ancestors: mempoolTx.ancestors, @@ -739,8 +748,9 @@ class WebsocketHandler { await statistics.runStatistics(); const _memPool = memPool.getMempool(); - - const isAccelerated = config.MEMPOOL_SERVICES.ACCELERATIONS && accelerationApi.isAcceleratedBlock(block, Object.values(mempool.getAccelerations())); + const candidateTxs = await memPool.getMempoolCandidates(); + let candidates: GbtCandidates | undefined = (memPool.limitGBT && candidateTxs) ? { txs: candidateTxs, added: [], removed: [] } : undefined; + let transactionIds: string[] = (memPool.limitGBT) ? Object.keys(candidates?.txs || {}) : Object.keys(_memPool); const accelerations = Object.values(mempool.getAccelerations()); await accelerationRepository.$indexAccelerationsForBlock(block, accelerations, transactions); @@ -751,31 +761,19 @@ class WebsocketHandler { if (config.MEMPOOL.AUDIT && memPool.isInSync()) { let projectedBlocks; - let auditMempool = _memPool; - // template calculation functions have mempool side effects, so calculate audits using - // a cloned copy of the mempool if we're running a different algorithm for mempool updates - const separateAudit = config.MEMPOOL.ADVANCED_GBT_AUDIT !== config.MEMPOOL.ADVANCED_GBT_MEMPOOL; - if (separateAudit) { - auditMempool = deepClone(_memPool); - if (config.MEMPOOL.ADVANCED_GBT_AUDIT) { - if (config.MEMPOOL.RUST_GBT) { - projectedBlocks = await mempoolBlocks.$oneOffRustBlockTemplates(auditMempool, isAccelerated, block.extras.pool.id); - } else { - projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, isAccelerated, block.extras.pool.id); - } + const auditMempool = _memPool; + const isAccelerated = config.MEMPOOL_SERVICES.ACCELERATIONS && accelerationApi.isAcceleratedBlock(block, Object.values(mempool.getAccelerations())); + + if ((config.MEMPOOL_SERVICES.ACCELERATIONS)) { + if (config.MEMPOOL.RUST_GBT) { + const added = memPool.limitGBT ? (candidates?.added || []) : []; + const removed = memPool.limitGBT ? (candidates?.removed || []) : []; + projectedBlocks = await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, auditMempool, added, removed, candidates, isAccelerated, block.extras.pool.id); } else { - projectedBlocks = mempoolBlocks.updateMempoolBlocks(auditMempool, false); + projectedBlocks = await mempoolBlocks.$makeBlockTemplates(transactionIds, auditMempool, candidates, false, isAccelerated, block.extras.pool.id); } } else { - if ((config.MEMPOOL_SERVICES.ACCELERATIONS)) { - if (config.MEMPOOL.RUST_GBT) { - projectedBlocks = await mempoolBlocks.$rustUpdateBlockTemplates(auditMempool, Object.keys(auditMempool).length, [], [], isAccelerated, block.extras.pool.id); - } else { - projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, isAccelerated, block.extras.pool.id); - } - } else { - projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions(); - } + projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions(); } if (Common.indexingEnabled()) { @@ -838,14 +836,23 @@ class WebsocketHandler { confirmedTxids[txId] = true; } - if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) { - if (config.MEMPOOL.RUST_GBT) { - await mempoolBlocks.$rustUpdateBlockTemplates(_memPool, Object.keys(_memPool).length, [], transactions, true); - } else { - await mempoolBlocks.$makeBlockTemplates(_memPool, true, config.MEMPOOL_SERVICES.ACCELERATIONS); - } + if (memPool.limitGBT) { + const minFeeMempool = memPool.limitGBT ? await bitcoinSecondClient.getRawMemPool() : null; + const minFeeTip = memPool.limitGBT ? await bitcoinSecondClient.getBlockCount() : -1; + candidates = await memPool.getNextCandidates(minFeeMempool, minFeeTip, transactions); + transactionIds = Object.keys(candidates?.txs || {}); } else { - mempoolBlocks.updateMempoolBlocks(_memPool, true); + candidates = undefined; + transactionIds = Object.keys(memPool.getMempool()); + } + + + if (config.MEMPOOL.RUST_GBT) { + const added = memPool.limitGBT ? (candidates?.added || []) : []; + const removed = memPool.limitGBT ? (candidates?.removed || []) : transactions; + await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, _memPool, added, removed, candidates, true); + } else { + await mempoolBlocks.$makeBlockTemplates(transactionIds, _memPool, candidates, true, config.MEMPOOL_SERVICES.ACCELERATIONS); } const mBlocks = mempoolBlocks.getMempoolBlocks(); const mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas(); diff --git a/backend/src/config.ts b/backend/src/config.ts index ee3645e58..41dd0aadb 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -32,9 +32,8 @@ interface IConfig { POOLS_JSON_URL: string, POOLS_JSON_TREE_URL: string, AUDIT: boolean; - ADVANCED_GBT_AUDIT: boolean; - ADVANCED_GBT_MEMPOOL: boolean; RUST_GBT: boolean; + LIMIT_GBT: boolean; CPFP_INDEXING: boolean; MAX_BLOCKS_BULK_QUERY: number; DISK_CACHE_BLOCK_INTERVAL: number; @@ -194,9 +193,8 @@ const defaults: IConfig = { 'POOLS_JSON_URL': 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json', 'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master', 'AUDIT': false, - 'ADVANCED_GBT_AUDIT': false, - 'ADVANCED_GBT_MEMPOOL': false, 'RUST_GBT': false, + 'LIMIT_GBT': false, 'CPFP_INDEXING': false, 'MAX_BLOCKS_BULK_QUERY': 0, 'DISK_CACHE_BLOCK_INTERVAL': 6, diff --git a/backend/src/index.ts b/backend/src/index.ts index 1988c7c56..ae5113029 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -45,6 +45,7 @@ import { formatBytes, getBytesUnit } from './utils/format'; import redisCache from './api/redis-cache'; import accelerationApi from './api/services/acceleration'; import bitcoinCoreRoutes from './api/bitcoin/bitcoin-core.routes'; +import bitcoinSecondClient from './api/bitcoin/bitcoin-second-client'; class Server { private wss: WebSocket.Server | undefined; @@ -215,11 +216,13 @@ class Server { } } const newMempool = await bitcoinApi.$getRawMempool(); + const minFeeMempool = memPool.limitGBT ? await bitcoinSecondClient.getRawMemPool() : null; + const minFeeTip = memPool.limitGBT ? await bitcoinSecondClient.getBlockCount() : -1; const newAccelerations = await accelerationApi.$fetchAccelerations(); const numHandledBlocks = await blocks.$updateBlocks(); const pollRate = config.MEMPOOL.POLL_RATE_MS * (indexer.indexerIsRunning() ? 10 : 1); if (numHandledBlocks === 0) { - await memPool.$updateMempool(newMempool, newAccelerations, pollRate); + await memPool.$updateMempool(newMempool, newAccelerations, minFeeMempool, minFeeTip, pollRate); } indexer.$run(); if (config.FIAT_PRICE.ENABLED) { diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index d04858607..a12351fe3 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -107,6 +107,7 @@ export interface MempoolTransactionExtended extends TransactionExtended { inputs?: number[]; lastBoosted?: number; cpfpDirty?: boolean; + cpfpUpdated?: number; } export interface AuditTransaction { @@ -143,6 +144,12 @@ export interface CompactThreadTransaction { dirty?: boolean; } +export interface GbtCandidates { + txs: { [txid: string ]: boolean }, + added: MempoolTransactionExtended[]; + removed: MempoolTransactionExtended[]; +} + export interface ThreadTransaction { txid: string; fee: number; @@ -181,6 +188,9 @@ export interface CpfpInfo { bestDescendant?: BestDescendant | null; descendants?: Ancestor[]; effectiveFeePerVsize?: number; + sigops?: number; + adjustedVsize?: number, + acceleration?: boolean, } export interface TransactionStripped { diff --git a/docker/README.md b/docker/README.md index 444324af8..c8ac91d38 100644 --- a/docker/README.md +++ b/docker/README.md @@ -109,8 +109,6 @@ Below we list all settings from `mempool-config.json` and the corresponding over "AUTOMATIC_BLOCK_REINDEXING": false, "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json", "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master", - "ADVANCED_GBT_AUDIT": false, - "ADVANCED_GBT_MEMPOOL": false, "CPFP_INDEXING": false, "MAX_BLOCKS_BULK_QUERY": 0, "DISK_CACHE_BLOCK_INTERVAL": 6, @@ -142,8 +140,6 @@ Corresponding `docker-compose.yml` overrides: MEMPOOL_AUTOMATIC_BLOCK_REINDEXING: "" MEMPOOL_POOLS_JSON_URL: "" MEMPOOL_POOLS_JSON_TREE_URL: "" - MEMPOOL_ADVANCED_GBT_AUDIT: "" - MEMPOOL_ADVANCED_GBT_MEMPOOL: "" MEMPOOL_CPFP_INDEXING: "" MEMPOOL_MAX_BLOCKS_BULK_QUERY: "" MEMPOOL_DISK_CACHE_BLOCK_INTERVAL: "" @@ -151,8 +147,6 @@ Corresponding `docker-compose.yml` overrides: ... ``` -`ADVANCED_GBT_AUDIT` AND `ADVANCED_GBT_MEMPOOL` enable a more accurate (but slower) block prediction algorithm for the block audit feature and the projected mempool-blocks respectively. - `CPFP_INDEXING` enables indexing CPFP (Child Pays For Parent) information for the last `INDEXING_BLOCKS_AMOUNT` blocks.
diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index f8935706c..0d6017788 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -26,9 +26,8 @@ "GOGGLES_INDEXING": __MEMPOOL_GOGGLES_INDEXING__, "AUTOMATIC_BLOCK_REINDEXING": __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__, "AUDIT": __MEMPOOL_AUDIT__, - "ADVANCED_GBT_AUDIT": __MEMPOOL_ADVANCED_GBT_AUDIT__, - "ADVANCED_GBT_MEMPOOL": __MEMPOOL_ADVANCED_GBT_MEMPOOL__, "RUST_GBT": __MEMPOOL_RUST_GBT__, + "LIMIT_GBT": __MEMPOOL_LIMIT_GBT__, "CPFP_INDEXING": __MEMPOOL_CPFP_INDEXING__, "MAX_BLOCKS_BULK_QUERY": __MEMPOOL_MAX_BLOCKS_BULK_QUERY__, "DISK_CACHE_BLOCK_INTERVAL": __MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__, diff --git a/docker/backend/start.sh b/docker/backend/start.sh index 50ecc17ab..aecdac8f1 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -29,9 +29,8 @@ __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__=${MEMPOOL_AUTOMATIC_BLOCK_REINDEXING:=fal __MEMPOOL_POOLS_JSON_URL__=${MEMPOOL_POOLS_JSON_URL:=https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json} __MEMPOOL_POOLS_JSON_TREE_URL__=${MEMPOOL_POOLS_JSON_TREE_URL:=https://api.github.com/repos/mempool/mining-pools/git/trees/master} __MEMPOOL_AUDIT__=${MEMPOOL_AUDIT:=false} -__MEMPOOL_ADVANCED_GBT_AUDIT__=${MEMPOOL_ADVANCED_GBT_AUDIT:=false} -__MEMPOOL_ADVANCED_GBT_MEMPOOL__=${MEMPOOL_ADVANCED_GBT_MEMPOOL:=false} __MEMPOOL_RUST_GBT__=${MEMPOOL_RUST_GBT:=false} +__MEMPOOL_LIMIT_GBT__=${MEMPOOL_LIMIT_GBT:=false} __MEMPOOL_CPFP_INDEXING__=${MEMPOOL_CPFP_INDEXING:=false} __MEMPOOL_MAX_BLOCKS_BULK_QUERY__=${MEMPOOL_MAX_BLOCKS_BULK_QUERY:=0} __MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__=${MEMPOOL_DISK_CACHE_BLOCK_INTERVAL:=6} @@ -189,9 +188,8 @@ sed -i "s!__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__!${__MEMPOOL_AUTOMATIC_BLOCK_REI sed -i "s!__MEMPOOL_POOLS_JSON_URL__!${__MEMPOOL_POOLS_JSON_URL__}!g" mempool-config.json sed -i "s!__MEMPOOL_POOLS_JSON_TREE_URL__!${__MEMPOOL_POOLS_JSON_TREE_URL__}!g" mempool-config.json sed -i "s!__MEMPOOL_AUDIT__!${__MEMPOOL_AUDIT__}!g" mempool-config.json -sed -i "s!__MEMPOOL_ADVANCED_GBT_MEMPOOL__!${__MEMPOOL_ADVANCED_GBT_MEMPOOL__}!g" mempool-config.json sed -i "s!__MEMPOOL_RUST_GBT__!${__MEMPOOL_RUST_GBT__}!g" mempool-config.json -sed -i "s!__MEMPOOL_ADVANCED_GBT_AUDIT__!${__MEMPOOL_ADVANCED_GBT_AUDIT__}!g" mempool-config.json +sed -i "s!__MEMPOOL_LIMIT_GBT__!${__MEMPOOL_LIMIT_GBT__}!g" mempool-config.json sed -i "s!__MEMPOOL_CPFP_INDEXING__!${__MEMPOOL_CPFP_INDEXING__}!g" mempool-config.json sed -i "s!__MEMPOOL_MAX_BLOCKS_BULK_QUERY__!${__MEMPOOL_MAX_BLOCKS_BULK_QUERY__}!g" mempool-config.json sed -i "s!__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__!${__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__}!g" mempool-config.json diff --git a/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts b/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts index ab6985a25..a5f6403ad 100644 --- a/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts +++ b/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts @@ -206,6 +206,10 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On update(add: TransactionStripped[], remove: string[], change: { txid: string, rate: number | undefined, acc: boolean | undefined }[], direction: string = 'left', resetLayout: boolean = false): void { if (this.scene) { + add = add.filter(tx => !this.scene.txs[tx.txid]); + remove = remove.filter(txid => this.scene.txs[txid]); + change = change.filter(tx => this.scene.txs[tx.txid]); + this.scene.update(add, remove, change, direction, resetLayout); this.start(); this.updateSearchHighlight(); diff --git a/production/mempool-config.mainnet.json b/production/mempool-config.mainnet.json index 5038d9bfb..385f8cbdc 100644 --- a/production/mempool-config.mainnet.json +++ b/production/mempool-config.mainnet.json @@ -15,8 +15,6 @@ "GOGGLES_INDEXING": true, "AUDIT": true, "CPFP_INDEXING": true, - "ADVANCED_GBT_AUDIT": true, - "ADVANCED_GBT_MEMPOOL": true, "RUST_GBT": true, "USE_SECOND_NODE_FOR_MINFEE": true, "DISK_CACHE_BLOCK_INTERVAL": 1, diff --git a/production/mempool-config.signet.json b/production/mempool-config.signet.json index 0a711d16f..6ebd9e8b3 100644 --- a/production/mempool-config.signet.json +++ b/production/mempool-config.signet.json @@ -9,8 +9,6 @@ "API_URL_PREFIX": "/api/v1/", "INDEXING_BLOCKS_AMOUNT": -1, "AUDIT": true, - "ADVANCED_GBT_AUDIT": true, - "ADVANCED_GBT_MEMPOOL": true, "RUST_GBT": true, "POLL_RATE_MS": 1000, "DISK_CACHE_BLOCK_INTERVAL": 1, diff --git a/production/mempool-config.testnet.json b/production/mempool-config.testnet.json index adc93c0e9..2394ac467 100644 --- a/production/mempool-config.testnet.json +++ b/production/mempool-config.testnet.json @@ -9,8 +9,6 @@ "API_URL_PREFIX": "/api/v1/", "INDEXING_BLOCKS_AMOUNT": -1, "AUDIT": true, - "ADVANCED_GBT_AUDIT": true, - "ADVANCED_GBT_MEMPOOL": true, "RUST_GBT": true, "POLL_RATE_MS": 1000, "DISK_CACHE_BLOCK_INTERVAL": 1,