From 1fd85b729d8928b84b9624b1c811372b426515ac Mon Sep 17 00:00:00 2001 From: Mononaut Date: Fri, 24 Mar 2023 09:47:08 +0900 Subject: [PATCH 1/4] handle stale transactions in block templates --- backend/src/api/disk-cache.ts | 4 +++- backend/src/api/mempool-blocks.ts | 15 +++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/backend/src/api/disk-cache.ts b/backend/src/api/disk-cache.ts index 7180c6f51..0ec502c08 100644 --- a/backend/src/api/disk-cache.ts +++ b/backend/src/api/disk-cache.ts @@ -43,7 +43,9 @@ class DiskCache { const mempool = memPool.getMempool(); const mempoolArray: TransactionExtended[] = []; for (const tx in mempool) { - mempoolArray.push(mempool[tx]); + if (mempool[tx] && !mempool[tx].deleteAfter) { + mempoolArray.push(mempool[tx]); + } } Common.shuffleArray(mempoolArray); diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 3c2feb0e2..ff502be24 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -151,7 +151,7 @@ class MempoolBlocks { // prepare a stripped down version of the mempool with only the minimum necessary data // to reduce the overhead of passing this data to the worker thread const strippedMempool: { [txid: string]: ThreadTransaction } = {}; - Object.values(newMempool).forEach(entry => { + Object.values(newMempool).filter(tx => !tx.deleteAfter).forEach(entry => { strippedMempool[entry.txid] = { txid: entry.txid, fee: entry.fee, @@ -186,7 +186,9 @@ class MempoolBlocks { this.txSelectionWorker?.once('error', reject); }); this.txSelectionWorker.postMessage({ type: 'set', mempool: strippedMempool }); - const { blocks, clusters } = await workerResultPromise; + let { blocks, clusters } = await workerResultPromise; + // filter out stale transactions + blocks = blocks.map(block => block.filter(tx => (tx.txid && tx.txid in newMempool))); // clean up thread error listener this.txSelectionWorker?.removeListener('error', threadErrorListener); @@ -228,7 +230,9 @@ class MempoolBlocks { this.txSelectionWorker?.once('error', reject); }); this.txSelectionWorker.postMessage({ type: 'update', added: addedStripped, removed }); - const { blocks, clusters } = await workerResultPromise; + let { blocks, clusters } = await workerResultPromise; + // filter out stale transactions + blocks = blocks.map(block => block.filter(tx => (tx.txid && tx.txid in newMempool))); // clean up thread error listener this.txSelectionWorker?.removeListener('error', threadErrorListener); @@ -243,7 +247,7 @@ class MempoolBlocks { // update this thread's mempool with the results blocks.forEach(block => { block.forEach(tx => { - if (tx.txid in mempool) { + if (tx.txid && tx.txid in mempool) { if (tx.effectiveFeePerVsize != null) { mempool[tx.txid].effectiveFeePerVsize = tx.effectiveFeePerVsize; } @@ -253,6 +257,9 @@ class MempoolBlocks { const cluster = clusters[tx.cpfpRoot]; let matched = false; cluster.forEach(txid => { + if (!txid || !mempool[txid]) { + return; + } if (txid === tx.txid) { matched = true; } else { From 28de93d0ff4e6c932b805f412e787515bbf45ce2 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Fri, 24 Mar 2023 09:48:08 +0900 Subject: [PATCH 2/4] move lazy tx deletion into main loop --- backend/src/api/mempool.ts | 3 +-- backend/src/index.ts | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index db5de82b2..1be1faceb 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -38,7 +38,6 @@ class Mempool { constructor() { setInterval(this.updateTxPerSecond.bind(this), 1000); - setInterval(this.deleteExpiredTransactions.bind(this), 20000); } /** @@ -256,7 +255,7 @@ class Mempool { } } - private deleteExpiredTransactions() { + public deleteExpiredTransactions() { const now = new Date().getTime(); for (const tx in this.mempoolCache) { const lazyDeleteAt = this.mempoolCache[tx].deleteAfter; diff --git a/backend/src/index.ts b/backend/src/index.ts index ccba83be9..a34ffd21b 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -178,6 +178,7 @@ class Server { logger.debug(msg); } } + memPool.deleteExpiredTransactions(); await blocks.$updateBlocks(); await memPool.$updateMempool(); indexer.$run(); From 61f24562fdeea946b59684274c169da34796f324 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Fri, 24 Mar 2023 09:49:02 +0900 Subject: [PATCH 3/4] tighten sanity checks in block audit --- backend/src/api/audit.ts | 68 ++++++++++++++++++++++------------------ 1 file changed, 38 insertions(+), 30 deletions(-) diff --git a/backend/src/api/audit.ts b/backend/src/api/audit.ts index d0d677740..639066af0 100644 --- a/backend/src/api/audit.ts +++ b/backend/src/api/audit.ts @@ -39,17 +39,19 @@ class Audit { } else { isCensored[txid] = true; } - displacedWeight += mempool[txid].weight; + displacedWeight += mempool[txid]?.weight || 0; } else { - matchedWeight += mempool[txid].weight; + matchedWeight += mempool[txid]?.weight || 0; } - projectedWeight += mempool[txid].weight; + projectedWeight += mempool[txid]?.weight || 0; inTemplate[txid] = true; } - displacedWeight += (4000 - transactions[0].weight); - projectedWeight += transactions[0].weight; - matchedWeight += transactions[0].weight; + if (transactions[0]) { + displacedWeight += (4000 - transactions[0].weight); + projectedWeight += transactions[0].weight; + matchedWeight += transactions[0].weight; + } // we can expect an honest miner to include 'displaced' transactions in place of recent arrivals and censored txs // these displaced transactions should occupy the first N weight units of the next projected block @@ -59,19 +61,22 @@ class Audit { let failures = 0; while (projectedBlocks[1] && index < projectedBlocks[1].transactionIds.length && failures < 500) { const txid = projectedBlocks[1].transactionIds[index]; - const fits = (mempool[txid].weight - displacedWeightRemaining) < 4000; - const feeMatches = mempool[txid].effectiveFeePerVsize >= lastFeeRate; - if (fits || feeMatches) { - isDisplaced[txid] = true; - if (fits) { - lastFeeRate = Math.min(lastFeeRate, mempool[txid].effectiveFeePerVsize); + const tx = mempool[txid]; + if (tx) { + const fits = (tx.weight - displacedWeightRemaining) < 4000; + const feeMatches = tx.effectiveFeePerVsize >= lastFeeRate; + if (fits || feeMatches) { + isDisplaced[txid] = true; + if (fits) { + lastFeeRate = Math.min(lastFeeRate, tx.effectiveFeePerVsize); + } + if (tx.firstSeen == null || (now - (tx?.firstSeen || 0)) > PROPAGATION_MARGIN) { + displacedWeightRemaining -= tx.weight; + } + failures = 0; + } else { + failures++; } - if (mempool[txid].firstSeen == null || (now - (mempool[txid]?.firstSeen || 0)) > PROPAGATION_MARGIN) { - displacedWeightRemaining -= mempool[txid].weight; - } - failures = 0; - } else { - failures++; } index++; } @@ -108,20 +113,23 @@ class Audit { index = projectedBlocks[0].transactionIds.length - 1; while (index >= 0) { const txid = projectedBlocks[0].transactionIds[index]; - if (overflowWeightRemaining > 0) { - if (isCensored[txid]) { - delete isCensored[txid]; - } - if (mempool[txid].effectiveFeePerVsize > maxOverflowRate) { - maxOverflowRate = mempool[txid].effectiveFeePerVsize; - rateThreshold = (Math.ceil(maxOverflowRate * 100) / 100) + 0.005; - } - } else if (mempool[txid].effectiveFeePerVsize <= rateThreshold) { // tolerance of 0.01 sat/vb + rounding - if (isCensored[txid]) { - delete isCensored[txid]; + const tx = mempool[txid]; + if (tx) { + if (overflowWeightRemaining > 0) { + if (isCensored[txid]) { + delete isCensored[txid]; + } + if (tx.effectiveFeePerVsize > maxOverflowRate) { + maxOverflowRate = tx.effectiveFeePerVsize; + rateThreshold = (Math.ceil(maxOverflowRate * 100) / 100) + 0.005; + } + } else if (tx.effectiveFeePerVsize <= rateThreshold) { // tolerance of 0.01 sat/vb + rounding + if (isCensored[txid]) { + delete isCensored[txid]; + } } + overflowWeightRemaining -= (mempool[txid]?.weight || 0); } - overflowWeightRemaining -= (mempool[txid]?.weight || 0); index--; } From 8486c1117d5dd178289602792870abf1025bb21b Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sun, 26 Mar 2023 05:41:31 +0900 Subject: [PATCH 4/4] log warnings for unexpectedly missing txs --- backend/src/api/audit.ts | 5 +++++ backend/src/api/mempool-blocks.ts | 13 +++++++++++++ 2 files changed, 18 insertions(+) diff --git a/backend/src/api/audit.ts b/backend/src/api/audit.ts index 639066af0..6e1cb3787 100644 --- a/backend/src/api/audit.ts +++ b/backend/src/api/audit.ts @@ -1,4 +1,5 @@ import config from '../config'; +import logger from '../logger'; import { TransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces'; const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners @@ -77,6 +78,8 @@ class Audit { } else { failures++; } + } else { + logger.warn('projected transaction missing from mempool cache'); } index++; } @@ -129,6 +132,8 @@ class Audit { } } overflowWeightRemaining -= (mempool[txid]?.weight || 0); + } else { + logger.warn('projected transaction missing from mempool cache'); } index--; } diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index ff502be24..0d6186ce1 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -188,7 +188,12 @@ class MempoolBlocks { this.txSelectionWorker.postMessage({ type: 'set', mempool: strippedMempool }); let { blocks, clusters } = await workerResultPromise; // filter out stale transactions + const unfilteredCount = blocks.reduce((total, block) => { return total + block.length; }, 0); blocks = blocks.map(block => block.filter(tx => (tx.txid && tx.txid in newMempool))); + const filteredCount = blocks.reduce((total, block) => { return total + block.length; }, 0); + if (filteredCount < unfilteredCount) { + logger.warn(`tx selection worker thread returned ${unfilteredCount - filteredCount} stale transactions from makeBlockTemplates`); + } // clean up thread error listener this.txSelectionWorker?.removeListener('error', threadErrorListener); @@ -232,7 +237,12 @@ class MempoolBlocks { this.txSelectionWorker.postMessage({ type: 'update', added: addedStripped, removed }); let { blocks, clusters } = await workerResultPromise; // filter out stale transactions + const unfilteredCount = blocks.reduce((total, block) => { return total + block.length; }, 0); blocks = blocks.map(block => block.filter(tx => (tx.txid && tx.txid in newMempool))); + const filteredCount = blocks.reduce((total, block) => { return total + block.length; }, 0); + if (filteredCount < unfilteredCount) { + logger.warn(`tx selection worker thread returned ${unfilteredCount - filteredCount} stale transactions from updateBlockTemplates`); + } // clean up thread error listener this.txSelectionWorker?.removeListener('error', threadErrorListener); @@ -258,6 +268,7 @@ class MempoolBlocks { let matched = false; cluster.forEach(txid => { if (!txid || !mempool[txid]) { + logger.warn('projected transaction ancestor missing from mempool cache'); return; } if (txid === tx.txid) { @@ -280,6 +291,8 @@ class MempoolBlocks { mempool[tx.txid].bestDescendant = null; } mempool[tx.txid].cpfpChecked = tx.cpfpChecked; + } else { + logger.warn('projected transaction missing from mempool cache'); } }); });