From 083bfdba06875fd8d254b16c6db9a6070b6b2d0d Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sat, 3 Jun 2023 16:54:12 -0400 Subject: [PATCH] Refactor accelerated audits --- backend/src/api/audit.ts | 4 ++-- backend/src/api/mempool-blocks.ts | 28 +++++++++++++----------- backend/src/api/mempool.ts | 2 +- backend/src/api/services/acceleration.ts | 20 ++++++++++++++++- backend/src/api/websocket-handler.ts | 17 ++++++++++---- 5 files changed, 50 insertions(+), 21 deletions(-) diff --git a/backend/src/api/audit.ts b/backend/src/api/audit.ts index 9710d0362..e78b2796f 100644 --- a/backend/src/api/audit.ts +++ b/backend/src/api/audit.ts @@ -6,7 +6,7 @@ import rbfCache from './rbf-cache'; const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners class Audit { - auditBlock(transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended }) + auditBlock(transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended }, useAccelerations: boolean = false) : { censored: string[], added: string[], fresh: string[], sigop: string[], fullrbf: string[], accelerated: string[], score: number, similarity: number } { if (!projectedBlocks?.[0]?.transactionIds || !mempool) { return { censored: [], added: [], fresh: [], sigop: [], fullrbf: [], accelerated: [], score: 0, similarity: 1 }; @@ -29,7 +29,7 @@ class Audit { const now = Math.round((Date.now() / 1000)); for (const tx of transactions) { inBlock[tx.txid] = tx; - if (tx.acceleration) { + if (mempool[tx.txid] && mempool[tx.txid].acceleration) { accelerated.push(tx.txid); } } diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index bc0def7a8..0ce0ea522 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -207,7 +207,7 @@ class MempoolBlocks { return mempoolBlockDeltas; } - public async $makeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false): Promise { + public async $makeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, useAccelerations: boolean = false): Promise { const start = Date.now(); // reset mempool short ids @@ -216,7 +216,7 @@ class MempoolBlocks { this.setUid(tx, true); } - const accelerations = mempool.getAccelerations(); + const accelerations = useAccelerations ? mempool.getAccelerations() : {}; // prepare a stripped down version of the mempool with only the minimum necessary data // to reduce the overhead of passing this data to the worker thread @@ -225,7 +225,7 @@ class MempoolBlocks { if (entry.uid !== null && entry.uid !== undefined) { const stripped = { uid: entry.uid, - fee: entry.fee + (accelerations[entry.txid] || 0), + fee: entry.fee + (useAccelerations ? (accelerations[entry.txid] || 0) : 0), weight: (entry.adjustedVsize * 4), sigops: entry.sigops, feePerVsize: entry.adjustedFeePerVsize || entry.feePerVsize, @@ -265,7 +265,7 @@ class MempoolBlocks { // clean up thread error listener this.txSelectionWorker?.removeListener('error', threadErrorListener); - const processed = this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), saveResults); + const processed = this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), accelerations, saveResults); logger.debug(`makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); @@ -276,17 +276,17 @@ class MempoolBlocks { return this.mempoolBlocks; } - public async $updateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], accelerationDelta: string[] = [], saveResults: boolean = false): Promise { + public async $updateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], accelerationDelta: string[] = [], saveResults: boolean = false, useAccelerations: boolean = false): Promise { if (!this.txSelectionWorker) { // need to reset the worker - await this.$makeBlockTemplates(newMempool, saveResults); + await this.$makeBlockTemplates(newMempool, saveResults, useAccelerations); return; } const start = Date.now(); - const accelerations = mempool.getAccelerations(); - const addedAndChanged: MempoolTransactionExtended[] = accelerationDelta.map(txid => newMempool[txid]).filter(tx => tx != null).concat(added); + const accelerations = useAccelerations ? mempool.getAccelerations() : {}; + const addedAndChanged: MempoolTransactionExtended[] = useAccelerations ? accelerationDelta.map(txid => newMempool[txid]).filter(tx => tx != null).concat(added) : added; for (const tx of addedAndChanged) { this.setUid(tx, true); @@ -298,7 +298,7 @@ class MempoolBlocks { const addedStripped: CompactThreadTransaction[] = addedAndChanged.filter(entry => entry.uid != null).map(entry => { return { uid: entry.uid || 0, - fee: entry.fee + (accelerations[entry.txid] || 0), + fee: entry.fee + (useAccelerations ? (accelerations[entry.txid] || 0) : 0), weight: (entry.adjustedVsize * 4), sigops: entry.sigops, feePerVsize: entry.adjustedFeePerVsize || entry.feePerVsize, @@ -325,7 +325,7 @@ class MempoolBlocks { // clean up thread error listener this.txSelectionWorker?.removeListener('error', threadErrorListener); - this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), saveResults); + this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), accelerations, saveResults); logger.debug(`updateBlockTemplates completed in ${(Date.now() - start) / 1000} seconds`); } catch (e) { logger.err('updateBlockTemplates failed. ' + (e instanceof Error ? e.message : e)); @@ -362,7 +362,7 @@ class MempoolBlocks { if (saveResults) { this.rustInitialized = true; } - const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, saveResults); + const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, {}, saveResults); logger.debug(`RUST makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); return processed; } catch (e) { @@ -414,7 +414,7 @@ class MempoolBlocks { if (mempoolSize !== resultMempoolSize) { throw new Error('GBT returned wrong number of transactions, cache is probably out of sync'); } else { - this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, true); + this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, {}, true); } this.removeUids(removedUids); logger.debug(`RUST updateBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); @@ -424,7 +424,7 @@ class MempoolBlocks { } } - private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], saveResults): MempoolBlockWithTransactions[] { + private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], accelerations, saveResults): MempoolBlockWithTransactions[] { for (const [txid, rate] of rates) { if (txid in mempool) { mempool[txid].effectiveFeePerVsize = rate; @@ -503,6 +503,8 @@ class MempoolBlocks { mempoolTx.cpfpChecked = true; } + mempoolTx.acceleration = accelerations[txid]; + // online calculation of stack-of-blocks fee stats if (hasBlockStack && blockIndex === lastBlockIndex && feeStatsCalculator) { feeStatsCalculator.processNext(mempoolTx); diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index b114c89fc..4efe17731 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -342,7 +342,7 @@ class Mempool { } try { - const newAccelerations = await accelerationApi.fetchAccelerations$(); + const newAccelerations = await accelerationApi.$fetchAccelerations(); const changed: string[] = []; diff --git a/backend/src/api/services/acceleration.ts b/backend/src/api/services/acceleration.ts index efbf32f5d..8c849cdd8 100644 --- a/backend/src/api/services/acceleration.ts +++ b/backend/src/api/services/acceleration.ts @@ -1,5 +1,6 @@ import { query } from '../../utils/axios-query'; import config from '../../config'; +import { BlockExtended, PoolTag } from '../../mempool.interfaces'; export interface Acceleration { txid: string, @@ -7,7 +8,7 @@ export interface Acceleration { } class AccelerationApi { - public async fetchAccelerations$(): Promise { + public async $fetchAccelerations(): Promise { if (config.MEMPOOL_SERVICES.ACCELERATIONS) { const response = await query(`${config.MEMPOOL_SERVICES.API}/accelerations`); return (response as Acceleration[]) || []; @@ -15,6 +16,23 @@ class AccelerationApi { return []; } } + + public async $fetchPools(): Promise { + if (config.MEMPOOL_SERVICES.ACCELERATIONS) { + const response = await query(`${config.MEMPOOL_SERVICES.API}/partners`); + return (response as PoolTag[]) || []; + } else { + return []; + } + } + + public async $isAcceleratedBlock(block: BlockExtended): Promise { + const pools = await this.$fetchPools(); + if (block?.extras?.pool?.id == null) { + return false; + } + return pools.reduce((match, tag) => match || tag.uniqueId === block.extras.pool.id, false); + } } export default new AccelerationApi(); \ No newline at end of file diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 3a90b29f4..51fccbcbb 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -21,7 +21,7 @@ import Audit from './audit'; import { deepClone } from '../utils/clone'; import priceUpdater from '../tasks/price-updater'; import { ApiPrice } from '../repositories/PricesRepository'; -import mempool from './mempool'; +import accelerationApi from './services/acceleration'; // valid 'want' subscriptions const wantable = [ @@ -392,7 +392,7 @@ class WebsocketHandler { if (config.MEMPOOL.RUST_GBT) { await mempoolBlocks.$rustUpdateBlockTemplates(newMempool, mempoolSize, newTransactions, deletedTransactions); } else { - await mempoolBlocks.$updateBlockTemplates(newMempool, newTransactions, deletedTransactions, accelerationDelta, true); + await mempoolBlocks.$updateBlockTemplates(newMempool, newTransactions, deletedTransactions, accelerationDelta, true, config.MEMPOOL_SERVICES.ACCELERATIONS); } } else { mempoolBlocks.updateMempoolBlocks(newMempool, true); @@ -648,6 +648,7 @@ class WebsocketHandler { if (config.MEMPOOL.AUDIT && memPool.isInSync()) { let projectedBlocks; let auditMempool = _memPool; + const isAccelerated = config.MEMPOOL_SERVICES.ACCELERATIONS && await accelerationApi.$isAcceleratedBlock(block); // template calculation functions have mempool side effects, so calculate audits using // a cloned copy of the mempool if we're running a different algorithm for mempool updates const separateAudit = config.MEMPOOL.ADVANCED_GBT_AUDIT !== config.MEMPOOL.ADVANCED_GBT_MEMPOOL; @@ -657,13 +658,17 @@ class WebsocketHandler { if (config.MEMPOOL.RUST_GBT) { projectedBlocks = await mempoolBlocks.$oneOffRustBlockTemplates(auditMempool); } else { - projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false); + projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, isAccelerated); } } else { projectedBlocks = mempoolBlocks.updateMempoolBlocks(auditMempool, false); } } else { - projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions(); + if ((config.MEMPOOL_SERVICES.ACCELERATIONS && !isAccelerated)) { + projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, isAccelerated); + } else { + projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions(); + } } if (Common.indexingEnabled()) { @@ -723,11 +728,15 @@ class WebsocketHandler { } if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) { +<<<<<<< HEAD if (config.MEMPOOL.RUST_GBT) { await mempoolBlocks.$rustUpdateBlockTemplates(_memPool, Object.keys(_memPool).length, [], transactions); } else { await mempoolBlocks.$makeBlockTemplates(_memPool, true); } +======= + await mempoolBlocks.$makeBlockTemplates(_memPool, true, config.MEMPOOL_SERVICES.ACCELERATIONS); +>>>>>>> 77b0a8ecc (Refactor accelerated audits) } else { mempoolBlocks.updateMempoolBlocks(_memPool, true); }