From e489f713eb46f8ece0d4190f1b49d72920d8ffdd Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 22 May 2023 18:16:58 -0400 Subject: [PATCH] include accelerated tx data in block audits --- backend/src/api/audit.ts | 11 ++++++++--- backend/src/api/database-migration.ts | 7 ++++++- backend/src/api/mempool-blocks.ts | 2 +- backend/src/api/websocket-handler.ts | 3 ++- backend/src/mempool.interfaces.ts | 1 + backend/src/repositories/BlocksAuditsRepository.ts | 9 ++++++--- .../app/components/block-overview-graph/tx-view.ts | 5 ++++- .../block-overview-tooltip.component.html | 1 + frontend/src/app/components/block/block.component.ts | 10 ++++++++++ frontend/src/app/interfaces/node-api.interface.ts | 3 ++- frontend/src/app/interfaces/websocket.interface.ts | 2 +- 11 files changed, 42 insertions(+), 12 deletions(-) diff --git a/backend/src/api/audit.ts b/backend/src/api/audit.ts index a909fc2b6..63da288a1 100644 --- a/backend/src/api/audit.ts +++ b/backend/src/api/audit.ts @@ -6,16 +6,17 @@ import rbfCache from './rbf-cache'; const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners class Audit { - auditBlock(transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended }) - : { censored: string[], added: string[], fresh: string[], sigop: string[], fullrbf: string[], score: number, similarity: number } { + auditBlock(transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended }, accelerations: { [txid: string]: number }) + : { censored: string[], added: string[], fresh: string[], sigop: string[], fullrbf: string[], accelerated: string[], score: number, similarity: number } { if (!projectedBlocks?.[0]?.transactionIds || !mempool) { - return { censored: [], added: [], fresh: [], sigop: [], fullrbf: [], score: 0, similarity: 1 }; + return { censored: [], added: [], fresh: [], sigop: [], fullrbf: [], accelerated: [], score: 0, similarity: 1 }; } const matches: string[] = []; // present in both mined block and template const added: string[] = []; // present in mined block, not in template const fresh: string[] = []; // missing, but firstSeen or lastBoosted within PROPAGATION_MARGIN const rbf: string[] = []; // either missing or present, and either part of a full-rbf replacement, or a conflict with the mined block + const accelerated: string[] = []; // prioritized by the mempool accelerator const isCensored = {}; // missing, without excuse const isDisplaced = {}; let displacedWeight = 0; @@ -28,6 +29,9 @@ class Audit { const now = Math.round((Date.now() / 1000)); for (const tx of transactions) { inBlock[tx.txid] = tx; + if (accelerations[tx.txid]) { + accelerated.push(tx.txid); + } } // coinbase is always expected if (transactions[0]) { @@ -149,6 +153,7 @@ class Audit { fresh, sigop: [], fullrbf: rbf, + accelerated, score, similarity, }; diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 7c7608aff..b7dc39493 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository'; import { RowDataPacket } from 'mysql2'; class DatabaseMigration { - private static currentVersion = 64; + private static currentVersion = 65; private queryTimeout = 3600_000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -548,6 +548,11 @@ class DatabaseMigration { await this.$executeQuery('ALTER TABLE `nodes` ADD features text NULL'); await this.updateToSchemaVersion(64); } + + if (databaseSchemaVersion < 65 && isBitcoin === true) { + await this.$executeQuery('ALTER TABLE `blocks_audits` ADD accelerated_txs JSON DEFAULT "[]"'); + await this.updateToSchemaVersion(65); + } } /** diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 08508310d..5ca5cff09 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -206,7 +206,7 @@ class MempoolBlocks { return mempoolBlockDeltas; } - public async $makeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false): Promise { + public async $makeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, accelerations: { [txid: string]: number } = {}): Promise { const start = Date.now(); // reset mempool short ids diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 56c8513cd..91004e292 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -666,7 +666,7 @@ class WebsocketHandler { } if (Common.indexingEnabled()) { - const { censored, added, fresh, sigop, fullrbf, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool); + const { censored, added, fresh, sigop, fullrbf, accelerated, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool, accelerations); const matchRate = Math.round(score * 100 * 100) / 100; const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions : []; @@ -695,6 +695,7 @@ class WebsocketHandler { freshTxs: fresh, sigopTxs: sigop, fullrbfTxs: fullrbf, + acceleratedTxs: accelerated, matchRate: matchRate, expectedFees: totalFees, expectedWeight: totalWeight, diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index 25e7f0387..d0549cd97 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -36,6 +36,7 @@ export interface BlockAudit { sigopTxs: string[], fullrbfTxs: string[], addedTxs: string[], + acceleratedTxs: string[], matchRate: number, expectedFees?: number, expectedWeight?: number, diff --git a/backend/src/repositories/BlocksAuditsRepository.ts b/backend/src/repositories/BlocksAuditsRepository.ts index f7a2a59b5..9c7568567 100644 --- a/backend/src/repositories/BlocksAuditsRepository.ts +++ b/backend/src/repositories/BlocksAuditsRepository.ts @@ -6,9 +6,9 @@ import { BlockAudit, AuditScore } from '../mempool.interfaces'; class BlocksAuditRepositories { public async $saveAudit(audit: BlockAudit): Promise { try { - await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, fresh_txs, sigop_txs, fullrbf_txs, match_rate, expected_fees, expected_weight) - VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs), - JSON.stringify(audit.addedTxs), JSON.stringify(audit.freshTxs), JSON.stringify(audit.sigopTxs), JSON.stringify(audit.fullrbfTxs), audit.matchRate, audit.expectedFees, audit.expectedWeight]); + await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, fresh_txs, sigop_txs, fullrbf_txs, accelerated_txs, match_rate, expected_fees, expected_weight) + VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs), + JSON.stringify(audit.addedTxs), JSON.stringify(audit.freshTxs), JSON.stringify(audit.sigopTxs), JSON.stringify(audit.fullrbfTxs), JSON.stringify(audit.acceleratedTxs), audit.matchRate, audit.expectedFees, audit.expectedWeight]); } catch (e: any) { if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart logger.debug(`Cannot save block audit for block ${audit.hash} because it has already been indexed, ignoring`); @@ -69,6 +69,7 @@ class BlocksAuditRepositories { fresh_txs as freshTxs, sigop_txs as sigopTxs, fullrbf_txs as fullrbfTxs, + accelerated_txs as acceleratedTxs, match_rate as matchRate, expected_fees as expectedFees, expected_weight as expectedWeight @@ -83,6 +84,8 @@ class BlocksAuditRepositories { rows[0].freshTxs = JSON.parse(rows[0].freshTxs); rows[0].sigopTxs = JSON.parse(rows[0].sigopTxs); rows[0].fullrbfTxs = JSON.parse(rows[0].fullrbfTxs); + rows[0].acceleratedTxs = JSON.parse(rows[0].acceleratedTxs); + rows[0].transactions = JSON.parse(rows[0].transactions); rows[0].template = JSON.parse(rows[0].template); return rows[0]; diff --git a/frontend/src/app/components/block-overview-graph/tx-view.ts b/frontend/src/app/components/block-overview-graph/tx-view.ts index 1b8c88704..ac93eee0c 100644 --- a/frontend/src/app/components/block-overview-graph/tx-view.ts +++ b/frontend/src/app/components/block-overview-graph/tx-view.ts @@ -17,6 +17,7 @@ const auditColors = { missing: darken(desaturate(hexToColor('f344df'), 0.3), 0.7), added: hexToColor('0099ff'), selected: darken(desaturate(hexToColor('0099ff'), 0.3), 0.7), + accelerated: hexToColor('8F5FF6'), }; // convert from this class's update format to TxSprite's update format @@ -38,7 +39,7 @@ export default class TxView implements TransactionStripped { value: number; feerate: number; rate?: number; - status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf'; + status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf' | 'accelerated'; context?: 'projected' | 'actual'; scene?: BlockScene; @@ -216,6 +217,8 @@ export default class TxView implements TransactionStripped { return auditColors.added; case 'selected': return marginalFeeColors[feeLevelIndex] || marginalFeeColors[mempoolFeeColors.length - 1]; + case 'accelerated': + return auditColors.accelerated; case 'found': if (this.context === 'projected') { return auditFeeColors[feeLevelIndex] || auditFeeColors[mempoolFeeColors.length - 1]; diff --git a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html index c62779b69..8a410f3df 100644 --- a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html +++ b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html @@ -54,6 +54,7 @@ Added Marginal fee rate Conflicting + Accelerated diff --git a/frontend/src/app/components/block/block.component.ts b/frontend/src/app/components/block/block.component.ts index ec9a49504..1e94b1a9c 100644 --- a/frontend/src/app/components/block/block.component.ts +++ b/frontend/src/app/components/block/block.component.ts @@ -340,6 +340,7 @@ export class BlockComponent implements OnInit, OnDestroy { const isFresh = {}; const isSigop = {}; const isRbf = {}; + const isAccelerated = {}; this.numMissing = 0; this.numUnexpected = 0; @@ -365,6 +366,9 @@ export class BlockComponent implements OnInit, OnDestroy { for (const txid of blockAudit.fullrbfTxs || []) { isRbf[txid] = true; } + for (const txid of blockAudit.acceleratedTxs || []) { + isAccelerated[txid] = true; + } // set transaction statuses for (const tx of blockAudit.template) { tx.context = 'projected'; @@ -389,6 +393,9 @@ export class BlockComponent implements OnInit, OnDestroy { isMissing[tx.txid] = true; this.numMissing++; } + if (isAccelerated[tx.txid]) { + tx.status = 'accelerated'; + } } for (const [index, tx] of transactions.entries()) { tx.context = 'actual'; @@ -405,6 +412,9 @@ export class BlockComponent implements OnInit, OnDestroy { isSelected[tx.txid] = true; this.numUnexpected++; } + if (isAccelerated[tx.txid]) { + tx.status = 'accelerated'; + } } for (const tx of transactions) { inBlock[tx.txid] = true; diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index 4249fd9db..7e7acfcf3 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -158,6 +158,7 @@ export interface BlockAudit extends BlockExtended { freshTxs: string[], sigopTxs: string[], fullrbfTxs: string[], + acceleratedTxs: string[], matchRate: number, expectedFees: number, expectedWeight: number, @@ -174,7 +175,7 @@ export interface TransactionStripped { vsize: number; value: number; rate?: number; // effective fee rate - status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf'; + status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf' | 'accelerated'; context?: 'projected' | 'actual'; } diff --git a/frontend/src/app/interfaces/websocket.interface.ts b/frontend/src/app/interfaces/websocket.interface.ts index e0ecdfeda..fb3c6d0c2 100644 --- a/frontend/src/app/interfaces/websocket.interface.ts +++ b/frontend/src/app/interfaces/websocket.interface.ts @@ -89,7 +89,7 @@ export interface TransactionStripped { vsize: number; value: number; rate?: number; // effective fee rate - status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf'; + status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf' | 'accelerated'; context?: 'projected' | 'actual'; }