Merge pull request #3879 from mempool/mononaut/audit-exclude-fullrbf

exclude fullrbf txs from audit and label in visualization

backend/src/api/audit.ts

@@ -1,19 +1,21 @@
 import config from '../config';
 import logger from '../logger';
 import { TransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces';
+import rbfCache from './rbf-cache';
 
 const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners
 
 class Audit {
   auditBlock(transactions: TransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: TransactionExtended })
-   : { censored: string[], added: string[], fresh: string[], sigop: string[], score: number, similarity: number } {
+   : { censored: string[], added: string[], fresh: string[], sigop: string[], fullrbf: string[], score: number, similarity: number } {
     if (!projectedBlocks?.[0]?.transactionIds || !mempool) {
-      return { censored: [], added: [], fresh: [], sigop: [], score: 0, similarity: 1 };
+      return { censored: [], added: [], fresh: [], sigop: [], fullrbf: [], score: 0, similarity: 1 };
     }
 
     const matches: string[] = []; // present in both mined block and template
     const added: string[] = []; // present in mined block, not in template
     const fresh: string[] = []; // missing, but firstSeen within PROPAGATION_MARGIN
+    const fullrbf: string[] = []; // either missing or present, and part of a fullrbf replacement
     const isCensored = {}; // missing, without excuse
     const isDisplaced = {};
     let displacedWeight = 0;
@@ -35,7 +37,9 @@ class Audit {
     for (const txid of projectedBlocks[0].transactionIds) {
       if (!inBlock[txid]) {
         // tx is recent, may have reached the miner too late for inclusion
-        if (mempool[txid]?.firstSeen != null && (now - (mempool[txid]?.firstSeen || 0)) <= PROPAGATION_MARGIN) {
+        if (rbfCache.isFullRbf(txid)) {
+          fullrbf.push(txid);
+        } else if (mempool[txid]?.firstSeen != null && (now - (mempool[txid]?.firstSeen || 0)) <= PROPAGATION_MARGIN) {
           fresh.push(txid);
         } else {
           isCensored[txid] = true;
@@ -91,7 +95,9 @@ class Audit {
       if (inTemplate[tx.txid]) {
         matches.push(tx.txid);
       } else {
-        if (!isDisplaced[tx.txid]) {
+        if (rbfCache.isFullRbf(tx.txid)) {
+          fullrbf.push(tx.txid);
+        } else if (!isDisplaced[tx.txid]) {
           added.push(tx.txid);
         }
         overflowWeight += tx.weight;
@@ -138,6 +144,7 @@ class Audit {
       added,
       fresh,
       sigop: [],
+      fullrbf,
       score,
       similarity,
     };

backend/src/api/database-migration.ts

@@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
 import { RowDataPacket } from 'mysql2';
 
 class DatabaseMigration {
-  private static currentVersion = 62;
+  private static currentVersion = 63;
   private queryTimeout = 3600_000;
   private statisticsAddedIndexed = false;
   private uniqueLogs: string[] = [];
@@ -539,6 +539,10 @@ class DatabaseMigration {
       await this.updateToSchemaVersion(62);
     }
 
+    if (databaseSchemaVersion < 63 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `blocks_audits` ADD fullrbf_txs JSON DEFAULT "[]"');
+      await this.updateToSchemaVersion(63);
+    }
   }
 
   /**

backend/src/api/rbf-cache.ts

@@ -169,6 +169,19 @@ class RbfCache {
     }
   }
 
+  // is the transaction involved in a full rbf replacement?
+  public isFullRbf(txid: string): boolean {
+    const treeId = this.treeMap.get(txid);
+    if (!treeId) {
+      return false;
+    }
+    const tree = this.rbfTrees.get(treeId);
+    if (!tree) {
+      return false;
+    }
+    return tree?.fullRbf;
+  }
+
   private cleanup(): void {
     const now = Date.now();
     for (const txid of this.expiring.keys()) {

backend/src/api/websocket-handler.ts

@@ -583,6 +583,10 @@ class WebsocketHandler {
 
     const _memPool = memPool.getMempool();
 
+    const rbfTransactions = Common.findMinedRbfTransactions(transactions, memPool.getSpendMap());
+    memPool.handleMinedRbfTransactions(rbfTransactions);
+    memPool.removeFromSpendMap(transactions);
+
     if (config.MEMPOOL.AUDIT) {
       let projectedBlocks;
       let auditMempool = _memPool;
@@ -605,7 +609,7 @@ class WebsocketHandler {
       }
 
       if (Common.indexingEnabled() && memPool.isInSync()) {
-        const { censored, added, fresh, sigop, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool);
+        const { censored, added, fresh, sigop, fullrbf, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool);
         const matchRate = Math.round(score * 100 * 100) / 100;
 
         const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions : [];
@@ -633,6 +637,7 @@ class WebsocketHandler {
           missingTxs: censored,
           freshTxs: fresh,
           sigopTxs: sigop,
+          fullrbfTxs: fullrbf,
           matchRate: matchRate,
           expectedFees: totalFees,
           expectedWeight: totalWeight,
@@ -652,10 +657,6 @@ class WebsocketHandler {
       }
     }
 
-    const rbfTransactions = Common.findMinedRbfTransactions(transactions, memPool.getSpendMap());
-    memPool.handleMinedRbfTransactions(rbfTransactions);
-    memPool.removeFromSpendMap(transactions);
-
     // Update mempool to remove transactions included in the new block
     for (const txId of txIds) {
       delete _memPool[txId];

backend/src/mempool.interfaces.ts

@@ -34,6 +34,7 @@ export interface BlockAudit {
   missingTxs: string[],
   freshTxs: string[],
   sigopTxs: string[],
+  fullrbfTxs: string[],
   addedTxs: string[],
   matchRate: number,
   expectedFees?: number,

backend/src/repositories/BlocksAuditsRepository.ts

@@ -6,9 +6,9 @@ import { BlockAudit, AuditScore } from '../mempool.interfaces';
 class BlocksAuditRepositories {
   public async $saveAudit(audit: BlockAudit): Promise<void> {
     try {
-      await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, fresh_txs, sigop_txs, match_rate, expected_fees, expected_weight)
+      await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, fresh_txs, sigop_txs, fullrbf_txs, match_rate, expected_fees, expected_weight)
-        VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs),
+        VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs),
-          JSON.stringify(audit.addedTxs), JSON.stringify(audit.freshTxs), JSON.stringify(audit.sigopTxs), audit.matchRate, audit.expectedFees, audit.expectedWeight]);
+          JSON.stringify(audit.addedTxs), JSON.stringify(audit.freshTxs), JSON.stringify(audit.sigopTxs), JSON.stringify(audit.fullrbfTxs), audit.matchRate, audit.expectedFees, audit.expectedWeight]);
     } catch (e: any) {
       if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
         logger.debug(`Cannot save block audit for block ${audit.hash} because it has already been indexed, ignoring`);
@@ -68,6 +68,7 @@ class BlocksAuditRepositories {
         added_txs as addedTxs,
         fresh_txs as freshTxs,
         sigop_txs as sigopTxs,
+        fullrbf_txs as fullrbfTxs,
         match_rate as matchRate,
         expected_fees as expectedFees,
         expected_weight as expectedWeight
@@ -81,6 +82,7 @@ class BlocksAuditRepositories {
         rows[0].addedTxs = JSON.parse(rows[0].addedTxs);
         rows[0].freshTxs = JSON.parse(rows[0].freshTxs);
         rows[0].sigopTxs = JSON.parse(rows[0].sigopTxs);
+        rows[0].fullrbfTxs = JSON.parse(rows[0].fullrbfTxs);
         rows[0].template = JSON.parse(rows[0].template);
 
         return rows[0];

frontend/src/app/components/block-overview-graph/tx-view.ts

@@ -37,7 +37,7 @@ export default class TxView implements TransactionStripped {
   value: number;
   feerate: number;
   rate?: number;
-  status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'added' | 'censored' | 'selected';
+  status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'added' | 'censored' | 'selected' | 'fullrbf';
   context?: 'projected' | 'actual';
   scene?: BlockScene;
 
@@ -172,6 +172,7 @@ export default class TxView implements TransactionStripped {
         return auditColors.censored;
       case 'missing':
       case 'sigop':
+      case 'fullrbf':
         return marginalFeeColors[feeLevelIndex] || marginalFeeColors[mempoolFeeColors.length - 1];
       case 'fresh':
         return auditColors.missing;

frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html

@@ -52,6 +52,7 @@
           <td *ngSwitchCase="'fresh'"><span class="badge badge-warning" i18n="transaction.audit.recently-broadcasted">Recently broadcasted</span></td>
           <td *ngSwitchCase="'added'"><span class="badge badge-warning" i18n="transaction.audit.added">Added</span></td>
           <td *ngSwitchCase="'selected'"><span class="badge badge-warning" i18n="transaction.audit.marginal">Marginal fee rate</span></td>
+          <td *ngSwitchCase="'fullrbf'"><span class="badge badge-warning" i18n="transaction.audit.fullrbf">Full RBF</span></td>
         </ng-container>
       </tr>
     </tbody>

frontend/src/app/components/block/block.component.ts

@@ -317,6 +317,7 @@ export class BlockComponent implements OnInit, OnDestroy {
         const isSelected = {};
         const isFresh = {};
         const isSigop = {};
+        const isFullRbf = {};
         this.numMissing = 0;
         this.numUnexpected = 0;
 
@@ -339,6 +340,9 @@ export class BlockComponent implements OnInit, OnDestroy {
           for (const txid of blockAudit.sigopTxs || []) {
             isSigop[txid] = true;
           }
+          for (const txid of blockAudit.fullrbfTxs || []) {
+            isFullRbf[txid] = true;
+          }
           // set transaction statuses
           for (const tx of blockAudit.template) {
             tx.context = 'projected';
@@ -347,7 +351,15 @@ export class BlockComponent implements OnInit, OnDestroy {
             } else if (inBlock[tx.txid]) {
               tx.status = 'found';
             } else {
-              tx.status = isFresh[tx.txid] ? 'fresh' : (isSigop[tx.txid] ? 'sigop' : 'missing');
+              if (isFresh[tx.txid]) {
+                tx.status = 'fresh';
+              } else if (isSigop[tx.txid]) {
+                tx.status = 'sigop';
+              } else if (isFullRbf[tx.txid]) {
+                tx.status = 'fullrbf';
+              } else {
+                tx.status = 'missing';
+              }
               isMissing[tx.txid] = true;
               this.numMissing++;
             }
@@ -360,6 +372,8 @@ export class BlockComponent implements OnInit, OnDestroy {
               tx.status = 'added';
             } else if (inTemplate[tx.txid]) {
               tx.status = 'found';
+            } else if (isFullRbf[tx.txid]) {
+              tx.status = 'fullrbf';
             } else {
               tx.status = 'selected';
               isSelected[tx.txid] = true;

frontend/src/app/interfaces/node-api.interface.ts

@@ -156,6 +156,7 @@ export interface BlockAudit extends BlockExtended {
   addedTxs: string[],
   freshTxs: string[],
   sigopTxs: string[],
+  fullrbfTxs: string[],
   matchRate: number,
   expectedFees: number,
   expectedWeight: number,
@@ -171,7 +172,7 @@ export interface TransactionStripped {
   fee: number;
   vsize: number;
   value: number;
-  status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'added' | 'censored' | 'selected';
+  status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'added' | 'censored' | 'selected' | 'fullrbf';
   context?: 'projected' | 'actual';
 }
 

frontend/src/app/interfaces/websocket.interface.ts

@@ -77,7 +77,7 @@ export interface TransactionStripped {
   vsize: number;
   value: number;
   rate?: number; // effective fee rate
-  status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'added' | 'censored' | 'selected';
+  status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'added' | 'censored' | 'selected' | 'fullrbf';
   context?: 'projected' | 'actual';
 }