Merge pull request #5354 from mempool/mononaut/v1-audits

v1 audits

backend/src/api/audit.ts

@@ -6,20 +6,22 @@ import rbfCache from './rbf-cache';
 const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners
 
 class Audit {
-  auditBlock(transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended }, useAccelerations: boolean = false)
+  auditBlock(height: number, transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended })
-   : { censored: string[], added: string[], prioritized: string[], fresh: string[], sigop: string[], fullrbf: string[], accelerated: string[], score: number, similarity: number } {
+   : { unseen: string[], censored: string[], added: string[], prioritized: string[], fresh: string[], sigop: string[], fullrbf: string[], accelerated: string[], score: number, similarity: number } {
     if (!projectedBlocks?.[0]?.transactionIds || !mempool) {
-      return { censored: [], added: [], prioritized: [], fresh: [], sigop: [], fullrbf: [], accelerated: [], score: 1, similarity: 1 };
+      return { unseen: [], censored: [], added: [], prioritized: [], fresh: [], sigop: [], fullrbf: [], accelerated: [], score: 1, similarity: 1 };
     }
 
     const matches: string[] = []; // present in both mined block and template
     const added: string[] = []; // present in mined block, not in template
-    const prioritized: string[] = [] // present in the mined block, not in the template, but further down in the mempool
+    const unseen: string[] = []; // present in the mined block, not in our mempool
+    const prioritized: string[] = []; // higher in the block than would be expected by in-band feerate alone
     const fresh: string[] = []; // missing, but firstSeen or lastBoosted within PROPAGATION_MARGIN
     const rbf: string[] = []; // either missing or present, and either part of a full-rbf replacement, or a conflict with the mined block
     const accelerated: string[] = []; // prioritized by the mempool accelerator
     const isCensored = {}; // missing, without excuse
     const isDisplaced = {};
+    const isAccelerated = {};
     let displacedWeight = 0;
     let matchedWeight = 0;
     let projectedWeight = 0;
@@ -32,6 +34,7 @@ class Audit {
       inBlock[tx.txid] = tx;
       if (mempool[tx.txid] && mempool[tx.txid].acceleration) {
         accelerated.push(tx.txid);
+        isAccelerated[tx.txid] = true;
       }
     }
     // coinbase is always expected
@@ -113,18 +116,41 @@ class Audit {
       } else {
         if (rbfCache.has(tx.txid)) {
           rbf.push(tx.txid);
-        } else if (!isDisplaced[tx.txid]) {
+          if (!mempool[tx.txid] && !rbfCache.getReplacedBy(tx.txid)) {
-          if (mempool[tx.txid]) {
+            unseen.push(tx.txid);
-            prioritized.push(tx.txid);
+          }
         } else {
+          if (mempool[tx.txid]) {
+            if (isDisplaced[tx.txid]) {
               added.push(tx.txid);
             }
+          } else {
+            unseen.push(tx.txid);
+          }
         }
         overflowWeight += tx.weight;
       }
       totalWeight += tx.weight;
     }
 
+
+    // identify "prioritized" transactions
+    let lastEffectiveRate = 0;
+    // Iterate over the mined template from bottom to top (excluding the coinbase)
+    // Transactions should appear in ascending order of mining priority.
+    for (let i = transactions.length - 1; i > 0; i--) {
+      const blockTx = transactions[i];
+      // If a tx has a lower in-band effective fee rate than the previous tx,
+      // it must have been prioritized out-of-band (in order to have a higher mining priority)
+      // so exclude from the analysis.
+      if ((blockTx.effectiveFeePerVsize || 0) < lastEffectiveRate) {
+        prioritized.push(blockTx.txid);
+        // accelerated txs may or may not have their prioritized fee rate applied, so don't use them as a reference
+      } else if (!isAccelerated[blockTx.txid]) {
+        lastEffectiveRate = blockTx.effectiveFeePerVsize || 0;
+      }
+    }
+
     // transactions missing from near the end of our template are probably not being censored
     let overflowWeightRemaining = overflowWeight - (config.MEMPOOL.BLOCK_WEIGHT_UNITS - totalWeight);
     let maxOverflowRate = 0;
@@ -165,6 +191,7 @@ class Audit {
     const similarity = projectedWeight ? matchedWeight / projectedWeight : 1;
 
     return {
+      unseen,
       censored: Object.keys(isCensored),
       added,
       prioritized,

backend/src/api/blocks.ts

@@ -33,6 +33,7 @@ import AccelerationRepository from '../repositories/AccelerationRepository';
 import { calculateFastBlockCpfp, calculateGoodBlockCpfp } from './cpfp';
 import mempool from './mempool';
 import CpfpRepository from '../repositories/CpfpRepository';
+import accelerationApi from './services/acceleration';
 
 class Blocks {
   private blocks: BlockExtended[] = [];
@@ -439,7 +440,7 @@ class Blocks {
 
 
         if (config.MEMPOOL.BACKEND === 'esplora') {
-          const txs = (await bitcoinApi.$getTxsForBlock(block.hash)).map(tx => transactionUtils.extendTransaction(tx));
+          const txs = (await bitcoinApi.$getTxsForBlock(block.hash)).map(tx => transactionUtils.extendMempoolTransaction(tx));
           const cpfpSummary = await this.$indexCPFP(block.hash, block.height, txs);
           if (cpfpSummary) {
             await this.$getStrippedBlockTransactions(block.hash, true, true, cpfpSummary, block.height); // This will index the block summary
@@ -904,7 +905,12 @@ class Blocks {
         }
       }
 
-      const cpfpSummary: CpfpSummary = calculateGoodBlockCpfp(block.height, transactions, Object.values(mempool.getAccelerations()).map(a => ({ txid: a.txid, max_bid: a.feeDelta })));
+      let accelerations = Object.values(mempool.getAccelerations());
+      if (accelerations?.length > 0) {
+        const pool = await this.$findBlockMiner(transactionUtils.stripCoinbaseTransaction(transactions[0]));
+        accelerations = accelerations.filter(a => a.pools.includes(pool.uniqueId));
+      }
+      const cpfpSummary: CpfpSummary = calculateGoodBlockCpfp(block.height, transactions, accelerations.map(a => ({ txid: a.txid, max_bid: a.feeDelta })));
       const blockExtended: BlockExtended = await this.$getBlockExtended(block, cpfpSummary.transactions);
       const blockSummary: BlockSummary = this.summarizeBlockTransactions(block.id, cpfpSummary.transactions);
       this.updateTimerProgress(timer, `got block data for ${this.currentBlockHeight}`);
@@ -927,12 +933,12 @@ class Blocks {
               const newBlock = await this.$indexBlock(lastBlock.height - i);
               this.blocks.push(newBlock);
               this.updateTimerProgress(timer, `reindexed block`);
-              let cpfpSummary;
+              let newCpfpSummary;
               if (config.MEMPOOL.CPFP_INDEXING) {
-                cpfpSummary = await this.$indexCPFP(newBlock.id, lastBlock.height - i);
+                newCpfpSummary = await this.$indexCPFP(newBlock.id, lastBlock.height - i);
                 this.updateTimerProgress(timer, `reindexed block cpfp`);
               }
-              await this.$getStrippedBlockTransactions(newBlock.id, true, true, cpfpSummary, newBlock.height);
+              await this.$getStrippedBlockTransactions(newBlock.id, true, true, newCpfpSummary, newBlock.height);
               this.updateTimerProgress(timer, `reindexed block summary`);
             }
             await mining.$indexDifficultyAdjustments();
@@ -981,7 +987,7 @@ class Blocks {
 
       // start async callbacks
       this.updateTimerProgress(timer, `starting async callbacks for ${this.currentBlockHeight}`);
-      const callbackPromises = this.newAsyncBlockCallbacks.map((cb) => cb(blockExtended, txIds, transactions));
+      const callbackPromises = this.newAsyncBlockCallbacks.map((cb) => cb(blockExtended, txIds, cpfpSummary.transactions));
 
       if (block.height % 2016 === 0) {
         if (Common.indexingEnabled()) {
@@ -1178,7 +1184,7 @@ class Blocks {
           };
         }),
       };
-      summaryVersion = 1;
+      summaryVersion = cpfpSummary.version;
     } else {
       if (config.MEMPOOL.BACKEND === 'esplora') {
         const txs = (await bitcoinApi.$getTxsForBlock(hash)).map(tx => transactionUtils.extendTransaction(tx));
@@ -1397,11 +1403,11 @@ class Blocks {
     return this.currentBlockHeight;
   }
 
-  public async $indexCPFP(hash: string, height: number, txs?: TransactionExtended[]): Promise<CpfpSummary | null> {
+  public async $indexCPFP(hash: string, height: number, txs?: MempoolTransactionExtended[]): Promise<CpfpSummary | null> {
     let transactions = txs;
     if (!transactions) {
       if (config.MEMPOOL.BACKEND === 'esplora') {
-        transactions = (await bitcoinApi.$getTxsForBlock(hash)).map(tx => transactionUtils.extendTransaction(tx));
+        transactions = (await bitcoinApi.$getTxsForBlock(hash)).map(tx => transactionUtils.extendMempoolTransaction(tx));
       }
       if (!transactions) {
         const block = await bitcoinClient.getBlock(hash, 2);
@@ -1413,7 +1419,7 @@ class Blocks {
     }
 
     if (transactions?.length != null) {
-      const summary = calculateFastBlockCpfp(height, transactions as TransactionExtended[]);
+      const summary = calculateFastBlockCpfp(height, transactions);
 
       await this.$saveCpfp(hash, height, summary);
 

backend/src/api/common.ts

@@ -1,6 +1,6 @@
 import * as bitcoinjs from 'bitcoinjs-lib';
 import { Request } from 'express';
-import { CpfpInfo, CpfpSummary, CpfpCluster, EffectiveFeeStats, MempoolBlockWithTransactions, TransactionExtended, MempoolTransactionExtended, TransactionStripped, WorkingEffectiveFeeStats, TransactionClassified, TransactionFlags } from '../mempool.interfaces';
+import { EffectiveFeeStats, MempoolBlockWithTransactions, TransactionExtended, MempoolTransactionExtended, TransactionStripped, WorkingEffectiveFeeStats, TransactionClassified, TransactionFlags } from '../mempool.interfaces';
 import config from '../config';
 import { NodeSocket } from '../repositories/NodesSocketsRepository';
 import { isIP } from 'net';

backend/src/api/cpfp.ts

@@ -6,7 +6,7 @@ import { Acceleration } from './acceleration/acceleration';
 const CPFP_UPDATE_INTERVAL = 60_000; // update CPFP info at most once per 60s per transaction
 const MAX_CLUSTER_ITERATIONS = 100;
 
-export function calculateFastBlockCpfp(height: number, transactions: TransactionExtended[], saveRelatives: boolean = false): CpfpSummary {
+export function calculateFastBlockCpfp(height: number, transactions: MempoolTransactionExtended[], saveRelatives: boolean = false): CpfpSummary {
   const clusters: CpfpCluster[] = []; // list of all cpfp clusters in this block
   const clusterMap: { [txid: string]: CpfpCluster } = {}; // map transactions to their cpfp cluster
   let clusterTxs: TransactionExtended[] = []; // working list of elements of the current cluster
@@ -93,6 +93,7 @@ export function calculateFastBlockCpfp(height: number, transactions: Transaction
   return {
     transactions,
     clusters,
+    version: 1,
   };
 }
 
@@ -159,6 +160,7 @@ export function calculateGoodBlockCpfp(height: number, transactions: MempoolTran
   return {
     transactions: transactions.map(tx => txMap[tx.txid]),
     clusters: clusterArray,
+    version: 2,
   };
 }
 

backend/src/api/database-migration.ts

@@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
 import { RowDataPacket } from 'mysql2';
 
 class DatabaseMigration {
-  private static currentVersion = 80;
+  private static currentVersion = 81;
   private queryTimeout = 3600_000;
   private statisticsAddedIndexed = false;
   private uniqueLogs: string[] = [];
@@ -691,6 +691,13 @@ class DatabaseMigration {
       await this.$executeQuery('ALTER TABLE `blocks` ADD coinbase_addresses JSON DEFAULT NULL');
       await this.updateToSchemaVersion(80);
     }
+
+    if (databaseSchemaVersion < 81) {
+      await this.$executeQuery('ALTER TABLE `blocks_audits` ADD version INT NOT NULL DEFAULT 0');
+      await this.$executeQuery('ALTER TABLE `blocks_audits` ADD INDEX `version` (`version`)');
+      await this.$executeQuery('ALTER TABLE `blocks_audits` ADD unseen_txs JSON DEFAULT "[]"');
+      await this.updateToSchemaVersion(81);
+    }
   }
 
   /**

backend/src/api/mini-miner.ts

@@ -337,7 +337,7 @@ export function makeBlockTemplate(candidates: MempoolTransactionExtended[], acce
   let failures = 0;
   while (mempoolArray.length || modified.length) {
     // skip invalid transactions
-    while (mempoolArray[0].used || mempoolArray[0].modified) {
+    while (mempoolArray[0]?.used || mempoolArray[0]?.modified) {
       mempoolArray.shift();
     }
 

backend/src/api/websocket-handler.ts

@@ -3,7 +3,7 @@ import * as WebSocket from 'ws';
 import {
   BlockExtended, TransactionExtended, MempoolTransactionExtended, WebsocketResponse,
   OptimizedStatistic, ILoadingIndicators, GbtCandidates, TxTrackingInfo,
-  MempoolBlockDelta, MempoolDelta, MempoolDeltaTxids
+  MempoolDelta, MempoolDeltaTxids
 } from '../mempool.interfaces';
 import blocks from './blocks';
 import memPool from './mempool';
@@ -933,6 +933,8 @@ class WebsocketHandler {
       throw new Error('No WebSocket.Server have been set');
     }
 
+    const blockTransactions = structuredClone(transactions);
+
     this.printLogs();
     await statistics.runStatistics();
 
@@ -942,7 +944,7 @@ class WebsocketHandler {
     let transactionIds: string[] = (memPool.limitGBT) ? Object.keys(candidates?.txs || {}) : Object.keys(_memPool);
 
     const accelerations = Object.values(mempool.getAccelerations());
-    await accelerationRepository.$indexAccelerationsForBlock(block, accelerations, transactions);
+    await accelerationRepository.$indexAccelerationsForBlock(block, accelerations, structuredClone(transactions));
 
     const rbfTransactions = Common.findMinedRbfTransactions(transactions, memPool.getSpendMap());
     memPool.handleMinedRbfTransactions(rbfTransactions);
@@ -962,7 +964,7 @@ class WebsocketHandler {
       }
 
       if (Common.indexingEnabled()) {
-        const { censored, added, prioritized, fresh, sigop, fullrbf, accelerated, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool);
+        const { unseen, censored, added, prioritized, fresh, sigop, fullrbf, accelerated, score, similarity } = Audit.auditBlock(block.height, blockTransactions, projectedBlocks, auditMempool);
         const matchRate = Math.round(score * 100 * 100) / 100;
 
         const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions : [];
@@ -984,9 +986,11 @@ class WebsocketHandler {
         });
 
         BlocksAuditsRepository.$saveAudit({
+          version: 1,
           time: block.timestamp,
           height: block.height,
           hash: block.id,
+          unseenTxs: unseen,
           addedTxs: added,
           prioritizedTxs: prioritized,
           missingTxs: censored,

backend/src/indexer.ts

@@ -10,6 +10,7 @@ import config from './config';
 import auditReplicator from './replication/AuditReplication';
 import statisticsReplicator from './replication/StatisticsReplication';
 import AccelerationRepository from './repositories/AccelerationRepository';
+import BlocksAuditsRepository from './repositories/BlocksAuditsRepository';
 
 export interface CoreIndex {
   name: string;
@@ -192,6 +193,7 @@ class Indexer {
       await auditReplicator.$sync();
       await statisticsReplicator.$sync();
       await AccelerationRepository.$indexPastAccelerations();
+      await BlocksAuditsRepository.$migrateAuditsV0toV1();
       // do not wait for classify blocks to finish
       blocks.$classifyBlocks();
     } catch (e) {

backend/src/mempool.interfaces.ts

@@ -29,9 +29,11 @@ export interface PoolStats extends PoolInfo {
 }
 
 export interface BlockAudit {
+  version: number,
   time: number,
   height: number,
   hash: string,
+  unseenTxs: string[],
   missingTxs: string[],
   freshTxs: string[],
   sigopTxs: string[],
@@ -383,8 +385,9 @@ export interface CpfpCluster {
 }
 
 export interface CpfpSummary {
-  transactions: TransactionExtended[];
+  transactions: MempoolTransactionExtended[];
   clusters: CpfpCluster[];
+  version: number;
 }
 
 export interface Statistic {

backend/src/replication/AuditReplication.ts

@@ -35,7 +35,7 @@ class AuditReplication {
     let totalSynced = 0;
     let totalMissed = 0;
     let loggerTimer = Date.now();
-    // process missing audits in batches of 
+    // process missing audits in batches of BATCH_SIZE
     for (let i = 0; i < missingAudits.length; i += BATCH_SIZE) {
       const slice = missingAudits.slice(i, i + BATCH_SIZE);
       const results = await Promise.all(slice.map(hash => this.$syncAudit(hash)));
@@ -109,9 +109,11 @@ class AuditReplication {
       version: 1,
     });
     await blocksAuditsRepository.$saveAudit({
+      version: auditSummary.version || 0,
       hash: blockHash,
       height: auditSummary.height,
       time: auditSummary.timestamp || auditSummary.time,
+      unseenTxs: auditSummary.unseenTxs || [],
       missingTxs: auditSummary.missingTxs || [],
       addedTxs: auditSummary.addedTxs || [],
       prioritizedTxs: auditSummary.prioritizedTxs || [],

backend/src/repositories/AccelerationRepository.ts

@@ -192,6 +192,7 @@ class AccelerationRepository {
     }
   }
 
+  // modifies block transactions
   public async $indexAccelerationsForBlock(block: BlockExtended, accelerations: Acceleration[], transactions: MempoolTransactionExtended[]): Promise<void> {
     const blockTxs: { [txid: string]: MempoolTransactionExtended } = {};
     for (const tx of transactions) {

backend/src/repositories/BlocksAuditsRepository.ts

@@ -1,13 +1,24 @@
-import blocks from '../api/blocks';
 import DB from '../database';
 import logger from '../logger';
-import { BlockAudit, AuditScore, TransactionAudit } from '../mempool.interfaces';
+import bitcoinApi from '../api/bitcoin/bitcoin-api-factory';
+import { BlockAudit, AuditScore, TransactionAudit, TransactionStripped } from '../mempool.interfaces';
+
+interface MigrationAudit {
+  version: number,
+  height: number,
+  id: string,
+  timestamp: number,
+  prioritizedTxs: string[],
+  acceleratedTxs: string[],
+  template: TransactionStripped[],
+  transactions: TransactionStripped[],
+}
 
 class BlocksAuditRepositories {
   public async $saveAudit(audit: BlockAudit): Promise<void> {
     try {
-      await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, prioritized_txs, fresh_txs, sigop_txs, fullrbf_txs, accelerated_txs, match_rate, expected_fees, expected_weight)
+      await DB.query(`INSERT INTO blocks_audits(version, time, height, hash, unseen_txs, missing_txs, added_txs, prioritized_txs, fresh_txs, sigop_txs, fullrbf_txs, accelerated_txs, match_rate, expected_fees, expected_weight)
-        VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs),
+        VALUE (?, FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [audit.version, audit.time, audit.height, audit.hash, JSON.stringify(audit.unseenTxs), JSON.stringify(audit.missingTxs),
           JSON.stringify(audit.addedTxs), JSON.stringify(audit.prioritizedTxs), JSON.stringify(audit.freshTxs), JSON.stringify(audit.sigopTxs), JSON.stringify(audit.fullrbfTxs), JSON.stringify(audit.acceleratedTxs), audit.matchRate, audit.expectedFees, audit.expectedWeight]);
     } catch (e: any) {
       if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
@@ -62,8 +73,13 @@ class BlocksAuditRepositories {
   public async $getBlockAudit(hash: string): Promise<BlockAudit | null> {
     try {
       const [rows]: any[] = await DB.query(
-        `SELECT blocks_audits.height, blocks_audits.hash as id, UNIX_TIMESTAMP(blocks_audits.time) as timestamp,
+        `SELECT
+          blocks_audits.version,
+          blocks_audits.height,
+          blocks_audits.hash as id,
+          UNIX_TIMESTAMP(blocks_audits.time) as timestamp,
           template,
+          unseen_txs as unseenTxs,
           missing_txs as missingTxs,
           added_txs as addedTxs,
           prioritized_txs as prioritizedTxs,
@@ -80,6 +96,7 @@ class BlocksAuditRepositories {
       `, [hash]);
       
       if (rows.length) {
+        rows[0].unseenTxs = JSON.parse(rows[0].unseenTxs);
         rows[0].missingTxs = JSON.parse(rows[0].missingTxs);
         rows[0].addedTxs = JSON.parse(rows[0].addedTxs);
         rows[0].prioritizedTxs = JSON.parse(rows[0].prioritizedTxs);
@@ -124,7 +141,7 @@ class BlocksAuditRepositories {
           conflict: isConflict,
           accelerated: isAccelerated,
           firstSeen,
-        }
+        };
       }
       return null;
     } catch (e: any) {
@@ -186,6 +203,96 @@ class BlocksAuditRepositories {
       throw e;
     }
   }
+
+  /**
+   * [INDEXING] Migrate audits from v0 to v1
+   */
+  public async $migrateAuditsV0toV1(): Promise<void> {
+    try {
+      let done = false;
+      let processed = 0;
+      let lastHeight;
+      while (!done) {
+        const [toMigrate]: MigrationAudit[][] = await DB.query(
+          `SELECT
+            blocks_audits.height as height,
+            blocks_audits.hash as id,
+            UNIX_TIMESTAMP(blocks_audits.time) as timestamp,
+            blocks_summaries.transactions as transactions,
+            blocks_templates.template as template,
+            blocks_audits.prioritized_txs as prioritizedTxs,
+            blocks_audits.accelerated_txs as acceleratedTxs
+          FROM blocks_audits
+          JOIN blocks_summaries ON blocks_summaries.id = blocks_audits.hash
+          JOIN blocks_templates ON blocks_templates.id = blocks_audits.hash
+          WHERE blocks_audits.version = 0
+          AND blocks_summaries.version = 2
+          ORDER BY blocks_audits.height DESC
+          LIMIT 100
+        `) as any[];
+
+        if (toMigrate.length <= 0 || lastHeight === toMigrate[0].height) {
+          done = true;
+          break;
+        }
+        lastHeight = toMigrate[0].height;
+
+        logger.info(`migrating ${toMigrate.length} audits to version 1`);
+
+        for (const audit of toMigrate) {
+          // unpack JSON-serialized transaction lists
+          audit.transactions = JSON.parse((audit.transactions as any as string) || '[]');
+          audit.template = JSON.parse((audit.template as any as string) || '[]');
+
+          // we know transactions in the template, or marked "prioritized" or "accelerated"
+          // were seen in our mempool before the block was mined.
+          const isSeen = new Set<string>();
+          for (const tx of audit.template) {
+            isSeen.add(tx.txid);
+          }
+          for (const txid of audit.prioritizedTxs) {
+            isSeen.add(txid);
+          }
+          for (const txid of audit.acceleratedTxs) {
+            isSeen.add(txid);
+          }
+          const unseenTxs = audit.transactions.slice(0).map(tx => tx.txid).filter(txid => !isSeen.has(txid));
+
+          // identify "prioritized" transactions
+          const prioritizedTxs: string[] = [];
+          let lastEffectiveRate = 0;
+          // Iterate over the mined template from bottom to top (excluding the coinbase)
+          // Transactions should appear in ascending order of mining priority.
+          for (let i = audit.transactions.length - 1; i > 0; i--) {
+            const blockTx = audit.transactions[i];
+            // If a tx has a lower in-band effective fee rate than the previous tx,
+            // it must have been prioritized out-of-band (in order to have a higher mining priority)
+            // so exclude from the analysis.
+            if ((blockTx.rate || 0) < lastEffectiveRate) {
+              prioritizedTxs.push(blockTx.txid);
+            } else {
+              lastEffectiveRate = blockTx.rate || 0;
+            }
+          }
+
+          // Update audit in the database
+          await DB.query(`
+            UPDATE blocks_audits SET
+              version = ?,
+              unseen_txs = ?,
+              prioritized_txs = ?
+            WHERE hash = ?
+          `, [1, JSON.stringify(unseenTxs), JSON.stringify(prioritizedTxs), audit.id]);
+        }
+
+        processed += toMigrate.length;
+      }
+
+      logger.info(`migrated ${processed} audits to version 1`);
+    } catch (e: any) {
+      logger.err(`Error while migrating audits from v0 to v1. Will try again later. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+  }
 }
 
 export default new BlocksAuditRepositories();

frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts

@@ -18,6 +18,7 @@ const unmatchedAuditColors = {
   censored: setOpacity(defaultAuditColors.censored, unmatchedOpacity),
   missing: setOpacity(defaultAuditColors.missing, unmatchedOpacity),
   added: setOpacity(defaultAuditColors.added, unmatchedOpacity),
+  added_prioritized: setOpacity(defaultAuditColors.added_prioritized, unmatchedOpacity),
   prioritized: setOpacity(defaultAuditColors.prioritized, unmatchedOpacity),
   accelerated: setOpacity(defaultAuditColors.accelerated, unmatchedOpacity),
 };
@@ -25,6 +26,7 @@ const unmatchedContrastAuditColors = {
   censored: setOpacity(contrastAuditColors.censored, unmatchedOpacity),
   missing: setOpacity(contrastAuditColors.missing, unmatchedOpacity),
   added: setOpacity(contrastAuditColors.added, unmatchedOpacity),
+  added_prioritized: setOpacity(contrastAuditColors.added_prioritized, unmatchedOpacity),
   prioritized: setOpacity(contrastAuditColors.prioritized, unmatchedOpacity),
   accelerated: setOpacity(contrastAuditColors.accelerated, unmatchedOpacity),
 };

frontend/src/app/components/block-overview-graph/tx-view.ts

@@ -33,7 +33,7 @@ export default class TxView implements TransactionStripped {
   flags: number;
   bigintFlags?: bigint | null = 0b00000100_00000000_00000000_00000000n;
   time?: number;
-  status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'prioritized' | 'censored' | 'selected' | 'rbf' | 'accelerated';
+  status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'added_prioritized' | 'prioritized' | 'censored' | 'selected' | 'rbf' | 'accelerated';
   context?: 'projected' | 'actual';
   scene?: BlockScene;
 

frontend/src/app/components/block-overview-graph/utils.ts

@@ -71,6 +71,7 @@ export const defaultAuditColors = {
   censored: hexToColor('f344df'),
   missing: darken(desaturate(hexToColor('f344df'), 0.3), 0.7),
   added: hexToColor('0099ff'),
+  added_prioritized: darken(desaturate(hexToColor('0099ff'), 0.15), 0.85),
   prioritized: darken(desaturate(hexToColor('0099ff'), 0.3), 0.7),
   accelerated: hexToColor('8f5ff6'),
 };
@@ -101,6 +102,7 @@ export const contrastAuditColors = {
   censored: hexToColor('ffa8ff'),
   missing: darken(desaturate(hexToColor('ffa8ff'), 0.3), 0.7),
   added: hexToColor('00bb98'),
+  added_prioritized: darken(desaturate(hexToColor('00bb98'), 0.15), 0.85),
   prioritized: darken(desaturate(hexToColor('00bb98'), 0.3), 0.7),
   accelerated: hexToColor('8f5ff6'),
 };
@@ -136,6 +138,8 @@ export function defaultColorFunction(
       return auditColors.missing;
     case 'added':
       return auditColors.added;
+    case 'added_prioritized':
+      return auditColors.added_prioritized;
     case 'prioritized':
       return auditColors.prioritized;
     case 'selected':

frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html

@@ -75,6 +75,10 @@
             <span *ngSwitchCase="'freshcpfp'" class="badge badge-warning" i18n="transaction.audit.recently-cpfped">Recently CPFP'd</span>
             <span *ngSwitchCase="'added'" class="badge badge-warning" i18n="tx-features.tag.added|Added">Added</span>
             <span *ngSwitchCase="'prioritized'" class="badge badge-warning" i18n="tx-features.tag.prioritized|Prioritized">Prioritized</span>
+            <ng-container *ngSwitchCase="'added_prioritized'">
+              <span class="badge badge-warning" i18n="tx-features.tag.added|Added">Added</span>
+              <span class="badge badge-warning ml-1" i18n="tx-features.tag.prioritized|Prioritized">Prioritized</span>
+            </ng-container>
             <span *ngSwitchCase="'selected'" class="badge badge-warning" i18n="transaction.audit.marginal">Marginal fee rate</span>
             <span *ngSwitchCase="'rbf'" class="badge badge-warning" i18n="tx-features.tag.conflict|Conflict">Conflict</span>
             <span *ngSwitchCase="'accelerated'" class="badge badge-accelerated" i18n="transaction.audit.accelerated">Accelerated</span>

frontend/src/app/components/block/block.component.ts

@@ -521,6 +521,7 @@ export class BlockComponent implements OnInit, OnDestroy {
     if (transactions && blockAudit) {
       const inTemplate = {};
       const inBlock = {};
+      const isUnseen = {};
       const isAdded = {};
       const isPrioritized = {};
       const isCensored = {};
@@ -543,6 +544,9 @@ export class BlockComponent implements OnInit, OnDestroy {
         for (const tx of transactions) {
           inBlock[tx.txid] = true;
         }
+        for (const txid of blockAudit.unseenTxs || []) {
+          isUnseen[txid] = true;
+        }
         for (const txid of blockAudit.addedTxs) {
           isAdded[txid] = true;
         }
@@ -592,18 +596,27 @@ export class BlockComponent implements OnInit, OnDestroy {
             tx.status = 'accelerated';
           }
         }
-        for (const [index, tx] of transactions.entries()) {
+        let anySeen = false;
+        for (let index = transactions.length - 1; index >= 0; index--) {
+          const tx = transactions[index];
           tx.context = 'actual';
           if (index === 0) {
             tx.status = null;
-          } else if (isAdded[tx.txid]) {
-            tx.status = 'added';
           } else if (isPrioritized[tx.txid]) {
+            if (isAdded[tx.txid] || (blockAudit.version > 0 && isUnseen[tx.txid])) {
+              tx.status = 'added_prioritized';
+            } else {
               tx.status = 'prioritized';
+            }
+          } else if (isAdded[tx.txid] && (blockAudit.version === 0 || isUnseen[tx.txid])) {
+            tx.status = 'added';
           } else if (inTemplate[tx.txid]) {
+            anySeen = true;
             tx.status = 'found';
           } else if (isRbf[tx.txid]) {
             tx.status = 'rbf';
+          } else if (isUnseen[tx.txid] && anySeen) {
+            tx.status = 'added';
           } else {
             tx.status = 'selected';
             isSelected[tx.txid] = true;

frontend/src/app/components/transaction/transaction.component.ts

@@ -411,10 +411,11 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
                 const isConflict = audit.fullrbfTxs.includes(txid);
                 const isExpected = audit.template.some(tx => tx.txid === txid);
                 const firstSeen = audit.template.find(tx => tx.txid === txid)?.time;
+                const wasSeen = audit.version === 1 ? !audit.unseenTxs.includes(txid) : (isExpected || isPrioritized || isAccelerated);
                 return {
-                  seen: isExpected || isPrioritized || isAccelerated,
+                  seen: wasSeen,
                   expected: isExpected,
-                  added: isAdded,
+                  added: isAdded && (audit.version === 0 || !wasSeen),
                   prioritized: isPrioritized,
                   conflict: isConflict,
                   accelerated: isAccelerated,

frontend/src/app/interfaces/node-api.interface.ts

@@ -211,6 +211,8 @@ export interface BlockExtended extends Block {
 }
 
 export interface BlockAudit extends BlockExtended {
+  version: number,
+  unseenTxs?: string[],
   missingTxs: string[],
   addedTxs: string[],
   prioritizedTxs: string[],
@@ -237,7 +239,7 @@ export interface TransactionStripped {
   acc?: boolean;
   flags?: number | null;
   time?: number;
-  status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'prioritized' | 'censored' | 'selected' | 'rbf' | 'accelerated';
+  status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'added_prioritized' | 'prioritized' | 'censored' | 'selected' | 'rbf' | 'accelerated';
   context?: 'projected' | 'actual';
 }