softsimon
GitHub
commit
2921c94520
@ -6,20 +6,22 @@ import rbfCache from './rbf-cache';
|
||||
const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners
|
||||
|
||||
class Audit {
|
||||
auditBlock(transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended }, useAccelerations: boolean = false)
|
||||
: { censored: string[], added: string[], prioritized: string[], fresh: string[], sigop: string[], fullrbf: string[], accelerated: string[], score: number, similarity: number } {
|
||||
auditBlock(height: number, transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended })
|
||||
: { unseen: string[], censored: string[], added: string[], prioritized: string[], fresh: string[], sigop: string[], fullrbf: string[], accelerated: string[], score: number, similarity: number } {
|
||||
if (!projectedBlocks?.[0]?.transactionIds || !mempool) {
|
||||
return { censored: [], added: [], prioritized: [], fresh: [], sigop: [], fullrbf: [], accelerated: [], score: 1, similarity: 1 };
|
||||
return { unseen: [], censored: [], added: [], prioritized: [], fresh: [], sigop: [], fullrbf: [], accelerated: [], score: 1, similarity: 1 };
|
||||
}
|
||||
|
||||
const matches: string[] = []; // present in both mined block and template
|
||||
const added: string[] = []; // present in mined block, not in template
|
||||
const prioritized: string[] = [] // present in the mined block, not in the template, but further down in the mempool
|
||||
const unseen: string[] = []; // present in the mined block, not in our mempool
|
||||
const prioritized: string[] = []; // higher in the block than would be expected by in-band feerate alone
|
||||
const fresh: string[] = []; // missing, but firstSeen or lastBoosted within PROPAGATION_MARGIN
|
||||
const rbf: string[] = []; // either missing or present, and either part of a full-rbf replacement, or a conflict with the mined block
|
||||
const accelerated: string[] = []; // prioritized by the mempool accelerator
|
||||
const isCensored = {}; // missing, without excuse
|
||||
const isDisplaced = {};
|
||||
const isAccelerated = {};
|
||||
let displacedWeight = 0;
|
||||
let matchedWeight = 0;
|
||||
let projectedWeight = 0;
|
||||
@ -32,6 +34,7 @@ class Audit {
|
||||
inBlock[tx.txid] = tx;
|
||||
if (mempool[tx.txid] && mempool[tx.txid].acceleration) {
|
||||
accelerated.push(tx.txid);
|
||||
isAccelerated[tx.txid] = true;
|
||||
}
|
||||
}
|
||||
// coinbase is always expected
|
||||
@ -113,11 +116,16 @@ class Audit {
|
||||
} else {
|
||||
if (rbfCache.has(tx.txid)) {
|
||||
rbf.push(tx.txid);
|
||||
} else if (!isDisplaced[tx.txid]) {
|
||||
if (!mempool[tx.txid] && !rbfCache.getReplacedBy(tx.txid)) {
|
||||
unseen.push(tx.txid);
|
||||
}
|
||||
} else {
|
||||
if (mempool[tx.txid]) {
|
||||
prioritized.push(tx.txid);
|
||||
if (isDisplaced[tx.txid]) {
|
||||
added.push(tx.txid);
|
||||
}
|
||||
} else {
|
||||
added.push(tx.txid);
|
||||
unseen.push(tx.txid);
|
||||
}
|
||||
}
|
||||
overflowWeight += tx.weight;
|
||||
@ -125,6 +133,24 @@ class Audit {
|
||||
totalWeight += tx.weight;
|
||||
}
|
||||
|
||||
|
||||
// identify "prioritized" transactions
|
||||
let lastEffectiveRate = 0;
|
||||
// Iterate over the mined template from bottom to top (excluding the coinbase)
|
||||
// Transactions should appear in ascending order of mining priority.
|
||||
for (let i = transactions.length - 1; i > 0; i--) {
|
||||
const blockTx = transactions[i];
|
||||
// If a tx has a lower in-band effective fee rate than the previous tx,
|
||||
// it must have been prioritized out-of-band (in order to have a higher mining priority)
|
||||
// so exclude from the analysis.
|
||||
if ((blockTx.effectiveFeePerVsize || 0) < lastEffectiveRate) {
|
||||
prioritized.push(blockTx.txid);
|
||||
// accelerated txs may or may not have their prioritized fee rate applied, so don't use them as a reference
|
||||
} else if (!isAccelerated[blockTx.txid]) {
|
||||
lastEffectiveRate = blockTx.effectiveFeePerVsize || 0;
|
||||
}
|
||||
}
|
||||
|
||||
// transactions missing from near the end of our template are probably not being censored
|
||||
let overflowWeightRemaining = overflowWeight - (config.MEMPOOL.BLOCK_WEIGHT_UNITS - totalWeight);
|
||||
let maxOverflowRate = 0;
|
||||
@ -165,6 +191,7 @@ class Audit {
|
||||
const similarity = projectedWeight ? matchedWeight / projectedWeight : 1;
|
||||
|
||||
return {
|
||||
unseen,
|
||||
censored: Object.keys(isCensored),
|
||||
added,
|
||||
prioritized,
|
||||
|
||||
@ -33,6 +33,7 @@ import AccelerationRepository from '../repositories/AccelerationRepository';
|
||||
import { calculateFastBlockCpfp, calculateGoodBlockCpfp } from './cpfp';
|
||||
import mempool from './mempool';
|
||||
import CpfpRepository from '../repositories/CpfpRepository';
|
||||
import accelerationApi from './services/acceleration';
|
||||
|
||||
class Blocks {
|
||||
private blocks: BlockExtended[] = [];
|
||||
@ -439,7 +440,7 @@ class Blocks {
|
||||
|
||||
|
||||
if (config.MEMPOOL.BACKEND === 'esplora') {
|
||||
const txs = (await bitcoinApi.$getTxsForBlock(block.hash)).map(tx => transactionUtils.extendTransaction(tx));
|
||||
const txs = (await bitcoinApi.$getTxsForBlock(block.hash)).map(tx => transactionUtils.extendMempoolTransaction(tx));
|
||||
const cpfpSummary = await this.$indexCPFP(block.hash, block.height, txs);
|
||||
if (cpfpSummary) {
|
||||
await this.$getStrippedBlockTransactions(block.hash, true, true, cpfpSummary, block.height); // This will index the block summary
|
||||
@ -904,7 +905,12 @@ class Blocks {
|
||||
}
|
||||
}
|
||||
|
||||
const cpfpSummary: CpfpSummary = calculateGoodBlockCpfp(block.height, transactions, Object.values(mempool.getAccelerations()).map(a => ({ txid: a.txid, max_bid: a.feeDelta })));
|
||||
let accelerations = Object.values(mempool.getAccelerations());
|
||||
if (accelerations?.length > 0) {
|
||||
const pool = await this.$findBlockMiner(transactionUtils.stripCoinbaseTransaction(transactions[0]));
|
||||
accelerations = accelerations.filter(a => a.pools.includes(pool.uniqueId));
|
||||
}
|
||||
const cpfpSummary: CpfpSummary = calculateGoodBlockCpfp(block.height, transactions, accelerations.map(a => ({ txid: a.txid, max_bid: a.feeDelta })));
|
||||
const blockExtended: BlockExtended = await this.$getBlockExtended(block, cpfpSummary.transactions);
|
||||
const blockSummary: BlockSummary = this.summarizeBlockTransactions(block.id, cpfpSummary.transactions);
|
||||
this.updateTimerProgress(timer, `got block data for ${this.currentBlockHeight}`);
|
||||
@ -927,12 +933,12 @@ class Blocks {
|
||||
const newBlock = await this.$indexBlock(lastBlock.height - i);
|
||||
this.blocks.push(newBlock);
|
||||
this.updateTimerProgress(timer, `reindexed block`);
|
||||
let cpfpSummary;
|
||||
let newCpfpSummary;
|
||||
if (config.MEMPOOL.CPFP_INDEXING) {
|
||||
cpfpSummary = await this.$indexCPFP(newBlock.id, lastBlock.height - i);
|
||||
newCpfpSummary = await this.$indexCPFP(newBlock.id, lastBlock.height - i);
|
||||
this.updateTimerProgress(timer, `reindexed block cpfp`);
|
||||
}
|
||||
await this.$getStrippedBlockTransactions(newBlock.id, true, true, cpfpSummary, newBlock.height);
|
||||
await this.$getStrippedBlockTransactions(newBlock.id, true, true, newCpfpSummary, newBlock.height);
|
||||
this.updateTimerProgress(timer, `reindexed block summary`);
|
||||
}
|
||||
await mining.$indexDifficultyAdjustments();
|
||||
@ -981,7 +987,7 @@ class Blocks {
|
||||
|
||||
// start async callbacks
|
||||
this.updateTimerProgress(timer, `starting async callbacks for ${this.currentBlockHeight}`);
|
||||
const callbackPromises = this.newAsyncBlockCallbacks.map((cb) => cb(blockExtended, txIds, transactions));
|
||||
const callbackPromises = this.newAsyncBlockCallbacks.map((cb) => cb(blockExtended, txIds, cpfpSummary.transactions));
|
||||
|
||||
if (block.height % 2016 === 0) {
|
||||
if (Common.indexingEnabled()) {
|
||||
@ -1178,7 +1184,7 @@ class Blocks {
|
||||
};
|
||||
}),
|
||||
};
|
||||
summaryVersion = 1;
|
||||
summaryVersion = cpfpSummary.version;
|
||||
} else {
|
||||
if (config.MEMPOOL.BACKEND === 'esplora') {
|
||||
const txs = (await bitcoinApi.$getTxsForBlock(hash)).map(tx => transactionUtils.extendTransaction(tx));
|
||||
@ -1397,11 +1403,11 @@ class Blocks {
|
||||
return this.currentBlockHeight;
|
||||
}
|
||||
|
||||
public async $indexCPFP(hash: string, height: number, txs?: TransactionExtended[]): Promise<CpfpSummary | null> {
|
||||
public async $indexCPFP(hash: string, height: number, txs?: MempoolTransactionExtended[]): Promise<CpfpSummary | null> {
|
||||
let transactions = txs;
|
||||
if (!transactions) {
|
||||
if (config.MEMPOOL.BACKEND === 'esplora') {
|
||||
transactions = (await bitcoinApi.$getTxsForBlock(hash)).map(tx => transactionUtils.extendTransaction(tx));
|
||||
transactions = (await bitcoinApi.$getTxsForBlock(hash)).map(tx => transactionUtils.extendMempoolTransaction(tx));
|
||||
}
|
||||
if (!transactions) {
|
||||
const block = await bitcoinClient.getBlock(hash, 2);
|
||||
@ -1413,7 +1419,7 @@ class Blocks {
|
||||
}
|
||||
|
||||
if (transactions?.length != null) {
|
||||
const summary = calculateFastBlockCpfp(height, transactions as TransactionExtended[]);
|
||||
const summary = calculateFastBlockCpfp(height, transactions);
|
||||
|
||||
await this.$saveCpfp(hash, height, summary);
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
import * as bitcoinjs from 'bitcoinjs-lib';
|
||||
import { Request } from 'express';
|
||||
import { CpfpInfo, CpfpSummary, CpfpCluster, EffectiveFeeStats, MempoolBlockWithTransactions, TransactionExtended, MempoolTransactionExtended, TransactionStripped, WorkingEffectiveFeeStats, TransactionClassified, TransactionFlags } from '../mempool.interfaces';
|
||||
import { EffectiveFeeStats, MempoolBlockWithTransactions, TransactionExtended, MempoolTransactionExtended, TransactionStripped, WorkingEffectiveFeeStats, TransactionClassified, TransactionFlags } from '../mempool.interfaces';
|
||||
import config from '../config';
|
||||
import { NodeSocket } from '../repositories/NodesSocketsRepository';
|
||||
import { isIP } from 'net';
|
||||
|
||||
@ -6,7 +6,7 @@ import { Acceleration } from './acceleration/acceleration';
|
||||
const CPFP_UPDATE_INTERVAL = 60_000; // update CPFP info at most once per 60s per transaction
|
||||
const MAX_CLUSTER_ITERATIONS = 100;
|
||||
|
||||
export function calculateFastBlockCpfp(height: number, transactions: TransactionExtended[], saveRelatives: boolean = false): CpfpSummary {
|
||||
export function calculateFastBlockCpfp(height: number, transactions: MempoolTransactionExtended[], saveRelatives: boolean = false): CpfpSummary {
|
||||
const clusters: CpfpCluster[] = []; // list of all cpfp clusters in this block
|
||||
const clusterMap: { [txid: string]: CpfpCluster } = {}; // map transactions to their cpfp cluster
|
||||
let clusterTxs: TransactionExtended[] = []; // working list of elements of the current cluster
|
||||
@ -93,6 +93,7 @@ export function calculateFastBlockCpfp(height: number, transactions: Transaction
|
||||
return {
|
||||
transactions,
|
||||
clusters,
|
||||
version: 1,
|
||||
};
|
||||
}
|
||||
|
||||
@ -159,6 +160,7 @@ export function calculateGoodBlockCpfp(height: number, transactions: MempoolTran
|
||||
return {
|
||||
transactions: transactions.map(tx => txMap[tx.txid]),
|
||||
clusters: clusterArray,
|
||||
version: 2,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
|
||||
import { RowDataPacket } from 'mysql2';
|
||||
|
||||
class DatabaseMigration {
|
||||
private static currentVersion = 80;
|
||||
private static currentVersion = 81;
|
||||
private queryTimeout = 3600_000;
|
||||
private statisticsAddedIndexed = false;
|
||||
private uniqueLogs: string[] = [];
|
||||
@ -691,6 +691,13 @@ class DatabaseMigration {
|
||||
await this.$executeQuery('ALTER TABLE `blocks` ADD coinbase_addresses JSON DEFAULT NULL');
|
||||
await this.updateToSchemaVersion(80);
|
||||
}
|
||||
|
||||
if (databaseSchemaVersion < 81) {
|
||||
await this.$executeQuery('ALTER TABLE `blocks_audits` ADD version INT NOT NULL DEFAULT 0');
|
||||
await this.$executeQuery('ALTER TABLE `blocks_audits` ADD INDEX `version` (`version`)');
|
||||
await this.$executeQuery('ALTER TABLE `blocks_audits` ADD unseen_txs JSON DEFAULT "[]"');
|
||||
await this.updateToSchemaVersion(81);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@ -337,7 +337,7 @@ export function makeBlockTemplate(candidates: MempoolTransactionExtended[], acce
|
||||
let failures = 0;
|
||||
while (mempoolArray.length || modified.length) {
|
||||
// skip invalid transactions
|
||||
while (mempoolArray[0].used || mempoolArray[0].modified) {
|
||||
while (mempoolArray[0]?.used || mempoolArray[0]?.modified) {
|
||||
mempoolArray.shift();
|
||||
}
|
||||
|
||||
|
||||
@ -3,7 +3,7 @@ import * as WebSocket from 'ws';
|
||||
import {
|
||||
BlockExtended, TransactionExtended, MempoolTransactionExtended, WebsocketResponse,
|
||||
OptimizedStatistic, ILoadingIndicators, GbtCandidates, TxTrackingInfo,
|
||||
MempoolBlockDelta, MempoolDelta, MempoolDeltaTxids
|
||||
MempoolDelta, MempoolDeltaTxids
|
||||
} from '../mempool.interfaces';
|
||||
import blocks from './blocks';
|
||||
import memPool from './mempool';
|
||||
@ -933,6 +933,8 @@ class WebsocketHandler {
|
||||
throw new Error('No WebSocket.Server have been set');
|
||||
}
|
||||
|
||||
const blockTransactions = structuredClone(transactions);
|
||||
|
||||
this.printLogs();
|
||||
await statistics.runStatistics();
|
||||
|
||||
@ -942,7 +944,7 @@ class WebsocketHandler {
|
||||
let transactionIds: string[] = (memPool.limitGBT) ? Object.keys(candidates?.txs || {}) : Object.keys(_memPool);
|
||||
|
||||
const accelerations = Object.values(mempool.getAccelerations());
|
||||
await accelerationRepository.$indexAccelerationsForBlock(block, accelerations, transactions);
|
||||
await accelerationRepository.$indexAccelerationsForBlock(block, accelerations, structuredClone(transactions));
|
||||
|
||||
const rbfTransactions = Common.findMinedRbfTransactions(transactions, memPool.getSpendMap());
|
||||
memPool.handleMinedRbfTransactions(rbfTransactions);
|
||||
@ -962,7 +964,7 @@ class WebsocketHandler {
|
||||
}
|
||||
|
||||
if (Common.indexingEnabled()) {
|
||||
const { censored, added, prioritized, fresh, sigop, fullrbf, accelerated, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool);
|
||||
const { unseen, censored, added, prioritized, fresh, sigop, fullrbf, accelerated, score, similarity } = Audit.auditBlock(block.height, blockTransactions, projectedBlocks, auditMempool);
|
||||
const matchRate = Math.round(score * 100 * 100) / 100;
|
||||
|
||||
const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions : [];
|
||||
@ -984,9 +986,11 @@ class WebsocketHandler {
|
||||
});
|
||||
|
||||
BlocksAuditsRepository.$saveAudit({
|
||||
version: 1,
|
||||
time: block.timestamp,
|
||||
height: block.height,
|
||||
hash: block.id,
|
||||
unseenTxs: unseen,
|
||||
addedTxs: added,
|
||||
prioritizedTxs: prioritized,
|
||||
missingTxs: censored,
|
||||
|
||||
@ -10,6 +10,7 @@ import config from './config';
|
||||
import auditReplicator from './replication/AuditReplication';
|
||||
import statisticsReplicator from './replication/StatisticsReplication';
|
||||
import AccelerationRepository from './repositories/AccelerationRepository';
|
||||
import BlocksAuditsRepository from './repositories/BlocksAuditsRepository';
|
||||
|
||||
export interface CoreIndex {
|
||||
name: string;
|
||||
@ -192,6 +193,7 @@ class Indexer {
|
||||
await auditReplicator.$sync();
|
||||
await statisticsReplicator.$sync();
|
||||
await AccelerationRepository.$indexPastAccelerations();
|
||||
await BlocksAuditsRepository.$migrateAuditsV0toV1();
|
||||
// do not wait for classify blocks to finish
|
||||
blocks.$classifyBlocks();
|
||||
} catch (e) {
|
||||
|
||||
@ -29,9 +29,11 @@ export interface PoolStats extends PoolInfo {
|
||||
}
|
||||
|
||||
export interface BlockAudit {
|
||||
version: number,
|
||||
time: number,
|
||||
height: number,
|
||||
hash: string,
|
||||
unseenTxs: string[],
|
||||
missingTxs: string[],
|
||||
freshTxs: string[],
|
||||
sigopTxs: string[],
|
||||
@ -383,8 +385,9 @@ export interface CpfpCluster {
|
||||
}
|
||||
|
||||
export interface CpfpSummary {
|
||||
transactions: TransactionExtended[];
|
||||
transactions: MempoolTransactionExtended[];
|
||||
clusters: CpfpCluster[];
|
||||
version: number;
|
||||
}
|
||||
|
||||
export interface Statistic {
|
||||
|
||||
@ -31,11 +31,11 @@ class AuditReplication {
|
||||
const missingAudits = await this.$getMissingAuditBlocks();
|
||||
|
||||
logger.debug(`Fetching missing audit data for ${missingAudits.length} blocks from trusted servers`, 'Replication');
|
||||
|
||||
|
||||
let totalSynced = 0;
|
||||
let totalMissed = 0;
|
||||
let loggerTimer = Date.now();
|
||||
// process missing audits in batches of
|
||||
// process missing audits in batches of BATCH_SIZE
|
||||
for (let i = 0; i < missingAudits.length; i += BATCH_SIZE) {
|
||||
const slice = missingAudits.slice(i, i + BATCH_SIZE);
|
||||
const results = await Promise.all(slice.map(hash => this.$syncAudit(hash)));
|
||||
@ -109,9 +109,11 @@ class AuditReplication {
|
||||
version: 1,
|
||||
});
|
||||
await blocksAuditsRepository.$saveAudit({
|
||||
version: auditSummary.version || 0,
|
||||
hash: blockHash,
|
||||
height: auditSummary.height,
|
||||
time: auditSummary.timestamp || auditSummary.time,
|
||||
unseenTxs: auditSummary.unseenTxs || [],
|
||||
missingTxs: auditSummary.missingTxs || [],
|
||||
addedTxs: auditSummary.addedTxs || [],
|
||||
prioritizedTxs: auditSummary.prioritizedTxs || [],
|
||||
|
||||
@ -192,6 +192,7 @@ class AccelerationRepository {
|
||||
}
|
||||
}
|
||||
|
||||
// modifies block transactions
|
||||
public async $indexAccelerationsForBlock(block: BlockExtended, accelerations: Acceleration[], transactions: MempoolTransactionExtended[]): Promise<void> {
|
||||
const blockTxs: { [txid: string]: MempoolTransactionExtended } = {};
|
||||
for (const tx of transactions) {
|
||||
|
||||
@ -1,13 +1,24 @@
|
||||
import blocks from '../api/blocks';
|
||||
import DB from '../database';
|
||||
import logger from '../logger';
|
||||
import { BlockAudit, AuditScore, TransactionAudit } from '../mempool.interfaces';
|
||||
import bitcoinApi from '../api/bitcoin/bitcoin-api-factory';
|
||||
import { BlockAudit, AuditScore, TransactionAudit, TransactionStripped } from '../mempool.interfaces';
|
||||
|
||||
interface MigrationAudit {
|
||||
version: number,
|
||||
height: number,
|
||||
id: string,
|
||||
timestamp: number,
|
||||
prioritizedTxs: string[],
|
||||
acceleratedTxs: string[],
|
||||
template: TransactionStripped[],
|
||||
transactions: TransactionStripped[],
|
||||
}
|
||||
|
||||
class BlocksAuditRepositories {
|
||||
public async $saveAudit(audit: BlockAudit): Promise<void> {
|
||||
try {
|
||||
await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, prioritized_txs, fresh_txs, sigop_txs, fullrbf_txs, accelerated_txs, match_rate, expected_fees, expected_weight)
|
||||
VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs),
|
||||
await DB.query(`INSERT INTO blocks_audits(version, time, height, hash, unseen_txs, missing_txs, added_txs, prioritized_txs, fresh_txs, sigop_txs, fullrbf_txs, accelerated_txs, match_rate, expected_fees, expected_weight)
|
||||
VALUE (?, FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [audit.version, audit.time, audit.height, audit.hash, JSON.stringify(audit.unseenTxs), JSON.stringify(audit.missingTxs),
|
||||
JSON.stringify(audit.addedTxs), JSON.stringify(audit.prioritizedTxs), JSON.stringify(audit.freshTxs), JSON.stringify(audit.sigopTxs), JSON.stringify(audit.fullrbfTxs), JSON.stringify(audit.acceleratedTxs), audit.matchRate, audit.expectedFees, audit.expectedWeight]);
|
||||
} catch (e: any) {
|
||||
if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
|
||||
@ -62,24 +73,30 @@ class BlocksAuditRepositories {
|
||||
public async $getBlockAudit(hash: string): Promise<BlockAudit | null> {
|
||||
try {
|
||||
const [rows]: any[] = await DB.query(
|
||||
`SELECT blocks_audits.height, blocks_audits.hash as id, UNIX_TIMESTAMP(blocks_audits.time) as timestamp,
|
||||
template,
|
||||
missing_txs as missingTxs,
|
||||
added_txs as addedTxs,
|
||||
prioritized_txs as prioritizedTxs,
|
||||
fresh_txs as freshTxs,
|
||||
sigop_txs as sigopTxs,
|
||||
fullrbf_txs as fullrbfTxs,
|
||||
accelerated_txs as acceleratedTxs,
|
||||
match_rate as matchRate,
|
||||
expected_fees as expectedFees,
|
||||
expected_weight as expectedWeight
|
||||
`SELECT
|
||||
blocks_audits.version,
|
||||
blocks_audits.height,
|
||||
blocks_audits.hash as id,
|
||||
UNIX_TIMESTAMP(blocks_audits.time) as timestamp,
|
||||
template,
|
||||
unseen_txs as unseenTxs,
|
||||
missing_txs as missingTxs,
|
||||
added_txs as addedTxs,
|
||||
prioritized_txs as prioritizedTxs,
|
||||
fresh_txs as freshTxs,
|
||||
sigop_txs as sigopTxs,
|
||||
fullrbf_txs as fullrbfTxs,
|
||||
accelerated_txs as acceleratedTxs,
|
||||
match_rate as matchRate,
|
||||
expected_fees as expectedFees,
|
||||
expected_weight as expectedWeight
|
||||
FROM blocks_audits
|
||||
JOIN blocks_templates ON blocks_templates.id = blocks_audits.hash
|
||||
WHERE blocks_audits.hash = ?
|
||||
`, [hash]);
|
||||
|
||||
if (rows.length) {
|
||||
rows[0].unseenTxs = JSON.parse(rows[0].unseenTxs);
|
||||
rows[0].missingTxs = JSON.parse(rows[0].missingTxs);
|
||||
rows[0].addedTxs = JSON.parse(rows[0].addedTxs);
|
||||
rows[0].prioritizedTxs = JSON.parse(rows[0].prioritizedTxs);
|
||||
@ -101,7 +118,7 @@ class BlocksAuditRepositories {
|
||||
public async $getBlockTxAudit(hash: string, txid: string): Promise<TransactionAudit | null> {
|
||||
try {
|
||||
const blockAudit = await this.$getBlockAudit(hash);
|
||||
|
||||
|
||||
if (blockAudit) {
|
||||
const isAdded = blockAudit.addedTxs.includes(txid);
|
||||
const isPrioritized = blockAudit.prioritizedTxs.includes(txid);
|
||||
@ -124,7 +141,7 @@ class BlocksAuditRepositories {
|
||||
conflict: isConflict,
|
||||
accelerated: isAccelerated,
|
||||
firstSeen,
|
||||
}
|
||||
};
|
||||
}
|
||||
return null;
|
||||
} catch (e: any) {
|
||||
@ -186,6 +203,96 @@ class BlocksAuditRepositories {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* [INDEXING] Migrate audits from v0 to v1
|
||||
*/
|
||||
public async $migrateAuditsV0toV1(): Promise<void> {
|
||||
try {
|
||||
let done = false;
|
||||
let processed = 0;
|
||||
let lastHeight;
|
||||
while (!done) {
|
||||
const [toMigrate]: MigrationAudit[][] = await DB.query(
|
||||
`SELECT
|
||||
blocks_audits.height as height,
|
||||
blocks_audits.hash as id,
|
||||
UNIX_TIMESTAMP(blocks_audits.time) as timestamp,
|
||||
blocks_summaries.transactions as transactions,
|
||||
blocks_templates.template as template,
|
||||
blocks_audits.prioritized_txs as prioritizedTxs,
|
||||
blocks_audits.accelerated_txs as acceleratedTxs
|
||||
FROM blocks_audits
|
||||
JOIN blocks_summaries ON blocks_summaries.id = blocks_audits.hash
|
||||
JOIN blocks_templates ON blocks_templates.id = blocks_audits.hash
|
||||
WHERE blocks_audits.version = 0
|
||||
AND blocks_summaries.version = 2
|
||||
ORDER BY blocks_audits.height DESC
|
||||
LIMIT 100
|
||||
`) as any[];
|
||||
|
||||
if (toMigrate.length <= 0 || lastHeight === toMigrate[0].height) {
|
||||
done = true;
|
||||
break;
|
||||
}
|
||||
lastHeight = toMigrate[0].height;
|
||||
|
||||
logger.info(`migrating ${toMigrate.length} audits to version 1`);
|
||||
|
||||
for (const audit of toMigrate) {
|
||||
// unpack JSON-serialized transaction lists
|
||||
audit.transactions = JSON.parse((audit.transactions as any as string) || '[]');
|
||||
audit.template = JSON.parse((audit.template as any as string) || '[]');
|
||||
|
||||
// we know transactions in the template, or marked "prioritized" or "accelerated"
|
||||
// were seen in our mempool before the block was mined.
|
||||
const isSeen = new Set<string>();
|
||||
for (const tx of audit.template) {
|
||||
isSeen.add(tx.txid);
|
||||
}
|
||||
for (const txid of audit.prioritizedTxs) {
|
||||
isSeen.add(txid);
|
||||
}
|
||||
for (const txid of audit.acceleratedTxs) {
|
||||
isSeen.add(txid);
|
||||
}
|
||||
const unseenTxs = audit.transactions.slice(0).map(tx => tx.txid).filter(txid => !isSeen.has(txid));
|
||||
|
||||
// identify "prioritized" transactions
|
||||
const prioritizedTxs: string[] = [];
|
||||
let lastEffectiveRate = 0;
|
||||
// Iterate over the mined template from bottom to top (excluding the coinbase)
|
||||
// Transactions should appear in ascending order of mining priority.
|
||||
for (let i = audit.transactions.length - 1; i > 0; i--) {
|
||||
const blockTx = audit.transactions[i];
|
||||
// If a tx has a lower in-band effective fee rate than the previous tx,
|
||||
// it must have been prioritized out-of-band (in order to have a higher mining priority)
|
||||
// so exclude from the analysis.
|
||||
if ((blockTx.rate || 0) < lastEffectiveRate) {
|
||||
prioritizedTxs.push(blockTx.txid);
|
||||
} else {
|
||||
lastEffectiveRate = blockTx.rate || 0;
|
||||
}
|
||||
}
|
||||
|
||||
// Update audit in the database
|
||||
await DB.query(`
|
||||
UPDATE blocks_audits SET
|
||||
version = ?,
|
||||
unseen_txs = ?,
|
||||
prioritized_txs = ?
|
||||
WHERE hash = ?
|
||||
`, [1, JSON.stringify(unseenTxs), JSON.stringify(prioritizedTxs), audit.id]);
|
||||
}
|
||||
|
||||
processed += toMigrate.length;
|
||||
}
|
||||
|
||||
logger.info(`migrated ${processed} audits to version 1`);
|
||||
} catch (e: any) {
|
||||
logger.err(`Error while migrating audits from v0 to v1. Will try again later. Reason: ` + (e instanceof Error ? e.message : e));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default new BlocksAuditRepositories();
|
||||
|
||||
@ -18,6 +18,7 @@ const unmatchedAuditColors = {
|
||||
censored: setOpacity(defaultAuditColors.censored, unmatchedOpacity),
|
||||
missing: setOpacity(defaultAuditColors.missing, unmatchedOpacity),
|
||||
added: setOpacity(defaultAuditColors.added, unmatchedOpacity),
|
||||
added_prioritized: setOpacity(defaultAuditColors.added_prioritized, unmatchedOpacity),
|
||||
prioritized: setOpacity(defaultAuditColors.prioritized, unmatchedOpacity),
|
||||
accelerated: setOpacity(defaultAuditColors.accelerated, unmatchedOpacity),
|
||||
};
|
||||
@ -25,6 +26,7 @@ const unmatchedContrastAuditColors = {
|
||||
censored: setOpacity(contrastAuditColors.censored, unmatchedOpacity),
|
||||
missing: setOpacity(contrastAuditColors.missing, unmatchedOpacity),
|
||||
added: setOpacity(contrastAuditColors.added, unmatchedOpacity),
|
||||
added_prioritized: setOpacity(contrastAuditColors.added_prioritized, unmatchedOpacity),
|
||||
prioritized: setOpacity(contrastAuditColors.prioritized, unmatchedOpacity),
|
||||
accelerated: setOpacity(contrastAuditColors.accelerated, unmatchedOpacity),
|
||||
};
|
||||
|
||||
@ -33,7 +33,7 @@ export default class TxView implements TransactionStripped {
|
||||
flags: number;
|
||||
bigintFlags?: bigint | null = 0b00000100_00000000_00000000_00000000n;
|
||||
time?: number;
|
||||
status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'prioritized' | 'censored' | 'selected' | 'rbf' | 'accelerated';
|
||||
status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'added_prioritized' | 'prioritized' | 'censored' | 'selected' | 'rbf' | 'accelerated';
|
||||
context?: 'projected' | 'actual';
|
||||
scene?: BlockScene;
|
||||
|
||||
|
||||
@ -71,6 +71,7 @@ export const defaultAuditColors = {
|
||||
censored: hexToColor('f344df'),
|
||||
missing: darken(desaturate(hexToColor('f344df'), 0.3), 0.7),
|
||||
added: hexToColor('0099ff'),
|
||||
added_prioritized: darken(desaturate(hexToColor('0099ff'), 0.15), 0.85),
|
||||
prioritized: darken(desaturate(hexToColor('0099ff'), 0.3), 0.7),
|
||||
accelerated: hexToColor('8f5ff6'),
|
||||
};
|
||||
@ -101,6 +102,7 @@ export const contrastAuditColors = {
|
||||
censored: hexToColor('ffa8ff'),
|
||||
missing: darken(desaturate(hexToColor('ffa8ff'), 0.3), 0.7),
|
||||
added: hexToColor('00bb98'),
|
||||
added_prioritized: darken(desaturate(hexToColor('00bb98'), 0.15), 0.85),
|
||||
prioritized: darken(desaturate(hexToColor('00bb98'), 0.3), 0.7),
|
||||
accelerated: hexToColor('8f5ff6'),
|
||||
};
|
||||
@ -136,6 +138,8 @@ export function defaultColorFunction(
|
||||
return auditColors.missing;
|
||||
case 'added':
|
||||
return auditColors.added;
|
||||
case 'added_prioritized':
|
||||
return auditColors.added_prioritized;
|
||||
case 'prioritized':
|
||||
return auditColors.prioritized;
|
||||
case 'selected':
|
||||
|
||||
@ -75,6 +75,10 @@
|
||||
<span *ngSwitchCase="'freshcpfp'" class="badge badge-warning" i18n="transaction.audit.recently-cpfped">Recently CPFP'd</span>
|
||||
<span *ngSwitchCase="'added'" class="badge badge-warning" i18n="tx-features.tag.added|Added">Added</span>
|
||||
<span *ngSwitchCase="'prioritized'" class="badge badge-warning" i18n="tx-features.tag.prioritized|Prioritized">Prioritized</span>
|
||||
<ng-container *ngSwitchCase="'added_prioritized'">
|
||||
<span class="badge badge-warning" i18n="tx-features.tag.added|Added">Added</span>
|
||||
<span class="badge badge-warning ml-1" i18n="tx-features.tag.prioritized|Prioritized">Prioritized</span>
|
||||
</ng-container>
|
||||
<span *ngSwitchCase="'selected'" class="badge badge-warning" i18n="transaction.audit.marginal">Marginal fee rate</span>
|
||||
<span *ngSwitchCase="'rbf'" class="badge badge-warning" i18n="tx-features.tag.conflict|Conflict">Conflict</span>
|
||||
<span *ngSwitchCase="'accelerated'" class="badge badge-accelerated" i18n="transaction.audit.accelerated">Accelerated</span>
|
||||
|
||||
@ -521,6 +521,7 @@ export class BlockComponent implements OnInit, OnDestroy {
|
||||
if (transactions && blockAudit) {
|
||||
const inTemplate = {};
|
||||
const inBlock = {};
|
||||
const isUnseen = {};
|
||||
const isAdded = {};
|
||||
const isPrioritized = {};
|
||||
const isCensored = {};
|
||||
@ -543,6 +544,9 @@ export class BlockComponent implements OnInit, OnDestroy {
|
||||
for (const tx of transactions) {
|
||||
inBlock[tx.txid] = true;
|
||||
}
|
||||
for (const txid of blockAudit.unseenTxs || []) {
|
||||
isUnseen[txid] = true;
|
||||
}
|
||||
for (const txid of blockAudit.addedTxs) {
|
||||
isAdded[txid] = true;
|
||||
}
|
||||
@ -592,18 +596,27 @@ export class BlockComponent implements OnInit, OnDestroy {
|
||||
tx.status = 'accelerated';
|
||||
}
|
||||
}
|
||||
for (const [index, tx] of transactions.entries()) {
|
||||
let anySeen = false;
|
||||
for (let index = transactions.length - 1; index >= 0; index--) {
|
||||
const tx = transactions[index];
|
||||
tx.context = 'actual';
|
||||
if (index === 0) {
|
||||
tx.status = null;
|
||||
} else if (isAdded[tx.txid]) {
|
||||
tx.status = 'added';
|
||||
} else if (isPrioritized[tx.txid]) {
|
||||
tx.status = 'prioritized';
|
||||
if (isAdded[tx.txid] || (blockAudit.version > 0 && isUnseen[tx.txid])) {
|
||||
tx.status = 'added_prioritized';
|
||||
} else {
|
||||
tx.status = 'prioritized';
|
||||
}
|
||||
} else if (isAdded[tx.txid] && (blockAudit.version === 0 || isUnseen[tx.txid])) {
|
||||
tx.status = 'added';
|
||||
} else if (inTemplate[tx.txid]) {
|
||||
anySeen = true;
|
||||
tx.status = 'found';
|
||||
} else if (isRbf[tx.txid]) {
|
||||
tx.status = 'rbf';
|
||||
} else if (isUnseen[tx.txid] && anySeen) {
|
||||
tx.status = 'added';
|
||||
} else {
|
||||
tx.status = 'selected';
|
||||
isSelected[tx.txid] = true;
|
||||
|
||||
@ -411,10 +411,11 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
|
||||
const isConflict = audit.fullrbfTxs.includes(txid);
|
||||
const isExpected = audit.template.some(tx => tx.txid === txid);
|
||||
const firstSeen = audit.template.find(tx => tx.txid === txid)?.time;
|
||||
const wasSeen = audit.version === 1 ? !audit.unseenTxs.includes(txid) : (isExpected || isPrioritized || isAccelerated);
|
||||
return {
|
||||
seen: isExpected || isPrioritized || isAccelerated,
|
||||
seen: wasSeen,
|
||||
expected: isExpected,
|
||||
added: isAdded,
|
||||
added: isAdded && (audit.version === 0 || !wasSeen),
|
||||
prioritized: isPrioritized,
|
||||
conflict: isConflict,
|
||||
accelerated: isAccelerated,
|
||||
|
||||
@ -211,6 +211,8 @@ export interface BlockExtended extends Block {
|
||||
}
|
||||
|
||||
export interface BlockAudit extends BlockExtended {
|
||||
version: number,
|
||||
unseenTxs?: string[],
|
||||
missingTxs: string[],
|
||||
addedTxs: string[],
|
||||
prioritizedTxs: string[],
|
||||
@ -237,7 +239,7 @@ export interface TransactionStripped {
|
||||
acc?: boolean;
|
||||
flags?: number | null;
|
||||
time?: number;
|
||||
status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'prioritized' | 'censored' | 'selected' | 'rbf' | 'accelerated';
|
||||
status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'added_prioritized' | 'prioritized' | 'censored' | 'selected' | 'rbf' | 'accelerated';
|
||||
context?: 'projected' | 'actual';
|
||||
}
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user