machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #4852 from mempool/mononaut/better-audits

Browse Source 
Better audits
This commit is contained in:
softsimon
2024-04-01 22:47:46 +09:00
committed by GitHub
parent 2048816c23 03255dd077
commit 10d4e5a600
30 changed files with 158 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
backend/src/api/audit.ts
View File

@@ -7,13 +7,14 @@ const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first
class Audit {
auditBlock(transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended }, useAccelerations: boolean = false)
: { censored: string[], added: string[], fresh: string[], sigop: string[], fullrbf: string[], accelerated: string[], score: number, similarity: number } {
: { censored: string[], added: string[], prioritized: string[], fresh: string[], sigop: string[], fullrbf: string[], accelerated: string[], score: number, similarity: number } {
if (!projectedBlocks?.[0]?.transactionIds || !mempool) {
return { censored: [], added: [], fresh: [], sigop: [], fullrbf: [], accelerated: [], score: 1, similarity: 1 };
return { censored: [], added: [], prioritized: [], fresh: [], sigop: [], fullrbf: [], accelerated: [], score: 1, similarity: 1 };
}
const matches: string[] = []; // present in both mined block and template
const added: string[] = []; // present in mined block, not in template
const prioritized: string[] = [] // present in the mined block, not in the template, but further down in the mempool
const fresh: string[] = []; // missing, but firstSeen or lastBoosted within PROPAGATION_MARGIN
const rbf: string[] = []; // either missing or present, and either part of a full-rbf replacement, or a conflict with the mined block
const accelerated: string[] = []; // prioritized by the mempool accelerator
@@ -68,20 +69,27 @@ class Audit {
// we can expect an honest miner to include 'displaced' transactions in place of recent arrivals and censored txs
// these displaced transactions should occupy the first N weight units of the next projected block
let displacedWeightRemaining = displacedWeight;
let displacedWeightRemaining = displacedWeight + 4000;
let index = 0;
let lastFeeRate = Infinity;
let failures = 0;
while (projectedBlocks[1] && index < projectedBlocks[1].transactionIds.length && failures < 500) {
const txid = projectedBlocks[1].transactionIds[index];
let blockIndex = 1;
while (projectedBlocks[blockIndex] && failures < 500) {
if (index >= projectedBlocks[blockIndex].transactionIds.length) {
index = 0;
blockIndex++;
}
const txid = projectedBlocks[blockIndex].transactionIds[index];
const tx = mempool[txid];
if (tx) {
const fits = (tx.weight - displacedWeightRemaining) < 4000;
const feeMatches = tx.effectiveFeePerVsize >= lastFeeRate;
// 0.005 margin of error for any remaining vsize rounding issues
const feeMatches = tx.effectiveFeePerVsize >= (lastFeeRate - 0.005);
if (fits || feeMatches) {
isDisplaced[txid] = true;
if (fits) {
lastFeeRate = Math.min(lastFeeRate, tx.effectiveFeePerVsize);
// (tx.effectiveFeePerVsize * tx.vsize) / Math.ceil(tx.vsize) attempts to correct for vsize rounding in the simple non-CPFP case
lastFeeRate = Math.min(lastFeeRate, (tx.effectiveFeePerVsize * tx.vsize) / Math.ceil(tx.vsize));
}
if (tx.firstSeen == null || (now - (tx?.firstSeen || 0)) > PROPAGATION_MARGIN) {
displacedWeightRemaining -= tx.weight;
@@ -106,7 +114,11 @@ class Audit {
if (rbfCache.has(tx.txid)) {
rbf.push(tx.txid);
} else if (!isDisplaced[tx.txid]) {
added.push(tx.txid);
if (mempool[tx.txid]) {
prioritized.push(tx.txid);
} else {
added.push(tx.txid);
}
}
overflowWeight += tx.weight;
}
@@ -155,6 +167,7 @@ class Audit {
return {
censored: Object.keys(isCensored),
added,
prioritized,
fresh,
sigop: [],
fullrbf: rbf,

1
backend/src/api/common.ts
View File

@@ -552,6 +552,7 @@ export class Common {
value: tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0),
acc: tx.acceleration || undefined,
rate: tx.effectiveFeePerVsize,
time: tx.firstSeen || undefined,
};
}

7
backend/src/api/database-migration.ts
View File

@@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
import { RowDataPacket } from 'mysql2';
class DatabaseMigration {
private static currentVersion = 75;
private static currentVersion = 76;
private queryTimeout = 3600_000;
private statisticsAddedIndexed = false;
private uniqueLogs: string[] = [];
@@ -654,6 +654,11 @@ class DatabaseMigration {
await this.$executeQuery('ALTER TABLE `prices` ADD `ZAR` float DEFAULT "-1"');
await this.updateToSchemaVersion(75);
}
if (databaseSchemaVersion < 76 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `blocks_audits` ADD prioritized_txs JSON DEFAULT "[]"');
await this.updateToSchemaVersion(76);
}
}
/**

4
backend/src/api/mempool-blocks.ts
View File

@@ -598,7 +598,8 @@ class MempoolBlocks {
tx.value,
Math.round((tx.rate || (tx.fee / tx.vsize)) * 100) / 100,
tx.flags,
1
tx.time || 0,
1,
];
} else {
return [
@@ -608,6 +609,7 @@ class MempoolBlocks {
tx.value,
Math.round((tx.rate || (tx.fee / tx.vsize)) * 100) / 100,
tx.flags,
tx.time || 0,
];
}
}

3
backend/src/api/websocket-handler.ts
View File

@@ -869,7 +869,7 @@ class WebsocketHandler {
}
if (Common.indexingEnabled()) {
const { censored, added, fresh, sigop, fullrbf, accelerated, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool);
const { censored, added, prioritized, fresh, sigop, fullrbf, accelerated, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool);
const matchRate = Math.round(score * 100 * 100) / 100;
const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions : [];
@@ -895,6 +895,7 @@ class WebsocketHandler {
height: block.height,
hash: block.id,
addedTxs: added,
prioritizedTxs: prioritized,
missingTxs: censored,
freshTxs: fresh,
sigopTxs: sigop,

4
backend/src/mempool.interfaces.ts
View File

@@ -37,6 +37,7 @@ export interface BlockAudit {
sigopTxs: string[],
fullrbfTxs: string[],
addedTxs: string[],
prioritizedTxs: string[],
acceleratedTxs: string[],
matchRate: number,
expectedFees?: number,
@@ -200,6 +201,7 @@ export interface TransactionStripped {
value: number;
acc?: boolean;
rate?: number; // effective fee rate
time?: number;
}
export interface TransactionClassified extends TransactionStripped {
@@ -207,7 +209,7 @@ export interface TransactionClassified extends TransactionStripped {
}
// [txid, fee, vsize, value, rate, flags, acceleration?]
export type TransactionCompressed = [string, number, number, number, number, number, 1?];
export type TransactionCompressed = [string, number, number, number, number, number, number, 1?];
// [txid, rate, flags, acceleration?]
export type MempoolDeltaChange = [string, number, number, (1|0)];

1
backend/src/replication/AuditReplication.ts
View File

@@ -114,6 +114,7 @@ class AuditReplication {
time: auditSummary.timestamp || auditSummary.time,
missingTxs: auditSummary.missingTxs || [],
addedTxs: auditSummary.addedTxs || [],
prioritizedTxs: auditSummary.prioritizedTxs || [],
freshTxs: auditSummary.freshTxs || [],
sigopTxs: auditSummary.sigopTxs || [],
fullrbfTxs: auditSummary.fullrbfTxs || [],

8
backend/src/repositories/BlocksAuditsRepository.ts
View File

@@ -6,9 +6,9 @@ import { BlockAudit, AuditScore } from '../mempool.interfaces';
class BlocksAuditRepositories {
public async $saveAudit(audit: BlockAudit): Promise<void> {
try {
await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, fresh_txs, sigop_txs, fullrbf_txs, accelerated_txs, match_rate, expected_fees, expected_weight)
VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs),
JSON.stringify(audit.addedTxs), JSON.stringify(audit.freshTxs), JSON.stringify(audit.sigopTxs), JSON.stringify(audit.fullrbfTxs), JSON.stringify(audit.acceleratedTxs), audit.matchRate, audit.expectedFees, audit.expectedWeight]);
await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, prioritized_txs, fresh_txs, sigop_txs, fullrbf_txs, accelerated_txs, match_rate, expected_fees, expected_weight)
VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs),
JSON.stringify(audit.addedTxs), JSON.stringify(audit.prioritizedTxs), JSON.stringify(audit.freshTxs), JSON.stringify(audit.sigopTxs), JSON.stringify(audit.fullrbfTxs), JSON.stringify(audit.acceleratedTxs), audit.matchRate, audit.expectedFees, audit.expectedWeight]);
} catch (e: any) {
if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
logger.debug(`Cannot save block audit for block ${audit.hash} because it has already been indexed, ignoring`);
@@ -66,6 +66,7 @@ class BlocksAuditRepositories {
template,
missing_txs as missingTxs,
added_txs as addedTxs,
prioritized_txs as prioritizedTxs,
fresh_txs as freshTxs,
sigop_txs as sigopTxs,
fullrbf_txs as fullrbfTxs,
@@ -81,6 +82,7 @@ class BlocksAuditRepositories {
if (rows.length) {
rows[0].missingTxs = JSON.parse(rows[0].missingTxs);
rows[0].addedTxs = JSON.parse(rows[0].addedTxs);
rows[0].prioritizedTxs = JSON.parse(rows[0].prioritizedTxs);
rows[0].freshTxs = JSON.parse(rows[0].freshTxs);
rows[0].sigopTxs = JSON.parse(rows[0].sigopTxs);
rows[0].fullrbfTxs = JSON.parse(rows[0].fullrbfTxs);