machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into mononaut/json-errors

Browse Source
This commit is contained in:
softsimon
2024-09-10 18:37:27 +04:00
committed by GitHub
parent b47e148677 31469ad361
commit 75316e60d0
51 changed files with 580 additions and 176 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
backend/src/api/audit.ts
View File

@@ -2,6 +2,7 @@ import config from '../config';
import logger from '../logger';
import { MempoolTransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces';
import rbfCache from './rbf-cache';
import transactionUtils from './transaction-utils';
const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners
@@ -15,7 +16,8 @@ class Audit {
const matches: string[] = []; // present in both mined block and template
const added: string[] = []; // present in mined block, not in template
const unseen: string[] = []; // present in the mined block, not in our mempool
const prioritized: string[] = []; // higher in the block than would be expected by in-band feerate alone
let prioritized: string[] = []; // higher in the block than would be expected by in-band feerate alone
let deprioritized: string[] = []; // lower in the block than would be expected by in-band feerate alone
const fresh: string[] = []; // missing, but firstSeen or lastBoosted within PROPAGATION_MARGIN
const rbf: string[] = []; // either missing or present, and either part of a full-rbf replacement, or a conflict with the mined block
const accelerated: string[] = []; // prioritized by the mempool accelerator
@@ -133,23 +135,7 @@ class Audit {
totalWeight += tx.weight;
}
// identify "prioritized" transactions
let lastEffectiveRate = 0;
// Iterate over the mined template from bottom to top (excluding the coinbase)
// Transactions should appear in ascending order of mining priority.
for (let i = transactions.length - 1; i > 0; i--) {
const blockTx = transactions[i];
// If a tx has a lower in-band effective fee rate than the previous tx,
// it must have been prioritized out-of-band (in order to have a higher mining priority)
// so exclude from the analysis.
if ((blockTx.effectiveFeePerVsize || 0) < lastEffectiveRate) {
prioritized.push(blockTx.txid);
// accelerated txs may or may not have their prioritized fee rate applied, so don't use them as a reference
} else if (!isAccelerated[blockTx.txid]) {
lastEffectiveRate = blockTx.effectiveFeePerVsize || 0;
}
}
({ prioritized, deprioritized } = transactionUtils.identifyPrioritizedTransactions(transactions, 'effectiveFeePerVsize'));
// transactions missing from near the end of our template are probably not being censored
let overflowWeightRemaining = overflowWeight - (config.MEMPOOL.BLOCK_WEIGHT_UNITS - totalWeight);

1
backend/src/api/bitcoin/bitcoin-api.ts
View File

@@ -323,6 +323,7 @@ class BitcoinApi implements AbstractBitcoinApi {
'witness_v1_taproot': 'v1_p2tr',
'nonstandard': 'nonstandard',
'multisig': 'multisig',
'anchor': 'anchor',
'nulldata': 'op_return'
};

16
backend/src/api/blocks.ts
View File

@@ -219,10 +219,10 @@ class Blocks {
};
}
public summarizeBlockTransactions(hash: string, transactions: TransactionExtended[]): BlockSummary {
public summarizeBlockTransactions(hash: string, height: number, transactions: TransactionExtended[]): BlockSummary {
return {
id: hash,
transactions: Common.classifyTransactions(transactions),
transactions: Common.classifyTransactions(transactions, height),
};
}
@@ -616,7 +616,7 @@ class Blocks {
// add CPFP
const cpfpSummary = calculateGoodBlockCpfp(height, txs, []);
// classify
const { transactions: classifiedTxs } = this.summarizeBlockTransactions(blockHash, cpfpSummary.transactions);
const { transactions: classifiedTxs } = this.summarizeBlockTransactions(blockHash, height, cpfpSummary.transactions);
await BlocksSummariesRepository.$saveTransactions(height, blockHash, classifiedTxs, 2);
if (unclassifiedBlocks[height].version < 2 && targetSummaryVersion === 2) {
const cpfpClusters = await CpfpRepository.$getClustersAt(height);
@@ -653,7 +653,7 @@ class Blocks {
}
const cpfpSummary = calculateGoodBlockCpfp(height, templateTxs?.filter(tx => tx['effectiveFeePerVsize'] != null) as MempoolTransactionExtended[], []);
// classify
const { transactions: classifiedTxs } = this.summarizeBlockTransactions(blockHash, cpfpSummary.transactions);
const { transactions: classifiedTxs } = this.summarizeBlockTransactions(blockHash, height, cpfpSummary.transactions);
const classifiedTxMap: { [txid: string]: TransactionClassified } = {};
for (const tx of classifiedTxs) {
classifiedTxMap[tx.txid] = tx;
@@ -912,7 +912,7 @@ class Blocks {
}
const cpfpSummary: CpfpSummary = calculateGoodBlockCpfp(block.height, transactions, accelerations.map(a => ({ txid: a.txid, max_bid: a.feeDelta })));
const blockExtended: BlockExtended = await this.$getBlockExtended(block, cpfpSummary.transactions);
const blockSummary: BlockSummary = this.summarizeBlockTransactions(block.id, cpfpSummary.transactions);
const blockSummary: BlockSummary = this.summarizeBlockTransactions(block.id, block.height, cpfpSummary.transactions);
this.updateTimerProgress(timer, `got block data for ${this.currentBlockHeight}`);
if (Common.indexingEnabled()) {
@@ -1169,7 +1169,7 @@ class Blocks {
transactions: cpfpSummary.transactions.map(tx => {
let flags: number = 0;
try {
flags = Common.getTransactionFlags(tx);
flags = Common.getTransactionFlags(tx, height);
} catch (e) {
logger.warn('Failed to classify transaction: ' + (e instanceof Error ? e.message : e));
}
@@ -1188,7 +1188,7 @@ class Blocks {
} else {
if (config.MEMPOOL.BACKEND === 'esplora') {
const txs = (await bitcoinApi.$getTxsForBlock(hash)).map(tx => transactionUtils.extendTransaction(tx));
summary = this.summarizeBlockTransactions(hash, txs);
summary = this.summarizeBlockTransactions(hash, height || 0, txs);
summaryVersion = 1;
} else {
// Call Core RPC
@@ -1324,7 +1324,7 @@ class Blocks {
let summaryVersion = 0;
if (config.MEMPOOL.BACKEND === 'esplora') {
const txs = (await bitcoinApi.$getTxsForBlock(cleanBlock.hash)).map(tx => transactionUtils.extendTransaction(tx));
summary = this.summarizeBlockTransactions(cleanBlock.hash, txs);
summary = this.summarizeBlockTransactions(cleanBlock.hash, cleanBlock.height, txs);
summaryVersion = 1;
} else {
// Call Core RPC

67
backend/src/api/common.ts
View File

@@ -10,7 +10,6 @@ import logger from '../logger';
import { getVarIntLength, opcodes, parseMultisigScript } from '../utils/bitcoin-script';
// Bitcoin Core default policy settings
const TX_MAX_STANDARD_VERSION = 2;
const MAX_STANDARD_TX_WEIGHT = 400_000;
const MAX_BLOCK_SIGOPS_COST = 80_000;
const MAX_STANDARD_TX_SIGOPS_COST = (MAX_BLOCK_SIGOPS_COST / 5);
@@ -200,10 +199,13 @@ export class Common {
*
* returns true early if any standardness rule is violated, otherwise false
* (except for non-mandatory-script-verify-flag and p2sh script evaluation rules which are *not* enforced)
*
* As standardness rules change, we'll need to apply the rules in force *at the time* to older blocks.
* For now, just pull out individual rules into versioned functions where necessary.
*/
static isNonStandard(tx: TransactionExtended): boolean {
static isNonStandard(tx: TransactionExtended, height?: number): boolean {
// version
if (tx.version > TX_MAX_STANDARD_VERSION) {
if (this.isNonStandardVersion(tx, height)) {
return true;
}
@@ -250,6 +252,8 @@ export class Common {
}
} else if (['unknown', 'provably_unspendable', 'empty'].includes(vin.prevout?.scriptpubkey_type || '')) {
return true;
} else if (this.isNonStandardAnchor(tx, height)) {
return true;
}
// TODO: bad-witness-nonstandard
}
@@ -335,6 +339,49 @@ export class Common {
return false;
}
// Individual versioned standardness rules
static V3_STANDARDNESS_ACTIVATION_HEIGHT = {
'testnet4': 42_000,
'testnet': 2_900_000,
'signet': 211_000,
'': 863_500,
};
static isNonStandardVersion(tx: TransactionExtended, height?: number): boolean {
let TX_MAX_STANDARD_VERSION = 3;
if (
height != null
&& this.V3_STANDARDNESS_ACTIVATION_HEIGHT[config.MEMPOOL.NETWORK]
&& height <= this.V3_STANDARDNESS_ACTIVATION_HEIGHT[config.MEMPOOL.NETWORK]
) {
// V3 transactions were non-standard to spend before v28.x (scheduled for 2024/09/30 https://github.com/bitcoin/bitcoin/issues/29891)
TX_MAX_STANDARD_VERSION = 2;
}
if (tx.version > TX_MAX_STANDARD_VERSION) {
return true;
}
return false;
}
static ANCHOR_STANDARDNESS_ACTIVATION_HEIGHT = {
'testnet4': 42_000,
'testnet': 2_900_000,
'signet': 211_000,
'': 863_500,
};
static isNonStandardAnchor(tx: TransactionExtended, height?: number): boolean {
if (
height != null
&& this.ANCHOR_STANDARDNESS_ACTIVATION_HEIGHT[config.MEMPOOL.NETWORK]
&& height <= this.ANCHOR_STANDARDNESS_ACTIVATION_HEIGHT[config.MEMPOOL.NETWORK]
) {
// anchor outputs were non-standard to spend before v28.x (scheduled for 2024/09/30 https://github.com/bitcoin/bitcoin/issues/29891)
return true;
}
return false;
}
static getNonWitnessSize(tx: TransactionExtended): number {
let weight = tx.weight;
let hasWitness = false;
@@ -415,7 +462,7 @@ export class Common {
return flags;
}
static getTransactionFlags(tx: TransactionExtended): number {
static getTransactionFlags(tx: TransactionExtended, height?: number): number {
let flags = tx.flags ? BigInt(tx.flags) : 0n;
// Update variable flags (CPFP, RBF)
@@ -548,7 +595,7 @@ export class Common {
if (hasFakePubkey) {
flags |= TransactionFlags.fake_pubkey;
}
// fast but bad heuristic to detect possible coinjoins
// (at least 5 inputs and 5 outputs, less than half of which are unique amounts, with no address reuse)
const addressReuse = Object.keys(reusedOutputAddresses).reduce((acc, key) => Math.max(acc, (reusedInputAddresses[key] || 0) + (reusedOutputAddresses[key] || 0)), 0) > 1;
@@ -564,17 +611,17 @@ export class Common {
flags |= TransactionFlags.batch_payout;
}
if (this.isNonStandard(tx)) {
if (this.isNonStandard(tx, height)) {
flags |= TransactionFlags.nonstandard;
}
return Number(flags);
}
static classifyTransaction(tx: TransactionExtended): TransactionClassified {
static classifyTransaction(tx: TransactionExtended, height?: number): TransactionClassified {
let flags = 0;
try {
flags = Common.getTransactionFlags(tx);
flags = Common.getTransactionFlags(tx, height);
} catch (e) {
logger.warn('Failed to add classification flags to transaction: ' + (e instanceof Error ? e.message : e));
}
@@ -585,8 +632,8 @@ export class Common {
};
}
static classifyTransactions(txs: TransactionExtended[]): TransactionClassified[] {
return txs.map(Common.classifyTransaction);
static classifyTransactions(txs: TransactionExtended[], height?: number): TransactionClassified[] {
return txs.map(tx => Common.classifyTransaction(tx, height));
}
static stripTransaction(tx: TransactionExtended): TransactionStripped {

29
backend/src/api/database-migration.ts
View File

@@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
import { RowDataPacket } from 'mysql2';
class DatabaseMigration {
private static currentVersion = 81;
private static currentVersion = 82;
private queryTimeout = 3600_000;
private statisticsAddedIndexed = false;
private uniqueLogs: string[] = [];
@@ -700,6 +700,11 @@ class DatabaseMigration {
await this.$executeQuery('ALTER TABLE `blocks_audits` ADD unseen_txs JSON DEFAULT "[]"');
await this.updateToSchemaVersion(81);
}
if (databaseSchemaVersion < 82 && isBitcoin === true && config.MEMPOOL.NETWORK === 'mainnet') {
await this.$fixBadV1AuditBlocks();
await this.updateToSchemaVersion(82);
}
}
/**
@@ -1314,6 +1319,28 @@ class DatabaseMigration {
logger.warn(`Failed to migrate cpfp transaction data`);
}
}
private async $fixBadV1AuditBlocks(): Promise<void> {
const badBlocks = [
'000000000000000000011ad49227fc8c9ba0ca96ad2ebce41a862f9a244478dc',
'000000000000000000010ac1f68b3080153f2826ffddc87ceffdd68ed97d6960',
'000000000000000000024cbdafeb2660ae8bd2947d166e7fe15d1689e86b2cf7',
'00000000000000000002e1dbfbf6ae057f331992a058b822644b368034f87286',
'0000000000000000000019973b2778f08ad6d21e083302ff0833d17066921ebb',
];
for (const hash of badBlocks) {
try {
await this.$executeQuery(`
UPDATE blocks_audits
SET prioritized_txs = '[]'
WHERE hash = '${hash}'
`, true);
} catch (e) {
continue;
}
}
}
}
export default new DatabaseMigration();

81
backend/src/api/transaction-utils.ts
View File

@@ -338,6 +338,87 @@ class TransactionUtils {
const positionOfScript = hasAnnex ? witness.length - 3 : witness.length - 2;
return witness[positionOfScript];
}
// calculate the most parsimonious set of prioritizations given a list of block transactions
// (i.e. the most likely prioritizations and deprioritizations)
public identifyPrioritizedTransactions(transactions: any[], rateKey: string): { prioritized: string[], deprioritized: string[] } {
// find the longest increasing subsequence of transactions
// (adapted from https://en.wikipedia.org/wiki/Longest_increasing_subsequence#Efficient_algorithms)
// should be O(n log n)
const X = transactions.slice(1).reverse().map((tx) => ({ txid: tx.txid, rate: tx[rateKey] })); // standard block order is by *decreasing* effective fee rate, but we want to iterate in increasing order (and skip the coinbase)
if (X.length < 2) {
return { prioritized: [], deprioritized: [] };
}
const N = X.length;
const P: number[] = new Array(N);
const M: number[] = new Array(N + 1);
M[0] = -1; // undefined so can be set to any value
let L = 0;
for (let i = 0; i < N; i++) {
// Binary search for the smallest positive l ≤ L
// such that X[M[l]].effectiveFeePerVsize > X[i].effectiveFeePerVsize
let lo = 1;
let hi = L + 1;
while (lo < hi) {
const mid = lo + Math.floor((hi - lo) / 2); // lo <= mid < hi
if (X[M[mid]].rate > X[i].rate) {
hi = mid;
} else { // if X[M[mid]].effectiveFeePerVsize < X[i].effectiveFeePerVsize
lo = mid + 1;
}
}
// After searching, lo == hi is 1 greater than the
// length of the longest prefix of X[i]
const newL = lo;
// The predecessor of X[i] is the last index of
// the subsequence of length newL-1
P[i] = M[newL - 1];
M[newL] = i;
if (newL > L) {
// If we found a subsequence longer than any we've
// found yet, update L
L = newL;
}
}
// Reconstruct the longest increasing subsequence
// It consists of the values of X at the L indices:
// ..., P[P[M[L]]], P[M[L]], M[L]
const LIS: any[] = new Array(L);
let k = M[L];
for (let j = L - 1; j >= 0; j--) {
LIS[j] = X[k];
k = P[k];
}
const lisMap = new Map<string, number>();
LIS.forEach((tx, index) => lisMap.set(tx.txid, index));
const prioritized: string[] = [];
const deprioritized: string[] = [];
let lastRate = X[0].rate;
for (const tx of X) {
if (lisMap.has(tx.txid)) {
lastRate = tx.rate;
} else {
if (Math.abs(tx.rate - lastRate) < 0.1) {
// skip if the rate is almost the same as the previous transaction
} else if (tx.rate <= lastRate) {
prioritized.push(tx.txid);
} else {
deprioritized.push(tx.txid);
}
}
}
return { prioritized, deprioritized };
}
}
export default new TransactionUtils();

2
backend/src/repositories/BlocksRepository.ts
View File

@@ -1106,7 +1106,7 @@ class BlocksRepository {
let summaryVersion = 0;
if (config.MEMPOOL.BACKEND === 'esplora') {
const txs = (await bitcoinApi.$getTxsForBlock(dbBlk.id)).map(tx => transactionUtils.extendTransaction(tx));
summary = blocks.summarizeBlockTransactions(dbBlk.id, txs);
summary = blocks.summarizeBlockTransactions(dbBlk.id, dbBlk.height, txs);
summaryVersion = 1;
} else {
// Call Core RPC

4
backend/src/utils/bitcoin-script.ts
View File

@@ -158,7 +158,7 @@ export function parseMultisigScript(script: string): void | { m: number, n: numb
if (!opN) {
return;
}
if (!opN.startsWith('OP_PUSHNUM_')) {
if (opN !== 'OP_0' && !opN.startsWith('OP_PUSHNUM_')) {
return;
}
const n = parseInt(opN.match(/[0-9]+/)?.[0] || '', 10);
@@ -178,7 +178,7 @@ export function parseMultisigScript(script: string): void | { m: number, n: numb
if (!opM) {
return;
}
if (!opM.startsWith('OP_PUSHNUM_')) {
if (opM !== 'OP_0' && !opM.startsWith('OP_PUSHNUM_')) {
return;
}
const m = parseInt(opM.match(/[0-9]+/)?.[0] || '', 10);