From 09e4e44e887d78950cb6b0f53916d2ecacb77457 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 29 May 2023 15:56:29 -0400 Subject: [PATCH] Count sigops & use adjusted vsizes in mempool projections --- backend/src/api/audit.ts | 13 +--- backend/src/api/blocks.ts | 21 +++--- backend/src/api/common.ts | 28 ++++---- backend/src/api/mempool-blocks.ts | 44 ++++++------ backend/src/api/mempool.ts | 47 ++++++------ backend/src/api/rbf-cache.ts | 10 +-- backend/src/api/transaction-utils.ts | 104 +++++++++++++++++++++++++-- backend/src/api/websocket-handler.ts | 16 ++--- backend/src/mempool.interfaces.ts | 6 ++ 9 files changed, 191 insertions(+), 98 deletions(-) diff --git a/backend/src/api/audit.ts b/backend/src/api/audit.ts index 1ec1ae65a..6c5f96988 100644 --- a/backend/src/api/audit.ts +++ b/backend/src/api/audit.ts @@ -14,7 +14,6 @@ class Audit { const matches: string[] = []; // present in both mined block and template const added: string[] = []; // present in mined block, not in template const fresh: string[] = []; // missing, but firstSeen within PROPAGATION_MARGIN - const sigop: string[] = []; // missing, but possibly has an adjusted vsize due to high sigop count const isCensored = {}; // missing, without excuse const isDisplaced = {}; let displacedWeight = 0; @@ -38,8 +37,6 @@ class Audit { // tx is recent, may have reached the miner too late for inclusion if (mempool[txid]?.firstSeen != null && (now - (mempool[txid]?.firstSeen || 0)) <= PROPAGATION_MARGIN) { fresh.push(txid); - } else if (this.isPossibleHighSigop(mempool[txid])) { - sigop.push(txid); } else { isCensored[txid] = true; } @@ -140,19 +137,11 @@ class Audit { censored: Object.keys(isCensored), added, fresh, - sigop, + sigop: [], score, similarity, }; } - - // Detect transactions with a possibly adjusted vsize due to high sigop count - // very rough heuristic based on number of OP_CHECKMULTISIG outputs - // will miss cases with other sources of sigops - isPossibleHighSigop(tx: TransactionExtended): boolean { - const numBareMultisig = tx.vout.reduce((count, vout) => count + (vout.scriptpubkey_asm.includes('OP_CHECKMULTISIG') ? 1 : 0), 0); - return (numBareMultisig * 400) > tx.vsize; - } } export default new Audit(); \ No newline at end of file diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index bce7983d3..fc12b5998 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -2,7 +2,7 @@ import config from '../config'; import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory'; import logger from '../logger'; import memPool from './mempool'; -import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionStripped, TransactionMinerInfo, CpfpSummary } from '../mempool.interfaces'; +import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionStripped, TransactionMinerInfo, CpfpSummary, MempoolTransactionExtended } from '../mempool.interfaces'; import { Common } from './common'; import diskCache from './disk-cache'; import transactionUtils from './transaction-utils'; @@ -76,6 +76,7 @@ class Blocks { blockHeight: number, onlyCoinbase: boolean, quiet: boolean = false, + addMempoolData: boolean = false, ): Promise { const transactions: TransactionExtended[] = []; const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash); @@ -96,14 +97,14 @@ class Blocks { logger.debug(`Indexing tx ${i + 1} of ${txIds.length} in block #${blockHeight}`); } try { - const tx = await transactionUtils.$getTransactionExtended(txIds[i]); + const tx = await transactionUtils.$getTransactionExtended(txIds[i], false, false, false, addMempoolData); transactions.push(tx); transactionsFetched++; } catch (e) { try { if (config.MEMPOOL.BACKEND === 'esplora') { // Try again with core - const tx = await transactionUtils.$getTransactionExtended(txIds[i], false, false, true); + const tx = await transactionUtils.$getTransactionExtended(txIds[i], false, false, true, addMempoolData); transactions.push(tx); transactionsFetched++; } else { @@ -126,11 +127,13 @@ class Blocks { } } - transactions.forEach((tx) => { - if (!tx.cpfpChecked) { - Common.setRelativesAndGetCpfpInfo(tx, mempool); // Child Pay For Parent - } - }); + if (addMempoolData) { + transactions.forEach((tx) => { + if (!tx.cpfpChecked) { + Common.setRelativesAndGetCpfpInfo(tx as MempoolTransactionExtended, mempool); // Child Pay For Parent + } + }); + } if (!quiet) { logger.debug(`${transactionsFound} of ${txIds.length} found in mempool. ${transactionsFetched} fetched through backend service.`); @@ -596,7 +599,7 @@ class Blocks { const verboseBlock = await bitcoinClient.getBlock(blockHash, 2); const block = BitcoinApi.convertBlock(verboseBlock); const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash); - const transactions = await this.$getTransactionsExtended(blockHash, block.height, false); + const transactions = await this.$getTransactionsExtended(blockHash, block.height, false, false, true); const cpfpSummary: CpfpSummary = Common.calculateCpfp(block.height, transactions); const blockExtended: BlockExtended = await this.$getBlockExtended(block, cpfpSummary.transactions); const blockSummary: BlockSummary = this.summarizeBlock(verboseBlock); diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index ba06c53b3..a2a74e907 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -1,4 +1,4 @@ -import { Ancestor, CpfpInfo, CpfpSummary, EffectiveFeeStats, MempoolBlockWithTransactions, TransactionExtended, TransactionStripped, WorkingEffectiveFeeStats } from '../mempool.interfaces'; +import { Ancestor, CpfpInfo, CpfpSummary, EffectiveFeeStats, MempoolBlockWithTransactions, TransactionExtended, MempoolTransactionExtended, TransactionStripped, WorkingEffectiveFeeStats } from '../mempool.interfaces'; import config from '../config'; import { NodeSocket } from '../repositories/NodesSocketsRepository'; import { isIP } from 'net'; @@ -57,15 +57,15 @@ export class Common { return arr; } - static findRbfTransactions(added: TransactionExtended[], deleted: TransactionExtended[]): { [txid: string]: TransactionExtended[] } { - const matches: { [txid: string]: TransactionExtended[] } = {}; + static findRbfTransactions(added: MempoolTransactionExtended[], deleted: MempoolTransactionExtended[]): { [txid: string]: MempoolTransactionExtended[] } { + const matches: { [txid: string]: MempoolTransactionExtended[] } = {}; added .forEach((addedTx) => { const foundMatches = deleted.filter((deletedTx) => { // The new tx must, absolutely speaking, pay at least as much fee as the replaced tx. return addedTx.fee > deletedTx.fee // The new transaction must pay more fee per kB than the replaced tx. - && addedTx.feePerVsize > deletedTx.feePerVsize + && addedTx.adjustedFeePerVsize > deletedTx.adjustedFeePerVsize // Spends one or more of the same inputs && deletedTx.vin.some((deletedVin) => addedTx.vin.some((vin) => vin.txid === deletedVin.txid && vin.vout === deletedVin.vout)); @@ -120,18 +120,18 @@ export class Common { } } - static setRelativesAndGetCpfpInfo(tx: TransactionExtended, memPool: { [txid: string]: TransactionExtended }): CpfpInfo { + static setRelativesAndGetCpfpInfo(tx: MempoolTransactionExtended, memPool: { [txid: string]: MempoolTransactionExtended }): CpfpInfo { const parents = this.findAllParents(tx, memPool); - const lowerFeeParents = parents.filter((parent) => parent.feePerVsize < tx.effectiveFeePerVsize); + const lowerFeeParents = parents.filter((parent) => parent.adjustedFeePerVsize < tx.effectiveFeePerVsize); - let totalWeight = tx.weight + lowerFeeParents.reduce((prev, val) => prev + val.weight, 0); + let totalWeight = (tx.adjustedVsize * 4) + lowerFeeParents.reduce((prev, val) => prev + (val.adjustedVsize * 4), 0); let totalFees = tx.fee + lowerFeeParents.reduce((prev, val) => prev + val.fee, 0); tx.ancestors = parents .map((t) => { return { txid: t.txid, - weight: t.weight, + weight: (t.adjustedVsize * 4), fee: t.fee, }; }); @@ -152,8 +152,8 @@ export class Common { } - private static findAllParents(tx: TransactionExtended, memPool: { [txid: string]: TransactionExtended }): TransactionExtended[] { - let parents: TransactionExtended[] = []; + private static findAllParents(tx: MempoolTransactionExtended, memPool: { [txid: string]: MempoolTransactionExtended }): MempoolTransactionExtended[] { + let parents: MempoolTransactionExtended[] = []; tx.vin.forEach((parent) => { if (parents.find((p) => p.txid === parent.txid)) { return; @@ -161,17 +161,17 @@ export class Common { const parentTx = memPool[parent.txid]; if (parentTx) { - if (tx.bestDescendant && tx.bestDescendant.fee / (tx.bestDescendant.weight / 4) > parentTx.feePerVsize) { + if (tx.bestDescendant && tx.bestDescendant.fee / (tx.bestDescendant.weight / 4) > parentTx.adjustedFeePerVsize) { if (parentTx.bestDescendant && parentTx.bestDescendant.fee < tx.fee + tx.bestDescendant.fee) { parentTx.bestDescendant = { - weight: tx.weight + tx.bestDescendant.weight, + weight: (tx.adjustedVsize * 4) + tx.bestDescendant.weight, fee: tx.fee + tx.bestDescendant.fee, txid: tx.txid, }; } - } else if (tx.feePerVsize > parentTx.feePerVsize) { + } else if (tx.adjustedFeePerVsize > parentTx.adjustedFeePerVsize) { parentTx.bestDescendant = { - weight: tx.weight, + weight: (tx.adjustedVsize * 4), fee: tx.fee, txid: tx.txid }; diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 803b7e56e..51f07fac6 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -1,5 +1,5 @@ import logger from '../logger'; -import { MempoolBlock, TransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats } from '../mempool.interfaces'; +import { MempoolBlock, MempoolTransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats } from '../mempool.interfaces'; import { Common, OnlineFeeStatsCalculator } from './common'; import config from '../config'; import { Worker } from 'worker_threads'; @@ -36,9 +36,9 @@ class MempoolBlocks { return this.mempoolBlockDeltas; } - public updateMempoolBlocks(memPool: { [txid: string]: TransactionExtended }, saveResults: boolean = false): MempoolBlockWithTransactions[] { + public updateMempoolBlocks(memPool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false): MempoolBlockWithTransactions[] { const latestMempool = memPool; - const memPoolArray: TransactionExtended[] = []; + const memPoolArray: MempoolTransactionExtended[] = []; for (const i in latestMempool) { if (latestMempool.hasOwnProperty(i)) { memPoolArray.push(latestMempool[i]); @@ -52,17 +52,17 @@ class MempoolBlocks { tx.ancestors = []; tx.cpfpChecked = false; if (!tx.effectiveFeePerVsize) { - tx.effectiveFeePerVsize = tx.feePerVsize; + tx.effectiveFeePerVsize = tx.adjustedFeePerVsize; } }); // First sort memPoolArray.sort((a, b) => { - if (a.feePerVsize === b.feePerVsize) { + if (a.adjustedFeePerVsize === b.adjustedFeePerVsize) { // tie-break by lexicographic txid order for stability return a.txid < b.txid ? -1 : 1; } else { - return b.feePerVsize - a.feePerVsize; + return b.adjustedFeePerVsize - a.adjustedFeePerVsize; } }); @@ -102,7 +102,7 @@ class MempoolBlocks { return blocks; } - private calculateMempoolBlocks(transactionsSorted: TransactionExtended[]): MempoolBlockWithTransactions[] { + private calculateMempoolBlocks(transactionsSorted: MempoolTransactionExtended[]): MempoolBlockWithTransactions[] { const mempoolBlocks: MempoolBlockWithTransactions[] = []; let feeStatsCalculator: OnlineFeeStatsCalculator = new OnlineFeeStatsCalculator(config.MEMPOOL.BLOCK_WEIGHT_UNITS); let onlineStats = false; @@ -112,7 +112,7 @@ class MempoolBlocks { let blockFees = 0; const sizeLimit = (config.MEMPOOL.BLOCK_WEIGHT_UNITS / 4) * 1.2; let transactionIds: string[] = []; - let transactions: TransactionExtended[] = []; + let transactions: MempoolTransactionExtended[] = []; transactionsSorted.forEach((tx, index) => { if (blockWeight + tx.weight <= config.MEMPOOL.BLOCK_WEIGHT_UNITS || mempoolBlocks.length === config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT - 1) { @@ -205,7 +205,7 @@ class MempoolBlocks { return mempoolBlockDeltas; } - public async $makeBlockTemplates(newMempool: { [txid: string]: TransactionExtended }, saveResults: boolean = false): Promise { + public async $makeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false): Promise { const start = Date.now(); // reset mempool short ids @@ -222,9 +222,9 @@ class MempoolBlocks { strippedMempool.set(entry.uid, { uid: entry.uid, fee: entry.fee, - weight: entry.weight, - feePerVsize: entry.fee / (entry.weight / 4), - effectiveFeePerVsize: entry.effectiveFeePerVsize || (entry.fee / (entry.weight / 4)), + weight: (entry.adjustedVsize * 4), + feePerVsize: entry.adjustedFeePerVsize || entry.feePerVsize, + effectiveFeePerVsize: entry.effectiveFeePerVsize || entry.adjustedFeePerVsize || entry.feePerVsize, inputs: entry.vin.map(v => this.getUid(newMempool[v.txid])).filter(uid => uid != null) as number[], }); } @@ -268,7 +268,7 @@ class MempoolBlocks { return this.mempoolBlocks; } - public async $updateBlockTemplates(newMempool: { [txid: string]: TransactionExtended }, added: TransactionExtended[], removed: TransactionExtended[], saveResults: boolean = false): Promise { + public async $updateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], saveResults: boolean = false): Promise { if (!this.txSelectionWorker) { // need to reset the worker await this.$makeBlockTemplates(newMempool, saveResults); @@ -287,9 +287,9 @@ class MempoolBlocks { return { uid: entry.uid || 0, fee: entry.fee, - weight: entry.weight, - feePerVsize: entry.fee / (entry.weight / 4), - effectiveFeePerVsize: entry.effectiveFeePerVsize || (entry.fee / (entry.weight / 4)), + weight: (entry.adjustedVsize * 4), + feePerVsize: entry.adjustedFeePerVsize || entry.feePerVsize, + effectiveFeePerVsize: entry.effectiveFeePerVsize || entry.adjustedFeePerVsize || entry.feePerVsize, inputs: entry.vin.map(v => this.getUid(newMempool[v.txid])).filter(uid => uid != null) as number[], }; }); @@ -341,12 +341,12 @@ class MempoolBlocks { for (let blockIndex = 0; blockIndex < blocks.length; blockIndex++) { const block: string[] = blocks[blockIndex]; let txid: string; - let mempoolTx: TransactionExtended; + let mempoolTx: MempoolTransactionExtended; let totalSize = 0; let totalVsize = 0; let totalWeight = 0; let totalFees = 0; - const transactions: TransactionExtended[] = []; + const transactions: MempoolTransactionExtended[] = []; for (let txIndex = 0; txIndex < block.length; txIndex++) { txid = block[txIndex]; if (txid) { @@ -397,7 +397,7 @@ class MempoolBlocks { const relative = { txid: txid, fee: mempool[txid].fee, - weight: mempool[txid].weight, + weight: (mempool[txid].adjustedVsize * 4), }; if (matched) { descendants.push(relative); @@ -426,7 +426,7 @@ class MempoolBlocks { return mempoolBlocks; } - private dataToMempoolBlocks(transactionIds: string[], transactions: TransactionExtended[], totalSize: number, totalWeight: number, totalFees: number, feeStats?: EffectiveFeeStats ): MempoolBlockWithTransactions { + private dataToMempoolBlocks(transactionIds: string[], transactions: MempoolTransactionExtended[], totalSize: number, totalWeight: number, totalFees: number, feeStats?: EffectiveFeeStats ): MempoolBlockWithTransactions { if (!feeStats) { feeStats = Common.calcEffectiveFeeStatistics(transactions); } @@ -447,7 +447,7 @@ class MempoolBlocks { this.nextUid = 1; } - private setUid(tx: TransactionExtended): number { + private setUid(tx: MempoolTransactionExtended): number { const uid = this.nextUid; this.nextUid++; this.uidMap.set(uid, tx.txid); @@ -455,7 +455,7 @@ class MempoolBlocks { return uid; } - private getUid(tx: TransactionExtended): number | void { + private getUid(tx: MempoolTransactionExtended): number | void { if (tx?.uid != null && this.uidMap.has(tx.uid)) { return tx.uid; } diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index fe84fb8e4..b5a565334 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -1,6 +1,6 @@ import config from '../config'; import bitcoinApi from './bitcoin/bitcoin-api-factory'; -import { TransactionExtended, VbytesPerSecond } from '../mempool.interfaces'; +import { MempoolTransactionExtended, VbytesPerSecond } from '../mempool.interfaces'; import logger from '../logger'; import { Common } from './common'; import transactionUtils from './transaction-utils'; @@ -13,14 +13,14 @@ import rbfCache from './rbf-cache'; class Mempool { private inSync: boolean = false; private mempoolCacheDelta: number = -1; - private mempoolCache: { [txId: string]: TransactionExtended } = {}; - private spendMap = new Map(); + private mempoolCache: { [txId: string]: MempoolTransactionExtended } = {}; + private spendMap = new Map(); private mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0, total_fee: 0, maxmempool: 300000000, mempoolminfee: 0.00001000, minrelaytxfee: 0.00001000 }; - private mempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[], - deletedTransactions: TransactionExtended[]) => void) | undefined; - private $asyncMempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[], - deletedTransactions: TransactionExtended[]) => Promise) | undefined; + private mempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, newTransactions: MempoolTransactionExtended[], + deletedTransactions: MempoolTransactionExtended[]) => void) | undefined; + private $asyncMempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, newTransactions: MempoolTransactionExtended[], + deletedTransactions: MempoolTransactionExtended[]) => Promise) | undefined; private txPerSecondArray: number[] = []; private txPerSecond: number = 0; @@ -64,26 +64,31 @@ class Mempool { return this.latestTransactions; } - public setMempoolChangedCallback(fn: (newMempool: { [txId: string]: TransactionExtended; }, - newTransactions: TransactionExtended[], deletedTransactions: TransactionExtended[]) => void) { + public setMempoolChangedCallback(fn: (newMempool: { [txId: string]: MempoolTransactionExtended; }, + newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[]) => void): void { this.mempoolChangedCallback = fn; } - public setAsyncMempoolChangedCallback(fn: (newMempool: { [txId: string]: TransactionExtended; }, - newTransactions: TransactionExtended[], deletedTransactions: TransactionExtended[]) => Promise) { + public setAsyncMempoolChangedCallback(fn: (newMempool: { [txId: string]: MempoolTransactionExtended; }, + newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[]) => Promise): void { this.$asyncMempoolChangedCallback = fn; } - public getMempool(): { [txid: string]: TransactionExtended } { + public getMempool(): { [txid: string]: MempoolTransactionExtended } { return this.mempoolCache; } - public getSpendMap(): Map { + public getSpendMap(): Map { return this.spendMap; } - public async $setMempool(mempoolData: { [txId: string]: TransactionExtended }) { + public async $setMempool(mempoolData: { [txId: string]: MempoolTransactionExtended }) { this.mempoolCache = mempoolData; + for (const txid of Object.keys(this.mempoolCache)) { + if (this.mempoolCache[txid].sigops == null || this.mempoolCache[txid].effectiveFeePerVsize == null) { + this.mempoolCache[txid] = transactionUtils.extendMempoolTransaction(this.mempoolCache[txid]); + } + } if (this.mempoolChangedCallback) { this.mempoolChangedCallback(this.mempoolCache, [], []); } @@ -133,7 +138,7 @@ class Mempool { const currentMempoolSize = Object.keys(this.mempoolCache).length; this.updateTimerProgress(timer, 'got raw mempool'); const diff = transactions.length - currentMempoolSize; - const newTransactions: TransactionExtended[] = []; + const newTransactions: MempoolTransactionExtended[] = []; this.mempoolCacheDelta = Math.abs(diff); @@ -155,7 +160,7 @@ class Mempool { for (const txid of transactions) { if (!this.mempoolCache[txid]) { try { - const transaction = await transactionUtils.$getTransactionExtended(txid); + const transaction = await transactionUtils.$getMempoolTransactionExtended(txid, false, false, false); this.updateTimerProgress(timer, 'fetched new transaction'); this.mempoolCache[txid] = transaction; if (this.inSync) { @@ -205,7 +210,7 @@ class Mempool { }, 1000 * 60 * config.MEMPOOL.CLEAR_PROTECTION_MINUTES); } - const deletedTransactions: TransactionExtended[] = []; + const deletedTransactions: MempoolTransactionExtended[] = []; if (this.mempoolProtection !== 1) { this.mempoolProtection = 0; @@ -273,7 +278,7 @@ class Mempool { } } - public handleRbfTransactions(rbfTransactions: { [txid: string]: TransactionExtended[]; }): void { + public handleRbfTransactions(rbfTransactions: { [txid: string]: MempoolTransactionExtended[]; }): void { for (const rbfTransaction in rbfTransactions) { if (this.mempoolCache[rbfTransaction] && rbfTransactions[rbfTransaction]?.length) { // Store replaced transactions @@ -282,7 +287,7 @@ class Mempool { } } - public handleMinedRbfTransactions(rbfTransactions: { [txid: string]: { replaced: TransactionExtended[], replacedBy: TransactionExtended }}): void { + public handleMinedRbfTransactions(rbfTransactions: { [txid: string]: { replaced: MempoolTransactionExtended[], replacedBy: MempoolTransactionExtended }}): void { for (const rbfTransaction in rbfTransactions) { if (rbfTransactions[rbfTransaction].replacedBy && rbfTransactions[rbfTransaction]?.replaced?.length) { // Store replaced transactions @@ -291,7 +296,7 @@ class Mempool { } } - public addToSpendMap(transactions: TransactionExtended[]): void { + public addToSpendMap(transactions: MempoolTransactionExtended[]): void { for (const tx of transactions) { for (const vin of tx.vin) { this.spendMap.set(`${vin.txid}:${vin.vout}`, tx); @@ -299,7 +304,7 @@ class Mempool { } } - public removeFromSpendMap(transactions: TransactionExtended[]): void { + public removeFromSpendMap(transactions: MempoolTransactionExtended[]): void { for (const tx of transactions) { for (const vin of tx.vin) { const key = `${vin.txid}:${vin.vout}`; diff --git a/backend/src/api/rbf-cache.ts b/backend/src/api/rbf-cache.ts index 51f8ffeca..f0a916c8c 100644 --- a/backend/src/api/rbf-cache.ts +++ b/backend/src/api/rbf-cache.ts @@ -1,5 +1,5 @@ import logger from "../logger"; -import { TransactionExtended, TransactionStripped } from "../mempool.interfaces"; +import { MempoolTransactionExtended, TransactionStripped } from "../mempool.interfaces"; import bitcoinApi from './bitcoin/bitcoin-api-factory'; import { Common } from "./common"; @@ -23,14 +23,14 @@ class RbfCache { private rbfTrees: Map = new Map(); // sequences of consecutive replacements private dirtyTrees: Set = new Set(); private treeMap: Map = new Map(); // map of txids to sequence ids - private txs: Map = new Map(); + private txs: Map = new Map(); private expiring: Map = new Map(); constructor() { setInterval(this.cleanup.bind(this), 1000 * 60 * 10); } - public add(replaced: TransactionExtended[], newTxExtended: TransactionExtended): void { + public add(replaced: MempoolTransactionExtended[], newTxExtended: MempoolTransactionExtended): void { if (!newTxExtended || !replaced?.length || this.txs.has(newTxExtended.txid)) { return; } @@ -92,7 +92,7 @@ class RbfCache { return this.replaces.get(txId); } - public getTx(txId: string): TransactionExtended | undefined { + public getTx(txId: string): MempoolTransactionExtended | undefined { return this.txs.get(txId); } @@ -272,7 +272,7 @@ class RbfCache { return deflated; } - async importTree(root, txid, deflated, txs: Map, mined: boolean = false): Promise { + async importTree(root, txid, deflated, txs: Map, mined: boolean = false): Promise { const treeInfo = deflated[txid]; const replaces: RbfTree[] = []; diff --git a/backend/src/api/transaction-utils.ts b/backend/src/api/transaction-utils.ts index fb69419fc..acb268b44 100644 --- a/backend/src/api/transaction-utils.ts +++ b/backend/src/api/transaction-utils.ts @@ -1,7 +1,8 @@ -import { TransactionExtended, TransactionMinerInfo } from '../mempool.interfaces'; +import { TransactionExtended, MempoolTransactionExtended, TransactionMinerInfo } from '../mempool.interfaces'; import { IEsploraApi } from './bitcoin/esplora-api.interface'; import { Common } from './common'; import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory'; +import * as bitcoinjs from 'bitcoinjs-lib'; class TransactionUtils { constructor() { } @@ -22,19 +23,27 @@ class TransactionUtils { } /** - * @param txId - * @param addPrevouts - * @param lazyPrevouts + * @param txId + * @param addPrevouts + * @param lazyPrevouts * @param forceCore - See https://github.com/mempool/mempool/issues/2904 */ - public async $getTransactionExtended(txId: string, addPrevouts = false, lazyPrevouts = false, forceCore = false): Promise { + public async $getTransactionExtended(txId: string, addPrevouts = false, lazyPrevouts = false, forceCore = false, addMempoolData = false): Promise { let transaction: IEsploraApi.Transaction; if (forceCore === true) { transaction = await bitcoinCoreApi.$getRawTransaction(txId, true); } else { transaction = await bitcoinApi.$getRawTransaction(txId, false, addPrevouts, lazyPrevouts); } - return this.extendTransaction(transaction); + if (addMempoolData || !transaction?.status?.confirmed) { + return this.extendMempoolTransaction(transaction); + } else { + return this.extendTransaction(transaction); + } + } + + public async $getMempoolTransactionExtended(txId: string, addPrevouts = false, lazyPrevouts = false, forceCore = false): Promise { + return (await this.$getTransactionExtended(txId, addPrevouts, lazyPrevouts, forceCore, true)) as MempoolTransactionExtended; } private extendTransaction(transaction: IEsploraApi.Transaction): TransactionExtended { @@ -50,7 +59,30 @@ class TransactionUtils { feePerVsize: feePerVbytes, effectiveFeePerVsize: feePerVbytes, }, transaction); - if (!transaction.status.confirmed) { + if (!transaction?.status?.confirmed) { + transactionExtended.firstSeen = Math.round((new Date().getTime() / 1000)); + } + return transactionExtended; + } + + public extendMempoolTransaction(transaction: IEsploraApi.Transaction): MempoolTransactionExtended { + const vsize = Math.ceil(transaction.weight / 4); + const sigops = this.countSigops(transaction); + // https://github.com/bitcoin/bitcoin/blob/e9262ea32a6e1d364fb7974844fadc36f931f8c6/src/policy/policy.cpp#L295-L298 + const adjustedVsize = Math.max(vsize, sigops * 5); // adjusted vsize = Max(weight, sigops * bytes_per_sigop) / witness_scale_factor + const feePerVbytes = Math.max(Common.isLiquid() ? 0.1 : 1, + (transaction.fee || 0) / vsize); + const adjustedFeePerVsize = Math.max(Common.isLiquid() ? 0.1 : 1, + (transaction.fee || 0) / adjustedVsize); + const transactionExtended: MempoolTransactionExtended = Object.assign(transaction, { + vsize: Math.round(transaction.weight / 4), + adjustedVsize, + sigops, + feePerVsize: feePerVbytes, + adjustedFeePerVsize: adjustedFeePerVsize, + effectiveFeePerVsize: adjustedFeePerVsize, + }); + if (!transaction?.status?.confirmed) { transactionExtended.firstSeen = Math.round((new Date().getTime() / 1000)); } return transactionExtended; @@ -63,6 +95,64 @@ class TransactionUtils { } return str; } + + public countScriptSigops(script: string, isRawScript: boolean = false, witness: boolean = false): number { + let sigops = 0; + // count OP_CHECKSIG and OP_CHECKSIGVERIFY + sigops += (script.match(/OP_CHECKSIG/g)?.length || 0); + + // count OP_CHECKMULTISIG and OP_CHECKMULTISIGVERIFY + if (isRawScript) { + // in scriptPubKey or scriptSig, always worth 20 + sigops += 20 * (script.match(/OP_CHECKMULTISIG/g)?.length || 0); + } else { + // in redeem scripts and witnesses, worth N if preceded by OP_N, 20 otherwise + const matches = script.matchAll(/(?:OP_(\d+))? OP_CHECKMULTISIG/g); + for (const match of matches) { + const n = parseInt(match[1]); + if (Number.isInteger(n)) { + sigops += n; + } else { + sigops += 20; + } + } + } + + return witness ? sigops : (sigops * 4); + } + + public countSigops(transaction: IEsploraApi.Transaction): number { + let sigops = 0; + + for (const input of transaction.vin) { + if (input.scriptsig_asm) { + sigops += this.countScriptSigops(input.scriptsig_asm, true); + } + if (input.prevout) { + switch (true) { + case input.prevout.scriptpubkey_type === 'p2sh' && input.witness?.length === 2 && input.scriptsig && input.scriptsig.startsWith('160014'): + case input.prevout.scriptpubkey_type === 'v0_p2wpkh': + sigops += 1; + break; + + case input.prevout?.scriptpubkey_type === 'p2sh' && input.witness?.length && input.scriptsig && input.scriptsig.startsWith('220020'): + case input.prevout.scriptpubkey_type === 'v0_p2wsh': + if (input.witness?.length) { + sigops += this.countScriptSigops(bitcoinjs.script.toASM(Buffer.from(input.witness[input.witness.length - 1], 'hex')), false, true); + } + break; + } + } + } + + for (const output of transaction.vout) { + if (output.scriptpubkey_asm) { + sigops += this.countScriptSigops(output.scriptpubkey_asm, true); + } + } + + return sigops; + } } export default new TransactionUtils(); diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index ca1bb01ff..89b819b08 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -1,7 +1,7 @@ import logger from '../logger'; import * as WebSocket from 'ws'; import { - BlockExtended, TransactionExtended, WebsocketResponse, + BlockExtended, TransactionExtended, MempoolTransactionExtended, WebsocketResponse, OptimizedStatistic, ILoadingIndicators } from '../mempool.interfaces'; import blocks from './blocks'; @@ -122,7 +122,7 @@ class WebsocketHandler { } else { // tx.prevout is missing from transactions when in bitcoind mode try { - const fullTx = await transactionUtils.$getTransactionExtended(tx.txid, true); + const fullTx = await transactionUtils.$getMempoolTransactionExtended(tx.txid, true); response['tx'] = fullTx; } catch (e) { logger.debug('Error finding transaction: ' + (e instanceof Error ? e.message : e)); @@ -130,7 +130,7 @@ class WebsocketHandler { } } else { try { - const fullTx = await transactionUtils.$getTransactionExtended(client['track-tx'], true); + const fullTx = await transactionUtils.$getMempoolTransactionExtended(client['track-tx'], true); response['tx'] = fullTx; } catch (e) { logger.debug('Error finding transaction. ' + (e instanceof Error ? e.message : e)); @@ -301,8 +301,8 @@ class WebsocketHandler { }); } - async $handleMempoolChange(newMempool: { [txid: string]: TransactionExtended }, - newTransactions: TransactionExtended[], deletedTransactions: TransactionExtended[]): Promise { + async $handleMempoolChange(newMempool: { [txid: string]: MempoolTransactionExtended }, + newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[]): Promise { if (!this.wss) { throw new Error('WebSocket.Server is not set'); } @@ -399,7 +399,7 @@ class WebsocketHandler { if (tx) { if (config.MEMPOOL.BACKEND !== 'esplora') { try { - const fullTx = await transactionUtils.$getTransactionExtended(tx.txid, true); + const fullTx = await transactionUtils.$getMempoolTransactionExtended(tx.txid, true); response['tx'] = JSON.stringify(fullTx); } catch (e) { logger.debug('Error finding transaction in mempool: ' + (e instanceof Error ? e.message : e)); @@ -419,7 +419,7 @@ class WebsocketHandler { if (someVin) { if (config.MEMPOOL.BACKEND !== 'esplora') { try { - const fullTx = await transactionUtils.$getTransactionExtended(tx.txid, true); + const fullTx = await transactionUtils.$getMempoolTransactionExtended(tx.txid, true); foundTransactions.push(fullTx); } catch (e) { logger.debug('Error finding transaction in mempool: ' + (e instanceof Error ? e.message : e)); @@ -433,7 +433,7 @@ class WebsocketHandler { if (someVout) { if (config.MEMPOOL.BACKEND !== 'esplora') { try { - const fullTx = await transactionUtils.$getTransactionExtended(tx.txid, true); + const fullTx = await transactionUtils.$getMempoolTransactionExtended(tx.txid, true); foundTransactions.push(fullTx); } catch (e) { logger.debug('Error finding transaction in mempool: ' + (e instanceof Error ? e.message : e)); diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index 14114fa2c..e7fba439d 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -88,6 +88,12 @@ export interface TransactionExtended extends IEsploraApi.Transaction { uid?: number; } +export interface MempoolTransactionExtended extends TransactionExtended { + sigops: number; + adjustedVsize: number; + adjustedFeePerVsize: number; +} + export interface AuditTransaction { uid: number; fee: number;