diff --git a/backend/src/api/acceleration.ts b/backend/src/api/acceleration.ts index fff8ffd3f..20c0e3b31 100644 --- a/backend/src/api/acceleration.ts +++ b/backend/src/api/acceleration.ts @@ -238,7 +238,7 @@ class AccelerationCosts { private convertToGraphTx(tx: MempoolTransactionExtended): GraphTx { return { txid: tx.txid, - vsize: tx.vsize, + vsize: Math.ceil(tx.weight / 4), weight: tx.weight, fees: { base: 0, // dummy @@ -256,7 +256,7 @@ class AccelerationCosts { ancestor: tx.fees.base, }, ancestorcount: 1, - ancestorsize: tx.vsize, + ancestorsize: Math.ceil(tx.weight / 4), ancestors: new Map(), ancestorRate: 0, individualRate: 0, @@ -493,7 +493,7 @@ interface MinerTransaction extends TemplateTransaction { * Build a block using an approximation of the transaction selection algorithm from Bitcoin Core * (see BlockAssembler in https://github.com/bitcoin/bitcoin/blob/master/src/node/miner.cpp) */ -function makeBlockTemplate(candidates: IEsploraApi.Transaction[], accelerations: Acceleration[], maxBlocks: number = 8, weightLimit: number = BLOCK_WEIGHT_UNITS, sigopLimit: number = BLOCK_SIGOPS): TemplateTransaction[] { +export function makeBlockTemplate(candidates: IEsploraApi.Transaction[], accelerations: Acceleration[], maxBlocks: number = 8, weightLimit: number = BLOCK_WEIGHT_UNITS, sigopLimit: number = BLOCK_SIGOPS): TemplateTransaction[] { const auditPool: Map = new Map(); const mempoolArray: MinerTransaction[] = []; diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 5365b61dc..38d65784f 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository'; import { RowDataPacket } from 'mysql2'; class DatabaseMigration { - private static currentVersion = 73; + private static currentVersion = 74; private queryTimeout = 3600_000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -619,6 +619,11 @@ class DatabaseMigration { this.uniqueLog(logger.notice, `'accelerations' table has been truncated`); await this.updateToSchemaVersion(73); } + + if (databaseSchemaVersion < 74 && config.MEMPOOL.NETWORK === 'mainnet') { + await this.$executeQuery(`INSERT INTO state(name, number) VALUE ('last_acceleration_block', 0);`); + await this.updateToSchemaVersion(74); + } } /** diff --git a/backend/src/api/services/acceleration.ts b/backend/src/api/services/acceleration.ts index 99bb963ee..f22959f3f 100644 --- a/backend/src/api/services/acceleration.ts +++ b/backend/src/api/services/acceleration.ts @@ -7,7 +7,26 @@ export interface Acceleration { txid: string, feeDelta: number, pools: number[], -} +}; + +export interface AccelerationHistory { + txid: string, + status: string, + feePaid: number, + added: number, + lastUpdated: number, + baseFee: number, + vsizeFee: number, + effectiveFee: number, + effectiveVsize: number, + feeDelta: number, + blockHash: string, + blockHeight: number, + pools: { + pool_unique_id: number, + username: string, + }[], +}; class AccelerationApi { public async $fetchAccelerations(): Promise { @@ -24,6 +43,27 @@ class AccelerationApi { } } + public async $fetchAccelerationHistory(page?: number, status?: string): Promise { + if (config.MEMPOOL_SERVICES.ACCELERATIONS) { + try { + const response = await axios.get(`${config.MEMPOOL_SERVICES.API}/accelerator/accelerations/history`, { + responseType: 'json', + timeout: 10000, + params: { + page, + status, + } + }); + return response.data as AccelerationHistory[]; + } catch (e) { + logger.warn('Failed to fetch acceleration history from the mempool services backend: ' + (e instanceof Error ? e.message : e)); + return null; + } + } else { + return []; + } + } + public isAcceleratedBlock(block: BlockExtended, accelerations: Acceleration[]): boolean { let anyAccelerated = false; for (let i = 0; i < accelerations.length && !anyAccelerated; i++) { diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 6711c88fb..925b676f1 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -24,7 +24,6 @@ import { ApiPrice } from '../repositories/PricesRepository'; import accelerationApi from './services/acceleration'; import mempool from './mempool'; import statistics from './statistics/statistics'; -import accelerationCosts from './acceleration'; import accelerationRepository from '../repositories/AccelerationRepository'; import bitcoinApi from './bitcoin/bitcoin-api-factory'; @@ -742,25 +741,8 @@ class WebsocketHandler { const isAccelerated = config.MEMPOOL_SERVICES.ACCELERATIONS && accelerationApi.isAcceleratedBlock(block, Object.values(mempool.getAccelerations())); - - if (isAccelerated) { - const blockTxs: { [txid: string]: MempoolTransactionExtended } = {}; - for (const tx of transactions) { - blockTxs[tx.txid] = tx; - } - const accelerations = Object.values(mempool.getAccelerations()); - const boostRate = accelerationCosts.calculateBoostRate( - accelerations.map(acc => ({ txid: acc.txid, max_bid: acc.feeDelta })), - transactions - ); - for (const acc of accelerations) { - if (blockTxs[acc.txid]) { - const tx = blockTxs[acc.txid]; - const accelerationInfo = accelerationCosts.getAccelerationInfo(tx, boostRate, transactions); - accelerationRepository.$saveAcceleration(accelerationInfo, block, block.extras.pool.id); - } - } - } + const accelerations = Object.values(mempool.getAccelerations()); + await accelerationRepository.$indexAccelerationsForBlock(block, accelerations, transactions); const rbfTransactions = Common.findMinedRbfTransactions(transactions, memPool.getSpendMap()); memPool.handleMinedRbfTransactions(rbfTransactions); diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index dcb91d010..37e9ad4f9 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -8,6 +8,7 @@ import priceUpdater from './tasks/price-updater'; import PricesRepository from './repositories/PricesRepository'; import config from './config'; import auditReplicator from './replication/AuditReplication'; +import AccelerationRepository from './repositories/AccelerationRepository'; export interface CoreIndex { name: string; @@ -185,6 +186,7 @@ class Indexer { await blocks.$generateCPFPDatabase(); await blocks.$generateAuditStats(); await auditReplicator.$sync(); + await AccelerationRepository.$indexPastAccelerations(); // do not wait for classify blocks to finish blocks.$classifyBlocks(); } catch (e) { diff --git a/backend/src/repositories/AccelerationRepository.ts b/backend/src/repositories/AccelerationRepository.ts index c98d007c5..bea6a394a 100644 --- a/backend/src/repositories/AccelerationRepository.ts +++ b/backend/src/repositories/AccelerationRepository.ts @@ -1,10 +1,16 @@ -import { AccelerationInfo } from '../api/acceleration'; -import { ResultSetHeader, RowDataPacket } from 'mysql2'; +import { AccelerationInfo, makeBlockTemplate } from '../api/acceleration'; +import { RowDataPacket } from 'mysql2'; import DB from '../database'; import logger from '../logger'; import { IEsploraApi } from '../api/bitcoin/esplora-api.interface'; import { Common } from '../api/common'; import config from '../config'; +import blocks from '../api/blocks'; +import accelerationApi, { Acceleration } from '../api/services/acceleration'; +import accelerationCosts from '../api/acceleration'; +import bitcoinApi from '../api/bitcoin/bitcoin-api-factory'; +import transactionUtils from '../api/transaction-utils'; +import { BlockExtended, MempoolTransactionExtended } from '../mempool.interfaces'; export interface PublicAcceleration { txid: string, @@ -21,19 +27,15 @@ export interface PublicAcceleration { } class AccelerationRepository { + private bidBoostV2Activated = 831580; + public async $saveAcceleration(acceleration: AccelerationInfo, block: IEsploraApi.Block, pool_id: number): Promise { try { await DB.query(` INSERT INTO accelerations(txid, added, height, pool, effective_vsize, effective_fee, boost_rate, boost_cost) VALUE (?, FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE - added = FROM_UNIXTIME(?), - height = ?, - pool = ?, - effective_vsize = ?, - effective_fee = ?, - boost_rate = ?, - boost_cost = ? + height = ? `, [ acceleration.txSummary.txid, block.timestamp, @@ -41,13 +43,9 @@ class AccelerationRepository { pool_id, acceleration.txSummary.effectiveVsize, acceleration.txSummary.effectiveFee, - acceleration.targetFeeRate, acceleration.cost, - block.timestamp, + acceleration.targetFeeRate, + acceleration.cost, block.height, - pool_id, - acceleration.txSummary.effectiveVsize, - acceleration.txSummary.effectiveFee, - acceleration.targetFeeRate, acceleration.cost, ]); } catch (e: any) { logger.err(`Cannot save acceleration (${acceleration.txSummary.txid}) into db. Reason: ` + (e instanceof Error ? e.message : e)); @@ -119,6 +117,167 @@ class AccelerationRepository { throw e; } } + + public async $getLastSyncedHeight(): Promise { + try { + const [rows] = await DB.query(` + SELECT * FROM state + WHERE name = 'last_acceleration_block' + `); + if (rows?.['length']) { + return rows[0].number; + } + } catch (e: any) { + logger.err(`Cannot find last acceleration sync height. Reason: ` + (e instanceof Error ? e.message : e)); + } + return 0; + } + + private async $setLastSyncedHeight(height: number): Promise { + try { + await DB.query(` + UPDATE state + SET number = ? + WHERE name = 'last_acceleration_block' + `, [height]); + } catch (e: any) { + logger.err(`Cannot update last acceleration sync height. Reason: ` + (e instanceof Error ? e.message : e)); + } + } + + public async $indexAccelerationsForBlock(block: BlockExtended, accelerations: Acceleration[], transactions: MempoolTransactionExtended[]): Promise { + const blockTxs: { [txid: string]: MempoolTransactionExtended } = {}; + for (const tx of transactions) { + blockTxs[tx.txid] = tx; + } + const successfulAccelerations = accelerations.filter(acc => acc.pools.includes(block.extras.pool.id)); + let boostRate: number | null = null; + for (const acc of successfulAccelerations) { + if (boostRate === null) { + boostRate = accelerationCosts.calculateBoostRate( + accelerations.map(acc => ({ txid: acc.txid, max_bid: acc.feeDelta })), + transactions + ); + } + if (blockTxs[acc.txid]) { + const tx = blockTxs[acc.txid]; + const accelerationInfo = accelerationCosts.getAccelerationInfo(tx, boostRate, transactions); + accelerationInfo.cost = Math.max(0, Math.min(acc.feeDelta, accelerationInfo.cost)); + this.$saveAcceleration(accelerationInfo, block, block.extras.pool.id); + } + } + const lastSyncedHeight = await this.$getLastSyncedHeight(); + // if we've missed any blocks, let the indexer catch up from the last synced height on the next run + if (block.height === lastSyncedHeight + 1) { + await this.$setLastSyncedHeight(block.height); + } + } + + /** + * [INDEXING] Backfill missing acceleration data + */ + async $indexPastAccelerations(): Promise { + if (config.MEMPOOL.NETWORK !== 'mainnet' || !config.MEMPOOL_SERVICES.ACCELERATIONS) { + // acceleration history disabled + return; + } + const lastSyncedHeight = await this.$getLastSyncedHeight(); + const currentHeight = blocks.getCurrentBlockHeight(); + if (currentHeight <= lastSyncedHeight) { + // already in sync + return; + } + + logger.debug(`Fetching accelerations between block ${lastSyncedHeight} and ${currentHeight}`); + + // Fetch accelerations from mempool.space since the last synced block; + const accelerationsByBlock = {}; + const blockHashes = {}; + let done = false; + let page = 1; + let count = 0; + try { + while (!done) { + const accelerations = await accelerationApi.$fetchAccelerationHistory(page); + page++; + if (!accelerations?.length) { + done = true; + break; + } + for (const acc of accelerations) { + if (acc.status !== 'mined' && acc.status !== 'completed') { + continue; + } + if (!lastSyncedHeight || acc.blockHeight > lastSyncedHeight) { + if (!accelerationsByBlock[acc.blockHeight]) { + accelerationsByBlock[acc.blockHeight] = []; + blockHashes[acc.blockHeight] = acc.blockHash; + } + accelerationsByBlock[acc.blockHeight].push(acc); + count++; + } else { + done = true; + } + } + } + } catch (e) { + logger.err(`Failed to fetch full acceleration history. Reason: ` + (e instanceof Error ? e.message : e)); + } + + logger.debug(`Indexing ${count} accelerations between block ${lastSyncedHeight} and ${currentHeight}`); + + // process accelerated blocks in order + const heights = Object.keys(accelerationsByBlock).map(key => parseInt(key)).sort((a,b) => a - b); + for (const height of heights) { + const accelerations = accelerationsByBlock[height]; + try { + const block = await blocks.$getBlock(blockHashes[height]) as BlockExtended; + const transactions = (await bitcoinApi.$getTxsForBlock(blockHashes[height])).map(tx => transactionUtils.extendMempoolTransaction(tx)); + + const blockTxs = {}; + for (const tx of transactions) { + blockTxs[tx.txid] = tx; + } + + let boostRate = 0; + // use Bid Boost V2 if active + if (height > this.bidBoostV2Activated) { + boostRate = accelerationCosts.calculateBoostRate( + accelerations.map(acc => ({ txid: acc.txid, max_bid: acc.feeDelta })), + transactions + ); + } else { + // default to Bid Boost V1 (median block fee rate) + const template = makeBlockTemplate( + transactions, + accelerations.map(acc => ({ txid: acc.txid, max_bid: acc.feeDelta })), + 1, + Infinity, + Infinity + ); + const feeStats = Common.calcEffectiveFeeStatistics(template); + boostRate = feeStats.medianFee; + } + for (const acc of accelerations) { + if (blockTxs[acc.txid]) { + const tx = blockTxs[acc.txid]; + const accelerationInfo = accelerationCosts.getAccelerationInfo(tx, boostRate, transactions); + accelerationInfo.cost = Math.max(0, Math.min(acc.feeDelta, accelerationInfo.cost)); + await this.$saveAcceleration(accelerationInfo, block, block.extras.pool.id); + } + } + await this.$setLastSyncedHeight(height); + } catch (e) { + logger.err(`Failed to process accelerations for block ${height}. Reason: ` + (e instanceof Error ? e.message : e)); + return; + } + logger.debug(`Indexed ${accelerations.length} accelerations in block ${height}`); + } + + await this.$setLastSyncedHeight(currentHeight); + + logger.debug(`Indexing accelerations completed`); + } } export default new AccelerationRepository();