diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index 9d8a7e900..9890654a5 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -3,7 +3,6 @@ "ENABLED": true, "NETWORK": "__MEMPOOL_NETWORK__", "BACKEND": "__MEMPOOL_BACKEND__", - "ENABLED": true, "BLOCKS_SUMMARIES_INDEXING": true, "HTTP_PORT": 1, "SPAWN_CLUSTER_PROCS": 2, @@ -28,7 +27,8 @@ "AUDIT": "__MEMPOOL_AUDIT__", "ADVANCED_GBT_AUDIT": "__MEMPOOL_ADVANCED_GBT_AUDIT__", "ADVANCED_GBT_MEMPOOL": "__MEMPOOL_ADVANCED_GBT_MEMPOOL__", - "CPFP_INDEXING": "__MEMPOOL_CPFP_INDEXING__" + "CPFP_INDEXING": "__MEMPOOL_CPFP_INDEXING__", + "MAX_BLOCKS_BULK_QUERY": "__MEMPOOL_MAX_BLOCKS_BULK_QUERY__" }, "CORE_RPC": { "HOST": "__CORE_RPC_HOST__", diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index 8b011d833..1e4c05ae3 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -41,6 +41,7 @@ describe('Mempool Backend Config', () => { ADVANCED_GBT_AUDIT: false, ADVANCED_GBT_MEMPOOL: false, CPFP_INDEXING: false, + MAX_BLOCKS_BULK_QUERY: 0, }); expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true }); diff --git a/backend/src/api/bitcoin/bitcoin-api.ts b/backend/src/api/bitcoin/bitcoin-api.ts index cad11aeda..117245ef8 100644 --- a/backend/src/api/bitcoin/bitcoin-api.ts +++ b/backend/src/api/bitcoin/bitcoin-api.ts @@ -28,6 +28,7 @@ class BitcoinApi implements AbstractBitcoinApi { size: block.size, weight: block.weight, previousblockhash: block.previousblockhash, + medianTime: block.mediantime, }; } diff --git a/backend/src/api/bitcoin/bitcoin.routes.ts b/backend/src/api/bitcoin/bitcoin.routes.ts index ea8154206..78d027663 100644 --- a/backend/src/api/bitcoin/bitcoin.routes.ts +++ b/backend/src/api/bitcoin/bitcoin.routes.ts @@ -95,6 +95,8 @@ class BitcoinRoutes { .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/summary', this.getStrippedBlockTransactions) .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/audit-summary', this.getBlockAuditSummary) .post(config.MEMPOOL.API_URL_PREFIX + 'psbt/addparents', this.postPsbtCompletion) + .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from', this.getBlocksByBulk.bind(this)) + .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from/:to', this.getBlocksByBulk.bind(this)) ; if (config.MEMPOOL.BACKEND !== 'esplora') { @@ -402,6 +404,41 @@ class BitcoinRoutes { } } + private async getBlocksByBulk(req: Request, res: Response) { + try { + if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { // Liquid, Bisq - Not implemented + return res.status(404).send(`This API is only available for Bitcoin networks`); + } + if (config.MEMPOOL.MAX_BLOCKS_BULK_QUERY <= 0) { + return res.status(404).send(`This API is disabled. Set config.MEMPOOL.MAX_BLOCKS_BULK_QUERY to a positive number to enable it.`); + } + if (!Common.indexingEnabled()) { + return res.status(404).send(`Indexing is required for this API`); + } + + const from = parseInt(req.params.from, 10); + if (!req.params.from || from < 0) { + return res.status(400).send(`Parameter 'from' must be a block height (integer)`); + } + const to = req.params.to === undefined ? await bitcoinApi.$getBlockHeightTip() : parseInt(req.params.to, 10); + if (to < 0) { + return res.status(400).send(`Parameter 'to' must be a block height (integer)`); + } + if (from > to) { + return res.status(400).send(`Parameter 'to' must be a higher block height than 'from'`); + } + if ((to - from + 1) > config.MEMPOOL.MAX_BLOCKS_BULK_QUERY) { + return res.status(400).send(`You can only query ${config.MEMPOOL.MAX_BLOCKS_BULK_QUERY} blocks at once.`); + } + + res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); + res.json(await blocks.$getBlocksBetweenHeight(from, to)); + + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + private async getLegacyBlocks(req: Request, res: Response) { try { const returnBlocks: IEsploraApi.Block[] = []; diff --git a/backend/src/api/bitcoin/esplora-api.interface.ts b/backend/src/api/bitcoin/esplora-api.interface.ts index 39f8cfd6f..eaf6476f4 100644 --- a/backend/src/api/bitcoin/esplora-api.interface.ts +++ b/backend/src/api/bitcoin/esplora-api.interface.ts @@ -88,6 +88,7 @@ export namespace IEsploraApi { size: number; weight: number; previousblockhash: string; + medianTime?: number; } export interface Address { diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index d110186f5..204419496 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -25,6 +25,7 @@ import mining from './mining/mining'; import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository'; import PricesRepository from '../repositories/PricesRepository'; import priceUpdater from '../tasks/price-updater'; +import chainTips from './chain-tips'; class Blocks { private blocks: BlockExtended[] = []; @@ -165,33 +166,80 @@ class Blocks { * @returns BlockExtended */ private async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise { - const blockExtended: BlockExtended = Object.assign({ extras: {} }, block); - blockExtended.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0); - blockExtended.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]); - blockExtended.extras.coinbaseRaw = blockExtended.extras.coinbaseTx.vin[0].scriptsig; - blockExtended.extras.usd = priceUpdater.latestPrices.USD; + const blk: BlockExtended = Object.assign({ extras: {} }, block); + blk.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0); + blk.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]); + blk.extras.coinbaseRaw = blk.extras.coinbaseTx.vin[0].scriptsig; + blk.extras.usd = priceUpdater.latestPrices.USD; + blk.extras.medianTimestamp = block.medianTime; + blk.extras.orphans = chainTips.getOrphanedBlocksAtHeight(blk.height); if (block.height === 0) { - blockExtended.extras.medianFee = 0; // 50th percentiles - blockExtended.extras.feeRange = [0, 0, 0, 0, 0, 0, 0]; - blockExtended.extras.totalFees = 0; - blockExtended.extras.avgFee = 0; - blockExtended.extras.avgFeeRate = 0; + blk.extras.medianFee = 0; // 50th percentiles + blk.extras.feeRange = [0, 0, 0, 0, 0, 0, 0]; + blk.extras.totalFees = 0; + blk.extras.avgFee = 0; + blk.extras.avgFeeRate = 0; + blk.extras.utxoSetChange = 0; + blk.extras.avgTxSize = 0; + blk.extras.totalInputs = 0; + blk.extras.totalOutputs = 1; + blk.extras.totalOutputAmt = 0; + blk.extras.segwitTotalTxs = 0; + blk.extras.segwitTotalSize = 0; + blk.extras.segwitTotalWeight = 0; } else { - const stats = await bitcoinClient.getBlockStats(block.id, [ - 'feerate_percentiles', 'minfeerate', 'maxfeerate', 'totalfee', 'avgfee', 'avgfeerate' - ]); - blockExtended.extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles - blockExtended.extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat(); - blockExtended.extras.totalFees = stats.totalfee; - blockExtended.extras.avgFee = stats.avgfee; - blockExtended.extras.avgFeeRate = stats.avgfeerate; + const stats = await bitcoinClient.getBlockStats(block.id); + blk.extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles + blk.extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat(); + blk.extras.totalFees = stats.totalfee; + blk.extras.avgFee = stats.avgfee; + blk.extras.avgFeeRate = stats.avgfeerate; + blk.extras.utxoSetChange = stats.utxo_increase; + blk.extras.avgTxSize = Math.round(stats.total_size / stats.txs * 100) * 0.01; + blk.extras.totalInputs = stats.ins; + blk.extras.totalOutputs = stats.outs; + blk.extras.totalOutputAmt = stats.total_out; + blk.extras.segwitTotalTxs = stats.swtxs; + blk.extras.segwitTotalSize = stats.swtotal_size; + blk.extras.segwitTotalWeight = stats.swtotal_weight; + } + + if (Common.blocksSummariesIndexingEnabled()) { + blk.extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(block.id); + if (blk.extras.feePercentiles !== null) { + blk.extras.medianFeeAmt = blk.extras.feePercentiles[3]; + } + } + + blk.extras.virtualSize = block.weight / 4.0; + if (blk.extras.coinbaseTx.vout.length > 0) { + blk.extras.coinbaseAddress = blk.extras.coinbaseTx.vout[0].scriptpubkey_address ?? null; + blk.extras.coinbaseSignature = blk.extras.coinbaseTx.vout[0].scriptpubkey_asm ?? null; + blk.extras.coinbaseSignatureAscii = transactionUtils.hex2ascii(blk.extras.coinbaseTx.vin[0].scriptsig) ?? null; + } else { + blk.extras.coinbaseAddress = null; + blk.extras.coinbaseSignature = null; + blk.extras.coinbaseSignatureAscii = null; + } + + const header = await bitcoinClient.getBlockHeader(block.id, false); + blk.extras.header = header; + + const coinStatsIndex = indexer.isCoreIndexReady('coinstatsindex'); + if (coinStatsIndex !== null && coinStatsIndex.best_block_height >= block.height) { + const txoutset = await bitcoinClient.getTxoutSetinfo('none', block.height); + blk.extras.utxoSetSize = txoutset.txouts, + blk.extras.totalInputAmt = Math.round(txoutset.block_info.prevout_spent * 100000000); + } else { + blk.extras.utxoSetSize = null; + blk.extras.totalInputAmt = null; } if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) { let pool: PoolTag; - if (blockExtended.extras?.coinbaseTx !== undefined) { - pool = await this.$findBlockMiner(blockExtended.extras?.coinbaseTx); + if (blk.extras?.coinbaseTx !== undefined) { + pool = await this.$findBlockMiner(blk.extras?.coinbaseTx); } else { if (config.DATABASE.ENABLED === true) { pool = await poolsRepository.$getUnknownPool(); @@ -201,10 +249,10 @@ class Blocks { } if (!pool) { // We should never have this situation in practise - logger.warn(`Cannot assign pool to block ${blockExtended.height} and 'unknown' pool does not exist. ` + + logger.warn(`Cannot assign pool to block ${blk.height} and 'unknown' pool does not exist. ` + `Check your "pools" table entries`); } else { - blockExtended.extras.pool = { + blk.extras.pool = { id: pool.id, name: pool.name, slug: pool.slug, @@ -214,12 +262,12 @@ class Blocks { if (config.MEMPOOL.AUDIT) { const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(block.id); if (auditScore != null) { - blockExtended.extras.matchRate = auditScore.matchRate; + blk.extras.matchRate = auditScore.matchRate; } } } - return blockExtended; + return blk; } /** @@ -500,6 +548,7 @@ class Blocks { } else { this.currentBlockHeight++; logger.debug(`New block found (#${this.currentBlockHeight})!`); + await chainTips.updateOrphanedBlocks(); } const blockHash = await bitcoinApi.$getBlockHash(this.currentBlockHeight); @@ -688,7 +737,6 @@ class Blocks { } public async $getBlocks(fromHeight?: number, limit: number = 15): Promise { - let currentHeight = fromHeight !== undefined ? fromHeight : this.currentBlockHeight; if (currentHeight > this.currentBlockHeight) { limit -= currentHeight - this.currentBlockHeight; @@ -728,6 +776,113 @@ class Blocks { return returnBlocks; } + /** + * Used for bulk block data query + * + * @param fromHeight + * @param toHeight + */ + public async $getBlocksBetweenHeight(fromHeight: number, toHeight: number): Promise { + if (!Common.indexingEnabled()) { + return []; + } + + const blocks: any[] = []; + + while (fromHeight <= toHeight) { + let block: any = await blocksRepository.$getBlockByHeight(fromHeight); + if (!block) { + await this.$indexBlock(fromHeight); + block = await blocksRepository.$getBlockByHeight(fromHeight); + if (!block) { + continue; + } + } + + // Cleanup fields before sending the response + const cleanBlock: any = { + height: block.height ?? null, + hash: block.id ?? null, + timestamp: block.blockTimestamp ?? null, + median_timestamp: block.medianTime ?? null, + previous_block_hash: block.previousblockhash ?? null, + difficulty: block.difficulty ?? null, + header: block.header ?? null, + version: block.version ?? null, + bits: block.bits ?? null, + nonce: block.nonce ?? null, + size: block.size ?? null, + weight: block.weight ?? null, + tx_count: block.tx_count ?? null, + merkle_root: block.merkle_root ?? null, + reward: block.reward ?? null, + total_fee_amt: block.fees ?? null, + avg_fee_amt: block.avg_fee ?? null, + median_fee_amt: block.median_fee_amt ?? null, + fee_amt_percentiles: block.fee_percentiles ?? null, + avg_fee_rate: block.avg_fee_rate ?? null, + median_fee_rate: block.median_fee ?? null, + fee_rate_percentiles: block.fee_span ?? null, + total_inputs: block.total_inputs ?? null, + total_input_amt: block.total_input_amt ?? null, + total_outputs: block.total_outputs ?? null, + total_output_amt: block.total_output_amt ?? null, + segwit_total_txs: block.segwit_total_txs ?? null, + segwit_total_size: block.segwit_total_size ?? null, + segwit_total_weight: block.segwit_total_weight ?? null, + avg_tx_size: block.avg_tx_size ?? null, + utxoset_change: block.utxoset_change ?? null, + utxoset_size: block.utxoset_size ?? null, + coinbase_raw: block.coinbase_raw ?? null, + coinbase_address: block.coinbase_address ?? null, + coinbase_signature: block.coinbase_signature ?? null, + coinbase_signature_ascii: block.coinbase_signature_ascii ?? null, + pool_slug: block.pool_slug ?? null, + }; + + if (Common.blocksSummariesIndexingEnabled() && cleanBlock.fee_amt_percentiles === null) { + cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash); + if (cleanBlock.fee_amt_percentiles === null) { + const block = await bitcoinClient.getBlock(cleanBlock.hash, 2); + const summary = this.summarizeBlock(block); + await BlocksSummariesRepository.$saveSummary({ height: block.height, mined: summary }); + cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash); + } + if (cleanBlock.fee_amt_percentiles !== null) { + cleanBlock.median_fee_amt = cleanBlock.fee_amt_percentiles[3]; + } + } + + cleanBlock.fee_amt_percentiles = { + 'min': cleanBlock.fee_amt_percentiles[0], + 'perc_10': cleanBlock.fee_amt_percentiles[1], + 'perc_25': cleanBlock.fee_amt_percentiles[2], + 'perc_50': cleanBlock.fee_amt_percentiles[3], + 'perc_75': cleanBlock.fee_amt_percentiles[4], + 'perc_90': cleanBlock.fee_amt_percentiles[5], + 'max': cleanBlock.fee_amt_percentiles[6], + }; + cleanBlock.fee_rate_percentiles = { + 'min': cleanBlock.fee_rate_percentiles[0], + 'perc_10': cleanBlock.fee_rate_percentiles[1], + 'perc_25': cleanBlock.fee_rate_percentiles[2], + 'perc_50': cleanBlock.fee_rate_percentiles[3], + 'perc_75': cleanBlock.fee_rate_percentiles[4], + 'perc_90': cleanBlock.fee_rate_percentiles[5], + 'max': cleanBlock.fee_rate_percentiles[6], + }; + + // Re-org can happen after indexing so we need to always get the + // latest state from core + cleanBlock.orphans = chainTips.getOrphanedBlocksAtHeight(cleanBlock.height); + + blocks.push(cleanBlock); + fromHeight++; + } + + return blocks; + } + public async $getBlockAuditSummary(hash: string): Promise { let summary; if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) { diff --git a/backend/src/api/chain-tips.ts b/backend/src/api/chain-tips.ts new file mode 100644 index 000000000..3384ebb19 --- /dev/null +++ b/backend/src/api/chain-tips.ts @@ -0,0 +1,57 @@ +import logger from "../logger"; +import bitcoinClient from "./bitcoin/bitcoin-client"; + +export interface ChainTip { + height: number; + hash: string; + branchlen: number; + status: 'invalid' | 'active' | 'valid-fork' | 'valid-headers' | 'headers-only'; +}; + +export interface OrphanedBlock { + height: number; + hash: string; + status: 'valid-fork' | 'valid-headers' | 'headers-only'; +} + +class ChainTips { + private chainTips: ChainTip[] = []; + private orphanedBlocks: OrphanedBlock[] = []; + + public async updateOrphanedBlocks(): Promise { + try { + this.chainTips = await bitcoinClient.getChainTips(); + this.orphanedBlocks = []; + + for (const chain of this.chainTips) { + if (chain.status === 'valid-fork' || chain.status === 'valid-headers') { + let block = await bitcoinClient.getBlock(chain.hash); + while (block && block.confirmations === -1) { + this.orphanedBlocks.push({ + height: block.height, + hash: block.hash, + status: chain.status + }); + block = await bitcoinClient.getBlock(block.previousblockhash); + } + } + } + + logger.debug(`Updated orphaned blocks cache. Found ${this.orphanedBlocks.length} orphaned blocks`); + } catch (e) { + logger.err(`Cannot get fetch orphaned blocks. Reason: ${e instanceof Error ? e.message : e}`); + } + } + + public getOrphanedBlocksAtHeight(height: number): OrphanedBlock[] { + const orphans: OrphanedBlock[] = []; + for (const block of this.orphanedBlocks) { + if (block.height === height) { + orphans.push(block); + } + } + return orphans; + } +} + +export default new ChainTips(); \ No newline at end of file diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 6e4221857..e732d15a5 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository'; import { RowDataPacket } from 'mysql2'; class DatabaseMigration { - private static currentVersion = 54; + private static currentVersion = 55; private queryTimeout = 3600_000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -483,6 +483,13 @@ class DatabaseMigration { } await this.updateToSchemaVersion(54); } + + if (databaseSchemaVersion < 55) { + await this.$executeQuery(this.getAdditionalBlocksDataQuery()); + this.uniqueLog(logger.notice, this.blocksTruncatedMessage); + await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index + await this.updateToSchemaVersion(55); + } } /** @@ -756,6 +763,28 @@ class DatabaseMigration { ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; } + private getAdditionalBlocksDataQuery(): string { + return `ALTER TABLE blocks + ADD median_timestamp timestamp NOT NULL, + ADD coinbase_address varchar(100) NULL, + ADD coinbase_signature varchar(500) NULL, + ADD coinbase_signature_ascii varchar(500) NULL, + ADD avg_tx_size double unsigned NOT NULL, + ADD total_inputs int unsigned NOT NULL, + ADD total_outputs int unsigned NOT NULL, + ADD total_output_amt bigint unsigned NOT NULL, + ADD fee_percentiles longtext NULL, + ADD median_fee_amt int unsigned NULL, + ADD segwit_total_txs int unsigned NOT NULL, + ADD segwit_total_size int unsigned NOT NULL, + ADD segwit_total_weight int unsigned NOT NULL, + ADD header varchar(160) NOT NULL, + ADD utxoset_change int NOT NULL, + ADD utxoset_size int unsigned NULL, + ADD total_input_amt bigint unsigned NULL + `; + } + private getCreateDailyStatsTableQuery(): string { return `CREATE TABLE IF NOT EXISTS hashrates ( hashrate_timestamp timestamp NOT NULL, diff --git a/backend/src/api/disk-cache.ts b/backend/src/api/disk-cache.ts index cf40d6952..a75fd43cc 100644 --- a/backend/src/api/disk-cache.ts +++ b/backend/src/api/disk-cache.ts @@ -9,7 +9,7 @@ import { TransactionExtended } from '../mempool.interfaces'; import { Common } from './common'; class DiskCache { - private cacheSchemaVersion = 1; + private cacheSchemaVersion = 2; private static FILE_NAME = config.MEMPOOL.CACHE_DIR + '/cache.json'; private static FILE_NAMES = config.MEMPOOL.CACHE_DIR + '/cache{number}.json'; diff --git a/backend/src/api/mining/mining.ts b/backend/src/api/mining/mining.ts index edcb5b2e5..f33a68dcb 100644 --- a/backend/src/api/mining/mining.ts +++ b/backend/src/api/mining/mining.ts @@ -172,7 +172,7 @@ class Mining { } /** - * [INDEXING] Generate weekly mining pool hashrate history + * Generate weekly mining pool hashrate history */ public async $generatePoolHashrateHistory(): Promise { const now = new Date(); @@ -279,7 +279,7 @@ class Mining { } /** - * [INDEXING] Generate daily hashrate data + * Generate daily hashrate data */ public async $generateNetworkHashrateHistory(): Promise { // We only run this once a day around midnight @@ -459,7 +459,7 @@ class Mining { /** * Create a link between blocks and the latest price at when they were mined */ - public async $indexBlockPrices() { + public async $indexBlockPrices(): Promise { if (this.blocksPriceIndexingRunning === true) { return; } @@ -520,6 +520,41 @@ class Mining { this.blocksPriceIndexingRunning = false; } + /** + * Index core coinstatsindex + */ + public async $indexCoinStatsIndex(): Promise { + let timer = new Date().getTime() / 1000; + let totalIndexed = 0; + + const blockchainInfo = await bitcoinClient.getBlockchainInfo(); + let currentBlockHeight = blockchainInfo.blocks; + + while (currentBlockHeight > 0) { + const indexedBlocks = await BlocksRepository.$getBlocksMissingCoinStatsIndex( + currentBlockHeight, currentBlockHeight - 10000); + + for (const block of indexedBlocks) { + const txoutset = await bitcoinClient.getTxoutSetinfo('none', block.height); + await BlocksRepository.$updateCoinStatsIndexData(block.hash, txoutset.txouts, + Math.round(txoutset.block_info.prevout_spent * 100000000)); + ++totalIndexed; + + const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer); + if (elapsedSeconds > 5) { + logger.info(`Indexing coinstatsindex data for block #${block.height}. Indexed ${totalIndexed} blocks.`, logger.tags.mining); + timer = new Date().getTime() / 1000; + } + } + + currentBlockHeight -= 10000; + } + + if (totalIndexed) { + logger.info(`Indexing missing coinstatsindex data completed`, logger.tags.mining); + } + } + private getDateMidnight(date: Date): Date { date.setUTCHours(0); date.setUTCMinutes(0); diff --git a/backend/src/api/transaction-utils.ts b/backend/src/api/transaction-utils.ts index fb5aeea42..fb69419fc 100644 --- a/backend/src/api/transaction-utils.ts +++ b/backend/src/api/transaction-utils.ts @@ -14,6 +14,7 @@ class TransactionUtils { vout: tx.vout .map((vout) => ({ scriptpubkey_address: vout.scriptpubkey_address, + scriptpubkey_asm: vout.scriptpubkey_asm, value: vout.value })) .filter((vout) => vout.value) diff --git a/backend/src/config.ts b/backend/src/config.ts index 2cda8d85b..ecd5c80aa 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -32,6 +32,7 @@ interface IConfig { ADVANCED_GBT_AUDIT: boolean; ADVANCED_GBT_MEMPOOL: boolean; CPFP_INDEXING: boolean; + MAX_BLOCKS_BULK_QUERY: number; }; ESPLORA: { REST_API_URL: string; @@ -153,6 +154,7 @@ const defaults: IConfig = { 'ADVANCED_GBT_AUDIT': false, 'ADVANCED_GBT_MEMPOOL': false, 'CPFP_INDEXING': false, + 'MAX_BLOCKS_BULK_QUERY': 0, }, 'ESPLORA': { 'REST_API_URL': 'http://127.0.0.1:3000', diff --git a/backend/src/database.ts b/backend/src/database.ts index c2fb0980b..a504eb0fa 100644 --- a/backend/src/database.ts +++ b/backend/src/database.ts @@ -24,7 +24,8 @@ import { FieldPacket, OkPacket, PoolOptions, ResultSetHeader, RowDataPacket } fr private checkDBFlag() { if (config.DATABASE.ENABLED === false) { - logger.err('Trying to use DB feature but config.DATABASE.ENABLED is set to false, please open an issue'); + const stack = new Error().stack; + logger.err(`Trying to use DB feature but config.DATABASE.ENABLED is set to false, please open an issue.\nStack trace: ${stack}}`); } } diff --git a/backend/src/index.ts b/backend/src/index.ts index 919c039c3..6ea3ddc43 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -36,6 +36,8 @@ import bitcoinRoutes from './api/bitcoin/bitcoin.routes'; import fundingTxFetcher from './tasks/lightning/sync-tasks/funding-tx-fetcher'; import forensicsService from './tasks/lightning/forensics.service'; import priceUpdater from './tasks/price-updater'; +import mining from './api/mining/mining'; +import chainTips from './api/chain-tips'; import { AxiosError } from 'axios'; class Server { @@ -133,6 +135,7 @@ class Server { } priceUpdater.$run(); + await chainTips.updateOrphanedBlocks(); this.setUpHttpApiRoutes(); diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index 22f3ce319..41c8024e0 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -8,18 +8,67 @@ import bitcoinClient from './api/bitcoin/bitcoin-client'; import priceUpdater from './tasks/price-updater'; import PricesRepository from './repositories/PricesRepository'; +export interface CoreIndex { + name: string; + synced: boolean; + best_block_height: number; +} + class Indexer { runIndexer = true; indexerRunning = false; tasksRunning: string[] = []; + coreIndexes: CoreIndex[] = []; - public reindex() { + /** + * Check which core index is available for indexing + */ + public async checkAvailableCoreIndexes(): Promise { + const updatedCoreIndexes: CoreIndex[] = []; + + const indexes: any = await bitcoinClient.getIndexInfo(); + for (const indexName in indexes) { + const newState = { + name: indexName, + synced: indexes[indexName].synced, + best_block_height: indexes[indexName].best_block_height, + }; + logger.info(`Core index '${indexName}' is ${indexes[indexName].synced ? 'synced' : 'not synced'}. Best block height is ${indexes[indexName].best_block_height}`); + updatedCoreIndexes.push(newState); + + if (indexName === 'coinstatsindex' && newState.synced === true) { + const previousState = this.isCoreIndexReady('coinstatsindex'); + // if (!previousState || previousState.synced === false) { + this.runSingleTask('coinStatsIndex'); + // } + } + } + + this.coreIndexes = updatedCoreIndexes; + } + + /** + * Return the best block height if a core index is available, or 0 if not + * + * @param name + * @returns + */ + public isCoreIndexReady(name: string): CoreIndex | null { + for (const index of this.coreIndexes) { + if (index.name === name && index.synced === true) { + return index; + } + } + return null; + } + + public reindex(): void { if (Common.indexingEnabled()) { this.runIndexer = true; } } - public async runSingleTask(task: 'blocksPrices') { + public async runSingleTask(task: 'blocksPrices' | 'coinStatsIndex'): Promise { if (!Common.indexingEnabled()) { return; } @@ -28,20 +77,27 @@ class Indexer { this.tasksRunning.push(task); const lastestPriceId = await PricesRepository.$getLatestPriceId(); if (priceUpdater.historyInserted === false || lastestPriceId === null) { - logger.debug(`Blocks prices indexer is waiting for the price updater to complete`) + logger.debug(`Blocks prices indexer is waiting for the price updater to complete`); setTimeout(() => { - this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task) + this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task); this.runSingleTask('blocksPrices'); }, 10000); } else { - logger.debug(`Blocks prices indexer will run now`) + logger.debug(`Blocks prices indexer will run now`); await mining.$indexBlockPrices(); - this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task) + this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task); } } + + if (task === 'coinStatsIndex' && !this.tasksRunning.includes(task)) { + this.tasksRunning.push(task); + logger.debug(`Indexing coinStatsIndex now`); + await mining.$indexCoinStatsIndex(); + this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task); + } } - public async $run() { + public async $run(): Promise { if (!Common.indexingEnabled() || this.runIndexer === false || this.indexerRunning === true || mempool.hasPriority() ) { @@ -57,7 +113,9 @@ class Indexer { this.runIndexer = false; this.indexerRunning = true; - logger.debug(`Running mining indexer`); + logger.info(`Running mining indexer`); + + await this.checkAvailableCoreIndexes(); try { await priceUpdater.$run(); @@ -93,7 +151,7 @@ class Indexer { setTimeout(() => this.reindex(), runEvery); } - async $resetHashratesIndexingState() { + async $resetHashratesIndexingState(): Promise { try { await HashratesRepository.$setLatestRun('last_hashrates_indexing', 0); await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', 0); diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index 6b258c173..cb95be98a 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -1,4 +1,5 @@ import { IEsploraApi } from './api/bitcoin/esplora-api.interface'; +import { OrphanedBlock } from './api/chain-tips'; import { HeapNode } from "./utils/pairing-heap"; export interface PoolTag { @@ -64,6 +65,7 @@ interface VinStrippedToScriptsig { interface VoutStrippedToScriptPubkey { scriptpubkey_address: string | undefined; + scriptpubkey_asm: string | undefined; value: number; } @@ -160,6 +162,27 @@ export interface BlockExtension { avgFeeRate?: number; coinbaseRaw?: string; usd?: number | null; + medianTimestamp?: number; + blockTime?: number; + orphans?: OrphanedBlock[] | null; + coinbaseAddress?: string | null; + coinbaseSignature?: string | null; + coinbaseSignatureAscii?: string | null; + virtualSize?: number; + avgTxSize?: number; + totalInputs?: number; + totalOutputs?: number; + totalOutputAmt?: number; + medianFeeAmt?: number | null; + feePercentiles?: number[] | null, + segwitTotalTxs?: number; + segwitTotalSize?: number; + segwitTotalWeight?: number; + header?: string; + utxoSetChange?: number; + // Requires coinstatsindex, will be set to NULL otherwise + utxoSetSize?: number | null; + totalInputAmt?: number | null; } export interface BlockExtended extends IEsploraApi.Block { diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index df98719b9..86dc006ff 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -18,17 +18,27 @@ class BlocksRepository { public async $saveBlockInDatabase(block: BlockExtended) { try { const query = `INSERT INTO blocks( - height, hash, blockTimestamp, size, - weight, tx_count, coinbase_raw, difficulty, - pool_id, fees, fee_span, median_fee, - reward, version, bits, nonce, - merkle_root, previous_block_hash, avg_fee, avg_fee_rate + height, hash, blockTimestamp, size, + weight, tx_count, coinbase_raw, difficulty, + pool_id, fees, fee_span, median_fee, + reward, version, bits, nonce, + merkle_root, previous_block_hash, avg_fee, avg_fee_rate, + median_timestamp, header, coinbase_address, + coinbase_signature, utxoset_size, utxoset_change, avg_tx_size, + total_inputs, total_outputs, total_input_amt, total_output_amt, + fee_percentiles, segwit_total_txs, segwit_total_size, segwit_total_weight, + median_fee_amt, coinbase_signature_ascii ) VALUE ( ?, ?, FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - ?, ?, ?, ? + ?, ?, ?, ?, + FROM_UNIXTIME(?), ?, ?, + ?, ?, ?, ?, + ?, ?, ?, ?, + ?, ?, ?, ?, + ?, ? )`; const params: any[] = [ @@ -52,6 +62,23 @@ class BlocksRepository { block.previousblockhash, block.extras.avgFee, block.extras.avgFeeRate, + block.extras.medianTimestamp, + block.extras.header, + block.extras.coinbaseAddress, + block.extras.coinbaseSignature, + block.extras.utxoSetSize, + block.extras.utxoSetChange, + block.extras.avgTxSize, + block.extras.totalInputs, + block.extras.totalOutputs, + block.extras.totalInputAmt, + block.extras.totalOutputAmt, + block.extras.feePercentiles ? JSON.stringify(block.extras.feePercentiles) : null, + block.extras.segwitTotalTxs, + block.extras.segwitTotalSize, + block.extras.segwitTotalWeight, + block.extras.medianFeeAmt, + block.extras.coinbaseSignatureAscii, ]; await DB.query(query, params); @@ -65,6 +92,33 @@ class BlocksRepository { } } + /** + * Save newly indexed data from core coinstatsindex + * + * @param utxoSetSize + * @param totalInputAmt + */ + public async $updateCoinStatsIndexData(blockHash: string, utxoSetSize: number, + totalInputAmt: number + ) : Promise { + try { + const query = ` + UPDATE blocks + SET utxoset_size = ?, total_input_amt = ? + WHERE hash = ? + `; + const params: any[] = [ + utxoSetSize, + totalInputAmt, + blockHash + ]; + await DB.query(query, params); + } catch (e: any) { + logger.err('Cannot update indexed block coinstatsindex. Reason: ' + (e instanceof Error ? e.message : e)); + throw e; + } + } + /** * Get all block height that have not been indexed between [startHeight, endHeight] */ @@ -310,32 +364,17 @@ class BlocksRepository { public async $getBlockByHeight(height: number): Promise { try { const [rows]: any[] = await DB.query(`SELECT - blocks.height, - hash, + blocks.*, hash as id, UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp, - size, - weight, - tx_count, - coinbase_raw, - difficulty, + UNIX_TIMESTAMP(blocks.median_timestamp) as medianTime, pools.id as pool_id, pools.name as pool_name, pools.link as pool_link, pools.slug as pool_slug, pools.addresses as pool_addresses, pools.regexes as pool_regexes, - fees, - fee_span, - median_fee, - reward, - version, - bits, - nonce, - merkle_root, - previous_block_hash as previousblockhash, - avg_fee, - avg_fee_rate + previous_block_hash as previousblockhash FROM blocks JOIN pools ON blocks.pool_id = pools.id WHERE blocks.height = ${height} @@ -346,6 +385,7 @@ class BlocksRepository { } rows[0].fee_span = JSON.parse(rows[0].fee_span); + rows[0].fee_percentiles = JSON.parse(rows[0].fee_percentiles); return rows[0]; } catch (e) { logger.err(`Cannot get indexed block ${height}. Reason: ` + (e instanceof Error ? e.message : e)); @@ -694,7 +734,6 @@ class BlocksRepository { logger.err('Cannot fetch CPFP unindexed blocks. Reason: ' + (e instanceof Error ? e.message : e)); throw e; } - return []; } /** @@ -741,7 +780,7 @@ class BlocksRepository { try { let query = `INSERT INTO blocks_prices(height, price_id) VALUES`; for (const price of blockPrices) { - query += ` (${price.height}, ${price.priceId}),` + query += ` (${price.height}, ${price.priceId}),`; } query = query.slice(0, -1); await DB.query(query); @@ -754,6 +793,43 @@ class BlocksRepository { } } } + + /** + * Get all indexed blocsk with missing coinstatsindex data + */ + public async $getBlocksMissingCoinStatsIndex(maxHeight: number, minHeight: number): Promise { + try { + const [blocks] = await DB.query(` + SELECT height, hash + FROM blocks + WHERE height >= ${minHeight} AND height <= ${maxHeight} AND + (utxoset_size IS NULL OR total_input_amt IS NULL) + `); + return blocks; + } catch (e) { + logger.err(`Cannot get blocks with missing coinstatsindex. Reason: ` + (e instanceof Error ? e.message : e)); + throw e; + } + } + + /** + * Save indexed median fee to avoid recomputing it later + * + * @param id + * @param feePercentiles + */ + public async $saveFeePercentilesForBlockId(id: string, feePercentiles: number[]): Promise { + try { + await DB.query(` + UPDATE blocks SET fee_percentiles = ?, median_fee_amt = ? + WHERE hash = ?`, + [JSON.stringify(feePercentiles), feePercentiles[3], id] + ); + } catch (e) { + logger.err(`Cannot update block fee_percentiles. Reason: ` + (e instanceof Error ? e.message : e)); + throw e; + } + } } export default new BlocksRepository(); diff --git a/backend/src/repositories/BlocksSummariesRepository.ts b/backend/src/repositories/BlocksSummariesRepository.ts index 1406a1d07..2724ddcf5 100644 --- a/backend/src/repositories/BlocksSummariesRepository.ts +++ b/backend/src/repositories/BlocksSummariesRepository.ts @@ -80,6 +80,48 @@ class BlocksSummariesRepository { logger.err('Cannot delete indexed blocks summaries. Reason: ' + (e instanceof Error ? e.message : e)); } } + + /** + * Get the fee percentiles if the block has already been indexed, [] otherwise + * + * @param id + */ + public async $getFeePercentilesByBlockId(id: string): Promise { + try { + const [rows]: any[] = await DB.query(` + SELECT transactions + FROM blocks_summaries + WHERE id = ?`, + [id] + ); + if (rows === null || rows.length === 0) { + return null; + } + + const transactions = JSON.parse(rows[0].transactions); + if (transactions === null) { + return null; + } + + transactions.shift(); // Ignore coinbase + transactions.sort((a: any, b: any) => a.fee - b.fee); + const fees = transactions.map((t: any) => t.fee); + + return [ + fees[0] ?? 0, // min + fees[Math.max(0, Math.floor(fees.length * 0.1) - 1)] ?? 0, // 10th + fees[Math.max(0, Math.floor(fees.length * 0.25) - 1)] ?? 0, // 25th + fees[Math.max(0, Math.floor(fees.length * 0.5) - 1)] ?? 0, // median + fees[Math.max(0, Math.floor(fees.length * 0.75) - 1)] ?? 0, // 75th + fees[Math.max(0, Math.floor(fees.length * 0.9) - 1)] ?? 0, // 90th + fees[fees.length - 1] ?? 0, // max + ]; + + } catch (e) { + logger.err(`Cannot get block summaries transactions. Reason: ` + (e instanceof Error ? e.message : e)); + return null; + } + } } export default new BlocksSummariesRepository(); diff --git a/backend/src/rpc-api/commands.ts b/backend/src/rpc-api/commands.ts index ea9bd7bf0..78f5e12f4 100644 --- a/backend/src/rpc-api/commands.ts +++ b/backend/src/rpc-api/commands.ts @@ -88,5 +88,7 @@ module.exports = { verifyTxOutProof: 'verifytxoutproof', // bitcoind v0.11.0+ walletLock: 'walletlock', walletPassphrase: 'walletpassphrase', - walletPassphraseChange: 'walletpassphrasechange' -} + walletPassphraseChange: 'walletpassphrasechange', + getTxoutSetinfo: 'gettxoutsetinfo', + getIndexInfo: 'getindexinfo', +}; diff --git a/docker/README.md b/docker/README.md index 69bb96030..168d4b1fa 100644 --- a/docker/README.md +++ b/docker/README.md @@ -111,6 +111,7 @@ Below we list all settings from `mempool-config.json` and the corresponding over "ADVANCED_GBT_AUDIT": false, "ADVANCED_GBT_MEMPOOL": false, "CPFP_INDEXING": false, + "MAX_BLOCKS_BULK_QUERY": 0, }, ``` @@ -141,6 +142,7 @@ Corresponding `docker-compose.yml` overrides: MEMPOOL_ADVANCED_GBT_AUDIT: "" MEMPOOL_ADVANCED_GBT_MEMPOOL: "" MEMPOOL_CPFP_INDEXING: "" + MAX_BLOCKS_BULK_QUERY: "" ... ``` diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index 904370f3e..d2aa75c69 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -25,7 +25,8 @@ "AUDIT": __MEMPOOL_AUDIT__, "ADVANCED_GBT_AUDIT": __MEMPOOL_ADVANCED_GBT_AUDIT__, "ADVANCED_GBT_MEMPOOL": __MEMPOOL_ADVANCED_GBT_MEMPOOL__, - "CPFP_INDEXING": __MEMPOOL_CPFP_INDEXING__ + "CPFP_INDEXING": __MEMPOOL_CPFP_INDEXING__, + "MAX_BLOCKS_BULK_QUERY": __MEMPOOL__MAX_BLOCKS_BULK_QUERY__ }, "CORE_RPC": { "HOST": "__CORE_RPC_HOST__", diff --git a/docker/backend/start.sh b/docker/backend/start.sh index 58b19898a..3ee542892 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -30,6 +30,7 @@ __MEMPOOL_AUDIT__=${MEMPOOL_AUDIT:=false} __MEMPOOL_ADVANCED_GBT_AUDIT__=${MEMPOOL_ADVANCED_GBT_AUDIT:=false} __MEMPOOL_ADVANCED_GBT_MEMPOOL__=${MEMPOOL_ADVANCED_GBT_MEMPOOL:=false} __MEMPOOL_CPFP_INDEXING__=${MEMPOOL_CPFP_INDEXING:=false} +__MEMPOOL_MAX_BLOCKS_BULK_QUERY__=${MEMPOOL_MAX_BLOCKS_BULK_QUERY:=0} # CORE_RPC __CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1} @@ -142,6 +143,7 @@ sed -i "s!__MEMPOOL_AUDIT__!${__MEMPOOL_AUDIT__}!g" mempool-config.json sed -i "s!__MEMPOOL_ADVANCED_GBT_MEMPOOL__!${__MEMPOOL_ADVANCED_GBT_MEMPOOL__}!g" mempool-config.json sed -i "s!__MEMPOOL_ADVANCED_GBT_AUDIT__!${__MEMPOOL_ADVANCED_GBT_AUDIT__}!g" mempool-config.json sed -i "s!__MEMPOOL_CPFP_INDEXING__!${__MEMPOOL_CPFP_INDEXING__}!g" mempool-config.json +sed -i "s!__MEMPOOL_MAX_BLOCKS_BULK_QUERY__!${__MEMPOOL_MAX_BLOCKS_BULK_QUERY__}!g" mempool-config.json sed -i "s/__CORE_RPC_HOST__/${__CORE_RPC_HOST__}/g" mempool-config.json sed -i "s/__CORE_RPC_PORT__/${__CORE_RPC_PORT__}/g" mempool-config.json diff --git a/production/bitcoin.conf b/production/bitcoin.conf index 46ab41b20..501f49f50 100644 --- a/production/bitcoin.conf +++ b/production/bitcoin.conf @@ -1,6 +1,7 @@ datadir=/bitcoin server=1 txindex=1 +coinstatsindex=1 listen=1 discover=1 par=16