From 73f76474dd7fa0d8f95e8fd02af8c923b6b7fa86 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Fri, 17 Feb 2023 21:21:21 +0900 Subject: [PATCH] Implemented coinstatsindex indexing --- backend/src/api/bitcoin/bitcoin.routes.ts | 7 +- .../src/api/bitcoin/esplora-api.interface.ts | 1 + backend/src/api/blocks.ts | 158 ++++++++++-------- backend/src/api/database-migration.ts | 29 +++- backend/src/api/mining/mining.ts | 41 ++++- backend/src/api/transaction-utils.ts | 1 + backend/src/index.ts | 1 + backend/src/indexer.ts | 76 ++++++++- backend/src/mempool.interfaces.ts | 21 +++ backend/src/repositories/BlocksRepository.ts | 107 +++++++++--- backend/src/rpc-api/commands.ts | 5 +- 11 files changed, 330 insertions(+), 117 deletions(-) diff --git a/backend/src/api/bitcoin/bitcoin.routes.ts b/backend/src/api/bitcoin/bitcoin.routes.ts index 0ff30376c..6d145e854 100644 --- a/backend/src/api/bitcoin/bitcoin.routes.ts +++ b/backend/src/api/bitcoin/bitcoin.routes.ts @@ -407,7 +407,10 @@ class BitcoinRoutes { private async getBlocksByBulk(req: Request, res: Response) { try { if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { // Liquid, Bisq - Not implemented - return res.status(404).send(`Not implemented`); + return res.status(404).send(`This API is only available for Bitcoin networks`); + } + if (!Common.indexingEnabled()) { + return res.status(404).send(`Indexing is required for this API`); } const from = parseInt(req.params.from, 10); @@ -423,7 +426,7 @@ class BitcoinRoutes { } res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); - res.json(await blocks.$getBlocksByBulk(from, to)); + res.json(await blocks.$getBlocksBetweenHeight(from, to)); } catch (e) { res.status(500).send(e instanceof Error ? e.message : e); diff --git a/backend/src/api/bitcoin/esplora-api.interface.ts b/backend/src/api/bitcoin/esplora-api.interface.ts index 39f8cfd6f..eaf6476f4 100644 --- a/backend/src/api/bitcoin/esplora-api.interface.ts +++ b/backend/src/api/bitcoin/esplora-api.interface.ts @@ -88,6 +88,7 @@ export namespace IEsploraApi { size: number; weight: number; previousblockhash: string; + medianTime?: number; } export interface Address { diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 5c8884c71..d950a9bd3 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -165,33 +165,75 @@ class Blocks { * @returns BlockExtended */ private async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise { - const blockExtended: BlockExtended = Object.assign({ extras: {} }, block); - blockExtended.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0); - blockExtended.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]); - blockExtended.extras.coinbaseRaw = blockExtended.extras.coinbaseTx.vin[0].scriptsig; - blockExtended.extras.usd = priceUpdater.latestPrices.USD; + const blk: BlockExtended = Object.assign({ extras: {} }, block); + blk.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0); + blk.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]); + blk.extras.coinbaseRaw = blk.extras.coinbaseTx.vin[0].scriptsig; + blk.extras.usd = priceUpdater.latestPrices.USD; if (block.height === 0) { - blockExtended.extras.medianFee = 0; // 50th percentiles - blockExtended.extras.feeRange = [0, 0, 0, 0, 0, 0, 0]; - blockExtended.extras.totalFees = 0; - blockExtended.extras.avgFee = 0; - blockExtended.extras.avgFeeRate = 0; + blk.extras.medianFee = 0; // 50th percentiles + blk.extras.feeRange = [0, 0, 0, 0, 0, 0, 0]; + blk.extras.totalFees = 0; + blk.extras.avgFee = 0; + blk.extras.avgFeeRate = 0; + blk.extras.utxoSetChange = 0; + blk.extras.avgTxSize = 0; + blk.extras.totalInputs = 0; + blk.extras.totalOutputs = 1; + blk.extras.totalOutputAmt = 0; + blk.extras.segwitTotalTxs = 0; + blk.extras.segwitTotalSize = 0; + blk.extras.segwitTotalWeight = 0; } else { - const stats = await bitcoinClient.getBlockStats(block.id, [ - 'feerate_percentiles', 'minfeerate', 'maxfeerate', 'totalfee', 'avgfee', 'avgfeerate' - ]); - blockExtended.extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles - blockExtended.extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat(); - blockExtended.extras.totalFees = stats.totalfee; - blockExtended.extras.avgFee = stats.avgfee; - blockExtended.extras.avgFeeRate = stats.avgfeerate; + const stats = await bitcoinClient.getBlockStats(block.id); + blk.extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles + blk.extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat(); + blk.extras.totalFees = stats.totalfee; + blk.extras.avgFee = stats.avgfee; + blk.extras.avgFeeRate = stats.avgfeerate; + blk.extras.utxoSetChange = stats.utxo_increase; + blk.extras.avgTxSize = Math.round(stats.total_size / stats.txs * 100) * 0.01; + blk.extras.totalInputs = stats.ins; + blk.extras.totalOutputs = stats.outs; + blk.extras.totalOutputAmt = stats.total_out; + blk.extras.segwitTotalTxs = stats.swtxs; + blk.extras.segwitTotalSize = stats.swtotal_size; + blk.extras.segwitTotalWeight = stats.swtotal_weight; + } + + blk.extras.feePercentiles = [], // TODO + blk.extras.medianFeeAmt = 0; // TODO + blk.extras.medianTimestamp = block.medianTime; // TODO + blk.extras.blockTime = 0; // TODO + blk.extras.orphaned = false; // TODO + + blk.extras.virtualSize = block.weight / 4.0; + if (blk.extras.coinbaseTx.vout.length > 0) { + blk.extras.coinbaseAddress = blk.extras.coinbaseTx.vout[0].scriptpubkey_address ?? null; + blk.extras.coinbaseSignature = blk.extras.coinbaseTx.vout[0].scriptpubkey_asm ?? null; + } else { + blk.extras.coinbaseAddress = null; + blk.extras.coinbaseSignature = null; + } + + const header = await bitcoinClient.getBlockHeader(block.id, false); + blk.extras.header = header; + + const coinStatsIndex = indexer.isCoreIndexReady('coinstatsindex'); + if (coinStatsIndex !== null && coinStatsIndex.best_block_height >= block.height) { + const txoutset = await bitcoinClient.getTxoutSetinfo('none', block.height); + blk.extras.utxoSetSize = txoutset.txouts, + blk.extras.totalInputAmt = Math.round(txoutset.block_info.prevout_spent * 100000000); + } else { + blk.extras.utxoSetSize = null; + blk.extras.totalInputAmt = null; } if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) { let pool: PoolTag; - if (blockExtended.extras?.coinbaseTx !== undefined) { - pool = await this.$findBlockMiner(blockExtended.extras?.coinbaseTx); + if (blk.extras?.coinbaseTx !== undefined) { + pool = await this.$findBlockMiner(blk.extras?.coinbaseTx); } else { if (config.DATABASE.ENABLED === true) { pool = await poolsRepository.$getUnknownPool(); @@ -201,10 +243,10 @@ class Blocks { } if (!pool) { // We should never have this situation in practise - logger.warn(`Cannot assign pool to block ${blockExtended.height} and 'unknown' pool does not exist. ` + + logger.warn(`Cannot assign pool to block ${blk.height} and 'unknown' pool does not exist. ` + `Check your "pools" table entries`); } else { - blockExtended.extras.pool = { + blk.extras.pool = { id: pool.id, name: pool.name, slug: pool.slug, @@ -214,12 +256,12 @@ class Blocks { if (config.MEMPOOL.AUDIT) { const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(block.id); if (auditScore != null) { - blockExtended.extras.matchRate = auditScore.matchRate; + blk.extras.matchRate = auditScore.matchRate; } } } - return blockExtended; + return blk; } /** @@ -727,60 +769,28 @@ class Blocks { return returnBlocks; } - public async $getBlocksByBulk(start: number, end: number) { - start = Math.max(1, start); + /** + * Used for bulk block data query + * + * @param fromHeight + * @param toHeight + */ + public async $getBlocksBetweenHeight(fromHeight: number, toHeight: number): Promise { + if (!Common.indexingEnabled()) { + return []; + } const blocks: any[] = []; - for (let i = end; i >= start; --i) { - const blockHash = await bitcoinApi.$getBlockHash(i); - const coreBlock = await bitcoinClient.getBlock(blockHash); - const electrsBlock = await bitcoinApi.$getBlock(blockHash); - const txs = await this.$getTransactionsExtended(blockHash, i, true); - const stats = await bitcoinClient.getBlockStats(blockHash); - const header = await bitcoinClient.getBlockHeader(blockHash, false); - const txoutset = await bitcoinClient.getTxoutSetinfo('none', i); - const formatted = { - blockhash: coreBlock.id, - blockheight: coreBlock.height, - prev_blockhash: coreBlock.previousblockhash, - timestamp: coreBlock.timestamp, - median_timestamp: coreBlock.mediantime, - // @ts-ignore - blocktime: coreBlock.time, - orphaned: null, - header: header, - version: coreBlock.version, - difficulty: coreBlock.difficulty, - merkle_root: coreBlock.merkle_root, - bits: coreBlock.bits, - nonce: coreBlock.nonce, - coinbase_scriptsig: txs[0].vin[0].scriptsig, - coinbase_address: txs[0].vout[0].scriptpubkey_address, - coinbase_signature: txs[0].vout[0].scriptpubkey_asm, - size: coreBlock.size, - virtual_size: coreBlock.weight / 4.0, - weight: coreBlock.weight, - utxoset_size: txoutset.txouts, - utxoset_change: stats.utxo_increase, - total_txs: coreBlock.tx_count, - avg_tx_size: Math.round(stats.total_size / stats.txs * 100) * 0.01, - total_inputs: stats.ins, - total_outputs: stats.outs, - total_input_amt: Math.round(txoutset.block_info.prevout_spent * 100000000), - total_output_amt: stats.total_out, - block_subsidy: txs[0].vout.reduce((acc, curr) => acc + curr.value, 0), - total_fee: stats.totalfee, - avg_feerate: stats.avgfeerate, - feerate_percentiles: [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat(), - avg_fee: stats.avgfee, - fee_percentiles: null, - segwit_total_txs: stats.swtxs, - segwit_total_size: stats.swtotal_size, - segwit_total_weight: stats.swtotal_weight, - }; - blocks.push(formatted); + while (fromHeight <= toHeight) { + let block = await blocksRepository.$getBlockByHeight(fromHeight); + if (!block) { + block = await this.$indexBlock(fromHeight); + } + blocks.push(block); + fromHeight++; } + return blocks; } diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 6e4221857..2c6adfd1b 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository'; import { RowDataPacket } from 'mysql2'; class DatabaseMigration { - private static currentVersion = 54; + private static currentVersion = 55; private queryTimeout = 3600_000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -483,6 +483,11 @@ class DatabaseMigration { } await this.updateToSchemaVersion(54); } + + if (databaseSchemaVersion < 55) { + await this.$executeQuery(this.getAdditionalBlocksDataQuery()); + await this.updateToSchemaVersion(55); + } } /** @@ -756,6 +761,28 @@ class DatabaseMigration { ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; } + private getAdditionalBlocksDataQuery(): string { + return `ALTER TABLE blocks + ADD median_timestamp timestamp NOT NULL, + ADD block_time int unsigned NOT NULL, + ADD coinbase_address varchar(100) NULL, + ADD coinbase_signature varchar(500) NULL, + ADD avg_tx_size double unsigned NOT NULL, + ADD total_inputs int unsigned NOT NULL, + ADD total_outputs int unsigned NOT NULL, + ADD total_output_amt bigint unsigned NOT NULL, + ADD fee_percentiles longtext NULL, + ADD median_fee_amt int unsigned NOT NULL, + ADD segwit_total_txs int unsigned NOT NULL, + ADD segwit_total_size int unsigned NOT NULL, + ADD segwit_total_weight int unsigned NOT NULL, + ADD header varchar(160) NOT NULL, + ADD utxoset_change int NOT NULL, + ADD utxoset_size int unsigned NULL, + ADD total_input_amt bigint unsigned NULL + `; + } + private getCreateDailyStatsTableQuery(): string { return `CREATE TABLE IF NOT EXISTS hashrates ( hashrate_timestamp timestamp NOT NULL, diff --git a/backend/src/api/mining/mining.ts b/backend/src/api/mining/mining.ts index edcb5b2e5..f33a68dcb 100644 --- a/backend/src/api/mining/mining.ts +++ b/backend/src/api/mining/mining.ts @@ -172,7 +172,7 @@ class Mining { } /** - * [INDEXING] Generate weekly mining pool hashrate history + * Generate weekly mining pool hashrate history */ public async $generatePoolHashrateHistory(): Promise { const now = new Date(); @@ -279,7 +279,7 @@ class Mining { } /** - * [INDEXING] Generate daily hashrate data + * Generate daily hashrate data */ public async $generateNetworkHashrateHistory(): Promise { // We only run this once a day around midnight @@ -459,7 +459,7 @@ class Mining { /** * Create a link between blocks and the latest price at when they were mined */ - public async $indexBlockPrices() { + public async $indexBlockPrices(): Promise { if (this.blocksPriceIndexingRunning === true) { return; } @@ -520,6 +520,41 @@ class Mining { this.blocksPriceIndexingRunning = false; } + /** + * Index core coinstatsindex + */ + public async $indexCoinStatsIndex(): Promise { + let timer = new Date().getTime() / 1000; + let totalIndexed = 0; + + const blockchainInfo = await bitcoinClient.getBlockchainInfo(); + let currentBlockHeight = blockchainInfo.blocks; + + while (currentBlockHeight > 0) { + const indexedBlocks = await BlocksRepository.$getBlocksMissingCoinStatsIndex( + currentBlockHeight, currentBlockHeight - 10000); + + for (const block of indexedBlocks) { + const txoutset = await bitcoinClient.getTxoutSetinfo('none', block.height); + await BlocksRepository.$updateCoinStatsIndexData(block.hash, txoutset.txouts, + Math.round(txoutset.block_info.prevout_spent * 100000000)); + ++totalIndexed; + + const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer); + if (elapsedSeconds > 5) { + logger.info(`Indexing coinstatsindex data for block #${block.height}. Indexed ${totalIndexed} blocks.`, logger.tags.mining); + timer = new Date().getTime() / 1000; + } + } + + currentBlockHeight -= 10000; + } + + if (totalIndexed) { + logger.info(`Indexing missing coinstatsindex data completed`, logger.tags.mining); + } + } + private getDateMidnight(date: Date): Date { date.setUTCHours(0); date.setUTCMinutes(0); diff --git a/backend/src/api/transaction-utils.ts b/backend/src/api/transaction-utils.ts index fb5aeea42..fb69419fc 100644 --- a/backend/src/api/transaction-utils.ts +++ b/backend/src/api/transaction-utils.ts @@ -14,6 +14,7 @@ class TransactionUtils { vout: tx.vout .map((vout) => ({ scriptpubkey_address: vout.scriptpubkey_address, + scriptpubkey_asm: vout.scriptpubkey_asm, value: vout.value })) .filter((vout) => vout.value) diff --git a/backend/src/index.ts b/backend/src/index.ts index 919c039c3..d8d46fc9f 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -36,6 +36,7 @@ import bitcoinRoutes from './api/bitcoin/bitcoin.routes'; import fundingTxFetcher from './tasks/lightning/sync-tasks/funding-tx-fetcher'; import forensicsService from './tasks/lightning/forensics.service'; import priceUpdater from './tasks/price-updater'; +import mining from './api/mining/mining'; import { AxiosError } from 'axios'; class Server { diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index 22f3ce319..41c8024e0 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -8,18 +8,67 @@ import bitcoinClient from './api/bitcoin/bitcoin-client'; import priceUpdater from './tasks/price-updater'; import PricesRepository from './repositories/PricesRepository'; +export interface CoreIndex { + name: string; + synced: boolean; + best_block_height: number; +} + class Indexer { runIndexer = true; indexerRunning = false; tasksRunning: string[] = []; + coreIndexes: CoreIndex[] = []; - public reindex() { + /** + * Check which core index is available for indexing + */ + public async checkAvailableCoreIndexes(): Promise { + const updatedCoreIndexes: CoreIndex[] = []; + + const indexes: any = await bitcoinClient.getIndexInfo(); + for (const indexName in indexes) { + const newState = { + name: indexName, + synced: indexes[indexName].synced, + best_block_height: indexes[indexName].best_block_height, + }; + logger.info(`Core index '${indexName}' is ${indexes[indexName].synced ? 'synced' : 'not synced'}. Best block height is ${indexes[indexName].best_block_height}`); + updatedCoreIndexes.push(newState); + + if (indexName === 'coinstatsindex' && newState.synced === true) { + const previousState = this.isCoreIndexReady('coinstatsindex'); + // if (!previousState || previousState.synced === false) { + this.runSingleTask('coinStatsIndex'); + // } + } + } + + this.coreIndexes = updatedCoreIndexes; + } + + /** + * Return the best block height if a core index is available, or 0 if not + * + * @param name + * @returns + */ + public isCoreIndexReady(name: string): CoreIndex | null { + for (const index of this.coreIndexes) { + if (index.name === name && index.synced === true) { + return index; + } + } + return null; + } + + public reindex(): void { if (Common.indexingEnabled()) { this.runIndexer = true; } } - public async runSingleTask(task: 'blocksPrices') { + public async runSingleTask(task: 'blocksPrices' | 'coinStatsIndex'): Promise { if (!Common.indexingEnabled()) { return; } @@ -28,20 +77,27 @@ class Indexer { this.tasksRunning.push(task); const lastestPriceId = await PricesRepository.$getLatestPriceId(); if (priceUpdater.historyInserted === false || lastestPriceId === null) { - logger.debug(`Blocks prices indexer is waiting for the price updater to complete`) + logger.debug(`Blocks prices indexer is waiting for the price updater to complete`); setTimeout(() => { - this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task) + this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task); this.runSingleTask('blocksPrices'); }, 10000); } else { - logger.debug(`Blocks prices indexer will run now`) + logger.debug(`Blocks prices indexer will run now`); await mining.$indexBlockPrices(); - this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task) + this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task); } } + + if (task === 'coinStatsIndex' && !this.tasksRunning.includes(task)) { + this.tasksRunning.push(task); + logger.debug(`Indexing coinStatsIndex now`); + await mining.$indexCoinStatsIndex(); + this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task); + } } - public async $run() { + public async $run(): Promise { if (!Common.indexingEnabled() || this.runIndexer === false || this.indexerRunning === true || mempool.hasPriority() ) { @@ -57,7 +113,9 @@ class Indexer { this.runIndexer = false; this.indexerRunning = true; - logger.debug(`Running mining indexer`); + logger.info(`Running mining indexer`); + + await this.checkAvailableCoreIndexes(); try { await priceUpdater.$run(); @@ -93,7 +151,7 @@ class Indexer { setTimeout(() => this.reindex(), runEvery); } - async $resetHashratesIndexingState() { + async $resetHashratesIndexingState(): Promise { try { await HashratesRepository.$setLatestRun('last_hashrates_indexing', 0); await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', 0); diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index 6b258c173..a1a9e1687 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -64,6 +64,7 @@ interface VinStrippedToScriptsig { interface VoutStrippedToScriptPubkey { scriptpubkey_address: string | undefined; + scriptpubkey_asm: string | undefined; value: number; } @@ -160,6 +161,26 @@ export interface BlockExtension { avgFeeRate?: number; coinbaseRaw?: string; usd?: number | null; + medianTimestamp?: number; + blockTime?: number; + orphaned?: boolean; + coinbaseAddress?: string | null; + coinbaseSignature?: string | null; + virtualSize?: number; + avgTxSize?: number; + totalInputs?: number; + totalOutputs?: number; + totalOutputAmt?: number; + medianFeeAmt?: number; + feePercentiles?: number[], + segwitTotalTxs?: number; + segwitTotalSize?: number; + segwitTotalWeight?: number; + header?: string; + utxoSetChange?: number; + // Requires coinstatsindex, will be set to NULL otherwise + utxoSetSize?: number | null; + totalInputAmt?: number | null; } export interface BlockExtended extends IEsploraApi.Block { diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index df98719b9..baaea38d9 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -18,17 +18,27 @@ class BlocksRepository { public async $saveBlockInDatabase(block: BlockExtended) { try { const query = `INSERT INTO blocks( - height, hash, blockTimestamp, size, - weight, tx_count, coinbase_raw, difficulty, - pool_id, fees, fee_span, median_fee, - reward, version, bits, nonce, - merkle_root, previous_block_hash, avg_fee, avg_fee_rate + height, hash, blockTimestamp, size, + weight, tx_count, coinbase_raw, difficulty, + pool_id, fees, fee_span, median_fee, + reward, version, bits, nonce, + merkle_root, previous_block_hash, avg_fee, avg_fee_rate, + median_timestamp, block_time, header, coinbase_address, + coinbase_signature, utxoset_size, utxoset_change, avg_tx_size, + total_inputs, total_outputs, total_input_amt, total_output_amt, + fee_percentiles, segwit_total_txs, segwit_total_size, segwit_total_weight, + median_fee_amt ) VALUE ( ?, ?, FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - ?, ?, ?, ? + ?, ?, ?, ?, + ?, ?, ?, ?, + ?, ?, ?, ?, + ?, ?, ?, ?, + ?, ?, ?, ?, + ? )`; const params: any[] = [ @@ -52,6 +62,23 @@ class BlocksRepository { block.previousblockhash, block.extras.avgFee, block.extras.avgFeeRate, + block.extras.medianTimestamp, + block.extras.blockTime, + block.extras.header, + block.extras.coinbaseAddress, + block.extras.coinbaseSignature, + block.extras.utxoSetSize, + block.extras.utxoSetChange, + block.extras.avgTxSize, + block.extras.totalInputs, + block.extras.totalOutputs, + block.extras.totalInputAmt, + block.extras.totalOutputAmt, + JSON.stringify(block.extras.feePercentiles), + block.extras.segwitTotalTxs, + block.extras.segwitTotalSize, + block.extras.segwitTotalWeight, + block.extras.medianFeeAmt, ]; await DB.query(query, params); @@ -65,6 +92,33 @@ class BlocksRepository { } } + /** + * Save newly indexed data from core coinstatsindex + * + * @param utxoSetSize + * @param totalInputAmt + */ + public async $updateCoinStatsIndexData(blockHash: string, utxoSetSize: number, + totalInputAmt: number + ) : Promise { + try { + const query = ` + UPDATE blocks + SET utxoset_size = ?, total_input_amt = ? + WHERE hash = ? + `; + const params: any[] = [ + utxoSetSize, + totalInputAmt, + blockHash + ]; + await DB.query(query, params); + } catch (e: any) { + logger.err('Cannot update indexed block coinstatsindex. Reason: ' + (e instanceof Error ? e.message : e)); + throw e; + } + } + /** * Get all block height that have not been indexed between [startHeight, endHeight] */ @@ -310,32 +364,16 @@ class BlocksRepository { public async $getBlockByHeight(height: number): Promise { try { const [rows]: any[] = await DB.query(`SELECT - blocks.height, - hash, + blocks.*, hash as id, UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp, - size, - weight, - tx_count, - coinbase_raw, - difficulty, pools.id as pool_id, pools.name as pool_name, pools.link as pool_link, pools.slug as pool_slug, pools.addresses as pool_addresses, pools.regexes as pool_regexes, - fees, - fee_span, - median_fee, - reward, - version, - bits, - nonce, - merkle_root, - previous_block_hash as previousblockhash, - avg_fee, - avg_fee_rate + previous_block_hash as previousblockhash FROM blocks JOIN pools ON blocks.pool_id = pools.id WHERE blocks.height = ${height} @@ -694,7 +732,6 @@ class BlocksRepository { logger.err('Cannot fetch CPFP unindexed blocks. Reason: ' + (e instanceof Error ? e.message : e)); throw e; } - return []; } /** @@ -741,7 +778,7 @@ class BlocksRepository { try { let query = `INSERT INTO blocks_prices(height, price_id) VALUES`; for (const price of blockPrices) { - query += ` (${price.height}, ${price.priceId}),` + query += ` (${price.height}, ${price.priceId}),`; } query = query.slice(0, -1); await DB.query(query); @@ -754,6 +791,24 @@ class BlocksRepository { } } } + + /** + * Get all indexed blocsk with missing coinstatsindex data + */ + public async $getBlocksMissingCoinStatsIndex(maxHeight: number, minHeight: number): Promise { + try { + const [blocks] = await DB.query(` + SELECT height, hash + FROM blocks + WHERE height >= ${minHeight} AND height <= ${maxHeight} AND + (utxoset_size IS NULL OR total_input_amt IS NULL) + `); + return blocks; + } catch (e) { + logger.err(`Cannot get blocks with missing coinstatsindex. Reason: ` + (e instanceof Error ? e.message : e)); + throw e; + } + } } export default new BlocksRepository(); diff --git a/backend/src/rpc-api/commands.ts b/backend/src/rpc-api/commands.ts index 5905a2bb6..78f5e12f4 100644 --- a/backend/src/rpc-api/commands.ts +++ b/backend/src/rpc-api/commands.ts @@ -89,5 +89,6 @@ module.exports = { walletLock: 'walletlock', walletPassphrase: 'walletpassphrase', walletPassphraseChange: 'walletpassphrasechange', - getTxoutSetinfo: 'gettxoutsetinfo' -} + getTxoutSetinfo: 'gettxoutsetinfo', + getIndexInfo: 'getindexinfo', +};