diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d0520f59c..2ff25b22f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,7 @@ jobs: if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')" strategy: matrix: - node: ["16.16.0", "18.14.1", "19.6.1"] + node: ["16.16.0", "18.14.1"] flavor: ["dev", "prod"] fail-fast: false runs-on: "ubuntu-latest" @@ -55,7 +55,7 @@ jobs: if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')" strategy: matrix: - node: ["16.16.0", "18.14.1", "19.6.1"] + node: ["16.16.0", "18.14.1"] flavor: ["dev", "prod"] fail-fast: false runs-on: "ubuntu-latest" diff --git a/.github/workflows/cypress.yml b/.github/workflows/cypress.yml index e8f6d1df1..bc66678d4 100644 --- a/.github/workflows/cypress.yml +++ b/.github/workflows/cypress.yml @@ -1,8 +1,11 @@ name: Cypress Tests on: + push: + branches: [master] pull_request: - types: [opened, review_requested, synchronize] + types: [opened, synchronize] + jobs: cypress: if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')" diff --git a/README.md b/README.md index cde9b5adb..d2f9f9382 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # The Mempool Open Source Project™ [![mempool](https://img.shields.io/endpoint?url=https://dashboard.cypress.io/badge/simple/ry4br7/master&style=flat-square)](https://dashboard.cypress.io/projects/ry4br7/runs) +https://user-images.githubusercontent.com/232186/222445818-234aa6c9-c233-4c52-b3f0-e32b8232893b.mp4 + Mempool is the fully-featured mempool visualizer, explorer, and API service running at [mempool.space](https://mempool.space/). It is an open-source project developed and operated for the benefit of the Bitcoin community, with a focus on the emerging transaction fee market that is evolving Bitcoin into a multi-layer ecosystem. diff --git a/backend/README.md b/backend/README.md index d00dc1812..be85d25af 100644 --- a/backend/README.md +++ b/backend/README.md @@ -160,7 +160,7 @@ npm install -g ts-node nodemon Then, run the watcher: ``` -nodemon src/index.ts --ignore cache/ --ignore pools.json +nodemon src/index.ts --ignore cache/ ``` `nodemon` should be in npm's global binary folder. If needed, you can determine where that is with `npm -g bin`. @@ -219,6 +219,16 @@ Generate block at regular interval (every 10 seconds in this example): watch -n 10 "./src/bitcoin-cli -regtest -rpcport=8332 generatetoaddress 1 $address" ``` +### Mining pools update + +By default, mining pools will be not automatically updated regularly (`config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING` is set to `false`). + +To manually update your mining pools, you can use the `--update-pools` command line flag when you run the nodejs backend. For example `npm run start --update-pools`. This will trigger the mining pools update and automatically re-index appropriate blocks. + +You can enabled the automatic mining pools update by settings `config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING` to `true` in your `mempool-config.json`. + +When a `coinbase tag` or `coinbase address` change is detected, all blocks tagged to the `unknown` mining pools (starting from height 130635) will be deleted from the `blocks` table. Additionaly, all blocks which were tagged to the pool which has been updated will also be deleted from the `blocks` table. Of course, those blocks will be automatically reindexed. + ### Re-index tables You can manually force the nodejs backend to drop all data from a specified set of tables for future re-index. This is mostly useful for the mining dashboard and the lightning explorer. @@ -235,4 +245,4 @@ Feb 13 14:55:27 [63246] WARN: Indexed data for "hashrates" tables wi Feb 13 14:55:32 [63246] NOTICE: Table hashrates has been truncated ``` -Reference: https://github.com/mempool/mempool/pull/1269 \ No newline at end of file +Reference: https://github.com/mempool/mempool/pull/1269 diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 5ebb25ea5..2369b64b5 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -22,7 +22,7 @@ "USER_AGENT": "mempool", "STDOUT_LOG_MIN_PRIORITY": "debug", "AUTOMATIC_BLOCK_REINDEXING": false, - "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json", + "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json", "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master", "AUDIT": false, "ADVANCED_GBT_AUDIT": false, diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index 9890654a5..2bf52cbcf 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -7,7 +7,7 @@ "HTTP_PORT": 1, "SPAWN_CLUSTER_PROCS": 2, "API_URL_PREFIX": "__MEMPOOL_API_URL_PREFIX__", - "AUTOMATIC_BLOCK_REINDEXING": true, + "AUTOMATIC_BLOCK_REINDEXING": false, "POLL_RATE_MS": 3, "CACHE_DIR": "__MEMPOOL_CACHE_DIR__", "CLEAR_PROTECTION_MINUTES": 4, diff --git a/backend/src/api/audit.ts b/backend/src/api/audit.ts index b6b36dbdc..5b67dc965 100644 --- a/backend/src/api/audit.ts +++ b/backend/src/api/audit.ts @@ -119,7 +119,8 @@ class Audit { } const numCensored = Object.keys(isCensored).length; - const score = matches.length > 0 ? (matches.length / (matches.length + numCensored)) : 0; + const numMatches = matches.length - 1; // adjust for coinbase tx + const score = numMatches > 0 ? (numMatches / (numMatches + numCensored)) : 0; return { censored: Object.keys(isCensored), diff --git a/backend/src/api/bitcoin/bitcoin-api.interface.ts b/backend/src/api/bitcoin/bitcoin-api.interface.ts index 54d666794..3afc22897 100644 --- a/backend/src/api/bitcoin/bitcoin-api.interface.ts +++ b/backend/src/api/bitcoin/bitcoin-api.interface.ts @@ -172,4 +172,35 @@ export namespace IBitcoinApi { } } + export interface BlockStats { + "avgfee": number; + "avgfeerate": number; + "avgtxsize": number; + "blockhash": string; + "feerate_percentiles": [number, number, number, number, number]; + "height": number; + "ins": number; + "maxfee": number; + "maxfeerate": number; + "maxtxsize": number; + "medianfee": number; + "mediantime": number; + "mediantxsize": number; + "minfee": number; + "minfeerate": number; + "mintxsize": number; + "outs": number; + "subsidy": number; + "swtotal_size": number; + "swtotal_weight": number; + "swtxs": number; + "time": number; + "total_out": number; + "total_size": number; + "total_weight": number; + "totalfee": number; + "txs": number; + "utxo_increase": number; + "utxo_size_inc": number; + } } diff --git a/backend/src/api/bitcoin/bitcoin-api.ts b/backend/src/api/bitcoin/bitcoin-api.ts index 117245ef8..e20fe9e34 100644 --- a/backend/src/api/bitcoin/bitcoin-api.ts +++ b/backend/src/api/bitcoin/bitcoin-api.ts @@ -28,7 +28,7 @@ class BitcoinApi implements AbstractBitcoinApi { size: block.size, weight: block.weight, previousblockhash: block.previousblockhash, - medianTime: block.mediantime, + mediantime: block.mediantime, }; } diff --git a/backend/src/api/bitcoin/bitcoin.routes.ts b/backend/src/api/bitcoin/bitcoin.routes.ts index 78d027663..2fc497650 100644 --- a/backend/src/api/bitcoin/bitcoin.routes.ts +++ b/backend/src/api/bitcoin/bitcoin.routes.ts @@ -217,7 +217,15 @@ class BitcoinRoutes { res.json(cpfpInfo); return; } else { - const cpfpInfo = await transactionRepository.$getCpfpInfo(req.params.txId); + let cpfpInfo; + if (config.DATABASE.ENABLED) { + cpfpInfo = await transactionRepository.$getCpfpInfo(req.params.txId); + } else { + res.json({ + ancestors: [] + }); + return; + } if (cpfpInfo) { res.json(cpfpInfo); return; diff --git a/backend/src/api/bitcoin/esplora-api.interface.ts b/backend/src/api/bitcoin/esplora-api.interface.ts index eaf6476f4..6d50bddfd 100644 --- a/backend/src/api/bitcoin/esplora-api.interface.ts +++ b/backend/src/api/bitcoin/esplora-api.interface.ts @@ -88,7 +88,7 @@ export namespace IEsploraApi { size: number; weight: number; previousblockhash: string; - medianTime?: number; + mediantime: number; } export interface Address { diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 204419496..aa33f1ff7 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -2,7 +2,7 @@ import config from '../config'; import bitcoinApi from './bitcoin/bitcoin-api-factory'; import logger from '../logger'; import memPool from './mempool'; -import { BlockExtended, BlockSummary, PoolTag, TransactionExtended, TransactionStripped, TransactionMinerInfo } from '../mempool.interfaces'; +import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionStripped, TransactionMinerInfo } from '../mempool.interfaces'; import { Common } from './common'; import diskCache from './disk-cache'; import transactionUtils from './transaction-utils'; @@ -13,7 +13,6 @@ import poolsRepository from '../repositories/PoolsRepository'; import blocksRepository from '../repositories/BlocksRepository'; import loadingIndicators from './loading-indicators'; import BitcoinApi from './bitcoin/bitcoin-api'; -import { prepareBlock } from '../utils/blocks-utils'; import BlocksRepository from '../repositories/BlocksRepository'; import HashratesRepository from '../repositories/HashratesRepository'; import indexer from '../indexer'; @@ -143,7 +142,7 @@ class Blocks { * @param block * @returns BlockSummary */ - private summarizeBlock(block: IBitcoinApi.VerboseBlock): BlockSummary { + public summarizeBlock(block: IBitcoinApi.VerboseBlock): BlockSummary { const stripped = block.tx.map((tx) => { return { txid: tx.txid, @@ -166,80 +165,81 @@ class Blocks { * @returns BlockExtended */ private async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise { - const blk: BlockExtended = Object.assign({ extras: {} }, block); - blk.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0); - blk.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]); - blk.extras.coinbaseRaw = blk.extras.coinbaseTx.vin[0].scriptsig; - blk.extras.usd = priceUpdater.latestPrices.USD; - blk.extras.medianTimestamp = block.medianTime; - blk.extras.orphans = chainTips.getOrphanedBlocksAtHeight(blk.height); + const coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]); + + const blk: Partial = Object.assign({}, block); + const extras: Partial = {}; + + extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0); + extras.coinbaseRaw = coinbaseTx.vin[0].scriptsig; + extras.orphans = chainTips.getOrphanedBlocksAtHeight(blk.height); if (block.height === 0) { - blk.extras.medianFee = 0; // 50th percentiles - blk.extras.feeRange = [0, 0, 0, 0, 0, 0, 0]; - blk.extras.totalFees = 0; - blk.extras.avgFee = 0; - blk.extras.avgFeeRate = 0; - blk.extras.utxoSetChange = 0; - blk.extras.avgTxSize = 0; - blk.extras.totalInputs = 0; - blk.extras.totalOutputs = 1; - blk.extras.totalOutputAmt = 0; - blk.extras.segwitTotalTxs = 0; - blk.extras.segwitTotalSize = 0; - blk.extras.segwitTotalWeight = 0; + extras.medianFee = 0; // 50th percentiles + extras.feeRange = [0, 0, 0, 0, 0, 0, 0]; + extras.totalFees = 0; + extras.avgFee = 0; + extras.avgFeeRate = 0; + extras.utxoSetChange = 0; + extras.avgTxSize = 0; + extras.totalInputs = 0; + extras.totalOutputs = 1; + extras.totalOutputAmt = 0; + extras.segwitTotalTxs = 0; + extras.segwitTotalSize = 0; + extras.segwitTotalWeight = 0; } else { - const stats = await bitcoinClient.getBlockStats(block.id); - blk.extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles - blk.extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat(); - blk.extras.totalFees = stats.totalfee; - blk.extras.avgFee = stats.avgfee; - blk.extras.avgFeeRate = stats.avgfeerate; - blk.extras.utxoSetChange = stats.utxo_increase; - blk.extras.avgTxSize = Math.round(stats.total_size / stats.txs * 100) * 0.01; - blk.extras.totalInputs = stats.ins; - blk.extras.totalOutputs = stats.outs; - blk.extras.totalOutputAmt = stats.total_out; - blk.extras.segwitTotalTxs = stats.swtxs; - blk.extras.segwitTotalSize = stats.swtotal_size; - blk.extras.segwitTotalWeight = stats.swtotal_weight; + const stats: IBitcoinApi.BlockStats = await bitcoinClient.getBlockStats(block.id); + extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles + extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat(); + extras.totalFees = stats.totalfee; + extras.avgFee = stats.avgfee; + extras.avgFeeRate = stats.avgfeerate; + extras.utxoSetChange = stats.utxo_increase; + extras.avgTxSize = Math.round(stats.total_size / stats.txs * 100) * 0.01; + extras.totalInputs = stats.ins; + extras.totalOutputs = stats.outs; + extras.totalOutputAmt = stats.total_out; + extras.segwitTotalTxs = stats.swtxs; + extras.segwitTotalSize = stats.swtotal_size; + extras.segwitTotalWeight = stats.swtotal_weight; } if (Common.blocksSummariesIndexingEnabled()) { - blk.extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(block.id); - if (blk.extras.feePercentiles !== null) { - blk.extras.medianFeeAmt = blk.extras.feePercentiles[3]; + extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(block.id); + if (extras.feePercentiles !== null) { + extras.medianFeeAmt = extras.feePercentiles[3]; } } - blk.extras.virtualSize = block.weight / 4.0; - if (blk.extras.coinbaseTx.vout.length > 0) { - blk.extras.coinbaseAddress = blk.extras.coinbaseTx.vout[0].scriptpubkey_address ?? null; - blk.extras.coinbaseSignature = blk.extras.coinbaseTx.vout[0].scriptpubkey_asm ?? null; - blk.extras.coinbaseSignatureAscii = transactionUtils.hex2ascii(blk.extras.coinbaseTx.vin[0].scriptsig) ?? null; + extras.virtualSize = block.weight / 4.0; + if (coinbaseTx?.vout.length > 0) { + extras.coinbaseAddress = coinbaseTx.vout[0].scriptpubkey_address ?? null; + extras.coinbaseSignature = coinbaseTx.vout[0].scriptpubkey_asm ?? null; + extras.coinbaseSignatureAscii = transactionUtils.hex2ascii(coinbaseTx.vin[0].scriptsig) ?? null; } else { - blk.extras.coinbaseAddress = null; - blk.extras.coinbaseSignature = null; - blk.extras.coinbaseSignatureAscii = null; + extras.coinbaseAddress = null; + extras.coinbaseSignature = null; + extras.coinbaseSignatureAscii = null; } const header = await bitcoinClient.getBlockHeader(block.id, false); - blk.extras.header = header; + extras.header = header; const coinStatsIndex = indexer.isCoreIndexReady('coinstatsindex'); if (coinStatsIndex !== null && coinStatsIndex.best_block_height >= block.height) { const txoutset = await bitcoinClient.getTxoutSetinfo('none', block.height); - blk.extras.utxoSetSize = txoutset.txouts, - blk.extras.totalInputAmt = Math.round(txoutset.block_info.prevout_spent * 100000000); + extras.utxoSetSize = txoutset.txouts, + extras.totalInputAmt = Math.round(txoutset.block_info.prevout_spent * 100000000); } else { - blk.extras.utxoSetSize = null; - blk.extras.totalInputAmt = null; + extras.utxoSetSize = null; + extras.totalInputAmt = null; } if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) { let pool: PoolTag; - if (blk.extras?.coinbaseTx !== undefined) { - pool = await this.$findBlockMiner(blk.extras?.coinbaseTx); + if (coinbaseTx !== undefined) { + pool = await this.$findBlockMiner(coinbaseTx); } else { if (config.DATABASE.ENABLED === true) { pool = await poolsRepository.$getUnknownPool(); @@ -252,22 +252,24 @@ class Blocks { logger.warn(`Cannot assign pool to block ${blk.height} and 'unknown' pool does not exist. ` + `Check your "pools" table entries`); } else { - blk.extras.pool = { - id: pool.id, + extras.pool = { + id: pool.uniqueId, name: pool.name, slug: pool.slug, }; } + extras.matchRate = null; if (config.MEMPOOL.AUDIT) { const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(block.id); if (auditScore != null) { - blk.extras.matchRate = auditScore.matchRate; + extras.matchRate = auditScore.matchRate; } } } - return blk; + blk.extras = extras; + return blk; } /** @@ -293,15 +295,18 @@ class Blocks { } else { pools = poolsParser.miningPools; } + for (let i = 0; i < pools.length; ++i) { if (address !== undefined) { - const addresses: string[] = JSON.parse(pools[i].addresses); + const addresses: string[] = typeof pools[i].addresses === 'string' ? + JSON.parse(pools[i].addresses) : pools[i].addresses; if (addresses.indexOf(address) !== -1) { return pools[i]; } } - const regexes: string[] = JSON.parse(pools[i].regexes); + const regexes: string[] = typeof pools[i].regexes === 'string' ? + JSON.parse(pools[i].regexes) : pools[i].regexes; for (let y = 0; y < regexes.length; ++y) { const regex = new RegExp(regexes[y], 'i'); const match = asciiScriptSig.match(regex); @@ -479,7 +484,7 @@ class Blocks { loadingIndicators.setProgress('block-indexing', progress, false); } const blockHash = await bitcoinApi.$getBlockHash(blockHeight); - const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash)); + const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash); const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, true); const blockExtended = await this.$getBlockExtended(block, transactions); @@ -527,13 +532,13 @@ class Blocks { if (blockchainInfo.blocks === blockchainInfo.headers) { const heightDiff = blockHeightTip % 2016; const blockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff); - const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash)); + const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash); this.lastDifficultyAdjustmentTime = block.timestamp; this.currentDifficulty = block.difficulty; if (blockHeightTip >= 2016) { const previousPeriodBlockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff - 2016); - const previousPeriodBlock = await bitcoinClient.getBlock(previousPeriodBlockHash) + const previousPeriodBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(previousPeriodBlockHash); this.previousDifficultyRetarget = (block.difficulty - previousPeriodBlock.difficulty) / previousPeriodBlock.difficulty * 100; logger.debug(`Initial difficulty adjustment data set.`); } @@ -565,18 +570,18 @@ class Blocks { if (Common.indexingEnabled()) { if (!fastForwarded) { const lastBlock = await blocksRepository.$getBlockByHeight(blockExtended.height - 1); - if (lastBlock !== null && blockExtended.previousblockhash !== lastBlock['hash']) { - logger.warn(`Chain divergence detected at block ${lastBlock['height']}, re-indexing most recent data`); + if (lastBlock !== null && blockExtended.previousblockhash !== lastBlock.id) { + logger.warn(`Chain divergence detected at block ${lastBlock.height}, re-indexing most recent data`); // We assume there won't be a reorg with more than 10 block depth - await BlocksRepository.$deleteBlocksFrom(lastBlock['height'] - 10); + await BlocksRepository.$deleteBlocksFrom(lastBlock.height - 10); await HashratesRepository.$deleteLastEntries(); - await BlocksSummariesRepository.$deleteBlocksFrom(lastBlock['height'] - 10); - await cpfpRepository.$deleteClustersFrom(lastBlock['height'] - 10); + await BlocksSummariesRepository.$deleteBlocksFrom(lastBlock.height - 10); + await cpfpRepository.$deleteClustersFrom(lastBlock.height - 10); for (let i = 10; i >= 0; --i) { - const newBlock = await this.$indexBlock(lastBlock['height'] - i); + const newBlock = await this.$indexBlock(lastBlock.height - i); await this.$getStrippedBlockTransactions(newBlock.id, true, true); if (config.MEMPOOL.CPFP_INDEXING) { - await this.$indexCPFP(newBlock.id, lastBlock['height'] - i); + await this.$indexCPFP(newBlock.id, lastBlock.height - i); } } await mining.$indexDifficultyAdjustments(); @@ -652,12 +657,12 @@ class Blocks { if (Common.indexingEnabled()) { const dbBlock = await blocksRepository.$getBlockByHeight(height); if (dbBlock !== null) { - return prepareBlock(dbBlock); + return dbBlock; } } const blockHash = await bitcoinApi.$getBlockHash(height); - const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash)); + const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash); const transactions = await this.$getTransactionsExtended(blockHash, block.height, true); const blockExtended = await this.$getBlockExtended(block, transactions); @@ -665,11 +670,11 @@ class Blocks { await blocksRepository.$saveBlockInDatabase(blockExtended); } - return prepareBlock(blockExtended); + return blockExtended; } /** - * Index a block by hash if it's missing from the database. Returns the block after indexing + * Get one block by its hash */ public async $getBlock(hash: string): Promise { // Check the memory cache @@ -678,31 +683,14 @@ class Blocks { return blockByHash; } - // Block has already been indexed - if (Common.indexingEnabled()) { - const dbBlock = await blocksRepository.$getBlockByHash(hash); - if (dbBlock != null) { - return prepareBlock(dbBlock); - } - } - - // Not Bitcoin network, return the block as it + // Not Bitcoin network, return the block as it from the bitcoin backend if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { return await bitcoinApi.$getBlock(hash); } - let block = await bitcoinClient.getBlock(hash); - block = prepareBlock(block); - // Bitcoin network, add our custom data on top - const transactions = await this.$getTransactionsExtended(hash, block.height, true); - const blockExtended = await this.$getBlockExtended(block, transactions); - if (Common.indexingEnabled()) { - delete(blockExtended['coinbaseTx']); - await blocksRepository.$saveBlockInDatabase(blockExtended); - } - - return blockExtended; + const block: IEsploraApi.Block = await bitcoinApi.$getBlock(hash); + return await this.$indexBlock(block.height); } public async $getStrippedBlockTransactions(hash: string, skipMemoryCache = false, @@ -736,6 +724,18 @@ class Blocks { return summary.transactions; } + /** + * Get 15 blocks + * + * Internally this function uses two methods to get the blocks, and + * the method is automatically selected: + * - Using previous block hash links + * - Using block height + * + * @param fromHeight + * @param limit + * @returns + */ public async $getBlocks(fromHeight?: number, limit: number = 15): Promise { let currentHeight = fromHeight !== undefined ? fromHeight : this.currentBlockHeight; if (currentHeight > this.currentBlockHeight) { @@ -748,27 +748,15 @@ class Blocks { return returnBlocks; } - // Check if block height exist in local cache to skip the hash lookup - const blockByHeight = this.getBlocks().find((b) => b.height === currentHeight); - let startFromHash: string | null = null; - if (blockByHeight) { - startFromHash = blockByHeight.id; - } else if (!Common.indexingEnabled()) { - startFromHash = await bitcoinApi.$getBlockHash(currentHeight); - } - - let nextHash = startFromHash; for (let i = 0; i < limit && currentHeight >= 0; i++) { let block = this.getBlocks().find((b) => b.height === currentHeight); if (block) { + // Using the memory cache (find by height) returnBlocks.push(block); - } else if (Common.indexingEnabled()) { + } else { + // Using indexing (find by height, index on the fly, save in database) block = await this.$indexBlock(currentHeight); returnBlocks.push(block); - } else if (nextHash != null) { - block = await this.$indexBlock(currentHeight); - nextHash = block.previousblockhash; - returnBlocks.push(block); } currentHeight--; } @@ -790,7 +778,7 @@ class Blocks { const blocks: any[] = []; while (fromHeight <= toHeight) { - let block: any = await blocksRepository.$getBlockByHeight(fromHeight); + let block: BlockExtended | null = await blocksRepository.$getBlockByHeight(fromHeight); if (!block) { await this.$indexBlock(fromHeight); block = await blocksRepository.$getBlockByHeight(fromHeight); @@ -803,11 +791,11 @@ class Blocks { const cleanBlock: any = { height: block.height ?? null, hash: block.id ?? null, - timestamp: block.blockTimestamp ?? null, - median_timestamp: block.medianTime ?? null, + timestamp: block.timestamp ?? null, + median_timestamp: block.mediantime ?? null, previous_block_hash: block.previousblockhash ?? null, difficulty: block.difficulty ?? null, - header: block.header ?? null, + header: block.extras.header ?? null, version: block.version ?? null, bits: block.bits ?? null, nonce: block.nonce ?? null, @@ -815,29 +803,30 @@ class Blocks { weight: block.weight ?? null, tx_count: block.tx_count ?? null, merkle_root: block.merkle_root ?? null, - reward: block.reward ?? null, - total_fee_amt: block.fees ?? null, - avg_fee_amt: block.avg_fee ?? null, - median_fee_amt: block.median_fee_amt ?? null, - fee_amt_percentiles: block.fee_percentiles ?? null, - avg_fee_rate: block.avg_fee_rate ?? null, - median_fee_rate: block.median_fee ?? null, - fee_rate_percentiles: block.fee_span ?? null, - total_inputs: block.total_inputs ?? null, - total_input_amt: block.total_input_amt ?? null, - total_outputs: block.total_outputs ?? null, - total_output_amt: block.total_output_amt ?? null, - segwit_total_txs: block.segwit_total_txs ?? null, - segwit_total_size: block.segwit_total_size ?? null, - segwit_total_weight: block.segwit_total_weight ?? null, - avg_tx_size: block.avg_tx_size ?? null, - utxoset_change: block.utxoset_change ?? null, - utxoset_size: block.utxoset_size ?? null, - coinbase_raw: block.coinbase_raw ?? null, - coinbase_address: block.coinbase_address ?? null, - coinbase_signature: block.coinbase_signature ?? null, - coinbase_signature_ascii: block.coinbase_signature_ascii ?? null, - pool_slug: block.pool_slug ?? null, + reward: block.extras.reward ?? null, + total_fee_amt: block.extras.totalFees ?? null, + avg_fee_amt: block.extras.avgFee ?? null, + median_fee_amt: block.extras.medianFeeAmt ?? null, + fee_amt_percentiles: block.extras.feePercentiles ?? null, + avg_fee_rate: block.extras.avgFeeRate ?? null, + median_fee_rate: block.extras.medianFee ?? null, + fee_rate_percentiles: block.extras.feeRange ?? null, + total_inputs: block.extras.totalInputs ?? null, + total_input_amt: block.extras.totalInputAmt ?? null, + total_outputs: block.extras.totalOutputs ?? null, + total_output_amt: block.extras.totalOutputAmt ?? null, + segwit_total_txs: block.extras.segwitTotalTxs ?? null, + segwit_total_size: block.extras.segwitTotalSize ?? null, + segwit_total_weight: block.extras.segwitTotalWeight ?? null, + avg_tx_size: block.extras.avgTxSize ?? null, + utxoset_change: block.extras.utxoSetChange ?? null, + utxoset_size: block.extras.utxoSetSize ?? null, + coinbase_raw: block.extras.coinbaseRaw ?? null, + coinbase_address: block.extras.coinbaseAddress ?? null, + coinbase_signature: block.extras.coinbaseSignature ?? null, + coinbase_signature_ascii: block.extras.coinbaseSignatureAscii ?? null, + pool_slug: block.extras.pool.slug ?? null, + pool_id: block.extras.pool.id ?? null, }; if (Common.blocksSummariesIndexingEnabled() && cleanBlock.fee_amt_percentiles === null) { diff --git a/backend/src/api/chain-tips.ts b/backend/src/api/chain-tips.ts index 3384ebb19..b68b0b281 100644 --- a/backend/src/api/chain-tips.ts +++ b/backend/src/api/chain-tips.ts @@ -1,5 +1,5 @@ -import logger from "../logger"; -import bitcoinClient from "./bitcoin/bitcoin-client"; +import logger from '../logger'; +import bitcoinClient from './bitcoin/bitcoin-client'; export interface ChainTip { height: number; @@ -43,7 +43,11 @@ class ChainTips { } } - public getOrphanedBlocksAtHeight(height: number): OrphanedBlock[] { + public getOrphanedBlocksAtHeight(height: number | undefined): OrphanedBlock[] { + if (height === undefined) { + return []; + } + const orphans: OrphanedBlock[] = []; for (const block of this.orphanedBlocks) { if (block.height === height) { diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index 954c1a17d..ad4eccabe 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -237,14 +237,21 @@ export class Common { ].join('x'); } - static utcDateToMysql(date?: number): string { + static utcDateToMysql(date?: number | null): string | null { + if (date === null) { + return null; + } const d = new Date((date || 0) * 1000); return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0]; } static findSocketNetwork(addr: string): {network: string | null, url: string} { let network: string | null = null; - let url = addr.split('://')[1]; + let url: string = addr; + + if (config.LIGHTNING.BACKEND === 'cln') { + url = addr.split('://')[1]; + } if (!url) { return { @@ -261,7 +268,7 @@ export class Common { } } else if (addr.indexOf('i2p') !== -1) { network = 'i2p'; - } else if (addr.indexOf('ipv4') !== -1) { + } else if (addr.indexOf('ipv4') !== -1 || (config.LIGHTNING.BACKEND === 'lnd' && isIP(url.split(':')[0]) === 4)) { const ipv = isIP(url.split(':')[0]); if (ipv === 4) { network = 'ipv4'; @@ -271,7 +278,7 @@ export class Common { url: addr, }; } - } else if (addr.indexOf('ipv6') !== -1) { + } else if (addr.indexOf('ipv6') !== -1 || (config.LIGHTNING.BACKEND === 'lnd' && url.indexOf(']:'))) { url = url.split('[')[1].split(']')[0]; const ipv = isIP(url); if (ipv === 6) { diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 3140ea358..d40637e74 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository'; import { RowDataPacket } from 'mysql2'; class DatabaseMigration { - private static currentVersion = 56; + private static currentVersion = 57; private queryTimeout = 3600_000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -500,6 +500,11 @@ class DatabaseMigration { this.uniqueLog(logger.notice, '`pools` table has been truncated`'); await this.updateToSchemaVersion(56); } + + if (databaseSchemaVersion < 57 && isBitcoin === true) { + await this.$executeQuery(`ALTER TABLE nodes MODIFY updated_at datetime NULL`); + await this.updateToSchemaVersion(57); + } } /** @@ -1012,26 +1017,16 @@ class DatabaseMigration { ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; } - public async $truncateIndexedData(tables: string[]) { - const allowedTables = ['blocks', 'hashrates', 'prices']; + public async $blocksReindexingTruncate(): Promise { + logger.warn(`Truncating pools, blocks and hashrates for re-indexing (using '--reindex-blocks'). You can cancel this command within 5 seconds`); + await Common.sleep$(5000); - try { - for (const table of tables) { - if (!allowedTables.includes(table)) { - logger.debug(`Table ${table} cannot to be re-indexed (not allowed)`); - continue; - } - - await this.$executeQuery(`TRUNCATE ${table}`, true); - if (table === 'hashrates') { - await this.$executeQuery('UPDATE state set number = 0 where name = "last_hashrates_indexing"', true); - } - logger.notice(`Table ${table} has been truncated`); - } - } catch (e) { - logger.warn(`Unable to erase indexed data`); - } - } + await this.$executeQuery(`TRUNCATE blocks`); + await this.$executeQuery(`TRUNCATE hashrates`); + await this.$executeQuery('DELETE FROM `pools`'); + await this.$executeQuery('ALTER TABLE pools AUTO_INCREMENT = 1'); + await this.$executeQuery(`UPDATE state SET string = NULL WHERE name = 'pools_json_sha'`); +} private async $convertCompactCpfpTables(): Promise { try { diff --git a/backend/src/api/disk-cache.ts b/backend/src/api/disk-cache.ts index c6af20e29..83d37fe3f 100644 --- a/backend/src/api/disk-cache.ts +++ b/backend/src/api/disk-cache.ts @@ -9,7 +9,7 @@ import { TransactionExtended } from '../mempool.interfaces'; import { Common } from './common'; class DiskCache { - private cacheSchemaVersion = 2; + private cacheSchemaVersion = 3; private static FILE_NAME = config.MEMPOOL.CACHE_DIR + '/cache.json'; private static FILE_NAMES = config.MEMPOOL.CACHE_DIR + '/cache{number}.json'; diff --git a/backend/src/api/explorer/channels.api.ts b/backend/src/api/explorer/channels.api.ts index 8314b3345..00d146770 100644 --- a/backend/src/api/explorer/channels.api.ts +++ b/backend/src/api/explorer/channels.api.ts @@ -559,6 +559,17 @@ class ChannelsApi { const policy1: Partial = channel.node1_policy || {}; const policy2: Partial = channel.node2_policy || {}; + // https://github.com/mempool/mempool/issues/3006 + if ((channel.last_update ?? 0) < 1514736061) { // January 1st 2018 + channel.last_update = null; + } + if ((policy1.last_update ?? 0) < 1514736061) { // January 1st 2018 + policy1.last_update = null; + } + if ((policy2.last_update ?? 0) < 1514736061) { // January 1st 2018 + policy2.last_update = null; + } + const query = `INSERT INTO channels ( id, diff --git a/backend/src/api/explorer/nodes.api.ts b/backend/src/api/explorer/nodes.api.ts index b3f83faa6..e1949a17c 100644 --- a/backend/src/api/explorer/nodes.api.ts +++ b/backend/src/api/explorer/nodes.api.ts @@ -228,7 +228,7 @@ class NodesApi { nodes.capacity FROM nodes ORDER BY capacity DESC - LIMIT 100 + LIMIT 6 `; [rows] = await DB.query(query); @@ -269,14 +269,26 @@ class NodesApi { let query: string; if (full === false) { query = ` - SELECT nodes.public_key as publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias, - nodes.channels + SELECT + nodes.public_key as publicKey, + IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias, + nodes.channels, + geo_names_city.names as city, geo_names_country.names as country, + geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision FROM nodes + LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country' + LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city' + LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code' + LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division' ORDER BY channels DESC - LIMIT 100; + LIMIT 6; `; [rows] = await DB.query(query); + for (let i = 0; i < rows.length; ++i) { + rows[i].country = JSON.parse(rows[i].country); + rows[i].city = JSON.parse(rows[i].city); + } } else { query = ` SELECT nodes.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias, @@ -630,6 +642,11 @@ class NodesApi { */ public async $saveNode(node: ILightningApi.Node): Promise { try { + // https://github.com/mempool/mempool/issues/3006 + if ((node.last_update ?? 0) < 1514736061) { // January 1st 2018 + node.last_update = null; + } + const sockets = (node.addresses?.map(a => a.addr).join(',')) ?? ''; const query = `INSERT INTO nodes( public_key, diff --git a/backend/src/api/lightning/lightning-api.interface.ts b/backend/src/api/lightning/lightning-api.interface.ts index 453e2fffc..cd5cb973d 100644 --- a/backend/src/api/lightning/lightning-api.interface.ts +++ b/backend/src/api/lightning/lightning-api.interface.ts @@ -21,7 +21,7 @@ export namespace ILightningApi { export interface Channel { channel_id: string; chan_point: string; - last_update: number; + last_update: number | null; node1_pub: string; node2_pub: string; capacity: string; @@ -36,11 +36,11 @@ export namespace ILightningApi { fee_rate_milli_msat: string; disabled: boolean; max_htlc_msat: string; - last_update: number; + last_update: number | null; } export interface Node { - last_update: number; + last_update: number | null; pub_key: string; alias: string; addresses: { diff --git a/backend/src/api/mining/mining.ts b/backend/src/api/mining/mining.ts index f33a68dcb..e9b569485 100644 --- a/backend/src/api/mining/mining.ts +++ b/backend/src/api/mining/mining.ts @@ -11,6 +11,8 @@ import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjust import config from '../../config'; import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository'; import PricesRepository from '../../repositories/PricesRepository'; +import bitcoinApiFactory from '../bitcoin/bitcoin-api-factory'; +import { IEsploraApi } from '../bitcoin/esplora-api.interface'; class Mining { blocksPriceIndexingRunning = false; @@ -189,8 +191,8 @@ class Mining { try { const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp; - const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0)); - const genesisTimestamp = genesisBlock.time * 1000; + const genesisBlock: IEsploraApi.Block = await bitcoinApiFactory.$getBlock(await bitcoinClient.getBlockHash(0)); + const genesisTimestamp = genesisBlock.timestamp * 1000; const indexedTimestamp = await HashratesRepository.$getWeeklyHashrateTimestamps(); const hashrates: any[] = []; @@ -292,8 +294,8 @@ class Mining { const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp; try { - const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0)); - const genesisTimestamp = genesisBlock.time * 1000; + const genesisBlock: IEsploraApi.Block = await bitcoinApiFactory.$getBlock(await bitcoinClient.getBlockHash(0)); + const genesisTimestamp = genesisBlock.timestamp * 1000; const indexedTimestamp = (await HashratesRepository.$getRawNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp); const lastMidnight = this.getDateMidnight(new Date()); let toTimestamp = Math.round(lastMidnight.getTime()); @@ -394,13 +396,13 @@ class Mining { } const blocks: any = await BlocksRepository.$getBlocksDifficulty(); - const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0)); + const genesisBlock: IEsploraApi.Block = await bitcoinApiFactory.$getBlock(await bitcoinClient.getBlockHash(0)); let currentDifficulty = genesisBlock.difficulty; let totalIndexed = 0; if (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT === -1 && indexedHeights[0] !== true) { await DifficultyAdjustmentsRepository.$saveAdjustments({ - time: genesisBlock.time, + time: genesisBlock.timestamp, height: 0, difficulty: currentDifficulty, adjustment: 0.0, diff --git a/backend/src/api/pools-parser.ts b/backend/src/api/pools-parser.ts index d1705b829..f17df385a 100644 --- a/backend/src/api/pools-parser.ts +++ b/backend/src/api/pools-parser.ts @@ -8,6 +8,7 @@ import diskCache from './disk-cache'; class PoolsParser { miningPools: any[] = []; unknownPool: any = { + 'id': 0, 'name': 'Unknown', 'link': 'https://learnmeabitcoin.com/technical/coinbase-transaction', 'regexes': '[]', @@ -27,6 +28,7 @@ class PoolsParser { public setMiningPools(pools): void { for (const pool of pools) { pool.regexes = pool.tags; + pool.slug = pool.name.replace(/[^a-z0-9]/gi, '').toLowerCase(); delete(pool.tags); } this.miningPools = pools; @@ -74,7 +76,7 @@ class PoolsParser { } } - logger.info('Mining pools.json import completed'); + logger.info('Mining pools-v2.json import completed'); } /** @@ -116,7 +118,7 @@ class PoolsParser { return; } - // Get oldest blocks mined by the pool and assume pools.json updates only concern most recent years + // Get oldest blocks mined by the pool and assume pools-v2.json updates only concern most recent years // Ignore early days of Bitcoin as there were no mining pool yet const [oldestPoolBlock]: any[] = await DB.query(` SELECT height diff --git a/backend/src/index.ts b/backend/src/index.ts index 6ea3ddc43..812b49e15 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -36,7 +36,6 @@ import bitcoinRoutes from './api/bitcoin/bitcoin.routes'; import fundingTxFetcher from './tasks/lightning/sync-tasks/funding-tx-fetcher'; import forensicsService from './tasks/lightning/forensics.service'; import priceUpdater from './tasks/price-updater'; -import mining from './api/mining/mining'; import chainTips from './api/chain-tips'; import { AxiosError } from 'axios'; @@ -84,11 +83,8 @@ class Server { if (config.DATABASE.ENABLED) { await DB.checkDbConnection(); try { - if (process.env.npm_config_reindex !== undefined) { // Re-index requests - const tables = process.env.npm_config_reindex.split(','); - logger.warn(`Indexed data for "${process.env.npm_config_reindex}" tables will be erased in 5 seconds (using '--reindex')`); - await Common.sleep$(5000); - await databaseMigration.$truncateIndexedData(tables); + if (process.env.npm_config_reindex_blocks === 'true') { // Re-index requests + await databaseMigration.$blocksReindexingTruncate(); } await databaseMigration.$initializeOrMigrateDatabase(); if (Common.indexingEnabled()) { @@ -183,7 +179,14 @@ class Server { setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS); this.currentBackendRetryInterval = 5; } catch (e: any) { - const loggerMsg = `runMainLoop error: ${(e instanceof Error ? e.message : e)}. Retrying in ${this.currentBackendRetryInterval} sec.`; + let loggerMsg = `Exception in runMainUpdateLoop(). Retrying in ${this.currentBackendRetryInterval} sec.`; + loggerMsg += ` Reason: ${(e instanceof Error ? e.message : e)}.`; + if (e?.stack) { + loggerMsg += ` Stack trace: ${e.stack}`; + } + // When we get a first Exception, only `logger.debug` it and retry after 5 seconds + // From the second Exception, `logger.warn` the Exception and increase the retry delay + // Maximum retry delay is 60 seconds if (this.currentBackendRetryInterval > 5) { logger.warn(loggerMsg); mempool.setOutOfSync(); @@ -203,8 +206,8 @@ class Server { try { await fundingTxFetcher.$init(); await networkSyncService.$startService(); - await forensicsService.$startService(); await lightningStatsUpdater.$startService(); + await forensicsService.$startService(); } catch(e) { logger.err(`Nodejs lightning backend crashed. Restarting in 1 minute. Reason: ${(e instanceof Error ? e.message : e)}`); await Common.sleep$(1000 * 60); diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index cb95be98a..a7937e01d 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -1,9 +1,10 @@ import { IEsploraApi } from './api/bitcoin/esplora-api.interface'; import { OrphanedBlock } from './api/chain-tips'; -import { HeapNode } from "./utils/pairing-heap"; +import { HeapNode } from './utils/pairing-heap'; export interface PoolTag { - id: number; // mysql row id + id: number; + uniqueId: number; name: string; link: string; regexes: string; // JSON array @@ -147,44 +148,44 @@ export interface TransactionStripped { } export interface BlockExtension { - totalFees?: number; - medianFee?: number; - feeRange?: number[]; - reward?: number; - coinbaseTx?: TransactionMinerInfo; - matchRate?: number; - pool?: { - id: number; + totalFees: number; + medianFee: number; // median fee rate + feeRange: number[]; // fee rate percentiles + reward: number; + matchRate: number | null; + pool: { + id: number; // Note - This is the `unique_id`, not to mix with the auto increment `id` name: string; slug: string; }; - avgFee?: number; - avgFeeRate?: number; - coinbaseRaw?: string; - usd?: number | null; - medianTimestamp?: number; - blockTime?: number; - orphans?: OrphanedBlock[] | null; - coinbaseAddress?: string | null; - coinbaseSignature?: string | null; - coinbaseSignatureAscii?: string | null; - virtualSize?: number; - avgTxSize?: number; - totalInputs?: number; - totalOutputs?: number; - totalOutputAmt?: number; - medianFeeAmt?: number | null; - feePercentiles?: number[] | null, - segwitTotalTxs?: number; - segwitTotalSize?: number; - segwitTotalWeight?: number; - header?: string; - utxoSetChange?: number; + avgFee: number; + avgFeeRate: number; + coinbaseRaw: string; + orphans: OrphanedBlock[] | null; + coinbaseAddress: string | null; + coinbaseSignature: string | null; + coinbaseSignatureAscii: string | null; + virtualSize: number; + avgTxSize: number; + totalInputs: number; + totalOutputs: number; + totalOutputAmt: number; + medianFeeAmt: number | null; // median fee in sats + feePercentiles: number[] | null, // fee percentiles in sats + segwitTotalTxs: number; + segwitTotalSize: number; + segwitTotalWeight: number; + header: string; + utxoSetChange: number; // Requires coinstatsindex, will be set to NULL otherwise - utxoSetSize?: number | null; - totalInputAmt?: number | null; + utxoSetSize: number | null; + totalInputAmt: number | null; } +/** + * Note: Everything that is added in here will be automatically returned through + * /api/v1/block and /api/v1/blocks APIs + */ export interface BlockExtended extends IEsploraApi.Block { extras: BlockExtension; } diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 86dc006ff..80df1ac92 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -1,8 +1,7 @@ -import { BlockExtended, BlockPrice } from '../mempool.interfaces'; +import { BlockExtended, BlockExtension, BlockPrice } from '../mempool.interfaces'; import DB from '../database'; import logger from '../logger'; import { Common } from '../api/common'; -import { prepareBlock } from '../utils/blocks-utils'; import PoolsRepository from './PoolsRepository'; import HashratesRepository from './HashratesRepository'; import { escape } from 'mysql2'; @@ -10,12 +9,60 @@ import BlocksSummariesRepository from './BlocksSummariesRepository'; import DifficultyAdjustmentsRepository from './DifficultyAdjustmentsRepository'; import bitcoinClient from '../api/bitcoin/bitcoin-client'; import config from '../config'; +import chainTips from '../api/chain-tips'; +import blocks from '../api/blocks'; +import BlocksAuditsRepository from './BlocksAuditsRepository'; + +const BLOCK_DB_FIELDS = ` + blocks.hash AS id, + blocks.height, + blocks.version, + UNIX_TIMESTAMP(blocks.blockTimestamp) AS timestamp, + blocks.bits, + blocks.nonce, + blocks.difficulty, + blocks.merkle_root, + blocks.tx_count, + blocks.size, + blocks.weight, + blocks.previous_block_hash AS previousblockhash, + UNIX_TIMESTAMP(blocks.median_timestamp) AS mediantime, + blocks.fees AS totalFees, + blocks.median_fee AS medianFee, + blocks.fee_span AS feeRange, + blocks.reward, + pools.unique_id AS poolId, + pools.name AS poolName, + pools.slug AS poolSlug, + blocks.avg_fee AS avgFee, + blocks.avg_fee_rate AS avgFeeRate, + blocks.coinbase_raw AS coinbaseRaw, + blocks.coinbase_address AS coinbaseAddress, + blocks.coinbase_signature AS coinbaseSignature, + blocks.coinbase_signature_ascii AS coinbaseSignatureAscii, + blocks.avg_tx_size AS avgTxSize, + blocks.total_inputs AS totalInputs, + blocks.total_outputs AS totalOutputs, + blocks.total_output_amt AS totalOutputAmt, + blocks.median_fee_amt AS medianFeeAmt, + blocks.fee_percentiles AS feePercentiles, + blocks.segwit_total_txs AS segwitTotalTxs, + blocks.segwit_total_size AS segwitTotalSize, + blocks.segwit_total_weight AS segwitTotalWeight, + blocks.header, + blocks.utxoset_change AS utxoSetChange, + blocks.utxoset_size AS utxoSetSize, + blocks.total_input_amt AS totalInputAmts +`; class BlocksRepository { /** * Save indexed block data in the database */ public async $saveBlockInDatabase(block: BlockExtended) { + const truncatedCoinbaseSignature = block?.extras?.coinbaseSignature?.substring(0, 500); + const truncatedCoinbaseSignatureAscii = block?.extras?.coinbaseSignatureAscii?.substring(0, 500); + try { const query = `INSERT INTO blocks( height, hash, blockTimestamp, size, @@ -41,6 +88,11 @@ class BlocksRepository { ?, ? )`; + const poolDbId = await PoolsRepository.$getPoolByUniqueId(block.extras.pool.id); + if (!poolDbId) { + throw Error(`Could not find a mining pool with the unique_id = ${block.extras.pool.id}. This error should never be printed.`); + } + const params: any[] = [ block.height, block.id, @@ -50,7 +102,7 @@ class BlocksRepository { block.tx_count, block.extras.coinbaseRaw, block.difficulty, - block.extras.pool?.id, // Should always be set to something + poolDbId.id, block.extras.totalFees, JSON.stringify(block.extras.feeRange), block.extras.medianFee, @@ -62,10 +114,10 @@ class BlocksRepository { block.previousblockhash, block.extras.avgFee, block.extras.avgFeeRate, - block.extras.medianTimestamp, + block.mediantime, block.extras.header, block.extras.coinbaseAddress, - block.extras.coinbaseSignature, + truncatedCoinbaseSignature, block.extras.utxoSetSize, block.extras.utxoSetChange, block.extras.avgTxSize, @@ -78,15 +130,15 @@ class BlocksRepository { block.extras.segwitTotalSize, block.extras.segwitTotalWeight, block.extras.medianFeeAmt, - block.extras.coinbaseSignatureAscii, + truncatedCoinbaseSignatureAscii, ]; await DB.query(query, params); } catch (e: any) { if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart - logger.debug(`$saveBlockInDatabase() - Block ${block.height} has already been indexed, ignoring`); + logger.debug(`$saveBlockInDatabase() - Block ${block.height} has already been indexed, ignoring`, logger.tags.mining); } else { - logger.err('Cannot save indexed block into db. Reason: ' + (e instanceof Error ? e.message : e)); + logger.err('Cannot save indexed block into db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining); throw e; } } @@ -304,34 +356,17 @@ class BlocksRepository { /** * Get blocks mined by a specific mining pool */ - public async $getBlocksByPool(slug: string, startHeight?: number): Promise { + public async $getBlocksByPool(slug: string, startHeight?: number): Promise { const pool = await PoolsRepository.$getPool(slug); if (!pool) { throw new Error('This mining pool does not exist ' + escape(slug)); } const params: any[] = []; - let query = ` SELECT - blocks.height, - hash as id, - UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp, - size, - weight, - tx_count, - coinbase_raw, - difficulty, - fees, - fee_span, - median_fee, - reward, - version, - bits, - nonce, - merkle_root, - previous_block_hash as previousblockhash, - avg_fee, - avg_fee_rate + let query = ` + SELECT ${BLOCK_DB_FIELDS} FROM blocks + JOIN pools ON blocks.pool_id = pools.id WHERE pool_id = ?`; params.push(pool.id); @@ -344,11 +379,11 @@ class BlocksRepository { LIMIT 10`; try { - const [rows] = await DB.query(query, params); + const [rows]: any[] = await DB.query(query, params); const blocks: BlockExtended[] = []; - for (const block of rows) { - blocks.push(prepareBlock(block)); + for (const block of rows) { + blocks.push(await this.formatDbBlockIntoExtendedBlock(block)); } return blocks; @@ -361,32 +396,21 @@ class BlocksRepository { /** * Get one block by height */ - public async $getBlockByHeight(height: number): Promise { + public async $getBlockByHeight(height: number): Promise { try { - const [rows]: any[] = await DB.query(`SELECT - blocks.*, - hash as id, - UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp, - UNIX_TIMESTAMP(blocks.median_timestamp) as medianTime, - pools.id as pool_id, - pools.name as pool_name, - pools.link as pool_link, - pools.slug as pool_slug, - pools.addresses as pool_addresses, - pools.regexes as pool_regexes, - previous_block_hash as previousblockhash + const [rows]: any[] = await DB.query(` + SELECT ${BLOCK_DB_FIELDS} FROM blocks JOIN pools ON blocks.pool_id = pools.id - WHERE blocks.height = ${height} - `); + WHERE blocks.height = ?`, + [height] + ); if (rows.length <= 0) { return null; } - rows[0].fee_span = JSON.parse(rows[0].fee_span); - rows[0].fee_percentiles = JSON.parse(rows[0].fee_percentiles); - return rows[0]; + return await this.formatDbBlockIntoExtendedBlock(rows[0]); } catch (e) { logger.err(`Cannot get indexed block ${height}. Reason: ` + (e instanceof Error ? e.message : e)); throw e; @@ -399,10 +423,7 @@ class BlocksRepository { public async $getBlockByHash(hash: string): Promise { try { const query = ` - SELECT *, blocks.height, UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp, hash as id, - pools.id as pool_id, pools.name as pool_name, pools.link as pool_link, pools.slug as pool_slug, - pools.addresses as pool_addresses, pools.regexes as pool_regexes, - previous_block_hash as previousblockhash + SELECT ${BLOCK_DB_FIELDS} FROM blocks JOIN pools ON blocks.pool_id = pools.id WHERE hash = ?; @@ -412,9 +433,8 @@ class BlocksRepository { if (rows.length <= 0) { return null; } - - rows[0].fee_span = JSON.parse(rows[0].fee_span); - return rows[0]; + + return await this.formatDbBlockIntoExtendedBlock(rows[0]); } catch (e) { logger.err(`Cannot get indexed block ${hash}. Reason: ` + (e instanceof Error ? e.message : e)); throw e; @@ -505,8 +525,15 @@ class BlocksRepository { public async $validateChain(): Promise { try { const start = new Date().getTime(); - const [blocks]: any[] = await DB.query(`SELECT height, hash, previous_block_hash, - UNIX_TIMESTAMP(blockTimestamp) as timestamp FROM blocks ORDER BY height`); + const [blocks]: any[] = await DB.query(` + SELECT + height, + hash, + previous_block_hash, + UNIX_TIMESTAMP(blockTimestamp) AS timestamp + FROM blocks + ORDER BY height + `); let partialMsg = false; let idx = 1; @@ -830,6 +857,95 @@ class BlocksRepository { throw e; } } + + /** + * Convert a mysql row block into a BlockExtended. Note that you + * must provide the correct field into dbBlk object param + * + * @param dbBlk + */ + private async formatDbBlockIntoExtendedBlock(dbBlk: any): Promise { + const blk: Partial = {}; + const extras: Partial = {}; + + // IEsploraApi.Block + blk.id = dbBlk.id; + blk.height = dbBlk.height; + blk.version = dbBlk.version; + blk.timestamp = dbBlk.timestamp; + blk.bits = dbBlk.bits; + blk.nonce = dbBlk.nonce; + blk.difficulty = dbBlk.difficulty; + blk.merkle_root = dbBlk.merkle_root; + blk.tx_count = dbBlk.tx_count; + blk.size = dbBlk.size; + blk.weight = dbBlk.weight; + blk.previousblockhash = dbBlk.previousblockhash; + blk.mediantime = dbBlk.mediantime; + + // BlockExtension + extras.totalFees = dbBlk.totalFees; + extras.medianFee = dbBlk.medianFee; + extras.feeRange = JSON.parse(dbBlk.feeRange); + extras.reward = dbBlk.reward; + extras.pool = { + id: dbBlk.poolId, + name: dbBlk.poolName, + slug: dbBlk.poolSlug, + }; + extras.avgFee = dbBlk.avgFee; + extras.avgFeeRate = dbBlk.avgFeeRate; + extras.coinbaseRaw = dbBlk.coinbaseRaw; + extras.coinbaseAddress = dbBlk.coinbaseAddress; + extras.coinbaseSignature = dbBlk.coinbaseSignature; + extras.coinbaseSignatureAscii = dbBlk.coinbaseSignatureAscii; + extras.avgTxSize = dbBlk.avgTxSize; + extras.totalInputs = dbBlk.totalInputs; + extras.totalOutputs = dbBlk.totalOutputs; + extras.totalOutputAmt = dbBlk.totalOutputAmt; + extras.medianFeeAmt = dbBlk.medianFeeAmt; + extras.feePercentiles = JSON.parse(dbBlk.feePercentiles); + extras.segwitTotalTxs = dbBlk.segwitTotalTxs; + extras.segwitTotalSize = dbBlk.segwitTotalSize; + extras.segwitTotalWeight = dbBlk.segwitTotalWeight; + extras.header = dbBlk.header, + extras.utxoSetChange = dbBlk.utxoSetChange; + extras.utxoSetSize = dbBlk.utxoSetSize; + extras.totalInputAmt = dbBlk.totalInputAmt; + extras.virtualSize = dbBlk.weight / 4.0; + + // Re-org can happen after indexing so we need to always get the + // latest state from core + extras.orphans = chainTips.getOrphanedBlocksAtHeight(dbBlk.height); + + // Match rate is not part of the blocks table, but it is part of APIs so we must include it + extras.matchRate = null; + if (config.MEMPOOL.AUDIT) { + const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(dbBlk.id); + if (auditScore != null) { + extras.matchRate = auditScore.matchRate; + } + } + + // If we're missing block summary related field, check if we can populate them on the fly now + if (Common.blocksSummariesIndexingEnabled() && + (extras.medianFeeAmt === null || extras.feePercentiles === null)) + { + extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id); + if (extras.feePercentiles === null) { + const block = await bitcoinClient.getBlock(dbBlk.id, 2); + const summary = blocks.summarizeBlock(block); + await BlocksSummariesRepository.$saveSummary({ height: block.height, mined: summary }); + extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id); + } + if (extras.feePercentiles !== null) { + extras.medianFeeAmt = extras.feePercentiles[3]; + } + } + + blk.extras = extras; + return blk; + } } export default new BlocksRepository(); diff --git a/backend/src/repositories/PoolsRepository.ts b/backend/src/repositories/PoolsRepository.ts index 63bddd497..293fd5e39 100644 --- a/backend/src/repositories/PoolsRepository.ts +++ b/backend/src/repositories/PoolsRepository.ts @@ -10,7 +10,7 @@ class PoolsRepository { * Get all pools tagging info */ public async $getPools(): Promise { - const [rows] = await DB.query('SELECT id, name, addresses, regexes, slug FROM pools;'); + const [rows] = await DB.query('SELECT id, unique_id as uniqueId, name, addresses, regexes, slug FROM pools'); return rows; } @@ -18,10 +18,10 @@ class PoolsRepository { * Get unknown pool tagging info */ public async $getUnknownPool(): Promise { - let [rows]: any[] = await DB.query('SELECT id, name, slug FROM pools where name = "Unknown"'); + let [rows]: any[] = await DB.query('SELECT id, unique_id as uniqueId, name, slug FROM pools where name = "Unknown"'); if (rows && rows.length === 0 && config.DATABASE.ENABLED) { await poolsParser.$insertUnknownPool(); - [rows] = await DB.query('SELECT id, name, slug FROM pools where name = "Unknown"'); + [rows] = await DB.query('SELECT id, unique_id as uniqueId, name, slug FROM pools where name = "Unknown"'); } return rows[0]; } @@ -99,7 +99,7 @@ class PoolsRepository { rows[0].regexes = JSON.parse(rows[0].regexes); } if (['testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) { - rows[0].addresses = []; // pools.json only contains mainnet addresses + rows[0].addresses = []; // pools-v2.json only contains mainnet addresses } else if (parse) { rows[0].addresses = JSON.parse(rows[0].addresses); } diff --git a/backend/src/repositories/PricesRepository.ts b/backend/src/repositories/PricesRepository.ts index 83336eaff..6493735ee 100644 --- a/backend/src/repositories/PricesRepository.ts +++ b/backend/src/repositories/PricesRepository.ts @@ -40,7 +40,7 @@ export const MAX_PRICES = { class PricesRepository { public async $savePrices(time: number, prices: IConversionRates): Promise { - if (prices.USD === 0) { + if (prices.USD === -1) { // Some historical price entries have no USD prices, so we just ignore them to avoid future UX issues // As of today there are only 4 (on 2013-09-05, 2013-0909, 2013-09-12 and 2013-09-26) so that's fine return; diff --git a/backend/src/tasks/lightning/network-sync.service.ts b/backend/src/tasks/lightning/network-sync.service.ts index fdef7ecae..3e5ae1366 100644 --- a/backend/src/tasks/lightning/network-sync.service.ts +++ b/backend/src/tasks/lightning/network-sync.service.ts @@ -72,7 +72,7 @@ class NetworkSyncService { const graphNodesPubkeys: string[] = []; for (const node of nodes) { const latestUpdated = await channelsApi.$getLatestChannelUpdateForNode(node.pub_key); - node.last_update = Math.max(node.last_update, latestUpdated); + node.last_update = Math.max(node.last_update ?? 0, latestUpdated); await nodesApi.$saveNode(node); graphNodesPubkeys.push(node.pub_key); diff --git a/backend/src/tasks/pools-updater.ts b/backend/src/tasks/pools-updater.ts index 8aa73376f..32de85f3a 100644 --- a/backend/src/tasks/pools-updater.ts +++ b/backend/src/tasks/pools-updater.ts @@ -17,11 +17,6 @@ class PoolsUpdater { treeUrl: string = config.MEMPOOL.POOLS_JSON_TREE_URL; public async updatePoolsJson(): Promise { - if (config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING === false) { - logger.info(`Not updating mining pools to avoid inconsistency because AUTOMATIC_BLOCK_REINDEXING is set to false`) - return; - } - if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { return; } @@ -36,12 +31,6 @@ class PoolsUpdater { this.lastRun = now; - if (config.SOCKS5PROXY.ENABLED) { - logger.info(`Updating latest mining pools from ${this.poolsUrl} over the Tor network`, logger.tags.mining); - } else { - logger.info(`Updating latest mining pools from ${this.poolsUrl} over clearnet`, logger.tags.mining); - } - try { const githubSha = await this.fetchPoolsSha(); // Fetch pools-v2.json sha from github if (githubSha === undefined) { @@ -57,10 +46,21 @@ class PoolsUpdater { return; } + // See backend README for more details about the mining pools update process + if (this.currentSha !== undefined && // If we don't have any mining pool, download it at least once + config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING !== true && // Automatic pools update is disabled + !process.env.npm_config_update_pools // We're not manually updating mining pool + ) { + logger.warn(`Updated mining pools data is available (${githubSha}) but AUTOMATIC_BLOCK_REINDEXING is disabled`); + logger.info(`You can update your mining pools using the --update-pools command flag. You may want to clear your nginx cache as well if applicable`); + return; + } + + const network = config.SOCKS5PROXY.ENABLED ? 'tor' : 'clearnet'; if (this.currentSha === undefined) { - logger.info(`Downloading pools-v2.json for the first time from ${this.poolsUrl}`, logger.tags.mining); + logger.info(`Downloading pools-v2.json for the first time from ${this.poolsUrl} over ${network}`, logger.tags.mining); } else { - logger.warn(`pools-v2.json is outdated, fetch latest from ${this.poolsUrl}`, logger.tags.mining); + logger.warn(`pools-v2.json is outdated, fetch latest from ${this.poolsUrl} over ${network}`, logger.tags.mining); } const poolsJson = await this.query(this.poolsUrl); if (poolsJson === undefined) { diff --git a/backend/src/utils/blocks-utils.ts b/backend/src/utils/blocks-utils.ts deleted file mode 100644 index 43a2fc964..000000000 --- a/backend/src/utils/blocks-utils.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { BlockExtended } from '../mempool.interfaces'; - -export function prepareBlock(block: any): BlockExtended { - return { - id: block.id ?? block.hash, // hash for indexed block - timestamp: block.timestamp ?? block.time ?? block.blockTimestamp, // blockTimestamp for indexed block - height: block.height, - version: block.version, - bits: (typeof block.bits === 'string' ? parseInt(block.bits, 16): block.bits), - nonce: block.nonce, - difficulty: block.difficulty, - merkle_root: block.merkle_root ?? block.merkleroot, - tx_count: block.tx_count ?? block.nTx, - size: block.size, - weight: block.weight, - previousblockhash: block.previousblockhash, - extras: { - coinbaseRaw: block.coinbase_raw ?? block.extras?.coinbaseRaw, - medianFee: block.medianFee ?? block.median_fee ?? block.extras?.medianFee, - feeRange: block.feeRange ?? block?.extras?.feeRange ?? block.fee_span, - reward: block.reward ?? block?.extras?.reward, - totalFees: block.totalFees ?? block?.fees ?? block?.extras?.totalFees, - avgFee: block?.extras?.avgFee ?? block.avg_fee, - avgFeeRate: block?.avgFeeRate ?? block.avg_fee_rate, - pool: block?.extras?.pool ?? (block?.pool_id ? { - id: block.pool_id, - name: block.pool_name, - slug: block.pool_slug, - } : undefined), - usd: block?.extras?.usd ?? block.usd ?? null, - } - }; -} diff --git a/docker/README.md b/docker/README.md index 168d4b1fa..468d8069b 100644 --- a/docker/README.md +++ b/docker/README.md @@ -102,11 +102,11 @@ Below we list all settings from `mempool-config.json` and the corresponding over "MEMPOOL_BLOCKS_AMOUNT": 8, "BLOCKS_SUMMARIES_INDEXING": false, "USE_SECOND_NODE_FOR_MINFEE": false, - "EXTERNAL_ASSETS": ["https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json"], + "EXTERNAL_ASSETS": [], "STDOUT_LOG_MIN_PRIORITY": "info", "INDEXING_BLOCKS_AMOUNT": false, "AUTOMATIC_BLOCK_REINDEXING": false, - "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json", + "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json", "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master", "ADVANCED_GBT_AUDIT": false, "ADVANCED_GBT_MEMPOOL": false, diff --git a/docker/backend/start.sh b/docker/backend/start.sh index 3ee542892..ee5069386 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -24,7 +24,7 @@ __MEMPOOL_USER_AGENT__=${MEMPOOL_USER_AGENT:=mempool} __MEMPOOL_STDOUT_LOG_MIN_PRIORITY__=${MEMPOOL_STDOUT_LOG_MIN_PRIORITY:=info} __MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=false} __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__=${MEMPOOL_AUTOMATIC_BLOCK_REINDEXING:=false} -__MEMPOOL_POOLS_JSON_URL__=${MEMPOOL_POOLS_JSON_URL:=https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json} +__MEMPOOL_POOLS_JSON_URL__=${MEMPOOL_POOLS_JSON_URL:=https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json} __MEMPOOL_POOLS_JSON_TREE_URL__=${MEMPOOL_POOLS_JSON_TREE_URL:=https://api.github.com/repos/mempool/mining-pools/git/trees/master} __MEMPOOL_AUDIT__=${MEMPOOL_AUDIT:=false} __MEMPOOL_ADVANCED_GBT_AUDIT__=${MEMPOOL_ADVANCED_GBT_AUDIT:=false} diff --git a/docker/frontend/entrypoint.sh b/docker/frontend/entrypoint.sh index 18cb782e9..45d852c45 100644 --- a/docker/frontend/entrypoint.sh +++ b/docker/frontend/entrypoint.sh @@ -35,6 +35,7 @@ __AUDIT__=${AUDIT:=false} __MAINNET_BLOCK_AUDIT_START_HEIGHT__=${MAINNET_BLOCK_AUDIT_START_HEIGHT:=0} __TESTNET_BLOCK_AUDIT_START_HEIGHT__=${TESTNET_BLOCK_AUDIT_START_HEIGHT:=0} __SIGNET_BLOCK_AUDIT_START_HEIGHT__=${SIGNET_BLOCK_AUDIT_START_HEIGHT:=0} +__HISTORICAL_PRICE__=${HISTORICAL_PRICE:=true} # Export as environment variables to be used by envsubst export __TESTNET_ENABLED__ @@ -60,6 +61,7 @@ export __AUDIT__ export __MAINNET_BLOCK_AUDIT_START_HEIGHT__ export __TESTNET_BLOCK_AUDIT_START_HEIGHT__ export __SIGNET_BLOCK_AUDIT_START_HEIGHT__ +export __HISTORICAL_PRICE__ folder=$(find /var/www/mempool -name "config.js" | xargs dirname) echo ${folder} diff --git a/frontend/mempool-frontend-config.sample.json b/frontend/mempool-frontend-config.sample.json index 9035315a4..084cbd0ef 100644 --- a/frontend/mempool-frontend-config.sample.json +++ b/frontend/mempool-frontend-config.sample.json @@ -21,5 +21,6 @@ "MAINNET_BLOCK_AUDIT_START_HEIGHT": 0, "TESTNET_BLOCK_AUDIT_START_HEIGHT": 0, "SIGNET_BLOCK_AUDIT_START_HEIGHT": 0, - "LIGHTNING": false + "LIGHTNING": false, + "HISTORICAL_PRICE": true } diff --git a/frontend/src/app/components/about/about.component.html b/frontend/src/app/components/about/about.component.html index 03323b6ed..23fed8dcd 100644 --- a/frontend/src/app/components/about/about.component.html +++ b/frontend/src/app/components/about/about.component.html @@ -13,19 +13,9 @@

Our mempool and blockchain explorer for the Bitcoin community, focusing on the transaction fee market and multi-layer ecosystem, completely self-hosted without any trusted third-parties.

- + -
+

Enterprise Sponsors 🚀

@@ -173,7 +163,7 @@
-
+

Community Sponsors ❤️

@@ -187,7 +177,7 @@
-
+ -
+

Community Alliances

@@ -297,7 +287,7 @@
-
+

Project Translators

@@ -311,7 +301,7 @@ -
+

Project Contributors

@@ -323,7 +313,7 @@
-
+

Project Members

@@ -336,7 +326,7 @@
-
+

Project Maintainers

@@ -383,6 +373,17 @@
Third-party Licenses Terms of Service +
diff --git a/frontend/src/app/components/about/about.component.scss b/frontend/src/app/components/about/about.component.scss index 8390ce0ba..35df3fc46 100644 --- a/frontend/src/app/components/about/about.component.scss +++ b/frontend/src/app/components/about/about.component.scss @@ -34,6 +34,12 @@ padding: 10px 15px 15px; } + video { + width: 640px; + max-width: 90%; + margin-top: 0; + } + .social-icons { a { margin: auto 10px; @@ -46,6 +52,7 @@ .maintainers { margin-top: 68px; margin-bottom: 68px; + scroll-margin: 30px; } .maintainers { @@ -117,6 +124,7 @@ .project-translators, .community-integrations-sponsor, .maintainers { + scroll-margin: 30px; .wrapper { display: inline-block; a { @@ -186,6 +194,11 @@ margin: 20px auto 30px; } } + .social-icons { + a { + margin: 45px 10px; + } + } } .footer-version { diff --git a/frontend/src/app/components/about/about.component.ts b/frontend/src/app/components/about/about.component.ts index d26efb411..0f71645d6 100644 --- a/frontend/src/app/components/about/about.component.ts +++ b/frontend/src/app/components/about/about.component.ts @@ -5,9 +5,10 @@ import { StateService } from '../../services/state.service'; import { Observable } from 'rxjs'; import { ApiService } from '../../services/api.service'; import { IBackendInfo } from '../../interfaces/websocket.interface'; -import { Router } from '@angular/router'; -import { map } from 'rxjs/operators'; +import { Router, ActivatedRoute } from '@angular/router'; +import { map, tap } from 'rxjs/operators'; import { ITranslators } from '../../interfaces/node-api.interface'; +import { DOCUMENT } from '@angular/common'; @Component({ selector: 'app-about', @@ -31,7 +32,9 @@ export class AboutComponent implements OnInit { public stateService: StateService, private apiService: ApiService, private router: Router, + private route: ActivatedRoute, @Inject(LOCALE_ID) public locale: string, + @Inject(DOCUMENT) private document: Document, ) { } ngOnInit() { @@ -39,17 +42,21 @@ export class AboutComponent implements OnInit { this.seoService.setTitle($localize`:@@004b222ff9ef9dd4771b777950ca1d0e4cd4348a:About`); this.websocketService.want(['blocks']); - this.sponsors$ = this.apiService.getDonation$(); + this.sponsors$ = this.apiService.getDonation$() + .pipe( + tap(() => this.goToAnchor()) + ); this.translators$ = this.apiService.getTranslators$() .pipe( map((translators) => { for (const t in translators) { if (translators[t] === '') { - delete translators[t] + delete translators[t]; } } return translators; - }) + }), + tap(() => this.goToAnchor()) ); this.allContributors$ = this.apiService.getContributor$().pipe( map((contributors) => { @@ -57,9 +64,24 @@ export class AboutComponent implements OnInit { regular: contributors.filter((user) => !user.core_constributor), core: contributors.filter((user) => user.core_constributor), }; - }) + }), + tap(() => this.goToAnchor()) ); } + + ngAfterViewInit() { + this.goToAnchor(); + } + + goToAnchor() { + setTimeout(() => { + if (this.route.snapshot.fragment) { + if (this.document.getElementById(this.route.snapshot.fragment)) { + this.document.getElementById(this.route.snapshot.fragment).scrollIntoView({behavior: 'smooth'}); + } + } + }, 1); + } sponsor(): void { if (this.officialMempoolSpace && this.stateService.env.BASE_MODULE === 'mempool') { diff --git a/frontend/src/app/components/amount/amount.component.ts b/frontend/src/app/components/amount/amount.component.ts index 927504012..479ae4791 100644 --- a/frontend/src/app/components/amount/amount.component.ts +++ b/frontend/src/app/components/amount/amount.component.ts @@ -1,7 +1,7 @@ import { Component, OnInit, OnDestroy, Input, ChangeDetectionStrategy, ChangeDetectorRef } from '@angular/core'; import { StateService } from '../../services/state.service'; import { Observable, Subscription } from 'rxjs'; -import { Price } from 'src/app/services/price.service'; +import { Price } from '../../services/price.service'; @Component({ selector: 'app-amount', diff --git a/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts b/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts index b46f7a3e7..b77792aee 100644 --- a/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts +++ b/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts @@ -5,7 +5,7 @@ import BlockScene from './block-scene'; import TxSprite from './tx-sprite'; import TxView from './tx-view'; import { Position } from './sprite-types'; -import { Price } from 'src/app/services/price.service'; +import { Price } from '../../services/price.service'; @Component({ selector: 'app-block-overview-graph', diff --git a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.ts b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.ts index 1bd2b8714..ea011d045 100644 --- a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.ts +++ b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.ts @@ -1,7 +1,7 @@ import { Component, ElementRef, ViewChild, Input, OnChanges, ChangeDetectionStrategy } from '@angular/core'; import { TransactionStripped } from '../../interfaces/websocket.interface'; import { Position } from '../../components/block-overview-graph/sprite-types.js'; -import { Price } from 'src/app/services/price.service'; +import { Price } from '../../services/price.service'; @Component({ selector: 'app-block-overview-tooltip', diff --git a/frontend/src/app/components/block/block.component.ts b/frontend/src/app/components/block/block.component.ts index 35f47de85..f5a0c93b0 100644 --- a/frontend/src/app/components/block/block.component.ts +++ b/frontend/src/app/components/block/block.component.ts @@ -13,7 +13,7 @@ import { BlockAudit, BlockExtended, TransactionStripped } from '../../interfaces import { ApiService } from '../../services/api.service'; import { BlockOverviewGraphComponent } from '../../components/block-overview-graph/block-overview-graph.component'; import { detectWebGL } from '../../shared/graphs.utils'; -import { PriceService, Price } from 'src/app/services/price.service'; +import { PriceService, Price } from '../../services/price.service'; @Component({ selector: 'app-block', diff --git a/frontend/src/app/components/start/start.component.ts b/frontend/src/app/components/start/start.component.ts index c7b8a83bc..0855cad05 100644 --- a/frontend/src/app/components/start/start.component.ts +++ b/frontend/src/app/components/start/start.component.ts @@ -267,6 +267,7 @@ export class StartComponent implements OnInit, OnDestroy { resetScroll(): void { this.scrollToBlock(this.chainTip); + this.blockchainContainer.nativeElement.scrollLeft = 0; } getPageIndexOf(height: number): number { diff --git a/frontend/src/app/components/transaction/transaction.component.ts b/frontend/src/app/components/transaction/transaction.component.ts index 4d036e131..4fedc3912 100644 --- a/frontend/src/app/components/transaction/transaction.component.ts +++ b/frontend/src/app/components/transaction/transaction.component.ts @@ -22,7 +22,7 @@ import { SeoService } from '../../services/seo.service'; import { BlockExtended, CpfpInfo } from '../../interfaces/node-api.interface'; import { LiquidUnblinding } from './liquid-ublinding'; import { RelativeUrlPipe } from '../../shared/pipes/relative-url/relative-url.pipe'; -import { Price, PriceService } from 'src/app/services/price.service'; +import { Price, PriceService } from '../../services/price.service'; @Component({ selector: 'app-transaction', diff --git a/frontend/src/app/components/transactions-list/transactions-list.component.ts b/frontend/src/app/components/transactions-list/transactions-list.component.ts index 6422d8507..c720d5960 100644 --- a/frontend/src/app/components/transactions-list/transactions-list.component.ts +++ b/frontend/src/app/components/transactions-list/transactions-list.component.ts @@ -9,7 +9,7 @@ import { AssetsService } from '../../services/assets.service'; import { filter, map, tap, switchMap, shareReplay } from 'rxjs/operators'; import { BlockExtended } from '../../interfaces/node-api.interface'; import { ApiService } from '../../services/api.service'; -import { PriceService } from 'src/app/services/price.service'; +import { PriceService } from '../../services/price.service'; @Component({ selector: 'app-transactions-list', diff --git a/frontend/src/app/components/tx-bowtie-graph-tooltip/tx-bowtie-graph-tooltip.component.ts b/frontend/src/app/components/tx-bowtie-graph-tooltip/tx-bowtie-graph-tooltip.component.ts index da8d91ab3..b02637ef0 100644 --- a/frontend/src/app/components/tx-bowtie-graph-tooltip/tx-bowtie-graph-tooltip.component.ts +++ b/frontend/src/app/components/tx-bowtie-graph-tooltip/tx-bowtie-graph-tooltip.component.ts @@ -1,6 +1,6 @@ import { Component, ElementRef, ViewChild, Input, OnChanges, OnInit } from '@angular/core'; import { tap } from 'rxjs'; -import { Price, PriceService } from 'src/app/services/price.service'; +import { Price, PriceService } from '../../services/price.service'; interface Xput { type: 'input' | 'output' | 'fee'; diff --git a/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.ts b/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.ts index 9c4991590..6be475243 100644 --- a/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.ts +++ b/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.ts @@ -199,8 +199,8 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges { this.outputs = this.initLines('out', voutWithFee, totalValue, this.maxStrands); this.middle = { - path: `M ${(this.width / 2) - this.midWidth} ${(this.height / 2) + 0.25} L ${(this.width / 2) + this.midWidth} ${(this.height / 2) + 0.25}`, - style: `stroke-width: ${this.combinedWeight + 0.5}; stroke: ${this.gradient[1]}` + path: `M ${(this.width / 2) - this.midWidth} ${(this.height / 2) + 0.5} L ${(this.width / 2) + this.midWidth} ${(this.height / 2) + 0.5}`, + style: `stroke-width: ${this.combinedWeight + 1}; stroke: ${this.gradient[1]}` }; this.hasLine = this.inputs.reduce((line, put) => line || !put.zeroValue, false) @@ -257,7 +257,7 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges { const lineParams = weights.map((w, i) => { return { weight: w, - thickness: xputs[i].value === 0 ? this.zeroValueThickness : Math.min(this.combinedWeight + 0.5, Math.max(this.minWeight - 1, w) + 1), + thickness: xputs[i].value === 0 ? this.zeroValueThickness : Math.max(this.minWeight - 1, w) + 1, offset: 0, innerY: 0, outerY: 0, @@ -269,7 +269,7 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges { // bounds of the middle segment const innerTop = (this.height / 2) - (this.combinedWeight / 2); - const innerBottom = innerTop + this.combinedWeight + 0.5; + const innerBottom = innerTop + this.combinedWeight; // tracks the visual bottom of the endpoints of the previous line let lastOuter = 0; let lastInner = innerTop; @@ -294,7 +294,7 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges { // set the vertical position of the (center of the) outer side of the line line.outerY = lastOuter + (line.thickness / 2); - line.innerY = Math.min(innerBottom - (line.thickness / 2), Math.max(innerTop + (line.thickness / 2), lastInner + (line.weight / 2))); + line.innerY = Math.min(innerBottom + (line.thickness / 2), Math.max(innerTop + (line.thickness / 2), lastInner + (line.weight / 2))); // special case to center single input/outputs if (xputs.length === 1) { diff --git a/frontend/src/app/docs/api-docs/api-docs-data.ts b/frontend/src/app/docs/api-docs/api-docs-data.ts index a6e8e418f..62d031613 100644 --- a/frontend/src/app/docs/api-docs/api-docs-data.ts +++ b/frontend/src/app/docs/api-docs/api-docs-data.ts @@ -2907,6 +2907,219 @@ export const restApiDocsData = [ } }, ... +]` + }, + codeSampleLiquid: emptyCodeSample, + codeSampleLiquidTestnet: emptyCodeSample, + codeSampleBisq: emptyCodeSample, + } + } + }, + { + type: "endpoint", + category: "blocks", + httpRequestMethod: "GET", + fragment: "get-blocks-bulk", + title: "GET Blocks (Bulk)", + description: { + default: "

Returns details on the range of blocks between :minHeight and :maxHeight, inclusive, up to 10 blocks. If :maxHeight is not specified, it defaults to the current tip.

To return data for more than 10 blocks, consider becoming an enterprise sponsor.

" + }, + urlString: "/v1/blocks-bulk/:minHeight[/:maxHeight]", + showConditions: bitcoinNetworks, + showJsExamples: showJsExamplesDefaultFalse, + codeExample: { + default: { + codeTemplate: { + curl: `/api/v1/blocks-bulk/%{1}/%{2}`, + commonJS: ``, + }, + codeSampleMainnet: { + esModule: [], + commonJS: [], + curl: [100000,100000], + response: `[ + { + "height": 100000, + "hash": "000000000003ba27aa200b1cecaad478d2b00432346c3f1f3986da1afd33e506", + "timestamp": 1293623863, + "median_timestamp": 1293622620, + "previous_block_hash": "000000000002d01c1fccc21636b607dfd930d31d01c3a62104612a1719011250", + "difficulty": 14484.1623612254, + "header": "0100000050120119172a610421a6c3011dd330d9df07b63616c2cc1f1cd00200000000006657a9252aacd5c0b2940996ecff952228c3067cc38d4885efb5a4ac4247e9f337221b4d4c86041b0f2b5710", + "version": 1, + "bits": 453281356, + "nonce": 274148111, + "size": 957, + "weight": 3828, + "tx_count": 4, + "merkle_root": "f3e94742aca4b5ef85488dc37c06c3282295ffec960994b2c0d5ac2a25a95766", + "reward": 5000000000, + "total_fee_amt": 0, + "avg_fee_amt": 0, + "median_fee_amt": 0, + "fee_amt_percentiles": { + "min": 0, + "perc_10": 0, + "perc_25": 0, + "perc_50": 0, + "perc_75": 0, + "perc_90": 0, + "max": 0 + }, + "avg_fee_rate": 0, + "median_fee_rate": 0, + "fee_rate_percentiles": { + "min": 0, + "perc_10": 0, + "perc_25": 0, + "perc_50": 0, + "perc_75": 0, + "perc_90": 0, + "max": 0 + }, + "total_inputs": 3, + "total_input_amt": 5301000000, + "total_outputs": 6, + "total_output_amt": 5301000000, + "segwit_total_txs": 0, + "segwit_total_size": 0, + "segwit_total_weight": 0, + "avg_tx_size": 185.25, + "utxoset_change": 3, + "utxoset_size": 71888, + "coinbase_raw": "044c86041b020602", + "coinbase_address": null, + "coinbase_signature": "OP_PUSHBYTES_65 041b0e8c2567c12536aa13357b79a073dc4444acb83c4ec7a0e2f99dd7457516c5817242da796924ca4e99947d087fedf9ce467cb9f7c6287078f801df276fdf84 OP_CHECKSIG", + "coinbase_signature_ascii": "\u0004L�\u0004\u001b\u0002\u0006\u0002", + "pool_slug": "unknown", + "orphans": [] + } +]`, + }, + codeSampleTestnet: { + esModule: [], + commonJS: [], + curl: [100000,100000], + response: `[ + { + "height": 100000, + "hash": "00000000009e2958c15ff9290d571bf9459e93b19765c6801ddeccadbb160a1e", + "timestamp": 1376123972, + "median_timestamp": 1677396660, + "previous_block_hash": "000000004956cc2edd1a8caa05eacfa3c69f4c490bfc9ace820257834115ab35", + "difficulty": 271.7576739288896, + "header": "0200000035ab154183570282ce9afc0b494c9fc6a3cfea05aa8c1add2ecc56490000000038ba3d78e4500a5a7570dbe61960398add4410d278b21cd9708e6d9743f374d544fc055227f1001c29c1ea3b", + "version": 2, + "bits": 469823783, + "nonce": 1005240617, + "size": 221, + "weight": 884, + "tx_count": 1, + "merkle_root": "d574f343976d8e70d91cb278d21044dd8a396019e6db70755a0a50e4783dba38", + "reward": 5000000000, + "total_fee_amt": 0, + "avg_fee_amt": 0, + "median_fee_amt": 0, + "fee_amt_percentiles": { + "min": 0, + "perc_10": 0, + "perc_25": 0, + "perc_50": 0, + "perc_75": 0, + "perc_90": 0, + "max": 0 + }, + "avg_fee_rate": 0, + "median_fee_rate": 0, + "fee_rate_percentiles": { + "min": 0, + "perc_10": 0, + "perc_25": 0, + "perc_50": 0, + "perc_75": 0, + "perc_90": 0, + "max": 0 + }, + "total_inputs": 0, + "total_input_amt": null, + "total_outputs": 1, + "total_output_amt": 0, + "segwit_total_txs": 0, + "segwit_total_size": 0, + "segwit_total_weight": 0, + "avg_tx_size": 0, + "utxoset_change": 1, + "utxoset_size": null, + "coinbase_raw": "03a08601000427f1001c046a510100522cfabe6d6d0000000000000000000068692066726f6d20706f6f6c7365727665726aac1eeeed88", + "coinbase_address": "mtkbaiLiUH3fvGJeSzuN3kUgmJzqinLejJ", + "coinbase_signature": "OP_DUP OP_HASH160 OP_PUSHBYTES_20 912e2b234f941f30b18afbb4fa46171214bf66c8 OP_EQUALVERIFY OP_CHECKSIG", + "coinbase_signature_ascii": "\u0003 �\u0001\u0000\u0004'ñ\u0000\u001c\u0004jQ\u0001\u0000R,ú¾mm\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000hi from poolserverj¬\u001eîí�", + "pool_slug": "unknown", + "orphans": [] + } +]` + }, + codeSampleSignet: { + esModule: [], + commonJS: [], + curl: [100000,100000], + response: `[ + { + "height": 100000, + "hash": "0000008753108390007b3f5c26e5d924191567e147876b84489b0c0cf133a0bf", + "timestamp": 1658421183, + "median_timestamp": 1658418056, + "previous_block_hash": "000000b962a13c3dd3f81917bc8646a0c98224adcd5124026d4fdfcb76a76d30", + "difficulty": 0.002781447610743506, + "header": "00000020306da776cbdf4f6d022451cdad2482c9a04686bc1719f8d33d3ca162b90000001367fb15320ebb1932fd589f8f38866b692ca8a4ad6100a4bc732d212916d0efbf7fd9628567011e47662d00", + "version": 536870912, + "bits": 503408517, + "nonce": 2975303, + "size": 343, + "weight": 1264, + "tx_count": 1, + "merkle_root": "efd01629212d73bca40061ada4a82c696b86388f9f58fd3219bb0e3215fb6713", + "reward": 5000000000, + "total_fee_amt": 0, + "avg_fee_amt": 0, + "median_fee_amt": 0, + "fee_amt_percentiles": { + "min": 0, + "perc_10": 0, + "perc_25": 0, + "perc_50": 0, + "perc_75": 0, + "perc_90": 0, + "max": 0 + }, + "avg_fee_rate": 0, + "median_fee_rate": 0, + "fee_rate_percentiles": { + "min": 0, + "perc_10": 0, + "perc_25": 0, + "perc_50": 0, + "perc_75": 0, + "perc_90": 0, + "max": 0 + }, + "total_inputs": 0, + "total_input_amt": null, + "total_outputs": 2, + "total_output_amt": 0, + "segwit_total_txs": 0, + "segwit_total_size": 0, + "segwit_total_weight": 0, + "avg_tx_size": 0, + "utxoset_change": 2, + "utxoset_size": null, + "coinbase_raw": "03a08601", + "coinbase_address": "tb1psfjl80vk0yp3agcq6ylueas29rau00mfq90mhejerpgccg33xhasd9gjyd", + "coinbase_signature": "OP_PUSHNUM_1 OP_PUSHBYTES_32 8265f3bd9679031ea300d13fccf60a28fbc7bf69015fbbe65918518c223135fb", + "coinbase_signature_ascii": "\u0003 �\u0001", + "pool_slug": "unknown", + "orphans": [] + } ]` }, codeSampleLiquid: emptyCodeSample, diff --git a/frontend/src/app/docs/api-docs/api-docs.component.html b/frontend/src/app/docs/api-docs/api-docs.component.html index 47332abc3..8c8d6ac36 100644 --- a/frontend/src/app/docs/api-docs/api-docs.component.html +++ b/frontend/src/app/docs/api-docs/api-docs.component.html @@ -39,7 +39,7 @@

Below is a reference for the {{ network.val === '' ? 'Bitcoin' : network.val.charAt(0).toUpperCase() + network.val.slice(1) }} REST API service.

-

Note that we enforce rate limits. If you exceed these limits, you will get a polite error encouraging you to run your own Mempool instance. If you repeatedly exceed the limits, you may be banned from accessing the service altogether.

+

Note that we enforce rate limits. If you exceed these limits, you will get an HTTP 429 error. If you repeatedly exceed the limits, you may be banned from accessing the service altogether. Consider an enterprise sponsorship if you need higher API limits.

{{ item.title }}

diff --git a/frontend/src/app/docs/docs/docs.component.html b/frontend/src/app/docs/docs/docs.component.html index b0c51ad16..cf3bdb070 100644 --- a/frontend/src/app/docs/docs/docs.component.html +++ b/frontend/src/app/docs/docs/docs.component.html @@ -32,7 +32,7 @@ -
  • +