diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d0520f59c..2ff25b22f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,7 @@ jobs: if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')" strategy: matrix: - node: ["16.16.0", "18.14.1", "19.6.1"] + node: ["16.16.0", "18.14.1"] flavor: ["dev", "prod"] fail-fast: false runs-on: "ubuntu-latest" @@ -55,7 +55,7 @@ jobs: if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')" strategy: matrix: - node: ["16.16.0", "18.14.1", "19.6.1"] + node: ["16.16.0", "18.14.1"] flavor: ["dev", "prod"] fail-fast: false runs-on: "ubuntu-latest" diff --git a/LICENSE b/LICENSE index 966417847..ac267d120 100644 --- a/LICENSE +++ b/LICENSE @@ -1,5 +1,5 @@ The Mempool Open Source Project -Copyright (c) 2019-2022 The Mempool Open Source Project Developers +Copyright (c) 2019-2023 The Mempool Open Source Project Developers This program is free software; you can redistribute it and/or modify it under the terms of (at your option) either: diff --git a/backend/README.md b/backend/README.md index d00dc1812..be85d25af 100644 --- a/backend/README.md +++ b/backend/README.md @@ -160,7 +160,7 @@ npm install -g ts-node nodemon Then, run the watcher: ``` -nodemon src/index.ts --ignore cache/ --ignore pools.json +nodemon src/index.ts --ignore cache/ ``` `nodemon` should be in npm's global binary folder. If needed, you can determine where that is with `npm -g bin`. @@ -219,6 +219,16 @@ Generate block at regular interval (every 10 seconds in this example): watch -n 10 "./src/bitcoin-cli -regtest -rpcport=8332 generatetoaddress 1 $address" ``` +### Mining pools update + +By default, mining pools will be not automatically updated regularly (`config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING` is set to `false`). + +To manually update your mining pools, you can use the `--update-pools` command line flag when you run the nodejs backend. For example `npm run start --update-pools`. This will trigger the mining pools update and automatically re-index appropriate blocks. + +You can enabled the automatic mining pools update by settings `config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING` to `true` in your `mempool-config.json`. + +When a `coinbase tag` or `coinbase address` change is detected, all blocks tagged to the `unknown` mining pools (starting from height 130635) will be deleted from the `blocks` table. Additionaly, all blocks which were tagged to the pool which has been updated will also be deleted from the `blocks` table. Of course, those blocks will be automatically reindexed. + ### Re-index tables You can manually force the nodejs backend to drop all data from a specified set of tables for future re-index. This is mostly useful for the mining dashboard and the lightning explorer. @@ -235,4 +245,4 @@ Feb 13 14:55:27 [63246] WARN: Indexed data for "hashrates" tables wi Feb 13 14:55:32 [63246] NOTICE: Table hashrates has been truncated ``` -Reference: https://github.com/mempool/mempool/pull/1269 \ No newline at end of file +Reference: https://github.com/mempool/mempool/pull/1269 diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 5ebb25ea5..2369b64b5 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -22,7 +22,7 @@ "USER_AGENT": "mempool", "STDOUT_LOG_MIN_PRIORITY": "debug", "AUTOMATIC_BLOCK_REINDEXING": false, - "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json", + "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json", "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master", "AUDIT": false, "ADVANCED_GBT_AUDIT": false, diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index 9890654a5..2bf52cbcf 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -7,7 +7,7 @@ "HTTP_PORT": 1, "SPAWN_CLUSTER_PROCS": 2, "API_URL_PREFIX": "__MEMPOOL_API_URL_PREFIX__", - "AUTOMATIC_BLOCK_REINDEXING": true, + "AUTOMATIC_BLOCK_REINDEXING": false, "POLL_RATE_MS": 3, "CACHE_DIR": "__MEMPOOL_CACHE_DIR__", "CLEAR_PROTECTION_MINUTES": 4, diff --git a/backend/src/api/audit.ts b/backend/src/api/audit.ts index b6b36dbdc..5b67dc965 100644 --- a/backend/src/api/audit.ts +++ b/backend/src/api/audit.ts @@ -119,7 +119,8 @@ class Audit { } const numCensored = Object.keys(isCensored).length; - const score = matches.length > 0 ? (matches.length / (matches.length + numCensored)) : 0; + const numMatches = matches.length - 1; // adjust for coinbase tx + const score = numMatches > 0 ? (numMatches / (numMatches + numCensored)) : 0; return { censored: Object.keys(isCensored), diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 401fb0b2c..f4801deb6 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -1017,26 +1017,16 @@ class DatabaseMigration { ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; } - public async $truncateIndexedData(tables: string[]) { - const allowedTables = ['blocks', 'hashrates', 'prices']; + public async $blocksReindexingTruncate(): Promise { + logger.warn(`Truncating pools, blocks and hashrates for re-indexing (using '--reindex-blocks'). You can cancel this command within 5 seconds`); + await Common.sleep$(5000); - try { - for (const table of tables) { - if (!allowedTables.includes(table)) { - logger.debug(`Table ${table} cannot to be re-indexed (not allowed)`); - continue; - } - - await this.$executeQuery(`TRUNCATE ${table}`, true); - if (table === 'hashrates') { - await this.$executeQuery('UPDATE state set number = 0 where name = "last_hashrates_indexing"', true); - } - logger.notice(`Table ${table} has been truncated`); - } - } catch (e) { - logger.warn(`Unable to erase indexed data`); - } - } + await this.$executeQuery(`TRUNCATE blocks`); + await this.$executeQuery(`TRUNCATE hashrates`); + await this.$executeQuery('DELETE FROM `pools`'); + await this.$executeQuery('ALTER TABLE pools AUTO_INCREMENT = 1'); + await this.$executeQuery(`UPDATE state SET string = NULL WHERE name = 'pools_json_sha'`); +} private async $convertCompactCpfpTables(): Promise { try { diff --git a/backend/src/api/pools-parser.ts b/backend/src/api/pools-parser.ts index 4e67ce98b..b34dcb7b8 100644 --- a/backend/src/api/pools-parser.ts +++ b/backend/src/api/pools-parser.ts @@ -73,7 +73,7 @@ class PoolsParser { } } - logger.info('Mining pools.json import completed'); + logger.info('Mining pools-v2.json import completed'); } /** @@ -115,7 +115,7 @@ class PoolsParser { return; } - // Get oldest blocks mined by the pool and assume pools.json updates only concern most recent years + // Get oldest blocks mined by the pool and assume pools-v2.json updates only concern most recent years // Ignore early days of Bitcoin as there were no mining pool yet const [oldestPoolBlock]: any[] = await DB.query(` SELECT height diff --git a/backend/src/index.ts b/backend/src/index.ts index 6ea3ddc43..6f259a2bd 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -36,7 +36,6 @@ import bitcoinRoutes from './api/bitcoin/bitcoin.routes'; import fundingTxFetcher from './tasks/lightning/sync-tasks/funding-tx-fetcher'; import forensicsService from './tasks/lightning/forensics.service'; import priceUpdater from './tasks/price-updater'; -import mining from './api/mining/mining'; import chainTips from './api/chain-tips'; import { AxiosError } from 'axios'; @@ -84,11 +83,8 @@ class Server { if (config.DATABASE.ENABLED) { await DB.checkDbConnection(); try { - if (process.env.npm_config_reindex !== undefined) { // Re-index requests - const tables = process.env.npm_config_reindex.split(','); - logger.warn(`Indexed data for "${process.env.npm_config_reindex}" tables will be erased in 5 seconds (using '--reindex')`); - await Common.sleep$(5000); - await databaseMigration.$truncateIndexedData(tables); + if (process.env.npm_config_reindex_blocks === 'true') { // Re-index requests + await databaseMigration.$blocksReindexingTruncate(); } await databaseMigration.$initializeOrMigrateDatabase(); if (Common.indexingEnabled()) { diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 86dc006ff..c7edb97cb 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -16,6 +16,9 @@ class BlocksRepository { * Save indexed block data in the database */ public async $saveBlockInDatabase(block: BlockExtended) { + const truncatedCoinbaseSignature = block?.extras?.coinbaseSignature?.substring(0, 500); + const truncatedCoinbaseSignatureAscii = block?.extras?.coinbaseSignatureAscii?.substring(0, 500); + try { const query = `INSERT INTO blocks( height, hash, blockTimestamp, size, @@ -65,7 +68,7 @@ class BlocksRepository { block.extras.medianTimestamp, block.extras.header, block.extras.coinbaseAddress, - block.extras.coinbaseSignature, + truncatedCoinbaseSignature, block.extras.utxoSetSize, block.extras.utxoSetChange, block.extras.avgTxSize, @@ -78,7 +81,7 @@ class BlocksRepository { block.extras.segwitTotalSize, block.extras.segwitTotalWeight, block.extras.medianFeeAmt, - block.extras.coinbaseSignatureAscii, + truncatedCoinbaseSignatureAscii, ]; await DB.query(query, params); diff --git a/backend/src/repositories/PoolsRepository.ts b/backend/src/repositories/PoolsRepository.ts index 63bddd497..236955d65 100644 --- a/backend/src/repositories/PoolsRepository.ts +++ b/backend/src/repositories/PoolsRepository.ts @@ -99,7 +99,7 @@ class PoolsRepository { rows[0].regexes = JSON.parse(rows[0].regexes); } if (['testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) { - rows[0].addresses = []; // pools.json only contains mainnet addresses + rows[0].addresses = []; // pools-v2.json only contains mainnet addresses } else if (parse) { rows[0].addresses = JSON.parse(rows[0].addresses); } diff --git a/backend/src/tasks/pools-updater.ts b/backend/src/tasks/pools-updater.ts index 8aa73376f..32de85f3a 100644 --- a/backend/src/tasks/pools-updater.ts +++ b/backend/src/tasks/pools-updater.ts @@ -17,11 +17,6 @@ class PoolsUpdater { treeUrl: string = config.MEMPOOL.POOLS_JSON_TREE_URL; public async updatePoolsJson(): Promise { - if (config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING === false) { - logger.info(`Not updating mining pools to avoid inconsistency because AUTOMATIC_BLOCK_REINDEXING is set to false`) - return; - } - if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { return; } @@ -36,12 +31,6 @@ class PoolsUpdater { this.lastRun = now; - if (config.SOCKS5PROXY.ENABLED) { - logger.info(`Updating latest mining pools from ${this.poolsUrl} over the Tor network`, logger.tags.mining); - } else { - logger.info(`Updating latest mining pools from ${this.poolsUrl} over clearnet`, logger.tags.mining); - } - try { const githubSha = await this.fetchPoolsSha(); // Fetch pools-v2.json sha from github if (githubSha === undefined) { @@ -57,10 +46,21 @@ class PoolsUpdater { return; } + // See backend README for more details about the mining pools update process + if (this.currentSha !== undefined && // If we don't have any mining pool, download it at least once + config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING !== true && // Automatic pools update is disabled + !process.env.npm_config_update_pools // We're not manually updating mining pool + ) { + logger.warn(`Updated mining pools data is available (${githubSha}) but AUTOMATIC_BLOCK_REINDEXING is disabled`); + logger.info(`You can update your mining pools using the --update-pools command flag. You may want to clear your nginx cache as well if applicable`); + return; + } + + const network = config.SOCKS5PROXY.ENABLED ? 'tor' : 'clearnet'; if (this.currentSha === undefined) { - logger.info(`Downloading pools-v2.json for the first time from ${this.poolsUrl}`, logger.tags.mining); + logger.info(`Downloading pools-v2.json for the first time from ${this.poolsUrl} over ${network}`, logger.tags.mining); } else { - logger.warn(`pools-v2.json is outdated, fetch latest from ${this.poolsUrl}`, logger.tags.mining); + logger.warn(`pools-v2.json is outdated, fetch latest from ${this.poolsUrl} over ${network}`, logger.tags.mining); } const poolsJson = await this.query(this.poolsUrl); if (poolsJson === undefined) { diff --git a/docker/README.md b/docker/README.md index 168d4b1fa..468d8069b 100644 --- a/docker/README.md +++ b/docker/README.md @@ -102,11 +102,11 @@ Below we list all settings from `mempool-config.json` and the corresponding over "MEMPOOL_BLOCKS_AMOUNT": 8, "BLOCKS_SUMMARIES_INDEXING": false, "USE_SECOND_NODE_FOR_MINFEE": false, - "EXTERNAL_ASSETS": ["https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json"], + "EXTERNAL_ASSETS": [], "STDOUT_LOG_MIN_PRIORITY": "info", "INDEXING_BLOCKS_AMOUNT": false, "AUTOMATIC_BLOCK_REINDEXING": false, - "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json", + "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json", "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master", "ADVANCED_GBT_AUDIT": false, "ADVANCED_GBT_MEMPOOL": false, diff --git a/docker/backend/start.sh b/docker/backend/start.sh index 3ee542892..ee5069386 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -24,7 +24,7 @@ __MEMPOOL_USER_AGENT__=${MEMPOOL_USER_AGENT:=mempool} __MEMPOOL_STDOUT_LOG_MIN_PRIORITY__=${MEMPOOL_STDOUT_LOG_MIN_PRIORITY:=info} __MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=false} __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__=${MEMPOOL_AUTOMATIC_BLOCK_REINDEXING:=false} -__MEMPOOL_POOLS_JSON_URL__=${MEMPOOL_POOLS_JSON_URL:=https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json} +__MEMPOOL_POOLS_JSON_URL__=${MEMPOOL_POOLS_JSON_URL:=https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json} __MEMPOOL_POOLS_JSON_TREE_URL__=${MEMPOOL_POOLS_JSON_TREE_URL:=https://api.github.com/repos/mempool/mining-pools/git/trees/master} __MEMPOOL_AUDIT__=${MEMPOOL_AUDIT:=false} __MEMPOOL_ADVANCED_GBT_AUDIT__=${MEMPOOL_ADVANCED_GBT_AUDIT:=false} diff --git a/frontend/src/app/components/about/about.component.html b/frontend/src/app/components/about/about.component.html index 876bec028..03323b6ed 100644 --- a/frontend/src/app/components/about/about.component.html +++ b/frontend/src/app/components/about/about.component.html @@ -352,7 +352,7 @@