Merge branch 'master' into nymkappa/bugfix/ignore-too-low-lightning-timestamps

backend/README.md

@@ -160,7 +160,7 @@ npm install -g ts-node nodemon
 Then, run the watcher:
 
 ```
-nodemon src/index.ts --ignore cache/ --ignore pools.json
+nodemon src/index.ts --ignore cache/
 ```
 
 `nodemon` should be in npm's global binary folder. If needed, you can determine where that is with `npm -g bin`.
@@ -219,6 +219,16 @@ Generate block at regular interval (every 10 seconds in this example):
    watch -n 10 "./src/bitcoin-cli -regtest -rpcport=8332 generatetoaddress 1 $address"
    ```
 
+### Mining pools update
+
+By default, mining pools will be not automatically updated regularly (`config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING` is set to `false`). 
+
+To manually update your mining pools, you can use the `--update-pools` command line flag when you run the nodejs backend. For example `npm run start --update-pools`. This will trigger the mining pools update and automatically re-index appropriate blocks.
+
+You can enabled the automatic mining pools update by settings `config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING` to `true` in your `mempool-config.json`.
+
+When a `coinbase tag` or `coinbase address` change is detected, all blocks tagged to the `unknown` mining pools (starting from height 130635) will be deleted from the `blocks` table. Additionaly, all blocks which were tagged to the pool which has been updated will also be deleted from the `blocks` table. Of course, those blocks will be automatically reindexed.
+
 ### Re-index tables
 
 You can manually force the nodejs backend to drop all data from a specified set of tables for future re-index. This is mostly useful for the mining dashboard and the lightning explorer.
@@ -235,4 +245,4 @@ Feb 13 14:55:27 [63246] WARN: <lightning> Indexed data for "hashrates" tables wi
 Feb 13 14:55:32 [63246] NOTICE: <lightning> Table hashrates has been truncated
 ```
 
-Reference: https://github.com/mempool/mempool/pull/1269
+Reference: https://github.com/mempool/mempool/pull/1269

backend/mempool-config.sample.json

@@ -22,7 +22,7 @@
     "USER_AGENT": "mempool",
     "STDOUT_LOG_MIN_PRIORITY": "debug",
     "AUTOMATIC_BLOCK_REINDEXING": false,
-    "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json",
+    "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json",
     "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
     "AUDIT": false,
     "ADVANCED_GBT_AUDIT": false,

backend/src/__fixtures__/mempool-config.template.json

@@ -7,7 +7,7 @@
     "HTTP_PORT": 1,
     "SPAWN_CLUSTER_PROCS": 2,
     "API_URL_PREFIX": "__MEMPOOL_API_URL_PREFIX__",
-    "AUTOMATIC_BLOCK_REINDEXING": true,
+    "AUTOMATIC_BLOCK_REINDEXING": false,
     "POLL_RATE_MS": 3,
     "CACHE_DIR": "__MEMPOOL_CACHE_DIR__",
     "CLEAR_PROTECTION_MINUTES": 4,

backend/src/api/audit.ts

@@ -119,7 +119,8 @@ class Audit {
     }
 
     const numCensored = Object.keys(isCensored).length;
-    const score = matches.length > 0 ? (matches.length / (matches.length + numCensored)) : 0;
+    const numMatches = matches.length - 1; // adjust for coinbase tx
+    const score = numMatches > 0 ? (numMatches / (numMatches + numCensored)) : 0;
 
     return {
       censored: Object.keys(isCensored),

backend/src/api/database-migration.ts

@@ -1017,26 +1017,16 @@ class DatabaseMigration {
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
 
-  public async $truncateIndexedData(tables: string[]) {
-    const allowedTables = ['blocks', 'hashrates', 'prices'];
+  public async $blocksReindexingTruncate(): Promise<void> {
+    logger.warn(`Truncating pools, blocks and hashrates for re-indexing (using '--reindex-blocks'). You can cancel this command within 5 seconds`);
+    await Common.sleep$(5000);
 
-    try {
-      for (const table of tables) {
-        if (!allowedTables.includes(table)) {
-          logger.debug(`Table ${table} cannot to be re-indexed (not allowed)`);
-          continue;
-        }
-
-        await this.$executeQuery(`TRUNCATE ${table}`, true);
-        if (table === 'hashrates') {
-          await this.$executeQuery('UPDATE state set number = 0 where name = "last_hashrates_indexing"', true);
-        }
-        logger.notice(`Table ${table} has been truncated`);
-      }
-    } catch (e) {
-      logger.warn(`Unable to erase indexed data`);
-    }
-  }
+    await this.$executeQuery(`TRUNCATE blocks`);
+    await this.$executeQuery(`TRUNCATE hashrates`);
+    await this.$executeQuery('DELETE FROM `pools`');
+    await this.$executeQuery('ALTER TABLE pools AUTO_INCREMENT = 1');
+    await this.$executeQuery(`UPDATE state SET string = NULL WHERE name = 'pools_json_sha'`);
+}
 
   private async $convertCompactCpfpTables(): Promise<void> {
     try {

backend/src/api/pools-parser.ts

@@ -73,7 +73,7 @@ class PoolsParser {
       }
     }
 
-    logger.info('Mining pools.json import completed');
+    logger.info('Mining pools-v2.json import completed');
   }
 
   /**
@@ -115,7 +115,7 @@ class PoolsParser {
       return;
     }
 
-    // Get oldest blocks mined by the pool and assume pools.json updates only concern most recent years
+    // Get oldest blocks mined by the pool and assume pools-v2.json updates only concern most recent years
     // Ignore early days of Bitcoin as there were no mining pool yet
     const [oldestPoolBlock]: any[] = await DB.query(`
       SELECT height

backend/src/index.ts

@@ -36,7 +36,6 @@ import bitcoinRoutes from './api/bitcoin/bitcoin.routes';
 import fundingTxFetcher from './tasks/lightning/sync-tasks/funding-tx-fetcher';
 import forensicsService from './tasks/lightning/forensics.service';
 import priceUpdater from './tasks/price-updater';
-import mining from './api/mining/mining';
 import chainTips from './api/chain-tips';
 import { AxiosError } from 'axios';
 
@@ -84,11 +83,8 @@ class Server {
     if (config.DATABASE.ENABLED) {
       await DB.checkDbConnection();
       try {
-        if (process.env.npm_config_reindex !== undefined) { // Re-index requests
-          const tables = process.env.npm_config_reindex.split(',');
-          logger.warn(`Indexed data for "${process.env.npm_config_reindex}" tables will be erased in 5 seconds (using '--reindex')`);
-          await Common.sleep$(5000);
-          await databaseMigration.$truncateIndexedData(tables);
+        if (process.env.npm_config_reindex_blocks === 'true') { // Re-index requests
+          await databaseMigration.$blocksReindexingTruncate();
         }
         await databaseMigration.$initializeOrMigrateDatabase();
         if (Common.indexingEnabled()) {

backend/src/repositories/BlocksRepository.ts

@@ -16,6 +16,9 @@ class BlocksRepository {
    * Save indexed block data in the database
    */
   public async $saveBlockInDatabase(block: BlockExtended) {
+    const truncatedCoinbaseSignature = block?.extras?.coinbaseSignature?.substring(0, 500);
+    const truncatedCoinbaseSignatureAscii = block?.extras?.coinbaseSignatureAscii?.substring(0, 500);
+
     try {
       const query = `INSERT INTO blocks(
         height,             hash,                blockTimestamp,    size,
@@ -65,7 +68,7 @@ class BlocksRepository {
         block.extras.medianTimestamp,
         block.extras.header,
         block.extras.coinbaseAddress,
-        block.extras.coinbaseSignature,
+        truncatedCoinbaseSignature,
         block.extras.utxoSetSize,
         block.extras.utxoSetChange,
         block.extras.avgTxSize,
@@ -78,7 +81,7 @@ class BlocksRepository {
         block.extras.segwitTotalSize,
         block.extras.segwitTotalWeight,
         block.extras.medianFeeAmt,
-        block.extras.coinbaseSignatureAscii,
+        truncatedCoinbaseSignatureAscii,
       ];
 
       await DB.query(query, params);

backend/src/repositories/PoolsRepository.ts

@@ -99,7 +99,7 @@ class PoolsRepository {
         rows[0].regexes = JSON.parse(rows[0].regexes);
       }
       if (['testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
-        rows[0].addresses = []; // pools.json only contains mainnet addresses
+        rows[0].addresses = []; // pools-v2.json only contains mainnet addresses
       } else if (parse) {
         rows[0].addresses = JSON.parse(rows[0].addresses);
       }

backend/src/tasks/pools-updater.ts

@@ -17,11 +17,6 @@ class PoolsUpdater {
   treeUrl: string = config.MEMPOOL.POOLS_JSON_TREE_URL;
 
   public async updatePoolsJson(): Promise<void> {
-    if (config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING === false) {
-      logger.info(`Not updating mining pools to avoid inconsistency because AUTOMATIC_BLOCK_REINDEXING is set to false`)
-      return;
-    }
-
     if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
       return;
     }
@@ -36,12 +31,6 @@ class PoolsUpdater {
 
     this.lastRun = now;
 
-    if (config.SOCKS5PROXY.ENABLED) {
-      logger.info(`Updating latest mining pools from ${this.poolsUrl} over the Tor network`, logger.tags.mining);
-    } else {
-      logger.info(`Updating latest mining pools from ${this.poolsUrl} over clearnet`, logger.tags.mining);
-    }
-
     try {
       const githubSha = await this.fetchPoolsSha(); // Fetch pools-v2.json sha from github
       if (githubSha === undefined) {
@@ -57,10 +46,21 @@ class PoolsUpdater {
         return;
       }
 
+      // See backend README for more details about the mining pools update process
+      if (this.currentSha !== undefined && // If we don't have any mining pool, download it at least once
+        config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING !== true && // Automatic pools update is disabled
+        !process.env.npm_config_update_pools // We're not manually updating mining pool
+      ) {
+        logger.warn(`Updated mining pools data is available (${githubSha}) but AUTOMATIC_BLOCK_REINDEXING is disabled`);
+        logger.info(`You can update your mining pools using the --update-pools command flag. You may want to clear your nginx cache as well if applicable`);
+        return;
+      }
+
+      const network = config.SOCKS5PROXY.ENABLED ? 'tor' : 'clearnet';
       if (this.currentSha === undefined) {
-        logger.info(`Downloading pools-v2.json for the first time from ${this.poolsUrl}`, logger.tags.mining);
+        logger.info(`Downloading pools-v2.json for the first time from ${this.poolsUrl} over ${network}`, logger.tags.mining);
       } else {
-        logger.warn(`pools-v2.json is outdated, fetch latest from ${this.poolsUrl}`, logger.tags.mining);
+        logger.warn(`pools-v2.json is outdated, fetch latest from ${this.poolsUrl} over ${network}`, logger.tags.mining);
       }
       const poolsJson = await this.query(this.poolsUrl);
       if (poolsJson === undefined) {