Merge pull request #5469 from mempool/nymkappa/configurable-pool-update

[mining] fix pools updater only running at start

backend/mempool-config.sample.json

@@ -27,6 +27,7 @@
     "AUTOMATIC_POOLS_UPDATE": false,
     "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json",
     "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
+    "POOLS_UPDATE_DELAY": 604800,
     "AUDIT": false,
     "RUST_GBT": true,
     "LIMIT_GBT": false,

backend/src/__fixtures__/mempool-config.template.json

@@ -28,6 +28,7 @@
     "INDEXING_BLOCKS_AMOUNT": 14,
     "POOLS_JSON_TREE_URL": "__MEMPOOL_POOLS_JSON_TREE_URL__",
     "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__",
+    "POOLS_UPDATE_DELAY": 604800,
     "AUDIT": true,
     "RUST_GBT": false,
     "LIMIT_GBT": false,

backend/src/__tests__/config.test.ts

@@ -41,6 +41,7 @@ describe('Mempool Backend Config', () => {
         STDOUT_LOG_MIN_PRIORITY: 'debug',
         POOLS_JSON_TREE_URL: 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
         POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json',
+        POOLS_UPDATE_DELAY: 604800,
         AUDIT: false,
         RUST_GBT: true,
         LIMIT_GBT: false,

backend/src/config.ts

@@ -32,6 +32,7 @@ interface IConfig {
     AUTOMATIC_POOLS_UPDATE: boolean;
     POOLS_JSON_URL: string,
     POOLS_JSON_TREE_URL: string,
+    POOLS_UPDATE_DELAY: number,
     AUDIT: boolean;
     RUST_GBT: boolean;
     LIMIT_GBT: boolean;
@@ -192,6 +193,7 @@ const defaults: IConfig = {
     'AUTOMATIC_POOLS_UPDATE': false,
     'POOLS_JSON_URL': 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json',
     'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
+    'POOLS_UPDATE_DELAY': 604800, // in seconds, default is one week
     'AUDIT': false,
     'RUST_GBT': true,
     'LIMIT_GBT': false,

backend/src/index.ts

@@ -211,6 +211,8 @@ class Server {
         }
       });
     }
+
+    poolsUpdater.$startService();
   }
 
   async runMainUpdateLoop(): Promise<void> {

backend/src/tasks/pools-updater.ts

@@ -6,16 +6,30 @@ import backendInfo from '../api/backend-info';
 import logger from '../logger';
 import { SocksProxyAgent } from 'socks-proxy-agent';
 import * as https from 'https';
+import { Common } from '../api/common';
 
 /**
  * Maintain the most recent version of pools-v2.json
  */
 class PoolsUpdater {
+  tag = 'PoolsUpdater';
+
   lastRun: number = 0;
   currentSha: string | null = null;
   poolsUrl: string = config.MEMPOOL.POOLS_JSON_URL;
   treeUrl: string = config.MEMPOOL.POOLS_JSON_TREE_URL;
 
+  public async $startService(): Promise<void> {
+    while ('Bitcoin is still alive') {
+      try {
+        await this.updatePoolsJson();
+      } catch (e: any) {
+        logger.info(`Exception ${e} in PoolsUpdater::$startService. Code: ${e.code}. Message: ${e.message}`, this.tag);
+      }
+      await Common.sleep$(10000);
+    }
+  }
+
   public async updatePoolsJson(): Promise<void> {
     if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false ||
       config.MEMPOOL.ENABLED === false
@@ -23,11 +37,8 @@ class PoolsUpdater {
       return;
     }
 
-    const oneWeek = 604800;
-    const oneDay = 86400;
-
     const now = new Date().getTime() / 1000;
-    if (now - this.lastRun < oneWeek) { // Execute the PoolsUpdate only once a week, or upon restart
+    if (now - this.lastRun < config.MEMPOOL.POOLS_UPDATE_DELAY) { // Execute the PoolsUpdate only once a week, or upon restart
       return;
     }
 
@@ -43,7 +54,7 @@ class PoolsUpdater {
         this.currentSha = await this.getShaFromDb();
       }
 
-      logger.debug(`pools-v2.json sha | Current: ${this.currentSha} | Github: ${githubSha}`);
+      logger.debug(`pools-v2.json sha | Current: ${this.currentSha} | Github: ${githubSha}`, this.tag);
       if (this.currentSha !== null && this.currentSha === githubSha) {
         return;
       }
@@ -53,16 +64,16 @@ class PoolsUpdater {
         config.MEMPOOL.AUTOMATIC_POOLS_UPDATE !== true && // Automatic pools update is disabled
         !process.env.npm_config_update_pools // We're not manually updating mining pool
       ) {
-        logger.warn(`Updated mining pools data is available (${githubSha}) but AUTOMATIC_POOLS_UPDATE is disabled`);
+        logger.warn(`Updated mining pools data is available (${githubSha}) but AUTOMATIC_POOLS_UPDATE is disabled`, this.tag);
-        logger.info(`You can update your mining pools using the --update-pools command flag. You may want to clear your nginx cache as well if applicable`);
+        logger.info(`You can update your mining pools using the --update-pools command flag. You may want to clear your nginx cache as well if applicable`, this.tag);
         return;
       }
 
       const network = config.SOCKS5PROXY.ENABLED ? 'tor' : 'clearnet';
       if (this.currentSha === null) {
-        logger.info(`Downloading pools-v2.json for the first time from ${this.poolsUrl} over ${network}`, logger.tags.mining);
+        logger.info(`Downloading pools-v2.json for the first time from ${this.poolsUrl} over ${network}`, this.tag);
       } else {
-        logger.warn(`pools-v2.json is outdated, fetching latest from ${this.poolsUrl} over ${network}`, logger.tags.mining);
+        logger.warn(`pools-v2.json is outdated, fetching latest from ${this.poolsUrl} over ${network}`, this.tag);
       }
       const poolsJson = await this.query(this.poolsUrl);
       if (poolsJson === undefined) {
@@ -71,7 +82,7 @@ class PoolsUpdater {
       poolsParser.setMiningPools(poolsJson);
 
       if (config.DATABASE.ENABLED === false) { // Don't run db operations
-        logger.info(`Mining pools-v2.json (${githubSha}) import completed (no database)`);
+        logger.info(`Mining pools-v2.json (${githubSha}) import completed (no database)`, this.tag);
         return;
       }
 
@@ -81,14 +92,14 @@ class PoolsUpdater {
         await this.updateDBSha(githubSha);
         await DB.query('COMMIT;');
       } catch (e) {
-        logger.err(`Could not migrate mining pools, rolling back. Exception: ${JSON.stringify(e)}`, logger.tags.mining);
+        logger.err(`Could not migrate mining pools, rolling back. Exception: ${JSON.stringify(e)}`, this.tag);
         await DB.query('ROLLBACK;');
       }
-      logger.info(`Mining pools-v2.json (${githubSha}) import completed`);
+      logger.info(`Mining pools-v2.json (${githubSha}) import completed`, this.tag);
 
     } catch (e) {
-      this.lastRun = now - (oneWeek - oneDay); // Try again in 24h instead of waiting next week
+      this.lastRun = now - 600; // Try again in 10 minutes
-      logger.err(`PoolsUpdater failed. Will try again in 24h. Exception: ${JSON.stringify(e)}`, logger.tags.mining);
+      logger.err(`PoolsUpdater failed. Will try again in 10 minutes. Exception: ${JSON.stringify(e)}`, this.tag);
     }
   }
 
@@ -102,7 +113,7 @@ class PoolsUpdater {
         await DB.query('DELETE FROM state where name="pools_json_sha"');
         await DB.query(`INSERT INTO state VALUES('pools_json_sha', NULL, '${githubSha}')`);
       } catch (e) {
-        logger.err('Cannot save github pools-v2.json sha into the db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
+        logger.err('Cannot save github pools-v2.json sha into the db. Reason: ' + (e instanceof Error ? e.message : e), this.tag);
       }
     }
   }
@@ -115,7 +126,7 @@ class PoolsUpdater {
       const [rows]: any[] = await DB.query('SELECT string FROM state WHERE name="pools_json_sha"');
       return (rows.length > 0 ? rows[0].string : null);
     } catch (e) {
-      logger.err('Cannot fetch pools-v2.json sha from db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
+      logger.err('Cannot fetch pools-v2.json sha from db. Reason: ' + (e instanceof Error ? e.message : e), this.tag);
       return null;
     }
   }
@@ -134,7 +145,7 @@ class PoolsUpdater {
       }
     }
 
-    logger.err(`Cannot find "pools-v2.json" in git tree (${this.treeUrl})`, logger.tags.mining);
+    logger.err(`Cannot find "pools-v2.json" in git tree (${this.treeUrl})`, this.tag);
     return null;
   }
 
@@ -186,7 +197,7 @@ class PoolsUpdater {
         }
         return data.data;
       } catch (e) {
-        logger.err('Could not connect to Github. Reason: ' + (e instanceof Error ? e.message : e));
+        logger.err('Could not connect to Github. Reason: ' + (e instanceof Error ? e.message : e), this.tag);
         retry++;
       }
       await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL);

docker/README.md

@@ -109,6 +109,7 @@ Below we list all settings from `mempool-config.json` and the corresponding over
     "AUTOMATIC_POOLS_UPDATE": false,
     "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json",
     "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
+    "POOLS_UPDATE_DELAY": 604800,
     "CPFP_INDEXING": false,
     "MAX_BLOCKS_BULK_QUERY": 0,
     "DISK_CACHE_BLOCK_INTERVAL": 6,
@@ -140,6 +141,7 @@ Corresponding `docker-compose.yml` overrides:
       MEMPOOL_AUTOMATIC_POOLS_UPDATE: ""
       MEMPOOL_POOLS_JSON_URL: ""
       MEMPOOL_POOLS_JSON_TREE_URL: ""
+      MEMPOOL_POOLS_UPDATE_DELAY: ""
       MEMPOOL_CPFP_INDEXING: ""
       MEMPOOL_MAX_BLOCKS_BULK_QUERY: ""
       MEMPOOL_DISK_CACHE_BLOCK_INTERVAL: ""

docker/backend/mempool-config.json

@@ -36,6 +36,7 @@
     "ALLOW_UNREACHABLE": __MEMPOOL_ALLOW_UNREACHABLE__,
     "POOLS_JSON_TREE_URL": "__MEMPOOL_POOLS_JSON_TREE_URL__",
     "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__",
+    "POOLS_UPDATE_DELAY": __MEMPOOL_POOLS_UPDATE_DELAY__,
     "PRICE_UPDATES_PER_HOUR": __MEMPOOL_PRICE_UPDATES_PER_HOUR__,
     "MAX_TRACKED_ADDRESSES": __MEMPOOL_MAX_TRACKED_ADDRESSES__
   },

docker/backend/start.sh

@@ -29,6 +29,7 @@ __MEMPOOL_STDOUT_LOG_MIN_PRIORITY__=${MEMPOOL_STDOUT_LOG_MIN_PRIORITY:=info}
 __MEMPOOL_AUTOMATIC_POOLS_UPDATE__=${MEMPOOL_AUTOMATIC_POOLS_UPDATE:=false}
 __MEMPOOL_POOLS_JSON_URL__=${MEMPOOL_POOLS_JSON_URL:=https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json}
 __MEMPOOL_POOLS_JSON_TREE_URL__=${MEMPOOL_POOLS_JSON_TREE_URL:=https://api.github.com/repos/mempool/mining-pools/git/trees/master}
+__MEMPOOL_POOLS_UPDATE_DELAY__=${MEMPOOL_POOLS_UPDATE_DELAY:=604800}
 __MEMPOOL_AUDIT__=${MEMPOOL_AUDIT:=false}
 __MEMPOOL_RUST_GBT__=${MEMPOOL_RUST_GBT:=true}
 __MEMPOOL_LIMIT_GBT__=${MEMPOOL_LIMIT_GBT:=false}
@@ -187,6 +188,7 @@ sed -i "s!__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__!${__MEMPOOL_STDOUT_LOG_MIN_PRIORIT
 sed -i "s!__MEMPOOL_AUTOMATIC_POOLS_UPDATE__!${__MEMPOOL_AUTOMATIC_POOLS_UPDATE__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_POOLS_JSON_URL__!${__MEMPOOL_POOLS_JSON_URL__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_POOLS_JSON_TREE_URL__!${__MEMPOOL_POOLS_JSON_TREE_URL__}!g" mempool-config.json
+sed -i "s!__MEMPOOL_POOLS_UPDATE_DELAY__!${__MEMPOOL_POOLS_UPDATE_DELAY__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_AUDIT__!${__MEMPOOL_AUDIT__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_RUST_GBT__!${__MEMPOOL_RUST_GBT__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_LIMIT_GBT__!${__MEMPOOL_LIMIT_GBT__}!g" mempool-config.json