From ae2ed8fdae41145d23af1f7ecd2330e40732dc15 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Tue, 20 Aug 2024 11:53:48 +0200 Subject: [PATCH 1/3] [mining] fix pools updater only running at start --- backend/src/config.ts | 2 ++ backend/src/index.ts | 1 + backend/src/tasks/pools-updater.ts | 9 +++------ 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/src/config.ts b/backend/src/config.ts index b0afe7f23..cf63a0c7d 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -32,6 +32,7 @@ interface IConfig { AUTOMATIC_POOLS_UPDATE: boolean; POOLS_JSON_URL: string, POOLS_JSON_TREE_URL: string, + POOLS_UPDATE_DELAY: number, AUDIT: boolean; RUST_GBT: boolean; LIMIT_GBT: boolean; @@ -192,6 +193,7 @@ const defaults: IConfig = { 'AUTOMATIC_POOLS_UPDATE': false, 'POOLS_JSON_URL': 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json', 'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master', + 'POOLS_UPDATE_DELAY': 604800, // in seconds, default is one week 'AUDIT': false, 'RUST_GBT': false, 'LIMIT_GBT': false, diff --git a/backend/src/index.ts b/backend/src/index.ts index 1d83c56a3..fa380b0c9 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -239,6 +239,7 @@ class Server { if (config.FIAT_PRICE.ENABLED) { priceUpdater.$run(); } + await poolsUpdater.updatePoolsJson(); // rerun immediately if we skipped the mempool update, otherwise wait POLL_RATE_MS const elapsed = Date.now() - start; diff --git a/backend/src/tasks/pools-updater.ts b/backend/src/tasks/pools-updater.ts index a3a3265c6..6b56fff95 100644 --- a/backend/src/tasks/pools-updater.ts +++ b/backend/src/tasks/pools-updater.ts @@ -23,11 +23,8 @@ class PoolsUpdater { return; } - const oneWeek = 604800; - const oneDay = 86400; - const now = new Date().getTime() / 1000; - if (now - this.lastRun < oneWeek) { // Execute the PoolsUpdate only once a week, or upon restart + if (now - this.lastRun < config.MEMPOOL.POOLS_UPDATE_DELAY) { // Execute the PoolsUpdate only once a week, or upon restart return; } @@ -87,8 +84,8 @@ class PoolsUpdater { logger.info(`Mining pools-v2.json (${githubSha}) import completed`); } catch (e) { - this.lastRun = now - (oneWeek - oneDay); // Try again in 24h instead of waiting next week - logger.err(`PoolsUpdater failed. Will try again in 24h. Exception: ${JSON.stringify(e)}`, logger.tags.mining); + this.lastRun = now - 600; // Try again in 10 minutes + logger.err(`PoolsUpdater failed. Will try again in 10 minutes. Exception: ${JSON.stringify(e)}`, logger.tags.mining); } } From 459639410081554ddf7a5f523f2790a700f9e329 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Tue, 20 Aug 2024 12:07:20 +0200 Subject: [PATCH 2/3] [mining] pool updater is now self contained service --- backend/src/index.ts | 3 ++- backend/src/tasks/pools-updater.ts | 40 ++++++++++++++++++++---------- 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/backend/src/index.ts b/backend/src/index.ts index fa380b0c9..446a6a140 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -211,6 +211,8 @@ class Server { } }); } + + poolsUpdater.$startService(); } async runMainUpdateLoop(): Promise { @@ -239,7 +241,6 @@ class Server { if (config.FIAT_PRICE.ENABLED) { priceUpdater.$run(); } - await poolsUpdater.updatePoolsJson(); // rerun immediately if we skipped the mempool update, otherwise wait POLL_RATE_MS const elapsed = Date.now() - start; diff --git a/backend/src/tasks/pools-updater.ts b/backend/src/tasks/pools-updater.ts index 6b56fff95..652383a2a 100644 --- a/backend/src/tasks/pools-updater.ts +++ b/backend/src/tasks/pools-updater.ts @@ -6,16 +6,30 @@ import backendInfo from '../api/backend-info'; import logger from '../logger'; import { SocksProxyAgent } from 'socks-proxy-agent'; import * as https from 'https'; +import { Common } from '../api/common'; /** * Maintain the most recent version of pools-v2.json */ class PoolsUpdater { + tag = 'PoolsUpdater'; + lastRun: number = 0; currentSha: string | null = null; poolsUrl: string = config.MEMPOOL.POOLS_JSON_URL; treeUrl: string = config.MEMPOOL.POOLS_JSON_TREE_URL; + public async $startService(): Promise { + while ('Bitcoin is still alive') { + try { + await this.updatePoolsJson(); + } catch (e: any) { + logger.info(`Exception ${e} in PoolsUpdater::$startService. Code: ${e.code}. Message: ${e.message}`, this.tag); + } + await Common.sleep$(10000); + } + } + public async updatePoolsJson(): Promise { if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false || config.MEMPOOL.ENABLED === false @@ -40,7 +54,7 @@ class PoolsUpdater { this.currentSha = await this.getShaFromDb(); } - logger.debug(`pools-v2.json sha | Current: ${this.currentSha} | Github: ${githubSha}`); + logger.debug(`pools-v2.json sha | Current: ${this.currentSha} | Github: ${githubSha}`, this.tag); if (this.currentSha !== null && this.currentSha === githubSha) { return; } @@ -50,16 +64,16 @@ class PoolsUpdater { config.MEMPOOL.AUTOMATIC_POOLS_UPDATE !== true && // Automatic pools update is disabled !process.env.npm_config_update_pools // We're not manually updating mining pool ) { - logger.warn(`Updated mining pools data is available (${githubSha}) but AUTOMATIC_POOLS_UPDATE is disabled`); - logger.info(`You can update your mining pools using the --update-pools command flag. You may want to clear your nginx cache as well if applicable`); + logger.warn(`Updated mining pools data is available (${githubSha}) but AUTOMATIC_POOLS_UPDATE is disabled`, this.tag); + logger.info(`You can update your mining pools using the --update-pools command flag. You may want to clear your nginx cache as well if applicable`, this.tag); return; } const network = config.SOCKS5PROXY.ENABLED ? 'tor' : 'clearnet'; if (this.currentSha === null) { - logger.info(`Downloading pools-v2.json for the first time from ${this.poolsUrl} over ${network}`, logger.tags.mining); + logger.info(`Downloading pools-v2.json for the first time from ${this.poolsUrl} over ${network}`, this.tag); } else { - logger.warn(`pools-v2.json is outdated, fetching latest from ${this.poolsUrl} over ${network}`, logger.tags.mining); + logger.warn(`pools-v2.json is outdated, fetching latest from ${this.poolsUrl} over ${network}`, this.tag); } const poolsJson = await this.query(this.poolsUrl); if (poolsJson === undefined) { @@ -68,7 +82,7 @@ class PoolsUpdater { poolsParser.setMiningPools(poolsJson); if (config.DATABASE.ENABLED === false) { // Don't run db operations - logger.info(`Mining pools-v2.json (${githubSha}) import completed (no database)`); + logger.info(`Mining pools-v2.json (${githubSha}) import completed (no database)`, this.tag); return; } @@ -78,14 +92,14 @@ class PoolsUpdater { await this.updateDBSha(githubSha); await DB.query('COMMIT;'); } catch (e) { - logger.err(`Could not migrate mining pools, rolling back. Exception: ${JSON.stringify(e)}`, logger.tags.mining); + logger.err(`Could not migrate mining pools, rolling back. Exception: ${JSON.stringify(e)}`, this.tag); await DB.query('ROLLBACK;'); } - logger.info(`Mining pools-v2.json (${githubSha}) import completed`); + logger.info(`Mining pools-v2.json (${githubSha}) import completed`, this.tag); } catch (e) { this.lastRun = now - 600; // Try again in 10 minutes - logger.err(`PoolsUpdater failed. Will try again in 10 minutes. Exception: ${JSON.stringify(e)}`, logger.tags.mining); + logger.err(`PoolsUpdater failed. Will try again in 10 minutes. Exception: ${JSON.stringify(e)}`, this.tag); } } @@ -99,7 +113,7 @@ class PoolsUpdater { await DB.query('DELETE FROM state where name="pools_json_sha"'); await DB.query(`INSERT INTO state VALUES('pools_json_sha', NULL, '${githubSha}')`); } catch (e) { - logger.err('Cannot save github pools-v2.json sha into the db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining); + logger.err('Cannot save github pools-v2.json sha into the db. Reason: ' + (e instanceof Error ? e.message : e), this.tag); } } } @@ -112,7 +126,7 @@ class PoolsUpdater { const [rows]: any[] = await DB.query('SELECT string FROM state WHERE name="pools_json_sha"'); return (rows.length > 0 ? rows[0].string : null); } catch (e) { - logger.err('Cannot fetch pools-v2.json sha from db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining); + logger.err('Cannot fetch pools-v2.json sha from db. Reason: ' + (e instanceof Error ? e.message : e), this.tag); return null; } } @@ -131,7 +145,7 @@ class PoolsUpdater { } } - logger.err(`Cannot find "pools-v2.json" in git tree (${this.treeUrl})`, logger.tags.mining); + logger.err(`Cannot find "pools-v2.json" in git tree (${this.treeUrl})`, this.tag); return null; } @@ -183,7 +197,7 @@ class PoolsUpdater { } return data.data; } catch (e) { - logger.err('Could not connect to Github. Reason: ' + (e instanceof Error ? e.message : e)); + logger.err('Could not connect to Github. Reason: ' + (e instanceof Error ? e.message : e), this.tag); retry++; } await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL); From 6db4afe878082c67e3bb7045cb78d4a9add567ef Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Tue, 20 Aug 2024 14:31:07 +0200 Subject: [PATCH 3/3] [mining] add POOLS_UPDATE_DELAY where needed --- backend/mempool-config.sample.json | 1 + backend/src/__fixtures__/mempool-config.template.json | 1 + backend/src/__tests__/config.test.ts | 1 + docker/README.md | 2 ++ docker/backend/mempool-config.json | 1 + docker/backend/start.sh | 2 ++ 6 files changed, 8 insertions(+) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 4650c1e64..8cd7f2ede 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -27,6 +27,7 @@ "AUTOMATIC_POOLS_UPDATE": false, "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json", "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master", + "POOLS_UPDATE_DELAY": 604800, "AUDIT": false, "RUST_GBT": false, "LIMIT_GBT": false, diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index 3796b7f22..00049725a 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -28,6 +28,7 @@ "INDEXING_BLOCKS_AMOUNT": 14, "POOLS_JSON_TREE_URL": "__MEMPOOL_POOLS_JSON_TREE_URL__", "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__", + "POOLS_UPDATE_DELAY": 604800, "AUDIT": true, "RUST_GBT": false, "LIMIT_GBT": false, diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index 050213143..d112ab6eb 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -41,6 +41,7 @@ describe('Mempool Backend Config', () => { STDOUT_LOG_MIN_PRIORITY: 'debug', POOLS_JSON_TREE_URL: 'https://api.github.com/repos/mempool/mining-pools/git/trees/master', POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json', + POOLS_UPDATE_DELAY: 604800, AUDIT: false, RUST_GBT: false, LIMIT_GBT: false, diff --git a/docker/README.md b/docker/README.md index ce1548e91..2658914eb 100644 --- a/docker/README.md +++ b/docker/README.md @@ -109,6 +109,7 @@ Below we list all settings from `mempool-config.json` and the corresponding over "AUTOMATIC_POOLS_UPDATE": false, "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json", "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master", + "POOLS_UPDATE_DELAY": 604800, "CPFP_INDEXING": false, "MAX_BLOCKS_BULK_QUERY": 0, "DISK_CACHE_BLOCK_INTERVAL": 6, @@ -140,6 +141,7 @@ Corresponding `docker-compose.yml` overrides: MEMPOOL_AUTOMATIC_POOLS_UPDATE: "" MEMPOOL_POOLS_JSON_URL: "" MEMPOOL_POOLS_JSON_TREE_URL: "" + MEMPOOL_POOLS_UPDATE_DELAY: "" MEMPOOL_CPFP_INDEXING: "" MEMPOOL_MAX_BLOCKS_BULK_QUERY: "" MEMPOOL_DISK_CACHE_BLOCK_INTERVAL: "" diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index 79cd14644..7b00d792a 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -36,6 +36,7 @@ "ALLOW_UNREACHABLE": __MEMPOOL_ALLOW_UNREACHABLE__, "POOLS_JSON_TREE_URL": "__MEMPOOL_POOLS_JSON_TREE_URL__", "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__", + "POOLS_UPDATE_DELAY": __MEMPOOL_POOLS_UPDATE_DELAY__, "PRICE_UPDATES_PER_HOUR": __MEMPOOL_PRICE_UPDATES_PER_HOUR__, "MAX_TRACKED_ADDRESSES": __MEMPOOL_MAX_TRACKED_ADDRESSES__ }, diff --git a/docker/backend/start.sh b/docker/backend/start.sh index 8033531ef..9c98fed44 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -29,6 +29,7 @@ __MEMPOOL_STDOUT_LOG_MIN_PRIORITY__=${MEMPOOL_STDOUT_LOG_MIN_PRIORITY:=info} __MEMPOOL_AUTOMATIC_POOLS_UPDATE__=${MEMPOOL_AUTOMATIC_POOLS_UPDATE:=false} __MEMPOOL_POOLS_JSON_URL__=${MEMPOOL_POOLS_JSON_URL:=https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json} __MEMPOOL_POOLS_JSON_TREE_URL__=${MEMPOOL_POOLS_JSON_TREE_URL:=https://api.github.com/repos/mempool/mining-pools/git/trees/master} +__MEMPOOL_POOLS_UPDATE_DELAY__=${MEMPOOL_POOLS_UPDATE_DELAY:=604800} __MEMPOOL_AUDIT__=${MEMPOOL_AUDIT:=false} __MEMPOOL_RUST_GBT__=${MEMPOOL_RUST_GBT:=false} __MEMPOOL_LIMIT_GBT__=${MEMPOOL_LIMIT_GBT:=false} @@ -187,6 +188,7 @@ sed -i "s!__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__!${__MEMPOOL_STDOUT_LOG_MIN_PRIORIT sed -i "s!__MEMPOOL_AUTOMATIC_POOLS_UPDATE__!${__MEMPOOL_AUTOMATIC_POOLS_UPDATE__}!g" mempool-config.json sed -i "s!__MEMPOOL_POOLS_JSON_URL__!${__MEMPOOL_POOLS_JSON_URL__}!g" mempool-config.json sed -i "s!__MEMPOOL_POOLS_JSON_TREE_URL__!${__MEMPOOL_POOLS_JSON_TREE_URL__}!g" mempool-config.json +sed -i "s!__MEMPOOL_POOLS_UPDATE_DELAY__!${__MEMPOOL_POOLS_UPDATE_DELAY__}!g" mempool-config.json sed -i "s!__MEMPOOL_AUDIT__!${__MEMPOOL_AUDIT__}!g" mempool-config.json sed -i "s!__MEMPOOL_RUST_GBT__!${__MEMPOOL_RUST_GBT__}!g" mempool-config.json sed -i "s!__MEMPOOL_LIMIT_GBT__!${__MEMPOOL_LIMIT_GBT__}!g" mempool-config.json