From 7f8696c88dcb7d3279d0787cdc29edbf82465d5b Mon Sep 17 00:00:00 2001 From: nymkappa Date: Thu, 12 May 2022 10:12:14 +0200 Subject: [PATCH 1/2] Make sure to re-index skipped block when backend is offline for too long --- backend/src/api/blocks.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 293862e93..1fe0ba1b9 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -201,13 +201,12 @@ class Blocks { return; } - this.reindexFlag = false; - const blockchainInfo = await bitcoinClient.getBlockchainInfo(); if (blockchainInfo.blocks !== blockchainInfo.headers) { // Wait for node to sync return; } + this.reindexFlag = false; this.blockIndexingStarted = true; this.blockIndexingCompleted = false; @@ -299,6 +298,7 @@ class Blocks { logger.info(`${blockHeightTip - this.currentBlockHeight} blocks since tip. Fast forwarding to the ${config.MEMPOOL.INITIAL_BLOCKS_AMOUNT} recent blocks`); this.currentBlockHeight = blockHeightTip - config.MEMPOOL.INITIAL_BLOCKS_AMOUNT; fastForwarded = true; + this.reindexFlag = true; // Make sure to index the skipped blocks #1619 } if (!this.lastDifficultyAdjustmentTime) { From 2a8314efc5306a46af39c09af66936f442b81d35 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Thu, 19 May 2022 16:40:38 +0200 Subject: [PATCH 2/2] Move indexing logic into `Indexer` class --- backend/src/api/blocks.ts | 23 ++++++----------- backend/src/api/mining.ts | 25 ++----------------- backend/src/index.ts | 28 +++------------------ backend/src/indexer.ts | 52 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 64 insertions(+), 64 deletions(-) create mode 100644 backend/src/indexer.ts diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 1fe0ba1b9..ba6fdff22 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -15,6 +15,7 @@ import BitcoinApi from './bitcoin/bitcoin-api'; import { prepareBlock } from '../utils/blocks-utils'; import BlocksRepository from '../repositories/BlocksRepository'; import HashratesRepository from '../repositories/HashratesRepository'; +import indexer from '../indexer'; class Blocks { private blocks: BlockExtended[] = []; @@ -23,9 +24,6 @@ class Blocks { private lastDifficultyAdjustmentTime = 0; private previousDifficultyRetarget = 0; private newBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => void)[] = []; - private blockIndexingStarted = false; - public blockIndexingCompleted = false; - public reindexFlag = false; constructor() { } @@ -197,23 +195,15 @@ class Blocks { * [INDEXING] Index all blocks metadata for the mining dashboard */ public async $generateBlockDatabase() { - if (this.blockIndexingStarted && !this.reindexFlag) { - return; - } - const blockchainInfo = await bitcoinClient.getBlockchainInfo(); if (blockchainInfo.blocks !== blockchainInfo.headers) { // Wait for node to sync return; } - this.reindexFlag = false; - this.blockIndexingStarted = true; - this.blockIndexingCompleted = false; - try { let currentBlockHeight = blockchainInfo.blocks; - let indexingBlockAmount = config.MEMPOOL.INDEXING_BLOCKS_AMOUNT; + let indexingBlockAmount = Math.min(config.MEMPOOL.INDEXING_BLOCKS_AMOUNT, blockchainInfo.blocks); if (indexingBlockAmount <= -1) { indexingBlockAmount = currentBlockHeight + 1; } @@ -274,14 +264,14 @@ class Blocks { loadingIndicators.setProgress('block-indexing', 100); } catch (e) { logger.err('Block indexing failed. Trying again later. Reason: ' + (e instanceof Error ? e.message : e)); - this.blockIndexingStarted = false; loadingIndicators.setProgress('block-indexing', 100); return; } const chainValid = await BlocksRepository.$validateChain(); - this.reindexFlag = !chainValid; - this.blockIndexingCompleted = chainValid; + if (!chainValid) { + indexer.reindex(); + } } public async $updateBlocks() { @@ -298,7 +288,8 @@ class Blocks { logger.info(`${blockHeightTip - this.currentBlockHeight} blocks since tip. Fast forwarding to the ${config.MEMPOOL.INITIAL_BLOCKS_AMOUNT} recent blocks`); this.currentBlockHeight = blockHeightTip - config.MEMPOOL.INITIAL_BLOCKS_AMOUNT; fastForwarded = true; - this.reindexFlag = true; // Make sure to index the skipped blocks #1619 + logger.info(`Re-indexing skipped blocks and corresponding hashrates data`); + indexer.reindex(); // Make sure to index the skipped blocks #1619 } if (!this.lastDifficultyAdjustmentTime) { diff --git a/backend/src/api/mining.ts b/backend/src/api/mining.ts index cbd412068..592ac34f1 100644 --- a/backend/src/api/mining.ts +++ b/backend/src/api/mining.ts @@ -4,14 +4,10 @@ import PoolsRepository from '../repositories/PoolsRepository'; import HashratesRepository from '../repositories/HashratesRepository'; import bitcoinClient from './bitcoin/bitcoin-client'; import logger from '../logger'; -import blocks from './blocks'; import { Common } from './common'; import loadingIndicators from './loading-indicators'; class Mining { - hashrateIndexingStarted = false; - weeklyHashrateIndexingStarted = false; - constructor() { } @@ -152,14 +148,9 @@ class Mining { * [INDEXING] Generate weekly mining pool hashrate history */ public async $generatePoolHashrateHistory(): Promise { - if (!blocks.blockIndexingCompleted || this.hashrateIndexingStarted || this.weeklyHashrateIndexingStarted) { - return; - } - const now = new Date(); try { - this.weeklyHashrateIndexingStarted = true; const lastestRunDate = await HashratesRepository.$getLatestRun('last_weekly_hashrates_indexing'); // Run only if: @@ -167,11 +158,9 @@ class Mining { // * we started a new week (around Monday midnight) const runIndexing = lastestRunDate === 0 || now.getUTCDay() === 1 && lastestRunDate !== now.getUTCDate(); if (!runIndexing) { - this.weeklyHashrateIndexingStarted = false; return; } } catch (e) { - this.weeklyHashrateIndexingStarted = false; throw e; } @@ -191,6 +180,7 @@ class Mining { const startedAt = new Date().getTime() / 1000; let timer = new Date().getTime() / 1000; + logger.debug(`Indexing weekly mining pool hashrate`); loadingIndicators.setProgress('weekly-hashrate-indexing', 0); while (toTimestamp > genesisTimestamp) { @@ -255,7 +245,6 @@ class Mining { ++indexedThisRun; ++totalIndexed; } - this.weeklyHashrateIndexingStarted = false; await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', new Date().getUTCDate()); if (newlyIndexed > 0) { logger.info(`Indexed ${newlyIndexed} pools weekly hashrate`); @@ -263,7 +252,6 @@ class Mining { loadingIndicators.setProgress('weekly-hashrate-indexing', 100); } catch (e) { loadingIndicators.setProgress('weekly-hashrate-indexing', 100); - this.weeklyHashrateIndexingStarted = false; throw e; } } @@ -272,22 +260,14 @@ class Mining { * [INDEXING] Generate daily hashrate data */ public async $generateNetworkHashrateHistory(): Promise { - if (!blocks.blockIndexingCompleted || this.hashrateIndexingStarted) { - return; - } - try { - this.hashrateIndexingStarted = true; - // We only run this once a day around midnight const latestRunDate = await HashratesRepository.$getLatestRun('last_hashrates_indexing'); const now = new Date().getUTCDate(); if (now === latestRunDate) { - this.hashrateIndexingStarted = false; return; } } catch (e) { - this.hashrateIndexingStarted = false; throw e; } @@ -305,6 +285,7 @@ class Mining { const startedAt = new Date().getTime() / 1000; let timer = new Date().getTime() / 1000; + logger.debug(`Indexing daily network hashrate`); loadingIndicators.setProgress('daily-hashrate-indexing', 0); while (toTimestamp > genesisTimestamp) { @@ -376,14 +357,12 @@ class Mining { await HashratesRepository.$saveHashrates(hashrates); await HashratesRepository.$setLatestRun('last_hashrates_indexing', new Date().getUTCDate()); - this.hashrateIndexingStarted = false; if (newlyIndexed > 0) { logger.info(`Indexed ${newlyIndexed} day of network hashrate`); } loadingIndicators.setProgress('daily-hashrate-indexing', 100); } catch (e) { loadingIndicators.setProgress('daily-hashrate-indexing', 100); - this.hashrateIndexingStarted = false; throw e; } } diff --git a/backend/src/index.ts b/backend/src/index.ts index c6eab6c17..4b7881730 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -29,6 +29,7 @@ import mining from './api/mining'; import HashratesRepository from './repositories/HashratesRepository'; import BlocksRepository from './repositories/BlocksRepository'; import poolsUpdater from './tasks/pools-updater'; +import indexer from './indexer'; class Server { private wss: WebSocket.Server | undefined; @@ -99,7 +100,7 @@ class Server { } await databaseMigration.$initializeOrMigrateDatabase(); if (Common.indexingEnabled()) { - await this.$resetHashratesIndexingState(); + await indexer.$resetHashratesIndexingState(); } } catch (e) { throw new Error(e instanceof Error ? e.message : 'Error'); @@ -154,7 +155,7 @@ class Server { await poolsUpdater.updatePoolsJson(); await blocks.$updateBlocks(); await memPool.$updateMempool(); - this.$runIndexingWhenReady(); + indexer.$run(); setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS); this.currentBackendRetryInterval = 5; @@ -173,29 +174,6 @@ class Server { } } - async $resetHashratesIndexingState() { - try { - await HashratesRepository.$setLatestRun('last_hashrates_indexing', 0); - await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', 0); - } catch (e) { - logger.err(`Cannot reset hashrate indexing timestamps. Reason: ` + (e instanceof Error ? e.message : e)); - } - } - - async $runIndexingWhenReady() { - if (!Common.indexingEnabled() || mempool.hasPriority()) { - return; - } - - try { - await blocks.$generateBlockDatabase(); - await mining.$generateNetworkHashrateHistory(); - await mining.$generatePoolHashrateHistory(); - } catch (e) { - logger.err(`Indexing failed, trying again later. Reason: ` + (e instanceof Error ? e.message : e)); - } - } - setUpWebsocketHandling() { if (this.wss) { websocketHandler.setWebsocketServer(this.wss); diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts new file mode 100644 index 000000000..7ddc2a47f --- /dev/null +++ b/backend/src/indexer.ts @@ -0,0 +1,52 @@ +import { Common } from './api/common'; +import blocks from './api/blocks'; +import mempool from './api/mempool'; +import mining from './api/mining'; +import logger from './logger'; +import HashratesRepository from './repositories/HashratesRepository'; + +class Indexer { + runIndexer = true; + indexerRunning = false; + + constructor() { + } + + public reindex() { + this.runIndexer = true; + } + + public async $run() { + if (!Common.indexingEnabled() || this.runIndexer === false || + this.indexerRunning === true || mempool.hasPriority() + ) { + return; + } + + this.runIndexer = false; + this.indexerRunning = true; + + try { + await blocks.$generateBlockDatabase(); + await this.$resetHashratesIndexingState(); + await mining.$generateNetworkHashrateHistory(); + await mining.$generatePoolHashrateHistory(); + } catch (e) { + this.reindex(); + logger.err(`Indexer failed, trying again later. Reason: ` + (e instanceof Error ? e.message : e)); + } + + this.indexerRunning = false; + } + + async $resetHashratesIndexingState() { + try { + await HashratesRepository.$setLatestRun('last_hashrates_indexing', 0); + await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', 0); + } catch (e) { + logger.err(`Cannot reset hashrate indexing timestamps. Reason: ` + (e instanceof Error ? e.message : e)); + } + } +} + +export default new Indexer();