diff --git a/backend/src/api/mining.ts b/backend/src/api/mining.ts index d2504274c..207625be0 100644 --- a/backend/src/api/mining.ts +++ b/backend/src/api/mining.ts @@ -173,26 +173,25 @@ class Mining { */ public async $generatePoolHashrateHistory(): Promise { const now = new Date(); + const lastestRunDate = await HashratesRepository.$getLatestRun('last_weekly_hashrates_indexing'); - try { - const lastestRunDate = await HashratesRepository.$getLatestRun('last_weekly_hashrates_indexing'); - - // Run only if: - // * lastestRunDate is set to 0 (node backend restart, reorg) - // * we started a new week (around Monday midnight) - const runIndexing = lastestRunDate === 0 || now.getUTCDay() === 1 && lastestRunDate !== now.getUTCDate(); - if (!runIndexing) { - return; - } - } catch (e) { - throw e; + // Run only if: + // * lastestRunDate is set to 0 (node backend restart, reorg) + // * we started a new week (around Monday midnight) + const runIndexing = lastestRunDate === 0 || now.getUTCDay() === 1 && lastestRunDate !== now.getUTCDate(); + if (!runIndexing) { + return; } try { + const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlockTimestamp()); + + const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0)); + const genesisTimestamp = genesisBlock.time * 1000; + const indexedTimestamp = await HashratesRepository.$getWeeklyHashrateTimestamps(); const hashrates: any[] = []; - const genesisTimestamp = 1231006505000; // bitcoin-cli getblock 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f - + const lastMonday = new Date(now.setDate(now.getDate() - (now.getDay() + 6) % 7)); const lastMondayMidnight = this.getDateMidnight(lastMonday); let toTimestamp = lastMondayMidnight.getTime(); @@ -207,7 +206,7 @@ class Mining { logger.debug(`Indexing weekly mining pool hashrate`); loadingIndicators.setProgress('weekly-hashrate-indexing', 0); - while (toTimestamp > genesisTimestamp) { + while (toTimestamp > genesisTimestamp && toTimestamp > oldestConsecutiveBlockTimestamp) { const fromTimestamp = toTimestamp - 604800000; // Skip already indexed weeks @@ -217,14 +216,6 @@ class Mining { continue; } - // Check if we have blocks for the previous week (which mean that the week - // we are currently indexing has complete data) - const blockStatsPreviousWeek: any = await BlocksRepository.$blockCountBetweenTimestamp( - null, (fromTimestamp - 604800000) / 1000, (toTimestamp - 604800000) / 1000); - if (blockStatsPreviousWeek.blockCount === 0) { // We are done indexing - break; - } - const blockStats: any = await BlocksRepository.$blockCountBetweenTimestamp( null, fromTimestamp / 1000, toTimestamp / 1000); const lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount, @@ -232,25 +223,27 @@ class Mining { let pools = await PoolsRepository.$getPoolsInfoBetween(fromTimestamp / 1000, toTimestamp / 1000); const totalBlocks = pools.reduce((acc, pool) => acc + pool.blockCount, 0); - pools = pools.map((pool: any) => { - pool.hashrate = (pool.blockCount / totalBlocks) * lastBlockHashrate; - pool.share = (pool.blockCount / totalBlocks); - return pool; - }); - - for (const pool of pools) { - hashrates.push({ - hashrateTimestamp: toTimestamp / 1000, - avgHashrate: pool['hashrate'], - poolId: pool.poolId, - share: pool['share'], - type: 'weekly', + if (totalBlocks > 0) { + pools = pools.map((pool: any) => { + pool.hashrate = (pool.blockCount / totalBlocks) * lastBlockHashrate; + pool.share = (pool.blockCount / totalBlocks); + return pool; }); - } - newlyIndexed += hashrates.length; - await HashratesRepository.$saveHashrates(hashrates); - hashrates.length = 0; + for (const pool of pools) { + hashrates.push({ + hashrateTimestamp: toTimestamp / 1000, + avgHashrate: pool['hashrate'] , + poolId: pool.poolId, + share: pool['share'], + type: 'weekly', + }); + } + + newlyIndexed += hashrates.length; + await HashratesRepository.$saveHashrates(hashrates); + hashrates.length = 0; + } const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer)); if (elapsedSeconds > 1) { @@ -285,20 +278,19 @@ class Mining { * [INDEXING] Generate daily hashrate data */ public async $generateNetworkHashrateHistory(): Promise { - try { - // We only run this once a day around midnight - const latestRunDate = await HashratesRepository.$getLatestRun('last_hashrates_indexing'); - const now = new Date().getUTCDate(); - if (now === latestRunDate) { - return; - } - } catch (e) { - throw e; + // We only run this once a day around midnight + const latestRunDate = await HashratesRepository.$getLatestRun('last_hashrates_indexing'); + const now = new Date().getUTCDate(); + if (now === latestRunDate) { + return; } + const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlockTimestamp()); + try { - const indexedTimestamp = (await HashratesRepository.$getNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp); - const genesisTimestamp = (config.MEMPOOL.NETWORK === 'signet') ? 1598918400000 : 1231006505000; // bitcoin-cli getblock 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f + const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0)); + const genesisTimestamp = genesisBlock.time * 1000; + const indexedTimestamp = (await HashratesRepository.$getRawNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp); const lastMidnight = this.getDateMidnight(new Date()); let toTimestamp = Math.round(lastMidnight.getTime()); const hashrates: any[] = []; @@ -313,7 +305,7 @@ class Mining { logger.debug(`Indexing daily network hashrate`); loadingIndicators.setProgress('daily-hashrate-indexing', 0); - while (toTimestamp > genesisTimestamp) { + while (toTimestamp > genesisTimestamp && toTimestamp > oldestConsecutiveBlockTimestamp) { const fromTimestamp = toTimestamp - 86400000; // Skip already indexed days @@ -323,17 +315,9 @@ class Mining { continue; } - // Check if we have blocks for the previous day (which mean that the day - // we are currently indexing has complete data) - const blockStatsPreviousDay: any = await BlocksRepository.$blockCountBetweenTimestamp( - null, (fromTimestamp - 86400000) / 1000, (toTimestamp - 86400000) / 1000); - if (blockStatsPreviousDay.blockCount === 0 && config.MEMPOOL.NETWORK === 'mainnet') { // We are done indexing - break; - } - const blockStats: any = await BlocksRepository.$blockCountBetweenTimestamp( null, fromTimestamp / 1000, toTimestamp / 1000); - const lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount, + const lastBlockHashrate = blockStats.blockCount === 0 ? 0 : await bitcoinClient.getNetworkHashPs(blockStats.blockCount, blockStats.lastBlockHeight); hashrates.push({ @@ -368,8 +352,8 @@ class Mining { ++totalIndexed; } - // Add genesis block manually on mainnet and testnet - if ('signet' !== config.MEMPOOL.NETWORK && toTimestamp <= genesisTimestamp && !indexedTimestamp.includes(genesisTimestamp)) { + // Add genesis block manually + if (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT === -1 && !indexedTimestamp.includes(genesisTimestamp / 1000)) { hashrates.push({ hashrateTimestamp: genesisTimestamp / 1000, avgHashrate: await bitcoinClient.getNetworkHashPs(1, 1), @@ -410,14 +394,18 @@ class Mining { let totalIndexed = 0; if (indexedHeights[0] !== true) { + const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0)); await DifficultyAdjustmentsRepository.$saveAdjustments({ - time: (config.MEMPOOL.NETWORK === 'signet') ? 1598918400 : 1231006505, + time: genesisBlock.time, height: 0, - difficulty: (config.MEMPOOL.NETWORK === 'signet') ? 0.001126515290698186 : 1.0, + difficulty: genesisBlock.difficulty, adjustment: 0.0, }); } + let totalBlockChecked = 0; + let timer = new Date().getTime() / 1000; + for (const block of blocks) { if (block.difficulty !== currentDifficulty) { if (block.height === 0 || indexedHeights[block.height] === true) { // Already indexed @@ -438,6 +426,14 @@ class Mining { totalIndexed++; currentDifficulty = block.difficulty; } + + totalBlockChecked++; + const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer)); + if (elapsedSeconds > 5) { + const progress = Math.round(totalBlockChecked / blocks.length * 100); + logger.info(`Indexing difficulty adjustment at block #${block.height} | Progress: ${progress}%`); + timer = new Date().getTime() / 1000; + } } if (totalIndexed > 0) { diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index 8e4e7e87f..e34c43e8d 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -35,6 +35,8 @@ class Indexer { this.runIndexer = false; this.indexerRunning = true; + logger.debug(`Running mining indexer`); + try { const chainValid = await blocks.$generateBlockDatabase(); if (chainValid === false) { @@ -54,9 +56,15 @@ class Indexer { this.indexerRunning = false; logger.err(`Indexer failed, trying again in 10 seconds. Reason: ` + (e instanceof Error ? e.message : e)); setTimeout(() => this.reindex(), 10000); + this.indexerRunning = false; + return; } this.indexerRunning = false; + + const runEvery = 1000 * 3600; // 1 hour + logger.debug(`Indexing completed. Next run planned at ${new Date(new Date().getTime() + runEvery).toUTCString()}`); + setTimeout(() => this.reindex(), runEvery); } async $resetHashratesIndexingState() { diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index c6b14ff51..c3f824d61 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -610,6 +610,24 @@ class BlocksRepository { throw e; } } + + /** + * Return the oldest block timestamp from a consecutive chain of block from the most recent one + */ + public async $getOldestConsecutiveBlockTimestamp(): Promise { + try { + const [rows]: any = await DB.query(`SELECT height, UNIX_TIMESTAMP(blockTimestamp) as timestamp FROM blocks ORDER BY height DESC`); + for (let i = 0; i < rows.length - 1; ++i) { + if (rows[i].height - rows[i + 1].height > 1) { + return rows[i].timestamp; + } + } + return rows[rows.length - 1].timestamp; + } catch (e) { + logger.err('Cannot generate block size and weight history. Reason: ' + (e instanceof Error ? e.message : e)); + throw e; + } + } } export default new BlocksRepository(); diff --git a/backend/src/repositories/HashratesRepository.ts b/backend/src/repositories/HashratesRepository.ts index 5e6048abc..e5a193477 100644 --- a/backend/src/repositories/HashratesRepository.ts +++ b/backend/src/repositories/HashratesRepository.ts @@ -1,6 +1,5 @@ import { escape } from 'mysql2'; import { Common } from '../api/common'; -import config from '../config'; import DB from '../database'; import logger from '../logger'; import PoolsRepository from './PoolsRepository'; @@ -30,6 +29,32 @@ class HashratesRepository { } } + public async $getRawNetworkDailyHashrate(interval: string | null): Promise { + interval = Common.getSqlInterval(interval); + + let query = `SELECT + UNIX_TIMESTAMP(hashrate_timestamp) as timestamp, + avg_hashrate as avgHashrate + FROM hashrates`; + + if (interval) { + query += ` WHERE hashrate_timestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW() + AND hashrates.type = 'daily'`; + } else { + query += ` WHERE hashrates.type = 'daily'`; + } + + query += ` ORDER by hashrate_timestamp`; + + try { + const [rows]: any[] = await DB.query(query); + return rows; + } catch (e) { + logger.err('Cannot fetch network hashrate history. Reason: ' + (e instanceof Error ? e.message : e)); + throw e; + } + } + public async $getNetworkDailyHashrate(interval: string | null): Promise { interval = Common.getSqlInterval(interval);