From 7314582dd191f40714f6469767c34ba8fea4522d Mon Sep 17 00:00:00 2001 From: nymkappa Date: Sun, 6 Mar 2022 12:32:16 +0100 Subject: [PATCH] Split network daily hashrate indexing and weekly pool hashrate indexing --- backend/src/api/database-migration.ts | 10 +- backend/src/api/mining.ts | 145 +++++++++++++----- backend/src/index.ts | 4 +- .../src/repositories/HashratesRepository.ts | 26 +++- 4 files changed, 139 insertions(+), 46 deletions(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 49e9ef9c4..dd5dea399 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -6,7 +6,7 @@ import logger from '../logger'; const sleep = (ms: number) => new Promise(res => setTimeout(res, ms)); class DatabaseMigration { - private static currentVersion = 8; + private static currentVersion = 9; private queryTimeout = 120000; private statisticsAddedIndexed = false; @@ -133,6 +133,10 @@ class DatabaseMigration { await this.$executeQuery(connection, 'ALTER TABLE `hashrates` ADD `type` enum("daily", "weekly") DEFAULT "daily"'); } + if (databaseSchemaVersion < 9) { + await this.$executeQuery(connection, 'ALTER TABLE `state` CHANGE `name` `name` varchar(100)') + } + connection.release(); } catch (e) { connection.release(); @@ -276,6 +280,10 @@ class DatabaseMigration { queries.push(`INSERT INTO state(name, number, string) VALUES ('last_hashrates_indexing', 0, NULL)`); } + if (version < 9) { + queries.push(`INSERT INTO state(name, number, string) VALUES ('last_weekly_hashrates_indexing', 0, NULL)`); + } + return queries; } diff --git a/backend/src/api/mining.ts b/backend/src/api/mining.ts index 1d2f47561..62fa8baca 100644 --- a/backend/src/api/mining.ts +++ b/backend/src/api/mining.ts @@ -8,6 +8,7 @@ import blocks from './blocks'; class Mining { hashrateIndexingStarted = false; + weeklyHashrateIndexingStarted = false; constructor() { } @@ -95,31 +96,112 @@ class Mining { } /** - * Generate daily hashrate data + * Generate weekly mining pool hashrate history */ - public async $generateNetworkHashrateHistory(): Promise { - // We only run this once a day - const latestTimestamp = await HashratesRepository.$getLatestRunTimestamp(); - const now = new Date().getTime() / 1000; - if (now - latestTimestamp < 86400) { + public async $generatePoolHashrateHistory(): Promise { + if (!blocks.blockIndexingCompleted || this.weeklyHashrateIndexingStarted) { return; } + // We only run this once a week + const latestTimestamp = await HashratesRepository.$getLatestRunTimestamp('last_weekly_hashrates_indexing'); + const now = new Date().getTime() / 1000; + if (now - latestTimestamp < 604800) { + return; + } + + try { + this.weeklyHashrateIndexingStarted = true; + + logger.info(`Indexing mining pools weekly hashrates`); + + const indexedTimestamp = await HashratesRepository.$getWeeklyHashrateTimestamps(); + const hashrates: any[] = []; + const genesisTimestamp = 1231006505; // bitcoin-cli getblock 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f + const lastWeekMidnight = this.getDateMidnight(new Date()); + lastWeekMidnight.setDate(lastWeekMidnight.getDate() - 7); + let toTimestamp = Math.round(lastWeekMidnight.getTime() / 1000); + + while (toTimestamp > genesisTimestamp) { + const fromTimestamp = toTimestamp - 604800; + + // Skip already indexed weeks + if (indexedTimestamp.includes(toTimestamp + 1)) { + toTimestamp -= 604800; + continue; + } + + const blockStats: any = await BlocksRepository.$blockCountBetweenTimestamp( + null, fromTimestamp, toTimestamp); + if (blockStats.blockCount === 0) { // We are done indexing, no blocks left + break; + } + + const fromTxt = new Date(1000 * fromTimestamp); + const toTxt = new Date(1000 * toTimestamp); + logger.debug(`Indexing pools hashrate between ${fromTxt.toUTCString()} and ${toTxt.toUTCString()}`) + + const lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount, + blockStats.lastBlockHeight); + + let pools = await PoolsRepository.$getPoolsInfoBetween(fromTimestamp, toTimestamp); + const totalBlocks = pools.reduce((acc, pool) => acc + pool.blockCount, 0); + pools = pools.map((pool: any) => { + pool.hashrate = (pool.blockCount / totalBlocks) * lastBlockHashrate; + pool.share = (pool.blockCount / totalBlocks); + return pool; + }); + + for (const pool of pools) { + hashrates.push({ + hashrateTimestamp: toTimestamp + 1, + avgHashrate: pool['hashrate'], + poolId: pool.poolId, + share: pool['share'], + type: 'weekly', + }); + } + + await HashratesRepository.$saveHashrates(hashrates); + hashrates.length = 0; + + toTimestamp -= 604800; + } + this.weeklyHashrateIndexingStarted = false; + await HashratesRepository.$setLatestRunTimestamp('last_weekly_hashrates_indexing'); + logger.info(`Weekly pools hashrate indexing completed`); + } catch (e) { + this.weeklyHashrateIndexingStarted = false; + throw e; + } + + } + + /** + * Generate daily hashrate data + */ + public async $generateNetworkHashrateHistory(): Promise { if (!blocks.blockIndexingCompleted || this.hashrateIndexingStarted) { return; } + // We only run this once a day + const latestTimestamp = await HashratesRepository.$getLatestRunTimestamp('last_hashrates_indexing'); + const now = new Date().getTime() / 1000; + if (now - latestTimestamp < 86400) { + return; + } + try { this.hashrateIndexingStarted = true; - logger.info(`Indexing hashrates`); + logger.info(`Indexing network daily hashrate`); const totalDayIndexed = (await BlocksRepository.$blockCount(null, null)) / 144; const indexedTimestamp = (await HashratesRepository.$getNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp); let startedAt = new Date().getTime() / 1000; const genesisTimestamp = 1231006505; // bitcoin-cli getblock 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f - const lastMidnight = new Date(); - lastMidnight.setUTCHours(0); lastMidnight.setUTCMinutes(0); lastMidnight.setUTCSeconds(0); lastMidnight.setUTCMilliseconds(0); + const lastMidnight = this.getDateMidnight(new Date()); let toTimestamp = Math.round(lastMidnight.getTime() / 1000); let indexedThisRun = 0; let totalIndexed = 0; @@ -128,6 +210,8 @@ class Mining { while (toTimestamp > genesisTimestamp) { const fromTimestamp = toTimestamp - 86400; + + // Skip already indexed weeks if (indexedTimestamp.includes(fromTimestamp)) { toTimestamp -= 86400; ++totalIndexed; @@ -140,31 +224,9 @@ class Mining { break; } - let lastBlockHashrate = 0; - lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount, + const lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount, blockStats.lastBlockHeight); - if (totalIndexed > 7 && totalIndexed % 7 === 0 && !indexedTimestamp.includes(fromTimestamp + 1)) { // Save weekly pools hashrate - logger.debug("Indexing weekly hashrates for mining pools"); - let pools = await PoolsRepository.$getPoolsInfoBetween(fromTimestamp - 604800, fromTimestamp); - const totalBlocks = pools.reduce((acc, pool) => acc + pool.blockCount, 0); - pools = pools.map((pool: any) => { - pool.hashrate = (pool.blockCount / totalBlocks) * lastBlockHashrate; - pool.share = (pool.blockCount / totalBlocks); - return pool; - }); - - for (const pool of pools) { - hashrates.push({ - hashrateTimestamp: fromTimestamp + 1, - avgHashrate: pool['hashrate'], - poolId: pool.poolId, - share: pool['share'], - type: 'weekly', - }); - } - } - hashrates.push({ hashrateTimestamp: fromTimestamp, avgHashrate: lastBlockHashrate, @@ -203,18 +265,25 @@ class Mining { }); } - if (hashrates.length > 0) { - await HashratesRepository.$saveHashrates(hashrates); - } - await HashratesRepository.$setLatestRunTimestamp(); - this.hashrateIndexingStarted = false; + await HashratesRepository.$saveHashrates(hashrates); - logger.info(`Hashrates indexing completed`); + await HashratesRepository.$setLatestRunTimestamp('last_hashrates_indexing'); + this.hashrateIndexingStarted = false; + logger.info(`Daily network hashrate indexing completed`); } catch (e) { this.hashrateIndexingStarted = false; throw e; } } + + private getDateMidnight(date: Date): Date { + date.setUTCHours(0); + date.setUTCMinutes(0); + date.setUTCSeconds(0); + date.setUTCMilliseconds(0); + + return date; + } } export default new Mining(); diff --git a/backend/src/index.ts b/backend/src/index.ts index 51cee34c4..158f2895f 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -167,7 +167,8 @@ class Server { } async $resetHashratesIndexingState() { - return await HashratesRepository.$setLatestRunTimestamp(0); + await HashratesRepository.$setLatestRunTimestamp('last_hashrates_indexing', 0); + await HashratesRepository.$setLatestRunTimestamp('last_week_hashrates_indexing', 0); } async $runIndexingWhenReady() { @@ -178,6 +179,7 @@ class Server { try { await blocks.$generateBlockDatabase(); await mining.$generateNetworkHashrateHistory(); + await mining.$generatePoolHashrateHistory(); } catch (e) { logger.err(`Unable to run indexing right now, trying again later. ` + e); } diff --git a/backend/src/repositories/HashratesRepository.ts b/backend/src/repositories/HashratesRepository.ts index 5a33d7e77..367733cfa 100644 --- a/backend/src/repositories/HashratesRepository.ts +++ b/backend/src/repositories/HashratesRepository.ts @@ -8,6 +8,10 @@ class HashratesRepository { * Save indexed block data in the database */ public async $saveHashrates(hashrates: any) { + if (hashrates.length === 0) { + return; + } + let query = `INSERT INTO hashrates(hashrate_timestamp, avg_hashrate, pool_id, share, type) VALUES`; @@ -53,6 +57,16 @@ class HashratesRepository { return rows; } + public async $getWeeklyHashrateTimestamps(): Promise { + const connection = await DB.pool.getConnection(); + + let query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp FROM hashrates where type = 'weekly' GROUP BY hashrate_timestamp`; + const [rows]: any[] = await connection.query(query); + connection.release(); + + return rows.map(row => row.timestamp); + } + /** * Returns the current biggest pool hashrate history */ @@ -83,18 +97,18 @@ class HashratesRepository { return rows; } - public async $setLatestRunTimestamp(val: any = null) { + public async $setLatestRunTimestamp(key: string, val: any = null) { const connection = await DB.pool.getConnection(); - const query = `UPDATE state SET number = ? WHERE name = 'last_hashrates_indexing'`; + const query = `UPDATE state SET number = ? WHERE name = ?`; - await connection.query(query, (val === null) ? [Math.round(new Date().getTime() / 1000)] : [val]); + await connection.query(query, (val === null) ? [Math.round(new Date().getTime() / 1000), key] : [val, key]); connection.release(); } - public async $getLatestRunTimestamp(): Promise { + public async $getLatestRunTimestamp(key: string): Promise { const connection = await DB.pool.getConnection(); - const query = `SELECT number FROM state WHERE name = 'last_hashrates_indexing'`; - const [rows] = await connection.query(query); + const query = `SELECT number FROM state WHERE name = ?`; + const [rows] = await connection.query(query, [key]); connection.release(); return rows[0]['number']; }