From 7314582dd191f40714f6469767c34ba8fea4522d Mon Sep 17 00:00:00 2001 From: nymkappa Date: Sun, 6 Mar 2022 12:32:16 +0100 Subject: [PATCH 1/8] Split network daily hashrate indexing and weekly pool hashrate indexing --- backend/src/api/database-migration.ts | 10 +- backend/src/api/mining.ts | 145 +++++++++++++----- backend/src/index.ts | 4 +- .../src/repositories/HashratesRepository.ts | 26 +++- 4 files changed, 139 insertions(+), 46 deletions(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 49e9ef9c4..dd5dea399 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -6,7 +6,7 @@ import logger from '../logger'; const sleep = (ms: number) => new Promise(res => setTimeout(res, ms)); class DatabaseMigration { - private static currentVersion = 8; + private static currentVersion = 9; private queryTimeout = 120000; private statisticsAddedIndexed = false; @@ -133,6 +133,10 @@ class DatabaseMigration { await this.$executeQuery(connection, 'ALTER TABLE `hashrates` ADD `type` enum("daily", "weekly") DEFAULT "daily"'); } + if (databaseSchemaVersion < 9) { + await this.$executeQuery(connection, 'ALTER TABLE `state` CHANGE `name` `name` varchar(100)') + } + connection.release(); } catch (e) { connection.release(); @@ -276,6 +280,10 @@ class DatabaseMigration { queries.push(`INSERT INTO state(name, number, string) VALUES ('last_hashrates_indexing', 0, NULL)`); } + if (version < 9) { + queries.push(`INSERT INTO state(name, number, string) VALUES ('last_weekly_hashrates_indexing', 0, NULL)`); + } + return queries; } diff --git a/backend/src/api/mining.ts b/backend/src/api/mining.ts index 1d2f47561..62fa8baca 100644 --- a/backend/src/api/mining.ts +++ b/backend/src/api/mining.ts @@ -8,6 +8,7 @@ import blocks from './blocks'; class Mining { hashrateIndexingStarted = false; + weeklyHashrateIndexingStarted = false; constructor() { } @@ -95,31 +96,112 @@ class Mining { } /** - * Generate daily hashrate data + * Generate weekly mining pool hashrate history */ - public async $generateNetworkHashrateHistory(): Promise { - // We only run this once a day - const latestTimestamp = await HashratesRepository.$getLatestRunTimestamp(); - const now = new Date().getTime() / 1000; - if (now - latestTimestamp < 86400) { + public async $generatePoolHashrateHistory(): Promise { + if (!blocks.blockIndexingCompleted || this.weeklyHashrateIndexingStarted) { return; } + // We only run this once a week + const latestTimestamp = await HashratesRepository.$getLatestRunTimestamp('last_weekly_hashrates_indexing'); + const now = new Date().getTime() / 1000; + if (now - latestTimestamp < 604800) { + return; + } + + try { + this.weeklyHashrateIndexingStarted = true; + + logger.info(`Indexing mining pools weekly hashrates`); + + const indexedTimestamp = await HashratesRepository.$getWeeklyHashrateTimestamps(); + const hashrates: any[] = []; + const genesisTimestamp = 1231006505; // bitcoin-cli getblock 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f + const lastWeekMidnight = this.getDateMidnight(new Date()); + lastWeekMidnight.setDate(lastWeekMidnight.getDate() - 7); + let toTimestamp = Math.round(lastWeekMidnight.getTime() / 1000); + + while (toTimestamp > genesisTimestamp) { + const fromTimestamp = toTimestamp - 604800; + + // Skip already indexed weeks + if (indexedTimestamp.includes(toTimestamp + 1)) { + toTimestamp -= 604800; + continue; + } + + const blockStats: any = await BlocksRepository.$blockCountBetweenTimestamp( + null, fromTimestamp, toTimestamp); + if (blockStats.blockCount === 0) { // We are done indexing, no blocks left + break; + } + + const fromTxt = new Date(1000 * fromTimestamp); + const toTxt = new Date(1000 * toTimestamp); + logger.debug(`Indexing pools hashrate between ${fromTxt.toUTCString()} and ${toTxt.toUTCString()}`) + + const lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount, + blockStats.lastBlockHeight); + + let pools = await PoolsRepository.$getPoolsInfoBetween(fromTimestamp, toTimestamp); + const totalBlocks = pools.reduce((acc, pool) => acc + pool.blockCount, 0); + pools = pools.map((pool: any) => { + pool.hashrate = (pool.blockCount / totalBlocks) * lastBlockHashrate; + pool.share = (pool.blockCount / totalBlocks); + return pool; + }); + + for (const pool of pools) { + hashrates.push({ + hashrateTimestamp: toTimestamp + 1, + avgHashrate: pool['hashrate'], + poolId: pool.poolId, + share: pool['share'], + type: 'weekly', + }); + } + + await HashratesRepository.$saveHashrates(hashrates); + hashrates.length = 0; + + toTimestamp -= 604800; + } + this.weeklyHashrateIndexingStarted = false; + await HashratesRepository.$setLatestRunTimestamp('last_weekly_hashrates_indexing'); + logger.info(`Weekly pools hashrate indexing completed`); + } catch (e) { + this.weeklyHashrateIndexingStarted = false; + throw e; + } + + } + + /** + * Generate daily hashrate data + */ + public async $generateNetworkHashrateHistory(): Promise { if (!blocks.blockIndexingCompleted || this.hashrateIndexingStarted) { return; } + // We only run this once a day + const latestTimestamp = await HashratesRepository.$getLatestRunTimestamp('last_hashrates_indexing'); + const now = new Date().getTime() / 1000; + if (now - latestTimestamp < 86400) { + return; + } + try { this.hashrateIndexingStarted = true; - logger.info(`Indexing hashrates`); + logger.info(`Indexing network daily hashrate`); const totalDayIndexed = (await BlocksRepository.$blockCount(null, null)) / 144; const indexedTimestamp = (await HashratesRepository.$getNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp); let startedAt = new Date().getTime() / 1000; const genesisTimestamp = 1231006505; // bitcoin-cli getblock 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f - const lastMidnight = new Date(); - lastMidnight.setUTCHours(0); lastMidnight.setUTCMinutes(0); lastMidnight.setUTCSeconds(0); lastMidnight.setUTCMilliseconds(0); + const lastMidnight = this.getDateMidnight(new Date()); let toTimestamp = Math.round(lastMidnight.getTime() / 1000); let indexedThisRun = 0; let totalIndexed = 0; @@ -128,6 +210,8 @@ class Mining { while (toTimestamp > genesisTimestamp) { const fromTimestamp = toTimestamp - 86400; + + // Skip already indexed weeks if (indexedTimestamp.includes(fromTimestamp)) { toTimestamp -= 86400; ++totalIndexed; @@ -140,31 +224,9 @@ class Mining { break; } - let lastBlockHashrate = 0; - lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount, + const lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount, blockStats.lastBlockHeight); - if (totalIndexed > 7 && totalIndexed % 7 === 0 && !indexedTimestamp.includes(fromTimestamp + 1)) { // Save weekly pools hashrate - logger.debug("Indexing weekly hashrates for mining pools"); - let pools = await PoolsRepository.$getPoolsInfoBetween(fromTimestamp - 604800, fromTimestamp); - const totalBlocks = pools.reduce((acc, pool) => acc + pool.blockCount, 0); - pools = pools.map((pool: any) => { - pool.hashrate = (pool.blockCount / totalBlocks) * lastBlockHashrate; - pool.share = (pool.blockCount / totalBlocks); - return pool; - }); - - for (const pool of pools) { - hashrates.push({ - hashrateTimestamp: fromTimestamp + 1, - avgHashrate: pool['hashrate'], - poolId: pool.poolId, - share: pool['share'], - type: 'weekly', - }); - } - } - hashrates.push({ hashrateTimestamp: fromTimestamp, avgHashrate: lastBlockHashrate, @@ -203,18 +265,25 @@ class Mining { }); } - if (hashrates.length > 0) { - await HashratesRepository.$saveHashrates(hashrates); - } - await HashratesRepository.$setLatestRunTimestamp(); - this.hashrateIndexingStarted = false; + await HashratesRepository.$saveHashrates(hashrates); - logger.info(`Hashrates indexing completed`); + await HashratesRepository.$setLatestRunTimestamp('last_hashrates_indexing'); + this.hashrateIndexingStarted = false; + logger.info(`Daily network hashrate indexing completed`); } catch (e) { this.hashrateIndexingStarted = false; throw e; } } + + private getDateMidnight(date: Date): Date { + date.setUTCHours(0); + date.setUTCMinutes(0); + date.setUTCSeconds(0); + date.setUTCMilliseconds(0); + + return date; + } } export default new Mining(); diff --git a/backend/src/index.ts b/backend/src/index.ts index 51cee34c4..158f2895f 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -167,7 +167,8 @@ class Server { } async $resetHashratesIndexingState() { - return await HashratesRepository.$setLatestRunTimestamp(0); + await HashratesRepository.$setLatestRunTimestamp('last_hashrates_indexing', 0); + await HashratesRepository.$setLatestRunTimestamp('last_week_hashrates_indexing', 0); } async $runIndexingWhenReady() { @@ -178,6 +179,7 @@ class Server { try { await blocks.$generateBlockDatabase(); await mining.$generateNetworkHashrateHistory(); + await mining.$generatePoolHashrateHistory(); } catch (e) { logger.err(`Unable to run indexing right now, trying again later. ` + e); } diff --git a/backend/src/repositories/HashratesRepository.ts b/backend/src/repositories/HashratesRepository.ts index 5a33d7e77..367733cfa 100644 --- a/backend/src/repositories/HashratesRepository.ts +++ b/backend/src/repositories/HashratesRepository.ts @@ -8,6 +8,10 @@ class HashratesRepository { * Save indexed block data in the database */ public async $saveHashrates(hashrates: any) { + if (hashrates.length === 0) { + return; + } + let query = `INSERT INTO hashrates(hashrate_timestamp, avg_hashrate, pool_id, share, type) VALUES`; @@ -53,6 +57,16 @@ class HashratesRepository { return rows; } + public async $getWeeklyHashrateTimestamps(): Promise { + const connection = await DB.pool.getConnection(); + + let query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp FROM hashrates where type = 'weekly' GROUP BY hashrate_timestamp`; + const [rows]: any[] = await connection.query(query); + connection.release(); + + return rows.map(row => row.timestamp); + } + /** * Returns the current biggest pool hashrate history */ @@ -83,18 +97,18 @@ class HashratesRepository { return rows; } - public async $setLatestRunTimestamp(val: any = null) { + public async $setLatestRunTimestamp(key: string, val: any = null) { const connection = await DB.pool.getConnection(); - const query = `UPDATE state SET number = ? WHERE name = 'last_hashrates_indexing'`; + const query = `UPDATE state SET number = ? WHERE name = ?`; - await connection.query(query, (val === null) ? [Math.round(new Date().getTime() / 1000)] : [val]); + await connection.query(query, (val === null) ? [Math.round(new Date().getTime() / 1000), key] : [val, key]); connection.release(); } - public async $getLatestRunTimestamp(): Promise { + public async $getLatestRunTimestamp(key: string): Promise { const connection = await DB.pool.getConnection(); - const query = `SELECT number FROM state WHERE name = 'last_hashrates_indexing'`; - const [rows] = await connection.query(query); + const query = `SELECT number FROM state WHERE name = ?`; + const [rows] = await connection.query(query, [key]); connection.release(); return rows[0]['number']; } From 8532d13a0df7455cad777501c0c232e46c0d52cb Mon Sep 17 00:00:00 2001 From: nymkappa Date: Sun, 6 Mar 2022 12:36:50 +0100 Subject: [PATCH 2/8] Update hashrate indexing logs --- backend/src/api/mining.ts | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/backend/src/api/mining.ts b/backend/src/api/mining.ts index 62fa8baca..a8a8af967 100644 --- a/backend/src/api/mining.ts +++ b/backend/src/api/mining.ts @@ -122,12 +122,18 @@ class Mining { lastWeekMidnight.setDate(lastWeekMidnight.getDate() - 7); let toTimestamp = Math.round(lastWeekMidnight.getTime() / 1000); + const totalWeekIndexed = (await BlocksRepository.$blockCount(null, null)) / 1008; + let indexedThisRun = 0; + let totalIndexed = 0; + let startedAt = new Date().getTime() / 1000; + while (toTimestamp > genesisTimestamp) { const fromTimestamp = toTimestamp - 604800; // Skip already indexed weeks if (indexedTimestamp.includes(toTimestamp + 1)) { toTimestamp -= 604800; + ++totalIndexed; continue; } @@ -137,10 +143,6 @@ class Mining { break; } - const fromTxt = new Date(1000 * fromTimestamp); - const toTxt = new Date(1000 * toTimestamp); - logger.debug(`Indexing pools hashrate between ${fromTxt.toUTCString()} and ${toTxt.toUTCString()}`) - const lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount, blockStats.lastBlockHeight); @@ -165,7 +167,19 @@ class Mining { await HashratesRepository.$saveHashrates(hashrates); hashrates.length = 0; + const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); + if (elapsedSeconds > 5) { + const weeksPerSeconds = (indexedThisRun / elapsedSeconds).toFixed(2); + const formattedDate = new Date(fromTimestamp * 1000).toUTCString(); + const weeksLeft = Math.round(totalWeekIndexed - totalIndexed); + logger.debug(`Getting weekly pool hashrate for ${formattedDate} | ~${weeksPerSeconds} weeks/sec | ~${weeksLeft} weeks left to index`); + startedAt = new Date().getTime() / 1000; + indexedThisRun = 0; + } + toTimestamp -= 604800; + ++indexedThisRun; + ++totalIndexed; } this.weeklyHashrateIndexingStarted = false; await HashratesRepository.$setLatestRunTimestamp('last_weekly_hashrates_indexing'); @@ -197,16 +211,16 @@ class Mining { logger.info(`Indexing network daily hashrate`); - const totalDayIndexed = (await BlocksRepository.$blockCount(null, null)) / 144; const indexedTimestamp = (await HashratesRepository.$getNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp); - let startedAt = new Date().getTime() / 1000; const genesisTimestamp = 1231006505; // bitcoin-cli getblock 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f const lastMidnight = this.getDateMidnight(new Date()); let toTimestamp = Math.round(lastMidnight.getTime() / 1000); + const hashrates: any[] = []; + + const totalDayIndexed = (await BlocksRepository.$blockCount(null, null)) / 144; let indexedThisRun = 0; let totalIndexed = 0; - - const hashrates: any[] = []; + let startedAt = new Date().getTime() / 1000; while (toTimestamp > genesisTimestamp) { const fromTimestamp = toTimestamp - 86400; @@ -245,7 +259,7 @@ class Mining { const daysPerSeconds = (indexedThisRun / elapsedSeconds).toFixed(2); const formattedDate = new Date(fromTimestamp * 1000).toUTCString(); const daysLeft = Math.round(totalDayIndexed - totalIndexed); - logger.debug(`Getting hashrate for ${formattedDate} | ~${daysPerSeconds} days/sec | ~${daysLeft} days left to index`); + logger.debug(`Getting network daily hashrate for ${formattedDate} | ~${daysPerSeconds} days/sec | ~${daysLeft} days left to index`); startedAt = new Date().getTime() / 1000; indexedThisRun = 0; } From 1ced44d9704ff862c149595e383a91aff72fe207 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Sun, 6 Mar 2022 12:47:16 +0100 Subject: [PATCH 3/8] Remove useless mining function wrapper in backend --- backend/src/api/blocks.ts | 2 +- backend/src/api/mining.ts | 25 ++----------------------- backend/src/routes.ts | 9 +++++---- 3 files changed, 8 insertions(+), 28 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index d6d37b5c4..f0c04455b 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -171,7 +171,7 @@ class Blocks { } /** - * Index all blocks metadata for the mining dashboard + * [INDEXING] Index all blocks metadata for the mining dashboard */ public async $generateBlockDatabase() { if (this.blockIndexingStarted) { diff --git a/backend/src/api/mining.ts b/backend/src/api/mining.ts index a8a8af967..18d82669f 100644 --- a/backend/src/api/mining.ts +++ b/backend/src/api/mining.ts @@ -75,28 +75,7 @@ class Mining { } /** - * Return the historical difficulty adjustments and oldest indexed block timestamp - */ - public async $getHistoricalDifficulty(interval: string | null): Promise { - return await BlocksRepository.$getBlocksDifficulty(interval); - } - - /** - * Return the historical hashrates and oldest indexed block timestamp - */ - public async $getNetworkHistoricalHashrates(interval: string | null): Promise { - return await HashratesRepository.$getNetworkDailyHashrate(interval); - } - - /** - * Return the historical hashrates and oldest indexed block timestamp for one or all pools - */ - public async $getPoolsHistoricalHashrates(interval: string | null, poolId: number): Promise { - return await HashratesRepository.$getPoolsWeeklyHashrate(interval); - } - - /** - * Generate weekly mining pool hashrate history + * [INDEXING] Generate weekly mining pool hashrate history */ public async $generatePoolHashrateHistory(): Promise { if (!blocks.blockIndexingCompleted || this.weeklyHashrateIndexingStarted) { @@ -192,7 +171,7 @@ class Mining { } /** - * Generate daily hashrate data + * [INDEXING] Generate daily hashrate data */ public async $generateNetworkHashrateHistory(): Promise { if (!blocks.blockIndexingCompleted || this.hashrateIndexingStarted) { diff --git a/backend/src/routes.ts b/backend/src/routes.ts index dea9da2f2..6b1a365b6 100644 --- a/backend/src/routes.ts +++ b/backend/src/routes.ts @@ -24,6 +24,7 @@ import miningStats from './api/mining'; import axios from 'axios'; import mining from './api/mining'; import BlocksRepository from './repositories/BlocksRepository'; +import HashratesRepository from './repositories/HashratesRepository'; class Routes { constructor() {} @@ -576,7 +577,7 @@ class Routes { public async $getHistoricalDifficulty(req: Request, res: Response) { try { - const stats = await mining.$getHistoricalDifficulty(req.params.interval ?? null); + const stats = await BlocksRepository.$getBlocksDifficulty(req.params.interval ?? null); res.header('Pragma', 'public'); res.header('Cache-control', 'public'); res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString()); @@ -588,7 +589,7 @@ class Routes { public async $getPoolsHistoricalHashrate(req: Request, res: Response) { try { - const hashrates = await mining.$getPoolsHistoricalHashrates(req.params.interval ?? null, parseInt(req.params.poolId, 10)); + const hashrates = await HashratesRepository.$getPoolsWeeklyHashrate(req.params.interval ?? null); const oldestIndexedBlockTimestamp = await BlocksRepository.$oldestBlockTimestamp(); res.header('Pragma', 'public'); res.header('Cache-control', 'public'); @@ -604,8 +605,8 @@ class Routes { public async $getHistoricalHashrate(req: Request, res: Response) { try { - const hashrates = await mining.$getNetworkHistoricalHashrates(req.params.interval ?? null); - const difficulty = await mining.$getHistoricalDifficulty(req.params.interval ?? null); + const hashrates = await HashratesRepository.$getNetworkDailyHashrate(req.params.interval ?? null); + const difficulty = await BlocksRepository.$getBlocksDifficulty(req.params.interval ?? null); const oldestIndexedBlockTimestamp = await BlocksRepository.$oldestBlockTimestamp(); res.header('Pragma', 'public'); res.header('Cache-control', 'public'); From 89411f23d8253fd20bd572fa2873a8a67264806a Mon Sep 17 00:00:00 2001 From: nymkappa Date: Sun, 6 Mar 2022 16:44:09 +0100 Subject: [PATCH 4/8] Set connection pool timezone to UTC - Close mysql connections upon error --- backend/src/api/database-migration.ts | 3 +- backend/src/api/mining.ts | 6 +- backend/src/database.ts | 1 + backend/src/repositories/BlocksRepository.ts | 172 ++++++++++++------ .../src/repositories/HashratesRepository.ts | 65 +++++-- backend/src/repositories/PoolsRepository.ts | 42 +++-- 6 files changed, 196 insertions(+), 93 deletions(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index dd5dea399..d072fe496 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -134,7 +134,8 @@ class DatabaseMigration { } if (databaseSchemaVersion < 9) { - await this.$executeQuery(connection, 'ALTER TABLE `state` CHANGE `name` `name` varchar(100)') + await this.$executeQuery(connection, 'ALTER TABLE `state` CHANGE `name` `name` varchar(100)'); + await this.$executeQuery(connection, 'ALTER TABLE `hashrates` ADD UNIQUE `hashrate_timestamp_pool_id` (`hashrate_timestamp`, `pool_id`)'); } connection.release(); diff --git a/backend/src/api/mining.ts b/backend/src/api/mining.ts index 18d82669f..f15c52c88 100644 --- a/backend/src/api/mining.ts +++ b/backend/src/api/mining.ts @@ -97,9 +97,8 @@ class Mining { const indexedTimestamp = await HashratesRepository.$getWeeklyHashrateTimestamps(); const hashrates: any[] = []; const genesisTimestamp = 1231006505; // bitcoin-cli getblock 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f - const lastWeekMidnight = this.getDateMidnight(new Date()); - lastWeekMidnight.setDate(lastWeekMidnight.getDate() - 7); - let toTimestamp = Math.round(lastWeekMidnight.getTime() / 1000); + const lastMidnight = this.getDateMidnight(new Date()); + let toTimestamp = Math.round((lastMidnight.getTime() - 604800) / 1000); const totalWeekIndexed = (await BlocksRepository.$blockCount(null, null)) / 1008; let indexedThisRun = 0; @@ -167,7 +166,6 @@ class Mining { this.weeklyHashrateIndexingStarted = false; throw e; } - } /** diff --git a/backend/src/database.ts b/backend/src/database.ts index b0d39b301..9f2655016 100644 --- a/backend/src/database.ts +++ b/backend/src/database.ts @@ -11,6 +11,7 @@ export class DB { password: config.DATABASE.PASSWORD, connectionLimit: 10, supportBigNumbers: true, + timezone: '+00:00', }); } diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index d91777880..844f62bad 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -53,15 +53,17 @@ class BlocksRepository { // logger.debug(query); await connection.query(query, params); + connection.release(); } catch (e: any) { + connection.release(); if (e.errno === 1062) { // ER_DUP_ENTRY logger.debug(`$saveBlockInDatabase() - Block ${block.height} has already been indexed, ignoring`); } else { + connection.release(); logger.err('$saveBlockInDatabase() error' + (e instanceof Error ? e.message : e)); + throw e; } } - - connection.release(); } /** @@ -73,20 +75,26 @@ class BlocksRepository { } const connection = await DB.pool.getConnection(); - const [rows]: any[] = await connection.query(` - SELECT height - FROM blocks - WHERE height <= ? AND height >= ? - ORDER BY height DESC; - `, [startHeight, endHeight]); - connection.release(); + try { + const [rows]: any[] = await connection.query(` + SELECT height + FROM blocks + WHERE height <= ? AND height >= ? + ORDER BY height DESC; + `, [startHeight, endHeight]); + connection.release(); - const indexedBlockHeights: number[] = []; - rows.forEach((row: any) => { indexedBlockHeights.push(row.height); }); - const seekedBlocks: number[] = Array.from(Array(startHeight - endHeight + 1).keys(), n => n + endHeight).reverse(); - const missingBlocksHeights = seekedBlocks.filter(x => indexedBlockHeights.indexOf(x) === -1); + const indexedBlockHeights: number[] = []; + rows.forEach((row: any) => { indexedBlockHeights.push(row.height); }); + const seekedBlocks: number[] = Array.from(Array(startHeight - endHeight + 1).keys(), n => n + endHeight).reverse(); + const missingBlocksHeights = seekedBlocks.filter(x => indexedBlockHeights.indexOf(x) === -1); - return missingBlocksHeights; + return missingBlocksHeights; + } catch (e) { + connection.release(); + logger.err('$getMissingBlocksBetweenHeights() error' + (e instanceof Error ? e.message : e)); + throw e; + } } /** @@ -111,10 +119,16 @@ class BlocksRepository { // logger.debug(query); const connection = await DB.pool.getConnection(); - const [rows] = await connection.query(query, params); - connection.release(); + try { + const [rows] = await connection.query(query, params); + connection.release(); - return rows; + return rows; + } catch (e) { + connection.release(); + logger.err('$getEmptyBlocks() error' + (e instanceof Error ? e.message : e)); + throw e; + } } /** @@ -143,10 +157,16 @@ class BlocksRepository { // logger.debug(query); const connection = await DB.pool.getConnection(); - const [rows] = await connection.query(query, params); - connection.release(); + try { + const [rows] = await connection.query(query, params); + connection.release(); - return rows[0].blockCount; + return rows[0].blockCount; + } catch (e) { + connection.release(); + logger.err('$blockCount() error' + (e instanceof Error ? e.message : e)); + throw e; + } } /** @@ -177,10 +197,16 @@ class BlocksRepository { // logger.debug(query); const connection = await DB.pool.getConnection(); - const [rows] = await connection.query(query, params); - connection.release(); + try { + const [rows] = await connection.query(query, params); + connection.release(); - return rows[0]; + return rows[0]; + } catch (e) { + connection.release(); + logger.err('$blockCountBetweenTimestamp() error' + (e instanceof Error ? e.message : e)); + throw e; + } } /** @@ -194,23 +220,26 @@ class BlocksRepository { // logger.debug(query); const connection = await DB.pool.getConnection(); - const [rows]: any[] = await connection.query(query); - connection.release(); + try { + const [rows]: any[] = await connection.query(query); + connection.release(); - if (rows.length <= 0) { - return -1; + if (rows.length <= 0) { + return -1; + } + + return rows[0].blockTimestamp; + } catch (e) { + connection.release(); + logger.err('$oldestBlockTimestamp() error' + (e instanceof Error ? e.message : e)); + throw e; } - - return rows[0].blockTimestamp; } /** * Get blocks mined by a specific mining pool */ - public async $getBlocksByPool( - poolId: number, - startHeight: number | null = null - ): Promise { + public async $getBlocksByPool(poolId: number, startHeight: number | null = null): Promise { const params: any[] = []; let query = `SELECT height, hash as id, tx_count, size, weight, pool_id, UNIX_TIMESTAMP(blockTimestamp) as timestamp, reward FROM blocks @@ -227,14 +256,20 @@ class BlocksRepository { // logger.debug(query); const connection = await DB.pool.getConnection(); - const [rows] = await connection.query(query, params); - connection.release(); + try { + const [rows] = await connection.query(query, params); + connection.release(); - for (const block of rows) { - delete block['blockTimestamp']; + for (const block of rows) { + delete block['blockTimestamp']; + } + + return rows; + } catch (e) { + connection.release(); + logger.err('$getBlocksByPool() error' + (e instanceof Error ? e.message : e)); + throw e; } - - return rows; } /** @@ -242,19 +277,25 @@ class BlocksRepository { */ public async $getBlockByHeight(height: number): Promise { const connection = await DB.pool.getConnection(); - const [rows]: any[] = await connection.query(` - SELECT *, UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp, pools.id as pool_id, pools.name as pool_name, pools.link as pool_link, pools.addresses as pool_addresses, pools.regexes as pool_regexes - FROM blocks - JOIN pools ON blocks.pool_id = pools.id - WHERE height = ${height}; - `); - connection.release(); + try { + const [rows]: any[] = await connection.query(` + SELECT *, UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp, pools.id as pool_id, pools.name as pool_name, pools.link as pool_link, pools.addresses as pool_addresses, pools.regexes as pool_regexes + FROM blocks + JOIN pools ON blocks.pool_id = pools.id + WHERE height = ${height}; + `); + connection.release(); - if (rows.length <= 0) { - return null; + if (rows.length <= 0) { + return null; + } + + return rows[0]; + } catch (e) { + connection.release(); + logger.err('$getBlockByHeight() error' + (e instanceof Error ? e.message : e)); + throw e; } - - return rows[0]; } /** @@ -297,21 +338,34 @@ class BlocksRepository { ORDER BY t.height `; - const [rows]: any[] = await connection.query(query); - connection.release(); + try { + const [rows]: any[] = await connection.query(query); + connection.release(); - for (let row of rows) { - delete row['rn']; + for (let row of rows) { + delete row['rn']; + } + + return rows; + } catch (e) { + connection.release(); + logger.err('$getBlocksDifficulty() error' + (e instanceof Error ? e.message : e)); + throw e; } - - return rows; } public async $getOldestIndexedBlockHeight(): Promise { const connection = await DB.pool.getConnection(); - const [rows]: any[] = await connection.query(`SELECT MIN(height) as minHeight FROM blocks`); - connection.release(); - return rows[0].minHeight; + try { + const [rows]: any[] = await connection.query(`SELECT MIN(height) as minHeight FROM blocks`); + connection.release(); + + return rows[0].minHeight; + } catch (e) { + connection.release(); + logger.err('$getOldestIndexedBlockHeight() error' + (e instanceof Error ? e.message : e)); + throw e; + } } } diff --git a/backend/src/repositories/HashratesRepository.ts b/backend/src/repositories/HashratesRepository.ts index 367733cfa..3523004d5 100644 --- a/backend/src/repositories/HashratesRepository.ts +++ b/backend/src/repositories/HashratesRepository.ts @@ -24,12 +24,12 @@ class HashratesRepository { try { // logger.debug(query); await connection.query(query); + connection.release(); } catch (e: any) { + connection.release(); logger.err('$saveHashrateInDatabase() error' + (e instanceof Error ? e.message : e)); throw e; } - - connection.release(); } public async $getNetworkDailyHashrate(interval: string | null): Promise { @@ -51,20 +51,33 @@ class HashratesRepository { query += ` ORDER by hashrate_timestamp`; - const [rows]: any[] = await connection.query(query); - connection.release(); + try { + const [rows]: any[] = await connection.query(query); + connection.release(); - return rows; + return rows; + } catch (e) { + connection.release(); + logger.err('$getNetworkDailyHashrate() error' + (e instanceof Error ? e.message : e)); + throw e; + } } public async $getWeeklyHashrateTimestamps(): Promise { const connection = await DB.pool.getConnection(); - let query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp FROM hashrates where type = 'weekly' GROUP BY hashrate_timestamp`; - const [rows]: any[] = await connection.query(query); - connection.release(); + const query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp FROM hashrates where type = 'weekly' GROUP BY hashrate_timestamp`; - return rows.map(row => row.timestamp); + try { + const [rows]: any[] = await connection.query(query); + connection.release(); + + return rows.map(row => row.timestamp); + } catch (e) { + connection.release(); + logger.err('$getWeeklyHashrateTimestamps() error' + (e instanceof Error ? e.message : e)); + throw e; + } } /** @@ -91,26 +104,44 @@ class HashratesRepository { query += ` ORDER by hashrate_timestamp, FIELD(pool_id, ${topPoolsId})`; - const [rows]: any[] = await connection.query(query); - connection.release(); + try { + const [rows]: any[] = await connection.query(query); + connection.release(); - return rows; + return rows; + } catch (e) { + connection.release(); + logger.err('$getPoolsWeeklyHashrate() error' + (e instanceof Error ? e.message : e)); + throw e; + } } public async $setLatestRunTimestamp(key: string, val: any = null) { const connection = await DB.pool.getConnection(); const query = `UPDATE state SET number = ? WHERE name = ?`; - await connection.query(query, (val === null) ? [Math.round(new Date().getTime() / 1000), key] : [val, key]); - connection.release(); + try { + await connection.query(query, (val === null) ? [Math.round(new Date().getTime() / 1000), key] : [val, key]); + connection.release(); + } catch (e) { + connection.release(); + } } public async $getLatestRunTimestamp(key: string): Promise { const connection = await DB.pool.getConnection(); const query = `SELECT number FROM state WHERE name = ?`; - const [rows] = await connection.query(query, [key]); - connection.release(); - return rows[0]['number']; + + try { + const [rows] = await connection.query(query, [key]); + connection.release(); + + return rows[0]['number']; + } catch (e) { + connection.release(); + logger.err('$setLatestRunTimestamp() error' + (e instanceof Error ? e.message : e)); + throw e; + } } } diff --git a/backend/src/repositories/PoolsRepository.ts b/backend/src/repositories/PoolsRepository.ts index ea617322a..3f904888d 100644 --- a/backend/src/repositories/PoolsRepository.ts +++ b/backend/src/repositories/PoolsRepository.ts @@ -43,26 +43,38 @@ class PoolsRepository { // logger.debug(query); const connection = await DB.pool.getConnection(); - const [rows] = await connection.query(query); - connection.release(); + try { + const [rows] = await connection.query(query); + connection.release(); - return rows; + return rows; + } catch (e) { + connection.release(); + logger.err('$getPoolsInfo() error' + (e instanceof Error ? e.message : e)); + throw e; + } } /** * Get basic pool info and block count between two timestamp */ public async $getPoolsInfoBetween(from: number, to: number): Promise { - let query = `SELECT COUNT(height) as blockCount, pools.id as poolId, pools.name as poolName + const query = `SELECT COUNT(height) as blockCount, pools.id as poolId, pools.name as poolName FROM pools LEFT JOIN blocks on pools.id = blocks.pool_id AND blocks.blockTimestamp BETWEEN FROM_UNIXTIME(?) AND FROM_UNIXTIME(?) GROUP BY pools.id`; const connection = await DB.pool.getConnection(); - const [rows] = await connection.query(query, [from, to]); - connection.release(); + try { + const [rows] = await connection.query(query, [from, to]); + connection.release(); - return rows; + return rows; + } catch (e) { + connection.release(); + logger.err('$getPoolsInfoBetween() error' + (e instanceof Error ? e.message : e)); + throw e; + } } /** @@ -76,13 +88,19 @@ class PoolsRepository { // logger.debug(query); const connection = await DB.pool.getConnection(); - const [rows] = await connection.query(query, [poolId]); - connection.release(); + try { + const [rows] = await connection.query(query, [poolId]); + connection.release(); - rows[0].regexes = JSON.parse(rows[0].regexes); - rows[0].addresses = JSON.parse(rows[0].addresses); + rows[0].regexes = JSON.parse(rows[0].regexes); + rows[0].addresses = JSON.parse(rows[0].addresses); - return rows[0]; + return rows[0]; + } catch (e) { + connection.release(); + logger.err('$getPool() error' + (e instanceof Error ? e.message : e)); + throw e; + } } } From 4b859eb4f641ac34347d61565679e4fa04e7a49e Mon Sep 17 00:00:00 2001 From: nymkappa Date: Sun, 6 Mar 2022 16:48:14 +0100 Subject: [PATCH 5/8] Re-index hashrates because we have different timestamp handling --- backend/src/api/database-migration.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index d072fe496..c66148fe0 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -134,6 +134,8 @@ class DatabaseMigration { } if (databaseSchemaVersion < 9) { + logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.'`); + await this.$executeQuery(connection, 'TRUNCATE hashrates;'); // Need to re-index await this.$executeQuery(connection, 'ALTER TABLE `state` CHANGE `name` `name` varchar(100)'); await this.$executeQuery(connection, 'ALTER TABLE `hashrates` ADD UNIQUE `hashrate_timestamp_pool_id` (`hashrate_timestamp`, `pool_id`)'); } From 2a170c07d11809212d803a88f2ac1825f777a33c Mon Sep 17 00:00:00 2001 From: nymkappa Date: Sun, 6 Mar 2022 16:50:59 +0100 Subject: [PATCH 6/8] Remove unnecessary await --- backend/src/index.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/src/index.ts b/backend/src/index.ts index 158f2895f..029e32c96 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -177,9 +177,9 @@ class Server { } try { - await blocks.$generateBlockDatabase(); - await mining.$generateNetworkHashrateHistory(); - await mining.$generatePoolHashrateHistory(); + blocks.$generateBlockDatabase(); + mining.$generateNetworkHashrateHistory(); + mining.$generatePoolHashrateHistory(); } catch (e) { logger.err(`Unable to run indexing right now, trying again later. ` + e); } From 3d1a10cdfca972c998fe613d9f83b211ecd5e198 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Sun, 6 Mar 2022 16:57:40 +0100 Subject: [PATCH 7/8] Use the latest timestamp in hashrate data ticker --- backend/src/api/mining.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/api/mining.ts b/backend/src/api/mining.ts index f15c52c88..4423e5f16 100644 --- a/backend/src/api/mining.ts +++ b/backend/src/api/mining.ts @@ -219,7 +219,7 @@ class Mining { blockStats.lastBlockHeight); hashrates.push({ - hashrateTimestamp: fromTimestamp, + hashrateTimestamp: toTimestamp, avgHashrate: lastBlockHashrate, poolId: null, share: 1, From 2570dbfab45c2a4459fe17df32c1e60911c143dd Mon Sep 17 00:00:00 2001 From: nymkappa Date: Sun, 6 Mar 2022 17:06:55 +0100 Subject: [PATCH 8/8] Fix incorrect state naming --- backend/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/index.ts b/backend/src/index.ts index 029e32c96..4ede865a6 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -168,7 +168,7 @@ class Server { async $resetHashratesIndexingState() { await HashratesRepository.$setLatestRunTimestamp('last_hashrates_indexing', 0); - await HashratesRepository.$setLatestRunTimestamp('last_week_hashrates_indexing', 0); + await HashratesRepository.$setLatestRunTimestamp('last_weekly_hashrates_indexing', 0); } async $runIndexingWhenReady() {