diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index bdad06961..c9c1da8e8 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -6,7 +6,7 @@ import logger from '../logger'; const sleep = (ms: number) => new Promise(res => setTimeout(res, ms)); class DatabaseMigration { - private static currentVersion = 13; + private static currentVersion = 14; private queryTimeout = 120000; private statisticsAddedIndexed = false; @@ -77,7 +77,7 @@ class DatabaseMigration { await this.$setStatisticsAddedIndexedFlag(databaseSchemaVersion); const isBitcoin = ['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK); - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); try { await this.$executeQuery(connection, this.getCreateElementsTableQuery(), await this.$checkIfTableExists('elements_pegs')); await this.$executeQuery(connection, this.getCreateStatisticsQuery(), await this.$checkIfTableExists('statistics')); @@ -168,6 +168,13 @@ class DatabaseMigration { await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `avg_fee_rate` BIGINT UNSIGNED NOT NULL DEFAULT "0"'); } + if (databaseSchemaVersion < 14 && isBitcoin === true) { + logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.`); + await this.$executeQuery(connection, 'TRUNCATE hashrates;'); // Need to re-index + await this.$executeQuery(connection, 'ALTER TABLE `hashrates` DROP FOREIGN KEY `hashrates_ibfk_1`'); + await this.$executeQuery(connection, 'ALTER TABLE `hashrates` MODIFY `pool_id` SMALLINT UNSIGNED NOT NULL DEFAULT "0"'); + } + connection.release(); } catch (e) { connection.release(); @@ -187,7 +194,7 @@ class DatabaseMigration { return; } - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); try { // We don't use "CREATE INDEX IF NOT EXISTS" because it is not supported on old mariadb version 5.X @@ -225,7 +232,7 @@ class DatabaseMigration { * Check if 'table' exists in the database */ private async $checkIfTableExists(table: string): Promise { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); const query = `SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = '${config.DATABASE.DATABASE}' AND TABLE_NAME = '${table}'`; const [rows] = await connection.query({ sql: query, timeout: this.queryTimeout }); connection.release(); @@ -236,7 +243,7 @@ class DatabaseMigration { * Get current database version */ private async $getSchemaVersionFromDatabase(): Promise { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); const query = `SELECT number FROM state WHERE name = 'schema_version';`; const [rows] = await this.$executeQuery(connection, query, true); connection.release(); @@ -247,7 +254,7 @@ class DatabaseMigration { * Create the `state` table */ private async $createMigrationStateTable(): Promise { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); try { const query = `CREATE TABLE IF NOT EXISTS state ( @@ -279,7 +286,7 @@ class DatabaseMigration { } transactionQueries.push(this.getUpdateToLatestSchemaVersionQuery()); - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); try { await this.$executeQuery(connection, 'START TRANSACTION;'); for (const query of transactionQueries) { @@ -330,7 +337,7 @@ class DatabaseMigration { * Print current database version */ private async $printDatabaseVersion() { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); try { const [rows] = await this.$executeQuery(connection, 'SELECT VERSION() as version;', true); logger.debug(`MIGRATIONS: Database engine version '${rows[0].version}'`); @@ -474,7 +481,7 @@ class DatabaseMigration { public async $truncateIndexedData(tables: string[]) { const allowedTables = ['blocks', 'hashrates']; - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); try { for (const table of tables) { if (!allowedTables.includes(table)) { diff --git a/backend/src/api/liquid/elements-parser.ts b/backend/src/api/liquid/elements-parser.ts index a2d4e1546..24c7ab949 100644 --- a/backend/src/api/liquid/elements-parser.ts +++ b/backend/src/api/liquid/elements-parser.ts @@ -33,7 +33,7 @@ class ElementsParser { } public async $getPegDataByMonth(): Promise { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); const query = `SELECT SUM(amount) AS amount, DATE_FORMAT(FROM_UNIXTIME(datetime), '%Y-%m-01') AS date FROM elements_pegs GROUP BY DATE_FORMAT(FROM_UNIXTIME(datetime), '%Y%m')`; const [rows] = await connection.query(query); connection.release(); @@ -79,7 +79,7 @@ class ElementsParser { protected async $savePegToDatabase(height: number, blockTime: number, amount: number, txid: string, txindex: number, bitcoinaddress: string, bitcointxid: string, bitcoinindex: number, final_tx: number): Promise { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); const query = `INSERT INTO elements_pegs( block, datetime, amount, txid, txindex, bitcoinaddress, bitcointxid, bitcoinindex, final_tx ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`; @@ -93,7 +93,7 @@ class ElementsParser { } protected async $getLatestBlockHeightFromDatabase(): Promise { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); const query = `SELECT number FROM state WHERE name = 'last_elements_block'`; const [rows] = await connection.query(query); connection.release(); @@ -101,7 +101,7 @@ class ElementsParser { } protected async $saveLatestBlockToDatabase(blockHeight: number) { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); const query = `UPDATE state SET number = ? WHERE name = 'last_elements_block'`; await connection.query(query, [blockHeight]); connection.release(); diff --git a/backend/src/api/mining.ts b/backend/src/api/mining.ts index afcc89220..512388b36 100644 --- a/backend/src/api/mining.ts +++ b/backend/src/api/mining.ts @@ -80,8 +80,8 @@ class Mining { // We only run this once a week const latestTimestamp = await HashratesRepository.$getLatestRunTimestamp('last_weekly_hashrates_indexing'); - const now = new Date().getTime() / 1000; - if (now - latestTimestamp < 604800) { + const now = new Date(); + if ((now.getTime() / 1000) - latestTimestamp < 604800) { return; } @@ -94,7 +94,6 @@ class Mining { const hashrates: any[] = []; const genesisTimestamp = 1231006505; // bitcoin-cli getblock 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f - const now = new Date(); const lastMonday = new Date(now.setDate(now.getDate() - (now.getDay() + 6) % 7)); const lastMondayMidnight = this.getDateMidnight(lastMonday); let toTimestamp = Math.round((lastMondayMidnight.getTime() - 604800) / 1000); @@ -108,7 +107,7 @@ class Mining { const fromTimestamp = toTimestamp - 604800; // Skip already indexed weeks - if (indexedTimestamp.includes(toTimestamp + 1)) { + if (indexedTimestamp.includes(toTimestamp)) { toTimestamp -= 604800; ++totalIndexed; continue; @@ -133,7 +132,7 @@ class Mining { for (const pool of pools) { hashrates.push({ - hashrateTimestamp: toTimestamp + 1, + hashrateTimestamp: toTimestamp, avgHashrate: pool['hashrate'], poolId: pool.poolId, share: pool['share'], @@ -202,7 +201,7 @@ class Mining { const fromTimestamp = toTimestamp - 86400; // Skip already indexed weeks - if (indexedTimestamp.includes(fromTimestamp)) { + if (indexedTimestamp.includes(toTimestamp)) { toTimestamp -= 86400; ++totalIndexed; continue; @@ -220,7 +219,7 @@ class Mining { hashrates.push({ hashrateTimestamp: toTimestamp, avgHashrate: lastBlockHashrate, - poolId: null, + poolId: 0, share: 1, type: 'daily', }); diff --git a/backend/src/api/pools-parser.ts b/backend/src/api/pools-parser.ts index 194ce0dd9..ff70c3cb9 100644 --- a/backend/src/api/pools-parser.ts +++ b/backend/src/api/pools-parser.ts @@ -66,7 +66,7 @@ class PoolsParser { logger.debug(`Found ${poolNames.length} unique mining pools`); // Get existing pools from the db - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); let existingPools; try { [existingPools] = await connection.query({ sql: 'SELECT * FROM pools;', timeout: 120000 }); @@ -152,7 +152,7 @@ class PoolsParser { * Manually add the 'unknown pool' */ private async insertUnknownPool() { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); try { const [rows]: any[] = await connection.query({ sql: 'SELECT name from pools where name="Unknown"', timeout: 120000 }); if (rows.length === 0) { diff --git a/backend/src/api/statistics.ts b/backend/src/api/statistics.ts index 886ad78ba..3d99adcb7 100644 --- a/backend/src/api/statistics.ts +++ b/backend/src/api/statistics.ts @@ -155,7 +155,7 @@ class Statistics { } private async $createZeroedStatistic(): Promise { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); try { const query = `INSERT INTO statistics( added, @@ -216,7 +216,7 @@ class Statistics { } private async $create(statistics: Statistic): Promise { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); try { const query = `INSERT INTO statistics( added, @@ -421,7 +421,7 @@ class Statistics { private async $get(id: number): Promise { try { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); const query = `SELECT *, UNIX_TIMESTAMP(added) as added FROM statistics WHERE id = ?`; const [rows] = await connection.query(query, [id]); connection.release(); @@ -435,7 +435,7 @@ class Statistics { public async $list2H(): Promise { try { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); const query = `SELECT *, UNIX_TIMESTAMP(added) as added FROM statistics ORDER BY statistics.added DESC LIMIT 120`; const [rows] = await connection.query({ sql: query, timeout: this.queryTimeout }); connection.release(); @@ -448,7 +448,7 @@ class Statistics { public async $list24H(): Promise { try { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); const query = `SELECT *, UNIX_TIMESTAMP(added) as added FROM statistics ORDER BY statistics.added DESC LIMIT 1440`; const [rows] = await connection.query({ sql: query, timeout: this.queryTimeout }); connection.release(); @@ -461,7 +461,7 @@ class Statistics { public async $list1W(): Promise { try { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); const query = this.getQueryForDaysAvg(300, '1 WEEK'); // 5m interval const [rows] = await connection.query({ sql: query, timeout: this.queryTimeout }); connection.release(); @@ -474,7 +474,7 @@ class Statistics { public async $list1M(): Promise { try { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); const query = this.getQueryForDaysAvg(1800, '1 MONTH'); // 30m interval const [rows] = await connection.query({ sql: query, timeout: this.queryTimeout }); connection.release(); @@ -487,7 +487,7 @@ class Statistics { public async $list3M(): Promise { try { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); const query = this.getQueryForDaysAvg(7200, '3 MONTH'); // 2h interval const [rows] = await connection.query({ sql: query, timeout: this.queryTimeout }); connection.release(); @@ -500,7 +500,7 @@ class Statistics { public async $list6M(): Promise { try { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); const query = this.getQueryForDaysAvg(10800, '6 MONTH'); // 3h interval const [rows] = await connection.query({ sql: query, timeout: this.queryTimeout }); connection.release(); @@ -513,7 +513,7 @@ class Statistics { public async $list1Y(): Promise { try { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); const query = this.getQueryForDays(28800, '1 YEAR'); // 8h interval const [rows] = await connection.query({ sql: query, timeout: this.queryTimeout }); connection.release(); @@ -526,7 +526,7 @@ class Statistics { public async $list2Y(): Promise { try { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); const query = this.getQueryForDays(28800, "2 YEAR"); // 8h interval const [rows] = await connection.query({ sql: query, timeout: this.queryTimeout }); connection.release(); @@ -539,7 +539,7 @@ class Statistics { public async $list3Y(): Promise { try { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); const query = this.getQueryForDays(43200, "3 YEAR"); // 12h interval const [rows] = await connection.query({ sql: query, timeout: this.queryTimeout }); connection.release(); diff --git a/backend/src/database.ts b/backend/src/database.ts index 9f2655016..596ce2364 100644 --- a/backend/src/database.ts +++ b/backend/src/database.ts @@ -1,5 +1,5 @@ import config from './config'; -import { createPool } from 'mysql2/promise'; +import { createPool, PoolConnection } from 'mysql2/promise'; import logger from './logger'; export class DB { @@ -11,13 +11,24 @@ export class DB { password: config.DATABASE.PASSWORD, connectionLimit: 10, supportBigNumbers: true, - timezone: '+00:00', }); + + static connectionsReady: number[] = []; + + static async getConnection() { + const connection: PoolConnection = await DB.pool.getConnection(); + const connectionId = connection['connection'].connectionId; + if (!DB.connectionsReady.includes(connectionId)) { + await connection.query(`SET time_zone='+00:00';`); + this.connectionsReady.push(connectionId); + } + return connection; + } } export async function checkDbConnection() { try { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); logger.info('Database connection established.'); connection.release(); } catch (e) { diff --git a/backend/src/index.ts b/backend/src/index.ts index 0a6080b16..d4b55e078 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -5,7 +5,7 @@ import * as WebSocket from 'ws'; import * as cluster from 'cluster'; import axios from 'axios'; -import { checkDbConnection } from './database'; +import { checkDbConnection, DB } from './database'; import config from './config'; import routes from './routes'; import blocks from './api/blocks'; @@ -180,8 +180,8 @@ class Server { try { blocks.$generateBlockDatabase(); - mining.$generateNetworkHashrateHistory(); - mining.$generatePoolHashrateHistory(); + await mining.$generateNetworkHashrateHistory(); + await mining.$generatePoolHashrateHistory(); } catch (e) { logger.err(`Unable to run indexing right now, trying again later. ` + e); } diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 041086f73..0af5e2252 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -8,7 +8,7 @@ class BlocksRepository { * Save indexed block data in the database */ public async $saveBlockInDatabase(block: BlockExtended) { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); try { const query = `INSERT INTO blocks( @@ -70,7 +70,7 @@ class BlocksRepository { return []; } - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); try { const [rows]: any[] = await connection.query(` SELECT height @@ -116,7 +116,7 @@ class BlocksRepository { query += ` GROUP by pools.id`; - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); try { const [rows] = await connection.query(query, params); connection.release(); @@ -154,7 +154,7 @@ class BlocksRepository { } // logger.debug(query); - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); try { const [rows] = await connection.query(query, params); connection.release(); @@ -194,7 +194,7 @@ class BlocksRepository { query += ` blockTimestamp BETWEEN FROM_UNIXTIME('${from}') AND FROM_UNIXTIME('${to}')`; // logger.debug(query); - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); try { const [rows] = await connection.query(query, params); connection.release(); @@ -217,7 +217,7 @@ class BlocksRepository { LIMIT 1;`; // logger.debug(query); - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); try { const [rows]: any[] = await connection.query(query); connection.release(); @@ -253,7 +253,7 @@ class BlocksRepository { LIMIT 10`; // logger.debug(query); - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); try { const [rows] = await connection.query(query, params); connection.release(); @@ -274,7 +274,7 @@ class BlocksRepository { * Get one block by height */ public async $getBlockByHeight(height: number): Promise { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); try { const [rows]: any[] = await connection.query(` SELECT *, UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp, @@ -305,7 +305,7 @@ class BlocksRepository { public async $getBlocksDifficulty(interval: string | null): Promise { interval = Common.getSqlInterval(interval); - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); // :D ... Yeah don't ask me about this one https://stackoverflow.com/a/40303162 // Basically, using temporary user defined fields, we are able to extract all @@ -356,7 +356,7 @@ class BlocksRepository { } public async $getOldestIndexedBlockHeight(): Promise { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); try { const [rows]: any[] = await connection.query(`SELECT MIN(height) as minHeight FROM blocks`); connection.release(); diff --git a/backend/src/repositories/HashratesRepository.ts b/backend/src/repositories/HashratesRepository.ts index 749d3cb57..5237e6cb7 100644 --- a/backend/src/repositories/HashratesRepository.ts +++ b/backend/src/repositories/HashratesRepository.ts @@ -20,9 +20,9 @@ class HashratesRepository { } query = query.slice(0, -1); - const connection = await DB.pool.getConnection(); + let connection; try { - // logger.debug(query); + connection = await DB.getConnection(); await connection.query(query); connection.release(); } catch (e: any) { @@ -35,18 +35,16 @@ class HashratesRepository { public async $getNetworkDailyHashrate(interval: string | null): Promise { interval = Common.getSqlInterval(interval); - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); let query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp, avg_hashrate as avgHashrate FROM hashrates`; if (interval) { query += ` WHERE hashrate_timestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW() - AND hashrates.type = 'daily' - AND pool_id IS NULL`; + AND hashrates.type = 'daily'`; } else { - query += ` WHERE hashrates.type = 'daily' - AND pool_id IS NULL`; + query += ` WHERE hashrates.type = 'daily'`; } query += ` ORDER by hashrate_timestamp`; @@ -64,9 +62,12 @@ class HashratesRepository { } public async $getWeeklyHashrateTimestamps(): Promise { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); - const query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp FROM hashrates where type = 'weekly' GROUP BY hashrate_timestamp`; + const query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp + FROM hashrates + WHERE type = 'weekly' + GROUP BY hashrate_timestamp`; try { const [rows]: any[] = await connection.query(query); @@ -86,7 +87,7 @@ class HashratesRepository { public async $getPoolsWeeklyHashrate(interval: string | null): Promise { interval = Common.getSqlInterval(interval); - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); const topPoolsId = (await PoolsRepository.$getPoolsInfo('1w')).map((pool) => pool.poolId); let query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp, avg_hashrate as avgHashrate, share, pools.name as poolName @@ -120,7 +121,7 @@ class HashratesRepository { * Returns a pool hashrate history */ public async $getPoolWeeklyHashrate(poolId: number): Promise { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); // Find hashrate boundaries let query = `SELECT MIN(hashrate_timestamp) as firstTimestamp, MAX(hashrate_timestamp) as lastTimestamp @@ -163,7 +164,7 @@ class HashratesRepository { } public async $setLatestRunTimestamp(key: string, val: any = null) { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); const query = `UPDATE state SET number = ? WHERE name = ?`; try { @@ -175,7 +176,7 @@ class HashratesRepository { } public async $getLatestRunTimestamp(key: string): Promise { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); const query = `SELECT number FROM state WHERE name = ?`; try { diff --git a/backend/src/repositories/PoolsRepository.ts b/backend/src/repositories/PoolsRepository.ts index 3f904888d..4c3fd67ce 100644 --- a/backend/src/repositories/PoolsRepository.ts +++ b/backend/src/repositories/PoolsRepository.ts @@ -8,7 +8,7 @@ class PoolsRepository { * Get all pools tagging info */ public async $getPools(): Promise { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); const [rows] = await connection.query('SELECT id, name, addresses, regexes FROM pools;'); connection.release(); return rows; @@ -18,7 +18,7 @@ class PoolsRepository { * Get unknown pool tagging info */ public async $getUnknownPool(): Promise { - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); const [rows] = await connection.query('SELECT id, name FROM pools where name = "Unknown"'); connection.release(); return rows[0]; @@ -42,7 +42,7 @@ class PoolsRepository { ORDER BY COUNT(height) DESC`; // logger.debug(query); - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); try { const [rows] = await connection.query(query); connection.release(); @@ -64,7 +64,7 @@ class PoolsRepository { LEFT JOIN blocks on pools.id = blocks.pool_id AND blocks.blockTimestamp BETWEEN FROM_UNIXTIME(?) AND FROM_UNIXTIME(?) GROUP BY pools.id`; - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); try { const [rows] = await connection.query(query, [from, to]); connection.release(); @@ -87,7 +87,7 @@ class PoolsRepository { WHERE pools.id = ?`; // logger.debug(query); - const connection = await DB.pool.getConnection(); + const connection = await DB.getConnection(); try { const [rows] = await connection.query(query, [poolId]); connection.release();