machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Set db connection to UTC - Fix hashrate indexing

Browse Source
This commit is contained in:
nymkappa 2022-03-12 14:47:33 +01:00
parent 456e6a7296
commit 33897b029f
No known key found for this signature in database
GPG Key ID: E155910B16E8BD04
10 changed files with 86 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
backend/src/api/database-migration.ts
View File

@ -6,7 +6,7 @@ import logger from '../logger';
const sleep = (ms: number) => new Promise(res => setTimeout(res, ms));
class DatabaseMigration {
private static currentVersion = 13;
private static currentVersion = 14;
private queryTimeout = 120000;
private statisticsAddedIndexed = false;
@ -77,7 +77,7 @@ class DatabaseMigration {
await this.$setStatisticsAddedIndexedFlag(databaseSchemaVersion);
const isBitcoin = ['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK);
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
try {
await this.$executeQuery(connection, this.getCreateElementsTableQuery(), await this.$checkIfTableExists('elements_pegs'));
await this.$executeQuery(connection, this.getCreateStatisticsQuery(), await this.$checkIfTableExists('statistics'));
@ -168,6 +168,13 @@ class DatabaseMigration {
await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `avg_fee_rate` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
}
if (databaseSchemaVersion < 14 && isBitcoin === true) {
logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.`);
await this.$executeQuery(connection, 'TRUNCATE hashrates;'); // Need to re-index
await this.$executeQuery(connection, 'ALTER TABLE `hashrates` DROP FOREIGN KEY `hashrates_ibfk_1`');
await this.$executeQuery(connection, 'ALTER TABLE `hashrates` MODIFY `pool_id` SMALLINT UNSIGNED NOT NULL DEFAULT "0"');
}
connection.release();
} catch (e) {
connection.release();
@ -187,7 +194,7 @@ class DatabaseMigration {
return;
}
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
try {
// We don't use "CREATE INDEX IF NOT EXISTS" because it is not supported on old mariadb version 5.X
@ -225,7 +232,7 @@ class DatabaseMigration {
* Check if 'table' exists in the database
*/
private async $checkIfTableExists(table: string): Promise<boolean> {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
const query = `SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = '${config.DATABASE.DATABASE}' AND TABLE_NAME = '${table}'`;
const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
connection.release();
@ -236,7 +243,7 @@ class DatabaseMigration {
* Get current database version
*/
private async $getSchemaVersionFromDatabase(): Promise<number> {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
const query = `SELECT number FROM state WHERE name = 'schema_version';`;
const [rows] = await this.$executeQuery(connection, query, true);
connection.release();
@ -247,7 +254,7 @@ class DatabaseMigration {
* Create the `state` table
*/
private async $createMigrationStateTable(): Promise<void> {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
try {
const query = `CREATE TABLE IF NOT EXISTS state (
@ -279,7 +286,7 @@ class DatabaseMigration {
}
transactionQueries.push(this.getUpdateToLatestSchemaVersionQuery());
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
try {
await this.$executeQuery(connection, 'START TRANSACTION;');
for (const query of transactionQueries) {
@ -330,7 +337,7 @@ class DatabaseMigration {
* Print current database version
*/
private async $printDatabaseVersion() {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
try {
const [rows] = await this.$executeQuery(connection, 'SELECT VERSION() as version;', true);
logger.debug(`MIGRATIONS: Database engine version '${rows[0].version}'`);
@ -474,7 +481,7 @@ class DatabaseMigration {
public async $truncateIndexedData(tables: string[]) {
const allowedTables = ['blocks', 'hashrates'];
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
try {
for (const table of tables) {
if (!allowedTables.includes(table)) {

8
backend/src/api/liquid/elements-parser.ts
View File

@ -33,7 +33,7 @@ class ElementsParser {
}
public async $getPegDataByMonth(): Promise<any> {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
const query = `SELECT SUM(amount) AS amount, DATE_FORMAT(FROM_UNIXTIME(datetime), '%Y-%m-01') AS date FROM elements_pegs GROUP BY DATE_FORMAT(FROM_UNIXTIME(datetime), '%Y%m')`;
const [rows] = await connection.query<any>(query);
connection.release();
@ -79,7 +79,7 @@ class ElementsParser {
protected async $savePegToDatabase(height: number, blockTime: number, amount: number, txid: string,
txindex: number, bitcoinaddress: string, bitcointxid: string, bitcoinindex: number, final_tx: number): Promise<void> {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
const query = `INSERT INTO elements_pegs(
block, datetime, amount, txid, txindex, bitcoinaddress, bitcointxid, bitcoinindex, final_tx
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`;
@ -93,7 +93,7 @@ class ElementsParser {
}
protected async $getLatestBlockHeightFromDatabase(): Promise<number> {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
const query = `SELECT number FROM state WHERE name = 'last_elements_block'`;
const [rows] = await connection.query<any>(query);
connection.release();
@ -101,7 +101,7 @@ class ElementsParser {
}
protected async $saveLatestBlockToDatabase(blockHeight: number) {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
const query = `UPDATE state SET number = ? WHERE name = 'last_elements_block'`;
await connection.query<any>(query, [blockHeight]);
connection.release();

13
backend/src/api/mining.ts
View File

@ -80,8 +80,8 @@ class Mining {
// We only run this once a week
const latestTimestamp = await HashratesRepository.$getLatestRunTimestamp('last_weekly_hashrates_indexing');
const now = new Date().getTime() / 1000;
if (now - latestTimestamp < 604800) {
const now = new Date();
if ((now.getTime() / 1000) - latestTimestamp < 604800) {
return;
}
@ -94,7 +94,6 @@ class Mining {
const hashrates: any[] = [];
const genesisTimestamp = 1231006505; // bitcoin-cli getblock 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f
const now = new Date();
const lastMonday = new Date(now.setDate(now.getDate() - (now.getDay() + 6) % 7));
const lastMondayMidnight = this.getDateMidnight(lastMonday);
let toTimestamp = Math.round((lastMondayMidnight.getTime() - 604800) / 1000);
@ -108,7 +107,7 @@ class Mining {
const fromTimestamp = toTimestamp - 604800;
// Skip already indexed weeks
if (indexedTimestamp.includes(toTimestamp + 1)) {
if (indexedTimestamp.includes(toTimestamp)) {
toTimestamp -= 604800;
++totalIndexed;
continue;
@ -133,7 +132,7 @@ class Mining {
for (const pool of pools) {
hashrates.push({
hashrateTimestamp: toTimestamp + 1,
hashrateTimestamp: toTimestamp,
avgHashrate: pool['hashrate'],
poolId: pool.poolId,
share: pool['share'],
@ -202,7 +201,7 @@ class Mining {
const fromTimestamp = toTimestamp - 86400;
// Skip already indexed weeks
if (indexedTimestamp.includes(fromTimestamp)) {
if (indexedTimestamp.includes(toTimestamp)) {
toTimestamp -= 86400;
++totalIndexed;
continue;
@ -220,7 +219,7 @@ class Mining {
hashrates.push({
hashrateTimestamp: toTimestamp,
avgHashrate: lastBlockHashrate,
poolId: null,
poolId: 0,
share: 1,
type: 'daily',
});

4
backend/src/api/pools-parser.ts
View File

@ -66,7 +66,7 @@ class PoolsParser {
logger.debug(`Found ${poolNames.length} unique mining pools`);
// Get existing pools from the db
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
let existingPools;
try {
[existingPools] = await connection.query<any>({ sql: 'SELECT * FROM pools;', timeout: 120000 });
@ -152,7 +152,7 @@ class PoolsParser {
* Manually add the 'unknown pool'
*/
private async insertUnknownPool() {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
try {
const [rows]: any[] = await connection.query({ sql: 'SELECT name from pools where name="Unknown"', timeout: 120000 });
if (rows.length === 0) {

24
backend/src/api/statistics.ts
View File

@ -155,7 +155,7 @@ class Statistics {
}
private async $createZeroedStatistic(): Promise<number | undefined> {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
try {
const query = `INSERT INTO statistics(
added,
@ -216,7 +216,7 @@ class Statistics {
}
private async $create(statistics: Statistic): Promise<number | undefined> {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
try {
const query = `INSERT INTO statistics(
added,
@ -421,7 +421,7 @@ class Statistics {
private async $get(id: number): Promise<OptimizedStatistic | undefined> {
try {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
const query = `SELECT *, UNIX_TIMESTAMP(added) as added FROM statistics WHERE id = ?`;
const [rows] = await connection.query<any>(query, [id]);
connection.release();
@ -435,7 +435,7 @@ class Statistics {
public async $list2H(): Promise<OptimizedStatistic[]> {
try {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
const query = `SELECT *, UNIX_TIMESTAMP(added) as added FROM statistics ORDER BY statistics.added DESC LIMIT 120`;
const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
connection.release();
@ -448,7 +448,7 @@ class Statistics {
public async $list24H(): Promise<OptimizedStatistic[]> {
try {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
const query = `SELECT *, UNIX_TIMESTAMP(added) as added FROM statistics ORDER BY statistics.added DESC LIMIT 1440`;
const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
connection.release();
@ -461,7 +461,7 @@ class Statistics {
public async $list1W(): Promise<OptimizedStatistic[]> {
try {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
const query = this.getQueryForDaysAvg(300, '1 WEEK'); // 5m interval
const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
connection.release();
@ -474,7 +474,7 @@ class Statistics {
public async $list1M(): Promise<OptimizedStatistic[]> {
try {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
const query = this.getQueryForDaysAvg(1800, '1 MONTH'); // 30m interval
const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
connection.release();
@ -487,7 +487,7 @@ class Statistics {
public async $list3M(): Promise<OptimizedStatistic[]> {
try {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
const query = this.getQueryForDaysAvg(7200, '3 MONTH'); // 2h interval
const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
connection.release();
@ -500,7 +500,7 @@ class Statistics {
public async $list6M(): Promise<OptimizedStatistic[]> {
try {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
const query = this.getQueryForDaysAvg(10800, '6 MONTH'); // 3h interval
const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
connection.release();
@ -513,7 +513,7 @@ class Statistics {
public async $list1Y(): Promise<OptimizedStatistic[]> {
try {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
const query = this.getQueryForDays(28800, '1 YEAR'); // 8h interval
const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
connection.release();
@ -526,7 +526,7 @@ class Statistics {
public async $list2Y(): Promise<OptimizedStatistic[]> {
try {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
const query = this.getQueryForDays(28800, "2 YEAR"); // 8h interval
const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
connection.release();
@ -539,7 +539,7 @@ class Statistics {
public async $list3Y(): Promise<OptimizedStatistic[]> {
try {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
const query = this.getQueryForDays(43200, "3 YEAR"); // 12h interval
const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
connection.release();

17
backend/src/database.ts
View File

@ -1,5 +1,5 @@
import config from './config';
import { createPool } from 'mysql2/promise';
import { createPool, PoolConnection } from 'mysql2/promise';
import logger from './logger';
export class DB {
@ -11,13 +11,24 @@ export class DB {
password: config.DATABASE.PASSWORD,
connectionLimit: 10,
supportBigNumbers: true,
timezone: '+00:00',
});
static connectionsReady: number[] = [];
static async getConnection() {
const connection: PoolConnection = await DB.pool.getConnection();
const connectionId = connection['connection'].connectionId;
if (!DB.connectionsReady.includes(connectionId)) {
await connection.query(`SET time_zone='+00:00';`);
this.connectionsReady.push(connectionId);
}
return connection;
}
}
export async function checkDbConnection() {
try {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
logger.info('Database connection established.');
connection.release();
} catch (e) {

6
backend/src/index.ts
View File

@ -5,7 +5,7 @@ import * as WebSocket from 'ws';
import * as cluster from 'cluster';
import axios from 'axios';
import { checkDbConnection } from './database';
import { checkDbConnection, DB } from './database';
import config from './config';
import routes from './routes';
import blocks from './api/blocks';
@ -180,8 +180,8 @@ class Server {
try {
blocks.$generateBlockDatabase();
mining.$generateNetworkHashrateHistory();
mining.$generatePoolHashrateHistory();
await mining.$generateNetworkHashrateHistory();
await mining.$generatePoolHashrateHistory();
} catch (e) {
logger.err(`Unable to run indexing right now, trying again later. ` + e);
}

20
backend/src/repositories/BlocksRepository.ts
View File

@ -8,7 +8,7 @@ class BlocksRepository {
* Save indexed block data in the database
*/
public async $saveBlockInDatabase(block: BlockExtended) {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
try {
const query = `INSERT INTO blocks(
@ -70,7 +70,7 @@ class BlocksRepository {
return [];
}
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
try {
const [rows]: any[] = await connection.query(`
SELECT height
@ -116,7 +116,7 @@ class BlocksRepository {
query += ` GROUP by pools.id`;
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
try {
const [rows] = await connection.query(query, params);
connection.release();
@ -154,7 +154,7 @@ class BlocksRepository {
}
// logger.debug(query);
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
try {
const [rows] = await connection.query(query, params);
connection.release();
@ -194,7 +194,7 @@ class BlocksRepository {
query += ` blockTimestamp BETWEEN FROM_UNIXTIME('${from}') AND FROM_UNIXTIME('${to}')`;
// logger.debug(query);
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
try {
const [rows] = await connection.query(query, params);
connection.release();
@ -217,7 +217,7 @@ class BlocksRepository {
LIMIT 1;`;
// logger.debug(query);
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
try {
const [rows]: any[] = await connection.query(query);
connection.release();
@ -253,7 +253,7 @@ class BlocksRepository {
LIMIT 10`;
// logger.debug(query);
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
try {
const [rows] = await connection.query(query, params);
connection.release();
@ -274,7 +274,7 @@ class BlocksRepository {
* Get one block by height
*/
public async $getBlockByHeight(height: number): Promise<object | null> {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
try {
const [rows]: any[] = await connection.query(`
SELECT *, UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp,
@ -305,7 +305,7 @@ class BlocksRepository {
public async $getBlocksDifficulty(interval: string | null): Promise<object[]> {
interval = Common.getSqlInterval(interval);
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
// :D ... Yeah don't ask me about this one https://stackoverflow.com/a/40303162
// Basically, using temporary user defined fields, we are able to extract all
@ -356,7 +356,7 @@ class BlocksRepository {
}
public async $getOldestIndexedBlockHeight(): Promise<number> {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
try {
const [rows]: any[] = await connection.query(`SELECT MIN(height) as minHeight FROM blocks`);
connection.release();

27
backend/src/repositories/HashratesRepository.ts
View File

@ -20,9 +20,9 @@ class HashratesRepository {
}
query = query.slice(0, -1);
const connection = await DB.pool.getConnection();
let connection;
try {
// logger.debug(query);
connection = await DB.getConnection();
await connection.query(query);
connection.release();
} catch (e: any) {
@ -35,18 +35,16 @@ class HashratesRepository {
public async $getNetworkDailyHashrate(interval: string | null): Promise<any[]> {
interval = Common.getSqlInterval(interval);
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
let query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp, avg_hashrate as avgHashrate
FROM hashrates`;
if (interval) {
query += ` WHERE hashrate_timestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()
AND hashrates.type = 'daily'
AND pool_id IS NULL`;
AND hashrates.type = 'daily'`;
} else {
query += ` WHERE hashrates.type = 'daily'
AND pool_id IS NULL`;
query += ` WHERE hashrates.type = 'daily'`;
}
query += ` ORDER by hashrate_timestamp`;
@ -64,9 +62,12 @@ class HashratesRepository {
}
public async $getWeeklyHashrateTimestamps(): Promise<number[]> {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
const query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp FROM hashrates where type = 'weekly' GROUP BY hashrate_timestamp`;
const query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp
FROM hashrates
WHERE type = 'weekly'
GROUP BY hashrate_timestamp`;
try {
const [rows]: any[] = await connection.query(query);
@ -86,7 +87,7 @@ class HashratesRepository {
public async $getPoolsWeeklyHashrate(interval: string | null): Promise<any[]> {
interval = Common.getSqlInterval(interval);
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
const topPoolsId = (await PoolsRepository.$getPoolsInfo('1w')).map((pool) => pool.poolId);
let query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp, avg_hashrate as avgHashrate, share, pools.name as poolName
@ -120,7 +121,7 @@ class HashratesRepository {
* Returns a pool hashrate history
*/
public async $getPoolWeeklyHashrate(poolId: number): Promise<any[]> {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
// Find hashrate boundaries
let query = `SELECT MIN(hashrate_timestamp) as firstTimestamp, MAX(hashrate_timestamp) as lastTimestamp
@ -163,7 +164,7 @@ class HashratesRepository {
}
public async $setLatestRunTimestamp(key: string, val: any = null) {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
const query = `UPDATE state SET number = ? WHERE name = ?`;
try {
@ -175,7 +176,7 @@ class HashratesRepository {
}
public async $getLatestRunTimestamp(key: string): Promise<number> {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
const query = `SELECT number FROM state WHERE name = ?`;
try {

10
backend/src/repositories/PoolsRepository.ts
View File

@ -8,7 +8,7 @@ class PoolsRepository {
* Get all pools tagging info
*/
public async $getPools(): Promise<PoolTag[]> {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
const [rows] = await connection.query('SELECT id, name, addresses, regexes FROM pools;');
connection.release();
return <PoolTag[]>rows;
@ -18,7 +18,7 @@ class PoolsRepository {
* Get unknown pool tagging info
*/
public async $getUnknownPool(): Promise<PoolTag> {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
const [rows] = await connection.query('SELECT id, name FROM pools where name = "Unknown"');
connection.release();
return <PoolTag>rows[0];
@ -42,7 +42,7 @@ class PoolsRepository {
ORDER BY COUNT(height) DESC`;
// logger.debug(query);
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
try {
const [rows] = await connection.query(query);
connection.release();
@ -64,7 +64,7 @@ class PoolsRepository {
LEFT JOIN blocks on pools.id = blocks.pool_id AND blocks.blockTimestamp BETWEEN FROM_UNIXTIME(?) AND FROM_UNIXTIME(?)
GROUP BY pools.id`;
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
try {
const [rows] = await connection.query(query, [from, to]);
connection.release();
@ -87,7 +87,7 @@ class PoolsRepository {
WHERE pools.id = ?`;
// logger.debug(query);
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
try {
const [rows] = await connection.query(query, [poolId]);
connection.release();