machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Set db connection to UTC - Fix hashrate indexing

Browse Source
This commit is contained in:
nymkappa
2022-03-12 14:47:33 +01:00
parent 456e6a7296
commit 33897b029f
10 changed files with 86 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
backend/src/api/database-migration.ts
View File

@@ -6,7 +6,7 @@ import logger from '../logger';
const sleep = (ms: number) => new Promise(res => setTimeout(res, ms));
class DatabaseMigration {
private static currentVersion = 13;
private static currentVersion = 14;
private queryTimeout = 120000;
private statisticsAddedIndexed = false;
@@ -77,7 +77,7 @@ class DatabaseMigration {
await this.$setStatisticsAddedIndexedFlag(databaseSchemaVersion);
const isBitcoin = ['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK);
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
try {
await this.$executeQuery(connection, this.getCreateElementsTableQuery(), await this.$checkIfTableExists('elements_pegs'));
await this.$executeQuery(connection, this.getCreateStatisticsQuery(), await this.$checkIfTableExists('statistics'));
@@ -168,6 +168,13 @@ class DatabaseMigration {
await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `avg_fee_rate` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
}
if (databaseSchemaVersion < 14 && isBitcoin === true) {
logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.`);
await this.$executeQuery(connection, 'TRUNCATE hashrates;'); // Need to re-index
await this.$executeQuery(connection, 'ALTER TABLE `hashrates` DROP FOREIGN KEY `hashrates_ibfk_1`');
await this.$executeQuery(connection, 'ALTER TABLE `hashrates` MODIFY `pool_id` SMALLINT UNSIGNED NOT NULL DEFAULT "0"');
}
connection.release();
} catch (e) {
connection.release();
@@ -187,7 +194,7 @@ class DatabaseMigration {
return;
}
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
try {
// We don't use "CREATE INDEX IF NOT EXISTS" because it is not supported on old mariadb version 5.X
@@ -225,7 +232,7 @@ class DatabaseMigration {
* Check if 'table' exists in the database
*/
private async $checkIfTableExists(table: string): Promise<boolean> {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
const query = `SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = '${config.DATABASE.DATABASE}' AND TABLE_NAME = '${table}'`;
const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
connection.release();
@@ -236,7 +243,7 @@ class DatabaseMigration {
* Get current database version
*/
private async $getSchemaVersionFromDatabase(): Promise<number> {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
const query = `SELECT number FROM state WHERE name = 'schema_version';`;
const [rows] = await this.$executeQuery(connection, query, true);
connection.release();
@@ -247,7 +254,7 @@ class DatabaseMigration {
* Create the `state` table
*/
private async $createMigrationStateTable(): Promise<void> {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
try {
const query = `CREATE TABLE IF NOT EXISTS state (
@@ -279,7 +286,7 @@ class DatabaseMigration {
}
transactionQueries.push(this.getUpdateToLatestSchemaVersionQuery());
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
try {
await this.$executeQuery(connection, 'START TRANSACTION;');
for (const query of transactionQueries) {
@@ -330,7 +337,7 @@ class DatabaseMigration {
* Print current database version
*/
private async $printDatabaseVersion() {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
try {
const [rows] = await this.$executeQuery(connection, 'SELECT VERSION() as version;', true);
logger.debug(`MIGRATIONS: Database engine version '${rows[0].version}'`);
@@ -474,7 +481,7 @@ class DatabaseMigration {
public async $truncateIndexedData(tables: string[]) {
const allowedTables = ['blocks', 'hashrates'];
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
try {
for (const table of tables) {
if (!allowedTables.includes(table)) {

8
backend/src/api/liquid/elements-parser.ts
View File

@@ -33,7 +33,7 @@ class ElementsParser {
}
public async $getPegDataByMonth(): Promise<any> {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
const query = `SELECT SUM(amount) AS amount, DATE_FORMAT(FROM_UNIXTIME(datetime), '%Y-%m-01') AS date FROM elements_pegs GROUP BY DATE_FORMAT(FROM_UNIXTIME(datetime), '%Y%m')`;
const [rows] = await connection.query<any>(query);
connection.release();
@@ -79,7 +79,7 @@ class ElementsParser {
protected async $savePegToDatabase(height: number, blockTime: number, amount: number, txid: string,
txindex: number, bitcoinaddress: string, bitcointxid: string, bitcoinindex: number, final_tx: number): Promise<void> {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
const query = `INSERT INTO elements_pegs(
block, datetime, amount, txid, txindex, bitcoinaddress, bitcointxid, bitcoinindex, final_tx
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`;
@@ -93,7 +93,7 @@ class ElementsParser {
}
protected async $getLatestBlockHeightFromDatabase(): Promise<number> {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
const query = `SELECT number FROM state WHERE name = 'last_elements_block'`;
const [rows] = await connection.query<any>(query);
connection.release();
@@ -101,7 +101,7 @@ class ElementsParser {
}
protected async $saveLatestBlockToDatabase(blockHeight: number) {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
const query = `UPDATE state SET number = ? WHERE name = 'last_elements_block'`;
await connection.query<any>(query, [blockHeight]);
connection.release();

13
backend/src/api/mining.ts
View File

@@ -80,8 +80,8 @@ class Mining {
// We only run this once a week
const latestTimestamp = await HashratesRepository.$getLatestRunTimestamp('last_weekly_hashrates_indexing');
const now = new Date().getTime() / 1000;
if (now - latestTimestamp < 604800) {
const now = new Date();
if ((now.getTime() / 1000) - latestTimestamp < 604800) {
return;
}
@@ -94,7 +94,6 @@ class Mining {
const hashrates: any[] = [];
const genesisTimestamp = 1231006505; // bitcoin-cli getblock 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f
const now = new Date();
const lastMonday = new Date(now.setDate(now.getDate() - (now.getDay() + 6) % 7));
const lastMondayMidnight = this.getDateMidnight(lastMonday);
let toTimestamp = Math.round((lastMondayMidnight.getTime() - 604800) / 1000);
@@ -108,7 +107,7 @@ class Mining {
const fromTimestamp = toTimestamp - 604800;
// Skip already indexed weeks
if (indexedTimestamp.includes(toTimestamp + 1)) {
if (indexedTimestamp.includes(toTimestamp)) {
toTimestamp -= 604800;
++totalIndexed;
continue;
@@ -133,7 +132,7 @@ class Mining {
for (const pool of pools) {
hashrates.push({
hashrateTimestamp: toTimestamp + 1,
hashrateTimestamp: toTimestamp,
avgHashrate: pool['hashrate'],
poolId: pool.poolId,
share: pool['share'],
@@ -202,7 +201,7 @@ class Mining {
const fromTimestamp = toTimestamp - 86400;
// Skip already indexed weeks
if (indexedTimestamp.includes(fromTimestamp)) {
if (indexedTimestamp.includes(toTimestamp)) {
toTimestamp -= 86400;
++totalIndexed;
continue;
@@ -220,7 +219,7 @@ class Mining {
hashrates.push({
hashrateTimestamp: toTimestamp,
avgHashrate: lastBlockHashrate,
poolId: null,
poolId: 0,
share: 1,
type: 'daily',
});

4
backend/src/api/pools-parser.ts
View File

@@ -66,7 +66,7 @@ class PoolsParser {
logger.debug(`Found ${poolNames.length} unique mining pools`);
// Get existing pools from the db
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
let existingPools;
try {
[existingPools] = await connection.query<any>({ sql: 'SELECT * FROM pools;', timeout: 120000 });
@@ -152,7 +152,7 @@ class PoolsParser {
* Manually add the 'unknown pool'
*/
private async insertUnknownPool() {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
try {
const [rows]: any[] = await connection.query({ sql: 'SELECT name from pools where name="Unknown"', timeout: 120000 });
if (rows.length === 0) {

24
backend/src/api/statistics.ts
View File

@@ -155,7 +155,7 @@ class Statistics {
}
private async $createZeroedStatistic(): Promise<number | undefined> {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
try {
const query = `INSERT INTO statistics(
added,
@@ -216,7 +216,7 @@ class Statistics {
}
private async $create(statistics: Statistic): Promise<number | undefined> {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
try {
const query = `INSERT INTO statistics(
added,
@@ -421,7 +421,7 @@ class Statistics {
private async $get(id: number): Promise<OptimizedStatistic | undefined> {
try {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
const query = `SELECT *, UNIX_TIMESTAMP(added) as added FROM statistics WHERE id = ?`;
const [rows] = await connection.query<any>(query, [id]);
connection.release();
@@ -435,7 +435,7 @@ class Statistics {
public async $list2H(): Promise<OptimizedStatistic[]> {
try {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
const query = `SELECT *, UNIX_TIMESTAMP(added) as added FROM statistics ORDER BY statistics.added DESC LIMIT 120`;
const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
connection.release();
@@ -448,7 +448,7 @@ class Statistics {
public async $list24H(): Promise<OptimizedStatistic[]> {
try {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
const query = `SELECT *, UNIX_TIMESTAMP(added) as added FROM statistics ORDER BY statistics.added DESC LIMIT 1440`;
const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
connection.release();
@@ -461,7 +461,7 @@ class Statistics {
public async $list1W(): Promise<OptimizedStatistic[]> {
try {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
const query = this.getQueryForDaysAvg(300, '1 WEEK'); // 5m interval
const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
connection.release();
@@ -474,7 +474,7 @@ class Statistics {
public async $list1M(): Promise<OptimizedStatistic[]> {
try {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
const query = this.getQueryForDaysAvg(1800, '1 MONTH'); // 30m interval
const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
connection.release();
@@ -487,7 +487,7 @@ class Statistics {
public async $list3M(): Promise<OptimizedStatistic[]> {
try {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
const query = this.getQueryForDaysAvg(7200, '3 MONTH'); // 2h interval
const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
connection.release();
@@ -500,7 +500,7 @@ class Statistics {
public async $list6M(): Promise<OptimizedStatistic[]> {
try {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
const query = this.getQueryForDaysAvg(10800, '6 MONTH'); // 3h interval
const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
connection.release();
@@ -513,7 +513,7 @@ class Statistics {
public async $list1Y(): Promise<OptimizedStatistic[]> {
try {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
const query = this.getQueryForDays(28800, '1 YEAR'); // 8h interval
const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
connection.release();
@@ -526,7 +526,7 @@ class Statistics {
public async $list2Y(): Promise<OptimizedStatistic[]> {
try {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
const query = this.getQueryForDays(28800, "2 YEAR"); // 8h interval
const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
connection.release();
@@ -539,7 +539,7 @@ class Statistics {
public async $list3Y(): Promise<OptimizedStatistic[]> {
try {
const connection = await DB.pool.getConnection();
const connection = await DB.getConnection();
const query = this.getQueryForDays(43200, "3 YEAR"); // 12h interval
const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
connection.release();