machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into regtest-1

Browse Source
This commit is contained in:
Antoni Spaanderman
2022-02-28 13:09:51 +01:00
committed by GitHub
parent cfaf8b037a 67b45e2040
commit 82462d47e7
72 changed files with 5463 additions and 5677 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
backend/src/api/blocks.ts
View File

@@ -20,6 +20,7 @@ class Blocks {
private previousDifficultyRetarget = 0;
private newBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => void)[] = [];
private blockIndexingStarted = false;
public blockIndexingCompleted = false;
constructor() { }
@@ -115,6 +116,9 @@ class Blocks {
Common.median(transactionsTmp.map((tx) => tx.effectiveFeePerVsize)) : 0;
blockExtended.extras.feeRange = transactionsTmp.length > 0 ?
Common.getFeesInRange(transactionsTmp, 8) : [0, 0];
blockExtended.extras.totalFees = transactionsTmp.reduce((acc, tx) => {
return acc + tx.fee;
}, 0)
if (Common.indexingEnabled()) {
let pool: PoolTag;
@@ -170,10 +174,7 @@ class Blocks {
* Index all blocks metadata for the mining dashboard
*/
public async $generateBlockDatabase() {
if (this.blockIndexingStarted === true ||
!Common.indexingEnabled() ||
memPool.hasPriority()
) {
if (this.blockIndexingStarted) {
return;
}
@@ -243,6 +244,8 @@ class Blocks {
logger.err('An error occured in $generateBlockDatabase(). Skipping block indexing. ' + e);
console.log(e);
}
this.blockIndexingCompleted = true;
}
public async $updateBlocks() {

83
backend/src/api/database-migration.ts
View File

@@ -6,7 +6,7 @@ import logger from '../logger';
const sleep = (ms: number) => new Promise(res => setTimeout(res, ms));
class DatabaseMigration {
private static currentVersion = 6;
private static currentVersion = 8;
private queryTimeout = 120000;
private statisticsAddedIndexed = false;
@@ -15,13 +15,13 @@ class DatabaseMigration {
* Entry point
*/
public async $initializeOrMigrateDatabase(): Promise<void> {
logger.info('MIGRATIONS: Running migrations');
logger.debug('MIGRATIONS: Running migrations');
await this.$printDatabaseVersion();
// First of all, if the `state` database does not exist, create it so we can track migration version
if (!await this.$checkIfTableExists('state')) {
logger.info('MIGRATIONS: `state` table does not exist. Creating it.');
logger.debug('MIGRATIONS: `state` table does not exist. Creating it.');
try {
await this.$createMigrationStateTable();
} catch (e) {
@@ -29,7 +29,7 @@ class DatabaseMigration {
await sleep(10000);
process.exit(-1);
}
logger.info('MIGRATIONS: `state` table initialized.');
logger.debug('MIGRATIONS: `state` table initialized.');
}
let databaseSchemaVersion = 0;
@@ -41,10 +41,10 @@ class DatabaseMigration {
process.exit(-1);
}
logger.info('MIGRATIONS: Current state.schema_version ' + databaseSchemaVersion);
logger.info('MIGRATIONS: Latest DatabaseMigration.version is ' + DatabaseMigration.currentVersion);
logger.debug('MIGRATIONS: Current state.schema_version ' + databaseSchemaVersion);
logger.debug('MIGRATIONS: Latest DatabaseMigration.version is ' + DatabaseMigration.currentVersion);
if (databaseSchemaVersion >= DatabaseMigration.currentVersion) {
logger.info('MIGRATIONS: Nothing to do.');
logger.debug('MIGRATIONS: Nothing to do.');
return;
}
@@ -58,10 +58,10 @@ class DatabaseMigration {
}
if (DatabaseMigration.currentVersion > databaseSchemaVersion) {
logger.info('MIGRATIONS: Upgrading datababse schema');
logger.notice('MIGRATIONS: Upgrading datababse schema');
try {
await this.$migrateTableSchemaFromVersion(databaseSchemaVersion);
logger.info(`MIGRATIONS: OK. Database schema have been migrated from version ${databaseSchemaVersion} to ${DatabaseMigration.currentVersion} (latest version)`);
logger.notice(`MIGRATIONS: OK. Database schema have been migrated from version ${databaseSchemaVersion} to ${DatabaseMigration.currentVersion} (latest version)`);
} catch (e) {
logger.err('MIGRATIONS: Unable to migrate database, aborting. ' + e);
}
@@ -92,11 +92,13 @@ class DatabaseMigration {
await this.$executeQuery(connection, this.getCreateBlocksTableQuery(), await this.$checkIfTableExists('blocks'));
}
if (databaseSchemaVersion < 5 && isBitcoin === true) {
logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.'`);
await this.$executeQuery(connection, 'TRUNCATE blocks;'); // Need to re-index
await this.$executeQuery(connection, 'ALTER TABLE blocks ADD `reward` double unsigned NOT NULL DEFAULT "0"');
}
if (databaseSchemaVersion < 6 && isBitcoin === true) {
logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.'`);
await this.$executeQuery(connection, 'TRUNCATE blocks;'); // Need to re-index
// Cleanup original blocks fields type
await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `height` integer unsigned NOT NULL DEFAULT "0"');
@@ -116,6 +118,21 @@ class DatabaseMigration {
await this.$executeQuery(connection, 'ALTER TABLE blocks ADD `merkle_root` varchar(65) NOT NULL DEFAULT ""');
await this.$executeQuery(connection, 'ALTER TABLE blocks ADD `previous_block_hash` varchar(65) NULL');
}
if (databaseSchemaVersion < 7 && isBitcoin === true) {
await this.$executeQuery(connection, 'DROP table IF EXISTS hashrates;');
await this.$executeQuery(connection, this.getCreateDailyStatsTableQuery(), await this.$checkIfTableExists('hashrates'));
}
if (databaseSchemaVersion < 8 && isBitcoin === true) {
logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.'`);
await this.$executeQuery(connection, 'TRUNCATE hashrates;'); // Need to re-index
await this.$executeQuery(connection, 'ALTER TABLE `hashrates` DROP INDEX `PRIMARY`');
await this.$executeQuery(connection, 'ALTER TABLE `hashrates` ADD `id` int NOT NULL AUTO_INCREMENT PRIMARY KEY FIRST');
await this.$executeQuery(connection, 'ALTER TABLE `hashrates` ADD `share` float NOT NULL DEFAULT "0"');
await this.$executeQuery(connection, 'ALTER TABLE `hashrates` ADD `type` enum("daily", "weekly") DEFAULT "daily"');
}
connection.release();
} catch (e) {
connection.release();
@@ -143,10 +160,10 @@ class DatabaseMigration {
WHERE table_schema=DATABASE() AND table_name='statistics' AND index_name='added';`;
const [rows] = await this.$executeQuery(connection, query, true);
if (rows[0].hasIndex === 0) {
logger.info('MIGRATIONS: `statistics.added` is not indexed');
logger.debug('MIGRATIONS: `statistics.added` is not indexed');
this.statisticsAddedIndexed = false;
} else if (rows[0].hasIndex === 1) {
logger.info('MIGRATIONS: `statistics.added` is already indexed');
logger.debug('MIGRATIONS: `statistics.added` is already indexed');
this.statisticsAddedIndexed = true;
}
} catch (e) {
@@ -164,7 +181,7 @@ class DatabaseMigration {
*/
private async $executeQuery(connection: PoolConnection, query: string, silent: boolean = false): Promise<any> {
if (!silent) {
logger.info('MIGRATIONS: Execute query:\n' + query);
logger.debug('MIGRATIONS: Execute query:\n' + query);
}
return connection.query<any>({ sql: query, timeout: this.queryTimeout });
}
@@ -255,6 +272,10 @@ class DatabaseMigration {
}
}
if (version < 7) {
queries.push(`INSERT INTO state(name, number, string) VALUES ('last_hashrates_indexing', 0, NULL)`);
}
return queries;
}
@@ -272,9 +293,9 @@ class DatabaseMigration {
const connection = await DB.pool.getConnection();
try {
const [rows] = await this.$executeQuery(connection, 'SELECT VERSION() as version;', true);
logger.info(`MIGRATIONS: Database engine version '${rows[0].version}'`);
logger.debug(`MIGRATIONS: Database engine version '${rows[0].version}'`);
} catch (e) {
logger.info(`MIGRATIONS: Could not fetch database engine version. ` + e);
logger.debug(`MIGRATIONS: Could not fetch database engine version. ` + e);
}
connection.release();
}
@@ -398,6 +419,40 @@ class DatabaseMigration {
FOREIGN KEY (pool_id) REFERENCES pools (id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}
private getCreateDailyStatsTableQuery(): string {
return `CREATE TABLE IF NOT EXISTS hashrates (
hashrate_timestamp timestamp NOT NULL,
avg_hashrate double unsigned DEFAULT '0',
pool_id smallint unsigned NULL,
PRIMARY KEY (hashrate_timestamp),
INDEX (pool_id),
FOREIGN KEY (pool_id) REFERENCES pools (id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}
public async $truncateIndexedData(tables: string[]) {
const allowedTables = ['blocks', 'hashrates'];
const connection = await DB.pool.getConnection();
try {
for (const table of tables) {
if (!allowedTables.includes(table)) {
logger.debug(`Table ${table} cannot to be re-indexed (not allowed)`);
continue;
};
await this.$executeQuery(connection, `TRUNCATE ${table}`, true);
if (table === 'hashrates') {
await this.$executeQuery(connection, 'UPDATE state set number = 0 where name = "last_hashrates_indexing"', true);
}
logger.notice(`Table ${table} has been truncated`);
}
} catch (e) {
logger.warn(`Unable to erase indexed data`);
}
connection.release();
}
}
export default new DatabaseMigration();

151
backend/src/api/mining.ts
View File

@@ -1,16 +1,21 @@
import { PoolInfo, PoolStats } from '../mempool.interfaces';
import BlocksRepository, { EmptyBlocks } from '../repositories/BlocksRepository';
import PoolsRepository from '../repositories/PoolsRepository';
import HashratesRepository from '../repositories/HashratesRepository';
import bitcoinClient from './bitcoin/bitcoin-client';
import logger from '../logger';
import blocks from './blocks';
class Mining {
hashrateIndexingStarted = false;
constructor() {
}
/**
* Generate high level overview of the pool ranks and general stats
*/
public async $getPoolsStats(interval: string | null) : Promise<object> {
public async $getPoolsStats(interval: string | null): Promise<object> {
const poolsStatistics = {};
const poolsInfo: PoolInfo[] = await PoolsRepository.$getPoolsInfo(interval);
@@ -26,8 +31,8 @@ class Mining {
link: poolInfo.link,
blockCount: poolInfo.blockCount,
rank: rank++,
emptyBlocks: 0,
}
emptyBlocks: 0
};
for (let i = 0; i < emptyBlocks.length; ++i) {
if (emptyBlocks[i].poolId === poolInfo.poolId) {
poolStat.emptyBlocks++;
@@ -37,15 +42,13 @@ class Mining {
});
poolsStatistics['pools'] = poolsStats;
const oldestBlock = new Date(await BlocksRepository.$oldestBlockTimestamp());
poolsStatistics['oldestIndexedBlockTimestamp'] = oldestBlock.getTime();
poolsStatistics['oldestIndexedBlockTimestamp'] = await BlocksRepository.$oldestBlockTimestamp();
const blockCount: number = await BlocksRepository.$blockCount(null, interval);
poolsStatistics['blockCount'] = blockCount;
const blockHeightTip = await bitcoinClient.getBlockCount();
const lastBlockHashrate = await bitcoinClient.getNetworkHashPs(120, blockHeightTip);
const lastBlockHashrate = await bitcoinClient.getNetworkHashPs(144, blockHeightTip);
poolsStatistics['lastEstimatedHashrate'] = lastBlockHashrate;
return poolsStatistics;
@@ -74,13 +77,137 @@ class Mining {
* Return the historical difficulty adjustments and oldest indexed block timestamp
*/
public async $getHistoricalDifficulty(interval: string | null): Promise<object> {
const difficultyAdjustments = await BlocksRepository.$getBlocksDifficulty(interval);
const oldestBlock = new Date(await BlocksRepository.$oldestBlockTimestamp());
return await BlocksRepository.$getBlocksDifficulty(interval);
}
return {
adjustments: difficultyAdjustments,
oldestIndexedBlockTimestamp: oldestBlock.getTime(),
/**
* Return the historical hashrates and oldest indexed block timestamp
*/
public async $getNetworkHistoricalHashrates(interval: string | null): Promise<object> {
return await HashratesRepository.$getNetworkDailyHashrate(interval);
}
/**
* Return the historical hashrates and oldest indexed block timestamp for one or all pools
*/
public async $getPoolsHistoricalHashrates(interval: string | null, poolId: number): Promise<object> {
return await HashratesRepository.$getPoolsWeeklyHashrate(interval);
}
/**
* Generate daily hashrate data
*/
public async $generateNetworkHashrateHistory(): Promise<void> {
// We only run this once a day
const latestTimestamp = await HashratesRepository.$getLatestRunTimestamp();
const now = new Date().getTime() / 1000;
if (now - latestTimestamp < 86400) {
return;
}
if (!blocks.blockIndexingCompleted || this.hashrateIndexingStarted) {
return;
}
this.hashrateIndexingStarted = true;
logger.info(`Indexing hashrates`);
const totalDayIndexed = (await BlocksRepository.$blockCount(null, null)) / 144;
const indexedTimestamp = (await HashratesRepository.$getNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp);
let startedAt = new Date().getTime() / 1000;
const genesisTimestamp = 1231006505; // bitcoin-cli getblock 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f
const lastMidnight = new Date();
lastMidnight.setUTCHours(0); lastMidnight.setUTCMinutes(0); lastMidnight.setUTCSeconds(0); lastMidnight.setUTCMilliseconds(0);
let toTimestamp = Math.round(lastMidnight.getTime() / 1000);
let indexedThisRun = 0;
let totalIndexed = 0;
const hashrates: any[] = [];
while (toTimestamp > genesisTimestamp) {
const fromTimestamp = toTimestamp - 86400;
if (indexedTimestamp.includes(fromTimestamp)) {
toTimestamp -= 86400;
++totalIndexed;
continue;
}
const blockStats: any = await BlocksRepository.$blockCountBetweenTimestamp(
null, fromTimestamp, toTimestamp);
if (blockStats.blockCount === 0) { // We are done indexing, no blocks left
break;
}
let lastBlockHashrate = 0;
lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount,
blockStats.lastBlockHeight);
if (totalIndexed % 7 === 0 && !indexedTimestamp.includes(fromTimestamp + 1)) { // Save weekly pools hashrate
logger.debug("Indexing weekly hashrates for mining pools");
let pools = await PoolsRepository.$getPoolsInfoBetween(fromTimestamp - 604800, fromTimestamp);
const totalBlocks = pools.reduce((acc, pool) => acc + pool.blockCount, 0);
pools = pools.map((pool: any) => {
pool.hashrate = (pool.blockCount / totalBlocks) * lastBlockHashrate;
pool.share = (pool.blockCount / totalBlocks);
return pool;
});
for (const pool of pools) {
hashrates.push({
hashrateTimestamp: fromTimestamp + 1,
avgHashrate: pool['hashrate'],
poolId: pool.poolId,
share: pool['share'],
type: 'weekly',
});
}
}
hashrates.push({
hashrateTimestamp: fromTimestamp,
avgHashrate: lastBlockHashrate,
poolId: null,
share: 1,
type: 'daily',
});
if (hashrates.length > 10) {
await HashratesRepository.$saveHashrates(hashrates);
hashrates.length = 0;
}
const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
if (elapsedSeconds > 5) {
const daysPerSeconds = (indexedThisRun / elapsedSeconds).toFixed(2);
const formattedDate = new Date(fromTimestamp * 1000).toUTCString();
const daysLeft = Math.round(totalDayIndexed - totalIndexed);
logger.debug(`Getting hashrate for ${formattedDate} | ~${daysPerSeconds} days/sec | ~${daysLeft} days left to index`);
startedAt = new Date().getTime() / 1000;
indexedThisRun = 0;
}
toTimestamp -= 86400;
++indexedThisRun;
++totalIndexed;
}
// Add genesis block manually
if (toTimestamp <= genesisTimestamp && !indexedTimestamp.includes(genesisTimestamp)) {
hashrates.push({
hashrateTimestamp: genesisTimestamp,
avgHashrate: await bitcoinClient.getNetworkHashPs(1, 1),
poolId: null,
type: 'daily',
});
}
if (hashrates.length > 0) {
await HashratesRepository.$saveHashrates(hashrates);
}
await HashratesRepository.$setLatestRunTimestamp();
this.hashrateIndexingStarted = false;
logger.info(`Hashrates indexing completed`);
}
}