From 4b9bfd6ca07654a5af01954f1a4f65bc60382238 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Tue, 18 Jan 2022 17:37:04 +0900 Subject: [PATCH] Basic block indexing WIP - Default mining pool icon - Only show mining hashrate on 1d scale --- backend/src/api/blocks.ts | 47 +++++++++---- backend/src/api/database-migration.ts | 4 +- backend/src/api/mining.ts | 33 ++++----- backend/src/index.ts | 8 ++- backend/src/repositories/BlocksRepository.ts | 49 +++++++++---- backend/src/repositories/PoolsRepository.ts | 2 +- .../pool-ranking/pool-ranking.component.html | 8 +-- .../pool-ranking/pool-ranking.component.ts | 20 +++--- .../src/app/interfaces/node-api.interface.ts | 2 + frontend/src/app/services/mining.service.ts | 53 +++++++++++++- .../src/resources/mining-pools/default.svg | 69 +++++++++++++++++++ .../src/resources/mining-pools/unknown.svg | 7 -- 12 files changed, 230 insertions(+), 72 deletions(-) create mode 100644 frontend/src/resources/mining-pools/default.svg delete mode 100644 frontend/src/resources/mining-pools/unknown.svg diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 4f1e17101..eda348ad8 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -7,10 +7,10 @@ import { Common } from './common'; import diskCache from './disk-cache'; import transactionUtils from './transaction-utils'; import bitcoinClient from './bitcoin/bitcoin-client'; -import { DB } from '../database'; import { IEsploraApi } from './bitcoin/esplora-api.interface'; import poolsRepository from '../repositories/PoolsRepository'; import blocksRepository from '../repositories/BlocksRepository'; +import BitcoinApi from './bitcoin/bitcoin-api'; class Blocks { private blocks: BlockExtended[] = []; @@ -146,22 +146,41 @@ class Blocks { * Index all blocks metadata for the mining dashboard */ public async $generateBlockDatabase() { - let currentBlockHeight = await bitcoinApi.$getBlockHeightTip(); - let maxBlocks = 1008*2; // tmp + let currentBlockHeight = await bitcoinClient.getBlockCount(); + const indexedBlockCount = await blocksRepository.$blockCount(); - while (currentBlockHeight-- > 0 && maxBlocks-- > 0) { - if (await blocksRepository.$isBlockAlreadyIndexed(currentBlockHeight)) { - // logger.debug(`Block #${currentBlockHeight} already indexed, skipping`); + logger.info(`Starting block indexing. Current tip at block #${currentBlockHeight}`); + logger.info(`Need to index ${currentBlockHeight - indexedBlockCount} blocks. Working on it!`); + + const chunkSize = 10000; + while (currentBlockHeight >= 0) { + const endBlock = Math.max(0, currentBlockHeight - chunkSize + 1); + const missingBlockHeights: number[] = await blocksRepository.$getMissingBlocksBetweenHeights( + currentBlockHeight, endBlock); + if (missingBlockHeights.length <= 0) { + logger.debug(`No missing blocks between #${currentBlockHeight} to #${endBlock}, moving on`); + currentBlockHeight -= chunkSize; continue; } - logger.debug(`Indexing block #${currentBlockHeight}`); - const blockHash = await bitcoinApi.$getBlockHash(currentBlockHeight); - const block = await bitcoinApi.$getBlock(blockHash); - const transactions = await this.$getTransactionsExtended(blockHash, block.height, true); - const blockExtended = this.getBlockExtended(block, transactions); - const miner = await this.$findBlockMiner(blockExtended.coinbaseTx); - const coinbase: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(transactions[0].txid, true); - await blocksRepository.$saveBlockInDatabase(blockExtended, blockHash, coinbase.hex, miner); + + logger.info(`Indexing ${chunkSize} blocks from #${currentBlockHeight} to #${endBlock}`); + + for (const blockHeight of missingBlockHeights) { + try { + logger.debug(`Indexing block #${blockHeight}`); + const blockHash = await bitcoinApi.$getBlockHash(blockHeight); + const block = await bitcoinApi.$getBlock(blockHash); + const transactions = await this.$getTransactionsExtended(blockHash, block.height, true); + const blockExtended = this.getBlockExtended(block, transactions); + const miner = await this.$findBlockMiner(blockExtended.coinbaseTx); + const coinbase: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(transactions[0].txid, true); + await blocksRepository.$saveBlockInDatabase(blockExtended, blockHash, coinbase.hex, miner); + } catch (e) { + logger.err(`Something went wrong while indexing blocks.` + e); + } + } + + currentBlockHeight -= chunkSize; } } diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 69807b414..c08b36ce1 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -6,7 +6,7 @@ import logger from '../logger'; const sleep = (ms: number) => new Promise(res => setTimeout(res, ms)); class DatabaseMigration { - private static currentVersion = 3; + private static currentVersion = 4; private queryTimeout = 120000; private statisticsAddedIndexed = false; @@ -85,6 +85,8 @@ class DatabaseMigration { } if (databaseSchemaVersion < 3) { await this.$executeQuery(connection, this.getCreatePoolsTableQuery(), await this.$checkIfTableExists('pools')); + } + if (databaseSchemaVersion < 4) { await this.$executeQuery(connection, this.getCreateBlocksTableQuery(), await this.$checkIfTableExists('blocks')); } connection.release(); diff --git a/backend/src/api/mining.ts b/backend/src/api/mining.ts index 6582221a5..b6d3d8750 100644 --- a/backend/src/api/mining.ts +++ b/backend/src/api/mining.ts @@ -1,33 +1,30 @@ -import { PoolInfo, PoolStats } from "../mempool.interfaces"; -import BlocksRepository, { EmptyBlocks } from "../repositories/BlocksRepository"; -import PoolsRepository from "../repositories/PoolsRepository"; -import bitcoinClient from "./bitcoin/bitcoin-client"; -import BitcoinApi from "./bitcoin/bitcoin-api"; +import { PoolInfo, PoolStats } from '../mempool.interfaces'; +import BlocksRepository, { EmptyBlocks } from '../repositories/BlocksRepository'; +import PoolsRepository from '../repositories/PoolsRepository'; +import bitcoinClient from './bitcoin/bitcoin-client'; +import BitcoinApi from './bitcoin/bitcoin-api'; class Mining { - private bitcoinApi: BitcoinApi; - constructor() { - this.bitcoinApi = new BitcoinApi(bitcoinClient); } /** * Generate high level overview of the pool ranks and general stats */ - public async $getPoolsStats(interval: string = "100 YEAR") : Promise { - let poolsStatistics = {}; + public async $getPoolsStats(interval: string = '100 YEAR') : Promise { + const poolsStatistics = {}; - const blockHeightTip = await this.bitcoinApi.$getBlockHeightTip(); - const lastBlockHashrate = await this.bitcoinApi.$getEstimatedHashrate(blockHeightTip); + const blockHeightTip = await bitcoinClient.getBlockCount(); + const lastBlockHashrate = await bitcoinClient.getNetworkHashPs(120, blockHeightTip); const poolsInfo: PoolInfo[] = await PoolsRepository.$getPoolsInfo(interval); const blockCount: number = await BlocksRepository.$blockCount(interval); const emptyBlocks: EmptyBlocks[] = await BlocksRepository.$countEmptyBlocks(interval); - let poolsStats: PoolStats[] = []; + const poolsStats: PoolStats[] = []; let rank = 1; poolsInfo.forEach((poolInfo: PoolInfo) => { - let poolStat: PoolStats = { + const poolStat: PoolStats = { poolId: poolInfo.poolId, // mysql row id name: poolInfo.name, link: poolInfo.link, @@ -41,11 +38,11 @@ class Mining { } } poolsStats.push(poolStat); - }) + }); - poolsStatistics["blockCount"] = blockCount; - poolsStatistics["lastEstimatedHashrate"] = lastBlockHashrate; - poolsStatistics["pools"] = poolsStats; + poolsStatistics['blockCount'] = blockCount; + poolsStatistics['lastEstimatedHashrate'] = lastBlockHashrate; + poolsStatistics['pools'] = poolsStats; return poolsStatistics; } diff --git a/backend/src/index.ts b/backend/src/index.ts index bf3abf01f..4839322d4 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -26,6 +26,7 @@ import poolsParser from './api/pools-parser'; import syncAssets from './sync-assets'; import icons from './api/liquid/icons'; import { Common } from './api/common'; +import bitcoinClient from './api/bitcoin/bitcoin-client'; class Server { private wss: WebSocket.Server | undefined; @@ -139,10 +140,13 @@ class Server { await blocks.$updateBlocks(); await memPool.$updateMempool(); - if (this.blockIndexingStarted === false/* && memPool.isInSync()*/) { + const blockchainInfo = await bitcoinClient.getBlockchainInfo(); + if (this.blockIndexingStarted === false + && memPool.isInSync() + && blockchainInfo.blocks === blockchainInfo.headers + ) { blocks.$generateBlockDatabase(); this.blockIndexingStarted = true; - logger.info("START OLDER BLOCK INDEXING"); } setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS); diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 9802a0a74..bbc701215 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -1,10 +1,10 @@ -import { BlockExtended, PoolTag } from "../mempool.interfaces"; -import { DB } from "../database"; -import logger from "../logger"; +import { BlockExtended, PoolTag } from '../mempool.interfaces'; +import { DB } from '../database'; +import logger from '../logger'; export interface EmptyBlocks { - emptyBlocks: number, - poolId: number, + emptyBlocks: number; + poolId: number; } class BlocksRepository { @@ -21,9 +21,9 @@ class BlocksRepository { try { const query = `INSERT INTO blocks( - height, hash, timestamp, size, - weight, tx_count, coinbase_raw, difficulty, - pool_id, fees, fee_span, median_fee + height, hash, blockTimestamp, size, + weight, tx_count, coinbase_raw, difficulty, + pool_id, fees, fee_span, median_fee ) VALUE ( ?, ?, FROM_UNIXTIME(?), ?, ?, ?, ?, ?, @@ -32,8 +32,8 @@ class BlocksRepository { const params: any[] = [ block.height, blockHash, block.timestamp, block.size, - block.weight, block.tx_count, coinbaseHex ? coinbaseHex : "", block.difficulty, - poolTag.id, 0, "[]", block.medianFee, + block.weight, block.tx_count, coinbaseHex ? coinbaseHex : '', block.difficulty, + poolTag.id, 0, '[]', block.medianFee, ]; await connection.query(query, params); @@ -61,15 +61,36 @@ class BlocksRepository { return exists; } + /** + * Get all block height that have not been indexed between [startHeight, endHeight] + */ + public async $getMissingBlocksBetweenHeights(startHeight: number, endHeight: number): Promise { + const connection = await DB.pool.getConnection(); + const [rows] : any[] = await connection.query(` + SELECT height + FROM blocks + WHERE height <= ${startHeight} AND height >= ${endHeight} + ORDER BY height DESC; + `); + connection.release(); + + const indexedBlockHeights: number[] = []; + rows.forEach((row: any) => { indexedBlockHeights.push(row.height); }); + const seekedBlocks: number[] = Array.from(Array(startHeight - endHeight + 1).keys(), n => n + endHeight).reverse(); + const missingBlocksHeights = seekedBlocks.filter(x => indexedBlockHeights.indexOf(x) === -1); + + return missingBlocksHeights; + } + /** * Count empty blocks for all pools */ - public async $countEmptyBlocks(interval: string = "100 YEAR") : Promise { + public async $countEmptyBlocks(interval: string = '100 YEAR'): Promise { const connection = await DB.pool.getConnection(); const [rows] = await connection.query(` SELECT pool_id as poolId FROM blocks - WHERE timestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW() + WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW() AND tx_count = 1; `); connection.release(); @@ -80,12 +101,12 @@ class BlocksRepository { /** * Get blocks count for a period */ - public async $blockCount(interval: string = "100 YEAR") : Promise { + public async $blockCount(interval: string = '100 YEAR'): Promise { const connection = await DB.pool.getConnection(); const [rows] = await connection.query(` SELECT count(height) as blockCount FROM blocks - WHERE timestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW(); + WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW(); `); connection.release(); diff --git a/backend/src/repositories/PoolsRepository.ts b/backend/src/repositories/PoolsRepository.ts index bf3ee18e0..b2a61dc19 100644 --- a/backend/src/repositories/PoolsRepository.ts +++ b/backend/src/repositories/PoolsRepository.ts @@ -31,7 +31,7 @@ class PoolsRepository { SELECT COUNT(height) as blockCount, pool_id as poolId, pools.name as name, pools.link as link FROM blocks JOIN pools on pools.id = pool_id - WHERE timestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW() + WHERE blocks.blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW() GROUP BY pool_id ORDER BY COUNT(height) DESC; `); diff --git a/frontend/src/app/components/pool-ranking/pool-ranking.component.html b/frontend/src/app/components/pool-ranking/pool-ranking.component.html index 195f4b6f6..2a48b045b 100644 --- a/frontend/src/app/components/pool-ranking/pool-ranking.component.html +++ b/frontend/src/app/components/pool-ranking/pool-ranking.component.html @@ -50,7 +50,7 @@ Rank Name - Hashrate + Hashrate Block Count (%) Empty Blocks (%) @@ -59,15 +59,15 @@ - All miners - {{ miningStats.lastEstimatedHashrate}} PH/s + {{ miningStats.lastEstimatedHashrate}} {{ miningStats.miningUnits.hashrateUnit }} {{ miningStats.blockCount }} {{ miningStats.totalEmptyBlock }} ({{ miningStats.totalEmptyBlockRatio }}%) {{ pool.rank }} - + {{ pool.name }} - {{ pool.lastEstimatedHashrate }} PH/s + {{ pool.lastEstimatedHashrate }} {{ miningStats.miningUnits.hashrateUnit }} {{ pool.blockCount }} ({{ pool.share }}%) {{ pool.emptyBlocks }} ({{ pool.emptyBlockRatio }}%) diff --git a/frontend/src/app/components/pool-ranking/pool-ranking.component.ts b/frontend/src/app/components/pool-ranking/pool-ranking.component.ts index 8dc1ab0fc..b193c5690 100644 --- a/frontend/src/app/components/pool-ranking/pool-ranking.component.ts +++ b/frontend/src/app/components/pool-ranking/pool-ranking.component.ts @@ -2,10 +2,9 @@ import { Component, OnDestroy, OnInit } from '@angular/core'; import { FormBuilder, FormGroup } from '@angular/forms'; import { EChartsOption } from 'echarts'; import { BehaviorSubject, Subscription } from 'rxjs'; -import { MiningStats } from 'src/app/interfaces/node-api.interface'; import { StateService } from 'src/app/services/state.service'; import { StorageService } from 'src/app/services/storage.service'; -import { MiningService } from '../../services/mining.service'; +import { MiningService, MiningStats } from '../../services/mining.service'; @Component({ selector: 'app-pool-ranking', @@ -70,7 +69,7 @@ export class PoolRankingComponent implements OnInit, OnDestroy { watchBlocks() { this.blocksSubscription = this.stateService.blocks$ - .subscribe(([block]) => { + .subscribe(() => { if (!this.miningStats) { return; } @@ -98,9 +97,14 @@ export class PoolRankingComponent implements OnInit, OnDestroy { label: { color: '#FFFFFF' }, tooltip: { formatter: () => { - return `${pool.name}
` + - pool.lastEstimatedHashrate.toString() + ' PH/s (' + pool.share + `%) -
(` + pool.blockCount.toString() + ` blocks)`; + if (this.poolsWindowPreference === '1d') { + return `${pool.name}
` + + pool.lastEstimatedHashrate.toString() + ' PH/s (' + pool.share + `%) +
(` + pool.blockCount.toString() + ` blocks)`; + } else { + return `${pool.name}
` + + pool.blockCount.toString() + ` blocks`; + } } } }); @@ -111,8 +115,8 @@ export class PoolRankingComponent implements OnInit, OnDestroy { prepareChartOptions() { this.chartOptions = { title: { - text: 'Hashrate distribution', - subtext: 'Estimated from the # of blocks mined', + text: (this.poolsWindowPreference === '1d') ? 'Hashrate distribution' : 'Block distribution', + subtext: (this.poolsWindowPreference === '1d') ? 'Estimated from the # of blocks mined' : null, left: 'center', textStyle: { color: '#FFFFFF', diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index 265ad59d9..ba0adc77f 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -71,6 +71,8 @@ export interface PoolsStats { pools: SinglePoolStats[], } +export interface ITranslators { [language: string]: string; } + export interface MiningStats { lastEstimatedHashrate: string, blockCount: number, diff --git a/frontend/src/app/services/mining.service.ts b/frontend/src/app/services/mining.service.ts index f388975a8..00cffbca1 100644 --- a/frontend/src/app/services/mining.service.ts +++ b/frontend/src/app/services/mining.service.ts @@ -1,8 +1,23 @@ import { Injectable } from '@angular/core'; import { Observable } from 'rxjs'; import { map } from 'rxjs/operators'; -import { MiningStats, PoolsStats } from '../interfaces/node-api.interface'; +import { PoolsStats, SinglePoolStats } from '../interfaces/node-api.interface'; import { ApiService } from '../services/api.service'; +import { StateService } from './state.service'; + +export interface MiningUnits { + hashrateDivider: number, + hashrateUnit: string, +} + +export interface MiningStats { + lastEstimatedHashrate: string, + blockCount: number, + totalEmptyBlock: number, + totalEmptyBlockRatio: string, + pools: SinglePoolStats[], + miningUnits: MiningUnits, +} @Injectable({ providedIn: 'root' @@ -10,6 +25,7 @@ import { ApiService } from '../services/api.service'; export class MiningService { constructor( + private stateService: StateService, private apiService: ApiService, ) { } @@ -19,7 +35,37 @@ export class MiningService { ); } + /** + * Set the hashrate power of ten we want to display + */ + public getMiningUnits() : MiningUnits { + const powerTable = { + 0: "H/s", + 3: "kH/s", + 6: "MH/s", + 9: "GH/s", + 12: "TH/s", + 15: "PH/s", + 18: "EH/s", + }; + + // I think it's fine to hardcode this since we don't have x1000 hashrate jump everyday + // If we want to support the mining dashboard for testnet, we can hardcode it too + let selectedPower = 15; + if (this.stateService.network === 'testnet') { + selectedPower = 12; + } + + return { + hashrateDivider: Math.pow(10, selectedPower), + hashrateUnit: powerTable[selectedPower], + }; + } + private generateMiningStats(stats: PoolsStats) : MiningStats { + const miningUnits = this.getMiningUnits(); + const hashrateDivider = miningUnits.hashrateDivider; + const totalEmptyBlock = Object.values(stats.pools).reduce((prev, cur) => { return prev + cur.emptyBlocks; }, 0); @@ -27,7 +73,7 @@ export class MiningService { const poolsStats = stats.pools.map((poolStat) => { return { share: (poolStat.blockCount / stats.blockCount * 100).toFixed(2), - lastEstimatedHashrate: (poolStat.blockCount / stats.blockCount * stats.lastEstimatedHashrate / Math.pow(10, 15)).toFixed(2), + lastEstimatedHashrate: (poolStat.blockCount / stats.blockCount * stats.lastEstimatedHashrate / hashrateDivider).toFixed(2), emptyBlockRatio: (poolStat.emptyBlocks / poolStat.blockCount * 100).toFixed(2), logo: `./resources/mining-pools/` + poolStat.name.toLowerCase().replace(' ', '').replace('.', '') + '.svg', ...poolStat @@ -35,11 +81,12 @@ export class MiningService { }); return { - lastEstimatedHashrate: (stats.lastEstimatedHashrate / Math.pow(10, 15)).toFixed(2), + lastEstimatedHashrate: (stats.lastEstimatedHashrate / hashrateDivider).toFixed(2), blockCount: stats.blockCount, totalEmptyBlock: totalEmptyBlock, totalEmptyBlockRatio: totalEmptyBlockRatio, pools: poolsStats, + miningUnits: miningUnits, }; } } diff --git a/frontend/src/resources/mining-pools/default.svg b/frontend/src/resources/mining-pools/default.svg new file mode 100644 index 000000000..84496f899 --- /dev/null +++ b/frontend/src/resources/mining-pools/default.svg @@ -0,0 +1,69 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/frontend/src/resources/mining-pools/unknown.svg b/frontend/src/resources/mining-pools/unknown.svg deleted file mode 100644 index d6661188a..000000000 --- a/frontend/src/resources/mining-pools/unknown.svg +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - -