From 1a22923cd8e5a21bf0c164fb11dcc74162bcc807 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Thu, 6 Jan 2022 02:26:07 +0900 Subject: [PATCH 01/24] Migrate pools.json to the database in one command - Updated latest pools.json file from Blockchain-Known-Pools master --- backend/package.json | 3 +- backend/src/api/database-migration.ts | 2 +- backend/src/index.ts | 1 + frontend/cypress/fixtures/pools.json | 72 ++++++++++++++++++++++++--- 4 files changed, 70 insertions(+), 8 deletions(-) diff --git a/backend/package.json b/backend/package.json index a5c9a32ff..594e25427 100644 --- a/backend/package.json +++ b/backend/package.json @@ -25,7 +25,8 @@ "build": "npm run tsc", "start": "node --max-old-space-size=2048 dist/index.js", "start-production": "node --max-old-space-size=4096 dist/index.js", - "test": "echo \"Error: no test specified\" && exit 1" + "test": "echo \"Error: no test specified\" && exit 1", + "migrate-pools": "npm run tsc ; node dist/api/pools-parser.js" }, "dependencies": { "@mempool/bitcoin": "^3.0.3", diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index a375b7bf4..efdf6755c 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -350,4 +350,4 @@ class DatabaseMigration { } } -export default new DatabaseMigration(); \ No newline at end of file +export default new DatabaseMigration(); diff --git a/backend/src/index.ts b/backend/src/index.ts index 9e4dcee35..420c60365 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -25,6 +25,7 @@ import databaseMigration from './api/database-migration'; import poolsParser from './api/pools-parser'; import syncAssets from './sync-assets'; import icons from './api/liquid/icons'; +import poolsParser from './api/pools-parser'; import { Common } from './api/common'; class Server { diff --git a/frontend/cypress/fixtures/pools.json b/frontend/cypress/fixtures/pools.json index ab198b6d0..55c91a99a 100644 --- a/frontend/cypress/fixtures/pools.json +++ b/frontend/cypress/fixtures/pools.json @@ -488,10 +488,14 @@ "name" : "Binance Pool", "link" : "https://pool.binance.com/" }, - "/Minerium.com/" : { + "/Mined in the USA by: /Minerium.com/" : { "name" : "Minerium", "link" : "https://www.minerium.com/" }, + "/Minerium.com/" : { + "name" : "Minerium", + "link" : "https://www.minerium.com/" + }, "/Buffett/": { "name" : "Lubian.com", "link" : "" @@ -504,15 +508,15 @@ "name" : "OKKONG", "link" : "https://hash.okkong.com" }, - "/TMSPOOL/" : { - "name" : "TMSPool", + "/AAOPOOL/" : { + "name" : "AAO Pool", "link" : "https://btc.tmspool.top" }, "/one_more_mcd/" : { "name" : "EMCDPool", "link" : "https://pool.emcd.io" }, - "/Foundry USA Pool #dropgold/" : { + "Foundry USA Pool" : { "name" : "Foundry USA", "link" : "https://foundrydigital.com/" }, @@ -539,9 +543,29 @@ "/PureBTC.COM/": { "name": "PureBTC.COM", "link": "https://purebtc.com" + }, + "MARA Pool": { + "name": "MARA Pool", + "link": "https://marapool.com" + }, + "KuCoinPool": { + "name": "KuCoinPool", + "link": "https://www.kucoin.com/mining-pool/" + }, + "Entrustus" : { + "name": "Entrust Charity Pool", + "link": "pool.entustus.org" } }, "payout_addresses" : { + "1MkCDCzHpBsYQivp8MxjY5AkTGG1f2baoe": { + "name": "Luxor", + "link": "https://mining.luxor.tech" + }, + "1ArTPjj6pV3aNRhLPjJVPYoxB98VLBzUmb": { + "name" : "KuCoinPool", + "link" : "https://www.kucoin.com/mining-pool/" + }, "3Bmb9Jig8A5kHdDSxvDZ6eryj3AXd3swuJ": { "name" : "NovaBlock", "link" : "https://novablock.com" @@ -606,7 +630,7 @@ "name" : "BitMinter", "link" : "http://bitminter.com/" }, - "15xiShqUqerfjFdyfgBH1K7Gwp6cbYmsTW " : { + "15xiShqUqerfjFdyfgBH1K7Gwp6cbYmsTW" : { "name" : "EclipseMC", "link" : "https://eclipsemc.com/" }, @@ -634,6 +658,14 @@ "name" : "Huobi.pool", "link" : "https://www.hpt.com/" }, + "1BDbsWi3Mrcjp1wdop3PWFNCNZtu4R7Hjy" : { + "name" : "EMCDPool", + "link" : "https://pool.emcd.io" + }, + "12QVFmJH2b4455YUHkMpEnWLeRY3eJ4Jb5" : { + "name" : "AAO Pool", + "link" : "https://btc.tmspool.top " + }, "1ALA5v7h49QT7WYLcRsxcXqXUqEqaWmkvw" : { "name" : "CloudHashing", "link" : "https://cloudhashing.com/" @@ -934,6 +966,22 @@ "name" : "Poolin", "link" : "https://www.poolin.com/" }, + "1E8CZo2S3CqWg1VZSJNFCTbtT8hZPuQ2kB" : { + "name" : "Poolin", + "link" : "https://www.poolin.com/" + }, + "14sA8jqYQgMRQV9zUtGFvpeMEw7YDn77SK" : { + "name" : "Poolin", + "link" : "https://www.poolin.com/" + }, + "1GNgwA8JfG7Kc8akJ8opdNWJUihqUztfPe" : { + "name" : "Poolin", + "link" : "https://www.poolin.com/" + }, + "17tUZLvy3X2557JGhceXRiij2TNYuhRr4r" : { + "name" : "Poolin", + "link" : "https://www.poolin.com/" + }, "12Taz8FFXQ3E2AGn3ZW1SZM5bLnYGX4xR6" : { "name" : "Tangpool", "link" : "http://www.tangpool.com/" @@ -1126,6 +1174,10 @@ "name" : "Binance Pool", "link" : "https://pool.binance.com/" }, + "1JvXhnHCi6XqcanvrZJ5s2Qiv4tsmm2UMy": { + "name" : "Binance Pool", + "link" : "https://pool.binance.com/" + }, "34Jpa4Eu3ApoPVUKNTN2WeuXVVq1jzxgPi": { "name" : "Lubian.com", "link" : "http://www.lubian.com/" @@ -1173,6 +1225,14 @@ "3CLigLYNkrtoNgNcUwTaKoUSHCwr9W851W": { "name": "Rawpool", "link": "https://www.rawpool.com" + }, + "bc1qf274x7penhcd8hsv3jcmwa5xxzjl2a6pa9pxwm": { + "name" : "F2Pool", + "link" : "https://www.f2pool.com/" + }, + "1A32KFEX7JNPmU1PVjrtiXRrTQcesT3Nf1": { + "name": "MARA Pool", + "link": "https://marapool.com" } } -} +} \ No newline at end of file From 031f69a403e69220d12d7388b8b5fcaafb30597b Mon Sep 17 00:00:00 2001 From: nymkappa Date: Mon, 3 Jan 2022 11:47:04 +0900 Subject: [PATCH 02/24] Add backend README - Backend watchers setup --- backend/README.md | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 backend/README.md diff --git a/backend/README.md b/backend/README.md new file mode 100644 index 000000000..a6d0fb60f --- /dev/null +++ b/backend/README.md @@ -0,0 +1,33 @@ +# Setup backend watchers + +The backend is static. Typescript scripts are compiled into the `dist` folder and served through a node web server. + +You can avoid the manual shutdown/recompile/restart command line cycle by setting up watchers. + +Make sure you are in the `backend` directory `cd backend`. + +1. Install nodemon +``` +sudo npm install -g nodemon +``` +2. [Optional] Add the following configuration into `tsconfig.json`. You can find watch options here https://www.typescriptlang.org/docs/handbook/configuring-watch.html +``` + "watchOptions": { + "watchFile": "useFsEvents", + "watchDirectory": "useFsEvents", + "fallbackPolling": "dynamicPriority", + "synchronousWatchDirectory": true, + "excludeDirectories": ["**/node_modules", "_build"], + "excludeFiles": ["build/fileWhichChangesOften.ts"] + } +``` +3. In one terminal, watch typescript scripts +``` +./node_modules/typescript/bin/tsc --watch +``` +4. In another terminal, watch compiled javascript +``` +nodemon --max-old-space-size=2048 dist/index.js +``` + +Everytime you save a backend `.ts` file, `tsc` will recompile it and genereate a new static `.js` file in the `dist` folder. `nodemon` will detect this new file and restart the node web server automatically. From 37031ec9139be270d7e93ae4ef6354700c3f7280 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Wed, 5 Jan 2022 15:41:14 +0900 Subject: [PATCH 03/24] Refactor blocks.ts and index 10k block headers at launch --- backend/src/api/blocks.ts | 183 +++++++++++++++---- backend/src/api/mempool.ts | 10 +- backend/src/api/transaction-utils.ts | 8 + backend/src/index.ts | 10 +- backend/src/mempool.interfaces.ts | 8 + backend/src/repositories/BlocksRepository.ts | 73 ++++++++ backend/src/repositories/PoolsRepository.ts | 30 +++ 7 files changed, 275 insertions(+), 47 deletions(-) create mode 100644 backend/src/repositories/BlocksRepository.ts create mode 100644 backend/src/repositories/PoolsRepository.ts diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 1043c344f..c9bb46754 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -2,11 +2,15 @@ import config from '../config'; import bitcoinApi from './bitcoin/bitcoin-api-factory'; import logger from '../logger'; import memPool from './mempool'; -import { BlockExtended, TransactionExtended } from '../mempool.interfaces'; +import { BlockExtended, PoolTag, TransactionExtended, TransactionMinerInfo } from '../mempool.interfaces'; import { Common } from './common'; import diskCache from './disk-cache'; import transactionUtils from './transaction-utils'; import bitcoinClient from './bitcoin/bitcoin-client'; +import { DB } from '../database'; +import { IEsploraApi } from './bitcoin/esplora-api.interface'; +import poolsRepository from '../repositories/PoolsRepository'; +import blocksRepository from '../repositories/BlocksRepository'; class Blocks { private blocks: BlockExtended[] = []; @@ -30,6 +34,137 @@ class Blocks { this.newBlockCallbacks.push(fn); } + /** + * Return the list of transaction for a block + * @param blockHash + * @param blockHeight + * @param onlyCoinbase - Set to true if you only need the coinbase transaction + * @returns Promise + */ + private async $getTransactionsExtended(blockHash: string, blockHeight: number, onlyCoinbase: boolean) : Promise { + const transactions: TransactionExtended[] = []; + const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash); + + const mempool = memPool.getMempool(); + let transactionsFound = 0; + let transactionsFetched = 0; + + for (let i = 0; i < txIds.length; i++) { + if (mempool[txIds[i]]) { + // We update blocks before the mempool (index.ts), therefore we can + // optimize here by directly fetching txs in the "outdated" mempool + transactions.push(mempool[txIds[i]]); + transactionsFound++; + } else if (config.MEMPOOL.BACKEND === 'esplora' || memPool.isInSync() || i === 0) { + // Otherwise we fetch the tx data through backend services (esplora, electrum, core rpc...) + if (i % (Math.round((txIds.length) / 10)) === 0 || i + 1 === txIds.length) { // Avoid log spam + logger.debug(`Indexing tx ${i + 1} of ${txIds.length} in block #${blockHeight}`); + } + try { + const tx = await transactionUtils.$getTransactionExtended(txIds[i]); + transactions.push(tx); + transactionsFetched++; + } catch (e) { + logger.debug('Error fetching block tx: ' + (e instanceof Error ? e.message : e)); + if (i === 0) { + throw new Error('Failed to fetch Coinbase transaction: ' + txIds[i]); + } + } + } + + if (onlyCoinbase === true) { + break; // Fetch the first transaction and exit + } + } + + transactions.forEach((tx) => { + if (!tx.cpfpChecked) { + Common.setRelativesAndGetCpfpInfo(tx, mempool); // Child Pay For Parent + } + }); + + logger.debug(`${transactionsFound} of ${txIds.length} found in mempool. ${transactionsFetched} fetched through backend service.`); + + return transactions; + } + + /** + * Return a block with additional data (reward, coinbase, fees...) + * @param block + * @param transactions + * @returns BlockExtended + */ + private getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]) : BlockExtended { + const blockExtended: BlockExtended = Object.assign({}, block); + blockExtended.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0); + blockExtended.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]); + + const transactionsTmp = [...transactions]; + transactionsTmp.shift(); + transactionsTmp.sort((a, b) => b.effectiveFeePerVsize - a.effectiveFeePerVsize); + blockExtended.medianFee = transactionsTmp.length > 0 ? Common.median(transactionsTmp.map((tx) => tx.effectiveFeePerVsize)) : 0; + blockExtended.feeRange = transactionsTmp.length > 0 ? Common.getFeesInRange(transactionsTmp, 8) : [0, 0]; + + return blockExtended; + } + + /** + * Try to find which miner found the block + * @param txMinerInfo + * @returns + */ + private async $findBlockMiner(txMinerInfo: TransactionMinerInfo | undefined) : Promise { + if (txMinerInfo === undefined) { + return poolsRepository.getUnknownPool(); + } + + const asciiScriptSig = transactionUtils.hex2ascii(txMinerInfo.vin[0].scriptsig); + const address = txMinerInfo.vout[0].scriptpubkey_address; + + const pools: PoolTag[] = await poolsRepository.$getPools(); + for (let i = 0; i < pools.length; ++i) { + if (address !== undefined) { + let addresses: string[] = JSON.parse(pools[i].addresses); + if (addresses.indexOf(address) !== -1) { + return pools[i]; + } + } + + let regexes: string[] = JSON.parse(pools[i].regexes); + for (let y = 0; y < regexes.length; ++y) { + let match = asciiScriptSig.match(regexes[y]); + if (match !== null) { + return pools[i]; + } + } + } + + return poolsRepository.getUnknownPool(); + } + + /** + * Index all blocks metadata for the mining dashboard + */ + public async $generateBlockDatabase() { + let currentBlockHeight = await bitcoinApi.$getBlockHeightTip(); + let maxBlocks = 100; // tmp + + while (currentBlockHeight-- > 0 && maxBlocks-- > 0) { + if (await blocksRepository.$isBlockAlreadyIndexed(currentBlockHeight)) { + // logger.debug(`Block #${currentBlockHeight} already indexed, skipping`); + continue; + } + logger.debug(`Indexing block #${currentBlockHeight}`); + const blockHash = await bitcoinApi.$getBlockHash(currentBlockHeight); + const block = await bitcoinApi.$getBlock(blockHash); + const transactions = await this.$getTransactionsExtended(blockHash, block.height, true); + const blockExtended = this.getBlockExtended(block, transactions); + const miner = await this.$findBlockMiner(blockExtended.coinbaseTx); + const coinbase: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(transactions[0].txid, true); + await blocksRepository.$saveBlockInDatabase(blockExtended, blockHash, coinbase.hex, miner); + } + } + public async $updateBlocks() { const blockHeightTip = await bitcoinApi.$getBlockHeightTip(); @@ -70,48 +205,14 @@ class Blocks { logger.debug(`New block found (#${this.currentBlockHeight})!`); } - const transactions: TransactionExtended[] = []; - const blockHash = await bitcoinApi.$getBlockHash(this.currentBlockHeight); const block = await bitcoinApi.$getBlock(blockHash); const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash); - - const mempool = memPool.getMempool(); - let transactionsFound = 0; - - for (let i = 0; i < txIds.length; i++) { - if (mempool[txIds[i]]) { - transactions.push(mempool[txIds[i]]); - transactionsFound++; - } else if (config.MEMPOOL.BACKEND === 'esplora' || memPool.isInSync() || i === 0) { - logger.debug(`Fetching block tx ${i} of ${txIds.length}`); - try { - const tx = await transactionUtils.$getTransactionExtended(txIds[i]); - transactions.push(tx); - } catch (e) { - logger.debug('Error fetching block tx: ' + (e instanceof Error ? e.message : e)); - if (i === 0) { - throw new Error('Failed to fetch Coinbase transaction: ' + txIds[i]); - } - } - } - } - - transactions.forEach((tx) => { - if (!tx.cpfpChecked) { - Common.setRelativesAndGetCpfpInfo(tx, mempool); - } - }); - - logger.debug(`${transactionsFound} of ${txIds.length} found in mempool. ${txIds.length - transactionsFound} not found.`); - - const blockExtended: BlockExtended = Object.assign({}, block); - blockExtended.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0); - blockExtended.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]); - transactions.shift(); - transactions.sort((a, b) => b.effectiveFeePerVsize - a.effectiveFeePerVsize); - blockExtended.medianFee = transactions.length > 0 ? Common.median(transactions.map((tx) => tx.effectiveFeePerVsize)) : 0; - blockExtended.feeRange = transactions.length > 0 ? Common.getFeesInRange(transactions, 8) : [0, 0]; + const transactions = await this.$getTransactionsExtended(blockHash, block.height, false); + const blockExtended: BlockExtended = this.getBlockExtended(block, transactions); + const miner = await this.$findBlockMiner(blockExtended.coinbaseTx); + const coinbase: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(transactions[0].txid, true); + await blocksRepository.$saveBlockInDatabase(blockExtended, blockHash, coinbase.hex, miner); if (block.height % 2016 === 0) { this.previousDifficultyRetarget = (block.difficulty - this.currentDifficulty) / this.currentDifficulty * 100; @@ -130,6 +231,8 @@ class Blocks { if (memPool.isInSync()) { diskCache.$saveCacheToDisk(); } + + return; } } diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index 4d6c35860..859912f6c 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -118,11 +118,11 @@ class Mempool { }); } hasChange = true; - if (diff > 0) { - logger.debug('Fetched transaction ' + txCount + ' / ' + diff); - } else { - logger.debug('Fetched transaction ' + txCount); - } + // if (diff > 0) { + // logger.debug('Fetched transaction ' + txCount + ' / ' + diff); + // } else { + // logger.debug('Fetched transaction ' + txCount); + // } newTransactions.push(transaction); } catch (e) { logger.debug('Error finding transaction in mempool: ' + (e instanceof Error ? e.message : e)); diff --git a/backend/src/api/transaction-utils.ts b/backend/src/api/transaction-utils.ts index 1496b810b..2e669d709 100644 --- a/backend/src/api/transaction-utils.ts +++ b/backend/src/api/transaction-utils.ts @@ -44,6 +44,14 @@ class TransactionUtils { } return transactionExtended; } + + public hex2ascii(hex: string) { + let str = ''; + for (let i = 0; i < hex.length; i += 2) { + str += String.fromCharCode(parseInt(hex.substr(i, 2), 16)); + } + return str; + } } export default new TransactionUtils(); diff --git a/backend/src/index.ts b/backend/src/index.ts index 420c60365..be26d53fe 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -25,7 +25,6 @@ import databaseMigration from './api/database-migration'; import poolsParser from './api/pools-parser'; import syncAssets from './sync-assets'; import icons from './api/liquid/icons'; -import poolsParser from './api/pools-parser'; import { Common } from './api/common'; class Server { @@ -33,6 +32,7 @@ class Server { private server: http.Server | undefined; private app: Express; private currentBackendRetryInterval = 5; + private blockIndexingStarted = false; constructor() { this.app = express(); @@ -90,7 +90,6 @@ class Server { await checkDbConnection(); try { await databaseMigration.$initializeOrMigrateDatabase(); - await poolsParser.migratePoolsJson(); } catch (e) { throw new Error(e instanceof Error ? e.message : 'Error'); } @@ -139,6 +138,13 @@ class Server { } await blocks.$updateBlocks(); await memPool.$updateMempool(); + + if (this.blockIndexingStarted === false/* && memPool.isInSync()*/) { + blocks.$generateBlockDatabase(); + this.blockIndexingStarted = true; + logger.info("START OLDER BLOCK INDEXING"); + } + setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS); this.currentBackendRetryInterval = 5; } catch (e) { diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index 2604a233c..ae921f073 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -1,5 +1,13 @@ +import { RowDataPacket } from 'mysql2'; import { IEsploraApi } from './api/bitcoin/esplora-api.interface'; +export interface PoolTag extends RowDataPacket { + name: string, + link: string, + regexes: string, + addresses: string, +} + export interface MempoolBlock { blockSize: number; blockVSize: number; diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts new file mode 100644 index 000000000..f62b261a1 --- /dev/null +++ b/backend/src/repositories/BlocksRepository.ts @@ -0,0 +1,73 @@ +import { IEsploraApi } from "../api/bitcoin/esplora-api.interface"; +import { BlockExtended, PoolTag } from "../mempool.interfaces"; +import { DB } from "../database"; +import logger from "../logger"; +import bitcoinApi from '../api/bitcoin/bitcoin-api-factory'; + +class BlocksRepository { + /** + * Save indexed block data in the database + * @param block + * @param blockHash + * @param coinbaseTxid + * @param poolTag + */ + public async $saveBlockInDatabase( + block: BlockExtended, + blockHash: string, + coinbaseHex: string | undefined, + poolTag: PoolTag + ) { + const connection = await DB.pool.getConnection(); + + try { + const query = `INSERT INTO blocks( + height, hash, timestamp, size, + weight, tx_count, coinbase_raw, difficulty, + pool_id, fees, fee_span, median_fee + ) VALUE ( + ?, ?, ?, ?, + ?, ?, ?, ?, + ?, ?, ?, ? + )`; + + const params: any[] = [ + block.height, blockHash, block.timestamp, block.size, + block.weight, block.tx_count, coinbaseHex ? coinbaseHex : "", block.difficulty, + poolTag.id, 0, "[]", block.medianFee, + ]; + + await connection.query(query, params); + } catch (e) { + console.log(e); + logger.err('$updateBlocksDatabase() error' + (e instanceof Error ? e.message : e)); + } + + connection.release(); + } + + /** + * Check if a block has already been indexed in the database. Query the databse directly. + * This can be cached/optimized if required later on to avoid too many db queries. + * @param blockHeight + * @returns + */ + public async $isBlockAlreadyIndexed(blockHeight: number) { + const connection = await DB.pool.getConnection(); + let exists = false; + + try { + const query = `SELECT height from blocks where blocks.height = ${blockHeight}`; + const [rows]: any[] = await connection.query(query); + exists = rows.length === 1; + } catch (e) { + console.log(e); + logger.err('$isBlockAlreadyIndexed() error' + (e instanceof Error ? e.message : e)); + } + connection.release(); + + return exists; + } +} + +export default new BlocksRepository(); \ No newline at end of file diff --git a/backend/src/repositories/PoolsRepository.ts b/backend/src/repositories/PoolsRepository.ts new file mode 100644 index 000000000..aa6f457f2 --- /dev/null +++ b/backend/src/repositories/PoolsRepository.ts @@ -0,0 +1,30 @@ +import { FieldPacket } from "mysql2"; +import { DB } from "../database"; +import { PoolTag } from "../mempool.interfaces" + +class PoolsRepository { + /** + * Get all pools tagging info + */ + public async $getPools() : Promise { + const connection = await DB.pool.getConnection(); + const [rows]: [PoolTag[], FieldPacket[]] = await connection.query("SELECT * FROM pools;"); + connection.release(); + return rows; + } + + /** + * Get unknown pool tagging info + */ + public getUnknownPool(): PoolTag { + return { + id: null, + name: 'Unknown', + link: 'rickroll?', + regexes: "[]", + addresses: "[]", + }; + } +} + +export default new PoolsRepository(); \ No newline at end of file From bfe9f99c35951325c57d3b13197b7119f641901c Mon Sep 17 00:00:00 2001 From: nymkappa Date: Thu, 6 Jan 2022 19:59:33 +0900 Subject: [PATCH 04/24] Generate mining basic pool ranking (sorted by block found) for a specified timeframe --- backend/src/api/bitcoin/bitcoin-api.ts | 5 + backend/src/api/blocks.ts | 6 +- backend/src/api/mining.ts | 53 +++++++++ backend/src/index.ts | 1 + backend/src/mempool.interfaces.ts | 20 +++- backend/src/repositories/BlocksRepository.ts | 57 +++++++--- backend/src/repositories/PoolsRepository.ts | 41 ++++--- backend/src/routes.ts | 10 ++ frontend/server.ts | 2 +- frontend/src/app/app-routing.module.ts | 5 + frontend/src/app/app.module.ts | 2 + .../pool-ranking/pool-ranking.component.html | 68 ++++++++++++ .../pool-ranking/pool-ranking.component.ts | 103 ++++++++++++++++++ .../src/app/interfaces/node-api.interface.ts | 13 +++ frontend/src/app/services/api.service.ts | 7 +- frontend/src/app/services/storage.service.ts | 21 ++-- 16 files changed, 366 insertions(+), 48 deletions(-) create mode 100644 backend/src/api/mining.ts create mode 100644 frontend/src/app/components/pool-ranking/pool-ranking.component.html create mode 100644 frontend/src/app/components/pool-ranking/pool-ranking.component.ts diff --git a/backend/src/api/bitcoin/bitcoin-api.ts b/backend/src/api/bitcoin/bitcoin-api.ts index b0a04116f..75d3e6e8f 100644 --- a/backend/src/api/bitcoin/bitcoin-api.ts +++ b/backend/src/api/bitcoin/bitcoin-api.ts @@ -115,6 +115,11 @@ class BitcoinApi implements AbstractBitcoinApi { return outSpends; } + $getEstimatedHashrate(blockHeight: number): Promise { + // 120 is the default block span in Core + return this.bitcoindClient.getNetworkHashPs(120, blockHeight); + } + protected async $convertTransaction(transaction: IBitcoinApi.Transaction, addPrevout: boolean): Promise { let esploraTransaction: IEsploraApi.Transaction = { txid: transaction.txid, diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index c9bb46754..4f1e17101 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -115,7 +115,7 @@ class Blocks { */ private async $findBlockMiner(txMinerInfo: TransactionMinerInfo | undefined) : Promise { if (txMinerInfo === undefined) { - return poolsRepository.getUnknownPool(); + return await poolsRepository.$getUnknownPool(); } const asciiScriptSig = transactionUtils.hex2ascii(txMinerInfo.vin[0].scriptsig); @@ -139,7 +139,7 @@ class Blocks { } } - return poolsRepository.getUnknownPool(); + return await poolsRepository.$getUnknownPool(); } /** @@ -147,7 +147,7 @@ class Blocks { */ public async $generateBlockDatabase() { let currentBlockHeight = await bitcoinApi.$getBlockHeightTip(); - let maxBlocks = 100; // tmp + let maxBlocks = 1008*2; // tmp while (currentBlockHeight-- > 0 && maxBlocks-- > 0) { if (await blocksRepository.$isBlockAlreadyIndexed(currentBlockHeight)) { diff --git a/backend/src/api/mining.ts b/backend/src/api/mining.ts new file mode 100644 index 000000000..d22a29d5b --- /dev/null +++ b/backend/src/api/mining.ts @@ -0,0 +1,53 @@ +import { PoolInfo, PoolStats } from "../mempool.interfaces"; +import BlocksRepository, { EmptyBlocks } from "../repositories/BlocksRepository"; +import PoolsRepository from "../repositories/PoolsRepository"; +import bitcoinClient from "./bitcoin/bitcoin-client"; +import BitcoinApi from "./bitcoin/bitcoin-api"; + +class Mining { + private bitcoinApi: BitcoinApi; + + constructor() { + this.bitcoinApi = new BitcoinApi(bitcoinClient); + } + + /** + * Generate high level overview of the pool ranks and general stats + */ + public async $getPoolsStats(interval: string = "100 YEAR") : Promise { + let poolsStatistics = {}; + + const lastBlockHashrate = await this.bitcoinApi.$getEstimatedHashrate(717960); + const poolsInfo: PoolInfo[] = await PoolsRepository.$getPoolsInfo(interval); + const blockCount: number = await BlocksRepository.$blockCount(interval); + const emptyBlocks: EmptyBlocks[] = await BlocksRepository.$countEmptyBlocks(interval); + + let poolsStats: PoolStats[] = []; + let rank = 1; + + poolsInfo.forEach((poolInfo: PoolInfo) => { + let poolStat: PoolStats = { + poolId: poolInfo.poolId, // mysql row id + name: poolInfo.name, + link: poolInfo.link, + blockCount: poolInfo.blockCount, + rank: rank++, + emptyBlocks: 0, + } + for (let i = 0; i < emptyBlocks.length; ++i) { + if (emptyBlocks[i].poolId === poolInfo.poolId) { + poolStat.emptyBlocks++; + } + } + poolsStats.push(poolStat); + }) + + poolsStatistics["blockCount"] = blockCount; + poolsStatistics["poolsStats"] = poolsStats; + poolsStatistics["lastEstimatedHashrate"] = lastBlockHashrate; + + return poolsStatistics; + } +} + +export default new Mining(); \ No newline at end of file diff --git a/backend/src/index.ts b/backend/src/index.ts index be26d53fe..bf3abf01f 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -261,6 +261,7 @@ class Server { .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/1y', routes.$getStatisticsByTime.bind(routes, '1y')) .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/2y', routes.$getStatisticsByTime.bind(routes, '2y')) .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/3y', routes.$getStatisticsByTime.bind(routes, '3y')) + .get(config.MEMPOOL.API_URL_PREFIX + 'pools', routes.getPools) ; } diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index ae921f073..5fb83d792 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -1,11 +1,23 @@ -import { RowDataPacket } from 'mysql2'; import { IEsploraApi } from './api/bitcoin/esplora-api.interface'; -export interface PoolTag extends RowDataPacket { +export interface PoolTag { + id: number | null, // mysql row id name: string, link: string, - regexes: string, - addresses: string, + regexes: string, // JSON array + addresses: string, // JSON array +} + +export interface PoolInfo { + poolId: number, // mysql row id + name: string, + link: string, + blockCount: number, +} + +export interface PoolStats extends PoolInfo { + rank: number, + emptyBlocks: number, } export interface MempoolBlock { diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index f62b261a1..9802a0a74 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -1,16 +1,15 @@ -import { IEsploraApi } from "../api/bitcoin/esplora-api.interface"; import { BlockExtended, PoolTag } from "../mempool.interfaces"; import { DB } from "../database"; import logger from "../logger"; -import bitcoinApi from '../api/bitcoin/bitcoin-api-factory'; + +export interface EmptyBlocks { + emptyBlocks: number, + poolId: number, +} class BlocksRepository { /** * Save indexed block data in the database - * @param block - * @param blockHash - * @param coinbaseTxid - * @param poolTag */ public async $saveBlockInDatabase( block: BlockExtended, @@ -26,7 +25,7 @@ class BlocksRepository { weight, tx_count, coinbase_raw, difficulty, pool_id, fees, fee_span, median_fee ) VALUE ( - ?, ?, ?, ?, + ?, ?, FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ? )`; @@ -49,25 +48,49 @@ class BlocksRepository { /** * Check if a block has already been indexed in the database. Query the databse directly. * This can be cached/optimized if required later on to avoid too many db queries. - * @param blockHeight - * @returns */ public async $isBlockAlreadyIndexed(blockHeight: number) { const connection = await DB.pool.getConnection(); let exists = false; - try { - const query = `SELECT height from blocks where blocks.height = ${blockHeight}`; - const [rows]: any[] = await connection.query(query); - exists = rows.length === 1; - } catch (e) { - console.log(e); - logger.err('$isBlockAlreadyIndexed() error' + (e instanceof Error ? e.message : e)); - } + const query = `SELECT height from blocks where blocks.height = ${blockHeight}`; + const [rows]: any[] = await connection.query(query); + exists = rows.length === 1; connection.release(); return exists; } + + /** + * Count empty blocks for all pools + */ + public async $countEmptyBlocks(interval: string = "100 YEAR") : Promise { + const connection = await DB.pool.getConnection(); + const [rows] = await connection.query(` + SELECT pool_id as poolId + FROM blocks + WHERE timestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW() + AND tx_count = 1; + `); + connection.release(); + + return rows; + } + + /** + * Get blocks count for a period + */ + public async $blockCount(interval: string = "100 YEAR") : Promise { + const connection = await DB.pool.getConnection(); + const [rows] = await connection.query(` + SELECT count(height) as blockCount + FROM blocks + WHERE timestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW(); + `); + connection.release(); + + return rows[0].blockCount; + } } export default new BlocksRepository(); \ No newline at end of file diff --git a/backend/src/repositories/PoolsRepository.ts b/backend/src/repositories/PoolsRepository.ts index aa6f457f2..bf3ee18e0 100644 --- a/backend/src/repositories/PoolsRepository.ts +++ b/backend/src/repositories/PoolsRepository.ts @@ -1,30 +1,43 @@ -import { FieldPacket } from "mysql2"; import { DB } from "../database"; -import { PoolTag } from "../mempool.interfaces" +import { PoolInfo, PoolTag } from "../mempool.interfaces" class PoolsRepository { /** * Get all pools tagging info */ - public async $getPools() : Promise { + public async $getPools(): Promise { const connection = await DB.pool.getConnection(); - const [rows]: [PoolTag[], FieldPacket[]] = await connection.query("SELECT * FROM pools;"); + const [rows] = await connection.query("SELECT * FROM pools;"); connection.release(); - return rows; + return rows; } /** * Get unknown pool tagging info */ - public getUnknownPool(): PoolTag { - return { - id: null, - name: 'Unknown', - link: 'rickroll?', - regexes: "[]", - addresses: "[]", - }; - } + public async $getUnknownPool(): Promise { + const connection = await DB.pool.getConnection(); + const [rows] = await connection.query("SELECT * FROM pools where name = 'Unknown'"); + connection.release(); + return rows[0]; + } + + /** + * Get basic pool info and block count + */ + public async $getPoolsInfo(interval: string = "100 YEARS"): Promise { + const connection = await DB.pool.getConnection(); + const [rows] = await connection.query(` + SELECT COUNT(height) as blockCount, pool_id as poolId, pools.name as name, pools.link as link + FROM blocks + JOIN pools on pools.id = pool_id + WHERE timestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW() + GROUP BY pool_id + ORDER BY COUNT(height) DESC; + `); + connection.release(); + return rows; + } } export default new PoolsRepository(); \ No newline at end of file diff --git a/backend/src/routes.ts b/backend/src/routes.ts index 1d98c9f4e..b7956cd64 100644 --- a/backend/src/routes.ts +++ b/backend/src/routes.ts @@ -20,6 +20,7 @@ import { Common } from './api/common'; import bitcoinClient from './api/bitcoin/bitcoin-client'; import elementsParser from './api/liquid/elements-parser'; import icons from './api/liquid/icons'; +import miningStats from './api/mining'; class Routes { constructor() {} @@ -531,6 +532,15 @@ class Routes { } } + public async getPools(req: Request, res: Response) { + try { + let stats = await miningStats.$getPoolsStats(req.query.interval as string); + res.json(stats); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + public async getBlock(req: Request, res: Response) { try { const result = await bitcoinApi.$getBlock(req.params.hash); diff --git a/frontend/server.ts b/frontend/server.ts index af27fcd08..059522a05 100644 --- a/frontend/server.ts +++ b/frontend/server.ts @@ -6,7 +6,6 @@ import * as express from 'express'; import * as fs from 'fs'; import * as path from 'path'; import * as domino from 'domino'; -import { createProxyMiddleware } from 'http-proxy-middleware'; import { join } from 'path'; import { AppServerModule } from './src/main.server'; @@ -66,6 +65,7 @@ export function app(locale: string): express.Express { server.get('/mempool-block/*', getLocalizedSSR(indexHtml)); server.get('/address/*', getLocalizedSSR(indexHtml)); server.get('/blocks', getLocalizedSSR(indexHtml)); + server.get('/pools', getLocalizedSSR(indexHtml)); server.get('/graphs', getLocalizedSSR(indexHtml)); server.get('/liquid', getLocalizedSSR(indexHtml)); server.get('/liquid/tx/*', getLocalizedSSR(indexHtml)); diff --git a/frontend/src/app/app-routing.module.ts b/frontend/src/app/app-routing.module.ts index 43705b85e..19e4ee7ab 100644 --- a/frontend/src/app/app-routing.module.ts +++ b/frontend/src/app/app-routing.module.ts @@ -22,6 +22,7 @@ import { BisqMasterPageComponent } from './components/bisq-master-page/bisq-mast import { SponsorComponent } from './components/sponsor/sponsor.component'; import { LiquidMasterPageComponent } from './components/liquid-master-page/liquid-master-page.component'; import { PushTransactionComponent } from './components/push-transaction/push-transaction.component'; +import { PoolRankingComponent } from './components/pool-ranking/pool-ranking.component'; let routes: Routes = [ { @@ -58,6 +59,10 @@ let routes: Routes = [ path: 'blocks', component: LatestBlocksComponent, }, + { + path: 'pools', + component: PoolRankingComponent, + }, { path: 'graphs', component: StatisticsComponent, diff --git a/frontend/src/app/app.module.ts b/frontend/src/app/app.module.ts index 3e2c40b25..b16c5bdea 100644 --- a/frontend/src/app/app.module.ts +++ b/frontend/src/app/app.module.ts @@ -37,6 +37,7 @@ import { IncomingTransactionsGraphComponent } from './components/incoming-transa import { TimeSpanComponent } from './components/time-span/time-span.component'; import { SeoService } from './services/seo.service'; import { MempoolGraphComponent } from './components/mempool-graph/mempool-graph.component'; +import { PoolRankingComponent } from './components/pool-ranking/pool-ranking.component'; import { LbtcPegsGraphComponent } from './components/lbtc-pegs-graph/lbtc-pegs-graph.component'; import { AssetComponent } from './components/asset/asset.component'; import { AssetsComponent } from './assets/assets.component'; @@ -91,6 +92,7 @@ import { NgbModule } from '@ng-bootstrap/ng-bootstrap'; FeeDistributionGraphComponent, IncomingTransactionsGraphComponent, MempoolGraphComponent, + PoolRankingComponent, LbtcPegsGraphComponent, AssetComponent, AssetsComponent, diff --git a/frontend/src/app/components/pool-ranking/pool-ranking.component.html b/frontend/src/app/components/pool-ranking/pool-ranking.component.html new file mode 100644 index 000000000..9d38ced93 --- /dev/null +++ b/frontend/src/app/components/pool-ranking/pool-ranking.component.html @@ -0,0 +1,68 @@ +
+
+ Pools +
+
+ + + + + + + + + + +
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + +
RankNameHashrateBlock Count (share)Empty Blocks (ratio)
-All miners{{ pools["lastEstimatedHashrate"]}} PH/s{{ pools["blockCount"] }}{{ pools["totalEmptyBlock"] }} ({{ pools["totalEmptyBlockRatio"] }}%)
{{ pool.rank }}{{ pool.name }}{{ pool.lastEstimatedHashrate }} PH/s{{ pool.blockCount }} ({{ pool.share }}%){{ pool.emptyBlocks }} ({{ pool.emptyBlockRatio }}%)
+ +
diff --git a/frontend/src/app/components/pool-ranking/pool-ranking.component.ts b/frontend/src/app/components/pool-ranking/pool-ranking.component.ts new file mode 100644 index 000000000..b1841e867 --- /dev/null +++ b/frontend/src/app/components/pool-ranking/pool-ranking.component.ts @@ -0,0 +1,103 @@ +import { Component, OnInit, ChangeDetectionStrategy } from '@angular/core'; +import { FormBuilder, FormGroup } from '@angular/forms'; +import { ActivatedRoute } from '@angular/router'; +import { merge, Observable, ObservableInput, of } from 'rxjs'; +import { map, share, switchMap, tap } from 'rxjs/operators'; +import { PoolsStats } from 'src/app/interfaces/node-api.interface'; +import { StorageService } from 'src/app/services/storage.service'; +import { ApiService } from '../../services/api.service'; + +@Component({ + selector: 'app-pool-ranking', + templateUrl: './pool-ranking.component.html', + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class PoolRankingComponent implements OnInit { + pools$: Observable + + radioGroupForm: FormGroup; + poolsWindowPreference: string; + + constructor( + private formBuilder: FormBuilder, + private route: ActivatedRoute, + private apiService: ApiService, + private storageService: StorageService, + ) { } + + ngOnInit(): void { + this.poolsWindowPreference = this.storageService.getValue('poolsWindowPreference') ? this.storageService.getValue('poolsWindowPreference').trim() : '2h'; + this.radioGroupForm = this.formBuilder.group({ + dateSpan: this.poolsWindowPreference + }); + + // Setup datespan triggers + this.route.fragment.subscribe((fragment) => { + if (['1d', '3d', '1w', '1m', '3m', '6m', '1y', '2y', '3y', 'all'].indexOf(fragment) > -1) { + this.radioGroupForm.controls.dateSpan.setValue(fragment, { emitEvent: false }); + } + }); + merge(of(''), this.radioGroupForm.controls.dateSpan.valueChanges) + .pipe(switchMap(() => this.onDateSpanChanged())) + .subscribe((pools: any) => { + console.log(pools); + }); + + // Fetch initial mining pool data + this.onDateSpanChanged(); + } + + ngOnChanges() { + } + + rendered() { + } + + savePoolsPreference() { + this.storageService.setValue('poolsWindowPreference', this.radioGroupForm.controls.dateSpan.value); + this.poolsWindowPreference = this.radioGroupForm.controls.dateSpan.value; + } + + onDateSpanChanged(): ObservableInput { + let interval: string; + console.log(this.poolsWindowPreference); + switch (this.poolsWindowPreference) { + case '1d': interval = '1 DAY'; break; + case '3d': interval = '3 DAY'; break; + case '1w': interval = '1 WEEK'; break; + case '1m': interval = '1 MONTH'; break; + case '3m': interval = '3 MONTH'; break; + case '6m': interval = '6 MONTH'; break; + case '1y': interval = '1 YEAR'; break; + case '2y': interval = '2 YEAR'; break; + case '3y': interval = '3 YEAR'; break; + case 'all': interval = '1000 YEAR'; break; + } + this.pools$ = this.apiService.listPools$(interval).pipe(map(res => this.computeMiningStats(res))); + return this.pools$; + } + + computeMiningStats(stats: PoolsStats) { + const totalEmptyBlock = Object.values(stats.poolsStats).reduce((prev, cur) => { + return prev + cur.emptyBlocks; + }, 0); + const totalEmptyBlockRatio = (totalEmptyBlock / stats.blockCount * 100).toFixed(2); + const poolsStats = stats.poolsStats.map((poolStat) => { + return { + share: (poolStat.blockCount / stats.blockCount * 100).toFixed(2), + lastEstimatedHashrate: (poolStat.blockCount / stats.blockCount * stats.lastEstimatedHashrate / Math.pow(10, 15)).toFixed(2), + emptyBlockRatio: (poolStat.emptyBlocks / poolStat.blockCount * 100).toFixed(2), + ...poolStat + } + }); + + return { + lastEstimatedHashrate: (stats.lastEstimatedHashrate / Math.pow(10, 15)).toFixed(2), + blockCount: stats.blockCount, + totalEmptyBlock: totalEmptyBlock, + totalEmptyBlockRatio: totalEmptyBlockRatio, + poolsStats: poolsStats, + } + } +} + diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index fa0ddeb77..669c8d251 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -51,3 +51,16 @@ export interface LiquidPegs { } export interface ITranslators { [language: string]: string; } + +export interface PoolsStats { + poolsStats: { + pooldId: number, + name: string, + link: string, + blockCount: number, + emptyBlocks: number, + rank: number, + }[], + blockCount: number, + lastEstimatedHashrate: number, +} diff --git a/frontend/src/app/services/api.service.ts b/frontend/src/app/services/api.service.ts index 05c6f074e..f17f32032 100644 --- a/frontend/src/app/services/api.service.ts +++ b/frontend/src/app/services/api.service.ts @@ -1,6 +1,6 @@ import { Injectable } from '@angular/core'; import { HttpClient, HttpParams } from '@angular/common/http'; -import { CpfpInfo, OptimizedMempoolStats, DifficultyAdjustment, AddressInformation, LiquidPegs, ITranslators } from '../interfaces/node-api.interface'; +import { CpfpInfo, OptimizedMempoolStats, DifficultyAdjustment, AddressInformation, LiquidPegs, ITranslators, PoolsStats } from '../interfaces/node-api.interface'; import { Observable } from 'rxjs'; import { StateService } from './state.service'; import { WebsocketResponse } from '../interfaces/websocket.interface'; @@ -120,4 +120,9 @@ export class ApiService { postTransaction$(hexPayload: string): Observable { return this.httpClient.post(this.apiBaseUrl + this.apiBasePath + '/api/tx', hexPayload, { responseType: 'text' as 'json'}); } + + listPools$(interval: string) : Observable { + const params = new HttpParams().set('interval', interval); + return this.httpClient.get(this.apiBaseUrl + this.apiBasePath + '/api/v1/pools', {params}); + } } diff --git a/frontend/src/app/services/storage.service.ts b/frontend/src/app/services/storage.service.ts index 7494784f6..6eff9d391 100644 --- a/frontend/src/app/services/storage.service.ts +++ b/frontend/src/app/services/storage.service.ts @@ -6,18 +6,23 @@ import { Router, ActivatedRoute } from '@angular/router'; }) export class StorageService { constructor(private router: Router, private route: ActivatedRoute) { - let graphWindowPreference: string = this.getValue('graphWindowPreference'); + this.setDefaultValueIfNeeded('graphWindowPreference', '2h'); + this.setDefaultValueIfNeeded('poolsWindowPreference', '1d'); + } + + setDefaultValueIfNeeded(key: string, defaultValue: string) { + let graphWindowPreference: string = this.getValue(key); if (graphWindowPreference === null) { // First visit to mempool.space - if (this.router.url.includes("graphs")) { - this.setValue('graphWindowPreference', this.route.snapshot.fragment ? this.route.snapshot.fragment : "2h"); + if (this.router.url.includes("graphs") || this.router.url.includes("pools")) { + this.setValue(key, this.route.snapshot.fragment ? this.route.snapshot.fragment : defaultValue); } else { - this.setValue('graphWindowPreference', "2h"); + this.setValue(key, defaultValue); } - } else if (this.router.url.includes("graphs")) { // Visit a different graphs#fragment from last visit - if (this.route.snapshot.fragment !== null && graphWindowPreference !== this.route.snapshot.fragment) { - this.setValue('graphWindowPreference', this.route.snapshot.fragment); - } + } else if (this.router.url.includes("graphs") || this.router.url.includes("pools")) { // Visit a different graphs#fragment from last visit + if (this.route.snapshot.fragment !== null && graphWindowPreference !== this.route.snapshot.fragment) { + this.setValue(key, this.route.snapshot.fragment); } + } } getValue(key: string): string { From 18a63933fa9cc49deea39f532cc6af452a7c888d Mon Sep 17 00:00:00 2001 From: nymkappa Date: Thu, 13 Jan 2022 12:18:39 +0900 Subject: [PATCH 05/24] Increment migration schema version to 3 and re-add `pools` and `blocks` table creation queries --- backend/src/api/database-migration.ts | 32 +++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index efdf6755c..69807b414 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -85,6 +85,7 @@ class DatabaseMigration { } if (databaseSchemaVersion < 3) { await this.$executeQuery(connection, this.getCreatePoolsTableQuery(), await this.$checkIfTableExists('pools')); + await this.$executeQuery(connection, this.getCreateBlocksTableQuery(), await this.$checkIfTableExists('blocks')); } connection.release(); } catch (e) { @@ -348,6 +349,37 @@ class DatabaseMigration { PRIMARY KEY (id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;`; } + + private getCreatePoolsTableQuery(): string { + return `CREATE TABLE IF NOT EXISTS pools ( + id int(11) NOT NULL AUTO_INCREMENT, + name varchar(50) NOT NULL, + link varchar(255) NOT NULL, + addresses text NOT NULL, + regexes text NOT NULL, + PRIMARY KEY (id) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;`; + } + + private getCreateBlocksTableQuery(): string { + return `CREATE TABLE IF NOT EXISTS blocks ( + height int(11) unsigned NOT NULL, + hash varchar(65) NOT NULL, + blockTimestamp timestamp NOT NULL, + size int(11) unsigned NOT NULL, + weight int(11) unsigned NOT NULL, + tx_count int(11) unsigned NOT NULL, + coinbase_raw text, + difficulty bigint(20) unsigned NOT NULL, + pool_id int(11) DEFAULT -1, + fees double unsigned NOT NULL, + fee_span json NOT NULL, + median_fee double unsigned NOT NULL, + PRIMARY KEY (height), + INDEX (pool_id), + FOREIGN KEY (pool_id) REFERENCES pools (id) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; + } } export default new DatabaseMigration(); From 0a267affafbbb79cb53cf7734077bd823b8af87e Mon Sep 17 00:00:00 2001 From: nymkappa Date: Fri, 14 Jan 2022 18:09:40 +0900 Subject: [PATCH 06/24] Add pie chart and rewrite the pool ranking component --- backend/src/api/mining.ts | 2 +- frontend/src/app/app.module.ts | 3 +- .../master-page/master-page.component.html | 4 +- .../pool-ranking/pool-ranking.component.html | 36 +-- .../pool-ranking/pool-ranking.component.ts | 214 ++++++++++++------ .../src/app/interfaces/node-api.interface.ts | 30 ++- frontend/src/app/services/mining.service.ts | 44 ++++ 7 files changed, 231 insertions(+), 102 deletions(-) create mode 100644 frontend/src/app/services/mining.service.ts diff --git a/backend/src/api/mining.ts b/backend/src/api/mining.ts index d22a29d5b..94d6c29a4 100644 --- a/backend/src/api/mining.ts +++ b/backend/src/api/mining.ts @@ -43,8 +43,8 @@ class Mining { }) poolsStatistics["blockCount"] = blockCount; - poolsStatistics["poolsStats"] = poolsStats; poolsStatistics["lastEstimatedHashrate"] = lastBlockHashrate; + poolsStatistics["pools"] = poolsStats; return poolsStatistics; } diff --git a/frontend/src/app/app.module.ts b/frontend/src/app/app.module.ts index b16c5bdea..f9eae0666 100644 --- a/frontend/src/app/app.module.ts +++ b/frontend/src/app/app.module.ts @@ -49,7 +49,7 @@ import { FeesBoxComponent } from './components/fees-box/fees-box.component'; import { DashboardComponent } from './dashboard/dashboard.component'; import { DifficultyComponent } from './components/difficulty/difficulty.component'; import { FontAwesomeModule, FaIconLibrary } from '@fortawesome/angular-fontawesome'; -import { faFilter, faAngleDown, faAngleUp, faAngleRight, faAngleLeft, faBolt, faChartArea, faCogs, faCubes, faDatabase, faExchangeAlt, faInfoCircle, +import { faFilter, faAngleDown, faAngleUp, faAngleRight, faAngleLeft, faBolt, faChartArea, faCogs, faCubes, faHammer, faDatabase, faExchangeAlt, faInfoCircle, faLink, faList, faSearch, faCaretUp, faCaretDown, faTachometerAlt, faThList, faTint, faTv, faAngleDoubleDown, faSortUp, faAngleDoubleUp, faChevronDown, faFileAlt, faRedoAlt, faArrowAltCircleRight, faExternalLinkAlt, faBook, faListUl } from '@fortawesome/free-solid-svg-icons'; import { ApiDocsComponent } from './components/docs/api-docs.component'; import { DocsComponent } from './components/docs/docs.component'; @@ -145,6 +145,7 @@ export class AppModule { library.addIcons(faTv); library.addIcons(faTachometerAlt); library.addIcons(faCubes); + library.addIcons(faHammer); library.addIcons(faCogs); library.addIcons(faThList); library.addIcons(faList); diff --git a/frontend/src/app/components/master-page/master-page.component.html b/frontend/src/app/components/master-page/master-page.component.html index f05b297c7..479f324c2 100644 --- a/frontend/src/app/components/master-page/master-page.component.html +++ b/frontend/src/app/components/master-page/master-page.component.html @@ -31,8 +31,8 @@ -