From 3cd150512862784d87e2a25756a9a08c35344e22 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Tue, 15 Mar 2022 20:57:27 +0100 Subject: [PATCH 01/13] Optimize RPC calls --- backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts | 2 +- backend/src/api/bitcoin/bitcoin-api.ts | 4 ++-- backend/src/api/blocks.ts | 6 ++++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts index 53c731e1f..266be5f1e 100644 --- a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts +++ b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts @@ -2,7 +2,7 @@ import { IEsploraApi } from './esplora-api.interface'; export interface AbstractBitcoinApi { $getRawMempool(): Promise; - $getRawTransaction(txId: string, skipConversion?: boolean, addPrevout?: boolean): Promise; + $getRawTransaction(txId: string, skipConversion?: boolean, addPrevout?: boolean, blockHash?: string): Promise; $getBlockHeightTip(): Promise; $getTxIdsForBlock(hash: string): Promise; $getBlockHash(height: number): Promise; diff --git a/backend/src/api/bitcoin/bitcoin-api.ts b/backend/src/api/bitcoin/bitcoin-api.ts index 8d66f82ef..27b021af0 100644 --- a/backend/src/api/bitcoin/bitcoin-api.ts +++ b/backend/src/api/bitcoin/bitcoin-api.ts @@ -14,14 +14,14 @@ class BitcoinApi implements AbstractBitcoinApi { this.bitcoindClient = bitcoinClient; } - $getRawTransaction(txId: string, skipConversion = false, addPrevout = false): Promise { + $getRawTransaction(txId: string, skipConversion = false, addPrevout = false, blockHash?: string): Promise { // If the transaction is in the mempool we already converted and fetched the fee. Only prevouts are missing const txInMempool = mempool.getMempool()[txId]; if (txInMempool && addPrevout) { return this.$addPrevouts(txInMempool); } - return this.bitcoindClient.getRawTransaction(txId, true) + return this.bitcoindClient.getRawTransaction(txId, true, blockHash) .then((transaction: IBitcoinApi.Transaction) => { if (skipConversion) { transaction.vout.forEach((vout) => { diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 8f066b5a4..bde5bcea9 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -109,7 +109,7 @@ class Blocks { blockExtended.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0); blockExtended.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]); - const coinbaseRaw: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(transactions[0].txid, true); + const coinbaseRaw: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(transactions[0].txid, true, false, block.id); blockExtended.extras.coinbaseRaw = coinbaseRaw.hex; if (block.height === 0) { @@ -119,7 +119,9 @@ class Blocks { blockExtended.extras.avgFee = 0; blockExtended.extras.avgFeeRate = 0; } else { - const stats = await bitcoinClient.getBlockStats(block.id); + const stats = await bitcoinClient.getBlockStats(block.id, [ + 'feerate_percentiles', 'minfeerate', 'maxfeerate', 'totalfee', 'avgfee', 'avgfeerate' + ]); blockExtended.extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles blockExtended.extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat(); blockExtended.extras.totalFees = stats.totalfee; From 94dbec46cfc62d42af575cd5259a26f1d9eaf1cb Mon Sep 17 00:00:00 2001 From: nymkappa Date: Tue, 15 Mar 2022 23:33:51 +0100 Subject: [PATCH 02/13] Index asciiScriptSig and display it in /mining/blocks --- backend/src/api/blocks.ts | 5 ++-- backend/src/api/database-migration.ts | 8 ++++++- backend/src/api/transaction-utils.ts | 19 +++++++++++---- backend/src/repositories/BlocksRepository.ts | 4 ++-- .../blocks-list/blocks-list.component.html | 13 ++++++---- .../blocks-list/blocks-list.component.scss | 24 +++++++++++++++++++ .../src/app/interfaces/node-api.interface.ts | 1 + .../shared/pipes/hex2ascii/hex2ascii.pipe.ts | 2 +- 8 files changed, 59 insertions(+), 17 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index bde5bcea9..700cc4b63 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -108,9 +108,7 @@ class Blocks { const blockExtended: BlockExtended = Object.assign({ extras: {} }, block); blockExtended.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0); blockExtended.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]); - - const coinbaseRaw: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(transactions[0].txid, true, false, block.id); - blockExtended.extras.coinbaseRaw = coinbaseRaw.hex; + blockExtended.extras.coinbaseRaw = transactionUtils.hex2ascii(blockExtended.extras.coinbaseTx.vin[0].scriptsig); if (block.height === 0) { blockExtended.extras.medianFee = 0; // 50th percentiles @@ -410,6 +408,7 @@ class Blocks { weight: block.weight, previousblockhash: block.previousblockhash, extras: { + coinbaseRaw: block.coinbase_raw ?? block.extras.coinbaseRaw, medianFee: block.medianFee ?? block.median_fee ?? block.extras?.medianFee, feeRange: block.feeRange ?? block.fee_range ?? block?.extras?.feeSpan, reward: block.reward ?? block?.extras?.reward, diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index c9c1da8e8..245f260f8 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -6,7 +6,7 @@ import logger from '../logger'; const sleep = (ms: number) => new Promise(res => setTimeout(res, ms)); class DatabaseMigration { - private static currentVersion = 14; + private static currentVersion = 15; private queryTimeout = 120000; private statisticsAddedIndexed = false; @@ -175,6 +175,12 @@ class DatabaseMigration { await this.$executeQuery(connection, 'ALTER TABLE `hashrates` MODIFY `pool_id` SMALLINT UNSIGNED NOT NULL DEFAULT "0"'); } + if (databaseSchemaVersion < 15 && isBitcoin === true) { + logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.`); + await this.$executeQuery(connection, 'TRUNCATE blocks;'); // Need to re-index + await this.$executeQuery(connection, 'ALTER TABLE `blocks` MODIFY `coinbase_raw` TEXT COLLATE "utf8mb4_general_ci" NULL '); + } + connection.release(); } catch (e) { connection.release(); diff --git a/backend/src/api/transaction-utils.ts b/backend/src/api/transaction-utils.ts index 2e669d709..b5def870c 100644 --- a/backend/src/api/transaction-utils.ts +++ b/backend/src/api/transaction-utils.ts @@ -45,12 +45,21 @@ class TransactionUtils { return transactionExtended; } - public hex2ascii(hex: string) { - let str = ''; - for (let i = 0; i < hex.length; i += 2) { - str += String.fromCharCode(parseInt(hex.substr(i, 2), 16)); + public hex2ascii(hex: string): string { + const opPush = hex.split(' ').filter((_, i, a) => i > 0 && /^OP_PUSH/.test(a[i - 1])); + + if (opPush[0]) { + hex = opPush[0]; } - return str; + + if (!hex) { + return ''; + } + const bytes: number[] = []; + for (let i = 0; i < hex.length; i += 2) { + bytes.push(parseInt(hex.substr(i, 2), 16)); + } + return new TextDecoder('utf8').decode(Uint8Array.from(bytes)); } } diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 0af5e2252..add99f1fd 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -32,7 +32,7 @@ class BlocksRepository { block.size, block.weight, block.tx_count, - block.extras.coinbaseRaw, + connection.escape(block.extras.coinbaseRaw), block.difficulty, block.extras.pool?.id, // Should always be set to something block.extras.totalFees, @@ -56,7 +56,7 @@ class BlocksRepository { logger.debug(`$saveBlockInDatabase() - Block ${block.height} has already been indexed, ignoring`); } else { connection.release(); - logger.err('$saveBlockInDatabase() error' + (e instanceof Error ? e.message : e)); + logger.err('$saveBlockInDatabase() error: ' + (e instanceof Error ? e.message : e)); throw e; } } diff --git a/frontend/src/app/components/blocks-list/blocks-list.component.html b/frontend/src/app/components/blocks-list/blocks-list.component.html index f50a5fff2..4e81c44f2 100644 --- a/frontend/src/app/components/blocks-list/blocks-list.component.html +++ b/frontend/src/app/components/blocks-list/blocks-list.component.html @@ -24,11 +24,14 @@ }} - - - {{ block.extras.pool.name }} - +
+ + + {{ block.extras.pool.name }} + + {{ block.extras.coinbaseRaw }} +
‎{{ block.timestamp * 1000 | date:'yyyy-MM-dd HH:mm' }} diff --git a/frontend/src/app/components/blocks-list/blocks-list.component.scss b/frontend/src/app/components/blocks-list/blocks-list.component.scss index 9414348c1..3ea6e42f9 100644 --- a/frontend/src/app/components/blocks-list/blocks-list.component.scss +++ b/frontend/src/app/components/blocks-list/blocks-list.component.scss @@ -122,3 +122,27 @@ td { display: none; } } + +/* Tooltip text */ +.tooltip-custom { + position: relative; +} + +.tooltip-custom .tooltiptext { + visibility: hidden; + background-color: black; + color: #fff; + text-align: center; + padding: 5px 0; + border-radius: 6px; + position: absolute; + z-index: 1; + // width: 120px; + top: -40px; + left: 0; +} + +/* Show the tooltip text when you mouse over the tooltip container */ +.tooltip-custom:hover .tooltiptext { + visibility: visible; +} \ No newline at end of file diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index 8caba7496..c8118add7 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -102,6 +102,7 @@ export interface BlockExtension { feeRange?: number[]; reward?: number; coinbaseTx?: Transaction; + coinbaseRaw?: string; matchRate?: number; pool?: { id: number; diff --git a/frontend/src/app/shared/pipes/hex2ascii/hex2ascii.pipe.ts b/frontend/src/app/shared/pipes/hex2ascii/hex2ascii.pipe.ts index 3d62ae7d6..f512a0815 100644 --- a/frontend/src/app/shared/pipes/hex2ascii/hex2ascii.pipe.ts +++ b/frontend/src/app/shared/pipes/hex2ascii/hex2ascii.pipe.ts @@ -15,7 +15,7 @@ export class Hex2asciiPipe implements PipeTransform { if (!hex) { return ''; } - let bytes: number[] = []; + const bytes: number[] = []; for (let i = 0; i < hex.length; i += 2) { bytes.push(parseInt(hex.substr(i, 2), 16)); } From ffb5db69a815e1741064e123973f3eb30dee0b71 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Wed, 16 Mar 2022 12:10:18 +0100 Subject: [PATCH 03/13] Store hex in coinbase raw - Improve scripsig parsing --- backend/src/api/blocks.ts | 2 +- backend/src/api/database-migration.ts | 6 ------ backend/src/api/transaction-utils.ts | 17 ++++------------- backend/src/repositories/BlocksRepository.ts | 2 +- .../blocks-list/blocks-list.component.html | 2 +- .../shared/pipes/hex2ascii/hex2ascii.pipe.ts | 2 +- 6 files changed, 8 insertions(+), 23 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 700cc4b63..922c0e639 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -108,7 +108,7 @@ class Blocks { const blockExtended: BlockExtended = Object.assign({ extras: {} }, block); blockExtended.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0); blockExtended.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]); - blockExtended.extras.coinbaseRaw = transactionUtils.hex2ascii(blockExtended.extras.coinbaseTx.vin[0].scriptsig); + blockExtended.extras.coinbaseRaw = blockExtended.extras.coinbaseTx.vin[0].scriptsig; if (block.height === 0) { blockExtended.extras.medianFee = 0; // 50th percentiles diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 245f260f8..ffa9041e3 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -175,12 +175,6 @@ class DatabaseMigration { await this.$executeQuery(connection, 'ALTER TABLE `hashrates` MODIFY `pool_id` SMALLINT UNSIGNED NOT NULL DEFAULT "0"'); } - if (databaseSchemaVersion < 15 && isBitcoin === true) { - logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.`); - await this.$executeQuery(connection, 'TRUNCATE blocks;'); // Need to re-index - await this.$executeQuery(connection, 'ALTER TABLE `blocks` MODIFY `coinbase_raw` TEXT COLLATE "utf8mb4_general_ci" NULL '); - } - connection.release(); } catch (e) { connection.release(); diff --git a/backend/src/api/transaction-utils.ts b/backend/src/api/transaction-utils.ts index b5def870c..2e669d709 100644 --- a/backend/src/api/transaction-utils.ts +++ b/backend/src/api/transaction-utils.ts @@ -45,21 +45,12 @@ class TransactionUtils { return transactionExtended; } - public hex2ascii(hex: string): string { - const opPush = hex.split(' ').filter((_, i, a) => i > 0 && /^OP_PUSH/.test(a[i - 1])); - - if (opPush[0]) { - hex = opPush[0]; - } - - if (!hex) { - return ''; - } - const bytes: number[] = []; + public hex2ascii(hex: string) { + let str = ''; for (let i = 0; i < hex.length; i += 2) { - bytes.push(parseInt(hex.substr(i, 2), 16)); + str += String.fromCharCode(parseInt(hex.substr(i, 2), 16)); } - return new TextDecoder('utf8').decode(Uint8Array.from(bytes)); + return str; } } diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index add99f1fd..4ac0a9d9d 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -32,7 +32,7 @@ class BlocksRepository { block.size, block.weight, block.tx_count, - connection.escape(block.extras.coinbaseRaw), + block.extras.coinbaseRaw, block.difficulty, block.extras.pool?.id, // Should always be set to something block.extras.totalFees, diff --git a/frontend/src/app/components/blocks-list/blocks-list.component.html b/frontend/src/app/components/blocks-list/blocks-list.component.html index 4e81c44f2..cb301e992 100644 --- a/frontend/src/app/components/blocks-list/blocks-list.component.html +++ b/frontend/src/app/components/blocks-list/blocks-list.component.html @@ -30,7 +30,7 @@ onError="this.src = './resources/mining-pools/default.svg'"> {{ block.extras.pool.name }} - {{ block.extras.coinbaseRaw }} + {{ block.extras.coinbaseRaw | hex2ascii }} diff --git a/frontend/src/app/shared/pipes/hex2ascii/hex2ascii.pipe.ts b/frontend/src/app/shared/pipes/hex2ascii/hex2ascii.pipe.ts index f512a0815..09b3df762 100644 --- a/frontend/src/app/shared/pipes/hex2ascii/hex2ascii.pipe.ts +++ b/frontend/src/app/shared/pipes/hex2ascii/hex2ascii.pipe.ts @@ -19,7 +19,7 @@ export class Hex2asciiPipe implements PipeTransform { for (let i = 0; i < hex.length; i += 2) { bytes.push(parseInt(hex.substr(i, 2), 16)); } - return new TextDecoder('utf8').decode(Uint8Array.from(bytes)); + return new TextDecoder('utf8').decode(Uint8Array.from(bytes)).replace(/\uFFFD/g, '').replace(/\\0/g, ''); } } From 92142cd531aa38f82f494f243c7a8a8cbbee81b2 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Wed, 16 Mar 2022 14:48:37 +0100 Subject: [PATCH 04/13] Hide tabs in /graphs page is mining not available --- frontend/src/app/components/graphs/graphs.component.html | 2 +- frontend/src/app/components/graphs/graphs.component.ts | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/frontend/src/app/components/graphs/graphs.component.html b/frontend/src/app/components/graphs/graphs.component.html index 9788437a9..d5cc61e91 100644 --- a/frontend/src/app/components/graphs/graphs.component.html +++ b/frontend/src/app/components/graphs/graphs.component.html @@ -1,4 +1,4 @@ -
    +
    • Mempool diff --git a/frontend/src/app/components/graphs/graphs.component.ts b/frontend/src/app/components/graphs/graphs.component.ts index 61fa65196..58bf58db1 100644 --- a/frontend/src/app/components/graphs/graphs.component.ts +++ b/frontend/src/app/components/graphs/graphs.component.ts @@ -1,4 +1,5 @@ import { Component, OnInit } from "@angular/core"; +import { StateService } from "src/app/services/state.service"; @Component({ selector: 'app-graphs', @@ -6,7 +7,7 @@ import { Component, OnInit } from "@angular/core"; styleUrls: ['./graphs.component.scss'], }) export class GraphsComponent implements OnInit { - constructor() { } + constructor(public stateService: StateService) { } ngOnInit(): void { From 71648bf01e215876086022f7c8b046b64c591f5e Mon Sep 17 00:00:00 2001 From: nymkappa Date: Wed, 16 Mar 2022 17:28:00 +0100 Subject: [PATCH 05/13] Force disable mining dashboard if base_module not mempool --- .../statistics/statistics.component.html | 2 +- .../statistics/statistics.component.scss | 18 ++++++++++-------- .../statistics/statistics.component.ts | 2 +- frontend/src/app/services/state.service.ts | 4 ++++ 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/frontend/src/app/components/statistics/statistics.component.html b/frontend/src/app/components/statistics/statistics.component.html index 0d2276e98..79dcc6fe2 100644 --- a/frontend/src/app/components/statistics/statistics.component.html +++ b/frontend/src/app/components/statistics/statistics.component.html @@ -4,7 +4,7 @@
      Mempool by vBytes (sat/vByte) -
      +