diff --git a/.github/workflows/cypress.yml b/.github/workflows/cypress.yml index 0cd95e568..732923ce2 100644 --- a/.github/workflows/cypress.yml +++ b/.github/workflows/cypress.yml @@ -21,7 +21,7 @@ jobs: - name: Setup node uses: actions/setup-node@v2 with: - node-version: 16.10.0 + node-version: 16.15.0 cache: 'npm' cache-dependency-path: frontend/package-lock.json - name: ${{ matrix.browser }} browser tests (Mempool) diff --git a/.nvmrc b/.nvmrc index 56bfee434..7fd023741 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -v16.10.0 +v16.15.0 diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 6af631382..1f27d8c0e 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -134,7 +134,7 @@ class Blocks { blockExtended.extras.avgFeeRate = stats.avgfeerate; } - if (Common.indexingEnabled()) { + if (['mainnet', 'testnet', 'signet', 'regtest'].includes(config.MEMPOOL.NETWORK)) { let pool: PoolTag; if (blockExtended.extras?.coinbaseTx !== undefined) { pool = await this.$findBlockMiner(blockExtended.extras?.coinbaseTx); @@ -143,7 +143,8 @@ class Blocks { } if (!pool) { // We should never have this situation in practise - logger.warn(`Cannot assign pool to block ${blockExtended.height} and 'unknown' pool does not exist. Check your "pools" table entries`); + logger.warn(`Cannot assign pool to block ${blockExtended.height} and 'unknown' pool does not exist. ` + + `Check your "pools" table entries`); return blockExtended; } @@ -389,6 +390,45 @@ class Blocks { return prepareBlock(blockExtended); } + /** + * Index a block by hash if it's missing from the database. Returns the block after indexing + */ + public async $getBlock(hash: string): Promise { + // Check the memory cache + const blockByHash = this.getBlocks().find((b) => b.id === hash); + if (blockByHash) { + return blockByHash; + } + + // Block has already been indexed + if (Common.indexingEnabled()) { + const dbBlock = await blocksRepository.$getBlockByHash(hash); + if (dbBlock != null) { + logger.info('GET BLOCK: already indexed'); + return prepareBlock(dbBlock); + } + } + + const block = await bitcoinApi.$getBlock(hash); + + // Not Bitcoin network, return the block as it + if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { + logger.info('GET BLOCK: using bitcoin backend'); + return block; + } + + // Bitcoin network, add our custom data on top + logger.info('GET BLOCK: index block on the fly'); + const transactions = await this.$getTransactionsExtended(hash, block.height, true); + const blockExtended = await this.$getBlockExtended(block, transactions); + if (Common.indexingEnabled()) { + delete(blockExtended['coinbaseTx']); + await blocksRepository.$saveBlockInDatabase(blockExtended); + } + + return blockExtended; + } + public async $getBlocksExtras(fromHeight?: number, limit: number = 15): Promise { // Note - This API is breaking if indexing is not available. For now it is okay because we only // use it for the mining pages, and mining pages should not be available if indexing is turned off. diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index 45ef5f576..d4b57f204 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -169,12 +169,12 @@ export class Common { default: return null; } } - + static indexingEnabled(): boolean { return ( - ['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) && + ['mainnet', 'testnet', 'signet', 'regtest'].includes(config.MEMPOOL.NETWORK) && config.DATABASE.ENABLED === true && - config.MEMPOOL.INDEXING_BLOCKS_AMOUNT != 0 + config.MEMPOOL.INDEXING_BLOCKS_AMOUNT !== 0 ); } } diff --git a/backend/src/index.ts b/backend/src/index.ts index 8560064a9..f2658a22a 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -350,7 +350,8 @@ class Server { this.app .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-extras', routes.getBlocksExtras) - .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-extras/:height', routes.getBlocksExtras); + .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-extras/:height', routes.getBlocksExtras) + .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash', routes.getBlock); if (config.MEMPOOL.BACKEND !== 'esplora') { this.app @@ -362,7 +363,6 @@ class Server { .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/hex', routes.getRawTransaction) .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/status', routes.getTransactionStatus) .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/outspends', routes.getTransactionOutspends) - .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash', routes.getBlock) .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/header', routes.getBlockHeader) .get(config.MEMPOOL.API_URL_PREFIX + 'blocks', routes.getBlocks) .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/:height', routes.getBlocks) diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 0792f130f..47a18fc9b 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -287,6 +287,7 @@ class BlocksRepository { return null; } + rows[0].fee_span = JSON.parse(rows[0].fee_span); return rows[0]; } catch (e) { logger.err(`Cannot get indexed block ${height}. Reason: ` + (e instanceof Error ? e.message : e)); @@ -294,6 +295,34 @@ class BlocksRepository { } } + /** + * Get one block by hash + */ + public async $getBlockByHash(hash: string): Promise { + try { + const query = ` + SELECT *, UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp, hash as id, + pools.id as pool_id, pools.name as pool_name, pools.link as pool_link, pools.slug as pool_slug, + pools.addresses as pool_addresses, pools.regexes as pool_regexes, + previous_block_hash as previousblockhash + FROM blocks + JOIN pools ON blocks.pool_id = pools.id + WHERE hash = '${hash}'; + `; + const [rows]: any[] = await DB.query(query); + + if (rows.length <= 0) { + return null; + } + + rows[0].fee_span = JSON.parse(rows[0].fee_span); + return rows[0]; + } catch (e) { + logger.err(`Cannot get indexed block ${hash}. Reason: ` + (e instanceof Error ? e.message : e)); + throw e; + } + } + /** * Return blocks difficulty */ @@ -397,18 +426,28 @@ class BlocksRepository { const [blocks]: any[] = await DB.query(`SELECT height, hash, previous_block_hash, UNIX_TIMESTAMP(blockTimestamp) as timestamp FROM blocks ORDER BY height`); - let currentHeight = 1; - while (currentHeight < blocks.length) { - if (blocks[currentHeight].previous_block_hash !== blocks[currentHeight - 1].hash) { - logger.warn(`Chain divergence detected at block ${blocks[currentHeight - 1].height}, re-indexing newer blocks and hashrates`); - await this.$deleteBlocksFrom(blocks[currentHeight - 1].height); - await HashratesRepository.$deleteHashratesFromTimestamp(blocks[currentHeight - 1].timestamp - 604800); + let partialMsg = false; + let idx = 1; + while (idx < blocks.length) { + if (blocks[idx].height - 1 !== blocks[idx - 1].height) { + if (partialMsg === false) { + logger.info('Some blocks are not indexed, skipping missing blocks during chain validation'); + partialMsg = true; + } + ++idx; + continue; + } + + if (blocks[idx].previous_block_hash !== blocks[idx - 1].hash) { + logger.warn(`Chain divergence detected at block ${blocks[idx - 1].height}, re-indexing newer blocks and hashrates`); + await this.$deleteBlocksFrom(blocks[idx - 1].height); + await HashratesRepository.$deleteHashratesFromTimestamp(blocks[idx - 1].timestamp - 604800); return false; } - ++currentHeight; + ++idx; } - logger.info(`${currentHeight} blocks hash validated in ${new Date().getTime() - start} ms`); + logger.info(`${idx} blocks hash validated in ${new Date().getTime() - start} ms`); return true; } catch (e) { logger.err('Cannot validate chain of block hash. Reason: ' + (e instanceof Error ? e.message : e)); @@ -457,7 +496,7 @@ class BlocksRepository { /** * Get the historical averaged block rewards */ - public async $getHistoricalBlockRewards(div: number, interval: string | null): Promise { + public async $getHistoricalBlockRewards(div: number, interval: string | null): Promise { try { let query = `SELECT CAST(AVG(height) as INT) as avg_height, diff --git a/backend/src/routes.ts b/backend/src/routes.ts index 72bf3b483..52f590775 100644 --- a/backend/src/routes.ts +++ b/backend/src/routes.ts @@ -702,8 +702,8 @@ class Routes { public async getBlock(req: Request, res: Response) { try { - const result = await bitcoinApi.$getBlock(req.params.hash); - res.json(result); + const block = await blocks.$getBlock(req.params.hash); + res.json(block); } catch (e) { res.status(500).send(e instanceof Error ? e.message : e); } @@ -727,7 +727,7 @@ class Routes { res.status(500).send(e instanceof Error ? e.message : e); } } - + public async getBlocks(req: Request, res: Response) { try { loadingIndicators.setProgress('blocks', 0); diff --git a/backend/src/utils/blocks-utils.ts b/backend/src/utils/blocks-utils.ts index 7b5c0b23a..8760a08c0 100644 --- a/backend/src/utils/blocks-utils.ts +++ b/backend/src/utils/blocks-utils.ts @@ -1,4 +1,4 @@ -import { BlockExtended } from "../mempool.interfaces"; +import { BlockExtended } from '../mempool.interfaces'; export function prepareBlock(block: any): BlockExtended { return { @@ -17,9 +17,11 @@ export function prepareBlock(block: any): BlockExtended { extras: { coinbaseRaw: block.coinbase_raw ?? block.extras.coinbaseRaw, medianFee: block.medianFee ?? block.median_fee ?? block.extras?.medianFee, - feeRange: block.feeRange ?? block.fee_range ?? block?.extras?.feeSpan, + feeRange: block.feeRange ?? block.fee_span, reward: block.reward ?? block?.extras?.reward, - totalFees: block.totalFees ?? block?.fees ?? block?.extras.totalFees, + totalFees: block.totalFees ?? block?.fees ?? block?.extras?.totalFees, + avgFee: block?.extras?.avgFee ?? block.avg_fee, + avgFeeRate: block?.avgFeeRate ?? block.avg_fee_rate, pool: block?.extras?.pool ?? (block?.pool_id ? { id: block.pool_id, name: block.pool_name, diff --git a/docker/README.md b/docker/README.md index 07590d0dd..37a2cc079 100644 --- a/docker/README.md +++ b/docker/README.md @@ -155,7 +155,7 @@ Corresponding `docker-compose.yml` overrides: environment: ELECTRUM_HOST: "" ELECTRUM_PORT: "" - ELECTRUM_TLS: "" + ELECTRUM_TLS_ENABLED: "" ... ``` diff --git a/docker/backend/Dockerfile b/docker/backend/Dockerfile index c013fc23a..31acff047 100644 --- a/docker/backend/Dockerfile +++ b/docker/backend/Dockerfile @@ -1,4 +1,4 @@ -FROM node:16.10.0-buster-slim AS builder +FROM node:16.15.0-buster-slim AS builder ARG commitHash ENV DOCKER_COMMIT_HASH=${commitHash} @@ -11,7 +11,7 @@ RUN apt-get install -y build-essential python3 pkg-config RUN npm install RUN npm run build -FROM node:16.10.0-buster-slim +FROM node:16.15.0-buster-slim WORKDIR /backend diff --git a/docker/backend/start.sh b/docker/backend/start.sh index 09672184d..960171e43 100644 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -28,7 +28,7 @@ __CORE_RPC_PASSWORD__=${CORE_RPC_PASSWORD:=mempool} # ELECTRUM __ELECTRUM_HOST__=${ELECTRUM_HOST:=127.0.0.1} __ELECTRUM_PORT__=${ELECTRUM_PORT:=50002} -__ELECTRUM_TLS_ENABLED__=${ELECTRUM_TLS:=false} +__ELECTRUM_TLS_ENABLED__=${ELECTRUM_TLS_ENABLED:=false} # ESPLORA __ESPLORA_REST_API_URL__=${ESPLORA_REST_API_URL:=http://127.0.0.1:3000} diff --git a/docker/frontend/Dockerfile b/docker/frontend/Dockerfile index 34c41119c..e2874ff4e 100644 --- a/docker/frontend/Dockerfile +++ b/docker/frontend/Dockerfile @@ -1,4 +1,4 @@ -FROM node:16.10.0-buster-slim AS builder +FROM node:16.15.0-buster-slim AS builder ARG commitHash ENV DOCKER_COMMIT_HASH=${commitHash} diff --git a/frontend/src/app/app.module.ts b/frontend/src/app/app.module.ts index 336cfead2..c9cd63c0c 100644 --- a/frontend/src/app/app.module.ts +++ b/frontend/src/app/app.module.ts @@ -40,7 +40,6 @@ import { AssetComponent } from './components/asset/asset.component'; import { AssetsComponent } from './components/assets/assets.component'; import { AssetsNavComponent } from './components/assets/assets-nav/assets-nav.component'; import { StatusViewComponent } from './components/status-view/status-view.component'; -import { MinerComponent } from './components/miner/miner.component'; import { SharedModule } from './shared/shared.module'; import { NgbTypeaheadModule } from '@ng-bootstrap/ng-bootstrap'; import { FeesBoxComponent } from './components/fees-box/fees-box.component'; @@ -108,7 +107,6 @@ import { BlockSizesWeightsGraphComponent } from './components/block-sizes-weight LbtcPegsGraphComponent, AssetComponent, AssetsComponent, - MinerComponent, StatusViewComponent, FeesBoxComponent, DashboardComponent, diff --git a/frontend/src/app/components/block/block.component.html b/frontend/src/app/components/block/block.component.html index 8b511b30c..87e45e088 100644 --- a/frontend/src/app/components/block/block.component.html +++ b/frontend/src/app/components/block/block.component.html @@ -81,15 +81,26 @@ Total fees - + + + + + + -   + +   + Subsidy + fees: - + + + + @@ -105,7 +116,18 @@ Miner - + + + {{ block.extras.pool.name }} + + + + + {{ block.extras.pool.name }} + + diff --git a/frontend/src/app/components/block/block.component.ts b/frontend/src/app/components/block/block.component.ts index 512b6b411..57417a5c3 100644 --- a/frontend/src/app/components/block/block.component.ts +++ b/frontend/src/app/components/block/block.component.ts @@ -10,6 +10,7 @@ import { SeoService } from 'src/app/services/seo.service'; import { WebsocketService } from 'src/app/services/websocket.service'; import { RelativeUrlPipe } from 'src/app/shared/pipes/relative-url/relative-url.pipe'; import { BlockExtended } from 'src/app/interfaces/node-api.interface'; +import { ApiService } from 'src/app/services/api.service'; @Component({ selector: 'app-block', @@ -31,7 +32,6 @@ export class BlockComponent implements OnInit, OnDestroy { blockSubsidy: number; fees: number; paginationMaxSize: number; - coinbaseTx: Transaction; page = 1; itemsPerPage: number; txsLoadingStatus$: Observable; @@ -50,10 +50,11 @@ export class BlockComponent implements OnInit, OnDestroy { private location: Location, private router: Router, private electrsApiService: ElectrsApiService, - private stateService: StateService, + public stateService: StateService, private seoService: SeoService, private websocketService: WebsocketService, private relativeUrlPipe: RelativeUrlPipe, + private apiService: ApiService ) { } ngOnInit() { @@ -88,7 +89,6 @@ export class BlockComponent implements OnInit, OnDestroy { const blockHash: string = params.get('id') || ''; this.block = undefined; this.page = 1; - this.coinbaseTx = undefined; this.error = undefined; this.fees = undefined; this.stateService.markBlock$.next({}); @@ -124,7 +124,7 @@ export class BlockComponent implements OnInit, OnDestroy { this.location.replaceState( this.router.createUrlTree([(this.network ? '/' + this.network : '') + '/block/', hash]).toString() ); - return this.electrsApiService.getBlock$(hash); + return this.apiService.getBlock$(hash); }) ); } @@ -134,7 +134,7 @@ export class BlockComponent implements OnInit, OnDestroy { return of(blockInCache); } - return this.electrsApiService.getBlock$(blockHash); + return this.apiService.getBlock$(blockHash); } }), tap((block: BlockExtended) => { @@ -145,7 +145,6 @@ export class BlockComponent implements OnInit, OnDestroy { this.seoService.setTitle($localize`:@@block.component.browser-title:Block ${block.height}:BLOCK_HEIGHT:: ${block.id}:BLOCK_ID:`); this.isLoadingBlock = false; - this.coinbaseTx = block?.extras?.coinbaseTx; this.setBlockSubsidy(); if (block?.extras?.reward !== undefined) { this.fees = block.extras.reward / 100000000 - this.blockSubsidy; @@ -167,9 +166,6 @@ export class BlockComponent implements OnInit, OnDestroy { if (this.fees === undefined && transactions[0]) { this.fees = transactions[0].vout.reduce((acc: number, curr: Vout) => acc + curr.value, 0) / 100000000 - this.blockSubsidy; } - if (!this.coinbaseTx && transactions[0]) { - this.coinbaseTx = transactions[0]; - } this.transactions = transactions; this.isLoadingTransactions = false; }, @@ -212,13 +208,10 @@ export class BlockComponent implements OnInit, OnDestroy { this.queryParamsSubscription.unsubscribe(); } + // TODO - Refactor this.fees/this.reward for liquid because it is not + // used anymore on Bitcoin networks (we use block.extras directly) setBlockSubsidy() { - if (this.network === 'liquid' || this.network === 'liquidtestnet') { - this.blockSubsidy = 0; - return; - } - const halvings = Math.floor(this.block.height / 210000); - this.blockSubsidy = 50 * 2 ** -halvings; + this.blockSubsidy = 0; } pageChange(page: number, target: HTMLElement) { diff --git a/frontend/src/app/components/blocks-list/blocks-list.component.html b/frontend/src/app/components/blocks-list/blocks-list.component.html index ee205aaf4..46d92f82f 100644 --- a/frontend/src/app/components/blocks-list/blocks-list.component.html +++ b/frontend/src/app/components/blocks-list/blocks-list.component.html @@ -29,7 +29,7 @@
+ onError="this.src = './resources/mining-pools/default.svg'" [alt]="'Logo of ' + block.extras.pool.name + ' mining pool'"> {{ block.extras.pool.name }} {{ block.extras.coinbaseRaw | hex2ascii }} diff --git a/frontend/src/app/components/master-page/master-page.component.html b/frontend/src/app/components/master-page/master-page.component.html index 98e490f37..1daabcfac 100644 --- a/frontend/src/app/components/master-page/master-page.component.html +++ b/frontend/src/app/components/master-page/master-page.component.html @@ -3,7 +3,7 @@