From 9f15e9a1d76a9a88f622a0890ac8e9e9a8e93415 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Wed, 1 Feb 2023 11:52:00 -0600 Subject: [PATCH 01/95] Fix blockchain scroll jumping on resize on mobile --- frontend/src/app/components/start/start.component.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/frontend/src/app/components/start/start.component.ts b/frontend/src/app/components/start/start.component.ts index d29372d97..20492669d 100644 --- a/frontend/src/app/components/start/start.component.ts +++ b/frontend/src/app/components/start/start.component.ts @@ -123,7 +123,7 @@ export class StartComponent implements OnInit, OnDestroy { this.minScrollWidth = this.firstPageWidth + (this.pageWidth * 2); if (firstVisibleBlock != null) { - this.scrollToBlock(firstVisibleBlock, offset); + this.scrollToBlock(firstVisibleBlock, offset + (this.isMobile ? this.blockWidth : 0)); } else { this.updatePages(); } @@ -178,8 +178,10 @@ export class StartComponent implements OnInit, OnDestroy { setTimeout(() => { this.scrollToBlock(height, blockOffset); }, 50); return; } - const targetHeight = this.isMobile ? height - 1 : height; - const viewingPageIndex = this.getPageIndexOf(targetHeight); + if (this.isMobile) { + blockOffset -= this.blockWidth; + } + const viewingPageIndex = this.getPageIndexOf(height); const pages = []; this.pageIndex = Math.max(viewingPageIndex - 1, 0); let viewingPage = this.getPageAt(viewingPageIndex); @@ -189,7 +191,7 @@ export class StartComponent implements OnInit, OnDestroy { viewingPage = this.getPageAt(viewingPageIndex); } const left = viewingPage.offset - this.getConvertedScrollOffset(); - const blockIndex = viewingPage.height - targetHeight; + const blockIndex = viewingPage.height - height; const targetOffset = (this.blockWidth * blockIndex) + left; let deltaOffset = targetOffset - blockOffset; From 77686821bae49f31a483facddc36eea40049927a Mon Sep 17 00:00:00 2001 From: Mononaut Date: Fri, 17 Feb 2023 10:12:33 -0600 Subject: [PATCH 02/95] Raise production memory limits Increases maxmempool production bitcoin.conf setting to 2GB, and raises `npm run start-production` nodejs memory limit to 8GB --- backend/package.json | 2 +- production/bitcoin.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/package.json b/backend/package.json index da72386fe..9c1b3a12a 100644 --- a/backend/package.json +++ b/backend/package.json @@ -27,7 +27,7 @@ "package": "npm run build && rm -rf package && mv dist package && mv node_modules package && npm run package-rm-build-deps", "package-rm-build-deps": "(cd package/node_modules; rm -r typescript @typescript-eslint)", "start": "node --max-old-space-size=2048 dist/index.js", - "start-production": "node --max-old-space-size=4096 dist/index.js", + "start-production": "node --max-old-space-size=16384 dist/index.js", "test": "./node_modules/.bin/jest --coverage", "lint": "./node_modules/.bin/eslint . --ext .ts", "lint:fix": "./node_modules/.bin/eslint . --ext .ts --fix", diff --git a/production/bitcoin.conf b/production/bitcoin.conf index 99fbaeed1..46ab41b20 100644 --- a/production/bitcoin.conf +++ b/production/bitcoin.conf @@ -5,7 +5,7 @@ listen=1 discover=1 par=16 dbcache=8192 -maxmempool=1337 +maxmempool=4096 mempoolexpiry=999999 maxconnections=42 onion=127.0.0.1:9050 From 224613e5ee8402d3bf159f1c8524fdd6ad6c871e Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Sat, 18 Feb 2023 21:17:58 +0900 Subject: [PATCH 03/95] Fix channel map not being updated (racing condition) --- .../nodes-channels-map/nodes-channels-map.component.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts b/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts index 4da4e3cb4..c998bdd2f 100644 --- a/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts +++ b/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts @@ -1,7 +1,7 @@ import { ChangeDetectionStrategy, Component, Input, Output, EventEmitter, NgZone, OnInit } from '@angular/core'; import { SeoService } from '../../services/seo.service'; import { ApiService } from '../../services/api.service'; -import { Observable, switchMap, tap, zip } from 'rxjs'; +import { delay, Observable, switchMap, tap, zip } from 'rxjs'; import { AssetsService } from '../../services/assets.service'; import { ActivatedRoute, ParamMap, Router } from '@angular/router'; import { RelativeUrlPipe } from '../../shared/pipes/relative-url/relative-url.pipe'; @@ -75,6 +75,7 @@ export class NodesChannelsMap implements OnInit { this.channelsObservable = this.activatedRoute.paramMap .pipe( + delay(100), switchMap((params: ParamMap) => { this.isLoading = true; if (this.style === 'channelpage' && this.channel.length === 0 || !this.hasLocation) { From 44865d18ae64e3ceb0f92f3fff19ac783da78a5a Mon Sep 17 00:00:00 2001 From: softsimon Date: Sun, 19 Feb 2023 17:24:43 +0700 Subject: [PATCH 04/95] Remove included in block --- .../app/components/transaction/transaction.component.html | 6 ------ 1 file changed, 6 deletions(-) diff --git a/frontend/src/app/components/transaction/transaction.component.html b/frontend/src/app/components/transaction/transaction.component.html index 64c31aece..f0b25d60e 100644 --- a/frontend/src/app/components/transaction/transaction.component.html +++ b/frontend/src/app/components/transaction/transaction.component.html @@ -61,12 +61,6 @@ - - Included in block - - {{ tx.status.block_height }} - - Confirmed From e0f149550af1fc20f6e358974869849a60fe56ba Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn <100320+knorrium@users.noreply.github.com> Date: Sun, 19 Feb 2023 13:50:12 -0800 Subject: [PATCH 05/95] Update node v16, LTS and current for CI --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 75884def7..d0520f59c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,7 @@ jobs: if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')" strategy: matrix: - node: ["16.16.0", "18.5.0"] + node: ["16.16.0", "18.14.1", "19.6.1"] flavor: ["dev", "prod"] fail-fast: false runs-on: "ubuntu-latest" @@ -55,7 +55,7 @@ jobs: if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')" strategy: matrix: - node: ["16.15.0", "18.5.0"] + node: ["16.16.0", "18.14.1", "19.6.1"] flavor: ["dev", "prod"] fail-fast: false runs-on: "ubuntu-latest" From ca15887e4b24d5e76fda4e9bada4216b70e748e4 Mon Sep 17 00:00:00 2001 From: wiz Date: Mon, 20 Feb 2023 15:58:07 +0900 Subject: [PATCH 06/95] ops: Fix another typo in build script credentials sed --- production/mempool-build-all | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/production/mempool-build-all b/production/mempool-build-all index 491c3e0b5..aa764da7d 100755 --- a/production/mempool-build-all +++ b/production/mempool-build-all @@ -106,7 +106,7 @@ build_backend() -e "s!__MEMPOOL_SIGNET_LIGHTNING_PASS__!${MEMPOOL_SIGNET_LIGHTNING_PASS}!" \ -e "s!__MEMPOOL_LIQUID_USER__!${MEMPOOL_LIQUID_USER}!" \ -e "s!__MEMPOOL_LIQUID_PASS__!${MEMPOOL_LIQUID_PASS}!" \ - -e "s!__MEMPOOL_LIQUIDTESTNET_USER__!${LIQUIDTESTNET_USER}!" \ + -e "s!__MEMPOOL_LIQUIDTESTNET_USER__!${MEMPOOL_LIQUIDTESTNET_USER}!" \ -e "s!__MEMPOOL_LIQUIDTESTNET_PASS__!${MEMPOOL_LIQUIDTESTNET_PASS}!" \ -e "s!__MEMPOOL_BISQ_USER__!${MEMPOOL_BISQ_USER}!" \ -e "s!__MEMPOOL_BISQ_PASS__!${MEMPOOL_BISQ_PASS}!" \ From 85251fcd5c950ae00b292dc020092a96f2df9d77 Mon Sep 17 00:00:00 2001 From: wiz Date: Mon, 20 Feb 2023 16:09:40 +0900 Subject: [PATCH 07/95] ops: Fix typo in mainnet lightning backend config --- production/mempool-config.mainnet-lightning.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/production/mempool-config.mainnet-lightning.json b/production/mempool-config.mainnet-lightning.json index a38509983..5bcadc584 100644 --- a/production/mempool-config.mainnet-lightning.json +++ b/production/mempool-config.mainnet-lightning.json @@ -45,6 +45,6 @@ "PORT": 3306, "USERNAME": "__MEMPOOL_MAINNET_LIGHTNING_USER__", "PASSWORD": "__MEMPOOL_MAINNET_LIGHTNING_PASS__", - "PASSWORD": "mempool_mainnet_lightning" + "DATABASE": "mempool_mainnet_lightning" } } From e3e7271c9d6e23e82a39fbdaec3476f21cc5647b Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Mon, 20 Feb 2023 17:31:07 +0900 Subject: [PATCH 08/95] Add avg mining pool block mathrate in pools stats API --- backend/src/api/mining/mining.ts | 1 + backend/src/mempool.interfaces.ts | 1 + backend/src/repositories/PoolsRepository.ts | 15 ++++++++++++--- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/backend/src/api/mining/mining.ts b/backend/src/api/mining/mining.ts index 83e810d43..8f698dfb6 100644 --- a/backend/src/api/mining/mining.ts +++ b/backend/src/api/mining/mining.ts @@ -100,6 +100,7 @@ class Mining { rank: rank++, emptyBlocks: emptyBlocksCount.length > 0 ? emptyBlocksCount[0]['count'] : 0, slug: poolInfo.slug, + avgMatchRate: Math.round(100 * poolInfo.avgMatchRate) / 100, }; poolsStats.push(poolStat); }); diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index f79786279..44bbf0ac2 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -16,6 +16,7 @@ export interface PoolInfo { link: string; blockCount: number; slug: string; + avgMatchRate: number; } export interface PoolStats extends PoolInfo { diff --git a/backend/src/repositories/PoolsRepository.ts b/backend/src/repositories/PoolsRepository.ts index c7cc6cba3..56cc2b3bc 100644 --- a/backend/src/repositories/PoolsRepository.ts +++ b/backend/src/repositories/PoolsRepository.ts @@ -27,16 +27,25 @@ class PoolsRepository { public async $getPoolsInfo(interval: string | null = null): Promise { interval = Common.getSqlInterval(interval); - let query = `SELECT COUNT(height) as blockCount, pool_id as poolId, pools.name as name, pools.link as link, slug + let query = ` + SELECT + COUNT(blocks.height) As blockCount, + pool_id AS poolId, + pools.name AS name, + pools.link AS link, + slug, + AVG(blocks_audits.match_rate) AS avgMatchRate FROM blocks - JOIN pools on pools.id = pool_id`; + JOIN pools on pools.id = pool_id + LEFT JOIN blocks_audits ON blocks_audits.height = blocks.height + `; if (interval) { query += ` WHERE blocks.blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`; } query += ` GROUP BY pool_id - ORDER BY COUNT(height) DESC`; + ORDER BY COUNT(blocks.height) DESC`; try { const [rows] = await DB.query(query); From 0dc2a598c3487223be8f2b24d98f017b444c9602 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Mon, 20 Feb 2023 17:47:45 +0900 Subject: [PATCH 09/95] Show avg block heath in pool ranking pie chart --- backend/src/api/mining/mining.ts | 2 +- backend/src/mempool.interfaces.ts | 2 +- .../pool-ranking/pool-ranking.component.html | 14 ++++++++++++++ .../pool-ranking/pool-ranking.component.ts | 6 ++++++ frontend/src/app/interfaces/node-api.interface.ts | 1 + 5 files changed, 23 insertions(+), 2 deletions(-) diff --git a/backend/src/api/mining/mining.ts b/backend/src/api/mining/mining.ts index 8f698dfb6..edcb5b2e5 100644 --- a/backend/src/api/mining/mining.ts +++ b/backend/src/api/mining/mining.ts @@ -100,7 +100,7 @@ class Mining { rank: rank++, emptyBlocks: emptyBlocksCount.length > 0 ? emptyBlocksCount[0]['count'] : 0, slug: poolInfo.slug, - avgMatchRate: Math.round(100 * poolInfo.avgMatchRate) / 100, + avgMatchRate: poolInfo.avgMatchRate !== null ? Math.round(100 * poolInfo.avgMatchRate) / 100 : null, }; poolsStats.push(poolStat); }); diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index 44bbf0ac2..6b258c173 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -16,7 +16,7 @@ export interface PoolInfo { link: string; blockCount: number; slug: string; - avgMatchRate: number; + avgMatchRate: number | null; } export interface PoolStats extends PoolInfo { diff --git a/frontend/src/app/components/pool-ranking/pool-ranking.component.html b/frontend/src/app/components/pool-ranking/pool-ranking.component.html index 48f2407e8..1d82867d8 100644 --- a/frontend/src/app/components/pool-ranking/pool-ranking.component.html +++ b/frontend/src/app/components/pool-ranking/pool-ranking.component.html @@ -92,6 +92,8 @@ Pool Hashrate Blocks + Avg Health Empty blocks @@ -105,6 +107,18 @@ {{ pool.lastEstimatedHashrate }} {{ miningStats.miningUnits.hashrateUnit }} {{ pool['blockText'] }} + + {{ pool.avgMatchRate }}% + + Unknown + + {{ pool.emptyBlocks }} ({{ pool.emptyBlockRatio }}%) diff --git a/frontend/src/app/components/pool-ranking/pool-ranking.component.ts b/frontend/src/app/components/pool-ranking/pool-ranking.component.ts index 214318bd5..135267b60 100644 --- a/frontend/src/app/components/pool-ranking/pool-ranking.component.ts +++ b/frontend/src/app/components/pool-ranking/pool-ranking.component.ts @@ -26,6 +26,8 @@ export class PoolRankingComponent implements OnInit { miningWindowPreference: string; radioGroupForm: UntypedFormGroup; + auditAvailable = false; + indexingAvailable = false; isLoading = true; chartOptions: EChartsOption = {}; chartInitOptions = { @@ -60,6 +62,10 @@ export class PoolRankingComponent implements OnInit { this.radioGroupForm = this.formBuilder.group({ dateSpan: this.miningWindowPreference }); this.radioGroupForm.controls.dateSpan.setValue(this.miningWindowPreference); + this.indexingAvailable = (this.stateService.env.BASE_MODULE === 'mempool' && + this.stateService.env.MINING_DASHBOARD === true); + this.auditAvailable = this.indexingAvailable && this.stateService.env.AUDIT; + this.route .fragment .subscribe((fragment) => { diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index c11cb5828..8fa30a723 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -73,6 +73,7 @@ export interface SinglePoolStats { emptyBlockRatio: string; logo: string; slug: string; + avgMatchRate: number; } export interface PoolsStats { blockCount: number; From 14be0fc54798724ce09b3f709e70e618837cf877 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Mon, 20 Feb 2023 18:13:56 +0900 Subject: [PATCH 10/95] Hide pool share on mobile in pool ranking --- .../app/components/pool-ranking/pool-ranking.component.html | 4 +++- .../app/components/pool-ranking/pool-ranking.component.ts | 6 ------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/frontend/src/app/components/pool-ranking/pool-ranking.component.html b/frontend/src/app/components/pool-ranking/pool-ranking.component.html index 1d82867d8..35ab709c5 100644 --- a/frontend/src/app/components/pool-ranking/pool-ranking.component.html +++ b/frontend/src/app/components/pool-ranking/pool-ranking.component.html @@ -106,7 +106,9 @@ {{ pool.name }} {{ pool.lastEstimatedHashrate }} {{ miningStats.miningUnits.hashrateUnit }} - {{ pool['blockText'] }} + + {{ pool.blockCount }} ({{ pool.share }}%) + { - data.pools = data.pools.map((pool: SinglePoolStats) => this.formatPoolUI(pool)); data['minersLuck'] = (100 * (data.blockCount / 1008)).toFixed(2); // luck 1w return data; }), @@ -110,11 +109,6 @@ export class PoolRankingComponent implements OnInit { ); } - formatPoolUI(pool: SinglePoolStats) { - pool['blockText'] = pool.blockCount.toString() + ` (${pool.share}%)`; - return pool; - } - generatePoolsChartSerieData(miningStats) { let poolShareThreshold = 0.5; if (isMobile()) { From 3c94755a695dc2b93a3e2650fe1e4fe8a5679789 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Tue, 21 Feb 2023 12:36:43 +0900 Subject: [PATCH 11/95] Use historical price for older blocks and transactions --- backend/src/api/mining/mining-routes.ts | 11 +- backend/src/repositories/PricesRepository.ts | 53 ++++++++ frontend/src/app/app.module.ts | 2 + .../components/amount/amount.component.html | 16 ++- .../app/components/amount/amount.component.ts | 2 + .../app/components/block/block.component.html | 14 +- .../app/components/block/block.component.ts | 20 ++- .../transaction/transaction.component.html | 2 +- .../transaction/transaction.component.ts | 18 ++- .../transactions-list.component.html | 12 +- .../transactions-list.component.ts | 8 ++ frontend/src/app/fiat/fiat.component.html | 15 ++- frontend/src/app/fiat/fiat.component.ts | 2 + .../src/app/interfaces/electrs.interface.ts | 2 + frontend/src/app/services/api.service.ts | 5 + frontend/src/app/services/price.service.ts | 121 ++++++++++++++++++ 16 files changed, 284 insertions(+), 19 deletions(-) create mode 100644 frontend/src/app/services/price.service.ts diff --git a/backend/src/api/mining/mining-routes.ts b/backend/src/api/mining/mining-routes.ts index 81c7b5a99..393ea119a 100644 --- a/backend/src/api/mining/mining-routes.ts +++ b/backend/src/api/mining/mining-routes.ts @@ -1,13 +1,13 @@ import { Application, Request, Response } from 'express'; import config from "../../config"; import logger from '../../logger'; -import audits from '../audit'; import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository'; import BlocksRepository from '../../repositories/BlocksRepository'; import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjustmentsRepository'; import HashratesRepository from '../../repositories/HashratesRepository'; import bitcoinClient from '../bitcoin/bitcoin-client'; import mining from "./mining"; +import PricesRepository from '../../repositories/PricesRepository'; class MiningRoutes { public initRoutes(app: Application) { @@ -32,9 +32,18 @@ class MiningRoutes { .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/score/:hash', this.$getBlockAuditScore) .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/:hash', this.$getBlockAudit) .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/timestamp/:timestamp', this.$getHeightFromTimestamp) + .get(config.MEMPOOL.API_URL_PREFIX + 'historical-price', this.$getHistoricalPrice) ; } + private async $getHistoricalPrice(req: Request, res: Response): Promise { + try { + res.status(200).send(await PricesRepository.$getHistoricalPrice()); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + private async $getPool(req: Request, res: Response): Promise { try { const stats = await mining.$getPoolStat(req.params.slug); diff --git a/backend/src/repositories/PricesRepository.ts b/backend/src/repositories/PricesRepository.ts index bc606e68b..639f16dc6 100644 --- a/backend/src/repositories/PricesRepository.ts +++ b/backend/src/repositories/PricesRepository.ts @@ -3,6 +3,31 @@ import logger from '../logger'; import { IConversionRates } from '../mempool.interfaces'; import priceUpdater from '../tasks/price-updater'; +export interface ApiPrice { + time?: number, + USD: number, + EUR: number, + GBP: number, + CAD: number, + CHF: number, + AUD: number, + JPY: number, +} + +export interface ExchangeRates { + USDEUR: number, + USDGBP: number, + USDCAD: number, + USDCHF: number, + USDAUD: number, + USDJPY: number, +} + +export interface Conversion { + prices: ApiPrice[], + exchangeRates: ExchangeRates; +} + class PricesRepository { public async $savePrices(time: number, prices: IConversionRates): Promise { if (prices.USD === 0) { @@ -60,6 +85,34 @@ class PricesRepository { } return rates[0]; } + + public async $getHistoricalPrice(): Promise { + try { + const [rates]: any[] = await DB.query(`SELECT *, UNIX_TIMESTAMP(time) as time FROM prices ORDER BY time DESC`); + if (!rates) { + throw Error(`Cannot get average historical price from the database`); + } + + // Compute fiat exchange rates + const latestPrice: ApiPrice = rates[0]; + const exchangeRates: ExchangeRates = { + USDEUR: Math.round(latestPrice.EUR / latestPrice.USD * 100) / 100, + USDGBP: Math.round(latestPrice.GBP / latestPrice.USD * 100) / 100, + USDCAD: Math.round(latestPrice.CAD / latestPrice.USD * 100) / 100, + USDCHF: Math.round(latestPrice.CHF / latestPrice.USD * 100) / 100, + USDAUD: Math.round(latestPrice.AUD / latestPrice.USD * 100) / 100, + USDJPY: Math.round(latestPrice.JPY / latestPrice.USD * 100) / 100, + }; + + return { + prices: rates, + exchangeRates: exchangeRates + }; + } catch (e) { + logger.err(`Cannot fetch averaged historical prices from the db. Reason ${e instanceof Error ? e.message : e}`); + return null; + } + } } export default new PricesRepository(); diff --git a/frontend/src/app/app.module.ts b/frontend/src/app/app.module.ts index f26b4a924..7afdba1f8 100644 --- a/frontend/src/app/app.module.ts +++ b/frontend/src/app/app.module.ts @@ -7,6 +7,7 @@ import { AppComponent } from './components/app/app.component'; import { ElectrsApiService } from './services/electrs-api.service'; import { StateService } from './services/state.service'; import { CacheService } from './services/cache.service'; +import { PriceService } from './services/price.service'; import { EnterpriseService } from './services/enterprise.service'; import { WebsocketService } from './services/websocket.service'; import { AudioService } from './services/audio.service'; @@ -26,6 +27,7 @@ const providers = [ ElectrsApiService, StateService, CacheService, + PriceService, WebsocketService, AudioService, SeoService, diff --git a/frontend/src/app/components/amount/amount.component.html b/frontend/src/app/components/amount/amount.component.html index 3526b554b..4a57e72e2 100644 --- a/frontend/src/app/components/amount/amount.component.html +++ b/frontend/src/app/components/amount/amount.component.html @@ -1,7 +1,19 @@ - {{ addPlus && satoshis >= 0 ? '+' : '' }}{{ (conversions ? conversions[currency] : 0) * satoshis / 100000000 | fiatCurrency : digitsInfo : currency }} + + {{ addPlus && satoshis >= 0 ? '+' : '' }} + {{ + ( + (blockConversion.price[currency] > 0 ? blockConversion.price[currency] : null) ?? + (blockConversion.price['USD'] * blockConversion.exchangeRates['USD' + currency]) ?? 0 + ) * satoshis / 100000000 | fiatCurrency : digitsInfo : currency + }} + + + {{ addPlus && satoshis >= 0 ? '+' : '' }}{{ (conversions ? conversions[currency] : 0) * satoshis / 100000000 | fiatCurrency : digitsInfo : currency }} + - + + Confidential diff --git a/frontend/src/app/components/amount/amount.component.ts b/frontend/src/app/components/amount/amount.component.ts index cfdc50468..927504012 100644 --- a/frontend/src/app/components/amount/amount.component.ts +++ b/frontend/src/app/components/amount/amount.component.ts @@ -1,6 +1,7 @@ import { Component, OnInit, OnDestroy, Input, ChangeDetectionStrategy, ChangeDetectorRef } from '@angular/core'; import { StateService } from '../../services/state.service'; import { Observable, Subscription } from 'rxjs'; +import { Price } from 'src/app/services/price.service'; @Component({ selector: 'app-amount', @@ -21,6 +22,7 @@ export class AmountComponent implements OnInit, OnDestroy { @Input() digitsInfo = '1.8-8'; @Input() noFiat = false; @Input() addPlus = false; + @Input() blockConversion: Price; constructor( private stateService: StateService, diff --git a/frontend/src/app/components/block/block.component.html b/frontend/src/app/components/block/block.component.html index 0fda470d6..bb4d2082c 100644 --- a/frontend/src/app/components/block/block.component.html +++ b/frontend/src/app/components/block/block.component.html @@ -124,7 +124,13 @@ Median fee - ~{{ block?.extras?.medianFee | number:'1.0-0' }} sat/vB + ~{{ block?.extras?.medianFee | number:'1.0-0' }} sat/vB + + + + @@ -132,13 +138,13 @@ - +   + [blockConversion]="blockConversion" [value]="fees * 100000000" digitsInfo="1.2-2"> @@ -147,7 +153,7 @@ - + diff --git a/frontend/src/app/components/block/block.component.ts b/frontend/src/app/components/block/block.component.ts index ba5dd8cf7..9e476ac61 100644 --- a/frontend/src/app/components/block/block.component.ts +++ b/frontend/src/app/components/block/block.component.ts @@ -13,6 +13,7 @@ import { BlockAudit, BlockExtended, TransactionStripped } from '../../interfaces import { ApiService } from '../../services/api.service'; import { BlockOverviewGraphComponent } from '../../components/block-overview-graph/block-overview-graph.component'; import { detectWebGL } from '../../shared/graphs.utils'; +import { PriceService, Price } from 'src/app/services/price.service'; @Component({ selector: 'app-block', @@ -81,6 +82,9 @@ export class BlockComponent implements OnInit, OnDestroy { timeLtr: boolean; childChangeSubscription: Subscription; auditPrefSubscription: Subscription; + + priceSubscription: Subscription; + blockConversion: Price; @ViewChildren('blockGraphProjected') blockGraphProjected: QueryList; @ViewChildren('blockGraphActual') blockGraphActual: QueryList; @@ -94,7 +98,8 @@ export class BlockComponent implements OnInit, OnDestroy { private seoService: SeoService, private websocketService: WebsocketService, private relativeUrlPipe: RelativeUrlPipe, - private apiService: ApiService + private apiService: ApiService, + private priceService: PriceService, ) { this.webGlEnabled = detectWebGL(); } @@ -432,6 +437,19 @@ export class BlockComponent implements OnInit, OnDestroy { } } }); + + if (this.priceSubscription) { + this.priceSubscription.unsubscribe(); + } + this.priceSubscription = block$.pipe( + switchMap((block) => { + return this.priceService.getPrices().pipe( + tap(() => { + this.blockConversion = this.priceService.getPriceForTimestamp(block.timestamp); + }) + ); + }) + ).subscribe(); } ngAfterViewInit(): void { diff --git a/frontend/src/app/components/transaction/transaction.component.html b/frontend/src/app/components/transaction/transaction.component.html index f0b25d60e..07234f7bc 100644 --- a/frontend/src/app/components/transaction/transaction.component.html +++ b/frontend/src/app/components/transaction/transaction.component.html @@ -469,7 +469,7 @@ Fee - {{ tx.fee | number }} sat + {{ tx.fee | number }} sat Fee rate diff --git a/frontend/src/app/components/transaction/transaction.component.ts b/frontend/src/app/components/transaction/transaction.component.ts index 7e1ae525e..06a4c5836 100644 --- a/frontend/src/app/components/transaction/transaction.component.ts +++ b/frontend/src/app/components/transaction/transaction.component.ts @@ -8,10 +8,11 @@ import { retryWhen, delay, map, - mergeMap + mergeMap, + tap } from 'rxjs/operators'; import { Transaction } from '../../interfaces/electrs.interface'; -import { of, merge, Subscription, Observable, Subject, timer, combineLatest, from, throwError } from 'rxjs'; +import { of, merge, Subscription, Observable, Subject, timer, from, throwError } from 'rxjs'; import { StateService } from '../../services/state.service'; import { CacheService } from '../../services/cache.service'; import { WebsocketService } from '../../services/websocket.service'; @@ -21,6 +22,7 @@ import { SeoService } from '../../services/seo.service'; import { BlockExtended, CpfpInfo } from '../../interfaces/node-api.interface'; import { LiquidUnblinding } from './liquid-ublinding'; import { RelativeUrlPipe } from '../../shared/pipes/relative-url/relative-url.pipe'; +import { Price, PriceService } from 'src/app/services/price.service'; @Component({ selector: 'app-transaction', @@ -69,7 +71,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { hideFlow: boolean = this.stateService.hideFlow.value; overrideFlowPreference: boolean = null; flowEnabled: boolean; - + blockConversion: Price; tooltipPosition: { x: number, y: number }; @ViewChild('graphContainer') @@ -85,7 +87,8 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { private websocketService: WebsocketService, private audioService: AudioService, private apiService: ApiService, - private seoService: SeoService + private seoService: SeoService, + private priceService: PriceService, ) {} ngOnInit() { @@ -323,6 +326,13 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { } this.fetchRbfHistory$.next(this.tx.txid); } + + this.priceService.getPrices().pipe( + tap(() => { + this.blockConversion = this.priceService.getPriceForTimestamp(tx.status.block_time); + }) + ).subscribe(); + setTimeout(() => { this.applyFragment(); }, 0); }, (error) => { diff --git a/frontend/src/app/components/transactions-list/transactions-list.component.html b/frontend/src/app/components/transactions-list/transactions-list.component.html index 7602e40cc..e5280d572 100644 --- a/frontend/src/app/components/transactions-list/transactions-list.component.html +++ b/frontend/src/app/components/transactions-list/transactions-list.component.html @@ -88,7 +88,7 @@ - + @@ -216,7 +216,7 @@ - + @@ -283,7 +283,9 @@
- {{ tx.fee / (tx.weight / 4) | feeRounding }} sat/vB  – {{ tx.fee | number }} sat + {{ tx.fee / (tx.weight / 4) | feeRounding }} sat/vB  – {{ tx.fee | number }} sat
Show more inputs to reveal fee data
@@ -301,12 +303,12 @@
diff --git a/frontend/src/app/components/transactions-list/transactions-list.component.ts b/frontend/src/app/components/transactions-list/transactions-list.component.ts index 67df2daa2..bfdaa02bc 100644 --- a/frontend/src/app/components/transactions-list/transactions-list.component.ts +++ b/frontend/src/app/components/transactions-list/transactions-list.component.ts @@ -9,6 +9,7 @@ import { AssetsService } from '../../services/assets.service'; import { filter, map, tap, switchMap } from 'rxjs/operators'; import { BlockExtended } from '../../interfaces/node-api.interface'; import { ApiService } from '../../services/api.service'; +import { PriceService } from 'src/app/services/price.service'; @Component({ selector: 'app-transactions-list', @@ -50,6 +51,7 @@ export class TransactionsListComponent implements OnInit, OnChanges { private apiService: ApiService, private assetsService: AssetsService, private ref: ChangeDetectorRef, + private priceService: PriceService, ) { } ngOnInit(): void { @@ -147,6 +149,12 @@ export class TransactionsListComponent implements OnInit, OnChanges { tx['addressValue'] = addressIn - addressOut; } + + this.priceService.getPrices().pipe( + tap(() => { + tx['price'] = this.priceService.getPriceForTimestamp(tx.status.block_time); + }) + ).subscribe(); }); const txIds = this.transactions.filter((tx) => !tx._outspends).map((tx) => tx.txid); if (txIds.length) { diff --git a/frontend/src/app/fiat/fiat.component.html b/frontend/src/app/fiat/fiat.component.html index 99a177cc0..a1bf79978 100644 --- a/frontend/src/app/fiat/fiat.component.html +++ b/frontend/src/app/fiat/fiat.component.html @@ -1 +1,14 @@ -{{ (conversions ? conversions[currency] : 0) * value / 100000000 | fiatCurrency : digitsInfo : currency }} \ No newline at end of file + + {{ + ( + (blockConversion.price[currency] > 0 ? blockConversion.price[currency] : null) ?? + (blockConversion.price['USD'] * blockConversion.exchangeRates['USD' + currency]) ?? 0 + ) * value / 100000000 | fiatCurrency : digitsInfo : currency + }} + + + + + {{ (conversions[currency] ?? conversions['USD'] ?? 0) * value / 100000000 | fiatCurrency : digitsInfo : currency }} + + \ No newline at end of file diff --git a/frontend/src/app/fiat/fiat.component.ts b/frontend/src/app/fiat/fiat.component.ts index bc0f6a0de..909b249c0 100644 --- a/frontend/src/app/fiat/fiat.component.ts +++ b/frontend/src/app/fiat/fiat.component.ts @@ -1,5 +1,6 @@ import { Component, OnInit, ChangeDetectionStrategy, Input, ChangeDetectorRef, OnDestroy } from '@angular/core'; import { Observable, Subscription } from 'rxjs'; +import { Price } from '../services/price.service'; import { StateService } from '../services/state.service'; @Component({ @@ -15,6 +16,7 @@ export class FiatComponent implements OnInit, OnDestroy { @Input() value: number; @Input() digitsInfo = '1.2-2'; + @Input() blockConversion: Price; constructor( private stateService: StateService, diff --git a/frontend/src/app/interfaces/electrs.interface.ts b/frontend/src/app/interfaces/electrs.interface.ts index 63dec7abd..dcccfb67c 100644 --- a/frontend/src/app/interfaces/electrs.interface.ts +++ b/frontend/src/app/interfaces/electrs.interface.ts @@ -1,3 +1,4 @@ +import { Price } from '../services/price.service'; import { IChannel } from './node-api.interface'; export interface Transaction { @@ -23,6 +24,7 @@ export interface Transaction { _deduced?: boolean; _outspends?: Outspend[]; _channels?: TransactionChannels; + price?: Price; } export interface TransactionChannels { diff --git a/frontend/src/app/services/api.service.ts b/frontend/src/app/services/api.service.ts index 04b2b72e2..2c74de361 100644 --- a/frontend/src/app/services/api.service.ts +++ b/frontend/src/app/services/api.service.ts @@ -6,6 +6,7 @@ import { Observable } from 'rxjs'; import { StateService } from './state.service'; import { WebsocketResponse } from '../interfaces/websocket.interface'; import { Outspend, Transaction } from '../interfaces/electrs.interface'; +import { Conversion } from './price.service'; @Injectable({ providedIn: 'root' @@ -303,4 +304,8 @@ export class ApiService { (style !== undefined ? `?style=${style}` : '') ); } + + getHistoricalPrice$(): Observable { + return this.httpClient.get( this.apiBaseUrl + this.apiBasePath + '/api/v1/historical-price'); + } } diff --git a/frontend/src/app/services/price.service.ts b/frontend/src/app/services/price.service.ts new file mode 100644 index 000000000..3320280e9 --- /dev/null +++ b/frontend/src/app/services/price.service.ts @@ -0,0 +1,121 @@ +import { Injectable } from '@angular/core'; +import { map, Observable, of, shareReplay } from 'rxjs'; +import { ApiService } from './api.service'; + +// nodejs backend interfaces +export interface ApiPrice { + time?: number, + USD: number, + EUR: number, + GBP: number, + CAD: number, + CHF: number, + AUD: number, + JPY: number, +} +export interface ExchangeRates { + USDEUR: number, + USDGBP: number, + USDCAD: number, + USDCHF: number, + USDAUD: number, + USDJPY: number, +} +export interface Conversion { + prices: ApiPrice[], + exchangeRates: ExchangeRates; +} + +// frontend interface +export interface Price { + price: ApiPrice, + exchangeRates: ExchangeRates, +} +export interface ConversionDict { + prices: { [timestamp: number]: ApiPrice } + exchangeRates: ExchangeRates; +} + +@Injectable({ + providedIn: 'root' +}) +export class PriceService { + historicalPrice: ConversionDict = { + prices: null, + exchangeRates: null, + }; + + constructor( + private apiService: ApiService + ) { + } + + getEmptyPrice(): Price { + return { + price: { + USD: 0, EUR: 0, GBP: 0, CAD: 0, CHF: 0, AUD: 0, JPY: 0, + }, + exchangeRates: { + USDEUR: 0, USDGBP: 0, USDCAD: 0, USDCHF: 0, USDAUD: 0, USDJPY: 0, + }, + }; + } + + /** + * Fetch prices from the nodejs backend only once + */ + getPrices(): Observable { + if (this.historicalPrice.prices) { + return of(null); + } + + return this.apiService.getHistoricalPrice$().pipe( + map((conversion: Conversion) => { + if (!this.historicalPrice.prices) { + this.historicalPrice.prices = Object(); + } + for (const price of conversion.prices) { + this.historicalPrice.prices[price.time] = { + USD: price.USD, EUR: price.EUR, GBP: price.GBP, CAD: price.CAD, + CHF: price.CHF, AUD: price.AUD, JPY: price.JPY + }; + } + this.historicalPrice.exchangeRates = conversion.exchangeRates; + return; + }), + shareReplay(), + ); + } + + /** + * Note: The first block with a price we have is block 68952 (using MtGox price history) + * + * @param blockTimestamp + */ + getPriceForTimestamp(blockTimestamp: number): Price | null { + const priceTimestamps = Object.keys(this.historicalPrice.prices); + priceTimestamps.push(Number.MAX_SAFE_INTEGER.toString()); + priceTimestamps.sort().reverse(); + + // Small trick here. Because latest blocks have higher timestamps than our + // latest price timestamp (we only insert once every hour), we have no price for them. + // Therefore we want to fallback to the websocket price by returning an undefined `price` field. + // Since this.historicalPrice.prices[Number.MAX_SAFE_INTEGER] does not exists + // it will return `undefined` and automatically use the websocket price. + // This way we can differenciate blocks without prices like the genesis block + // vs ones without a price (yet) like the latest blocks + + for (const t of priceTimestamps) { + const priceTimestamp = parseInt(t, 10); + if (blockTimestamp > priceTimestamp) { + return { + price: this.historicalPrice.prices[priceTimestamp], + exchangeRates: this.historicalPrice.exchangeRates, + }; + } + } + + return this.getEmptyPrice(); + } +} + From c949fee49f9e61188b22f5153b36782de47327c4 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Tue, 21 Feb 2023 17:25:39 +0900 Subject: [PATCH 12/95] Fix liquid infinite scrolling --- .../blockchain-blocks/blockchain-blocks.component.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html b/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html index 17c5709fd..d943c7ef5 100644 --- a/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html +++ b/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html @@ -14,7 +14,7 @@ }}
-
+
~{{ block?.extras?.medianFee | number:feeRounding }} sat/vB
From de117e30f2034a77a0c7b926824410b95e5f5dc0 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Tue, 21 Feb 2023 17:51:46 +0900 Subject: [PATCH 13/95] Use mempool chart color palette on hashrate history --- .../hashrates-chart-pools/hashrate-chart-pools.component.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.ts b/frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.ts index ed3683e9b..df7780fee 100644 --- a/frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.ts +++ b/frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.ts @@ -5,7 +5,7 @@ import { delay, map, retryWhen, share, startWith, switchMap, tap } from 'rxjs/op import { ApiService } from '../../services/api.service'; import { SeoService } from '../../services/seo.service'; import { UntypedFormBuilder, UntypedFormGroup } from '@angular/forms'; -import { poolsColor } from '../../app.constants'; +import { chartColors, poolsColor } from '../../app.constants'; import { StorageService } from '../../services/storage.service'; import { MiningService } from '../../services/mining.service'; import { download } from '../../shared/graphs.utils'; @@ -173,6 +173,7 @@ export class HashrateChartPoolsComponent implements OnInit { this.chartOptions = { title: title, animation: false, + color: chartColors, grid: { right: this.right, left: this.left, From b2162130d9354d2125677f2b76e9955b5fb29c1d Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Tue, 21 Feb 2023 18:09:41 +0900 Subject: [PATCH 14/95] Keep block alignment if fee/fee-range is missing --- .../blockchain-blocks.component.html | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html b/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html index d943c7ef5..400016d02 100644 --- a/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html +++ b/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html @@ -14,20 +14,26 @@ }}
-
+
~{{ block?.extras?.medianFee | number:feeRounding }} sat/vB
+ +
+   +
+
+ *ngIf="block?.extras?.feeRange; else emptyfeespan"> {{ block?.extras?.feeRange?.[1] | number:feeRounding }} - {{ block?.extras?.feeRange[block?.extras?.feeRange?.length - 1] | number:feeRounding }} sat/vB
-
-   -
+ +
+   +
+
From 952e540d65e7c3a96ca3f323d0b703ee074aa43a Mon Sep 17 00:00:00 2001 From: wiz Date: Tue, 21 Feb 2023 18:41:12 +0900 Subject: [PATCH 15/95] ops: Add 250ms delay between warm cache requests --- production/nginx-cache-warmer | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/production/nginx-cache-warmer b/production/nginx-cache-warmer index 27d1e3a8f..db025a137 100755 --- a/production/nginx-cache-warmer +++ b/production/nginx-cache-warmer @@ -120,14 +120,16 @@ do for url in / \ do warm "https://${hostname}${url}" + sleep 0.25 # wait 250ms between queries to not DoS mariadb done for slug in $slugs do warm "https://${hostname}/api/v1/mining/pool/${slug}" + sleep 0.25 # wait 250ms between queries to not DoS mariadb warm "https://${hostname}/api/v1/mining/pool/${slug}/hashrate" + sleep 0.25 # wait 250ms between queries to not DoS mariadb warm "https://${hostname}/api/v1/mining/pool/${slug}/blocks" + sleep 0.25 # wait 250ms between queries to not DoS mariadb done - - sleep 10 done From cf1bf9f0c578cd3272c0ee305affa0bd1067862e Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Tue, 21 Feb 2023 18:47:27 +0900 Subject: [PATCH 16/95] Add missing formControl name --- .../block-rewards-graph/block-rewards-graph.component.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.html b/frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.html index 4134f6310..a1cfdc427 100644 --- a/frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.html +++ b/frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.html @@ -31,7 +31,7 @@ 3Y
From 05594675c0f49e2ff6dc24c12cdca500297ff80b Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Tue, 21 Feb 2023 18:48:09 +0900 Subject: [PATCH 17/95] Disable timespan controls while isLoading --- .../block-fee-rates-graph.component.html | 2 +- .../block-fee-rates-graph.component.scss | 5 +++++ .../block-fees-graph/block-fees-graph.component.html | 2 +- .../block-fees-graph/block-fees-graph.component.scss | 5 +++++ .../block-fees-graph/block-fees-graph.component.ts | 5 +++-- .../block-prediction-graph.component.html | 2 +- .../block-prediction-graph.component.scss | 5 +++++ .../block-rewards-graph/block-rewards-graph.component.html | 2 +- .../block-rewards-graph/block-rewards-graph.component.scss | 5 +++++ .../block-rewards-graph/block-rewards-graph.component.ts | 2 +- .../block-sizes-weights-graph.component.html | 2 +- .../block-sizes-weights-graph.component.scss | 5 +++++ .../components/hashrate-chart/hashrate-chart.component.html | 2 +- .../components/hashrate-chart/hashrate-chart.component.scss | 5 +++++ .../hashrate-chart-pools.component.html | 2 +- .../hashrate-chart-pools.component.scss | 5 +++++ .../app/components/pool-ranking/pool-ranking.component.html | 4 ++-- .../app/components/pool-ranking/pool-ranking.component.scss | 5 +++++ .../app/components/pool-ranking/pool-ranking.component.ts | 1 + 19 files changed, 54 insertions(+), 12 deletions(-) diff --git a/frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.html b/frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.html index 679854535..77c35cea8 100644 --- a/frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.html +++ b/frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.html @@ -10,7 +10,7 @@
-
+
diff --git a/frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.scss b/frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.scss index f916bfc79..ec1755e7d 100644 --- a/frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.scss +++ b/frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.scss @@ -78,3 +78,8 @@ } } } + +.disabled { + pointer-events: none; + opacity: 0.5; +} \ No newline at end of file diff --git a/frontend/src/app/components/block-fees-graph/block-fees-graph.component.html b/frontend/src/app/components/block-fees-graph/block-fees-graph.component.html index cbe742ed4..76071be96 100644 --- a/frontend/src/app/components/block-fees-graph/block-fees-graph.component.html +++ b/frontend/src/app/components/block-fees-graph/block-fees-graph.component.html @@ -10,7 +10,7 @@
-
+
diff --git a/frontend/src/app/components/block-fees-graph/block-fees-graph.component.scss b/frontend/src/app/components/block-fees-graph/block-fees-graph.component.scss index f916bfc79..ec1755e7d 100644 --- a/frontend/src/app/components/block-fees-graph/block-fees-graph.component.scss +++ b/frontend/src/app/components/block-fees-graph/block-fees-graph.component.scss @@ -78,3 +78,8 @@ } } } + +.disabled { + pointer-events: none; + opacity: 0.5; +} \ No newline at end of file diff --git a/frontend/src/app/components/block-fees-graph/block-fees-graph.component.ts b/frontend/src/app/components/block-fees-graph/block-fees-graph.component.ts index 42667126f..96bd0697c 100644 --- a/frontend/src/app/components/block-fees-graph/block-fees-graph.component.ts +++ b/frontend/src/app/components/block-fees-graph/block-fees-graph.component.ts @@ -4,9 +4,9 @@ import { Observable, Subscription } from 'rxjs'; import { map, share, startWith, switchMap, tap } from 'rxjs/operators'; import { ApiService } from '../../services/api.service'; import { SeoService } from '../../services/seo.service'; -import { formatCurrency, formatNumber, getCurrencySymbol } from '@angular/common'; +import { formatNumber } from '@angular/common'; import { UntypedFormBuilder, UntypedFormGroup } from '@angular/forms'; -import { download, formatterXAxis, formatterXAxisLabel, formatterXAxisTimeCategory } from '../../shared/graphs.utils'; +import { download, formatterXAxis } from '../../shared/graphs.utils'; import { StateService } from '../../services/state.service'; import { StorageService } from '../../services/storage.service'; import { MiningService } from '../../services/mining.service'; @@ -92,6 +92,7 @@ export class BlockFeesGraphComponent implements OnInit { .pipe( startWith(this.radioGroupForm.controls.dateSpan.value), switchMap((timespan) => { + this.isLoading = true; this.storageService.setValue('miningWindowPreference', timespan); this.timespan = timespan; this.isLoading = true; diff --git a/frontend/src/app/components/block-prediction-graph/block-prediction-graph.component.html b/frontend/src/app/components/block-prediction-graph/block-prediction-graph.component.html index fc67b5d98..7dcd81c69 100644 --- a/frontend/src/app/components/block-prediction-graph/block-prediction-graph.component.html +++ b/frontend/src/app/components/block-prediction-graph/block-prediction-graph.component.html @@ -10,7 +10,7 @@
-
+
diff --git a/frontend/src/app/components/block-prediction-graph/block-prediction-graph.component.scss b/frontend/src/app/components/block-prediction-graph/block-prediction-graph.component.scss index f916bfc79..ec1755e7d 100644 --- a/frontend/src/app/components/block-prediction-graph/block-prediction-graph.component.scss +++ b/frontend/src/app/components/block-prediction-graph/block-prediction-graph.component.scss @@ -78,3 +78,8 @@ } } } + +.disabled { + pointer-events: none; + opacity: 0.5; +} \ No newline at end of file diff --git a/frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.html b/frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.html index a1cfdc427..198153583 100644 --- a/frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.html +++ b/frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.html @@ -11,7 +11,7 @@
-
+
diff --git a/frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.scss b/frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.scss index f916bfc79..ec1755e7d 100644 --- a/frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.scss +++ b/frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.scss @@ -78,3 +78,8 @@ } } } + +.disabled { + pointer-events: none; + opacity: 0.5; +} \ No newline at end of file diff --git a/frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.ts b/frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.ts index 0e5e339fa..ca1853633 100644 --- a/frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.ts +++ b/frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.ts @@ -80,7 +80,7 @@ export class BlockRewardsGraphComponent implements OnInit { this.route .fragment .subscribe((fragment) => { - if (['3m', '6m', '1y', '2y', '3y', 'all'].indexOf(fragment) > -1) { + if (['1m', '3m', '6m', '1y', '2y', '3y', 'all'].indexOf(fragment) > -1) { this.radioGroupForm.controls.dateSpan.setValue(fragment, { emitEvent: false }); } }); diff --git a/frontend/src/app/components/block-sizes-weights-graph/block-sizes-weights-graph.component.html b/frontend/src/app/components/block-sizes-weights-graph/block-sizes-weights-graph.component.html index b6787a3cc..122b5e7ca 100644 --- a/frontend/src/app/components/block-sizes-weights-graph/block-sizes-weights-graph.component.html +++ b/frontend/src/app/components/block-sizes-weights-graph/block-sizes-weights-graph.component.html @@ -9,7 +9,7 @@
-
+
diff --git a/frontend/src/app/components/block-sizes-weights-graph/block-sizes-weights-graph.component.scss b/frontend/src/app/components/block-sizes-weights-graph/block-sizes-weights-graph.component.scss index e5e4bfd9a..85765e0e1 100644 --- a/frontend/src/app/components/block-sizes-weights-graph/block-sizes-weights-graph.component.scss +++ b/frontend/src/app/components/block-sizes-weights-graph/block-sizes-weights-graph.component.scss @@ -78,3 +78,8 @@ } } } + +.disabled { + pointer-events: none; + opacity: 0.5; +} \ No newline at end of file diff --git a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.html b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.html index cf22548a6..83f8a3a4c 100644 --- a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.html +++ b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.html @@ -31,7 +31,7 @@
-
+
diff --git a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.scss b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.scss index 8718caf9b..154d46fa6 100644 --- a/frontend/src/app/components/hashrate-chart/hashrate-chart.component.scss +++ b/frontend/src/app/components/hashrate-chart/hashrate-chart.component.scss @@ -131,4 +131,9 @@ display: block; max-width: 80px; margin: 15px auto 3px; +} + +.disabled { + pointer-events: none; + opacity: 0.5; } \ No newline at end of file diff --git a/frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.html b/frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.html index 107e30147..bbdc745fe 100644 --- a/frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.html +++ b/frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.html @@ -11,7 +11,7 @@
-
+
diff --git a/frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.scss b/frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.scss index b59e21af3..00414e4df 100644 --- a/frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.scss +++ b/frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.scss @@ -82,3 +82,8 @@ .loadingGraphs.widget { top: 75%; } + +.disabled { + pointer-events: none; + opacity: 0.5; +} \ No newline at end of file diff --git a/frontend/src/app/components/pool-ranking/pool-ranking.component.html b/frontend/src/app/components/pool-ranking/pool-ranking.component.html index 35ab709c5..32a186fb8 100644 --- a/frontend/src/app/components/pool-ranking/pool-ranking.component.html +++ b/frontend/src/app/components/pool-ranking/pool-ranking.component.html @@ -40,7 +40,7 @@
-
+
@@ -104,7 +104,7 @@ {{ pool.name }} - {{ pool.lastEstimatedHashrate }} {{ + {{ pool.lastEstimatedHashrate }} {{ miningStats.miningUnits.hashrateUnit }} {{ pool.blockCount }} ({{ pool.share }}%) diff --git a/frontend/src/app/components/pool-ranking/pool-ranking.component.scss b/frontend/src/app/components/pool-ranking/pool-ranking.component.scss index 8cb82d92d..277c7d4ad 100644 --- a/frontend/src/app/components/pool-ranking/pool-ranking.component.scss +++ b/frontend/src/app/components/pool-ranking/pool-ranking.component.scss @@ -139,3 +139,8 @@ max-width: 80px; margin: 15px auto 3px; } + +.disabled { + pointer-events: none; + opacity: 0.5; +} \ No newline at end of file diff --git a/frontend/src/app/components/pool-ranking/pool-ranking.component.ts b/frontend/src/app/components/pool-ranking/pool-ranking.component.ts index ed27b6de7..8284d81da 100644 --- a/frontend/src/app/components/pool-ranking/pool-ranking.component.ts +++ b/frontend/src/app/components/pool-ranking/pool-ranking.component.ts @@ -79,6 +79,7 @@ export class PoolRankingComponent implements OnInit { .pipe( startWith(this.radioGroupForm.controls.dateSpan.value), // (trigger when the page loads) tap((value) => { + this.isLoading = true; this.timespan = value; if (!this.widget) { this.storageService.setValue('miningWindowPreference', value); From 8f51e20b2e944fde17ddefd587703e6b744540f9 Mon Sep 17 00:00:00 2001 From: softsimon Date: Tue, 21 Feb 2023 17:14:51 +0400 Subject: [PATCH 18/95] Fixes a syntax error with certain keywords --- backend/src/api/explorer/nodes.api.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/backend/src/api/explorer/nodes.api.ts b/backend/src/api/explorer/nodes.api.ts index d292aa2e3..0d641d3cf 100644 --- a/backend/src/api/explorer/nodes.api.ts +++ b/backend/src/api/explorer/nodes.api.ts @@ -362,7 +362,12 @@ class NodesApi { public async $searchNodeByPublicKeyOrAlias(search: string) { try { const publicKeySearch = search.replace('%', '') + '%'; - const aliasSearch = search.replace(/[-_.]/g, ' ').replace(/[^a-zA-Z0-9 ]/g, '').split(' ').map((search) => '+' + search + '*').join(' '); + const aliasSearch = search + .replace(/[-_.]/g, ' ') + .replace(/[^a-zA-Z0-9 ]/g, '') + .split(' ') + .filter(key => key.length) + .map((search) => '+' + search + '*').join(' '); const query = `SELECT public_key, alias, capacity, channels, status FROM nodes WHERE public_key LIKE ? OR MATCH alias_search AGAINST (? IN BOOLEAN MODE) ORDER BY capacity DESC LIMIT 10`; const [rows]: any = await DB.query(query, [publicKeySearch, aliasSearch]); return rows; From e8d54f254bfea23d0586f9bd9bc077a23f321f68 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 21 Feb 2023 21:59:31 -0600 Subject: [PATCH 19/95] pixel-perfect flow diagrams --- .../tx-bowtie-graph/tx-bowtie-graph.component.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.ts b/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.ts index 49d97dd40..cd1fc6855 100644 --- a/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.ts +++ b/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.ts @@ -196,8 +196,8 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges { this.outputs = this.initLines('out', voutWithFee, totalValue, this.maxStrands); this.middle = { - path: `M ${(this.width / 2) - this.midWidth} ${(this.height / 2) + 0.5} L ${(this.width / 2) + this.midWidth} ${(this.height / 2) + 0.5}`, - style: `stroke-width: ${this.combinedWeight + 1}; stroke: ${this.gradient[1]}` + path: `M ${(this.width / 2) - this.midWidth} ${(this.height / 2) + 0.25} L ${(this.width / 2) + this.midWidth} ${(this.height / 2) + 0.25}`, + style: `stroke-width: ${this.combinedWeight + 0.5}; stroke: ${this.gradient[1]}` }; this.hasLine = this.inputs.reduce((line, put) => line || !put.zeroValue, false) @@ -254,7 +254,7 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges { const lineParams = weights.map((w, i) => { return { weight: w, - thickness: xputs[i].value === 0 ? this.zeroValueThickness : Math.max(this.minWeight - 1, w) + 1, + thickness: xputs[i].value === 0 ? this.zeroValueThickness : Math.min(this.combinedWeight + 0.5, Math.max(this.minWeight - 1, w) + 1), offset: 0, innerY: 0, outerY: 0, @@ -266,7 +266,7 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges { // bounds of the middle segment const innerTop = (this.height / 2) - (this.combinedWeight / 2); - const innerBottom = innerTop + this.combinedWeight; + const innerBottom = innerTop + this.combinedWeight + 0.5; // tracks the visual bottom of the endpoints of the previous line let lastOuter = 0; let lastInner = innerTop; @@ -291,7 +291,7 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges { // set the vertical position of the (center of the) outer side of the line line.outerY = lastOuter + (line.thickness / 2); - line.innerY = Math.min(innerBottom + (line.thickness / 2), Math.max(innerTop + (line.thickness / 2), lastInner + (line.weight / 2))); + line.innerY = Math.min(innerBottom - (line.thickness / 2), Math.max(innerTop + (line.thickness / 2), lastInner + (line.weight / 2))); // special case to center single input/outputs if (xputs.length === 1) { From 2ff930ef3e4c8ce0792e7eaaf788b9a4bd1c5cd9 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 21 Feb 2023 22:01:30 -0600 Subject: [PATCH 20/95] Ignore coinbase tx in block health calculation --- backend/src/api/audit.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/src/api/audit.ts b/backend/src/api/audit.ts index b6b36dbdc..5b67dc965 100644 --- a/backend/src/api/audit.ts +++ b/backend/src/api/audit.ts @@ -119,7 +119,8 @@ class Audit { } const numCensored = Object.keys(isCensored).length; - const score = matches.length > 0 ? (matches.length / (matches.length + numCensored)) : 0; + const numMatches = matches.length - 1; // adjust for coinbase tx + const score = numMatches > 0 ? (numMatches / (numMatches + numCensored)) : 0; return { censored: Object.keys(isCensored), From 32b38e6cd1a4748b8f99b26bf9c8fe6f0b678348 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 21 Feb 2023 22:07:12 -0600 Subject: [PATCH 21/95] increase size of mempool_byte_weight db column --- backend/src/api/database-migration.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 66216c64a..a0200c98c 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository'; import { RowDataPacket } from 'mysql2'; class DatabaseMigration { - private static currentVersion = 52; + private static currentVersion = 53; private queryTimeout = 3600_000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -468,6 +468,11 @@ class DatabaseMigration { logger.warn('' + (e instanceof Error ? e.message : e)); } } + + if (databaseSchemaVersion < 53) { + await this.$executeQuery('ALTER TABLE statistics MODIFY mempool_byte_weight bigint(20) UNSIGNED NOT NULL'); + await this.updateToSchemaVersion(53); + } } /** From 437350aaff352ec504053f9f333648b13cbbfff6 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Wed, 22 Feb 2023 14:06:08 +0900 Subject: [PATCH 22/95] Unknown pool color #FDD835 --- frontend/src/app/app.constants.ts | 2 +- .../hashrates-chart-pools/hashrate-chart-pools.component.ts | 2 +- .../src/app/components/pool-ranking/pool-ranking.component.ts | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/frontend/src/app/app.constants.ts b/frontend/src/app/app.constants.ts index 8a954166e..779eab62e 100644 --- a/frontend/src/app/app.constants.ts +++ b/frontend/src/app/app.constants.ts @@ -72,7 +72,7 @@ export const chartColors = [ ]; export const poolsColor = { - 'unknown': '#9C9C9C', + 'unknown': '#FDD835', }; export const feeLevels = [1, 2, 3, 4, 5, 6, 8, 10, 12, 15, 20, 30, 40, 50, 60, 70, 80, 90, 100, 125, 150, 175, 200, diff --git a/frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.ts b/frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.ts index df7780fee..e7e3685d3 100644 --- a/frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.ts +++ b/frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.ts @@ -173,7 +173,7 @@ export class HashrateChartPoolsComponent implements OnInit { this.chartOptions = { title: title, animation: false, - color: chartColors, + color: chartColors.filter(color => color !== '#FDD835'), grid: { right: this.right, left: this.left, diff --git a/frontend/src/app/components/pool-ranking/pool-ranking.component.ts b/frontend/src/app/components/pool-ranking/pool-ranking.component.ts index 8284d81da..a3f23c031 100644 --- a/frontend/src/app/components/pool-ranking/pool-ranking.component.ts +++ b/frontend/src/app/components/pool-ranking/pool-ranking.component.ts @@ -4,7 +4,6 @@ import { ActivatedRoute, Router } from '@angular/router'; import { EChartsOption, PieSeriesOption } from 'echarts'; import { concat, Observable } from 'rxjs'; import { map, share, startWith, switchMap, tap } from 'rxjs/operators'; -import { SinglePoolStats } from '../../interfaces/node-api.interface'; import { SeoService } from '../../services/seo.service'; import { StorageService } from '../..//services/storage.service'; import { MiningService, MiningStats } from '../../services/mining.service'; @@ -220,7 +219,7 @@ export class PoolRankingComponent implements OnInit { this.chartOptions = { animation: false, - color: chartColors, + color: chartColors.filter(color => color !== '#FDD835'), tooltip: { trigger: 'item', textStyle: { From b6792784e85caac81a65d2a408881b7fc759c51d Mon Sep 17 00:00:00 2001 From: softsimon Date: Wed, 22 Feb 2023 10:19:19 +0400 Subject: [PATCH 23/95] Adding regex comments. --- backend/src/api/explorer/nodes.api.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/api/explorer/nodes.api.ts b/backend/src/api/explorer/nodes.api.ts index 0d641d3cf..36bbf26d2 100644 --- a/backend/src/api/explorer/nodes.api.ts +++ b/backend/src/api/explorer/nodes.api.ts @@ -363,8 +363,8 @@ class NodesApi { try { const publicKeySearch = search.replace('%', '') + '%'; const aliasSearch = search - .replace(/[-_.]/g, ' ') - .replace(/[^a-zA-Z0-9 ]/g, '') + .replace(/[-_.]/g, ' ') // Replace all -_. characters with empty space. Eg: "ln.nicehash" becomes "ln nicehash". + .replace(/[^a-zA-Z0-9 ]/g, '') // Remove all special characters and keep just A to Z, 0 to 9. .split(' ') .filter(key => key.length) .map((search) => '+' + search + '*').join(' '); From 2246a6f3cefba9f789f9192d533a29afab8d5bba Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Wed, 22 Feb 2023 20:42:32 +0900 Subject: [PATCH 24/95] Fix 'NaN' price for unconfirmed transaction since we have no block timestamp --- frontend/src/app/services/price.service.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frontend/src/app/services/price.service.ts b/frontend/src/app/services/price.service.ts index 3320280e9..fe6d67bb6 100644 --- a/frontend/src/app/services/price.service.ts +++ b/frontend/src/app/services/price.service.ts @@ -93,6 +93,10 @@ export class PriceService { * @param blockTimestamp */ getPriceForTimestamp(blockTimestamp: number): Price | null { + if (!blockTimestamp) { + return undefined; + } + const priceTimestamps = Object.keys(this.historicalPrice.prices); priceTimestamps.push(Number.MAX_SAFE_INTEGER.toString()); priceTimestamps.sort().reverse(); From f44eacd5d5d428afaae151320735f078714f1895 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Thu, 23 Feb 2023 09:50:34 +0900 Subject: [PATCH 25/95] Redo/Fix completely failed PR #3092 + add PR #3105 --- backend/src/repositories/PricesRepository.ts | 18 ++++ backend/src/tasks/price-updater.ts | 20 ++--- .../components/amount/amount.component.html | 2 +- .../app/components/block/block.component.ts | 6 +- .../transaction/transaction.component.ts | 6 +- .../transactions-list.component.ts | 8 +- frontend/src/app/fiat/fiat.component.html | 2 +- frontend/src/app/services/price.service.ts | 90 +++++++++---------- 8 files changed, 79 insertions(+), 73 deletions(-) diff --git a/backend/src/repositories/PricesRepository.ts b/backend/src/repositories/PricesRepository.ts index 639f16dc6..f460a504f 100644 --- a/backend/src/repositories/PricesRepository.ts +++ b/backend/src/repositories/PricesRepository.ts @@ -28,6 +28,16 @@ export interface Conversion { exchangeRates: ExchangeRates; } +export const MAX_PRICES = { + USD: 100000000, + EUR: 100000000, + GBP: 100000000, + CAD: 100000000, + CHF: 100000000, + AUD: 100000000, + JPY: 10000000000, +}; + class PricesRepository { public async $savePrices(time: number, prices: IConversionRates): Promise { if (prices.USD === 0) { @@ -36,6 +46,14 @@ class PricesRepository { return; } + // Sanity check + for (const currency of Object.keys(prices)) { + if (prices[currency] < -1 || prices[currency] > MAX_PRICES[currency]) { // We use -1 to mark a "missing data, so it's a valid entry" + logger.info(`Ignore BTC${currency} price of ${prices[currency]}`); + prices[currency] = 0; + } + } + try { await DB.query(` INSERT INTO prices(time, USD, EUR, GBP, CAD, CHF, AUD, JPY) diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index 939a1ea85..b39e152ae 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -3,7 +3,7 @@ import path from 'path'; import config from '../config'; import logger from '../logger'; import { IConversionRates } from '../mempool.interfaces'; -import PricesRepository from '../repositories/PricesRepository'; +import PricesRepository, { MAX_PRICES } from '../repositories/PricesRepository'; import BitfinexApi from './price-feeds/bitfinex-api'; import BitflyerApi from './price-feeds/bitflyer-api'; import CoinbaseApi from './price-feeds/coinbase-api'; @@ -46,13 +46,13 @@ class PriceUpdater { public getEmptyPricesObj(): IConversionRates { return { - USD: 0, - EUR: 0, - GBP: 0, - CAD: 0, - CHF: 0, - AUD: 0, - JPY: 0, + USD: -1, + EUR: -1, + GBP: -1, + CAD: -1, + CHF: -1, + AUD: -1, + JPY: -1, }; } @@ -115,7 +115,7 @@ class PriceUpdater { if (feed.currencies.includes(currency)) { try { const price = await feed.$fetchPrice(currency); - if (price > 0) { + if (price > -1 && price < MAX_PRICES[currency]) { prices.push(price); } logger.debug(`${feed.name} BTC/${currency} price: ${price}`, logger.tags.mining); @@ -239,7 +239,7 @@ class PriceUpdater { for (const currency of this.currencies) { const price = historicalEntry[time][currency]; - if (price > 0) { + if (price > -1 && price < MAX_PRICES[currency]) { grouped[time][currency].push(typeof price === 'string' ? parseInt(price, 10) : price); } } diff --git a/frontend/src/app/components/amount/amount.component.html b/frontend/src/app/components/amount/amount.component.html index 4a57e72e2..ce9c02d78 100644 --- a/frontend/src/app/components/amount/amount.component.html +++ b/frontend/src/app/components/amount/amount.component.html @@ -3,7 +3,7 @@ {{ addPlus && satoshis >= 0 ? '+' : '' }} {{ ( - (blockConversion.price[currency] > 0 ? blockConversion.price[currency] : null) ?? + (blockConversion.price[currency] >= 0 ? blockConversion.price[currency] : null) ?? (blockConversion.price['USD'] * blockConversion.exchangeRates['USD' + currency]) ?? 0 ) * satoshis / 100000000 | fiatCurrency : digitsInfo : currency }} diff --git a/frontend/src/app/components/block/block.component.ts b/frontend/src/app/components/block/block.component.ts index 9e476ac61..5e0465fe1 100644 --- a/frontend/src/app/components/block/block.component.ts +++ b/frontend/src/app/components/block/block.component.ts @@ -443,9 +443,9 @@ export class BlockComponent implements OnInit, OnDestroy { } this.priceSubscription = block$.pipe( switchMap((block) => { - return this.priceService.getPrices().pipe( - tap(() => { - this.blockConversion = this.priceService.getPriceForTimestamp(block.timestamp); + return this.priceService.getBlockPrice$(block.timestamp).pipe( + tap((price) => { + this.blockConversion = price; }) ); }) diff --git a/frontend/src/app/components/transaction/transaction.component.ts b/frontend/src/app/components/transaction/transaction.component.ts index 06a4c5836..0c3a2b331 100644 --- a/frontend/src/app/components/transaction/transaction.component.ts +++ b/frontend/src/app/components/transaction/transaction.component.ts @@ -327,9 +327,9 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { this.fetchRbfHistory$.next(this.tx.txid); } - this.priceService.getPrices().pipe( - tap(() => { - this.blockConversion = this.priceService.getPriceForTimestamp(tx.status.block_time); + this.priceService.getBlockPrice$(tx.status.block_time).pipe( + tap((price) => { + this.blockConversion = price; }) ).subscribe(); diff --git a/frontend/src/app/components/transactions-list/transactions-list.component.ts b/frontend/src/app/components/transactions-list/transactions-list.component.ts index bfdaa02bc..6422d8507 100644 --- a/frontend/src/app/components/transactions-list/transactions-list.component.ts +++ b/frontend/src/app/components/transactions-list/transactions-list.component.ts @@ -6,7 +6,7 @@ import { Outspend, Transaction, Vin, Vout } from '../../interfaces/electrs.inter import { ElectrsApiService } from '../../services/electrs-api.service'; import { environment } from '../../../environments/environment'; import { AssetsService } from '../../services/assets.service'; -import { filter, map, tap, switchMap } from 'rxjs/operators'; +import { filter, map, tap, switchMap, shareReplay } from 'rxjs/operators'; import { BlockExtended } from '../../interfaces/node-api.interface'; import { ApiService } from '../../services/api.service'; import { PriceService } from 'src/app/services/price.service'; @@ -150,10 +150,8 @@ export class TransactionsListComponent implements OnInit, OnChanges { tx['addressValue'] = addressIn - addressOut; } - this.priceService.getPrices().pipe( - tap(() => { - tx['price'] = this.priceService.getPriceForTimestamp(tx.status.block_time); - }) + this.priceService.getBlockPrice$(tx.status.block_time).pipe( + tap((price) => tx['price'] = price) ).subscribe(); }); const txIds = this.transactions.filter((tx) => !tx._outspends).map((tx) => tx.txid); diff --git a/frontend/src/app/fiat/fiat.component.html b/frontend/src/app/fiat/fiat.component.html index a1bf79978..998153d29 100644 --- a/frontend/src/app/fiat/fiat.component.html +++ b/frontend/src/app/fiat/fiat.component.html @@ -1,7 +1,7 @@ {{ ( - (blockConversion.price[currency] > 0 ? blockConversion.price[currency] : null) ?? + (blockConversion.price[currency] >= 0 ? blockConversion.price[currency] : null) ?? (blockConversion.price['USD'] * blockConversion.exchangeRates['USD' + currency]) ?? 0 ) * value / 100000000 | fiatCurrency : digitsInfo : currency }} diff --git a/frontend/src/app/services/price.service.ts b/frontend/src/app/services/price.service.ts index fe6d67bb6..409ff05fd 100644 --- a/frontend/src/app/services/price.service.ts +++ b/frontend/src/app/services/price.service.ts @@ -1,5 +1,5 @@ import { Injectable } from '@angular/core'; -import { map, Observable, of, shareReplay } from 'rxjs'; +import { map, Observable, of, share, shareReplay, tap } from 'rxjs'; import { ApiService } from './api.service'; // nodejs backend interfaces @@ -40,6 +40,8 @@ export interface ConversionDict { providedIn: 'root' }) export class PriceService { + priceObservable$: Observable; + historicalPrice: ConversionDict = { prices: null, exchangeRates: null, @@ -61,65 +63,53 @@ export class PriceService { }; } - /** - * Fetch prices from the nodejs backend only once - */ - getPrices(): Observable { - if (this.historicalPrice.prices) { - return of(null); + getBlockPrice$(blockTimestamp: number): Observable { + if (!this.priceObservable$) { + this.priceObservable$ = this.apiService.getHistoricalPrice$().pipe(shareReplay()); } - return this.apiService.getHistoricalPrice$().pipe( - map((conversion: Conversion) => { - if (!this.historicalPrice.prices) { - this.historicalPrice.prices = Object(); + return this.priceObservable$.pipe( + map((conversion) => { + if (!blockTimestamp) { + return undefined; } + + const historicalPrice = { + prices: {}, + exchangeRates: conversion.exchangeRates, + }; for (const price of conversion.prices) { - this.historicalPrice.prices[price.time] = { + historicalPrice.prices[price.time] = { USD: price.USD, EUR: price.EUR, GBP: price.GBP, CAD: price.CAD, CHF: price.CHF, AUD: price.AUD, JPY: price.JPY }; } - this.historicalPrice.exchangeRates = conversion.exchangeRates; - return; - }), - shareReplay(), - ); - } - /** - * Note: The first block with a price we have is block 68952 (using MtGox price history) - * - * @param blockTimestamp - */ - getPriceForTimestamp(blockTimestamp: number): Price | null { - if (!blockTimestamp) { - return undefined; - } - - const priceTimestamps = Object.keys(this.historicalPrice.prices); - priceTimestamps.push(Number.MAX_SAFE_INTEGER.toString()); - priceTimestamps.sort().reverse(); + const priceTimestamps = Object.keys(historicalPrice.prices); + priceTimestamps.push(Number.MAX_SAFE_INTEGER.toString()); + priceTimestamps.sort().reverse(); + + // Small trick here. Because latest blocks have higher timestamps than our + // latest price timestamp (we only insert once every hour), we have no price for them. + // Therefore we want to fallback to the websocket price by returning an undefined `price` field. + // Since historicalPrice.prices[Number.MAX_SAFE_INTEGER] does not exists + // it will return `undefined` and automatically use the websocket price. + // This way we can differenciate blocks without prices like the genesis block + // vs ones without a price (yet) like the latest blocks - // Small trick here. Because latest blocks have higher timestamps than our - // latest price timestamp (we only insert once every hour), we have no price for them. - // Therefore we want to fallback to the websocket price by returning an undefined `price` field. - // Since this.historicalPrice.prices[Number.MAX_SAFE_INTEGER] does not exists - // it will return `undefined` and automatically use the websocket price. - // This way we can differenciate blocks without prices like the genesis block - // vs ones without a price (yet) like the latest blocks - - for (const t of priceTimestamps) { - const priceTimestamp = parseInt(t, 10); - if (blockTimestamp > priceTimestamp) { - return { - price: this.historicalPrice.prices[priceTimestamp], - exchangeRates: this.historicalPrice.exchangeRates, - }; - } - } - - return this.getEmptyPrice(); + for (const t of priceTimestamps) { + const priceTimestamp = parseInt(t, 10); + if (blockTimestamp > priceTimestamp) { + return { + price: historicalPrice.prices[priceTimestamp], + exchangeRates: historicalPrice.exchangeRates, + }; + } + } + + return this.getEmptyPrice(); + }) + ); } } From 62e1fa03c1509a76c6e4131cbe557b4d2b32d640 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Thu, 23 Feb 2023 10:04:31 +0900 Subject: [PATCH 26/95] Cache price API for 5 minutes --- backend/src/api/mining/mining-routes.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/backend/src/api/mining/mining-routes.ts b/backend/src/api/mining/mining-routes.ts index 393ea119a..aaeec6bab 100644 --- a/backend/src/api/mining/mining-routes.ts +++ b/backend/src/api/mining/mining-routes.ts @@ -38,6 +38,9 @@ class MiningRoutes { private async $getHistoricalPrice(req: Request, res: Response): Promise { try { + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString()); res.status(200).send(await PricesRepository.$getHistoricalPrice()); } catch (e) { res.status(500).send(e instanceof Error ? e.message : e); From f6c7839524c7b19ae7536b649b79441fd4559dd3 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Thu, 23 Feb 2023 10:46:18 +0900 Subject: [PATCH 27/95] Show historical price on transaction bowtie chart --- .../tx-bowtie-graph-tooltip.component.html | 2 +- .../tx-bowtie-graph-tooltip.component.ts | 17 ++++++++++++++--- .../tx-bowtie-graph.component.ts | 3 +++ 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/frontend/src/app/components/tx-bowtie-graph-tooltip/tx-bowtie-graph-tooltip.component.html b/frontend/src/app/components/tx-bowtie-graph-tooltip/tx-bowtie-graph-tooltip.component.html index 25e9ccc1f..395c38f88 100644 --- a/frontend/src/app/components/tx-bowtie-graph-tooltip/tx-bowtie-graph-tooltip.component.html +++ b/frontend/src/app/components/tx-bowtie-graph-tooltip/tx-bowtie-graph-tooltip.component.html @@ -56,7 +56,7 @@

Confidential

-

+

diff --git a/frontend/src/app/components/tx-bowtie-graph-tooltip/tx-bowtie-graph-tooltip.component.ts b/frontend/src/app/components/tx-bowtie-graph-tooltip/tx-bowtie-graph-tooltip.component.ts index 54c58ffab..a27885c1e 100644 --- a/frontend/src/app/components/tx-bowtie-graph-tooltip/tx-bowtie-graph-tooltip.component.ts +++ b/frontend/src/app/components/tx-bowtie-graph-tooltip/tx-bowtie-graph-tooltip.component.ts @@ -1,5 +1,6 @@ -import { Component, ElementRef, ViewChild, Input, OnChanges, ChangeDetectionStrategy } from '@angular/core'; -import { TransactionStripped } from '../../interfaces/websocket.interface'; +import { Component, ElementRef, ViewChild, Input, OnChanges, OnInit } from '@angular/core'; +import { tap } from 'rxjs'; +import { Price, PriceService } from 'src/app/services/price.service'; interface Xput { type: 'input' | 'output' | 'fee'; @@ -14,6 +15,7 @@ interface Xput { pegin?: boolean; pegout?: string; confidential?: boolean; + timestamp?: number; } @Component({ @@ -27,12 +29,21 @@ export class TxBowtieGraphTooltipComponent implements OnChanges { @Input() isConnector: boolean = false; tooltipPosition = { x: 0, y: 0 }; + blockConversion: Price; @ViewChild('tooltip') tooltipElement: ElementRef; - constructor() {} + constructor(private priceService: PriceService) {} ngOnChanges(changes): void { + if (changes.line?.currentValue) { + this.priceService.getBlockPrice$(changes.line?.currentValue.timestamp).pipe( + tap((price) => { + this.blockConversion = price; + }) + ).subscribe(); + } + if (changes.cursorPosition && changes.cursorPosition.currentValue) { let x = Math.max(10, changes.cursorPosition.currentValue.x - 50); let y = changes.cursorPosition.currentValue.y + 20; diff --git a/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.ts b/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.ts index 49d97dd40..6be475243 100644 --- a/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.ts +++ b/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.ts @@ -29,6 +29,7 @@ interface Xput { pegin?: boolean; pegout?: string; confidential?: boolean; + timestamp?: number; } @Component({ @@ -152,6 +153,7 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges { index: i, pegout: v?.pegout?.scriptpubkey_address, confidential: (this.isLiquid && v?.value === undefined), + timestamp: this.tx.status.block_time } as Xput; }); @@ -171,6 +173,7 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges { coinbase: v?.is_coinbase, pegin: v?.is_pegin, confidential: (this.isLiquid && v?.prevout?.value === undefined), + timestamp: this.tx.status.block_time } as Xput; }); From 57498209993a4511a85af4a42c789ab3ab112812 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Thu, 23 Feb 2023 13:13:20 +0900 Subject: [PATCH 28/95] Optimize price API response size reduce the number of query to that API --- backend/src/api/mining/mining-routes.ts | 8 +- backend/src/repositories/PricesRepository.ts | 45 ++++++- .../transaction/transaction.component.ts | 2 +- .../tx-bowtie-graph-tooltip.component.ts | 2 +- frontend/src/app/services/api.service.ts | 7 +- frontend/src/app/services/price.service.ts | 118 +++++++++++------- 6 files changed, 132 insertions(+), 50 deletions(-) diff --git a/backend/src/api/mining/mining-routes.ts b/backend/src/api/mining/mining-routes.ts index aaeec6bab..f7f392068 100644 --- a/backend/src/api/mining/mining-routes.ts +++ b/backend/src/api/mining/mining-routes.ts @@ -41,7 +41,13 @@ class MiningRoutes { res.header('Pragma', 'public'); res.header('Cache-control', 'public'); res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString()); - res.status(200).send(await PricesRepository.$getHistoricalPrice()); + if (req.query.timestamp) { + res.status(200).send(await PricesRepository.$getNearestHistoricalPrice( + parseInt(req.query.timestamp ?? 0, 10) + )); + } else { + res.status(200).send(await PricesRepository.$getHistoricalPrices()); + } } catch (e) { res.status(500).send(e instanceof Error ? e.message : e); } diff --git a/backend/src/repositories/PricesRepository.ts b/backend/src/repositories/PricesRepository.ts index f460a504f..83336eaff 100644 --- a/backend/src/repositories/PricesRepository.ts +++ b/backend/src/repositories/PricesRepository.ts @@ -104,9 +104,48 @@ class PricesRepository { return rates[0]; } - public async $getHistoricalPrice(): Promise { + public async $getNearestHistoricalPrice(timestamp: number | undefined): Promise { try { - const [rates]: any[] = await DB.query(`SELECT *, UNIX_TIMESTAMP(time) as time FROM prices ORDER BY time DESC`); + const [rates]: any[] = await DB.query(` + SELECT *, UNIX_TIMESTAMP(time) AS time + FROM prices + WHERE UNIX_TIMESTAMP(time) < ? + ORDER BY time DESC + LIMIT 1`, + [timestamp] + ); + if (!rates) { + throw Error(`Cannot get single historical price from the database`); + } + + // Compute fiat exchange rates + const latestPrice = await this.$getLatestConversionRates(); + const exchangeRates: ExchangeRates = { + USDEUR: Math.round(latestPrice.EUR / latestPrice.USD * 100) / 100, + USDGBP: Math.round(latestPrice.GBP / latestPrice.USD * 100) / 100, + USDCAD: Math.round(latestPrice.CAD / latestPrice.USD * 100) / 100, + USDCHF: Math.round(latestPrice.CHF / latestPrice.USD * 100) / 100, + USDAUD: Math.round(latestPrice.AUD / latestPrice.USD * 100) / 100, + USDJPY: Math.round(latestPrice.JPY / latestPrice.USD * 100) / 100, + }; + + return { + prices: rates, + exchangeRates: exchangeRates + }; + } catch (e) { + logger.err(`Cannot fetch single historical prices from the db. Reason ${e instanceof Error ? e.message : e}`); + return null; + } + } + + public async $getHistoricalPrices(): Promise { + try { + const [rates]: any[] = await DB.query(` + SELECT *, UNIX_TIMESTAMP(time) AS time + FROM prices + ORDER BY time DESC + `); if (!rates) { throw Error(`Cannot get average historical price from the database`); } @@ -127,7 +166,7 @@ class PricesRepository { exchangeRates: exchangeRates }; } catch (e) { - logger.err(`Cannot fetch averaged historical prices from the db. Reason ${e instanceof Error ? e.message : e}`); + logger.err(`Cannot fetch historical prices from the db. Reason ${e instanceof Error ? e.message : e}`); return null; } } diff --git a/frontend/src/app/components/transaction/transaction.component.ts b/frontend/src/app/components/transaction/transaction.component.ts index 0c3a2b331..4d036e131 100644 --- a/frontend/src/app/components/transaction/transaction.component.ts +++ b/frontend/src/app/components/transaction/transaction.component.ts @@ -327,7 +327,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { this.fetchRbfHistory$.next(this.tx.txid); } - this.priceService.getBlockPrice$(tx.status.block_time).pipe( + this.priceService.getBlockPrice$(tx.status.block_time, true).pipe( tap((price) => { this.blockConversion = price; }) diff --git a/frontend/src/app/components/tx-bowtie-graph-tooltip/tx-bowtie-graph-tooltip.component.ts b/frontend/src/app/components/tx-bowtie-graph-tooltip/tx-bowtie-graph-tooltip.component.ts index a27885c1e..da8d91ab3 100644 --- a/frontend/src/app/components/tx-bowtie-graph-tooltip/tx-bowtie-graph-tooltip.component.ts +++ b/frontend/src/app/components/tx-bowtie-graph-tooltip/tx-bowtie-graph-tooltip.component.ts @@ -37,7 +37,7 @@ export class TxBowtieGraphTooltipComponent implements OnChanges { ngOnChanges(changes): void { if (changes.line?.currentValue) { - this.priceService.getBlockPrice$(changes.line?.currentValue.timestamp).pipe( + this.priceService.getBlockPrice$(changes.line?.currentValue.timestamp, true).pipe( tap((price) => { this.blockConversion = price; }) diff --git a/frontend/src/app/services/api.service.ts b/frontend/src/app/services/api.service.ts index 2c74de361..840fd5070 100644 --- a/frontend/src/app/services/api.service.ts +++ b/frontend/src/app/services/api.service.ts @@ -305,7 +305,10 @@ export class ApiService { ); } - getHistoricalPrice$(): Observable { - return this.httpClient.get( this.apiBaseUrl + this.apiBasePath + '/api/v1/historical-price'); + getHistoricalPrice$(timestamp: number | undefined): Observable { + return this.httpClient.get( + this.apiBaseUrl + this.apiBasePath + '/api/v1/historical-price' + + (timestamp ? `?timestamp=${timestamp}` : '') + ); } } diff --git a/frontend/src/app/services/price.service.ts b/frontend/src/app/services/price.service.ts index 409ff05fd..ef1a973a1 100644 --- a/frontend/src/app/services/price.service.ts +++ b/frontend/src/app/services/price.service.ts @@ -41,6 +41,10 @@ export interface ConversionDict { }) export class PriceService { priceObservable$: Observable; + singlePriceObservable$: Observable; + + lastQueriedTimestamp: number; + lastPriceHistoryUpdate: number; historicalPrice: ConversionDict = { prices: null, @@ -63,53 +67,83 @@ export class PriceService { }; } - getBlockPrice$(blockTimestamp: number): Observable { - if (!this.priceObservable$) { - this.priceObservable$ = this.apiService.getHistoricalPrice$().pipe(shareReplay()); + getBlockPrice$(blockTimestamp: number, singlePrice = false): Observable { + const now = new Date().getTime() / 1000; + + /** + * Query nearest price for a specific blockTimestamp. The observable is invalidated if we + * query a different timestamp than the last one + */ + if (singlePrice) { + if (!this.singlePriceObservable$ || (this.singlePriceObservable$ && blockTimestamp !== this.lastQueriedTimestamp)) { + this.singlePriceObservable$ = this.apiService.getHistoricalPrice$(blockTimestamp).pipe(shareReplay()); + this.lastQueriedTimestamp = blockTimestamp; + } + + return this.singlePriceObservable$.pipe( + map((conversion) => { + return { + price: { + USD: conversion.prices[0].USD, EUR: conversion.prices[0].EUR, GBP: conversion.prices[0].GBP, CAD: conversion.prices[0].CAD, + CHF: conversion.prices[0].CHF, AUD: conversion.prices[0].AUD, JPY: conversion.prices[0].JPY + }, + exchangeRates: conversion.exchangeRates, + }; + }) + ); } - return this.priceObservable$.pipe( - map((conversion) => { - if (!blockTimestamp) { - return undefined; - } + /** + * Query all price history only once. The observable is invalidated after 1 hour + */ + else { + if (!this.priceObservable$ || (this.priceObservable$ && (now - this.lastPriceHistoryUpdate > 3600))) { + this.priceObservable$ = this.apiService.getHistoricalPrice$(undefined).pipe(shareReplay()); + this.lastPriceHistoryUpdate = new Date().getTime() / 1000; + } - const historicalPrice = { - prices: {}, - exchangeRates: conversion.exchangeRates, - }; - for (const price of conversion.prices) { - historicalPrice.prices[price.time] = { - USD: price.USD, EUR: price.EUR, GBP: price.GBP, CAD: price.CAD, - CHF: price.CHF, AUD: price.AUD, JPY: price.JPY + return this.priceObservable$.pipe( + map((conversion) => { + if (!blockTimestamp) { + return undefined; + } + + const historicalPrice = { + prices: {}, + exchangeRates: conversion.exchangeRates, }; - } - - const priceTimestamps = Object.keys(historicalPrice.prices); - priceTimestamps.push(Number.MAX_SAFE_INTEGER.toString()); - priceTimestamps.sort().reverse(); - - // Small trick here. Because latest blocks have higher timestamps than our - // latest price timestamp (we only insert once every hour), we have no price for them. - // Therefore we want to fallback to the websocket price by returning an undefined `price` field. - // Since historicalPrice.prices[Number.MAX_SAFE_INTEGER] does not exists - // it will return `undefined` and automatically use the websocket price. - // This way we can differenciate blocks without prices like the genesis block - // vs ones without a price (yet) like the latest blocks - - for (const t of priceTimestamps) { - const priceTimestamp = parseInt(t, 10); - if (blockTimestamp > priceTimestamp) { - return { - price: historicalPrice.prices[priceTimestamp], - exchangeRates: historicalPrice.exchangeRates, + for (const price of conversion.prices) { + historicalPrice.prices[price.time] = { + USD: price.USD, EUR: price.EUR, GBP: price.GBP, CAD: price.CAD, + CHF: price.CHF, AUD: price.AUD, JPY: price.JPY }; } - } - - return this.getEmptyPrice(); - }) - ); + + const priceTimestamps = Object.keys(historicalPrice.prices); + priceTimestamps.push(Number.MAX_SAFE_INTEGER.toString()); + priceTimestamps.sort().reverse(); + + // Small trick here. Because latest blocks have higher timestamps than our + // latest price timestamp (we only insert once every hour), we have no price for them. + // Therefore we want to fallback to the websocket price by returning an undefined `price` field. + // Since historicalPrice.prices[Number.MAX_SAFE_INTEGER] does not exists + // it will return `undefined` and automatically use the websocket price. + // This way we can differenciate blocks without prices like the genesis block + // vs ones without a price (yet) like the latest blocks + + for (const t of priceTimestamps) { + const priceTimestamp = parseInt(t, 10); + if (blockTimestamp > priceTimestamp) { + return { + price: historicalPrice.prices[priceTimestamp], + exchangeRates: historicalPrice.exchangeRates, + }; + } + } + + return this.getEmptyPrice(); + }) + ); + } } } - From 5b5de958286eea5531903e43fc0093c9a2edf417 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Thu, 23 Feb 2023 13:34:48 +0900 Subject: [PATCH 29/95] Show historical price on block overview graph --- .../block-overview-graph/block-overview-graph.component.html | 1 + .../block-overview-graph/block-overview-graph.component.ts | 2 ++ .../block-overview-tooltip.component.html | 4 ++-- .../block-overview-tooltip.component.ts | 2 ++ frontend/src/app/components/block/block.component.html | 1 + 5 files changed, 8 insertions(+), 2 deletions(-) diff --git a/frontend/src/app/components/block-overview-graph/block-overview-graph.component.html b/frontend/src/app/components/block-overview-graph/block-overview-graph.component.html index 54cd995aa..2a357843b 100644 --- a/frontend/src/app/components/block-overview-graph/block-overview-graph.component.html +++ b/frontend/src/app/components/block-overview-graph/block-overview-graph.component.html @@ -10,5 +10,6 @@ [cursorPosition]="tooltipPosition" [clickable]="!!selectedTx" [auditEnabled]="auditHighlighting" + [blockConversion]="blockConversion" >
diff --git a/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts b/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts index 7c71d36fe..b46f7a3e7 100644 --- a/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts +++ b/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts @@ -5,6 +5,7 @@ import BlockScene from './block-scene'; import TxSprite from './tx-sprite'; import TxView from './tx-view'; import { Position } from './sprite-types'; +import { Price } from 'src/app/services/price.service'; @Component({ selector: 'app-block-overview-graph', @@ -21,6 +22,7 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On @Input() mirrorTxid: string | void; @Input() unavailable: boolean = false; @Input() auditHighlighting: boolean = false; + @Input() blockConversion: Price; @Output() txClickEvent = new EventEmitter(); @Output() txHoverEvent = new EventEmitter(); @Output() readyEvent = new EventEmitter(); diff --git a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html index 826eaaf8f..2fa626a95 100644 --- a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html +++ b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html @@ -16,11 +16,11 @@ Amount - + Fee - {{ fee | number }} sat   + {{ fee | number }} sat   Fee rate diff --git a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.ts b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.ts index 6702c4d62..1bd2b8714 100644 --- a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.ts +++ b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.ts @@ -1,6 +1,7 @@ import { Component, ElementRef, ViewChild, Input, OnChanges, ChangeDetectionStrategy } from '@angular/core'; import { TransactionStripped } from '../../interfaces/websocket.interface'; import { Position } from '../../components/block-overview-graph/sprite-types.js'; +import { Price } from 'src/app/services/price.service'; @Component({ selector: 'app-block-overview-tooltip', @@ -12,6 +13,7 @@ export class BlockOverviewTooltipComponent implements OnChanges { @Input() cursorPosition: Position; @Input() clickable: boolean; @Input() auditEnabled: boolean = false; + @Input() blockConversion: Price; txid = ''; fee = 0; diff --git a/frontend/src/app/components/block/block.component.html b/frontend/src/app/components/block/block.component.html index bb4d2082c..4c7e4684a 100644 --- a/frontend/src/app/components/block/block.component.html +++ b/frontend/src/app/components/block/block.component.html @@ -108,6 +108,7 @@ [blockLimit]="stateService.blockVSize" [orientation]="'top'" [flip]="false" + [blockConversion]="blockConversion" (txClickEvent)="onTxClick($event)" > From 10bfb512159f34c12ea9cf8a6948fa20a970a1b0 Mon Sep 17 00:00:00 2001 From: wiz Date: Thu, 23 Feb 2023 13:37:09 +0900 Subject: [PATCH 30/95] Add simon's comment to $searchNodeByPublicKeyOrAlias() --- backend/src/api/explorer/nodes.api.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/src/api/explorer/nodes.api.ts b/backend/src/api/explorer/nodes.api.ts index 36bbf26d2..b3f83faa6 100644 --- a/backend/src/api/explorer/nodes.api.ts +++ b/backend/src/api/explorer/nodes.api.ts @@ -368,6 +368,7 @@ class NodesApi { .split(' ') .filter(key => key.length) .map((search) => '+' + search + '*').join(' '); + // %keyword% is wildcard search and can't be indexed so it's slower as the node database grow. keyword% can be indexed but then you can't search for "Nicehash" and get result for ln.nicehash.com. So we use fulltext index for words "ln, nicehash, com" and nicehash* will find it instantly. const query = `SELECT public_key, alias, capacity, channels, status FROM nodes WHERE public_key LIKE ? OR MATCH alias_search AGAINST (? IN BOOLEAN MODE) ORDER BY capacity DESC LIMIT 10`; const [rows]: any = await DB.query(query, [publicKeySearch, aliasSearch]); return rows; From 5cfd715d4a59bbbc71088b7191785f6306900018 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Thu, 23 Feb 2023 14:50:20 +0900 Subject: [PATCH 31/95] Add database migration to re-index prices with negative values support --- backend/src/api/database-migration.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 66216c64a..42cf5ee9f 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository'; import { RowDataPacket } from 'mysql2'; class DatabaseMigration { - private static currentVersion = 52; + private static currentVersion = 53; private queryTimeout = 3600_000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -468,6 +468,13 @@ class DatabaseMigration { logger.warn('' + (e instanceof Error ? e.message : e)); } } + + if (databaseSchemaVersion < 53) { + this.uniqueLog(logger.notice, `'prices' table has been truncated`); + this.uniqueLog(logger.notice, `'blocks_prices' table has been truncated`); + await this.$executeQuery(`TRUNCATE prices`); + await this.$executeQuery(`TRUNCATE blocks_prices`); + } } /** From 2d0333233345bedfd79cc2c3007945206a48fa46 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Thu, 23 Feb 2023 14:52:29 +0900 Subject: [PATCH 32/95] Add missing db schema incrementation --- backend/src/api/database-migration.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 7b56ce361..4dc59a22d 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -479,6 +479,7 @@ class DatabaseMigration { this.uniqueLog(logger.notice, `'blocks_prices' table has been truncated`); await this.$executeQuery(`TRUNCATE prices`); await this.$executeQuery(`TRUNCATE blocks_prices`); + await this.updateToSchemaVersion(54); } } From ee54e782f8c99faab9cae476241a172c2d41394a Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Thu, 23 Feb 2023 15:06:57 +0900 Subject: [PATCH 33/95] Only display reward and block fee charts in USD due to missing historical data --- backend/src/repositories/BlocksRepository.ts | 4 ++-- .../block-fees-graph.component.ts | 15 ++------------- .../block-rewards-graph.component.ts | 16 +++------------- 3 files changed, 7 insertions(+), 28 deletions(-) diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 355187e21..df98719b9 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -521,7 +521,7 @@ class BlocksRepository { CAST(AVG(blocks.height) as INT) as avgHeight, CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp, CAST(AVG(fees) as INT) as avgFees, - prices.* + prices.USD FROM blocks JOIN blocks_prices on blocks_prices.height = blocks.height JOIN prices on prices.id = blocks_prices.price_id @@ -550,7 +550,7 @@ class BlocksRepository { CAST(AVG(blocks.height) as INT) as avgHeight, CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp, CAST(AVG(reward) as INT) as avgRewards, - prices.* + prices.USD FROM blocks JOIN blocks_prices on blocks_prices.height = blocks.height JOIN prices on prices.id = blocks_prices.price_id diff --git a/frontend/src/app/components/block-fees-graph/block-fees-graph.component.ts b/frontend/src/app/components/block-fees-graph/block-fees-graph.component.ts index 96bd0697c..051d24848 100644 --- a/frontend/src/app/components/block-fees-graph/block-fees-graph.component.ts +++ b/frontend/src/app/components/block-fees-graph/block-fees-graph.component.ts @@ -1,19 +1,17 @@ import { ChangeDetectionStrategy, Component, Inject, Input, LOCALE_ID, OnInit } from '@angular/core'; import { EChartsOption, graphic } from 'echarts'; -import { Observable, Subscription } from 'rxjs'; +import { Observable } from 'rxjs'; import { map, share, startWith, switchMap, tap } from 'rxjs/operators'; import { ApiService } from '../../services/api.service'; import { SeoService } from '../../services/seo.service'; import { formatNumber } from '@angular/common'; import { UntypedFormBuilder, UntypedFormGroup } from '@angular/forms'; import { download, formatterXAxis } from '../../shared/graphs.utils'; -import { StateService } from '../../services/state.service'; import { StorageService } from '../../services/storage.service'; import { MiningService } from '../../services/mining.service'; import { ActivatedRoute } from '@angular/router'; import { FiatShortenerPipe } from '../../shared/pipes/fiat-shortener.pipe'; import { FiatCurrencyPipe } from '../../shared/pipes/fiat-currency.pipe'; -import { fiatCurrencies } from '../../app.constants'; @Component({ selector: 'app-block-fees-graph', @@ -47,7 +45,6 @@ export class BlockFeesGraphComponent implements OnInit { timespan = ''; chartInstance: any = undefined; - currencySubscription: Subscription; currency: string; constructor( @@ -57,21 +54,13 @@ export class BlockFeesGraphComponent implements OnInit { private formBuilder: UntypedFormBuilder, private storageService: StorageService, private miningService: MiningService, - private stateService: StateService, private route: ActivatedRoute, private fiatShortenerPipe: FiatShortenerPipe, private fiatCurrencyPipe: FiatCurrencyPipe, ) { this.radioGroupForm = this.formBuilder.group({ dateSpan: '1y' }); this.radioGroupForm.controls.dateSpan.setValue('1y'); - - this.currencySubscription = this.stateService.fiatCurrency$.subscribe((fiat) => { - if (fiat && fiatCurrencies[fiat]?.indexed) { - this.currency = fiat; - } else { - this.currency = 'USD'; - } - }); + this.currency = 'USD'; } ngOnInit(): void { diff --git a/frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.ts b/frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.ts index ca1853633..2d8a6f858 100644 --- a/frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.ts +++ b/frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.ts @@ -1,19 +1,17 @@ import { ChangeDetectionStrategy, Component, Inject, Input, LOCALE_ID, OnInit } from '@angular/core'; import { EChartsOption, graphic } from 'echarts'; -import { Observable, Subscription } from 'rxjs'; +import { Observable } from 'rxjs'; import { map, share, startWith, switchMap, tap } from 'rxjs/operators'; import { ApiService } from '../../services/api.service'; import { SeoService } from '../../services/seo.service'; import { formatNumber } from '@angular/common'; import { UntypedFormBuilder, UntypedFormGroup } from '@angular/forms'; -import { download, formatterXAxis, formatterXAxisLabel, formatterXAxisTimeCategory } from '../../shared/graphs.utils'; +import { download, formatterXAxis } from '../../shared/graphs.utils'; import { MiningService } from '../../services/mining.service'; -import { StateService } from '../../services/state.service'; import { StorageService } from '../../services/storage.service'; import { ActivatedRoute } from '@angular/router'; import { FiatShortenerPipe } from '../../shared/pipes/fiat-shortener.pipe'; import { FiatCurrencyPipe } from '../../shared/pipes/fiat-currency.pipe'; -import { fiatCurrencies } from '../../app.constants'; @Component({ selector: 'app-block-rewards-graph', @@ -47,7 +45,6 @@ export class BlockRewardsGraphComponent implements OnInit { timespan = ''; chartInstance: any = undefined; - currencySubscription: Subscription; currency: string; constructor( @@ -56,19 +53,12 @@ export class BlockRewardsGraphComponent implements OnInit { private apiService: ApiService, private formBuilder: UntypedFormBuilder, private miningService: MiningService, - private stateService: StateService, private storageService: StorageService, private route: ActivatedRoute, private fiatShortenerPipe: FiatShortenerPipe, private fiatCurrencyPipe: FiatCurrencyPipe, ) { - this.currencySubscription = this.stateService.fiatCurrency$.subscribe((fiat) => { - if (fiat && fiatCurrencies[fiat]?.indexed) { - this.currency = fiat; - } else { - this.currency = 'USD'; - } - }); + this.currency = 'USD'; } ngOnInit(): void { From ffe02c250953a980b0df40e7f0cc96081e754390 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Thu, 23 Feb 2023 16:01:56 +0900 Subject: [PATCH 34/95] Fix lightning chart widget layout --- .../nodes-networks-chart/nodes-networks-chart.component.ts | 6 +++--- .../lightning-statistics-chart.component.ts | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/frontend/src/app/lightning/nodes-networks-chart/nodes-networks-chart.component.ts b/frontend/src/app/lightning/nodes-networks-chart/nodes-networks-chart.component.ts index 20ce5cc6f..0ff9f4af1 100644 --- a/frontend/src/app/lightning/nodes-networks-chart/nodes-networks-chart.component.ts +++ b/frontend/src/app/lightning/nodes-networks-chart/nodes-networks-chart.component.ts @@ -130,7 +130,7 @@ export class NodesNetworksChartComponent implements OnInit { }, text: $localize`:@@b420668a91f8ebaf6e6409c4ba87f1d45961d2bd:Lightning Nodes Per Network`, left: 'center', - top: 11, + top: 0, zlevel: 10, }; } @@ -227,8 +227,8 @@ export class NodesNetworksChartComponent implements OnInit { title: title, animation: false, grid: { - height: this.widget ? 100 : undefined, - top: this.widget ? 10 : 40, + height: this.widget ? 90 : undefined, + top: this.widget ? 20 : 40, bottom: this.widget ? 0 : 70, right: (isMobile() && this.widget) ? 35 : this.right, left: (isMobile() && this.widget) ? 40 :this.left, diff --git a/frontend/src/app/lightning/statistics-chart/lightning-statistics-chart.component.ts b/frontend/src/app/lightning/statistics-chart/lightning-statistics-chart.component.ts index 916483781..b93ee1f3d 100644 --- a/frontend/src/app/lightning/statistics-chart/lightning-statistics-chart.component.ts +++ b/frontend/src/app/lightning/statistics-chart/lightning-statistics-chart.component.ts @@ -121,7 +121,7 @@ export class LightningStatisticsChartComponent implements OnInit { }, text: $localize`:@@ea8db27e6db64f8b940711948c001a1100e5fe9f:Lightning Network Capacity`, left: 'center', - top: 11, + top: 0, zlevel: 10, }; } @@ -137,8 +137,8 @@ export class LightningStatisticsChartComponent implements OnInit { ]), ], grid: { - height: this.widget ? 100 : undefined, - top: this.widget ? 10 : 40, + height: this.widget ? 90 : undefined, + top: this.widget ? 20 : 40, bottom: this.widget ? 0 : 70, right: (isMobile() && this.widget) ? 35 : this.right, left: (isMobile() && this.widget) ? 40 :this.left, From 58f886b33714d811296581c2572e39f53c6f423e Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Thu, 23 Feb 2023 18:40:13 +0900 Subject: [PATCH 35/95] If we don't have a price for "single price" query then return empty price --- frontend/src/app/services/price.service.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/frontend/src/app/services/price.service.ts b/frontend/src/app/services/price.service.ts index ef1a973a1..8a93d9554 100644 --- a/frontend/src/app/services/price.service.ts +++ b/frontend/src/app/services/price.service.ts @@ -82,6 +82,9 @@ export class PriceService { return this.singlePriceObservable$.pipe( map((conversion) => { + if (conversion.prices.length <= 0) { + return this.getEmptyPrice(); + } return { price: { USD: conversion.prices[0].USD, EUR: conversion.prices[0].EUR, GBP: conversion.prices[0].GBP, CAD: conversion.prices[0].CAD, From f9fe096669daeda56e96eb5e0bd5810086b42b8e Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Thu, 23 Feb 2023 18:43:32 +0900 Subject: [PATCH 36/95] Unsubscribe priceSubscription onDestroy --- frontend/src/app/components/block/block.component.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/src/app/components/block/block.component.ts b/frontend/src/app/components/block/block.component.ts index 5e0465fe1..35f47de85 100644 --- a/frontend/src/app/components/block/block.component.ts +++ b/frontend/src/app/components/block/block.component.ts @@ -471,6 +471,7 @@ export class BlockComponent implements OnInit, OnDestroy { this.auditSubscription?.unsubscribe(); this.unsubscribeNextBlockSubscriptions(); this.childChangeSubscription?.unsubscribe(); + this.priceSubscription?.unsubscribe(); } unsubscribeNextBlockSubscriptions() { From bb5fd4b1b1e094cf015bc11c9fd48db307cf05a2 Mon Sep 17 00:00:00 2001 From: hunicus <93150691+hunicus@users.noreply.github.com> Date: Thu, 23 Feb 2023 07:39:57 -0500 Subject: [PATCH 37/95] Fit translator avatars neatly on 2 lines --- frontend/src/app/components/about/about.component.scss | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/frontend/src/app/components/about/about.component.scss b/frontend/src/app/components/about/about.component.scss index 42ecded1c..8390ce0ba 100644 --- a/frontend/src/app/components/about/about.component.scss +++ b/frontend/src/app/components/about/about.component.scss @@ -145,6 +145,13 @@ } } + .project-translators .wrapper { + a img { + width: 72px; + height: 72px; + } + } + .copyright { text-align: left; max-width: 620px; From 8eca1e5f7e90a925dc988dfe10cf8255624f48fd Mon Sep 17 00:00:00 2001 From: Mononaut Date: Wed, 8 Feb 2023 19:07:59 -0600 Subject: [PATCH 38/95] Handle network interruptions in scrollable blockchain --- .../blockchain-blocks.component.html | 10 +++++----- .../blockchain-blocks.component.scss | 4 ++++ .../blockchain-blocks/blockchain-blocks.component.ts | 2 ++ .../components/blockchain/blockchain.component.html | 2 +- .../components/blockchain/blockchain.component.ts | 12 +++++++++++- frontend/src/app/components/start/start.component.ts | 4 +++- frontend/src/app/services/cache.service.ts | 7 ++++++- 7 files changed, 32 insertions(+), 9 deletions(-) diff --git a/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html b/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html index 400016d02..deb915f26 100644 --- a/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html +++ b/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html @@ -1,6 +1,6 @@
+ *ngIf="static || (loadingBlocks$ | async) === false; else loadingBlocksTemplate">
- +
- +
- -
+ +
diff --git a/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.scss b/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.scss index 64bfd2379..5db452470 100644 --- a/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.scss +++ b/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.scss @@ -137,6 +137,10 @@ opacity: 1; } +.loading .bitcoin-block.mined-block { + background: #2d3348; +} + @keyframes opacityPulse { 0% {opacity: 0.7;} 50% {opacity: 1.0;} diff --git a/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.ts b/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.ts index fb37cf72a..39f68c22e 100644 --- a/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.ts +++ b/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.ts @@ -22,6 +22,8 @@ export class BlockchainBlocksComponent implements OnInit, OnChanges, OnDestroy { @Input() offset: number = 0; @Input() height: number = 0; @Input() count: number = 8; + @Input() loadingTip: boolean = false; + @Input() connected: boolean = true; specialBlocks = specialBlocks; network = ''; diff --git a/frontend/src/app/components/blockchain/blockchain.component.html b/frontend/src/app/components/blockchain/blockchain.component.html index ad2e5e86a..0c4a1cbb7 100644 --- a/frontend/src/app/components/blockchain/blockchain.component.html +++ b/frontend/src/app/components/blockchain/blockchain.component.html @@ -6,7 +6,7 @@ - +
diff --git a/frontend/src/app/components/blockchain/blockchain.component.ts b/frontend/src/app/components/blockchain/blockchain.component.ts index 0ad3625ea..ab9875a4c 100644 --- a/frontend/src/app/components/blockchain/blockchain.component.ts +++ b/frontend/src/app/components/blockchain/blockchain.component.ts @@ -1,5 +1,5 @@ import { Component, OnInit, OnDestroy, ChangeDetectionStrategy, Input, OnChanges, SimpleChanges } from '@angular/core'; -import { Subscription } from 'rxjs'; +import { firstValueFrom, Subscription } from 'rxjs'; import { StateService } from '../../services/state.service'; @Component({ @@ -18,6 +18,9 @@ export class BlockchainComponent implements OnInit, OnDestroy { timeLtrSubscription: Subscription; timeLtr: boolean = this.stateService.timeLtr.value; ltrTransitionEnabled = false; + connectionStateSubscription: Subscription; + loadingTip: boolean = true; + connected: boolean = true; constructor( public stateService: StateService, @@ -28,10 +31,17 @@ export class BlockchainComponent implements OnInit, OnDestroy { this.timeLtrSubscription = this.stateService.timeLtr.subscribe((ltr) => { this.timeLtr = !!ltr; }); + this.connectionStateSubscription = this.stateService.connectionState$.subscribe(state => { + this.connected = (state === 2); + }) + firstValueFrom(this.stateService.chainTip$).then(tip => { + this.loadingTip = false; + }); } ngOnDestroy() { this.timeLtrSubscription.unsubscribe(); + this.connectionStateSubscription.unsubscribe(); } trackByPageFn(index: number, item: { index: number }) { diff --git a/frontend/src/app/components/start/start.component.ts b/frontend/src/app/components/start/start.component.ts index 28f9cf6aa..c7b8a83bc 100644 --- a/frontend/src/app/components/start/start.component.ts +++ b/frontend/src/app/components/start/start.component.ts @@ -21,6 +21,7 @@ export class StartComponent implements OnInit, OnDestroy { timeLtr: boolean = this.stateService.timeLtr.value; chainTipSubscription: Subscription; chainTip: number = -1; + tipIsSet: boolean = false; markBlockSubscription: Subscription; blockCounterSubscription: Subscription; @ViewChild('blockchainContainer') blockchainContainer: ElementRef; @@ -58,6 +59,7 @@ export class StartComponent implements OnInit, OnDestroy { }); this.chainTipSubscription = this.stateService.chainTip$.subscribe((height) => { this.chainTip = height; + this.tipIsSet = true; this.updatePages(); if (this.pendingMark != null) { this.scrollToBlock(this.pendingMark); @@ -66,7 +68,7 @@ export class StartComponent implements OnInit, OnDestroy { }); this.markBlockSubscription = this.stateService.markBlock$.subscribe((mark) => { if (mark?.blockHeight != null) { - if (this.chainTip >=0) { + if (this.tipIsSet) { if (!this.blockInViewport(mark.blockHeight)) { this.scrollToBlock(mark.blockHeight); } diff --git a/frontend/src/app/services/cache.service.ts b/frontend/src/app/services/cache.service.ts index be37164dd..15ef99859 100644 --- a/frontend/src/app/services/cache.service.ts +++ b/frontend/src/app/services/cache.service.ts @@ -62,7 +62,12 @@ export class CacheService { for (let i = 0; i < chunkSize; i++) { this.blockLoading[maxHeight - i] = true; } - const result = await firstValueFrom(this.apiService.getBlocks$(maxHeight)); + let result; + try { + result = await firstValueFrom(this.apiService.getBlocks$(maxHeight)); + } catch (e) { + console.log("failed to load blocks: ", e.message); + } for (let i = 0; i < chunkSize; i++) { delete this.blockLoading[maxHeight - i]; } From ee265be55e19aae9e722e61458c5811ec5f3387a Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 14 Feb 2023 12:51:41 -0600 Subject: [PATCH 39/95] Show skeleton loader for all blocks while offline --- .../blockchain-blocks/blockchain-blocks.component.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html b/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html index deb915f26..52805b750 100644 --- a/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html +++ b/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html @@ -2,7 +2,7 @@ [style.left]="static ? (offset || 0) + 'px' : null" *ngIf="static || (loadingBlocks$ | async) === false; else loadingBlocksTemplate">
- +
- +
@@ -68,10 +68,10 @@ - +
+ [ngStyle]="emptyBlockStyles[i]">
From c65674479abfbe56a54af9c271877d186d9639e4 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Thu, 23 Feb 2023 14:31:01 -0600 Subject: [PATCH 40/95] fix gaps in loading blockchain --- .../blockchain-blocks/blockchain-blocks.component.html | 2 +- .../blockchain-blocks/blockchain-blocks.component.ts | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html b/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html index 52805b750..d5decd415 100644 --- a/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html +++ b/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html @@ -71,7 +71,7 @@
+ [ngStyle]="convertStyleForLoadingBlock(blockStyles[i])">
diff --git a/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.ts b/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.ts index 39f68c22e..3feaf6c2d 100644 --- a/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.ts +++ b/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.ts @@ -290,6 +290,13 @@ export class BlockchainBlocksComponent implements OnInit, OnChanges, OnDestroy { }; } + convertStyleForLoadingBlock(style) { + return { + ...style, + background: "#2d3348", + }; + } + getStyleForLoadingBlock(index: number, animateEnterFrom: number = 0) { const addLeft = animateEnterFrom || 0; From 6f68c1666ff04bb666b7e6a9546062e9fac5504a Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Fri, 24 Feb 2023 09:55:39 +0900 Subject: [PATCH 41/95] Add border input in the qr code component --- frontend/src/app/components/qrcode/qrcode.component.html | 2 +- frontend/src/app/components/qrcode/qrcode.component.ts | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/components/qrcode/qrcode.component.html b/frontend/src/app/components/qrcode/qrcode.component.html index d7886b907..56f32f42c 100644 --- a/frontend/src/app/components/qrcode/qrcode.component.html +++ b/frontend/src/app/components/qrcode/qrcode.component.html @@ -1,4 +1,4 @@
- +
diff --git a/frontend/src/app/components/qrcode/qrcode.component.ts b/frontend/src/app/components/qrcode/qrcode.component.ts index e8ebac904..dad7522c6 100644 --- a/frontend/src/app/components/qrcode/qrcode.component.ts +++ b/frontend/src/app/components/qrcode/qrcode.component.ts @@ -12,6 +12,7 @@ export class QrcodeComponent implements AfterViewInit { @Input() data: string; @Input() size = 125; @Input() imageUrl: string; + @Input() border = 0; @ViewChild('canvas') canvas: ElementRef; qrcodeObject: any; From 4d7c69dd733a6bdd4b69a6dcd97846d776724deb Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Fri, 24 Feb 2023 10:41:17 +0900 Subject: [PATCH 42/95] Fix DB migration 54 breaking liquid --- backend/src/api/database-migration.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 4dc59a22d..6e4221857 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -86,7 +86,7 @@ class DatabaseMigration { try { await this.$migrateTableSchemaFromVersion(databaseSchemaVersion); if (databaseSchemaVersion === 0) { - logger.notice(`MIGRATIONS: OK. Database schema has been properly initialized to version ${DatabaseMigration.currentVersion} (latest version)`); + logger.notice(`MIGRATIONS: OK. Database schema has been properly initialized to version ${DatabaseMigration.currentVersion} (latest version)`); } else { logger.notice(`MIGRATIONS: OK. Database schema have been migrated from version ${databaseSchemaVersion} to ${DatabaseMigration.currentVersion} (latest version)`); } @@ -300,7 +300,7 @@ class DatabaseMigration { await this.$executeQuery('ALTER TABLE `lightning_stats` ADD med_base_fee_mtokens bigint(20) unsigned NOT NULL DEFAULT "0"'); await this.updateToSchemaVersion(27); } - + if (databaseSchemaVersion < 28 && isBitcoin === true) { if (config.LIGHTNING.ENABLED) { this.uniqueLog(logger.notice, `'lightning_stats' and 'node_stats' tables have been truncated.`); @@ -464,7 +464,7 @@ class DatabaseMigration { await this.$executeQuery('DROP TABLE IF EXISTS `transactions`'); await this.$executeQuery('DROP TABLE IF EXISTS `cpfp_clusters`'); await this.updateToSchemaVersion(52); - } catch(e) { + } catch (e) { logger.warn('' + (e instanceof Error ? e.message : e)); } } @@ -476,9 +476,11 @@ class DatabaseMigration { if (databaseSchemaVersion < 54) { this.uniqueLog(logger.notice, `'prices' table has been truncated`); - this.uniqueLog(logger.notice, `'blocks_prices' table has been truncated`); await this.$executeQuery(`TRUNCATE prices`); - await this.$executeQuery(`TRUNCATE blocks_prices`); + if (isBitcoin === true) { + this.uniqueLog(logger.notice, `'blocks_prices' table has been truncated`); + await this.$executeQuery(`TRUNCATE blocks_prices`); + } await this.updateToSchemaVersion(54); } } @@ -604,7 +606,7 @@ class DatabaseMigration { queries.push(`INSERT INTO state(name, number, string) VALUES ('last_hashrates_indexing', 0, NULL)`); } - if (version < 9 && isBitcoin === true) { + if (version < 9 && isBitcoin === true) { queries.push(`INSERT INTO state(name, number, string) VALUES ('last_weekly_hashrates_indexing', 0, NULL)`); } From b4d0e20d7545b382dfc8915a85041bc4f3ee6587 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Fri, 24 Feb 2023 12:12:50 +0900 Subject: [PATCH 43/95] Only query historical price if we're running mempool BASE_MODULE --- frontend/src/app/services/price.service.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/services/price.service.ts b/frontend/src/app/services/price.service.ts index 8a93d9554..e3ec93c8b 100644 --- a/frontend/src/app/services/price.service.ts +++ b/frontend/src/app/services/price.service.ts @@ -1,6 +1,7 @@ import { Injectable } from '@angular/core'; import { map, Observable, of, share, shareReplay, tap } from 'rxjs'; import { ApiService } from './api.service'; +import { StateService } from './state.service'; // nodejs backend interfaces export interface ApiPrice { @@ -52,7 +53,8 @@ export class PriceService { }; constructor( - private apiService: ApiService + private apiService: ApiService, + private stateService: StateService ) { } @@ -68,6 +70,10 @@ export class PriceService { } getBlockPrice$(blockTimestamp: number, singlePrice = false): Observable { + if (this.stateService.env.BASE_MODULE !== 'mempool') { + return of(undefined); + } + const now = new Date().getTime() / 1000; /** From b50e97357368c904c55a5f9c4a51c1267bc9969f Mon Sep 17 00:00:00 2001 From: wiz Date: Sat, 25 Feb 2023 12:14:07 +0900 Subject: [PATCH 44/95] ops: Enable CPFP indexing for mainnet --- production/mempool-config.mainnet.json | 1 + 1 file changed, 1 insertion(+) diff --git a/production/mempool-config.mainnet.json b/production/mempool-config.mainnet.json index 1258e62fb..658437edc 100644 --- a/production/mempool-config.mainnet.json +++ b/production/mempool-config.mainnet.json @@ -11,6 +11,7 @@ "INDEXING_BLOCKS_AMOUNT": -1, "BLOCKS_SUMMARIES_INDEXING": true, "AUDIT": true, + "CPFP_INDEXING": true, "ADVANCED_GBT_AUDIT": true, "ADVANCED_GBT_MEMPOOL": false, "USE_SECOND_NODE_FOR_MINFEE": true From 8df247626638347d65d83c34739acebaa738fdfc Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Sat, 25 Feb 2023 13:38:09 +0900 Subject: [PATCH 45/95] Improve error handling on channel component --- .../lightning/channel/channel.component.html | 47 +++++++++---------- .../lightning/channel/channel.component.ts | 4 +- .../app/lightning/node/node.component.html | 23 +++++---- 3 files changed, 39 insertions(+), 35 deletions(-) diff --git a/frontend/src/app/lightning/channel/channel.component.html b/frontend/src/app/lightning/channel/channel.component.html index f52b85762..96af4ab67 100644 --- a/frontend/src/app/lightning/channel/channel.component.html +++ b/frontend/src/app/lightning/channel/channel.component.html @@ -1,25 +1,32 @@
-
Lightning channel
-
-

{{ channel.short_id }}

- - {{ channel.id }} - - -
-
- Inactive - Active - Closed - -
+ + +
Lightning channel
+
+

{{ channel.short_id }}

+ + {{ channel.id }} + + +
+
+ Inactive + Active + Closed + +
+
+
+ No channel found for short id "{{ channel.short_id }}" +
+ -
+
@@ -65,7 +72,7 @@
-
+
@@ -104,14 +111,6 @@
- -
- Error loading data. -

- {{ error.status }}: {{ error.error }} -
-
-
Lightning channel
diff --git a/frontend/src/app/lightning/channel/channel.component.ts b/frontend/src/app/lightning/channel/channel.component.ts index 379e8a005..d57aa3f01 100644 --- a/frontend/src/app/lightning/channel/channel.component.ts +++ b/frontend/src/app/lightning/channel/channel.component.ts @@ -38,7 +38,9 @@ export class ChannelComponent implements OnInit { }), catchError((err) => { this.error = err; - return of(null); + return [{ + short_id: params.get('short_id') + }]; }) ); }), diff --git a/frontend/src/app/lightning/node/node.component.html b/frontend/src/app/lightning/node/node.component.html index 575614c10..1519eb1da 100644 --- a/frontend/src/app/lightning/node/node.component.html +++ b/frontend/src/app/lightning/node/node.component.html @@ -1,15 +1,18 @@
-
Lightning node
-
-

{{ node.alias }}

- - - - - + + +
Lightning node
+
+

{{ node.alias }}

+ + + + + + - -
+
+
From 9a246c68de699675d3ddead751ef8d5c23d1ae57 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Sat, 25 Feb 2023 13:43:48 +0900 Subject: [PATCH 46/95] Center wrapping error message on mobile --- frontend/src/app/lightning/channel/channel.component.html | 2 +- frontend/src/app/lightning/node/node.component.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/app/lightning/channel/channel.component.html b/frontend/src/app/lightning/channel/channel.component.html index 96af4ab67..951cb8090 100644 --- a/frontend/src/app/lightning/channel/channel.component.html +++ b/frontend/src/app/lightning/channel/channel.component.html @@ -20,7 +20,7 @@
- No channel found for short id "{{ channel.short_id }}" + No channel found for short id "{{ channel.short_id }}"
- No node found for public key "{{ node.public_key | shortenString : 12}}" + No node found for public key "{{ node.public_key | shortenString : 12}}"
From c44896f53e46a7725ced8e9239d7cce9580bd337 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Thu, 16 Feb 2023 15:36:16 +0900 Subject: [PATCH 47/95] Get blocks data set by bulk (non indexed) --- backend/src/api/bitcoin/bitcoin.routes.ts | 28 +++++++++++ backend/src/api/blocks.ts | 58 ++++++++++++++++++++++- backend/src/rpc-api/commands.ts | 3 +- 3 files changed, 87 insertions(+), 2 deletions(-) diff --git a/backend/src/api/bitcoin/bitcoin.routes.ts b/backend/src/api/bitcoin/bitcoin.routes.ts index ea8154206..0ff30376c 100644 --- a/backend/src/api/bitcoin/bitcoin.routes.ts +++ b/backend/src/api/bitcoin/bitcoin.routes.ts @@ -95,6 +95,8 @@ class BitcoinRoutes { .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/summary', this.getStrippedBlockTransactions) .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/audit-summary', this.getBlockAuditSummary) .post(config.MEMPOOL.API_URL_PREFIX + 'psbt/addparents', this.postPsbtCompletion) + .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from', this.getBlocksByBulk.bind(this)) + .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from/:to', this.getBlocksByBulk.bind(this)) ; if (config.MEMPOOL.BACKEND !== 'esplora') { @@ -402,6 +404,32 @@ class BitcoinRoutes { } } + private async getBlocksByBulk(req: Request, res: Response) { + try { + if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { // Liquid, Bisq - Not implemented + return res.status(404).send(`Not implemented`); + } + + const from = parseInt(req.params.from, 10); + if (!from) { + return res.status(400).send(`Parameter 'from' must be a block height (integer)`); + } + const to = req.params.to === undefined ? await bitcoinApi.$getBlockHeightTip() : parseInt(req.params.to, 10); + if (!to) { + return res.status(400).send(`Parameter 'to' must be a block height (integer)`); + } + if (from > to) { + return res.status(400).send(`Parameter 'to' must be a higher block height than 'from'`); + } + + res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); + res.json(await blocks.$getBlocksByBulk(from, to)); + + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + private async getLegacyBlocks(req: Request, res: Response) { try { const returnBlocks: IEsploraApi.Block[] = []; diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index d110186f5..5c8884c71 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -688,7 +688,6 @@ class Blocks { } public async $getBlocks(fromHeight?: number, limit: number = 15): Promise { - let currentHeight = fromHeight !== undefined ? fromHeight : this.currentBlockHeight; if (currentHeight > this.currentBlockHeight) { limit -= currentHeight - this.currentBlockHeight; @@ -728,6 +727,63 @@ class Blocks { return returnBlocks; } + public async $getBlocksByBulk(start: number, end: number) { + start = Math.max(1, start); + + const blocks: any[] = []; + for (let i = end; i >= start; --i) { + const blockHash = await bitcoinApi.$getBlockHash(i); + const coreBlock = await bitcoinClient.getBlock(blockHash); + const electrsBlock = await bitcoinApi.$getBlock(blockHash); + const txs = await this.$getTransactionsExtended(blockHash, i, true); + const stats = await bitcoinClient.getBlockStats(blockHash); + const header = await bitcoinClient.getBlockHeader(blockHash, false); + const txoutset = await bitcoinClient.getTxoutSetinfo('none', i); + + const formatted = { + blockhash: coreBlock.id, + blockheight: coreBlock.height, + prev_blockhash: coreBlock.previousblockhash, + timestamp: coreBlock.timestamp, + median_timestamp: coreBlock.mediantime, + // @ts-ignore + blocktime: coreBlock.time, + orphaned: null, + header: header, + version: coreBlock.version, + difficulty: coreBlock.difficulty, + merkle_root: coreBlock.merkle_root, + bits: coreBlock.bits, + nonce: coreBlock.nonce, + coinbase_scriptsig: txs[0].vin[0].scriptsig, + coinbase_address: txs[0].vout[0].scriptpubkey_address, + coinbase_signature: txs[0].vout[0].scriptpubkey_asm, + size: coreBlock.size, + virtual_size: coreBlock.weight / 4.0, + weight: coreBlock.weight, + utxoset_size: txoutset.txouts, + utxoset_change: stats.utxo_increase, + total_txs: coreBlock.tx_count, + avg_tx_size: Math.round(stats.total_size / stats.txs * 100) * 0.01, + total_inputs: stats.ins, + total_outputs: stats.outs, + total_input_amt: Math.round(txoutset.block_info.prevout_spent * 100000000), + total_output_amt: stats.total_out, + block_subsidy: txs[0].vout.reduce((acc, curr) => acc + curr.value, 0), + total_fee: stats.totalfee, + avg_feerate: stats.avgfeerate, + feerate_percentiles: [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat(), + avg_fee: stats.avgfee, + fee_percentiles: null, + segwit_total_txs: stats.swtxs, + segwit_total_size: stats.swtotal_size, + segwit_total_weight: stats.swtotal_weight, + }; + blocks.push(formatted); + } + return blocks; + } + public async $getBlockAuditSummary(hash: string): Promise { let summary; if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) { diff --git a/backend/src/rpc-api/commands.ts b/backend/src/rpc-api/commands.ts index ea9bd7bf0..5905a2bb6 100644 --- a/backend/src/rpc-api/commands.ts +++ b/backend/src/rpc-api/commands.ts @@ -88,5 +88,6 @@ module.exports = { verifyTxOutProof: 'verifytxoutproof', // bitcoind v0.11.0+ walletLock: 'walletlock', walletPassphrase: 'walletpassphrase', - walletPassphraseChange: 'walletpassphrasechange' + walletPassphraseChange: 'walletpassphrasechange', + getTxoutSetinfo: 'gettxoutsetinfo' } From 73f76474dd7fa0d8f95e8fd02af8c923b6b7fa86 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Fri, 17 Feb 2023 21:21:21 +0900 Subject: [PATCH 48/95] Implemented coinstatsindex indexing --- backend/src/api/bitcoin/bitcoin.routes.ts | 7 +- .../src/api/bitcoin/esplora-api.interface.ts | 1 + backend/src/api/blocks.ts | 158 ++++++++++-------- backend/src/api/database-migration.ts | 29 +++- backend/src/api/mining/mining.ts | 41 ++++- backend/src/api/transaction-utils.ts | 1 + backend/src/index.ts | 1 + backend/src/indexer.ts | 76 ++++++++- backend/src/mempool.interfaces.ts | 21 +++ backend/src/repositories/BlocksRepository.ts | 107 +++++++++--- backend/src/rpc-api/commands.ts | 5 +- 11 files changed, 330 insertions(+), 117 deletions(-) diff --git a/backend/src/api/bitcoin/bitcoin.routes.ts b/backend/src/api/bitcoin/bitcoin.routes.ts index 0ff30376c..6d145e854 100644 --- a/backend/src/api/bitcoin/bitcoin.routes.ts +++ b/backend/src/api/bitcoin/bitcoin.routes.ts @@ -407,7 +407,10 @@ class BitcoinRoutes { private async getBlocksByBulk(req: Request, res: Response) { try { if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { // Liquid, Bisq - Not implemented - return res.status(404).send(`Not implemented`); + return res.status(404).send(`This API is only available for Bitcoin networks`); + } + if (!Common.indexingEnabled()) { + return res.status(404).send(`Indexing is required for this API`); } const from = parseInt(req.params.from, 10); @@ -423,7 +426,7 @@ class BitcoinRoutes { } res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); - res.json(await blocks.$getBlocksByBulk(from, to)); + res.json(await blocks.$getBlocksBetweenHeight(from, to)); } catch (e) { res.status(500).send(e instanceof Error ? e.message : e); diff --git a/backend/src/api/bitcoin/esplora-api.interface.ts b/backend/src/api/bitcoin/esplora-api.interface.ts index 39f8cfd6f..eaf6476f4 100644 --- a/backend/src/api/bitcoin/esplora-api.interface.ts +++ b/backend/src/api/bitcoin/esplora-api.interface.ts @@ -88,6 +88,7 @@ export namespace IEsploraApi { size: number; weight: number; previousblockhash: string; + medianTime?: number; } export interface Address { diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 5c8884c71..d950a9bd3 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -165,33 +165,75 @@ class Blocks { * @returns BlockExtended */ private async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise { - const blockExtended: BlockExtended = Object.assign({ extras: {} }, block); - blockExtended.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0); - blockExtended.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]); - blockExtended.extras.coinbaseRaw = blockExtended.extras.coinbaseTx.vin[0].scriptsig; - blockExtended.extras.usd = priceUpdater.latestPrices.USD; + const blk: BlockExtended = Object.assign({ extras: {} }, block); + blk.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0); + blk.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]); + blk.extras.coinbaseRaw = blk.extras.coinbaseTx.vin[0].scriptsig; + blk.extras.usd = priceUpdater.latestPrices.USD; if (block.height === 0) { - blockExtended.extras.medianFee = 0; // 50th percentiles - blockExtended.extras.feeRange = [0, 0, 0, 0, 0, 0, 0]; - blockExtended.extras.totalFees = 0; - blockExtended.extras.avgFee = 0; - blockExtended.extras.avgFeeRate = 0; + blk.extras.medianFee = 0; // 50th percentiles + blk.extras.feeRange = [0, 0, 0, 0, 0, 0, 0]; + blk.extras.totalFees = 0; + blk.extras.avgFee = 0; + blk.extras.avgFeeRate = 0; + blk.extras.utxoSetChange = 0; + blk.extras.avgTxSize = 0; + blk.extras.totalInputs = 0; + blk.extras.totalOutputs = 1; + blk.extras.totalOutputAmt = 0; + blk.extras.segwitTotalTxs = 0; + blk.extras.segwitTotalSize = 0; + blk.extras.segwitTotalWeight = 0; } else { - const stats = await bitcoinClient.getBlockStats(block.id, [ - 'feerate_percentiles', 'minfeerate', 'maxfeerate', 'totalfee', 'avgfee', 'avgfeerate' - ]); - blockExtended.extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles - blockExtended.extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat(); - blockExtended.extras.totalFees = stats.totalfee; - blockExtended.extras.avgFee = stats.avgfee; - blockExtended.extras.avgFeeRate = stats.avgfeerate; + const stats = await bitcoinClient.getBlockStats(block.id); + blk.extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles + blk.extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat(); + blk.extras.totalFees = stats.totalfee; + blk.extras.avgFee = stats.avgfee; + blk.extras.avgFeeRate = stats.avgfeerate; + blk.extras.utxoSetChange = stats.utxo_increase; + blk.extras.avgTxSize = Math.round(stats.total_size / stats.txs * 100) * 0.01; + blk.extras.totalInputs = stats.ins; + blk.extras.totalOutputs = stats.outs; + blk.extras.totalOutputAmt = stats.total_out; + blk.extras.segwitTotalTxs = stats.swtxs; + blk.extras.segwitTotalSize = stats.swtotal_size; + blk.extras.segwitTotalWeight = stats.swtotal_weight; + } + + blk.extras.feePercentiles = [], // TODO + blk.extras.medianFeeAmt = 0; // TODO + blk.extras.medianTimestamp = block.medianTime; // TODO + blk.extras.blockTime = 0; // TODO + blk.extras.orphaned = false; // TODO + + blk.extras.virtualSize = block.weight / 4.0; + if (blk.extras.coinbaseTx.vout.length > 0) { + blk.extras.coinbaseAddress = blk.extras.coinbaseTx.vout[0].scriptpubkey_address ?? null; + blk.extras.coinbaseSignature = blk.extras.coinbaseTx.vout[0].scriptpubkey_asm ?? null; + } else { + blk.extras.coinbaseAddress = null; + blk.extras.coinbaseSignature = null; + } + + const header = await bitcoinClient.getBlockHeader(block.id, false); + blk.extras.header = header; + + const coinStatsIndex = indexer.isCoreIndexReady('coinstatsindex'); + if (coinStatsIndex !== null && coinStatsIndex.best_block_height >= block.height) { + const txoutset = await bitcoinClient.getTxoutSetinfo('none', block.height); + blk.extras.utxoSetSize = txoutset.txouts, + blk.extras.totalInputAmt = Math.round(txoutset.block_info.prevout_spent * 100000000); + } else { + blk.extras.utxoSetSize = null; + blk.extras.totalInputAmt = null; } if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) { let pool: PoolTag; - if (blockExtended.extras?.coinbaseTx !== undefined) { - pool = await this.$findBlockMiner(blockExtended.extras?.coinbaseTx); + if (blk.extras?.coinbaseTx !== undefined) { + pool = await this.$findBlockMiner(blk.extras?.coinbaseTx); } else { if (config.DATABASE.ENABLED === true) { pool = await poolsRepository.$getUnknownPool(); @@ -201,10 +243,10 @@ class Blocks { } if (!pool) { // We should never have this situation in practise - logger.warn(`Cannot assign pool to block ${blockExtended.height} and 'unknown' pool does not exist. ` + + logger.warn(`Cannot assign pool to block ${blk.height} and 'unknown' pool does not exist. ` + `Check your "pools" table entries`); } else { - blockExtended.extras.pool = { + blk.extras.pool = { id: pool.id, name: pool.name, slug: pool.slug, @@ -214,12 +256,12 @@ class Blocks { if (config.MEMPOOL.AUDIT) { const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(block.id); if (auditScore != null) { - blockExtended.extras.matchRate = auditScore.matchRate; + blk.extras.matchRate = auditScore.matchRate; } } } - return blockExtended; + return blk; } /** @@ -727,60 +769,28 @@ class Blocks { return returnBlocks; } - public async $getBlocksByBulk(start: number, end: number) { - start = Math.max(1, start); + /** + * Used for bulk block data query + * + * @param fromHeight + * @param toHeight + */ + public async $getBlocksBetweenHeight(fromHeight: number, toHeight: number): Promise { + if (!Common.indexingEnabled()) { + return []; + } const blocks: any[] = []; - for (let i = end; i >= start; --i) { - const blockHash = await bitcoinApi.$getBlockHash(i); - const coreBlock = await bitcoinClient.getBlock(blockHash); - const electrsBlock = await bitcoinApi.$getBlock(blockHash); - const txs = await this.$getTransactionsExtended(blockHash, i, true); - const stats = await bitcoinClient.getBlockStats(blockHash); - const header = await bitcoinClient.getBlockHeader(blockHash, false); - const txoutset = await bitcoinClient.getTxoutSetinfo('none', i); - const formatted = { - blockhash: coreBlock.id, - blockheight: coreBlock.height, - prev_blockhash: coreBlock.previousblockhash, - timestamp: coreBlock.timestamp, - median_timestamp: coreBlock.mediantime, - // @ts-ignore - blocktime: coreBlock.time, - orphaned: null, - header: header, - version: coreBlock.version, - difficulty: coreBlock.difficulty, - merkle_root: coreBlock.merkle_root, - bits: coreBlock.bits, - nonce: coreBlock.nonce, - coinbase_scriptsig: txs[0].vin[0].scriptsig, - coinbase_address: txs[0].vout[0].scriptpubkey_address, - coinbase_signature: txs[0].vout[0].scriptpubkey_asm, - size: coreBlock.size, - virtual_size: coreBlock.weight / 4.0, - weight: coreBlock.weight, - utxoset_size: txoutset.txouts, - utxoset_change: stats.utxo_increase, - total_txs: coreBlock.tx_count, - avg_tx_size: Math.round(stats.total_size / stats.txs * 100) * 0.01, - total_inputs: stats.ins, - total_outputs: stats.outs, - total_input_amt: Math.round(txoutset.block_info.prevout_spent * 100000000), - total_output_amt: stats.total_out, - block_subsidy: txs[0].vout.reduce((acc, curr) => acc + curr.value, 0), - total_fee: stats.totalfee, - avg_feerate: stats.avgfeerate, - feerate_percentiles: [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat(), - avg_fee: stats.avgfee, - fee_percentiles: null, - segwit_total_txs: stats.swtxs, - segwit_total_size: stats.swtotal_size, - segwit_total_weight: stats.swtotal_weight, - }; - blocks.push(formatted); + while (fromHeight <= toHeight) { + let block = await blocksRepository.$getBlockByHeight(fromHeight); + if (!block) { + block = await this.$indexBlock(fromHeight); + } + blocks.push(block); + fromHeight++; } + return blocks; } diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 6e4221857..2c6adfd1b 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository'; import { RowDataPacket } from 'mysql2'; class DatabaseMigration { - private static currentVersion = 54; + private static currentVersion = 55; private queryTimeout = 3600_000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -483,6 +483,11 @@ class DatabaseMigration { } await this.updateToSchemaVersion(54); } + + if (databaseSchemaVersion < 55) { + await this.$executeQuery(this.getAdditionalBlocksDataQuery()); + await this.updateToSchemaVersion(55); + } } /** @@ -756,6 +761,28 @@ class DatabaseMigration { ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; } + private getAdditionalBlocksDataQuery(): string { + return `ALTER TABLE blocks + ADD median_timestamp timestamp NOT NULL, + ADD block_time int unsigned NOT NULL, + ADD coinbase_address varchar(100) NULL, + ADD coinbase_signature varchar(500) NULL, + ADD avg_tx_size double unsigned NOT NULL, + ADD total_inputs int unsigned NOT NULL, + ADD total_outputs int unsigned NOT NULL, + ADD total_output_amt bigint unsigned NOT NULL, + ADD fee_percentiles longtext NULL, + ADD median_fee_amt int unsigned NOT NULL, + ADD segwit_total_txs int unsigned NOT NULL, + ADD segwit_total_size int unsigned NOT NULL, + ADD segwit_total_weight int unsigned NOT NULL, + ADD header varchar(160) NOT NULL, + ADD utxoset_change int NOT NULL, + ADD utxoset_size int unsigned NULL, + ADD total_input_amt bigint unsigned NULL + `; + } + private getCreateDailyStatsTableQuery(): string { return `CREATE TABLE IF NOT EXISTS hashrates ( hashrate_timestamp timestamp NOT NULL, diff --git a/backend/src/api/mining/mining.ts b/backend/src/api/mining/mining.ts index edcb5b2e5..f33a68dcb 100644 --- a/backend/src/api/mining/mining.ts +++ b/backend/src/api/mining/mining.ts @@ -172,7 +172,7 @@ class Mining { } /** - * [INDEXING] Generate weekly mining pool hashrate history + * Generate weekly mining pool hashrate history */ public async $generatePoolHashrateHistory(): Promise { const now = new Date(); @@ -279,7 +279,7 @@ class Mining { } /** - * [INDEXING] Generate daily hashrate data + * Generate daily hashrate data */ public async $generateNetworkHashrateHistory(): Promise { // We only run this once a day around midnight @@ -459,7 +459,7 @@ class Mining { /** * Create a link between blocks and the latest price at when they were mined */ - public async $indexBlockPrices() { + public async $indexBlockPrices(): Promise { if (this.blocksPriceIndexingRunning === true) { return; } @@ -520,6 +520,41 @@ class Mining { this.blocksPriceIndexingRunning = false; } + /** + * Index core coinstatsindex + */ + public async $indexCoinStatsIndex(): Promise { + let timer = new Date().getTime() / 1000; + let totalIndexed = 0; + + const blockchainInfo = await bitcoinClient.getBlockchainInfo(); + let currentBlockHeight = blockchainInfo.blocks; + + while (currentBlockHeight > 0) { + const indexedBlocks = await BlocksRepository.$getBlocksMissingCoinStatsIndex( + currentBlockHeight, currentBlockHeight - 10000); + + for (const block of indexedBlocks) { + const txoutset = await bitcoinClient.getTxoutSetinfo('none', block.height); + await BlocksRepository.$updateCoinStatsIndexData(block.hash, txoutset.txouts, + Math.round(txoutset.block_info.prevout_spent * 100000000)); + ++totalIndexed; + + const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer); + if (elapsedSeconds > 5) { + logger.info(`Indexing coinstatsindex data for block #${block.height}. Indexed ${totalIndexed} blocks.`, logger.tags.mining); + timer = new Date().getTime() / 1000; + } + } + + currentBlockHeight -= 10000; + } + + if (totalIndexed) { + logger.info(`Indexing missing coinstatsindex data completed`, logger.tags.mining); + } + } + private getDateMidnight(date: Date): Date { date.setUTCHours(0); date.setUTCMinutes(0); diff --git a/backend/src/api/transaction-utils.ts b/backend/src/api/transaction-utils.ts index fb5aeea42..fb69419fc 100644 --- a/backend/src/api/transaction-utils.ts +++ b/backend/src/api/transaction-utils.ts @@ -14,6 +14,7 @@ class TransactionUtils { vout: tx.vout .map((vout) => ({ scriptpubkey_address: vout.scriptpubkey_address, + scriptpubkey_asm: vout.scriptpubkey_asm, value: vout.value })) .filter((vout) => vout.value) diff --git a/backend/src/index.ts b/backend/src/index.ts index 919c039c3..d8d46fc9f 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -36,6 +36,7 @@ import bitcoinRoutes from './api/bitcoin/bitcoin.routes'; import fundingTxFetcher from './tasks/lightning/sync-tasks/funding-tx-fetcher'; import forensicsService from './tasks/lightning/forensics.service'; import priceUpdater from './tasks/price-updater'; +import mining from './api/mining/mining'; import { AxiosError } from 'axios'; class Server { diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index 22f3ce319..41c8024e0 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -8,18 +8,67 @@ import bitcoinClient from './api/bitcoin/bitcoin-client'; import priceUpdater from './tasks/price-updater'; import PricesRepository from './repositories/PricesRepository'; +export interface CoreIndex { + name: string; + synced: boolean; + best_block_height: number; +} + class Indexer { runIndexer = true; indexerRunning = false; tasksRunning: string[] = []; + coreIndexes: CoreIndex[] = []; - public reindex() { + /** + * Check which core index is available for indexing + */ + public async checkAvailableCoreIndexes(): Promise { + const updatedCoreIndexes: CoreIndex[] = []; + + const indexes: any = await bitcoinClient.getIndexInfo(); + for (const indexName in indexes) { + const newState = { + name: indexName, + synced: indexes[indexName].synced, + best_block_height: indexes[indexName].best_block_height, + }; + logger.info(`Core index '${indexName}' is ${indexes[indexName].synced ? 'synced' : 'not synced'}. Best block height is ${indexes[indexName].best_block_height}`); + updatedCoreIndexes.push(newState); + + if (indexName === 'coinstatsindex' && newState.synced === true) { + const previousState = this.isCoreIndexReady('coinstatsindex'); + // if (!previousState || previousState.synced === false) { + this.runSingleTask('coinStatsIndex'); + // } + } + } + + this.coreIndexes = updatedCoreIndexes; + } + + /** + * Return the best block height if a core index is available, or 0 if not + * + * @param name + * @returns + */ + public isCoreIndexReady(name: string): CoreIndex | null { + for (const index of this.coreIndexes) { + if (index.name === name && index.synced === true) { + return index; + } + } + return null; + } + + public reindex(): void { if (Common.indexingEnabled()) { this.runIndexer = true; } } - public async runSingleTask(task: 'blocksPrices') { + public async runSingleTask(task: 'blocksPrices' | 'coinStatsIndex'): Promise { if (!Common.indexingEnabled()) { return; } @@ -28,20 +77,27 @@ class Indexer { this.tasksRunning.push(task); const lastestPriceId = await PricesRepository.$getLatestPriceId(); if (priceUpdater.historyInserted === false || lastestPriceId === null) { - logger.debug(`Blocks prices indexer is waiting for the price updater to complete`) + logger.debug(`Blocks prices indexer is waiting for the price updater to complete`); setTimeout(() => { - this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task) + this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task); this.runSingleTask('blocksPrices'); }, 10000); } else { - logger.debug(`Blocks prices indexer will run now`) + logger.debug(`Blocks prices indexer will run now`); await mining.$indexBlockPrices(); - this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task) + this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task); } } + + if (task === 'coinStatsIndex' && !this.tasksRunning.includes(task)) { + this.tasksRunning.push(task); + logger.debug(`Indexing coinStatsIndex now`); + await mining.$indexCoinStatsIndex(); + this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task); + } } - public async $run() { + public async $run(): Promise { if (!Common.indexingEnabled() || this.runIndexer === false || this.indexerRunning === true || mempool.hasPriority() ) { @@ -57,7 +113,9 @@ class Indexer { this.runIndexer = false; this.indexerRunning = true; - logger.debug(`Running mining indexer`); + logger.info(`Running mining indexer`); + + await this.checkAvailableCoreIndexes(); try { await priceUpdater.$run(); @@ -93,7 +151,7 @@ class Indexer { setTimeout(() => this.reindex(), runEvery); } - async $resetHashratesIndexingState() { + async $resetHashratesIndexingState(): Promise { try { await HashratesRepository.$setLatestRun('last_hashrates_indexing', 0); await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', 0); diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index 6b258c173..a1a9e1687 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -64,6 +64,7 @@ interface VinStrippedToScriptsig { interface VoutStrippedToScriptPubkey { scriptpubkey_address: string | undefined; + scriptpubkey_asm: string | undefined; value: number; } @@ -160,6 +161,26 @@ export interface BlockExtension { avgFeeRate?: number; coinbaseRaw?: string; usd?: number | null; + medianTimestamp?: number; + blockTime?: number; + orphaned?: boolean; + coinbaseAddress?: string | null; + coinbaseSignature?: string | null; + virtualSize?: number; + avgTxSize?: number; + totalInputs?: number; + totalOutputs?: number; + totalOutputAmt?: number; + medianFeeAmt?: number; + feePercentiles?: number[], + segwitTotalTxs?: number; + segwitTotalSize?: number; + segwitTotalWeight?: number; + header?: string; + utxoSetChange?: number; + // Requires coinstatsindex, will be set to NULL otherwise + utxoSetSize?: number | null; + totalInputAmt?: number | null; } export interface BlockExtended extends IEsploraApi.Block { diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index df98719b9..baaea38d9 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -18,17 +18,27 @@ class BlocksRepository { public async $saveBlockInDatabase(block: BlockExtended) { try { const query = `INSERT INTO blocks( - height, hash, blockTimestamp, size, - weight, tx_count, coinbase_raw, difficulty, - pool_id, fees, fee_span, median_fee, - reward, version, bits, nonce, - merkle_root, previous_block_hash, avg_fee, avg_fee_rate + height, hash, blockTimestamp, size, + weight, tx_count, coinbase_raw, difficulty, + pool_id, fees, fee_span, median_fee, + reward, version, bits, nonce, + merkle_root, previous_block_hash, avg_fee, avg_fee_rate, + median_timestamp, block_time, header, coinbase_address, + coinbase_signature, utxoset_size, utxoset_change, avg_tx_size, + total_inputs, total_outputs, total_input_amt, total_output_amt, + fee_percentiles, segwit_total_txs, segwit_total_size, segwit_total_weight, + median_fee_amt ) VALUE ( ?, ?, FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - ?, ?, ?, ? + ?, ?, ?, ?, + ?, ?, ?, ?, + ?, ?, ?, ?, + ?, ?, ?, ?, + ?, ?, ?, ?, + ? )`; const params: any[] = [ @@ -52,6 +62,23 @@ class BlocksRepository { block.previousblockhash, block.extras.avgFee, block.extras.avgFeeRate, + block.extras.medianTimestamp, + block.extras.blockTime, + block.extras.header, + block.extras.coinbaseAddress, + block.extras.coinbaseSignature, + block.extras.utxoSetSize, + block.extras.utxoSetChange, + block.extras.avgTxSize, + block.extras.totalInputs, + block.extras.totalOutputs, + block.extras.totalInputAmt, + block.extras.totalOutputAmt, + JSON.stringify(block.extras.feePercentiles), + block.extras.segwitTotalTxs, + block.extras.segwitTotalSize, + block.extras.segwitTotalWeight, + block.extras.medianFeeAmt, ]; await DB.query(query, params); @@ -65,6 +92,33 @@ class BlocksRepository { } } + /** + * Save newly indexed data from core coinstatsindex + * + * @param utxoSetSize + * @param totalInputAmt + */ + public async $updateCoinStatsIndexData(blockHash: string, utxoSetSize: number, + totalInputAmt: number + ) : Promise { + try { + const query = ` + UPDATE blocks + SET utxoset_size = ?, total_input_amt = ? + WHERE hash = ? + `; + const params: any[] = [ + utxoSetSize, + totalInputAmt, + blockHash + ]; + await DB.query(query, params); + } catch (e: any) { + logger.err('Cannot update indexed block coinstatsindex. Reason: ' + (e instanceof Error ? e.message : e)); + throw e; + } + } + /** * Get all block height that have not been indexed between [startHeight, endHeight] */ @@ -310,32 +364,16 @@ class BlocksRepository { public async $getBlockByHeight(height: number): Promise { try { const [rows]: any[] = await DB.query(`SELECT - blocks.height, - hash, + blocks.*, hash as id, UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp, - size, - weight, - tx_count, - coinbase_raw, - difficulty, pools.id as pool_id, pools.name as pool_name, pools.link as pool_link, pools.slug as pool_slug, pools.addresses as pool_addresses, pools.regexes as pool_regexes, - fees, - fee_span, - median_fee, - reward, - version, - bits, - nonce, - merkle_root, - previous_block_hash as previousblockhash, - avg_fee, - avg_fee_rate + previous_block_hash as previousblockhash FROM blocks JOIN pools ON blocks.pool_id = pools.id WHERE blocks.height = ${height} @@ -694,7 +732,6 @@ class BlocksRepository { logger.err('Cannot fetch CPFP unindexed blocks. Reason: ' + (e instanceof Error ? e.message : e)); throw e; } - return []; } /** @@ -741,7 +778,7 @@ class BlocksRepository { try { let query = `INSERT INTO blocks_prices(height, price_id) VALUES`; for (const price of blockPrices) { - query += ` (${price.height}, ${price.priceId}),` + query += ` (${price.height}, ${price.priceId}),`; } query = query.slice(0, -1); await DB.query(query); @@ -754,6 +791,24 @@ class BlocksRepository { } } } + + /** + * Get all indexed blocsk with missing coinstatsindex data + */ + public async $getBlocksMissingCoinStatsIndex(maxHeight: number, minHeight: number): Promise { + try { + const [blocks] = await DB.query(` + SELECT height, hash + FROM blocks + WHERE height >= ${minHeight} AND height <= ${maxHeight} AND + (utxoset_size IS NULL OR total_input_amt IS NULL) + `); + return blocks; + } catch (e) { + logger.err(`Cannot get blocks with missing coinstatsindex. Reason: ` + (e instanceof Error ? e.message : e)); + throw e; + } + } } export default new BlocksRepository(); diff --git a/backend/src/rpc-api/commands.ts b/backend/src/rpc-api/commands.ts index 5905a2bb6..78f5e12f4 100644 --- a/backend/src/rpc-api/commands.ts +++ b/backend/src/rpc-api/commands.ts @@ -89,5 +89,6 @@ module.exports = { walletLock: 'walletlock', walletPassphrase: 'walletpassphrase', walletPassphraseChange: 'walletpassphrasechange', - getTxoutSetinfo: 'gettxoutsetinfo' -} + getTxoutSetinfo: 'gettxoutsetinfo', + getIndexInfo: 'getindexinfo', +}; From 8612dd2d73cd173d02e8d01d640ddfb8c89f74c1 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Sat, 18 Feb 2023 08:28:59 +0900 Subject: [PATCH 49/95] Remove unescessary data from the blocks-bulk API --- backend/src/api/blocks.ts | 10 +++++++++- backend/src/repositories/BlocksRepository.ts | 1 + 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index d950a9bd3..9c1c1d05b 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -783,10 +783,18 @@ class Blocks { const blocks: any[] = []; while (fromHeight <= toHeight) { - let block = await blocksRepository.$getBlockByHeight(fromHeight); + let block: any = await blocksRepository.$getBlockByHeight(fromHeight); if (!block) { block = await this.$indexBlock(fromHeight); } + + delete(block.hash); + delete(block.previous_block_hash); + delete(block.pool_name); + delete(block.pool_link); + delete(block.pool_addresses); + delete(block.pool_regexes); + blocks.push(block); fromHeight++; } diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index baaea38d9..cc6fdeb08 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -384,6 +384,7 @@ class BlocksRepository { } rows[0].fee_span = JSON.parse(rows[0].fee_span); + rows[0].fee_percentiles = JSON.parse(rows[0].fee_percentiles); return rows[0]; } catch (e) { logger.err(`Cannot get indexed block ${height}. Reason: ` + (e instanceof Error ? e.message : e)); From 8f716a1d8c7e5eeb4370e36119cc2446b37355b5 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Sat, 18 Feb 2023 08:42:38 +0900 Subject: [PATCH 50/95] Fix median timestamp field - Fix reponse format when block is indexed on the fly --- backend/src/api/blocks.ts | 9 +++++++-- backend/src/repositories/BlocksRepository.ts | 1 + 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 9c1c1d05b..006d5f055 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -170,6 +170,7 @@ class Blocks { blk.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]); blk.extras.coinbaseRaw = blk.extras.coinbaseTx.vin[0].scriptsig; blk.extras.usd = priceUpdater.latestPrices.USD; + blk.extras.medianTimestamp = block.medianTime; if (block.height === 0) { blk.extras.medianFee = 0; // 50th percentiles @@ -204,7 +205,6 @@ class Blocks { blk.extras.feePercentiles = [], // TODO blk.extras.medianFeeAmt = 0; // TODO - blk.extras.medianTimestamp = block.medianTime; // TODO blk.extras.blockTime = 0; // TODO blk.extras.orphaned = false; // TODO @@ -785,7 +785,11 @@ class Blocks { while (fromHeight <= toHeight) { let block: any = await blocksRepository.$getBlockByHeight(fromHeight); if (!block) { - block = await this.$indexBlock(fromHeight); + await this.$indexBlock(fromHeight); + block = await blocksRepository.$getBlockByHeight(fromHeight); + if (!block) { + continue; + } } delete(block.hash); @@ -794,6 +798,7 @@ class Blocks { delete(block.pool_link); delete(block.pool_addresses); delete(block.pool_regexes); + delete(block.median_timestamp); blocks.push(block); fromHeight++; diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index cc6fdeb08..d7811f601 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -367,6 +367,7 @@ class BlocksRepository { blocks.*, hash as id, UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp, + UNIX_TIMESTAMP(blocks.median_timestamp) as medianTime, pools.id as pool_id, pools.name as pool_name, pools.link as pool_link, From 458f24c9f21a84269c4fd6e67ec57eef84ed4845 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Sat, 18 Feb 2023 11:26:13 +0900 Subject: [PATCH 51/95] Compute median fee and fee percentiles in sats --- backend/src/api/blocks.ts | 18 ++++++-- backend/src/api/database-migration.ts | 2 +- backend/src/mempool.interfaces.ts | 4 +- backend/src/repositories/BlocksRepository.ts | 19 +++++++++ .../repositories/BlocksSummariesRepository.ts | 42 +++++++++++++++++++ 5 files changed, 79 insertions(+), 6 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 006d5f055..ccf7bd2f4 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -203,10 +203,13 @@ class Blocks { blk.extras.segwitTotalWeight = stats.swtotal_weight; } - blk.extras.feePercentiles = [], // TODO - blk.extras.medianFeeAmt = 0; // TODO blk.extras.blockTime = 0; // TODO blk.extras.orphaned = false; // TODO + + blk.extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(block.id); + if (blk.extras.feePercentiles !== null) { + blk.extras.medianFeeAmt = blk.extras.feePercentiles[3]; + } blk.extras.virtualSize = block.weight / 4.0; if (blk.extras.coinbaseTx.vout.length > 0) { @@ -791,7 +794,6 @@ class Blocks { continue; } } - delete(block.hash); delete(block.previous_block_hash); delete(block.pool_name); @@ -800,6 +802,16 @@ class Blocks { delete(block.pool_regexes); delete(block.median_timestamp); + // This requires `blocks_summaries` to be available. It takes a very long + // time to index this table so we just try to serve the data the best we can + if (block.fee_percentiles === null) { + block.fee_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(block.id); + if (block.fee_percentiles !== null) { + block.median_fee_amt = block.fee_percentiles[3]; + await blocksRepository.$saveFeePercentilesForBlockId(block.id, block.fee_percentiles); + } + } + blocks.push(block); fromHeight++; } diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 2c6adfd1b..352abfbfe 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -772,7 +772,7 @@ class DatabaseMigration { ADD total_outputs int unsigned NOT NULL, ADD total_output_amt bigint unsigned NOT NULL, ADD fee_percentiles longtext NULL, - ADD median_fee_amt int unsigned NOT NULL, + ADD median_fee_amt int unsigned NULL, ADD segwit_total_txs int unsigned NOT NULL, ADD segwit_total_size int unsigned NOT NULL, ADD segwit_total_weight int unsigned NOT NULL, diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index a1a9e1687..a7e7c4ec6 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -171,8 +171,8 @@ export interface BlockExtension { totalInputs?: number; totalOutputs?: number; totalOutputAmt?: number; - medianFeeAmt?: number; - feePercentiles?: number[], + medianFeeAmt?: number | null; + feePercentiles?: number[] | null, segwitTotalTxs?: number; segwitTotalSize?: number; segwitTotalWeight?: number; diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index d7811f601..cc0b43fe9 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -811,6 +811,25 @@ class BlocksRepository { throw e; } } + + /** + * Save indexed median fee to avoid recomputing it later + * + * @param id + * @param feePercentiles + */ + public async $saveFeePercentilesForBlockId(id: string, feePercentiles: number[]): Promise { + try { + await DB.query(` + UPDATE blocks SET fee_percentiles = ?, median_fee_amt = ? + WHERE hash = ?`, + [JSON.stringify(feePercentiles), feePercentiles[3], id] + ); + } catch (e) { + logger.err(`Cannot update block fee_percentiles. Reason: ` + (e instanceof Error ? e.message : e)); + throw e; + } + } } export default new BlocksRepository(); diff --git a/backend/src/repositories/BlocksSummariesRepository.ts b/backend/src/repositories/BlocksSummariesRepository.ts index 1406a1d07..ebc83b7dd 100644 --- a/backend/src/repositories/BlocksSummariesRepository.ts +++ b/backend/src/repositories/BlocksSummariesRepository.ts @@ -80,6 +80,48 @@ class BlocksSummariesRepository { logger.err('Cannot delete indexed blocks summaries. Reason: ' + (e instanceof Error ? e.message : e)); } } + + /** + * Get the fee percentiles if the block has already been indexed, [] otherwise + * + * @param id + */ + public async $getFeePercentilesByBlockId(id: string): Promise { + try { + const [rows]: any[] = await DB.query(` + SELECT transactions + FROM blocks_summaries + WHERE id = ?`, + [id] + ); + if (rows === null || rows.length === 0) { + return null; + } + + const transactions = JSON.parse(rows[0].transactions); + if (transactions === null) { + return null; + } + + transactions.shift(); // Ignore coinbase + transactions.sort((a: any, b: any) => a.fee - b.fee); + const fees = transactions.map((t: any) => t.fee); + + return [ + fees[0], // min + fees[Math.max(0, Math.floor(fees.length * 0.1) - 1)], // 10th + fees[Math.max(0, Math.floor(fees.length * 0.25) - 1)], // 25th + fees[Math.max(0, Math.floor(fees.length * 0.5) - 1)], // median + fees[Math.max(0, Math.floor(fees.length * 0.75) - 1)], // 75th + fees[Math.max(0, Math.floor(fees.length * 0.9) - 1)], // 90th + fees[fees.length - 1], // max + ]; + + } catch (e) { + logger.err(`Cannot get block summaries transactions. Reason: ` + (e instanceof Error ? e.message : e)); + return null; + } + } } export default new BlocksSummariesRepository(); From 281899f5514417224be4275873b4804a3323dd7e Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Sat, 18 Feb 2023 14:10:07 +0900 Subject: [PATCH 52/95] List orphaned blocks in the new blocks-bulk API --- backend/src/api/blocks.ts | 8 ++++- backend/src/api/chain-tips.ts | 53 +++++++++++++++++++++++++++++++ backend/src/index.ts | 2 ++ backend/src/mempool.interfaces.ts | 3 +- 4 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 backend/src/api/chain-tips.ts diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index ccf7bd2f4..25c199de9 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -25,6 +25,7 @@ import mining from './mining/mining'; import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository'; import PricesRepository from '../repositories/PricesRepository'; import priceUpdater from '../tasks/price-updater'; +import chainTips from './chain-tips'; class Blocks { private blocks: BlockExtended[] = []; @@ -171,6 +172,7 @@ class Blocks { blk.extras.coinbaseRaw = blk.extras.coinbaseTx.vin[0].scriptsig; blk.extras.usd = priceUpdater.latestPrices.USD; blk.extras.medianTimestamp = block.medianTime; + blk.extras.orphans = chainTips.getOrphanedBlocksAtHeight(blk.height); if (block.height === 0) { blk.extras.medianFee = 0; // 50th percentiles @@ -204,7 +206,6 @@ class Blocks { } blk.extras.blockTime = 0; // TODO - blk.extras.orphaned = false; // TODO blk.extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(block.id); if (blk.extras.feePercentiles !== null) { @@ -545,6 +546,7 @@ class Blocks { } else { this.currentBlockHeight++; logger.debug(`New block found (#${this.currentBlockHeight})!`); + await chainTips.updateOrphanedBlocks(); } const blockHash = await bitcoinApi.$getBlockHash(this.currentBlockHeight); @@ -812,6 +814,10 @@ class Blocks { } } + // Re-org can happen after indexing so we need to always get the + // latest state from core + block.orphans = chainTips.getOrphanedBlocksAtHeight(block.height); + blocks.push(block); fromHeight++; } diff --git a/backend/src/api/chain-tips.ts b/backend/src/api/chain-tips.ts new file mode 100644 index 000000000..5b0aa8a5c --- /dev/null +++ b/backend/src/api/chain-tips.ts @@ -0,0 +1,53 @@ +import logger from "../logger"; +import bitcoinClient from "./bitcoin/bitcoin-client"; + +export interface ChainTip { + height: number; + hash: string; + branchlen: number; + status: 'invalid' | 'active' | 'valid-fork' | 'valid-headers' | 'headers-only'; +}; + +export interface OrphanedBlock { + height: number; + hash: string; + status: 'valid-fork' | 'valid-headers' | 'headers-only'; +} + +class ChainTips { + private chainTips: ChainTip[] = []; + private orphanedBlocks: OrphanedBlock[] = []; + + public async updateOrphanedBlocks(): Promise { + this.chainTips = await bitcoinClient.getChainTips(); + this.orphanedBlocks = []; + + for (const chain of this.chainTips) { + if (chain.status === 'valid-fork' || chain.status === 'valid-headers' || chain.status === 'headers-only') { + let block = await bitcoinClient.getBlock(chain.hash); + while (block && block.confirmations === -1) { + this.orphanedBlocks.push({ + height: block.height, + hash: block.hash, + status: chain.status + }); + block = await bitcoinClient.getBlock(block.previousblockhash); + } + } + } + + logger.debug(`Updated orphaned blocks cache. Found ${this.orphanedBlocks.length} orphaned blocks`); + } + + public getOrphanedBlocksAtHeight(height: number): OrphanedBlock[] { + const orphans: OrphanedBlock[] = []; + for (const block of this.orphanedBlocks) { + if (block.height === height) { + orphans.push(block); + } + } + return orphans; + } +} + +export default new ChainTips(); \ No newline at end of file diff --git a/backend/src/index.ts b/backend/src/index.ts index d8d46fc9f..6ea3ddc43 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -37,6 +37,7 @@ import fundingTxFetcher from './tasks/lightning/sync-tasks/funding-tx-fetcher'; import forensicsService from './tasks/lightning/forensics.service'; import priceUpdater from './tasks/price-updater'; import mining from './api/mining/mining'; +import chainTips from './api/chain-tips'; import { AxiosError } from 'axios'; class Server { @@ -134,6 +135,7 @@ class Server { } priceUpdater.$run(); + await chainTips.updateOrphanedBlocks(); this.setUpHttpApiRoutes(); diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index a7e7c4ec6..e139bde8f 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -1,4 +1,5 @@ import { IEsploraApi } from './api/bitcoin/esplora-api.interface'; +import { OrphanedBlock } from './api/chain-tips'; import { HeapNode } from "./utils/pairing-heap"; export interface PoolTag { @@ -163,7 +164,7 @@ export interface BlockExtension { usd?: number | null; medianTimestamp?: number; blockTime?: number; - orphaned?: boolean; + orphans?: OrphanedBlock[] | null; coinbaseAddress?: string | null; coinbaseSignature?: string | null; virtualSize?: number; From e2fe39f241432ed6ec387952f5570071f7b81658 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Sat, 18 Feb 2023 14:53:21 +0900 Subject: [PATCH 53/95] Wrap orphaned blocks updater into try/catch --- backend/src/api/chain-tips.ts | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/backend/src/api/chain-tips.ts b/backend/src/api/chain-tips.ts index 5b0aa8a5c..92f148c8e 100644 --- a/backend/src/api/chain-tips.ts +++ b/backend/src/api/chain-tips.ts @@ -19,24 +19,28 @@ class ChainTips { private orphanedBlocks: OrphanedBlock[] = []; public async updateOrphanedBlocks(): Promise { - this.chainTips = await bitcoinClient.getChainTips(); - this.orphanedBlocks = []; + try { + this.chainTips = await bitcoinClient.getChainTips(); + this.orphanedBlocks = []; - for (const chain of this.chainTips) { - if (chain.status === 'valid-fork' || chain.status === 'valid-headers' || chain.status === 'headers-only') { - let block = await bitcoinClient.getBlock(chain.hash); - while (block && block.confirmations === -1) { - this.orphanedBlocks.push({ - height: block.height, - hash: block.hash, - status: chain.status - }); - block = await bitcoinClient.getBlock(block.previousblockhash); + for (const chain of this.chainTips) { + if (chain.status === 'valid-fork' || chain.status === 'valid-headers' || chain.status === 'headers-only') { + let block = await bitcoinClient.getBlock(chain.hash); + while (block && block.confirmations === -1) { + this.orphanedBlocks.push({ + height: block.height, + hash: block.hash, + status: chain.status + }); + block = await bitcoinClient.getBlock(block.previousblockhash); + } } } - } - logger.debug(`Updated orphaned blocks cache. Found ${this.orphanedBlocks.length} orphaned blocks`); + logger.debug(`Updated orphaned blocks cache. Found ${this.orphanedBlocks.length} orphaned blocks`); + } catch (e) { + logger.err(`Cannot get fetch orphaned blocks. Reason: ${e instanceof Error ? e.message : e}`); + } } public getOrphanedBlocksAtHeight(height: number): OrphanedBlock[] { From 6965c8f41ba3d2a358f17f27294f12fd0798bca8 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Sun, 19 Feb 2023 18:38:23 +0900 Subject: [PATCH 54/95] Fix median time indexing --- backend/src/api/bitcoin/bitcoin-api.ts | 1 + backend/src/repositories/BlocksRepository.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/src/api/bitcoin/bitcoin-api.ts b/backend/src/api/bitcoin/bitcoin-api.ts index cad11aeda..117245ef8 100644 --- a/backend/src/api/bitcoin/bitcoin-api.ts +++ b/backend/src/api/bitcoin/bitcoin-api.ts @@ -28,6 +28,7 @@ class BitcoinApi implements AbstractBitcoinApi { size: block.size, weight: block.weight, previousblockhash: block.previousblockhash, + medianTime: block.mediantime, }; } diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index cc0b43fe9..20331897c 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -34,7 +34,7 @@ class BlocksRepository { ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - ?, ?, ?, ?, + FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, From eceedf0bdfa6fd806596d2f707bc3f437de94c9d Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Sun, 19 Feb 2023 19:17:51 +0900 Subject: [PATCH 55/95] Dont compute fee percentile / median fee when indexing is disabled because we need summaries --- backend/src/api/blocks.ts | 8 +++++--- backend/src/database.ts | 3 ++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 25c199de9..8a11bccc5 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -207,9 +207,11 @@ class Blocks { blk.extras.blockTime = 0; // TODO - blk.extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(block.id); - if (blk.extras.feePercentiles !== null) { - blk.extras.medianFeeAmt = blk.extras.feePercentiles[3]; + if (Common.indexingEnabled()) { + blk.extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(block.id); + if (blk.extras.feePercentiles !== null) { + blk.extras.medianFeeAmt = blk.extras.feePercentiles[3]; + } } blk.extras.virtualSize = block.weight / 4.0; diff --git a/backend/src/database.ts b/backend/src/database.ts index c2fb0980b..a504eb0fa 100644 --- a/backend/src/database.ts +++ b/backend/src/database.ts @@ -24,7 +24,8 @@ import { FieldPacket, OkPacket, PoolOptions, ResultSetHeader, RowDataPacket } fr private checkDBFlag() { if (config.DATABASE.ENABLED === false) { - logger.err('Trying to use DB feature but config.DATABASE.ENABLED is set to false, please open an issue'); + const stack = new Error().stack; + logger.err(`Trying to use DB feature but config.DATABASE.ENABLED is set to false, please open an issue.\nStack trace: ${stack}}`); } } From b2eaa7efb1636ddcfb37fc3ab1c7671c82be91d1 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Mon, 20 Feb 2023 11:59:38 +0900 Subject: [PATCH 56/95] Fix fee percentiles indexing --- backend/src/repositories/BlocksRepository.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 20331897c..1f244d7cd 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -74,7 +74,7 @@ class BlocksRepository { block.extras.totalOutputs, block.extras.totalInputAmt, block.extras.totalOutputAmt, - JSON.stringify(block.extras.feePercentiles), + block.extras.feePercentiles ? JSON.stringify(block.extras.feePercentiles) : null, block.extras.segwitTotalTxs, block.extras.segwitTotalSize, block.extras.segwitTotalWeight, From 75a99568bfff64d5192df21f9093bc3e412b07fd Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Thu, 23 Feb 2023 08:50:30 +0900 Subject: [PATCH 57/95] Index coinbase signature in ascii --- backend/src/api/blocks.ts | 2 ++ backend/src/api/database-migration.ts | 1 + backend/src/mempool.interfaces.ts | 1 + backend/src/repositories/BlocksRepository.ts | 5 +++-- 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 8a11bccc5..3d33642ce 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -218,9 +218,11 @@ class Blocks { if (blk.extras.coinbaseTx.vout.length > 0) { blk.extras.coinbaseAddress = blk.extras.coinbaseTx.vout[0].scriptpubkey_address ?? null; blk.extras.coinbaseSignature = blk.extras.coinbaseTx.vout[0].scriptpubkey_asm ?? null; + blk.extras.coinbaseSignatureAscii = transactionUtils.hex2ascii(blk.extras.coinbaseTx.vin[0].scriptsig) ?? null; } else { blk.extras.coinbaseAddress = null; blk.extras.coinbaseSignature = null; + blk.extras.coinbaseSignatureAscii = null; } const header = await bitcoinClient.getBlockHeader(block.id, false); diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 352abfbfe..c965ef420 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -767,6 +767,7 @@ class DatabaseMigration { ADD block_time int unsigned NOT NULL, ADD coinbase_address varchar(100) NULL, ADD coinbase_signature varchar(500) NULL, + ADD coinbase_signature_ascii varchar(500) NULL, ADD avg_tx_size double unsigned NOT NULL, ADD total_inputs int unsigned NOT NULL, ADD total_outputs int unsigned NOT NULL, diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index e139bde8f..cb95be98a 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -167,6 +167,7 @@ export interface BlockExtension { orphans?: OrphanedBlock[] | null; coinbaseAddress?: string | null; coinbaseSignature?: string | null; + coinbaseSignatureAscii?: string | null; virtualSize?: number; avgTxSize?: number; totalInputs?: number; diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 1f244d7cd..e2362b67d 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -27,7 +27,7 @@ class BlocksRepository { coinbase_signature, utxoset_size, utxoset_change, avg_tx_size, total_inputs, total_outputs, total_input_amt, total_output_amt, fee_percentiles, segwit_total_txs, segwit_total_size, segwit_total_weight, - median_fee_amt + median_fee_amt, coinbase_signature_ascii ) VALUE ( ?, ?, FROM_UNIXTIME(?), ?, ?, ?, ?, ?, @@ -38,7 +38,7 @@ class BlocksRepository { ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - ? + ?, ? )`; const params: any[] = [ @@ -79,6 +79,7 @@ class BlocksRepository { block.extras.segwitTotalSize, block.extras.segwitTotalWeight, block.extras.medianFeeAmt, + block.extras.coinbaseSignatureAscii, ]; await DB.query(query, params); From 086ee68b520156e8b1fe6b63dc2cbe78ea4e2e5f Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Fri, 24 Feb 2023 10:29:58 +0900 Subject: [PATCH 58/95] Remove `block_time` from indexed fields --- backend/src/api/blocks.ts | 2 -- backend/src/api/database-migration.ts | 1 - 2 files changed, 3 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 3d33642ce..ba3927ab7 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -205,8 +205,6 @@ class Blocks { blk.extras.segwitTotalWeight = stats.swtotal_weight; } - blk.extras.blockTime = 0; // TODO - if (Common.indexingEnabled()) { blk.extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(block.id); if (blk.extras.feePercentiles !== null) { diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index c965ef420..6e6e6855f 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -764,7 +764,6 @@ class DatabaseMigration { private getAdditionalBlocksDataQuery(): string { return `ALTER TABLE blocks ADD median_timestamp timestamp NOT NULL, - ADD block_time int unsigned NOT NULL, ADD coinbase_address varchar(100) NULL, ADD coinbase_signature varchar(500) NULL, ADD coinbase_signature_ascii varchar(500) NULL, From a0488dba7664acc26e7517e3b019b8dd28d43324 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Fri, 24 Feb 2023 11:33:36 +0900 Subject: [PATCH 59/95] Cleanup block before sending response in /blocks-bulk API Remove block_time Index summaries on the fly --- backend/src/api/blocks.ts | 70 +++++++++++++++----- backend/src/repositories/BlocksRepository.ts | 5 +- 2 files changed, 55 insertions(+), 20 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index ba3927ab7..459b3903e 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -205,7 +205,7 @@ class Blocks { blk.extras.segwitTotalWeight = stats.swtotal_weight; } - if (Common.indexingEnabled()) { + if (Common.blocksSummariesIndexingEnabled()) { blk.extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(block.id); if (blk.extras.feePercentiles !== null) { blk.extras.medianFeeAmt = blk.extras.feePercentiles[3]; @@ -798,29 +798,65 @@ class Blocks { continue; } } - delete(block.hash); - delete(block.previous_block_hash); - delete(block.pool_name); - delete(block.pool_link); - delete(block.pool_addresses); - delete(block.pool_regexes); - delete(block.median_timestamp); - // This requires `blocks_summaries` to be available. It takes a very long - // time to index this table so we just try to serve the data the best we can - if (block.fee_percentiles === null) { - block.fee_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(block.id); - if (block.fee_percentiles !== null) { - block.median_fee_amt = block.fee_percentiles[3]; - await blocksRepository.$saveFeePercentilesForBlockId(block.id, block.fee_percentiles); + // Cleanup fields before sending the response + const cleanBlock: any = { + height: block.height ?? null, + hash: block.id ?? null, + timestamp: block.blockTimestamp ?? null, + median_timestamp: block.medianTime ?? null, + previousblockhash: block.previousblockhash ?? null, + difficulty: block.difficulty ?? null, + header: block.header ?? null, + version: block.version ?? null, + bits: block.bits ?? null, + nonce: block.nonce ?? null, + size: block.size ?? null, + weight: block.weight ?? null, + tx_count: block.tx_count ?? null, + merkle_root: block.merkle_root ?? null, + reward: block.reward ?? null, + total_fee_amt: block.fees ?? null, + avg_fee_amt: block.avg_fee ?? null, + median_fee_amt: block.median_fee_amt ?? null, + fee_amt_percentiles: block.fee_percentiles ?? null, + avg_fee_rate: block.avg_fee_rate ?? null, + median_fee_rate: block.median_fee ?? null, + fee_rate_percentiles: block.fee_span ?? null, + total_inputs: block.total_inputs ?? null, + total_input_amt: block.total_input_amt ?? null, + total_outputs: block.total_outputs ?? null, + total_output_amt: block.total_output_amt ?? null, + segwit_total_txs: block.segwit_total_txs ?? null, + segwit_total_size: block.segwit_total_size ?? null, + segwit_total_weight: block.segwit_total_weight ?? null, + avg_tx_size: block.avg_tx_size ?? null, + utxoset_change: block.utxoset_change ?? null, + utxoset_size: block.utxoset_size ?? null, + coinbase_raw: block.coinbase_raw ?? null, + coinbase_address: block.coinbase_address ?? null, + coinbase_signature: block.coinbase_signature ?? null, + pool_slug: block.pool_slug ?? null, + }; + + if (Common.blocksSummariesIndexingEnabled() && cleanBlock.fee_amt_percentiles === null) { + cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash); + if (cleanBlock.fee_amt_percentiles === null) { + const block = await bitcoinClient.getBlock(cleanBlock.hash, 2); + const summary = this.summarizeBlock(block); + await BlocksSummariesRepository.$saveSummary({ height: block.height, mined: summary }); + cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash); + } + if (cleanBlock.fee_amt_percentiles !== null) { + cleanBlock.median_fee_amt = cleanBlock.fee_amt_percentiles[3]; } } // Re-org can happen after indexing so we need to always get the // latest state from core - block.orphans = chainTips.getOrphanedBlocksAtHeight(block.height); + cleanBlock.orphans = chainTips.getOrphanedBlocksAtHeight(cleanBlock.height); - blocks.push(block); + blocks.push(cleanBlock); fromHeight++; } diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index e2362b67d..86dc006ff 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -23,7 +23,7 @@ class BlocksRepository { pool_id, fees, fee_span, median_fee, reward, version, bits, nonce, merkle_root, previous_block_hash, avg_fee, avg_fee_rate, - median_timestamp, block_time, header, coinbase_address, + median_timestamp, header, coinbase_address, coinbase_signature, utxoset_size, utxoset_change, avg_tx_size, total_inputs, total_outputs, total_input_amt, total_output_amt, fee_percentiles, segwit_total_txs, segwit_total_size, segwit_total_weight, @@ -34,7 +34,7 @@ class BlocksRepository { ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - FROM_UNIXTIME(?), ?, ?, ?, + FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, @@ -63,7 +63,6 @@ class BlocksRepository { block.extras.avgFee, block.extras.avgFeeRate, block.extras.medianTimestamp, - block.extras.blockTime, block.extras.header, block.extras.coinbaseAddress, block.extras.coinbaseSignature, From 0bf4d5218326373cdafb195ae3c08235419f76fd Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Fri, 24 Feb 2023 11:41:54 +0900 Subject: [PATCH 60/95] Return zeroed out `fee_amt_percentiles` if there is no transaction --- .../src/repositories/BlocksSummariesRepository.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/backend/src/repositories/BlocksSummariesRepository.ts b/backend/src/repositories/BlocksSummariesRepository.ts index ebc83b7dd..2724ddcf5 100644 --- a/backend/src/repositories/BlocksSummariesRepository.ts +++ b/backend/src/repositories/BlocksSummariesRepository.ts @@ -108,13 +108,13 @@ class BlocksSummariesRepository { const fees = transactions.map((t: any) => t.fee); return [ - fees[0], // min - fees[Math.max(0, Math.floor(fees.length * 0.1) - 1)], // 10th - fees[Math.max(0, Math.floor(fees.length * 0.25) - 1)], // 25th - fees[Math.max(0, Math.floor(fees.length * 0.5) - 1)], // median - fees[Math.max(0, Math.floor(fees.length * 0.75) - 1)], // 75th - fees[Math.max(0, Math.floor(fees.length * 0.9) - 1)], // 90th - fees[fees.length - 1], // max + fees[0] ?? 0, // min + fees[Math.max(0, Math.floor(fees.length * 0.1) - 1)] ?? 0, // 10th + fees[Math.max(0, Math.floor(fees.length * 0.25) - 1)] ?? 0, // 25th + fees[Math.max(0, Math.floor(fees.length * 0.5) - 1)] ?? 0, // median + fees[Math.max(0, Math.floor(fees.length * 0.75) - 1)] ?? 0, // 75th + fees[Math.max(0, Math.floor(fees.length * 0.9) - 1)] ?? 0, // 90th + fees[fees.length - 1] ?? 0, // max ]; } catch (e) { From aa1114926c2296d79b9429a6e0ca47cb7b117c62 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Fri, 24 Feb 2023 11:43:38 +0900 Subject: [PATCH 61/95] `previousblockhash` -> `previous_block_hash` --- backend/src/api/blocks.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 459b3903e..2a026a303 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -805,7 +805,7 @@ class Blocks { hash: block.id ?? null, timestamp: block.blockTimestamp ?? null, median_timestamp: block.medianTime ?? null, - previousblockhash: block.previousblockhash ?? null, + previous_block_hash: block.previousblockhash ?? null, difficulty: block.difficulty ?? null, header: block.header ?? null, version: block.version ?? null, From e19db4ae35e8d4f3a81c1a5b7c1ba5802d24bcd2 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Fri, 24 Feb 2023 11:46:47 +0900 Subject: [PATCH 62/95] Add missing `coinbase_signature_ascii` --- backend/src/api/blocks.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 2a026a303..fb38e0d7e 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -836,6 +836,7 @@ class Blocks { coinbase_raw: block.coinbase_raw ?? null, coinbase_address: block.coinbase_address ?? null, coinbase_signature: block.coinbase_signature ?? null, + coinbase_signature_ascii: block.coinbase_signature_ascii ?? null, pool_slug: block.pool_slug ?? null, }; From ed8cf89fee51bf54a39f60421255711d0d981509 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Fri, 24 Feb 2023 12:48:55 +0900 Subject: [PATCH 63/95] Format percentiles in a more verbose way --- backend/src/api/blocks.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index fb38e0d7e..204419496 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -853,6 +853,25 @@ class Blocks { } } + cleanBlock.fee_amt_percentiles = { + 'min': cleanBlock.fee_amt_percentiles[0], + 'perc_10': cleanBlock.fee_amt_percentiles[1], + 'perc_25': cleanBlock.fee_amt_percentiles[2], + 'perc_50': cleanBlock.fee_amt_percentiles[3], + 'perc_75': cleanBlock.fee_amt_percentiles[4], + 'perc_90': cleanBlock.fee_amt_percentiles[5], + 'max': cleanBlock.fee_amt_percentiles[6], + }; + cleanBlock.fee_rate_percentiles = { + 'min': cleanBlock.fee_rate_percentiles[0], + 'perc_10': cleanBlock.fee_rate_percentiles[1], + 'perc_25': cleanBlock.fee_rate_percentiles[2], + 'perc_50': cleanBlock.fee_rate_percentiles[3], + 'perc_75': cleanBlock.fee_rate_percentiles[4], + 'perc_90': cleanBlock.fee_rate_percentiles[5], + 'max': cleanBlock.fee_rate_percentiles[6], + }; + // Re-org can happen after indexing so we need to always get the // latest state from core cleanBlock.orphans = chainTips.getOrphanedBlocksAtHeight(cleanBlock.height); From 6c3a273e7588d53a495abfd3dccceaf0b9995ce7 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Fri, 24 Feb 2023 14:24:47 +0900 Subject: [PATCH 64/95] Enabled coinstatsindex=1 --- production/bitcoin.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/production/bitcoin.conf b/production/bitcoin.conf index 46ab41b20..501f49f50 100644 --- a/production/bitcoin.conf +++ b/production/bitcoin.conf @@ -1,6 +1,7 @@ datadir=/bitcoin server=1 txindex=1 +coinstatsindex=1 listen=1 discover=1 par=16 From 822362c10584cc2eb4bc376685bd19e50be3b4cb Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Fri, 24 Feb 2023 14:30:59 +0900 Subject: [PATCH 65/95] Increase cache schema version --- backend/src/api/disk-cache.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/api/disk-cache.ts b/backend/src/api/disk-cache.ts index cf40d6952..a75fd43cc 100644 --- a/backend/src/api/disk-cache.ts +++ b/backend/src/api/disk-cache.ts @@ -9,7 +9,7 @@ import { TransactionExtended } from '../mempool.interfaces'; import { Common } from './common'; class DiskCache { - private cacheSchemaVersion = 1; + private cacheSchemaVersion = 2; private static FILE_NAME = config.MEMPOOL.CACHE_DIR + '/cache.json'; private static FILE_NAMES = config.MEMPOOL.CACHE_DIR + '/cache{number}.json'; From ad4cbd60d5623dde7b8940065775dd3e24be79bc Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Fri, 24 Feb 2023 14:40:28 +0900 Subject: [PATCH 66/95] Do not download orphaned block if `headers-only` --- backend/src/api/chain-tips.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/api/chain-tips.ts b/backend/src/api/chain-tips.ts index 92f148c8e..3384ebb19 100644 --- a/backend/src/api/chain-tips.ts +++ b/backend/src/api/chain-tips.ts @@ -24,7 +24,7 @@ class ChainTips { this.orphanedBlocks = []; for (const chain of this.chainTips) { - if (chain.status === 'valid-fork' || chain.status === 'valid-headers' || chain.status === 'headers-only') { + if (chain.status === 'valid-fork' || chain.status === 'valid-headers') { let block = await bitcoinClient.getBlock(chain.hash); while (block && block.confirmations === -1) { this.orphanedBlocks.push({ From 5d7c9f93153c501fe9d10080290510a877a146d7 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Fri, 24 Feb 2023 15:06:47 +0900 Subject: [PATCH 67/95] Add config.MEMPOOOL.MAX_BLOCKS_BULK_QUERY parameter (default to 0, API disable) --- backend/src/__fixtures__/mempool-config.template.json | 3 ++- backend/src/__tests__/config.test.ts | 1 + backend/src/api/bitcoin/bitcoin.routes.ts | 10 ++++++++-- backend/src/config.ts | 2 ++ docker/README.md | 2 ++ docker/backend/mempool-config.json | 3 ++- docker/backend/start.sh | 2 ++ 7 files changed, 19 insertions(+), 4 deletions(-) diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index 9d8a7e900..fa7ea7d21 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -28,7 +28,8 @@ "AUDIT": "__MEMPOOL_AUDIT__", "ADVANCED_GBT_AUDIT": "__MEMPOOL_ADVANCED_GBT_AUDIT__", "ADVANCED_GBT_MEMPOOL": "__MEMPOOL_ADVANCED_GBT_MEMPOOL__", - "CPFP_INDEXING": "__MEMPOOL_CPFP_INDEXING__" + "CPFP_INDEXING": "__MEMPOOL_CPFP_INDEXING__", + "MAX_BLOCKS_BULK_QUERY": "__MEMPOOL_MAX_BLOCKS_BULK_QUERY__" }, "CORE_RPC": { "HOST": "__CORE_RPC_HOST__", diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index 8b011d833..1e4c05ae3 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -41,6 +41,7 @@ describe('Mempool Backend Config', () => { ADVANCED_GBT_AUDIT: false, ADVANCED_GBT_MEMPOOL: false, CPFP_INDEXING: false, + MAX_BLOCKS_BULK_QUERY: 0, }); expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true }); diff --git a/backend/src/api/bitcoin/bitcoin.routes.ts b/backend/src/api/bitcoin/bitcoin.routes.ts index 6d145e854..78d027663 100644 --- a/backend/src/api/bitcoin/bitcoin.routes.ts +++ b/backend/src/api/bitcoin/bitcoin.routes.ts @@ -409,21 +409,27 @@ class BitcoinRoutes { if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { // Liquid, Bisq - Not implemented return res.status(404).send(`This API is only available for Bitcoin networks`); } + if (config.MEMPOOL.MAX_BLOCKS_BULK_QUERY <= 0) { + return res.status(404).send(`This API is disabled. Set config.MEMPOOL.MAX_BLOCKS_BULK_QUERY to a positive number to enable it.`); + } if (!Common.indexingEnabled()) { return res.status(404).send(`Indexing is required for this API`); } const from = parseInt(req.params.from, 10); - if (!from) { + if (!req.params.from || from < 0) { return res.status(400).send(`Parameter 'from' must be a block height (integer)`); } const to = req.params.to === undefined ? await bitcoinApi.$getBlockHeightTip() : parseInt(req.params.to, 10); - if (!to) { + if (to < 0) { return res.status(400).send(`Parameter 'to' must be a block height (integer)`); } if (from > to) { return res.status(400).send(`Parameter 'to' must be a higher block height than 'from'`); } + if ((to - from + 1) > config.MEMPOOL.MAX_BLOCKS_BULK_QUERY) { + return res.status(400).send(`You can only query ${config.MEMPOOL.MAX_BLOCKS_BULK_QUERY} blocks at once.`); + } res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); res.json(await blocks.$getBlocksBetweenHeight(from, to)); diff --git a/backend/src/config.ts b/backend/src/config.ts index 2cda8d85b..ecd5c80aa 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -32,6 +32,7 @@ interface IConfig { ADVANCED_GBT_AUDIT: boolean; ADVANCED_GBT_MEMPOOL: boolean; CPFP_INDEXING: boolean; + MAX_BLOCKS_BULK_QUERY: number; }; ESPLORA: { REST_API_URL: string; @@ -153,6 +154,7 @@ const defaults: IConfig = { 'ADVANCED_GBT_AUDIT': false, 'ADVANCED_GBT_MEMPOOL': false, 'CPFP_INDEXING': false, + 'MAX_BLOCKS_BULK_QUERY': 0, }, 'ESPLORA': { 'REST_API_URL': 'http://127.0.0.1:3000', diff --git a/docker/README.md b/docker/README.md index 69bb96030..168d4b1fa 100644 --- a/docker/README.md +++ b/docker/README.md @@ -111,6 +111,7 @@ Below we list all settings from `mempool-config.json` and the corresponding over "ADVANCED_GBT_AUDIT": false, "ADVANCED_GBT_MEMPOOL": false, "CPFP_INDEXING": false, + "MAX_BLOCKS_BULK_QUERY": 0, }, ``` @@ -141,6 +142,7 @@ Corresponding `docker-compose.yml` overrides: MEMPOOL_ADVANCED_GBT_AUDIT: "" MEMPOOL_ADVANCED_GBT_MEMPOOL: "" MEMPOOL_CPFP_INDEXING: "" + MAX_BLOCKS_BULK_QUERY: "" ... ``` diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index 904370f3e..d2aa75c69 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -25,7 +25,8 @@ "AUDIT": __MEMPOOL_AUDIT__, "ADVANCED_GBT_AUDIT": __MEMPOOL_ADVANCED_GBT_AUDIT__, "ADVANCED_GBT_MEMPOOL": __MEMPOOL_ADVANCED_GBT_MEMPOOL__, - "CPFP_INDEXING": __MEMPOOL_CPFP_INDEXING__ + "CPFP_INDEXING": __MEMPOOL_CPFP_INDEXING__, + "MAX_BLOCKS_BULK_QUERY": __MEMPOOL__MAX_BLOCKS_BULK_QUERY__ }, "CORE_RPC": { "HOST": "__CORE_RPC_HOST__", diff --git a/docker/backend/start.sh b/docker/backend/start.sh index 58b19898a..3ee542892 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -30,6 +30,7 @@ __MEMPOOL_AUDIT__=${MEMPOOL_AUDIT:=false} __MEMPOOL_ADVANCED_GBT_AUDIT__=${MEMPOOL_ADVANCED_GBT_AUDIT:=false} __MEMPOOL_ADVANCED_GBT_MEMPOOL__=${MEMPOOL_ADVANCED_GBT_MEMPOOL:=false} __MEMPOOL_CPFP_INDEXING__=${MEMPOOL_CPFP_INDEXING:=false} +__MEMPOOL_MAX_BLOCKS_BULK_QUERY__=${MEMPOOL_MAX_BLOCKS_BULK_QUERY:=0} # CORE_RPC __CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1} @@ -142,6 +143,7 @@ sed -i "s!__MEMPOOL_AUDIT__!${__MEMPOOL_AUDIT__}!g" mempool-config.json sed -i "s!__MEMPOOL_ADVANCED_GBT_MEMPOOL__!${__MEMPOOL_ADVANCED_GBT_MEMPOOL__}!g" mempool-config.json sed -i "s!__MEMPOOL_ADVANCED_GBT_AUDIT__!${__MEMPOOL_ADVANCED_GBT_AUDIT__}!g" mempool-config.json sed -i "s!__MEMPOOL_CPFP_INDEXING__!${__MEMPOOL_CPFP_INDEXING__}!g" mempool-config.json +sed -i "s!__MEMPOOL_MAX_BLOCKS_BULK_QUERY__!${__MEMPOOL_MAX_BLOCKS_BULK_QUERY__}!g" mempool-config.json sed -i "s/__CORE_RPC_HOST__/${__CORE_RPC_HOST__}/g" mempool-config.json sed -i "s/__CORE_RPC_PORT__/${__CORE_RPC_PORT__}/g" mempool-config.json From 8d9568016ed63608719d720dcb66ee69da514599 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Fri, 24 Feb 2023 15:14:35 +0900 Subject: [PATCH 68/95] Remove duplicated entry in backend/src/__fixtures__/mempool-config.template.json --- backend/src/__fixtures__/mempool-config.template.json | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index fa7ea7d21..9890654a5 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -3,7 +3,6 @@ "ENABLED": true, "NETWORK": "__MEMPOOL_NETWORK__", "BACKEND": "__MEMPOOL_BACKEND__", - "ENABLED": true, "BLOCKS_SUMMARIES_INDEXING": true, "HTTP_PORT": 1, "SPAWN_CLUSTER_PROCS": 2, From 210f939e653767a53e11ab4025ed088427da00f7 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Sat, 25 Feb 2023 13:59:37 +0900 Subject: [PATCH 69/95] Add missing truncate blocks table --- backend/src/api/database-migration.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 6e6e6855f..e732d15a5 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -486,6 +486,8 @@ class DatabaseMigration { if (databaseSchemaVersion < 55) { await this.$executeQuery(this.getAdditionalBlocksDataQuery()); + this.uniqueLog(logger.notice, this.blocksTruncatedMessage); + await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index await this.updateToSchemaVersion(55); } } From d3fdef256c422e5866815753b7345a5b749aeee5 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Mon, 2 Jan 2023 13:25:40 +0100 Subject: [PATCH 70/95] Rewrite mining pools parser - Re-index blocks table --- backend/src/api/database-migration.ts | 14 +- backend/src/api/pools-parser.ts | 309 +++++--------------- backend/src/repositories/PoolsRepository.ts | 129 +++++++- backend/src/tasks/pools-updater.ts | 27 +- 4 files changed, 231 insertions(+), 248 deletions(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index e732d15a5..6f9da8cc1 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -62,8 +62,8 @@ class DatabaseMigration { if (databaseSchemaVersion <= 2) { // Disable some spam logs when they're not relevant - this.uniqueLogs.push(this.blocksTruncatedMessage); - this.uniqueLogs.push(this.hashratesTruncatedMessage); + this.uniqueLog(logger.notice, this.blocksTruncatedMessage); + this.uniqueLog(logger.notice, this.hashratesTruncatedMessage); } logger.debug('MIGRATIONS: Current state.schema_version ' + databaseSchemaVersion); @@ -490,6 +490,16 @@ class DatabaseMigration { await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index await this.updateToSchemaVersion(55); } + + if (databaseSchemaVersion < 56) { + await this.$executeQuery('ALTER TABLE pools ADD unique_id int NOT NULL DEFAULT -1'); + await this.$executeQuery('TRUNCATE TABLE `blocks`'); + this.uniqueLog(logger.notice, this.blocksTruncatedMessage); + await this.$executeQuery('DELETE FROM `pools`'); + await this.$executeQuery('ALTER TABLE pools AUTO_INCREMENT = 1'); + this.uniqueLog(logger.notice, '`pools` table has been truncated`'); + await this.updateToSchemaVersion(56); + } } /** diff --git a/backend/src/api/pools-parser.ts b/backend/src/api/pools-parser.ts index e37414bbe..4ba38876d 100644 --- a/backend/src/api/pools-parser.ts +++ b/backend/src/api/pools-parser.ts @@ -1,15 +1,8 @@ import DB from '../database'; import logger from '../logger'; import config from '../config'; -import BlocksRepository from '../repositories/BlocksRepository'; - -interface Pool { - name: string; - link: string; - regexes: string[]; - addresses: string[]; - slug: string; -} +import PoolsRepository from '../repositories/PoolsRepository'; +import { PoolTag } from '../mempool.interfaces'; class PoolsParser { miningPools: any[] = []; @@ -20,270 +13,118 @@ class PoolsParser { 'addresses': '[]', 'slug': 'unknown' }; - slugWarnFlag = false; + + public setMiningPools(pools): void { + this.miningPools = pools; + } /** - * Parse the pools.json file, consolidate the data and dump it into the database + * Populate our db with updated mining pool definition + * @param pools */ - public async migratePoolsJson(poolsJson: object): Promise { - if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { - return; - } + public async migratePoolsJson(pools: any[]): Promise { + await this.$insertUnknownPool(); - // First we save every entries without paying attention to pool duplication - const poolsDuplicated: Pool[] = []; - - const coinbaseTags = Object.entries(poolsJson['coinbase_tags']); - for (let i = 0; i < coinbaseTags.length; ++i) { - poolsDuplicated.push({ - 'name': (coinbaseTags[i][1]).name, - 'link': (coinbaseTags[i][1]).link, - 'regexes': [coinbaseTags[i][0]], - 'addresses': [], - 'slug': '' - }); - } - const addressesTags = Object.entries(poolsJson['payout_addresses']); - for (let i = 0; i < addressesTags.length; ++i) { - poolsDuplicated.push({ - 'name': (addressesTags[i][1]).name, - 'link': (addressesTags[i][1]).link, - 'regexes': [], - 'addresses': [addressesTags[i][0]], - 'slug': '' - }); - } - - // Then, we find unique mining pool names - const poolNames: string[] = []; - for (let i = 0; i < poolsDuplicated.length; ++i) { - if (poolNames.indexOf(poolsDuplicated[i].name) === -1) { - poolNames.push(poolsDuplicated[i].name); + for (const pool of pools) { + if (!pool.id) { + logger.info(`Mining pool ${pool.name} has no unique 'id' defined. Skipping.`); + continue; } - } - logger.debug(`Found ${poolNames.length} unique mining pools`, logger.tags.mining); - // Get existing pools from the db - let existingPools; - try { - if (config.DATABASE.ENABLED === true) { - [existingPools] = await DB.query({ sql: 'SELECT * FROM pools;', timeout: 120000 }); + const poolDB = await PoolsRepository.$getPoolByUniqueId(pool.id, false); + if (!poolDB) { + // New mining pool + const slug = pool.name.replace(/[^a-z0-9]/gi, '').toLowerCase(); + logger.debug(`Inserting new mining pool ${pool.name}`); + await PoolsRepository.$insertNewMiningPool(pool, slug); } else { - existingPools = []; - } - } catch (e) { - logger.err('Cannot get existing pools from the database, skipping pools.json import', logger.tags.mining); - return; - } - - this.miningPools = []; - - // Finally, we generate the final consolidated pools data - const finalPoolDataAdd: Pool[] = []; - const finalPoolDataUpdate: Pool[] = []; - const finalPoolDataRename: Pool[] = []; - for (let i = 0; i < poolNames.length; ++i) { - let allAddresses: string[] = []; - let allRegexes: string[] = []; - const match = poolsDuplicated.filter((pool: Pool) => pool.name === poolNames[i]); - - for (let y = 0; y < match.length; ++y) { - allAddresses = allAddresses.concat(match[y].addresses); - allRegexes = allRegexes.concat(match[y].regexes); - } - - const finalPoolName = poolNames[i].replace(`'`, `''`); // To support single quote in names when doing db queries - - let slug: string | undefined; - try { - slug = poolsJson['slugs'][poolNames[i]]; - } catch (e) { - if (this.slugWarnFlag === false) { - logger.warn(`pools.json does not seem to contain the 'slugs' object`, logger.tags.mining); - this.slugWarnFlag = true; + if (poolDB.name !== pool.name) { + // Pool has been renamed + const newSlug = pool.name.replace(/[^a-z0-9]/gi, '').toLowerCase(); + logger.warn(`Renaming ${poolDB.name} mining pool to ${pool.name}. Slug has been updated. Maybe you want to make a redirection from 'https://mempool.space/mining/pool/${poolDB.slug}' to 'https://mempool.space/mining/pool/${newSlug}`); + await PoolsRepository.$renameMiningPool(poolDB.id, newSlug, pool.name); } - } - - if (slug === undefined) { - // Only keep alphanumerical - slug = poolNames[i].replace(/[^a-z0-9]/gi, '').toLowerCase(); - logger.warn(`No slug found for '${poolNames[i]}', generating it => '${slug}'`, logger.tags.mining); - } - - const poolObj = { - 'name': finalPoolName, - 'link': match[0].link, - 'regexes': allRegexes, - 'addresses': allAddresses, - 'slug': slug - }; - - const existingPool = existingPools.find((pool) => pool.name === poolNames[i]); - if (existingPool !== undefined) { - // Check if any data was actually updated - const equals = (a, b) => - a.length === b.length && - a.every((v, i) => v === b[i]); - if (!equals(JSON.parse(existingPool.addresses), poolObj.addresses) || !equals(JSON.parse(existingPool.regexes), poolObj.regexes)) { - finalPoolDataUpdate.push(poolObj); + if (poolDB.link !== pool.link) { + // Pool link has changed + logger.debug(`Updating link for ${pool.name} mining pool`); + await PoolsRepository.$updateMiningPoolLink(poolDB.id, pool.link); } - } else if (config.DATABASE.ENABLED) { - // Double check that if we're not just renaming a pool (same address same regex) - const [poolToRename]: any[] = await DB.query(` - SELECT * FROM pools - WHERE addresses = ? OR regexes = ?`, - [JSON.stringify(poolObj.addresses), JSON.stringify(poolObj.regexes)] - ); - if (poolToRename && poolToRename.length > 0) { - // We're actually renaming an existing pool - finalPoolDataRename.push({ - 'name': poolObj.name, - 'link': poolObj.link, - 'regexes': allRegexes, - 'addresses': allAddresses, - 'slug': slug - }); - logger.debug(`Rename '${poolToRename[0].name}' mining pool to ${poolObj.name}`, logger.tags.mining); - } else { - logger.debug(`Add '${finalPoolName}' mining pool`, logger.tags.mining); - finalPoolDataAdd.push(poolObj); + if (JSON.stringify(pool.addresses) !== poolDB.addresses || + JSON.stringify(pool.tags) !== poolDB.regexes) { + // Pool addresses changed or coinbase tags changed + logger.notice(`Updating addresses and/or coinbase tags for ${pool.name} mining pool. If 'AUTOMATIC_BLOCK_REINDEXING' is enabled, we will re-index its blocks and 'unknown' blocks`); + await PoolsRepository.$updateMiningPoolTags(poolDB.id, pool.addresses, pool.tags); + await this.$deleteBlocksForPool(poolDB); } } - - this.miningPools.push({ - 'name': finalPoolName, - 'link': match[0].link, - 'regexes': JSON.stringify(allRegexes), - 'addresses': JSON.stringify(allAddresses), - 'slug': slug - }); - } - - if (config.DATABASE.ENABLED === false) { // Don't run db operations - logger.info('Mining pools.json import completed (no database)', logger.tags.mining); - return; - } - - if (finalPoolDataAdd.length > 0 || finalPoolDataUpdate.length > 0 || - finalPoolDataRename.length > 0 - ) { - logger.debug(`Update pools table now`, logger.tags.mining); - - // Add new mining pools into the database - let queryAdd: string = 'INSERT INTO pools(name, link, regexes, addresses, slug) VALUES '; - for (let i = 0; i < finalPoolDataAdd.length; ++i) { - queryAdd += `('${finalPoolDataAdd[i].name}', '${finalPoolDataAdd[i].link}', - '${JSON.stringify(finalPoolDataAdd[i].regexes)}', '${JSON.stringify(finalPoolDataAdd[i].addresses)}', - ${JSON.stringify(finalPoolDataAdd[i].slug)}),`; - } - queryAdd = queryAdd.slice(0, -1) + ';'; - - // Updated existing mining pools in the database - const updateQueries: string[] = []; - for (let i = 0; i < finalPoolDataUpdate.length; ++i) { - updateQueries.push(` - UPDATE pools - SET name='${finalPoolDataUpdate[i].name}', link='${finalPoolDataUpdate[i].link}', - regexes='${JSON.stringify(finalPoolDataUpdate[i].regexes)}', addresses='${JSON.stringify(finalPoolDataUpdate[i].addresses)}', - slug='${finalPoolDataUpdate[i].slug}' - WHERE name='${finalPoolDataUpdate[i].name}' - ;`); - } - - // Rename mining pools - const renameQueries: string[] = []; - for (let i = 0; i < finalPoolDataRename.length; ++i) { - renameQueries.push(` - UPDATE pools - SET name='${finalPoolDataRename[i].name}', link='${finalPoolDataRename[i].link}', - slug='${finalPoolDataRename[i].slug}' - WHERE regexes='${JSON.stringify(finalPoolDataRename[i].regexes)}' - AND addresses='${JSON.stringify(finalPoolDataRename[i].addresses)}' - ;`); - } - - try { - if (finalPoolDataAdd.length > 0 || updateQueries.length > 0) { - await this.$deleteBlocskToReindex(finalPoolDataUpdate); - } - - if (finalPoolDataAdd.length > 0) { - await DB.query({ sql: queryAdd, timeout: 120000 }); - } - for (const query of updateQueries) { - await DB.query({ sql: query, timeout: 120000 }); - } - for (const query of renameQueries) { - await DB.query({ sql: query, timeout: 120000 }); - } - await this.insertUnknownPool(); - logger.info('Mining pools.json import completed', logger.tags.mining); - } catch (e) { - logger.err(`Cannot import pools in the database`, logger.tags.mining); - throw e; - } } - try { - await this.insertUnknownPool(); - } catch (e) { - logger.err(`Cannot insert unknown pool in the database`, logger.tags.mining); - throw e; - } + logger.info('Mining pools.json import completed'); } /** * Manually add the 'unknown pool' */ - private async insertUnknownPool() { + public async $insertUnknownPool(): Promise { + if (!config.DATABASE.ENABLED) { + return; + } + try { const [rows]: any[] = await DB.query({ sql: 'SELECT name from pools where name="Unknown"', timeout: 120000 }); if (rows.length === 0) { await DB.query({ - sql: `INSERT INTO pools(name, link, regexes, addresses, slug) - VALUES("Unknown", "https://learnmeabitcoin.com/technical/coinbase-transaction", "[]", "[]", "unknown"); + sql: `INSERT INTO pools(name, link, regexes, addresses, slug, unique_id) + VALUES("${this.unknownPool.name}", "${this.unknownPool.link}", "[]", "[]", "${this.unknownPool.slug}", 0); `}); } else { await DB.query(`UPDATE pools - SET name='Unknown', link='https://learnmeabitcoin.com/technical/coinbase-transaction', + SET name='${this.unknownPool.name}', link='${this.unknownPool.link}', regexes='[]', addresses='[]', - slug='unknown' - WHERE name='Unknown' + slug='${this.unknownPool.slug}', + unique_id=0 + WHERE slug='${this.unknownPool.slug}' `); } } catch (e) { - logger.err('Unable to insert "Unknown" mining pool', logger.tags.mining); + logger.err(`Unable to insert or update "Unknown" mining pool. Reason: ${e instanceof Error ? e.message : e}`); } } /** - * Delete blocks which needs to be reindexed + * Delete indexed blocks for an updated mining pool + * + * @param pool */ - private async $deleteBlocskToReindex(finalPoolDataUpdate: any[]) { + private async $deleteBlocksForPool(pool: PoolTag): Promise { if (config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING === false) { return; } - const blockCount = await BlocksRepository.$blockCount(null, null); - if (blockCount === 0) { - return; - } - - for (const updatedPool of finalPoolDataUpdate) { - const [pool]: any[] = await DB.query(`SELECT id, name from pools where slug = "${updatedPool.slug}"`); - if (pool.length > 0) { - logger.notice(`Deleting blocks from ${pool[0].name} mining pool for future re-indexing`, logger.tags.mining); - await DB.query(`DELETE FROM blocks WHERE pool_id = ${pool[0].id}`); - } - } - - // Ignore early days of Bitcoin as there were not mining pool yet - logger.notice(`Deleting blocks with unknown mining pool from height 130635 for future re-indexing`, logger.tags.mining); + // Get oldest blocks mined by the pool and assume pools.json updates only concern most recent years + // Ignore early days of Bitcoin as there were no mining pool yet + const [oldestPoolBlock]: any[] = await DB.query(` + SELECT height + FROM blocks + WHERE pool_id = ? + ORDER BY height + LIMIT 1`, + [pool.id] + ); + const oldestBlockHeight = oldestPoolBlock.length ?? 0 > 0 ? oldestPoolBlock[0].height : 130635; const [unknownPool] = await DB.query(`SELECT id from pools where slug = "unknown"`); - await DB.query(`DELETE FROM blocks WHERE pool_id = ${unknownPool[0].id} AND height > 130635`); - - logger.notice(`Truncating hashrates for future re-indexing`, logger.tags.mining); - await DB.query(`DELETE FROM hashrates`); + logger.notice(`Deleting blocks with unknown mining pool from height ${oldestBlockHeight} for re-indexing`); + await DB.query(` + DELETE FROM blocks + WHERE pool_id = ? AND height >= ${oldestBlockHeight}`, + [unknownPool[0].id] + ); + logger.notice(`Deleting blocks from ${pool.name} mining pool for re-indexing`); + await DB.query(` + DELETE FROM blocks + WHERE pool_id = ?`, + [pool.id] + ); } } diff --git a/backend/src/repositories/PoolsRepository.ts b/backend/src/repositories/PoolsRepository.ts index 56cc2b3bc..35319157e 100644 --- a/backend/src/repositories/PoolsRepository.ts +++ b/backend/src/repositories/PoolsRepository.ts @@ -1,4 +1,5 @@ import { Common } from '../api/common'; +import poolsParser from '../api/pools-parser'; import config from '../config'; import DB from '../database'; import logger from '../logger'; @@ -17,7 +18,11 @@ class PoolsRepository { * Get unknown pool tagging info */ public async $getUnknownPool(): Promise { - const [rows] = await DB.query('SELECT id, name, slug FROM pools where name = "Unknown"'); + let [rows]: any[] = await DB.query('SELECT id, name, slug FROM pools where name = "Unknown"'); + if (rows && rows.length === 0 && config.DATABASE.ENABLED) { + await poolsParser.$insertUnknownPool(); + [rows] = await DB.query('SELECT id, name, slug FROM pools where name = "Unknown"'); + } return rows[0]; } @@ -59,7 +64,7 @@ class PoolsRepository { /** * Get basic pool info and block count between two timestamp */ - public async $getPoolsInfoBetween(from: number, to: number): Promise { + public async $getPoolsInfoBetween(from: number, to: number): Promise { const query = `SELECT COUNT(height) as blockCount, pools.id as poolId, pools.name as poolName FROM pools LEFT JOIN blocks on pools.id = blocks.pool_id AND blocks.blockTimestamp BETWEEN FROM_UNIXTIME(?) AND FROM_UNIXTIME(?) @@ -75,9 +80,9 @@ class PoolsRepository { } /** - * Get mining pool statistics for one pool + * Get a mining pool info */ - public async $getPool(slug: string): Promise { + public async $getPool(slug: string, parse: boolean = true): Promise { const query = ` SELECT * FROM pools @@ -90,10 +95,12 @@ class PoolsRepository { return null; } - rows[0].regexes = JSON.parse(rows[0].regexes); + if (parse) { + rows[0].regexes = JSON.parse(rows[0].regexes); + } if (['testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) { rows[0].addresses = []; // pools.json only contains mainnet addresses - } else { + } else if (parse) { rows[0].addresses = JSON.parse(rows[0].addresses); } @@ -103,6 +110,116 @@ class PoolsRepository { throw e; } } + + /** + * Get a mining pool info by its unique id + */ + public async $getPoolByUniqueId(id: number, parse: boolean = true): Promise { + const query = ` + SELECT * + FROM pools + WHERE pools.unique_id = ?`; + + try { + const [rows]: any[] = await DB.query(query, [id]); + + if (rows.length < 1) { + return null; + } + + if (parse) { + rows[0].regexes = JSON.parse(rows[0].regexes); + } + if (['testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) { + rows[0].addresses = []; // pools.json only contains mainnet addresses + } else if (parse) { + rows[0].addresses = JSON.parse(rows[0].addresses); + } + + return rows[0]; + } catch (e) { + logger.err('Cannot get pool from db. Reason: ' + (e instanceof Error ? e.message : e)); + throw e; + } + } + + /** + * Insert a new mining pool in the database + * + * @param pool + */ + public async $insertNewMiningPool(pool: any, slug: string): Promise { + try { + await DB.query(` + INSERT INTO pools + SET name = ?, link = ?, addresses = ?, regexes = ?, slug = ?, unique_id = ?`, + [pool.name, pool.link, JSON.stringify(pool.addresses), JSON.stringify(pool.tags), slug, pool.id] + ); + } catch (e: any) { + logger.err(`Cannot insert new mining pool into db. Reason: ` + (e instanceof Error ? e.message : e)); + } + } + + /** + * Rename an existing mining pool + * + * @param dbId + * @param newSlug + * @param newName + */ + public async $renameMiningPool(dbId: number, newSlug: string, newName: string): Promise { + try { + await DB.query(` + UPDATE pools + SET slug = ?, name = ? + WHERE id = ?`, + [newSlug, newName, dbId] + ); + } catch (e: any) { + logger.err(`Cannot rename mining pool id ${dbId}. Reason: ` + (e instanceof Error ? e.message : e)); + } + } + + /** + * Update an exisiting mining pool link + * + * @param dbId + * @param newLink + */ + public async $updateMiningPoolLink(dbId: number, newLink: string): Promise { + try { + await DB.query(` + UPDATE pools + SET link = ? + WHERE id = ?`, + [newLink, dbId] + ); + } catch (e: any) { + logger.err(`Cannot update link for mining pool id ${dbId}. Reason: ` + (e instanceof Error ? e.message : e)); + } + + } + + /** + * Update an existing mining pool addresses or coinbase tags + * + * @param dbId + * @param addresses + * @param regexes + */ + public async $updateMiningPoolTags(dbId: number, addresses: string, regexes: string): Promise { + try { + await DB.query(` + UPDATE pools + SET addresses = ?, regexes = ? + WHERE id = ?`, + [JSON.stringify(addresses), JSON.stringify(regexes), dbId] + ); + } catch (e: any) { + logger.err(`Cannot update mining pool id ${dbId}. Reason: ` + (e instanceof Error ? e.message : e)); + } + } + } export default new PoolsRepository(); diff --git a/backend/src/tasks/pools-updater.ts b/backend/src/tasks/pools-updater.ts index 086a00cea..8e78c44e6 100644 --- a/backend/src/tasks/pools-updater.ts +++ b/backend/src/tasks/pools-updater.ts @@ -61,9 +61,24 @@ class PoolsUpdater { if (poolsJson === undefined) { return; } - await poolsParser.migratePoolsJson(poolsJson); - await this.updateDBSha(githubSha); - logger.notice(`PoolsUpdater completed`, logger.tags.mining); + poolsParser.setMiningPools(poolsJson); + + if (config.DATABASE.ENABLED === false) { // Don't run db operations + logger.info('Mining pools.json import completed (no database)'); + return; + } + + try { + await DB.query('START TRANSACTION;'); + await poolsParser.migratePoolsJson(poolsJson); + await this.updateDBSha(githubSha); + await DB.query('START TRANSACTION;'); + await DB.query('COMMIT;'); + } catch (e) { + logger.err(`Could not migrate mining pools, rolling back. Reason: ${e instanceof Error ? e.message : e}`); + await DB.query('ROLLBACK;'); + } + logger.notice('PoolsUpdater completed'); } catch (e) { this.lastRun = now - (oneWeek - oneDay); // Try again in 24h instead of waiting next week @@ -106,8 +121,8 @@ class PoolsUpdater { const response = await this.query(this.treeUrl); if (response !== undefined) { - for (const file of response['tree']) { - if (file['path'] === 'pools.json') { + for (const file of response) { + if (file['name'] === 'pool-list.json') { return file['sha']; } } @@ -120,7 +135,7 @@ class PoolsUpdater { /** * Http request wrapper */ - private async query(path): Promise { + private async query(path): Promise { type axiosOptions = { headers: { 'User-Agent': string From d87fb04a920bd4c630a58ec38c0ff016e8ec08ed Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Sun, 12 Feb 2023 22:15:24 +0900 Subject: [PATCH 71/95] Point to the new mining pool files pools-v2.json --- backend/src/config.ts | 2 +- backend/src/tasks/pools-updater.ts | 28 ++++++++++++++-------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/backend/src/config.ts b/backend/src/config.ts index ecd5c80aa..8ccd7e2e4 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -148,7 +148,7 @@ const defaults: IConfig = { 'USER_AGENT': 'mempool', 'STDOUT_LOG_MIN_PRIORITY': 'debug', 'AUTOMATIC_BLOCK_REINDEXING': false, - 'POOLS_JSON_URL': 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json', + 'POOLS_JSON_URL': 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json', 'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master', 'AUDIT': false, 'ADVANCED_GBT_AUDIT': false, diff --git a/backend/src/tasks/pools-updater.ts b/backend/src/tasks/pools-updater.ts index 8e78c44e6..a58e2177a 100644 --- a/backend/src/tasks/pools-updater.ts +++ b/backend/src/tasks/pools-updater.ts @@ -8,7 +8,7 @@ import { SocksProxyAgent } from 'socks-proxy-agent'; import * as https from 'https'; /** - * Maintain the most recent version of pools.json + * Maintain the most recent version of pools-v2.json */ class PoolsUpdater { lastRun: number = 0; @@ -38,7 +38,7 @@ class PoolsUpdater { } try { - const githubSha = await this.fetchPoolsSha(); // Fetch pools.json sha from github + const githubSha = await this.fetchPoolsSha(); // Fetch pools-v2.json sha from github if (githubSha === undefined) { return; } @@ -47,15 +47,15 @@ class PoolsUpdater { this.currentSha = await this.getShaFromDb(); } - logger.debug(`Pools.json sha | Current: ${this.currentSha} | Github: ${githubSha}`); + logger.debug(`pools-v2.json sha | Current: ${this.currentSha} | Github: ${githubSha}`); if (this.currentSha !== undefined && this.currentSha === githubSha) { return; } if (this.currentSha === undefined) { - logger.info(`Downloading pools.json for the first time from ${this.poolsUrl}`, logger.tags.mining); + logger.info(`Downloading pools-v2.json for the first time from ${this.poolsUrl}`, logger.tags.mining); } else { - logger.warn(`Pools.json is outdated, fetch latest from ${this.poolsUrl}`, logger.tags.mining); + logger.warn(`pools-v2.json is outdated, fetch latest from ${this.poolsUrl}`, logger.tags.mining); } const poolsJson = await this.query(this.poolsUrl); if (poolsJson === undefined) { @@ -64,7 +64,7 @@ class PoolsUpdater { poolsParser.setMiningPools(poolsJson); if (config.DATABASE.ENABLED === false) { // Don't run db operations - logger.info('Mining pools.json import completed (no database)'); + logger.info('Mining pools-v2.json import completed (no database)'); return; } @@ -87,7 +87,7 @@ class PoolsUpdater { } /** - * Fetch our latest pools.json sha from the db + * Fetch our latest pools-v2.json sha from the db */ private async updateDBSha(githubSha: string): Promise { this.currentSha = githubSha; @@ -96,39 +96,39 @@ class PoolsUpdater { await DB.query('DELETE FROM state where name="pools_json_sha"'); await DB.query(`INSERT INTO state VALUES('pools_json_sha', NULL, '${githubSha}')`); } catch (e) { - logger.err('Cannot save github pools.json sha into the db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining); + logger.err('Cannot save github pools-v2.json sha into the db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining); } } } /** - * Fetch our latest pools.json sha from the db + * Fetch our latest pools-v2.json sha from the db */ private async getShaFromDb(): Promise { try { const [rows]: any[] = await DB.query('SELECT string FROM state WHERE name="pools_json_sha"'); return (rows.length > 0 ? rows[0].string : undefined); } catch (e) { - logger.err('Cannot fetch pools.json sha from db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining); + logger.err('Cannot fetch pools-v2.json sha from db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining); return undefined; } } /** - * Fetch our latest pools.json sha from github + * Fetch our latest pools-v2.json sha from github */ private async fetchPoolsSha(): Promise { const response = await this.query(this.treeUrl); if (response !== undefined) { - for (const file of response) { - if (file['name'] === 'pool-list.json') { + for (const file of response['tree']) { + if (file['path'] === 'pools-v2.json') { return file['sha']; } } } - logger.err(`Cannot find "pools.json" in git tree (${this.treeUrl})`, logger.tags.mining); + logger.err(`Cannot find "pools-v2.json" in git tree (${this.treeUrl})`, logger.tags.mining); return undefined; } From 117aa1375d4570c14df319b7776aa44e97544219 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Sun, 12 Feb 2023 22:44:04 +0900 Subject: [PATCH 72/95] Disable mining pools update if AUTOMATIC_BLOCK_REINDEXING is not set - Re-index unknown blocks when a new pool is added --- backend/src/api/pools-parser.ts | 11 +++++++++++ backend/src/tasks/pools-updater.ts | 5 +++++ 2 files changed, 16 insertions(+) diff --git a/backend/src/api/pools-parser.ts b/backend/src/api/pools-parser.ts index 4ba38876d..7632f1207 100644 --- a/backend/src/api/pools-parser.ts +++ b/backend/src/api/pools-parser.ts @@ -37,6 +37,7 @@ class PoolsParser { const slug = pool.name.replace(/[^a-z0-9]/gi, '').toLowerCase(); logger.debug(`Inserting new mining pool ${pool.name}`); await PoolsRepository.$insertNewMiningPool(pool, slug); + await this.$deleteUnknownBlocks(); } else { if (poolDB.name !== pool.name) { // Pool has been renamed @@ -126,6 +127,16 @@ class PoolsParser { [pool.id] ); } + + private async $deleteUnknownBlocks(): Promise { + const [unknownPool] = await DB.query(`SELECT id from pools where slug = "unknown"`); + logger.notice(`Deleting blocks with unknown mining pool from height 130635 for re-indexing`); + await DB.query(` + DELETE FROM blocks + WHERE pool_id = ? AND height >= 130635`, + [unknownPool[0].id] + ); + } } export default new PoolsParser(); diff --git a/backend/src/tasks/pools-updater.ts b/backend/src/tasks/pools-updater.ts index a58e2177a..cbe137163 100644 --- a/backend/src/tasks/pools-updater.ts +++ b/backend/src/tasks/pools-updater.ts @@ -17,6 +17,11 @@ class PoolsUpdater { treeUrl: string = config.MEMPOOL.POOLS_JSON_TREE_URL; public async updatePoolsJson(): Promise { + if (config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING === false) { + logger.info(`Not updating mining pools to avoid inconsistency because AUTOMATIC_BLOCK_REINDEXING is set to false`) + return; + } + if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { return; } From 6cd42cfc73b346daa64992201e8312dcb722f403 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Mon, 13 Feb 2023 10:16:41 +0900 Subject: [PATCH 73/95] Update missing POOLS_JSON_URL config --- backend/src/__tests__/config.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index 1e4c05ae3..5717808dd 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -36,7 +36,7 @@ describe('Mempool Backend Config', () => { USER_AGENT: 'mempool', STDOUT_LOG_MIN_PRIORITY: 'debug', POOLS_JSON_TREE_URL: 'https://api.github.com/repos/mempool/mining-pools/git/trees/master', - POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json', + POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json', AUDIT: false, ADVANCED_GBT_AUDIT: false, ADVANCED_GBT_MEMPOOL: false, From c2f5cb95290a020fc7f72912c24e3916f5558e18 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Thu, 16 Feb 2023 09:09:14 +0900 Subject: [PATCH 74/95] Update pool parser to work with no database --- backend/src/api/pools-parser.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/backend/src/api/pools-parser.ts b/backend/src/api/pools-parser.ts index 7632f1207..f322578cb 100644 --- a/backend/src/api/pools-parser.ts +++ b/backend/src/api/pools-parser.ts @@ -16,6 +16,11 @@ class PoolsParser { public setMiningPools(pools): void { this.miningPools = pools; + for (const pool of this.miningPools) { + pool.regexes = JSON.stringify(pool.tags); + pool.addresses = JSON.stringify(pool.addresses); + delete pool.tags; + } } /** From ad9e42db2640cb762e1635550ebd42c601a71210 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Fri, 24 Feb 2023 21:35:13 +0900 Subject: [PATCH 75/95] Use regexes instead of tags --- backend/src/api/pools-parser.ts | 30 +++++++++++++-------- backend/src/repositories/PoolsRepository.ts | 2 +- backend/src/tasks/pools-updater.ts | 2 +- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/backend/src/api/pools-parser.ts b/backend/src/api/pools-parser.ts index f322578cb..4e67ce98b 100644 --- a/backend/src/api/pools-parser.ts +++ b/backend/src/api/pools-parser.ts @@ -13,24 +13,32 @@ class PoolsParser { 'addresses': '[]', 'slug': 'unknown' }; + private uniqueLogs: string[] = []; + + private uniqueLog(loggerFunction: any, msg: string): void { + if (this.uniqueLogs.includes(msg)) { + return; + } + this.uniqueLogs.push(msg); + loggerFunction(msg); + } public setMiningPools(pools): void { - this.miningPools = pools; - for (const pool of this.miningPools) { - pool.regexes = JSON.stringify(pool.tags); - pool.addresses = JSON.stringify(pool.addresses); - delete pool.tags; + for (const pool of pools) { + pool.regexes = pool.tags; + delete(pool.tags); } + this.miningPools = pools; } /** * Populate our db with updated mining pool definition * @param pools */ - public async migratePoolsJson(pools: any[]): Promise { + public async migratePoolsJson(): Promise { await this.$insertUnknownPool(); - for (const pool of pools) { + for (const pool of this.miningPools) { if (!pool.id) { logger.info(`Mining pool ${pool.name} has no unique 'id' defined. Skipping.`); continue; @@ -56,10 +64,10 @@ class PoolsParser { await PoolsRepository.$updateMiningPoolLink(poolDB.id, pool.link); } if (JSON.stringify(pool.addresses) !== poolDB.addresses || - JSON.stringify(pool.tags) !== poolDB.regexes) { + JSON.stringify(pool.regexes) !== poolDB.regexes) { // Pool addresses changed or coinbase tags changed logger.notice(`Updating addresses and/or coinbase tags for ${pool.name} mining pool. If 'AUTOMATIC_BLOCK_REINDEXING' is enabled, we will re-index its blocks and 'unknown' blocks`); - await PoolsRepository.$updateMiningPoolTags(poolDB.id, pool.addresses, pool.tags); + await PoolsRepository.$updateMiningPoolTags(poolDB.id, pool.addresses, pool.regexes); await this.$deleteBlocksForPool(poolDB); } } @@ -119,7 +127,7 @@ class PoolsParser { ); const oldestBlockHeight = oldestPoolBlock.length ?? 0 > 0 ? oldestPoolBlock[0].height : 130635; const [unknownPool] = await DB.query(`SELECT id from pools where slug = "unknown"`); - logger.notice(`Deleting blocks with unknown mining pool from height ${oldestBlockHeight} for re-indexing`); + this.uniqueLog(logger.notice, `Deleting blocks with unknown mining pool from height ${oldestBlockHeight} for re-indexing`); await DB.query(` DELETE FROM blocks WHERE pool_id = ? AND height >= ${oldestBlockHeight}`, @@ -135,7 +143,7 @@ class PoolsParser { private async $deleteUnknownBlocks(): Promise { const [unknownPool] = await DB.query(`SELECT id from pools where slug = "unknown"`); - logger.notice(`Deleting blocks with unknown mining pool from height 130635 for re-indexing`); + this.uniqueLog(logger.notice, `Deleting blocks with unknown mining pool from height 130635 for re-indexing`); await DB.query(` DELETE FROM blocks WHERE pool_id = ? AND height >= 130635`, diff --git a/backend/src/repositories/PoolsRepository.ts b/backend/src/repositories/PoolsRepository.ts index 35319157e..63bddd497 100644 --- a/backend/src/repositories/PoolsRepository.ts +++ b/backend/src/repositories/PoolsRepository.ts @@ -153,7 +153,7 @@ class PoolsRepository { await DB.query(` INSERT INTO pools SET name = ?, link = ?, addresses = ?, regexes = ?, slug = ?, unique_id = ?`, - [pool.name, pool.link, JSON.stringify(pool.addresses), JSON.stringify(pool.tags), slug, pool.id] + [pool.name, pool.link, JSON.stringify(pool.addresses), JSON.stringify(pool.regexes), slug, pool.id] ); } catch (e: any) { logger.err(`Cannot insert new mining pool into db. Reason: ` + (e instanceof Error ? e.message : e)); diff --git a/backend/src/tasks/pools-updater.ts b/backend/src/tasks/pools-updater.ts index cbe137163..6d5a86559 100644 --- a/backend/src/tasks/pools-updater.ts +++ b/backend/src/tasks/pools-updater.ts @@ -75,7 +75,7 @@ class PoolsUpdater { try { await DB.query('START TRANSACTION;'); - await poolsParser.migratePoolsJson(poolsJson); + await poolsParser.migratePoolsJson(); await this.updateDBSha(githubSha); await DB.query('START TRANSACTION;'); await DB.query('COMMIT;'); From 3d38064dbbc7e76e3204b0d339161575ef3d7b22 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Sat, 25 Feb 2023 16:48:11 +0900 Subject: [PATCH 76/95] Increase db schema version to 56 --- backend/src/api/database-migration.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 6f9da8cc1..3140ea358 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository'; import { RowDataPacket } from 'mysql2'; class DatabaseMigration { - private static currentVersion = 55; + private static currentVersion = 56; private queryTimeout = 3600_000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; From 2363a397f1f4ea3ee366f395a8ad8aaae1576c20 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Sat, 25 Feb 2023 17:05:58 +0900 Subject: [PATCH 77/95] Remove duplicated db transaction --- backend/src/tasks/pools-updater.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/src/tasks/pools-updater.ts b/backend/src/tasks/pools-updater.ts index 6d5a86559..1ac87e695 100644 --- a/backend/src/tasks/pools-updater.ts +++ b/backend/src/tasks/pools-updater.ts @@ -77,7 +77,6 @@ class PoolsUpdater { await DB.query('START TRANSACTION;'); await poolsParser.migratePoolsJson(); await this.updateDBSha(githubSha); - await DB.query('START TRANSACTION;'); await DB.query('COMMIT;'); } catch (e) { logger.err(`Could not migrate mining pools, rolling back. Reason: ${e instanceof Error ? e.message : e}`); From 9395a5031e4742269f12e3c7e99b1fee099682d6 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Sat, 25 Feb 2023 17:12:50 +0900 Subject: [PATCH 78/95] Log the whole exception in pool parser --- backend/src/tasks/pools-updater.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/tasks/pools-updater.ts b/backend/src/tasks/pools-updater.ts index 1ac87e695..8aa73376f 100644 --- a/backend/src/tasks/pools-updater.ts +++ b/backend/src/tasks/pools-updater.ts @@ -79,14 +79,14 @@ class PoolsUpdater { await this.updateDBSha(githubSha); await DB.query('COMMIT;'); } catch (e) { - logger.err(`Could not migrate mining pools, rolling back. Reason: ${e instanceof Error ? e.message : e}`); + logger.err(`Could not migrate mining pools, rolling back. Exception: ${JSON.stringify(e)}`, logger.tags.mining); await DB.query('ROLLBACK;'); } logger.notice('PoolsUpdater completed'); } catch (e) { this.lastRun = now - (oneWeek - oneDay); // Try again in 24h instead of waiting next week - logger.err(`PoolsUpdater failed. Will try again in 24h. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.mining); + logger.err(`PoolsUpdater failed. Will try again in 24h. Exception: ${JSON.stringify(e)}`, logger.tags.mining); } } From 333aef5e94ce66360e2d6066b8e5de2eb2d33d74 Mon Sep 17 00:00:00 2001 From: hunicus <93150691+hunicus@users.noreply.github.com> Date: Sat, 25 Feb 2023 04:23:45 -0500 Subject: [PATCH 79/95] Update legal notices for 2023 --- LICENSE | 2 +- frontend/src/app/components/about/about.component.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LICENSE b/LICENSE index 966417847..ac267d120 100644 --- a/LICENSE +++ b/LICENSE @@ -1,5 +1,5 @@ The Mempool Open Source Project -Copyright (c) 2019-2022 The Mempool Open Source Project Developers +Copyright (c) 2019-2023 The Mempool Open Source Project Developers This program is free software; you can redistribute it and/or modify it under the terms of (at your option) either: diff --git a/frontend/src/app/components/about/about.component.html b/frontend/src/app/components/about/about.component.html index 876bec028..03323b6ed 100644 --- a/frontend/src/app/components/about/about.component.html +++ b/frontend/src/app/components/about/about.component.html @@ -352,7 +352,7 @@