diff --git a/backend/README.md b/backend/README.md index 3d7c23eaa..d00dc1812 100644 --- a/backend/README.md +++ b/backend/README.md @@ -218,3 +218,21 @@ Generate block at regular interval (every 10 seconds in this example): ``` watch -n 10 "./src/bitcoin-cli -regtest -rpcport=8332 generatetoaddress 1 $address" ``` + +### Re-index tables + +You can manually force the nodejs backend to drop all data from a specified set of tables for future re-index. This is mostly useful for the mining dashboard and the lightning explorer. + +Use the `--reindex` command to specify a list of comma separated table which will be truncated at start. Note that a 5 seconds delay will be observed before truncating tables in order to give you a chance to cancel (CTRL+C) in case of misuse of the command. + +Usage: +``` +npm run start --reindex=blocks,hashrates +``` +Example output: +``` +Feb 13 14:55:27 [63246] WARN: Indexed data for "hashrates" tables will be erased in 5 seconds (using '--reindex') +Feb 13 14:55:32 [63246] NOTICE: Table hashrates has been truncated +``` + +Reference: https://github.com/mempool/mempool/pull/1269 \ No newline at end of file diff --git a/backend/src/api/bitcoin/esplora-api.ts b/backend/src/api/bitcoin/esplora-api.ts index 37f6e5892..0366695d1 100644 --- a/backend/src/api/bitcoin/esplora-api.ts +++ b/backend/src/api/bitcoin/esplora-api.ts @@ -1,8 +1,13 @@ import config from '../../config'; import axios, { AxiosRequestConfig } from 'axios'; +import http from 'http'; import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory'; import { IEsploraApi } from './esplora-api.interface'; +const axiosConnection = axios.create({ + httpAgent: new http.Agent({ keepAlive: true }) +}); + class ElectrsApi implements AbstractBitcoinApi { axiosConfig: AxiosRequestConfig = { timeout: 10000, @@ -11,52 +16,52 @@ class ElectrsApi implements AbstractBitcoinApi { constructor() { } $getRawMempool(): Promise { - return axios.get(config.ESPLORA.REST_API_URL + '/mempool/txids', this.axiosConfig) + return axiosConnection.get(config.ESPLORA.REST_API_URL + '/mempool/txids', this.axiosConfig) .then((response) => response.data); } $getRawTransaction(txId: string): Promise { - return axios.get(config.ESPLORA.REST_API_URL + '/tx/' + txId, this.axiosConfig) + return axiosConnection.get(config.ESPLORA.REST_API_URL + '/tx/' + txId, this.axiosConfig) .then((response) => response.data); } $getTransactionHex(txId: string): Promise { - return axios.get(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/hex', this.axiosConfig) + return axiosConnection.get(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/hex', this.axiosConfig) .then((response) => response.data); } $getBlockHeightTip(): Promise { - return axios.get(config.ESPLORA.REST_API_URL + '/blocks/tip/height', this.axiosConfig) + return axiosConnection.get(config.ESPLORA.REST_API_URL + '/blocks/tip/height', this.axiosConfig) .then((response) => response.data); } $getBlockHashTip(): Promise { - return axios.get(config.ESPLORA.REST_API_URL + '/blocks/tip/hash', this.axiosConfig) + return axiosConnection.get(config.ESPLORA.REST_API_URL + '/blocks/tip/hash', this.axiosConfig) .then((response) => response.data); } $getTxIdsForBlock(hash: string): Promise { - return axios.get(config.ESPLORA.REST_API_URL + '/block/' + hash + '/txids', this.axiosConfig) + return axiosConnection.get(config.ESPLORA.REST_API_URL + '/block/' + hash + '/txids', this.axiosConfig) .then((response) => response.data); } $getBlockHash(height: number): Promise { - return axios.get(config.ESPLORA.REST_API_URL + '/block-height/' + height, this.axiosConfig) + return axiosConnection.get(config.ESPLORA.REST_API_URL + '/block-height/' + height, this.axiosConfig) .then((response) => response.data); } $getBlockHeader(hash: string): Promise { - return axios.get(config.ESPLORA.REST_API_URL + '/block/' + hash + '/header', this.axiosConfig) + return axiosConnection.get(config.ESPLORA.REST_API_URL + '/block/' + hash + '/header', this.axiosConfig) .then((response) => response.data); } $getBlock(hash: string): Promise { - return axios.get(config.ESPLORA.REST_API_URL + '/block/' + hash, this.axiosConfig) + return axiosConnection.get(config.ESPLORA.REST_API_URL + '/block/' + hash, this.axiosConfig) .then((response) => response.data); } $getRawBlock(hash: string): Promise { - return axios.get(config.ESPLORA.REST_API_URL + '/block/' + hash + "/raw", { ...this.axiosConfig, responseType: 'arraybuffer' }) + return axiosConnection.get(config.ESPLORA.REST_API_URL + '/block/' + hash + "/raw", { ...this.axiosConfig, responseType: 'arraybuffer' }) .then((response) => { return Buffer.from(response.data); }); } @@ -77,12 +82,12 @@ class ElectrsApi implements AbstractBitcoinApi { } $getOutspend(txId: string, vout: number): Promise { - return axios.get(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspend/' + vout, this.axiosConfig) + return axiosConnection.get(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspend/' + vout, this.axiosConfig) .then((response) => response.data); } $getOutspends(txId: string): Promise { - return axios.get(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspends', this.axiosConfig) + return axiosConnection.get(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspends', this.axiosConfig) .then((response) => response.data); } diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 84c919e5d..3f2a7b49e 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -610,7 +610,9 @@ class Blocks { const transactions = await this.$getTransactionsExtended(blockHash, block.height, true); const blockExtended = await this.$getBlockExtended(block, transactions); - await blocksRepository.$saveBlockInDatabase(blockExtended); + if (Common.indexingEnabled()) { + await blocksRepository.$saveBlockInDatabase(blockExtended); + } return prepareBlock(blockExtended); } @@ -714,7 +716,7 @@ class Blocks { block = await this.$indexBlock(currentHeight); returnBlocks.push(block); } else if (nextHash != null) { - block = prepareBlock(await bitcoinClient.getBlock(nextHash)); + block = await this.$indexBlock(currentHeight); nextHash = block.previousblockhash; returnBlocks.push(block); } diff --git a/backend/src/index.ts b/backend/src/index.ts index a81275066..919c039c3 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -36,6 +36,7 @@ import bitcoinRoutes from './api/bitcoin/bitcoin.routes'; import fundingTxFetcher from './tasks/lightning/sync-tasks/funding-tx-fetcher'; import forensicsService from './tasks/lightning/forensics.service'; import priceUpdater from './tasks/price-updater'; +import { AxiosError } from 'axios'; class Server { private wss: WebSocket.Server | undefined; @@ -178,7 +179,7 @@ class Server { setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS); this.currentBackendRetryInterval = 5; - } catch (e) { + } catch (e: any) { const loggerMsg = `runMainLoop error: ${(e instanceof Error ? e.message : e)}. Retrying in ${this.currentBackendRetryInterval} sec.`; if (this.currentBackendRetryInterval > 5) { logger.warn(loggerMsg); @@ -186,7 +187,9 @@ class Server { } else { logger.debug(loggerMsg); } - logger.debug(JSON.stringify(e)); + if (e instanceof AxiosError) { + logger.debug(`AxiosError: ${e?.message}`); + } setTimeout(this.runMainUpdateLoop.bind(this), 1000 * this.currentBackendRetryInterval); this.currentBackendRetryInterval *= 2; this.currentBackendRetryInterval = Math.min(this.currentBackendRetryInterval, 60); diff --git a/backend/src/tasks/price-feeds/bitfinex-api.ts b/backend/src/tasks/price-feeds/bitfinex-api.ts index 04bd47732..0e06c3af7 100644 --- a/backend/src/tasks/price-feeds/bitfinex-api.ts +++ b/backend/src/tasks/price-feeds/bitfinex-api.ts @@ -13,7 +13,11 @@ class BitfinexApi implements PriceFeed { public async $fetchPrice(currency): Promise { const response = await query(this.url + currency); - return response ? parseInt(response['last_price'], 10) : -1; + if (response && response['last_price']) { + return parseInt(response['last_price'], 10); + } else { + return -1; + } } public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise { diff --git a/backend/src/tasks/price-feeds/bitflyer-api.ts b/backend/src/tasks/price-feeds/bitflyer-api.ts index 143fbe8d9..72b2e6adf 100644 --- a/backend/src/tasks/price-feeds/bitflyer-api.ts +++ b/backend/src/tasks/price-feeds/bitflyer-api.ts @@ -13,7 +13,11 @@ class BitflyerApi implements PriceFeed { public async $fetchPrice(currency): Promise { const response = await query(this.url + currency); - return response ? parseInt(response['ltp'], 10) : -1; + if (response && response['ltp']) { + return parseInt(response['ltp'], 10); + } else { + return -1; + } } public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise { diff --git a/backend/src/tasks/price-feeds/coinbase-api.ts b/backend/src/tasks/price-feeds/coinbase-api.ts index ef28b0d80..424ac8867 100644 --- a/backend/src/tasks/price-feeds/coinbase-api.ts +++ b/backend/src/tasks/price-feeds/coinbase-api.ts @@ -13,7 +13,11 @@ class CoinbaseApi implements PriceFeed { public async $fetchPrice(currency): Promise { const response = await query(this.url + currency); - return response ? parseInt(response['data']['amount'], 10) : -1; + if (response && response['data'] && response['data']['amount']) { + return parseInt(response['data']['amount'], 10); + } else { + return -1; + } } public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise { diff --git a/backend/src/tasks/price-feeds/gemini-api.ts b/backend/src/tasks/price-feeds/gemini-api.ts index abd8e0939..fc86dc0a3 100644 --- a/backend/src/tasks/price-feeds/gemini-api.ts +++ b/backend/src/tasks/price-feeds/gemini-api.ts @@ -13,7 +13,11 @@ class GeminiApi implements PriceFeed { public async $fetchPrice(currency): Promise { const response = await query(this.url + currency); - return response ? parseInt(response['last'], 10) : -1; + if (response && response['last']) { + return parseInt(response['last'], 10); + } else { + return -1; + } } public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise { diff --git a/backend/src/tasks/price-feeds/kraken-api.ts b/backend/src/tasks/price-feeds/kraken-api.ts index ea02a772d..c6b3c0c11 100644 --- a/backend/src/tasks/price-feeds/kraken-api.ts +++ b/backend/src/tasks/price-feeds/kraken-api.ts @@ -23,7 +23,14 @@ class KrakenApi implements PriceFeed { public async $fetchPrice(currency): Promise { const response = await query(this.url + currency); - return response ? parseInt(response['result'][this.getTicker(currency)]['c'][0], 10) : -1; + const ticker = this.getTicker(currency); + if (response && response['result'] && response['result'][ticker] && + response['result'][ticker]['c'] && response['result'][ticker]['c'].length > 0 + ) { + return parseInt(response['result'][ticker]['c'][0], 10); + } else { + return -1; + } } public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise { diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index e42c5887a..939a1ea85 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -130,7 +130,11 @@ class PriceUpdater { // Compute average price, non weighted prices = prices.filter(price => price > 0); - this.latestPrices[currency] = Math.round((prices.reduce((partialSum, a) => partialSum + a, 0)) / prices.length); + if (prices.length === 0) { + this.latestPrices[currency] = -1; + } else { + this.latestPrices[currency] = Math.round((prices.reduce((partialSum, a) => partialSum + a, 0)) / prices.length); + } } logger.info(`Latest BTC fiat averaged price: ${JSON.stringify(this.latestPrices)}`); diff --git a/backend/src/utils/blocks-utils.ts b/backend/src/utils/blocks-utils.ts index b933d6ae7..43a2fc964 100644 --- a/backend/src/utils/blocks-utils.ts +++ b/backend/src/utils/blocks-utils.ts @@ -17,7 +17,7 @@ export function prepareBlock(block: any): BlockExtended { extras: { coinbaseRaw: block.coinbase_raw ?? block.extras?.coinbaseRaw, medianFee: block.medianFee ?? block.median_fee ?? block.extras?.medianFee, - feeRange: block.feeRange ?? block.fee_span, + feeRange: block.feeRange ?? block?.extras?.feeRange ?? block.fee_span, reward: block.reward ?? block?.extras?.reward, totalFees: block.totalFees ?? block?.fees ?? block?.extras?.totalFees, avgFee: block?.extras?.avgFee ?? block.avg_fee, diff --git a/frontend/.tx/config b/frontend/.tx/config index 3016be606..825e77ddc 100644 --- a/frontend/.tx/config +++ b/frontend/.tx/config @@ -1,7 +1,9 @@ [main] host = https://www.transifex.com -[mempool.frontend-src-locale-messages-xlf--master] +[o:mempool:p:mempool:r:frontend-src-locale-messages-xlf--master] file_filter = frontend/src/locale/messages..xlf +source_file = frontend/src/locale/messages.en-US.xlf source_lang = en-US -type = XLIFF +type = XLIFF + diff --git a/frontend/src/app/components/address/address.component.html b/frontend/src/app/components/address/address.component.html index d3b23315a..ae2d7ba9c 100644 --- a/frontend/src/app/components/address/address.component.html +++ b/frontend/src/app/components/address/address.component.html @@ -1,4 +1,4 @@ -
+

Address