Merge branch 'master' into mononaut/optimize-mempool-block-7

backend/README.md

@@ -218,3 +218,21 @@ Generate block at regular interval (every 10 seconds in this example):
    ```
    watch -n 10 "./src/bitcoin-cli -regtest -rpcport=8332 generatetoaddress 1 $address"
    ```
+
+### Re-index tables
+
+You can manually force the nodejs backend to drop all data from a specified set of tables for future re-index. This is mostly useful for the mining dashboard and the lightning explorer.
+
+Use the `--reindex` command to specify a list of comma separated table which will be truncated at start. Note that a 5 seconds delay will be observed before truncating tables in order to give you a chance to cancel (CTRL+C) in case of misuse of the command.
+
+Usage:
+```
+npm run start --reindex=blocks,hashrates
+```
+Example output:
+```
+Feb 13 14:55:27 [63246] WARN: <lightning> Indexed data for "hashrates" tables will be erased in 5 seconds (using '--reindex')
+Feb 13 14:55:32 [63246] NOTICE: <lightning> Table hashrates has been truncated
+```
+
+Reference: https://github.com/mempool/mempool/pull/1269

backend/mempool-config.sample.json

@@ -15,7 +15,6 @@
     "MEMPOOL_BLOCKS_AMOUNT": 8,
     "INDEXING_BLOCKS_AMOUNT": 11000,
     "BLOCKS_SUMMARIES_INDEXING": false,
-    "PRICE_FEED_UPDATE_INTERVAL": 600,
     "USE_SECOND_NODE_FOR_MINFEE": false,
     "EXTERNAL_ASSETS": [],
     "EXTERNAL_MAX_RETRY": 1,

backend/src/__fixtures__/mempool-config.template.json

@@ -16,7 +16,6 @@
     "BLOCK_WEIGHT_UNITS": 6,
     "INITIAL_BLOCKS_AMOUNT": 7,
     "MEMPOOL_BLOCKS_AMOUNT": 8,
-    "PRICE_FEED_UPDATE_INTERVAL": 9,
     "USE_SECOND_NODE_FOR_MINFEE": 10,
     "EXTERNAL_ASSETS": 11,
     "EXTERNAL_MAX_RETRY": 12,

backend/src/__tests__/config.test.ts

@@ -29,7 +29,6 @@ describe('Mempool Backend Config', () => {
         INITIAL_BLOCKS_AMOUNT: 8,
         MEMPOOL_BLOCKS_AMOUNT: 8,
         INDEXING_BLOCKS_AMOUNT: 11000,
-        PRICE_FEED_UPDATE_INTERVAL: 600,
         USE_SECOND_NODE_FOR_MINFEE: false,
         EXTERNAL_ASSETS: [],
         EXTERNAL_MAX_RETRY: 1,

backend/src/api/bitcoin/esplora-api.ts

@@ -1,8 +1,13 @@
 import config from '../../config';
 import axios, { AxiosRequestConfig } from 'axios';
+import http from 'http';
 import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
 import { IEsploraApi } from './esplora-api.interface';
 
+const axiosConnection = axios.create({
+  httpAgent: new http.Agent({ keepAlive: true })
+});
+
 class ElectrsApi implements AbstractBitcoinApi {
   axiosConfig: AxiosRequestConfig = {
     timeout: 10000,
@@ -11,52 +16,52 @@ class ElectrsApi implements AbstractBitcoinApi {
   constructor() { }
 
   $getRawMempool(): Promise<IEsploraApi.Transaction['txid'][]> {
-    return axios.get<IEsploraApi.Transaction['txid'][]>(config.ESPLORA.REST_API_URL + '/mempool/txids', this.axiosConfig)
+    return axiosConnection.get<IEsploraApi.Transaction['txid'][]>(config.ESPLORA.REST_API_URL + '/mempool/txids', this.axiosConfig)
       .then((response) => response.data);
   }
 
   $getRawTransaction(txId: string): Promise<IEsploraApi.Transaction> {
-    return axios.get<IEsploraApi.Transaction>(config.ESPLORA.REST_API_URL + '/tx/' + txId, this.axiosConfig)
+    return axiosConnection.get<IEsploraApi.Transaction>(config.ESPLORA.REST_API_URL + '/tx/' + txId, this.axiosConfig)
       .then((response) => response.data);
   }
 
   $getTransactionHex(txId: string): Promise<string> {
-    return axios.get<string>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/hex', this.axiosConfig)
+    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/hex', this.axiosConfig)
       .then((response) => response.data);
   }
 
   $getBlockHeightTip(): Promise<number> {
-    return axios.get<number>(config.ESPLORA.REST_API_URL + '/blocks/tip/height', this.axiosConfig)
+    return axiosConnection.get<number>(config.ESPLORA.REST_API_URL + '/blocks/tip/height', this.axiosConfig)
       .then((response) => response.data);
   }
 
   $getBlockHashTip(): Promise<string> {
-    return axios.get<string>(config.ESPLORA.REST_API_URL + '/blocks/tip/hash', this.axiosConfig)
+    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/blocks/tip/hash', this.axiosConfig)
       .then((response) => response.data);
   }
 
   $getTxIdsForBlock(hash: string): Promise<string[]> {
-    return axios.get<string[]>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/txids', this.axiosConfig)
+    return axiosConnection.get<string[]>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/txids', this.axiosConfig)
       .then((response) => response.data);
   }
 
   $getBlockHash(height: number): Promise<string> {
-    return axios.get<string>(config.ESPLORA.REST_API_URL + '/block-height/' + height, this.axiosConfig)
+    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/block-height/' + height, this.axiosConfig)
       .then((response) => response.data);
   }
 
   $getBlockHeader(hash: string): Promise<string> {
-    return axios.get<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/header', this.axiosConfig)
+    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/header', this.axiosConfig)
       .then((response) => response.data);
   }
 
   $getBlock(hash: string): Promise<IEsploraApi.Block> {
-    return axios.get<IEsploraApi.Block>(config.ESPLORA.REST_API_URL + '/block/' + hash, this.axiosConfig)
+    return axiosConnection.get<IEsploraApi.Block>(config.ESPLORA.REST_API_URL + '/block/' + hash, this.axiosConfig)
       .then((response) => response.data);
   }
 
   $getRawBlock(hash: string): Promise<Buffer> {
-    return axios.get<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + "/raw", { ...this.axiosConfig, responseType: 'arraybuffer' })
+    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + "/raw", { ...this.axiosConfig, responseType: 'arraybuffer' })
       .then((response) => { return Buffer.from(response.data); });
   }
 
@@ -77,12 +82,12 @@ class ElectrsApi implements AbstractBitcoinApi {
   }
 
   $getOutspend(txId: string, vout: number): Promise<IEsploraApi.Outspend> {
-    return axios.get<IEsploraApi.Outspend>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspend/' + vout, this.axiosConfig)
+    return axiosConnection.get<IEsploraApi.Outspend>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspend/' + vout, this.axiosConfig)
       .then((response) => response.data);
   }
 
   $getOutspends(txId: string): Promise<IEsploraApi.Outspend[]> {
-    return axios.get<IEsploraApi.Outspend[]>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspends', this.axiosConfig)
+    return axiosConnection.get<IEsploraApi.Outspend[]>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspends', this.axiosConfig)
       .then((response) => response.data);
   }
 

backend/src/api/blocks.ts

@@ -600,9 +600,11 @@ class Blocks {
    * Index a block if it's missing from the database. Returns the block after indexing
    */
   public async $indexBlock(height: number): Promise<BlockExtended> {
-    const dbBlock = await blocksRepository.$getBlockByHeight(height);
-    if (dbBlock != null) {
-      return prepareBlock(dbBlock);
+    if (Common.indexingEnabled()) {
+      const dbBlock = await blocksRepository.$getBlockByHeight(height);
+      if (dbBlock !== null) {
+        return prepareBlock(dbBlock);
+      }
     }
 
     const blockHash = await bitcoinApi.$getBlockHash(height);
@@ -610,7 +612,9 @@ class Blocks {
     const transactions = await this.$getTransactionsExtended(blockHash, block.height, true);
     const blockExtended = await this.$getBlockExtended(block, transactions);
 
-    await blocksRepository.$saveBlockInDatabase(blockExtended);
+    if (Common.indexingEnabled()) {
+      await blocksRepository.$saveBlockInDatabase(blockExtended);
+    }
 
     return prepareBlock(blockExtended);
   }
@@ -714,7 +718,7 @@ class Blocks {
         block = await this.$indexBlock(currentHeight);
         returnBlocks.push(block);
       } else if (nextHash != null) {
-        block = prepareBlock(await bitcoinClient.getBlock(nextHash));
+        block = await this.$indexBlock(currentHeight);
         nextHash = block.previousblockhash;
         returnBlocks.push(block);
       }

backend/src/config.ts

@@ -19,7 +19,6 @@ interface IConfig {
     MEMPOOL_BLOCKS_AMOUNT: number;
     INDEXING_BLOCKS_AMOUNT: number;
     BLOCKS_SUMMARIES_INDEXING: boolean;
-    PRICE_FEED_UPDATE_INTERVAL: number;
     USE_SECOND_NODE_FOR_MINFEE: boolean;
     EXTERNAL_ASSETS: string[];
     EXTERNAL_MAX_RETRY: number;
@@ -141,7 +140,6 @@ const defaults: IConfig = {
     'MEMPOOL_BLOCKS_AMOUNT': 8,
     'INDEXING_BLOCKS_AMOUNT': 11000, // 0 = disable indexing, -1 = index all blocks
     'BLOCKS_SUMMARIES_INDEXING': false,
-    'PRICE_FEED_UPDATE_INTERVAL': 600,
     'USE_SECOND_NODE_FOR_MINFEE': false,
     'EXTERNAL_ASSETS': [],
     'EXTERNAL_MAX_RETRY': 1,

backend/src/index.ts

@@ -36,6 +36,7 @@ import bitcoinRoutes from './api/bitcoin/bitcoin.routes';
 import fundingTxFetcher from './tasks/lightning/sync-tasks/funding-tx-fetcher';
 import forensicsService from './tasks/lightning/forensics.service';
 import priceUpdater from './tasks/price-updater';
+import { AxiosError } from 'axios';
 
 class Server {
   private wss: WebSocket.Server | undefined;
@@ -178,7 +179,7 @@ class Server {
 
       setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS);
       this.currentBackendRetryInterval = 5;
-    } catch (e) {
+    } catch (e: any) {
       const loggerMsg = `runMainLoop error: ${(e instanceof Error ? e.message : e)}. Retrying in ${this.currentBackendRetryInterval} sec.`;
       if (this.currentBackendRetryInterval > 5) {
         logger.warn(loggerMsg);
@@ -186,7 +187,9 @@ class Server {
       } else {
         logger.debug(loggerMsg);
       }
-      logger.debug(JSON.stringify(e));
+      if (e instanceof AxiosError) {
+        logger.debug(`AxiosError: ${e?.message}`);
+      }
       setTimeout(this.runMainUpdateLoop.bind(this), 1000 * this.currentBackendRetryInterval);
       this.currentBackendRetryInterval *= 2;
       this.currentBackendRetryInterval = Math.min(this.currentBackendRetryInterval, 60);

backend/src/tasks/price-feeds/bitfinex-api.ts

@@ -13,7 +13,11 @@ class BitfinexApi implements PriceFeed {
 
   public async $fetchPrice(currency): Promise<number> {
     const response = await query(this.url + currency);
-    return response ? parseInt(response['last_price'], 10) : -1;
+    if (response && response['last_price']) {
+      return parseInt(response['last_price'], 10);
+    } else {
+      return -1;
+    }
   }
 
   public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {

backend/src/tasks/price-feeds/bitflyer-api.ts

@@ -13,7 +13,11 @@ class BitflyerApi implements PriceFeed {
 
   public async $fetchPrice(currency): Promise<number> {
     const response = await query(this.url + currency);
-    return response ? parseInt(response['ltp'], 10) : -1;
+    if (response && response['ltp']) {
+      return parseInt(response['ltp'], 10);
+    } else {
+      return -1;
+    }
   }
 
   public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {

backend/src/tasks/price-feeds/coinbase-api.ts

@@ -13,7 +13,11 @@ class CoinbaseApi implements PriceFeed {
 
   public async $fetchPrice(currency): Promise<number> {
     const response = await query(this.url + currency);
-    return response ? parseInt(response['data']['amount'], 10) : -1;
+    if (response && response['data'] && response['data']['amount']) {
+      return parseInt(response['data']['amount'], 10);
+    } else {
+      return -1;
+    }
   }
 
   public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {

backend/src/tasks/price-feeds/gemini-api.ts

@@ -13,7 +13,11 @@ class GeminiApi implements PriceFeed {
 
   public async $fetchPrice(currency): Promise<number> {
     const response = await query(this.url + currency);
-    return response ? parseInt(response['last'], 10) : -1;
+    if (response && response['last']) {
+      return parseInt(response['last'], 10);
+    } else {
+      return -1;
+    }
   }
 
   public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {

backend/src/tasks/price-feeds/kraken-api.ts

@@ -23,7 +23,14 @@ class KrakenApi implements PriceFeed {
 
   public async $fetchPrice(currency): Promise<number> {
     const response = await query(this.url + currency);
-    return response ? parseInt(response['result'][this.getTicker(currency)]['c'][0], 10) : -1;
+    const ticker = this.getTicker(currency);
+    if (response && response['result'] && response['result'][ticker] &&
+      response['result'][ticker]['c'] && response['result'][ticker]['c'].length > 0
+    ) {
+      return parseInt(response['result'][ticker]['c'][0], 10);
+    } else {
+      return -1;
+    }
   }
 
   public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {

backend/src/tasks/price-updater.ts

@@ -130,7 +130,11 @@ class PriceUpdater {
 
       // Compute average price, non weighted
       prices = prices.filter(price => price > 0);
-      this.latestPrices[currency] = Math.round((prices.reduce((partialSum, a) => partialSum + a, 0)) / prices.length);
+      if (prices.length === 0) {
+        this.latestPrices[currency] = -1;
+      } else {
+        this.latestPrices[currency] = Math.round((prices.reduce((partialSum, a) => partialSum + a, 0)) / prices.length);
+      }
     }
 
     logger.info(`Latest BTC fiat averaged price: ${JSON.stringify(this.latestPrices)}`);

backend/src/utils/blocks-utils.ts

@@ -17,7 +17,7 @@ export function prepareBlock(block: any): BlockExtended {
     extras: {
       coinbaseRaw: block.coinbase_raw ?? block.extras?.coinbaseRaw,
       medianFee: block.medianFee ?? block.median_fee ?? block.extras?.medianFee,
-      feeRange: block.feeRange ?? block.fee_span,
+      feeRange: block.feeRange ?? block?.extras?.feeRange ?? block.fee_span,
       reward: block.reward ?? block?.extras?.reward,
       totalFees: block.totalFees ?? block?.fees ?? block?.extras?.totalFees,
       avgFee: block?.extras?.avgFee ?? block.avg_fee,