Improve blockchain animations during IBD

Fix indexing error

README.md

@@ -1,13 +1,13 @@
 # The Mempool Open Source Project™ [![mempool](https://img.shields.io/endpoint?url=https://dashboard.cypress.io/badge/simple/ry4br7/master&style=flat-square)](https://dashboard.cypress.io/projects/ry4br7/runs)
 
-https://user-images.githubusercontent.com/232186/222445818-234aa6c9-c233-4c52-b3f0-e32b8232893b.mp4
+https://user-images.githubusercontent.com/93150691/226236121-375ea64f-b4a1-4cc0-8fad-a6fb33226840.mp4
+
+<br>
 
 Mempool is the fully-featured mempool visualizer, explorer, and API service running at [mempool.space](https://mempool.space/). 
 
 It is an open-source project developed and operated for the benefit of the Bitcoin community, with a focus on the emerging transaction fee market that is evolving Bitcoin into a multi-layer ecosystem.
 
-![mempool](https://mempool.space/resources/screenshots/v2.4.0-dashboard.png)
-
 # Installation Methods
 
 Mempool can be self-hosted on a wide variety of your own hardware, ranging from a simple one-click installation on a Raspberry Pi full-node distro all the way to a robust production instance on a powerful FreeBSD server. 

backend/mempool-config.sample.json

@@ -27,13 +27,15 @@
     "AUDIT": false,
     "ADVANCED_GBT_AUDIT": false,
     "ADVANCED_GBT_MEMPOOL": false,
-    "CPFP_INDEXING": false
+    "CPFP_INDEXING": false,
+    "DISK_CACHE_BLOCK_INTERVAL": 6
   },
   "CORE_RPC": {
     "HOST": "127.0.0.1",
     "PORT": 8332,
     "USERNAME": "mempool",
-    "PASSWORD": "mempool"
+    "PASSWORD": "mempool",
+    "TIMEOUT": 60000
   },
   "ELECTRUM": {
     "HOST": "127.0.0.1",
@@ -41,13 +43,15 @@
     "TLS_ENABLED": true
   },
   "ESPLORA": {
-    "REST_API_URL": "http://127.0.0.1:3000"
+    "REST_API_URL": "http://127.0.0.1:3000",
+    "UNIX_SOCKET_PATH": "/tmp/esplora-bitcoin-mainnet"
   },
   "SECOND_CORE_RPC": {
     "HOST": "127.0.0.1",
     "PORT": 8332,
     "USERNAME": "mempool",
-    "PASSWORD": "mempool"
+    "PASSWORD": "mempool",
+    "TIMEOUT": 60000
   },
   "DATABASE": {
     "ENABLED": true,
@@ -91,7 +95,8 @@
   "LND": {
     "TLS_CERT_PATH": "tls.cert",
     "MACAROON_PATH": "readonly.macaroon",
-    "REST_API_URL": "https://localhost:8080"
+    "REST_API_URL": "https://localhost:8080",
+    "TIMEOUT": 10000
   },
   "CLIGHTNING": {
     "SOCKET": "lightning-rpc"

backend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mempool-backend",
-  "version": "2.5.0-dev",
+  "version": "2.6.0-dev",
   "description": "Bitcoin mempool visualizer and blockchain explorer backend",
   "license": "GNU Affero General Public License v3.0",
   "homepage": "https://mempool.space",
@@ -34,35 +34,35 @@
     "prettier": "./node_modules/.bin/prettier --write \"src/**/*.{js,ts}\""
   },
   "dependencies": {
-    "@babel/core": "^7.20.12",
+    "@babel/core": "^7.21.3",
-    "@mempool/electrum-client": "^1.1.7",
+    "@mempool/electrum-client": "1.1.9",
-    "@types/node": "^16.18.11",
+    "@types/node": "^18.15.3",
     "axios": "~0.27.2",
     "bitcoinjs-lib": "~6.1.0",
     "crypto-js": "~4.1.1",
     "express": "~4.18.2",
     "maxmind": "~4.3.8",
-    "mysql2": "~2.3.3",
+    "mysql2": "~3.2.0",
     "node-worker-threads-pool": "~1.5.1",
     "socks-proxy-agent": "~7.0.0",
     "typescript": "~4.7.4",
-    "ws": "~8.11.0"
+    "ws": "~8.13.0"
   },
   "devDependencies": {
-    "@babel/core": "^7.20.7",
+    "@babel/core": "^7.21.3",
     "@babel/code-frame": "^7.18.6",
     "@types/compression": "^1.7.2",
     "@types/crypto-js": "^4.1.1",
     "@types/express": "^4.17.15",
-    "@types/jest": "^29.2.5",
+    "@types/jest": "^29.5.0",
     "@types/ws": "~8.5.4",
-    "@typescript-eslint/eslint-plugin": "^5.48.1",
+    "@typescript-eslint/eslint-plugin": "^5.55.0",
-    "@typescript-eslint/parser": "^5.48.1",
+    "@typescript-eslint/parser": "^5.55.0",
-    "eslint": "^8.31.0",
+    "eslint": "^8.36.0",
-    "eslint-config-prettier": "^8.5.0",
+    "eslint-config-prettier": "^8.7.0",
-    "jest": "^29.3.1",
+    "jest": "^29.5.0",
-    "prettier": "^2.8.2",
+    "prettier": "^2.8.4",
-    "ts-jest": "^29.0.3",
+    "ts-jest": "^29.0.5",
     "ts-node": "^10.9.1"
   }
 }

backend/src/__fixtures__/mempool-config.template.json

@@ -28,13 +28,15 @@
     "ADVANCED_GBT_AUDIT": "__MEMPOOL_ADVANCED_GBT_AUDIT__",
     "ADVANCED_GBT_MEMPOOL": "__MEMPOOL_ADVANCED_GBT_MEMPOOL__",
     "CPFP_INDEXING": "__MEMPOOL_CPFP_INDEXING__",
-    "MAX_BLOCKS_BULK_QUERY": "__MEMPOOL_MAX_BLOCKS_BULK_QUERY__"
+    "MAX_BLOCKS_BULK_QUERY": "__MEMPOOL_MAX_BLOCKS_BULK_QUERY__",
+    "DISK_CACHE_BLOCK_INTERVAL": "__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__"
   },
   "CORE_RPC": {
     "HOST": "__CORE_RPC_HOST__",
     "PORT": 15,
     "USERNAME": "__CORE_RPC_USERNAME__",
-    "PASSWORD": "__CORE_RPC_PASSWORD__"
+    "PASSWORD": "__CORE_RPC_PASSWORD__",
+    "TIMEOUT": "__CORE_RPC_TIMEOUT__"
   },
   "ELECTRUM": {
     "HOST": "__ELECTRUM_HOST__",
@@ -42,13 +44,15 @@
     "TLS_ENABLED": true
   },
   "ESPLORA": {
-    "REST_API_URL": "__ESPLORA_REST_API_URL__"
+    "REST_API_URL": "__ESPLORA_REST_API_URL__",
+    "UNIX_SOCKET_PATH": "__ESPLORA_UNIX_SOCKET_PATH__"
   },
   "SECOND_CORE_RPC": {
     "HOST": "__SECOND_CORE_RPC_HOST__",
     "PORT": 17,
     "USERNAME": "__SECOND_CORE_RPC_USERNAME__",
-    "PASSWORD": "__SECOND_CORE_RPC_PASSWORD__"
+    "PASSWORD": "__SECOND_CORE_RPC_PASSWORD__",
+    "TIMEOUT": "__SECOND_CORE_RPC_TIMEOUT__"
   },
   "DATABASE": {
     "ENABLED": false,
@@ -107,7 +111,8 @@
   "LND": {
     "TLS_CERT_PATH": "",
     "MACAROON_PATH": "",
-    "REST_API_URL": "https://localhost:8080"
+    "REST_API_URL": "https://localhost:8080",
+    "TIMEOUT": 10000
   },
   "CLIGHTNING": {
     "SOCKET": "__CLIGHTNING_SOCKET__"

backend/src/__tests__/config.test.ts

@@ -42,24 +42,27 @@ describe('Mempool Backend Config', () => {
         ADVANCED_GBT_MEMPOOL: false,
         CPFP_INDEXING: false,
         MAX_BLOCKS_BULK_QUERY: 0,
+        DISK_CACHE_BLOCK_INTERVAL: 6,
       });
 
       expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true });
 
-      expect(config.ESPLORA).toStrictEqual({ REST_API_URL: 'http://127.0.0.1:3000' });
+      expect(config.ESPLORA).toStrictEqual({ REST_API_URL: 'http://127.0.0.1:3000', UNIX_SOCKET_PATH: null });
 
       expect(config.CORE_RPC).toStrictEqual({
         HOST: '127.0.0.1',
         PORT: 8332,
         USERNAME: 'mempool',
-        PASSWORD: 'mempool'
+        PASSWORD: 'mempool',
+        TIMEOUT: 60000
       });
 
       expect(config.SECOND_CORE_RPC).toStrictEqual({
         HOST: '127.0.0.1',
         PORT: 8332,
         USERNAME: 'mempool',
-        PASSWORD: 'mempool'
+        PASSWORD: 'mempool',
+        TIMEOUT: 60000
       });
 
       expect(config.DATABASE).toStrictEqual({
@@ -108,10 +111,10 @@ describe('Mempool Backend Config', () => {
       });
 
       expect(config.MAXMIND).toStrictEqual({
-        ENABLED: true,
+        ENABLED: false,
-        GEOLITE2_CITY: './backend/GeoIP/GeoLite2-City.mmdb',
+        GEOLITE2_CITY: '/usr/local/share/GeoIP/GeoLite2-City.mmdb',
-        GEOLITE2_ASN: './backend/GeoIP/GeoLite2-ASN.mmdb',
+        GEOLITE2_ASN: '/usr/local/share/GeoIP/GeoLite2-ASN.mmdb',
-        GEOIP2_ISP: ''
+        GEOIP2_ISP: '/usr/local/share/GeoIP/GeoIP2-ISP.mmdb'
       });
     });
   });

backend/src/api/audit.ts

@@ -1,4 +1,5 @@
 import config from '../config';
+import logger from '../logger';
 import { TransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces';
 
 const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners
@@ -39,17 +40,19 @@ class Audit {
         } else {
           isCensored[txid] = true;
         }
-        displacedWeight += mempool[txid].weight;
+        displacedWeight += mempool[txid]?.weight || 0;
       } else {
-        matchedWeight += mempool[txid].weight;
+        matchedWeight += mempool[txid]?.weight || 0;
       }
-      projectedWeight += mempool[txid].weight;
+      projectedWeight += mempool[txid]?.weight || 0;
       inTemplate[txid] = true;
     }
 
-    displacedWeight += (4000 - transactions[0].weight);
+    if (transactions[0]) {
-    projectedWeight += transactions[0].weight;
+      displacedWeight += (4000 - transactions[0].weight);
-    matchedWeight += transactions[0].weight;
+      projectedWeight += transactions[0].weight;
+      matchedWeight += transactions[0].weight;
+    }
 
     // we can expect an honest miner to include 'displaced' transactions in place of recent arrivals and censored txs
     // these displaced transactions should occupy the first N weight units of the next projected block
@@ -59,19 +62,24 @@ class Audit {
     let failures = 0;
     while (projectedBlocks[1] && index < projectedBlocks[1].transactionIds.length && failures < 500) {
       const txid = projectedBlocks[1].transactionIds[index];
-      const fits = (mempool[txid].weight - displacedWeightRemaining) < 4000;
+      const tx = mempool[txid];
-      const feeMatches = mempool[txid].effectiveFeePerVsize >= lastFeeRate;
+      if (tx) {
-      if (fits || feeMatches) {
+        const fits = (tx.weight - displacedWeightRemaining) < 4000;
-        isDisplaced[txid] = true;
+        const feeMatches = tx.effectiveFeePerVsize >= lastFeeRate;
-        if (fits) {
+        if (fits || feeMatches) {
-          lastFeeRate = Math.min(lastFeeRate, mempool[txid].effectiveFeePerVsize);
+          isDisplaced[txid] = true;
+          if (fits) {
+            lastFeeRate = Math.min(lastFeeRate, tx.effectiveFeePerVsize);
+          }
+          if (tx.firstSeen == null || (now - (tx?.firstSeen || 0)) > PROPAGATION_MARGIN) {
+            displacedWeightRemaining -= tx.weight;
+          }
+          failures = 0;
+        } else {
+          failures++;
         }
-        if (mempool[txid].firstSeen == null || (now - (mempool[txid]?.firstSeen || 0)) > PROPAGATION_MARGIN) {
-          displacedWeightRemaining -= mempool[txid].weight;
-        }
-        failures = 0;
       } else {
-        failures++;
+        logger.warn('projected transaction missing from mempool cache');
       }
       index++;
     }
@@ -108,20 +116,25 @@ class Audit {
     index = projectedBlocks[0].transactionIds.length - 1;
     while (index >= 0) {
       const txid = projectedBlocks[0].transactionIds[index];
-      if (overflowWeightRemaining > 0) {
+      const tx = mempool[txid];
-        if (isCensored[txid]) {
+      if (tx) {
-          delete isCensored[txid];
+        if (overflowWeightRemaining > 0) {
-        }
+          if (isCensored[txid]) {
-        if (mempool[txid].effectiveFeePerVsize > maxOverflowRate) {
+            delete isCensored[txid];
-          maxOverflowRate = mempool[txid].effectiveFeePerVsize;
+          }
-          rateThreshold = (Math.ceil(maxOverflowRate * 100) / 100) + 0.005;
+          if (tx.effectiveFeePerVsize > maxOverflowRate) {
-        }
+            maxOverflowRate = tx.effectiveFeePerVsize;
-      } else if (mempool[txid].effectiveFeePerVsize <= rateThreshold) { // tolerance of 0.01 sat/vb + rounding
+            rateThreshold = (Math.ceil(maxOverflowRate * 100) / 100) + 0.005;
-        if (isCensored[txid]) {
+          }
-          delete isCensored[txid];
+        } else if (tx.effectiveFeePerVsize <= rateThreshold) { // tolerance of 0.01 sat/vb + rounding
+          if (isCensored[txid]) {
+            delete isCensored[txid];
+          }
         }
+        overflowWeightRemaining -= (mempool[txid]?.weight || 0);
+      } else {
+        logger.warn('projected transaction missing from mempool cache');
       }
-      overflowWeightRemaining -= (mempool[txid]?.weight || 0);
       index--;
     }
 

backend/src/api/bitcoin/bitcoin-client.ts

@@ -7,7 +7,7 @@ const nodeRpcCredentials: BitcoinRpcCredentials = {
   port: config.CORE_RPC.PORT,
   user: config.CORE_RPC.USERNAME,
   pass: config.CORE_RPC.PASSWORD,
-  timeout: 60000,
+  timeout: config.CORE_RPC.TIMEOUT,
 };
 
 export default new bitcoin.Client(nodeRpcCredentials);

backend/src/api/bitcoin/bitcoin-second-client.ts

@@ -7,7 +7,7 @@ const nodeRpcCredentials: BitcoinRpcCredentials = {
   port: config.SECOND_CORE_RPC.PORT,
   user: config.SECOND_CORE_RPC.USERNAME,
   pass: config.SECOND_CORE_RPC.PASSWORD,
-  timeout: 60000,
+  timeout: config.SECOND_CORE_RPC.TIMEOUT,
 };
 
 export default new bitcoin.Client(nodeRpcCredentials);

backend/src/api/bitcoin/electrum-api.ts

@@ -16,7 +16,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
     super(bitcoinClient);
 
     const electrumConfig = { client: 'mempool-v2', version: '1.4' };
-    const electrumPersistencePolicy = { retryPeriod: 10000, maxRetry: 1000, callback: null };
+    const electrumPersistencePolicy = { retryPeriod: 1000, maxRetry: Number.MAX_SAFE_INTEGER, callback: null };
 
     const electrumCallbacks = {
       onConnect: (client, versionInfo) => { logger.info(`Connected to Electrum Server at ${config.ELECTRUM.HOST}:${config.ELECTRUM.PORT} (${JSON.stringify(versionInfo)})`); },

backend/src/api/bitcoin/esplora-api.ts

@@ -3,65 +3,102 @@ import axios, { AxiosRequestConfig } from 'axios';
 import http from 'http';
 import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
 import { IEsploraApi } from './esplora-api.interface';
+import logger from '../../logger';
 
 const axiosConnection = axios.create({
-  httpAgent: new http.Agent({ keepAlive: true })
+  httpAgent: new http.Agent({ keepAlive: true, })
 });
 
 class ElectrsApi implements AbstractBitcoinApi {
-  axiosConfig: AxiosRequestConfig = {
+  private axiosConfigWithUnixSocket: AxiosRequestConfig = config.ESPLORA.UNIX_SOCKET_PATH ? {
+    socketPath: config.ESPLORA.UNIX_SOCKET_PATH,
+    timeout: 10000,
+  } : {
+    timeout: 10000,
+  };
+  private axiosConfigTcpSocketOnly: AxiosRequestConfig = {
     timeout: 10000,
   };
 
-  constructor() { }
+  unixSocketRetryTimeout;
+  activeAxiosConfig;
+
+  constructor() {
+    this.activeAxiosConfig = this.axiosConfigWithUnixSocket;
+  }
+
+  fallbackToTcpSocket() {
+    if (!this.unixSocketRetryTimeout) {
+      logger.err(`Unable to connect to esplora unix socket. Falling back to tcp socket. Retrying unix socket in ${config.ESPLORA.RETRY_UNIX_SOCKET_AFTER / 1000} seconds`);
+      // Retry the unix socket after a few seconds
+      this.unixSocketRetryTimeout = setTimeout(() => {
+        logger.info(`Retrying to use unix socket for esplora now (applied for the next query)`);
+        this.activeAxiosConfig = this.axiosConfigWithUnixSocket;
+        this.unixSocketRetryTimeout = undefined;
+      }, config.ESPLORA.RETRY_UNIX_SOCKET_AFTER);
+    }
+
+    // Use the TCP socket (reach a different esplora instance through nginx)
+    this.activeAxiosConfig = this.axiosConfigTcpSocketOnly;
+  }
+
+  $queryWrapper<T>(url, responseType = 'json'): Promise<T> {
+    return axiosConnection.get<T>(url, { ...this.activeAxiosConfig, responseType: responseType })
+      .then((response) => response.data)
+      .catch((e) => {
+        if (e?.code === 'ECONNREFUSED') {
+          this.fallbackToTcpSocket();
+          // Retry immediately
+          return axiosConnection.get<T>(url, this.activeAxiosConfig)
+            .then((response) => response.data)
+            .catch((e) => {
+              logger.warn(`Cannot query esplora through the unix socket nor the tcp socket. Exception ${e}`);
+              throw e;
+            });
+        } else {
+          throw e;
+        }
+      });
+  }
 
   $getRawMempool(): Promise<IEsploraApi.Transaction['txid'][]> {
-    return axiosConnection.get<IEsploraApi.Transaction['txid'][]>(config.ESPLORA.REST_API_URL + '/mempool/txids', this.axiosConfig)
+    return this.$queryWrapper<IEsploraApi.Transaction['txid'][]>(config.ESPLORA.REST_API_URL + '/mempool/txids');
-      .then((response) => response.data);
   }
 
   $getRawTransaction(txId: string): Promise<IEsploraApi.Transaction> {
-    return axiosConnection.get<IEsploraApi.Transaction>(config.ESPLORA.REST_API_URL + '/tx/' + txId, this.axiosConfig)
+    return this.$queryWrapper<IEsploraApi.Transaction>(config.ESPLORA.REST_API_URL + '/tx/' + txId);
-      .then((response) => response.data);
   }
 
   $getTransactionHex(txId: string): Promise<string> {
-    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/hex', this.axiosConfig)
+    return this.$queryWrapper<string>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/hex');
-      .then((response) => response.data);
   }
 
   $getBlockHeightTip(): Promise<number> {
-    return axiosConnection.get<number>(config.ESPLORA.REST_API_URL + '/blocks/tip/height', this.axiosConfig)
+    return this.$queryWrapper<number>(config.ESPLORA.REST_API_URL + '/blocks/tip/height');
-      .then((response) => response.data);
   }
 
   $getBlockHashTip(): Promise<string> {
-    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/blocks/tip/hash', this.axiosConfig)
+    return this.$queryWrapper<string>(config.ESPLORA.REST_API_URL + '/blocks/tip/hash');
-      .then((response) => response.data);
   }
 
   $getTxIdsForBlock(hash: string): Promise<string[]> {
-    return axiosConnection.get<string[]>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/txids', this.axiosConfig)
+    return this.$queryWrapper<string[]>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/txids');
-      .then((response) => response.data);
   }
 
   $getBlockHash(height: number): Promise<string> {
-    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/block-height/' + height, this.axiosConfig)
+    return this.$queryWrapper<string>(config.ESPLORA.REST_API_URL + '/block-height/' + height);
-      .then((response) => response.data);
   }
 
   $getBlockHeader(hash: string): Promise<string> {
-    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/header', this.axiosConfig)
+    return this.$queryWrapper<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/header');
-      .then((response) => response.data);
   }
 
   $getBlock(hash: string): Promise<IEsploraApi.Block> {
-    return axiosConnection.get<IEsploraApi.Block>(config.ESPLORA.REST_API_URL + '/block/' + hash, this.axiosConfig)
+    return this.$queryWrapper<IEsploraApi.Block>(config.ESPLORA.REST_API_URL + '/block/' + hash);
-      .then((response) => response.data);
   }
 
   $getRawBlock(hash: string): Promise<Buffer> {
-    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + "/raw", { ...this.axiosConfig, responseType: 'arraybuffer' })
+    return this.$queryWrapper<any>(config.ESPLORA.REST_API_URL + '/block/' + hash + "/raw", 'arraybuffer')
       .then((response) => { return Buffer.from(response.data); });
   }
 
@@ -82,13 +119,11 @@ class ElectrsApi implements AbstractBitcoinApi {
   }
 
   $getOutspend(txId: string, vout: number): Promise<IEsploraApi.Outspend> {
-    return axiosConnection.get<IEsploraApi.Outspend>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspend/' + vout, this.axiosConfig)
+    return this.$queryWrapper<IEsploraApi.Outspend>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspend/' + vout);
-      .then((response) => response.data);
   }
 
   $getOutspends(txId: string): Promise<IEsploraApi.Outspend[]> {
-    return axiosConnection.get<IEsploraApi.Outspend[]>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspends', this.axiosConfig)
+    return this.$queryWrapper<IEsploraApi.Outspend[]>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspends');
-      .then((response) => response.data);
   }
 
   async $getBatchedOutspends(txId: string[]): Promise<IEsploraApi.Outspend[][]> {

backend/src/api/blocks.ts

@@ -2,7 +2,7 @@ import config from '../config';
 import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory';
 import logger from '../logger';
 import memPool from './mempool';
-import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionStripped, TransactionMinerInfo } from '../mempool.interfaces';
+import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionStripped, TransactionMinerInfo, CpfpSummary } from '../mempool.interfaces';
 import { Common } from './common';
 import diskCache from './disk-cache';
 import transactionUtils from './transaction-utils';
@@ -200,8 +200,15 @@ class Blocks {
       extras.segwitTotalWeight = 0;
     } else {
       const stats: IBitcoinApi.BlockStats = await bitcoinClient.getBlockStats(block.id);
-      extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles
+      let feeStats = {
-      extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat();
+        medianFee: stats.feerate_percentiles[2], // 50th percentiles
+        feeRange: [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat(),
+      };
+      if (transactions?.length > 1) {
+        feeStats = Common.calcEffectiveFeeStatistics(transactions);
+      }
+      extras.medianFee = feeStats.medianFee;
+      extras.feeRange = feeStats.feeRange;
       extras.totalFees = stats.totalfee;
       extras.avgFee = stats.avgfee;
       extras.avgFeeRate = stats.avgfeerate;
@@ -403,12 +410,13 @@ class Blocks {
     try {
       // Get all indexed block hash
       const unindexedBlockHeights = await blocksRepository.$getCPFPUnindexedBlocks();
-      logger.info(`Indexing cpfp data for ${unindexedBlockHeights.length} blocks`);
 
       if (!unindexedBlockHeights?.length) {
         return;
       }
 
+      logger.info(`Indexing cpfp data for ${unindexedBlockHeights.length} blocks`);
+
       // Logging
       let count = 0;
       let countThisRun = 0;
@@ -558,7 +566,7 @@ class Blocks {
     }
 
     while (this.currentBlockHeight < blockHeightTip) {
-      if (this.currentBlockHeight < blockHeightTip - config.MEMPOOL.INITIAL_BLOCKS_AMOUNT) {
+      if (this.currentBlockHeight === 0) {
         this.currentBlockHeight = blockHeightTip;
       } else {
         this.currentBlockHeight++;
@@ -571,7 +579,8 @@ class Blocks {
       const block = BitcoinApi.convertBlock(verboseBlock);
       const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash);
       const transactions = await this.$getTransactionsExtended(blockHash, block.height, false);
-      const blockExtended: BlockExtended = await this.$getBlockExtended(block, transactions);
+      const cpfpSummary: CpfpSummary = Common.calculateCpfp(block.height, transactions);
+      const blockExtended: BlockExtended = await this.$getBlockExtended(block, cpfpSummary.transactions);
       const blockSummary: BlockSummary = this.summarizeBlock(verboseBlock);
 
       // start async callbacks
@@ -581,11 +590,10 @@ class Blocks {
         if (!fastForwarded) {
           const lastBlock = await blocksRepository.$getBlockByHeight(blockExtended.height - 1);
           if (lastBlock !== null && blockExtended.previousblockhash !== lastBlock.id) {
-            logger.warn(`Chain divergence detected at block ${lastBlock.height}, re-indexing most recent data`);
+            logger.warn(`Chain divergence detected at block ${lastBlock.height}, re-indexing most recent data`, logger.tags.mining);
             // We assume there won't be a reorg with more than 10 block depth
             await BlocksRepository.$deleteBlocksFrom(lastBlock.height - 10);
             await HashratesRepository.$deleteLastEntries();
-            await BlocksSummariesRepository.$deleteBlocksFrom(lastBlock.height - 10);
             await cpfpRepository.$deleteClustersFrom(lastBlock.height - 10);
             for (let i = 10; i >= 0; --i) {
               const newBlock = await this.$indexBlock(lastBlock.height - i);
@@ -596,7 +604,7 @@ class Blocks {
             }
             await mining.$indexDifficultyAdjustments();
             await DifficultyAdjustmentsRepository.$deleteLastAdjustment();
-            logger.info(`Re-indexed 10 blocks and summaries. Also re-indexed the last difficulty adjustments. Will re-index latest hashrates in a few seconds.`);
+            logger.info(`Re-indexed 10 blocks and summaries. Also re-indexed the last difficulty adjustments. Will re-index latest hashrates in a few seconds.`, logger.tags.mining);
             indexer.reindex();
           }
           await blocksRepository.$saveBlockInDatabase(blockExtended);
@@ -608,7 +616,7 @@ class Blocks {
               priceId: lastestPriceId,
             }]);
           } else {
-            logger.info(`Cannot save block price for ${blockExtended.height} because the price updater hasnt completed yet. Trying again in 10 seconds.`, logger.tags.mining);
+            logger.debug(`Cannot save block price for ${blockExtended.height} because the price updater hasnt completed yet. Trying again in 10 seconds.`, logger.tags.mining);
             setTimeout(() => {
               indexer.runSingleTask('blocksPrices');
             }, 10000);
@@ -619,7 +627,7 @@ class Blocks {
             await this.$getStrippedBlockTransactions(blockExtended.id, true);
           }
           if (config.MEMPOOL.CPFP_INDEXING) {
-            this.$indexCPFP(blockExtended.id, this.currentBlockHeight);
+            this.$saveCpfp(blockExtended.id, this.currentBlockHeight, cpfpSummary);
           }
         }
       }
@@ -651,7 +659,7 @@ class Blocks {
       if (this.newBlockCallbacks.length) {
         this.newBlockCallbacks.forEach((cb) => cb(blockExtended, txIds, transactions));
       }
-      if (!memPool.hasPriority() && (block.height % 6 === 0)) {
+      if (!memPool.hasPriority() && (block.height % config.MEMPOOL.DISK_CACHE_BLOCK_INTERVAL === 0)) {
         diskCache.$saveCacheToDisk();
       }
 
@@ -728,7 +736,7 @@ class Blocks {
 
     // Index the response if needed
     if (Common.blocksSummariesIndexingEnabled() === true) {
-      await BlocksSummariesRepository.$saveSummary({height: block.height, mined: summary});
+      await BlocksSummariesRepository.$saveTransactions(block.height, block.hash, summary.transactions);
     }
 
     return summary.transactions;
@@ -844,11 +852,12 @@ class Blocks {
         if (cleanBlock.fee_amt_percentiles === null) {
           const block = await bitcoinClient.getBlock(cleanBlock.hash, 2);
           const summary = this.summarizeBlock(block);
-          await BlocksSummariesRepository.$saveSummary({ height: block.height, mined: summary });
+          await BlocksSummariesRepository.$saveTransactions(cleanBlock.height, cleanBlock.hash, summary.transactions);
           cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash);
         }
         if (cleanBlock.fee_amt_percentiles !== null) {
           cleanBlock.median_fee_amt = cleanBlock.fee_amt_percentiles[3];
+          await blocksRepository.$updateFeeAmounts(cleanBlock.hash, cleanBlock.fee_amt_percentiles, cleanBlock.median_fee_amt);
         }
       }
 
@@ -913,42 +922,20 @@ class Blocks {
   public async $indexCPFP(hash: string, height: number): Promise<void> {
     const block = await bitcoinClient.getBlock(hash, 2);
     const transactions = block.tx.map(tx => {
-      tx.vsize = tx.weight / 4;
       tx.fee *= 100_000_000;
       return tx;
     });
 
-    const clusters: any[] = [];
+    const summary = Common.calculateCpfp(height, transactions);
 
-    let cluster: TransactionStripped[] = [];
+    await this.$saveCpfp(hash, height, summary);
-    let ancestors: { [txid: string]: boolean } = {};
+
-    for (let i = transactions.length - 1; i >= 0; i--) {
+    const effectiveFeeStats = Common.calcEffectiveFeeStatistics(summary.transactions);
-      const tx = transactions[i];
+    await blocksRepository.$saveEffectiveFeeStats(hash, effectiveFeeStats);
-      if (!ancestors[tx.txid]) {
+  }
-        let totalFee = 0;
+
-        let totalVSize = 0;
+  public async $saveCpfp(hash: string, height: number, cpfpSummary: CpfpSummary): Promise<void> {
-        cluster.forEach(tx => {
+    const result = await cpfpRepository.$batchSaveClusters(cpfpSummary.clusters);
-          totalFee += tx?.fee || 0;
-          totalVSize += tx.vsize;
-        });
-        const effectiveFeePerVsize = totalFee / totalVSize;
-        if (cluster.length > 1) {
-          clusters.push({
-            root: cluster[0].txid,
-            height,
-            txs: cluster.map(tx => { return { txid: tx.txid, weight: tx.vsize * 4, fee: tx.fee || 0 }; }),
-            effectiveFeePerVsize,
-          });
-        }
-        cluster = [];
-        ancestors = {};
-      }
-      cluster.push(tx);
-      tx.vin.forEach(vin => {
-        ancestors[vin.txid] = true;
-      });
-    }
-    const result = await cpfpRepository.$batchSaveClusters(clusters);
     if (!result) {
       await cpfpRepository.$insertProgressMarker(height);
     }

backend/src/api/common.ts

@@ -1,4 +1,4 @@
-import { CpfpInfo, MempoolBlockWithTransactions, TransactionExtended, TransactionStripped } from '../mempool.interfaces';
+import { Ancestor, CpfpInfo, CpfpSummary, EffectiveFeeStats, MempoolBlockWithTransactions, TransactionExtended, TransactionStripped } from '../mempool.interfaces';
 import config from '../config';
 import { NodeSocket } from '../repositories/NodesSocketsRepository';
 import { isIP } from 'net';
@@ -345,4 +345,99 @@ export class Common {
       };
     }
   }
+
+  static calculateCpfp(height: number, transactions: TransactionExtended[]): CpfpSummary {
+    const clusters: { root: string, height: number, txs: Ancestor[], effectiveFeePerVsize: number }[] = [];
+    let cluster: TransactionExtended[] = [];
+    let ancestors: { [txid: string]: boolean } = {};
+    const txMap = {};
+    for (let i = transactions.length - 1; i >= 0; i--) {
+      const tx = transactions[i];
+      txMap[tx.txid] = tx;
+      if (!ancestors[tx.txid]) {
+        let totalFee = 0;
+        let totalVSize = 0;
+        cluster.forEach(tx => {
+          totalFee += tx?.fee || 0;
+          totalVSize += (tx.weight / 4);
+        });
+        const effectiveFeePerVsize = totalFee / totalVSize;
+        if (cluster.length > 1) {
+          clusters.push({
+            root: cluster[0].txid,
+            height,
+            txs: cluster.map(tx => { return { txid: tx.txid, weight: tx.weight, fee: tx.fee || 0 }; }),
+            effectiveFeePerVsize,
+          });
+        }
+        cluster.forEach(tx => {
+          txMap[tx.txid].effectiveFeePerVsize = effectiveFeePerVsize;
+        });
+        cluster = [];
+        ancestors = {};
+      }
+      cluster.push(tx);
+      tx.vin.forEach(vin => {
+        ancestors[vin.txid] = true;
+      });
+    }
+    return {
+      transactions,
+      clusters,
+    };
+  }
+
+  static calcEffectiveFeeStatistics(transactions: { weight: number, fee: number, effectiveFeePerVsize?: number, txid: string }[]): EffectiveFeeStats {
+    const sortedTxs = transactions.map(tx => { return { txid: tx.txid, weight: tx.weight, rate: tx.effectiveFeePerVsize || ((tx.fee || 0) / (tx.weight / 4)) }; }).sort((a, b) => a.rate - b.rate);
+
+    let weightCount = 0;
+    let medianFee = 0;
+    let medianWeight = 0;
+
+    // calculate the "medianFee" as the average fee rate of the middle 10000 weight units of transactions
+    const leftBound = 1995000;
+    const rightBound = 2005000;
+    for (let i = 0; i < sortedTxs.length && weightCount < rightBound; i++) {
+      const left = weightCount;
+      const right = weightCount + sortedTxs[i].weight;
+      if (right > leftBound) {
+        const weight = Math.min(right, rightBound) - Math.max(left, leftBound);
+        medianFee += (sortedTxs[i].rate * (weight / 4) );
+        medianWeight += weight;
+      }
+      weightCount += sortedTxs[i].weight;
+    }
+    const medianFeeRate = medianWeight ? (medianFee / (medianWeight / 4)) : 0;
+
+    // minimum effective fee heuristic:
+    // lowest of
+    // a) the 1st percentile of effective fee rates
+    // b) the minimum effective fee rate in the last 2% of transactions (in block order)
+    const minFee = Math.min(
+      Common.getNthPercentile(1, sortedTxs).rate,
+      transactions.slice(-transactions.length / 50).reduce((min, tx) => { return Math.min(min, tx.effectiveFeePerVsize || ((tx.fee || 0) / (tx.weight / 4))); }, Infinity)
+    );
+
+    // maximum effective fee heuristic:
+    // highest of
+    // a) the 99th percentile of effective fee rates
+    // b) the maximum effective fee rate in the first 2% of transactions (in block order)
+    const maxFee = Math.max(
+      Common.getNthPercentile(99, sortedTxs).rate,
+      transactions.slice(0, transactions.length / 50).reduce((max, tx) => { return Math.max(max, tx.effectiveFeePerVsize || ((tx.fee || 0) / (tx.weight / 4))); }, 0)
+    );
+
+    return {
+      medianFee: medianFeeRate,
+      feeRange: [
+        minFee,
+        [10,25,50,75,90].map(n => Common.getNthPercentile(n, sortedTxs).rate),
+        maxFee,
+      ].flat(),
+    };
+  }
+
+  static getNthPercentile(n: number, sortedDistribution: any[]): any {
+    return sortedDistribution[Math.floor((sortedDistribution.length - 1) * (n / 100))];
+  }
 }

backend/src/api/database-migration.ts

@@ -497,6 +497,7 @@ class DatabaseMigration {
       this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
       await this.$executeQuery('DELETE FROM `pools`');
       await this.$executeQuery('ALTER TABLE pools AUTO_INCREMENT = 1');
+      await this.$executeQuery(`UPDATE state SET string = NULL WHERE name = 'pools_json_sha'`);
       this.uniqueLog(logger.notice, '`pools` table has been truncated`');
       await this.updateToSchemaVersion(56);
     }

backend/src/api/difficulty-adjustment.ts

@@ -24,12 +24,11 @@ export function calcDifficultyAdjustment(
   network: string,
   latestBlockTimestamp: number,
 ): DifficultyAdjustment {
-  const ESTIMATE_LAG_BLOCKS = 146; // For first 7.2% of epoch, don't estimate.
   const EPOCH_BLOCK_LENGTH = 2016; // Bitcoin mainnet
   const BLOCK_SECONDS_TARGET = 600; // Bitcoin mainnet
   const TESTNET_MAX_BLOCK_SECONDS = 1200; // Bitcoin testnet
 
-  const diffSeconds = nowSeconds - DATime;
+  const diffSeconds = Math.max(0, nowSeconds - DATime);
   const blocksInEpoch = (blockHeight >= 0) ? blockHeight % EPOCH_BLOCK_LENGTH : 0;
   const progressPercent = (blockHeight >= 0) ? blocksInEpoch / EPOCH_BLOCK_LENGTH * 100 : 100;
   const remainingBlocks = EPOCH_BLOCK_LENGTH - blocksInEpoch;
@@ -37,18 +36,16 @@ export function calcDifficultyAdjustment(
   const expectedBlocks = diffSeconds / BLOCK_SECONDS_TARGET;
 
   let difficultyChange = 0;
-  let timeAvgSecs = diffSeconds / blocksInEpoch;
+  let timeAvgSecs = blocksInEpoch ? diffSeconds / blocksInEpoch : BLOCK_SECONDS_TARGET;
-  // Only calculate the estimate once we have 7.2% of blocks in current epoch
+
-  if (blocksInEpoch >= ESTIMATE_LAG_BLOCKS) {
+  difficultyChange = (BLOCK_SECONDS_TARGET / timeAvgSecs - 1) * 100;
-    difficultyChange = (BLOCK_SECONDS_TARGET / timeAvgSecs - 1) * 100;
+  // Max increase is x4 (+300%)
-    // Max increase is x4 (+300%)
+  if (difficultyChange > 300) {
-    if (difficultyChange > 300) {
+    difficultyChange = 300;
-      difficultyChange = 300;
+  }
-    }
+  // Max decrease is /4 (-75%)
-    // Max decrease is /4 (-75%)
+  if (difficultyChange < -75) {
-    if (difficultyChange < -75) {
+    difficultyChange = -75;
-      difficultyChange = -75;
-    }
   }
 
   // Testnet difficulty is set to 1 after 20 minutes of no blocks,

backend/src/api/disk-cache.ts

@@ -19,20 +19,16 @@ class DiskCache {
   private isWritingCache = false;
 
   constructor() {
-    if (!cluster.isMaster) {
+    if (!cluster.isPrimary) {
       return;
     }
     process.on('SIGINT', (e) => {
-      this.saveCacheToDiskSync();
+      this.$saveCacheToDisk(true);
-      process.exit(2);
+      process.exit(0);
-    });
-    process.on('SIGTERM', (e) => {
-      this.saveCacheToDiskSync();
-      process.exit(2);
     });
   }
 
-  async $saveCacheToDisk(): Promise<void> {
+  async $saveCacheToDisk(sync: boolean = false): Promise<void> {
     if (!cluster.isPrimary) {
       return;
     }
@@ -41,81 +37,61 @@ class DiskCache {
       return;
     }
     try {
-      logger.debug('Writing mempool and blocks data to disk cache (async)...');
+      logger.debug(`Writing mempool and blocks data to disk cache (${ sync ? 'sync' : 'async' })...`);
       this.isWritingCache = true;
 
       const mempool = memPool.getMempool();
       const mempoolArray: TransactionExtended[] = [];
       for (const tx in mempool) {
-        mempoolArray.push(mempool[tx]);
+        if (mempool[tx] && !mempool[tx].deleteAfter) {
+          mempoolArray.push(mempool[tx]);
+        }
       }
 
       Common.shuffleArray(mempoolArray);
 
       const chunkSize = Math.floor(mempoolArray.length / DiskCache.CHUNK_FILES);
 
-      await fsPromises.writeFile(DiskCache.FILE_NAME, JSON.stringify({
+      if (sync) {
-        network: config.MEMPOOL.NETWORK,
+        fs.writeFileSync(DiskCache.TMP_FILE_NAME, JSON.stringify({
-        cacheSchemaVersion: this.cacheSchemaVersion,
+          network: config.MEMPOOL.NETWORK,
-        blocks: blocks.getBlocks(),
+          cacheSchemaVersion: this.cacheSchemaVersion,
-        blockSummaries: blocks.getBlockSummaries(),
+          blocks: blocks.getBlocks(),
-        mempool: {},
+          blockSummaries: blocks.getBlockSummaries(),
-        mempoolArray: mempoolArray.splice(0, chunkSize),
-      }), { flag: 'w' });
-      for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
-        await fsPromises.writeFile(DiskCache.FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
           mempool: {},
           mempoolArray: mempoolArray.splice(0, chunkSize),
         }), { flag: 'w' });
-      }
+        for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
-      logger.debug('Mempool and blocks data saved to disk cache');
+          fs.writeFileSync(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
-      this.isWritingCache = false;
+            mempool: {},
-    } catch (e) {
+            mempoolArray: mempoolArray.splice(0, chunkSize),
-      logger.warn('Error writing to cache file: ' + (e instanceof Error ? e.message : e));
+          }), { flag: 'w' });
-      this.isWritingCache = false;
+        }
-    }
-  }
 
-  saveCacheToDiskSync(): void {
+        fs.renameSync(DiskCache.TMP_FILE_NAME, DiskCache.FILE_NAME);
-    if (!cluster.isPrimary) {
+        for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
-      return;
+          fs.renameSync(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), DiskCache.FILE_NAMES.replace('{number}', i.toString()));
-    }
+        }
-    if (this.isWritingCache) {
+      } else {
-      logger.debug('Saving cache already in progress. Skipping.');
+        await fsPromises.writeFile(DiskCache.TMP_FILE_NAME, JSON.stringify({
-      return;
+          network: config.MEMPOOL.NETWORK,
-    }
+          cacheSchemaVersion: this.cacheSchemaVersion,
-    try {
+          blocks: blocks.getBlocks(),
-      logger.debug('Writing mempool and blocks data to disk cache (sync)...');
+          blockSummaries: blocks.getBlockSummaries(),
-      this.isWritingCache = true;
-
-      const mempool = memPool.getMempool();
-      const mempoolArray: TransactionExtended[] = [];
-      for (const tx in mempool) {
-        mempoolArray.push(mempool[tx]);
-      }
-
-      Common.shuffleArray(mempoolArray);
-
-      const chunkSize = Math.floor(mempoolArray.length / DiskCache.CHUNK_FILES);
-
-      fs.writeFileSync(DiskCache.TMP_FILE_NAME, JSON.stringify({
-        network: config.MEMPOOL.NETWORK,
-        cacheSchemaVersion: this.cacheSchemaVersion,
-        blocks: blocks.getBlocks(),
-        blockSummaries: blocks.getBlockSummaries(),
-        mempool: {},
-        mempoolArray: mempoolArray.splice(0, chunkSize),
-      }), { flag: 'w' });
-      for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
-        fs.writeFileSync(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
           mempool: {},
           mempoolArray: mempoolArray.splice(0, chunkSize),
         }), { flag: 'w' });
-      }
+        for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
+          await fsPromises.writeFile(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
+            mempool: {},
+            mempoolArray: mempoolArray.splice(0, chunkSize),
+          }), { flag: 'w' });
+        }
 
-      fs.renameSync(DiskCache.TMP_FILE_NAME, DiskCache.FILE_NAME);
+        await fsPromises.rename(DiskCache.TMP_FILE_NAME, DiskCache.FILE_NAME);
-      for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
+        for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
-        fs.renameSync(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), DiskCache.FILE_NAMES.replace('{number}', i.toString()));
+          await fsPromises.rename(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), DiskCache.FILE_NAMES.replace('{number}', i.toString()));
+        }
       }
 
       logger.debug('Mempool and blocks data saved to disk cache');
@@ -188,7 +164,7 @@ class DiskCache {
             }
           }
         } catch (e) {
-          logger.info('Error parsing ' + fileName + '. Skipping. Reason: ' + (e instanceof Error ? e.message : e));
+          logger.err('Error parsing ' + fileName + '. Skipping. Reason: ' + (e instanceof Error ? e.message : e));
         }
       }
 

backend/src/api/lightning/lnd/lnd-api.ts

@@ -4,21 +4,29 @@ import * as fs from 'fs';
 import { AbstractLightningApi } from '../lightning-api-abstract-factory';
 import { ILightningApi } from '../lightning-api.interface';
 import config from '../../../config';
+import logger from '../../../logger';
 
 class LndApi implements AbstractLightningApi {
   axiosConfig: AxiosRequestConfig = {};
 
   constructor() {
-    if (config.LIGHTNING.ENABLED) {
+    if (!config.LIGHTNING.ENABLED) {
+      return;
+    }
+    try {
       this.axiosConfig = {
         headers: {
-          'Grpc-Metadata-macaroon': fs.readFileSync(config.LND.MACAROON_PATH).toString('hex')
+          'Grpc-Metadata-macaroon': fs.readFileSync(config.LND.MACAROON_PATH).toString('hex'),
         },
         httpsAgent: new Agent({
           ca: fs.readFileSync(config.LND.TLS_CERT_PATH)
         }),
-        timeout: 10000
+        timeout: config.LND.TIMEOUT
       };
+    } catch (e) {
+      config.LIGHTNING.ENABLED = false;
+      logger.updateNetwork();
+      logger.err(`Could not initialize LND Macaroon/TLS Cert. Disabling LIGHTNING. ` + (e instanceof Error ? e.message : e));
     }
   }
 

backend/src/api/liquid/icons.ts

@@ -2,7 +2,7 @@ import * as fs from 'fs';
 import logger from '../../logger';
 
 class Icons {
-  private static FILE_NAME = './icons.json';
+  private static FILE_NAME = '/elements/asset_registry_db/icons.json';
   private iconIds: string[] = [];
   private icons: { [assetId: string]: string; } = {};
 

backend/src/api/mempool-blocks.ts

@@ -59,7 +59,7 @@ class MempoolBlocks {
     // Loop through and traverse all ancestors and sum up all the sizes + fees
     // Pass down size + fee to all unconfirmed children
     let sizes = 0;
-    memPoolArray.forEach((tx, i) => {
+    memPoolArray.forEach((tx) => {
       sizes += tx.weight;
       if (sizes > 4000000 * 8) {
         return;
@@ -74,7 +74,7 @@ class MempoolBlocks {
     const time = end - start;
     logger.debug('Mempool blocks calculated in ' + time / 1000 + ' seconds');
 
-    const blocks = this.calculateMempoolBlocks(memPoolArray, this.mempoolBlocks);
+    const blocks = this.calculateMempoolBlocks(memPoolArray);
 
     if (saveResults) {
       const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, blocks);
@@ -85,26 +85,23 @@ class MempoolBlocks {
     return blocks;
   }
 
-  private calculateMempoolBlocks(transactionsSorted: TransactionExtended[], prevBlocks: MempoolBlockWithTransactions[]): MempoolBlockWithTransactions[] {
+  private calculateMempoolBlocks(transactionsSorted: TransactionExtended[]): MempoolBlockWithTransactions[] {
     const mempoolBlocks: MempoolBlockWithTransactions[] = [];
     let blockWeight = 0;
-    let blockSize = 0;
     let transactions: TransactionExtended[] = [];
     transactionsSorted.forEach((tx) => {
       if (blockWeight + tx.weight <= config.MEMPOOL.BLOCK_WEIGHT_UNITS
         || mempoolBlocks.length === config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT - 1) {
         blockWeight += tx.weight;
-        blockSize += tx.size;
         transactions.push(tx);
       } else {
-        mempoolBlocks.push(this.dataToMempoolBlocks(transactions, mempoolBlocks.length));
+        mempoolBlocks.push(this.dataToMempoolBlocks(transactions));
         blockWeight = tx.weight;
-        blockSize = tx.size;
         transactions = [tx];
       }
     });
     if (transactions.length) {
-      mempoolBlocks.push(this.dataToMempoolBlocks(transactions, mempoolBlocks.length));
+      mempoolBlocks.push(this.dataToMempoolBlocks(transactions));
     }
 
     return mempoolBlocks;
@@ -151,7 +148,7 @@ class MempoolBlocks {
     // prepare a stripped down version of the mempool with only the minimum necessary data
     // to reduce the overhead of passing this data to the worker thread
     const strippedMempool: { [txid: string]: ThreadTransaction } = {};
-    Object.values(newMempool).forEach(entry => {
+    Object.values(newMempool).filter(tx => !tx.deleteAfter).forEach(entry => {
       strippedMempool[entry.txid] = {
         txid: entry.txid,
         fee: entry.fee,
@@ -186,7 +183,14 @@ class MempoolBlocks {
         this.txSelectionWorker?.once('error', reject);
       });
       this.txSelectionWorker.postMessage({ type: 'set', mempool: strippedMempool });
-      const { blocks, clusters } = await workerResultPromise;
+      let { blocks, clusters } = await workerResultPromise;
+      // filter out stale transactions
+      const unfilteredCount = blocks.reduce((total, block) => { return total + block.length; }, 0);
+      blocks = blocks.map(block => block.filter(tx => (tx.txid && tx.txid in newMempool)));
+      const filteredCount = blocks.reduce((total, block) => { return total + block.length; }, 0);
+      if (filteredCount < unfilteredCount) {
+        logger.warn(`tx selection worker thread returned ${unfilteredCount - filteredCount} stale transactions from makeBlockTemplates`);
+      }
 
       // clean up thread error listener
       this.txSelectionWorker?.removeListener('error', threadErrorListener);
@@ -228,7 +232,14 @@ class MempoolBlocks {
         this.txSelectionWorker?.once('error', reject);
       });
       this.txSelectionWorker.postMessage({ type: 'update', added: addedStripped, removed });
-      const { blocks, clusters } = await workerResultPromise;
+      let { blocks, clusters } = await workerResultPromise;
+      // filter out stale transactions
+      const unfilteredCount = blocks.reduce((total, block) => { return total + block.length; }, 0);
+      blocks = blocks.map(block => block.filter(tx => (tx.txid && tx.txid in newMempool)));
+      const filteredCount = blocks.reduce((total, block) => { return total + block.length; }, 0);
+      if (filteredCount < unfilteredCount) {
+        logger.warn(`tx selection worker thread returned ${unfilteredCount - filteredCount} stale transactions from updateBlockTemplates`);
+      }
 
       // clean up thread error listener
       this.txSelectionWorker?.removeListener('error', threadErrorListener);
@@ -243,7 +254,7 @@ class MempoolBlocks {
     // update this thread's mempool with the results
     blocks.forEach(block => {
       block.forEach(tx => {
-        if (tx.txid in mempool) {
+        if (tx.txid && tx.txid in mempool) {
           if (tx.effectiveFeePerVsize != null) {
             mempool[tx.txid].effectiveFeePerVsize = tx.effectiveFeePerVsize;
           }
@@ -253,6 +264,10 @@ class MempoolBlocks {
             const cluster = clusters[tx.cpfpRoot];
             let matched = false;
             cluster.forEach(txid => {
+              if (!txid || !mempool[txid]) {
+                logger.warn('projected transaction ancestor missing from mempool cache');
+                return;
+              }
               if (txid === tx.txid) {
                 matched = true;
               } else {
@@ -273,15 +288,17 @@ class MempoolBlocks {
             mempool[tx.txid].bestDescendant = null;
           }
           mempool[tx.txid].cpfpChecked = tx.cpfpChecked;
+        } else {
+          logger.warn('projected transaction missing from mempool cache');
         }
       });
     });
 
     // unpack the condensed blocks into proper mempool blocks
-    const mempoolBlocks = blocks.map((transactions, blockIndex) => {
+    const mempoolBlocks = blocks.map((transactions) => {
       return this.dataToMempoolBlocks(transactions.map(tx => {
         return mempool[tx.txid] || null;
-      }).filter(tx => !!tx), blockIndex);
+      }).filter(tx => !!tx));
     });
 
     if (saveResults) {
@@ -293,7 +310,7 @@ class MempoolBlocks {
     return mempoolBlocks;
   }
 
-  private dataToMempoolBlocks(transactions: TransactionExtended[], blocksIndex: number): MempoolBlockWithTransactions {
+  private dataToMempoolBlocks(transactions: TransactionExtended[]): MempoolBlockWithTransactions {
     let totalSize = 0;
     let totalWeight = 0;
     const fitTransactions: TransactionExtended[] = [];
@@ -304,22 +321,14 @@ class MempoolBlocks {
         fitTransactions.push(tx);
       }
     });
-    let rangeLength = 4;
+    const feeStats = Common.calcEffectiveFeeStatistics(transactions);
-    if (blocksIndex === 0) {
-      rangeLength = 8;
-    }
-    if (transactions.length > 4000) {
-      rangeLength = 6;
-    } else if (transactions.length > 10000) {
-      rangeLength = 8;
-    }
     return {
       blockSize: totalSize,
       blockVSize: totalWeight / 4,
       nTx: transactions.length,
       totalFees: transactions.reduce((acc, cur) => acc + cur.fee, 0),
-      medianFee: Common.percentile(transactions.map((tx) => tx.effectiveFeePerVsize), config.MEMPOOL.RECOMMENDED_FEE_PERCENTILE),
+      medianFee: feeStats.medianFee, // Common.percentile(transactions.map((tx) => tx.effectiveFeePerVsize), config.MEMPOOL.RECOMMENDED_FEE_PERCENTILE),
-      feeRange: Common.getFeesInRange(transactions, rangeLength),
+      feeRange: feeStats.feeRange, //Common.getFeesInRange(transactions, rangeLength),
       transactionIds: transactions.map((tx) => tx.txid),
       transactions: fitTransactions.map((tx) => Common.stripTransaction(tx)),
     };

backend/src/api/mempool.ts

@@ -38,7 +38,6 @@ class Mempool {
 
   constructor() {
     setInterval(this.updateTxPerSecond.bind(this), 1000);
-    setInterval(this.deleteExpiredTransactions.bind(this), 20000);
   }
 
   /**
@@ -256,7 +255,7 @@ class Mempool {
     }
   }
 
-  private deleteExpiredTransactions() {
+  public deleteExpiredTransactions() {
     const now = new Date().getTime();
     for (const tx in this.mempoolCache) {
       const lazyDeleteAt = this.mempoolCache[tx].deleteAfter;

backend/src/api/mining/mining.ts

@@ -452,7 +452,7 @@ class Mining {
       const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
       if (elapsedSeconds > 5) {
         const progress = Math.round(totalBlockChecked / blocks.length * 100);
-        logger.info(`Indexing difficulty adjustment at block #${block.height} | Progress: ${progress}%`, logger.tags.mining);
+        logger.debug(`Indexing difficulty adjustment at block #${block.height} | Progress: ${progress}%`, logger.tags.mining);
         timer = new Date().getTime() / 1000;
       }
     }
@@ -558,8 +558,10 @@ class Mining {
       currentBlockHeight -= 10000;
     }
 
-    if (totalIndexed) {
+    if (totalIndexed > 0) {
-      logger.info(`Indexing missing coinstatsindex data completed`, logger.tags.mining);
+      logger.info(`Indexing missing coinstatsindex data completed. Indexed ${totalIndexed}`, logger.tags.mining);
+    } else {
+      logger.debug(`Indexing missing coinstatsindex data completed. Indexed 0.`, logger.tags.mining);
     }
   }
 

backend/src/api/websocket-handler.ts

@@ -211,6 +211,7 @@ class WebsocketHandler {
     if (!_blocks) {
       _blocks = blocks.getBlocks().slice(-config.MEMPOOL.INITIAL_BLOCKS_AMOUNT);
     }
+    const da = difficultyAdjustment.getDifficultyAdjustment();
     return {
       'mempoolInfo': memPool.getMempoolInfo(),
       'vBytesPerSecond': memPool.getVBytesPerSecond(),
@@ -220,7 +221,7 @@ class WebsocketHandler {
       'transactions': memPool.getLatestTransactions(),
       'backendInfo': backendInfo.getBackendInfo(),
       'loadingIndicators': loadingIndicators.getLoadingIndicators(),
-      'da': difficultyAdjustment.getDifficultyAdjustment(),
+      'da': da?.previousTime ? da : undefined,
       'fees': feeApi.getRecommendedFee(),
       ...this.extraInitProperties
     };
@@ -278,7 +279,9 @@ class WebsocketHandler {
         response['mempoolInfo'] = mempoolInfo;
         response['vBytesPerSecond'] = vBytesPerSecond;
         response['transactions'] = newTransactions.slice(0, 6).map((tx) => Common.stripTransaction(tx));
-        response['da'] = da;
+        if (da?.previousTime) {
+          response['da'] = da;
+        }
         response['fees'] = recommendedFees;
       }
 
@@ -505,7 +508,7 @@ class WebsocketHandler {
       const response = {
         'block': block,
         'mempoolInfo': memPool.getMempoolInfo(),
-        'da': da,
+        'da': da?.previousTime ? da : undefined,
         'fees': fees,
       };
 

backend/src/config.ts

@@ -33,9 +33,12 @@ interface IConfig {
     ADVANCED_GBT_MEMPOOL: boolean;
     CPFP_INDEXING: boolean;
     MAX_BLOCKS_BULK_QUERY: number;
+    DISK_CACHE_BLOCK_INTERVAL: number;
   };
   ESPLORA: {
     REST_API_URL: string;
+    UNIX_SOCKET_PATH: string | void | null;
+    RETRY_UNIX_SOCKET_AFTER: number;
   };
   LIGHTNING: {
     ENABLED: boolean;
@@ -51,6 +54,7 @@ interface IConfig {
     TLS_CERT_PATH: string;
     MACAROON_PATH: string;
     REST_API_URL: string;
+    TIMEOUT: number;
   };
   CLIGHTNING: {
     SOCKET: string;
@@ -65,12 +69,14 @@ interface IConfig {
     PORT: number;
     USERNAME: string;
     PASSWORD: string;
+    TIMEOUT: number;
   };
   SECOND_CORE_RPC: {
     HOST: string;
     PORT: number;
     USERNAME: string;
     PASSWORD: string;
+    TIMEOUT: number;
   };
   DATABASE: {
     ENABLED: boolean;
@@ -155,9 +161,12 @@ const defaults: IConfig = {
     'ADVANCED_GBT_MEMPOOL': false,
     'CPFP_INDEXING': false,
     'MAX_BLOCKS_BULK_QUERY': 0,
+    'DISK_CACHE_BLOCK_INTERVAL': 6,
   },
   'ESPLORA': {
     'REST_API_URL': 'http://127.0.0.1:3000',
+    'UNIX_SOCKET_PATH': null,
+    'RETRY_UNIX_SOCKET_AFTER': 30000,
   },
   'ELECTRUM': {
     'HOST': '127.0.0.1',
@@ -168,13 +177,15 @@ const defaults: IConfig = {
     'HOST': '127.0.0.1',
     'PORT': 8332,
     'USERNAME': 'mempool',
-    'PASSWORD': 'mempool'
+    'PASSWORD': 'mempool',
+    'TIMEOUT': 60000,
   },
   'SECOND_CORE_RPC': {
     'HOST': '127.0.0.1',
     'PORT': 8332,
     'USERNAME': 'mempool',
-    'PASSWORD': 'mempool'
+    'PASSWORD': 'mempool',
+    'TIMEOUT': 60000,
   },
   'DATABASE': {
     'ENABLED': true,
@@ -214,6 +225,7 @@ const defaults: IConfig = {
     'TLS_CERT_PATH': '',
     'MACAROON_PATH': '',
     'REST_API_URL': 'https://localhost:8080',
+    'TIMEOUT': 10000,
   },
   'CLIGHTNING': {
     'SOCKET': '',

backend/src/index.ts

@@ -45,7 +45,8 @@ class Server {
   private wss: WebSocket.Server | undefined;
   private server: http.Server | undefined;
   private app: Application;
-  private currentBackendRetryInterval = 5;
+  private currentBackendRetryInterval = 1;
+  private backendRetryCount = 0;
 
   private maxHeapSize: number = 0;
   private heapLogInterval: number = 60;
@@ -178,22 +179,23 @@ class Server {
           logger.debug(msg);
         }
       }
+      memPool.deleteExpiredTransactions();
       await blocks.$updateBlocks();
       await memPool.$updateMempool();
       indexer.$run();
 
       setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS);
-      this.currentBackendRetryInterval = 5;
+      this.backendRetryCount = 0;
     } catch (e: any) {
-      let loggerMsg = `Exception in runMainUpdateLoop(). Retrying in ${this.currentBackendRetryInterval} sec.`;
+      this.backendRetryCount++;
+      let loggerMsg = `Exception in runMainUpdateLoop() (count: ${this.backendRetryCount}). Retrying in ${this.currentBackendRetryInterval} sec.`;
       loggerMsg += ` Reason: ${(e instanceof Error ? e.message : e)}.`;
       if (e?.stack) {
         loggerMsg += ` Stack trace: ${e.stack}`;
       }
       // When we get a first Exception, only `logger.debug` it and retry after 5 seconds
       // From the second Exception, `logger.warn` the Exception and increase the retry delay
-      // Maximum retry delay is 60 seconds
+      if (this.backendRetryCount >= 5) {
-      if (this.currentBackendRetryInterval > 5) {
         logger.warn(loggerMsg);
         mempool.setOutOfSync();
       } else {
@@ -203,8 +205,6 @@ class Server {
         logger.debug(`AxiosError: ${e?.message}`);
       }
       setTimeout(this.runMainUpdateLoop.bind(this), 1000 * this.currentBackendRetryInterval);
-      this.currentBackendRetryInterval *= 2;
-      this.currentBackendRetryInterval = Math.min(this.currentBackendRetryInterval, 60);
     }
   }
 
@@ -215,11 +215,11 @@ class Server {
       await lightningStatsUpdater.$startService();
       await forensicsService.$startService();
     } catch(e) {
-      logger.err(`Nodejs lightning backend crashed. Restarting in 1 minute. Reason: ${(e instanceof Error ? e.message : e)}`);
+      logger.err(`Exception in $runLightningBackend. Restarting in 1 minute. Reason: ${(e instanceof Error ? e.message : e)}`);
       await Common.sleep$(1000 * 60);
       this.$runLightningBackend();
     };
-}
+  }
 
   setUpWebsocketHandling(): void {
     if (this.wss) {
@@ -275,7 +275,7 @@ class Server {
 
     if (!this.warnedHeapCritical && this.maxHeapSize > warnThreshold) {
       this.warnedHeapCritical = true;
-      logger.warn(`Used ${(this.maxHeapSize / stats.heap_size_limit).toFixed(2)}% of heap limit (${formatBytes(this.maxHeapSize, byteUnits, true)} / ${formatBytes(stats.heap_size_limit, byteUnits)})!`);
+      logger.warn(`Used ${(this.maxHeapSize / stats.heap_size_limit * 100).toFixed(2)}% of heap limit (${formatBytes(this.maxHeapSize, byteUnits, true)} / ${formatBytes(stats.heap_size_limit, byteUnits)})!`);
     }
     if (this.lastHeapLogTime === null || (now - this.lastHeapLogTime) > (this.heapLogInterval * 1000)) {
       logger.debug(`Memory usage: ${formatBytes(this.maxHeapSize, byteUnits)} / ${formatBytes(stats.heap_size_limit, byteUnits)}`);

backend/src/logger.ts

@@ -69,6 +69,10 @@ class Logger {
     this.network = this.getNetwork();
   }
 
+  public updateNetwork(): void {
+    this.network = this.getNetwork();
+  }
+
   private addprio(prio): void {
     this[prio] = (function(_this) {
       return function(msg, tag?: string) {

backend/src/mempool.interfaces.ts

@@ -214,6 +214,16 @@ export interface MempoolStats {
   tx_count: number;
 }
 
+export interface EffectiveFeeStats {
+  medianFee: number; // median effective fee rate
+  feeRange: number[]; // 2nd, 10th, 25th, 50th, 75th, 90th, 98th percentiles
+}
+
+export interface CpfpSummary {
+  transactions: TransactionExtended[];
+  clusters: { root: string, height: number, txs: Ancestor[], effectiveFeePerVsize: number }[];
+}
+
 export interface Statistic {
   id?: number;
   added: string;
@@ -309,9 +319,11 @@ export interface IDifficultyAdjustment {
   remainingBlocks: number;
   remainingTime: number;
   previousRetarget: number;
+  previousTime: number;
   nextRetargetHeight: number;
   timeAvg: number;
   timeOffset: number;
+  expectedBlocks: number;
 }
 
 export interface IndexedDifficultyAdjustment {

backend/src/repositories/BlocksRepository.ts

@@ -1,4 +1,4 @@
-import { BlockExtended, BlockExtension, BlockPrice } from '../mempool.interfaces';
+import { BlockExtended, BlockExtension, BlockPrice, EffectiveFeeStats } from '../mempool.interfaces';
 import DB from '../database';
 import logger from '../logger';
 import { Common } from '../api/common';
@@ -13,6 +13,48 @@ import chainTips from '../api/chain-tips';
 import blocks from '../api/blocks';
 import BlocksAuditsRepository from './BlocksAuditsRepository';
 
+interface DatabaseBlock {
+  id: string;
+  height: number;
+  version: number;
+  timestamp: number;
+  bits: number;
+  nonce: number;
+  difficulty: number;
+  merkle_root: string;
+  tx_count: number;
+  size: number;
+  weight: number;
+  previousblockhash: string;
+  mediantime: number;
+  totalFees: number;
+  medianFee: number;
+  feeRange: string;
+  reward: number;
+  poolId: number;
+  poolName: string;
+  poolSlug: string;
+  avgFee: number;
+  avgFeeRate: number;
+  coinbaseRaw: string;
+  coinbaseAddress: string;
+  coinbaseSignature: string;
+  coinbaseSignatureAscii: string;
+  avgTxSize: number;
+  totalInputs: number;
+  totalOutputs: number;
+  totalOutputAmt: number;
+  medianFeeAmt: number;
+  feePercentiles: string;
+  segwitTotalTxs: number;
+  segwitTotalSize: number;
+  segwitTotalWeight: number;
+  header: string;
+  utxoSetChange: number;
+  utxoSetSize: number;
+  totalInputAmt: number;
+}
+
 const BLOCK_DB_FIELDS = `
   blocks.hash AS id,
   blocks.height,
@@ -52,7 +94,7 @@ const BLOCK_DB_FIELDS = `
   blocks.header,
   blocks.utxoset_change AS utxoSetChange,
   blocks.utxoset_size AS utxoSetSize,
-  blocks.total_input_amt AS totalInputAmts
+  blocks.total_input_amt AS totalInputAmt
 `;
 
 class BlocksRepository {
@@ -171,6 +213,32 @@ class BlocksRepository {
     }
   }
 
+  /**
+   * Update missing fee amounts fields
+   *
+   * @param blockHash 
+   * @param feeAmtPercentiles 
+   * @param medianFeeAmt 
+   */
+  public async $updateFeeAmounts(blockHash: string, feeAmtPercentiles, medianFeeAmt) : Promise<void> {
+    try {
+      const query = `
+        UPDATE blocks
+        SET fee_percentiles = ?, median_fee_amt = ?
+        WHERE hash = ?
+      `;
+      const params: any[] = [
+        JSON.stringify(feeAmtPercentiles),
+        medianFeeAmt,
+        blockHash
+      ];
+      await DB.query(query, params);
+    } catch (e: any) {
+      logger.err(`Cannot update fee amounts for block ${blockHash}. Reason: ' + ${e instanceof Error ? e.message : e}`);
+      throw e;
+    }
+  }
+
   /**
    * Get all block height that have not been indexed between [startHeight, endHeight]
    */
@@ -432,7 +500,7 @@ class BlocksRepository {
 
       const blocks: BlockExtended[] = [];
       for (const block of rows) {
-        blocks.push(await this.formatDbBlockIntoExtendedBlock(block));
+        blocks.push(await this.formatDbBlockIntoExtendedBlock(block as DatabaseBlock));
       }
 
       return blocks;
@@ -459,37 +527,13 @@ class BlocksRepository {
         return null;
       }
 
-      return await this.formatDbBlockIntoExtendedBlock(rows[0]);  
+      return await this.formatDbBlockIntoExtendedBlock(rows[0] as DatabaseBlock);  
     } catch (e) {
       logger.err(`Cannot get indexed block ${height}. Reason: ` + (e instanceof Error ? e.message : e));
       throw e;
     }
   }
 
-  /**
-   * Get one block by hash
-   */
-  public async $getBlockByHash(hash: string): Promise<object | null> {
-    try {
-      const query = `
-        SELECT ${BLOCK_DB_FIELDS}
-        FROM blocks
-        JOIN pools ON blocks.pool_id = pools.id
-        WHERE hash = ?;
-      `;
-      const [rows]: any[] = await DB.query(query, [hash]);
-
-      if (rows.length <= 0) {
-        return null;
-      }
- 
-      return await this.formatDbBlockIntoExtendedBlock(rows[0]);
-    } catch (e) {
-      logger.err(`Cannot get indexed block ${hash}. Reason: ` + (e instanceof Error ? e.message : e));
-      throw e;
-    }
-  }
-
   /**
    * Return blocks difficulty
    */
@@ -599,7 +643,6 @@ class BlocksRepository {
         if (blocks[idx].previous_block_hash !== blocks[idx - 1].hash) {
           logger.warn(`Chain divergence detected at block ${blocks[idx - 1].height}`);
           await this.$deleteBlocksFrom(blocks[idx - 1].height);
-          await BlocksSummariesRepository.$deleteBlocksFrom(blocks[idx - 1].height);
           await HashratesRepository.$deleteHashratesFromTimestamp(blocks[idx - 1].timestamp - 604800);
           await DifficultyAdjustmentsRepository.$deleteAdjustementsFromHeight(blocks[idx - 1].height);
           return false;
@@ -619,7 +662,7 @@ class BlocksRepository {
    * Delete blocks from the database from blockHeight
    */
   public async $deleteBlocksFrom(blockHeight: number) {
-    logger.info(`Delete newer blocks from height ${blockHeight} from the database`);
+    logger.info(`Delete newer blocks from height ${blockHeight} from the database`, logger.tags.mining);
 
     try {
       await DB.query(`DELETE FROM blocks where height >= ${blockHeight}`);
@@ -908,13 +951,32 @@ class BlocksRepository {
     }
   }
 
+  /**
+   * Save indexed effective fee statistics
+   * 
+   * @param id 
+   * @param feeStats 
+   */
+  public async $saveEffectiveFeeStats(id: string, feeStats: EffectiveFeeStats): Promise<void> {
+    try {
+      await DB.query(`
+        UPDATE blocks SET median_fee = ?, fee_span = ?
+        WHERE hash = ?`,
+        [feeStats.medianFee, JSON.stringify(feeStats.feeRange), id]
+      );
+    } catch (e) {
+      logger.err(`Cannot update block fee stats. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
   /**
    * Convert a mysql row block into a BlockExtended. Note that you
    * must provide the correct field into dbBlk object param
    * 
    * @param dbBlk 
    */
-  private async formatDbBlockIntoExtendedBlock(dbBlk: any): Promise<BlockExtended> {
+  private async formatDbBlockIntoExtendedBlock(dbBlk: DatabaseBlock): Promise<BlockExtended> {
     const blk: Partial<BlockExtended> = {};
     const extras: Partial<BlockExtension> = {};
 
@@ -978,6 +1040,7 @@ class BlocksRepository {
     }
 
     // If we're missing block summary related field, check if we can populate them on the fly now
+    // This is for example triggered upon re-org
     if (Common.blocksSummariesIndexingEnabled() &&
       (extras.medianFeeAmt === null || extras.feePercentiles === null))
     {
@@ -985,11 +1048,12 @@ class BlocksRepository {
       if (extras.feePercentiles === null) {
         const block = await bitcoinClient.getBlock(dbBlk.id, 2);
         const summary = blocks.summarizeBlock(block);
-        await BlocksSummariesRepository.$saveSummary({ height: block.height, mined: summary });
+        await BlocksSummariesRepository.$saveTransactions(dbBlk.height, dbBlk.id, summary.transactions);
         extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id);
       }
       if (extras.feePercentiles !== null) {
         extras.medianFeeAmt = extras.feePercentiles[3];
+        await this.$updateFeeAmounts(dbBlk.id, extras.feePercentiles, extras.medianFeeAmt);
       }
     }
 

backend/src/repositories/BlocksSummariesRepository.ts

@@ -1,6 +1,6 @@
 import DB from '../database';
 import logger from '../logger';
-import { BlockSummary } from '../mempool.interfaces';
+import { BlockSummary, TransactionStripped } from '../mempool.interfaces';
 
 class BlocksSummariesRepository {
   public async $getByBlockId(id: string): Promise<BlockSummary | undefined> {
@@ -17,23 +17,17 @@ class BlocksSummariesRepository {
     return undefined;
   }
 
-  public async $saveSummary(params: { height: number, mined?: BlockSummary}) {
+  public async $saveTransactions(blockHeight: number, blockId: string, transactions: TransactionStripped[]): Promise<void> {
-    const blockId = params.mined?.id;
     try {
-      const transactions = JSON.stringify(params.mined?.transactions || []);
+      const transactionsStr = JSON.stringify(transactions);
       await DB.query(`
-        INSERT INTO blocks_summaries (height, id, transactions, template)
+        INSERT INTO blocks_summaries
-        VALUE (?, ?, ?, ?)
+        SET height = ?, transactions = ?, id = ?
-        ON DUPLICATE KEY UPDATE
+        ON DUPLICATE KEY UPDATE transactions = ?`,
-          transactions = ?
+        [blockHeight, transactionsStr, blockId, transactionsStr]);
-      `, [params.height, blockId, transactions, '[]', transactions]);
     } catch (e: any) {
-      if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
+      logger.debug(`Cannot save block summary transactions for ${blockId}. Reason: ${e instanceof Error ? e.message : e}`);
-        logger.debug(`Cannot save block summary for ${blockId} because it has already been indexed, ignoring`);
+      throw e;
-      } else {
-        logger.debug(`Cannot save block summary for ${blockId}. Reason: ${e instanceof Error ? e.message : e}`);
-        throw e;
-      }
     }
   }
 
@@ -68,19 +62,6 @@ class BlocksSummariesRepository {
     return [];
   }
 
-  /**
-   * Delete blocks from the database from blockHeight
-   */
-  public async $deleteBlocksFrom(blockHeight: number) {
-    logger.info(`Delete newer blocks summary from height ${blockHeight} from the database`);
-
-    try {
-      await DB.query(`DELETE FROM blocks_summaries where height >= ${blockHeight}`);
-    } catch (e) {
-      logger.err('Cannot delete indexed blocks summaries. Reason: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
   /**
    * Get the fee percentiles if the block has already been indexed, [] otherwise
    * 

backend/src/repositories/CpfpRepository.ts

@@ -48,7 +48,7 @@ class CpfpRepository {
     }
   }
 
-  public async $batchSaveClusters(clusters: { root: string, height: number, txs: any, effectiveFeePerVsize: number}[]): Promise<boolean> {
+  public async $batchSaveClusters(clusters: { root: string, height: number, txs: Ancestor[], effectiveFeePerVsize: number }[]): Promise<boolean> {
     try {
       const clusterValues: any[] = [];
       const txs: any[] = [];

backend/src/repositories/HashratesRepository.ts

@@ -220,7 +220,7 @@ class HashratesRepository {
    * Delete hashrates from the database from timestamp
    */
   public async $deleteHashratesFromTimestamp(timestamp: number) {
-    logger.info(`Delete newer hashrates from timestamp ${new Date(timestamp * 1000).toUTCString()} from the database`);
+    logger.info(`Delete newer hashrates from timestamp ${new Date(timestamp * 1000).toUTCString()} from the database`, logger.tags.mining);
 
     try {
       await DB.query(`DELETE FROM hashrates WHERE hashrate_timestamp >= FROM_UNIXTIME(?)`, [timestamp]);

backend/src/repositories/PricesRepository.ts

@@ -160,7 +160,7 @@ class PricesRepository {
 
       // Compute fiat exchange rates
       let latestPrice = rates[0] as ApiPrice;
-      if (latestPrice.USD === -1) {
+      if (!latestPrice || latestPrice.USD === -1) {
         latestPrice = priceUpdater.getEmptyPricesObj();
       }
 

backend/src/tasks/lightning/forensics.service.ts

@@ -27,7 +27,7 @@ class ForensicsService {
 
   private async $runTasks(): Promise<void> {
     try {
-      logger.info(`Running forensics scans`);
+      logger.debug(`Running forensics scans`);
 
       if (config.MEMPOOL.BACKEND === 'esplora') {
         await this.$runClosedChannelsForensics(false);
@@ -73,7 +73,7 @@ class ForensicsService {
     let progress = 0;
 
     try {
-      logger.info(`Started running closed channel forensics...`);
+      logger.debug(`Started running closed channel forensics...`);
       let channels;
       if (onlyNewChannels) {
         channels = await channelsApi.$getClosedChannelsWithoutReason();
@@ -156,7 +156,7 @@ class ForensicsService {
           this.loggerTimer = new Date().getTime() / 1000;
         }
       }
-      logger.info(`Closed channels forensics scan complete.`);
+      logger.debug(`Closed channels forensics scan complete.`);
     } catch (e) {
       logger.err('$runClosedChannelsForensics() error: ' + (e instanceof Error ? e.message : e));
     }
@@ -217,7 +217,7 @@ class ForensicsService {
     let progress = 0;
 
     try {
-      logger.info(`Started running open channel forensics...`);
+      logger.debug(`Started running open channel forensics...`);
       const channels = await channelsApi.$getChannelsWithoutSourceChecked();
 
       for (const openChannel of channels) {
@@ -266,7 +266,7 @@ class ForensicsService {
         }
       }
 
-      logger.info(`Open channels forensics scan complete.`);
+      logger.debug(`Open channels forensics scan complete.`);
     } catch (e) {
       logger.err('$runOpenedChannelsForensics() error: ' + (e instanceof Error ? e.message : e));
     } finally {

backend/src/tasks/lightning/network-sync.service.ts

@@ -283,7 +283,7 @@ class NetworkSyncService {
       } else {
         log += ` for the first time`;
       }
-      logger.info(`${log}`, logger.tags.ln);
+      logger.debug(`${log}`, logger.tags.ln);
 
       const channels = await channelsApi.$getChannelsByStatus([0, 1]);
       for (const channel of channels) {

backend/src/tasks/lightning/stats-updater.service.ts

@@ -22,12 +22,15 @@ class LightningStatsUpdater {
    * Update the latest entry for each node every config.LIGHTNING.STATS_REFRESH_INTERVAL seconds
    */
   private async $logStatsDaily(): Promise<void> {
-    const date = new Date();
+    try {
-    Common.setDateMidnight(date);
+      const date = new Date();
-    const networkGraph = await lightningApi.$getNetworkGraph();
+      Common.setDateMidnight(date);
-    await LightningStatsImporter.computeNetworkStats(date.getTime() / 1000, networkGraph);
+      const networkGraph = await lightningApi.$getNetworkGraph();
-    
+      await LightningStatsImporter.computeNetworkStats(date.getTime() / 1000, networkGraph);
-    logger.debug(`Updated latest network stats`, logger.tags.ln);
+      logger.debug(`Updated latest network stats`, logger.tags.ln);
+    } catch (e) {
+      logger.err(`Exception in $logStatsDaily. Reason: ${(e instanceof Error ? e.message : e)}`);
+    }
   }
 }
 

backend/src/tasks/lightning/sync-tasks/stats-importer.ts

@@ -15,16 +15,20 @@ class LightningStatsImporter {
   topologiesFolder = config.LIGHTNING.TOPOLOGY_FOLDER;
 
   async $run(): Promise<void> {
-    const [channels]: any[] = await DB.query('SELECT short_id from channels;');
+    try {
-    logger.info(`Caching funding txs for currently existing channels`, logger.tags.ln);
+      const [channels]: any[] = await DB.query('SELECT short_id from channels;');
-    await fundingTxFetcher.$fetchChannelsFundingTxs(channels.map(channel => channel.short_id));
+      logger.info(`Caching funding txs for currently existing channels`, logger.tags.ln);
+      await fundingTxFetcher.$fetchChannelsFundingTxs(channels.map(channel => channel.short_id));
 
-    if (config.MEMPOOL.NETWORK !== 'mainnet' || config.DATABASE.ENABLED === false) {
+      if (config.MEMPOOL.NETWORK !== 'mainnet' || config.DATABASE.ENABLED === false) {
-      return;
+        return;
+      }
+
+      await this.$importHistoricalLightningStats();
+      await this.$cleanupIncorrectSnapshot();
+    } catch (e) {
+      logger.err(`Exception in LightningStatsImporter::$run(). ${e}`);
     }
-
-    await this.$importHistoricalLightningStats();
-    await this.$cleanupIncorrectSnapshot();
   }
 
   /**

backend/src/tasks/pools-updater.ts

@@ -62,7 +62,7 @@ class PoolsUpdater {
       if (this.currentSha === null) {
         logger.info(`Downloading pools-v2.json for the first time from ${this.poolsUrl} over ${network}`, logger.tags.mining);
       } else {
-        logger.warn(`pools-v2.json is outdated, fetch latest from ${this.poolsUrl} over ${network}`, logger.tags.mining);
+        logger.warn(`pools-v2.json is outdated, fetching latest from ${this.poolsUrl} over ${network}`, logger.tags.mining);
       }
       const poolsJson = await this.query(this.poolsUrl);
       if (poolsJson === undefined) {

backend/src/tasks/price-updater.ts

@@ -222,7 +222,7 @@ class PriceUpdater {
   private async $insertMissingRecentPrices(type: 'hour' | 'day'): Promise<void> {
     const existingPriceTimes = await PricesRepository.$getPricesTimes();
 
-    logger.info(`Fetching ${type === 'day' ? 'dai' : 'hour'}ly price history from exchanges and saving missing ones into the database`, logger.tags.mining);
+    logger.debug(`Fetching ${type === 'day' ? 'dai' : 'hour'}ly price history from exchanges and saving missing ones into the database`, logger.tags.mining);
 
     const historicalPrices: PriceHistory[] = [];
 

backend/tsconfig.json

@@ -5,6 +5,7 @@
     "types": ["node", "jest"],
     "lib": ["es2019", "dom"],
     "strict": true,
+    "skipLibCheck": true,
     "noImplicitAny": false,
     "sourceMap": false,
     "outDir": "dist",

docker/README.md

@@ -34,6 +34,7 @@ If you want to use different credentials, specify them in the `docker-compose.ym
       CORE_RPC_PORT: "8332"
       CORE_RPC_USERNAME: "customuser"
       CORE_RPC_PASSWORD: "custompassword"
+      CORE_RPC_TIMEOUT: "60000"
 ```
 
 The IP address in the example above refers to Docker's default gateway IP address so that the container can hit the `bitcoind` instance running on the host machine. If your setup is different, update it accordingly.
@@ -112,6 +113,7 @@ Below we list all settings from `mempool-config.json` and the corresponding over
     "ADVANCED_GBT_MEMPOOL": false,
     "CPFP_INDEXING": false,
     "MAX_BLOCKS_BULK_QUERY": 0,
+    "DISK_CACHE_BLOCK_INTERVAL": 6
   },
 ```
 
@@ -143,6 +145,7 @@ Corresponding `docker-compose.yml` overrides:
       MEMPOOL_ADVANCED_GBT_MEMPOOL: ""
       MEMPOOL_CPFP_INDEXING: ""
       MAX_BLOCKS_BULK_QUERY: ""
+      DISK_CACHE_BLOCK_INTERVAL: ""
       ...
 ```
 
@@ -158,7 +161,8 @@ Corresponding `docker-compose.yml` overrides:
     "HOST": "127.0.0.1",
     "PORT": 8332,
     "USERNAME": "mempool",
-    "PASSWORD": "mempool"
+    "PASSWORD": "mempool",
+    "TIMEOUT": 60000
   },
 ```
 
@@ -170,6 +174,7 @@ Corresponding `docker-compose.yml` overrides:
       CORE_RPC_PORT: ""
       CORE_RPC_USERNAME: ""
       CORE_RPC_PASSWORD: ""
+      CORE_RPC_TIMEOUT: 60000
       ...
 ```
 
@@ -199,7 +204,8 @@ Corresponding `docker-compose.yml` overrides:
 `mempool-config.json`:
 ```json
   "ESPLORA": {
-    "REST_API_URL": "http://127.0.0.1:3000"
+    "REST_API_URL": "http://127.0.0.1:3000",
+    "UNIX_SOCKET_PATH": "/tmp/esplora-socket"
   },
 ```
 
@@ -208,6 +214,7 @@ Corresponding `docker-compose.yml` overrides:
   api:
     environment:
       ESPLORA_REST_API_URL: ""
+      ESPLORA_UNIX_SOCKET_PATH: ""
       ...
 ```
 
@@ -219,7 +226,8 @@ Corresponding `docker-compose.yml` overrides:
     "HOST": "127.0.0.1",
     "PORT": 8332,
     "USERNAME": "mempool",
-    "PASSWORD": "mempool"
+    "PASSWORD": "mempool",
+    "TIMEOUT": 60000
   },
 ```
 
@@ -231,6 +239,7 @@ Corresponding `docker-compose.yml` overrides:
       SECOND_CORE_RPC_PORT: ""
       SECOND_CORE_RPC_USERNAME: ""
       SECOND_CORE_RPC_PASSWORD: ""
+      SECOND_CORE_RPC_TIMEOUT: ""
       ...
 ```
 
@@ -403,6 +412,7 @@ Corresponding `docker-compose.yml` overrides:
     "TLS_CERT_PATH": ""
     "MACAROON_PATH": ""
     "REST_API_URL": "https://localhost:8080"
+    "TIMEOUT": 10000
   }
 ```
 
@@ -413,6 +423,7 @@ Corresponding `docker-compose.yml` overrides:
       LND_TLS_CERT_PATH: ""
       LND_MACAROON_PATH: ""
       LND_REST_API_URL: "https://localhost:8080"
+      LND_TIMEOUT: 10000
       ...
 ```
 

docker/backend/mempool-config.json

@@ -26,13 +26,15 @@
     "ADVANCED_GBT_AUDIT": __MEMPOOL_ADVANCED_GBT_AUDIT__,
     "ADVANCED_GBT_MEMPOOL": __MEMPOOL_ADVANCED_GBT_MEMPOOL__,
     "CPFP_INDEXING": __MEMPOOL_CPFP_INDEXING__,
-    "MAX_BLOCKS_BULK_QUERY": __MEMPOOL_MAX_BLOCKS_BULK_QUERY__
+    "MAX_BLOCKS_BULK_QUERY": __MEMPOOL_MAX_BLOCKS_BULK_QUERY__,
+    "DISK_CACHE_BLOCK_INTERVAL": __MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__
   },
   "CORE_RPC": {
     "HOST": "__CORE_RPC_HOST__",
     "PORT": __CORE_RPC_PORT__,
     "USERNAME": "__CORE_RPC_USERNAME__",
-    "PASSWORD": "__CORE_RPC_PASSWORD__"
+    "PASSWORD": "__CORE_RPC_PASSWORD__",
+    "TIMEOUT": __CORE_RPC_TIMEOUT__
   },
   "ELECTRUM": {
     "HOST": "__ELECTRUM_HOST__",
@@ -40,13 +42,15 @@
     "TLS_ENABLED": __ELECTRUM_TLS_ENABLED__
   },
   "ESPLORA": {
-    "REST_API_URL": "__ESPLORA_REST_API_URL__"
+    "REST_API_URL": "__ESPLORA_REST_API_URL__",
+    "UNIX_SOCKET_PATH": "__ESPLORA_UNIX_SOCKET_PATH__"
   },
   "SECOND_CORE_RPC": {
     "HOST": "__SECOND_CORE_RPC_HOST__",
     "PORT": __SECOND_CORE_RPC_PORT__,
     "USERNAME": "__SECOND_CORE_RPC_USERNAME__",
-    "PASSWORD": "__SECOND_CORE_RPC_PASSWORD__"
+    "PASSWORD": "__SECOND_CORE_RPC_PASSWORD__",
+    "TIMEOUT": __SECOND_CORE_RPC_TIMEOUT__
   },
   "DATABASE": {
     "ENABLED": __DATABASE_ENABLED__,
@@ -83,7 +87,8 @@
   "LND": {
     "TLS_CERT_PATH": "__LND_TLS_CERT_PATH__",
     "MACAROON_PATH": "__LND_MACAROON_PATH__",
-    "REST_API_URL": "__LND_REST_API_URL__"
+    "REST_API_URL": "__LND_REST_API_URL__",
+    "TIMEOUT": "__LND_TIMEOUT__"
   },
   "CLIGHTNING": {
     "SOCKET": "__CLIGHTNING_SOCKET__"

docker/backend/start.sh

@@ -30,12 +30,14 @@ __MEMPOOL_ADVANCED_GBT_AUDIT__=${MEMPOOL_ADVANCED_GBT_AUDIT:=false}
 __MEMPOOL_ADVANCED_GBT_MEMPOOL__=${MEMPOOL_ADVANCED_GBT_MEMPOOL:=false}
 __MEMPOOL_CPFP_INDEXING__=${MEMPOOL_CPFP_INDEXING:=false}
 __MEMPOOL_MAX_BLOCKS_BULK_QUERY__=${MEMPOOL_MAX_BLOCKS_BULK_QUERY:=0}
+__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__=${MEMPOOL_DISK_CACHE_BLOCK_INTERVAL:=6}
 
 # CORE_RPC
 __CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1}
 __CORE_RPC_PORT__=${CORE_RPC_PORT:=8332}
 __CORE_RPC_USERNAME__=${CORE_RPC_USERNAME:=mempool}
 __CORE_RPC_PASSWORD__=${CORE_RPC_PASSWORD:=mempool}
+__CORE_RPC_TIMEOUT__=${CORE_RPC_TIMEOUT:=60000}
 
 # ELECTRUM
 __ELECTRUM_HOST__=${ELECTRUM_HOST:=127.0.0.1}
@@ -44,12 +46,14 @@ __ELECTRUM_TLS_ENABLED__=${ELECTRUM_TLS_ENABLED:=false}
 
 # ESPLORA
 __ESPLORA_REST_API_URL__=${ESPLORA_REST_API_URL:=http://127.0.0.1:3000}
+__ESPLORA_UNIX_SOCKET_PATH__=${ESPLORA_UNIX_SOCKET_PATH:=null}
 
 # SECOND_CORE_RPC
 __SECOND_CORE_RPC_HOST__=${SECOND_CORE_RPC_HOST:=127.0.0.1}
 __SECOND_CORE_RPC_PORT__=${SECOND_CORE_RPC_PORT:=8332}
 __SECOND_CORE_RPC_USERNAME__=${SECOND_CORE_RPC_USERNAME:=mempool}
 __SECOND_CORE_RPC_PASSWORD__=${SECOND_CORE_RPC_PASSWORD:=mempool}
+__SECOND_CORE_RPC_TIMEOUT__=${SECOND_CORE_RPC_TIMEOUT:=60000}
 
 # DATABASE
 __DATABASE_ENABLED__=${DATABASE_ENABLED:=true}
@@ -107,6 +111,7 @@ __LIGHTNING_LOGGER_UPDATE_INTERVAL__=${LIGHTNING_LOGGER_UPDATE_INTERVAL:=30}
 __LND_TLS_CERT_PATH__=${LND_TLS_CERT_PATH:=""}
 __LND_MACAROON_PATH__=${LND_MACAROON_PATH:=""}
 __LND_REST_API_URL__=${LND_REST_API_URL:="https://localhost:8080"}
+__LND_TIMEOUT__=${LND_TIMEOUT:=10000}
 
 # CLN
 __CLIGHTNING_SOCKET__=${CLIGHTNING_SOCKET:=""}
@@ -149,22 +154,26 @@ sed -i "s!__MEMPOOL_ADVANCED_GBT_MEMPOOL__!${__MEMPOOL_ADVANCED_GBT_MEMPOOL__}!g
 sed -i "s!__MEMPOOL_ADVANCED_GBT_AUDIT__!${__MEMPOOL_ADVANCED_GBT_AUDIT__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_CPFP_INDEXING__!${__MEMPOOL_CPFP_INDEXING__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_MAX_BLOCKS_BULK_QUERY__!${__MEMPOOL_MAX_BLOCKS_BULK_QUERY__}!g" mempool-config.json
+sed -i "s!__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__!${__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__}!g" mempool-config.json
 
 sed -i "s/__CORE_RPC_HOST__/${__CORE_RPC_HOST__}/g" mempool-config.json
 sed -i "s/__CORE_RPC_PORT__/${__CORE_RPC_PORT__}/g" mempool-config.json
 sed -i "s/__CORE_RPC_USERNAME__/${__CORE_RPC_USERNAME__}/g" mempool-config.json
 sed -i "s/__CORE_RPC_PASSWORD__/${__CORE_RPC_PASSWORD__}/g" mempool-config.json
+sed -i "s/__CORE_RPC_TIMEOUT__/${__CORE_RPC_TIMEOUT__}/g" mempool-config.json
 
 sed -i "s/__ELECTRUM_HOST__/${__ELECTRUM_HOST__}/g" mempool-config.json
 sed -i "s/__ELECTRUM_PORT__/${__ELECTRUM_PORT__}/g" mempool-config.json
 sed -i "s/__ELECTRUM_TLS_ENABLED__/${__ELECTRUM_TLS_ENABLED__}/g" mempool-config.json
 
 sed -i "s!__ESPLORA_REST_API_URL__!${__ESPLORA_REST_API_URL__}!g" mempool-config.json
+sed -i "s!__ESPLORA_UNIX_SOCKET_PATH__!${__ESPLORA_UNIX_SOCKET_PATH__}!g" mempool-config.json
 
 sed -i "s/__SECOND_CORE_RPC_HOST__/${__SECOND_CORE_RPC_HOST__}/g" mempool-config.json
 sed -i "s/__SECOND_CORE_RPC_PORT__/${__SECOND_CORE_RPC_PORT__}/g" mempool-config.json
 sed -i "s/__SECOND_CORE_RPC_USERNAME__/${__SECOND_CORE_RPC_USERNAME__}/g" mempool-config.json
 sed -i "s/__SECOND_CORE_RPC_PASSWORD__/${__SECOND_CORE_RPC_PASSWORD__}/g" mempool-config.json
+sed -i "s/__SECOND_CORE_RPC_TIMEOUT__/${__SECOND_CORE_RPC_TIMEOUT__}/g" mempool-config.json
 
 sed -i "s/__DATABASE_ENABLED__/${__DATABASE_ENABLED__}/g" mempool-config.json
 sed -i "s/__DATABASE_HOST__/${__DATABASE_HOST__}/g" mempool-config.json
@@ -216,6 +225,7 @@ sed -i "s!__LIGHTNING_LOGGER_UPDATE_INTERVAL__!${__LIGHTNING_LOGGER_UPDATE_INTER
 sed -i "s!__LND_TLS_CERT_PATH__!${__LND_TLS_CERT_PATH__}!g" mempool-config.json
 sed -i "s!__LND_MACAROON_PATH__!${__LND_MACAROON_PATH__}!g" mempool-config.json
 sed -i "s!__LND_REST_API_URL__!${__LND_REST_API_URL__}!g" mempool-config.json
+sed -i "s!__LND_TIMEOUT__!${__LND_TIMEOUT__}!g" mempool-config.json
 
 # CLN
 sed -i "s!__CLIGHTNING_SOCKET__!${__CLIGHTNING_SOCKET__}!g" mempool-config.json

docker/frontend/entrypoint.sh

@@ -10,6 +10,10 @@ cp /etc/nginx/nginx.conf /patch/nginx.conf
 sed -i "s/__MEMPOOL_FRONTEND_HTTP_PORT__/${__MEMPOOL_FRONTEND_HTTP_PORT__}/g" /patch/nginx.conf
 cat /patch/nginx.conf > /etc/nginx/nginx.conf
 
+if [ "${LIGHTNING_DETECTED_PORT}" != "" ];then
+  export LIGHTNING=true
+fi
+
 # Runtime overrides - read env vars defined in docker compose
 
 __TESTNET_ENABLED__=${TESTNET_ENABLED:=false}

frontend/README.md

@@ -106,6 +106,7 @@ https://www.transifex.com/mempool/mempool/dashboard/
 
 * Arabic @baro0k
 * Czech @pixelmade2
+* Danish @pierrevendelboe
 * German @Emzy
 * English (default)
 * Spanish @maxhodler @bisqes
@@ -113,6 +114,7 @@ https://www.transifex.com/mempool/mempool/dashboard/
 * French @Bayernatoor
 * Korean @kcalvinalvinn @sogoagain
 * Italian @HodlBits
+* Lithuanian @eimze21
 * Hebrew @rapidlab309
 * Georgian @wyd_idk
 * Hungarian @btcdragonlord

frontend/cypress/e2e/liquid/liquid.spec.ts

@@ -158,10 +158,10 @@ describe('Liquid', () => {
       it('show empty unblinded TX', () => {
         cy.visit(`${basePath}/tx/f2f41c0850e8e7e3f1af233161fd596662e67c11ef10ed15943884186fbb7f46#blinded=`);
         cy.waitForSkeletonGone();
-        cy.get('.table-tx-vin tr:nth-child(1)').should('have.class', '');
+        cy.get('.table-tx-vin tr:nth-child(1)').should('have.class', 'ng-star-inserted');
         cy.get('.table-tx-vin tr:nth-child(1) .amount').should('contain.text', 'Confidential');
-        cy.get('.table-tx-vout tr:nth-child(1)').should('have.class', '');
+        cy.get('.table-tx-vout tr:nth-child(1)').should('have.class', 'ng-star-inserted');
-        cy.get('.table-tx-vout tr:nth-child(2)').should('have.class', '');
+        cy.get('.table-tx-vout tr:nth-child(2)').should('have.class', 'ng-star-inserted');
         cy.get('.table-tx-vout tr:nth-child(1) .amount').should('contain.text', 'Confidential');
         cy.get('.table-tx-vout tr:nth-child(2) .amount').should('contain.text', 'Confidential');
       });
@@ -169,8 +169,8 @@ describe('Liquid', () => {
       it('show invalid unblinded TX hex', () => {
         cy.visit(`${basePath}/tx/f2f41c0850e8e7e3f1af233161fd596662e67c11ef10ed15943884186fbb7f46#blinded=123`);
         cy.waitForSkeletonGone();
-        cy.get('.table-tx-vin tr').should('have.class', '');
+        cy.get('.table-tx-vin tr').should('have.class', 'ng-star-inserted');
-        cy.get('.table-tx-vout tr').should('have.class', '');
+        cy.get('.table-tx-vout tr').should('have.class', 'ng-star-inserted');
         cy.get('.error-unblinded').contains('Error: Invalid blinding data (invalid hex)');
       });
 

frontend/cypress/e2e/liquidtestnet/liquidtestnet.spec.ts

@@ -109,10 +109,10 @@ describe('Liquid Testnet', () => {
       it('show empty unblinded TX', () => {
         cy.visit(`${basePath}/tx/c3d908ab77891e4c569b0df71aae90f4720b157019ebb20db176f4f9c4d626b8#blinded=`);
         cy.waitForSkeletonGone();
-        cy.get('.table-tx-vin tr:nth-child(1)').should('have.class', '');
+        cy.get('.table-tx-vin tr:nth-child(1)').should('have.class', 'ng-star-inserted');
         cy.get('.table-tx-vin tr:nth-child(1) .amount').should('contain.text', 'Confidential');
-        cy.get('.table-tx-vout tr:nth-child(1)').should('have.class', '');
+        cy.get('.table-tx-vout tr:nth-child(1)').should('have.class', 'ng-star-inserted');
-        cy.get('.table-tx-vout tr:nth-child(2)').should('have.class', '');
+        cy.get('.table-tx-vout tr:nth-child(2)').should('have.class', 'ng-star-inserted');
         cy.get('.table-tx-vout tr:nth-child(1) .amount').should('contain.text', 'Confidential');
         cy.get('.table-tx-vout tr:nth-child(2) .amount').should('contain.text', 'Confidential');
       });
@@ -120,8 +120,8 @@ describe('Liquid Testnet', () => {
       it('show invalid unblinded TX hex', () => {
         cy.visit(`${basePath}/tx/2477f220eef1d03f8ffa4a2861c275d155c3562adf0d79523aeeb0c59ee611ba#blinded=5000`);
         cy.waitForSkeletonGone();
-        cy.get('.table-tx-vin tr').should('have.class', '');
+        cy.get('.table-tx-vin tr').should('have.class', 'ng-star-inserted');
-        cy.get('.table-tx-vout tr').should('have.class', '');
+        cy.get('.table-tx-vout tr').should('have.class', 'ng-star-inserted');
         cy.get('.error-unblinded').contains('Error: Invalid blinding data (invalid hex)');
       });
 

frontend/cypress/e2e/mainnet/mainnet.spec.ts

@@ -127,7 +127,7 @@ describe('Mainnet', () => {
 
         cy.get('.search-box-container > .form-control').type('S').then(() => {
           cy.wait('@search-1wizS');
-          cy.get('app-search-results button.dropdown-item').should('have.length', 5);
+          cy.get('app-search-results button.dropdown-item').should('have.length', 6);
         });
 
         cy.get('.search-box-container > .form-control').type('A').then(() => {

frontend/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "mempool-frontend",
-  "version": "2.5.0-dev",
+  "version": "2.6.0-dev",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "mempool-frontend",
-      "version": "2.5.0-dev",
+      "version": "2.6.0-dev",
       "license": "GNU Affero General Public License v3.0",
       "dependencies": {
         "@angular-devkit/build-angular": "^14.2.10",

frontend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mempool-frontend",
-  "version": "2.5.0-dev",
+  "version": "2.6.0-dev",
   "description": "Bitcoin mempool visualizer and blockchain explorer backend",
   "license": "GNU Affero General Public License v3.0",
   "homepage": "https://mempool.space",

frontend/src/app/components/about/about.component.html

@@ -201,12 +201,12 @@
         <span>Umbrel</span>
       </a>
       <a href="https://github.com/rootzoll/raspiblitz" target="_blank" title="RaspiBlitz">
-        <img class="image" src="/resources/profile/raspiblitz.jpg" />
+        <img class="image" src="/resources/profile/raspiblitz.svg" />
         <span>RaspiBlitz</span>
       </a>
-      <a href="https://github.com/mynodebtc/mynode" target="_blank" title="MyNode">
+      <a href="https://github.com/mynodebtc/mynode" target="_blank" title="myNode">
-        <img class="image" src="/resources/profile/mynodebtc.jpg" />
+        <img class="image" src="/resources/profile/mynodebtc.png" />
-        <span>MyNode</span>
+        <span>myNode</span>
       </a>
       <a href="https://github.com/RoninDojo/RoninDojo" target="_blank" title="RoninDojo">
         <img class="image" src="/resources/profile/ronindojo.png" />
@@ -253,7 +253,7 @@
         <span>Sparrow</span>
       </a>
       <a href="https://github.com/ACINQ/phoenix" target="_blank" title="Phoenix Wallet by ACINQ">
-        <img class="image" src="/resources/profile/phoenix.jpg" />
+        <img class="image not-rounded" src="/resources/profile/phoenix.svg" />
         <span>Phoenix</span>
       </a>
       <a href="https://github.com/lnbits/lnbits-legend" target="_blank" title="LNbits">

frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts

@@ -23,7 +23,7 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On
   @Input() unavailable: boolean = false;
   @Input() auditHighlighting: boolean = false;
   @Input() blockConversion: Price;
-  @Output() txClickEvent = new EventEmitter<TransactionStripped>();
+  @Output() txClickEvent = new EventEmitter<{ tx: TransactionStripped, keyModifier: boolean}>();
   @Output() txHoverEvent = new EventEmitter<string>();
   @Output() readyEvent = new EventEmitter();
 
@@ -326,7 +326,9 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On
     if (event.target === this.canvas.nativeElement && event.pointerType === 'touch') {
       this.setPreviewTx(event.offsetX, event.offsetY, true);
     } else if (event.target === this.canvas.nativeElement) {
-      this.onTxClick(event.offsetX, event.offsetY);
+      const keyMod = event.shiftKey || event.ctrlKey || event.metaKey;
+      const middleClick = event.which === 2 || event.button === 1;
+      this.onTxClick(event.offsetX, event.offsetY, keyMod || middleClick);
     }
   }
 
@@ -409,12 +411,12 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On
     }
   }
 
-  onTxClick(cssX: number, cssY: number) {
+  onTxClick(cssX: number, cssY: number, keyModifier: boolean = false) {
     const x = cssX * window.devicePixelRatio;
     const y = cssY * window.devicePixelRatio;
     const selected = this.scene.getTxAt({ x, y });
     if (selected && selected.txid) {
-      this.txClickEvent.emit(selected);
+      this.txClickEvent.emit({ tx: selected, keyModifier });
     }
   }
 

frontend/src/app/components/block/block.component.ts

@@ -612,9 +612,13 @@ export class BlockComponent implements OnInit, OnDestroy {
     });
   }
 
-  onTxClick(event: TransactionStripped): void {
+  onTxClick(event: { tx: TransactionStripped, keyModifier: boolean }): void {
-    const url = new RelativeUrlPipe(this.stateService).transform(`/tx/${event.txid}`);
+    const url = new RelativeUrlPipe(this.stateService).transform(`/tx/${event.tx.txid}`);
-    this.router.navigate([url]);
+    if (!event.keyModifier) {
+      this.router.navigate([url]);
+    } else {
+      window.open(url, '_blank');
+    }
   }
 
   onTxHover(txid: string): void {

frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html

@@ -25,7 +25,7 @@
           </ng-template>
           <div [attr.data-cy]="'bitcoin-block-' + offset + '-index-' + i + '-fee-span'" class="fee-span"
             *ngIf="block?.extras?.feeRange; else emptyfeespan">
-            {{ block?.extras?.feeRange?.[1] | number:feeRounding }} - {{
+            {{ block?.extras?.feeRange?.[0] | number:feeRounding }} - {{
             block?.extras?.feeRange[block?.extras?.feeRange?.length - 1] | number:feeRounding }} <ng-container
               i18n="shared.sat-vbyte|sat/vB">sat/vB</ng-container>
           </div>

frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.ts

@@ -22,6 +22,7 @@ export class BlockchainBlocksComponent implements OnInit, OnChanges, OnDestroy {
   @Input() offset: number = 0;
   @Input() height: number = 0; // max height of blocks in chunk (dynamic blocks only)
   @Input() count: number = 8; // number of blocks in this chunk (dynamic blocks only)
+  @Input() dynamicBlockCount: number = 8; // number of blocks in the dynamic block chunk
   @Input() loadingTip: boolean = false;
   @Input() connected: boolean = true;
   
@@ -45,7 +46,6 @@ export class BlockchainBlocksComponent implements OnInit, OnChanges, OnDestroy {
   feeRounding = '1.0-0';
   arrowVisible = false;
   arrowLeftPx = 30;
-  blocksFilled = false;
   arrowTransition = '1s';
   showMiningInfo = false;
   timeLtrSubscription: Subscription;
@@ -96,14 +96,13 @@ export class BlockchainBlocksComponent implements OnInit, OnChanges, OnDestroy {
     this.tabHiddenSubscription = this.stateService.isTabHidden$.subscribe((tabHidden) => this.tabHidden = tabHidden);
     if (!this.static) {
       this.blocksSubscription = this.stateService.blocks$
-        .subscribe(([block, txConfirmed]) => {
+        .subscribe(([block, txConfirmed, batch]) => {
           if (this.blocks.some((b) => b.height === block.height)) {
             return;
           }
 
           if (this.blocks.length && block.height !== this.blocks[0].height + 1) {
             this.blocks = [];
-            this.blocksFilled = false;
           }
 
           this.blocks.unshift(block);
@@ -117,20 +116,18 @@ export class BlockchainBlocksComponent implements OnInit, OnChanges, OnDestroy {
           }
 
           this.blockStyles = [];
-          if (this.blocksFilled && block.height > this.chainTip) {
+          this.blocks.forEach((b, i) => {
-            this.blocks.forEach((b, i) => this.blockStyles.push(this.getStyleForBlock(b, i, i ? -155 : -205)));
+            if (i === 0 && !batch && block.height > this.chainTip) {
-            setTimeout(() => {
+              this.blockStyles.push(this.getStyleForBlock(b, i, -205));
-              this.blockStyles = [];
+              setTimeout(() => {
-              this.blocks.forEach((b, i) => this.blockStyles.push(this.getStyleForBlock(b, i)));
+                this.blockStyles = [];
-              this.cd.markForCheck();
+                this.blocks.forEach((b, i) => this.blockStyles.push(this.getStyleForBlock(b, i)));
-            }, 50);
+                this.cd.markForCheck();
-          } else {
+              }, 50);
-            this.blocks.forEach((b, i) => this.blockStyles.push(this.getStyleForBlock(b, i)));
+            } else {
-          }
+              this.blockStyles.push(this.getStyleForBlock(b, i));
-
+            }
-          if (this.blocks.length === this.dynamicBlocksAmount) {
+          });
-            this.blocksFilled = true;
-          }
 
           this.chainTip = Math.max(this.chainTip, block.height);
           this.cd.markForCheck();
@@ -160,7 +157,7 @@ export class BlockchainBlocksComponent implements OnInit, OnChanges, OnDestroy {
 
   ngOnChanges(changes: SimpleChanges): void {
     if (this.static) {
-      const animateSlide = changes.height && (changes.height.currentValue === changes.height.previousValue + 1);
+      const animateSlide = (changes.dynamicBlockCount && changes.dynamicBlockCount.previousValue != null) || (changes.height && (changes.height.currentValue === changes.height.previousValue + 1));
       this.updateStaticBlocks(animateSlide);
     }
   }
@@ -227,8 +224,8 @@ export class BlockchainBlocksComponent implements OnInit, OnChanges, OnDestroy {
         block = this.cacheService.getCachedBlock(height) || null;
       }
       this.blocks.push(block || {
-        placeholder: height < 0,
+        placeholder: !isNaN(height) && height < 0,
-        loading: height >= 0,
+        loading: isNaN(height) || height >= 0,
         id: '',
         height,
         version: 0,
@@ -311,6 +308,7 @@ export class BlockchainBlocksComponent implements OnInit, OnChanges, OnDestroy {
     return {
       left: addLeft + (155 * index) + 'px',
       background: "#2d3348",
+      transition: animateEnterFrom ? 'background 2s, transform 1s' : null,
     };
   }
 
@@ -318,6 +316,7 @@ export class BlockchainBlocksComponent implements OnInit, OnChanges, OnDestroy {
     const addLeft = animateEnterFrom || 0;
     return {
       left: addLeft + (155 * index) + 'px',
+      transition: animateEnterFrom ? 'background 2s, transform 1s' : null,
     };
   }
 
@@ -326,6 +325,7 @@ export class BlockchainBlocksComponent implements OnInit, OnChanges, OnDestroy {
 
     return {
       left: addLeft + 155 * this.emptyBlocks.indexOf(block) + 'px',
+      transition: animateEnterFrom ? 'background 2s, transform 1s' : null,
       background: "#2d3348",
     };
   }

frontend/src/app/components/blockchain/blockchain.component.html

@@ -6,7 +6,7 @@
         <app-mempool-blocks [hidden]="pageIndex > 0"></app-mempool-blocks>
         <app-blockchain-blocks [hidden]="pageIndex > 0"></app-blockchain-blocks>
         <ng-container *ngFor="let page of pages; trackBy: trackByPageFn">
-          <app-blockchain-blocks [static]="true" [offset]="page.offset" [height]="page.height" [count]="blocksPerPage" [loadingTip]="loadingTip" [connected]="connected"></app-blockchain-blocks>
+          <app-blockchain-blocks [static]="true" [offset]="page.offset" [height]="page.height" [dynamicBlockCount]="dynamicBlockCount" [count]="blocksPerPage" [loadingTip]="loadingTip" [connected]="connected"></app-blockchain-blocks>
         </ng-container>
       </div>
       <div id="divider" [hidden]="pageIndex > 0">

frontend/src/app/components/blockchain/blockchain.component.ts

@@ -12,6 +12,7 @@ export class BlockchainComponent implements OnInit, OnDestroy {
   @Input() pages: any[] = [];
   @Input() pageIndex: number;
   @Input() blocksPerPage: number = 8;
+  @Input() dynamicBlockCount: number = 8;
   @Input() minScrollWidth: number = 0;
 
   network: string;

frontend/src/app/components/blocks-list/blocks-list.component.html

@@ -26,7 +26,7 @@
       </thead>
       <tbody *ngIf="blocks$ | async as blocks; else skeleton" [style]="isLoading ? 'opacity: 0.75' : ''">
         <tr *ngFor="let block of blocks; let i= index; trackBy: trackByBlock">
-          <td class="text-left" [class]="widget ? 'widget' : ''">
+          <td class="height text-left" [class]="widget ? 'widget' : ''">
             <a [routerLink]="['/block' | relativeUrl, block.id]" [state]="{ data: { block: block } }">{{ block.height }}</a>
           </td>
           <td  *ngIf="indexingAvailable" class="pool text-left" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}">
@@ -89,7 +89,6 @@
               <span class="skeleton-loader" style="max-width: 75px"></span>
             </td>
             <td *ngIf="indexingAvailable" class="pool text-left" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}">
-              <img width="1" height="25" style="opacity: 0">
               <span class="skeleton-loader" style="max-width: 125px"></span>
             </td>
             <td class="timestamp" *ngIf="!widget" [class]="indexingAvailable ? '' : 'legacy'">
@@ -98,7 +97,7 @@
             <td class="mined" *ngIf="!widget" [class]="indexingAvailable ? '' : 'legacy'">
               <span class="skeleton-loader" style="max-width: 125px"></span>
             </td>
-            <td *ngIf="auditAvailable" class="health text-left" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}">
+            <td *ngIf="auditAvailable" class="health text-right" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}">
               <span class="skeleton-loader" style="max-width: 75px"></span>
             </td>
             <td *ngIf="indexingAvailable" class="reward text-right" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}">

frontend/src/app/components/blocks-list/blocks-list.component.scss

@@ -51,7 +51,12 @@ tr, td, th {
 .pool.widget {
   width: 40%;
   padding-left: 24px;
-  @media (max-width: 376px) {
+  @media (min-width: 768px) AND (max-width: 926px) {
+    padding-left: 0px;
+    width: 60%;
+  }
+  @media (max-width: 430px) {
+    padding-left: 0px;
     width: 60%;
   }
 }
@@ -59,6 +64,10 @@ tr, td, th {
   display: inline-block;
   vertical-align: text-top;
   padding-left: 10px;
+  overflow: hidden;
+  text-overflow: ellipsis;
+  white-space: nowrap;
+  max-width: 160px;
 }
 
 .height {
@@ -69,6 +78,12 @@ tr, td, th {
   @media (max-width: 576px) {
     width: 10%;
   }
+  @media (min-width: 768px) AND (max-width: 926px) {
+    width: 30%;
+  }
+  @media (max-width: 430px) {
+    width: 30%;
+  }
 }
 .height.legacy {
   width: 15%;
@@ -92,7 +107,7 @@ tr, td, th {
 
 .mined {
   width: 13%;
-  @media (max-width: 576px) {
+  @media (max-width: 730px) {
     display: none;
   }
 }
@@ -138,7 +153,7 @@ tr, td, th {
 
 .fees {
   width: 8%;
-  @media (max-width: 650px) {
+  @media (max-width: 820px) {
     display: none;
   }
 }
@@ -163,6 +178,16 @@ tr, td, th {
     width: 30%;
     padding-right: 0;
   }
+  @media (min-width: 768px) AND (max-width: 926px) {
+    overflow: hidden;
+    text-overflow: ellipsis;
+    max-width: 90px;
+  }
+  @media (max-width: 430px) {
+    overflow: hidden;
+    text-overflow: ellipsis;
+    max-width: 90px;
+  }
 }
 
 .size {
@@ -189,10 +214,10 @@ tr, td, th {
 
 .health {
   width: 10%;
-  @media (max-width: 1000px) {
+  @media (max-width: 1105px) {
     width: 13%;
   }
-  @media (max-width: 950px) {
+  @media (max-width: 560px) {
     display: none;
   }
 
@@ -202,7 +227,7 @@ tr, td, th {
 }
 .health.widget {
   width: 25%;
-  @media (max-width: 1000px) {
+  @media (max-width: 1105px) {
     display: none;
   }
   @media (max-width: 767px) {
@@ -242,4 +267,4 @@ tr, td, th {
 	vertical-align: middle;
 	max-width: 50vw;
   text-align: left;
-}
+}

frontend/src/app/components/difficulty-adjustments-table/difficulty-adjustments-table.component.html

@@ -2,17 +2,18 @@
   <table class="table latest-adjustments">
     <thead>
       <tr>
-        <th class="d-none d-md-block" i18n="block.height">Height</th>
+        <th class="" i18n="block.height">Height</th>
-        <th i18n="mining.adjusted" class="text-left">Adjusted</th>
+        <th class="date text-left" i18n="mining.adjusted">Adjusted</th>
-        <th i18n="mining.difficulty" class="text-right">Difficulty</th>
+        <th class="text-right" i18n="mining.difficulty">Difficulty</th>
-        <th i18n="mining.change" class="text-right">Change</th>
+        <th class="text-right" i18n="mining.change">Change</th>
       </tr>
     </thead>
     <tbody *ngIf="(hashrateObservable$ | async) as data">
       <tr *ngFor="let diffChange of data">
-        <td class="d-none d-md-block"><a [routerLink]="['/block' | relativeUrl, diffChange.height]">{{ diffChange.height
+        <td class="">
-            }}</a></td>
+          <a [routerLink]="['/block' | relativeUrl, diffChange.height]">{{ diffChange.height }}</a>
-        <td class="text-left">
+        </td>
+        <td class="date text-left">
           <app-time kind="since" [time]="diffChange.timestamp" [fastRender]="true"></app-time>
         </td>
         <td class="text-right">{{ diffChange.difficultyShorten }}</td>
@@ -23,8 +24,8 @@
     </tbody>
     <tbody *ngIf="isLoading">
       <tr *ngFor="let item of [1,2,3,4,5,6]">
-        <td class="d-none d-md-block w-75"><span class="skeleton-loader"></span></td>
+        <td class=""><span class="skeleton-loader"></span></td>
-        <td class="text-left"><span class="skeleton-loader w-75"></span></td>
+        <td class="date text-left"><span class="skeleton-loader w-75"></span></td>
         <td class="text-right"><span class="skeleton-loader w-75"></span></td>
         <td class="text-right"><span class="skeleton-loader w-75"></span></td>
       </tr>

frontend/src/app/components/difficulty-adjustments-table/difficulty-adjustments-table.component.scss

@@ -17,3 +17,12 @@
     }
   }
 }
+
+.date {
+  @media (min-width: 767px) AND (max-width: 991px) {
+    display: none;
+  }
+  @media (max-width: 500px) {
+    display: none;
+  }
+}

frontend/src/app/components/difficulty-mining/difficulty-mining.component.html

@@ -47,7 +47,8 @@
           </div>
         </div>
         <div class="item" *ngIf="showHalving">
-          <h5 class="card-title" i18n="difficulty-box.next-halving">Next Halving</h5>
+          <h5 class="card-title" i18n="difficulty-box.next-halving" i18n-ngbTooltip="difficulty-box.next-halving"
+          ngbTooltip="Next Halving" placement="bottom" #averagefee [disableTooltip]="!isEllipsisActive(averagefee)">Next Halving</h5>
           <div class="card-text">
             <ng-container *ngTemplateOutlet="epochData.blocksUntilHalving === 1 ? blocksSingular : blocksPlural; context: {$implicit: epochData.blocksUntilHalving }"></ng-container>
             <ng-template #blocksPlural let-i i18n="shared.blocks">{{ i }} <span class="shared-block">blocks</span></ng-template>
@@ -77,7 +78,7 @@
       </div>
     </div>
     <div class="item">
-      <h5 class="card-title" i18n="difficulty-box.current-period">Current Period</h5>
+      <h5 class="card-title" i18n="difficulty-box.next-halving">Next Halving</h5>
       <div class="card-text">
         <div class="skeleton-loader"></div>
         <div class="skeleton-loader"></div>

frontend/src/app/components/difficulty-mining/difficulty-mining.component.scss

@@ -10,6 +10,7 @@
   .item {
     padding: 0 5px;
     width: 100%;
+    max-width: 150px;
     &:nth-child(1) {
       display: none;
       @media (min-width: 485px) {
@@ -85,6 +86,9 @@
 .card-title {
   color: #4a68b9;
   font-size: 1rem;
+  overflow: hidden;
+  text-overflow: ellipsis;
+  white-space: nowrap;
 }
 
 .progress {
@@ -152,4 +156,5 @@
 
 .symbol {
   font-size: 13px;
+  white-space: nowrap;
 }

frontend/src/app/components/difficulty-mining/difficulty-mining.component.ts

@@ -83,4 +83,8 @@ export class DifficultyMiningComponent implements OnInit {
       })
     );
   }
+
+  isEllipsisActive(e): boolean {
+    return (e.offsetWidth < e.scrollWidth);
+  }
 }

frontend/src/app/components/difficulty/difficulty.component.html

@@ -42,7 +42,7 @@
             <div class="symbol" i18n="difficulty-box.average-block-time">Average block time</div>
           </div>
           <div class="item">
-            <div *ngIf="epochData.remainingBlocks < 1870; else recentlyAdjusted" class="card-text" [ngStyle]="{'color': epochData.colorAdjustments}">
+            <div *ngIf="epochData.remainingBlocks < 1870; else recentlyAdjusted" class="card-text bigger" [ngStyle]="{'color': epochData.colorAdjustments}">
               <span *ngIf="epochData.change > 0; else arrowDownDifficulty" >
                 <fa-icon class="retarget-sign" [icon]="['fas', 'caret-up']" [fixedWidth]="true"></fa-icon>
               </span>

frontend/src/app/components/difficulty/difficulty.component.scss

@@ -30,9 +30,14 @@
     }
   }
   .card-text {
-    font-size: 20px;
+    font-size: 18px;
     margin: auto;
     position: relative;
+    margin-bottom: 0.2rem;
+    &.bigger {
+      font-size: 20px;
+      margin-bottom: 0;
+    }
   }
 }
 
@@ -160,6 +165,7 @@
 
 .symbol {
   font-size: 13px;
+  white-space: nowrap;
 }
 
 .epoch-progress {

frontend/src/app/components/liquid-master-page/liquid-master-page.component.html

@@ -90,6 +90,8 @@
 </nav>
 </header>
 
+<app-testnet-alert *ngIf="network.val === 'liquidtestnet'"></app-testnet-alert>
+
 <br />
 
 <router-outlet></router-outlet>

frontend/src/app/components/master-page/master-page.component.html

@@ -62,6 +62,8 @@
 </nav>
 </header>
 
+<app-testnet-alert *ngIf="network.val === 'testnet' || network.val === 'signet'"></app-testnet-alert>
+
 <br />
 
 <router-outlet></router-outlet>

frontend/src/app/components/master-page/master-page.component.scss

@@ -192,4 +192,4 @@ nav {
     margin: 33px 0px 0px -19px;
     font-size: 7px;
   }
-}
+}

frontend/src/app/components/master-page/master-page.component.ts

@@ -1,4 +1,4 @@
-import { Component, Inject, OnInit } from '@angular/core';
+import { Component, OnInit } from '@angular/core';
 import { Env, StateService } from '../../services/state.service';
 import { Observable, merge, of } from 'rxjs';
 import { LanguageService } from '../../services/language.service';

frontend/src/app/components/mempool-block-overview/mempool-block-overview.component.ts

@@ -107,8 +107,12 @@ export class MempoolBlockOverviewComponent implements OnInit, OnDestroy, OnChang
     this.isLoading$.next(false);
   }
 
-  onTxClick(event: TransactionStripped): void {
+  onTxClick(event: { tx: TransactionStripped, keyModifier: boolean }): void {
-    const url = new RelativeUrlPipe(this.stateService).transform(`/tx/${event.txid}`);
+    const url = new RelativeUrlPipe(this.stateService).transform(`/tx/${event.tx.txid}`);
-    this.router.navigate([url]);
+    if (!event.keyModifier) {
+      this.router.navigate([url]);
+    } else {
+      window.open(url, '_blank');
+    }
   }
 }

frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts

@@ -105,7 +105,6 @@ export class MempoolBlocksComponent implements OnInit, OnDestroy {
     });
     this.reduceMempoolBlocksToFitScreen(this.mempoolBlocks);
     this.stateService.isTabHidden$.subscribe((tabHidden) => this.tabHidden = tabHidden);
-    this.loadingBlocks$ = this.stateService.isLoadingWebSocket$;
 
     this.mempoolBlocks$ = merge(
       of(true),
@@ -141,6 +140,13 @@ export class MempoolBlocksComponent implements OnInit, OnDestroy {
         })
       );
 
+    this.loadingBlocks$ = combineLatest([
+      this.stateService.isLoadingWebSocket$,
+      this.mempoolBlocks$
+    ]).pipe(map(([loading, mempoolBlocks]) => {
+      return loading || !mempoolBlocks.length;
+    }));
+
     this.difficultyAdjustments$ = this.stateService.difficultyAdjustment$
       .pipe(
         map((da) => {

frontend/src/app/components/pool-ranking/pool-ranking.component.html

@@ -32,7 +32,7 @@
   </div>
 
   <div class="card-header" *ngIf="!widget">
-    <div class="d-flex d-md-block align-items-baseline">
+    <div class="d-flex d-md-table-cell align-items-baseline">
       <span i18n="mining.pools">Pools Ranking</span>
       <button class="btn p-0 pl-2" style="margin: 0 0 4px 0px" (click)="onSaveChart()">
         <fa-icon [icon]="['fas', 'download']" [fixedWidth]="true"></fa-icon>
@@ -87,19 +87,19 @@
     <table *ngIf="widget === false" class="table table-borderless text-center pools-table">
       <thead>
         <tr>
-          <th class="d-none d-md-block" i18n="mining.rank">Rank</th>
+          <th class="d-none d-md-table-cell" i18n="mining.rank">Rank</th>
           <th class=""></th>
           <th class="" i18n="mining.pool-name">Pool</th>
           <th class="" *ngIf="this.miningWindowPreference === '24h'" i18n="mining.hashrate">Hashrate</th>
           <th class="" i18n="master-page.blocks">Blocks</th>
           <th *ngIf="auditAvailable" class="health text-right widget" i18n="latest-blocks.avg_health"
             i18n-ngbTooltip="latest-blocks.avg_health" ngbTooltip="Avg Health" placement="bottom" #health [disableTooltip]="!isEllipsisActive(health)">Avg Health</th>
-          <th class="d-none d-md-block" i18n="mining.empty-blocks">Empty blocks</th>
+          <th class="d-none d-md-table-cell" i18n="mining.empty-blocks">Empty blocks</th>
         </tr>
       </thead>
       <tbody [attr.data-cy]="'pools-table'" *ngIf="(miningStatsObservable$ | async) as miningStats">
         <tr *ngFor="let pool of miningStats.pools">
-          <td class="d-none d-md-block">{{ pool.rank }}</td>
+          <td class="d-none d-md-table-cell">{{ pool.rank }}</td>
           <td class="text-right">
             <img width="25" height="25" src="{{ pool.logo }}" [alt]="pool.name + ' mining pool logo'" onError="this.src = '/resources/mining-pools/default.svg'">
           </td>
@@ -107,7 +107,7 @@
           <td class="" *ngIf="this.miningWindowPreference === '24h'">{{ pool.lastEstimatedHashrate }} {{
             miningStats.miningUnits.hashrateUnit }}</td>
           <td class="d-flex justify-content-center">
-            {{ pool.blockCount }}<span class="d-none d-md-block">&nbsp;({{ pool.share }}%)</span>
+            {{ pool.blockCount }}<span class="d-none d-md-table-cell">&nbsp;({{ pool.share }}%)</span>
           </td>
           <td *ngIf="auditAvailable" class="health text-right" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}">
             <a
@@ -121,16 +121,16 @@
               <span class="health-badge badge badge-secondary" i18n="unknown">Unknown</span>
             </ng-template>
           </td>
-          <td class="d-none d-md-block">{{ pool.emptyBlocks }} ({{ pool.emptyBlockRatio }}%)</td>
+          <td class="d-none d-md-table-cell">{{ pool.emptyBlocks }} ({{ pool.emptyBlockRatio }}%)</td>
         </tr>
         <tr style="border-top: 1px solid #555">
-          <td class="d-none d-md-block"></td>
+          <td class="d-none d-md-table-cell"></td>
           <td class="text-right"></td>
           <td class=""><b i18n="mining.all-miners">All miners</b></td>
           <td class="" *ngIf="this.miningWindowPreference === '24h'"><b>{{ miningStats.lastEstimatedHashrate}} {{
               miningStats.miningUnits.hashrateUnit }}</b></td>
           <td class=""><b>{{ miningStats.blockCount }}</b></td>
-          <td class="d-none d-md-block"><b>{{ miningStats.totalEmptyBlock }} ({{ miningStats.totalEmptyBlockRatio
+          <td class="d-none d-md-table-cell"><b>{{ miningStats.totalEmptyBlock }} ({{ miningStats.totalEmptyBlockRatio
               }}%)</b></td>
         </tr>
       </tbody>

frontend/src/app/components/pool/pool.component.html

@@ -94,13 +94,13 @@
                       <tr>
                         <th scope="col" class="block-count-title text-center" style="width: 33%" i18n="mining.reward">Reward</th>
                         <th scope="col" class="block-count-title text-center" style="width: 33%" i18n="mining.hashrate">Hashrate (24h)</th>
-                        <th scope="col" class="block-count-title text-center" style="width: 33%" i18n="latest-blocks.avg_health">Avg Health</th>
+                        <th scope="col" class="block-count-title text-center" style="width: 33%" i18n="latest-blocks.avg_health" *ngIf="auditAvailable">Avg Health</th>
                       </tr>
                     </thead>
                     <tbody>
                       <td class="text-center"><app-amount [satoshis]="poolStats.totalReward" digitsInfo="1.0-0" [noFiat]="true"></app-amount></td>
                       <td class="text-center">{{ poolStats.estimatedHashrate | amountShortener : 1 : 'H/s' }}</td>
-                      <td class="text-center"><span class="health-badge badge" [class.badge-success]="poolStats.avgBlockHealth >= 99"
+                      <td class="text-center" *ngIf="auditAvailable; else emptyTd"><span class="health-badge badge" [class.badge-success]="poolStats.avgBlockHealth >= 99"
                           [class.badge-warning]="poolStats.avgBlockHealth >= 75 && poolStats.avgBlockHealth < 99" [class.badge-danger]="poolStats.avgBlockHealth < 75"
                           *ngIf="poolStats.avgBlockHealth != null; else nullHealth">{{ poolStats.avgBlockHealth }}%</span>
                           <ng-template #nullHealth>
@@ -119,13 +119,13 @@
                       <tr>
                         <th scope="col" class="block-count-title text-center" style="width: 33%" i18n="mining.reward">Reward</th>
                         <th scope="col" class="block-count-title text-center" style="width: 33%" i18n="mining.hashrate">Hashrate (24h)</th>
-                        <th scope="col" class="block-count-title text-center" style="width: 33%" i18n="latest-blocks.avg_health">Avg Health</th>
+                        <th scope="col" class="block-count-title text-center" style="width: 33%" i18n="latest-blocks.avg_health" *ngIf="auditAvailable">Avg Health</th>
                       </tr>
                     </thead>
                     <tbody>
                       <td class="text-center"><app-amount [satoshis]="poolStats.totalReward" digitsInfo="1.0-0" [noFiat]="true"></app-amount></td>
                       <td class="text-center">{{ poolStats.estimatedHashrate | amountShortener : 1 : 'H/s' }}</td>
-                      <td class="text-center"><span class="health-badge badge" [class.badge-success]="poolStats.avgBlockHealth >= 99"
+                      <td *ngIf="auditAvailable; else emptyTd" class="text-center"><span class="health-badge badge" [class.badge-success]="poolStats.avgBlockHealth >= 99"
                           [class.badge-warning]="poolStats.avgBlockHealth >= 75 && poolStats.avgBlockHealth < 99" [class.badge-danger]="poolStats.avgBlockHealth < 75"
                           *ngIf="poolStats.avgBlockHealth != null; else nullHealth">{{ poolStats.avgBlockHealth }}%</span>
                           <ng-template #nullHealth>
@@ -384,7 +384,7 @@
                       <tr>
                         <th scope="col" class="block-count-title text-center" style="width: 33%" i18n="mining.total-reward">Reward</th>
                         <th scope="col" class="block-count-title text-center" style="width: 33%" i18n="mining.estimated">Hashrate (24h)</th>
-                        <th scope="col" class="block-count-title text-center" style="width: 33%" i18n="mining.luck">Avg Health</th>
+                        <th scope="col" class="block-count-title text-center" style="width: 33%" i18n="mining.luck" *ngIf="auditAvailable">Avg Health</th>
                       </tr>
                     </thead>
                     <tbody>
@@ -394,7 +394,7 @@
                       <td class="text-center">
                         <div class="skeleton-loader data"></div>
                       </td>
-                      <td class="text-center">
+                      <td class="text-center" *ngIf="auditAvailable">
                         <div class="skeleton-loader data"></div>
                       </td>
                     </tbody>
@@ -409,7 +409,7 @@
                       <tr>
                         <th scope="col" class="block-count-title text-center" style="width: 33%" i18n="mining.total-reward">Reward</th>
                         <th scope="col" class="block-count-title text-center" style="width: 33%" i18n="mining.estimated">Hashrate (24h)</th>
-                        <th scope="col" class="block-count-title text-center" style="width: 33%" i18n="mining.luck">Avg Health</th>
+                        <th scope="col" class="block-count-title text-center" style="width: 33%" i18n="mining.luck" *ngIf="auditAvailable">Avg Health</th>
                       </tr>
                     </thead>
                     <tbody>
@@ -419,7 +419,7 @@
                       <td class="text-center">
                         <div class="skeleton-loader data"></div>
                       </td>
-                      <td class="text-center">
+                      <td class="text-center" *ngIf="auditAvailable">
                         <div class="skeleton-loader data"></div>
                       </td>
                     </tbody>
@@ -485,4 +485,8 @@
       </div>
     </div>
   </div>
+</ng-template>
+
+<ng-template #emptyTd>
+  <td class="text-center"></td>
 </ng-template>

frontend/src/app/components/reward-stats/reward-stats.component.html

@@ -50,14 +50,14 @@
       </div>
     </div>
     <div class="item">
-      <h5 class="card-title" i18n="mining.rewards-per-tx">Reward Per Tx</h5>
+      <h5 class="card-title" i18n="mining.fees-per-block">Avg Block Fees</h5>
       <div class="card-text">
         <div class="skeleton-loader"></div>
         <div class="skeleton-loader"></div>
       </div>
     </div>
     <div class="item">
-      <h5 class="card-title" i18n="mining.average-fee">Reward Per Tx</h5>
+      <h5 class="card-title" i18n="mining.average-fee">Avg Tx Fee</h5>
       <div class="card-text">
         <div class="skeleton-loader"></div>
         <div class="skeleton-loader"></div>

frontend/src/app/components/start/start.component.html

@@ -16,7 +16,7 @@
     (dragstart)="onDragStart($event)"
     (scroll)="onScroll($event)"
   >
-    <app-blockchain [pageIndex]="pageIndex" [pages]="pages" [blocksPerPage]="blocksPerPage" [minScrollWidth]="minScrollWidth"></app-blockchain>
+    <app-blockchain [pageIndex]="pageIndex" [pages]="pages" [dynamicBlockCount]="dynamicBlocksAmount" [blocksPerPage]="blocksPerPage" [minScrollWidth]="minScrollWidth"></app-blockchain>
   </div>
   <div class="reset-scroll" [class.hidden]="pageIndex === 0" (click)="resetScroll()">
     <fa-icon [icon]="['fas', 'circle-left']" [fixedWidth]="true"></fa-icon>

frontend/src/app/components/start/start.component.ts

@@ -2,6 +2,7 @@ import { Component, ElementRef, HostListener, OnInit, OnDestroy, ViewChild } fro
 import { Subscription } from 'rxjs';
 import { StateService } from '../../services/state.service';
 import { specialBlocks } from '../../app.constants';
+import { BlockExtended } from '../../interfaces/node-api.interface';
 
 @Component({
   selector: 'app-start',
@@ -29,7 +30,8 @@ export class StartComponent implements OnInit, OnDestroy {
   isMobile: boolean = false;
   isiOS: boolean = false;
   blockWidth = 155;
-  dynamicBlocksAmount: number = 8;
+  blocks: BlockExtended[] = [];
+  dynamicBlocksAmount: number;
   blockCount: number = 0;
   blocksPerPage: number = 1;
   pageWidth: number;
@@ -49,15 +51,28 @@ export class StartComponent implements OnInit, OnDestroy {
   }
 
   ngOnInit() {
-    this.firstPageWidth = 40 + (this.blockWidth * this.dynamicBlocksAmount);
+    this.firstPageWidth = 40 + (this.blockWidth * (this.dynamicBlocksAmount || 0));
-    this.blockCounterSubscription = this.stateService.blocks$.subscribe(() => {
+
-      this.blockCount++;
+    this.blockCounterSubscription = this.stateService.blocks$.subscribe(([block, txConfirmed, batch]) => {
-      this.dynamicBlocksAmount = Math.min(this.blockCount, this.stateService.env.KEEP_BLOCKS_AMOUNT, 8);
+      if (this.blocks.some((b) => b.height === block.height)) {
+        return;
+      }
+
+      if (this.blocks.length && block.height !== this.blocks[0].height + 1) {
+        this.blocks = [];
+      }
+
+      this.blocks.unshift(block);
+      this.blocks = this.blocks.slice(0, Math.min(8, this.stateService.env.KEEP_BLOCKS_AMOUNT));
+
+      this.dynamicBlocksAmount = this.blocks.length;
       this.firstPageWidth = 40 + (this.blockWidth * this.dynamicBlocksAmount);
-      if (this.blockCount <= Math.min(8, this.stateService.env.KEEP_BLOCKS_AMOUNT)) {
+
+      if (this.blocks.length <= Math.min(8, this.stateService.env.KEEP_BLOCKS_AMOUNT)) {
         this.onResize();
       }
     });
+
     this.onResize();
     this.updatePages();
     this.timeLtrSubscription = this.stateService.timeLtr.subscribe((ltr) => {

frontend/src/app/components/transaction/transaction-preview.component.html

@@ -8,10 +8,7 @@
     </div>
     <div *ngIf="network !== 'liquid' && network !== 'liquidtestnet'" class="features">
       <app-tx-features [tx]="tx"></app-tx-features>
-      <span *ngIf="cpfpInfo && (cpfpInfo.bestDescendant || cpfpInfo.descendants.length)" class="badge badge-primary mr-1">
+      <span *ngIf="cpfpInfo && (cpfpInfo?.bestDescendant || cpfpInfo?.descendants?.length || cpfpInfo?.ancestors?.length)" class="badge badge-primary ml-1 mr-1">
-        CPFP
-      </span>
-      <span *ngIf="cpfpInfo && !cpfpInfo.bestDescendant && !cpfpInfo.descendants.length && cpfpInfo.ancestors.length" class="badge badge-info mr-1">
         CPFP
       </span>
     </div>

frontend/src/app/components/transactions-list/transactions-list.component.html

@@ -1,3 +1,5 @@
+<div infiniteScroll [alwaysCallback]="true" [infiniteScrollDistance]="2" [infiniteScrollUpDistance]="1.5" [infiniteScrollThrottle]="50" (scrolled)="onScroll()">
+
 <ng-container *ngFor="let tx of transactions; let i = index; trackBy: trackByFn">
   <div *ngIf="!transactionPage" class="header-bg box tx-page-container">
     <a class="tx-link" [routerLink]="['/tx/' | relativeUrl, tx.txid]">
@@ -11,7 +13,7 @@
     </div>
   </div>
 
-  <div class="header-bg box" infiniteScroll [alwaysCallback]="true" [infiniteScrollDistance]="2" [infiniteScrollUpDistance]="1.5" [infiniteScrollThrottle]="50" (scrolled)="onScroll()" [attr.data-cy]="'tx-' + i">
+  <div class="header-bg box" [attr.data-cy]="'tx-' + i">
 
     <div *ngIf="errorUnblinded" class="error-unblinded">{{ errorUnblinded }}</div>
     <div class="row">
@@ -321,6 +323,8 @@
 
 </ng-container>
 
+</div>
+
 <ng-template #assetBox let-item>
   {{ item.value / pow(10, assetsMinimal[item.asset][3]) | number: '1.' + assetsMinimal[item.asset][3] + '-' + assetsMinimal[item.asset][3] }} {{ assetsMinimal[item.asset][1] }}
   <br />

frontend/src/app/components/transactions-list/transactions-list.component.ts

@@ -182,14 +182,7 @@ export class TransactionsListComponent implements OnInit, OnChanges {
   }
 
   onScroll(): void {
-    const scrollHeight = document.body.scrollHeight;
+    this.loadMore.emit();
-    const scrollTop = document.documentElement.scrollTop;
-    if (scrollHeight > 0){
-      const percentageScrolled = scrollTop * 100 / scrollHeight;
-      if (percentageScrolled > 70){
-        this.loadMore.emit();
-      }
-    }
   }
 
   haveBlindedOutputValues(tx: Transaction): boolean {

frontend/src/app/components/tx-bowtie-graph-tooltip/tx-bowtie-graph-tooltip.component.html

@@ -29,7 +29,7 @@
   <ng-template #pegout>
     <ng-container *ngIf="line.pegout; else normal">
       <p *ngIf="!isConnector">Peg Out</p>
-      <p *ngIf="line.value != null"><app-amount [satoshis]="line.value"></app-amount></p>
+      <p *ngIf="line.displayValue != null"><app-amount [satoshis]="line.displayValue"></app-amount></p>
       <p class="address">
         <app-truncate [text]="line.pegout"></app-truncate>
       </p>
@@ -55,18 +55,18 @@
             <p *ngSwitchCase="'output'"><span i18n="transaction.input">Input</span>&nbsp; #{{ line.vin + 1 }}</p>
           </ng-container>
       </ng-container>
-      <p *ngIf="line.value == null && line.confidential" i18n="shared.confidential">Confidential</p>
+      <p *ngIf="line.displayValue == null && line.confidential" i18n="shared.confidential">Confidential</p>
-      <p *ngIf="line.value != null">
+      <p *ngIf="line.displayValue != null">
         <ng-template [ngIf]="line.asset && line.asset !== nativeAssetId" [ngIfElse]="defaultOutput">
           <div *ngIf="assetsMinimal && assetsMinimal[line.asset] else assetNotFound">
             <ng-container *ngTemplateOutlet="assetBox; context:{ $implicit: line }"></ng-container>
           </div>
           <ng-template #assetNotFound>
-            {{ line.value }} <span class="symbol">{{ line.asset | slice : 0 : 7 }}</span>
+            {{ line.displayValue }} <span class="symbol">{{ line.asset | slice : 0 : 7 }}</span>
           </ng-template>
         </ng-template>
         <ng-template #defaultOutput>
-          <app-amount [blockConversion]="blockConversion" [satoshis]="line.value"></app-amount>
+          <app-amount [blockConversion]="blockConversion" [satoshis]="line.displayValue"></app-amount>
         </ng-template>
       </p>
       <p *ngIf="line.type !== 'fee' && line.address" class="address">
@@ -76,5 +76,5 @@
 </div>
 
 <ng-template #assetBox let-item>
-  {{ item.value / pow(10, assetsMinimal[item.asset][3]) | number: '1.' + assetsMinimal[item.asset][3] + '-' + assetsMinimal[item.asset][3] }} <span class="symbol">{{ assetsMinimal[item.asset][1] }}</span>
+  {{ item.displayValue / pow(10, assetsMinimal[item.asset][3]) | number: '1.' + assetsMinimal[item.asset][3] + '-' + assetsMinimal[item.asset][3] }} <span class="symbol">{{ assetsMinimal[item.asset][1] }}</span>
 </ng-template>

frontend/src/app/components/tx-bowtie-graph-tooltip/tx-bowtie-graph-tooltip.component.ts

@@ -7,6 +7,7 @@ import { environment } from '../../../environments/environment';
 interface Xput {
   type: 'input' | 'output' | 'fee';
   value?: number;
+  displayValue?: number;
   index?: number;
   txid?: string;
   vin?: number;

frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.ts

@@ -1,12 +1,13 @@
 import { Component, OnInit, Input, OnChanges, HostListener, Inject, LOCALE_ID } from '@angular/core';
 import { StateService } from '../../services/state.service';
-import { Outspend, Transaction } from '../../interfaces/electrs.interface';
+import { Outspend, Transaction, Vin, Vout } from '../../interfaces/electrs.interface';
 import { Router } from '@angular/router';
 import { ReplaySubject, merge, Subscription, of } from 'rxjs';
 import { tap, switchMap } from 'rxjs/operators';
 import { ApiService } from '../../services/api.service';
 import { RelativeUrlPipe } from '../../shared/pipes/relative-url/relative-url.pipe';
 import { AssetsService } from '../../services/assets.service';
+import { environment } from '../../../environments/environment';
 
 interface SvgLine {
   path: string;
@@ -20,6 +21,7 @@ interface SvgLine {
 interface Xput {
   type: 'input' | 'output' | 'fee';
   value?: number;
+  displayValue?: number;
   index?: number;
   txid?: string;
   vin?: number;
@@ -74,6 +76,7 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges {
   zeroValueThickness = 20;
   hasLine: boolean;
   assetsMinimal: any;
+  nativeAssetId = this.stateService.network === 'liquidtestnet' ? environment.nativeTestAssetId : environment.nativeAssetId;
 
   outspendsSubscription: Subscription;
   refreshOutspends$: ReplaySubject<string> = new ReplaySubject();
@@ -167,7 +170,8 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges {
     let voutWithFee = this.tx.vout.map((v, i) => {
       return {
         type: v.scriptpubkey_type === 'fee' ? 'fee' : 'output',
-        value: v?.value,
+        value: this.getOutputValue(v),
+        displayValue: v?.value,
         address: v?.scriptpubkey_address || v?.scriptpubkey_type?.toUpperCase(),
         index: i,
         pegout: v?.pegout?.scriptpubkey_address,
@@ -185,7 +189,8 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges {
     let truncatedInputs = this.tx.vin.map((v, i) => {
       return {
         type: 'input',
-        value: v?.prevout?.value || (v?.is_coinbase && !totalValue ? 0 : undefined),
+        value: (v?.is_coinbase && !totalValue ? 0 : this.getInputValue(v)),
+        displayValue: v?.prevout?.value,
         txid: v.txid,
         vout: v.vout,
         address: v?.prevout?.scriptpubkey_address || v?.prevout?.scriptpubkey_type?.toUpperCase(),
@@ -229,14 +234,14 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges {
   }
 
   calcTotalValue(tx: Transaction): number {
-    const totalOutput = this.tx.vout.reduce((acc, v) => (v.value == null ? 0 : v.value) + acc, 0);
+    let totalOutput = this.tx.vout.reduce((acc, v) => (this.getOutputValue(v) || 0) + acc, 0);
     // simple sum of outputs + fee for bitcoin
     if (!this.isLiquid) {
       return this.tx.fee ? totalOutput + this.tx.fee : totalOutput;
     } else {
-      const totalInput = this.tx.vin.reduce((acc, v) => (v?.prevout?.value == null ? 0 : v.prevout.value) + acc, 0);
+      const totalInput = this.tx.vin.reduce((acc, v) => (this.getInputValue(v) || 0) + acc, 0);
-      const confidentialInputCount = this.tx.vin.reduce((acc, v) => acc + (v?.prevout?.value == null ? 1 : 0), 0);
+      const confidentialInputCount = this.tx.vin.reduce((acc, v) => acc + (this.isUnknownInputValue(v) ? 1 : 0), 0);
-      const confidentialOutputCount = this.tx.vout.reduce((acc, v) => acc + (v.value == null ? 1 : 0), 0);
+      const confidentialOutputCount = this.tx.vout.reduce((acc, v) => acc + (this.isUnknownOutputValue(v) ? 1 : 0), 0);
 
       // if there are unknowns on both sides, the total is indeterminate, so we'll just fudge it
       if (confidentialInputCount && confidentialOutputCount) {
@@ -456,6 +461,34 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges {
     }
   }
 
+  getOutputValue(v: Vout): number | void {
+    if (!v) {
+      return null;
+    } else if (this.isLiquid && v.asset !== this.nativeAssetId) {
+      return null;
+    } else {
+      return v.value;
+    }
+  }
+
+  getInputValue(v: Vin): number | void {
+    if (!v?.prevout) {
+      return null;
+    } else if (this.isLiquid && v.prevout.asset !== this.nativeAssetId) {
+      return null;
+    } else {
+      return v.prevout.value;
+    }
+  }
+
+  isUnknownInputValue(v: Vin): boolean {
+    return v?.prevout?.value == null || this.isLiquid && v?.prevout?.asset !== this.nativeAssetId;
+  }
+
+  isUnknownOutputValue(v: Vout): boolean {
+    return v?.value == null || this.isLiquid && v?.asset !== this.nativeAssetId;
+  }
+
   @HostListener('pointermove', ['$event'])
   onPointerMove(event) {
     if (this.dir === 'rtl') {

frontend/src/app/docs/api-docs/api-docs-data.ts

@@ -8863,7 +8863,7 @@ export const faqData = [
     type: "endpoint",
     category: "advanced",
     showConditions: bitcoinNetworks,
-    fragment: "how-big-is-mempool-used-by-mempool.space",
+    fragment: "how-big-is-mempool-used-by-mempool-space",
     title: "How big is the mempool used by mempool.space?",
     options: { officialOnly: true },
   },

frontend/src/app/docs/api-docs/api-docs.component.html

@@ -207,7 +207,7 @@
   <p>When a Bitcoin transaction is made, it is stored in a Bitcoin node's mempool before it is confirmed into a block. When the rate of incoming transactions exceeds the rate transactions are confirmed, the mempool grows in size.</p><p>By default, Bitcoin Core allocates 300MB of memory for its mempool, so when a node's mempool grows big enough to use all 300MB of allocated memory, we say it's "full".</p><p>Once a node's mempool is using all of its allocated memory, it will start rejecting new transactions below a certain feerate threshold—so when this is the case, be extra sure to set a feerate that (at a minimum) exceeds that threshold. The current threshold feerate (and memory usage) are displayed right on Mempool's front page.</p>
 </ng-template>
 
-<ng-template type="how-big-is-mempool-used-by-mempool.space">
+<ng-template type="how-big-is-mempool-used-by-mempool-space">
   <p>mempool.space uses multiple Bitcoin nodes to obtain data: some with the default 300MB mempool memory limit (call these Small Nodes) and others with a much larger mempool memory limit (call these Big Nodes).</p>
   <p>Many nodes on the Bitcoin network are configured to run with the default 300MB mempool memory setting. When all 300MB of memory are used up, such nodes will reject transactions below a certain threshold feerate. Running Small Nodes allows mempool.space to tell you what this threshold feerate is—this is the "Purging" feerate that shows on the front page when mempools are full, which you can use to be reasonably sure that your transaction will be widely propagated.</p>
     <p>Big Node mempools are so big that they don't need to reject (or purge) transactions. Such nodes allow for mempool.space to provide you with information on any pending transaction it has received—no matter how congested the mempool is, and no matter how low-feerate or low-priority the transaction is.</p>

frontend/src/app/fiat/fiat.component.html

@@ -1,4 +1,4 @@
-<span class="green-color" *ngIf="blockConversion; else noblockconversion">
+<span [class]="colorClass" *ngIf="blockConversion; else noblockconversion">
   {{
     (
       (blockConversion.price[currency] > -1 ? blockConversion.price[currency] : null) ??
@@ -8,7 +8,7 @@
 </span>
 
 <ng-template #noblockconversion>
-  <span class="green-color" *ngIf="(conversions$ | async) as conversions">
+  <span [class]="colorClass" *ngIf="(conversions$ | async) as conversions">
     {{ (conversions[currency] > -1 ? conversions[currency] : 0) * value / 100000000 | fiatCurrency : digitsInfo : currency }}
   </span>
 </ng-template>

frontend/src/app/fiat/fiat.component.scss

@@ -1,3 +0,0 @@
-.green-color {
-  color: #3bcc49;
-}

frontend/src/app/fiat/fiat.component.ts

@@ -6,7 +6,7 @@ import { StateService } from '../services/state.service';
 @Component({
   selector: 'app-fiat',
   templateUrl: './fiat.component.html',
-  styleUrls: ['./fiat.component.scss'],
+  styleUrls: [],
   changeDetection: ChangeDetectionStrategy.OnPush,
 })
 export class FiatComponent implements OnInit, OnDestroy {
@@ -17,6 +17,7 @@ export class FiatComponent implements OnInit, OnDestroy {
   @Input() value: number;
   @Input() digitsInfo = '1.2-2';
   @Input() blockConversion: Price;
+  @Input() colorClass = 'green-color';
 
   constructor(
     private stateService: StateService,

frontend/src/app/lightning/node-fee-chart/node-fee-chart.component.html

@@ -2,6 +2,6 @@
   <h2 i18n="lightning.node-fee-distribution">Fee distribution</h2>
   <div class="chart" echarts [initOpts]="chartInitOptions" [options]="chartOptions" (chartInit)="onChartInit($event)"></div>
   <div class="text-center loadingGraphs" *ngIf="isLoading">
-    <div class="spinner-border text-light"></div>d
+    <div class="spinner-border text-light"></div>
   </div>
 </div>

frontend/src/app/lightning/node-fee-chart/node-fee-chart.component.ts

@@ -101,8 +101,15 @@ export class NodeFeeChartComponent implements OnInit {
   }
 
   prepareChartOptions(outgoingData, incomingData): void {
+    let sum = outgoingData.reduce((accumulator, object) => {
+      return accumulator + object.count;
+    }, 0);
+    sum += incomingData.reduce((accumulator, object) => {
+      return accumulator + object.count;
+    }, 0);
+
     let title: object;
-    if (outgoingData.length === 0) {
+    if (sum === 0) {
       title = {
         textStyle: {
           color: 'grey',
@@ -115,7 +122,7 @@ export class NodeFeeChartComponent implements OnInit {
     }
 
     this.chartOptions = {
-      title: outgoingData.length === 0 ? title : undefined,
+      title: sum === 0 ? title : undefined,
       animation: false,
       grid: {
         top: 30,
@@ -151,7 +158,7 @@ export class NodeFeeChartComponent implements OnInit {
           `;
         }
       },
-      xAxis: outgoingData.length === 0 ? undefined : {
+      xAxis: sum === 0 ? undefined : {
         type: 'category',
         axisLine: { onZero: true },
         axisLabel: {
@@ -163,7 +170,7 @@ export class NodeFeeChartComponent implements OnInit {
         },
         data: outgoingData.map(bucket => bucket.label)
       },
-      legend: outgoingData.length === 0 ? undefined : {
+      legend: sum === 0 ? undefined : {
         padding: 10,
         data: [
           {
@@ -184,7 +191,7 @@ export class NodeFeeChartComponent implements OnInit {
           },
         ],
       },
-      yAxis: outgoingData.length === 0 ? undefined : [
+      yAxis: sum === 0 ? undefined : [
         {
           type: 'value',
           axisLabel: {
@@ -202,7 +209,7 @@ export class NodeFeeChartComponent implements OnInit {
           },
         },
       ],
-      series: outgoingData.length === 0 ? undefined : [
+      series: sum === 0 ? undefined : [
         {
           zlevel: 0,
           name: $localize`Outgoing Fees`,

frontend/src/app/lightning/node-statistics-chart/node-statistics-chart.component.ts

@@ -75,13 +75,13 @@ export class NodeStatisticsChartComponent implements OnInit {
 
   prepareChartOptions(data) {
     let title: object;
-    if (data.channels.length === 0) {
+    if (data.channels.length < 2) {
       title = {
         textStyle: {
           color: 'grey',
           fontSize: 15
         },
-        text: `Loading`,
+        text: $localize`No data to display yet. Try again later.`,
         left: 'center',
         top: 'center'
       };
@@ -135,14 +135,14 @@ export class NodeStatisticsChartComponent implements OnInit {
           return tooltip;
         }
       },
-      xAxis: data.channels.length === 0 ? undefined : {
+      xAxis: data.channels.length < 2 ? undefined : {
         type: 'time',
         splitNumber: this.isMobile() ? 5 : 10,
         axisLabel: {
           hideOverlap: true,
         }
       },
-      legend: data.channels.length === 0 ? undefined : {
+      legend: data.channels.length < 2 ? undefined : {
         padding: 10,
         data: [
           {
@@ -167,7 +167,7 @@ export class NodeStatisticsChartComponent implements OnInit {
           'Capacity': true,
         }
       },
-      yAxis: data.channels.length === 0 ? undefined : [
+      yAxis: data.channels.length < 2 ? undefined : [
         {
           type: 'value',
           axisLabel: {
@@ -198,7 +198,7 @@ export class NodeStatisticsChartComponent implements OnInit {
           }
         }
       ],
-      series: data.channels.length === 0 ? [] : [
+      series: data.channels.length < 2 ? [] : [
         {
           zlevel: 1,
           name: $localize`:@@807cf11e6ac1cde912496f764c176bdfdd6b7e19:Channels`,

frontend/src/app/lightning/node/node-preview.component.html

@@ -12,7 +12,7 @@
     </div>
   </div>
   <div class="row">
-    <div class="col-md">
+    <div class="col-md table-col">
       <a class="subtitle" [routerLink]="['/lightning/node' | relativeUrl, node.public_key]">{{ node.public_key }}</a>
       <table class="table table-borderless table-striped">
         <tbody>

frontend/src/app/lightning/node/node-preview.component.scss

@@ -18,6 +18,10 @@
   }
 }
 
+.table-col {
+  max-width: calc(100% - 470px);
+}
+
 .map-col {
   flex-grow: 0;
   flex-shrink: 0;

frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.scss

@@ -133,3 +133,10 @@
     top: 450px;
   }
 }
+
+.indexing-message {
+  position: absolute;
+  width: 100%;
+  text-align: center;
+  margin-top: 100px;
+}

frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts

@@ -204,24 +204,33 @@ export class NodesChannelsMap implements OnInit {
 
   prepareChartOptions(nodes, channels) {
     let title: object;
-    if (channels.length === 0 && !this.placeholder) {
+    if (channels.length === 0) {
-      this.chartOptions = null;
+      if (!this.placeholder) {
-      return;
+        this.isLoading = false;
-    }
+        title = {
-
+          textStyle: {
-    // empty map fallback
+            color: 'white',
-    if (channels.length === 0 && this.placeholder) {
+            fontSize: 18
-      title = {
+          },
-        textStyle: {
+          text: $localize`No data to display yet. Try again later.`,
-          color: 'white',
+          left: 'center',
-          fontSize: 18
+          top: 'center'
-        },
+        };
-        text: $localize`No geolocation data available`,
+        this.zoom = 1.5;
-        left: 'center',
+        this.center = [0, 20];
-        top: 'center'
+      } else { // used for Node and Channel preview components
-      };
+        title = {
-      this.zoom = 1.5;
+          textStyle: {
-      this.center = [0, 20];
+            color: 'white',
+            fontSize: 18
+          },
+          text: $localize`No geolocation data available`,
+          left: 'center',
+          top: 'center'
+        };
+        this.zoom = 1.5;
+        this.center = [0, 20];
+      }
     }
 
     this.chartOptions = {

frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.html

@@ -38,7 +38,7 @@
     </small>
   </div>
 
-  <div [class]="!widget ? 'bottom-padding' : 'pb-0'" class="container pb-lg-0">
+  <div *ngIf="!indexingInProgress else indexing" [class]="!widget ? 'bottom-padding' : 'pb-0'" class="container pb-lg-0">
     <div [class]="widget ? 'chart-widget' : 'chart'" echarts [initOpts]="chartInitOptions" [options]="chartOptions"
       (chartInit)="onChartInit($event)">
     </div>
@@ -99,3 +99,7 @@
     </div>
   </div>
 </ng-template>
+
+<ng-template #indexing>
+  <div class="indexing-message" i18n="lightning.indexing-in-progress">Indexing in progress</div>
+</ng-template>

frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.scss

@@ -167,4 +167,14 @@
     padding-left: 105px;
     padding-right: 105px;
   }
-}
+}
+
+.indexing-message {
+  font-size: 15px;
+  color: grey;
+  font-weight: bold;
+  width: 100%;
+  padding-top: 100px;
+  text-align: center;
+  height: 240px;
+}

frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.ts

@@ -29,6 +29,7 @@ export class NodesPerISPChartComponent implements OnInit {
   sortBy = 'capacity';
   showUnknown = false;
   chartInstance = undefined;
+  indexingInProgress = false;
 
   @HostBinding('attr.dir') dir = 'ltr';
 
@@ -88,6 +89,8 @@ export class NodesPerISPChartComponent implements OnInit {
 
                 this.prepareChartOptions(data.ispRanking);
 
+                this.indexingInProgress = !data.ispRanking.length;
+
                 return {
                   taggedISP: data.ispRanking.length,
                   clearnetCapacity: data.clearnetCapacity,

frontend/src/app/services/state.service.ts

@@ -89,7 +89,7 @@ export class StateService {
 
   networkChanged$ = new ReplaySubject<string>(1);
   lightningChanged$ = new ReplaySubject<boolean>(1);
-  blocks$: ReplaySubject<[BlockExtended, boolean]>;
+  blocks$: ReplaySubject<[BlockExtended, boolean, boolean]>;
   transactions$ = new ReplaySubject<TransactionStripped>(6);
   conversions$ = new ReplaySubject<any>(1);
   bsqPrice$ = new ReplaySubject<number>(1);
@@ -157,7 +157,7 @@ export class StateService {
       }
     });
 
-    this.blocks$ = new ReplaySubject<[BlockExtended, boolean]>(this.env.KEEP_BLOCKS_AMOUNT);
+    this.blocks$ = new ReplaySubject<[BlockExtended, boolean, boolean]>(this.env.KEEP_BLOCKS_AMOUNT);
 
     if (this.env.BASE_MODULE === 'bisq') {
       this.network = this.env.BASE_MODULE;

frontend/src/app/services/storage.service.ts

@@ -12,20 +12,22 @@ export class StorageService {
 
   setDefaultValueIfNeeded(key: string, defaultValue: string) {
     const graphWindowPreference: string = this.getValue(key);
+    const fragment = window.location.hash.replace('#', '');
+
     if (graphWindowPreference === null) { // First visit to mempool.space
-      if (this.router.url.includes('graphs') && key === 'graphWindowPreference' ||
+      if (window.location.pathname.includes('graphs') && key === 'graphWindowPreference' ||
-        this.router.url.includes('pools') && key === 'miningWindowPreference'
+        window.location.pathname.includes('pools') && key === 'miningWindowPreference'
       ) {
-        this.setValue(key, this.route.snapshot.fragment ? this.route.snapshot.fragment : defaultValue);
+        this.setValue(key, fragment ? fragment : defaultValue);
       } else {
         this.setValue(key, defaultValue);
       }
-    } else if (this.router.url.includes('graphs') && key === 'graphWindowPreference' ||
+    } else if (window.location.pathname.includes('graphs') && key === 'graphWindowPreference' ||
-      this.router.url.includes('pools') && key === 'miningWindowPreference'
+      window.location.pathname.includes('pools') && key === 'miningWindowPreference'
     ) {
       // Visit a different graphs#fragment from last visit
-      if (this.route.snapshot.fragment !== null && graphWindowPreference !== this.route.snapshot.fragment) {
+      if (fragment !== null && graphWindowPreference !== fragment) {
-        this.setValue(key, this.route.snapshot.fragment);
+        this.setValue(key, fragment);
       }
     }
   }

frontend/src/app/services/websocket.service.ts

@@ -228,7 +228,7 @@ export class WebsocketService {
       blocks.forEach((block: BlockExtended) => {
         if (block.height > this.stateService.latestBlockHeight) {
           maxHeight = Math.max(maxHeight, block.height);
-          this.stateService.blocks$.next([block, false]);
+          this.stateService.blocks$.next([block, false, true]);
         }
       });
       this.stateService.updateChainTip(maxHeight);
@@ -241,7 +241,7 @@ export class WebsocketService {
     if (response.block) {
       if (response.block.height > this.stateService.latestBlockHeight) {
         this.stateService.updateChainTip(response.block.height);
-        this.stateService.blocks$.next([response.block, !!response.txConfirmed]);
+        this.stateService.blocks$.next([response.block, !!response.txConfirmed, false]);
       }
 
       if (response.txConfirmed) {

frontend/src/app/shared/components/testnet-alert/testnet-alert.component.html

@@ -0,0 +1,8 @@
+<div class="container p-lg-0 pb-0" style="max-width: 100%; margin-top: 7px" *ngIf="storageService.getValue('hideWarning') !== 'hidden'">
+  <div class="alert alert-danger mb-0 text-center">
+    <div class="message-container" i18n="warning-testnet">This is a test network. Coins have no value.</div>
+    <button type="button" class="close" (click)="dismissWarning()">
+      <span aria-hidden="true">&times;</span>
+    </button>
+  </div>
+</div>