reduce number of GBT candidates

More precise durations in difficulty components

README.md

@@ -1,13 +1,13 @@
 # The Mempool Open Source Project™ [![mempool](https://img.shields.io/endpoint?url=https://dashboard.cypress.io/badge/simple/ry4br7/master&style=flat-square)](https://dashboard.cypress.io/projects/ry4br7/runs)
 
-https://user-images.githubusercontent.com/232186/222445818-234aa6c9-c233-4c52-b3f0-e32b8232893b.mp4
+https://user-images.githubusercontent.com/93150691/226236121-375ea64f-b4a1-4cc0-8fad-a6fb33226840.mp4
+
+<br>
 
 Mempool is the fully-featured mempool visualizer, explorer, and API service running at [mempool.space](https://mempool.space/). 
 
 It is an open-source project developed and operated for the benefit of the Bitcoin community, with a focus on the emerging transaction fee market that is evolving Bitcoin into a multi-layer ecosystem.
 
-![mempool](https://mempool.space/resources/screenshots/v2.4.0-dashboard.png)
-
 # Installation Methods
 
 Mempool can be self-hosted on a wide variety of your own hardware, ranging from a simple one-click installation on a Raspberry Pi full-node distro all the way to a robust production instance on a powerful FreeBSD server. 

backend/mempool-config.sample.json

@@ -27,13 +27,15 @@
     "AUDIT": false,
     "ADVANCED_GBT_AUDIT": false,
     "ADVANCED_GBT_MEMPOOL": false,
-    "CPFP_INDEXING": false
+    "CPFP_INDEXING": false,
+    "DISK_CACHE_BLOCK_INTERVAL": 6
   },
   "CORE_RPC": {
     "HOST": "127.0.0.1",
     "PORT": 8332,
     "USERNAME": "mempool",
-    "PASSWORD": "mempool"
+    "PASSWORD": "mempool",
+    "TIMEOUT": 60000
   },
   "ELECTRUM": {
     "HOST": "127.0.0.1",
@@ -41,13 +43,16 @@
     "TLS_ENABLED": true
   },
   "ESPLORA": {
-    "REST_API_URL": "http://127.0.0.1:3000"
+    "REST_API_URL": "http://127.0.0.1:3000",
+    "UNIX_SOCKET_PATH": "/tmp/esplora-bitcoin-mainnet",
+    "RETRY_UNIX_SOCKET_AFTER": 30000
   },
   "SECOND_CORE_RPC": {
     "HOST": "127.0.0.1",
     "PORT": 8332,
     "USERNAME": "mempool",
-    "PASSWORD": "mempool"
+    "PASSWORD": "mempool",
+    "TIMEOUT": 60000
   },
   "DATABASE": {
     "ENABLED": true,
@@ -56,7 +61,8 @@
     "SOCKET": "/var/run/mysql/mysql.sock",
     "DATABASE": "mempool",
     "USERNAME": "mempool",
-    "PASSWORD": "mempool"
+    "PASSWORD": "mempool",
+    "TIMEOUT": 180000
   },
   "SYSLOG": {
     "ENABLED": true,
@@ -91,7 +97,8 @@
   "LND": {
     "TLS_CERT_PATH": "tls.cert",
     "MACAROON_PATH": "readonly.macaroon",
-    "REST_API_URL": "https://localhost:8080"
+    "REST_API_URL": "https://localhost:8080",
+    "TIMEOUT": 10000
   },
   "CLIGHTNING": {
     "SOCKET": "lightning-rpc"

backend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mempool-backend",
-  "version": "2.5.0-dev",
+  "version": "2.6.0-dev",
   "description": "Bitcoin mempool visualizer and blockchain explorer backend",
   "license": "GNU Affero General Public License v3.0",
   "homepage": "https://mempool.space",
@@ -34,35 +34,35 @@
     "prettier": "./node_modules/.bin/prettier --write \"src/**/*.{js,ts}\""
   },
   "dependencies": {
-    "@babel/core": "^7.20.12",
-    "@mempool/electrum-client": "^1.1.7",
-    "@types/node": "^16.18.11",
+    "@babel/core": "^7.21.3",
+    "@mempool/electrum-client": "1.1.9",
+    "@types/node": "^18.15.3",
     "axios": "~0.27.2",
     "bitcoinjs-lib": "~6.1.0",
     "crypto-js": "~4.1.1",
     "express": "~4.18.2",
     "maxmind": "~4.3.8",
-    "mysql2": "~2.3.3",
+    "mysql2": "~3.2.0",
     "node-worker-threads-pool": "~1.5.1",
     "socks-proxy-agent": "~7.0.0",
     "typescript": "~4.7.4",
-    "ws": "~8.11.0"
+    "ws": "~8.13.0"
   },
   "devDependencies": {
-    "@babel/core": "^7.20.7",
+    "@babel/core": "^7.21.3",
     "@babel/code-frame": "^7.18.6",
     "@types/compression": "^1.7.2",
     "@types/crypto-js": "^4.1.1",
     "@types/express": "^4.17.15",
-    "@types/jest": "^29.2.5",
+    "@types/jest": "^29.5.0",
     "@types/ws": "~8.5.4",
-    "@typescript-eslint/eslint-plugin": "^5.48.1",
-    "@typescript-eslint/parser": "^5.48.1",
-    "eslint": "^8.31.0",
-    "eslint-config-prettier": "^8.5.0",
-    "jest": "^29.3.1",
-    "prettier": "^2.8.2",
-    "ts-jest": "^29.0.3",
+    "@typescript-eslint/eslint-plugin": "^5.55.0",
+    "@typescript-eslint/parser": "^5.55.0",
+    "eslint": "^8.36.0",
+    "eslint-config-prettier": "^8.7.0",
+    "jest": "^29.5.0",
+    "prettier": "^2.8.4",
+    "ts-jest": "^29.0.5",
     "ts-node": "^10.9.1"
   }
 }

backend/src/__fixtures__/mempool-config.template.json

@@ -28,13 +28,15 @@
     "ADVANCED_GBT_AUDIT": "__MEMPOOL_ADVANCED_GBT_AUDIT__",
     "ADVANCED_GBT_MEMPOOL": "__MEMPOOL_ADVANCED_GBT_MEMPOOL__",
     "CPFP_INDEXING": "__MEMPOOL_CPFP_INDEXING__",
-    "MAX_BLOCKS_BULK_QUERY": "__MEMPOOL_MAX_BLOCKS_BULK_QUERY__"
+    "MAX_BLOCKS_BULK_QUERY": "__MEMPOOL_MAX_BLOCKS_BULK_QUERY__",
+    "DISK_CACHE_BLOCK_INTERVAL": "__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__"
   },
   "CORE_RPC": {
     "HOST": "__CORE_RPC_HOST__",
     "PORT": 15,
     "USERNAME": "__CORE_RPC_USERNAME__",
-    "PASSWORD": "__CORE_RPC_PASSWORD__"
+    "PASSWORD": "__CORE_RPC_PASSWORD__",
+    "TIMEOUT": "__CORE_RPC_TIMEOUT__"
   },
   "ELECTRUM": {
     "HOST": "__ELECTRUM_HOST__",
@@ -42,13 +44,16 @@
     "TLS_ENABLED": true
   },
   "ESPLORA": {
-    "REST_API_URL": "__ESPLORA_REST_API_URL__"
+    "REST_API_URL": "__ESPLORA_REST_API_URL__",
+    "UNIX_SOCKET_PATH": "__ESPLORA_UNIX_SOCKET_PATH__",
+    "RETRY_UNIX_SOCKET_AFTER": "__ESPLORA_RETRY_UNIX_SOCKET_AFTER__"
   },
   "SECOND_CORE_RPC": {
     "HOST": "__SECOND_CORE_RPC_HOST__",
     "PORT": 17,
     "USERNAME": "__SECOND_CORE_RPC_USERNAME__",
-    "PASSWORD": "__SECOND_CORE_RPC_PASSWORD__"
+    "PASSWORD": "__SECOND_CORE_RPC_PASSWORD__",
+    "TIMEOUT": "__SECOND_CORE_RPC_TIMEOUT__"
   },
   "DATABASE": {
     "ENABLED": false,
@@ -57,7 +62,8 @@
     "PORT": 18,
     "DATABASE": "__DATABASE_DATABASE__",
     "USERNAME": "__DATABASE_USERNAME__",
-    "PASSWORD": "__DATABASE_PASSWORD__"
+    "PASSWORD": "__DATABASE_PASSWORD__",
+    "TIMEOUT": "__DATABASE_TIMEOUT__"
   },
   "SYSLOG": {
     "ENABLED": false,
@@ -107,7 +113,8 @@
   "LND": {
     "TLS_CERT_PATH": "",
     "MACAROON_PATH": "",
-    "REST_API_URL": "https://localhost:8080"
+    "REST_API_URL": "https://localhost:8080",
+    "TIMEOUT": 10000
   },
   "CLIGHTNING": {
     "SOCKET": "__CLIGHTNING_SOCKET__"

backend/src/__tests__/config.test.ts

@@ -42,24 +42,27 @@ describe('Mempool Backend Config', () => {
         ADVANCED_GBT_MEMPOOL: false,
         CPFP_INDEXING: false,
         MAX_BLOCKS_BULK_QUERY: 0,
+        DISK_CACHE_BLOCK_INTERVAL: 6,
       });
 
       expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true });
 
-      expect(config.ESPLORA).toStrictEqual({ REST_API_URL: 'http://127.0.0.1:3000' });
+      expect(config.ESPLORA).toStrictEqual({ REST_API_URL: 'http://127.0.0.1:3000', UNIX_SOCKET_PATH: null, RETRY_UNIX_SOCKET_AFTER: 30000 });
 
       expect(config.CORE_RPC).toStrictEqual({
         HOST: '127.0.0.1',
         PORT: 8332,
         USERNAME: 'mempool',
-        PASSWORD: 'mempool'
+        PASSWORD: 'mempool',
+        TIMEOUT: 60000
       });
 
       expect(config.SECOND_CORE_RPC).toStrictEqual({
         HOST: '127.0.0.1',
         PORT: 8332,
         USERNAME: 'mempool',
-        PASSWORD: 'mempool'
+        PASSWORD: 'mempool',
+        TIMEOUT: 60000
       });
 
       expect(config.DATABASE).toStrictEqual({
@@ -69,7 +72,8 @@ describe('Mempool Backend Config', () => {
         PORT: 3306,
         DATABASE: 'mempool',
         USERNAME: 'mempool',
-        PASSWORD: 'mempool'
+        PASSWORD: 'mempool',
+        TIMEOUT: 180000,
       });
 
       expect(config.SYSLOG).toStrictEqual({
@@ -108,10 +112,10 @@ describe('Mempool Backend Config', () => {
       });
 
       expect(config.MAXMIND).toStrictEqual({
-        ENABLED: true,
-        GEOLITE2_CITY: './backend/GeoIP/GeoLite2-City.mmdb',
-        GEOLITE2_ASN: './backend/GeoIP/GeoLite2-ASN.mmdb',
-        GEOIP2_ISP: ''
+        ENABLED: false,
+        GEOLITE2_CITY: '/usr/local/share/GeoIP/GeoLite2-City.mmdb',
+        GEOLITE2_ASN: '/usr/local/share/GeoIP/GeoLite2-ASN.mmdb',
+        GEOIP2_ISP: '/usr/local/share/GeoIP/GeoIP2-ISP.mmdb'
       });
     });
   });

backend/src/api/audit.ts

@@ -1,4 +1,5 @@
 import config from '../config';
+import logger from '../logger';
 import { TransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces';
 
 const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners
@@ -39,17 +40,19 @@ class Audit {
         } else {
           isCensored[txid] = true;
         }
-        displacedWeight += mempool[txid].weight;
+        displacedWeight += mempool[txid]?.weight || 0;
       } else {
-        matchedWeight += mempool[txid].weight;
+        matchedWeight += mempool[txid]?.weight || 0;
       }
-      projectedWeight += mempool[txid].weight;
+      projectedWeight += mempool[txid]?.weight || 0;
       inTemplate[txid] = true;
     }
 
-    displacedWeight += (4000 - transactions[0].weight);
-    projectedWeight += transactions[0].weight;
-    matchedWeight += transactions[0].weight;
+    if (transactions[0]) {
+      displacedWeight += (4000 - transactions[0].weight);
+      projectedWeight += transactions[0].weight;
+      matchedWeight += transactions[0].weight;
+    }
 
     // we can expect an honest miner to include 'displaced' transactions in place of recent arrivals and censored txs
     // these displaced transactions should occupy the first N weight units of the next projected block
@@ -59,19 +62,24 @@ class Audit {
     let failures = 0;
     while (projectedBlocks[1] && index < projectedBlocks[1].transactionIds.length && failures < 500) {
       const txid = projectedBlocks[1].transactionIds[index];
-      const fits = (mempool[txid].weight - displacedWeightRemaining) < 4000;
-      const feeMatches = mempool[txid].effectiveFeePerVsize >= lastFeeRate;
-      if (fits || feeMatches) {
-        isDisplaced[txid] = true;
-        if (fits) {
-          lastFeeRate = Math.min(lastFeeRate, mempool[txid].effectiveFeePerVsize);
+      const tx = mempool[txid];
+      if (tx) {
+        const fits = (tx.weight - displacedWeightRemaining) < 4000;
+        const feeMatches = tx.effectiveFeePerVsize >= lastFeeRate;
+        if (fits || feeMatches) {
+          isDisplaced[txid] = true;
+          if (fits) {
+            lastFeeRate = Math.min(lastFeeRate, tx.effectiveFeePerVsize);
+          }
+          if (tx.firstSeen == null || (now - (tx?.firstSeen || 0)) > PROPAGATION_MARGIN) {
+            displacedWeightRemaining -= tx.weight;
+          }
+          failures = 0;
+        } else {
+          failures++;
         }
-        if (mempool[txid].firstSeen == null || (now - (mempool[txid]?.firstSeen || 0)) > PROPAGATION_MARGIN) {
-          displacedWeightRemaining -= mempool[txid].weight;
-        }
-        failures = 0;
       } else {
-        failures++;
+        logger.warn('projected transaction missing from mempool cache');
       }
       index++;
     }
@@ -85,17 +93,7 @@ class Audit {
       } else {
         if (!isDisplaced[tx.txid]) {
           added.push(tx.txid);
-        } else {
         }
-        let blockIndex = -1;
-        let index = -1;
-        projectedBlocks.forEach((block, bi) => {
-          const i = block.transactionIds.indexOf(tx.txid);
-          if (i >= 0) {
-            blockIndex = bi;
-            index = i;
-          }
-        });
         overflowWeight += tx.weight;
       }
       totalWeight += tx.weight;
@@ -108,20 +106,25 @@ class Audit {
     index = projectedBlocks[0].transactionIds.length - 1;
     while (index >= 0) {
       const txid = projectedBlocks[0].transactionIds[index];
-      if (overflowWeightRemaining > 0) {
-        if (isCensored[txid]) {
-          delete isCensored[txid];
-        }
-        if (mempool[txid].effectiveFeePerVsize > maxOverflowRate) {
-          maxOverflowRate = mempool[txid].effectiveFeePerVsize;
-          rateThreshold = (Math.ceil(maxOverflowRate * 100) / 100) + 0.005;
-        }
-      } else if (mempool[txid].effectiveFeePerVsize <= rateThreshold) { // tolerance of 0.01 sat/vb + rounding
-        if (isCensored[txid]) {
-          delete isCensored[txid];
+      const tx = mempool[txid];
+      if (tx) {
+        if (overflowWeightRemaining > 0) {
+          if (isCensored[txid]) {
+            delete isCensored[txid];
+          }
+          if (tx.effectiveFeePerVsize > maxOverflowRate) {
+            maxOverflowRate = tx.effectiveFeePerVsize;
+            rateThreshold = (Math.ceil(maxOverflowRate * 100) / 100) + 0.005;
+          }
+        } else if (tx.effectiveFeePerVsize <= rateThreshold) { // tolerance of 0.01 sat/vb + rounding
+          if (isCensored[txid]) {
+            delete isCensored[txid];
+          }
         }
+        overflowWeightRemaining -= (mempool[txid]?.weight || 0);
+      } else {
+        logger.warn('projected transaction missing from mempool cache');
       }
-      overflowWeightRemaining -= (mempool[txid]?.weight || 0);
       index--;
     }
 

backend/src/api/bitcoin/bitcoin-client.ts

@@ -7,7 +7,7 @@ const nodeRpcCredentials: BitcoinRpcCredentials = {
   port: config.CORE_RPC.PORT,
   user: config.CORE_RPC.USERNAME,
   pass: config.CORE_RPC.PASSWORD,
-  timeout: 60000,
+  timeout: config.CORE_RPC.TIMEOUT,
 };
 
 export default new bitcoin.Client(nodeRpcCredentials);

backend/src/api/bitcoin/bitcoin-second-client.ts

@@ -7,7 +7,7 @@ const nodeRpcCredentials: BitcoinRpcCredentials = {
   port: config.SECOND_CORE_RPC.PORT,
   user: config.SECOND_CORE_RPC.USERNAME,
   pass: config.SECOND_CORE_RPC.PASSWORD,
-  timeout: 60000,
+  timeout: config.SECOND_CORE_RPC.TIMEOUT,
 };
 
 export default new bitcoin.Client(nodeRpcCredentials);

backend/src/api/bitcoin/bitcoin.routes.ts

@@ -32,8 +32,10 @@ class BitcoinRoutes {
       .get(config.MEMPOOL.API_URL_PREFIX + 'backend-info', this.getBackendInfo)
       .get(config.MEMPOOL.API_URL_PREFIX + 'init-data', this.getInitData)
       .get(config.MEMPOOL.API_URL_PREFIX + 'validate-address/:address', this.validateAddress)
-      .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/replaces', this.getRbfHistory)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/rbf', this.getRbfHistory)
       .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/cached', this.getCachedTx)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'replacements', this.getRbfReplacements)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'fullrbf/replacements', this.getFullRbfReplacements)
       .post(config.MEMPOOL.API_URL_PREFIX + 'tx/push', this.$postTransactionForm)
       .get(config.MEMPOOL.API_URL_PREFIX + 'donations', async (req, res) => {
         try {
@@ -94,6 +96,7 @@ class BitcoinRoutes {
       .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash', this.getBlock)
       .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/summary', this.getStrippedBlockTransactions)
       .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/audit-summary', this.getBlockAuditSummary)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/height', this.getBlockTipHeight)
       .post(config.MEMPOOL.API_URL_PREFIX + 'psbt/addparents', this.postPsbtCompletion)
       .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from', this.getBlocksByBulk.bind(this))
       .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from/:to', this.getBlocksByBulk.bind(this))
@@ -110,7 +113,6 @@ class BitcoinRoutes {
           .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/status', this.getTransactionStatus)
           .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/outspends', this.getTransactionOutspends)
           .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/header', this.getBlockHeader)
-          .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/height', this.getBlockTipHeight)
           .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/hash', this.getBlockTipHash)
           .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/raw', this.getRawBlock)
           .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txids', this.getTxIdsForBlock)
@@ -589,10 +591,14 @@ class BitcoinRoutes {
     }
   }
 
-  private async getBlockTipHeight(req: Request, res: Response) {
+  private getBlockTipHeight(req: Request, res: Response) {
     try {
-      const result = await bitcoinApi.$getBlockHeightTip();
-      res.json(result);
+      const result = blocks.getCurrentBlockHeight();
+      if (!result) {
+        return res.status(503).send(`Service Temporarily Unavailable`);
+      }
+      res.setHeader('content-type', 'text/plain');
+      res.send(result.toString());
     } catch (e) {
       res.status(500).send(e instanceof Error ? e.message : e);
     }
@@ -638,8 +644,30 @@ class BitcoinRoutes {
 
   private async getRbfHistory(req: Request, res: Response) {
     try {
-      const result = rbfCache.getReplaces(req.params.txId);
-      res.json(result || []);
+      const replacements = rbfCache.getRbfTree(req.params.txId) || null;
+      const replaces = rbfCache.getReplaces(req.params.txId) || null;
+      res.json({
+        replacements,
+        replaces
+      });
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getRbfReplacements(req: Request, res: Response) {
+    try {
+      const result = rbfCache.getRbfTrees(false);
+      res.json(result);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getFullRbfReplacements(req: Request, res: Response) {
+    try {
+      const result = rbfCache.getRbfTrees(true);
+      res.json(result);
     } catch (e) {
       res.status(500).send(e instanceof Error ? e.message : e);
     }

backend/src/api/bitcoin/electrum-api.ts

@@ -16,7 +16,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
     super(bitcoinClient);
 
     const electrumConfig = { client: 'mempool-v2', version: '1.4' };
-    const electrumPersistencePolicy = { retryPeriod: 10000, maxRetry: 1000, callback: null };
+    const electrumPersistencePolicy = { retryPeriod: 1000, maxRetry: Number.MAX_SAFE_INTEGER, callback: null };
 
     const electrumCallbacks = {
       onConnect: (client, versionInfo) => { logger.info(`Connected to Electrum Server at ${config.ELECTRUM.HOST}:${config.ELECTRUM.PORT} (${JSON.stringify(versionInfo)})`); },

backend/src/api/bitcoin/esplora-api.ts

@@ -3,65 +3,102 @@ import axios, { AxiosRequestConfig } from 'axios';
 import http from 'http';
 import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
 import { IEsploraApi } from './esplora-api.interface';
+import logger from '../../logger';
 
 const axiosConnection = axios.create({
-  httpAgent: new http.Agent({ keepAlive: true })
+  httpAgent: new http.Agent({ keepAlive: true, })
 });
 
 class ElectrsApi implements AbstractBitcoinApi {
-  axiosConfig: AxiosRequestConfig = {
+  private axiosConfigWithUnixSocket: AxiosRequestConfig = config.ESPLORA.UNIX_SOCKET_PATH ? {
+    socketPath: config.ESPLORA.UNIX_SOCKET_PATH,
+    timeout: 10000,
+  } : {
+    timeout: 10000,
+  };
+  private axiosConfigTcpSocketOnly: AxiosRequestConfig = {
     timeout: 10000,
   };
 
-  constructor() { }
+  unixSocketRetryTimeout;
+  activeAxiosConfig;
+
+  constructor() {
+    this.activeAxiosConfig = this.axiosConfigWithUnixSocket;
+  }
+
+  fallbackToTcpSocket() {
+    if (!this.unixSocketRetryTimeout) {
+      logger.err(`Unable to connect to esplora unix socket. Falling back to tcp socket. Retrying unix socket in ${config.ESPLORA.RETRY_UNIX_SOCKET_AFTER / 1000} seconds`);
+      // Retry the unix socket after a few seconds
+      this.unixSocketRetryTimeout = setTimeout(() => {
+        logger.info(`Retrying to use unix socket for esplora now (applied for the next query)`);
+        this.activeAxiosConfig = this.axiosConfigWithUnixSocket;
+        this.unixSocketRetryTimeout = undefined;
+      }, config.ESPLORA.RETRY_UNIX_SOCKET_AFTER);
+    }
+
+    // Use the TCP socket (reach a different esplora instance through nginx)
+    this.activeAxiosConfig = this.axiosConfigTcpSocketOnly;
+  }
+
+  $queryWrapper<T>(url, responseType = 'json'): Promise<T> {
+    return axiosConnection.get<T>(url, { ...this.activeAxiosConfig, responseType: responseType })
+      .then((response) => response.data)
+      .catch((e) => {
+        if (e?.code === 'ECONNREFUSED') {
+          this.fallbackToTcpSocket();
+          // Retry immediately
+          return axiosConnection.get<T>(url, this.activeAxiosConfig)
+            .then((response) => response.data)
+            .catch((e) => {
+              logger.warn(`Cannot query esplora through the unix socket nor the tcp socket. Exception ${e}`);
+              throw e;
+            });
+        } else {
+          throw e;
+        }
+      });
+  }
 
   $getRawMempool(): Promise<IEsploraApi.Transaction['txid'][]> {
-    return axiosConnection.get<IEsploraApi.Transaction['txid'][]>(config.ESPLORA.REST_API_URL + '/mempool/txids', this.axiosConfig)
-      .then((response) => response.data);
+    return this.$queryWrapper<IEsploraApi.Transaction['txid'][]>(config.ESPLORA.REST_API_URL + '/mempool/txids');
   }
 
   $getRawTransaction(txId: string): Promise<IEsploraApi.Transaction> {
-    return axiosConnection.get<IEsploraApi.Transaction>(config.ESPLORA.REST_API_URL + '/tx/' + txId, this.axiosConfig)
-      .then((response) => response.data);
+    return this.$queryWrapper<IEsploraApi.Transaction>(config.ESPLORA.REST_API_URL + '/tx/' + txId);
   }
 
   $getTransactionHex(txId: string): Promise<string> {
-    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/hex', this.axiosConfig)
-      .then((response) => response.data);
+    return this.$queryWrapper<string>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/hex');
   }
 
   $getBlockHeightTip(): Promise<number> {
-    return axiosConnection.get<number>(config.ESPLORA.REST_API_URL + '/blocks/tip/height', this.axiosConfig)
-      .then((response) => response.data);
+    return this.$queryWrapper<number>(config.ESPLORA.REST_API_URL + '/blocks/tip/height');
   }
 
   $getBlockHashTip(): Promise<string> {
-    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/blocks/tip/hash', this.axiosConfig)
-      .then((response) => response.data);
+    return this.$queryWrapper<string>(config.ESPLORA.REST_API_URL + '/blocks/tip/hash');
   }
 
   $getTxIdsForBlock(hash: string): Promise<string[]> {
-    return axiosConnection.get<string[]>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/txids', this.axiosConfig)
-      .then((response) => response.data);
+    return this.$queryWrapper<string[]>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/txids');
   }
 
   $getBlockHash(height: number): Promise<string> {
-    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/block-height/' + height, this.axiosConfig)
-      .then((response) => response.data);
+    return this.$queryWrapper<string>(config.ESPLORA.REST_API_URL + '/block-height/' + height);
   }
 
   $getBlockHeader(hash: string): Promise<string> {
-    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/header', this.axiosConfig)
-      .then((response) => response.data);
+    return this.$queryWrapper<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/header');
   }
 
   $getBlock(hash: string): Promise<IEsploraApi.Block> {
-    return axiosConnection.get<IEsploraApi.Block>(config.ESPLORA.REST_API_URL + '/block/' + hash, this.axiosConfig)
-      .then((response) => response.data);
+    return this.$queryWrapper<IEsploraApi.Block>(config.ESPLORA.REST_API_URL + '/block/' + hash);
   }
 
   $getRawBlock(hash: string): Promise<Buffer> {
-    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + "/raw", { ...this.axiosConfig, responseType: 'arraybuffer' })
+    return this.$queryWrapper<any>(config.ESPLORA.REST_API_URL + '/block/' + hash + "/raw", 'arraybuffer')
       .then((response) => { return Buffer.from(response.data); });
   }
 
@@ -82,13 +119,11 @@ class ElectrsApi implements AbstractBitcoinApi {
   }
 
   $getOutspend(txId: string, vout: number): Promise<IEsploraApi.Outspend> {
-    return axiosConnection.get<IEsploraApi.Outspend>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspend/' + vout, this.axiosConfig)
-      .then((response) => response.data);
+    return this.$queryWrapper<IEsploraApi.Outspend>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspend/' + vout);
   }
 
   $getOutspends(txId: string): Promise<IEsploraApi.Outspend[]> {
-    return axiosConnection.get<IEsploraApi.Outspend[]>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspends', this.axiosConfig)
-      .then((response) => response.data);
+    return this.$queryWrapper<IEsploraApi.Outspend[]>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspends');
   }
 
   async $getBatchedOutspends(txId: string[]): Promise<IEsploraApi.Outspend[][]> {

backend/src/api/blocks.ts

@@ -2,7 +2,7 @@ import config from '../config';
 import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory';
 import logger from '../logger';
 import memPool from './mempool';
-import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionStripped, TransactionMinerInfo } from '../mempool.interfaces';
+import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionStripped, TransactionMinerInfo, CpfpSummary } from '../mempool.interfaces';
 import { Common } from './common';
 import diskCache from './disk-cache';
 import transactionUtils from './transaction-utils';
@@ -36,6 +36,8 @@ class Blocks {
   private newBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => void)[] = [];
   private newAsyncBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => Promise<void>)[] = [];
 
+  private mainLoopTimeout: number = 120000;
+
   constructor() { }
 
   public getBlocks(): BlockExtended[] {
@@ -200,8 +202,15 @@ class Blocks {
       extras.segwitTotalWeight = 0;
     } else {
       const stats: IBitcoinApi.BlockStats = await bitcoinClient.getBlockStats(block.id);
-      extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles
-      extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat();
+      let feeStats = {
+        medianFee: stats.feerate_percentiles[2], // 50th percentiles
+        feeRange: [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat(),
+      };
+      if (transactions?.length > 1) {
+        feeStats = Common.calcEffectiveFeeStatistics(transactions);
+      }
+      extras.medianFee = feeStats.medianFee;
+      extras.feeRange = feeStats.feeRange;
       extras.totalFees = stats.totalfee;
       extras.avgFee = stats.avgfee;
       extras.avgFeeRate = stats.avgfeerate;
@@ -403,12 +412,13 @@ class Blocks {
     try {
       // Get all indexed block hash
       const unindexedBlockHeights = await blocksRepository.$getCPFPUnindexedBlocks();
-      logger.info(`Indexing cpfp data for ${unindexedBlockHeights.length} blocks`);
 
       if (!unindexedBlockHeights?.length) {
         return;
       }
 
+      logger.info(`Indexing cpfp data for ${unindexedBlockHeights.length} blocks`);
+
       // Logging
       let count = 0;
       let countThisRun = 0;
@@ -519,9 +529,16 @@ class Blocks {
     return await BlocksRepository.$validateChain();
   }
 
-  public async $updateBlocks() {
+  public async $updateBlocks(): Promise<number> {
+    // warn if this run stalls the main loop for more than 2 minutes
+    const timer = this.startTimer();
+
+    diskCache.lock();
+
     let fastForwarded = false;
+    let handledBlocks = 0;
     const blockHeightTip = await bitcoinApi.$getBlockHeightTip();
+    this.updateTimerProgress(timer, 'got block height tip');
 
     if (this.blocks.length === 0) {
       this.currentBlockHeight = Math.max(blockHeightTip - config.MEMPOOL.INITIAL_BLOCKS_AMOUNT, -1);
@@ -539,16 +556,21 @@ class Blocks {
 
     if (!this.lastDifficultyAdjustmentTime) {
       const blockchainInfo = await bitcoinClient.getBlockchainInfo();
+      this.updateTimerProgress(timer, 'got blockchain info for initial difficulty adjustment');
       if (blockchainInfo.blocks === blockchainInfo.headers) {
         const heightDiff = blockHeightTip % 2016;
         const blockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff);
+        this.updateTimerProgress(timer, 'got block hash for initial difficulty adjustment');
         const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash);
+        this.updateTimerProgress(timer, 'got block for initial difficulty adjustment');
         this.lastDifficultyAdjustmentTime = block.timestamp;
         this.currentDifficulty = block.difficulty;
 
         if (blockHeightTip >= 2016) {
           const previousPeriodBlockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff - 2016);
+          this.updateTimerProgress(timer, 'got previous block hash for initial difficulty adjustment');
           const previousPeriodBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(previousPeriodBlockHash);
+          this.updateTimerProgress(timer, 'got previous block for initial difficulty adjustment');
           this.previousDifficultyRetarget = (block.difficulty - previousPeriodBlock.difficulty) / previousPeriodBlock.difficulty * 100;
           logger.debug(`Initial difficulty adjustment data set.`);
         }
@@ -558,57 +580,71 @@ class Blocks {
     }
 
     while (this.currentBlockHeight < blockHeightTip) {
-      if (this.currentBlockHeight < blockHeightTip - config.MEMPOOL.INITIAL_BLOCKS_AMOUNT) {
+      if (this.currentBlockHeight === 0) {
         this.currentBlockHeight = blockHeightTip;
       } else {
         this.currentBlockHeight++;
         logger.debug(`New block found (#${this.currentBlockHeight})!`);
+        this.updateTimerProgress(timer, `getting orphaned blocks for ${this.currentBlockHeight}`);
         await chainTips.updateOrphanedBlocks();
       }
 
+      this.updateTimerProgress(timer, `getting block data for ${this.currentBlockHeight}`);
       const blockHash = await bitcoinApi.$getBlockHash(this.currentBlockHeight);
       const verboseBlock = await bitcoinClient.getBlock(blockHash, 2);
       const block = BitcoinApi.convertBlock(verboseBlock);
       const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash);
       const transactions = await this.$getTransactionsExtended(blockHash, block.height, false);
-      const blockExtended: BlockExtended = await this.$getBlockExtended(block, transactions);
+      const cpfpSummary: CpfpSummary = Common.calculateCpfp(block.height, transactions);
+      const blockExtended: BlockExtended = await this.$getBlockExtended(block, cpfpSummary.transactions);
       const blockSummary: BlockSummary = this.summarizeBlock(verboseBlock);
+      this.updateTimerProgress(timer, `got block data for ${this.currentBlockHeight}`);
 
       // start async callbacks
+      this.updateTimerProgress(timer, `starting async callbacks for ${this.currentBlockHeight}`);
       const callbackPromises = this.newAsyncBlockCallbacks.map((cb) => cb(blockExtended, txIds, transactions));
 
       if (Common.indexingEnabled()) {
         if (!fastForwarded) {
           const lastBlock = await blocksRepository.$getBlockByHeight(blockExtended.height - 1);
+          this.updateTimerProgress(timer, `got block by height for ${this.currentBlockHeight}`);
           if (lastBlock !== null && blockExtended.previousblockhash !== lastBlock.id) {
-            logger.warn(`Chain divergence detected at block ${lastBlock.height}, re-indexing most recent data`);
+            logger.warn(`Chain divergence detected at block ${lastBlock.height}, re-indexing most recent data`, logger.tags.mining);
             // We assume there won't be a reorg with more than 10 block depth
+            this.updateTimerProgress(timer, `rolling back diverged chain from ${this.currentBlockHeight}`);
             await BlocksRepository.$deleteBlocksFrom(lastBlock.height - 10);
             await HashratesRepository.$deleteLastEntries();
-            await BlocksSummariesRepository.$deleteBlocksFrom(lastBlock.height - 10);
             await cpfpRepository.$deleteClustersFrom(lastBlock.height - 10);
+            this.updateTimerProgress(timer, `rolled back chain divergence from ${this.currentBlockHeight}`);
             for (let i = 10; i >= 0; --i) {
               const newBlock = await this.$indexBlock(lastBlock.height - i);
+              this.updateTimerProgress(timer, `reindexed block`);
               await this.$getStrippedBlockTransactions(newBlock.id, true, true);
+              this.updateTimerProgress(timer, `reindexed block summary`);
               if (config.MEMPOOL.CPFP_INDEXING) {
                 await this.$indexCPFP(newBlock.id, lastBlock.height - i);
+                this.updateTimerProgress(timer, `reindexed block cpfp`);
               }
             }
             await mining.$indexDifficultyAdjustments();
             await DifficultyAdjustmentsRepository.$deleteLastAdjustment();
-            logger.info(`Re-indexed 10 blocks and summaries. Also re-indexed the last difficulty adjustments. Will re-index latest hashrates in a few seconds.`);
+            this.updateTimerProgress(timer, `reindexed difficulty adjustments`);
+            logger.info(`Re-indexed 10 blocks and summaries. Also re-indexed the last difficulty adjustments. Will re-index latest hashrates in a few seconds.`, logger.tags.mining);
             indexer.reindex();
           }
           await blocksRepository.$saveBlockInDatabase(blockExtended);
+          this.updateTimerProgress(timer, `saved ${this.currentBlockHeight} to database`);
 
           const lastestPriceId = await PricesRepository.$getLatestPriceId();
+          this.updateTimerProgress(timer, `got latest price id ${this.currentBlockHeight}`);
           if (priceUpdater.historyInserted === true && lastestPriceId !== null) {
             await blocksRepository.$saveBlockPrices([{
               height: blockExtended.height,
               priceId: lastestPriceId,
             }]);
+            this.updateTimerProgress(timer, `saved prices for ${this.currentBlockHeight}`);
           } else {
-            logger.info(`Cannot save block price for ${blockExtended.height} because the price updater hasnt completed yet. Trying again in 10 seconds.`, logger.tags.mining);
+            logger.debug(`Cannot save block price for ${blockExtended.height} because the price updater hasnt completed yet. Trying again in 10 seconds.`, logger.tags.mining);
             setTimeout(() => {
               indexer.runSingleTask('blocksPrices');
             }, 10000);
@@ -617,9 +653,11 @@ class Blocks {
           // Save blocks summary for visualization if it's enabled
           if (Common.blocksSummariesIndexingEnabled() === true) {
             await this.$getStrippedBlockTransactions(blockExtended.id, true);
+            this.updateTimerProgress(timer, `saved block summary for ${this.currentBlockHeight}`);
           }
           if (config.MEMPOOL.CPFP_INDEXING) {
-            this.$indexCPFP(blockExtended.id, this.currentBlockHeight);
+            this.$saveCpfp(blockExtended.id, this.currentBlockHeight, cpfpSummary);
+            this.updateTimerProgress(timer, `saved cpfp for ${this.currentBlockHeight}`);
           }
         }
       }
@@ -632,6 +670,7 @@ class Blocks {
             difficulty: block.difficulty,
             adjustment: Math.round((block.difficulty / this.currentDifficulty) * 1000000) / 1000000, // Remove float point noise
           });
+          this.updateTimerProgress(timer, `saved difficulty adjustment for ${this.currentBlockHeight}`);
         }
 
         this.previousDifficultyRetarget = (block.difficulty - this.currentDifficulty) / this.currentDifficulty * 100;
@@ -651,12 +690,44 @@ class Blocks {
       if (this.newBlockCallbacks.length) {
         this.newBlockCallbacks.forEach((cb) => cb(blockExtended, txIds, transactions));
       }
-      if (!memPool.hasPriority() && (block.height % 6 === 0)) {
+      if (!memPool.hasPriority() && (block.height % config.MEMPOOL.DISK_CACHE_BLOCK_INTERVAL === 0)) {
         diskCache.$saveCacheToDisk();
       }
 
       // wait for pending async callbacks to finish
+      this.updateTimerProgress(timer, `waiting for async callbacks to complete for ${this.currentBlockHeight}`);
       await Promise.all(callbackPromises);
+      this.updateTimerProgress(timer, `async callbacks completed for ${this.currentBlockHeight}`);
+
+      handledBlocks++;
+    }
+
+    diskCache.unlock();
+
+    this.clearTimer(timer);
+
+    return handledBlocks;
+  }
+
+  private startTimer() {
+    const state: any = {
+      start: Date.now(),
+      progress: 'begin $updateBlocks',
+      timer: null,
+    };
+    state.timer = setTimeout(() => {
+      logger.err(`$updateBlocks stalled at "${state.progress}"`);
+    }, this.mainLoopTimeout);
+    return state;
+  }
+
+  private updateTimerProgress(state, msg) {
+    state.progress = msg;
+  }
+
+  private clearTimer(state) {
+    if (state.timer) {
+      clearTimeout(state.timer);
     }
   }
 
@@ -728,7 +799,7 @@ class Blocks {
 
     // Index the response if needed
     if (Common.blocksSummariesIndexingEnabled() === true) {
-      await BlocksSummariesRepository.$saveSummary({height: block.height, mined: summary});
+      await BlocksSummariesRepository.$saveTransactions(block.height, block.hash, summary.transactions);
     }
 
     return summary.transactions;
@@ -844,11 +915,12 @@ class Blocks {
         if (cleanBlock.fee_amt_percentiles === null) {
           const block = await bitcoinClient.getBlock(cleanBlock.hash, 2);
           const summary = this.summarizeBlock(block);
-          await BlocksSummariesRepository.$saveSummary({ height: block.height, mined: summary });
+          await BlocksSummariesRepository.$saveTransactions(cleanBlock.height, cleanBlock.hash, summary.transactions);
           cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash);
         }
         if (cleanBlock.fee_amt_percentiles !== null) {
           cleanBlock.median_fee_amt = cleanBlock.fee_amt_percentiles[3];
+          await blocksRepository.$updateFeeAmounts(cleanBlock.hash, cleanBlock.fee_amt_percentiles, cleanBlock.median_fee_amt);
         }
       }
 
@@ -913,42 +985,20 @@ class Blocks {
   public async $indexCPFP(hash: string, height: number): Promise<void> {
     const block = await bitcoinClient.getBlock(hash, 2);
     const transactions = block.tx.map(tx => {
-      tx.vsize = tx.weight / 4;
       tx.fee *= 100_000_000;
       return tx;
     });
 
-    const clusters: any[] = [];
+    const summary = Common.calculateCpfp(height, transactions);
 
-    let cluster: TransactionStripped[] = [];
-    let ancestors: { [txid: string]: boolean } = {};
-    for (let i = transactions.length - 1; i >= 0; i--) {
-      const tx = transactions[i];
-      if (!ancestors[tx.txid]) {
-        let totalFee = 0;
-        let totalVSize = 0;
-        cluster.forEach(tx => {
-          totalFee += tx?.fee || 0;
-          totalVSize += tx.vsize;
-        });
-        const effectiveFeePerVsize = totalFee / totalVSize;
-        if (cluster.length > 1) {
-          clusters.push({
-            root: cluster[0].txid,
-            height,
-            txs: cluster.map(tx => { return { txid: tx.txid, weight: tx.vsize * 4, fee: tx.fee || 0 }; }),
-            effectiveFeePerVsize,
-          });
-        }
-        cluster = [];
-        ancestors = {};
-      }
-      cluster.push(tx);
-      tx.vin.forEach(vin => {
-        ancestors[vin.txid] = true;
-      });
-    }
-    const result = await cpfpRepository.$batchSaveClusters(clusters);
+    await this.$saveCpfp(hash, height, summary);
+
+    const effectiveFeeStats = Common.calcEffectiveFeeStatistics(summary.transactions);
+    await blocksRepository.$saveEffectiveFeeStats(hash, effectiveFeeStats);
+  }
+
+  public async $saveCpfp(hash: string, height: number, cpfpSummary: CpfpSummary): Promise<void> {
+    const result = await cpfpRepository.$batchSaveClusters(cpfpSummary.clusters);
     if (!result) {
       await cpfpRepository.$insertProgressMarker(height);
     }

backend/src/api/common.ts

@@ -1,4 +1,4 @@
-import { CpfpInfo, MempoolBlockWithTransactions, TransactionExtended, TransactionStripped } from '../mempool.interfaces';
+import { Ancestor, CpfpInfo, CpfpSummary, EffectiveFeeStats, MempoolBlockWithTransactions, TransactionExtended, TransactionStripped } from '../mempool.interfaces';
 import config from '../config';
 import { NodeSocket } from '../repositories/NodesSocketsRepository';
 import { isIP } from 'net';
@@ -57,11 +57,11 @@ export class Common {
     return arr;
   }
 
-  static findRbfTransactions(added: TransactionExtended[], deleted: TransactionExtended[]): { [txid: string]: TransactionExtended } {
-    const matches: { [txid: string]: TransactionExtended } = {};
-    deleted
-      .forEach((deletedTx) => {
-        const foundMatches = added.find((addedTx) => {
+  static findRbfTransactions(added: TransactionExtended[], deleted: TransactionExtended[]): { [txid: string]: TransactionExtended[] } {
+    const matches: { [txid: string]: TransactionExtended[] } = {};
+    added
+      .forEach((addedTx) => {
+        const foundMatches = deleted.filter((deletedTx) => {
           // The new tx must, absolutely speaking, pay at least as much fee as the replaced tx.
           return addedTx.fee > deletedTx.fee
             // The new transaction must pay more fee per kB than the replaced tx.
@@ -70,8 +70,8 @@ export class Common {
             && deletedTx.vin.some((deletedVin) =>
               addedTx.vin.some((vin) => vin.txid === deletedVin.txid && vin.vout === deletedVin.vout));
             });
-        if (foundMatches) {
-          matches[deletedTx.txid] = foundMatches;
+        if (foundMatches?.length) {
+          matches[addedTx.txid] = foundMatches;
         }
       });
     return matches;
@@ -83,6 +83,7 @@ export class Common {
       fee: tx.fee,
       vsize: tx.weight / 4,
       value: tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0),
+      rate: tx.effectiveFeePerVsize,
     };
   }
 
@@ -345,4 +346,99 @@ export class Common {
       };
     }
   }
+
+  static calculateCpfp(height: number, transactions: TransactionExtended[]): CpfpSummary {
+    const clusters: { root: string, height: number, txs: Ancestor[], effectiveFeePerVsize: number }[] = [];
+    let cluster: TransactionExtended[] = [];
+    let ancestors: { [txid: string]: boolean } = {};
+    const txMap = {};
+    for (let i = transactions.length - 1; i >= 0; i--) {
+      const tx = transactions[i];
+      txMap[tx.txid] = tx;
+      if (!ancestors[tx.txid]) {
+        let totalFee = 0;
+        let totalVSize = 0;
+        cluster.forEach(tx => {
+          totalFee += tx?.fee || 0;
+          totalVSize += (tx.weight / 4);
+        });
+        const effectiveFeePerVsize = totalFee / totalVSize;
+        if (cluster.length > 1) {
+          clusters.push({
+            root: cluster[0].txid,
+            height,
+            txs: cluster.map(tx => { return { txid: tx.txid, weight: tx.weight, fee: tx.fee || 0 }; }),
+            effectiveFeePerVsize,
+          });
+        }
+        cluster.forEach(tx => {
+          txMap[tx.txid].effectiveFeePerVsize = effectiveFeePerVsize;
+        });
+        cluster = [];
+        ancestors = {};
+      }
+      cluster.push(tx);
+      tx.vin.forEach(vin => {
+        ancestors[vin.txid] = true;
+      });
+    }
+    return {
+      transactions,
+      clusters,
+    };
+  }
+
+  static calcEffectiveFeeStatistics(transactions: { weight: number, fee: number, effectiveFeePerVsize?: number, txid: string }[]): EffectiveFeeStats {
+    const sortedTxs = transactions.map(tx => { return { txid: tx.txid, weight: tx.weight, rate: tx.effectiveFeePerVsize || ((tx.fee || 0) / (tx.weight / 4)) }; }).sort((a, b) => a.rate - b.rate);
+
+    let weightCount = 0;
+    let medianFee = 0;
+    let medianWeight = 0;
+
+    // calculate the "medianFee" as the average fee rate of the middle 10000 weight units of transactions
+    const leftBound = 1995000;
+    const rightBound = 2005000;
+    for (let i = 0; i < sortedTxs.length && weightCount < rightBound; i++) {
+      const left = weightCount;
+      const right = weightCount + sortedTxs[i].weight;
+      if (right > leftBound) {
+        const weight = Math.min(right, rightBound) - Math.max(left, leftBound);
+        medianFee += (sortedTxs[i].rate * (weight / 4) );
+        medianWeight += weight;
+      }
+      weightCount += sortedTxs[i].weight;
+    }
+    const medianFeeRate = medianWeight ? (medianFee / (medianWeight / 4)) : 0;
+
+    // minimum effective fee heuristic:
+    // lowest of
+    // a) the 1st percentile of effective fee rates
+    // b) the minimum effective fee rate in the last 2% of transactions (in block order)
+    const minFee = Math.min(
+      Common.getNthPercentile(1, sortedTxs).rate,
+      transactions.slice(-transactions.length / 50).reduce((min, tx) => { return Math.min(min, tx.effectiveFeePerVsize || ((tx.fee || 0) / (tx.weight / 4))); }, Infinity)
+    );
+
+    // maximum effective fee heuristic:
+    // highest of
+    // a) the 99th percentile of effective fee rates
+    // b) the maximum effective fee rate in the first 2% of transactions (in block order)
+    const maxFee = Math.max(
+      Common.getNthPercentile(99, sortedTxs).rate,
+      transactions.slice(0, transactions.length / 50).reduce((max, tx) => { return Math.max(max, tx.effectiveFeePerVsize || ((tx.fee || 0) / (tx.weight / 4))); }, 0)
+    );
+
+    return {
+      medianFee: medianFeeRate,
+      feeRange: [
+        minFee,
+        [10,25,50,75,90].map(n => Common.getNthPercentile(n, sortedTxs).rate),
+        maxFee,
+      ].flat(),
+    };
+  }
+
+  static getNthPercentile(n: number, sortedDistribution: any[]): any {
+    return sortedDistribution[Math.floor((sortedDistribution.length - 1) * (n / 100))];
+  }
 }

backend/src/api/database-migration.ts

@@ -497,6 +497,7 @@ class DatabaseMigration {
       this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
       await this.$executeQuery('DELETE FROM `pools`');
       await this.$executeQuery('ALTER TABLE pools AUTO_INCREMENT = 1');
+      await this.$executeQuery(`UPDATE state SET string = NULL WHERE name = 'pools_json_sha'`);
       this.uniqueLog(logger.notice, '`pools` table has been truncated`');
       await this.updateToSchemaVersion(56);
     }

backend/src/api/difficulty-adjustment.ts

@@ -24,12 +24,11 @@ export function calcDifficultyAdjustment(
   network: string,
   latestBlockTimestamp: number,
 ): DifficultyAdjustment {
-  const ESTIMATE_LAG_BLOCKS = 146; // For first 7.2% of epoch, don't estimate.
   const EPOCH_BLOCK_LENGTH = 2016; // Bitcoin mainnet
   const BLOCK_SECONDS_TARGET = 600; // Bitcoin mainnet
   const TESTNET_MAX_BLOCK_SECONDS = 1200; // Bitcoin testnet
 
-  const diffSeconds = nowSeconds - DATime;
+  const diffSeconds = Math.max(0, nowSeconds - DATime);
   const blocksInEpoch = (blockHeight >= 0) ? blockHeight % EPOCH_BLOCK_LENGTH : 0;
   const progressPercent = (blockHeight >= 0) ? blocksInEpoch / EPOCH_BLOCK_LENGTH * 100 : 100;
   const remainingBlocks = EPOCH_BLOCK_LENGTH - blocksInEpoch;
@@ -37,18 +36,16 @@ export function calcDifficultyAdjustment(
   const expectedBlocks = diffSeconds / BLOCK_SECONDS_TARGET;
 
   let difficultyChange = 0;
-  let timeAvgSecs = diffSeconds / blocksInEpoch;
-  // Only calculate the estimate once we have 7.2% of blocks in current epoch
-  if (blocksInEpoch >= ESTIMATE_LAG_BLOCKS) {
-    difficultyChange = (BLOCK_SECONDS_TARGET / timeAvgSecs - 1) * 100;
-    // Max increase is x4 (+300%)
-    if (difficultyChange > 300) {
-      difficultyChange = 300;
-    }
-    // Max decrease is /4 (-75%)
-    if (difficultyChange < -75) {
-      difficultyChange = -75;
-    }
+  let timeAvgSecs = blocksInEpoch ? diffSeconds / blocksInEpoch : BLOCK_SECONDS_TARGET;
+
+  difficultyChange = (BLOCK_SECONDS_TARGET / timeAvgSecs - 1) * 100;
+  // Max increase is x4 (+300%)
+  if (difficultyChange > 300) {
+    difficultyChange = 300;
+  }
+  // Max decrease is /4 (-75%)
+  if (difficultyChange < -75) {
+    difficultyChange = -75;
   }
 
   // Testnet difficulty is set to 1 after 20 minutes of no blocks,

backend/src/api/disk-cache.ts

@@ -7,32 +7,37 @@ import logger from '../logger';
 import config from '../config';
 import { TransactionExtended } from '../mempool.interfaces';
 import { Common } from './common';
+import rbfCache from './rbf-cache';
 
 class DiskCache {
   private cacheSchemaVersion = 3;
+  private rbfCacheSchemaVersion = 1;
 
   private static TMP_FILE_NAME = config.MEMPOOL.CACHE_DIR + '/tmp-cache.json';
   private static TMP_FILE_NAMES = config.MEMPOOL.CACHE_DIR + '/tmp-cache{number}.json';
   private static FILE_NAME = config.MEMPOOL.CACHE_DIR + '/cache.json';
   private static FILE_NAMES = config.MEMPOOL.CACHE_DIR + '/cache{number}.json';
+  private static TMP_RBF_FILE_NAME = config.MEMPOOL.CACHE_DIR + '/tmp-rbfcache.json';
+  private static RBF_FILE_NAME = config.MEMPOOL.CACHE_DIR + '/rbfcache.json';
   private static CHUNK_FILES = 25;
   private isWritingCache = false;
 
+  private semaphore: { resume: (() => void)[], locks: number } = {
+    resume: [],
+    locks: 0,
+  };
+
   constructor() {
-    if (!cluster.isMaster) {
+    if (!cluster.isPrimary) {
       return;
     }
     process.on('SIGINT', (e) => {
-      this.saveCacheToDiskSync();
-      process.exit(2);
-    });
-    process.on('SIGTERM', (e) => {
-      this.saveCacheToDiskSync();
-      process.exit(2);
+      this.$saveCacheToDisk(true);
+      process.exit(0);
     });
   }
 
-  async $saveCacheToDisk(): Promise<void> {
+  async $saveCacheToDisk(sync: boolean = false): Promise<void> {
     if (!cluster.isPrimary) {
       return;
     }
@@ -41,87 +46,95 @@ class DiskCache {
       return;
     }
     try {
-      logger.debug('Writing mempool and blocks data to disk cache (async)...');
+      logger.debug(`Writing mempool and blocks data to disk cache (${ sync ? 'sync' : 'async' })...`);
       this.isWritingCache = true;
 
       const mempool = memPool.getMempool();
       const mempoolArray: TransactionExtended[] = [];
       for (const tx in mempool) {
-        mempoolArray.push(mempool[tx]);
+        if (mempool[tx]) {
+          mempoolArray.push(mempool[tx]);
+        }
       }
 
       Common.shuffleArray(mempoolArray);
 
       const chunkSize = Math.floor(mempoolArray.length / DiskCache.CHUNK_FILES);
 
-      await fsPromises.writeFile(DiskCache.FILE_NAME, JSON.stringify({
-        network: config.MEMPOOL.NETWORK,
-        cacheSchemaVersion: this.cacheSchemaVersion,
-        blocks: blocks.getBlocks(),
-        blockSummaries: blocks.getBlockSummaries(),
-        mempool: {},
-        mempoolArray: mempoolArray.splice(0, chunkSize),
-      }), { flag: 'w' });
-      for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
-        await fsPromises.writeFile(DiskCache.FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
+      if (sync) {
+        fs.writeFileSync(DiskCache.TMP_FILE_NAME, JSON.stringify({
+          network: config.MEMPOOL.NETWORK,
+          cacheSchemaVersion: this.cacheSchemaVersion,
+          blocks: blocks.getBlocks(),
+          blockSummaries: blocks.getBlockSummaries(),
           mempool: {},
           mempoolArray: mempoolArray.splice(0, chunkSize),
         }), { flag: 'w' });
+        for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
+          fs.writeFileSync(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
+            mempool: {},
+            mempoolArray: mempoolArray.splice(0, chunkSize),
+          }), { flag: 'w' });
+        }
+
+        fs.renameSync(DiskCache.TMP_FILE_NAME, DiskCache.FILE_NAME);
+        for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
+          fs.renameSync(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), DiskCache.FILE_NAMES.replace('{number}', i.toString()));
+        }
+      } else {
+        await this.$yield();
+        await fsPromises.writeFile(DiskCache.TMP_FILE_NAME, JSON.stringify({
+          network: config.MEMPOOL.NETWORK,
+          cacheSchemaVersion: this.cacheSchemaVersion,
+          blocks: blocks.getBlocks(),
+          blockSummaries: blocks.getBlockSummaries(),
+          mempool: {},
+          mempoolArray: mempoolArray.splice(0, chunkSize),
+        }), { flag: 'w' });
+        for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
+          await this.$yield();
+          await fsPromises.writeFile(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
+            mempool: {},
+            mempoolArray: mempoolArray.splice(0, chunkSize),
+          }), { flag: 'w' });
+        }
+
+        await fsPromises.rename(DiskCache.TMP_FILE_NAME, DiskCache.FILE_NAME);
+        for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
+          await fsPromises.rename(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), DiskCache.FILE_NAMES.replace('{number}', i.toString()));
+        }
       }
+
       logger.debug('Mempool and blocks data saved to disk cache');
       this.isWritingCache = false;
     } catch (e) {
       logger.warn('Error writing to cache file: ' + (e instanceof Error ? e.message : e));
       this.isWritingCache = false;
     }
-  }
 
-  saveCacheToDiskSync(): void {
-    if (!cluster.isPrimary) {
-      return;
-    }
-    if (this.isWritingCache) {
-      logger.debug('Saving cache already in progress. Skipping.');
-      return;
-    }
     try {
-      logger.debug('Writing mempool and blocks data to disk cache (sync)...');
+      logger.debug('Writing rbf data to disk cache (async)...');
       this.isWritingCache = true;
-
-      const mempool = memPool.getMempool();
-      const mempoolArray: TransactionExtended[] = [];
-      for (const tx in mempool) {
-        mempoolArray.push(mempool[tx]);
-      }
-
-      Common.shuffleArray(mempoolArray);
-
-      const chunkSize = Math.floor(mempoolArray.length / DiskCache.CHUNK_FILES);
-
-      fs.writeFileSync(DiskCache.TMP_FILE_NAME, JSON.stringify({
-        network: config.MEMPOOL.NETWORK,
-        cacheSchemaVersion: this.cacheSchemaVersion,
-        blocks: blocks.getBlocks(),
-        blockSummaries: blocks.getBlockSummaries(),
-        mempool: {},
-        mempoolArray: mempoolArray.splice(0, chunkSize),
-      }), { flag: 'w' });
-      for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
-        fs.writeFileSync(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
-          mempool: {},
-          mempoolArray: mempoolArray.splice(0, chunkSize),
+      const rbfData = rbfCache.dump();
+      if (sync) {
+        fs.writeFileSync(DiskCache.TMP_RBF_FILE_NAME, JSON.stringify({
+          network: config.MEMPOOL.NETWORK,
+          rbfCacheSchemaVersion: this.rbfCacheSchemaVersion,
+          rbf: rbfData,
         }), { flag: 'w' });
+        fs.renameSync(DiskCache.TMP_RBF_FILE_NAME, DiskCache.RBF_FILE_NAME);
+      } else {
+        await fsPromises.writeFile(DiskCache.TMP_RBF_FILE_NAME, JSON.stringify({
+          network: config.MEMPOOL.NETWORK,
+          rbfCacheSchemaVersion: this.rbfCacheSchemaVersion,
+          rbf: rbfData,
+        }), { flag: 'w' });
+        await fsPromises.rename(DiskCache.TMP_RBF_FILE_NAME, DiskCache.RBF_FILE_NAME);
       }
-
-      fs.renameSync(DiskCache.TMP_FILE_NAME, DiskCache.FILE_NAME);
-      for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
-        fs.renameSync(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), DiskCache.FILE_NAMES.replace('{number}', i.toString()));
-      }
-
-      logger.debug('Mempool and blocks data saved to disk cache');
+      logger.debug('Rbf data saved to disk cache');
       this.isWritingCache = false;
     } catch (e) {
-      logger.warn('Error writing to cache file: ' + (e instanceof Error ? e.message : e));
+      logger.warn('Error writing rbf data to cache file: ' + (e instanceof Error ? e.message : e));
       this.isWritingCache = false;
     }
   }
@@ -148,7 +161,19 @@ class DiskCache {
     }
   }
 
-  loadMempoolCache(): void {
+  wipeRbfCache() {
+    logger.notice(`Wipping nodejs backend cache/rbfcache.json file`);
+
+    try {
+      fs.unlinkSync(DiskCache.RBF_FILE_NAME);
+    } catch (e: any) {
+      if (e?.code !== 'ENOENT') {
+        logger.err(`Cannot wipe cache file ${DiskCache.RBF_FILE_NAME}. Exception ${JSON.stringify(e)}`);
+      }
+    }
+  }
+
+  async $loadMempoolCache(): Promise<void> {
     if (!fs.existsSync(DiskCache.FILE_NAME)) {
       return;
     }
@@ -188,16 +213,65 @@ class DiskCache {
             }
           }
         } catch (e) {
-          logger.info('Error parsing ' + fileName + '. Skipping. Reason: ' + (e instanceof Error ? e.message : e));
+          logger.err('Error parsing ' + fileName + '. Skipping. Reason: ' + (e instanceof Error ? e.message : e));
         }
       }
 
-      memPool.setMempool(data.mempool);
+      await memPool.$setMempool(data.mempool);
       blocks.setBlocks(data.blocks);
       blocks.setBlockSummaries(data.blockSummaries || []);
     } catch (e) {
       logger.warn('Failed to parse mempoool and blocks cache. Skipping. Reason: ' + (e instanceof Error ? e.message : e));
     }
+
+    try {
+      let rbfData: any = {};
+      const rbfCacheData = fs.readFileSync(DiskCache.RBF_FILE_NAME, 'utf8');
+      if (rbfCacheData) {
+        logger.info('Restoring rbf data from disk cache');
+        rbfData = JSON.parse(rbfCacheData);
+        if (rbfData.rbfCacheSchemaVersion === undefined || rbfData.rbfCacheSchemaVersion !== this.rbfCacheSchemaVersion) {
+          logger.notice('Rbf disk cache contains an outdated schema version. Clearing it and skipping the cache loading.');
+          return this.wipeRbfCache();
+        }
+        if (rbfData.network && rbfData.network !== config.MEMPOOL.NETWORK) {
+          logger.notice('Rbf disk cache contains data from a different network. Clearing it and skipping the cache loading.');
+          return this.wipeRbfCache();
+        }
+      }
+
+      if (rbfData?.rbf) {
+        rbfCache.load(rbfData.rbf);
+      }
+    } catch (e) {
+      logger.warn('Failed to parse rbf cache. Skipping. Reason: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  private $yield(): Promise<void> {
+    if (this.semaphore.locks) {
+      logger.debug('Pause writing mempool and blocks data to disk cache (async)');
+      return new Promise((resolve) => {
+        this.semaphore.resume.push(resolve);
+      });
+    } else {
+      return Promise.resolve();
+    }
+  }
+
+  public lock(): void {
+    this.semaphore.locks++;
+  }
+
+  public unlock(): void {
+    this.semaphore.locks = Math.max(0, this.semaphore.locks - 1);
+    if (!this.semaphore.locks && this.semaphore.resume.length) {
+      const nextResume = this.semaphore.resume.shift();
+      if (nextResume) {
+        logger.debug('Resume writing mempool and blocks data to disk cache (async)');
+        nextResume();
+      }
+    }
   }
 }
 

backend/src/api/lightning/lnd/lnd-api.ts

@@ -4,21 +4,29 @@ import * as fs from 'fs';
 import { AbstractLightningApi } from '../lightning-api-abstract-factory';
 import { ILightningApi } from '../lightning-api.interface';
 import config from '../../../config';
+import logger from '../../../logger';
 
 class LndApi implements AbstractLightningApi {
   axiosConfig: AxiosRequestConfig = {};
 
   constructor() {
-    if (config.LIGHTNING.ENABLED) {
+    if (!config.LIGHTNING.ENABLED) {
+      return;
+    }
+    try {
       this.axiosConfig = {
         headers: {
-          'Grpc-Metadata-macaroon': fs.readFileSync(config.LND.MACAROON_PATH).toString('hex')
+          'Grpc-Metadata-macaroon': fs.readFileSync(config.LND.MACAROON_PATH).toString('hex'),
         },
         httpsAgent: new Agent({
           ca: fs.readFileSync(config.LND.TLS_CERT_PATH)
         }),
-        timeout: 10000
+        timeout: config.LND.TIMEOUT
       };
+    } catch (e) {
+      config.LIGHTNING.ENABLED = false;
+      logger.updateNetwork();
+      logger.err(`Could not initialize LND Macaroon/TLS Cert. Disabling LIGHTNING. ` + (e instanceof Error ? e.message : e));
     }
   }
 

backend/src/api/liquid/icons.ts

@@ -2,7 +2,7 @@ import * as fs from 'fs';
 import logger from '../../logger';
 
 class Icons {
-  private static FILE_NAME = './icons.json';
+  private static FILE_NAME = '/elements/asset_registry_db/icons.json';
   private iconIds: string[] = [];
   private icons: { [assetId: string]: string; } = {};
 

backend/src/api/mempool-blocks.ts

@@ -1,6 +1,7 @@
 import logger from '../logger';
 import { MempoolBlock, TransactionExtended, ThreadTransaction, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor } from '../mempool.interfaces';
 import { Common } from './common';
+import Mempool from './mempool';
 import config from '../config';
 import { Worker } from 'worker_threads';
 import path from 'path';
@@ -10,6 +11,9 @@ class MempoolBlocks {
   private mempoolBlockDeltas: MempoolBlockDelta[] = [];
   private txSelectionWorker: Worker | null = null;
 
+  private filteredTxs: Map<string, TransactionExtended> = new Map();
+  private minFee: number = 0;
+
   constructor() {}
 
   public getMempoolBlocks(): MempoolBlock[] {
@@ -54,12 +58,19 @@ class MempoolBlocks {
     });
 
     // First sort
-    memPoolArray.sort((a, b) => b.feePerVsize - a.feePerVsize);
+    memPoolArray.sort((a, b) => {
+      if (a.feePerVsize === b.feePerVsize) {
+        // tie-break by lexicographic txid order for stability
+        return a.txid < b.txid ? -1 : 1;
+      } else {
+        return b.feePerVsize - a.feePerVsize;
+      }
+    });
 
     // Loop through and traverse all ancestors and sum up all the sizes + fees
     // Pass down size + fee to all unconfirmed children
     let sizes = 0;
-    memPoolArray.forEach((tx, i) => {
+    memPoolArray.forEach((tx) => {
       sizes += tx.weight;
       if (sizes > 4000000 * 8) {
         return;
@@ -68,43 +79,60 @@ class MempoolBlocks {
     });
 
     // Final sort, by effective fee
-    memPoolArray.sort((a, b) => b.effectiveFeePerVsize - a.effectiveFeePerVsize);
+    memPoolArray.sort((a, b) => {
+      if (a.effectiveFeePerVsize === b.effectiveFeePerVsize) {
+        // tie-break by lexicographic txid order for stability
+        return a.txid < b.txid ? -1 : 1;
+      } else {
+        return b.effectiveFeePerVsize - a.effectiveFeePerVsize;
+      }
+    });
 
     const end = new Date().getTime();
     const time = end - start;
     logger.debug('Mempool blocks calculated in ' + time / 1000 + ' seconds');
 
-    const blocks = this.calculateMempoolBlocks(memPoolArray, this.mempoolBlocks);
+    const blocks = this.calculateMempoolBlocks(memPoolArray);
 
     if (saveResults) {
       const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, blocks);
       this.mempoolBlocks = blocks;
       this.mempoolBlockDeltas = deltas;
+      this.updateMinFee(this.mempoolBlocks);
     }
 
     return blocks;
   }
 
-  private calculateMempoolBlocks(transactionsSorted: TransactionExtended[], prevBlocks: MempoolBlockWithTransactions[]): MempoolBlockWithTransactions[] {
+  private calculateMempoolBlocks(transactionsSorted: TransactionExtended[]): MempoolBlockWithTransactions[] {
     const mempoolBlocks: MempoolBlockWithTransactions[] = [];
     let blockWeight = 0;
-    let blockSize = 0;
+    let blockVsize = 0;
     let transactions: TransactionExtended[] = [];
     transactionsSorted.forEach((tx) => {
       if (blockWeight + tx.weight <= config.MEMPOOL.BLOCK_WEIGHT_UNITS
         || mempoolBlocks.length === config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT - 1) {
+        tx.position = {
+          block: mempoolBlocks.length,
+          vsize: blockVsize + (tx.vsize / 2),
+        };
         blockWeight += tx.weight;
-        blockSize += tx.size;
+        blockVsize += tx.vsize;
         transactions.push(tx);
       } else {
-        mempoolBlocks.push(this.dataToMempoolBlocks(transactions, mempoolBlocks.length));
+        mempoolBlocks.push(this.dataToMempoolBlocks(transactions));
+        blockVsize = 0;
+        tx.position = {
+          block: mempoolBlocks.length,
+          vsize: blockVsize + (tx.vsize / 2),
+        };
+        blockVsize += tx.vsize;
         blockWeight = tx.weight;
-        blockSize = tx.size;
         transactions = [tx];
       }
     });
     if (transactions.length) {
-      mempoolBlocks.push(this.dataToMempoolBlocks(transactions, mempoolBlocks.length));
+      mempoolBlocks.push(this.dataToMempoolBlocks(transactions));
     }
 
     return mempoolBlocks;
@@ -115,6 +143,7 @@ class MempoolBlocks {
     for (let i = 0; i < Math.max(mempoolBlocks.length, prevBlocks.length); i++) {
       let added: TransactionStripped[] = [];
       let removed: string[] = [];
+      const changed: { txid: string, rate: number | undefined }[] = [];
       if (mempoolBlocks[i] && !prevBlocks[i]) {
         added = mempoolBlocks[i].transactions;
       } else if (!mempoolBlocks[i] && prevBlocks[i]) {
@@ -123,7 +152,7 @@ class MempoolBlocks {
         const prevIds = {};
         const newIds = {};
         prevBlocks[i].transactions.forEach(tx => {
-          prevIds[tx.txid] = true;
+          prevIds[tx.txid] = tx;
         });
         mempoolBlocks[i].transactions.forEach(tx => {
           newIds[tx.txid] = true;
@@ -136,30 +165,44 @@ class MempoolBlocks {
         mempoolBlocks[i].transactions.forEach(tx => {
           if (!prevIds[tx.txid]) {
             added.push(tx);
+          } else if (tx.rate !== prevIds[tx.txid].rate) {
+            changed.push({ txid: tx.txid, rate: tx.rate });
           }
         });
       }
       mempoolBlockDeltas.push({
         added,
-        removed
+        removed,
+        changed,
       });
     }
     return mempoolBlockDeltas;
   }
 
-  public async makeBlockTemplates(newMempool: { [txid: string]: TransactionExtended }, saveResults: boolean = false): Promise<MempoolBlockWithTransactions[]> {
+  public async $makeBlockTemplates(newMempool: { [txid: string]: TransactionExtended }, saveResults: boolean = false): Promise<MempoolBlockWithTransactions[]> {
+    // Identify txs that can't be CPFP'd
+    this.markRelatives(newMempool);
+
+    // Track filtered transactions
+    this.filteredTxs.clear();
+    const filterFee = this.getFilterFee();
+ 
     // prepare a stripped down version of the mempool with only the minimum necessary data
     // to reduce the overhead of passing this data to the worker thread
     const strippedMempool: { [txid: string]: ThreadTransaction } = {};
     Object.values(newMempool).forEach(entry => {
-      strippedMempool[entry.txid] = {
-        txid: entry.txid,
-        fee: entry.fee,
-        weight: entry.weight,
-        feePerVsize: entry.fee / (entry.weight / 4),
-        effectiveFeePerVsize: entry.fee / (entry.weight / 4),
-        vin: entry.vin.map(v => v.txid),
-      };
+      if (entry.hasRelatives || entry.feePerVsize >= filterFee) {
+        strippedMempool[entry.txid] = {
+          txid: entry.txid,
+          fee: entry.fee,
+          weight: entry.weight,
+          feePerVsize: entry.fee / (entry.weight / 4),
+          effectiveFeePerVsize: entry.fee / (entry.weight / 4),
+          vin: entry.vin.map(v => v.txid),
+        };
+      } else {
+        this.filteredTxs.set(entry.txid, entry);
+      }
     });
 
     // (re)initialize tx selection worker thread
@@ -186,7 +229,14 @@ class MempoolBlocks {
         this.txSelectionWorker?.once('error', reject);
       });
       this.txSelectionWorker.postMessage({ type: 'set', mempool: strippedMempool });
-      const { blocks, clusters } = await workerResultPromise;
+      let { blocks, clusters } = await workerResultPromise;
+      // filter out stale transactions
+      const unfilteredCount = blocks.reduce((total, block) => { return total + block.length; }, 0);
+      blocks = blocks.map(block => block.filter(tx => (tx.txid && tx.txid in newMempool)));
+      const filteredCount = blocks.reduce((total, block) => { return total + block.length; }, 0);
+      if (filteredCount < unfilteredCount) {
+        logger.warn(`tx selection worker thread returned ${unfilteredCount - filteredCount} stale transactions from makeBlockTemplates`);
+      }
 
       // clean up thread error listener
       this.txSelectionWorker?.removeListener('error', threadErrorListener);
@@ -198,15 +248,20 @@ class MempoolBlocks {
     return this.mempoolBlocks;
   }
 
-  public async updateBlockTemplates(newMempool: { [txid: string]: TransactionExtended }, added: TransactionExtended[], removed: string[], saveResults: boolean = false): Promise<void> {
+  public async $updateBlockTemplates(newMempool: { [txid: string]: TransactionExtended }, added: TransactionExtended[], removed: string[], saveResults: boolean = false): Promise<void> {
     if (!this.txSelectionWorker) {
       // need to reset the worker
-      this.makeBlockTemplates(newMempool, saveResults);
+      await this.$makeBlockTemplates(newMempool, saveResults);
       return;
     }
+
+
+    this.updateMarkedRelatives(newMempool, added);
+    const filterDiff = this.updateFilteredTxs(newMempool, this.getFilterFee());
+
     // prepare a stripped down version of the mempool with only the minimum necessary data
     // to reduce the overhead of passing this data to the worker thread
-    const addedStripped: ThreadTransaction[] = added.map(entry => {
+    const addedStripped: ThreadTransaction[] = added.concat(filterDiff.included).map(entry => {
       return {
         txid: entry.txid,
         fee: entry.fee,
@@ -216,6 +271,7 @@ class MempoolBlocks {
         vin: entry.vin.map(v => v.txid),
       };
     });
+    const removedIds = removed.concat(filterDiff.filtered);
 
     // run the block construction algorithm in a separate thread, and wait for a result
     let threadErrorListener;
@@ -227,8 +283,15 @@ class MempoolBlocks {
         });
         this.txSelectionWorker?.once('error', reject);
       });
-      this.txSelectionWorker.postMessage({ type: 'update', added: addedStripped, removed });
-      const { blocks, clusters } = await workerResultPromise;
+      this.txSelectionWorker.postMessage({ type: 'update', added: addedStripped, removed: removedIds });
+      let { blocks, clusters } = await workerResultPromise;
+      // filter out stale transactions
+      const unfilteredCount = blocks.reduce((total, block) => { return total + block.length; }, 0);
+      blocks = blocks.map(block => block.filter(tx => (tx.txid && tx.txid in newMempool)));
+      const filteredCount = blocks.reduce((total, block) => { return total + block.length; }, 0);
+      if (filteredCount < unfilteredCount) {
+        logger.warn(`tx selection worker thread returned ${unfilteredCount - filteredCount} stale transactions from updateBlockTemplates`);
+      }
 
       // clean up thread error listener
       this.txSelectionWorker?.removeListener('error', threadErrorListener);
@@ -241,9 +304,17 @@ class MempoolBlocks {
 
   private processBlockTemplates(mempool, blocks, clusters, saveResults): MempoolBlockWithTransactions[] {
     // update this thread's mempool with the results
-    blocks.forEach(block => {
+    blocks.forEach((block, blockIndex) => {
+      let runningVsize = 0;
       block.forEach(tx => {
-        if (tx.txid in mempool) {
+        if (tx.txid && tx.txid in mempool) {
+          // save position in projected blocks
+          mempool[tx.txid].position = {
+            block: blockIndex,
+            vsize: runningVsize + (mempool[tx.txid].vsize / 2),
+          };
+          runningVsize += mempool[tx.txid].vsize;
+
           if (tx.effectiveFeePerVsize != null) {
             mempool[tx.txid].effectiveFeePerVsize = tx.effectiveFeePerVsize;
           }
@@ -253,6 +324,10 @@ class MempoolBlocks {
             const cluster = clusters[tx.cpfpRoot];
             let matched = false;
             cluster.forEach(txid => {
+              if (!txid || !mempool[txid]) {
+                logger.warn('projected transaction ancestor missing from mempool cache');
+                return;
+              }
               if (txid === tx.txid) {
                 matched = true;
               } else {
@@ -273,27 +348,35 @@ class MempoolBlocks {
             mempool[tx.txid].bestDescendant = null;
           }
           mempool[tx.txid].cpfpChecked = tx.cpfpChecked;
+        } else {
+          logger.warn('projected transaction missing from mempool cache');
         }
       });
     });
 
+    if (blocks.length) {
+      const sortedFiltered = Array.from(this.filteredTxs.values()).sort((a, b) => b.feePerVsize - a.feePerVsize);
+      blocks[blocks.length - 1] = blocks[blocks.length - 1].concat(sortedFiltered);
+    }
+
     // unpack the condensed blocks into proper mempool blocks
-    const mempoolBlocks = blocks.map((transactions, blockIndex) => {
+    const mempoolBlocks = blocks.map((transactions) => {
       return this.dataToMempoolBlocks(transactions.map(tx => {
         return mempool[tx.txid] || null;
-      }).filter(tx => !!tx), blockIndex);
+      }).filter(tx => !!tx));
     });
 
     if (saveResults) {
       const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, mempoolBlocks);
       this.mempoolBlocks = mempoolBlocks;
       this.mempoolBlockDeltas = deltas;
+      this.updateMinFee(this.mempoolBlocks);
     }
 
     return mempoolBlocks;
   }
 
-  private dataToMempoolBlocks(transactions: TransactionExtended[], blocksIndex: number): MempoolBlockWithTransactions {
+  private dataToMempoolBlocks(transactions: TransactionExtended[]): MempoolBlockWithTransactions {
     let totalSize = 0;
     let totalWeight = 0;
     const fitTransactions: TransactionExtended[] = [];
@@ -304,26 +387,96 @@ class MempoolBlocks {
         fitTransactions.push(tx);
       }
     });
-    let rangeLength = 4;
-    if (blocksIndex === 0) {
-      rangeLength = 8;
-    }
-    if (transactions.length > 4000) {
-      rangeLength = 6;
-    } else if (transactions.length > 10000) {
-      rangeLength = 8;
-    }
+    const feeStats = Common.calcEffectiveFeeStatistics(transactions);
     return {
       blockSize: totalSize,
       blockVSize: totalWeight / 4,
       nTx: transactions.length,
       totalFees: transactions.reduce((acc, cur) => acc + cur.fee, 0),
-      medianFee: Common.percentile(transactions.map((tx) => tx.effectiveFeePerVsize), config.MEMPOOL.RECOMMENDED_FEE_PERCENTILE),
-      feeRange: Common.getFeesInRange(transactions, rangeLength),
+      medianFee: feeStats.medianFee, // Common.percentile(transactions.map((tx) => tx.effectiveFeePerVsize), config.MEMPOOL.RECOMMENDED_FEE_PERCENTILE),
+      feeRange: feeStats.feeRange, //Common.getFeesInRange(transactions, rangeLength),
       transactionIds: transactions.map((tx) => tx.txid),
       transactions: fitTransactions.map((tx) => Common.stripTransaction(tx)),
     };
   }
+
+  // Mark all transactions with in-mempool relatives
+  private markRelatives(mempool: { [txid: string]: TransactionExtended }): void {
+    for (const tx of Object.values(mempool)) {
+      if (!tx.hasRelatives) {
+        let hasRelatives = false;
+        tx.vin.forEach(parent => {
+          if (mempool[parent.txid] != null) {
+            hasRelatives = true;
+            mempool[parent.txid].hasRelatives = true;
+          } 
+        });
+        tx.hasRelatives = hasRelatives;
+      }
+    }
+  }
+
+  private updateMarkedRelatives(mempool: { [txid: string]: TransactionExtended }, added: TransactionExtended[]): void {
+    const newFiltered: TransactionExtended[] = [];
+    for (const tx of added) {
+      if (!tx.hasRelatives) {
+        let hasRelatives = false;
+        tx.vin.forEach(parent => {
+          if (mempool[parent.txid] != null) {
+            hasRelatives = true;
+            mempool[parent.txid].hasRelatives = true;
+          } 
+        });
+        tx.hasRelatives = hasRelatives;
+      }
+    }
+  }
+
+  private updateFilteredTxs(mempool: { [txid: string]: TransactionExtended }, filterFee: number): { filtered: string[], included: TransactionExtended[] } {
+    const filtered: string[] = [];
+    const included: TransactionExtended[] = [];
+    for (const tx of Object.values(mempool)) {
+      if (!tx.hasRelatives) {
+        if (tx.feePerVsize < filterFee) {
+          // filter out tx
+          if (!this.filteredTxs.has(tx.txid)) {
+            this.filteredTxs.set(tx.txid, tx);
+            filtered.push(tx.txid);
+          }
+        } else {
+          // include tx
+          if (this.filteredTxs.has(tx.txid)) {
+            this.filteredTxs.delete(tx.txid);
+            included.push(tx);
+          }
+        }
+      }
+    }
+    return { filtered, included };
+  }
+
+  private updateMinFee(mempoolBlocks: MempoolBlockWithTransactions[]): void {
+    let totalFeeRate = 0;
+    let totalSize = 0;
+    let count = 0;
+    if (mempoolBlocks.length === 8) {
+      const lastBlock = mempoolBlocks[mempoolBlocks.length - 1];
+      for (let i = 0; i < lastBlock.transactions.length && totalSize < 16_000_000; i++) {
+        totalFeeRate += lastBlock.transactions[i].rate || (lastBlock.transactions[i].fee / lastBlock.transactions[i].vsize);
+        totalSize += lastBlock.transactions[i].vsize;
+        count++;
+      }
+      this.minFee = count ? (totalFeeRate / count) : 1;
+    } else {
+      this.minFee = 1;
+    }
+  }
+
+  private getFilterFee(): number {
+    const purgingBelow = Mempool.getMempoolInfo().mempoolminfee * 100000;
+    const filterFee = Math.max(purgingBelow, this.minFee);
+    return filterFee;
+  }
 }
 
 export default new MempoolBlocks();

backend/src/api/mempool.ts

@@ -12,7 +12,6 @@ import rbfCache from './rbf-cache';
 
 class Mempool {
   private static WEBSOCKET_REFRESH_RATE_MS = 10000;
-  private static LAZY_DELETE_AFTER_SECONDS = 30;
   private inSync: boolean = false;
   private mempoolCacheDelta: number = -1;
   private mempoolCache: { [txId: string]: TransactionExtended } = {};
@@ -20,7 +19,7 @@ class Mempool {
                                                     maxmempool: 300000000, mempoolminfee: 0.00001000, minrelaytxfee: 0.00001000 };
   private mempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[],
     deletedTransactions: TransactionExtended[]) => void) | undefined;
-  private asyncMempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[],
+  private $asyncMempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[],
     deletedTransactions: TransactionExtended[]) => Promise<void>) | undefined;
 
   private txPerSecondArray: number[] = [];
@@ -36,9 +35,10 @@ class Mempool {
   private timer = new Date().getTime();
   private missingTxCount = 0;
 
+  private mainLoopTimeout: number = 120000;
+
   constructor() {
     setInterval(this.updateTxPerSecond.bind(this), 1000);
-    setInterval(this.deleteExpiredTransactions.bind(this), 20000);
   }
 
   /**
@@ -72,20 +72,20 @@ class Mempool {
 
   public setAsyncMempoolChangedCallback(fn: (newMempool: { [txId: string]: TransactionExtended; },
     newTransactions: TransactionExtended[], deletedTransactions: TransactionExtended[]) => Promise<void>) {
-    this.asyncMempoolChangedCallback = fn;
+    this.$asyncMempoolChangedCallback = fn;
   }
 
   public getMempool(): { [txid: string]: TransactionExtended } {
     return this.mempoolCache;
   }
 
-  public setMempool(mempoolData: { [txId: string]: TransactionExtended }) {
+  public async $setMempool(mempoolData: { [txId: string]: TransactionExtended }) {
     this.mempoolCache = mempoolData;
     if (this.mempoolChangedCallback) {
       this.mempoolChangedCallback(this.mempoolCache, [], []);
     }
-    if (this.asyncMempoolChangedCallback) {
-      this.asyncMempoolChangedCallback(this.mempoolCache, [], []);
+    if (this.$asyncMempoolChangedCallback) {
+      await this.$asyncMempoolChangedCallback(this.mempoolCache, [], []);
     }
   }
 
@@ -118,12 +118,16 @@ class Mempool {
     return txTimes;
   }
 
-  public async $updateMempool(): Promise<void> {
+  public async $updateMempool(transactions: string[]): Promise<void> {
     logger.debug(`Updating mempool...`);
+
+    // warn if this run stalls the main loop for more than 2 minutes
+    const timer = this.startTimer();
+
     const start = new Date().getTime();
     let hasChange: boolean = false;
     const currentMempoolSize = Object.keys(this.mempoolCache).length;
-    const transactions = await bitcoinApi.$getRawMempool();
+    this.updateTimerProgress(timer, 'got raw mempool');
     const diff = transactions.length - currentMempoolSize;
     const newTransactions: TransactionExtended[] = [];
 
@@ -147,6 +151,7 @@ class Mempool {
       if (!this.mempoolCache[txid]) {
         try {
           const transaction = await transactionUtils.$getTransactionExtended(txid);
+          this.updateTimerProgress(timer, 'fetched new transaction');
           this.mempoolCache[txid] = transaction;
           if (this.inSync) {
             this.txPerSecondArray.push(new Date().getTime());
@@ -200,13 +205,15 @@ class Mempool {
       const transactionsObject = {};
       transactions.forEach((txId) => transactionsObject[txId] = true);
 
-      // Flag transactions for lazy deletion
+      // Delete evicted transactions from mempool
       for (const tx in this.mempoolCache) {
-        if (!transactionsObject[tx] && !this.mempoolCache[tx].deleteAfter) {
+        if (!transactionsObject[tx]) {
           deletedTransactions.push(this.mempoolCache[tx]);
-          this.mempoolCache[tx].deleteAfter = new Date().getTime() + Mempool.LAZY_DELETE_AFTER_SECONDS * 1000;
         }
       }
+      for (const tx of deletedTransactions) {
+        delete this.mempoolCache[tx.txid];
+      }
     }
 
     const newTransactionsStripped = newTransactions.map((tx) => Common.stripTransaction(tx));
@@ -223,22 +230,46 @@ class Mempool {
     if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) {
       this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions);
     }
-    if (this.asyncMempoolChangedCallback && (hasChange || deletedTransactions.length)) {
-      await this.asyncMempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions);
+    if (this.$asyncMempoolChangedCallback && (hasChange || deletedTransactions.length)) {
+      this.updateTimerProgress(timer, 'running async mempool callback');
+      await this.$asyncMempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions);
+      this.updateTimerProgress(timer, 'completed async mempool callback');
     }
 
     const end = new Date().getTime();
     const time = end - start;
     logger.debug(`Mempool updated in ${time / 1000} seconds. New size: ${Object.keys(this.mempoolCache).length} (${diff > 0 ? '+' + diff : diff})`);
+
+    this.clearTimer(timer);
   }
 
-  public handleRbfTransactions(rbfTransactions: { [txid: string]: TransactionExtended; }) {
+  private startTimer() {
+    const state: any = {
+      start: Date.now(),
+      progress: 'begin $updateMempool',
+      timer: null,
+    };
+    state.timer = setTimeout(() => {
+      logger.err(`$updateMempool stalled at "${state.progress}"`);
+    }, this.mainLoopTimeout);
+    return state;
+  }
+
+  private updateTimerProgress(state, msg) {
+    state.progress = msg;
+  }
+
+  private clearTimer(state) {
+    if (state.timer) {
+      clearTimeout(state.timer);
+    }
+  }
+
+  public handleRbfTransactions(rbfTransactions: { [txid: string]: TransactionExtended[]; }): void {
     for (const rbfTransaction in rbfTransactions) {
-      if (this.mempoolCache[rbfTransaction]) {
+      if (this.mempoolCache[rbfTransaction] && rbfTransactions[rbfTransaction]?.length) {
         // Store replaced transactions
-        rbfCache.add(this.mempoolCache[rbfTransaction], rbfTransactions[rbfTransaction].txid);
-        // Erase the replaced transactions from the local mempool
-        delete this.mempoolCache[rbfTransaction];
+        rbfCache.add(rbfTransactions[rbfTransaction], this.mempoolCache[rbfTransaction]);
       }
     }
   }
@@ -256,17 +287,6 @@ class Mempool {
     }
   }
 
-  private deleteExpiredTransactions() {
-    const now = new Date().getTime();
-    for (const tx in this.mempoolCache) {
-      const lazyDeleteAt = this.mempoolCache[tx].deleteAfter;
-      if (lazyDeleteAt && lazyDeleteAt < now) {
-        delete this.mempoolCache[tx];
-        rbfCache.evict(tx);
-      }
-    }
-  }
-
   private $getMempoolInfo() {
     if (config.MEMPOOL.USE_SECOND_NODE_FOR_MINFEE) {
       return Promise.all([

backend/src/api/mining/mining.ts

@@ -452,7 +452,7 @@ class Mining {
       const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
       if (elapsedSeconds > 5) {
         const progress = Math.round(totalBlockChecked / blocks.length * 100);
-        logger.info(`Indexing difficulty adjustment at block #${block.height} | Progress: ${progress}%`, logger.tags.mining);
+        logger.debug(`Indexing difficulty adjustment at block #${block.height} | Progress: ${progress}%`, logger.tags.mining);
         timer = new Date().getTime() / 1000;
       }
     }
@@ -558,8 +558,10 @@ class Mining {
       currentBlockHeight -= 10000;
     }
 
-    if (totalIndexed) {
-      logger.info(`Indexing missing coinstatsindex data completed`, logger.tags.mining);
+    if (totalIndexed > 0) {
+      logger.info(`Indexing missing coinstatsindex data completed. Indexed ${totalIndexed}`, logger.tags.mining);
+    } else {
+      logger.debug(`Indexing missing coinstatsindex data completed. Indexed 0.`, logger.tags.mining);
     }
   }
 

backend/src/api/rbf-cache.ts

@@ -1,65 +1,341 @@
-import { TransactionExtended } from "../mempool.interfaces";
+import logger from "../logger";
+import { TransactionExtended, TransactionStripped } from "../mempool.interfaces";
+import bitcoinApi from './bitcoin/bitcoin-api-factory';
+import { Common } from "./common";
+
+interface RbfTransaction extends TransactionStripped {
+  rbf?: boolean;
+  mined?: boolean;
+}
+
+interface RbfTree {
+  tx: RbfTransaction;
+  time: number;
+  interval?: number;
+  mined?: boolean;
+  fullRbf: boolean;
+  replaces: RbfTree[];
+}
 
 class RbfCache {
-  private replacedBy: { [txid: string]: string; } = {};
-  private replaces: { [txid: string]: string[] } = {};
-  private txs: { [txid: string]: TransactionExtended } = {};
-  private expiring: { [txid: string]: Date } = {};
+  private replacedBy: Map<string, string> = new Map();
+  private replaces: Map<string, string[]> = new Map();
+  private rbfTrees: Map<string, RbfTree> = new Map(); // sequences of consecutive replacements
+  private dirtyTrees: Set<string> = new Set();
+  private treeMap: Map<string, string> = new Map(); // map of txids to sequence ids
+  private txs: Map<string, TransactionExtended> = new Map();
+  private expiring: Map<string, number> = new Map();
 
   constructor() {
-    setInterval(this.cleanup.bind(this), 1000 * 60 * 60);
+    setInterval(this.cleanup.bind(this), 1000 * 60 * 10);
   }
 
-  public add(replacedTx: TransactionExtended, newTxId: string): void {
-    this.replacedBy[replacedTx.txid] = newTxId;
-    this.txs[replacedTx.txid] = replacedTx;
-    if (!this.replaces[newTxId]) {
-      this.replaces[newTxId] = [];
+  public add(replaced: TransactionExtended[], newTxExtended: TransactionExtended): void {
+    if (!newTxExtended || !replaced?.length) {
+      return;
     }
-    this.replaces[newTxId].push(replacedTx.txid);
+
+    const newTx = Common.stripTransaction(newTxExtended) as RbfTransaction;
+    const newTime = newTxExtended.firstSeen || Date.now();
+    newTx.rbf = newTxExtended.vin.some((v) => v.sequence < 0xfffffffe);
+    this.txs.set(newTx.txid, newTxExtended);
+
+    // maintain rbf trees
+    let fullRbf = false;
+    const replacedTrees: RbfTree[] = [];
+    for (const replacedTxExtended of replaced) {
+      const replacedTx = Common.stripTransaction(replacedTxExtended) as RbfTransaction;
+      replacedTx.rbf = replacedTxExtended.vin.some((v) => v.sequence < 0xfffffffe);
+      this.replacedBy.set(replacedTx.txid, newTx.txid);
+      if (this.treeMap.has(replacedTx.txid)) {
+        const treeId = this.treeMap.get(replacedTx.txid);
+        if (treeId) {
+          const tree = this.rbfTrees.get(treeId);
+          this.rbfTrees.delete(treeId);
+          if (tree) {
+            tree.interval = newTime - tree?.time;
+            replacedTrees.push(tree);
+            fullRbf = fullRbf || tree.fullRbf;
+          }
+        }
+      } else {
+        const replacedTime = replacedTxExtended.firstSeen || Date.now();
+        replacedTrees.push({
+          tx: replacedTx,
+          time: replacedTime,
+          interval: newTime - replacedTime,
+          fullRbf: !replacedTx.rbf,
+          replaces: [],
+        });
+        fullRbf = fullRbf || !replacedTx.rbf;
+        this.txs.set(replacedTx.txid, replacedTxExtended);
+      }
+    }
+    const treeId = replacedTrees[0].tx.txid;
+    const newTree = {
+      tx: newTx,
+      time: newTxExtended.firstSeen || Date.now(),
+      fullRbf,
+      replaces: replacedTrees
+    };
+    this.rbfTrees.set(treeId, newTree);
+    this.updateTreeMap(treeId, newTree);
+    this.replaces.set(newTx.txid, replacedTrees.map(tree => tree.tx.txid));
+    this.dirtyTrees.add(treeId);
   }
 
   public getReplacedBy(txId: string): string | undefined {
-    return this.replacedBy[txId];
+    return this.replacedBy.get(txId);
   }
 
   public getReplaces(txId: string): string[] | undefined {
-    return this.replaces[txId];
+    return this.replaces.get(txId);
   }
 
   public getTx(txId: string): TransactionExtended | undefined {
-    return this.txs[txId];
+    return this.txs.get(txId);
+  }
+
+  public getRbfTree(txId: string): RbfTree | void {
+    return this.rbfTrees.get(this.treeMap.get(txId) || '');
+  }
+
+  // get a paginated list of RbfTrees
+  // ordered by most recent replacement time
+  public getRbfTrees(onlyFullRbf: boolean, after?: string): RbfTree[] {
+    const limit = 25;
+    const trees: RbfTree[] = [];
+    const used = new Set<string>();
+    const replacements: string[][] = Array.from(this.replacedBy).reverse();
+    const afterTree = after ? this.treeMap.get(after) : null;
+    let ready = !afterTree;
+    for (let i = 0; i < replacements.length && trees.length <= limit - 1; i++) {
+      const txid = replacements[i][1];
+      const treeId = this.treeMap.get(txid) || '';
+      if (treeId === afterTree) {
+        ready = true;
+      } else if (ready) {
+        if (!used.has(treeId)) {
+          const tree = this.rbfTrees.get(treeId);
+          used.add(treeId);
+          if (tree && (!onlyFullRbf || tree.fullRbf)) {
+            trees.push(tree);
+          }
+        }
+      }
+    }
+    return trees;
+  }
+
+  // get map of rbf trees that have been updated since the last call
+  public getRbfChanges(): { trees: {[id: string]: RbfTree }, map: { [txid: string]: string }} {
+    const changes: { trees: {[id: string]: RbfTree }, map: { [txid: string]: string }} = {
+      trees: {},
+      map: {},
+    };
+    this.dirtyTrees.forEach(id => {
+      const tree = this.rbfTrees.get(id);
+      if (tree) {
+        changes.trees[id] = tree;
+        this.getTransactionsInTree(tree).forEach(tx => {
+          changes.map[tx.txid] = id;
+        });
+      }
+    });
+    this.dirtyTrees = new Set();
+    return changes;
+  }
+
+  public mined(txid): void {
+    if (!this.txs.has(txid)) {
+      return;
+    }
+    const treeId = this.treeMap.get(txid);
+    if (treeId && this.rbfTrees.has(treeId)) {
+      const tree = this.rbfTrees.get(treeId);
+      if (tree) {
+        this.setTreeMined(tree, txid);
+        tree.mined = true;
+        this.dirtyTrees.add(treeId);
+      }
+    }
+    this.evict(txid);
   }
 
   // flag a transaction as removed from the mempool
-  public evict(txid): void {
-    this.expiring[txid] = new Date(Date.now() + 1000 * 86400); // 24 hours
+  public evict(txid: string, fast: boolean = false): void {
+    if (this.txs.has(txid) && (fast || !this.expiring.has(txid))) {
+      this.expiring.set(txid, fast ? Date.now() + (1000 * 60 * 10) : Date.now() + (1000 * 86400)); // 24 hours
+    }
   }
 
   private cleanup(): void {
-    const currentDate = new Date();
-    for (const txid in this.expiring) {
-      if (this.expiring[txid] < currentDate) {
-        delete this.expiring[txid];
+    const now = Date.now();
+    for (const txid of this.expiring.keys()) {
+      if ((this.expiring.get(txid) || 0) < now) {
+        this.expiring.delete(txid);
         this.remove(txid);
       }
     }
+    logger.debug(`rbf cache contains ${this.txs.size} txs, ${this.expiring.size} due to expire`);
   }
 
   // remove a transaction & all previous versions from the cache
   private remove(txid): void {
-    // don't remove a transaction while a newer version remains in the mempool
-    if (this.replaces[txid] && !this.replacedBy[txid]) {
-      const replaces = this.replaces[txid];
-      delete this.replaces[txid];
-      for (const tx of replaces) {
+    // don't remove a transaction if a newer version remains in the mempool
+    if (!this.replacedBy.has(txid)) {
+      const replaces = this.replaces.get(txid);
+      this.replaces.delete(txid);
+      this.treeMap.delete(txid);
+      this.txs.delete(txid);
+      this.expiring.delete(txid);
+      for (const tx of (replaces || [])) {
         // recursively remove prior versions from the cache
-        delete this.replacedBy[tx];
-        delete this.txs[tx];
+        this.replacedBy.delete(tx);
+        // if this is the id of a tree, remove that too
+        if (this.treeMap.get(tx) === tx) {
+          this.rbfTrees.delete(tx);
+        }
         this.remove(tx);
       }
     }
   }
+
+  private updateTreeMap(newId: string, tree: RbfTree): void {
+    this.treeMap.set(tree.tx.txid, newId);
+    tree.replaces.forEach(subtree => {
+      this.updateTreeMap(newId, subtree);
+    });
+  }
+
+  private getTransactionsInTree(tree: RbfTree, txs: RbfTransaction[] = []): RbfTransaction[] {
+    txs.push(tree.tx);
+    tree.replaces.forEach(subtree => {
+      this.getTransactionsInTree(subtree, txs);
+    });
+    return txs;
+  }
+
+  private setTreeMined(tree: RbfTree, txid: string): void {
+    if (tree.tx.txid === txid) {
+      tree.tx.mined = true;
+    } else {
+      tree.replaces.forEach(subtree => {
+        this.setTreeMined(subtree, txid);
+      });
+    }
+  }
+
+  public dump(): any {
+    const trees = Array.from(this.rbfTrees.values()).map((tree: RbfTree) => { return this.exportTree(tree); });
+
+    return {
+      txs: Array.from(this.txs.entries()),
+      trees,
+      expiring: Array.from(this.expiring.entries()),
+    };
+  }
+
+  public async load({ txs, trees, expiring }): Promise<void> {
+    txs.forEach(txEntry => {
+      this.txs.set(txEntry[0], txEntry[1]);
+    });
+    for (const deflatedTree of trees) {
+      await this.importTree(deflatedTree.root, deflatedTree.root, deflatedTree, this.txs);
+    }
+    expiring.forEach(expiringEntry => {
+      if (this.txs.has(expiringEntry[0])) {
+        this.expiring.set(expiringEntry[0], new Date(expiringEntry[1]).getTime());
+      }
+    });
+    this.cleanup();
+  }
+
+  exportTree(tree: RbfTree, deflated: any = null) {
+    if (!deflated) {
+      deflated = {
+        root: tree.tx.txid,
+      };
+    }
+    deflated[tree.tx.txid] = {
+      tx: tree.tx.txid,
+      txMined: tree.tx.mined,
+      time: tree.time,
+      interval: tree.interval,
+      mined: tree.mined,
+      fullRbf: tree.fullRbf,
+      replaces: tree.replaces.map(child => child.tx.txid),
+    };
+    tree.replaces.forEach(child => {
+      this.exportTree(child, deflated);
+    });
+    return deflated;
+  }
+
+  async importTree(root, txid, deflated, txs: Map<string, TransactionExtended>, mined: boolean = false): Promise<RbfTree | void> {
+    const treeInfo = deflated[txid];
+    const replaces: RbfTree[] = [];
+
+    // check if any transactions in this tree have already been confirmed
+    mined = mined || treeInfo.mined;
+    let exists = mined;
+    if (!mined) {
+      try {
+        const apiTx = await bitcoinApi.$getRawTransaction(txid);
+        if (apiTx) {
+          exists = true;
+        }
+        if (apiTx?.status?.confirmed) {
+          mined = true;
+          treeInfo.txMined = true;
+          this.evict(txid, true);
+        }
+      } catch (e) {
+        // most transactions do not exist
+      }
+    }
+
+    // if the root tx is not in the mempool or the blockchain
+    // evict this tree as soon as possible
+    if (root === txid && !exists) {
+      this.evict(txid, true);
+    }
+
+    // recursively reconstruct child trees
+    for (const childId of treeInfo.replaces) {
+      const replaced = await this.importTree(root, childId, deflated, txs, mined);
+      if (replaced) {
+        this.replacedBy.set(replaced.tx.txid, txid);
+        replaces.push(replaced);
+        if (replaced.mined) {
+          mined = true;
+        }
+      }
+    }
+    this.replaces.set(txid, replaces.map(t => t.tx.txid));
+
+    const tx = txs.get(txid);
+    if (!tx) {
+      return;
+    }
+    const strippedTx = Common.stripTransaction(tx) as RbfTransaction;
+    strippedTx.rbf = tx.vin.some((v) => v.sequence < 0xfffffffe);
+    strippedTx.mined = treeInfo.txMined;
+    const tree = {
+      tx: strippedTx,
+      time: treeInfo.time,
+      interval: treeInfo.interval,
+      mined: mined,
+      fullRbf: treeInfo.fullRbf,
+      replaces,
+    };
+    this.treeMap.set(txid, root);
+    if (root === txid) {
+      this.rbfTrees.set(root, tree);
+      this.dirtyTrees.add(root);
+    }
+    return tree;
+  }
 }
 
 export default new RbfCache();

backend/src/api/tx-selection-worker.ts

@@ -2,7 +2,6 @@ import config from '../config';
 import logger from '../logger';
 import { ThreadTransaction, MempoolBlockWithTransactions, AuditTransaction } from '../mempool.interfaces';
 import { PairingHeap } from '../utils/pairing-heap';
-import { Common } from './common';
 import { parentPort } from 'worker_threads';
 
 let mempool: { [txid: string]: ThreadTransaction } = {};
@@ -72,7 +71,14 @@ function makeBlockTemplates(mempool: { [txid: string]: ThreadTransaction })
   }
 
   // Sort by descending ancestor score
-  mempoolArray.sort((a, b) => (b.score || 0) - (a.score || 0));
+  mempoolArray.sort((a, b) => {
+    if (b.score === a.score) {
+      // tie-break by lexicographic txid order for stability
+      return a.txid < b.txid ? -1 : 1;
+    } else {
+      return (b.score || 0) - (a.score || 0);
+    }
+  });
 
   // Build blocks by greedily choosing the highest feerate package
   // (i.e. the package rooted in the transaction with the best ancestor score)
@@ -80,7 +86,14 @@ function makeBlockTemplates(mempool: { [txid: string]: ThreadTransaction })
   let blockWeight = 4000;
   let blockSize = 0;
   let transactions: AuditTransaction[] = [];
-  const modified: PairingHeap<AuditTransaction> = new PairingHeap((a, b): boolean => (a.score || 0) > (b.score || 0));
+  const modified: PairingHeap<AuditTransaction> = new PairingHeap((a, b): boolean => {
+    if (a.score === b.score) {
+      // tie-break by lexicographic txid order for stability
+      return a.txid > b.txid;
+    } else {
+      return (a.score || 0) > (b.score || 0);
+    }
+  });
   let overflow: AuditTransaction[] = [];
   let failures = 0;
   let top = 0;
@@ -107,7 +120,7 @@ function makeBlockTemplates(mempool: { [txid: string]: ThreadTransaction })
 
     if (nextTx && !nextTx?.used) {
       // Check if the package fits into this block
-      if (blockWeight + nextTx.ancestorWeight < config.MEMPOOL.BLOCK_WEIGHT_UNITS) {
+      if (blocks.length >= 7 || (blockWeight + nextTx.ancestorWeight < config.MEMPOOL.BLOCK_WEIGHT_UNITS)) {
         const ancestors: AuditTransaction[] = Array.from(nextTx.ancestorMap.values());
         // sort ancestors by dependency graph (equivalent to sorting by ascending ancestor count)
         const sortedTxSet = [...ancestors.sort((a, b) => { return (a.ancestorMap.size || 0) - (b.ancestorMap.size || 0); }), nextTx];
@@ -175,34 +188,14 @@ function makeBlockTemplates(mempool: { [txid: string]: ThreadTransaction })
       overflow = [];
     }
   }
-  // pack any leftover transactions into the last block
-  for (const tx of overflow) {
-    if (!tx || tx?.used) {
-      continue;
-    }
-    blockWeight += tx.weight;
-    const mempoolTx = mempool[tx.txid];
-    // update original copy of this tx with effective fee rate & relatives data
-    mempoolTx.effectiveFeePerVsize = tx.score;
-    if (tx.ancestorMap.size > 0) {
-      cpfpClusters[tx.txid] = Array.from(tx.ancestorMap?.values()).map(a => a.txid);
-      mempoolTx.cpfpRoot = tx.txid;
-    }
-    mempoolTx.cpfpChecked = true;
-    transactions.push(tx);
-    tx.used = true;
+
+  if (overflow.length > 0) {
+    logger.warn('GBT overflow list unexpectedly non-empty after final block constructed');
   }
-  const blockTransactions = transactions.map(t => mempool[t.txid]);
-  restOfArray.forEach(tx => {
-    blockWeight += tx.weight;
-    tx.effectiveFeePerVsize = tx.feePerVsize;
-    tx.cpfpChecked = false;
-    blockTransactions.push(tx);
-  });
-  if (blockTransactions.length) {
-    blocks.push(blockTransactions);
+  // add the final unbounded block if it contains any transactions
+  if (transactions.length > 0) {
+    blocks.push(transactions.map(t => mempool[t.txid]));
   }
-  transactions = [];
 
   const end = Date.now();
   const time = end - start;

backend/src/api/websocket-handler.ts

@@ -26,6 +26,10 @@ class WebsocketHandler {
   private wss: WebSocket.Server | undefined;
   private extraInitProperties = {};
 
+  private numClients = 0;
+  private numConnected = 0;
+  private numDisconnected = 0;
+
   constructor() { }
 
   setWebsocketServer(wss: WebSocket.Server) {
@@ -42,7 +46,11 @@ class WebsocketHandler {
     }
 
     this.wss.on('connection', (client: WebSocket) => {
+      this.numConnected++;
       client.on('error', logger.info);
+      client.on('close', () => {
+        this.numDisconnected++;
+      });
       client.on('message', async (message: string) => {
         try {
           const parsedMessage: WebsocketResponse = JSON.parse(message);
@@ -58,9 +66,10 @@ class WebsocketHandler {
           if (parsedMessage && parsedMessage['track-tx']) {
             if (/^[a-fA-F0-9]{64}$/.test(parsedMessage['track-tx'])) {
               client['track-tx'] = parsedMessage['track-tx'];
+              const trackTxid = client['track-tx'];
               // Client is telling the transaction wasn't found
               if (parsedMessage['watch-mempool']) {
-                const rbfCacheTxid = rbfCache.getReplacedBy(client['track-tx']);
+                const rbfCacheTxid = rbfCache.getReplacedBy(trackTxid);
                 if (rbfCacheTxid) {
                   response['txReplaced'] = {
                     txid: rbfCacheTxid,
@@ -68,7 +77,7 @@ class WebsocketHandler {
                   client['track-tx'] = null;
                 } else {
                   // It might have appeared before we had the time to start watching for it
-                  const tx = memPool.getMempool()[client['track-tx']];
+                  const tx = memPool.getMempool()[trackTxid];
                   if (tx) {
                     if (config.MEMPOOL.BACKEND === 'esplora') {
                       response['tx'] = tx;
@@ -92,6 +101,13 @@ class WebsocketHandler {
                   }
                 }
               }
+              const tx = memPool.getMempool()[trackTxid];
+              if (tx && tx.position) {
+                response['txPosition'] = {
+                  txid: trackTxid,
+                  position: tx.position,
+                };
+              }
             } else {
               client['track-tx'] = null;
             }
@@ -132,6 +148,14 @@ class WebsocketHandler {
             }
           }
 
+          if (parsedMessage && parsedMessage['track-rbf'] !== undefined) {
+            if (['all', 'fullRbf'].includes(parsedMessage['track-rbf'])) {
+              client['track-rbf'] = parsedMessage['track-rbf'];
+            } else {
+              client['track-rbf'] = false;
+            }
+          }
+
           if (parsedMessage.action === 'init') {
             const _blocks = blocks.getBlocks().slice(-config.MEMPOOL.INITIAL_BLOCKS_AMOUNT);
             if (!_blocks) {
@@ -211,6 +235,7 @@ class WebsocketHandler {
     if (!_blocks) {
       _blocks = blocks.getBlocks().slice(-config.MEMPOOL.INITIAL_BLOCKS_AMOUNT);
     }
+    const da = difficultyAdjustment.getDifficultyAdjustment();
     return {
       'mempoolInfo': memPool.getMempoolInfo(),
       'vBytesPerSecond': memPool.getVBytesPerSecond(),
@@ -220,7 +245,7 @@ class WebsocketHandler {
       'transactions': memPool.getLatestTransactions(),
       'backendInfo': backendInfo.getBackendInfo(),
       'loadingIndicators': loadingIndicators.getLoadingIndicators(),
-      'da': difficultyAdjustment.getDifficultyAdjustment(),
+      'da': da?.previousTime ? da : undefined,
       'fees': feeApi.getRecommendedFee(),
       ...this.extraInitProperties
     };
@@ -231,6 +256,8 @@ class WebsocketHandler {
       throw new Error('WebSocket.Server is not set');
     }
 
+    this.printLogs();
+
     this.wss.clients.forEach((client) => {
       if (client.readyState !== WebSocket.OPEN) {
         return;
@@ -246,14 +273,16 @@ class WebsocketHandler {
     });
   }
 
-  async handleMempoolChange(newMempool: { [txid: string]: TransactionExtended },
+  async $handleMempoolChange(newMempool: { [txid: string]: TransactionExtended },
     newTransactions: TransactionExtended[], deletedTransactions: TransactionExtended[]): Promise<void> {
     if (!this.wss) {
       throw new Error('WebSocket.Server is not set');
     }
 
+    this.printLogs();
+
     if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) {
-      await mempoolBlocks.updateBlockTemplates(newMempool, newTransactions, deletedTransactions.map(tx => tx.txid), true);
+      await mempoolBlocks.$updateBlockTemplates(newMempool, newTransactions, deletedTransactions.map(tx => tx.txid), true);
     } else {
       mempoolBlocks.updateMempoolBlocks(newMempool, true);
     }
@@ -265,6 +294,16 @@ class WebsocketHandler {
     const rbfTransactions = Common.findRbfTransactions(newTransactions, deletedTransactions);
     const da = difficultyAdjustment.getDifficultyAdjustment();
     memPool.handleRbfTransactions(rbfTransactions);
+    const rbfChanges = rbfCache.getRbfChanges();
+    let rbfReplacements;
+    let fullRbfReplacements;
+    if (Object.keys(rbfChanges.trees).length) {
+      rbfReplacements = rbfCache.getRbfTrees(false);
+      fullRbfReplacements = rbfCache.getRbfTrees(true);
+    }
+    for (const deletedTx of deletedTransactions) {
+      rbfCache.evict(deletedTx.txid);
+    }
     const recommendedFees = feeApi.getRecommendedFee();
 
     this.wss.clients.forEach(async (client) => {
@@ -278,7 +317,9 @@ class WebsocketHandler {
         response['mempoolInfo'] = mempoolInfo;
         response['vBytesPerSecond'] = vBytesPerSecond;
         response['transactions'] = newTransactions.slice(0, 6).map((tx) => Common.stripTransaction(tx));
-        response['da'] = da;
+        if (da?.previousTime) {
+          response['da'] = da;
+        }
         response['fees'] = recommendedFees;
       }
 
@@ -371,9 +412,10 @@ class WebsocketHandler {
       }
 
       if (client['track-tx']) {
+        const trackTxid = client['track-tx'];
         const outspends: object = {};
         newTransactions.forEach((tx) => tx.vin.forEach((vin, i) => {
-          if (vin.txid === client['track-tx']) {
+          if (vin.txid === trackTxid) {
             outspends[vin.vout] = {
               vin: i,
               txid: tx.txid,
@@ -385,16 +427,25 @@ class WebsocketHandler {
           response['utxoSpent'] = outspends;
         }
 
-        if (rbfTransactions[client['track-tx']]) {
-          for (const rbfTransaction in rbfTransactions) {
-            if (client['track-tx'] === rbfTransaction) {
-              response['rbfTransaction'] = {
-                txid: rbfTransactions[rbfTransaction].txid,
-              };
-              break;
-            }
+        const rbfReplacedBy = rbfCache.getReplacedBy(client['track-tx']);
+        if (rbfReplacedBy) {
+          response['rbfTransaction'] = {
+            txid: rbfReplacedBy,
           }
         }
+
+        const rbfChange = rbfChanges.map[client['track-tx']];
+        if (rbfChange) {
+          response['rbfInfo'] = rbfChanges.trees[rbfChange];
+        }
+
+        const mempoolTx = newMempool[trackTxid];
+        if (mempoolTx && mempoolTx.position) {
+          response['txPosition'] = {
+            txid: trackTxid,
+            position: mempoolTx.position,
+          };
+        }
       }
 
       if (client['track-mempool-block'] >= 0) {
@@ -407,6 +458,12 @@ class WebsocketHandler {
         }
       }
 
+      if (client['track-rbf'] === 'all' && rbfReplacements) {
+        response['rbfLatest'] = rbfReplacements;
+      } else if (client['track-rbf'] === 'fullRbf' && fullRbfReplacements) {
+        response['rbfLatest'] = fullRbfReplacements;
+      }
+
       if (Object.keys(response).length) {
         client.send(JSON.stringify(response));
       }
@@ -418,17 +475,25 @@ class WebsocketHandler {
       throw new Error('WebSocket.Server is not set');
     }
 
+    this.printLogs();
+
     const _memPool = memPool.getMempool();
 
     if (config.MEMPOOL.AUDIT) {
       let projectedBlocks;
+      let auditMempool = _memPool;
       // template calculation functions have mempool side effects, so calculate audits using
       // a cloned copy of the mempool if we're running a different algorithm for mempool updates
-      const auditMempool = (config.MEMPOOL.ADVANCED_GBT_AUDIT === config.MEMPOOL.ADVANCED_GBT_MEMPOOL) ? _memPool : deepClone(_memPool);
-      if (config.MEMPOOL.ADVANCED_GBT_AUDIT) {
-        projectedBlocks = await mempoolBlocks.makeBlockTemplates(auditMempool, false);
+      const separateAudit = config.MEMPOOL.ADVANCED_GBT_AUDIT !== config.MEMPOOL.ADVANCED_GBT_MEMPOOL;
+      if (separateAudit) {
+        auditMempool = deepClone(_memPool);
+        if (config.MEMPOOL.ADVANCED_GBT_AUDIT) {
+          projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false);
+        } else {
+          projectedBlocks = mempoolBlocks.updateMempoolBlocks(auditMempool, false);
+        }
       } else {
-        projectedBlocks = mempoolBlocks.updateMempoolBlocks(auditMempool, false);
+        projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
       }
 
       if (Common.indexingEnabled() && memPool.isInSync()) {
@@ -474,16 +539,14 @@ class WebsocketHandler {
       }
     }
 
-    const removed: string[] = [];
     // Update mempool to remove transactions included in the new block
     for (const txId of txIds) {
       delete _memPool[txId];
-      removed.push(txId);
-      rbfCache.evict(txId);
+      rbfCache.mined(txId);
     }
 
     if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) {
-      await mempoolBlocks.updateBlockTemplates(_memPool, [], removed, true);
+      await mempoolBlocks.$makeBlockTemplates(_memPool, true);
     } else {
       mempoolBlocks.updateMempoolBlocks(_memPool, true);
     }
@@ -505,7 +568,7 @@ class WebsocketHandler {
       const response = {
         'block': block,
         'mempoolInfo': memPool.getMempoolInfo(),
-        'da': da,
+        'da': da?.previousTime ? da : undefined,
         'fees': fees,
       };
 
@@ -513,8 +576,19 @@ class WebsocketHandler {
         response['mempool-blocks'] = mBlocks;
       }
 
-      if (client['track-tx'] && txIds.indexOf(client['track-tx']) > -1) {
-        response['txConfirmed'] = true;
+      if (client['track-tx']) {
+        const trackTxid = client['track-tx'];
+        if (txIds.indexOf(trackTxid) > -1) {
+          response['txConfirmed'] = true;
+        } else {
+          const mempoolTx = _memPool[trackTxid];
+          if (mempoolTx && mempoolTx.position) {
+            response['txPosition'] = {
+              txid: trackTxid,
+              position: mempoolTx.position,
+            };
+          }
+        }
       }
 
       if (client['track-address']) {
@@ -594,6 +668,17 @@ class WebsocketHandler {
       client.send(JSON.stringify(response));
     });
   }
+
+  private printLogs(): void {
+    if (this.wss) {
+      const count = this.wss?.clients?.size || 0;
+      const diff = count - this.numClients;
+      this.numClients = count;
+      logger.debug(`${count} websocket clients | ${this.numConnected} connected | ${this.numDisconnected} disconnected | (${diff >= 0 ? '+' : ''}${diff})`);
+      this.numConnected = 0;
+      this.numDisconnected = 0;
+    }
+  }
 }
 
 export default new WebsocketHandler();

backend/src/config.ts

@@ -33,9 +33,12 @@ interface IConfig {
     ADVANCED_GBT_MEMPOOL: boolean;
     CPFP_INDEXING: boolean;
     MAX_BLOCKS_BULK_QUERY: number;
+    DISK_CACHE_BLOCK_INTERVAL: number;
   };
   ESPLORA: {
     REST_API_URL: string;
+    UNIX_SOCKET_PATH: string | void | null;
+    RETRY_UNIX_SOCKET_AFTER: number;
   };
   LIGHTNING: {
     ENABLED: boolean;
@@ -51,6 +54,7 @@ interface IConfig {
     TLS_CERT_PATH: string;
     MACAROON_PATH: string;
     REST_API_URL: string;
+    TIMEOUT: number;
   };
   CLIGHTNING: {
     SOCKET: string;
@@ -65,12 +69,14 @@ interface IConfig {
     PORT: number;
     USERNAME: string;
     PASSWORD: string;
+    TIMEOUT: number;
   };
   SECOND_CORE_RPC: {
     HOST: string;
     PORT: number;
     USERNAME: string;
     PASSWORD: string;
+    TIMEOUT: number;
   };
   DATABASE: {
     ENABLED: boolean;
@@ -80,6 +86,7 @@ interface IConfig {
     DATABASE: string;
     USERNAME: string;
     PASSWORD: string;
+    TIMEOUT: number;
   };
   SYSLOG: {
     ENABLED: boolean;
@@ -155,9 +162,12 @@ const defaults: IConfig = {
     'ADVANCED_GBT_MEMPOOL': false,
     'CPFP_INDEXING': false,
     'MAX_BLOCKS_BULK_QUERY': 0,
+    'DISK_CACHE_BLOCK_INTERVAL': 6,
   },
   'ESPLORA': {
     'REST_API_URL': 'http://127.0.0.1:3000',
+    'UNIX_SOCKET_PATH': null,
+    'RETRY_UNIX_SOCKET_AFTER': 30000,
   },
   'ELECTRUM': {
     'HOST': '127.0.0.1',
@@ -168,13 +178,15 @@ const defaults: IConfig = {
     'HOST': '127.0.0.1',
     'PORT': 8332,
     'USERNAME': 'mempool',
-    'PASSWORD': 'mempool'
+    'PASSWORD': 'mempool',
+    'TIMEOUT': 60000,
   },
   'SECOND_CORE_RPC': {
     'HOST': '127.0.0.1',
     'PORT': 8332,
     'USERNAME': 'mempool',
-    'PASSWORD': 'mempool'
+    'PASSWORD': 'mempool',
+    'TIMEOUT': 60000,
   },
   'DATABASE': {
     'ENABLED': true,
@@ -183,7 +195,8 @@ const defaults: IConfig = {
     'PORT': 3306,
     'DATABASE': 'mempool',
     'USERNAME': 'mempool',
-    'PASSWORD': 'mempool'
+    'PASSWORD': 'mempool',
+    'TIMEOUT': 180000,
   },
   'SYSLOG': {
     'ENABLED': true,
@@ -214,6 +227,7 @@ const defaults: IConfig = {
     'TLS_CERT_PATH': '',
     'MACAROON_PATH': '',
     'REST_API_URL': 'https://localhost:8080',
+    'TIMEOUT': 10000,
   },
   'CLIGHTNING': {
     'SOCKET': '',

backend/src/database.ts

@@ -33,8 +33,32 @@ import { FieldPacket, OkPacket, PoolOptions, ResultSetHeader, RowDataPacket } fr
     OkPacket[] | ResultSetHeader>(query, params?): Promise<[T, FieldPacket[]]>
   {
     this.checkDBFlag();
-    const pool = await this.getPool();
-    return pool.query(query, params);
+    let hardTimeout;
+    if (query?.timeout != null) {
+      hardTimeout = Math.floor(query.timeout * 1.1);
+    } else {
+      hardTimeout = config.DATABASE.TIMEOUT;
+    }
+    if (hardTimeout > 0) {
+      return new Promise((resolve, reject) => {
+        const timer = setTimeout(() => {
+          reject(new Error(`DB query failed to return, reject or time out within ${hardTimeout / 1000}s - ${query?.sql?.slice(0, 160) || (typeof(query) === 'string' || query instanceof String ? query?.slice(0, 160) : 'unknown query')}`));
+        }, hardTimeout);
+
+        this.getPool().then(pool => {
+          return pool.query(query, params) as Promise<[T, FieldPacket[]]>;
+        }).then(result => {
+          resolve(result);
+        }).catch(error => {
+          reject(error);
+        }).finally(() => {
+          clearTimeout(timer);
+        });
+      });
+    } else {
+      const pool = await this.getPool();
+      return pool.query(query, params);
+    }
   }
 
   public async checkDbConnection() {

backend/src/index.ts

@@ -2,6 +2,7 @@ import express from 'express';
 import { Application, Request, Response, NextFunction } from 'express';
 import * as http from 'http';
 import * as WebSocket from 'ws';
+import bitcoinApi from './api/bitcoin/bitcoin-api-factory';
 import cluster from 'cluster';
 import DB from './database';
 import config from './config';
@@ -45,7 +46,8 @@ class Server {
   private wss: WebSocket.Server | undefined;
   private server: http.Server | undefined;
   private app: Application;
-  private currentBackendRetryInterval = 5;
+  private currentBackendRetryInterval = 1;
+  private backendRetryCount = 0;
 
   private maxHeapSize: number = 0;
   private heapLogInterval: number = 60;
@@ -120,7 +122,7 @@ class Server {
     await poolsUpdater.updatePoolsJson(); // Needs to be done before loading the disk cache because we sometimes wipe it
     await syncAssets.syncAssets$();
     if (config.MEMPOOL.ENABLED) {
-      diskCache.loadMempoolCache();
+      await diskCache.$loadMempoolCache();
     }
 
     if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED && cluster.isPrimary) {
@@ -178,22 +180,26 @@ class Server {
           logger.debug(msg);
         }
       }
-      await blocks.$updateBlocks();
-      await memPool.$updateMempool();
+      const newMempool = await bitcoinApi.$getRawMempool();
+      const numHandledBlocks = await blocks.$updateBlocks();
+      if (numHandledBlocks === 0) {
+        await memPool.$updateMempool(newMempool);
+      }
       indexer.$run();
 
-      setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS);
-      this.currentBackendRetryInterval = 5;
+      // rerun immediately if we skipped the mempool update, otherwise wait POLL_RATE_MS
+      setTimeout(this.runMainUpdateLoop.bind(this), numHandledBlocks > 0 ? 1 : config.MEMPOOL.POLL_RATE_MS);
+      this.backendRetryCount = 0;
     } catch (e: any) {
-      let loggerMsg = `Exception in runMainUpdateLoop(). Retrying in ${this.currentBackendRetryInterval} sec.`;
+      this.backendRetryCount++;
+      let loggerMsg = `Exception in runMainUpdateLoop() (count: ${this.backendRetryCount}). Retrying in ${this.currentBackendRetryInterval} sec.`;
       loggerMsg += ` Reason: ${(e instanceof Error ? e.message : e)}.`;
       if (e?.stack) {
         loggerMsg += ` Stack trace: ${e.stack}`;
       }
       // When we get a first Exception, only `logger.debug` it and retry after 5 seconds
       // From the second Exception, `logger.warn` the Exception and increase the retry delay
-      // Maximum retry delay is 60 seconds
-      if (this.currentBackendRetryInterval > 5) {
+      if (this.backendRetryCount >= 5) {
         logger.warn(loggerMsg);
         mempool.setOutOfSync();
       } else {
@@ -203,8 +209,8 @@ class Server {
         logger.debug(`AxiosError: ${e?.message}`);
       }
       setTimeout(this.runMainUpdateLoop.bind(this), 1000 * this.currentBackendRetryInterval);
-      this.currentBackendRetryInterval *= 2;
-      this.currentBackendRetryInterval = Math.min(this.currentBackendRetryInterval, 60);
+    } finally {
+      diskCache.unlock();
     }
   }
 
@@ -215,11 +221,11 @@ class Server {
       await lightningStatsUpdater.$startService();
       await forensicsService.$startService();
     } catch(e) {
-      logger.err(`Nodejs lightning backend crashed. Restarting in 1 minute. Reason: ${(e instanceof Error ? e.message : e)}`);
+      logger.err(`Exception in $runLightningBackend. Restarting in 1 minute. Reason: ${(e instanceof Error ? e.message : e)}`);
       await Common.sleep$(1000 * 60);
       this.$runLightningBackend();
     };
-}
+  }
 
   setUpWebsocketHandling(): void {
     if (this.wss) {
@@ -237,7 +243,7 @@ class Server {
     websocketHandler.setupConnectionHandling();
     if (config.MEMPOOL.ENABLED) {
       statistics.setNewStatisticsEntryCallback(websocketHandler.handleNewStatistic.bind(websocketHandler));
-      memPool.setAsyncMempoolChangedCallback(websocketHandler.handleMempoolChange.bind(websocketHandler));
+      memPool.setAsyncMempoolChangedCallback(websocketHandler.$handleMempoolChange.bind(websocketHandler));
       blocks.setNewAsyncBlockCallback(websocketHandler.handleNewBlock.bind(websocketHandler));
     }
     priceUpdater.setRatesChangedCallback(websocketHandler.handleNewConversionRates.bind(websocketHandler));
@@ -275,7 +281,7 @@ class Server {
 
     if (!this.warnedHeapCritical && this.maxHeapSize > warnThreshold) {
       this.warnedHeapCritical = true;
-      logger.warn(`Used ${(this.maxHeapSize / stats.heap_size_limit).toFixed(2)}% of heap limit (${formatBytes(this.maxHeapSize, byteUnits, true)} / ${formatBytes(stats.heap_size_limit, byteUnits)})!`);
+      logger.warn(`Used ${(this.maxHeapSize / stats.heap_size_limit * 100).toFixed(2)}% of heap limit (${formatBytes(this.maxHeapSize, byteUnits, true)} / ${formatBytes(stats.heap_size_limit, byteUnits)})!`);
     }
     if (this.lastHeapLogTime === null || (now - this.lastHeapLogTime) > (this.heapLogInterval * 1000)) {
       logger.debug(`Memory usage: ${formatBytes(this.maxHeapSize, byteUnits)} / ${formatBytes(stats.heap_size_limit, byteUnits)}`);

backend/src/logger.ts

@@ -69,6 +69,10 @@ class Logger {
     this.network = this.getNetwork();
   }
 
+  public updateNetwork(): void {
+    this.network = this.getNetwork();
+  }
+
   private addprio(prio): void {
     this[prio] = (function(_this) {
       return function(msg, tag?: string) {

backend/src/mempool.interfaces.ts

@@ -58,6 +58,7 @@ export interface MempoolBlockWithTransactions extends MempoolBlock {
 export interface MempoolBlockDelta {
   added: TransactionStripped[];
   removed: string[];
+  changed: { txid: string, rate: number | undefined }[];
 }
 
 interface VinStrippedToScriptsig {
@@ -79,7 +80,11 @@ export interface TransactionExtended extends IEsploraApi.Transaction {
   descendants?: Ancestor[];
   bestDescendant?: BestDescendant | null;
   cpfpChecked?: boolean;
-  deleteAfter?: number;
+  hasRelatives?: boolean;
+  position?: {
+    block: number,
+    vsize: number,
+  };
 }
 
 export interface AuditTransaction {
@@ -145,6 +150,7 @@ export interface TransactionStripped {
   fee: number;
   vsize: number;
   value: number;
+  rate?: number; // effective fee rate
 }
 
 export interface BlockExtension {
@@ -214,6 +220,16 @@ export interface MempoolStats {
   tx_count: number;
 }
 
+export interface EffectiveFeeStats {
+  medianFee: number; // median effective fee rate
+  feeRange: number[]; // 2nd, 10th, 25th, 50th, 75th, 90th, 98th percentiles
+}
+
+export interface CpfpSummary {
+  transactions: TransactionExtended[];
+  clusters: { root: string, height: number, txs: Ancestor[], effectiveFeePerVsize: number }[];
+}
+
 export interface Statistic {
   id?: number;
   added: string;
@@ -309,9 +325,11 @@ export interface IDifficultyAdjustment {
   remainingBlocks: number;
   remainingTime: number;
   previousRetarget: number;
+  previousTime: number;
   nextRetargetHeight: number;
   timeAvg: number;
   timeOffset: number;
+  expectedBlocks: number;
 }
 
 export interface IndexedDifficultyAdjustment {

backend/src/repositories/BlocksRepository.ts

@@ -1,4 +1,4 @@
-import { BlockExtended, BlockExtension, BlockPrice } from '../mempool.interfaces';
+import { BlockExtended, BlockExtension, BlockPrice, EffectiveFeeStats } from '../mempool.interfaces';
 import DB from '../database';
 import logger from '../logger';
 import { Common } from '../api/common';
@@ -13,6 +13,48 @@ import chainTips from '../api/chain-tips';
 import blocks from '../api/blocks';
 import BlocksAuditsRepository from './BlocksAuditsRepository';
 
+interface DatabaseBlock {
+  id: string;
+  height: number;
+  version: number;
+  timestamp: number;
+  bits: number;
+  nonce: number;
+  difficulty: number;
+  merkle_root: string;
+  tx_count: number;
+  size: number;
+  weight: number;
+  previousblockhash: string;
+  mediantime: number;
+  totalFees: number;
+  medianFee: number;
+  feeRange: string;
+  reward: number;
+  poolId: number;
+  poolName: string;
+  poolSlug: string;
+  avgFee: number;
+  avgFeeRate: number;
+  coinbaseRaw: string;
+  coinbaseAddress: string;
+  coinbaseSignature: string;
+  coinbaseSignatureAscii: string;
+  avgTxSize: number;
+  totalInputs: number;
+  totalOutputs: number;
+  totalOutputAmt: number;
+  medianFeeAmt: number;
+  feePercentiles: string;
+  segwitTotalTxs: number;
+  segwitTotalSize: number;
+  segwitTotalWeight: number;
+  header: string;
+  utxoSetChange: number;
+  utxoSetSize: number;
+  totalInputAmt: number;
+}
+
 const BLOCK_DB_FIELDS = `
   blocks.hash AS id,
   blocks.height,
@@ -52,7 +94,7 @@ const BLOCK_DB_FIELDS = `
   blocks.header,
   blocks.utxoset_change AS utxoSetChange,
   blocks.utxoset_size AS utxoSetSize,
-  blocks.total_input_amt AS totalInputAmts
+  blocks.total_input_amt AS totalInputAmt
 `;
 
 class BlocksRepository {
@@ -171,6 +213,32 @@ class BlocksRepository {
     }
   }
 
+  /**
+   * Update missing fee amounts fields
+   *
+   * @param blockHash 
+   * @param feeAmtPercentiles 
+   * @param medianFeeAmt 
+   */
+  public async $updateFeeAmounts(blockHash: string, feeAmtPercentiles, medianFeeAmt) : Promise<void> {
+    try {
+      const query = `
+        UPDATE blocks
+        SET fee_percentiles = ?, median_fee_amt = ?
+        WHERE hash = ?
+      `;
+      const params: any[] = [
+        JSON.stringify(feeAmtPercentiles),
+        medianFeeAmt,
+        blockHash
+      ];
+      await DB.query(query, params);
+    } catch (e: any) {
+      logger.err(`Cannot update fee amounts for block ${blockHash}. Reason: ' + ${e instanceof Error ? e.message : e}`);
+      throw e;
+    }
+  }
+
   /**
    * Get all block height that have not been indexed between [startHeight, endHeight]
    */
@@ -432,7 +500,7 @@ class BlocksRepository {
 
       const blocks: BlockExtended[] = [];
       for (const block of rows) {
-        blocks.push(await this.formatDbBlockIntoExtendedBlock(block));
+        blocks.push(await this.formatDbBlockIntoExtendedBlock(block as DatabaseBlock));
       }
 
       return blocks;
@@ -459,37 +527,13 @@ class BlocksRepository {
         return null;
       }
 
-      return await this.formatDbBlockIntoExtendedBlock(rows[0]);  
+      return await this.formatDbBlockIntoExtendedBlock(rows[0] as DatabaseBlock);  
     } catch (e) {
       logger.err(`Cannot get indexed block ${height}. Reason: ` + (e instanceof Error ? e.message : e));
       throw e;
     }
   }
 
-  /**
-   * Get one block by hash
-   */
-  public async $getBlockByHash(hash: string): Promise<object | null> {
-    try {
-      const query = `
-        SELECT ${BLOCK_DB_FIELDS}
-        FROM blocks
-        JOIN pools ON blocks.pool_id = pools.id
-        WHERE hash = ?;
-      `;
-      const [rows]: any[] = await DB.query(query, [hash]);
-
-      if (rows.length <= 0) {
-        return null;
-      }
- 
-      return await this.formatDbBlockIntoExtendedBlock(rows[0]);
-    } catch (e) {
-      logger.err(`Cannot get indexed block ${hash}. Reason: ` + (e instanceof Error ? e.message : e));
-      throw e;
-    }
-  }
-
   /**
    * Return blocks difficulty
    */
@@ -599,7 +643,6 @@ class BlocksRepository {
         if (blocks[idx].previous_block_hash !== blocks[idx - 1].hash) {
           logger.warn(`Chain divergence detected at block ${blocks[idx - 1].height}`);
           await this.$deleteBlocksFrom(blocks[idx - 1].height);
-          await BlocksSummariesRepository.$deleteBlocksFrom(blocks[idx - 1].height);
           await HashratesRepository.$deleteHashratesFromTimestamp(blocks[idx - 1].timestamp - 604800);
           await DifficultyAdjustmentsRepository.$deleteAdjustementsFromHeight(blocks[idx - 1].height);
           return false;
@@ -619,7 +662,7 @@ class BlocksRepository {
    * Delete blocks from the database from blockHeight
    */
   public async $deleteBlocksFrom(blockHeight: number) {
-    logger.info(`Delete newer blocks from height ${blockHeight} from the database`);
+    logger.info(`Delete newer blocks from height ${blockHeight} from the database`, logger.tags.mining);
 
     try {
       await DB.query(`DELETE FROM blocks where height >= ${blockHeight}`);
@@ -908,13 +951,32 @@ class BlocksRepository {
     }
   }
 
+  /**
+   * Save indexed effective fee statistics
+   * 
+   * @param id 
+   * @param feeStats 
+   */
+  public async $saveEffectiveFeeStats(id: string, feeStats: EffectiveFeeStats): Promise<void> {
+    try {
+      await DB.query(`
+        UPDATE blocks SET median_fee = ?, fee_span = ?
+        WHERE hash = ?`,
+        [feeStats.medianFee, JSON.stringify(feeStats.feeRange), id]
+      );
+    } catch (e) {
+      logger.err(`Cannot update block fee stats. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
   /**
    * Convert a mysql row block into a BlockExtended. Note that you
    * must provide the correct field into dbBlk object param
    * 
    * @param dbBlk 
    */
-  private async formatDbBlockIntoExtendedBlock(dbBlk: any): Promise<BlockExtended> {
+  private async formatDbBlockIntoExtendedBlock(dbBlk: DatabaseBlock): Promise<BlockExtended> {
     const blk: Partial<BlockExtended> = {};
     const extras: Partial<BlockExtension> = {};
 
@@ -978,6 +1040,7 @@ class BlocksRepository {
     }
 
     // If we're missing block summary related field, check if we can populate them on the fly now
+    // This is for example triggered upon re-org
     if (Common.blocksSummariesIndexingEnabled() &&
       (extras.medianFeeAmt === null || extras.feePercentiles === null))
     {
@@ -985,11 +1048,12 @@ class BlocksRepository {
       if (extras.feePercentiles === null) {
         const block = await bitcoinClient.getBlock(dbBlk.id, 2);
         const summary = blocks.summarizeBlock(block);
-        await BlocksSummariesRepository.$saveSummary({ height: block.height, mined: summary });
+        await BlocksSummariesRepository.$saveTransactions(dbBlk.height, dbBlk.id, summary.transactions);
         extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id);
       }
       if (extras.feePercentiles !== null) {
         extras.medianFeeAmt = extras.feePercentiles[3];
+        await this.$updateFeeAmounts(dbBlk.id, extras.feePercentiles, extras.medianFeeAmt);
       }
     }
 

backend/src/repositories/BlocksSummariesRepository.ts

@@ -1,6 +1,6 @@
 import DB from '../database';
 import logger from '../logger';
-import { BlockSummary } from '../mempool.interfaces';
+import { BlockSummary, TransactionStripped } from '../mempool.interfaces';
 
 class BlocksSummariesRepository {
   public async $getByBlockId(id: string): Promise<BlockSummary | undefined> {
@@ -17,23 +17,17 @@ class BlocksSummariesRepository {
     return undefined;
   }
 
-  public async $saveSummary(params: { height: number, mined?: BlockSummary}) {
-    const blockId = params.mined?.id;
+  public async $saveTransactions(blockHeight: number, blockId: string, transactions: TransactionStripped[]): Promise<void> {
     try {
-      const transactions = JSON.stringify(params.mined?.transactions || []);
+      const transactionsStr = JSON.stringify(transactions);
       await DB.query(`
-        INSERT INTO blocks_summaries (height, id, transactions, template)
-        VALUE (?, ?, ?, ?)
-        ON DUPLICATE KEY UPDATE
-          transactions = ?
-      `, [params.height, blockId, transactions, '[]', transactions]);
+        INSERT INTO blocks_summaries
+        SET height = ?, transactions = ?, id = ?
+        ON DUPLICATE KEY UPDATE transactions = ?`,
+        [blockHeight, transactionsStr, blockId, transactionsStr]);
     } catch (e: any) {
-      if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
-        logger.debug(`Cannot save block summary for ${blockId} because it has already been indexed, ignoring`);
-      } else {
-        logger.debug(`Cannot save block summary for ${blockId}. Reason: ${e instanceof Error ? e.message : e}`);
-        throw e;
-      }
+      logger.debug(`Cannot save block summary transactions for ${blockId}. Reason: ${e instanceof Error ? e.message : e}`);
+      throw e;
     }
   }
 
@@ -68,19 +62,6 @@ class BlocksSummariesRepository {
     return [];
   }
 
-  /**
-   * Delete blocks from the database from blockHeight
-   */
-  public async $deleteBlocksFrom(blockHeight: number) {
-    logger.info(`Delete newer blocks summary from height ${blockHeight} from the database`);
-
-    try {
-      await DB.query(`DELETE FROM blocks_summaries where height >= ${blockHeight}`);
-    } catch (e) {
-      logger.err('Cannot delete indexed blocks summaries. Reason: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
   /**
    * Get the fee percentiles if the block has already been indexed, [] otherwise
    * 

backend/src/repositories/CpfpRepository.ts

@@ -48,7 +48,7 @@ class CpfpRepository {
     }
   }
 
-  public async $batchSaveClusters(clusters: { root: string, height: number, txs: any, effectiveFeePerVsize: number}[]): Promise<boolean> {
+  public async $batchSaveClusters(clusters: { root: string, height: number, txs: Ancestor[], effectiveFeePerVsize: number }[]): Promise<boolean> {
     try {
       const clusterValues: any[] = [];
       const txs: any[] = [];

backend/src/repositories/HashratesRepository.ts

@@ -220,7 +220,7 @@ class HashratesRepository {
    * Delete hashrates from the database from timestamp
    */
   public async $deleteHashratesFromTimestamp(timestamp: number) {
-    logger.info(`Delete newer hashrates from timestamp ${new Date(timestamp * 1000).toUTCString()} from the database`);
+    logger.info(`Delete newer hashrates from timestamp ${new Date(timestamp * 1000).toUTCString()} from the database`, logger.tags.mining);
 
     try {
       await DB.query(`DELETE FROM hashrates WHERE hashrate_timestamp >= FROM_UNIXTIME(?)`, [timestamp]);

backend/src/repositories/PricesRepository.ts

@@ -160,7 +160,7 @@ class PricesRepository {
 
       // Compute fiat exchange rates
       let latestPrice = rates[0] as ApiPrice;
-      if (latestPrice.USD === -1) {
+      if (!latestPrice || latestPrice.USD === -1) {
         latestPrice = priceUpdater.getEmptyPricesObj();
       }
 

backend/src/tasks/lightning/forensics.service.ts

@@ -27,7 +27,7 @@ class ForensicsService {
 
   private async $runTasks(): Promise<void> {
     try {
-      logger.info(`Running forensics scans`);
+      logger.debug(`Running forensics scans`);
 
       if (config.MEMPOOL.BACKEND === 'esplora') {
         await this.$runClosedChannelsForensics(false);
@@ -73,7 +73,7 @@ class ForensicsService {
     let progress = 0;
 
     try {
-      logger.info(`Started running closed channel forensics...`);
+      logger.debug(`Started running closed channel forensics...`);
       let channels;
       if (onlyNewChannels) {
         channels = await channelsApi.$getClosedChannelsWithoutReason();
@@ -152,11 +152,11 @@ class ForensicsService {
         ++progress;
         const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
         if (elapsedSeconds > 10) {
-          logger.info(`Updating channel closed channel forensics ${progress}/${channels.length}`);
+          logger.debug(`Updating channel closed channel forensics ${progress}/${channels.length}`);
           this.loggerTimer = new Date().getTime() / 1000;
         }
       }
-      logger.info(`Closed channels forensics scan complete.`);
+      logger.debug(`Closed channels forensics scan complete.`);
     } catch (e) {
       logger.err('$runClosedChannelsForensics() error: ' + (e instanceof Error ? e.message : e));
     }
@@ -217,7 +217,7 @@ class ForensicsService {
     let progress = 0;
 
     try {
-      logger.info(`Started running open channel forensics...`);
+      logger.debug(`Started running open channel forensics...`);
       const channels = await channelsApi.$getChannelsWithoutSourceChecked();
 
       for (const openChannel of channels) {
@@ -257,7 +257,7 @@ class ForensicsService {
         ++progress;
         const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
         if (elapsedSeconds > 10) {
-          logger.info(`Updating opened channel forensics ${progress}/${channels?.length}`);
+          logger.debug(`Updating opened channel forensics ${progress}/${channels?.length}`);
           this.loggerTimer = new Date().getTime() / 1000;
           this.truncateTempCache();
         }
@@ -266,7 +266,7 @@ class ForensicsService {
         }
       }
 
-      logger.info(`Open channels forensics scan complete.`);
+      logger.debug(`Open channels forensics scan complete.`);
     } catch (e) {
       logger.err('$runOpenedChannelsForensics() error: ' + (e instanceof Error ? e.message : e));
     } finally {

backend/src/tasks/lightning/network-sync.service.ts

@@ -283,7 +283,7 @@ class NetworkSyncService {
       } else {
         log += ` for the first time`;
       }
-      logger.info(`${log}`, logger.tags.ln);
+      logger.debug(`${log}`, logger.tags.ln);
 
       const channels = await channelsApi.$getChannelsByStatus([0, 1]);
       for (const channel of channels) {
@@ -300,7 +300,7 @@ class NetworkSyncService {
         ++progress;
         const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
         if (elapsedSeconds > config.LIGHTNING.LOGGER_UPDATE_INTERVAL) {
-          logger.info(`Checking if channel has been closed ${progress}/${channels.length}`, logger.tags.ln);
+          logger.debug(`Checking if channel has been closed ${progress}/${channels.length}`, logger.tags.ln);
           this.loggerTimer = new Date().getTime() / 1000;
         }
       }

backend/src/tasks/lightning/stats-updater.service.ts

@@ -22,12 +22,15 @@ class LightningStatsUpdater {
    * Update the latest entry for each node every config.LIGHTNING.STATS_REFRESH_INTERVAL seconds
    */
   private async $logStatsDaily(): Promise<void> {
-    const date = new Date();
-    Common.setDateMidnight(date);
-    const networkGraph = await lightningApi.$getNetworkGraph();
-    await LightningStatsImporter.computeNetworkStats(date.getTime() / 1000, networkGraph);
-    
-    logger.debug(`Updated latest network stats`, logger.tags.ln);
+    try {
+      const date = new Date();
+      Common.setDateMidnight(date);
+      const networkGraph = await lightningApi.$getNetworkGraph();
+      await LightningStatsImporter.computeNetworkStats(date.getTime() / 1000, networkGraph);
+      logger.debug(`Updated latest network stats`, logger.tags.ln);
+    } catch (e) {
+      logger.err(`Exception in $logStatsDaily. Reason: ${(e instanceof Error ? e.message : e)}`);
+    }
   }
 }
 

backend/src/tasks/lightning/sync-tasks/stats-importer.ts

@@ -15,16 +15,20 @@ class LightningStatsImporter {
   topologiesFolder = config.LIGHTNING.TOPOLOGY_FOLDER;
 
   async $run(): Promise<void> {
-    const [channels]: any[] = await DB.query('SELECT short_id from channels;');
-    logger.info(`Caching funding txs for currently existing channels`, logger.tags.ln);
-    await fundingTxFetcher.$fetchChannelsFundingTxs(channels.map(channel => channel.short_id));
+    try {
+      const [channels]: any[] = await DB.query('SELECT short_id from channels;');
+      logger.info(`Caching funding txs for currently existing channels`, logger.tags.ln);
+      await fundingTxFetcher.$fetchChannelsFundingTxs(channels.map(channel => channel.short_id));
 
-    if (config.MEMPOOL.NETWORK !== 'mainnet' || config.DATABASE.ENABLED === false) {
-      return;
+      if (config.MEMPOOL.NETWORK !== 'mainnet' || config.DATABASE.ENABLED === false) {
+        return;
+      }
+
+      await this.$importHistoricalLightningStats();
+      await this.$cleanupIncorrectSnapshot();
+    } catch (e) {
+      logger.err(`Exception in LightningStatsImporter::$run(). ${e}`);
     }
-
-    await this.$importHistoricalLightningStats();
-    await this.$cleanupIncorrectSnapshot();
   }
 
   /**

backend/src/tasks/pools-updater.ts

@@ -62,7 +62,7 @@ class PoolsUpdater {
       if (this.currentSha === null) {
         logger.info(`Downloading pools-v2.json for the first time from ${this.poolsUrl} over ${network}`, logger.tags.mining);
       } else {
-        logger.warn(`pools-v2.json is outdated, fetch latest from ${this.poolsUrl} over ${network}`, logger.tags.mining);
+        logger.warn(`pools-v2.json is outdated, fetching latest from ${this.poolsUrl} over ${network}`, logger.tags.mining);
       }
       const poolsJson = await this.query(this.poolsUrl);
       if (poolsJson === undefined) {

backend/src/tasks/price-updater.ts

@@ -222,7 +222,7 @@ class PriceUpdater {
   private async $insertMissingRecentPrices(type: 'hour' | 'day'): Promise<void> {
     const existingPriceTimes = await PricesRepository.$getPricesTimes();
 
-    logger.info(`Fetching ${type === 'day' ? 'dai' : 'hour'}ly price history from exchanges and saving missing ones into the database`, logger.tags.mining);
+    logger.debug(`Fetching ${type === 'day' ? 'dai' : 'hour'}ly price history from exchanges and saving missing ones into the database`, logger.tags.mining);
 
     const historicalPrices: PriceHistory[] = [];
 

backend/tsconfig.json

@@ -5,6 +5,7 @@
     "types": ["node", "jest"],
     "lib": ["es2019", "dom"],
     "strict": true,
+    "skipLibCheck": true,
     "noImplicitAny": false,
     "sourceMap": false,
     "outDir": "dist",

docker/README.md

@@ -34,6 +34,7 @@ If you want to use different credentials, specify them in the `docker-compose.ym
       CORE_RPC_PORT: "8332"
       CORE_RPC_USERNAME: "customuser"
       CORE_RPC_PASSWORD: "custompassword"
+      CORE_RPC_TIMEOUT: "60000"
 ```
 
 The IP address in the example above refers to Docker's default gateway IP address so that the container can hit the `bitcoind` instance running on the host machine. If your setup is different, update it accordingly.
@@ -112,6 +113,7 @@ Below we list all settings from `mempool-config.json` and the corresponding over
     "ADVANCED_GBT_MEMPOOL": false,
     "CPFP_INDEXING": false,
     "MAX_BLOCKS_BULK_QUERY": 0,
+    "DISK_CACHE_BLOCK_INTERVAL": 6
   },
 ```
 
@@ -143,6 +145,7 @@ Corresponding `docker-compose.yml` overrides:
       MEMPOOL_ADVANCED_GBT_MEMPOOL: ""
       MEMPOOL_CPFP_INDEXING: ""
       MAX_BLOCKS_BULK_QUERY: ""
+      DISK_CACHE_BLOCK_INTERVAL: ""
       ...
 ```
 
@@ -158,7 +161,8 @@ Corresponding `docker-compose.yml` overrides:
     "HOST": "127.0.0.1",
     "PORT": 8332,
     "USERNAME": "mempool",
-    "PASSWORD": "mempool"
+    "PASSWORD": "mempool",
+    "TIMEOUT": 60000
   },
 ```
 
@@ -170,6 +174,7 @@ Corresponding `docker-compose.yml` overrides:
       CORE_RPC_PORT: ""
       CORE_RPC_USERNAME: ""
       CORE_RPC_PASSWORD: ""
+      CORE_RPC_TIMEOUT: 60000
       ...
 ```
 
@@ -199,7 +204,9 @@ Corresponding `docker-compose.yml` overrides:
 `mempool-config.json`:
 ```json
   "ESPLORA": {
-    "REST_API_URL": "http://127.0.0.1:3000"
+    "REST_API_URL": "http://127.0.0.1:3000",
+    "UNIX_SOCKET_PATH": "/tmp/esplora-socket",
+    "RETRY_UNIX_SOCKET_AFTER": 30000
   },
 ```
 
@@ -208,6 +215,8 @@ Corresponding `docker-compose.yml` overrides:
   api:
     environment:
       ESPLORA_REST_API_URL: ""
+      ESPLORA_UNIX_SOCKET_PATH: ""
+      ESPLORA_RETRY_UNIX_SOCKET_AFTER: ""
       ...
 ```
 
@@ -219,7 +228,8 @@ Corresponding `docker-compose.yml` overrides:
     "HOST": "127.0.0.1",
     "PORT": 8332,
     "USERNAME": "mempool",
-    "PASSWORD": "mempool"
+    "PASSWORD": "mempool",
+    "TIMEOUT": 60000
   },
 ```
 
@@ -231,6 +241,7 @@ Corresponding `docker-compose.yml` overrides:
       SECOND_CORE_RPC_PORT: ""
       SECOND_CORE_RPC_USERNAME: ""
       SECOND_CORE_RPC_PASSWORD: ""
+      SECOND_CORE_RPC_TIMEOUT: ""
       ...
 ```
 
@@ -258,6 +269,7 @@ Corresponding `docker-compose.yml` overrides:
       DATABASE_DATABASE: ""
       DATABASE_USERNAME: ""
       DATABASE_PASSWORD: ""
+      DATABASE_TIMEOUT: ""
       ...
 ```
 
@@ -403,6 +415,7 @@ Corresponding `docker-compose.yml` overrides:
     "TLS_CERT_PATH": ""
     "MACAROON_PATH": ""
     "REST_API_URL": "https://localhost:8080"
+    "TIMEOUT": 10000
   }
 ```
 
@@ -413,6 +426,7 @@ Corresponding `docker-compose.yml` overrides:
       LND_TLS_CERT_PATH: ""
       LND_MACAROON_PATH: ""
       LND_REST_API_URL: "https://localhost:8080"
+      LND_TIMEOUT: 10000
       ...
 ```
 
@@ -450,8 +464,8 @@ Corresponding `docker-compose.yml` overrides:
   api:
     environment:
       MAXMIND_ENABLED: true,
-      MAXMIND_GEOLITE2_CITY: "./backend/GeoIP/GeoLite2-City.mmdb",
-      MAXMIND_GEOLITE2_ASN": "./backend/GeoIP/GeoLite2-ASN.mmdb",
-      MAXMIND_GEOIP2_ISP": "./backend/GeoIP/GeoIP2-ISP.mmdb"
+      MAXMIND_GEOLITE2_CITY: "/backend/GeoIP/GeoLite2-City.mmdb",
+      MAXMIND_GEOLITE2_ASN": "/backend/GeoIP/GeoLite2-ASN.mmdb",
+      MAXMIND_GEOIP2_ISP": "/backend/GeoIP/GeoIP2-ISP.mmdb"
       ...
 ```

docker/backend/mempool-config.json

@@ -26,13 +26,15 @@
     "ADVANCED_GBT_AUDIT": __MEMPOOL_ADVANCED_GBT_AUDIT__,
     "ADVANCED_GBT_MEMPOOL": __MEMPOOL_ADVANCED_GBT_MEMPOOL__,
     "CPFP_INDEXING": __MEMPOOL_CPFP_INDEXING__,
-    "MAX_BLOCKS_BULK_QUERY": __MEMPOOL_MAX_BLOCKS_BULK_QUERY__
+    "MAX_BLOCKS_BULK_QUERY": __MEMPOOL_MAX_BLOCKS_BULK_QUERY__,
+    "DISK_CACHE_BLOCK_INTERVAL": __MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__
   },
   "CORE_RPC": {
     "HOST": "__CORE_RPC_HOST__",
     "PORT": __CORE_RPC_PORT__,
     "USERNAME": "__CORE_RPC_USERNAME__",
-    "PASSWORD": "__CORE_RPC_PASSWORD__"
+    "PASSWORD": "__CORE_RPC_PASSWORD__",
+    "TIMEOUT": __CORE_RPC_TIMEOUT__
   },
   "ELECTRUM": {
     "HOST": "__ELECTRUM_HOST__",
@@ -40,13 +42,16 @@
     "TLS_ENABLED": __ELECTRUM_TLS_ENABLED__
   },
   "ESPLORA": {
-    "REST_API_URL": "__ESPLORA_REST_API_URL__"
+    "REST_API_URL": "__ESPLORA_REST_API_URL__",
+    "UNIX_SOCKET_PATH": "__ESPLORA_UNIX_SOCKET_PATH__",
+    "RETRY_UNIX_SOCKET_AFTER": __ESPLORA_RETRY_UNIX_SOCKET_AFTER__
   },
   "SECOND_CORE_RPC": {
     "HOST": "__SECOND_CORE_RPC_HOST__",
     "PORT": __SECOND_CORE_RPC_PORT__,
     "USERNAME": "__SECOND_CORE_RPC_USERNAME__",
-    "PASSWORD": "__SECOND_CORE_RPC_PASSWORD__"
+    "PASSWORD": "__SECOND_CORE_RPC_PASSWORD__",
+    "TIMEOUT": __SECOND_CORE_RPC_TIMEOUT__
   },
   "DATABASE": {
     "ENABLED": __DATABASE_ENABLED__,
@@ -55,7 +60,8 @@
     "PORT": __DATABASE_PORT__,
     "DATABASE": "__DATABASE_DATABASE__",
     "USERNAME": "__DATABASE_USERNAME__",
-    "PASSWORD": "__DATABASE_PASSWORD__"
+    "PASSWORD": "__DATABASE_PASSWORD__",
+    "TIMEOUT": "__DATABASE_TIMEOUT__"
   },
   "SYSLOG": {
     "ENABLED": __SYSLOG_ENABLED__,
@@ -83,7 +89,8 @@
   "LND": {
     "TLS_CERT_PATH": "__LND_TLS_CERT_PATH__",
     "MACAROON_PATH": "__LND_MACAROON_PATH__",
-    "REST_API_URL": "__LND_REST_API_URL__"
+    "REST_API_URL": "__LND_REST_API_URL__",
+    "TIMEOUT": "__LND_TIMEOUT__"
   },
   "CLIGHTNING": {
     "SOCKET": "__CLIGHTNING_SOCKET__"

docker/backend/start.sh

@@ -30,12 +30,14 @@ __MEMPOOL_ADVANCED_GBT_AUDIT__=${MEMPOOL_ADVANCED_GBT_AUDIT:=false}
 __MEMPOOL_ADVANCED_GBT_MEMPOOL__=${MEMPOOL_ADVANCED_GBT_MEMPOOL:=false}
 __MEMPOOL_CPFP_INDEXING__=${MEMPOOL_CPFP_INDEXING:=false}
 __MEMPOOL_MAX_BLOCKS_BULK_QUERY__=${MEMPOOL_MAX_BLOCKS_BULK_QUERY:=0}
+__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__=${MEMPOOL_DISK_CACHE_BLOCK_INTERVAL:=6}
 
 # CORE_RPC
 __CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1}
 __CORE_RPC_PORT__=${CORE_RPC_PORT:=8332}
 __CORE_RPC_USERNAME__=${CORE_RPC_USERNAME:=mempool}
 __CORE_RPC_PASSWORD__=${CORE_RPC_PASSWORD:=mempool}
+__CORE_RPC_TIMEOUT__=${CORE_RPC_TIMEOUT:=60000}
 
 # ELECTRUM
 __ELECTRUM_HOST__=${ELECTRUM_HOST:=127.0.0.1}
@@ -44,12 +46,15 @@ __ELECTRUM_TLS_ENABLED__=${ELECTRUM_TLS_ENABLED:=false}
 
 # ESPLORA
 __ESPLORA_REST_API_URL__=${ESPLORA_REST_API_URL:=http://127.0.0.1:3000}
+__ESPLORA_UNIX_SOCKET_PATH__=${ESPLORA_UNIX_SOCKET_PATH:=null}
+__ESPLORA_RETRY_UNIX_SOCKET_AFTER__=${ESPLORA_RETRY_UNIX_SOCKET_AFTER:=30000}
 
 # SECOND_CORE_RPC
 __SECOND_CORE_RPC_HOST__=${SECOND_CORE_RPC_HOST:=127.0.0.1}
 __SECOND_CORE_RPC_PORT__=${SECOND_CORE_RPC_PORT:=8332}
 __SECOND_CORE_RPC_USERNAME__=${SECOND_CORE_RPC_USERNAME:=mempool}
 __SECOND_CORE_RPC_PASSWORD__=${SECOND_CORE_RPC_PASSWORD:=mempool}
+__SECOND_CORE_RPC_TIMEOUT__=${SECOND_CORE_RPC_TIMEOUT:=60000}
 
 # DATABASE
 __DATABASE_ENABLED__=${DATABASE_ENABLED:=true}
@@ -59,6 +64,7 @@ __DATABASE_PORT__=${DATABASE_PORT:=3306}
 __DATABASE_DATABASE__=${DATABASE_DATABASE:=mempool}
 __DATABASE_USERNAME__=${DATABASE_USERNAME:=mempool}
 __DATABASE_PASSWORD__=${DATABASE_PASSWORD:=mempool}
+__DATABASE_TIMEOUT__=${DATABASE_TIMEOUT:=180000}
 
 # SYSLOG
 __SYSLOG_ENABLED__=${SYSLOG_ENABLED:=false}
@@ -107,14 +113,15 @@ __LIGHTNING_LOGGER_UPDATE_INTERVAL__=${LIGHTNING_LOGGER_UPDATE_INTERVAL:=30}
 __LND_TLS_CERT_PATH__=${LND_TLS_CERT_PATH:=""}
 __LND_MACAROON_PATH__=${LND_MACAROON_PATH:=""}
 __LND_REST_API_URL__=${LND_REST_API_URL:="https://localhost:8080"}
+__LND_TIMEOUT__=${LND_TIMEOUT:=10000}
 
 # CLN
 __CLIGHTNING_SOCKET__=${CLIGHTNING_SOCKET:=""}
 
 # MAXMIND
 __MAXMIND_ENABLED__=${MAXMIND_ENABLED:=true}
-__MAXMIND_GEOLITE2_CITY__=${MAXMIND_GEOLITE2_CITY:="./backend/GeoIP/GeoLite2-City.mmdb"}
-__MAXMIND_GEOLITE2_ASN__=${MAXMIND_GEOLITE2_ASN:="./backend/GeoIP/GeoLite2-ASN.mmdb"}
+__MAXMIND_GEOLITE2_CITY__=${MAXMIND_GEOLITE2_CITY:="/backend/GeoIP/GeoLite2-City.mmdb"}
+__MAXMIND_GEOLITE2_ASN__=${MAXMIND_GEOLITE2_ASN:="/backend/GeoIP/GeoLite2-ASN.mmdb"}
 __MAXMIND_GEOIP2_ISP__=${MAXMIND_GEOIP2_ISP:=""}
 
 
@@ -149,22 +156,27 @@ sed -i "s!__MEMPOOL_ADVANCED_GBT_MEMPOOL__!${__MEMPOOL_ADVANCED_GBT_MEMPOOL__}!g
 sed -i "s!__MEMPOOL_ADVANCED_GBT_AUDIT__!${__MEMPOOL_ADVANCED_GBT_AUDIT__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_CPFP_INDEXING__!${__MEMPOOL_CPFP_INDEXING__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_MAX_BLOCKS_BULK_QUERY__!${__MEMPOOL_MAX_BLOCKS_BULK_QUERY__}!g" mempool-config.json
+sed -i "s!__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__!${__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__}!g" mempool-config.json
 
 sed -i "s/__CORE_RPC_HOST__/${__CORE_RPC_HOST__}/g" mempool-config.json
 sed -i "s/__CORE_RPC_PORT__/${__CORE_RPC_PORT__}/g" mempool-config.json
 sed -i "s/__CORE_RPC_USERNAME__/${__CORE_RPC_USERNAME__}/g" mempool-config.json
 sed -i "s/__CORE_RPC_PASSWORD__/${__CORE_RPC_PASSWORD__}/g" mempool-config.json
+sed -i "s/__CORE_RPC_TIMEOUT__/${__CORE_RPC_TIMEOUT__}/g" mempool-config.json
 
 sed -i "s/__ELECTRUM_HOST__/${__ELECTRUM_HOST__}/g" mempool-config.json
 sed -i "s/__ELECTRUM_PORT__/${__ELECTRUM_PORT__}/g" mempool-config.json
 sed -i "s/__ELECTRUM_TLS_ENABLED__/${__ELECTRUM_TLS_ENABLED__}/g" mempool-config.json
 
 sed -i "s!__ESPLORA_REST_API_URL__!${__ESPLORA_REST_API_URL__}!g" mempool-config.json
+sed -i "s!__ESPLORA_UNIX_SOCKET_PATH__!${__ESPLORA_UNIX_SOCKET_PATH__}!g" mempool-config.json
+sed -i "s!__ESPLORA_RETRY_UNIX_SOCKET_AFTER__!${__ESPLORA_RETRY_UNIX_SOCKET_AFTER__}!g" mempool-config.json
 
 sed -i "s/__SECOND_CORE_RPC_HOST__/${__SECOND_CORE_RPC_HOST__}/g" mempool-config.json
 sed -i "s/__SECOND_CORE_RPC_PORT__/${__SECOND_CORE_RPC_PORT__}/g" mempool-config.json
 sed -i "s/__SECOND_CORE_RPC_USERNAME__/${__SECOND_CORE_RPC_USERNAME__}/g" mempool-config.json
 sed -i "s/__SECOND_CORE_RPC_PASSWORD__/${__SECOND_CORE_RPC_PASSWORD__}/g" mempool-config.json
+sed -i "s/__SECOND_CORE_RPC_TIMEOUT__/${__SECOND_CORE_RPC_TIMEOUT__}/g" mempool-config.json
 
 sed -i "s/__DATABASE_ENABLED__/${__DATABASE_ENABLED__}/g" mempool-config.json
 sed -i "s/__DATABASE_HOST__/${__DATABASE_HOST__}/g" mempool-config.json
@@ -216,6 +228,7 @@ sed -i "s!__LIGHTNING_LOGGER_UPDATE_INTERVAL__!${__LIGHTNING_LOGGER_UPDATE_INTER
 sed -i "s!__LND_TLS_CERT_PATH__!${__LND_TLS_CERT_PATH__}!g" mempool-config.json
 sed -i "s!__LND_MACAROON_PATH__!${__LND_MACAROON_PATH__}!g" mempool-config.json
 sed -i "s!__LND_REST_API_URL__!${__LND_REST_API_URL__}!g" mempool-config.json
+sed -i "s!__LND_TIMEOUT__!${__LND_TIMEOUT__}!g" mempool-config.json
 
 # CLN
 sed -i "s!__CLIGHTNING_SOCKET__!${__CLIGHTNING_SOCKET__}!g" mempool-config.json

docker/frontend/entrypoint.sh

@@ -10,6 +10,10 @@ cp /etc/nginx/nginx.conf /patch/nginx.conf
 sed -i "s/__MEMPOOL_FRONTEND_HTTP_PORT__/${__MEMPOOL_FRONTEND_HTTP_PORT__}/g" /patch/nginx.conf
 cat /patch/nginx.conf > /etc/nginx/nginx.conf
 
+if [ "${LIGHTNING_DETECTED_PORT}" != "" ];then
+  export LIGHTNING=true
+fi
+
 # Runtime overrides - read env vars defined in docker compose
 
 __TESTNET_ENABLED__=${TESTNET_ENABLED:=false}
@@ -35,6 +39,7 @@ __AUDIT__=${AUDIT:=false}
 __MAINNET_BLOCK_AUDIT_START_HEIGHT__=${MAINNET_BLOCK_AUDIT_START_HEIGHT:=0}
 __TESTNET_BLOCK_AUDIT_START_HEIGHT__=${TESTNET_BLOCK_AUDIT_START_HEIGHT:=0}
 __SIGNET_BLOCK_AUDIT_START_HEIGHT__=${SIGNET_BLOCK_AUDIT_START_HEIGHT:=0}
+__FULL_RBF_ENABLED__=${FULL_RBF_ENABLED:=false}
 __HISTORICAL_PRICE__=${HISTORICAL_PRICE:=true}
 
 # Export as environment variables to be used by envsubst
@@ -61,6 +66,7 @@ export __AUDIT__
 export __MAINNET_BLOCK_AUDIT_START_HEIGHT__
 export __TESTNET_BLOCK_AUDIT_START_HEIGHT__
 export __SIGNET_BLOCK_AUDIT_START_HEIGHT__
+export __FULL_RBF_ENABLED__
 export __HISTORICAL_PRICE__
 
 folder=$(find /var/www/mempool -name "config.js" | xargs dirname)

frontend/README.md

@@ -106,6 +106,7 @@ https://www.transifex.com/mempool/mempool/dashboard/
 
 * Arabic @baro0k
 * Czech @pixelmade2
+* Danish @pierrevendelboe
 * German @Emzy
 * English (default)
 * Spanish @maxhodler @bisqes
@@ -113,6 +114,7 @@ https://www.transifex.com/mempool/mempool/dashboard/
 * French @Bayernatoor
 * Korean @kcalvinalvinn @sogoagain
 * Italian @HodlBits
+* Lithuanian @eimze21
 * Hebrew @rapidlab309
 * Georgian @wyd_idk
 * Hungarian @btcdragonlord

frontend/cypress/e2e/liquid/liquid.spec.ts

@@ -158,10 +158,10 @@ describe('Liquid', () => {
       it('show empty unblinded TX', () => {
         cy.visit(`${basePath}/tx/f2f41c0850e8e7e3f1af233161fd596662e67c11ef10ed15943884186fbb7f46#blinded=`);
         cy.waitForSkeletonGone();
-        cy.get('.table-tx-vin tr:nth-child(1)').should('have.class', '');
+        cy.get('.table-tx-vin tr:nth-child(1)').should('have.class', 'ng-star-inserted');
         cy.get('.table-tx-vin tr:nth-child(1) .amount').should('contain.text', 'Confidential');
-        cy.get('.table-tx-vout tr:nth-child(1)').should('have.class', '');
-        cy.get('.table-tx-vout tr:nth-child(2)').should('have.class', '');
+        cy.get('.table-tx-vout tr:nth-child(1)').should('have.class', 'ng-star-inserted');
+        cy.get('.table-tx-vout tr:nth-child(2)').should('have.class', 'ng-star-inserted');
         cy.get('.table-tx-vout tr:nth-child(1) .amount').should('contain.text', 'Confidential');
         cy.get('.table-tx-vout tr:nth-child(2) .amount').should('contain.text', 'Confidential');
       });
@@ -169,8 +169,8 @@ describe('Liquid', () => {
       it('show invalid unblinded TX hex', () => {
         cy.visit(`${basePath}/tx/f2f41c0850e8e7e3f1af233161fd596662e67c11ef10ed15943884186fbb7f46#blinded=123`);
         cy.waitForSkeletonGone();
-        cy.get('.table-tx-vin tr').should('have.class', '');
-        cy.get('.table-tx-vout tr').should('have.class', '');
+        cy.get('.table-tx-vin tr').should('have.class', 'ng-star-inserted');
+        cy.get('.table-tx-vout tr').should('have.class', 'ng-star-inserted');
         cy.get('.error-unblinded').contains('Error: Invalid blinding data (invalid hex)');
       });
 

frontend/cypress/e2e/liquidtestnet/liquidtestnet.spec.ts

@@ -109,10 +109,10 @@ describe('Liquid Testnet', () => {
       it('show empty unblinded TX', () => {
         cy.visit(`${basePath}/tx/c3d908ab77891e4c569b0df71aae90f4720b157019ebb20db176f4f9c4d626b8#blinded=`);
         cy.waitForSkeletonGone();
-        cy.get('.table-tx-vin tr:nth-child(1)').should('have.class', '');
+        cy.get('.table-tx-vin tr:nth-child(1)').should('have.class', 'ng-star-inserted');
         cy.get('.table-tx-vin tr:nth-child(1) .amount').should('contain.text', 'Confidential');
-        cy.get('.table-tx-vout tr:nth-child(1)').should('have.class', '');
-        cy.get('.table-tx-vout tr:nth-child(2)').should('have.class', '');
+        cy.get('.table-tx-vout tr:nth-child(1)').should('have.class', 'ng-star-inserted');
+        cy.get('.table-tx-vout tr:nth-child(2)').should('have.class', 'ng-star-inserted');
         cy.get('.table-tx-vout tr:nth-child(1) .amount').should('contain.text', 'Confidential');
         cy.get('.table-tx-vout tr:nth-child(2) .amount').should('contain.text', 'Confidential');
       });
@@ -120,8 +120,8 @@ describe('Liquid Testnet', () => {
       it('show invalid unblinded TX hex', () => {
         cy.visit(`${basePath}/tx/2477f220eef1d03f8ffa4a2861c275d155c3562adf0d79523aeeb0c59ee611ba#blinded=5000`);
         cy.waitForSkeletonGone();
-        cy.get('.table-tx-vin tr').should('have.class', '');
-        cy.get('.table-tx-vout tr').should('have.class', '');
+        cy.get('.table-tx-vin tr').should('have.class', 'ng-star-inserted');
+        cy.get('.table-tx-vout tr').should('have.class', 'ng-star-inserted');
         cy.get('.error-unblinded').contains('Error: Invalid blinding data (invalid hex)');
       });
 

frontend/cypress/e2e/mainnet/mainnet.spec.ts

@@ -127,7 +127,7 @@ describe('Mainnet', () => {
 
         cy.get('.search-box-container > .form-control').type('S').then(() => {
           cy.wait('@search-1wizS');
-          cy.get('app-search-results button.dropdown-item').should('have.length', 5);
+          cy.get('app-search-results button.dropdown-item').should('have.length', 6);
         });
 
         cy.get('.search-box-container > .form-control').type('A').then(() => {

frontend/mempool-frontend-config.sample.json

@@ -22,5 +22,6 @@
   "TESTNET_BLOCK_AUDIT_START_HEIGHT": 0,
   "SIGNET_BLOCK_AUDIT_START_HEIGHT": 0,
   "LIGHTNING": false,
+  "FULL_RBF_ENABLED": false,
   "HISTORICAL_PRICE": true
 }

frontend/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "mempool-frontend",
-  "version": "2.5.0-dev",
+  "version": "2.6.0-dev",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "mempool-frontend",
-      "version": "2.5.0-dev",
+      "version": "2.6.0-dev",
       "license": "GNU Affero General Public License v3.0",
       "dependencies": {
         "@angular-devkit/build-angular": "^14.2.10",

frontend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mempool-frontend",
-  "version": "2.5.0-dev",
+  "version": "2.6.0-dev",
   "description": "Bitcoin mempool visualizer and blockchain explorer backend",
   "license": "GNU Affero General Public License v3.0",
   "homepage": "https://mempool.space",

frontend/src/app/app-routing.module.ts

@@ -14,6 +14,7 @@ import { TrademarkPolicyComponent } from './components/trademark-policy/trademar
 import { BisqMasterPageComponent } from './components/bisq-master-page/bisq-master-page.component';
 import { PushTransactionComponent } from './components/push-transaction/push-transaction.component';
 import { BlocksList } from './components/blocks-list/blocks-list.component';
+import { RbfList } from './components/rbf-list/rbf-list.component';
 import { LiquidMasterPageComponent } from './components/liquid-master-page/liquid-master-page.component';
 import { AssetGroupComponent } from './components/assets/asset-group/asset-group.component';
 import { AssetsFeaturedComponent } from './components/assets/assets-featured/assets-featured.component';
@@ -56,6 +57,10 @@ let routes: Routes = [
             path: 'blocks',
             component: BlocksList,
           },
+          {
+            path: 'rbf',
+            component: RbfList,
+          },
           {
             path: 'terms-of-service',
             component: TermsOfServiceComponent
@@ -162,6 +167,10 @@ let routes: Routes = [
             path: 'blocks',
             component: BlocksList,
           },
+          {
+            path: 'rbf',
+            component: RbfList,
+          },
           {
             path: 'terms-of-service',
             component: TermsOfServiceComponent
@@ -264,6 +273,10 @@ let routes: Routes = [
         path: 'blocks',
         component: BlocksList,
       },
+      {
+        path: 'rbf',
+        component: RbfList,
+      },
       {
         path: 'terms-of-service',
         component: TermsOfServiceComponent

frontend/src/app/bisq/bisq-main-dashboard/bisq-main-dashboard.component.html

@@ -107,12 +107,7 @@
 
   <app-language-selector></app-language-selector>
 
-  <div class="text-small text-center mt-3">
-    <a [routerLink]="['/terms-of-service']" i18n="shared.terms-of-service|Terms of Service">Terms of Service</a>
-    |
-    <a [routerLink]="['/privacy-policy']" i18n="shared.privacy-policy|Privacy Policy">Privacy Policy</a>
-  </div>
-
+  <app-global-footer></app-global-footer>
 </div>
 
 <ng-template #loadingTmpl>

frontend/src/app/components/about/about.component.html

@@ -107,22 +107,7 @@
         <span>Blockstream</span>
       </a>
       <a href="https://unchained.com/" target="_blank" title="Unchained">
-        <svg xmlns="http://www.w3.org/2000/svg" version="1.1" x="0px" y="0px" viewBox="0 0 216 216" class="image" style="enable-background:new 0 0 216 216;">
-        <style type="text/css">
-          .ucst0{fill:#002248;}
-          .ucst1{opacity:0.5;fill:#FFFFFF;}
-          .ucst2{fill:#FFFFFF;}
-          .ucst3{opacity:0.75;fill:#FFFFFF;}
-        </style>
-        <rect class="ucst0" width="216" height="216"/>
-        <g>
-          <g>
-            <path class="ucst1" d="M108,39.5V108l59.3,34.2V73.8L108,39.5z M126.9,95.4c0,2,1.1,3.8,2.8,4.8l27.9,16l0,10.8L125,108.2c-4.6-2.6-7.4-7.5-7.4-12.8l-0.1-22.7c0-1.9,0.5-3.7,1.4-5.3c0.9-1.5,2.2-2.9,3.8-3.8c3.3-1.9,7.2-1.9,10.5,0l24.5,14.2l-0.2,10.7l-29-16.8c-0.5-0.3-0.9-0.2-1.2,0c-0.3,0.2-0.6,0.5-0.6,1L126.9,95.4z"/>
-            <path class="ucst2" d="M108,39.5L48.7,73.8v68.5L108,108V39.5z M99.7,93.1c0,5.3-2.8,10.2-7.4,12.8l-19.6,11.4c-1.7,1-3.5,1.4-5.3,1.5c-1.8,0-3.6-0.5-5.2-1.4c-3.3-1.9-5.3-5.3-5.3-9.1V80l9.4-5.2l-0.1,33.5c0,0.6,0.3,0.9,0.6,1c0.3,0.2,0.7,0.3,1.2,0l19.6-11.4c1.7-1,2.8-2.8,2.8-4.8L90.3,61l9.4-5.4L99.7,93.1z"/>
-            <path class="ucst3" d="M108,108l-59.3,34.2l59.3,34.2l59.3-34.2L108,108z M133.8,152l-24.5,14.2l-9.2-5.5l29.1-16.7c0.5-0.3,0.6-0.7,0.6-1c0-0.3-0.1-0.7-0.6-1l-19.7-11.2c-1.7-1-3.8-1-5.5,0l-27.8,16.1l-9.4-5.4l32.6-18.7c4.6-2.6,10.2-2.6,14.8,0l19.7,11.2c1.7,0.9,3,2.3,3.9,3.9c0.9,1.5,1.4,3.3,1.4,5.2C139.1,146.7,137.1,150.1,133.8,152z"/>
-          </g>
-        </g>
-        </svg>
+        <svg id="Layer_1" width="78" height="78" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 156.68 156.68"><defs><style>.cls-unchained-1{fill:#fff;}</style></defs><path class="cls-unchained-1" d="m78.34,0C35.07,0,0,35.07,0,78.34s35.07,78.34,78.34,78.34,78.34-35.07,78.34-78.34S121.6,0,78.34,0ZM20.23,109.5c-4.99-9.28-7.81-19.89-7.81-31.16C12.42,41.93,41.93,12.42,78.34,12.42c33.15,0,60.58,24.46,65.23,56.32h-37.48c-45.29,0-71.19,20.05-85.85,40.76Zm58.11,34.76c-12.42,0-24.04-3.44-33.96-9.41,3.94-8.85,9.11-18.7,15.84-28.9,20.99-31.8,52.2-31.19,76.49-31.19h7.45c.06,1.18.1,2.38.1,3.58,0,36.41-29.51,65.92-65.92,65.92Z"/><path class="cls-unchained-1" d="m91.98,42.4l-3.62-1.18c-3.94-1.29-7.03-4.38-8.32-8.32l-1.18-3.63c-.13-.39-.68-.39-.81,0l-1.18,3.63c-1.29,3.94-4.38,7.03-8.32,8.32l-3.62,1.18c-.39.13-.39.68,0,.81l3.62,1.18c3.94,1.29,7.03,4.38,8.32,8.32l1.18,3.63c.13.39.68.39.81,0l1.18-3.63c1.29-3.94,4.38-7.03,8.32-8.32l3.62-1.18c.39-.13.39-.68,0-.81Z"/></svg>
         <span>Unchained</span>
       </a>
       <a href="https://gemini.com/" target="_blank" title="Gemini">
@@ -201,12 +186,12 @@
         <span>Umbrel</span>
       </a>
       <a href="https://github.com/rootzoll/raspiblitz" target="_blank" title="RaspiBlitz">
-        <img class="image" src="/resources/profile/raspiblitz.jpg" />
+        <img class="image" src="/resources/profile/raspiblitz.svg" />
         <span>RaspiBlitz</span>
       </a>
-      <a href="https://github.com/mynodebtc/mynode" target="_blank" title="MyNode">
-        <img class="image" src="/resources/profile/mynodebtc.jpg" />
-        <span>MyNode</span>
+      <a href="https://github.com/mynodebtc/mynode" target="_blank" title="myNode">
+        <img class="image" src="/resources/profile/mynodebtc.png" />
+        <span>myNode</span>
       </a>
       <a href="https://github.com/RoninDojo/RoninDojo" target="_blank" title="RoninDojo">
         <img class="image" src="/resources/profile/ronindojo.png" />
@@ -253,7 +238,7 @@
         <span>Sparrow</span>
       </a>
       <a href="https://github.com/ACINQ/phoenix" target="_blank" title="Phoenix Wallet by ACINQ">
-        <img class="image" src="/resources/profile/phoenix.jpg" />
+        <img class="image not-rounded" src="/resources/profile/phoenix.svg" />
         <span>Phoenix</span>
       </a>
       <a href="https://github.com/lnbits/lnbits-legend" target="_blank" title="LNbits">
@@ -408,7 +393,6 @@
 
   <div class="footer-links">
     <a href="/3rdpartylicenses.txt">Third-party Licenses</a>
-    <a [routerLink]="['/terms-of-service']" i18n="shared.terms-of-service|Terms of Service">Terms of Service</a>
     <div class="social-icons">
       <a target="_blank" href="https://github.com/mempool/mempool">
         <svg aria-hidden="true" focusable="false" data-prefix="fab" data-icon="github" class="svg-inline--fa fa-github fa-w-16 fa-2x" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 496 512"><path fill="currentColor" d="M165.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6zm-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3zm44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9zM244.8 8C106.1 8 0 113.3 0 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C428.2 457.8 496 362.9 496 252 496 113.3 383.5 8 244.8 8zM97.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1zm-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7zm32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1zm-11.4-14.7c-1.6 1-1.6 3.6 0 5.9 1.6 2.3 4.3 3.3 5.6 2.3 1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2z"></path></svg>
@@ -432,9 +416,12 @@
     {{ (backendInfo$ | async)?.hostname }} (v{{ (backendInfo$ | async )?.version }}) [<a href="https://github.com/mempool/mempool/commit/{{ (backendInfo$ | async )?.gitCommit | slice:0:8 }}">{{ (backendInfo$ | async )?.gitCommit | slice:0:8 }}</a>]
   </div>
 
+  <br>
 </div>
 
+
 <ng-template #loadingSponsors>
   <br>
   <div class="spinner-border text-light"></div>
 </ng-template>
+

frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.scss

@@ -4,6 +4,9 @@
   @media (min-width: 465px) {
     font-size: 20px;
   }
+  @media (min-width: 992px) {
+    height: 40px;
+  }
 }
 
 .main-title {
@@ -18,17 +21,19 @@
 }
 
 .full-container {
+  display: flex;
+  flex-direction: column;
   padding: 0px 15px;
   width: 100%;
-  min-height: 500px;
-  height: calc(100% - 150px);
-  @media (max-width: 992px) {
-    padding-bottom: 100px;
-  };
+  height: calc(100vh - 250px);
+  @media (min-width: 992px) {
+    height: calc(100vh - 150px);
+  }
 }
 
 .chart {
-  width: 100%;
+  display: flex;
+  flex: 1;
   height: 100%;
   padding-bottom: 20px;
   padding-right: 10px;

frontend/src/app/components/block-fees-graph/block-fees-graph.component.scss

@@ -4,6 +4,9 @@
   @media (min-width: 465px) {
     font-size: 20px;
   }
+  @media (min-width: 992px) {
+    height: 40px;
+  }
 }
 
 .main-title {
@@ -18,18 +21,20 @@
 }
 
 .full-container {
+  display: flex;
+  flex-direction: column;
   padding: 0px 15px;
   width: 100%;
-  min-height: 500px;
-  height: calc(100% - 150px);
-  @media (max-width: 992px) {
-    padding-bottom: 100px;
-  };
+  height: calc(100vh - 250px);
+  @media (min-width: 992px) {
+    height: calc(100vh - 150px);
+  }
 }
 
 .chart {
+  display: flex;
+  flex: 1;
   width: 100%;
-  height: 100%;
   padding-bottom: 20px;
   padding-right: 10px;
   @media (max-width: 992px) {

frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts

@@ -23,7 +23,7 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On
   @Input() unavailable: boolean = false;
   @Input() auditHighlighting: boolean = false;
   @Input() blockConversion: Price;
-  @Output() txClickEvent = new EventEmitter<TransactionStripped>();
+  @Output() txClickEvent = new EventEmitter<{ tx: TransactionStripped, keyModifier: boolean}>();
   @Output() txHoverEvent = new EventEmitter<string>();
   @Output() readyEvent = new EventEmitter();
 
@@ -132,9 +132,9 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On
     }
   }
 
-  update(add: TransactionStripped[], remove: string[], direction: string = 'left', resetLayout: boolean = false): void {
+  update(add: TransactionStripped[], remove: string[], change: { txid: string, rate: number | undefined }[], direction: string = 'left', resetLayout: boolean = false): void {
     if (this.scene) {
-      this.scene.update(add, remove, direction, resetLayout);
+      this.scene.update(add, remove, change, direction, resetLayout);
       this.start();
     }
   }
@@ -326,7 +326,9 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On
     if (event.target === this.canvas.nativeElement && event.pointerType === 'touch') {
       this.setPreviewTx(event.offsetX, event.offsetY, true);
     } else if (event.target === this.canvas.nativeElement) {
-      this.onTxClick(event.offsetX, event.offsetY);
+      const keyMod = event.shiftKey || event.ctrlKey || event.metaKey;
+      const middleClick = event.which === 2 || event.button === 1;
+      this.onTxClick(event.offsetX, event.offsetY, keyMod || middleClick);
     }
   }
 
@@ -409,12 +411,12 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On
     }
   }
 
-  onTxClick(cssX: number, cssY: number) {
+  onTxClick(cssX: number, cssY: number, keyModifier: boolean = false) {
     const x = cssX * window.devicePixelRatio;
     const y = cssY * window.devicePixelRatio;
     const selected = this.scene.getTxAt({ x, y });
     if (selected && selected.txid) {
-      this.txClickEvent.emit(selected);
+      this.txClickEvent.emit({ tx: selected, keyModifier });
     }
   }
 

frontend/src/app/components/block-overview-graph/block-scene.ts

@@ -150,7 +150,7 @@ export default class BlockScene {
     this.updateAll(startTime, 200, direction);
   }
 
-  update(add: TransactionStripped[], remove: string[], direction: string = 'left', resetLayout: boolean = false): void {
+  update(add: TransactionStripped[], remove: string[], change: { txid: string, rate: number | undefined }[], direction: string = 'left', resetLayout: boolean = false): void {
     const startTime = performance.now();
     const removed = this.removeBatch(remove, startTime, direction);
 
@@ -172,6 +172,15 @@ export default class BlockScene {
         this.place(tx);
       });
     } else {
+      // update effective rates
+      change.forEach(tx => {
+        if (this.txs[tx.txid]) {
+          this.txs[tx.txid].feerate = tx.rate || (this.txs[tx.txid].fee / this.txs[tx.txid].vsize);
+          this.txs[tx.txid].rate = tx.rate;
+          this.txs[tx.txid].dirty = true;
+        }
+      });
+
       // try to insert new txs directly
       const remaining = [];
       add.map(tx => new TxView(tx, this)).sort(feeRateDescending).forEach(tx => {

frontend/src/app/components/block-overview-graph/tx-view.ts

@@ -36,6 +36,7 @@ export default class TxView implements TransactionStripped {
   vsize: number;
   value: number;
   feerate: number;
+  rate?: number;
   status?: 'found' | 'missing' | 'fresh' | 'added' | 'censored' | 'selected';
   context?: 'projected' | 'actual';
   scene?: BlockScene;
@@ -58,7 +59,8 @@ export default class TxView implements TransactionStripped {
     this.fee = tx.fee;
     this.vsize = tx.vsize;
     this.value = tx.value;
-    this.feerate = tx.fee / tx.vsize;
+    this.feerate = tx.rate || (tx.fee / tx.vsize); // sort by effective fee rate where available
+    this.rate = tx.rate;
     this.status = tx.status;
     this.initialised = false;
     this.vertexArray = scene.vertexArray;
@@ -157,7 +159,8 @@ export default class TxView implements TransactionStripped {
   }
 
   getColor(): Color {
-    const feeLevelIndex = feeLevels.findIndex((feeLvl) => Math.max(1, this.feerate) < feeLvl) - 1;
+    const rate = this.fee / this.vsize; // color by simple single-tx fee rate
+    const feeLevelIndex = feeLevels.findIndex((feeLvl) => Math.max(1, rate) < feeLvl) - 1;
     const feeLevelColor = feeColors[feeLevelIndex] || feeColors[mempoolFeeColors.length - 1];
     // Normal mode
     if (!this.scene?.highlightingEnabled) {

frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html

@@ -28,6 +28,12 @@
           {{ feeRate | feeRounding }} <span class="symbol" i18n="shared.sat-vbyte|sat/vB">sat/vB</span>
         </td>
       </tr>
+      <tr *ngIf="effectiveRate && effectiveRate !== feeRate">
+        <td class="td-width" i18n="transaction.effective-fee-rate|Effective transaction fee rate">Effective fee rate</td>
+        <td>
+          {{ effectiveRate | feeRounding }} <span class="symbol" i18n="shared.sat-vbyte|sat/vB">sat/vB</span>
+        </td>
+      </tr>
       <tr>
         <td class="td-width" i18n="transaction.vsize|Transaction Virtual Size">Virtual size</td>
         <td [innerHTML]="'&lrm;' + (vsize | vbytes: 2)"></td>

frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.ts

@@ -20,6 +20,7 @@ export class BlockOverviewTooltipComponent implements OnChanges {
   value = 0;
   vsize = 1;
   feeRate = 0;
+  effectiveRate;
 
   tooltipPosition: Position = { x: 0, y: 0 };
 
@@ -51,6 +52,7 @@ export class BlockOverviewTooltipComponent implements OnChanges {
       this.value = tx.value || 0;
       this.vsize = tx.vsize || 1;
       this.feeRate = this.fee / this.vsize;
+      this.effectiveRate = tx.rate;
     }
   }
 }

frontend/src/app/components/block-prediction-graph/block-prediction-graph.component.scss

@@ -4,6 +4,9 @@
   @media (min-width: 465px) {
     font-size: 20px;
   }
+  @media (min-width: 992px) {
+    height: 40px;
+  }
 }
 
 .main-title {
@@ -18,17 +21,19 @@
 }
 
 .full-container {
+  display: flex;
+  flex-direction: column;
   padding: 0px 15px;
   width: 100%;
-  min-height: 500px;
-  height: calc(100% - 150px);
-  @media (max-width: 992px) {
-    padding-bottom: 100px;
-  };
+  height: calc(100vh - 250px);
+  @media (min-width: 992px) {
+    height: calc(100vh - 150px);
+  }
 }
 
 .chart {
-  width: 100%;
+  display: flex;
+  flex: 1;
   height: 100%;
   padding-bottom: 20px;
   padding-right: 10px;

frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.scss

@@ -4,6 +4,9 @@
   @media (min-width: 465px) {
     font-size: 20px;
   }
+  @media (min-width: 992px) {
+    height: 40px;
+  }
 }
 
 .main-title {
@@ -18,17 +21,19 @@
 }
 
 .full-container {
+  display: flex;
+  flex-direction: column;
   padding: 0px 15px;
   width: 100%;
-  min-height: 500px;
-  height: calc(100% - 150px);
-  @media (max-width: 992px) {
-    padding-bottom: 100px;
-  };
+  height: calc(100vh - 250px);
+  @media (min-width: 992px) {
+    height: calc(100vh - 150px);
+  }
 }
 
 .chart {
-  width: 100%;
+  display: flex;
+  flex: 1;
   height: 100%;
   padding-bottom: 20px;
   padding-right: 10px;

frontend/src/app/components/block-sizes-weights-graph/block-sizes-weights-graph.component.scss

@@ -4,6 +4,9 @@
   @media (min-width: 465px) {
     font-size: 20px;
   }
+  @media (min-width: 992px) {
+    height: 40px;
+  }
 }
 
 .main-title {
@@ -18,17 +21,19 @@
 }
 
 .full-container {
+  display: flex;
+  flex-direction: column;
   padding: 0px 15px;
   width: 100%;
-  min-height: 500px;
-  height: calc(100% - 150px);
-  @media (max-width: 992px) {
-    padding-bottom: 100px;
-  };
+  height: calc(100vh - 250px);
+  @media (min-width: 992px) {
+    height: calc(100vh - 150px);
+  }
 }
 
 .chart {
-  width: 100%;
+  display: flex;
+  flex: 1;
   height: 100%;
   padding-bottom: 20px;
   padding-right: 10px;

frontend/src/app/components/block/block.component.ts

@@ -612,9 +612,13 @@ export class BlockComponent implements OnInit, OnDestroy {
     });
   }
 
-  onTxClick(event: TransactionStripped): void {
-    const url = new RelativeUrlPipe(this.stateService).transform(`/tx/${event.txid}`);
-    this.router.navigate([url]);
+  onTxClick(event: { tx: TransactionStripped, keyModifier: boolean }): void {
+    const url = new RelativeUrlPipe(this.stateService).transform(`/tx/${event.tx.txid}`);
+    if (!event.keyModifier) {
+      this.router.navigate([url]);
+    } else {
+      window.open(url, '_blank');
+    }
   }
 
   onTxHover(txid: string): void {

frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html

@@ -25,7 +25,7 @@
           </ng-template>
           <div [attr.data-cy]="'bitcoin-block-' + offset + '-index-' + i + '-fee-span'" class="fee-span"
             *ngIf="block?.extras?.feeRange; else emptyfeespan">
-            {{ block?.extras?.feeRange?.[1] | number:feeRounding }} - {{
+            {{ block?.extras?.feeRange?.[0] | number:feeRounding }} - {{
             block?.extras?.feeRange[block?.extras?.feeRange?.length - 1] | number:feeRounding }} <ng-container
               i18n="shared.sat-vbyte|sat/vB">sat/vB</ng-container>
           </div>

frontend/src/app/components/blocks-list/blocks-list.component.html

@@ -26,7 +26,7 @@
       </thead>
       <tbody *ngIf="blocks$ | async as blocks; else skeleton" [style]="isLoading ? 'opacity: 0.75' : ''">
         <tr *ngFor="let block of blocks; let i= index; trackBy: trackByBlock">
-          <td class="text-left" [class]="widget ? 'widget' : ''">
+          <td class="height text-left" [class]="widget ? 'widget' : ''">
             <a [routerLink]="['/block' | relativeUrl, block.id]" [state]="{ data: { block: block } }">{{ block.height }}</a>
           </td>
           <td  *ngIf="indexingAvailable" class="pool text-left" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}">
@@ -89,7 +89,6 @@
               <span class="skeleton-loader" style="max-width: 75px"></span>
             </td>
             <td *ngIf="indexingAvailable" class="pool text-left" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}">
-              <img width="1" height="25" style="opacity: 0">
               <span class="skeleton-loader" style="max-width: 125px"></span>
             </td>
             <td class="timestamp" *ngIf="!widget" [class]="indexingAvailable ? '' : 'legacy'">
@@ -98,7 +97,7 @@
             <td class="mined" *ngIf="!widget" [class]="indexingAvailable ? '' : 'legacy'">
               <span class="skeleton-loader" style="max-width: 125px"></span>
             </td>
-            <td *ngIf="auditAvailable" class="health text-left" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}">
+            <td *ngIf="auditAvailable" class="health text-right" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}">
               <span class="skeleton-loader" style="max-width: 75px"></span>
             </td>
             <td *ngIf="indexingAvailable" class="reward text-right" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}">

frontend/src/app/components/blocks-list/blocks-list.component.scss

@@ -51,7 +51,12 @@ tr, td, th {
 .pool.widget {
   width: 40%;
   padding-left: 24px;
-  @media (max-width: 376px) {
+  @media (min-width: 768px) AND (max-width: 926px) {
+    padding-left: 0px;
+    width: 60%;
+  }
+  @media (max-width: 430px) {
+    padding-left: 0px;
     width: 60%;
   }
 }
@@ -59,6 +64,10 @@ tr, td, th {
   display: inline-block;
   vertical-align: text-top;
   padding-left: 10px;
+  overflow: hidden;
+  text-overflow: ellipsis;
+  white-space: nowrap;
+  max-width: 160px;
 }
 
 .height {
@@ -69,6 +78,12 @@ tr, td, th {
   @media (max-width: 576px) {
     width: 10%;
   }
+  @media (min-width: 768px) AND (max-width: 926px) {
+    width: 30%;
+  }
+  @media (max-width: 430px) {
+    width: 30%;
+  }
 }
 .height.legacy {
   width: 15%;
@@ -92,7 +107,7 @@ tr, td, th {
 
 .mined {
   width: 13%;
-  @media (max-width: 576px) {
+  @media (max-width: 730px) {
     display: none;
   }
 }
@@ -138,7 +153,7 @@ tr, td, th {
 
 .fees {
   width: 8%;
-  @media (max-width: 650px) {
+  @media (max-width: 820px) {
     display: none;
   }
 }
@@ -163,6 +178,16 @@ tr, td, th {
     width: 30%;
     padding-right: 0;
   }
+  @media (min-width: 768px) AND (max-width: 926px) {
+    overflow: hidden;
+    text-overflow: ellipsis;
+    max-width: 90px;
+  }
+  @media (max-width: 430px) {
+    overflow: hidden;
+    text-overflow: ellipsis;
+    max-width: 90px;
+  }
 }
 
 .size {
@@ -189,10 +214,10 @@ tr, td, th {
 
 .health {
   width: 10%;
-  @media (max-width: 1000px) {
+  @media (max-width: 1105px) {
     width: 13%;
   }
-  @media (max-width: 950px) {
+  @media (max-width: 560px) {
     display: none;
   }
 
@@ -202,7 +227,7 @@ tr, td, th {
 }
 .health.widget {
   width: 25%;
-  @media (max-width: 1000px) {
+  @media (max-width: 1105px) {
     display: none;
   }
   @media (max-width: 767px) {
@@ -242,4 +267,4 @@ tr, td, th {
 	vertical-align: middle;
 	max-width: 50vw;
   text-align: left;
-}
+}

frontend/src/app/components/difficulty-adjustments-table/difficulty-adjustments-table.component.html

@@ -2,18 +2,19 @@
   <table class="table latest-adjustments">
     <thead>
       <tr>
-        <th class="d-none d-md-block" i18n="block.height">Height</th>
-        <th i18n="mining.adjusted" class="text-left">Adjusted</th>
-        <th i18n="mining.difficulty" class="text-right">Difficulty</th>
-        <th i18n="mining.change" class="text-right">Change</th>
+        <th class="" i18n="block.height">Height</th>
+        <th class="date text-left" i18n="mining.adjusted">Adjusted</th>
+        <th class="text-right" i18n="mining.difficulty">Difficulty</th>
+        <th class="text-right" i18n="mining.change">Change</th>
       </tr>
     </thead>
     <tbody *ngIf="(hashrateObservable$ | async) as data">
       <tr *ngFor="let diffChange of data">
-        <td class="d-none d-md-block"><a [routerLink]="['/block' | relativeUrl, diffChange.height]">{{ diffChange.height
-            }}</a></td>
-        <td class="text-left">
-          <app-time kind="since" [time]="diffChange.timestamp" [fastRender]="true"></app-time>
+        <td class="">
+          <a [routerLink]="['/block' | relativeUrl, diffChange.height]">{{ diffChange.height }}</a>
+        </td>
+        <td class="date text-left">
+          <app-time kind="since" [time]="diffChange.timestamp" [fastRender]="true" [precision]="1"></app-time>
         </td>
         <td class="text-right">{{ diffChange.difficultyShorten }}</td>
         <td class="text-right" [style]="diffChange.change >= 0 ? 'color: #42B747' : 'color: #B74242'">
@@ -23,8 +24,8 @@
     </tbody>
     <tbody *ngIf="isLoading">
       <tr *ngFor="let item of [1,2,3,4,5,6]">
-        <td class="d-none d-md-block w-75"><span class="skeleton-loader"></span></td>
-        <td class="text-left"><span class="skeleton-loader w-75"></span></td>
+        <td class=""><span class="skeleton-loader"></span></td>
+        <td class="date text-left"><span class="skeleton-loader w-75"></span></td>
         <td class="text-right"><span class="skeleton-loader w-75"></span></td>
         <td class="text-right"><span class="skeleton-loader w-75"></span></td>
       </tr>

frontend/src/app/components/difficulty-adjustments-table/difficulty-adjustments-table.component.scss

@@ -17,3 +17,12 @@
     }
   }
 }
+
+.date {
+  @media (min-width: 767px) AND (max-width: 991px) {
+    display: none;
+  }
+  @media (max-width: 500px) {
+    display: none;
+  }
+}

frontend/src/app/components/difficulty-mining/difficulty-mining.component.html

@@ -10,7 +10,7 @@
             <ng-template #blocksPlural let-i i18n="shared.blocks">{{ i }} <span class="shared-block">blocks</span></ng-template>
             <ng-template #blocksSingular let-i i18n="shared.block">{{ i }} <span class="shared-block">block</span></ng-template>
           </div>
-          <div class="symbol"><app-time kind="until" [time]="epochData.estimatedRetargetDate" [fastRender]="true"></app-time></div>
+          <div class="symbol"><app-time kind="until" [time]="epochData.estimatedRetargetDate" [fastRender]="true" [precision]="1"></app-time></div>
         </div>
         <div class="item">
           <h5 class="card-title" i18n="difficulty-box.estimate">Estimate</h5>
@@ -47,13 +47,14 @@
           </div>
         </div>
         <div class="item" *ngIf="showHalving">
-          <h5 class="card-title" i18n="difficulty-box.next-halving">Next Halving</h5>
+          <h5 class="card-title" i18n="difficulty-box.next-halving" i18n-ngbTooltip="difficulty-box.next-halving"
+          ngbTooltip="Next Halving" placement="bottom" #averagefee [disableTooltip]="!isEllipsisActive(averagefee)">Next Halving</h5>
           <div class="card-text">
             <ng-container *ngTemplateOutlet="epochData.blocksUntilHalving === 1 ? blocksSingular : blocksPlural; context: {$implicit: epochData.blocksUntilHalving }"></ng-container>
             <ng-template #blocksPlural let-i i18n="shared.blocks">{{ i }} <span class="shared-block">blocks</span></ng-template>
             <ng-template #blocksSingular let-i i18n="shared.block">{{ i }} <span class="shared-block">block</span></ng-template>
           </div>
-          <div class="symbol"><app-time kind="until" [time]="epochData.timeUntilHalving" [fastRender]="true"></app-time></div>
+          <div class="symbol"><app-time kind="until" [time]="epochData.timeUntilHalving" [fastRender]="true" [precision]="1"></app-time></div>
         </div>
       </div>
     </div>
@@ -77,7 +78,7 @@
       </div>
     </div>
     <div class="item">
-      <h5 class="card-title" i18n="difficulty-box.current-period">Current Period</h5>
+      <h5 class="card-title" i18n="difficulty-box.next-halving">Next Halving</h5>
       <div class="card-text">
         <div class="skeleton-loader"></div>
         <div class="skeleton-loader"></div>

frontend/src/app/components/difficulty-mining/difficulty-mining.component.scss

@@ -10,6 +10,7 @@
   .item {
     padding: 0 5px;
     width: 100%;
+    max-width: 150px;
     &:nth-child(1) {
       display: none;
       @media (min-width: 485px) {
@@ -85,6 +86,9 @@
 .card-title {
   color: #4a68b9;
   font-size: 1rem;
+  overflow: hidden;
+  text-overflow: ellipsis;
+  white-space: nowrap;
 }
 
 .progress {
@@ -152,4 +156,5 @@
 
 .symbol {
   font-size: 13px;
+  white-space: nowrap;
 }

frontend/src/app/components/difficulty-mining/difficulty-mining.component.ts

@@ -83,4 +83,8 @@ export class DifficultyMiningComponent implements OnInit {
       })
     );
   }
+
+  isEllipsisActive(e): boolean {
+    return (e.offsetWidth < e.scrollWidth);
+  }
 }

frontend/src/app/components/difficulty/difficulty.component.html

@@ -37,12 +37,12 @@
         <div class="difficulty-stats">
           <div class="item">
             <div class="card-text">
-              ~<app-time [time]="epochData.timeAvg / 1000" [forceFloorOnTimeIntervals]="['minute']" [fractionDigits]="1"></app-time>
+              ~<app-time [time]="epochData.timeAvg / 1000" [fractionDigits]="1"></app-time>
             </div>
             <div class="symbol" i18n="difficulty-box.average-block-time">Average block time</div>
           </div>
           <div class="item">
-            <div *ngIf="epochData.remainingBlocks < 1870; else recentlyAdjusted" class="card-text" [ngStyle]="{'color': epochData.colorAdjustments}">
+            <div *ngIf="epochData.remainingBlocks < 1870; else recentlyAdjusted" class="card-text bigger" [ngStyle]="{'color': epochData.colorAdjustments}">
               <span *ngIf="epochData.change > 0; else arrowDownDifficulty" >
                 <fa-icon class="retarget-sign" [icon]="['fas', 'caret-up']" [fixedWidth]="true"></fa-icon>
               </span>
@@ -68,7 +68,7 @@
             </div>
           </div>
           <div class="item">
-            <div class="card-text"><app-time kind="until" [time]="epochData.estimatedRetargetDate" [fastRender]="true"></app-time></div>
+            <div class="card-text"><app-time kind="until" [time]="epochData.estimatedRetargetDate" [fastRender]="true" [precision]="1"></app-time></div>
             <div class="symbol">
               {{ epochData.retargetDateString }}
             </div>

frontend/src/app/components/difficulty/difficulty.component.scss

@@ -30,9 +30,14 @@
     }
   }
   .card-text {
-    font-size: 20px;
+    font-size: 18px;
     margin: auto;
     position: relative;
+    margin-bottom: 0.2rem;
+    &.bigger {
+      font-size: 20px;
+      margin-bottom: 0;
+    }
   }
 }
 
@@ -160,6 +165,7 @@
 
 .symbol {
   font-size: 13px;
+  white-space: nowrap;
 }
 
 .epoch-progress {

frontend/src/app/components/hashrate-chart/hashrate-chart.component.scss

@@ -4,6 +4,9 @@
   @media (min-width: 465px) {
     font-size: 20px;
   }
+  @media (min-width: 992px) {
+    height: 40px;
+  }
 }
 
 .main-title {
@@ -18,16 +21,19 @@
 }
 
 .full-container {
+  display: flex;
+  flex-direction: column;
   padding: 0px 15px;
   width: 100%;
-  min-height: 500px;
-  height: calc(100% - 150px);
-  @media (max-width: 992px) {
-    padding-bottom: 100px;
-  };
+  height: calc(100vh - 250px);
+  @media (min-width: 992px) {
+    height: calc(100vh - 150px);
+  }
 }
 
 .chart {
+  display: flex;
+  flex: 1;
   width: 100%;
   height: 100%;
   padding-bottom: 20px;

frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.scss

@@ -4,6 +4,9 @@
   @media (min-width: 465px) {
     font-size: 20px;
   }
+  @media (min-width: 992px) {
+    height: 40px;
+  }
 }
 
 .main-title {
@@ -18,18 +21,20 @@
 }
 
 .full-container {
+  display: flex;
+  flex-direction: column;
   padding: 0px 15px;
   width: 100%;
-  min-height: 500px;
-  height: calc(100% - 150px);
-  @media (max-width: 992px) {
-    padding-bottom: 100px;
-  };
+  height: calc(100vh - 250px);
+  @media (min-width: 992px) {
+    height: calc(100vh - 150px);
+  }
 }
 
 .chart {
+  display: flex;
+  flex: 1;
   width: 100%;
-  height: 100%;
   padding-bottom: 20px;
   padding-right: 10px;
   @media (max-width: 992px) {

frontend/src/app/components/liquid-master-page/liquid-master-page.component.html

@@ -90,9 +90,12 @@
 </nav>
 </header>
 
+<app-testnet-alert *ngIf="network.val === 'liquidtestnet'"></app-testnet-alert>
+
 <br />
 
 <router-outlet></router-outlet>
 
-<br>
+<app-global-footer></app-global-footer>
+
 </ng-container>

frontend/src/app/components/master-page/master-page.component.html

@@ -62,10 +62,11 @@
 </nav>
 </header>
 
-<br />
+<app-testnet-alert *ngIf="network.val === 'testnet' || network.val === 'signet'"></app-testnet-alert>
 
-<router-outlet></router-outlet>
-
-<br>
+<main style="margin-top: 24px;">
+  <router-outlet></router-outlet>
+</main>
 
+<app-global-footer></app-global-footer>
 </ng-container>

frontend/src/app/components/master-page/master-page.component.scss

@@ -192,4 +192,4 @@ nav {
     margin: 33px 0px 0px -19px;
     font-size: 7px;
   }
-}
+}

frontend/src/app/components/master-page/master-page.component.ts

@@ -1,4 +1,4 @@
-import { Component, Inject, OnInit } from '@angular/core';
+import { Component, OnInit } from '@angular/core';
 import { Env, StateService } from '../../services/state.service';
 import { Observable, merge, of } from 'rxjs';
 import { LanguageService } from '../../services/language.service';

frontend/src/app/components/mempool-block-overview/mempool-block-overview.component.ts

@@ -99,7 +99,7 @@ export class MempoolBlockOverviewComponent implements OnInit, OnDestroy, OnChang
       const direction = (this.blockIndex == null || this.index < this.blockIndex) ? this.poolDirection : this.chainDirection;
       this.blockGraph.replace(delta.added, direction);
     } else {
-      this.blockGraph.update(delta.added, delta.removed, blockMined ? this.chainDirection : this.poolDirection, blockMined);
+      this.blockGraph.update(delta.added, delta.removed, delta.changed || [], blockMined ? this.chainDirection : this.poolDirection, blockMined);
     }
 
     this.lastBlockHeight = this.stateService.latestBlockHeight;
@@ -107,8 +107,12 @@ export class MempoolBlockOverviewComponent implements OnInit, OnDestroy, OnChang
     this.isLoading$.next(false);
   }
 
-  onTxClick(event: TransactionStripped): void {
-    const url = new RelativeUrlPipe(this.stateService).transform(`/tx/${event.txid}`);
-    this.router.navigate([url]);
+  onTxClick(event: { tx: TransactionStripped, keyModifier: boolean }): void {
+    const url = new RelativeUrlPipe(this.stateService).transform(`/tx/${event.tx.txid}`);
+    if (!event.keyModifier) {
+      this.router.navigate([url]);
+    } else {
+      window.open(url, '_blank');
+    }
   }
 }

frontend/src/app/components/mempool-blocks/mempool-blocks.component.html

@@ -2,7 +2,7 @@
   <div class="mempool-blocks-container" [class.time-ltr]="timeLtr" *ngIf="(difficultyAdjustments$ | async) as da;">
     <div class="flashing">
       <ng-template ngFor let-projectedBlock [ngForOf]="mempoolBlocks$ | async" let-i="index" [ngForTrackBy]="trackByFn">
-        <div @blockEntryTrigger [@.disabled]="!animateEntry" [attr.data-cy]="'mempool-block-' + i" class="bitcoin-block text-center mempool-block" id="mempool-block-{{ i }}" [ngStyle]="mempoolBlockStyles[i]" [class.blink-bg]="projectedBlock.blink">
+        <div @blockEntryTrigger [@.disabled]="i > 0 || !animateEntry" [attr.data-cy]="'mempool-block-' + i" class="bitcoin-block text-center mempool-block" id="mempool-block-{{ i }}" [ngStyle]="mempoolBlockStyles[i]" [class.blink-bg]="projectedBlock.blink">
           <a draggable="false" [routerLink]="['/mempool-block/' | relativeUrl, i]"
             class="blockLink" [ngClass]="{'disabled': (this.stateService.blockScrolling$ | async)}">&nbsp;</a>
           <div class="block-body">
@@ -26,7 +26,7 @@
                 <app-time kind="until" [time]="(1 * i) + now + 61000" [fastRender]="false" [fixedRender]="true"></app-time>
               </ng-template>
               <ng-template #timeDiffMainnet>
-                <app-time kind="until" [time]="da.timeAvg * (i + 1) + now + da.timeOffset" [fastRender]="false" [fixedRender]="true" [forceFloorOnTimeIntervals]="['hour']"></app-time>
+                <app-time kind="until" [time]="da.timeAvg * (i + 1) + now + da.timeOffset" [fastRender]="false" [fixedRender]="true"></app-time>
               </ng-template>
             </div>
             <ng-template #mergedBlock>

frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts

@@ -1,4 +1,4 @@
-import { Component, OnInit, OnDestroy, ChangeDetectionStrategy, ChangeDetectorRef } from '@angular/core';
+import { Component, OnInit, OnDestroy, ChangeDetectionStrategy, ChangeDetectorRef, HostListener } from '@angular/core';
 import { Subscription, Observable, fromEvent, merge, of, combineLatest } from 'rxjs';
 import { MempoolBlock } from '../../interfaces/websocket.interface';
 import { StateService } from '../../services/state.service';
@@ -8,7 +8,7 @@ import { feeLevels, mempoolFeeColors } from '../../app.constants';
 import { specialBlocks } from '../../app.constants';
 import { RelativeUrlPipe } from '../../shared/pipes/relative-url/relative-url.pipe';
 import { Location } from '@angular/common';
-import { DifficultyAdjustment } from '../../interfaces/node-api.interface';
+import { DifficultyAdjustment, MempoolPosition } from '../../interfaces/node-api.interface';
 import { animate, style, transition, trigger } from '@angular/animations';
 
 @Component({
@@ -58,6 +58,7 @@ export class MempoolBlocksComponent implements OnInit, OnDestroy {
   transition = 'background 2s, right 2s, transform 1s';
 
   markIndex: number;
+  txPosition: MempoolPosition;
   txFeePerVSize: number;
 
   resetTransitionTimeout: number;
@@ -152,10 +153,14 @@ export class MempoolBlocksComponent implements OnInit, OnDestroy {
     this.markBlocksSubscription = this.stateService.markBlock$
       .subscribe((state) => {
         this.markIndex = undefined;
+        this.txPosition = undefined;
         this.txFeePerVSize = undefined;
         if (state.mempoolBlockIndex !== undefined) {
           this.markIndex = state.mempoolBlockIndex;
         }
+        if (state.mempoolPosition) {
+          this.txPosition = state.mempoolPosition;
+        }
         if (state.txFeePerVSize) {
           this.txFeePerVSize = state.txFeePerVSize;
         }
@@ -222,8 +227,13 @@ export class MempoolBlocksComponent implements OnInit, OnDestroy {
     clearTimeout(this.resetTransitionTimeout);
   }
 
+  @HostListener('window:resize', ['$event'])
+  onResize(): void {
+    this.animateEntry = false;
+  }
+
   trackByFn(index: number, block: MempoolBlock) {
-    return (block.isStack) ? 'stack' : block.index;
+    return (block.isStack) ? `stack-${block.index}` : block.index;
   }
 
   reduceMempoolBlocksToFitScreen(blocks: MempoolBlock[]): MempoolBlock[] {
@@ -297,7 +307,7 @@ export class MempoolBlocksComponent implements OnInit, OnDestroy {
   }
 
   calculateTransactionPosition() {
-    if ((!this.txFeePerVSize && (this.markIndex === undefined || this.markIndex === -1)) || !this.mempoolBlocks) {
+    if ((!this.txPosition && !this.txFeePerVSize && (this.markIndex === undefined || this.markIndex === -1)) || !this.mempoolBlocks) {
       this.arrowVisible = false;
       return;
     } else if (this.markIndex > -1) {
@@ -315,33 +325,43 @@ export class MempoolBlocksComponent implements OnInit, OnDestroy {
 
     this.arrowVisible = true;
 
-    let found = false;
-    for (let txInBlockIndex = 0; txInBlockIndex < this.mempoolBlocks.length && !found; txInBlockIndex++) {
-      const block = this.mempoolBlocks[txInBlockIndex];
-      for (let i = 0; i < block.feeRange.length - 1 && !found; i++) {
-        if (this.txFeePerVSize < block.feeRange[i + 1] && this.txFeePerVSize >= block.feeRange[i]) {
-          const feeRangeIndex = i;
-          const feeRangeChunkSize = 1 / (block.feeRange.length - 1);
+    if (this.txPosition) {
+      if (this.txPosition.block >= this.mempoolBlocks.length) {
+        this.rightPosition = ((this.mempoolBlocks.length - 1) * (this.blockWidth + this.blockPadding)) + this.blockWidth;
+      } else {
+        const positionInBlock = Math.min(1, this.txPosition.vsize / this.stateService.blockVSize) * this.blockWidth;
+        const positionOfBlock = this.txPosition.block * (this.blockWidth + this.blockPadding);
+        this.rightPosition = positionOfBlock + positionInBlock;
+      }
+    } else {
+      let found = false;
+      for (let txInBlockIndex = 0; txInBlockIndex < this.mempoolBlocks.length && !found; txInBlockIndex++) {
+        const block = this.mempoolBlocks[txInBlockIndex];
+        for (let i = 0; i < block.feeRange.length - 1 && !found; i++) {
+          if (this.txFeePerVSize < block.feeRange[i + 1] && this.txFeePerVSize >= block.feeRange[i]) {
+            const feeRangeIndex = i;
+            const feeRangeChunkSize = 1 / (block.feeRange.length - 1);
 
-          const txFee = this.txFeePerVSize - block.feeRange[i];
-          const max = block.feeRange[i + 1] - block.feeRange[i];
-          const blockLocation = txFee / max;
+            const txFee = this.txFeePerVSize - block.feeRange[i];
+            const max = block.feeRange[i + 1] - block.feeRange[i];
+            const blockLocation = txFee / max;
 
-          const chunkPositionOffset = blockLocation * feeRangeChunkSize;
-          const feePosition = feeRangeChunkSize * feeRangeIndex + chunkPositionOffset;
+            const chunkPositionOffset = blockLocation * feeRangeChunkSize;
+            const feePosition = feeRangeChunkSize * feeRangeIndex + chunkPositionOffset;
 
-          const blockedFilledPercentage = (block.blockVSize > this.stateService.blockVSize ? this.stateService.blockVSize : block.blockVSize) / this.stateService.blockVSize;
-          const arrowRightPosition = txInBlockIndex * (this.blockWidth + this.blockPadding)
-            + ((1 - feePosition) * blockedFilledPercentage * this.blockWidth);
+            const blockedFilledPercentage = (block.blockVSize > this.stateService.blockVSize ? this.stateService.blockVSize : block.blockVSize) / this.stateService.blockVSize;
+            const arrowRightPosition = txInBlockIndex * (this.blockWidth + this.blockPadding)
+              + ((1 - feePosition) * blockedFilledPercentage * this.blockWidth);
 
-          this.rightPosition = arrowRightPosition;
+            this.rightPosition = arrowRightPosition;
+            found = true;
+          }
+        }
+        if (this.txFeePerVSize >= block.feeRange[block.feeRange.length - 1]) {
+          this.rightPosition = txInBlockIndex * (this.blockWidth + this.blockPadding);
           found = true;
         }
       }
-      if (this.txFeePerVSize >= block.feeRange[block.feeRange.length - 1]) {
-        this.rightPosition = txInBlockIndex * (this.blockWidth + this.blockPadding);
-        found = true;
-      }
     }
   }
 

frontend/src/app/components/mining-dashboard/mining-dashboard.component.html

@@ -73,24 +73,4 @@
     </div>
 
   </div>
-
-  <div class="pref-selectors">
-    <div class="selector">
-      <app-language-selector></app-language-selector>
-    </div>
-    <div class="selector">
-      <app-fiat-selector></app-fiat-selector>
-    </div>
-  </div>
-
-  <div class="terms-of-service">
-    <a [routerLink]="['/terms-of-service']" i18n="shared.terms-of-service|Terms of Service">Terms of Service</a>
-    |
-    <a [routerLink]="['/privacy-policy']" i18n="shared.privacy-policy|Privacy Policy">Privacy Policy</a>
-    |
-    <a [routerLink]="['/tx/push' | relativeUrl]" i18n="shared.broadcast-transaction|Broadcast Transaction">Broadcast Transaction</a>
-  </div>
-
-  <br>
-
 </div>

frontend/src/app/components/mining-dashboard/mining-dashboard.component.scss

@@ -1,10 +1,6 @@
 .dashboard-container {
-  padding-bottom: 60px;
   text-align: center;
   margin-top: 0.5rem;
-  @media (min-width: 992px) {
-    padding-bottom: 0px;
-  }
   .col {
     margin-bottom: 1.5rem;
   }
@@ -104,22 +100,3 @@
   text-decoration: none;
   color: inherit;
 }
-
-.terms-of-service {
-  margin-top: 1rem;
-}
-
-.pref-selectors {
-  display: flex;
-  flex-direction: row;
-  flex-wrap: wrap;
-  justify-content: center;
-
-  .selector {
-    margin-left: .5em;
-    margin-bottom: .5em;
-    &:first {
-      margin-left: 0;
-    }
-  }
-}

frontend/src/app/components/pool-ranking/pool-ranking.component.html

@@ -32,7 +32,7 @@
   </div>
 
   <div class="card-header" *ngIf="!widget">
-    <div class="d-flex d-md-block align-items-baseline">
+    <div class="d-flex d-md-table-cell align-items-baseline">
       <span i18n="mining.pools">Pools Ranking</span>
       <button class="btn p-0 pl-2" style="margin: 0 0 4px 0px" (click)="onSaveChart()">
         <fa-icon [icon]="['fas', 'download']" [fixedWidth]="true"></fa-icon>
@@ -75,7 +75,7 @@
     </form>
   </div>
 
-  <div [class]="!widget ? 'bottom-padding' : 'pb-0'" class="container pb-lg-0">
+  <div [class]="!widget ? '' : 'pb-0'" class="container pb-lg-0">
     <div [class]="widget ? 'chart-widget' : 'chart'" echarts [initOpts]="chartInitOptions" [options]="chartOptions"
       (chartInit)="onChartInit($event)">
     </div>
@@ -87,19 +87,19 @@
     <table *ngIf="widget === false" class="table table-borderless text-center pools-table">
       <thead>
         <tr>
-          <th class="d-none d-md-block" i18n="mining.rank">Rank</th>
+          <th class="d-none d-md-table-cell" i18n="mining.rank">Rank</th>
           <th class=""></th>
           <th class="" i18n="mining.pool-name">Pool</th>
           <th class="" *ngIf="this.miningWindowPreference === '24h'" i18n="mining.hashrate">Hashrate</th>
           <th class="" i18n="master-page.blocks">Blocks</th>
           <th *ngIf="auditAvailable" class="health text-right widget" i18n="latest-blocks.avg_health"
             i18n-ngbTooltip="latest-blocks.avg_health" ngbTooltip="Avg Health" placement="bottom" #health [disableTooltip]="!isEllipsisActive(health)">Avg Health</th>
-          <th class="d-none d-md-block" i18n="mining.empty-blocks">Empty blocks</th>
+          <th class="d-none d-md-table-cell" i18n="mining.empty-blocks">Empty blocks</th>
         </tr>
       </thead>
       <tbody [attr.data-cy]="'pools-table'" *ngIf="(miningStatsObservable$ | async) as miningStats">
         <tr *ngFor="let pool of miningStats.pools">
-          <td class="d-none d-md-block">{{ pool.rank }}</td>
+          <td class="d-none d-md-table-cell">{{ pool.rank }}</td>
           <td class="text-right">
             <img width="25" height="25" src="{{ pool.logo }}" [alt]="pool.name + ' mining pool logo'" onError="this.src = '/resources/mining-pools/default.svg'">
           </td>
@@ -107,7 +107,7 @@
           <td class="" *ngIf="this.miningWindowPreference === '24h'">{{ pool.lastEstimatedHashrate }} {{
             miningStats.miningUnits.hashrateUnit }}</td>
           <td class="d-flex justify-content-center">
-            {{ pool.blockCount }}<span class="d-none d-md-block">&nbsp;({{ pool.share }}%)</span>
+            {{ pool.blockCount }}<span class="d-none d-md-table-cell">&nbsp;({{ pool.share }}%)</span>
           </td>
           <td *ngIf="auditAvailable" class="health text-right" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}">
             <a
@@ -121,22 +121,21 @@
               <span class="health-badge badge badge-secondary" i18n="unknown">Unknown</span>
             </ng-template>
           </td>
-          <td class="d-none d-md-block">{{ pool.emptyBlocks }} ({{ pool.emptyBlockRatio }}%)</td>
+          <td class="d-none d-md-table-cell">{{ pool.emptyBlocks }} ({{ pool.emptyBlockRatio }}%)</td>
         </tr>
         <tr style="border-top: 1px solid #555">
-          <td class="d-none d-md-block"></td>
+          <td class="d-none d-md-table-cell"></td>
           <td class="text-right"></td>
           <td class=""><b i18n="mining.all-miners">All miners</b></td>
           <td class="" *ngIf="this.miningWindowPreference === '24h'"><b>{{ miningStats.lastEstimatedHashrate}} {{
               miningStats.miningUnits.hashrateUnit }}</b></td>
           <td class=""><b>{{ miningStats.blockCount }}</b></td>
-          <td class="d-none d-md-block"><b>{{ miningStats.totalEmptyBlock }} ({{ miningStats.totalEmptyBlockRatio
+          <td class="d-none d-md-table-cell"><b>{{ miningStats.totalEmptyBlock }} ({{ miningStats.totalEmptyBlockRatio
               }}%)</b></td>
         </tr>
       </tbody>
     </table>
   </div>
-
 </div>
 
 

frontend/src/app/components/pool-ranking/pool-ranking.component.scss

@@ -10,6 +10,7 @@
   padding: 0px 15px;
   width: 100%;
   height: calc(100% - 140px);
+  padding-bottom: 20px;
   @media (max-width: 992px) {
     height: calc(100% - 190px);
   };
@@ -33,15 +34,6 @@
   }
 }
 
-.bottom-padding {
-  @media (max-width: 992px) {
-    padding-bottom: 65px
-  };
-  @media (max-width: 576px) {
-    padding-bottom: 65px
-  };
-}
-
 @media (max-width: 767.98px) {
   .pools-table th,
   .pools-table td {

frontend/src/app/components/pool/pool.component.html

@@ -94,13 +94,13 @@
                       <tr>
                         <th scope="col" class="block-count-title text-center" style="width: 33%" i18n="mining.reward">Reward</th>
                         <th scope="col" class="block-count-title text-center" style="width: 33%" i18n="mining.hashrate">Hashrate (24h)</th>
-                        <th scope="col" class="block-count-title text-center" style="width: 33%" i18n="latest-blocks.avg_health">Avg Health</th>
+                        <th scope="col" class="block-count-title text-center" style="width: 33%" i18n="latest-blocks.avg_health" *ngIf="auditAvailable">Avg Health</th>
                       </tr>
                     </thead>
                     <tbody>
                       <td class="text-center"><app-amount [satoshis]="poolStats.totalReward" digitsInfo="1.0-0" [noFiat]="true"></app-amount></td>
                       <td class="text-center">{{ poolStats.estimatedHashrate | amountShortener : 1 : 'H/s' }}</td>
-                      <td class="text-center"><span class="health-badge badge" [class.badge-success]="poolStats.avgBlockHealth >= 99"
+                      <td class="text-center" *ngIf="auditAvailable; else emptyTd"><span class="health-badge badge" [class.badge-success]="poolStats.avgBlockHealth >= 99"
                           [class.badge-warning]="poolStats.avgBlockHealth >= 75 && poolStats.avgBlockHealth < 99" [class.badge-danger]="poolStats.avgBlockHealth < 75"
                           *ngIf="poolStats.avgBlockHealth != null; else nullHealth">{{ poolStats.avgBlockHealth }}%</span>
                           <ng-template #nullHealth>
@@ -119,13 +119,13 @@
                       <tr>
                         <th scope="col" class="block-count-title text-center" style="width: 33%" i18n="mining.reward">Reward</th>
                         <th scope="col" class="block-count-title text-center" style="width: 33%" i18n="mining.hashrate">Hashrate (24h)</th>
-                        <th scope="col" class="block-count-title text-center" style="width: 33%" i18n="latest-blocks.avg_health">Avg Health</th>
+                        <th scope="col" class="block-count-title text-center" style="width: 33%" i18n="latest-blocks.avg_health" *ngIf="auditAvailable">Avg Health</th>
                       </tr>
                     </thead>
                     <tbody>
                       <td class="text-center"><app-amount [satoshis]="poolStats.totalReward" digitsInfo="1.0-0" [noFiat]="true"></app-amount></td>
                       <td class="text-center">{{ poolStats.estimatedHashrate | amountShortener : 1 : 'H/s' }}</td>
-                      <td class="text-center"><span class="health-badge badge" [class.badge-success]="poolStats.avgBlockHealth >= 99"
+                      <td *ngIf="auditAvailable; else emptyTd" class="text-center"><span class="health-badge badge" [class.badge-success]="poolStats.avgBlockHealth >= 99"
                           [class.badge-warning]="poolStats.avgBlockHealth >= 75 && poolStats.avgBlockHealth < 99" [class.badge-danger]="poolStats.avgBlockHealth < 75"
                           *ngIf="poolStats.avgBlockHealth != null; else nullHealth">{{ poolStats.avgBlockHealth }}%</span>
                           <ng-template #nullHealth>
@@ -384,7 +384,7 @@
                       <tr>
                         <th scope="col" class="block-count-title text-center" style="width: 33%" i18n="mining.total-reward">Reward</th>
                         <th scope="col" class="block-count-title text-center" style="width: 33%" i18n="mining.estimated">Hashrate (24h)</th>
-                        <th scope="col" class="block-count-title text-center" style="width: 33%" i18n="mining.luck">Avg Health</th>
+                        <th scope="col" class="block-count-title text-center" style="width: 33%" i18n="mining.luck" *ngIf="auditAvailable">Avg Health</th>
                       </tr>
                     </thead>
                     <tbody>
@@ -394,7 +394,7 @@
                       <td class="text-center">
                         <div class="skeleton-loader data"></div>
                       </td>
-                      <td class="text-center">
+                      <td class="text-center" *ngIf="auditAvailable">
                         <div class="skeleton-loader data"></div>
                       </td>
                     </tbody>
@@ -409,7 +409,7 @@
                       <tr>
                         <th scope="col" class="block-count-title text-center" style="width: 33%" i18n="mining.total-reward">Reward</th>
                         <th scope="col" class="block-count-title text-center" style="width: 33%" i18n="mining.estimated">Hashrate (24h)</th>
-                        <th scope="col" class="block-count-title text-center" style="width: 33%" i18n="mining.luck">Avg Health</th>
+                        <th scope="col" class="block-count-title text-center" style="width: 33%" i18n="mining.luck" *ngIf="auditAvailable">Avg Health</th>
                       </tr>
                     </thead>
                     <tbody>
@@ -419,7 +419,7 @@
                       <td class="text-center">
                         <div class="skeleton-loader data"></div>
                       </td>
-                      <td class="text-center">
+                      <td class="text-center" *ngIf="auditAvailable">
                         <div class="skeleton-loader data"></div>
                       </td>
                     </tbody>
@@ -485,4 +485,8 @@
       </div>
     </div>
   </div>
+</ng-template>
+
+<ng-template #emptyTd>
+  <td class="text-center"></td>
 </ng-template>

frontend/src/app/components/rbf-list/rbf-list.component.html

@@ -0,0 +1,46 @@
+<div class="container-xl" style="min-height: 335px">
+  <h1 class="float-left" i18n="page.rbf-replacements">RBF Replacements</h1>
+  <div *ngIf="isLoading" class="spinner-border ml-3" role="status"></div>
+
+  <div class="mode-toggle float-right" *ngIf="fullRbfEnabled">
+    <form class="formRadioGroup">
+      <div class="btn-group btn-group-toggle" name="radioBasic">
+        <label class="btn btn-primary btn-sm" [class.active]="!fullRbf">
+          <input type="radio" [value]="'All'" fragment="" [routerLink]="[]"> All
+        </label>
+        <label class="btn btn-primary btn-sm" [class.active]="fullRbf">
+          <input type="radio" [value]="'Full RBF'" fragment="fullrbf" [routerLink]="[]"> Full RBF
+        </label>
+      </div>
+    </form>
+  </div>
+
+  <div class="clearfix"></div>
+
+  <div class="rbf-trees" style="min-height: 295px">
+    <ng-container *ngIf="rbfTrees$ | async as trees">
+      <div *ngFor="let tree of trees" class="tree">
+        <p class="info">
+          <span class="type">
+            <span *ngIf="isMined(tree)" class="badge badge-success" i18n="transaction.rbf.mined">Mined</span>
+            <span *ngIf="isFullRbf(tree)" class="badge badge-info" i18n="transaction.full-rbf">Full RBF</span>
+          </span>
+          <app-time kind="since" [time]="tree.time"></app-time>
+        </p>
+        <div class="timeline-wrapper" [class.mined]="isMined(tree)">
+          <app-rbf-timeline [replacements]="tree"></app-rbf-timeline>
+        </div>
+      </div>
+
+      <div class="no-replacements" *ngIf="!trees?.length">
+        <p i18n="rbf.no-replacements-yet">there are no replacements in the mempool yet!</p>
+      </div>
+    </ng-container>
+    
+    <!-- <ngb-pagination class="pagination-container float-right mt-2" [class]="isLoading ? 'disabled' : ''"
+      [collectionSize]="blocksCount" [rotate]="true" [maxSize]="maxSize" [pageSize]="15" [(page)]="page"
+      (pageChange)="pageChange(page)" [boundaryLinks]="true" [ellipses]="false">
+    </ngb-pagination> -->
+  </div>
+  
+</div>

frontend/src/app/components/rbf-list/rbf-list.component.scss

@@ -0,0 +1,35 @@
+.spinner-border {
+  height: 25px;
+  width: 25px;
+  margin-top: 13px;
+}
+
+.rbf-trees {
+  .info {
+    display: flex;
+    flex-direction: row;
+    justify-content: space-between;
+    align-items: baseline;
+    margin: 0;
+    margin-bottom: 0.5em;
+
+    .type {
+      .badge {
+        margin-left: .5em;
+      }
+    }
+  }
+
+  .tree {
+    margin-bottom: 1em;
+  }
+
+  .timeline-wrapper.mined {
+    border: solid 4px #1a9436;
+  }
+
+  .no-replacements {
+    margin: 1em;
+    text-align: center;
+  }
+}

frontend/src/app/components/rbf-list/rbf-list.component.ts

@@ -0,0 +1,81 @@
+import { Component, OnInit, ChangeDetectionStrategy, OnDestroy } from '@angular/core';
+import { ActivatedRoute, Router } from '@angular/router';
+import { BehaviorSubject, EMPTY, merge, Observable, Subscription } from 'rxjs';
+import { catchError, switchMap, tap } from 'rxjs/operators';
+import { WebsocketService } from '../../services/websocket.service';
+import { RbfTree } from '../../interfaces/node-api.interface';
+import { ApiService } from '../../services/api.service';
+import { StateService } from '../../services/state.service';
+
+@Component({
+  selector: 'app-rbf-list',
+  templateUrl: './rbf-list.component.html',
+  styleUrls: ['./rbf-list.component.scss'],
+  changeDetection: ChangeDetectionStrategy.OnPush,
+})
+export class RbfList implements OnInit, OnDestroy {
+  rbfTrees$: Observable<RbfTree[]>;
+  nextRbfSubject = new BehaviorSubject(null);
+  urlFragmentSubscription: Subscription;
+  fullRbfEnabled: boolean;
+  fullRbf: boolean;
+  isLoading = true;
+
+  constructor(
+    private route: ActivatedRoute,
+    private router: Router,
+    private apiService: ApiService,
+    public stateService: StateService,
+    private websocketService: WebsocketService,
+  ) {
+    this.fullRbfEnabled = stateService.env.FULL_RBF_ENABLED;
+  }
+
+  ngOnInit(): void {
+    this.urlFragmentSubscription = this.route.fragment.subscribe((fragment) => {
+      this.fullRbf = (fragment === 'fullrbf');
+      this.websocketService.startTrackRbf(this.fullRbf ? 'fullRbf' : 'all');
+      this.nextRbfSubject.next(null);
+    });
+
+    this.rbfTrees$ = merge(
+      this.nextRbfSubject.pipe(
+        switchMap(() => {
+          return this.apiService.getRbfList$(this.fullRbf);
+        }),
+        catchError((e) => {
+          return EMPTY;
+        })
+      ),
+      this.stateService.rbfLatest$
+    )
+    .pipe(
+      tap(() => {
+        this.isLoading = false;
+      })
+    );
+  }
+
+  toggleFullRbf(event) {
+    this.router.navigate([], {
+      relativeTo: this.route,
+      fragment: this.fullRbf ? null : 'fullrbf'
+    });
+  }
+
+  isFullRbf(tree: RbfTree): boolean {
+    return tree.fullRbf;
+  }
+
+  isMined(tree: RbfTree): boolean {
+    return tree.mined;
+  }
+
+  // pageChange(page: number) {
+  //   this.fromTreeSubject.next(this.lastTreeId);
+  // }
+
+  ngOnDestroy(): void {
+    this.websocketService.stopTrackRbf();
+  }
+}

frontend/src/app/components/rbf-timeline/rbf-timeline-tooltip.component.html

@@ -0,0 +1,38 @@
+<div
+  #tooltip
+  *ngIf="rbfInfo && tooltipPosition !== null"
+  class="rbf-tooltip"
+  [style.left]="tooltipPosition.x + 'px'"
+  [style.top]="tooltipPosition.y + 'px'"
+>
+  <table>
+    <tbody>
+      <tr>
+        <td class="td-width" i18n="shared.transaction">Transaction</td>
+        <td>
+          <a [routerLink]="['/tx/' | relativeUrl, rbfInfo.tx.txid]">{{ rbfInfo.tx.txid | shortenString : 16}}</a>
+        </td>
+      </tr>
+      <tr>
+        <td class="td-width" i18n="transaction.first-seen|Transaction first seen">First seen</td>
+        <td><i><app-time kind="since" [time]="rbfInfo.time" [fastRender]="true"></app-time></i></td>
+      </tr>
+      <tr>
+        <td class="td-width" i18n="transaction.fee|Transaction fee">Fee</td>
+        <td>{{ rbfInfo.tx.fee | number }} <span class="symbol" i18n="shared.sat|sat">sat</span></td>
+      </tr>
+      <tr>
+        <td class="td-width" i18n="transaction.vsize|Transaction Virtual Size">Virtual size</td>
+        <td [innerHTML]="'&lrm;' + (rbfInfo.tx.vsize | vbytes: 2)"></td>
+      </tr>
+      <tr>
+        <td class="td-width" i18n="transaction.status|Transaction Status">Status</td>
+        <td>
+          <span *ngIf="rbfInfo.tx.rbf; else rbfDisabled" class="badge badge-success" i18n="rbfInfo-features.tag.rbf|RBF">RBF</span>
+          <ng-template #rbfDisabled><span class="badge badge-danger mr-1"><del i18n="rbfInfo-features.tag.rbf|RBF">RBF</del></span></ng-template>
+          <span *ngIf="rbfInfo.tx.mined" class="badge badge-success" i18n="transaction.rbf.mined">Mined</span>
+        </td>
+      </tr>
+    </tbody>
+  </table>
+</div>

frontend/src/app/components/rbf-timeline/rbf-timeline-tooltip.component.scss

@@ -0,0 +1,25 @@
+.rbf-tooltip {
+  position: fixed;
+  z-index: 3;
+  background: rgba(#11131f, 0.95);
+  border-radius: 4px;
+  box-shadow: 1px 1px 10px rgba(0,0,0,0.5);
+  color: #b1b1b1;
+  display: flex;
+  flex-direction: column;
+  justify-content: space-between;
+  padding: 10px 15px;
+  text-align: left;
+  pointer-events: none;
+
+  .badge {
+    margin-right: 1em;
+    &:last-child {
+      margin-right: 0;
+    }
+  }
+}
+
+.td-width {
+  padding-right: 10px;
+}

frontend/src/app/components/rbf-timeline/rbf-timeline-tooltip.component.ts

@@ -0,0 +1,35 @@
+import { Component, ElementRef, ViewChild, Input, OnChanges } from '@angular/core';
+import { RbfInfo } from '../../interfaces/node-api.interface';
+
+@Component({
+  selector: 'app-rbf-timeline-tooltip',
+  templateUrl: './rbf-timeline-tooltip.component.html',
+  styleUrls: ['./rbf-timeline-tooltip.component.scss'],
+})
+export class RbfTimelineTooltipComponent implements OnChanges {
+  @Input() rbfInfo: RbfInfo | void;
+  @Input() cursorPosition: { x: number, y: number };
+
+  tooltipPosition = null;
+
+  @ViewChild('tooltip') tooltipElement: ElementRef<HTMLCanvasElement>;
+
+  constructor() {}
+
+  ngOnChanges(changes): void {
+    if (changes.cursorPosition && changes.cursorPosition.currentValue) {
+      let x = Math.max(10, changes.cursorPosition.currentValue.x - 50);
+      let y = changes.cursorPosition.currentValue.y + 20;
+      if (this.tooltipElement) {
+        const elementBounds = this.tooltipElement.nativeElement.getBoundingClientRect();
+        if ((x + elementBounds.width) > (window.innerWidth - 10)) {
+          x = Math.max(0, window.innerWidth - elementBounds.width - 10);
+        }
+        if (y + elementBounds.height > (window.innerHeight - 20)) {
+          y = y - elementBounds.height - 20;
+        }
+      }
+      this.tooltipPosition = { x, y };
+    }
+  }
+}

frontend/src/app/components/rbf-timeline/rbf-timeline.component.html

@@ -0,0 +1,73 @@
+<div class="rbf-timeline box" [class.mined]="replacements.mined">
+  <div class="timeline-wrapper">
+    <div class="timeline" *ngFor="let timeline of rows">
+      <div class="intervals">
+        <ng-container *ngFor="let cell of timeline; let i = index;">
+          <div class="node-spacer"></div>
+          <ng-container *ngIf="i < timeline.length - 1">
+            <div class="interval" *ngIf="cell.replacement?.interval != null; else intervalSpacer">
+              <div class="interval-time">
+                <app-time [time]="cell.replacement.interval" [relative]="false"></app-time>
+              </div>
+            </div>
+          </ng-container>
+        </ng-container>
+      </div>
+      <div class="nodes">
+        <ng-container *ngFor="let cell of timeline; let i = index;">
+          <ng-container *ngIf="cell.replacement; else nonNode">
+            <div class="node"
+              [id]="'node-'+cell.replacement.tx.txid"
+              [class.selected]="txid === cell.replacement.tx.txid"
+              [class.mined]="cell.replacement.tx.mined"
+              [class.first-node]="cell.first"
+            >
+              <div class="track"></div>
+              <a class="shape-border"
+                [class.rbf]="cell.replacement.tx.rbf"
+                [routerLink]="['/tx/' | relativeUrl, cell.replacement.tx.txid]"
+                (pointerover)="onHover($event, cell.replacement);"
+                (pointerout)="onBlur($event);"
+              >
+                <div class="shape"></div>
+              </a>
+              <span class="fee-rate">{{ cell.replacement.tx.fee / (cell.replacement.tx.vsize) | feeRounding }} <span class="symbol" i18n="shared.sat-vbyte|sat/vB">sat/vB</span></span>
+            </div>
+          </ng-container>
+          <ng-template #nonNode>
+            <ng-container [ngSwitch]="cell.connector">
+              <div class="connector" *ngSwitchCase="'pipe'"><div class="pipe"></div></div>
+              <div class="connector" *ngSwitchCase="'corner'"><div class="corner"></div></div>
+              <div class="node-spacer" *ngSwitchDefault></div>
+            </ng-container>
+          </ng-template>
+          <ng-container *ngIf="i < timeline.length - 1">
+            <div class="interval-spacer" *ngIf="cell.replacement?.interval != null; else intervalSpacer">
+              <div class="track"></div>
+            </div>
+          </ng-container>
+        </ng-container>
+      </div>
+    </div>
+  </div>
+
+  <ng-template #nodeSpacer>
+    <div class="node-spacer"></div>
+  </ng-template>
+
+  <ng-template #intervalSpacer>
+    <div class="interval-spacer"></div>
+  </ng-template>
+
+  <app-rbf-timeline-tooltip
+    [rbfInfo]="hoverInfo"
+    [cursorPosition]="tooltipPosition"
+  ></app-rbf-timeline-tooltip>
+
+  <!-- <app-rbf-timeline-tooltip
+    *ngIf=[tooltip]
+    [line]="hoverLine"
+    [cursorPosition]="tooltipPosition"
+    [isConnector]="hoverConnector"
+  ></app-rbf-timeline-tooltip> -->
+</div>

frontend/src/app/components/rbf-timeline/rbf-timeline.component.scss

@@ -0,0 +1,193 @@
+.rbf-timeline {
+  position: relative;
+  width: 100%;
+  padding: 1em 0;
+
+  &::after, &::before {
+    content: '';
+    display: block;
+    position: absolute;
+    top: 0;
+    bottom: 0;
+    width: 2em;
+    z-index: 2;
+  }
+
+  &::before {
+    left: 0;
+    background: linear-gradient(to right, #24273e, #24273e, transparent);
+  }
+
+  &::after {
+    right: 0;
+    background: linear-gradient(to left, #24273e, #24273e, transparent);
+  }
+
+  .timeline-wrapper {
+    position: relative;
+    width: calc(100% - 2em);
+    margin: auto;
+    overflow-x: auto;
+    -ms-overflow-style: none;
+    scrollbar-width: none;
+
+    &::-webkit-scrollbar {
+      display: none;
+    }
+  }
+
+  .intervals, .nodes {
+    min-width: 100%;
+    display: flex;
+    flex-direction: row;
+    justify-content: space-between;
+    align-items: flex-start;
+    text-align: center;
+
+    .node, .node-spacer, .connector {
+      width: 6em;
+      min-width: 6em;
+      flex-grow: 1;
+    }
+
+    .interval, .interval-spacer {
+      width: 8em;
+      min-width: 5em;
+      max-width: 8em;
+      height: 32px;
+      display: flex;
+      flex-direction: row;
+      justify-content: center;
+      align-items: flex-end;
+    }
+
+    .interval {
+      overflow: visible;
+    }
+
+    .interval-time {
+      font-size: 12px;
+      line-height: 16px;
+      white-space: nowrap;
+    }
+  }
+
+  .node, .interval-spacer {
+    position: relative;
+    .track {
+      position: absolute;
+      height: 10px;
+      left: -5px;
+      right: -5px;
+      top: 0;
+      transform: translateY(-50%);
+      background: #105fb0;
+      border-radius: 5px;
+    }
+    &.first-node {
+      .track {
+        left: 50%;
+      }
+    }
+    &:last-child {
+      .track {
+        right: 50%;
+      }
+    }
+  }
+
+  .nodes {
+    position: relative;
+    margin-top: 1em;
+    .node {
+      .shape-border {
+        display: block;
+        margin: auto;
+        height: calc(1em + 8px);
+        width: calc(1em + 8px);
+        margin-bottom: -8px;
+        transform: translateY(-50%);
+        border-radius: 10%;
+        cursor: pointer;
+        padding: 4px;
+        background: transparent;
+        transition: background-color 300ms, padding 300ms;
+
+        .shape {
+          width: 100%;
+          height: 100%;
+          border-radius: 10%;
+          background: white;
+          transition: background-color 300ms, border 300ms;
+        }
+
+        &.rbf, &.rbf .shape {
+          border-radius: 50%;
+        }
+      }
+
+      .symbol::ng-deep {
+        display: block;
+        margin-top: -0.5em;
+      }
+
+      &.selected {
+        .shape-border {
+          background: #9339f4;
+        }
+      }
+
+      &.mined {
+        .shape-border {
+          background: #1a9436;
+        }
+      }
+
+      .shape-border:hover {
+        padding: 0px;
+        .shape {
+          background: #1bd8f4;
+        }
+      }
+
+      &.selected.mined {
+        .shape-border {
+          background: #1a9436;
+          height: calc(1em + 16px);
+          width: calc(1em + 16px);
+
+          .shape {
+            border: solid 4px #9339f4;
+          }
+
+          &:hover {
+            padding: 4px;
+            .shape {
+              border-width: 1px;
+              border-color: #1bd8f4
+            }
+          }
+        }
+      }
+    }
+
+    .connector {
+      position: relative;
+      height: 10px;
+
+      .corner, .pipe {
+        position: absolute;
+        left: -10px;
+        width: 20px;
+        height: 108px;
+        bottom: 50%;
+        border-right: solid 10px #105fb0;
+      }
+
+      .corner {
+        border-bottom: solid 10px #105fb0;
+        border-bottom-right-radius: 10px;
+      }
+    }
+  }
+}

frontend/src/app/components/rbf-timeline/rbf-timeline.component.ts

@@ -0,0 +1,191 @@
+import { Component, Input, OnInit, OnChanges, Inject, LOCALE_ID, HostListener } from '@angular/core';
+import { Router } from '@angular/router';
+import { RbfInfo, RbfTree } from '../../interfaces/node-api.interface';
+import { StateService } from '../../services/state.service';
+import { ApiService } from '../../services/api.service';
+
+type Connector = 'pipe' | 'corner';
+
+interface TimelineCell {
+  replacement?: RbfInfo,
+  connector?: Connector,
+  first?: boolean,
+}
+
+@Component({
+  selector: 'app-rbf-timeline',
+  templateUrl: './rbf-timeline.component.html',
+  styleUrls: ['./rbf-timeline.component.scss'],
+})
+export class RbfTimelineComponent implements OnInit, OnChanges {
+  @Input() replacements: RbfTree;
+  @Input() txid: string;
+  rows: TimelineCell[][] = [];
+
+  hoverInfo: RbfInfo | void = null;
+  tooltipPosition = null;
+
+  dir: 'rtl' | 'ltr' = 'ltr';
+
+  constructor(
+    private router: Router,
+    private stateService: StateService,
+    private apiService: ApiService,
+    @Inject(LOCALE_ID) private locale: string,
+  ) {
+    if (this.locale.startsWith('ar') || this.locale.startsWith('fa') || this.locale.startsWith('he')) {
+      this.dir = 'rtl';
+    }
+  }
+
+  ngOnInit(): void {
+    this.rows = this.buildTimelines(this.replacements);
+  }
+
+  ngOnChanges(changes): void {
+    this.rows = this.buildTimelines(this.replacements);
+    if (changes.txid) {
+      setTimeout(() => { this.scrollToSelected(); });
+    }
+  }
+
+  // converts a tree of RBF events into a format that can be more easily rendered in HTML
+  buildTimelines(tree: RbfTree): TimelineCell[][] {
+    if (!tree) return [];
+
+    const split = this.splitTimelines(tree);
+    const timelines = this.prepareTimelines(split);
+    return this.connectTimelines(timelines);
+  }
+
+  // splits a tree into N leaf-to-root paths
+  splitTimelines(tree: RbfTree, tail: RbfInfo[] = []): RbfInfo[][] {
+    const replacements = [...tail, tree];
+    if (tree.replaces.length) {
+      return [].concat(...tree.replaces.map(subtree => this.splitTimelines(subtree, replacements)));
+    } else {
+      return [[...replacements]];
+    }
+  }
+
+  // merges separate leaf-to-root paths into a coherent forking timeline
+  // represented as a 2D array of Rbf events
+  prepareTimelines(lines: RbfInfo[][]): RbfInfo[][] {
+    lines.sort((a, b) => b.length - a.length);
+
+    const rows = lines.map(() => []);
+    let lineGroups = [lines];
+    let done = false;
+    let column = 0; // sanity check for while loop stopping condition
+    while (!done && column < 100) {
+      // iterate over timelines element-by-element
+      // at each step, group lines which share a common transaction at their head
+      // (i.e. lines terminating in the same replacement event)
+      let index = 0;
+      let emptyCount = 0;
+      const nextGroups = [];
+      for (const group of lineGroups) {
+        const toMerge: { [txid: string]: RbfInfo[][] } = {};
+        let emptyInGroup = 0;
+        let first = true;
+        for (const line of group) {
+          const head = line.shift() || null;
+          if (first) {
+            // only insert the first instance of the replacement node
+            rows[index].unshift(head);
+            first = false;
+          } else {
+            // substitute duplicates with empty cells
+            // (we'll fill these in with connecting lines later)
+            rows[index].unshift(null);
+          }
+          // group the tails of the remaining lines for the next iteration
+          if (line.length) {
+            const nextId = line[0].tx.txid;
+            if (!toMerge[nextId]) {
+              toMerge[nextId] = [];
+            }
+            toMerge[nextId].push(line);
+          } else {
+            emptyInGroup++;
+          }
+          index++;
+        }
+        for (const merged of Object.values(toMerge).sort((a, b) => b.length - a.length)) {
+          nextGroups.push(merged);
+        }
+        for (let i = 0; i < emptyInGroup; i++) {
+          nextGroups.push([[]]);
+        }
+        emptyCount += emptyInGroup;
+        lineGroups = nextGroups;
+        done = (emptyCount >= rows.length);
+      }
+      column++;
+    }
+    return rows;
+  }
+
+  // annotates a 2D timeline array with info needed to draw connecting lines for multi-replacements
+  connectTimelines(timelines: RbfInfo[][]): TimelineCell[][] {
+    const rows: TimelineCell[][] = [];
+    timelines.forEach((lines, row) => {
+      rows.push([]);
+      let started = false;
+      let finished = false;
+      lines.forEach((replacement, column) => {
+        const cell: TimelineCell = {};
+        if (replacement) {
+          cell.replacement = replacement;
+        }
+        rows[row].push(cell);
+        if (replacement) {
+          if (!started) {
+            cell.first = true;
+            started = true;
+          }
+        } else if (started && !finished) {
+          if (column < timelines[row].length) {
+            let matched = false;
+            for (let i = row; i >= 0 && !matched; i--) {
+              const nextCell = rows[i][column];
+              if (nextCell.replacement) {
+                matched = true;
+              } else if (i === row) {
+                rows[i][column] = {
+                  connector: 'corner'
+                };
+              } else if (nextCell.connector !== 'corner') {
+                rows[i][column] = {
+                  connector: 'pipe'
+                };
+              }
+            }
+          }
+          finished = true;
+        }
+      });
+    });
+    return rows;
+  }
+
+  scrollToSelected() {
+    const node = document.getElementById('node-' + this.txid);
+    if (node) {
+      node.scrollIntoView({ block: 'nearest', inline: 'center' });
+    }
+  }
+
+  @HostListener('pointermove', ['$event'])
+  onPointerMove(event) {
+    this.tooltipPosition = { x: event.clientX, y: event.clientY };
+  }
+
+  onHover(event, replacement): void {
+    this.hoverInfo = replacement;
+  }
+
+  onBlur(event): void {
+    this.hoverInfo = null;
+  }
+}