Use minfee second node to supply alt rbf policy mempool txs

Support inner links in truncated string component

.github/dependabot.yml

@@ -1,20 +1,47 @@
 version: 2
 updates:
-- package-ecosystem: npm
+  - package-ecosystem: npm
-  directory: "/backend"
+    directory: "/backend"
-  schedule:
+    schedule:
-    interval: daily
+      interval: daily
-  open-pull-requests-limit: 10
+    open-pull-requests-limit: 10
-- package-ecosystem: npm
+    ignore:
-  directory: "/frontend"
+      - dependency-name: "*"
-  schedule:
+        update-types: ["version-update:semver-major"]
-    interval: daily
+    allow:
-  open-pull-requests-limit: 10
+      - dependency-type: "production"
-- package-ecosystem: docker
+
-  directory: "/docker/backend"
+  - package-ecosystem: npm
-  schedule:
+    directory: "/frontend"
-    interval: weekly
+    schedule:
-- package-ecosystem: "github-actions"
+      interval: daily
-  directory: "/"
+    open-pull-requests-limit: 10
-  schedule:
+    ignore:
-      interval: "weekly"
+      - dependency-name: "*"
+        update-types: ["version-update:semver-major"]
+    allow:
+      - dependency-type: "production"
+
+  - package-ecosystem: docker
+    directory: "/docker/backend"
+    schedule:
+      interval: weekly
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-major"]
+
+  - package-ecosystem: docker
+    directory: "/docker/frontend"
+    schedule:
+      interval: weekly
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-major"]
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: weekly
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-major"]

.github/workflows/cypress.yml

@@ -2,7 +2,7 @@ name: Cypress Tests
 
 on:
   pull_request:
-    types: [ opened, review_requested, synchronize ]
+    types: [opened, review_requested, synchronize]
 jobs:
   cypress:
     if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')"
@@ -24,36 +24,36 @@ jobs:
           - module: "bisq"
             spec: |
               cypress/e2e/bisq/bisq.spec.ts
-          
+
     name: E2E tests for ${{ matrix.module }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           path: ${{ matrix.module }}
-          
+
       - name: Setup node
-        uses: actions/setup-node@v2
+        uses: actions/setup-node@v3
         with:
           node-version: 16.15.0
-          cache: 'npm'
+          cache: "npm"
           cache-dependency-path: ${{ matrix.module }}/frontend/package-lock.json
 
       - name: Chrome browser tests (${{ matrix.module }})
-        uses: cypress-io/github-action@v4
+        uses: cypress-io/github-action@v5
         with:
           tag: ${{ github.event_name }}
           working-directory: ${{ matrix.module }}/frontend
           build: npm run config:defaults:${{ matrix.module }}
           start: npm run start:local-staging
-          wait-on: 'http://localhost:4200'
+          wait-on: "http://localhost:4200"
           wait-on-timeout: 120
           record: true
           parallel: true
           spec: ${{ matrix.spec }}
           group: Tests on Chrome (${{ matrix.module }})
           browser: "chrome"
-          ci-build-id: '${{ github.sha }}-${{ github.workflow }}-${{ github.event_name }}'
+          ci-build-id: "${{ github.sha }}-${{ github.workflow }}-${{ github.event_name }}"
         env:
           COMMIT_INFO_MESSAGE: ${{ github.event.pull_request.title }}
           CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}

.github/workflows/on-tag.yml

@@ -31,7 +31,7 @@ jobs:
         run: |
           sudo swapoff /mnt/swapfile
           sudo rm -v /mnt/swapfile
-          sudo fallocate -l 10G /mnt/swapfile
+          sudo fallocate -l 13G /mnt/swapfile
           sudo chmod 600 /mnt/swapfile
           sudo mkswap /mnt/swapfile
           sudo swapon /mnt/swapfile
@@ -68,24 +68,24 @@ jobs:
         run: echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin
 
       - name: Checkout project
-        uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
+        uses: actions/checkout@v3
 
       - name: Init repo for Dockerization
         run: docker/init.sh "$TAG"
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 # v1
+        uses: docker/setup-qemu-action@v2
         id: qemu
 
       - name: Setup Docker buildx action
-        uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 # v1
+        uses: docker/setup-buildx-action@v2
         id: buildx
 
       - name: Available platforms
         run: echo ${{ steps.buildx.outputs.platforms }}
 
       - name: Cache Docker layers
-        uses: actions/cache@661fd3eb7f2f20d8c7c84bc2b0509efd7a826628 # v2
+        uses: actions/cache@v3
         id: cache
         with:
           path: /tmp/.buildx-cache

.gitignore

@@ -3,3 +3,5 @@ data
 docker-compose.yml
 backend/mempool-config.json
 *.swp
+frontend/src/resources/config.template.js
+frontend/src/resources/config.js

backend/mempool-config.sample.json

@@ -2,6 +2,7 @@
   "MEMPOOL": {
     "NETWORK": "mainnet",
     "BACKEND": "electrum",
+    "ENABLED": true,
     "HTTP_PORT": 8999,
     "SPAWN_CLUSTER_PROCS": 0,
     "API_URL_PREFIX": "/api/v1/",
@@ -23,7 +24,11 @@
     "STDOUT_LOG_MIN_PRIORITY": "debug",
     "AUTOMATIC_BLOCK_REINDEXING": false,
     "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json",
-    "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master"
+    "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
+    "ADVANCED_GBT_AUDIT": false,
+    "ADVANCED_GBT_MEMPOOL": false,
+    "CPFP_INDEXING": false,
+    "RBF_DUAL_NODE": false
   },
   "CORE_RPC": {
     "HOST": "127.0.0.1",
@@ -80,7 +85,9 @@
     "BACKEND": "lnd",
     "STATS_REFRESH_INTERVAL": 600,
     "GRAPH_REFRESH_INTERVAL": 600,
-    "LOGGER_UPDATE_INTERVAL": 30
+    "LOGGER_UPDATE_INTERVAL": 30,
+    "FORENSICS_INTERVAL": 43200,
+    "FORENSICS_RATE_LIMIT": 20
   },
   "LND": {
     "TLS_CERT_PATH": "tls.cert",

backend/package.json

@@ -34,35 +34,35 @@
     "prettier": "./node_modules/.bin/prettier --write \"src/**/*.{js,ts}\""
   },
   "dependencies": {
-    "@babel/core": "^7.18.6",
+    "@babel/core": "^7.20.12",
     "@mempool/electrum-client": "^1.1.7",
-    "@types/node": "^16.11.41",
+    "@types/node": "^16.18.11",
     "axios": "~0.27.2",
-    "bitcoinjs-lib": "6.0.2",
+    "bitcoinjs-lib": "~6.1.0",
-    "crypto-js": "^4.0.0",
+    "crypto-js": "~4.1.1",
-    "express": "^4.18.0",
+    "express": "~4.18.2",
-    "maxmind": "^4.3.6",
+    "maxmind": "~4.3.8",
-    "mysql2": "2.3.3",
+    "mysql2": "~2.3.3",
-    "node-worker-threads-pool": "^1.5.1",
+    "node-worker-threads-pool": "~1.5.1",
     "socks-proxy-agent": "~7.0.0",
     "typescript": "~4.7.4",
-    "ws": "~8.8.0"
+    "ws": "~8.11.0"
   },
   "devDependencies": {
-    "@babel/core": "^7.18.6",
+    "@babel/core": "^7.20.7",
     "@babel/code-frame": "^7.18.6",
     "@types/compression": "^1.7.2",
     "@types/crypto-js": "^4.1.1",
-    "@types/express": "^4.17.13",
+    "@types/express": "^4.17.15",
-    "@types/jest": "^28.1.4",
+    "@types/jest": "^29.2.5",
-    "@types/ws": "~8.5.3",
+    "@types/ws": "~8.5.4",
-    "@typescript-eslint/eslint-plugin": "^5.30.5",
+    "@typescript-eslint/eslint-plugin": "^5.48.1",
-    "@typescript-eslint/parser": "^5.30.5",
+    "@typescript-eslint/parser": "^5.48.1",
-    "eslint": "^8.19.0",
+    "eslint": "^8.31.0",
     "eslint-config-prettier": "^8.5.0",
-    "jest": "^28.1.2",
+    "jest": "^29.3.1",
-    "prettier": "^2.7.1",
+    "prettier": "^2.8.2",
-    "ts-jest": "^28.0.5",
+    "ts-jest": "^29.0.3",
-    "ts-node": "^10.8.2"
+    "ts-node": "^10.9.1"
   }
 }

backend/src/__fixtures__/mempool-config.template.json

@@ -1,7 +1,9 @@
 {
   "MEMPOOL": {
+    "ENABLED": true,
     "NETWORK": "__MEMPOOL_NETWORK__",
     "BACKEND": "__MEMPOOL_BACKEND__",
+    "ENABLED": true,
     "BLOCKS_SUMMARIES_INDEXING": true,
     "HTTP_PORT": 1,
     "SPAWN_CLUSTER_PROCS": 2,
@@ -23,7 +25,11 @@
     "STDOUT_LOG_MIN_PRIORITY": "__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__",
     "INDEXING_BLOCKS_AMOUNT": 14,
     "POOLS_JSON_TREE_URL": "__POOLS_JSON_TREE_URL__",
-    "POOLS_JSON_URL": "__POOLS_JSON_URL__"
+    "POOLS_JSON_URL": "__POOLS_JSON_URL__",
+    "ADVANCED_GBT_AUDIT": "__ADVANCED_GBT_AUDIT__",
+    "ADVANCED_GBT_MEMPOOL": "__ADVANCED_GBT_MEMPOOL__",
+    "CPFP_INDEXING": "__CPFP_INDEXING__",
+    "RBF_DUAL_NODE": "__RBF_DUAL_NODE__"
   },
   "CORE_RPC": {
     "HOST": "__CORE_RPC_HOST__",
@@ -95,7 +101,9 @@
     "TOPOLOGY_FOLDER": "__LIGHTNING_TOPOLOGY_FOLDER__",
     "STATS_REFRESH_INTERVAL": 600,
     "GRAPH_REFRESH_INTERVAL": 600,
-    "LOGGER_UPDATE_INTERVAL": 30
+    "LOGGER_UPDATE_INTERVAL": 30,
+    "FORENSICS_INTERVAL": 43200,
+    "FORENSICS_RATE_LIMIT": "__FORENSICS_RATE_LIMIT__"
   },
   "LND": {
     "TLS_CERT_PATH": "",

backend/src/__tests__/config.test.ts

@@ -13,6 +13,7 @@ describe('Mempool Backend Config', () => {
       const config = jest.requireActual('../config').default;
 
       expect(config.MEMPOOL).toStrictEqual({
+        ENABLED: true,
         NETWORK: 'mainnet',
         BACKEND: 'none',
         BLOCKS_SUMMARIES_INDEXING: false,
@@ -36,7 +37,11 @@ describe('Mempool Backend Config', () => {
         USER_AGENT: 'mempool',
         STDOUT_LOG_MIN_PRIORITY: 'debug',
         POOLS_JSON_TREE_URL: 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
-        POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json'
+        POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json',
+        ADVANCED_GBT_AUDIT: false,
+        ADVANCED_GBT_MEMPOOL: false,
+        CPFP_INDEXING: false,
+        RBF_DUAL_NODE: false,
       });
 
       expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true });

backend/src/api/alt-mempool.ts

@@ -0,0 +1,175 @@
+import config from '../config';
+import { TransactionExtended } from '../mempool.interfaces';
+import logger from '../logger';
+import { Common } from './common';
+import { IBitcoinApi } from './bitcoin/bitcoin-api.interface';
+import BitcoinApi from './bitcoin/bitcoin-api';
+import bitcoinApi from './bitcoin/bitcoin-api-factory';
+import bitcoinSecondClient from './bitcoin/bitcoin-second-client';
+import { IEsploraApi } from './bitcoin/esplora-api.interface';
+import { Mempool } from './mempool';
+
+class AltMempool extends Mempool {
+  private bitcoinSecondApi: BitcoinApi;
+
+  constructor() {
+    super();
+    this.bitcoinSecondApi = new BitcoinApi(bitcoinSecondClient, this, bitcoinApi);
+  }
+
+  protected init(): void {
+    // override
+  }
+
+  public setOutOfSync(): void {
+    this.inSync = false;
+  }
+
+  public setMempool(mempoolData: { [txId: string]: TransactionExtended }): void {
+    this.mempoolCache = mempoolData;
+  }
+
+  public getFirstSeenForTransactions(txIds: string[]): number[] {
+    const txTimes: number[] = [];
+    txIds.forEach((txId: string) => {
+      const tx = this.mempoolCache[txId];
+      if (tx && tx.firstSeen) {
+        txTimes.push(tx.firstSeen);
+      } else {
+        txTimes.push(0);
+      }
+    });
+    return txTimes;
+  }
+
+  public async $updateMempool(): Promise<void> {
+    logger.debug(`Updating alternative mempool...`);
+    const start = new Date().getTime();
+    const currentMempoolSize = Object.keys(this.mempoolCache).length;
+    const transactions = await this.bitcoinSecondApi.$getRawMempool();
+    const diff = transactions.length - currentMempoolSize;
+
+    this.mempoolCacheDelta = Math.abs(diff);
+    const loadingMempool = this.mempoolCacheDelta > 100;
+
+    for (const txid of transactions) {
+      if (!this.mempoolCache[txid]) {
+        try {
+          const transaction = await this.$fetchTransaction(txid);
+          this.mempoolCache[txid] = transaction;
+          if (loadingMempool && Object.keys(this.mempoolCache).length % 50 === 0) {
+            logger.info(`loaded ${Object.keys(this.mempoolCache).length}/${transactions.length} alternative mempool transactions`);
+          }
+        } catch (e) {
+          logger.debug(`Error finding transaction '${txid}' in the alternative mempool: ` + (e instanceof Error ? e.message : e));
+        }
+      }
+
+      if ((new Date().getTime()) - start > Mempool.WEBSOCKET_REFRESH_RATE_MS) {
+        break;
+      }
+    }
+    if (loadingMempool) {
+      logger.info(`loaded ${Object.keys(this.mempoolCache).length}/${transactions.length} alternative mempool transactions`);
+    }
+
+    // Prevent mempool from clear on bitcoind restart by delaying the deletion
+    if (this.mempoolProtection === 0
+      && currentMempoolSize > 20000
+      && transactions.length / currentMempoolSize <= 0.80
+    ) {
+      this.mempoolProtection = 1;
+      setTimeout(() => {
+        this.mempoolProtection = 2;
+      }, 1000 * 60 * config.MEMPOOL.CLEAR_PROTECTION_MINUTES);
+    }
+
+    const deletedTransactions: string[] = [];
+
+    if (this.mempoolProtection !== 1) {
+      this.mempoolProtection = 0;
+      // Index object for faster search
+      const transactionsObject = {};
+      transactions.forEach((txId) => transactionsObject[txId] = true);
+
+      // Flag transactions for lazy deletion
+      for (const tx in this.mempoolCache) {
+        if (!transactionsObject[tx] && !this.mempoolCache[tx].deleteAfter) {
+          deletedTransactions.push(this.mempoolCache[tx].txid);
+        }
+      }
+      for (const txid of deletedTransactions) {
+        delete this.mempoolCache[txid];
+      }
+    }
+
+    this.mempoolCacheDelta = Math.abs(transactions.length - Object.keys(this.mempoolCache).length);
+
+    const end = new Date().getTime();
+    const time = end - start;
+    logger.debug(`Alt mempool updated in ${time / 1000} seconds. New size: ${Object.keys(this.mempoolCache).length} (${diff > 0 ? '+' + diff : diff})`);
+  }
+
+  public getTransaction(txid: string): TransactionExtended {
+    return this.mempoolCache[txid] || null;
+  }
+
+  protected async $fetchTransaction(txid: string): Promise<TransactionExtended> {
+    const rawTx = await this.bitcoinSecondApi.$getRawTransaction(txid, false, true, false);
+    return this.extendTransaction(rawTx);
+  }
+
+  protected extendTransaction(transaction: IEsploraApi.Transaction): TransactionExtended {
+    // @ts-ignore
+    if (transaction.vsize) {
+      // @ts-ignore
+      return transaction;
+    }
+    const feePerVbytes = Math.max(Common.isLiquid() ? 0.1 : 1,
+      (transaction.fee || 0) / (transaction.weight / 4));
+    const transactionExtended: TransactionExtended = Object.assign({
+      vsize: Math.round(transaction.weight / 4),
+      feePerVsize: feePerVbytes,
+      effectiveFeePerVsize: feePerVbytes,
+    }, transaction);
+    if (!transaction.status.confirmed) {
+      transactionExtended.firstSeen = Math.round((new Date().getTime() / 1000));
+    }
+    return transactionExtended;
+  }
+
+  public async $updateMemPoolInfo(): Promise<void> {
+    this.mempoolInfo = await this.$getMempoolInfo();
+  }
+
+  public getMempoolInfo(): IBitcoinApi.MempoolInfo {
+    return this.mempoolInfo;
+  }
+
+  public getTxPerSecond(): number {
+    return this.txPerSecond;
+  }
+
+  public getVBytesPerSecond(): number {
+    return this.vBytesPerSecond;
+  }
+
+  public handleRbfTransactions(rbfTransactions: { [txid: string]: TransactionExtended; }): void {
+    for (const rbfTransaction in rbfTransactions) {
+      if (this.mempoolCache[rbfTransaction]) {
+        // Erase the replaced transactions from the local mempool
+        delete this.mempoolCache[rbfTransaction];
+      }
+    }
+  }
+
+  protected updateTxPerSecond(): void {}
+
+  protected deleteExpiredTransactions(): void {}
+
+  protected $getMempoolInfo(): any {
+    return bitcoinSecondClient.getMempoolInfo();
+  }
+}
+
+export default new AltMempool();

backend/src/api/audit.ts

@@ -0,0 +1,133 @@
+import config from '../config';
+import { TransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces';
+
+const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners
+
+class Audit {
+  auditBlock(transactions: TransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: TransactionExtended })
+   : { censored: string[], added: string[], fresh: string[], score: number } {
+    if (!projectedBlocks?.[0]?.transactionIds || !mempool) {
+      return { censored: [], added: [], fresh: [], score: 0 };
+    }
+
+    const matches: string[] = []; // present in both mined block and template
+    const added: string[] = []; // present in mined block, not in template
+    const fresh: string[] = []; // missing, but firstSeen within PROPAGATION_MARGIN
+    const isCensored = {}; // missing, without excuse
+    const isDisplaced = {};
+    let displacedWeight = 0;
+
+    const inBlock = {};
+    const inTemplate = {};
+
+    const now = Math.round((Date.now() / 1000));
+    for (const tx of transactions) {
+      inBlock[tx.txid] = tx;
+    }
+    // coinbase is always expected
+    if (transactions[0]) {
+      inTemplate[transactions[0].txid] = true;
+    }
+    // look for transactions that were expected in the template, but missing from the mined block
+    for (const txid of projectedBlocks[0].transactionIds) {
+      if (!inBlock[txid]) {
+        // tx is recent, may have reached the miner too late for inclusion
+        if (mempool[txid]?.firstSeen != null && (now - (mempool[txid]?.firstSeen || 0)) <= PROPAGATION_MARGIN) {
+          fresh.push(txid);
+        } else {
+          isCensored[txid] = true;
+        }
+        displacedWeight += mempool[txid].weight;
+      }
+      inTemplate[txid] = true;
+    }
+
+    displacedWeight += (4000 - transactions[0].weight);
+
+    // we can expect an honest miner to include 'displaced' transactions in place of recent arrivals and censored txs
+    // these displaced transactions should occupy the first N weight units of the next projected block
+    let displacedWeightRemaining = displacedWeight;
+    let index = 0;
+    let lastFeeRate = Infinity;
+    let failures = 0;
+    while (projectedBlocks[1] && index < projectedBlocks[1].transactionIds.length && failures < 500) {
+      const txid = projectedBlocks[1].transactionIds[index];
+      const fits = (mempool[txid].weight - displacedWeightRemaining) < 4000;
+      const feeMatches = mempool[txid].effectiveFeePerVsize >= lastFeeRate;
+      if (fits || feeMatches) {
+        isDisplaced[txid] = true;
+        if (fits) {
+          lastFeeRate = Math.min(lastFeeRate, mempool[txid].effectiveFeePerVsize);
+        }
+        if (mempool[txid].firstSeen == null || (now - (mempool[txid]?.firstSeen || 0)) > PROPAGATION_MARGIN) {
+          displacedWeightRemaining -= mempool[txid].weight;
+        }
+        failures = 0;
+      } else {
+        failures++;
+      }
+      index++;
+    }
+
+    // mark unexpected transactions in the mined block as 'added'
+    let overflowWeight = 0;
+    let totalWeight = 0;
+    for (const tx of transactions) {
+      if (inTemplate[tx.txid]) {
+        matches.push(tx.txid);
+      } else {
+        if (!isDisplaced[tx.txid]) {
+          added.push(tx.txid);
+        } else {
+        }
+        let blockIndex = -1;
+        let index = -1;
+        projectedBlocks.forEach((block, bi) => {
+          const i = block.transactionIds.indexOf(tx.txid);
+          if (i >= 0) {
+            blockIndex = bi;
+            index = i;
+          }
+        });
+        overflowWeight += tx.weight;
+      }
+      totalWeight += tx.weight;
+    }
+
+    // transactions missing from near the end of our template are probably not being censored
+    let overflowWeightRemaining = overflowWeight - (config.MEMPOOL.BLOCK_WEIGHT_UNITS - totalWeight);
+    let maxOverflowRate = 0;
+    let rateThreshold = 0;
+    index = projectedBlocks[0].transactionIds.length - 1;
+    while (index >= 0) {
+      const txid = projectedBlocks[0].transactionIds[index];
+      if (overflowWeightRemaining > 0) {
+        if (isCensored[txid]) {
+          delete isCensored[txid];
+        }
+        if (mempool[txid].effectiveFeePerVsize > maxOverflowRate) {
+          maxOverflowRate = mempool[txid].effectiveFeePerVsize;
+          rateThreshold = (Math.ceil(maxOverflowRate * 100) / 100) + 0.005;
+        }
+      } else if (mempool[txid].effectiveFeePerVsize <= rateThreshold) { // tolerance of 0.01 sat/vb + rounding
+        if (isCensored[txid]) {
+          delete isCensored[txid];
+        }
+      }
+      overflowWeightRemaining -= (mempool[txid]?.weight || 0);
+      index--;
+    }
+
+    const numCensored = Object.keys(isCensored).length;
+    const score = matches.length > 0 ? (matches.length / (matches.length + numCensored)) : 0;
+
+    return {
+      censored: Object.keys(isCensored),
+      added,
+      fresh,
+      score
+    };
+  }
+}
+
+export default new Audit();

backend/src/api/backend-info.ts

@@ -2,6 +2,7 @@ import fs from 'fs';
 import path from 'path';
 import os from 'os';
 import { IBackendInfo } from '../mempool.interfaces';
+import config from '../config';
 
 class BackendInfo {
   private backendInfo: IBackendInfo;
@@ -22,7 +23,8 @@ class BackendInfo {
     this.backendInfo = {
       hostname: os.hostname(),
       version: versionInfo.version,
-      gitCommit: versionInfo.gitCommit
+      gitCommit: versionInfo.gitCommit,
+      lightning: config.LIGHTNING.ENABLED
     };
   }
 

backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts

@@ -3,13 +3,14 @@ import { IEsploraApi } from './esplora-api.interface';
 export interface AbstractBitcoinApi {
   $getRawMempool(): Promise<IEsploraApi.Transaction['txid'][]>;
   $getRawTransaction(txId: string, skipConversion?: boolean, addPrevout?: boolean, lazyPrevouts?: boolean): Promise<IEsploraApi.Transaction>;
+  $getTransactionHex(txId: string): Promise<string>;
   $getBlockHeightTip(): Promise<number>;
   $getBlockHashTip(): Promise<string>;
   $getTxIdsForBlock(hash: string): Promise<string[]>;
   $getBlockHash(height: number): Promise<string>;
   $getBlockHeader(hash: string): Promise<string>;
   $getBlock(hash: string): Promise<IEsploraApi.Block>;
-  $getRawBlock(hash: string): Promise<string>;
+  $getRawBlock(hash: string): Promise<Buffer>;
   $getAddress(address: string): Promise<IEsploraApi.Address>;
   $getAddressTransactions(address: string, lastSeenTxId: string): Promise<IEsploraApi.Transaction[]>;
   $getAddressPrefix(prefix: string): string[];

backend/src/api/bitcoin/bitcoin-api-factory.ts

@@ -4,17 +4,20 @@ import EsploraApi from './esplora-api';
 import BitcoinApi from './bitcoin-api';
 import ElectrumApi from './electrum-api';
 import bitcoinClient from './bitcoin-client';
+import mempool from '../mempool';
 
 function bitcoinApiFactory(): AbstractBitcoinApi {
   switch (config.MEMPOOL.BACKEND) {
     case 'esplora':
       return new EsploraApi();
     case 'electrum':
-      return new ElectrumApi(bitcoinClient);
+      return new ElectrumApi(bitcoinClient, mempool);
     case 'none':
     default:
-      return new BitcoinApi(bitcoinClient);
+      return new BitcoinApi(bitcoinClient, mempool);
   }
 }
 
+export const bitcoinCoreApi = new BitcoinApi(bitcoinClient, mempool);
+
 export default bitcoinApiFactory();

backend/src/api/bitcoin/bitcoin-api.ts

@@ -3,15 +3,18 @@ import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
 import { IBitcoinApi } from './bitcoin-api.interface';
 import { IEsploraApi } from './esplora-api.interface';
 import blocks from '../blocks';
-import mempool from '../mempool';
+import { MempoolBlock, TransactionExtended } from '../../mempool.interfaces';
-import { TransactionExtended } from '../../mempool.interfaces';
 
 class BitcoinApi implements AbstractBitcoinApi {
+  private mempool: any = null;
   private rawMempoolCache: IBitcoinApi.RawMempool | null = null;
   protected bitcoindClient: any;
+  protected backupBitcoinApi: BitcoinApi;
 
-  constructor(bitcoinClient: any) {
+  constructor(bitcoinClient: any, mempool: any, backupBitcoinApi: any = null) {
     this.bitcoindClient = bitcoinClient;
+    this.mempool = mempool;
+    this.backupBitcoinApi = backupBitcoinApi;
   }
 
   static convertBlock(block: IBitcoinApi.Block): IEsploraApi.Block {
@@ -34,7 +37,7 @@ class BitcoinApi implements AbstractBitcoinApi {
 
   $getRawTransaction(txId: string, skipConversion = false, addPrevout = false, lazyPrevouts = false): Promise<IEsploraApi.Transaction> {
     // If the transaction is in the mempool we already converted and fetched the fee. Only prevouts are missing
-    const txInMempool = mempool.getMempool()[txId];
+    const txInMempool = this.mempool.getMempool()[txId];
     if (txInMempool && addPrevout) {
       return this.$addPrevouts(txInMempool);
     }
@@ -57,6 +60,11 @@ class BitcoinApi implements AbstractBitcoinApi {
       });
   }
 
+  $getTransactionHex(txId: string): Promise<string> {
+    return this.$getRawTransaction(txId, true)
+      .then((tx) => tx.hex || '');
+  }
+
   $getBlockHeightTip(): Promise<number> {
     return this.bitcoindClient.getChainTips()
       .then((result: IBitcoinApi.ChainTips[]) => {
@@ -76,7 +84,7 @@ class BitcoinApi implements AbstractBitcoinApi {
       .then((rpcBlock: IBitcoinApi.Block) => rpcBlock.tx);
   }
 
-  $getRawBlock(hash: string): Promise<string> {
+  $getRawBlock(hash: string): Promise<Buffer> {
     return this.bitcoindClient.getBlock(hash, 0)
       .then((raw: string) => Buffer.from(raw, "hex"));
   }
@@ -113,7 +121,7 @@ class BitcoinApi implements AbstractBitcoinApi {
 
   $getAddressPrefix(prefix: string): string[] {
     const found: { [address: string]: string } = {};
-    const mp = mempool.getMempool();
+    const mp = this.mempool.getMempool();
     for (const tx in mp) {
       for (const vout of mp[tx].vout) {
         if (vout.scriptpubkey_address.indexOf(prefix) === 0) {
@@ -255,7 +263,7 @@ class BitcoinApi implements AbstractBitcoinApi {
       return transaction;
     }
     let mempoolEntry: IBitcoinApi.MempoolEntry;
-    if (!mempool.isInSync() && !this.rawMempoolCache) {
+    if (!this.mempool.isInSync() && !this.rawMempoolCache) {
       this.rawMempoolCache = await this.$getRawMempoolVerbose();
     }
     if (this.rawMempoolCache && this.rawMempoolCache[transaction.txid]) {
@@ -311,10 +319,23 @@ class BitcoinApi implements AbstractBitcoinApi {
         transaction.vin[i].lazy = true;
         continue;
       }
-      const innerTx = await this.$getRawTransaction(transaction.vin[i].txid, false, false);
+      let innerTx;
-      transaction.vin[i].prevout = innerTx.vout[transaction.vin[i].vout];
+      try {
-      this.addInnerScriptsToVin(transaction.vin[i]);
+        innerTx = await this.$getRawTransaction(transaction.vin[i].txid, false, false);
-      totalIn += innerTx.vout[transaction.vin[i].vout].value;
+      } catch (e) {
+        if (this.backupBitcoinApi) {
+          // input tx is confirmed, but the preferred client has txindex=0, so fetch from the backup client instead.
+          const backupTx = await this.backupBitcoinApi.$getRawTransaction(transaction.vin[i].txid);
+          innerTx = JSON.parse(JSON.stringify(backupTx));
+        } else {
+          throw e;
+        }
+      }
+      if (innerTx) {
+        transaction.vin[i].prevout = innerTx.vout[transaction.vin[i].vout];
+        this.addInnerScriptsToVin(transaction.vin[i]);
+        totalIn += innerTx.vout[transaction.vin[i].vout].value;
+      }
     }
     if (lazyPrevouts && transaction.vin.length > 12) {
       transaction.fee = -1;

backend/src/api/bitcoin/bitcoin.routes.ts

@@ -1,5 +1,6 @@
 import { Application, Request, Response } from 'express';
 import axios from 'axios';
+import * as bitcoinjs from 'bitcoinjs-lib';
 import config from '../../config';
 import websocketHandler from '../websocket-handler';
 import mempool from '../mempool';
@@ -16,19 +17,25 @@ import logger from '../../logger';
 import blocks from '../blocks';
 import bitcoinClient from './bitcoin-client';
 import difficultyAdjustment from '../difficulty-adjustment';
+import transactionRepository from '../../repositories/TransactionRepository';
+import rbfCache from '../rbf-cache';
+import altMempool from '../alt-mempool';
 
 class BitcoinRoutes {
   public initRoutes(app: Application) {
     app
       .get(config.MEMPOOL.API_URL_PREFIX + 'transaction-times', this.getTransactionTimes)
       .get(config.MEMPOOL.API_URL_PREFIX + 'outspends', this.$getBatchedOutspends)
-      .get(config.MEMPOOL.API_URL_PREFIX + 'cpfp/:txId', this.getCpfpInfo)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'cpfp/:txId', this.$getCpfpInfo)
       .get(config.MEMPOOL.API_URL_PREFIX + 'difficulty-adjustment', this.getDifficultyChange)
       .get(config.MEMPOOL.API_URL_PREFIX + 'fees/recommended', this.getRecommendedFees)
       .get(config.MEMPOOL.API_URL_PREFIX + 'fees/mempool-blocks', this.getMempoolBlocks)
       .get(config.MEMPOOL.API_URL_PREFIX + 'backend-info', this.getBackendInfo)
       .get(config.MEMPOOL.API_URL_PREFIX + 'init-data', this.getInitData)
       .get(config.MEMPOOL.API_URL_PREFIX + 'validate-address/:address', this.validateAddress)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'tx/alt/:txId', this.getAltTransaction)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/replaces', this.getRbfHistory)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/cached', this.getCachedTx)
       .post(config.MEMPOOL.API_URL_PREFIX + 'tx/push', this.$postTransactionForm)
       .get(config.MEMPOOL.API_URL_PREFIX + 'donations', async (req, res) => {
         try {
@@ -87,7 +94,9 @@ class BitcoinRoutes {
       .get(config.MEMPOOL.API_URL_PREFIX + 'blocks', this.getBlocks.bind(this))
       .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/:height', this.getBlocks.bind(this))
       .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash', this.getBlock)
-      .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/summary', this.getStrippedBlockTransactions);
+      .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/summary', this.getStrippedBlockTransactions)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/audit-summary', this.getBlockAuditSummary)
+      .post(config.MEMPOOL.API_URL_PREFIX + 'psbt/addparents', this.postPsbtCompletion)
       ;
 
       if (config.MEMPOOL.BACKEND !== 'esplora') {
@@ -185,29 +194,36 @@ class BitcoinRoutes {
     }
   }
 
-  private getCpfpInfo(req: Request, res: Response) {
+  private async $getCpfpInfo(req: Request, res: Response) {
     if (!/^[a-fA-F0-9]{64}$/.test(req.params.txId)) {
       res.status(501).send(`Invalid transaction ID.`);
       return;
     }
 
     const tx = mempool.getMempool()[req.params.txId];
-    if (!tx) {
+    if (tx) {
-      res.status(404).send(`Transaction doesn't exist in the mempool.`);
+      if (tx?.cpfpChecked) {
+        res.json({
+          ancestors: tx.ancestors,
+          bestDescendant: tx.bestDescendant || null,
+          descendants: tx.descendants || null,
+          effectiveFeePerVsize: tx.effectiveFeePerVsize || null,
+        });
+        return;
+      }
+
+      const cpfpInfo = Common.setRelativesAndGetCpfpInfo(tx, mempool.getMempool());
+
+      res.json(cpfpInfo);
       return;
+    } else {
+      const cpfpInfo = await transactionRepository.$getCpfpInfo(req.params.txId);
+      if (cpfpInfo) {
+        res.json(cpfpInfo);
+        return;
+      }
     }
-
+    res.status(404).send(`Transaction has no CPFP info available.`);
-    if (tx.cpfpChecked) {
-      res.json({
-        ancestors: tx.ancestors,
-        bestDescendant: tx.bestDescendant || null,
-      });
-      return;
-    }
-
-    const cpfpInfo = Common.setRelativesAndGetCpfpInfo(tx, mempool.getMempool());
-
-    res.json(cpfpInfo);
   }
 
   private getBackendInfo(req: Request, res: Response) {
@@ -227,6 +243,23 @@ class BitcoinRoutes {
     }
   }
 
+  private async getAltTransaction(req: Request, res: Response) {
+    try {
+      const transaction = altMempool.getTransaction(req.params.txId);
+      if (transaction) {
+        res.json(transaction);
+      } else {
+        res.status(404).send('No such transaction in the alternate mempool');
+      }
+    } catch (e) {
+      let statusCode = 500;
+      if (e instanceof Error && e instanceof Error && e.message && e.message.indexOf('No such mempool or blockchain transaction') > -1) {
+        statusCode = 404;
+      }
+      res.status(statusCode).send(e instanceof Error ? e.message : e);
+    }
+  }
+
   private async getRawTransaction(req: Request, res: Response) {
     try {
       const transaction: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(req.params.txId, true);
@@ -241,6 +274,74 @@ class BitcoinRoutes {
     }
   }
 
+  /**
+   * Takes the PSBT as text/plain body, parses it, and adds the full
+   * parent transaction to each input that doesn't already have it.
+   * This is used for BTCPayServer / Trezor users which need access to
+   * the full parent transaction even with segwit inputs.
+   * It will respond with a text/plain PSBT in the same format (hex|base64).
+   */
+  private async postPsbtCompletion(req: Request, res: Response): Promise<void> {
+    res.setHeader('content-type', 'text/plain');
+    const notFoundError = `Couldn't get transaction hex for parent of input`;
+    try {
+      let psbt: bitcoinjs.Psbt;
+      let format: 'hex' | 'base64';
+      let isModified = false;
+      try {
+        psbt = bitcoinjs.Psbt.fromBase64(req.body);
+        format = 'base64';
+      } catch (e1) {
+        try {
+          psbt = bitcoinjs.Psbt.fromHex(req.body);
+          format = 'hex';
+        } catch (e2) {
+          throw new Error(`Unable to parse PSBT`);
+        }
+      }
+      for (const [index, input] of psbt.data.inputs.entries()) {
+        if (!input.nonWitnessUtxo) {
+          // Buffer.from ensures it won't be modified in place by reverse()
+          const txid = Buffer.from(psbt.txInputs[index].hash)
+            .reverse()
+            .toString('hex');
+
+          let transactionHex: string;
+          // If missing transaction, return 404 status error
+          try {
+            transactionHex = await bitcoinApi.$getTransactionHex(txid);
+            if (!transactionHex) {
+              throw new Error('');
+            }
+          } catch (err) {
+            throw new Error(`${notFoundError} #${index} @ ${txid}`);
+          }
+
+          psbt.updateInput(index, {
+            nonWitnessUtxo: Buffer.from(transactionHex, 'hex'),
+          });
+          if (!isModified) {
+            isModified = true;
+          }
+        }
+      }
+      if (isModified) {
+        res.send(format === 'hex' ? psbt.toHex() : psbt.toBase64());
+      } else {
+        // Not modified
+        // 422 Unprocessable Entity
+        // https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/422
+        res.status(422).send(`Psbt had no missing nonWitnessUtxos.`);
+      }
+    } catch (e: any) {
+      if (e instanceof Error && new RegExp(notFoundError).test(e.message)) {
+        res.status(404).send(e.message);
+      } else {
+        res.status(500).send(e instanceof Error ? e.message : e);
+      }
+    }
+  }
+
   private async getTransactionStatus(req: Request, res: Response) {
     try {
       const transaction = await transactionUtils.$getTransactionExtended(req.params.txId, true);
@@ -254,6 +355,16 @@ class BitcoinRoutes {
     }
   }
 
+  private async getStrippedBlockTransactions(req: Request, res: Response) {
+    try {
+      const transactions = await blocks.$getStrippedBlockTransactions(req.params.hash);
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24 * 30).toUTCString());
+      res.json(transactions);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
   private async getBlock(req: Request, res: Response) {
     try {
       const block = await blocks.$getBlock(req.params.hash);
@@ -286,9 +397,9 @@ class BitcoinRoutes {
     }
   }
 
-  private async getStrippedBlockTransactions(req: Request, res: Response) {
+  private async getBlockAuditSummary(req: Request, res: Response) {
     try {
-      const transactions = await blocks.$getStrippedBlockTransactions(req.params.hash);
+      const transactions = await blocks.$getBlockAuditSummary(req.params.hash);
       res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24 * 30).toUTCString());
       res.json(transactions);
     } catch (e) {
@@ -313,7 +424,8 @@ class BitcoinRoutes {
   private async getLegacyBlocks(req: Request, res: Response) {
     try {
       const returnBlocks: IEsploraApi.Block[] = [];
-      const fromHeight = parseInt(req.params.height, 10) || blocks.getCurrentBlockHeight();
+      const tip = blocks.getCurrentBlockHeight();
+      const fromHeight = Math.min(parseInt(req.params.height, 10) || tip, tip);
 
       // Check if block height exist in local cache to skip the hash lookup
       const blockByHeight = blocks.getBlocks().find((b) => b.height === fromHeight);
@@ -499,6 +611,28 @@ class BitcoinRoutes {
     }
   }
 
+  private async getRbfHistory(req: Request, res: Response) {
+    try {
+      const result = rbfCache.getReplaces(req.params.txId);
+      res.json(result || []);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getCachedTx(req: Request, res: Response) {
+    try {
+      const result = rbfCache.getTx(req.params.txId);
+      if (result) {
+        res.json(result);
+      } else {
+        res.status(404).send('not found');
+      }
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
   private async getTransactionOutspends(req: Request, res: Response) {
     try {
       const result = await bitcoinApi.$getOutspends(req.params.txId);

backend/src/api/bitcoin/electrum-api.ts

@@ -12,8 +12,8 @@ import memoryCache from '../memory-cache';
 class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
   private electrumClient: any;
 
-  constructor(bitcoinClient: any) {
+  constructor(bitcoinClient: any, mempool: any) {
-    super(bitcoinClient);
+    super(bitcoinClient, mempool);
 
     const electrumConfig = { client: 'mempool-v2', version: '1.4' };
     const electrumPersistencePolicy = { retryPeriod: 10000, maxRetry: 1000, callback: null };

backend/src/api/bitcoin/esplora-api.ts

@@ -20,6 +20,11 @@ class ElectrsApi implements AbstractBitcoinApi {
       .then((response) => response.data);
   }
 
+  $getTransactionHex(txId: string): Promise<string> {
+    return axios.get<string>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/hex', this.axiosConfig)
+      .then((response) => response.data);
+  }
+
   $getBlockHeightTip(): Promise<number> {
     return axios.get<number>(config.ESPLORA.REST_API_URL + '/blocks/tip/height', this.axiosConfig)
       .then((response) => response.data);
@@ -50,9 +55,9 @@ class ElectrsApi implements AbstractBitcoinApi {
       .then((response) => response.data);
   }
 
-  $getRawBlock(hash: string): Promise<string> {
+  $getRawBlock(hash: string): Promise<Buffer> {
-    return axios.get<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + "/raw", this.axiosConfig)
+    return axios.get<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + "/raw", { ...this.axiosConfig, responseType: 'arraybuffer' })
-      .then((response) => response.data);
+      .then((response) => { return Buffer.from(response.data); });
   }
 
   $getAddress(address: string): Promise<IEsploraApi.Address> {

backend/src/api/blocks.ts

@@ -20,6 +20,8 @@ import indexer from '../indexer';
 import fiatConversion from './fiat-conversion';
 import poolsParser from './pools-parser';
 import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
+import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository';
+import cpfpRepository from '../repositories/CpfpRepository';
 import mining from './mining/mining';
 import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository';
 import PricesRepository from '../repositories/PricesRepository';
@@ -33,6 +35,7 @@ class Blocks {
   private lastDifficultyAdjustmentTime = 0;
   private previousDifficultyRetarget = 0;
   private newBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => void)[] = [];
+  private newAsyncBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => Promise<void>)[] = [];
 
   constructor() { }
 
@@ -56,6 +59,10 @@ class Blocks {
     this.newBlockCallbacks.push(fn);
   }
 
+  public setNewAsyncBlockCallback(fn: (block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => Promise<void>) {
+    this.newAsyncBlockCallbacks.push(fn);
+  }
+
   /**
    * Return the list of transaction for a block
    * @param blockHash
@@ -92,12 +99,23 @@ class Blocks {
           transactions.push(tx);
           transactionsFetched++;
         } catch (e) {
-          if (i === 0) {
+          try {
-            const msg = `Cannot fetch coinbase tx ${txIds[i]}. Reason: ` + (e instanceof Error ? e.message : e); 
+            if (config.MEMPOOL.BACKEND === 'esplora') {
-            logger.err(msg);
+              // Try again with core
-            throw new Error(msg);
+              const tx = await transactionUtils.$getTransactionExtended(txIds[i], false, false, true);
-          } else {
+              transactions.push(tx);
-            logger.err(`Cannot fetch tx ${txIds[i]}. Reason: ` + (e instanceof Error ? e.message : e));
+              transactionsFetched++;
+            } else {
+              throw e;
+            }
+          } catch (e) {
+            if (i === 0) {
+              const msg = `Cannot fetch coinbase tx ${txIds[i]}. Reason: ` + (e instanceof Error ? e.message : e); 
+              logger.err(msg);
+              throw new Error(msg);
+            } else {
+              logger.err(`Cannot fetch tx ${txIds[i]}. Reason: ` + (e instanceof Error ? e.message : e));
+            }
           }
         }
       }
@@ -129,7 +147,7 @@ class Blocks {
     const stripped = block.tx.map((tx) => {
       return {
         txid: tx.txid,
-        vsize: tx.vsize,
+        vsize: tx.weight / 4,
         fee: tx.fee ? Math.round(tx.fee * 100000000) : 0,
         value: Math.round(tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0) * 100000000)
       };
@@ -186,14 +204,18 @@ class Blocks {
       if (!pool) { // We should never have this situation in practise
         logger.warn(`Cannot assign pool to block ${blockExtended.height} and 'unknown' pool does not exist. ` +
           `Check your "pools" table entries`);
-        return blockExtended;
+      } else {
+        blockExtended.extras.pool = {
+          id: pool.id,
+          name: pool.name,
+          slug: pool.slug,
+        };
       }
 
-      blockExtended.extras.pool = {
+      const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(block.id);
-        id: pool.id,
+      if (auditScore != null) {
-        name: pool.name,
+        blockExtended.extras.matchRate = auditScore.matchRate;
-        slug: pool.slug,
+      }
-      };
     }
 
     return blockExtended;
@@ -250,7 +272,7 @@ class Blocks {
   /**
    * [INDEXING] Index all blocks summaries for the block txs visualization
    */
-  public async $generateBlocksSummariesDatabase() {
+  public async $generateBlocksSummariesDatabase(): Promise<void> {
     if (Common.blocksSummariesIndexingEnabled() === false) {
       return;
     }
@@ -283,7 +305,7 @@ class Blocks {
           const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
           const blockPerSeconds = Math.max(1, indexedThisRun / elapsedSeconds);
           const progress = Math.round(totalIndexed / indexedBlocks.length * 10000) / 100;
-          logger.debug(`Indexing block summary for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexedBlocks.length} (${progress}%) | elapsed: ${runningFor} seconds`);
+          logger.debug(`Indexing block summary for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexedBlocks.length} (${progress}%) | elapsed: ${runningFor} seconds`, logger.tags.mining);
           timer = new Date().getTime() / 1000;
           indexedThisRun = 0;
         }
@@ -296,12 +318,60 @@ class Blocks {
         newlyIndexed++;
       }
       if (newlyIndexed > 0) {
-        logger.notice(`Blocks summaries indexing completed: indexed ${newlyIndexed} blocks`);
+        logger.notice(`Blocks summaries indexing completed: indexed ${newlyIndexed} blocks`, logger.tags.mining);
       } else {
-        logger.debug(`Blocks summaries indexing completed: indexed ${newlyIndexed} blocks`);
+        logger.debug(`Blocks summaries indexing completed: indexed ${newlyIndexed} blocks`, logger.tags.mining);
       }
     } catch (e) {
-      logger.err(`Blocks summaries indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`);
+      logger.err(`Blocks summaries indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`, logger.tags.mining);
+      throw e;
+    }
+  }
+
+  /**
+   * [INDEXING] Index transaction CPFP data for all blocks
+   */
+   public async $generateCPFPDatabase(): Promise<void> {
+    if (Common.cpfpIndexingEnabled() === false) {
+      return;
+    }
+
+    try {
+      // Get all indexed block hash
+      const unindexedBlockHeights = await blocksRepository.$getCPFPUnindexedBlocks();
+      logger.info(`Indexing cpfp data for ${unindexedBlockHeights.length} blocks`);
+
+      if (!unindexedBlockHeights?.length) {
+        return;
+      }
+
+      // Logging
+      let count = 0;
+      let countThisRun = 0;
+      let timer = new Date().getTime() / 1000;
+      const startedAt = new Date().getTime() / 1000;
+      for (const height of unindexedBlockHeights) {
+        // Logging
+        const hash = await bitcoinApi.$getBlockHash(height);
+        const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer);
+        if (elapsedSeconds > 5) {
+          const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
+          const blockPerSeconds = (countThisRun / elapsedSeconds);
+          const progress = Math.round(count / unindexedBlockHeights.length * 10000) / 100;
+          logger.debug(`Indexing cpfp clusters for #${height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${count}/${unindexedBlockHeights.length} (${progress}%) | elapsed: ${runningFor} seconds`);
+          timer = new Date().getTime() / 1000;
+          countThisRun = 0;
+        }
+
+        await this.$indexCPFP(hash, height); // Calculate and save CPFP data for transactions in this block
+
+        // Logging
+        count++;
+        countThisRun++;
+      }
+      logger.notice(`CPFP indexing completed: indexed ${count} blocks`);
+    } catch (e) {
+      logger.err(`CPFP indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`);
       throw e;
     }
   }
@@ -321,7 +391,7 @@ class Blocks {
 
       const lastBlockToIndex = Math.max(0, currentBlockHeight - indexingBlockAmount + 1);
 
-      logger.debug(`Indexing blocks from #${currentBlockHeight} to #${lastBlockToIndex}`);
+      logger.debug(`Indexing blocks from #${currentBlockHeight} to #${lastBlockToIndex}`, logger.tags.mining);
       loadingIndicators.setProgress('block-indexing', 0);
 
       const chunkSize = 10000;
@@ -341,7 +411,7 @@ class Blocks {
           continue;
         }
 
-        logger.info(`Indexing ${missingBlockHeights.length} blocks from #${currentBlockHeight} to #${endBlock}`);
+        logger.info(`Indexing ${missingBlockHeights.length} blocks from #${currentBlockHeight} to #${endBlock}`, logger.tags.mining);
 
         for (const blockHeight of missingBlockHeights) {
           if (blockHeight < lastBlockToIndex) {
@@ -349,12 +419,12 @@ class Blocks {
           }
           ++indexedThisRun;
           ++totalIndexed;
-          const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
+          const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer);
           if (elapsedSeconds > 5 || blockHeight === lastBlockToIndex) {
             const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
             const blockPerSeconds = Math.max(1, indexedThisRun / elapsedSeconds);
             const progress = Math.round(totalIndexed / indexingBlockAmount * 10000) / 100;
-            logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${runningFor} seconds`);
+            logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${runningFor} seconds`, logger.tags.mining);
             timer = new Date().getTime() / 1000;
             indexedThisRun = 0;
             loadingIndicators.setProgress('block-indexing', progress, false);
@@ -371,13 +441,13 @@ class Blocks {
         currentBlockHeight -= chunkSize;
       }
       if (newlyIndexed > 0) {
-        logger.notice(`Block indexing completed: indexed ${newlyIndexed} blocks`);
+        logger.notice(`Block indexing completed: indexed ${newlyIndexed} blocks`, logger.tags.mining);
       } else {
-        logger.debug(`Block indexing completed: indexed ${newlyIndexed} blocks`);
+        logger.debug(`Block indexing completed: indexed ${newlyIndexed} blocks`, logger.tags.mining);
       }
       loadingIndicators.setProgress('block-indexing', 100);
     } catch (e) {
-      logger.err('Block indexing failed. Trying again in 10 seconds. Reason: ' + (e instanceof Error ? e.message : e));
+      logger.err('Block indexing failed. Trying again in 10 seconds. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
       loadingIndicators.setProgress('block-indexing', 100);
       throw e;
     }
@@ -439,6 +509,9 @@ class Blocks {
       const blockExtended: BlockExtended = await this.$getBlockExtended(block, transactions);
       const blockSummary: BlockSummary = this.summarizeBlock(verboseBlock);
 
+      // start async callbacks
+      const callbackPromises = this.newAsyncBlockCallbacks.map((cb) => cb(blockExtended, txIds, transactions));
+
       if (Common.indexingEnabled()) {
         if (!fastForwarded) {
           const lastBlock = await blocksRepository.$getBlockByHeight(blockExtended.height - 1);
@@ -448,9 +521,13 @@ class Blocks {
             await BlocksRepository.$deleteBlocksFrom(lastBlock['height'] - 10);
             await HashratesRepository.$deleteLastEntries();
             await BlocksSummariesRepository.$deleteBlocksFrom(lastBlock['height'] - 10);
+            await cpfpRepository.$deleteClustersFrom(lastBlock['height'] - 10);
             for (let i = 10; i >= 0; --i) {
               const newBlock = await this.$indexBlock(lastBlock['height'] - i);
               await this.$getStrippedBlockTransactions(newBlock.id, true, true);
+              if (config.MEMPOOL.CPFP_INDEXING) {
+                await this.$indexCPFP(newBlock.id, lastBlock['height'] - i);
+              }
             }
             await mining.$indexDifficultyAdjustments();
             await DifficultyAdjustmentsRepository.$deleteLastAdjustment();
@@ -466,7 +543,7 @@ class Blocks {
               priceId: lastestPriceId,
             }]);
           } else {
-            logger.info(`Cannot save block price for ${blockExtended.height} because the price updater hasnt completed yet. Trying again in 10 seconds.`)
+            logger.info(`Cannot save block price for ${blockExtended.height} because the price updater hasnt completed yet. Trying again in 10 seconds.`, logger.tags.mining);
             setTimeout(() => {
               indexer.runSingleTask('blocksPrices');
             }, 10000);
@@ -476,6 +553,9 @@ class Blocks {
           if (Common.blocksSummariesIndexingEnabled() === true) {
             await this.$getStrippedBlockTransactions(blockExtended.id, true);
           }
+          if (config.MEMPOOL.CPFP_INDEXING) {
+            this.$indexCPFP(blockExtended.id, this.currentBlockHeight);
+          }
         }
       }
 
@@ -509,6 +589,9 @@ class Blocks {
       if (!memPool.hasPriority()) {
         diskCache.$saveCacheToDisk();
       }
+
+      // wait for pending async callbacks to finish
+      await Promise.all(callbackPromises);
     }
   }
 
@@ -574,7 +657,7 @@ class Blocks {
     if (skipMemoryCache === false) {
       // Check the memory cache
       const cachedSummary = this.getBlockSummaries().find((b) => b.id === hash);
-      if (cachedSummary) {
+      if (cachedSummary?.transactions?.length) {
         return cachedSummary.transactions;
       }
     }
@@ -582,7 +665,7 @@ class Blocks {
     // Check if it's indexed in db
     if (skipDBLookup === false && Common.blocksSummariesIndexingEnabled() === true) {
       const indexedSummary = await BlocksSummariesRepository.$getByBlockId(hash);
-      if (indexedSummary !== undefined) {
+      if (indexedSummary !== undefined && indexedSummary?.transactions?.length) {
         return indexedSummary.transactions;
       }
     }
@@ -600,7 +683,12 @@ class Blocks {
   }
 
   public async $getBlocks(fromHeight?: number, limit: number = 15): Promise<BlockExtended[]> {
-    let currentHeight = fromHeight !== undefined ? fromHeight : await blocksRepository.$mostRecentBlockHeight();
+
+    let currentHeight = fromHeight !== undefined ? fromHeight : this.currentBlockHeight;
+    if (currentHeight > this.currentBlockHeight) {
+      limit -= currentHeight - this.currentBlockHeight;
+      currentHeight = this.currentBlockHeight;
+    }
     const returnBlocks: BlockExtended[] = [];
 
     if (currentHeight < 0) {
@@ -635,6 +723,22 @@ class Blocks {
     return returnBlocks;
   }
 
+  public async $getBlockAuditSummary(hash: string): Promise<any> {
+    let summary;
+    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
+      summary = await BlocksAuditsRepository.$getBlockAudit(hash);
+    }
+
+    // fallback to non-audited transaction summary
+    if (!summary?.transactions?.length) {
+      const strippedTransactions = await this.$getStrippedBlockTransactions(hash);
+      summary = {
+        transactions: strippedTransactions
+      };
+    }
+    return summary;
+  }
+
   public getLastDifficultyAdjustmentTime(): number {
     return this.lastDifficultyAdjustmentTime;
   }
@@ -646,6 +750,50 @@ class Blocks {
   public getCurrentBlockHeight(): number {
     return this.currentBlockHeight;
   }
+
+  public async $indexCPFP(hash: string, height: number): Promise<void> {
+    const block = await bitcoinClient.getBlock(hash, 2);
+    const transactions = block.tx.map(tx => {
+      tx.vsize = tx.weight / 4;
+      tx.fee *= 100_000_000;
+      return tx;
+    });
+
+    const clusters: any[] = [];
+
+    let cluster: TransactionStripped[] = [];
+    let ancestors: { [txid: string]: boolean } = {};
+    for (let i = transactions.length - 1; i >= 0; i--) {
+      const tx = transactions[i];
+      if (!ancestors[tx.txid]) {
+        let totalFee = 0;
+        let totalVSize = 0;
+        cluster.forEach(tx => {
+          totalFee += tx?.fee || 0;
+          totalVSize += tx.vsize;
+        });
+        const effectiveFeePerVsize = totalFee / totalVSize;
+        if (cluster.length > 1) {
+          clusters.push({
+            root: cluster[0].txid,
+            height,
+            txs: cluster.map(tx => { return { txid: tx.txid, weight: tx.vsize * 4, fee: tx.fee || 0 }; }),
+            effectiveFeePerVsize,
+          });
+        }
+        cluster = [];
+        ancestors = {};
+      }
+      cluster.push(tx);
+      tx.vin.forEach(vin => {
+        ancestors[vin.txid] = true;
+      });
+    }
+    const result = await cpfpRepository.$batchSaveClusters(clusters);
+    if (!result) {
+      await cpfpRepository.$insertProgressMarker(height);
+    }
+  }
 }
 
 export default new Blocks();

backend/src/api/common.ts

@@ -51,8 +51,6 @@ export class Common {
   static findRbfTransactions(added: TransactionExtended[], deleted: TransactionExtended[]): { [txid: string]: TransactionExtended } {
     const matches: { [txid: string]: TransactionExtended } = {};
     deleted
-      // The replaced tx must have at least one input with nSequence < maxint-1 (That’s the opt-in)
-      .filter((tx) => tx.vin.some((vin) => vin.sequence < 0xfffffffe))
       .forEach((deletedTx) => {
         const foundMatches = added.find((addedTx) => {
           // The new tx must, absolutely speaking, pay at least as much fee as the replaced tx.
@@ -61,7 +59,7 @@ export class Common {
             && addedTx.feePerVsize > deletedTx.feePerVsize
             // Spends one or more of the same inputs
             && deletedTx.vin.some((deletedVin) =>
-              addedTx.vin.some((vin) => vin.txid === deletedVin.txid));
+              addedTx.vin.some((vin) => vin.txid === deletedVin.txid && vin.vout === deletedVin.vout));
             });
         if (foundMatches) {
           matches[deletedTx.txid] = foundMatches;
@@ -187,6 +185,13 @@ export class Common {
     );
   }
 
+  static cpfpIndexingEnabled(): boolean {
+    return (
+      Common.indexingEnabled() &&
+      config.MEMPOOL.CPFP_INDEXING === true
+    );
+  }
+
   static setDateMidnight(date: Date): void {
     date.setUTCHours(0);
     date.setUTCMinutes(0);

backend/src/api/database-migration.ts

@@ -2,10 +2,13 @@ import config from '../config';
 import DB from '../database';
 import logger from '../logger';
 import { Common } from './common';
+import blocksRepository from '../repositories/BlocksRepository';
+import cpfpRepository from '../repositories/CpfpRepository';
+import { RowDataPacket } from 'mysql2';
 
 class DatabaseMigration {
-  private static currentVersion = 40;
+  private static currentVersion = 52;
-  private queryTimeout = 120000;
+  private queryTimeout = 3600_000;
   private statisticsAddedIndexed = false;
   private uniqueLogs: string[] = [];
 
@@ -107,18 +110,22 @@ class DatabaseMigration {
     await this.$executeQuery(this.getCreateStatisticsQuery(), await this.$checkIfTableExists('statistics'));
     if (databaseSchemaVersion < 2 && this.statisticsAddedIndexed === false) {
       await this.$executeQuery(`CREATE INDEX added ON statistics (added);`);
+      await this.updateToSchemaVersion(2);
     }
     if (databaseSchemaVersion < 3) {
       await this.$executeQuery(this.getCreatePoolsTableQuery(), await this.$checkIfTableExists('pools'));
+      await this.updateToSchemaVersion(3);
     }
     if (databaseSchemaVersion < 4) {
       await this.$executeQuery('DROP table IF EXISTS blocks;');
       await this.$executeQuery(this.getCreateBlocksTableQuery(), await this.$checkIfTableExists('blocks'));
+      await this.updateToSchemaVersion(4);
     }
     if (databaseSchemaVersion < 5 && isBitcoin === true) {
       this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
       await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index
       await this.$executeQuery('ALTER TABLE blocks ADD `reward` double unsigned NOT NULL DEFAULT "0"');
+      await this.updateToSchemaVersion(5);
     }
 
     if (databaseSchemaVersion < 6 && isBitcoin === true) {
@@ -141,11 +148,13 @@ class DatabaseMigration {
       await this.$executeQuery('ALTER TABLE blocks ADD `nonce` bigint unsigned NOT NULL DEFAULT "0"');
       await this.$executeQuery('ALTER TABLE blocks ADD `merkle_root` varchar(65) NOT NULL DEFAULT ""');
       await this.$executeQuery('ALTER TABLE blocks ADD `previous_block_hash` varchar(65) NULL');
+      await this.updateToSchemaVersion(6);
     }
 
     if (databaseSchemaVersion < 7 && isBitcoin === true) {
       await this.$executeQuery('DROP table IF EXISTS hashrates;');
       await this.$executeQuery(this.getCreateDailyStatsTableQuery(), await this.$checkIfTableExists('hashrates'));
+      await this.updateToSchemaVersion(7);
     }
 
     if (databaseSchemaVersion < 8 && isBitcoin === true) {
@@ -155,6 +164,7 @@ class DatabaseMigration {
       await this.$executeQuery('ALTER TABLE `hashrates` ADD `id` int NOT NULL AUTO_INCREMENT PRIMARY KEY FIRST');
       await this.$executeQuery('ALTER TABLE `hashrates` ADD `share` float NOT NULL DEFAULT "0"');
       await this.$executeQuery('ALTER TABLE `hashrates` ADD `type` enum("daily", "weekly") DEFAULT "daily"');
+      await this.updateToSchemaVersion(8);
     }
 
     if (databaseSchemaVersion < 9 && isBitcoin === true) {
@@ -162,10 +172,12 @@ class DatabaseMigration {
       await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index
       await this.$executeQuery('ALTER TABLE `state` CHANGE `name` `name` varchar(100)');
       await this.$executeQuery('ALTER TABLE `hashrates` ADD UNIQUE `hashrate_timestamp_pool_id` (`hashrate_timestamp`, `pool_id`)');
+      await this.updateToSchemaVersion(9);
     }
 
     if (databaseSchemaVersion < 10 && isBitcoin === true) {
       await this.$executeQuery('ALTER TABLE `blocks` ADD INDEX `blockTimestamp` (`blockTimestamp`)');
+      await this.updateToSchemaVersion(10);
     }
 
     if (databaseSchemaVersion < 11 && isBitcoin === true) {
@@ -178,11 +190,13 @@ class DatabaseMigration {
       await this.$executeQuery('ALTER TABLE blocks MODIFY `reward` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
       await this.$executeQuery('ALTER TABLE blocks MODIFY `median_fee` INT UNSIGNED NOT NULL DEFAULT "0"');
       await this.$executeQuery('ALTER TABLE blocks MODIFY `fees` INT UNSIGNED NOT NULL DEFAULT "0"');
+      await this.updateToSchemaVersion(11);
     }
 
     if (databaseSchemaVersion < 12 && isBitcoin === true) {
       // No need to re-index because the new data type can contain larger values
       await this.$executeQuery('ALTER TABLE blocks MODIFY `fees` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
+      await this.updateToSchemaVersion(12);
     }
 
     if (databaseSchemaVersion < 13 && isBitcoin === true) {
@@ -190,6 +204,7 @@ class DatabaseMigration {
       await this.$executeQuery('ALTER TABLE blocks MODIFY `median_fee` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
       await this.$executeQuery('ALTER TABLE blocks MODIFY `avg_fee` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
       await this.$executeQuery('ALTER TABLE blocks MODIFY `avg_fee_rate` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
+      await this.updateToSchemaVersion(13);
     }
 
     if (databaseSchemaVersion < 14 && isBitcoin === true) {
@@ -197,37 +212,45 @@ class DatabaseMigration {
       await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index
       await this.$executeQuery('ALTER TABLE `hashrates` DROP FOREIGN KEY `hashrates_ibfk_1`');
       await this.$executeQuery('ALTER TABLE `hashrates` MODIFY `pool_id` SMALLINT UNSIGNED NOT NULL DEFAULT "0"');
+      await this.updateToSchemaVersion(14);
     }
 
     if (databaseSchemaVersion < 16 && isBitcoin === true) {
       this.uniqueLog(logger.notice, this.hashratesTruncatedMessage);
       await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index because we changed timestamps
+      await this.updateToSchemaVersion(16);
     }
 
     if (databaseSchemaVersion < 17 && isBitcoin === true) {
       await this.$executeQuery('ALTER TABLE `pools` ADD `slug` CHAR(50) NULL');
+      await this.updateToSchemaVersion(17);
     }
 
     if (databaseSchemaVersion < 18 && isBitcoin === true) {
       await this.$executeQuery('ALTER TABLE `blocks` ADD INDEX `hash` (`hash`);');
+      await this.updateToSchemaVersion(18);
     }
 
     if (databaseSchemaVersion < 19) {
       await this.$executeQuery(this.getCreateRatesTableQuery(), await this.$checkIfTableExists('rates'));
+      await this.updateToSchemaVersion(19);
     }
 
     if (databaseSchemaVersion < 20 && isBitcoin === true) {
       await this.$executeQuery(this.getCreateBlocksSummariesTableQuery(), await this.$checkIfTableExists('blocks_summaries'));
+      await this.updateToSchemaVersion(20);
     }
 
     if (databaseSchemaVersion < 21) {
       await this.$executeQuery('DROP TABLE IF EXISTS `rates`');
       await this.$executeQuery(this.getCreatePricesTableQuery(), await this.$checkIfTableExists('prices'));
+      await this.updateToSchemaVersion(21);
     }
 
     if (databaseSchemaVersion < 22 && isBitcoin === true) {
       await this.$executeQuery('DROP TABLE IF EXISTS `difficulty_adjustments`');
       await this.$executeQuery(this.getCreateDifficultyAdjustmentsTableQuery(), await this.$checkIfTableExists('difficulty_adjustments'));
+      await this.updateToSchemaVersion(22);
     }
 
     if (databaseSchemaVersion < 23) {
@@ -240,11 +263,13 @@ class DatabaseMigration {
       await this.$executeQuery('ALTER TABLE `prices` ADD `CHF` float DEFAULT "0"');
       await this.$executeQuery('ALTER TABLE `prices` ADD `AUD` float DEFAULT "0"');
       await this.$executeQuery('ALTER TABLE `prices` ADD `JPY` float DEFAULT "0"');
+      await this.updateToSchemaVersion(23);
     }
 
     if (databaseSchemaVersion < 24 && isBitcoin == true) {
       await this.$executeQuery('DROP TABLE IF EXISTS `blocks_audits`');
       await this.$executeQuery(this.getCreateBlocksAuditsTableQuery(), await this.$checkIfTableExists('blocks_audits'));
+      await this.updateToSchemaVersion(24);
     }
 
     if (databaseSchemaVersion < 25 && isBitcoin === true) {
@@ -252,6 +277,7 @@ class DatabaseMigration {
       await this.$executeQuery(this.getCreateNodesQuery(), await this.$checkIfTableExists('nodes'));
       await this.$executeQuery(this.getCreateChannelsQuery(), await this.$checkIfTableExists('channels'));
       await this.$executeQuery(this.getCreateNodesStatsQuery(), await this.$checkIfTableExists('node_stats'));
+      await this.updateToSchemaVersion(25);
     }
 
     if (databaseSchemaVersion < 26 && isBitcoin === true) {
@@ -262,6 +288,7 @@ class DatabaseMigration {
       await this.$executeQuery('ALTER TABLE `lightning_stats` ADD tor_nodes int(11) NOT NULL DEFAULT "0"');
       await this.$executeQuery('ALTER TABLE `lightning_stats` ADD clearnet_nodes int(11) NOT NULL DEFAULT "0"');
       await this.$executeQuery('ALTER TABLE `lightning_stats` ADD unannounced_nodes int(11) NOT NULL DEFAULT "0"');
+      await this.updateToSchemaVersion(26);
     }
 
     if (databaseSchemaVersion < 27 && isBitcoin === true) {
@@ -271,6 +298,7 @@ class DatabaseMigration {
       await this.$executeQuery('ALTER TABLE `lightning_stats` ADD med_capacity bigint(20) unsigned NOT NULL DEFAULT "0"');
       await this.$executeQuery('ALTER TABLE `lightning_stats` ADD med_fee_rate int(11) unsigned NOT NULL DEFAULT "0"');
       await this.$executeQuery('ALTER TABLE `lightning_stats` ADD med_base_fee_mtokens bigint(20) unsigned NOT NULL DEFAULT "0"');
+      await this.updateToSchemaVersion(27);
     }
     
     if (databaseSchemaVersion < 28 && isBitcoin === true) {
@@ -280,6 +308,7 @@ class DatabaseMigration {
       await this.$executeQuery(`TRUNCATE lightning_stats`);
       await this.$executeQuery(`TRUNCATE node_stats`);
       await this.$executeQuery(`ALTER TABLE lightning_stats MODIFY added DATE`);
+      await this.updateToSchemaVersion(28);
     }
 
     if (databaseSchemaVersion < 29 && isBitcoin === true) {
@@ -291,41 +320,50 @@ class DatabaseMigration {
       await this.$executeQuery('ALTER TABLE `nodes` ADD subdivision_id int(11) unsigned NULL DEFAULT NULL');
       await this.$executeQuery('ALTER TABLE `nodes` ADD longitude double NULL DEFAULT NULL');
       await this.$executeQuery('ALTER TABLE `nodes` ADD latitude double NULL DEFAULT NULL');
+      await this.updateToSchemaVersion(29);
     }
 
     if (databaseSchemaVersion < 30 && isBitcoin === true) {
       await this.$executeQuery('ALTER TABLE `geo_names` CHANGE `type` `type` enum("city","country","division","continent","as_organization") NOT NULL');
+      await this.updateToSchemaVersion(30);
     }
 
     if (databaseSchemaVersion < 31 && isBitcoin == true) { // Link blocks to prices
       await this.$executeQuery('ALTER TABLE `prices` ADD `id` int NULL AUTO_INCREMENT UNIQUE');
       await this.$executeQuery('DROP TABLE IF EXISTS `blocks_prices`');
       await this.$executeQuery(this.getCreateBlocksPricesTableQuery(), await this.$checkIfTableExists('blocks_prices'));
+      await this.updateToSchemaVersion(31);
     }
 
     if (databaseSchemaVersion < 32 && isBitcoin == true) {
       await this.$executeQuery('ALTER TABLE `blocks_summaries` ADD `template` JSON DEFAULT "[]"');
+      await this.updateToSchemaVersion(32);
     }
 
     if (databaseSchemaVersion < 33 && isBitcoin == true) {
       await this.$executeQuery('ALTER TABLE `geo_names` CHANGE `type` `type` enum("city","country","division","continent","as_organization", "country_iso_code") NOT NULL');
+      await this.updateToSchemaVersion(33);
     }
 
     if (databaseSchemaVersion < 34 && isBitcoin == true) {
       await this.$executeQuery('ALTER TABLE `lightning_stats` ADD clearnet_tor_nodes int(11) NOT NULL DEFAULT "0"');
+      await this.updateToSchemaVersion(34);
     }
 
     if (databaseSchemaVersion < 35 && isBitcoin == true) {
       await this.$executeQuery('DELETE from `lightning_stats` WHERE added > "2021-09-19"');
       await this.$executeQuery('ALTER TABLE `lightning_stats` ADD CONSTRAINT added_unique UNIQUE (added);');
+      await this.updateToSchemaVersion(35);
     }
 
     if (databaseSchemaVersion < 36 && isBitcoin == true) {
       await this.$executeQuery('ALTER TABLE `nodes` ADD status TINYINT NOT NULL DEFAULT "1"');
+      await this.updateToSchemaVersion(36);
     }
 
     if (databaseSchemaVersion < 37 && isBitcoin == true) {
       await this.$executeQuery(this.getCreateLNNodesSocketsTableQuery(), await this.$checkIfTableExists('nodes_sockets'));
+      await this.updateToSchemaVersion(37);
     }
 
     if (databaseSchemaVersion < 38 && isBitcoin == true) {
@@ -336,17 +374,99 @@ class DatabaseMigration {
       await this.$executeQuery(`TRUNCATE node_stats`);
       await this.$executeQuery('ALTER TABLE `lightning_stats` CHANGE `added` `added` timestamp NULL');
       await this.$executeQuery('ALTER TABLE `node_stats` CHANGE `added` `added` timestamp NULL');
+      await this.updateToSchemaVersion(38);
     }
 
     if (databaseSchemaVersion < 39 && isBitcoin === true) {
       await this.$executeQuery('ALTER TABLE `nodes` ADD alias_search TEXT NULL DEFAULT NULL AFTER `alias`');
       await this.$executeQuery('ALTER TABLE nodes ADD FULLTEXT(alias_search)');
+      await this.updateToSchemaVersion(39);
     }
 
     if (databaseSchemaVersion < 40 && isBitcoin === true) {
       await this.$executeQuery('ALTER TABLE `nodes` ADD capacity bigint(20) unsigned DEFAULT NULL');
       await this.$executeQuery('ALTER TABLE `nodes` ADD channels int(11) unsigned DEFAULT NULL');
       await this.$executeQuery('ALTER TABLE `nodes` ADD INDEX `capacity` (`capacity`);');
+      await this.updateToSchemaVersion(40);
+    }
+
+    if (databaseSchemaVersion < 41 && isBitcoin === true) {
+      await this.$executeQuery('UPDATE channels SET closing_reason = NULL WHERE closing_reason = 1');
+      await this.updateToSchemaVersion(41);
+    }
+
+    if (databaseSchemaVersion < 42 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `channels` ADD closing_resolved tinyint(1) DEFAULT 0');
+      await this.updateToSchemaVersion(42);
+    }
+
+    if (databaseSchemaVersion < 43 && isBitcoin === true) {
+      await this.$executeQuery(this.getCreateLNNodeRecordsTableQuery(), await this.$checkIfTableExists('nodes_records'));
+      await this.updateToSchemaVersion(43);
+    }
+
+    if (databaseSchemaVersion < 44 && isBitcoin === true) {
+      await this.$executeQuery('UPDATE blocks_summaries SET template = NULL');
+      await this.updateToSchemaVersion(44);
+    }
+
+    if (databaseSchemaVersion < 45 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `blocks_audits` ADD fresh_txs JSON DEFAULT "[]"');
+      await this.updateToSchemaVersion(45);
+    }
+
+    if (databaseSchemaVersion < 46) {
+      await this.$executeQuery(`ALTER TABLE blocks MODIFY blockTimestamp timestamp NOT NULL DEFAULT 0`);
+      await this.updateToSchemaVersion(46);
+    }
+
+    if (databaseSchemaVersion < 47) {
+      await this.$executeQuery('ALTER TABLE `blocks` ADD cpfp_indexed tinyint(1) DEFAULT 0');
+      await this.$executeQuery(this.getCreateCPFPTableQuery(), await this.$checkIfTableExists('cpfp_clusters'));
+      await this.$executeQuery(this.getCreateTransactionsTableQuery(), await this.$checkIfTableExists('transactions'));
+      await this.updateToSchemaVersion(47);
+    }
+
+    if (databaseSchemaVersion < 48 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `channels` ADD source_checked tinyint(1) DEFAULT 0');
+      await this.$executeQuery('ALTER TABLE `channels` ADD closing_fee bigint(20) unsigned DEFAULT 0');
+      await this.$executeQuery('ALTER TABLE `channels` ADD node1_funding_balance bigint(20) unsigned DEFAULT 0');
+      await this.$executeQuery('ALTER TABLE `channels` ADD node2_funding_balance bigint(20) unsigned DEFAULT 0');
+      await this.$executeQuery('ALTER TABLE `channels` ADD node1_closing_balance bigint(20) unsigned DEFAULT 0');
+      await this.$executeQuery('ALTER TABLE `channels` ADD node2_closing_balance bigint(20) unsigned DEFAULT 0');
+      await this.$executeQuery('ALTER TABLE `channels` ADD funding_ratio float unsigned DEFAULT NULL');
+      await this.$executeQuery('ALTER TABLE `channels` ADD closed_by varchar(66) DEFAULT NULL');
+      await this.$executeQuery('ALTER TABLE `channels` ADD single_funded tinyint(1) DEFAULT 0');
+      await this.$executeQuery('ALTER TABLE `channels` ADD outputs JSON DEFAULT "[]"');
+      await this.updateToSchemaVersion(48);
+    }
+
+    if (databaseSchemaVersion < 49 && isBitcoin === true) {
+      await this.$executeQuery('TRUNCATE TABLE `blocks_audits`');
+      await this.updateToSchemaVersion(49);
+    }
+
+    if (databaseSchemaVersion < 50) {
+      await this.$executeQuery('ALTER TABLE `blocks` DROP COLUMN `cpfp_indexed`');
+      await this.updateToSchemaVersion(50);
+    }
+
+    if (databaseSchemaVersion < 51) {
+      await this.$executeQuery('ALTER TABLE `cpfp_clusters` ADD INDEX `height` (`height`)');
+      await this.updateToSchemaVersion(51);
+    }
+
+    if (databaseSchemaVersion < 52) {
+      await this.$executeQuery(this.getCreateCompactCPFPTableQuery(), await this.$checkIfTableExists('compact_cpfp_clusters'));
+      await this.$executeQuery(this.getCreateCompactTransactionsTableQuery(), await this.$checkIfTableExists('compact_transactions'));
+      try {
+        await this.$convertCompactCpfpTables();
+        await this.$executeQuery('DROP TABLE IF EXISTS `cpfp_clusters`');
+        await this.$executeQuery('DROP TABLE IF EXISTS `transactions`');
+        await this.updateToSchemaVersion(52);
+      } catch(e) {
+        logger.warn('' + (e instanceof Error ? e.message : e));
+      }
     }
   }
 
@@ -485,6 +605,10 @@ class DatabaseMigration {
     return `UPDATE state SET number = ${DatabaseMigration.currentVersion} WHERE name = 'schema_version';`;
   }
 
+  private async updateToSchemaVersion(version): Promise<void> {
+    await this.$executeQuery(`UPDATE state SET number = ${version} WHERE name = 'schema_version';`);
+  }
+
   /**
    * Print current database version
    */
@@ -783,6 +907,57 @@ class DatabaseMigration {
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
 
+  private getCreateLNNodeRecordsTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS nodes_records (
+      public_key varchar(66) NOT NULL,
+      type int(10) unsigned NOT NULL,
+      payload blob NOT NULL,
+      UNIQUE KEY public_key_type (public_key, type),
+      INDEX (public_key),
+      FOREIGN KEY (public_key)
+        REFERENCES nodes (public_key)
+        ON DELETE CASCADE
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
+  private getCreateCPFPTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS cpfp_clusters (
+      root varchar(65) NOT NULL,
+      height int(10) NOT NULL,
+      txs JSON DEFAULT NULL,
+      fee_rate double unsigned NOT NULL,
+      PRIMARY KEY (root)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
+  private getCreateTransactionsTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS transactions (
+      txid varchar(65) NOT NULL,
+      cluster varchar(65) DEFAULT NULL,
+      PRIMARY KEY (txid),
+      FOREIGN KEY (cluster) REFERENCES cpfp_clusters (root) ON DELETE SET NULL
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
+  private getCreateCompactCPFPTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS compact_cpfp_clusters (
+      root binary(32) NOT NULL,
+      height int(10) NOT NULL,
+      txs BLOB DEFAULT NULL,
+      fee_rate float unsigned,
+      PRIMARY KEY (root),
+      INDEX (height)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
+  private getCreateCompactTransactionsTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS compact_transactions (
+      txid binary(32) NOT NULL,
+      cluster binary(32) DEFAULT NULL,
+      PRIMARY KEY (txid)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
   public async $truncateIndexedData(tables: string[]) {
     const allowedTables = ['blocks', 'hashrates', 'prices'];
 
@@ -803,6 +978,49 @@ class DatabaseMigration {
       logger.warn(`Unable to erase indexed data`);
     }
   }
+
+  private async $convertCompactCpfpTables(): Promise<void> {
+    try {
+      const batchSize = 250;
+      const maxHeight = await blocksRepository.$mostRecentBlockHeight() || 0;
+      const [minHeightRows]: any = await DB.query(`SELECT MIN(height) AS minHeight from cpfp_clusters`);
+      const minHeight = (minHeightRows.length && minHeightRows[0].minHeight != null) ? minHeightRows[0].minHeight : maxHeight;
+      let height = maxHeight;
+
+      // Logging
+      let timer = new Date().getTime() / 1000;
+      const startedAt = new Date().getTime() / 1000;
+
+      while (height > minHeight) {
+        const [rows] = await DB.query(
+          `
+            SELECT * from cpfp_clusters
+            WHERE height <= ? AND height > ?
+            ORDER BY height
+          `,
+          [height, height - batchSize]
+        ) as RowDataPacket[][];
+        if (rows?.length) {
+          await cpfpRepository.$batchSaveClusters(rows.map(row => {
+            return {
+              root: row.root,
+              height: row.height,
+              txs: JSON.parse(row.txs),
+              effectiveFeePerVsize: row.fee_rate,
+            };
+          }));
+        }
+
+        const elapsed = new Date().getTime() / 1000 - timer;
+        const runningFor = new Date().getTime() / 1000 - startedAt;
+        logger.debug(`Migrated cpfp data from block ${height} to ${height - batchSize} in ${elapsed.toFixed(2)} seconds | total elapsed: ${runningFor.toFixed(2)} seconds`);
+        timer = new Date().getTime() / 1000;
+        height -= batchSize;
+      }
+    } catch (e) {
+      logger.warn(`Failed to migrate cpfp transaction data`);
+    }
+  }
 }
 
 export default new DatabaseMigration();

backend/src/api/explorer/channels.api.ts

@@ -117,6 +117,32 @@ class ChannelsApi {
     }
   }
 
+  public async $getUnresolvedClosedChannels(): Promise<any[]> {
+    try {
+      const query = `SELECT * FROM channels WHERE status = 2 AND closing_reason = 2 AND closing_resolved = 0 AND closing_transaction_id != ''`;
+      const [rows]: any = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('$getUnresolvedClosedChannels error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getChannelsWithoutSourceChecked(): Promise<any[]> {
+    try {
+      const query = `
+        SELECT channels.*
+        FROM channels
+        WHERE channels.source_checked != 1
+      `;
+      const [rows]: any = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('$getUnresolvedClosedChannels error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
   public async $getChannelsWithoutCreatedDate(): Promise<any[]> {
     try {
       const query = `SELECT * FROM channels WHERE created IS NULL`;
@@ -246,6 +272,108 @@ class ChannelsApi {
     }
   }
 
+  public async $getChannelByClosingId(transactionId: string): Promise<any> {
+    try {
+      const query = `
+        SELECT
+          channels.*
+        FROM channels
+        WHERE channels.closing_transaction_id = ?
+      `;
+      const [rows]: any = await DB.query(query, [transactionId]);
+      if (rows.length > 0) {
+        rows[0].outputs = JSON.parse(rows[0].outputs);
+        return rows[0];
+      }
+    } catch (e) {
+      logger.err('$getChannelByClosingId error: ' + (e instanceof Error ? e.message : e));
+      // don't throw - this data isn't essential
+    }
+  }
+
+  public async $getChannelsByOpeningId(transactionId: string): Promise<any> {
+    try {
+      const query = `
+        SELECT
+          channels.*
+        FROM channels
+        WHERE channels.transaction_id = ?
+      `;
+      const [rows]: any = await DB.query(query, [transactionId]);
+      if (rows.length > 0) {
+        return rows.map(row => {
+          row.outputs = JSON.parse(row.outputs);
+          return row;
+        });
+      }
+    } catch (e) {
+      logger.err('$getChannelsByOpeningId error: ' + (e instanceof Error ? e.message : e));
+      // don't throw - this data isn't essential
+    }
+  }
+
+  public async $updateClosingInfo(channelInfo: { id: string, node1_closing_balance: number, node2_closing_balance: number, closed_by: string | null, closing_fee: number, outputs: ILightningApi.ForensicOutput[]}): Promise<void> {
+    try {
+      const query = `
+        UPDATE channels SET
+          node1_closing_balance = ?,
+          node2_closing_balance = ?,
+          closed_by = ?,
+          closing_fee = ?,
+          outputs = ?
+        WHERE channels.id = ?
+      `;
+      await DB.query<ResultSetHeader>(query, [
+        channelInfo.node1_closing_balance || 0,
+        channelInfo.node2_closing_balance || 0,
+        channelInfo.closed_by,
+        channelInfo.closing_fee || 0,
+        JSON.stringify(channelInfo.outputs),
+        channelInfo.id,
+      ]);
+    } catch (e) {
+      logger.err('$updateClosingInfo error: ' + (e instanceof Error ? e.message : e));
+      // don't throw - this data isn't essential
+    }
+  }
+
+  public async $updateOpeningInfo(channelInfo: { id: string, node1_funding_balance: number, node2_funding_balance: number, funding_ratio: number, single_funded: boolean | void }): Promise<void> {
+    try {
+      const query = `
+        UPDATE channels SET
+          node1_funding_balance = ?,
+          node2_funding_balance = ?,
+          funding_ratio = ?,
+          single_funded = ?
+        WHERE channels.id = ?
+      `;
+      await DB.query<ResultSetHeader>(query, [
+        channelInfo.node1_funding_balance || 0,
+        channelInfo.node2_funding_balance || 0,
+        channelInfo.funding_ratio,
+        channelInfo.single_funded ? 1 : 0,
+        channelInfo.id,
+      ]);
+    } catch (e) {
+      logger.err('$updateOpeningInfo error: ' + (e instanceof Error ? e.message : e));
+      // don't throw - this data isn't essential
+    }
+  }
+
+  public async $markChannelSourceChecked(id: string): Promise<void> {
+    try {
+      const query = `
+        UPDATE channels
+        SET source_checked = 1
+        WHERE id = ?
+      `;
+      await DB.query<ResultSetHeader>(query, [id]);
+    } catch (e) {
+      logger.err('$markChannelSourceChecked error: ' + (e instanceof Error ? e.message : e));
+      // don't throw - this data isn't essential
+    }
+  }
+
   public async $getChannelsForNode(public_key: string, index: number, length: number, status: string): Promise<any[]> {
     try {
       let channelStatusFilter;
@@ -374,11 +502,15 @@ class ChannelsApi {
       'transaction_id': channel.transaction_id,
       'transaction_vout': channel.transaction_vout,
       'closing_transaction_id': channel.closing_transaction_id,
+      'closing_fee': channel.closing_fee,
       'closing_reason': channel.closing_reason,
       'closing_date': channel.closing_date,
       'updated_at': channel.updated_at,
       'created': channel.created,
       'status': channel.status,
+      'funding_ratio': channel.funding_ratio,
+      'closed_by': channel.closed_by,
+      'single_funded': !!channel.single_funded,
       'node_left': {
         'alias': channel.alias_left,
         'public_key': channel.node1_public_key,
@@ -393,6 +525,9 @@ class ChannelsApi {
         'updated_at': channel.node1_updated_at,
         'longitude': channel.node1_longitude,
         'latitude': channel.node1_latitude,
+        'funding_balance': channel.node1_funding_balance,
+        'closing_balance': channel.node1_closing_balance,
+        'initiated_close': channel.closed_by === channel.node1_public_key ? true : undefined,
       },
       'node_right': {
         'alias': channel.alias_right,
@@ -408,6 +543,9 @@ class ChannelsApi {
         'updated_at': channel.node2_updated_at,
         'longitude': channel.node2_longitude,
         'latitude': channel.node2_latitude,
+        'funding_balance': channel.node2_funding_balance,
+        'closing_balance': channel.node2_closing_balance,
+        'initiated_close': channel.closed_by === channel.node2_public_key ? true : undefined,
       },
     };
   }
@@ -532,9 +670,7 @@ class ChannelsApi {
         AND status != 2
       `);
       if (result[0].changedRows ?? 0 > 0) {
-        logger.info(`Marked ${result[0].changedRows} channels as inactive because they are not in the graph`);
+        logger.debug(`Marked ${result[0].changedRows} channels as inactive because they are not in the graph`, logger.tags.ln);
-      } else {
-        logger.debug(`Marked ${result[0].changedRows} channels as inactive because they are not in the graph`);
       }
     } catch (e) {
       logger.err('$setChannelsInactive() error: ' + (e instanceof Error ? e.message : e));

backend/src/api/explorer/nodes.api.ts

@@ -14,8 +14,8 @@ class NodesApi {
         nodes.longitude, nodes.latitude,
         geo_names_country.names as country, geo_names_iso.names as isoCode
         FROM nodes
-        LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
+        JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
-        LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+        JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
         WHERE status = 1 AND nodes.as_number IS NOT NULL
         ORDER BY capacity
       `;
@@ -105,6 +105,18 @@ class NodesApi {
         node.closed_channel_count = rows[0].closed_channel_count;
       }
 
+      // Custom records
+      query = `
+        SELECT type, payload
+        FROM nodes_records
+        WHERE public_key = ?
+      `;
+      [rows] = await DB.query(query, [public_key]);
+      node.custom_records = {};
+      for (const record of rows) {
+        node.custom_records[record.type] = Buffer.from(record.payload, 'binary').toString('hex');
+      }
+
       return node;
     } catch (e) {
       logger.err(`Cannot get node information for ${public_key}. Reason: ${(e instanceof Error ? e.message : e)}`);
@@ -129,6 +141,56 @@ class NodesApi {
     }
   }
 
+  public async $getFeeHistogram(node_public_key: string): Promise<unknown> {
+    try {
+      const inQuery = `
+        SELECT CASE WHEN fee_rate <= 10.0 THEN CEIL(fee_rate)
+                    WHEN (fee_rate > 10.0 and fee_rate <= 100.0) THEN CEIL(fee_rate / 10.0) * 10.0
+                    WHEN (fee_rate > 100.0 and fee_rate <= 1000.0) THEN CEIL(fee_rate / 100.0) * 100.0
+                    WHEN fee_rate > 1000.0 THEN CEIL(fee_rate / 1000.0) * 1000.0
+               END as bucket,
+               count(short_id) as count,
+               sum(capacity) as capacity
+        FROM (
+          SELECT CASE WHEN node1_public_key = ? THEN node2_fee_rate WHEN node2_public_key = ? THEN node1_fee_rate END as fee_rate,
+                 short_id as short_id,
+                 capacity as capacity
+          FROM channels
+          WHERE status = 1 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)
+        ) as fee_rate_table
+        GROUP BY bucket;
+      `;
+      const [inRows]: any[] = await DB.query(inQuery, [node_public_key, node_public_key, node_public_key, node_public_key]);
+
+      const outQuery = `
+        SELECT CASE WHEN fee_rate <= 10.0 THEN CEIL(fee_rate)
+                    WHEN (fee_rate > 10.0 and fee_rate <= 100.0) THEN CEIL(fee_rate / 10.0) * 10.0
+                    WHEN (fee_rate > 100.0 and fee_rate <= 1000.0) THEN CEIL(fee_rate / 100.0) * 100.0
+                    WHEN fee_rate > 1000.0 THEN CEIL(fee_rate / 1000.0) * 1000.0
+               END as bucket,
+               count(short_id) as count,
+               sum(capacity) as capacity
+        FROM (
+          SELECT CASE WHEN node1_public_key = ? THEN node1_fee_rate WHEN node2_public_key = ? THEN node2_fee_rate END as fee_rate,
+                 short_id as short_id,
+                 capacity as capacity
+          FROM channels
+          WHERE status = 1 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)
+        ) as fee_rate_table
+        GROUP BY bucket;
+      `;
+      const [outRows]: any[] = await DB.query(outQuery, [node_public_key, node_public_key, node_public_key, node_public_key]);
+
+      return {
+        incoming: inRows.length > 0 ? inRows : [],
+        outgoing: outRows.length > 0 ? outRows : [],
+      };
+    } catch (e) {
+      logger.err(`Cannot get node fee distribution for ${node_public_key}. Reason: ${(e instanceof Error ? e.message : e)}`);
+      throw e;
+    }
+  }
+
   public async $getAllNodes(): Promise<any> {
     try {
       const query = `SELECT * FROM nodes`;
@@ -462,7 +524,41 @@ class NodesApi {
 
   public async $getNodesPerISP(ISPId: string) {
     try {
-      const query = `
+      let query = `
+        SELECT channels.node1_public_key AS node1PublicKey, isp1.id as isp1ID,
+          channels.node2_public_key AS node2PublicKey, isp2.id as isp2ID
+        FROM channels
+        JOIN nodes node1 ON node1.public_key = channels.node1_public_key
+        JOIN nodes node2 ON node2.public_key = channels.node2_public_key
+        JOIN geo_names isp1 ON isp1.id = node1.as_number
+        JOIN geo_names isp2 ON isp2.id = node2.as_number
+        WHERE channels.status = 1 AND (node1.as_number IN (?) OR node2.as_number IN (?))
+        ORDER BY short_id DESC
+      `;
+
+      const IPSIds = ISPId.split(',');
+      const [rows]: any = await DB.query(query, [IPSIds, IPSIds]);
+      if (!rows || rows.length === 0) {
+        return [];
+      }
+
+      const nodes = {};
+
+      const intISPIds: number[] = [];
+      for (const ispId of IPSIds) {
+        intISPIds.push(parseInt(ispId, 10));
+      }
+
+      for (const channel of rows) {
+        if (intISPIds.includes(channel.isp1ID)) {
+          nodes[channel.node1PublicKey] = true;
+        }
+        if (intISPIds.includes(channel.isp2ID)) {
+          nodes[channel.node2PublicKey] = true;
+        }
+      }
+
+      query = `
         SELECT nodes.public_key, CAST(COALESCE(nodes.capacity, 0) as INT) as capacity, CAST(COALESCE(nodes.channels, 0) as INT) as channels,
           nodes.alias, UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at,
           geo_names_city.names as city, geo_names_country.names as country,
@@ -473,17 +569,18 @@ class NodesApi {
         LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
         LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
         LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
-        WHERE nodes.as_number IN (?)
+        WHERE nodes.public_key IN (?)
         ORDER BY capacity DESC
       `;
 
-      const [rows]: any = await DB.query(query, [ISPId.split(',')]);
+      const [rows2]: any = await DB.query(query, [Object.keys(nodes)]);
-      for (let i = 0; i < rows.length; ++i) {
+      for (let i = 0; i < rows2.length; ++i) {
-        rows[i].country = JSON.parse(rows[i].country);
+        rows2[i].country = JSON.parse(rows2[i].country);
-        rows[i].city = JSON.parse(rows[i].city);
+        rows2[i].city = JSON.parse(rows2[i].city);
-        rows[i].subdivision = JSON.parse(rows[i].subdivision);
+        rows2[i].subdivision = JSON.parse(rows2[i].subdivision);
       }
-      return rows;
+      return rows2;
+
     } catch (e) {
       logger.err(`Cannot get nodes for ISP id ${ISPId}. Reason: ${e instanceof Error ? e.message : e}`);
       throw e;
@@ -588,9 +685,7 @@ class NodesApi {
         )
       `);
       if (result[0].changedRows ?? 0 > 0) {
-        logger.info(`Marked ${result[0].changedRows} nodes as inactive because they are not in the graph`);
+        logger.debug(`Marked ${result[0].changedRows} nodes as inactive because they are not in the graph`, logger.tags.ln);
-      } else {
-        logger.debug(`Marked ${result[0].changedRows} nodes as inactive because they are not in the graph`);
       }
     } catch (e) {
       logger.err('$setNodesInactive() error: ' + (e instanceof Error ? e.message : e));

backend/src/api/explorer/nodes.routes.ts

@@ -20,6 +20,7 @@ class NodesRoutes {
       .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings/connectivity', this.$getTopNodesByChannels)
       .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings/age', this.$getOldestNodes)
       .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/:public_key/statistics', this.$getHistoricalNodeStats)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/:public_key/fees/histogram', this.$getFeeHistogram)
       .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/:public_key', this.$getNode)
       .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/group/:name', this.$getNodeGroup)
     ;
@@ -40,13 +41,70 @@ class NodesRoutes {
       let nodes: any[] = [];
       switch (config.MEMPOOL.NETWORK) {
         case 'testnet':
-          nodesList = ['032c7c7819276c4f706a04df1a0f1e10a5495994a7be4c1d3d28ca766e5a2b957b', '025a7e38c2834dd843591a4d23d5f09cdeb77ddca85f673c2d944a14220ff14cf7', '0395e2731a1673ef21d7a16a727c4fc4d4c35a861c428ce2c819c53d2b81c8bd55', '032ab2028c0b614c6d87824e2373529652fd7e4221b4c70cc4da7c7005c49afcf0', '029001b22fe70b48bee12d014df91982eb85ff1bd404ec772d5c83c4ee3e88d2c3', '0212e2848d79f928411da5f2ff0a8c95ec6ccb5a09d2031b6f71e91309dcde63af', '03e871a2229523d34f76e6311ff197cfe7f26c2fbec13554b93a46f4e710c47dab', '032202ec98d976b0e928bd1d91924e8bd3eab07231fc39feb3737b010071073df8', '02fa7c5a948d03d563a9f36940c2205a814e594d17c0042ced242c71a857d72605', '039c14fdec2d958e3d14cebf657451bbd9e039196615785e82c917f274e3fb2205', '033589bbcb233ffc416cefd5437c7f37e9d7cb7942d405e39e72c4c846d9b37f18', '029293110441c6e2eacb57e1255bf6ef05c41a6a676fe474922d33c19f98a7d584'];
+          nodesList = [
+            '032c7c7819276c4f706a04df1a0f1e10a5495994a7be4c1d3d28ca766e5a2b957b',
+            '025a7e38c2834dd843591a4d23d5f09cdeb77ddca85f673c2d944a14220ff14cf7',
+            '0395e2731a1673ef21d7a16a727c4fc4d4c35a861c428ce2c819c53d2b81c8bd55',
+            '032ab2028c0b614c6d87824e2373529652fd7e4221b4c70cc4da7c7005c49afcf0',
+            '029001b22fe70b48bee12d014df91982eb85ff1bd404ec772d5c83c4ee3e88d2c3',
+            '0212e2848d79f928411da5f2ff0a8c95ec6ccb5a09d2031b6f71e91309dcde63af',
+            '03e871a2229523d34f76e6311ff197cfe7f26c2fbec13554b93a46f4e710c47dab',
+            '032202ec98d976b0e928bd1d91924e8bd3eab07231fc39feb3737b010071073df8',
+            '02fa7c5a948d03d563a9f36940c2205a814e594d17c0042ced242c71a857d72605',
+            '039c14fdec2d958e3d14cebf657451bbd9e039196615785e82c917f274e3fb2205',
+            '033589bbcb233ffc416cefd5437c7f37e9d7cb7942d405e39e72c4c846d9b37f18',
+            '029293110441c6e2eacb57e1255bf6ef05c41a6a676fe474922d33c19f98a7d584',
+            '0235ad0b56ed8c42c4354444c24e971c05e769ec0b5fb0ccea42880095dc02ea2c',
+            '029700819a37afea630f80e6cc461f3fd3c4ace2598a21cfbbe64d1c78d0ee69a5',
+            '02c2d8b2dbf87c7894af2f1d321290e2fe6db5446cd35323987cee98f06e2e0075',
+            '030b0ca1ea7b1075716d2a555630e6fd47ef11bc7391fe68963ec06cf370a5e382',
+            '031adb9eb2d66693f85fa31a4adca0319ba68219f3ad5f9a2ef9b34a6b40755fa1',
+            '02ccd07faa47eda810ecf5591ccf5ca50f6c1034d0d175052898d32a00b9bae24f',
+          ];
           break;
         case 'signet':
-          nodesList = ['03ddab321b760433cbf561b615ef62ac7d318630c5f51d523aaf5395b90b751956', '033d92c7bfd213ef1b34c90e985fb5dc77f9ec2409d391492484e57a44c4aca1de', '02ad010dda54253c1eb9efe38b0760657a3b43ecad62198c359c051c9d99d45781', '025196512905b8a3f1597428b867bec63ec9a95e5089eb7dc7e63e2d2691669029', '027c625aa1fbe3768db68ebcb05b53b6dc0ce68b7b54b8900d326d167363e684fe', '03f1629af3101fcc56b7aac2667016be84e3defbf3d0c8719f836c9b41c9a57a43', '02dfb81e2f7a3c4c9e8a51b70ef82b4a24549cc2fab1f5b2fd636501774a918991', '02d01ccf832944c68f10d39006093769c5b8bda886d561b128534e313d729fdb34', '02499ed23027d4698a6904ff4ec1b6085a61f10b9a6937f90438f9947e38e8ea86', '038310e3a786340f2bd7770704c7ccfe560fd163d9a1c99d67894597419d12cbf7', '03e5e9d879b72c7d67ecd483bae023bd33e695bb32b981a4021260f7b9d62bc761', '028d16e1a0ace4c0c0a421536d8d32ce484dfe6e2f726b7b0e7c30f12a195f8cc7'];
+          nodesList = [
+            '03ddab321b760433cbf561b615ef62ac7d318630c5f51d523aaf5395b90b751956',
+            '033d92c7bfd213ef1b34c90e985fb5dc77f9ec2409d391492484e57a44c4aca1de',
+            '02ad010dda54253c1eb9efe38b0760657a3b43ecad62198c359c051c9d99d45781',
+            '025196512905b8a3f1597428b867bec63ec9a95e5089eb7dc7e63e2d2691669029',
+            '027c625aa1fbe3768db68ebcb05b53b6dc0ce68b7b54b8900d326d167363e684fe',
+            '03f1629af3101fcc56b7aac2667016be84e3defbf3d0c8719f836c9b41c9a57a43',
+            '02dfb81e2f7a3c4c9e8a51b70ef82b4a24549cc2fab1f5b2fd636501774a918991',
+            '02d01ccf832944c68f10d39006093769c5b8bda886d561b128534e313d729fdb34',
+            '02499ed23027d4698a6904ff4ec1b6085a61f10b9a6937f90438f9947e38e8ea86',
+            '038310e3a786340f2bd7770704c7ccfe560fd163d9a1c99d67894597419d12cbf7',
+            '03e5e9d879b72c7d67ecd483bae023bd33e695bb32b981a4021260f7b9d62bc761',
+            '028d16e1a0ace4c0c0a421536d8d32ce484dfe6e2f726b7b0e7c30f12a195f8cc7',
+            '02ff690d06c187ab994bf83c5a2114fe5bf50112c2c817af0f788f736be9fa2070',
+            '02a9f570c51a2526a5ee85802e88f9281bed771eb66a0c8a7d898430dd5d0eae45',
+            '038c3de773255d3bd7a50e31e58d423baac5c90826a74d75e64b74c95475de1097',
+            '0242c7f7d315095f37ad1421ae0a2fc967d4cbe65b61b079c5395a769436959853',
+            '02a909e70eb03742f12666ebb1f56ac42a5fbaab0c0e8b5b1df4aa9f10f8a09240',
+            '03a26efa12489803c07f3ac2f1dba63812e38f0f6e866ce3ebb34df7de1f458cd2',
+          ];
           break;
         default:
-          nodesList = ['03fbc17549ec667bccf397ababbcb4cdc0e3394345e4773079ab2774612ec9be61', '03da9a8623241ccf95f19cd645c6cecd4019ac91570e976eb0a128bebbc4d8a437', '03ca5340cf85cb2e7cf076e489f785410838de174e40be62723e8a60972ad75144', '0238bd27f02d67d6c51e269692bc8c9a32357a00e7777cba7f4f1f18a2a700b108', '03f983dcabed6baa1eab5b56c8b2e8fdc846ab3fd931155377897335e85a9fa57c', '03e399589533581e48796e29a825839a010036a61b20744fda929d6709fcbffcc5', '021f5288b5f72c42cd0d8801086af7ce09a816d8ee9a4c47a4b436399b26cb601a', '032b01b7585f781420cd4148841a82831ba37fa952342052cec16750852d4f2dd9', '02848036488d4b8fb1f1c4064261ec36151f43b085f0b51bd239ade3ddfc940c34', '02b6b1640fe029e304c216951af9fbefdb23b0bdc9baaf327540d31b6107841fdf', '03694289827203a5b3156d753071ddd5bf92e371f5a462943f9555eef6d2d6606c', '0283d850db7c3e8ea7cc9c4abc7afaab12bbdf72b677dcba1d608350d2537d7d43'];
+          nodesList = [
+            '03fbc17549ec667bccf397ababbcb4cdc0e3394345e4773079ab2774612ec9be61',
+            '03da9a8623241ccf95f19cd645c6cecd4019ac91570e976eb0a128bebbc4d8a437',
+            '03ca5340cf85cb2e7cf076e489f785410838de174e40be62723e8a60972ad75144',
+            '0238bd27f02d67d6c51e269692bc8c9a32357a00e7777cba7f4f1f18a2a700b108',
+            '03f983dcabed6baa1eab5b56c8b2e8fdc846ab3fd931155377897335e85a9fa57c',
+            '03e399589533581e48796e29a825839a010036a61b20744fda929d6709fcbffcc5',
+            '021f5288b5f72c42cd0d8801086af7ce09a816d8ee9a4c47a4b436399b26cb601a',
+            '032b01b7585f781420cd4148841a82831ba37fa952342052cec16750852d4f2dd9',
+            '02848036488d4b8fb1f1c4064261ec36151f43b085f0b51bd239ade3ddfc940c34',
+            '02b6b1640fe029e304c216951af9fbefdb23b0bdc9baaf327540d31b6107841fdf',
+            '03694289827203a5b3156d753071ddd5bf92e371f5a462943f9555eef6d2d6606c',
+            '0283d850db7c3e8ea7cc9c4abc7afaab12bbdf72b677dcba1d608350d2537d7d43',
+            '02521287789f851268a39c9eccc9d6180d2c614315b583c9e6ae0addbd6d79df06',
+            '0258c2a7b7f8af2585b4411b1ec945f70988f30412bb1df179de941f14d0b1bc3e',
+            '03c3389ff1a896f84d921ed01a19fc99c6724ce8dc4b960cd3b7b2362b62cd60d7',
+            '038d118996b3eaa15dcd317b32a539c9ecfdd7698f204acf8a087336af655a9192',
+            '02a928903d93d78877dacc3642b696128a3636e9566dd42d2d132325b2c8891c09',
+            '0328cd17f3a9d3d90b532ade0d1a67e05eb8a51835b3dce0a2e38eac04b5a62a57',
+          ];
       }
 
       for (let pubKey of nodesList) {
@@ -95,6 +153,22 @@ class NodesRoutes {
     }
   }
 
+  private async $getFeeHistogram(req: Request, res: Response) {
+    try {
+      const node = await nodesApi.$getFeeHistogram(req.params.public_key);
+      if (!node) {
+        res.status(404).send('Node not found');
+        return;
+      }
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(node);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
   private async $getNodesRanking(req: Request, res: Response): Promise<void> {
     try {
       const topCapacityNodes = await nodesApi.$getTopCapacityNodes(false);

backend/src/api/lightning/clightning/clightning-client.ts

@@ -141,13 +141,13 @@ export default class CLightningClient extends EventEmitter implements AbstractLi
       // main data directory provided, default to using the bitcoin mainnet subdirectory
       // to be removed in v0.2.0
       else if (fExists(rpcPath, 'bitcoin', 'lightning-rpc')) {
-        logger.warn(`[CLightningClient] ${rpcPath}/lightning-rpc is missing, using the bitcoin mainnet subdirectory at ${rpcPath}/bitcoin instead.`)
+        logger.warn(`${rpcPath}/lightning-rpc is missing, using the bitcoin mainnet subdirectory at ${rpcPath}/bitcoin instead.`, logger.tags.ln)
-        logger.warn(`[CLightningClient] specifying the main lightning data directory is deprecated, please specify the network directory explicitly.\n`)
+        logger.warn(`specifying the main lightning data directory is deprecated, please specify the network directory explicitly.\n`, logger.tags.ln)
         rpcPath = path.join(rpcPath, 'bitcoin', 'lightning-rpc')
       }
     }
 
-    logger.debug(`[CLightningClient] Connecting to ${rpcPath}`);
+    logger.debug(`Connecting to ${rpcPath}`, logger.tags.ln);
 
     super();
     this.rpcPath = rpcPath;
@@ -172,19 +172,19 @@ export default class CLightningClient extends EventEmitter implements AbstractLi
 
     this.clientConnectionPromise = new Promise<void>(resolve => {
       _self.client.on('connect', () => {
-        logger.info(`[CLightningClient] Lightning client connected`);
+        logger.info(`CLightning client connected`, logger.tags.ln);
         _self.reconnectWait = 1;
         resolve();
       });
 
       _self.client.on('end', () => {
-        logger.err('[CLightningClient] Lightning client connection closed, reconnecting');
+        logger.err(`CLightning client connection closed, reconnecting`, logger.tags.ln);
         _self.increaseWaitTime();
         _self.reconnect();
       });
 
       _self.client.on('error', error => {
-        logger.err(`[CLightningClient] Lightning client connection error: ${error}`);
+        logger.err(`CLightning client connection error: ${error}`, logger.tags.ln);
         _self.increaseWaitTime();
         _self.reconnect();
       });
@@ -196,7 +196,6 @@ export default class CLightningClient extends EventEmitter implements AbstractLi
         return;
       }
       const data = JSON.parse(line);
-      // logger.debug(`[CLightningClient] #${data.id} <-- ${JSON.stringify(data.error || data.result)}`);
       _self.emit('res:' + data.id, data);
     });
   }
@@ -217,7 +216,7 @@ export default class CLightningClient extends EventEmitter implements AbstractLi
     }
 
     this.reconnectTimeout = setTimeout(() => {
-      logger.debug('[CLightningClient] Trying to reconnect...');
+      logger.debug(`Trying to reconnect...`, logger.tags.ln);
 
       _self.client.connect(_self.rpcPath);
       _self.reconnectTimeout = null;
@@ -235,7 +234,6 @@ export default class CLightningClient extends EventEmitter implements AbstractLi
       id: '' + callInt
     };
 
-    // logger.debug(`[CLightningClient] #${callInt} --> ${method} ${args}`);
 
     // Wait for the client to connect
     return this.clientConnectionPromise

backend/src/api/lightning/clightning/clightning-convert.ts

@@ -2,11 +2,21 @@ import { ILightningApi } from '../lightning-api.interface';
 import FundingTxFetcher from '../../../tasks/lightning/sync-tasks/funding-tx-fetcher';
 import logger from '../../../logger';
 import { Common } from '../../common';
+import config from '../../../config';
 
 /**
  * Convert a clightning "listnode" entry to a lnd node entry
  */
 export function convertNode(clNode: any): ILightningApi.Node {
+  let custom_records: { [type: number]: string } | undefined = undefined;
+  if (clNode.option_will_fund) {
+    try {
+      custom_records = { '1': Buffer.from(clNode.option_will_fund.compact_lease || '', 'hex').toString('base64') };
+    } catch (e) {
+      logger.err(`Cannot decode option_will_fund compact_lease for ${clNode.nodeid}). Reason: ` + (e instanceof Error ? e.message : e));
+      custom_records = undefined;
+    }
+  }
   return {
     alias: clNode.alias ?? '',
     color: `#${clNode.color ?? ''}`,
@@ -23,6 +33,7 @@ export function convertNode(clNode: any): ILightningApi.Node {
       };
     }) ?? [],
     last_update: clNode?.last_timestamp ?? 0,
+    custom_records
   };
 }
 
@@ -30,7 +41,7 @@ export function convertNode(clNode: any): ILightningApi.Node {
  * Convert clightning "listchannels" response to lnd "describegraph.edges" format
  */
 export async function convertAndmergeBidirectionalChannels(clChannels: any[]): Promise<ILightningApi.Channel[]> {
-  logger.info('Converting clightning nodes and channels to lnd graph format');
+  logger.debug(`Converting clightning nodes and channels to lnd graph format`, logger.tags.ln);
 
   let loggerTimer = new Date().getTime() / 1000;
   let channelProcessed = 0;
@@ -52,8 +63,8 @@ export async function convertAndmergeBidirectionalChannels(clChannels: any[]): P
     }
 
     const elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer);
-    if (elapsedSeconds > 10) {
+    if (elapsedSeconds > config.LIGHTNING.LOGGER_UPDATE_INTERVAL) {
-      logger.info(`Building complete channels from clightning output. Channels processed: ${channelProcessed + 1} of ${clChannels.length}`);
+      logger.info(`Building complete channels from clightning output. Channels processed: ${channelProcessed + 1} of ${clChannels.length}`, logger.tags.ln);
       loggerTimer = new Date().getTime() / 1000;
     }
 
@@ -66,10 +77,12 @@ export async function convertAndmergeBidirectionalChannels(clChannels: any[]): P
     consolidatedChannelList.push(await buildIncompleteChannel(clChannelsDict[short_channel_id]));
 
     const elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer);
-    if (elapsedSeconds > 10) {
+    if (elapsedSeconds > config.LIGHTNING.LOGGER_UPDATE_INTERVAL) {
       logger.info(`Building partial channels from clightning output. Channels processed: ${channelProcessed + 1} of ${keys.length}`);
       loggerTimer = new Date().getTime() / 1000;
     }
+
+    channelProcessed++;
   }
 
   return consolidatedChannelList;

backend/src/api/lightning/lightning-api.interface.ts

@@ -49,6 +49,7 @@ export namespace ILightningApi {
     }[];
     color: string;
     features: { [key: number]: Feature };
+    custom_records?: { [type: number]: string };
   }
 
   export interface Info {
@@ -82,4 +83,10 @@ export namespace ILightningApi {
     is_required: boolean;
     is_known: boolean;
   }
+
+  export interface ForensicOutput {
+    node?: 1 | 2;
+    type: number;
+    value: number;
+  }
 }

backend/src/api/mempool-blocks.ts

@@ -1,11 +1,14 @@
 import logger from '../logger';
-import { MempoolBlock, TransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta } from '../mempool.interfaces';
+import { MempoolBlock, TransactionExtended, ThreadTransaction, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor } from '../mempool.interfaces';
 import { Common } from './common';
 import config from '../config';
+import { Worker } from 'worker_threads';
+import path from 'path';
 
 class MempoolBlocks {
   private mempoolBlocks: MempoolBlockWithTransactions[] = [];
   private mempoolBlockDeltas: MempoolBlockDelta[] = [];
+  private txSelectionWorker: Worker | null = null;
 
   constructor() {}
 
@@ -71,15 +74,15 @@ class MempoolBlocks {
     const time = end - start;
     logger.debug('Mempool blocks calculated in ' + time / 1000 + ' seconds');
 
-    const { blocks, deltas } = this.calculateMempoolBlocks(memPoolArray, this.mempoolBlocks);
+    const blocks = this.calculateMempoolBlocks(memPoolArray, this.mempoolBlocks);
+    const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, blocks);
+
     this.mempoolBlocks = blocks;
     this.mempoolBlockDeltas = deltas;
   }
 
-  private calculateMempoolBlocks(transactionsSorted: TransactionExtended[], prevBlocks: MempoolBlockWithTransactions[]):
+  private calculateMempoolBlocks(transactionsSorted: TransactionExtended[], prevBlocks: MempoolBlockWithTransactions[]): MempoolBlockWithTransactions[] {
-    { blocks: MempoolBlockWithTransactions[], deltas: MempoolBlockDelta[] } {
     const mempoolBlocks: MempoolBlockWithTransactions[] = [];
-    const mempoolBlockDeltas: MempoolBlockDelta[] = [];
     let blockWeight = 0;
     let blockSize = 0;
     let transactions: TransactionExtended[] = [];
@@ -99,7 +102,12 @@ class MempoolBlocks {
     if (transactions.length) {
       mempoolBlocks.push(this.dataToMempoolBlocks(transactions, blockSize, blockWeight, mempoolBlocks.length));
     }
-    // Calculate change from previous block states
+
+    return mempoolBlocks;
+  }
+
+  private calculateMempoolDeltas(prevBlocks: MempoolBlockWithTransactions[], mempoolBlocks: MempoolBlockWithTransactions[]): MempoolBlockDelta[] {
+    const mempoolBlockDeltas: MempoolBlockDelta[] = [];
     for (let i = 0; i < Math.max(mempoolBlocks.length, prevBlocks.length); i++) {
       let added: TransactionStripped[] = [];
       let removed: string[] = [];
@@ -132,14 +140,162 @@ class MempoolBlocks {
         removed
       });
     }
-    return {
+    return mempoolBlockDeltas;
-      blocks: mempoolBlocks,
+  }
-      deltas: mempoolBlockDeltas
+
-    };
+  public async makeBlockTemplates(newMempool: { [txid: string]: TransactionExtended }): Promise<void> {
+    // prepare a stripped down version of the mempool with only the minimum necessary data
+    // to reduce the overhead of passing this data to the worker thread
+    const strippedMempool: { [txid: string]: ThreadTransaction } = {};
+    Object.values(newMempool).forEach(entry => {
+      strippedMempool[entry.txid] = {
+        txid: entry.txid,
+        fee: entry.fee,
+        weight: entry.weight,
+        feePerVsize: entry.fee / (entry.weight / 4),
+        effectiveFeePerVsize: entry.fee / (entry.weight / 4),
+        vin: entry.vin.map(v => v.txid),
+      };
+    });
+
+    // (re)initialize tx selection worker thread
+    if (!this.txSelectionWorker) {
+      this.txSelectionWorker = new Worker(path.resolve(__dirname, './tx-selection-worker.js'));
+      // if the thread throws an unexpected error, or exits for any other reason,
+      // reset worker state so that it will be re-initialized on the next run
+      this.txSelectionWorker.once('error', () => {
+        this.txSelectionWorker = null;
+      });
+      this.txSelectionWorker.once('exit', () => {
+        this.txSelectionWorker = null;
+      });
+    }
+
+    // run the block construction algorithm in a separate thread, and wait for a result
+    let threadErrorListener;
+    try {
+      const workerResultPromise = new Promise<{ blocks: ThreadTransaction[][], clusters: { [root: string]: string[] } }>((resolve, reject) => {
+        threadErrorListener = reject;
+        this.txSelectionWorker?.once('message', (result): void => {
+          resolve(result);
+        });
+        this.txSelectionWorker?.once('error', reject);
+      });
+      this.txSelectionWorker.postMessage({ type: 'set', mempool: strippedMempool });
+      const { blocks, clusters } = await workerResultPromise;
+
+      this.processBlockTemplates(newMempool, blocks, clusters);
+
+      // clean up thread error listener
+      this.txSelectionWorker?.removeListener('error', threadErrorListener);
+    } catch (e) {
+      logger.err('makeBlockTemplates failed. ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  public async updateBlockTemplates(newMempool: { [txid: string]: TransactionExtended }, added: TransactionExtended[], removed: string[]): Promise<void> {
+    if (!this.txSelectionWorker) {
+      // need to reset the worker
+      return this.makeBlockTemplates(newMempool);
+    }
+    // prepare a stripped down version of the mempool with only the minimum necessary data
+    // to reduce the overhead of passing this data to the worker thread
+    const addedStripped: ThreadTransaction[] = added.map(entry => {
+      return {
+        txid: entry.txid,
+        fee: entry.fee,
+        weight: entry.weight,
+        feePerVsize: entry.fee / (entry.weight / 4),
+        effectiveFeePerVsize: entry.fee / (entry.weight / 4),
+        vin: entry.vin.map(v => v.txid),
+      };
+    });
+
+    // run the block construction algorithm in a separate thread, and wait for a result
+    let threadErrorListener;
+    try {
+      const workerResultPromise = new Promise<{ blocks: ThreadTransaction[][], clusters: { [root: string]: string[] } }>((resolve, reject) => {
+        threadErrorListener = reject;
+        this.txSelectionWorker?.once('message', (result): void => {
+          resolve(result);
+        });
+        this.txSelectionWorker?.once('error', reject);
+      });
+      this.txSelectionWorker.postMessage({ type: 'update', added: addedStripped, removed });
+      const { blocks, clusters } = await workerResultPromise;
+
+      this.processBlockTemplates(newMempool, blocks, clusters);
+
+      // clean up thread error listener
+      this.txSelectionWorker?.removeListener('error', threadErrorListener);
+    } catch (e) {
+      logger.err('updateBlockTemplates failed. ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  private processBlockTemplates(mempool, blocks, clusters): void {
+    // update this thread's mempool with the results
+    blocks.forEach(block => {
+      block.forEach(tx => {
+        if (tx.txid in mempool) {
+          if (tx.effectiveFeePerVsize != null) {
+            mempool[tx.txid].effectiveFeePerVsize = tx.effectiveFeePerVsize;
+          }
+          if (tx.cpfpRoot && tx.cpfpRoot in clusters) {
+            const ancestors: Ancestor[] = [];
+            const descendants: Ancestor[] = [];
+            const cluster = clusters[tx.cpfpRoot];
+            let matched = false;
+            cluster.forEach(txid => {
+              if (txid === tx.txid) {
+                matched = true;
+              } else if (mempool[txid]) {
+                const relative = {
+                  txid: txid,
+                  fee: mempool[txid].fee,
+                  weight: mempool[txid].weight,
+                };
+                if (matched) {
+                  descendants.push(relative);
+                } else {
+                  ancestors.push(relative);
+                }
+              }
+            });
+            mempool[tx.txid].ancestors = ancestors;
+            mempool[tx.txid].descendants = descendants;
+            mempool[tx.txid].bestDescendant = null;
+          }
+          mempool[tx.txid].cpfpChecked = tx.cpfpChecked;
+        }
+      });
+    });
+
+    // unpack the condensed blocks into proper mempool blocks
+    const mempoolBlocks = blocks.map((transactions, blockIndex) => {
+      return this.dataToMempoolBlocks(transactions.map(tx => {
+        return mempool[tx.txid] || null;
+      }).filter(tx => !!tx), undefined, undefined, blockIndex);
+    });
+
+    const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, mempoolBlocks);
+
+    this.mempoolBlocks = mempoolBlocks;
+    this.mempoolBlockDeltas = deltas;
   }
 
   private dataToMempoolBlocks(transactions: TransactionExtended[],
-    blockSize: number, blockWeight: number, blocksIndex: number): MempoolBlockWithTransactions {
+    blockSize: number | undefined, blockWeight: number | undefined, blocksIndex: number): MempoolBlockWithTransactions {
+    let totalSize = blockSize || 0;
+    let totalWeight = blockWeight || 0;
+    if (blockSize === undefined && blockWeight === undefined) {
+      totalSize = 0;
+      totalWeight = 0;
+      transactions.forEach(tx => {
+        totalSize += tx.size;
+        totalWeight += tx.weight;
+      });
+    }
     let rangeLength = 4;
     if (blocksIndex === 0) {
       rangeLength = 8;
@@ -150,8 +306,8 @@ class MempoolBlocks {
       rangeLength = 8;
     }
     return {
-      blockSize: blockSize,
+      blockSize: totalSize,
-      blockVSize: blockWeight / 4,
+      blockVSize: totalWeight / 4,
       nTx: transactions.length,
       totalFees: transactions.reduce((acc, cur) => acc + cur.fee, 0),
       medianFee: Common.percentile(transactions.map((tx) => tx.effectiveFeePerVsize), config.MEMPOOL.RECOMMENDED_FEE_PERCENTILE),

backend/src/api/mempool.ts

@@ -10,26 +10,32 @@ import bitcoinClient from './bitcoin/bitcoin-client';
 import bitcoinSecondClient from './bitcoin/bitcoin-second-client';
 import rbfCache from './rbf-cache';
 
-class Mempool {
+export class Mempool {
-  private static WEBSOCKET_REFRESH_RATE_MS = 10000;
+  protected static WEBSOCKET_REFRESH_RATE_MS = 10000;
-  private static LAZY_DELETE_AFTER_SECONDS = 30;
+  protected static LAZY_DELETE_AFTER_SECONDS = 30;
-  private inSync: boolean = false;
+  protected inSync: boolean = false;
-  private mempoolCacheDelta: number = -1;
+  protected mempoolCacheDelta: number = -1;
-  private mempoolCache: { [txId: string]: TransactionExtended } = {};
+  protected mempoolCache: { [txId: string]: TransactionExtended } = {};
-  private mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0, total_fee: 0,
+  protected mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0, total_fee: 0,
                                                     maxmempool: 300000000, mempoolminfee: 0.00001000, minrelaytxfee: 0.00001000 };
-  private mempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[],
+  protected mempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[],
     deletedTransactions: TransactionExtended[]) => void) | undefined;
+  protected asyncMempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[],
+    deletedTransactions: TransactionExtended[]) => Promise<void>) | undefined;
 
-  private txPerSecondArray: number[] = [];
+  protected txPerSecondArray: number[] = [];
-  private txPerSecond: number = 0;
+  protected txPerSecond: number = 0;
 
-  private vBytesPerSecondArray: VbytesPerSecond[] = [];
+  protected vBytesPerSecondArray: VbytesPerSecond[] = [];
-  private vBytesPerSecond: number = 0;
+  protected vBytesPerSecond: number = 0;
-  private mempoolProtection = 0;
+  protected mempoolProtection = 0;
-  private latestTransactions: any[] = [];
+  protected latestTransactions: any[] = [];
 
   constructor() {
+    this.init();
+  }
+
+  protected init(): void {
     setInterval(this.updateTxPerSecond.bind(this), 1000);
     setInterval(this.deleteExpiredTransactions.bind(this), 20000);
   }
@@ -63,6 +69,11 @@ class Mempool {
     this.mempoolChangedCallback = fn;
   }
 
+  public setAsyncMempoolChangedCallback(fn: (newMempool: { [txId: string]: TransactionExtended; },
+    newTransactions: TransactionExtended[], deletedTransactions: TransactionExtended[]) => Promise<void>) {
+    this.asyncMempoolChangedCallback = fn;
+  }
+
   public getMempool(): { [txid: string]: TransactionExtended } {
     return this.mempoolCache;
   }
@@ -72,6 +83,9 @@ class Mempool {
     if (this.mempoolChangedCallback) {
       this.mempoolChangedCallback(this.mempoolCache, [], []);
     }
+    if (this.asyncMempoolChangedCallback) {
+      this.asyncMempoolChangedCallback(this.mempoolCache, [], []);
+    }
   }
 
   public async $updateMemPoolInfo() {
@@ -103,12 +117,11 @@ class Mempool {
     return txTimes;
   }
 
-  public async $updateMempool() {
+  public async $updateMempool(): Promise<void> {
-    logger.debug('Updating mempool');
+    logger.debug(`Updating mempool...`);
     const start = new Date().getTime();
     let hasChange: boolean = false;
     const currentMempoolSize = Object.keys(this.mempoolCache).length;
-    let txCount = 0;
     const transactions = await bitcoinApi.$getRawMempool();
     const diff = transactions.length - currentMempoolSize;
     const newTransactions: TransactionExtended[] = [];
@@ -124,7 +137,6 @@ class Mempool {
         try {
           const transaction = await transactionUtils.$getTransactionExtended(txid);
           this.mempoolCache[txid] = transaction;
-          txCount++;
           if (this.inSync) {
             this.txPerSecondArray.push(new Date().getTime());
             this.vBytesPerSecondArray.push({
@@ -133,14 +145,9 @@ class Mempool {
             });
           }
           hasChange = true;
-          if (diff > 0) {
-            logger.debug('Fetched transaction ' + txCount + ' / ' + diff);
-          } else {
-            logger.debug('Fetched transaction ' + txCount);
-          }
           newTransactions.push(transaction);
         } catch (e) {
-          logger.debug('Error finding transaction in mempool: ' + (e instanceof Error ? e.message : e));
+          logger.debug(`Error finding transaction '${txid}' in the mempool: ` + (e instanceof Error ? e.message : e));
         }
       }
 
@@ -194,25 +201,27 @@ class Mempool {
     if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) {
       this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions);
     }
+    if (this.asyncMempoolChangedCallback && (hasChange || deletedTransactions.length)) {
+      await this.asyncMempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions);
+    }
 
     const end = new Date().getTime();
     const time = end - start;
-    logger.debug(`New mempool size: ${Object.keys(this.mempoolCache).length} Change: ${diff}`);
+    logger.debug(`Mempool updated in ${time / 1000} seconds. New size: ${Object.keys(this.mempoolCache).length} (${diff > 0 ? '+' + diff : diff})`);
-    logger.debug('Mempool updated in ' + time / 1000 + ' seconds');
   }
 
   public handleRbfTransactions(rbfTransactions: { [txid: string]: TransactionExtended; }) {
     for (const rbfTransaction in rbfTransactions) {
       if (this.mempoolCache[rbfTransaction]) {
         // Store replaced transactions
-        rbfCache.add(rbfTransaction, rbfTransactions[rbfTransaction].txid);
+        rbfCache.add(this.mempoolCache[rbfTransaction], rbfTransactions[rbfTransaction].txid);
         // Erase the replaced transactions from the local mempool
         delete this.mempoolCache[rbfTransaction];
       }
     }
   }
 
-  private updateTxPerSecond() {
+  protected updateTxPerSecond() {
     const nowMinusTimeSpan = new Date().getTime() - (1000 * config.STATISTICS.TX_PER_SECOND_SAMPLE_PERIOD);
     this.txPerSecondArray = this.txPerSecondArray.filter((unixTime) => unixTime > nowMinusTimeSpan);
     this.txPerSecond = this.txPerSecondArray.length / config.STATISTICS.TX_PER_SECOND_SAMPLE_PERIOD || 0;
@@ -225,17 +234,18 @@ class Mempool {
     }
   }
 
-  private deleteExpiredTransactions() {
+  protected deleteExpiredTransactions() {
     const now = new Date().getTime();
     for (const tx in this.mempoolCache) {
       const lazyDeleteAt = this.mempoolCache[tx].deleteAfter;
       if (lazyDeleteAt && lazyDeleteAt < now) {
         delete this.mempoolCache[tx];
+        rbfCache.evict(tx);
       }
     }
   }
 
-  private $getMempoolInfo() {
+  protected $getMempoolInfo() {
     if (config.MEMPOOL.USE_SECOND_NODE_FOR_MINFEE) {
       return Promise.all([
         bitcoinClient.getMempoolInfo(),

backend/src/api/mining/mining-routes.ts

@@ -1,6 +1,7 @@
 import { Application, Request, Response } from 'express';
 import config from "../../config";
 import logger from '../../logger';
+import audits from '../audit';
 import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository';
 import BlocksRepository from '../../repositories/BlocksRepository';
 import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjustmentsRepository';
@@ -26,7 +27,11 @@ class MiningRoutes {
       .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/sizes-weights/:interval', this.$getHistoricalBlockSizeAndWeight)
       .get(config.MEMPOOL.API_URL_PREFIX + 'mining/difficulty-adjustments/:interval', this.$getDifficultyAdjustments)
       .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/predictions/:interval', this.$getHistoricalBlockPrediction)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/scores', this.$getBlockAuditScores)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/scores/:height', this.$getBlockAuditScores)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/score/:hash', this.$getBlockAuditScore)
       .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/:hash', this.$getBlockAudit)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/timestamp/:timestamp', this.$getHeightFromTimestamp)
     ;
   }
 
@@ -238,6 +243,12 @@ class MiningRoutes {
   public async $getBlockAudit(req: Request, res: Response) {
     try {
       const audit = await BlocksAuditsRepository.$getBlockAudit(req.params.hash);
+
+      if (!audit) {
+        res.status(404).send(`This block has not been audited.`);
+        return;
+      }
+
       res.header('Pragma', 'public');
       res.header('Cache-control', 'public');
       res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24).toUTCString());
@@ -246,6 +257,55 @@ class MiningRoutes {
       res.status(500).send(e instanceof Error ? e.message : e);
     }
   }
+
+  private async $getHeightFromTimestamp(req: Request, res: Response) {
+    try {
+      const timestamp = parseInt(req.params.timestamp, 10);
+      // This will prevent people from entering milliseconds etc.
+      // Block timestamps are allowed to be up to 2 hours off, so 24 hours
+      // will never put the maximum value before the most recent block
+      const nowPlus1day = Math.floor(Date.now() / 1000) + 60 * 60 * 24;
+      // Prevent non-integers that are not seconds
+      if (!/^[1-9][0-9]*$/.test(req.params.timestamp) || timestamp > nowPlus1day) {
+        throw new Error(`Invalid timestamp, value must be Unix seconds`);
+      }
+      const result = await BlocksRepository.$getBlockHeightFromTimestamp(
+        timestamp,
+      );
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
+      res.json(result);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getBlockAuditScores(req: Request, res: Response) {
+    try {
+      let height = req.params.height === undefined ? undefined : parseInt(req.params.height, 10);
+      if (height == null) {
+        height = await BlocksRepository.$mostRecentBlockHeight();
+      }
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(await BlocksAuditsRepository.$getBlockAuditScores(height, height - 15));
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  public async $getBlockAuditScore(req: Request, res: Response) {
+    try {
+      const audit = await BlocksAuditsRepository.$getBlockAuditScore(req.params.hash);
+
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24).toUTCString());
+      res.json(audit || 'null');
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
 }
 
 export default new MiningRoutes();

backend/src/api/mining/mining.ts

@@ -265,9 +265,9 @@ class Mining {
       }
       await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', new Date().getUTCDate());
       if (newlyIndexed > 0) {
-        logger.notice(`Weekly mining pools hashrates indexing completed: indexed ${newlyIndexed}`);
+        logger.notice(`Weekly mining pools hashrates indexing completed: indexed ${newlyIndexed}`, logger.tags.mining);
       } else {
-        logger.debug(`Weekly mining pools hashrates indexing completed: indexed ${newlyIndexed}`);
+        logger.debug(`Weekly mining pools hashrates indexing completed: indexed ${newlyIndexed}`, logger.tags.mining);
       }
       loadingIndicators.setProgress('weekly-hashrate-indexing', 100);
     } catch (e) {
@@ -370,14 +370,14 @@ class Mining {
 
       await HashratesRepository.$setLatestRun('last_hashrates_indexing', new Date().getUTCDate());
       if (newlyIndexed > 0) {
-        logger.notice(`Daily network hashrate indexing completed: indexed ${newlyIndexed} days`);
+        logger.notice(`Daily network hashrate indexing completed: indexed ${newlyIndexed} days`, logger.tags.mining);
       } else {
-        logger.debug(`Daily network hashrate indexing completed: indexed ${newlyIndexed} days`);
+        logger.debug(`Daily network hashrate indexing completed: indexed ${newlyIndexed} days`, logger.tags.mining);
       }
       loadingIndicators.setProgress('daily-hashrate-indexing', 100);
     } catch (e) {
       loadingIndicators.setProgress('daily-hashrate-indexing', 100);
-      logger.err(`Daily network hashrate indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`);
+      logger.err(`Daily network hashrate indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`, logger.tags.mining);
       throw e;
     }
   }
@@ -449,9 +449,9 @@ class Mining {
     }
 
     if (totalIndexed > 0) {
-      logger.notice(`Indexed ${totalIndexed} difficulty adjustments`);
+      logger.notice(`Indexed ${totalIndexed} difficulty adjustments`, logger.tags.mining);
     } else {
-      logger.debug(`Indexed ${totalIndexed} difficulty adjustments`);
+      logger.debug(`Indexed ${totalIndexed} difficulty adjustments`, logger.tags.mining);
     }
   }
 

backend/src/api/pools-parser.ts

@@ -14,10 +14,10 @@ interface Pool {
 class PoolsParser {
   miningPools: any[] = [];
   unknownPool: any = {
-    'name': "Unknown",
+    'name': 'Unknown',
-    'link': "https://learnmeabitcoin.com/technical/coinbase-transaction",
+    'link': 'https://learnmeabitcoin.com/technical/coinbase-transaction',
-    'regexes': "[]",
+    'regexes': '[]',
-    'addresses': "[]",
+    'addresses': '[]',
     'slug': 'unknown'
   };
   slugWarnFlag = false;
@@ -25,7 +25,7 @@ class PoolsParser {
   /**
    * Parse the pools.json file, consolidate the data and dump it into the database
    */
-  public async migratePoolsJson(poolsJson: object) {
+  public async migratePoolsJson(poolsJson: object): Promise<void> {
     if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
       return;
     }
@@ -61,7 +61,7 @@ class PoolsParser {
         poolNames.push(poolsDuplicated[i].name);
       }
     }
-    logger.debug(`Found ${poolNames.length} unique mining pools`);
+    logger.debug(`Found ${poolNames.length} unique mining pools`, logger.tags.mining);
 
     // Get existing pools from the db
     let existingPools;
@@ -72,7 +72,7 @@ class PoolsParser {
         existingPools = [];
       }
     } catch (e) {
-      logger.err('Cannot get existing pools from the database, skipping pools.json import');
+      logger.err('Cannot get existing pools from the database, skipping pools.json import', logger.tags.mining);
       return;
     }
 
@@ -81,6 +81,7 @@ class PoolsParser {
     // Finally, we generate the final consolidated pools data
     const finalPoolDataAdd: Pool[] = [];
     const finalPoolDataUpdate: Pool[] = [];
+    const finalPoolDataRename: Pool[] = [];
     for (let i = 0; i < poolNames.length; ++i) {
       let allAddresses: string[] = [];
       let allRegexes: string[] = [];
@@ -98,7 +99,7 @@ class PoolsParser {
         slug = poolsJson['slugs'][poolNames[i]];
       } catch (e) {
         if (this.slugWarnFlag === false) {
-          logger.warn(`pools.json does not seem to contain the 'slugs' object`);
+          logger.warn(`pools.json does not seem to contain the 'slugs' object`, logger.tags.mining);
           this.slugWarnFlag = true;
         }
       }
@@ -106,7 +107,7 @@ class PoolsParser {
       if (slug === undefined) {
         // Only keep alphanumerical
         slug = poolNames[i].replace(/[^a-z0-9]/gi, '').toLowerCase();
-        logger.warn(`No slug found for '${poolNames[i]}', generating it => '${slug}'`);
+        logger.warn(`No slug found for '${poolNames[i]}', generating it => '${slug}'`, logger.tags.mining);
       }
 
       const poolObj = {
@@ -127,8 +128,26 @@ class PoolsParser {
           finalPoolDataUpdate.push(poolObj);
         }
       } else {
-        logger.debug(`Add '${finalPoolName}' mining pool`);
+        // Double check that if we're not just renaming a pool (same address same regex)
-        finalPoolDataAdd.push(poolObj);
+        const [poolToRename]: any[] = await DB.query(`
+          SELECT * FROM pools
+          WHERE addresses = ? OR regexes = ?`,
+          [JSON.stringify(poolObj.addresses), JSON.stringify(poolObj.regexes)]
+        );
+        if (poolToRename && poolToRename.length > 0) {
+          // We're actually renaming an existing pool
+          finalPoolDataRename.push({
+            'name': poolObj.name,
+            'link': poolObj.link,
+            'regexes': allRegexes,
+            'addresses': allAddresses,
+            'slug': slug
+          });
+          logger.debug(`Rename '${poolToRename[0].name}' mining pool to ${poolObj.name}`, logger.tags.mining);
+        } else {
+          logger.debug(`Add '${finalPoolName}' mining pool`, logger.tags.mining);
+          finalPoolDataAdd.push(poolObj);
+        }
       }
 
       this.miningPools.push({
@@ -141,12 +160,14 @@ class PoolsParser {
     }
 
     if (config.DATABASE.ENABLED === false) { // Don't run db operations
-      logger.info('Mining pools.json import completed (no database)');
+      logger.info('Mining pools.json import completed (no database)', logger.tags.mining);
       return;
     }
 
-    if (finalPoolDataAdd.length > 0 || finalPoolDataUpdate.length > 0) {    
+    if (finalPoolDataAdd.length > 0 || finalPoolDataUpdate.length > 0 ||
-      logger.debug(`Update pools table now`);
+      finalPoolDataRename.length > 0
+    ) {    
+      logger.debug(`Update pools table now`, logger.tags.mining);
 
       // Add new mining pools into the database
       let queryAdd: string = 'INSERT INTO pools(name, link, regexes, addresses, slug) VALUES ';
@@ -169,8 +190,22 @@ class PoolsParser {
         ;`);
       }
 
+      // Rename mining pools
+      const renameQueries: string[] = [];
+      for (let i = 0; i < finalPoolDataRename.length; ++i) {
+        renameQueries.push(`
+          UPDATE pools
+          SET name='${finalPoolDataRename[i].name}', link='${finalPoolDataRename[i].link}',
+            slug='${finalPoolDataRename[i].slug}'
+          WHERE regexes='${JSON.stringify(finalPoolDataRename[i].regexes)}'
+            AND addresses='${JSON.stringify(finalPoolDataRename[i].addresses)}'
+        ;`);
+      }
+
       try {
-        await this.$deleteBlocskToReindex(finalPoolDataUpdate);
+        if (finalPoolDataAdd.length > 0 || updateQueries.length > 0) {
+          await this.$deleteBlocskToReindex(finalPoolDataUpdate);
+        }
 
         if (finalPoolDataAdd.length > 0) {
           await DB.query({ sql: queryAdd, timeout: 120000 });
@@ -178,10 +213,13 @@ class PoolsParser {
         for (const query of updateQueries) {
           await DB.query({ sql: query, timeout: 120000 });
         }
+        for (const query of renameQueries) {
+          await DB.query({ sql: query, timeout: 120000 });
+        }
         await this.insertUnknownPool();
-        logger.info('Mining pools.json import completed');
+        logger.info('Mining pools.json import completed', logger.tags.mining);
       } catch (e) {
-        logger.err(`Cannot import pools in the database`);
+        logger.err(`Cannot import pools in the database`, logger.tags.mining);
         throw e;
       }
     }
@@ -189,7 +227,7 @@ class PoolsParser {
     try {
       await this.insertUnknownPool();
     } catch (e) {
-      logger.err(`Cannot insert unknown pool in the database`);
+      logger.err(`Cannot insert unknown pool in the database`, logger.tags.mining);
       throw e;
     }
   }
@@ -214,7 +252,7 @@ class PoolsParser {
         `);
       }
     } catch (e) {
-      logger.err('Unable to insert "Unknown" mining pool');
+      logger.err('Unable to insert "Unknown" mining pool', logger.tags.mining);
     }
   }
 
@@ -234,17 +272,17 @@ class PoolsParser {
     for (const updatedPool of finalPoolDataUpdate) {
       const [pool]: any[] = await DB.query(`SELECT id, name from pools where slug = "${updatedPool.slug}"`);
       if (pool.length > 0) {
-        logger.notice(`Deleting blocks from ${pool[0].name} mining pool for future re-indexing`);
+        logger.notice(`Deleting blocks from ${pool[0].name} mining pool for future re-indexing`, logger.tags.mining);
         await DB.query(`DELETE FROM blocks WHERE pool_id = ${pool[0].id}`);
       }
     }
 
     // Ignore early days of Bitcoin as there were not mining pool yet
-    logger.notice('Deleting blocks with unknown mining pool from height 130635 for future re-indexing');
+    logger.notice(`Deleting blocks with unknown mining pool from height 130635 for future re-indexing`, logger.tags.mining);
     const [unknownPool] = await DB.query(`SELECT id from pools where slug = "unknown"`);
     await DB.query(`DELETE FROM blocks WHERE pool_id = ${unknownPool[0].id} AND height > 130635`);
 
-    logger.notice('Truncating hashrates for future re-indexing');
+    logger.notice(`Truncating hashrates for future re-indexing`, logger.tags.mining);
     await DB.query(`DELETE FROM hashrates`);
   }
 }

backend/src/api/rbf-cache.ts

@@ -1,31 +1,62 @@
-export interface CachedRbf {
+import { TransactionExtended } from "../mempool.interfaces";
-  txid: string;
-  expires: Date;
-}
 
 class RbfCache {
-  private cache: { [txid: string]: CachedRbf; } = {};
+  private replacedBy: { [txid: string]: string; } = {};
+  private replaces: { [txid: string]: string[] } = {};
+  private txs: { [txid: string]: TransactionExtended } = {};
+  private expiring: { [txid: string]: Date } = {};
 
   constructor() {
     setInterval(this.cleanup.bind(this), 1000 * 60 * 60);
   }
 
-  public add(replacedTxId: string, newTxId: string): void {
+  public add(replacedTx: TransactionExtended, newTxId: string): void {
-    this.cache[replacedTxId] = {
+    this.replacedBy[replacedTx.txid] = newTxId;
-      expires: new Date(Date.now() + 1000 * 604800), // 1 week
+    this.txs[replacedTx.txid] = replacedTx;
-      txid: newTxId,
+    if (!this.replaces[newTxId]) {
-    };
+      this.replaces[newTxId] = [];
+    }
+    this.replaces[newTxId].push(replacedTx.txid);
   }
 
-  public get(txId: string): CachedRbf | undefined {
+  public getReplacedBy(txId: string): string | undefined {
-    return this.cache[txId];
+    return this.replacedBy[txId];
+  }
+
+  public getReplaces(txId: string): string[] | undefined {
+    return this.replaces[txId];
+  }
+
+  public getTx(txId: string): TransactionExtended | undefined {
+    return this.txs[txId];
+  }
+
+  // flag a transaction as removed from the mempool
+  public evict(txid): void {
+    this.expiring[txid] = new Date(Date.now() + 1000 * 86400); // 24 hours
   }
 
   private cleanup(): void {
     const currentDate = new Date();
-    for (const c in this.cache) {
+    for (const txid in this.expiring) {
-      if (this.cache[c].expires < currentDate) {
+      if (this.expiring[txid] < currentDate) {
-        delete this.cache[c];
+        delete this.expiring[txid];
+        this.remove(txid);
+      }
+    }
+  }
+
+  // remove a transaction & all previous versions from the cache
+  private remove(txid): void {
+    // don't remove a transaction while a newer version remains in the mempool
+    if (this.replaces[txid] && !this.replacedBy[txid]) {
+      const replaces = this.replaces[txid];
+      delete this.replaces[txid];
+      for (const tx of replaces) {
+        // recursively remove prior versions from the cache
+        delete this.replacedBy[tx];
+        delete this.txs[tx];
+        this.remove(tx);
       }
     }
   }

backend/src/api/transaction-utils.ts

@@ -1,8 +1,7 @@
-import bitcoinApi from './bitcoin/bitcoin-api-factory';
 import { TransactionExtended, TransactionMinerInfo } from '../mempool.interfaces';
 import { IEsploraApi } from './bitcoin/esplora-api.interface';
-import config from '../config';
 import { Common } from './common';
+import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory';
 
 class TransactionUtils {
   constructor() { }
@@ -21,8 +20,19 @@ class TransactionUtils {
     };
   }
 
-  public async $getTransactionExtended(txId: string, addPrevouts = false, lazyPrevouts = false): Promise<TransactionExtended> {
+  /**
-    const transaction: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(txId, false, addPrevouts, lazyPrevouts);
+   * @param txId 
+   * @param addPrevouts 
+   * @param lazyPrevouts 
+   * @param forceCore - See https://github.com/mempool/mempool/issues/2904
+   */
+  public async $getTransactionExtended(txId: string, addPrevouts = false, lazyPrevouts = false, forceCore = false): Promise<TransactionExtended> {
+    let transaction: IEsploraApi.Transaction;
+    if (forceCore === true) {
+      transaction  = await bitcoinCoreApi.$getRawTransaction(txId, true);
+    } else {
+      transaction  = await bitcoinApi.$getRawTransaction(txId, false, addPrevouts, lazyPrevouts);
+    }
     return this.extendTransaction(transaction);
   }
 

backend/src/api/tx-selection-worker.ts

@@ -0,0 +1,294 @@
+import config from '../config';
+import logger from '../logger';
+import { ThreadTransaction, MempoolBlockWithTransactions, AuditTransaction } from '../mempool.interfaces';
+import { PairingHeap } from '../utils/pairing-heap';
+import { Common } from './common';
+import { parentPort } from 'worker_threads';
+
+let mempool: { [txid: string]: ThreadTransaction } = {};
+
+if (parentPort) {
+  parentPort.on('message', (params) => {
+    if (params.type === 'set') {
+      mempool = params.mempool;
+    } else if (params.type === 'update') {
+      params.added.forEach(tx => {
+        mempool[tx.txid] = tx;
+      });
+      params.removed.forEach(txid => {
+        delete mempool[txid];
+      });
+    }
+    
+    const { blocks, clusters } = makeBlockTemplates(mempool);
+
+    // return the result to main thread.
+    if (parentPort) {
+     parentPort.postMessage({ blocks, clusters });
+    }
+  });
+}
+
+/*
+* Build projected mempool blocks using an approximation of the transaction selection algorithm from Bitcoin Core
+* (see BlockAssembler in https://github.com/bitcoin/bitcoin/blob/master/src/node/miner.cpp)
+*/
+function makeBlockTemplates(mempool: { [txid: string]: ThreadTransaction })
+  : { blocks: ThreadTransaction[][], clusters: { [root: string]: string[] } } {
+  const start = Date.now();
+  const auditPool: { [txid: string]: AuditTransaction } = {};
+  const mempoolArray: AuditTransaction[] = [];
+  const restOfArray: ThreadTransaction[] = [];
+  const cpfpClusters: { [root: string]: string[] } = {};
+  
+  // grab the top feerate txs up to maxWeight
+  Object.values(mempool).sort((a, b) => b.feePerVsize - a.feePerVsize).forEach(tx => {
+    // initializing everything up front helps V8 optimize property access later
+    auditPool[tx.txid] = {
+      txid: tx.txid,
+      fee: tx.fee,
+      weight: tx.weight,
+      feePerVsize: tx.feePerVsize,
+      effectiveFeePerVsize: tx.feePerVsize,
+      vin: tx.vin,
+      relativesSet: false,
+      ancestorMap: new Map<string, AuditTransaction>(),
+      children: new Set<AuditTransaction>(),
+      ancestorFee: 0,
+      ancestorWeight: 0,
+      score: 0,
+      used: false,
+      modified: false,
+      modifiedNode: null,
+    };
+    mempoolArray.push(auditPool[tx.txid]);
+  });
+
+  // Build relatives graph & calculate ancestor scores
+  for (const tx of mempoolArray) {
+    if (!tx.relativesSet) {
+      setRelatives(tx, auditPool);
+    }
+  }
+
+  // Sort by descending ancestor score
+  mempoolArray.sort((a, b) => (b.score || 0) - (a.score || 0));
+
+  // Build blocks by greedily choosing the highest feerate package
+  // (i.e. the package rooted in the transaction with the best ancestor score)
+  const blocks: ThreadTransaction[][] = [];
+  let blockWeight = 4000;
+  let blockSize = 0;
+  let transactions: AuditTransaction[] = [];
+  const modified: PairingHeap<AuditTransaction> = new PairingHeap((a, b): boolean => (a.score || 0) > (b.score || 0));
+  let overflow: AuditTransaction[] = [];
+  let failures = 0;
+  let top = 0;
+  while ((top < mempoolArray.length || !modified.isEmpty())) {
+    // skip invalid transactions
+    while (top < mempoolArray.length && (mempoolArray[top].used || mempoolArray[top].modified)) {
+      top++;
+    }
+
+    // Select best next package
+    let nextTx;
+    const nextPoolTx = mempoolArray[top];
+    const nextModifiedTx = modified.peek();
+    if (nextPoolTx && (!nextModifiedTx || (nextPoolTx.score || 0) > (nextModifiedTx.score || 0))) {
+      nextTx = nextPoolTx;
+      top++;
+    } else {
+      modified.pop();
+      if (nextModifiedTx) {
+        nextTx = nextModifiedTx;
+        nextTx.modifiedNode = undefined;
+      }
+    }
+
+    if (nextTx && !nextTx?.used) {
+      // Check if the package fits into this block
+      if (blockWeight + nextTx.ancestorWeight < config.MEMPOOL.BLOCK_WEIGHT_UNITS) {
+        const ancestors: AuditTransaction[] = Array.from(nextTx.ancestorMap.values());
+        // sort ancestors by dependency graph (equivalent to sorting by ascending ancestor count)
+        const sortedTxSet = [...ancestors.sort((a, b) => { return (a.ancestorMap.size || 0) - (b.ancestorMap.size || 0); }), nextTx];
+        let isCluster = false;
+        if (sortedTxSet.length > 1) {
+          cpfpClusters[nextTx.txid] = sortedTxSet.map(tx => tx.txid);
+          isCluster = true;
+        }
+        const effectiveFeeRate = nextTx.ancestorFee / (nextTx.ancestorWeight / 4);
+        const used: AuditTransaction[] = [];
+        while (sortedTxSet.length) {
+          const ancestor = sortedTxSet.pop();
+          const mempoolTx = mempool[ancestor.txid];
+          ancestor.used = true;
+          ancestor.usedBy = nextTx.txid;
+          // update original copy of this tx with effective fee rate & relatives data
+          mempoolTx.effectiveFeePerVsize = effectiveFeeRate;
+          if (isCluster) {
+            mempoolTx.cpfpRoot = nextTx.txid;
+          }
+          mempoolTx.cpfpChecked = true;
+          transactions.push(ancestor);
+          blockSize += ancestor.size;
+          blockWeight += ancestor.weight;
+          used.push(ancestor);
+        }
+
+        // remove these as valid package ancestors for any descendants remaining in the mempool
+        if (used.length) {
+          used.forEach(tx => {
+            updateDescendants(tx, auditPool, modified);
+          });
+        }
+
+        failures = 0;
+      } else {
+        // hold this package in an overflow list while we check for smaller options
+        overflow.push(nextTx);
+        failures++;
+      }
+    }
+
+    // this block is full
+    const exceededPackageTries = failures > 1000 && blockWeight > (config.MEMPOOL.BLOCK_WEIGHT_UNITS - 4000);
+    const queueEmpty = top >= mempoolArray.length && modified.isEmpty();
+    if ((exceededPackageTries || queueEmpty) && blocks.length < 7) {
+      // construct this block
+      if (transactions.length) {
+        blocks.push(transactions.map(t => mempool[t.txid]));
+      }
+      // reset for the next block
+      transactions = [];
+      blockSize = 0;
+      blockWeight = 4000;
+
+      // 'overflow' packages didn't fit in this block, but are valid candidates for the next
+      for (const overflowTx of overflow.reverse()) {
+        if (overflowTx.modified) {
+          overflowTx.modifiedNode = modified.add(overflowTx);
+        } else {
+          top--;
+          mempoolArray[top] = overflowTx;
+        }
+      }
+      overflow = [];
+    }
+  }
+  // pack any leftover transactions into the last block
+  for (const tx of overflow) {
+    if (!tx || tx?.used) {
+      continue;
+    }
+    blockWeight += tx.weight;
+    const mempoolTx = mempool[tx.txid];
+    // update original copy of this tx with effective fee rate & relatives data
+    mempoolTx.effectiveFeePerVsize = tx.score;
+    if (tx.ancestorMap.size > 0) {
+      cpfpClusters[tx.txid] = Array.from(tx.ancestorMap?.values()).map(a => a.txid);
+      mempoolTx.cpfpRoot = tx.txid;
+    }
+    mempoolTx.cpfpChecked = true;
+    transactions.push(tx);
+    tx.used = true;
+  }
+  const blockTransactions = transactions.map(t => mempool[t.txid]);
+  restOfArray.forEach(tx => {
+    blockWeight += tx.weight;
+    tx.effectiveFeePerVsize = tx.feePerVsize;
+    tx.cpfpChecked = false;
+    blockTransactions.push(tx);
+  });
+  if (blockTransactions.length) {
+    blocks.push(blockTransactions);
+  }
+  transactions = [];
+
+  const end = Date.now();
+  const time = end - start;
+  logger.debug('Mempool templates calculated in ' + time / 1000 + ' seconds');
+
+  return { blocks, clusters: cpfpClusters };
+}
+
+// traverse in-mempool ancestors
+// recursion unavoidable, but should be limited to depth < 25 by mempool policy
+function setRelatives(
+  tx: AuditTransaction,
+  mempool: { [txid: string]: AuditTransaction },
+): void {
+  for (const parent of tx.vin) {
+    const parentTx = mempool[parent];
+    if (parentTx && !tx.ancestorMap?.has(parent)) {
+      tx.ancestorMap.set(parent, parentTx);
+      parentTx.children.add(tx);
+      // visit each node only once
+      if (!parentTx.relativesSet) {
+        setRelatives(parentTx, mempool);
+      }
+      parentTx.ancestorMap.forEach((ancestor) => {
+        tx.ancestorMap.set(ancestor.txid, ancestor);
+      });
+    }
+  };
+  tx.ancestorFee = tx.fee || 0;
+  tx.ancestorWeight = tx.weight || 0;
+  tx.ancestorMap.forEach((ancestor) => {
+    tx.ancestorFee += ancestor.fee;
+    tx.ancestorWeight += ancestor.weight;
+  });
+  tx.score = tx.ancestorFee / ((tx.ancestorWeight / 4) || 1);
+  tx.relativesSet = true;
+}
+
+// iterate over remaining descendants, removing the root as a valid ancestor & updating the ancestor score
+// avoids recursion to limit call stack depth
+function updateDescendants(
+  rootTx: AuditTransaction,
+  mempool: { [txid: string]: AuditTransaction },
+  modified: PairingHeap<AuditTransaction>,
+): void {
+  const descendantSet: Set<AuditTransaction> = new Set();
+  // stack of nodes left to visit
+  const descendants: AuditTransaction[] = [];
+  let descendantTx;
+  let tmpScore;
+  rootTx.children.forEach(childTx => {
+    if (!descendantSet.has(childTx)) {
+      descendants.push(childTx);
+      descendantSet.add(childTx);
+    }
+  });
+  while (descendants.length) {
+    descendantTx = descendants.pop();
+    if (descendantTx && descendantTx.ancestorMap && descendantTx.ancestorMap.has(rootTx.txid)) {
+      // remove tx as ancestor
+      descendantTx.ancestorMap.delete(rootTx.txid);
+      descendantTx.ancestorFee -= rootTx.fee;
+      descendantTx.ancestorWeight -= rootTx.weight;
+      tmpScore = descendantTx.score;
+      descendantTx.score = descendantTx.ancestorFee / (descendantTx.ancestorWeight / 4);
+
+      if (!descendantTx.modifiedNode) {
+        descendantTx.modified = true;
+        descendantTx.modifiedNode = modified.add(descendantTx);
+      } else {
+        // rebalance modified heap if score has changed
+        if (descendantTx.score < tmpScore) {
+          modified.decreasePriority(descendantTx.modifiedNode);
+        } else if (descendantTx.score > tmpScore) {
+          modified.increasePriority(descendantTx.modifiedNode);
+        }
+      }
+
+      // add this node's children to the stack
+      descendantTx.children.forEach(childTx => {
+        // visit each node only once
+        if (!descendantSet.has(childTx)) {
+          descendants.push(childTx);
+          descendantSet.add(childTx);
+        }
+      });
+    }
+  }
+}

backend/src/api/websocket-handler.ts

@@ -18,6 +18,7 @@ import difficultyAdjustment from './difficulty-adjustment';
 import feeApi from './fee-api';
 import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository';
 import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
+import Audit from './audit';
 
 class WebsocketHandler {
   private wss: WebSocket.Server | undefined;
@@ -57,10 +58,10 @@ class WebsocketHandler {
               client['track-tx'] = parsedMessage['track-tx'];
               // Client is telling the transaction wasn't found
               if (parsedMessage['watch-mempool']) {
-                const rbfCacheTx = rbfCache.get(client['track-tx']);
+                const rbfCacheTxid = rbfCache.getReplacedBy(client['track-tx']);
-                if (rbfCacheTx) {
+                if (rbfCacheTxid) {
                   response['txReplaced'] = {
-                    txid: rbfCacheTx.txid,
+                    txid: rbfCacheTxid,
                   };
                   client['track-tx'] = null;
                 } else {
@@ -243,13 +244,18 @@ class WebsocketHandler {
     });
   }
 
-  handleMempoolChange(newMempool: { [txid: string]: TransactionExtended },
+  async handleMempoolChange(newMempool: { [txid: string]: TransactionExtended },
-    newTransactions: TransactionExtended[], deletedTransactions: TransactionExtended[]) {
+    newTransactions: TransactionExtended[], deletedTransactions: TransactionExtended[]): Promise<void> {
     if (!this.wss) {
       throw new Error('WebSocket.Server is not set');
     }
 
-    mempoolBlocks.updateMempoolBlocks(newMempool);
+    if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) {
+      await mempoolBlocks.updateBlockTemplates(newMempool, newTransactions, deletedTransactions.map(tx => tx.txid));
+    } else {
+      mempoolBlocks.updateMempoolBlocks(newMempool);
+    }
+
     const mBlocks = mempoolBlocks.getMempoolBlocks();
     const mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();
     const mempoolInfo = memPool.getMempoolInfo();
@@ -404,76 +410,74 @@ class WebsocketHandler {
       }
     });
   }
-
+ 
-  handleNewBlock(block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) {
+  async handleNewBlock(block: BlockExtended, txIds: string[], transactions: TransactionExtended[]): Promise<void> {
     if (!this.wss) {
       throw new Error('WebSocket.Server is not set');
     }
 
-    let mBlocks: undefined | MempoolBlock[];
-    let mBlockDeltas: undefined | MempoolBlockDelta[];
-    let matchRate = 0;
     const _memPool = memPool.getMempool();
-    const _mempoolBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
 
-    if (_mempoolBlocks[0]) {
+    if (config.MEMPOOL.ADVANCED_GBT_AUDIT) {
-      const matches: string[] = [];
+      await mempoolBlocks.makeBlockTemplates(_memPool);
-      const added: string[] = [];
+    } else {
-      const missing: string[] = [];
-
-      for (const txId of txIds) {
-        if (_mempoolBlocks[0].transactionIds.indexOf(txId) > -1) {
-          matches.push(txId);
-        } else {
-          added.push(txId);
-        }
-        delete _memPool[txId];
-      }
-
-      for (const txId of _mempoolBlocks[0].transactionIds) {
-        if (matches.includes(txId) || added.includes(txId)) {
-          continue;
-        }
-        missing.push(txId);
-      }
-
-      matchRate = Math.round((Math.max(0, matches.length - missing.length - added.length) / txIds.length * 100) * 100) / 100;
       mempoolBlocks.updateMempoolBlocks(_memPool);
-      mBlocks = mempoolBlocks.getMempoolBlocks();
+    }
-      mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();
 
-      if (Common.indexingEnabled()) {
+    if (Common.indexingEnabled() && memPool.isInSync()) {
-        const stripped = _mempoolBlocks[0].transactions.map((tx) => {
+      const projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
-          return {
-            txid: tx.txid,
-            vsize: tx.vsize,
-            fee: tx.fee ? Math.round(tx.fee) : 0,
-            value: tx.value,
-          };
-        });  
-        BlocksSummariesRepository.$saveSummary({
-          height: block.height,
-          template: {
-            id: block.id,
-            transactions: stripped
-          }
-        });
 
-        BlocksAuditsRepository.$saveAudit({
+      const { censored, added, fresh, score } = Audit.auditBlock(transactions, projectedBlocks, _memPool);
-          time: block.timestamp,
+      const matchRate = Math.round(score * 100 * 100) / 100;
-          height: block.height,
+
-          hash: block.id,
+      const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions.map((tx) => {
-          addedTxs: added,
+        return {
-          missingTxs: missing,
+          txid: tx.txid,
-          matchRate: matchRate,
+          vsize: tx.vsize,
-        });
+          fee: tx.fee ? Math.round(tx.fee) : 0,
+          value: tx.value,
+        };
+      }) : [];
+
+      BlocksSummariesRepository.$saveTemplate({
+        height: block.height,
+        template: {
+          id: block.id,
+          transactions: stripped
+        }
+      });
+
+      BlocksAuditsRepository.$saveAudit({
+        time: block.timestamp,
+        height: block.height,
+        hash: block.id,
+        addedTxs: added,
+        missingTxs: censored,
+        freshTxs: fresh,
+        matchRate: matchRate,
+      });
+
+      if (block.extras) {
+        block.extras.matchRate = matchRate;
       }
     }
 
-    if (block.extras) {
+    const removed: string[] = [];
-      block.extras.matchRate = matchRate;
+    // Update mempool to remove transactions included in the new block
+    for (const txId of txIds) {
+      delete _memPool[txId];
+      removed.push(txId);
+      rbfCache.evict(txId);
     }
 
+    if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) {
+      await mempoolBlocks.updateBlockTemplates(_memPool, [], removed);
+    } else {
+      mempoolBlocks.updateMempoolBlocks(_memPool);
+    }
+    const mBlocks = mempoolBlocks.getMempoolBlocks();
+    const mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();
+
     const da = difficultyAdjustment.getDifficultyAdjustment();
     const fees = feeApi.getRecommendedFee();
 

backend/src/config.ts

@@ -4,6 +4,7 @@ const configFromFile = require(
 
 interface IConfig {
   MEMPOOL: {
+    ENABLED: boolean;
     NETWORK: 'mainnet' | 'testnet' | 'signet' | 'liquid' | 'liquidtestnet';
     BACKEND: 'esplora' | 'electrum' | 'none';
     HTTP_PORT: number;
@@ -28,6 +29,10 @@ interface IConfig {
     AUTOMATIC_BLOCK_REINDEXING: boolean;
     POOLS_JSON_URL: string,
     POOLS_JSON_TREE_URL: string,
+    ADVANCED_GBT_AUDIT: boolean;
+    ADVANCED_GBT_MEMPOOL: boolean;
+    CPFP_INDEXING: boolean;
+    RBF_DUAL_NODE: boolean;
   };
   ESPLORA: {
     REST_API_URL: string;
@@ -39,6 +44,8 @@ interface IConfig {
     STATS_REFRESH_INTERVAL: number;
     GRAPH_REFRESH_INTERVAL: number;
     LOGGER_UPDATE_INTERVAL: number;
+    FORENSICS_INTERVAL: number;
+    FORENSICS_RATE_LIMIT: number;
   };
   LND: {
     TLS_CERT_PATH: string;
@@ -119,6 +126,7 @@ interface IConfig {
 
 const defaults: IConfig = {
   'MEMPOOL': {
+    'ENABLED': true,
     'NETWORK': 'mainnet',
     'BACKEND': 'none',
     'HTTP_PORT': 8999,
@@ -143,6 +151,10 @@ const defaults: IConfig = {
     'AUTOMATIC_BLOCK_REINDEXING': false,
     'POOLS_JSON_URL': 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json',
     'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
+    'ADVANCED_GBT_AUDIT': false,
+    'ADVANCED_GBT_MEMPOOL': false,
+    'CPFP_INDEXING': false,
+    'RBF_DUAL_NODE': false,
   },
   'ESPLORA': {
     'REST_API_URL': 'http://127.0.0.1:3000',
@@ -195,6 +207,8 @@ const defaults: IConfig = {
     'STATS_REFRESH_INTERVAL': 600,
     'GRAPH_REFRESH_INTERVAL': 600,
     'LOGGER_UPDATE_INTERVAL': 30,
+    'FORENSICS_INTERVAL': 43200,
+    'FORENSICS_RATE_LIMIT': 20,
   },
   'LND': {
     'TLS_CERT_PATH': '',
@@ -224,11 +238,11 @@ const defaults: IConfig = {
     'BISQ_URL': 'https://bisq.markets/api',
     'BISQ_ONION': 'http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api'
   },
-  "MAXMIND": {
+  'MAXMIND': {
     'ENABLED': false,
-    "GEOLITE2_CITY": "/usr/local/share/GeoIP/GeoLite2-City.mmdb",
+    'GEOLITE2_CITY': '/usr/local/share/GeoIP/GeoLite2-City.mmdb',
-    "GEOLITE2_ASN": "/usr/local/share/GeoIP/GeoLite2-ASN.mmdb",
+    'GEOLITE2_ASN': '/usr/local/share/GeoIP/GeoLite2-ASN.mmdb',
-    "GEOIP2_ISP": "/usr/local/share/GeoIP/GeoIP2-ISP.mmdb"
+    'GEOIP2_ISP': '/usr/local/share/GeoIP/GeoIP2-ISP.mmdb'
   },
 };
 

backend/src/index.ts

@@ -1,4 +1,4 @@
-import express from "express";
+import express from 'express';
 import { Application, Request, Response, NextFunction } from 'express';
 import * as http from 'http';
 import * as WebSocket from 'ws';
@@ -17,6 +17,7 @@ import logger from './logger';
 import backendInfo from './api/backend-info';
 import loadingIndicators from './api/loading-indicators';
 import mempool from './api/mempool';
+import altMempool from './api/alt-mempool';
 import elementsParser from './api/liquid/elements-parser';
 import databaseMigration from './api/database-migration';
 import syncAssets from './sync-assets';
@@ -34,7 +35,8 @@ import miningRoutes from './api/mining/mining-routes';
 import bisqRoutes from './api/bisq/bisq.routes';
 import liquidRoutes from './api/liquid/liquid.routes';
 import bitcoinRoutes from './api/bitcoin/bitcoin.routes';
-import fundingTxFetcher from "./tasks/lightning/sync-tasks/funding-tx-fetcher";
+import fundingTxFetcher from './tasks/lightning/sync-tasks/funding-tx-fetcher';
+import forensicsService from './tasks/lightning/forensics.service';
 
 class Server {
   private wss: WebSocket.Server | undefined;
@@ -74,7 +76,7 @@ class Server {
     }
   }
 
-  async startServer(worker = false) {
+  async startServer(worker = false): Promise<void> {
     logger.notice(`Starting Mempool Server${worker ? ' (worker)' : ''}... (${backendInfo.getShortCommitHash()})`);
 
     this.app
@@ -83,7 +85,7 @@ class Server {
         next();
       })
       .use(express.urlencoded({ extended: true }))
-      .use(express.text())
+      .use(express.text({ type: ['text/plain', 'application/base64'] }))
       ;
 
     this.server = http.createServer(this.app);
@@ -92,7 +94,9 @@ class Server {
     this.setUpWebsocketHandling();
 
     await syncAssets.syncAssets$();
-    diskCache.loadMempoolCache();
+    if (config.MEMPOOL.ENABLED) {
+      diskCache.loadMempoolCache();
+    }
 
     if (config.DATABASE.ENABLED) {
       await DB.checkDbConnection();
@@ -127,7 +131,10 @@ class Server {
     fiatConversion.startService();
 
     this.setUpHttpApiRoutes();
-    this.runMainUpdateLoop();
+
+    if (config.MEMPOOL.ENABLED) {
+      this.runMainUpdateLoop();
+    }
 
     if (config.BISQ.ENABLED) {
       bisq.startBisqService();
@@ -149,7 +156,7 @@ class Server {
     });
   }
 
-  async runMainUpdateLoop() {
+  async runMainUpdateLoop(): Promise<void> {
     try {
       try {
         await memPool.$updateMemPoolInfo();
@@ -164,6 +171,9 @@ class Server {
       await poolsUpdater.updatePoolsJson();
       await blocks.$updateBlocks();
       await memPool.$updateMempool();
+      if (config.MEMPOOL.RBF_DUAL_NODE) {
+        await altMempool.$updateMempool();
+      }
       indexer.$run();
 
       setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS);
@@ -183,10 +193,11 @@ class Server {
     }
   }
 
-  async $runLightningBackend() {
+  async $runLightningBackend(): Promise<void> {
     try {
       await fundingTxFetcher.$init();
       await networkSyncService.$startService();
+      await forensicsService.$startService();
       await lightningStatsUpdater.$startService();
     } catch(e) {
       logger.err(`Nodejs lightning backend crashed. Restarting in 1 minute. Reason: ${(e instanceof Error ? e.message : e)}`);
@@ -195,7 +206,7 @@ class Server {
     };
 }
 
-  setUpWebsocketHandling() {
+  setUpWebsocketHandling(): void {
     if (this.wss) {
       websocketHandler.setWebsocketServer(this.wss);
     }
@@ -209,19 +220,21 @@ class Server {
       });
     }
     websocketHandler.setupConnectionHandling();
-    statistics.setNewStatisticsEntryCallback(websocketHandler.handleNewStatistic.bind(websocketHandler));
+    if (config.MEMPOOL.ENABLED) {
-    blocks.setNewBlockCallback(websocketHandler.handleNewBlock.bind(websocketHandler));
+      statistics.setNewStatisticsEntryCallback(websocketHandler.handleNewStatistic.bind(websocketHandler));
-    memPool.setMempoolChangedCallback(websocketHandler.handleMempoolChange.bind(websocketHandler));
+      memPool.setAsyncMempoolChangedCallback(websocketHandler.handleMempoolChange.bind(websocketHandler));
+      blocks.setNewAsyncBlockCallback(websocketHandler.handleNewBlock.bind(websocketHandler));
+    }
     fiatConversion.setProgressChangedCallback(websocketHandler.handleNewConversionRates.bind(websocketHandler));
     loadingIndicators.setProgressChangedCallback(websocketHandler.handleLoadingChanged.bind(websocketHandler));
   }
-
+  
-  setUpHttpApiRoutes() {
+  setUpHttpApiRoutes(): void {
     bitcoinRoutes.initRoutes(this.app);
-    if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED) {
+    if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED && config.MEMPOOL.ENABLED) {
       statisticsRoutes.initRoutes(this.app);
     }
-    if (Common.indexingEnabled()) {
+    if (Common.indexingEnabled() && config.MEMPOOL.ENABLED) {
       miningRoutes.initRoutes(this.app);
     }
     if (config.BISQ.ENABLED) {
@@ -238,4 +251,4 @@ class Server {
   }
 }
 
-const server = new Server();
+((): Server => new Server())();

backend/src/indexer.ts

@@ -77,6 +77,7 @@ class Indexer {
       await mining.$generateNetworkHashrateHistory();
       await mining.$generatePoolHashrateHistory();
       await blocks.$generateBlocksSummariesDatabase();
+      await blocks.$generateCPFPDatabase();
     } catch (e) {
       this.indexerRunning = false;
       logger.err(`Indexer failed, trying again in 10 seconds. Reason: ` + (e instanceof Error ? e.message : e));

backend/src/logger.ts

@@ -32,22 +32,27 @@ class Logger {
     local7: 23
   };
 
+  public tags = {
+    mining: 'Mining',
+    ln: 'Lightning',
+  };  
+
   // @ts-ignore
-  public emerg: ((msg: string) => void);
+  public emerg: ((msg: string, tag?: string) => void);
   // @ts-ignore
-  public alert: ((msg: string) => void);
+  public alert: ((msg: string, tag?: string) => void);
   // @ts-ignore
-  public crit: ((msg: string) => void);
+  public crit: ((msg: string, tag?: string) => void);
   // @ts-ignore
-  public err: ((msg: string) => void);
+  public err: ((msg: string, tag?: string) => void);
   // @ts-ignore
-  public warn: ((msg: string) => void);
+  public warn: ((msg: string, tag?: string) => void);
   // @ts-ignore
-  public notice: ((msg: string) => void);
+  public notice: ((msg: string, tag?: string) => void);
   // @ts-ignore
-  public info: ((msg: string) => void);
+  public info: ((msg: string, tag?: string) => void);
   // @ts-ignore
-  public debug: ((msg: string) => void);
+  public debug: ((msg: string, tag?: string) => void);
 
   private name = 'mempool';
   private client: dgram.Socket;
@@ -66,8 +71,8 @@ class Logger {
 
   private addprio(prio): void {
     this[prio] = (function(_this) {
-      return function(msg) {
+      return function(msg, tag?: string) {
-        return _this.msg(prio, msg);
+        return _this.msg(prio, msg, tag);
       };
     })(this);
   }
@@ -85,7 +90,7 @@ class Logger {
     return '';
   }
 
-  private msg(priority, msg) {
+  private msg(priority, msg, tag?: string) {
     let consolemsg, prionum, syslogmsg;
     if (typeof msg === 'string' && msg.length > 0) {
       while (msg[msg.length - 1].charCodeAt(0) === 10) {
@@ -94,10 +99,10 @@ class Logger {
     }
     const network = this.network ? ' <' + this.network + '>' : '';
     prionum = Logger.priorities[priority] || Logger.priorities.info;
-    consolemsg = `${this.ts()} [${process.pid}] ${priority.toUpperCase()}:${network} ${msg}`;
+    consolemsg = `${this.ts()} [${process.pid}] ${priority.toUpperCase()}:${network} ${tag ? '[' + tag + '] ' : ''}${msg}`;
 
     if (config.SYSLOG.ENABLED && Logger.priorities[priority] <= Logger.priorities[config.SYSLOG.MIN_PRIORITY]) {
-      syslogmsg = `<${(Logger.facilities[config.SYSLOG.FACILITY] * 8 + prionum)}> ${this.name}[${process.pid}]: ${priority.toUpperCase()}${network} ${msg}`;
+      syslogmsg = `<${(Logger.facilities[config.SYSLOG.FACILITY] * 8 + prionum)}> ${this.name}[${process.pid}]: ${priority.toUpperCase()}${network} ${tag ? '[' + tag + '] ' : ''}${msg}`;
       this.syslog(syslogmsg);
     }
     if (Logger.priorities[priority] > Logger.priorities[config.MEMPOOL.STDOUT_LOG_MIN_PRIORITY]) {

backend/src/mempool.interfaces.ts

@@ -1,4 +1,5 @@
 import { IEsploraApi } from './api/bitcoin/esplora-api.interface';
+import { HeapNode } from "./utils/pairing-heap";
 
 export interface PoolTag {
   id: number; // mysql row id
@@ -27,10 +28,16 @@ export interface BlockAudit {
   height: number,
   hash: string,
   missingTxs: string[],
+  freshTxs: string[],
   addedTxs: string[],
   matchRate: number,
 }
 
+export interface AuditScore {
+  hash: string,
+  matchRate?: number,
+}
+
 export interface MempoolBlock {
   blockSize: number;
   blockVSize: number;
@@ -65,17 +72,57 @@ export interface TransactionExtended extends IEsploraApi.Transaction {
   firstSeen?: number;
   effectiveFeePerVsize: number;
   ancestors?: Ancestor[];
+  descendants?: Ancestor[];
   bestDescendant?: BestDescendant | null;
   cpfpChecked?: boolean;
   deleteAfter?: number;
 }
 
-interface Ancestor {
+export interface AuditTransaction {
+  txid: string;
+  fee: number;
+  weight: number;
+  feePerVsize: number;
+  effectiveFeePerVsize: number;
+  vin: string[];
+  relativesSet: boolean;
+  ancestorMap: Map<string, AuditTransaction>;
+  children: Set<AuditTransaction>;
+  ancestorFee: number;
+  ancestorWeight: number;
+  score: number;
+  used: boolean;
+  modified: boolean;
+  modifiedNode: HeapNode<AuditTransaction>;
+}
+
+export interface ThreadTransaction {
+  txid: string;
+  fee: number;
+  weight: number;
+  feePerVsize: number;
+  effectiveFeePerVsize?: number;
+  vin: string[];
+  cpfpRoot?: string;
+  cpfpChecked?: boolean;
+}
+
+export interface Ancestor {
   txid: string;
   weight: number;
   fee: number;
 }
 
+export interface TransactionSet {
+  fee: number;
+  weight: number;
+  score: number;
+  children?: Set<string>;
+  available?: boolean;
+  modified?: boolean;
+  modifiedNode?: HeapNode<string>;
+}
+
 interface BestDescendant {
   txid: string;
   weight: number;
@@ -84,7 +131,9 @@ interface BestDescendant {
 
 export interface CpfpInfo {
   ancestors: Ancestor[];
-  bestDescendant: BestDescendant | null;
+  bestDescendant?: BestDescendant | null;
+  descendants?: Ancestor[];
+  effectiveFeePerVsize?: number;
 }
 
 export interface TransactionStripped {
@@ -225,6 +274,7 @@ export interface IBackendInfo {
   hostname: string;
   gitCommit: string;
   version: string;
+  lightning: boolean;
 }
 
 export interface IDifficultyAdjustment {
@@ -288,4 +338,4 @@ export interface IOldestNodes {
   updatedAt?: number,
   city?: any,
   country?: any,
-}
+}

backend/src/repositories/BlocksAuditsRepository.ts

@@ -1,13 +1,14 @@
+import blocks from '../api/blocks';
 import DB from '../database';
 import logger from '../logger';
-import { BlockAudit } from '../mempool.interfaces';
+import { BlockAudit, AuditScore } from '../mempool.interfaces';
 
 class BlocksAuditRepositories {
   public async $saveAudit(audit: BlockAudit): Promise<void> {
     try {
-      await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, match_rate)
+      await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, fresh_txs, match_rate)
-        VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs),
+        VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs),
-          JSON.stringify(audit.addedTxs), audit.matchRate]);
+          JSON.stringify(audit.addedTxs), JSON.stringify(audit.freshTxs), audit.matchRate]);
     } catch (e: any) {
       if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
         logger.debug(`Cannot save block audit for block ${audit.hash} because it has already been indexed, ignoring`);
@@ -51,24 +52,58 @@ class BlocksAuditRepositories {
       const [rows]: any[] = await DB.query(
         `SELECT blocks.height, blocks.hash as id, UNIX_TIMESTAMP(blocks.blockTimestamp) as timestamp, blocks.size,
         blocks.weight, blocks.tx_count,
-        transactions, template, missing_txs as missingTxs, added_txs as addedTxs, match_rate as matchRate
+        transactions, template, missing_txs as missingTxs, added_txs as addedTxs, fresh_txs as freshTxs, match_rate as matchRate
         FROM blocks_audits
         JOIN blocks ON blocks.hash = blocks_audits.hash
         JOIN blocks_summaries ON blocks_summaries.id = blocks_audits.hash
         WHERE blocks_audits.hash = "${hash}"
       `);
       
-      rows[0].missingTxs = JSON.parse(rows[0].missingTxs);
+      if (rows.length) {
-      rows[0].addedTxs = JSON.parse(rows[0].addedTxs);
+        rows[0].missingTxs = JSON.parse(rows[0].missingTxs);
-      rows[0].transactions = JSON.parse(rows[0].transactions);
+        rows[0].addedTxs = JSON.parse(rows[0].addedTxs);
-      rows[0].template = JSON.parse(rows[0].template);
+        rows[0].freshTxs = JSON.parse(rows[0].freshTxs);
-            
+        rows[0].transactions = JSON.parse(rows[0].transactions);
+        rows[0].template = JSON.parse(rows[0].template);
+
+        if (rows[0].transactions.length) {
+          return rows[0];
+        }
+      }
+      return null;
+    } catch (e: any) {
+      logger.err(`Cannot fetch block audit from db. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getBlockAuditScore(hash: string): Promise<AuditScore> {
+    try {
+      const [rows]: any[] = await DB.query(
+        `SELECT hash, match_rate as matchRate
+        FROM blocks_audits
+        WHERE blocks_audits.hash = "${hash}"
+      `);
       return rows[0];
     } catch (e: any) {
       logger.err(`Cannot fetch block audit from db. Reason: ` + (e instanceof Error ? e.message : e));
       throw e;
     }
   }
+
+  public async $getBlockAuditScores(maxHeight: number, minHeight: number): Promise<AuditScore[]> {
+    try {
+      const [rows]: any[] = await DB.query(
+        `SELECT hash, match_rate as matchRate
+        FROM blocks_audits
+        WHERE blocks_audits.height BETWEEN ? AND ?
+      `, [minHeight, maxHeight]);
+      return rows;
+    } catch (e: any) {
+      logger.err(`Cannot fetch block audit from db. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
 }
 
 export default new BlocksAuditRepositories();

backend/src/repositories/BlocksRepository.ts

@@ -8,6 +8,8 @@ import HashratesRepository from './HashratesRepository';
 import { escape } from 'mysql2';
 import BlocksSummariesRepository from './BlocksSummariesRepository';
 import DifficultyAdjustmentsRepository from './DifficultyAdjustmentsRepository';
+import bitcoinClient from '../api/bitcoin/bitcoin-client';
+import config from '../config';
 
 class BlocksRepository {
   /**
@@ -392,6 +394,36 @@ class BlocksRepository {
     }
   }
 
+  /**
+   * Get the first block at or directly after a given timestamp
+   * @param timestamp number unix time in seconds
+   * @returns The height and timestamp of a block (timestamp might vary from given timestamp)
+   */
+  public async $getBlockHeightFromTimestamp(
+    timestamp: number,
+  ): Promise<{ height: number; hash: string; timestamp: number }> {
+    try {
+      // Get first block at or after the given timestamp
+      const query = `SELECT height, hash, blockTimestamp as timestamp FROM blocks
+        WHERE blockTimestamp <= FROM_UNIXTIME(?)
+        ORDER BY blockTimestamp DESC
+        LIMIT 1`;
+      const params = [timestamp];
+      const [rows]: any[][] = await DB.query(query, params);
+      if (rows.length === 0) {
+        throw new Error(`No block was found before timestamp ${timestamp}`);
+      }
+
+      return rows[0];
+    } catch (e) {
+      logger.err(
+        'Cannot get block height from timestamp from the db. Reason: ' +
+          (e instanceof Error ? e.message : e),
+      );
+      throw e;
+    }
+  }
+
   /**
    * Return blocks height
    */
@@ -632,6 +664,39 @@ class BlocksRepository {
     }
   }
 
+  /**
+   * Get a list of blocks that have not had CPFP data indexed
+   */
+   public async $getCPFPUnindexedBlocks(): Promise<any[]> {
+    try {
+      const blockchainInfo = await bitcoinClient.getBlockchainInfo();
+      const currentBlockHeight = blockchainInfo.blocks;
+      let indexingBlockAmount = Math.min(config.MEMPOOL.INDEXING_BLOCKS_AMOUNT, currentBlockHeight);
+      if (indexingBlockAmount <= -1) {
+        indexingBlockAmount = currentBlockHeight + 1;
+      }
+      const minHeight = Math.max(0, currentBlockHeight - indexingBlockAmount + 1);
+
+      const [rows]: any[] = await DB.query(`
+        SELECT height
+        FROM compact_cpfp_clusters
+        WHERE height <= ? AND height >= ?
+        ORDER BY height DESC;
+      `, [currentBlockHeight, minHeight]);
+
+      const indexedHeights = {};
+      rows.forEach((row) => { indexedHeights[row.height] = true; });
+      const allHeights: number[] = Array.from(Array(currentBlockHeight - minHeight + 1).keys(), n => n + minHeight).reverse();
+      const unindexedHeights = allHeights.filter(x => !indexedHeights[x]);
+
+      return unindexedHeights;
+    } catch (e) {
+      logger.err('Cannot fetch CPFP unindexed blocks. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+    return [];
+  }
+
   /**
    * Return the oldest block  from a consecutive chain of block from the most recent one
    */

backend/src/repositories/BlocksSummariesRepository.ts

@@ -17,19 +17,16 @@ class BlocksSummariesRepository {
     return undefined;
   }
 
-  public async $saveSummary(params: { height: number, mined?: BlockSummary, template?: BlockSummary}) {
+  public async $saveSummary(params: { height: number, mined?: BlockSummary}) {
-    const blockId = params.mined?.id ?? params.template?.id;
+    const blockId = params.mined?.id;
     try {
-      const [dbSummary]: any[] = await DB.query(`SELECT * FROM blocks_summaries WHERE id = "${blockId}"`);
+      const transactions = JSON.stringify(params.mined?.transactions || []);
-      if (dbSummary.length === 0) { // First insertion
+      await DB.query(`
-        await DB.query(`INSERT INTO blocks_summaries VALUE (?, ?, ?, ?)`, [
+        INSERT INTO blocks_summaries (height, id, transactions, template)
-          params.height, blockId, JSON.stringify(params.mined?.transactions ?? []), JSON.stringify(params.template?.transactions ?? [])
+        VALUE (?, ?, ?, ?)
-        ]);
+        ON DUPLICATE KEY UPDATE
-      } else if (params.mined !== undefined) { // Update mined block summary
+          transactions = ?
-        await DB.query(`UPDATE blocks_summaries SET transactions = ? WHERE id = "${params.mined.id}"`, [JSON.stringify(params.mined.transactions)]);
+      `, [params.height, blockId, transactions, '[]', transactions]);
-      } else if (params.template !== undefined) { // Update template block summary
-        await DB.query(`UPDATE blocks_summaries SET template = ? WHERE id = "${params.template.id}"`, [JSON.stringify(params.template?.transactions)]);
-      }
     } catch (e: any) {
       if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
         logger.debug(`Cannot save block summary for ${blockId} because it has already been indexed, ignoring`);
@@ -40,6 +37,26 @@ class BlocksSummariesRepository {
     }
   }
 
+  public async $saveTemplate(params: { height: number, template: BlockSummary}) {
+    const blockId = params.template?.id;
+    try {
+      const transactions = JSON.stringify(params.template?.transactions || []);
+      await DB.query(`
+        INSERT INTO blocks_summaries (height, id, transactions, template)
+        VALUE (?, ?, ?, ?)
+        ON DUPLICATE KEY UPDATE
+          template = ?
+      `, [params.height, blockId, '[]', transactions, transactions]);
+    } catch (e: any) {
+      if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
+        logger.debug(`Cannot save block template for ${blockId} because it has already been indexed, ignoring`);
+      } else {
+        logger.debug(`Cannot save block template for ${blockId}. Reason: ${e instanceof Error ? e.message : e}`);
+        throw e;
+      }
+    }
+  }
+
   public async $getIndexedSummariesId(): Promise<string[]> {
     try {
       const [rows]: any[] = await DB.query(`SELECT id from blocks_summaries`);

backend/src/repositories/CpfpRepository.ts

@@ -0,0 +1,224 @@
+import cluster, { Cluster } from 'cluster';
+import { RowDataPacket } from 'mysql2';
+import DB from '../database';
+import logger from '../logger';
+import { Ancestor } from '../mempool.interfaces';
+import transactionRepository from '../repositories/TransactionRepository';
+
+class CpfpRepository {
+  public async $saveCluster(clusterRoot: string, height: number, txs: Ancestor[], effectiveFeePerVsize: number): Promise<boolean> {
+    if (!txs[0]) {
+      return false;
+    }
+    // skip clusters of transactions with the same fees
+    const roundedEffectiveFee = Math.round(effectiveFeePerVsize * 100) / 100;
+    const equalFee = txs.reduce((acc, tx) => {
+      return (acc && Math.round(((tx.fee || 0) / (tx.weight / 4)) * 100) / 100 === roundedEffectiveFee);
+    }, true);
+    if (equalFee) {
+      return false;
+    }
+
+    try {
+      const packedTxs = Buffer.from(this.pack(txs));
+      await DB.query(
+        `
+          INSERT INTO compact_cpfp_clusters(root, height, txs, fee_rate)
+          VALUE (UNHEX(?), ?, ?, ?)
+          ON DUPLICATE KEY UPDATE
+            height = ?,
+            txs = ?,
+            fee_rate = ?
+        `,
+        [clusterRoot, height, packedTxs, effectiveFeePerVsize, height, packedTxs, effectiveFeePerVsize]
+      );
+      const maxChunk = 10;
+      let chunkIndex = 0;
+      while (chunkIndex < txs.length) {
+        const chunk = txs.slice(chunkIndex, chunkIndex + maxChunk).map(tx => {
+          return { txid: tx.txid, cluster: clusterRoot };
+        });
+        await transactionRepository.$batchSetCluster(chunk);
+        chunkIndex += maxChunk;
+      }
+      return true;
+    } catch (e: any) {
+      logger.err(`Cannot save cpfp cluster into db. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $batchSaveClusters(clusters: { root: string, height: number, txs: any, effectiveFeePerVsize: number}[]): Promise<boolean> {
+    try {
+      const clusterValues: any[] = [];
+      const txs: any[] = [];
+
+      for (const cluster of clusters) {
+        if (cluster.txs?.length > 1) {
+          const roundedEffectiveFee = Math.round(cluster.effectiveFeePerVsize * 100) / 100;
+          const equalFee = cluster.txs.reduce((acc, tx) => {
+            return (acc && Math.round(((tx.fee || 0) / (tx.weight / 4)) * 100) / 100 === roundedEffectiveFee);
+          }, true);
+          if (!equalFee) {
+            clusterValues.push([
+              cluster.root,
+              cluster.height,
+              Buffer.from(this.pack(cluster.txs)),
+              cluster.effectiveFeePerVsize
+            ]);
+            for (const tx of cluster.txs) {
+              txs.push({ txid: tx.txid, cluster: cluster.root });
+            }
+          }
+        }
+      }
+
+      if (!clusterValues.length) {
+        return false;
+      }
+
+      const maxChunk = 100;
+      let chunkIndex = 0;
+      // insert transactions in batches of up to 100 rows
+      while (chunkIndex < txs.length) {
+        const chunk = txs.slice(chunkIndex, chunkIndex + maxChunk);
+        await transactionRepository.$batchSetCluster(chunk);
+        chunkIndex += maxChunk;
+      }
+
+      chunkIndex = 0;
+      // insert clusters in batches of up to 100 rows
+      while (chunkIndex < clusterValues.length) {
+        const chunk = clusterValues.slice(chunkIndex, chunkIndex + maxChunk);
+        let query = `
+            INSERT IGNORE INTO compact_cpfp_clusters(root, height, txs, fee_rate)
+            VALUES
+        `;
+        query += chunk.map(chunk => {
+          return (' (UNHEX(?), ?, ?, ?)');
+        }) + ';';
+        const values = chunk.flat();
+        await DB.query(
+          query,
+          values
+        );
+        chunkIndex += maxChunk;
+      }
+      return true;
+    } catch (e: any) {
+      logger.err(`Cannot save cpfp clusters into db. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getCluster(clusterRoot: string): Promise<Cluster> {
+    const [clusterRows]: any = await DB.query(
+      `
+        SELECT *
+        FROM compact_cpfp_clusters
+        WHERE root = UNHEX(?)
+      `,
+      [clusterRoot]
+    );
+    const cluster = clusterRows[0];
+    cluster.txs = this.unpack(cluster.txs);
+    return cluster;
+  }
+
+  public async $deleteClustersFrom(height: number): Promise<void> {
+    logger.info(`Delete newer cpfp clusters from height ${height} from the database`);
+    try {
+      const [rows] = await DB.query(
+        `
+          SELECT txs, height, root from compact_cpfp_clusters
+          WHERE height >= ?
+        `,
+        [height]
+      ) as RowDataPacket[][];
+      if (rows?.length) {
+        for (let clusterToDelete of rows) {
+          const txs = this.unpack(clusterToDelete.txs);
+          for (let tx of txs) {
+            await transactionRepository.$removeTransaction(tx.txid);
+          }
+        }
+      }
+      await DB.query(
+        `
+          DELETE from compact_cpfp_clusters
+          WHERE height >= ?
+        `,
+        [height]
+      );
+    } catch (e: any) {
+      logger.err(`Cannot delete cpfp clusters from db. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  // insert a dummy row to mark that we've indexed as far as this block
+  public async $insertProgressMarker(height: number): Promise<void> {
+    try {
+      const [rows]: any = await DB.query(
+        `
+          SELECT root
+          FROM compact_cpfp_clusters
+          WHERE height = ?
+        `,
+        [height]
+      );
+      if (!rows?.length) {
+        const rootBuffer = Buffer.alloc(32);
+        rootBuffer.writeInt32LE(height);
+        await DB.query(
+          `
+            INSERT INTO compact_cpfp_clusters(root, height, fee_rate)
+            VALUE (?, ?, ?)
+          `,
+          [rootBuffer, height, 0]
+        );
+      }
+    } catch (e: any) {
+      logger.err(`Cannot insert cpfp progress marker. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public pack(txs: Ancestor[]): ArrayBuffer {
+    const buf = new ArrayBuffer(44 * txs.length);
+    const view = new DataView(buf);
+    txs.forEach((tx, i) => {
+      const offset = i * 44;
+      for (let x = 0; x < 32; x++) {
+        // store txid in little-endian
+        view.setUint8(offset + (31 - x), parseInt(tx.txid.slice(x * 2, (x * 2) + 2), 16));
+      }
+      view.setUint32(offset + 32, tx.weight);
+      view.setBigUint64(offset + 36, BigInt(Math.round(tx.fee)));
+    });
+    return buf;
+  }
+
+  public unpack(buf: Buffer): Ancestor[] {
+    if (!buf) {
+      return [];
+    }
+
+    const arrayBuffer = buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength);
+    const txs: Ancestor[] = [];
+    const view = new DataView(arrayBuffer);
+    for (let offset = 0; offset < arrayBuffer.byteLength; offset += 44) {
+      const txid = Array.from(new Uint8Array(arrayBuffer, offset, 32)).reverse().map(b => b.toString(16).padStart(2, '0')).join('');
+      const weight = view.getUint32(offset + 32);
+      const fee = Number(view.getBigUint64(offset + 36));
+      txs.push({
+        txid,
+        weight,
+        fee
+      });
+    }
+    return txs;
+  }
+}
+
+export default new CpfpRepository();

backend/src/repositories/NodeRecordsRepository.ts

@@ -0,0 +1,67 @@
+import { ResultSetHeader, RowDataPacket } from 'mysql2';
+import DB from '../database';
+import logger from '../logger';
+
+export interface NodeRecord {
+  publicKey: string; // node public key
+  type: number; // TLV extension record type
+  payload: string; // base64 record payload
+}
+
+class NodesRecordsRepository {
+  public async $saveRecord(record: NodeRecord): Promise<void> {
+    try {
+      const payloadBytes = Buffer.from(record.payload, 'base64');
+      await DB.query(`
+        INSERT INTO nodes_records(public_key, type, payload)
+        VALUE (?, ?, ?)
+        ON DUPLICATE KEY UPDATE
+          payload = ?
+      `, [record.publicKey, record.type, payloadBytes, payloadBytes]);
+    } catch (e: any) {
+      if (e.errno !== 1062) { // ER_DUP_ENTRY - Not an issue, just ignore this
+        logger.err(`Cannot save node record (${[record.publicKey, record.type, record.payload]}) into db. Reason: ` + (e instanceof Error ? e.message : e));
+        // We don't throw, not a critical issue if we miss some nodes records
+      }
+    }
+   }
+
+  public async $getRecordTypes(publicKey: string): Promise<any> {
+    try {
+      const query = `
+        SELECT type FROM nodes_records
+        WHERE public_key = ?
+      `;
+      const [rows] = await DB.query<RowDataPacket[][]>(query, [publicKey]);
+      return rows.map(row => row['type']);
+    } catch (e) {
+      logger.err(`Cannot retrieve custom records for ${publicKey} from db. Reason: ` + (e instanceof Error ? e.message : e));
+      return [];
+    }
+  }
+
+  public async $deleteUnusedRecords(publicKey: string, recordTypes: number[]): Promise<number> {
+    try {
+      let query;
+      if (recordTypes.length) {
+        query = `
+          DELETE FROM nodes_records
+          WHERE public_key = ?
+          AND type NOT IN (${recordTypes.map(type => `${type}`).join(',')})
+        `;
+      } else {
+        query = `
+          DELETE FROM nodes_records
+          WHERE public_key = ?
+        `;
+      }
+      const [result] = await DB.query<ResultSetHeader>(query, [publicKey]);
+      return result.affectedRows;
+    } catch (e) {
+      logger.err(`Cannot delete unused custom records for ${publicKey} from db. Reason: ` + (e instanceof Error ? e.message : e));
+      return 0;
+    }
+  }
+}
+
+export default new NodesRecordsRepository();

backend/src/repositories/TransactionRepository.ts

@@ -0,0 +1,115 @@
+import DB from '../database';
+import logger from '../logger';
+import { Ancestor, CpfpInfo } from '../mempool.interfaces';
+import cpfpRepository from './CpfpRepository';
+
+interface CpfpSummary {
+  txid: string;
+  cluster: string;
+  root: string;
+  txs: Ancestor[];
+  height: number;
+  fee_rate: number;
+}
+
+class TransactionRepository {
+  public async $setCluster(txid: string, clusterRoot: string): Promise<void> {
+    try {
+      await DB.query(
+        `
+          INSERT INTO compact_transactions
+          (
+            txid,
+            cluster
+          )
+          VALUE (UNHEX(?), UNHEX(?))
+          ON DUPLICATE KEY UPDATE
+            cluster = UNHEX(?)
+        ;`,
+        [txid, clusterRoot, clusterRoot]
+      );
+    } catch (e: any) {
+      logger.err(`Cannot save transaction cpfp cluster into db. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $batchSetCluster(txs): Promise<void> {
+    try {
+      let query = `
+          INSERT IGNORE INTO compact_transactions
+          (
+            txid,
+            cluster
+          )
+          VALUES
+      `;
+      query += txs.map(tx => {
+        return (' (UNHEX(?), UNHEX(?))');
+      }) + ';';
+      const values = txs.map(tx => [tx.txid, tx.cluster]).flat();
+      await DB.query(
+        query,
+        values
+      );
+    } catch (e: any) {
+      logger.err(`Cannot save cpfp transactions into db. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getCpfpInfo(txid: string): Promise<CpfpInfo | void> {
+    try {
+      const [txRows]: any = await DB.query(
+        `
+          SELECT HEX(txid) as id, HEX(cluster) as root
+          FROM compact_transactions
+          WHERE txid = UNHEX(?)
+        `,
+        [txid]
+      );
+      if (txRows.length && txRows[0].root != null) {
+        const txid = txRows[0].id.toLowerCase();
+        const clusterId = txRows[0].root.toLowerCase();
+        const cluster = await cpfpRepository.$getCluster(clusterId);
+        return this.convertCpfp(txid, cluster);
+      }
+    } catch (e) {
+      logger.err('Cannot get transaction cpfp info from db. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $removeTransaction(txid: string): Promise<void> {
+    await DB.query(
+      `
+        DELETE FROM compact_transactions
+        WHERE txid = UNHEX(?)
+      `,
+      [txid]
+    );
+  }
+
+  private convertCpfp(txid, cluster): CpfpInfo {
+    const descendants: Ancestor[] = [];
+    const ancestors: Ancestor[] = [];
+    let matched = false;
+
+    for (const tx of cluster.txs) {
+      if (tx.txid === txid) {
+        matched = true;
+      } else if (!matched) {
+        descendants.push(tx);
+      } else {
+        ancestors.push(tx);
+      }
+    }
+    return {
+      descendants,
+      ancestors,
+    };
+  }
+}
+
+export default new TransactionRepository();
+

backend/src/tasks/lightning/forensics.service.ts

@@ -0,0 +1,457 @@
+import DB from '../../database';
+import logger from '../../logger';
+import channelsApi from '../../api/explorer/channels.api';
+import bitcoinApi from '../../api/bitcoin/bitcoin-api-factory';
+import config from '../../config';
+import { IEsploraApi } from '../../api/bitcoin/esplora-api.interface';
+import { Common } from '../../api/common';
+import { ILightningApi } from '../../api/lightning/lightning-api.interface';
+
+const tempCacheSize = 10000;
+
+class ForensicsService {
+  loggerTimer = 0;
+  closedChannelsScanBlock = 0;
+  txCache: { [txid: string]: IEsploraApi.Transaction } = {};
+  tempCached: string[] = [];
+
+  constructor() {}
+
+  public async $startService(): Promise<void> {
+    logger.info('Starting lightning network forensics service');
+
+    this.loggerTimer = new Date().getTime() / 1000;
+
+    await this.$runTasks();
+  }
+
+  private async $runTasks(): Promise<void> {
+    try {
+      logger.info(`Running forensics scans`);
+
+      if (config.MEMPOOL.BACKEND === 'esplora') {
+        await this.$runClosedChannelsForensics(false);
+        await this.$runOpenedChannelsForensics();
+      }
+
+    } catch (e) {
+      logger.err('ForensicsService.$runTasks() error: ' + (e instanceof Error ? e.message : e));
+    }
+
+    setTimeout(() => { this.$runTasks(); }, 1000 * config.LIGHTNING.FORENSICS_INTERVAL);
+  }
+
+  /*
+    1. Mutually closed
+    2. Forced closed
+    3. Forced closed with penalty
+
+    ┌────────────────────────────────────┐       ┌────────────────────────────┐
+    │ outputs contain revocation script? ├──yes──► force close w/ penalty = 3 │
+    └──────────────┬─────────────────────┘       └────────────────────────────┘
+                   no
+    ┌──────────────▼──────────────────────────┐
+    │ outputs contain other lightning script? ├──┐
+    └──────────────┬──────────────────────────┘  │
+                   no                           yes
+    ┌──────────────▼─────────────┐               │
+    │ sequence starts with 0x80  │      ┌────────▼────────┐
+    │           and              ├──────► force close = 2 │
+    │ locktime starts with 0x20? │      └─────────────────┘
+    └──────────────┬─────────────┘
+                   no
+         ┌─────────▼────────┐
+         │ mutual close = 1 │
+         └──────────────────┘
+  */
+
+  public async $runClosedChannelsForensics(onlyNewChannels: boolean = false): Promise<void> {
+    if (config.MEMPOOL.BACKEND !== 'esplora') {
+      return;
+    }
+
+    let progress = 0;
+
+    try {
+      logger.info(`Started running closed channel forensics...`);
+      let channels;
+      if (onlyNewChannels) {
+        channels = await channelsApi.$getClosedChannelsWithoutReason();
+      } else {
+        channels = await channelsApi.$getUnresolvedClosedChannels();
+      }
+
+      for (const channel of channels) {
+        let reason = 0;
+        let resolvedForceClose = false;
+        // Only Esplora backend can retrieve spent transaction outputs
+        const cached: string[] = [];
+        try {
+          let outspends: IEsploraApi.Outspend[] | undefined;
+          try {
+            outspends = await bitcoinApi.$getOutspends(channel.closing_transaction_id);
+            await Common.sleep$(config.LIGHTNING.FORENSICS_RATE_LIMIT);
+          } catch (e) {
+            logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + channel.closing_transaction_id + '/outspends'}. Reason ${e instanceof Error ? e.message : e}`);
+            continue;
+          }
+          const lightningScriptReasons: number[] = [];
+          for (const outspend of outspends) {
+            if (outspend.spent && outspend.txid) {
+              let spendingTx = await this.fetchTransaction(outspend.txid);
+              if (!spendingTx) {
+                continue;
+              }
+              cached.push(spendingTx.txid);
+              const lightningScript = this.findLightningScript(spendingTx.vin[outspend.vin || 0]);
+              lightningScriptReasons.push(lightningScript);
+            }
+          }
+          const filteredReasons = lightningScriptReasons.filter((r) => r !== 1);
+          if (filteredReasons.length) {
+            if (filteredReasons.some((r) => r === 2 || r === 4)) {
+              reason = 3;
+            } else {
+              reason = 2;
+              resolvedForceClose = true;
+            }
+          } else {
+            /*
+              We can detect a commitment transaction (force close) by reading Sequence and Locktime
+              https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction
+            */
+            let closingTx = await this.fetchTransaction(channel.closing_transaction_id, true);
+            if (!closingTx) {
+              continue;
+            }
+            cached.push(closingTx.txid);
+            const sequenceHex: string = closingTx.vin[0].sequence.toString(16);
+            const locktimeHex: string = closingTx.locktime.toString(16);
+            if (sequenceHex.substring(0, 2) === '80' && locktimeHex.substring(0, 2) === '20') {
+              reason = 2; // Here we can't be sure if it's a penalty or not
+            } else {
+              reason = 1;
+            }
+          }
+          if (reason) {
+            logger.debug('Setting closing reason ' + reason + ' for channel: ' + channel.id + '.');
+            await DB.query(`UPDATE channels SET closing_reason = ? WHERE id = ?`, [reason, channel.id]);
+            if (reason === 2 && resolvedForceClose) {
+              await DB.query(`UPDATE channels SET closing_resolved = ? WHERE id = ?`, [true, channel.id]);
+            }
+            if (reason !== 2 || resolvedForceClose) {
+              cached.forEach(txid => {
+                delete this.txCache[txid];
+              });
+            }
+          }
+        } catch (e) {
+          logger.err(`$runClosedChannelsForensics() failed for channel ${channel.short_id}. Reason: ${e instanceof Error ? e.message : e}`);
+        }
+
+        ++progress;
+        const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
+        if (elapsedSeconds > 10) {
+          logger.info(`Updating channel closed channel forensics ${progress}/${channels.length}`);
+          this.loggerTimer = new Date().getTime() / 1000;
+        }
+      }
+      logger.info(`Closed channels forensics scan complete.`);
+    } catch (e) {
+      logger.err('$runClosedChannelsForensics() error: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  private findLightningScript(vin: IEsploraApi.Vin): number {
+    const topElement = vin.witness?.length > 2 ? vin.witness[vin.witness.length - 2] : null;
+      if (/^OP_IF OP_PUSHBYTES_33 \w{66} OP_ELSE OP_PUSH(NUM_\d+|BYTES_(1 \w{2}|2 \w{4})) OP_CSV OP_DROP OP_PUSHBYTES_33 \w{66} OP_ENDIF OP_CHECKSIG$/.test(vin.inner_witnessscript_asm)) {
+        // https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction-outputs
+        if (topElement === '01') {
+          // top element is '01' to get in the revocation path
+          // 'Revoked Lightning Force Close';
+          // Penalty force closed
+          return 2;
+        } else {
+          // top element is '', this is a delayed to_local output
+          // 'Lightning Force Close';
+          return 3;
+        }
+      } else if (
+        /^OP_DUP OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUAL OP_IF OP_CHECKSIG OP_ELSE OP_PUSHBYTES_33 \w{66} OP_SWAP OP_SIZE OP_PUSHBYTES_1 20 OP_EQUAL OP_NOTIF OP_DROP OP_PUSHNUM_2 OP_SWAP OP_PUSHBYTES_33 \w{66} OP_PUSHNUM_2 OP_CHECKMULTISIG OP_ELSE OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUALVERIFY OP_CHECKSIG OP_ENDIF (OP_PUSHNUM_1 OP_CSV OP_DROP |)OP_ENDIF$/.test(vin.inner_witnessscript_asm) ||
+        /^OP_DUP OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUAL OP_IF OP_CHECKSIG OP_ELSE OP_PUSHBYTES_33 \w{66} OP_SWAP OP_SIZE OP_PUSHBYTES_1 20 OP_EQUAL OP_IF OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUALVERIFY OP_PUSHNUM_2 OP_SWAP OP_PUSHBYTES_33 \w{66} OP_PUSHNUM_2 OP_CHECKMULTISIG OP_ELSE OP_DROP OP_PUSHBYTES_3 \w{6} OP_CLTV OP_DROP OP_CHECKSIG OP_ENDIF (OP_PUSHNUM_1 OP_CSV OP_DROP |)OP_ENDIF$/.test(vin.inner_witnessscript_asm)
+      ) {
+        // https://github.com/lightning/bolts/blob/master/03-transactions.md#offered-htlc-outputs
+        // https://github.com/lightning/bolts/blob/master/03-transactions.md#received-htlc-outputs
+        if (topElement?.length === 66) {
+          // top element is a public key
+          // 'Revoked Lightning HTLC'; Penalty force closed
+          return 4;
+        } else if (topElement) {
+          // top element is a preimage
+          // 'Lightning HTLC';
+          return 5;
+        } else {
+          // top element is '' to get in the expiry of the script
+          // 'Expired Lightning HTLC';
+          return 6;
+        }
+      } else if (/^OP_PUSHBYTES_33 \w{66} OP_CHECKSIG OP_IFDUP OP_NOTIF OP_PUSHNUM_16 OP_CSV OP_ENDIF$/.test(vin.inner_witnessscript_asm)) {
+        // https://github.com/lightning/bolts/blob/master/03-transactions.md#to_local_anchor-and-to_remote_anchor-output-option_anchors
+        if (topElement) {
+          // top element is a signature
+          // 'Lightning Anchor';
+          return 7;
+        } else {
+          // top element is '', it has been swept after 16 blocks
+          // 'Swept Lightning Anchor';
+          return 8;
+        }
+      }
+      return 1;
+  }
+
+  // If a channel open tx spends funds from a another channel transaction,
+  // we can attribute that output to a specific counterparty
+  private async $runOpenedChannelsForensics(): Promise<void> {
+    const runTimer = Date.now();
+    let progress = 0;
+
+    try {
+      logger.info(`Started running open channel forensics...`);
+      const channels = await channelsApi.$getChannelsWithoutSourceChecked();
+
+      for (const openChannel of channels) {
+        let openTx = await this.fetchTransaction(openChannel.transaction_id, true);
+        if (!openTx) {
+          continue;
+        }
+        for (const input of openTx.vin) {
+          const closeChannel = await channelsApi.$getChannelByClosingId(input.txid);
+          if (closeChannel) {
+            // this input directly spends a channel close output
+            await this.$attributeChannelBalances(closeChannel, openChannel, input);
+          } else {
+            const prevOpenChannels = await channelsApi.$getChannelsByOpeningId(input.txid);
+            if (prevOpenChannels?.length) {
+              // this input spends a channel open change output
+              for (const prevOpenChannel of prevOpenChannels) {
+                await this.$attributeChannelBalances(prevOpenChannel, openChannel, input, null, null, true);
+              }
+            } else {
+              // check if this input spends any swept channel close outputs
+              await this.$attributeSweptChannelCloses(openChannel, input);
+            }
+          }
+        }
+        // calculate how much of the total input value is attributable to the channel open output
+        openChannel.funding_ratio = openTx.vout[openChannel.transaction_vout].value / ((openTx.vout.reduce((sum, v) => sum + v.value, 0) || 1) + openTx.fee);
+        // save changes to the opening channel, and mark it as checked
+        if (openTx?.vin?.length === 1) {
+          openChannel.single_funded = true;
+        }
+        if (openChannel.node1_funding_balance || openChannel.node2_funding_balance || openChannel.node1_closing_balance || openChannel.node2_closing_balance || openChannel.closed_by) {
+          await channelsApi.$updateOpeningInfo(openChannel);
+        }
+        await channelsApi.$markChannelSourceChecked(openChannel.id);
+
+        ++progress;
+        const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
+        if (elapsedSeconds > 10) {
+          logger.info(`Updating opened channel forensics ${progress}/${channels?.length}`);
+          this.loggerTimer = new Date().getTime() / 1000;
+          this.truncateTempCache();
+        }
+        if (Date.now() - runTimer > (config.LIGHTNING.FORENSICS_INTERVAL * 1000)) {
+          break;
+        }
+      }
+
+      logger.info(`Open channels forensics scan complete.`);
+    } catch (e) {
+      logger.err('$runOpenedChannelsForensics() error: ' + (e instanceof Error ? e.message : e));
+    } finally {
+      this.clearTempCache();
+    }
+  }
+
+  // Check if a channel open tx input spends the result of a swept channel close output
+  private async $attributeSweptChannelCloses(openChannel: ILightningApi.Channel, input: IEsploraApi.Vin): Promise<void> {
+    let sweepTx = await this.fetchTransaction(input.txid, true);
+    if (!sweepTx) {
+      logger.err(`couldn't find input transaction for channel forensics ${openChannel.channel_id} ${input.txid}`);
+      return;
+    }
+    const openContribution = sweepTx.vout[input.vout].value;
+    for (const sweepInput of sweepTx.vin) {
+      const lnScriptType = this.findLightningScript(sweepInput);
+      if (lnScriptType > 1) {
+        const closeChannel = await channelsApi.$getChannelByClosingId(sweepInput.txid);
+        if (closeChannel) {
+          const initiator = (lnScriptType === 2 || lnScriptType === 4) ? 'remote' : (lnScriptType === 3 ? 'local' : null);
+          await this.$attributeChannelBalances(closeChannel, openChannel, sweepInput, openContribution, initiator);
+        }
+      }
+    }
+  }
+
+  private async $attributeChannelBalances(
+    prevChannel, openChannel, input: IEsploraApi.Vin, openContribution: number | null = null,
+    initiator: 'remote' | 'local' | null = null, linkedOpenings: boolean = false
+  ): Promise<void> {
+    // figure out which node controls the input/output
+    let openSide;
+    let prevLocal;
+    let prevRemote;
+    let matched = false;
+    let ambiguous = false; // if counterparties are the same in both channels, we can't tell them apart
+    if (openChannel.node1_public_key === prevChannel.node1_public_key) {
+      openSide = 1;
+      prevLocal = 1;
+      prevRemote = 2;
+      matched = true;
+    } else if (openChannel.node1_public_key === prevChannel.node2_public_key) {
+      openSide = 1;
+      prevLocal = 2;
+      prevRemote = 1;
+      matched = true;
+    }
+    if (openChannel.node2_public_key === prevChannel.node1_public_key) {
+      openSide = 2;
+      prevLocal = 1;
+      prevRemote = 2;
+      if (matched) {
+        ambiguous = true;
+      }
+      matched = true;
+    } else if (openChannel.node2_public_key === prevChannel.node2_public_key) {
+      openSide = 2;
+      prevLocal = 2;
+      prevRemote = 1;
+      if (matched) {
+        ambiguous = true;
+      }
+      matched = true;
+    }
+
+    if (matched && !ambiguous) {
+      // fetch closing channel transaction and perform forensics on the outputs
+      let prevChannelTx = await this.fetchTransaction(input.txid, true);
+      let outspends: IEsploraApi.Outspend[] | undefined;
+      try {
+        outspends = await bitcoinApi.$getOutspends(input.txid);
+        await Common.sleep$(config.LIGHTNING.FORENSICS_RATE_LIMIT);
+      } catch (e) {
+        logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + input.txid + '/outspends'}. Reason ${e instanceof Error ? e.message : e}`);
+      }
+      if (!outspends || !prevChannelTx) {
+        return;
+      }
+      if (!linkedOpenings) {
+        if (!prevChannel.outputs || !prevChannel.outputs.length) {
+          prevChannel.outputs = prevChannelTx.vout.map(vout => {
+            return {
+              type: 0,
+              value: vout.value,
+            };
+          });
+        }
+        for (let i = 0; i < outspends?.length; i++) {
+          const outspend = outspends[i];
+          const output = prevChannel.outputs[i];
+          if (outspend.spent && outspend.txid) {
+            try {
+              const spendingTx = await this.fetchTransaction(outspend.txid, true);
+              if (spendingTx) {
+                output.type = this.findLightningScript(spendingTx.vin[outspend.vin || 0]);
+              }
+            } catch (e) {
+              logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + outspend.txid}. Reason ${e instanceof Error ? e.message : e}`);
+            }
+          } else {
+            output.type = 0;
+          }
+        }
+
+        // attribute outputs to each counterparty, and sum up total known balances
+        prevChannel.outputs[input.vout].node = prevLocal;
+        const isPenalty = prevChannel.outputs.filter((out) => out.type === 2 || out.type === 4)?.length > 0;
+        const normalOutput = [1,3].includes(prevChannel.outputs[input.vout].type);
+        const mutualClose = ((prevChannel.status === 2 || prevChannel.status === 'closed') && prevChannel.closing_reason === 1);
+        let localClosingBalance = 0;
+        let remoteClosingBalance = 0;
+        for (const output of prevChannel.outputs) {
+          if (isPenalty) {
+            // penalty close, so local node takes everything
+            localClosingBalance += output.value;
+          } else if (output.node) {
+            // this output determinstically linked to one of the counterparties
+            if (output.node === prevLocal) {
+              localClosingBalance += output.value;
+            } else {
+              remoteClosingBalance += output.value;
+            }
+          } else if (normalOutput && (output.type === 1 || output.type === 3 || (mutualClose && prevChannel.outputs.length === 2))) {
+            // local node had one main output, therefore remote node takes the other
+            remoteClosingBalance += output.value;
+          }
+        }
+        prevChannel[`node${prevLocal}_closing_balance`] = localClosingBalance;
+        prevChannel[`node${prevRemote}_closing_balance`] = remoteClosingBalance;
+        prevChannel.closing_fee = prevChannelTx.fee;
+
+        if (initiator && !linkedOpenings) {
+          const initiatorSide = initiator === 'remote' ? prevRemote : prevLocal;
+          prevChannel.closed_by = prevChannel[`node${initiatorSide}_public_key`];
+        }
+  
+        // save changes to the closing channel
+        await channelsApi.$updateClosingInfo(prevChannel);
+      } else {
+        if (prevChannelTx.vin.length <= 1) {
+          prevChannel[`node${prevLocal}_funding_balance`] = prevChannel.capacity;
+          prevChannel.single_funded = true;
+          prevChannel.funding_ratio = 1;
+          // save changes to the closing channel
+          await channelsApi.$updateOpeningInfo(prevChannel);
+        }
+      }
+      openChannel[`node${openSide}_funding_balance`] = openChannel[`node${openSide}_funding_balance`] + (openContribution || prevChannelTx?.vout[input.vout]?.value || 0);
+    }
+  }
+
+  async fetchTransaction(txid: string, temp: boolean = false): Promise<IEsploraApi.Transaction | null> {
+    let tx = this.txCache[txid];
+    if (!tx) {
+      try {
+        tx = await bitcoinApi.$getRawTransaction(txid);
+        this.txCache[txid] = tx;
+        if (temp) {
+          this.tempCached.push(txid);
+        }
+        await Common.sleep$(config.LIGHTNING.FORENSICS_RATE_LIMIT);
+      } catch (e) {
+        logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + txid + '/outspends'}. Reason ${e instanceof Error ? e.message : e}`);
+        return null;
+      }
+    }
+    return tx;
+  }
+
+  clearTempCache(): void {
+    for (const txid of this.tempCached) {
+      delete this.txCache[txid];
+    }
+    this.tempCached = [];
+  }
+
+  truncateTempCache(): void {
+    if (this.tempCached.length > tempCacheSize) {
+      const removed = this.tempCached.splice(0, this.tempCached.length - tempCacheSize);
+      for (const txid of removed) {
+        delete this.txCache[txid];
+      }
+    }
+  }
+}
+
+export default new ForensicsService();

backend/src/tasks/lightning/network-sync.service.ts

@@ -13,6 +13,8 @@ import fundingTxFetcher from './sync-tasks/funding-tx-fetcher';
 import NodesSocketsRepository from '../../repositories/NodesSocketsRepository';
 import { Common } from '../../api/common';
 import blocks from '../../api/blocks';
+import NodeRecordsRepository from '../../repositories/NodeRecordsRepository';
+import forensicsService from './forensics.service';
 
 class NetworkSyncService {
   loggerTimer = 0;
@@ -21,7 +23,7 @@ class NetworkSyncService {
   constructor() {}
 
   public async $startService(): Promise<void> {
-    logger.info('Starting lightning network sync service');
+    logger.info(`Starting lightning network sync service`, logger.tags.ln);
 
     this.loggerTimer = new Date().getTime() / 1000;
 
@@ -29,12 +31,13 @@ class NetworkSyncService {
   }
 
   private async $runTasks(): Promise<void> {
+    const taskStartTime = Date.now();
     try {
-      logger.info(`Updating nodes and channels`);
+      logger.debug(`Updating nodes and channels`, logger.tags.ln);
 
       const networkGraph = await lightningApi.$getNetworkGraph();
       if (networkGraph.nodes.length === 0 || networkGraph.edges.length === 0) {
-        logger.info(`LN Network graph is empty, retrying in 10 seconds`);
+        logger.info(`LN Network graph is empty, retrying in 10 seconds`, logger.tags.ln);
         setTimeout(() => { this.$runTasks(); }, 10000);
         return;
       }
@@ -45,15 +48,17 @@ class NetworkSyncService {
       await this.$lookUpCreationDateFromChain();
       await this.$updateNodeFirstSeen();
       await this.$scanForClosedChannels();
+      
       if (config.MEMPOOL.BACKEND === 'esplora') {
-        await this.$runClosedChannelsForensics();
+        // run forensics on new channels only
+        await forensicsService.$runClosedChannelsForensics(true);
       }
 
     } catch (e) {
-      logger.err('$runTasks() error: ' + (e instanceof Error ? e.message : e));
+      logger.err(`$runTasks() error: ${e instanceof Error ? e.message : e}`, logger.tags.ln);
     }
 
-    setTimeout(() => { this.$runTasks(); }, 1000 * config.LIGHTNING.GRAPH_REFRESH_INTERVAL);
+    setTimeout(() => { this.$runTasks(); }, Math.max(1, (1000 * config.LIGHTNING.GRAPH_REFRESH_INTERVAL) - (Date.now() - taskStartTime)));
   }
 
   /**
@@ -63,6 +68,7 @@ class NetworkSyncService {
     let progress = 0;
 
     let deletedSockets = 0;
+    let deletedRecords = 0;
     const graphNodesPubkeys: string[] = [];
     for (const node of nodes) {
       const latestUpdated = await channelsApi.$getLatestChannelUpdateForNode(node.pub_key);
@@ -73,8 +79,8 @@ class NetworkSyncService {
       ++progress;
 
       const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
-      if (elapsedSeconds > 10) {
+      if (elapsedSeconds > config.LIGHTNING.LOGGER_UPDATE_INTERVAL) {
-        logger.info(`Updating node ${progress}/${nodes.length}`);
+        logger.debug(`Updating node ${progress}/${nodes.length}`, logger.tags.ln);
         this.loggerTimer = new Date().getTime() / 1000;
       }
 
@@ -84,8 +90,23 @@ class NetworkSyncService {
         addresses.push(socket.addr);
       }
       deletedSockets += await NodesSocketsRepository.$deleteUnusedSockets(node.pub_key, addresses);
+
+      const oldRecordTypes = await NodeRecordsRepository.$getRecordTypes(node.pub_key);
+      const customRecordTypes: number[] = [];
+      for (const [type, payload] of Object.entries(node.custom_records || {})) {
+        const numericalType = parseInt(type);
+        await NodeRecordsRepository.$saveRecord({
+          publicKey: node.pub_key,
+          type: numericalType,
+          payload,
+        });
+        customRecordTypes.push(numericalType);
+      }
+      if (oldRecordTypes.reduce((changed, type) => changed || customRecordTypes.indexOf(type) === -1, false)) {
+        deletedRecords += await NodeRecordsRepository.$deleteUnusedRecords(node.pub_key, customRecordTypes);
+      }
     }
-    logger.info(`${progress} nodes updated. ${deletedSockets} sockets deleted`);
+    logger.debug(`${progress} nodes updated. ${deletedSockets} sockets deleted. ${deletedRecords} custom records deleted.`);
 
     // If a channel if not present in the graph, mark it as inactive
     await nodesApi.$setNodesInactive(graphNodesPubkeys);
@@ -117,18 +138,18 @@ class NetworkSyncService {
         ++progress;
 
         const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
-        if (elapsedSeconds > 10) {
+        if (elapsedSeconds > config.LIGHTNING.LOGGER_UPDATE_INTERVAL) {
-          logger.info(`Updating channel ${progress}/${channels.length}`);
+          logger.debug(`Updating channel ${progress}/${channels.length}`, logger.tags.ln);
           this.loggerTimer = new Date().getTime() / 1000;
         }
       }
 
-      logger.info(`${progress} channels updated`);
+      logger.debug(`${progress} channels updated`, logger.tags.ln);
 
       // If a channel if not present in the graph, mark it as inactive
       await channelsApi.$setChannelsInactive(graphChannelsIds);
     } catch (e) {
-      logger.err(`Cannot update channel list. Reason: ${(e instanceof Error ? e.message : e)}`);
+      logger.err(` Cannot update channel list. Reason: ${(e instanceof Error ? e.message : e)}`, logger.tags.ln);
     }
   }
 
@@ -163,26 +184,28 @@ class NetworkSyncService {
         if (lowest < node.first_seen) {
           const query = `UPDATE nodes SET first_seen = FROM_UNIXTIME(?) WHERE public_key = ?`;
           const params = [lowest, node.public_key];
+          ++updated;
           await DB.query(query, params);
         }
         ++progress;
         const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
-        if (elapsedSeconds > 10) {
+        if (elapsedSeconds > config.LIGHTNING.LOGGER_UPDATE_INTERVAL) {
-          logger.info(`Updating node first seen date ${progress}/${nodes.length}`);
+          logger.debug(`Updating node first seen date ${progress}/${nodes.length}`, logger.tags.ln);
           this.loggerTimer = new Date().getTime() / 1000;
-          ++updated;
         }
       }
-      logger.info(`Updated ${updated} node first seen dates`);
+      if (updated > 0) {
+        logger.debug(`Updated ${updated} node first seen dates`, logger.tags.ln);
+      }
     } catch (e) {
-      logger.err('$updateNodeFirstSeen() error: ' + (e instanceof Error ? e.message : e));
+      logger.err(`$updateNodeFirstSeen() error: ${e instanceof Error ? e.message : e}`, logger.tags.ln);
     }
   }
 
   private async $lookUpCreationDateFromChain(): Promise<void> {
     let progress = 0;
 
-    logger.info(`Running channel creation date lookup`);
+    logger.debug(`Running channel creation date lookup`, logger.tags.ln);
     try {
       const channels = await channelsApi.$getChannelsWithoutCreatedDate();
       for (const channel of channels) {
@@ -193,14 +216,17 @@ class NetworkSyncService {
         );
         ++progress;
         const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
-        if (elapsedSeconds > 10) {
+        if (elapsedSeconds > config.LIGHTNING.LOGGER_UPDATE_INTERVAL) {
-          logger.info(`Updating channel creation date ${progress}/${channels.length}`);
+          logger.debug(`Updating channel creation date ${progress}/${channels.length}`, logger.tags.ln);
           this.loggerTimer = new Date().getTime() / 1000;
         }
       }
-      logger.info(`Updated ${channels.length} channels' creation date`);
+
+      if (channels.length > 0) {
+        logger.debug(`Updated ${channels.length} channels' creation date`, logger.tags.ln);
+      }      
     } catch (e) {
-      logger.err('$lookUpCreationDateFromChain() error: ' + (e instanceof Error ? e.message : e));
+      logger.err(`$lookUpCreationDateFromChain() error: ${e instanceof Error ? e.message : e}`, logger.tags.ln);
     }
   }
 
@@ -209,7 +235,7 @@ class NetworkSyncService {
    * mark that channel as inactive
    */
   private async $deactivateChannelsWithoutActiveNodes(): Promise<void> {
-    logger.info(`Find channels which nodes are offline`);
+    logger.debug(`Find channels which nodes are offline`, logger.tags.ln);
 
     try {
       const result = await DB.query<ResultSetHeader>(`
@@ -232,12 +258,10 @@ class NetworkSyncService {
         `);
 
       if (result[0].changedRows ?? 0 > 0) {
-        logger.info(`Marked ${result[0].changedRows} channels as inactive because they are not linked to any active node`);
+        logger.debug(`Marked ${result[0].changedRows} channels as inactive because they are not linked to any active node`, logger.tags.ln);
-      } else {
-        logger.debug(`Marked ${result[0].changedRows} channels as inactive because they are not linked to any active node`);
       }
     } catch (e) {
-      logger.err('$deactivateChannelsWithoutActiveNodes() error: ' + (e instanceof Error ? e.message : e));
+      logger.err(`$deactivateChannelsWithoutActiveNodes() error: ${e instanceof Error ? e.message : e}`, logger.tags.ln);
     }
   }
 
@@ -256,13 +280,13 @@ class NetworkSyncService {
       } else {
         log += ` for the first time`;
       }
-      logger.info(log);
+      logger.info(`${log}`, logger.tags.ln);
 
       const channels = await channelsApi.$getChannelsByStatus([0, 1]);
       for (const channel of channels) {
         const spendingTx = await bitcoinApi.$getOutspend(channel.transaction_id, channel.transaction_vout);
         if (spendingTx.spent === true && spendingTx.status?.confirmed === true) {
-          logger.debug('Marking channel: ' + channel.id + ' as closed.');
+          logger.debug(`Marking channel: ${channel.id} as closed.`, logger.tags.ln);
           await DB.query(`UPDATE channels SET status = 2, closing_date = FROM_UNIXTIME(?) WHERE id = ?`,
             [spendingTx.status.block_time, channel.id]);
           if (spendingTx.txid && !channel.closing_transaction_id) {
@@ -272,160 +296,18 @@ class NetworkSyncService {
 
         ++progress;
         const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
-        if (elapsedSeconds > 10) {
+        if (elapsedSeconds > config.LIGHTNING.LOGGER_UPDATE_INTERVAL) {
-          logger.info(`Checking if channel has been closed ${progress}/${channels.length}`);
+          logger.info(`Checking if channel has been closed ${progress}/${channels.length}`, logger.tags.ln);
           this.loggerTimer = new Date().getTime() / 1000;
         }
       }
 
       this.closedChannelsScanBlock = blocks.getCurrentBlockHeight();
-      logger.info(`Closed channels scan completed at block ${this.closedChannelsScanBlock}`);
+      logger.debug(`Closed channels scan completed at block ${this.closedChannelsScanBlock}`, logger.tags.ln);
     } catch (e) {
-      logger.err('$scanForClosedChannels() error: ' + (e instanceof Error ? e.message : e));
+      logger.err(`$scanForClosedChannels() error: ${e instanceof Error ? e.message : e}`, logger.tags.ln);
     }
   }
-
-  /*
-    1. Mutually closed
-    2. Forced closed
-    3. Forced closed with penalty
-  */
-
-  private async $runClosedChannelsForensics(): Promise<void> {
-    if (!config.ESPLORA.REST_API_URL) {
-      return;
-    }
-
-    let progress = 0;
-
-    try {
-      logger.info(`Started running closed channel forensics...`);
-      const channels = await channelsApi.$getClosedChannelsWithoutReason();
-      for (const channel of channels) {
-        let reason = 0;
-        // Only Esplora backend can retrieve spent transaction outputs
-        try {
-          let outspends: IEsploraApi.Outspend[] | undefined;
-          try {
-            outspends = await bitcoinApi.$getOutspends(channel.closing_transaction_id);
-          } catch (e) {
-            logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + channel.closing_transaction_id + '/outspends'}. Reason ${e instanceof Error ? e.message : e}`);
-            continue;
-          }
-          const lightningScriptReasons: number[] = [];
-          for (const outspend of outspends) {
-            if (outspend.spent && outspend.txid) {
-              let spendingTx: IEsploraApi.Transaction | undefined;
-              try {
-                spendingTx = await bitcoinApi.$getRawTransaction(outspend.txid);
-              } catch (e) {
-                logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + outspend.txid}. Reason ${e instanceof Error ? e.message : e}`);
-                continue;
-              }
-              const lightningScript = this.findLightningScript(spendingTx.vin[outspend.vin || 0]);
-              lightningScriptReasons.push(lightningScript);
-            }
-          }
-          if (lightningScriptReasons.length === outspends.length
-            && lightningScriptReasons.filter((r) => r === 1).length === outspends.length) {
-            reason = 1;
-          } else {
-            const filteredReasons = lightningScriptReasons.filter((r) => r !== 1);
-            if (filteredReasons.length) {
-              if (filteredReasons.some((r) => r === 2 || r === 4)) {
-                reason = 3;
-              } else {
-                reason = 2;
-              }
-            } else {
-              /*
-                We can detect a commitment transaction (force close) by reading Sequence and Locktime
-                https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction
-              */
-              let closingTx: IEsploraApi.Transaction | undefined;
-              try {
-                closingTx = await bitcoinApi.$getRawTransaction(channel.closing_transaction_id);
-              } catch (e) {
-                logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + channel.closing_transaction_id}. Reason ${e instanceof Error ? e.message : e}`);
-                continue;
-              }
-              const sequenceHex: string = closingTx.vin[0].sequence.toString(16);
-              const locktimeHex: string = closingTx.locktime.toString(16);
-              if (sequenceHex.substring(0, 2) === '80' && locktimeHex.substring(0, 2) === '20') {
-                reason = 2; // Here we can't be sure if it's a penalty or not
-              } else {
-                reason = 1;
-              }
-            }
-          }
-          if (reason) {
-            logger.debug('Setting closing reason ' + reason + ' for channel: ' + channel.id + '.');
-            await DB.query(`UPDATE channels SET closing_reason = ? WHERE id = ?`, [reason, channel.id]);
-          }
-        } catch (e) {
-          logger.err(`$runClosedChannelsForensics() failed for channel ${channel.short_id}. Reason: ${e instanceof Error ? e.message : e}`);
-        }
-
-        ++progress;
-        const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
-        if (elapsedSeconds > 10) {
-          logger.info(`Updating channel closed channel forensics ${progress}/${channels.length}`);
-          this.loggerTimer = new Date().getTime() / 1000;
-        }
-      }
-      logger.info(`Closed channels forensics scan complete.`);
-    } catch (e) {
-      logger.err('$runClosedChannelsForensics() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  private findLightningScript(vin: IEsploraApi.Vin): number {
-    const topElement = vin.witness[vin.witness.length - 2];
-      if (/^OP_IF OP_PUSHBYTES_33 \w{66} OP_ELSE OP_PUSH(NUM_\d+|BYTES_(1 \w{2}|2 \w{4})) OP_CSV OP_DROP OP_PUSHBYTES_33 \w{66} OP_ENDIF OP_CHECKSIG$/.test(vin.inner_witnessscript_asm)) {
-        // https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction-outputs
-        if (topElement === '01') {
-          // top element is '01' to get in the revocation path
-          // 'Revoked Lightning Force Close';
-          // Penalty force closed
-          return 2;
-        } else {
-          // top element is '', this is a delayed to_local output
-          // 'Lightning Force Close';
-          return 3;
-        }
-      } else if (
-        /^OP_DUP OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUAL OP_IF OP_CHECKSIG OP_ELSE OP_PUSHBYTES_33 \w{66} OP_SWAP OP_SIZE OP_PUSHBYTES_1 20 OP_EQUAL OP_NOTIF OP_DROP OP_PUSHNUM_2 OP_SWAP OP_PUSHBYTES_33 \w{66} OP_PUSHNUM_2 OP_CHECKMULTISIG OP_ELSE OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUALVERIFY OP_CHECKSIG OP_ENDIF (OP_PUSHNUM_1 OP_CSV OP_DROP |)OP_ENDIF$/.test(vin.inner_witnessscript_asm) ||
-        /^OP_DUP OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUAL OP_IF OP_CHECKSIG OP_ELSE OP_PUSHBYTES_33 \w{66} OP_SWAP OP_SIZE OP_PUSHBYTES_1 20 OP_EQUAL OP_IF OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUALVERIFY OP_PUSHNUM_2 OP_SWAP OP_PUSHBYTES_33 \w{66} OP_PUSHNUM_2 OP_CHECKMULTISIG OP_ELSE OP_DROP OP_PUSHBYTES_3 \w{6} OP_CLTV OP_DROP OP_CHECKSIG OP_ENDIF (OP_PUSHNUM_1 OP_CSV OP_DROP |)OP_ENDIF$/.test(vin.inner_witnessscript_asm)
-      ) {
-        // https://github.com/lightning/bolts/blob/master/03-transactions.md#offered-htlc-outputs
-        // https://github.com/lightning/bolts/blob/master/03-transactions.md#received-htlc-outputs
-        if (topElement.length === 66) {
-          // top element is a public key
-          // 'Revoked Lightning HTLC'; Penalty force closed
-          return 4;
-        } else if (topElement) {
-          // top element is a preimage
-          // 'Lightning HTLC';
-          return 5;
-        } else {
-          // top element is '' to get in the expiry of the script
-          // 'Expired Lightning HTLC';
-          return 6;
-        }
-      } else if (/^OP_PUSHBYTES_33 \w{66} OP_CHECKSIG OP_IFDUP OP_NOTIF OP_PUSHNUM_16 OP_CSV OP_ENDIF$/.test(vin.inner_witnessscript_asm)) {
-        // https://github.com/lightning/bolts/blob/master/03-transactions.md#to_local_anchor-and-to_remote_anchor-output-option_anchors
-        if (topElement) {
-          // top element is a signature
-          // 'Lightning Anchor';
-          return 7;
-        } else {
-          // top element is '', it has been swept after 16 blocks
-          // 'Swept Lightning Anchor';
-          return 8;
-        }
-      }
-      return 1;
-  }
 }
 
 export default new NetworkSyncService();

backend/src/tasks/lightning/stats-updater.service.ts

@@ -6,7 +6,7 @@ import { Common } from '../../api/common';
 
 class LightningStatsUpdater {
   public async $startService(): Promise<void> {
-    logger.info('Starting Lightning Stats service');
+    logger.info(`Starting Lightning Stats service`, logger.tags.ln);
 
     await this.$runTasks();
     LightningStatsImporter.$run();
@@ -27,7 +27,7 @@ class LightningStatsUpdater {
     const networkGraph = await lightningApi.$getNetworkGraph();
     await LightningStatsImporter.computeNetworkStats(date.getTime() / 1000, networkGraph);
     
-    logger.info(`Updated latest network stats`);
+    logger.debug(`Updated latest network stats`, logger.tags.ln);
   }
 }
 

backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts

@@ -21,10 +21,10 @@ class FundingTxFetcher {
       try {
         this.fundingTxCache = JSON.parse(await fsPromises.readFile(CACHE_FILE_NAME, 'utf-8'));
       } catch (e) {
-        logger.err(`Unable to parse channels funding txs disk cache. Starting from scratch`);
+        logger.err(`Unable to parse channels funding txs disk cache. Starting from scratch`, logger.tags.ln);
         this.fundingTxCache = {};
       }
-      logger.debug(`Imported ${Object.keys(this.fundingTxCache).length} funding tx amount from the disk cache`);
+      logger.debug(`Imported ${Object.keys(this.fundingTxCache).length} funding tx amount from the disk cache`, logger.tags.ln);
     }
   }
 
@@ -44,26 +44,27 @@ class FundingTxFetcher {
       ++channelProcessed;
 
       let elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer);
-      if (elapsedSeconds > 10) {
+      if (elapsedSeconds > config.LIGHTNING.LOGGER_UPDATE_INTERVAL) {
         elapsedSeconds = Math.round((new Date().getTime() / 1000) - globalTimer);
         logger.info(`Indexing channels funding tx ${channelProcessed + 1} of ${channelIds.length} ` +
           `(${Math.floor(channelProcessed / channelIds.length * 10000) / 100}%) | ` +
-          `elapsed: ${elapsedSeconds} seconds`
+          `elapsed: ${elapsedSeconds} seconds`,
+          logger.tags.ln
         );
         loggerTimer = new Date().getTime() / 1000;
       }
 
       elapsedSeconds = Math.round((new Date().getTime() / 1000) - cacheTimer);
       if (elapsedSeconds > 60) {
-        logger.debug(`Saving ${Object.keys(this.fundingTxCache).length} funding txs cache into disk`);
+        logger.debug(`Saving ${Object.keys(this.fundingTxCache).length} funding txs cache into disk`, logger.tags.ln);
         fsPromises.writeFile(CACHE_FILE_NAME, JSON.stringify(this.fundingTxCache));
         cacheTimer = new Date().getTime() / 1000;
       }
     }
 
     if (this.channelNewlyProcessed > 0) {
-      logger.info(`Indexed ${this.channelNewlyProcessed} additional channels funding tx`);
+      logger.info(`Indexed ${this.channelNewlyProcessed} additional channels funding tx`, logger.tags.ln);
-      logger.debug(`Saving ${Object.keys(this.fundingTxCache).length} funding txs cache into disk`);
+      logger.debug(`Saving ${Object.keys(this.fundingTxCache).length} funding txs cache into disk`, logger.tags.ln);
       fsPromises.writeFile(CACHE_FILE_NAME, JSON.stringify(this.fundingTxCache));
     }
 

backend/src/tasks/lightning/sync-tasks/node-locations.ts

@@ -6,6 +6,7 @@ import DB from '../../../database';
 import logger from '../../../logger';
 import { ResultSetHeader } from 'mysql2';
 import * as IPCheck from '../../../utils/ipcheck.js';
+import { Reader } from 'mmdb-lib';
 
 export async function $lookupNodeLocation(): Promise<void> {
   let loggerTimer = new Date().getTime() / 1000;
@@ -13,12 +14,15 @@ export async function $lookupNodeLocation(): Promise<void> {
   let nodesUpdated = 0;
   let geoNamesInserted = 0;
 
-  logger.info(`Running node location updater using Maxmind`);
+  logger.debug(`Running node location updater using Maxmind`, logger.tags.ln);
   try {
     const nodes = await nodesApi.$getAllNodes();
     const lookupCity = await maxmind.open<CityResponse>(config.MAXMIND.GEOLITE2_CITY);
     const lookupAsn = await maxmind.open<AsnResponse>(config.MAXMIND.GEOLITE2_ASN);
-    const lookupIsp = await maxmind.open<IspResponse>(config.MAXMIND.GEOIP2_ISP);
+    let lookupIsp: Reader<IspResponse> | null = null;
+    try {
+      lookupIsp = await maxmind.open<IspResponse>(config.MAXMIND.GEOIP2_ISP);
+    } catch (e) { }
 
     for (const node of nodes) {
       const sockets: string[] = node.sockets.split(',');
@@ -29,7 +33,10 @@ export async function $lookupNodeLocation(): Promise<void> {
         if (hasClearnet && ip !== '127.0.1.1' && ip !== '127.0.0.1') {
           const city = lookupCity.get(ip);
           const asn = lookupAsn.get(ip);
-          const isp = lookupIsp.get(ip);
+          let isp: IspResponse | null = null;
+          if (lookupIsp) {
+            isp = lookupIsp.get(ip);
+          }
 
           let asOverwrite: any | undefined;
           if (asn && (IPCheck.match(ip, '170.75.160.0/20') || IPCheck.match(ip, '172.81.176.0/21'))) {
@@ -145,8 +152,8 @@ export async function $lookupNodeLocation(): Promise<void> {
 
           ++progress;
           const elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer);
-          if (elapsedSeconds > 10) {
+          if (elapsedSeconds > config.LIGHTNING.LOGGER_UPDATE_INTERVAL) {
-            logger.info(`Updating node location data ${progress}/${nodes.length}`);
+            logger.debug(`Updating node location data ${progress}/${nodes.length}`);
             loggerTimer = new Date().getTime() / 1000;
           }
         }
@@ -154,9 +161,7 @@ export async function $lookupNodeLocation(): Promise<void> {
     }
 
     if (nodesUpdated > 0) {
-      logger.info(`${nodesUpdated} nodes maxmind data updated, ${geoNamesInserted} geo names inserted`);
+      logger.debug(`${nodesUpdated} nodes maxmind data updated, ${geoNamesInserted} geo names inserted`, logger.tags.ln);
-    } else {
-      logger.debug(`${nodesUpdated} nodes maxmind data updated, ${geoNamesInserted} geo names inserted`);
     }
   } catch (e) {
     logger.err('$lookupNodeLocation() error: ' + (e instanceof Error ? e.message : e));

backend/src/tasks/lightning/sync-tasks/stats-importer.ts

@@ -8,7 +8,6 @@ import { isIP } from 'net';
 import { Common } from '../../../api/common';
 import channelsApi from '../../../api/explorer/channels.api';
 import nodesApi from '../../../api/explorer/nodes.api';
-import { ResultSetHeader } from 'mysql2';
 
 const fsPromises = promises;
 
@@ -17,7 +16,7 @@ class LightningStatsImporter {
 
   async $run(): Promise<void> {
     const [channels]: any[] = await DB.query('SELECT short_id from channels;');
-    logger.info('Caching funding txs for currently existing channels');
+    logger.info(`Caching funding txs for currently existing channels`, logger.tags.ln);
     await fundingTxFetcher.$fetchChannelsFundingTxs(channels.map(channel => channel.short_id));
 
     if (config.MEMPOOL.NETWORK !== 'mainnet' || config.DATABASE.ENABLED === false) {
@@ -108,7 +107,7 @@ class LightningStatsImporter {
 
       const tx = await fundingTxFetcher.$fetchChannelOpenTx(short_id);
       if (!tx) {
-        logger.err(`Unable to fetch funding tx for channel ${short_id}. Capacity and creation date is unknown. Skipping channel.`);
+        logger.err(`Unable to fetch funding tx for channel ${short_id}. Capacity and creation date is unknown. Skipping channel.`, logger.tags.ln);
         continue;
       }
 
@@ -310,13 +309,18 @@ class LightningStatsImporter {
    * Import topology files LN historical data into the database
    */
   async $importHistoricalLightningStats(): Promise<void> {
+    if (!config.LIGHTNING.TOPOLOGY_FOLDER) {
+      logger.info(`Lightning topology folder is not set. Not importing historical LN stats`);
+      return;
+    }
+
     logger.debug('Run the historical importer');
     try {
       let fileList: string[] = [];
       try {
         fileList = await fsPromises.readdir(this.topologiesFolder);
       } catch (e) {
-        logger.err(`Unable to open topology folder at ${this.topologiesFolder}`);
+        logger.err(`Unable to open topology folder at ${this.topologiesFolder}`, logger.tags.ln);
         throw e;
       }
       // Insert history from the most recent to the oldest
@@ -354,7 +358,7 @@ class LightningStatsImporter {
           continue;
         }
 
-        logger.debug(`Reading ${this.topologiesFolder}/${filename}`);
+        logger.debug(`Reading ${this.topologiesFolder}/${filename}`, logger.tags.ln);
         let fileContent = '';
         try {
           fileContent = await fsPromises.readFile(`${this.topologiesFolder}/${filename}`, 'utf8');
@@ -363,7 +367,7 @@ class LightningStatsImporter {
             totalProcessed++;
             continue;
           }
-          logger.err(`Unable to open ${this.topologiesFolder}/${filename}`);
+          logger.err(`Unable to open ${this.topologiesFolder}/${filename}`, logger.tags.ln);
           totalProcessed++;
           continue;
         }
@@ -373,7 +377,7 @@ class LightningStatsImporter {
           graph = JSON.parse(fileContent);
           graph = await this.cleanupTopology(graph);
         } catch (e) {
-          logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content. Reason: ${e instanceof Error ? e.message : e}`);
+          logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.ln);
           totalProcessed++;
           continue;
         }
@@ -385,20 +389,20 @@ class LightningStatsImporter {
         }
 
         if (!logStarted) {
-          logger.info(`Founds a topology file that we did not import. Importing historical lightning stats now.`);
+          logger.info(`Founds a topology file that we did not import. Importing historical lightning stats now.`, logger.tags.ln);
           logStarted = true;
         }
         
         const datestr = `${new Date(timestamp * 1000).toUTCString()} (${timestamp})`;
-        logger.debug(`${datestr}: Found ${graph.nodes.length} nodes and ${graph.edges.length} channels`);
+        logger.debug(`${datestr}: Found ${graph.nodes.length} nodes and ${graph.edges.length} channels`, logger.tags.ln);
 
         totalProcessed++;
 
         if (processed > 10) {
-          logger.info(`Generating LN network stats for ${datestr}. Processed ${totalProcessed}/${fileList.length} files`);
+          logger.info(`Generating LN network stats for ${datestr}. Processed ${totalProcessed}/${fileList.length} files`, logger.tags.ln);
           processed = 0;
         } else {
-          logger.debug(`Generating LN network stats for ${datestr}. Processed ${totalProcessed}/${fileList.length} files`);
+          logger.debug(`Generating LN network stats for ${datestr}. Processed ${totalProcessed}/${fileList.length} files`, logger.tags.ln);
         }
         await fundingTxFetcher.$fetchChannelsFundingTxs(graph.edges.map(channel => channel.channel_id.slice(0, -2)));
         const stat = await this.computeNetworkStats(timestamp, graph, true);
@@ -407,10 +411,10 @@ class LightningStatsImporter {
       }
 
       if (totalProcessed > 0) {
-        logger.info(`Lightning network stats historical import completed`);
+        logger.notice(`Lightning network stats historical import completed`, logger.tags.ln);
       }
     } catch (e) {
-      logger.err(`Lightning network stats historical failed. Reason: ${e instanceof Error ? e.message : e}`);
+      logger.err(`Lightning network stats historical failed. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.ln);
     }
   }
 

backend/src/tasks/pools-updater.ts

@@ -32,9 +32,9 @@ class PoolsUpdater {
     this.lastRun = now;
 
     if (config.SOCKS5PROXY.ENABLED) {
-      logger.info(`Updating latest mining pools from ${this.poolsUrl} over the Tor network`);
+      logger.info(`Updating latest mining pools from ${this.poolsUrl} over the Tor network`, logger.tags.mining);
     } else {
-      logger.info(`Updating latest mining pools from ${this.poolsUrl} over clearnet`);
+      logger.info(`Updating latest mining pools from ${this.poolsUrl} over clearnet`, logger.tags.mining);
     }
 
     try {
@@ -53,9 +53,9 @@ class PoolsUpdater {
       }
 
       if (this.currentSha === undefined) {
-        logger.info(`Downloading pools.json for the first time from ${this.poolsUrl}`);
+        logger.info(`Downloading pools.json for the first time from ${this.poolsUrl}`, logger.tags.mining);
       } else {
-        logger.warn(`Pools.json is outdated, fetch latest from ${this.poolsUrl}`);
+        logger.warn(`Pools.json is outdated, fetch latest from ${this.poolsUrl}`, logger.tags.mining);
       }
       const poolsJson = await this.query(this.poolsUrl);
       if (poolsJson === undefined) {
@@ -63,11 +63,11 @@ class PoolsUpdater {
       }
       await poolsParser.migratePoolsJson(poolsJson);
       await this.updateDBSha(githubSha);
-      logger.notice('PoolsUpdater completed');
+      logger.notice(`PoolsUpdater completed`, logger.tags.mining);
 
     } catch (e) {
       this.lastRun = now - (oneWeek - oneDay); // Try again in 24h instead of waiting next week
-      logger.err('PoolsUpdater failed. Will try again in 24h. Reason: ' + (e instanceof Error ? e.message : e));
+      logger.err(`PoolsUpdater failed. Will try again in 24h. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.mining);
     }
   }
 
@@ -81,7 +81,7 @@ class PoolsUpdater {
         await DB.query('DELETE FROM state where name="pools_json_sha"');
         await DB.query(`INSERT INTO state VALUES('pools_json_sha', NULL, '${githubSha}')`);
       } catch (e) {
-        logger.err('Cannot save github pools.json sha into the db. Reason: ' + (e instanceof Error ? e.message : e));
+        logger.err('Cannot save github pools.json sha into the db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
       }
     }
   }
@@ -94,7 +94,7 @@ class PoolsUpdater {
       const [rows]: any[] = await DB.query('SELECT string FROM state WHERE name="pools_json_sha"');
       return (rows.length > 0 ? rows[0].string : undefined);
     } catch (e) {
-      logger.err('Cannot fetch pools.json sha from db. Reason: ' + (e instanceof Error ? e.message : e));
+      logger.err('Cannot fetch pools.json sha from db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
       return undefined;
     }
   }
@@ -113,7 +113,7 @@ class PoolsUpdater {
       }
     }
 
-    logger.err(`Cannot find "pools.json" in git tree (${this.treeUrl})`);
+    logger.err(`Cannot find "pools.json" in git tree (${this.treeUrl})`, logger.tags.mining);
     return undefined;
   }
 

backend/src/tasks/price-feeds/ftx-api.ts

@@ -1,43 +0,0 @@
-import { query } from '../../utils/axios-query';
-import priceUpdater, { PriceFeed, PriceHistory } from '../price-updater';
-
-class FtxApi implements PriceFeed {
-  public name: string = 'FTX';
-  public currencies: string[] = ['USD', 'BRZ', 'EUR', 'JPY', 'AUD'];
-
-  public url: string = 'https://ftx.com/api/markets/BTC/';
-  public urlHist: string = 'https://ftx.com/api/markets/BTC/{CURRENCY}/candles?resolution={GRANULARITY}';
-
-  constructor() {
-  }
-
-  public async $fetchPrice(currency): Promise<number> {
-    const response = await query(this.url + currency);
-    return response ? parseInt(response['result']['last'], 10) : -1;
-  }
-
-  public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {
-    const priceHistory: PriceHistory = {};
-
-    for (const currency of currencies) {
-      if (this.currencies.includes(currency) === false) {
-        continue;
-      }
-
-      const response = await query(this.urlHist.replace('{GRANULARITY}', type === 'hour' ? '3600' : '86400').replace('{CURRENCY}', currency));
-      const pricesRaw = response ? response['result'] : [];
-
-      for (const price of pricesRaw as any[]) {
-        const time = Math.round(price['time'] / 1000);
-        if (priceHistory[time] === undefined) {
-          priceHistory[time] = priceUpdater.getEmptyPricesObj();
-        }
-        priceHistory[time][currency] = price['close'];
-      }
-    }
-
-    return priceHistory;
-  }
-}
-
-export default FtxApi;

backend/src/tasks/price-feeds/kraken-api.ts

@@ -91,7 +91,7 @@ class KrakenApi implements PriceFeed {
     }
 
     if (Object.keys(priceHistory).length > 0) {
-      logger.notice(`Inserted ${Object.keys(priceHistory).length} Kraken EUR, USD, GBP, JPY, CAD, CHF and AUD weekly price history into db`);
+      logger.notice(`Inserted ${Object.keys(priceHistory).length} Kraken EUR, USD, GBP, JPY, CAD, CHF and AUD weekly price history into db`, logger.tags.mining);
     }
   }
 }

backend/src/tasks/price-updater.ts

@@ -1,13 +1,11 @@
 import * as fs from 'fs';
 import path from "path";
-import { Common } from '../api/common';
 import config from '../config';
 import logger from '../logger';
 import PricesRepository from '../repositories/PricesRepository';
 import BitfinexApi from './price-feeds/bitfinex-api';
 import BitflyerApi from './price-feeds/bitflyer-api';
 import CoinbaseApi from './price-feeds/coinbase-api';
-import FtxApi from './price-feeds/ftx-api';
 import GeminiApi from './price-feeds/gemini-api';
 import KrakenApi from './price-feeds/kraken-api';
 
@@ -48,7 +46,6 @@ class PriceUpdater {
     this.latestPrices = this.getEmptyPricesObj();
 
     this.feeds.push(new BitflyerApi()); // Does not have historical endpoint
-    this.feeds.push(new FtxApi());
     this.feeds.push(new KrakenApi());
     this.feeds.push(new CoinbaseApi());
     this.feeds.push(new BitfinexApi());
@@ -85,7 +82,7 @@ class PriceUpdater {
         await this.$updatePrice();
       }
     } catch (e) {
-      logger.err(`Cannot save BTC prices in db. Reason: ${e instanceof Error ? e.message : e}`);
+      logger.err(`Cannot save BTC prices in db. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.mining);
     }
 
     this.running = false;
@@ -118,14 +115,14 @@ class PriceUpdater {
             if (price > 0) {
               prices.push(price);
             }
-            logger.debug(`${feed.name} BTC/${currency} price: ${price}`);
+            logger.debug(`${feed.name} BTC/${currency} price: ${price}`, logger.tags.mining);
           } catch (e) {
-            logger.debug(`Could not fetch BTC/${currency} price at ${feed.name}. Reason: ${(e instanceof Error ? e.message : e)}`);
+            logger.debug(`Could not fetch BTC/${currency} price at ${feed.name}. Reason: ${(e instanceof Error ? e.message : e)}`, logger.tags.mining);
           }
         }
       }
       if (prices.length === 1) {
-        logger.debug(`Only ${prices.length} feed available for BTC/${currency} price`);
+        logger.debug(`Only ${prices.length} feed available for BTC/${currency} price`, logger.tags.mining);
       }
 
       // Compute average price, non weighted
@@ -178,9 +175,9 @@ class PriceUpdater {
       ++insertedCount;
     }
     if (insertedCount > 0) {
-      logger.notice(`Inserted ${insertedCount} MtGox USD weekly price history into db`);
+      logger.notice(`Inserted ${insertedCount} MtGox USD weekly price history into db`, logger.tags.mining);
     } else {
-      logger.debug(`Inserted ${insertedCount} MtGox USD weekly price history into db`);
+      logger.debug(`Inserted ${insertedCount} MtGox USD weekly price history into db`, logger.tags.mining);
     }
 
     // Insert Kraken weekly prices
@@ -201,7 +198,7 @@ class PriceUpdater {
   private async $insertMissingRecentPrices(type: 'hour' | 'day'): Promise<void> {
     const existingPriceTimes = await PricesRepository.$getPricesTimes();
 
-    logger.info(`Fetching ${type === 'day' ? 'dai' : 'hour'}ly price history from exchanges and saving missing ones into the database, this may take a while`);
+    logger.info(`Fetching ${type === 'day' ? 'dai' : 'hour'}ly price history from exchanges and saving missing ones into the database`, logger.tags.mining);
 
     const historicalPrices: PriceHistory[] = [];
 
@@ -210,7 +207,7 @@ class PriceUpdater {
       try {
         historicalPrices.push(await feed.$fetchRecentPrice(this.currencies, type));
       } catch (e) {
-        logger.err(`Cannot fetch hourly historical price from ${feed.name}. Ignoring this feed. Reason: ${e instanceof Error ? e.message : e}`);
+        logger.err(`Cannot fetch hourly historical price from ${feed.name}. Ignoring this feed. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.mining);
       }
     }
 
@@ -255,9 +252,9 @@ class PriceUpdater {
     }
 
     if (totalInserted > 0) {
-      logger.notice(`Inserted ${totalInserted} ${type === 'day' ? 'dai' : 'hour'}ly historical prices into the db`);
+      logger.notice(`Inserted ${totalInserted} ${type === 'day' ? 'dai' : 'hour'}ly historical prices into the db`, logger.tags.mining);
     } else {
-      logger.debug(`Inserted ${totalInserted} ${type === 'day' ? 'dai' : 'hour'}ly historical prices into the db`);
+      logger.debug(`Inserted ${totalInserted} ${type === 'day' ? 'dai' : 'hour'}ly historical prices into the db`, logger.tags.mining);
     }
   }
 }

backend/src/utils/pairing-heap.ts

@@ -0,0 +1,174 @@
+export type HeapNode<T> = {
+  element: T
+  child?: HeapNode<T>
+  next?: HeapNode<T>
+  prev?: HeapNode<T>
+} | null | undefined;
+
+// minimal pairing heap priority queue implementation
+export class PairingHeap<T> {
+  private root: HeapNode<T> = null;
+  private comparator: (a: T, b: T) => boolean;
+
+  // comparator function should return 'true' if a is higher priority than b
+  constructor(comparator: (a: T, b: T) => boolean) {
+    this.comparator = comparator;
+  }
+
+  isEmpty(): boolean {
+    return !this.root;
+  }
+
+  add(element: T): HeapNode<T> {
+    const node: HeapNode<T> = {
+      element
+    };
+
+    this.root = this.meld(this.root, node);
+
+    return node;
+  }
+
+  // returns the top priority element without modifying the queue
+  peek(): T | void {
+    return this.root?.element;
+  }
+
+  // removes and returns the top priority element
+  pop(): T | void {
+    let element;
+    if (this.root) {
+      const node = this.root;
+      element = node.element;
+      this.root = this.mergePairs(node.child);
+    }
+    return element;
+  }
+
+  deleteNode(node: HeapNode<T>): void {
+    if (!node) {
+      return;
+    }
+
+    if (node === this.root) {
+      this.root = this.mergePairs(node.child);
+    }
+    else {
+      if (node.prev) {
+        if (node.prev.child === node) {
+          node.prev.child = node.next;
+        }
+        else {
+          node.prev.next = node.next;
+        }
+      }
+      if (node.next) {
+        node.next.prev = node.prev;
+      }
+      this.root = this.meld(this.root, this.mergePairs(node.child));
+    }
+
+    node.child = null;
+    node.prev = null;
+    node.next = null;
+  }
+
+  // fix the heap after increasing the priority of a given node
+  increasePriority(node: HeapNode<T>): void {
+    // already the top priority element
+    if (!node || node === this.root) {
+      return;
+    }
+    // extract from siblings
+    if (node.prev) {
+      if (node.prev?.child === node) {
+        if (this.comparator(node.prev.element, node.element)) {
+          // already in a valid position
+          return;
+        }
+        node.prev.child = node.next;
+      }
+      else {
+        node.prev.next = node.next;
+      }
+    }
+    if (node.next) {
+      node.next.prev = node.prev;
+    }
+
+    this.root = this.meld(this.root, node);
+  }
+
+  decreasePriority(node: HeapNode<T>): void {
+    this.deleteNode(node);
+    this.root = this.meld(this.root, node);
+  }
+
+  meld(a: HeapNode<T>, b: HeapNode<T>): HeapNode<T> {
+    if (!a) {
+      return b;
+    }
+    if (!b || a === b) {
+      return a;
+    }
+
+    let parent: HeapNode<T> = b;
+    let child: HeapNode<T> = a;
+    if (this.comparator(a.element, b.element)) {
+      parent = a;
+      child = b;
+    }
+
+    child.next = parent.child;
+    if (parent.child) {
+      parent.child.prev = child;
+    }
+    child.prev = parent;
+    parent.child = child;
+
+    parent.next = null;
+    parent.prev = null;
+
+    return parent;
+  }
+
+  mergePairs(node: HeapNode<T>): HeapNode<T> {
+    if (!node) {
+      return null;
+    }
+
+    let current: HeapNode<T> = node;
+    let next: HeapNode<T>;
+    let nextCurrent: HeapNode<T>;
+    let pairs: HeapNode<T>;
+    let melded: HeapNode<T>;
+    while (current) {
+      next = current.next;
+      if (next) {
+        nextCurrent = next.next;
+        melded = this.meld(current, next);
+        if (melded) {
+          melded.prev = pairs;
+        }
+        pairs = melded;
+      }
+      else {
+        nextCurrent = null;
+        current.prev = pairs;
+        pairs = current;
+        break;
+      }
+      current = nextCurrent;
+    }
+
+    melded = null;
+    let prev: HeapNode<T>;
+    while (pairs) {
+      prev = pairs.prev;
+      melded = this.meld(melded, pairs);
+      pairs = prev;
+    }
+
+    return melded;
+  }
+}

docker/README.md

@@ -89,6 +89,7 @@ Below we list all settings from `mempool-config.json` and the corresponding over
   "MEMPOOL": {
     "NETWORK": "mainnet",
     "BACKEND": "electrum",
+    "ENABLED": true,
     "HTTP_PORT": 8999,
     "SPAWN_CLUSTER_PROCS": 0,
     "API_URL_PREFIX": "/api/v1/",
@@ -99,12 +100,18 @@ Below we list all settings from `mempool-config.json` and the corresponding over
     "BLOCK_WEIGHT_UNITS": 4000000,
     "INITIAL_BLOCKS_AMOUNT": 8,
     "MEMPOOL_BLOCKS_AMOUNT": 8,
+    "BLOCKS_SUMMARIES_INDEXING": false,
     "PRICE_FEED_UPDATE_INTERVAL": 600,
     "USE_SECOND_NODE_FOR_MINFEE": false,
     "EXTERNAL_ASSETS": ["https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json"],
     "STDOUT_LOG_MIN_PRIORITY": "info",
+    "INDEXING_BLOCKS_AMOUNT": false,
+    "AUTOMATIC_BLOCK_REINDEXING": false,
     "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json",
-    "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master"
+    "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
+    "ADVANCED_GBT_AUDIT": false,
+    "ADVANCED_GBT_MEMPOOL": false,
+    "CPFP_INDEXING": false,
   },
 ```
 
@@ -124,15 +131,25 @@ Corresponding `docker-compose.yml` overrides:
       MEMPOOL_BLOCK_WEIGHT_UNITS: ""
       MEMPOOL_INITIAL_BLOCKS_AMOUNT: ""
       MEMPOOL_MEMPOOL_BLOCKS_AMOUNT: ""
+      MEMPOOL_BLOCKS_SUMMARIES_INDEXING: ""
       MEMPOOL_PRICE_FEED_UPDATE_INTERVAL: ""
       MEMPOOL_USE_SECOND_NODE_FOR_MINFEE: ""
       MEMPOOL_EXTERNAL_ASSETS: ""
       MEMPOOL_STDOUT_LOG_MIN_PRIORITY: ""
+      MEMPOOL_INDEXING_BLOCKS_AMOUNT: ""
+      MEMPOOL_AUTOMATIC_BLOCK_REINDEXING: ""
       MEMPOOL_POOLS_JSON_URL: ""
       MEMPOOL_POOLS_JSON_TREE_URL: ""
+      MEMPOOL_ADVANCED_GBT_AUDIT: ""
+      MEMPOOL_ADVANCED_GBT_MEMPOOL: ""
+      MEMPOOL_CPFP_INDEXING: ""
       ...
 ```
 
+`ADVANCED_GBT_AUDIT` AND `ADVANCED_GBT_MEMPOOL` enable a more accurate (but slower) block prediction algorithm for the block audit feature and the projected mempool-blocks respectively.
+
+`CPFP_INDEXING` enables indexing CPFP (Child Pays For Parent) information for the last `INDEXING_BLOCKS_AMOUNT` blocks.
+
 <br/>
 
 `mempool-config.json`:

docker/backend/mempool-config.json

@@ -2,6 +2,7 @@
   "MEMPOOL": {
     "NETWORK": "__MEMPOOL_NETWORK__",
     "BACKEND": "__MEMPOOL_BACKEND__",
+    "ENABLED": __MEMPOOL_ENABLED__,
     "HTTP_PORT": __MEMPOOL_HTTP_PORT__,
     "SPAWN_CLUSTER_PROCS": __MEMPOOL_SPAWN_CLUSTER_PROCS__,
     "API_URL_PREFIX": "__MEMPOOL_API_URL_PREFIX__",
@@ -21,7 +22,11 @@
     "STDOUT_LOG_MIN_PRIORITY": "__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__",
     "INDEXING_BLOCKS_AMOUNT": __MEMPOOL_INDEXING_BLOCKS_AMOUNT__,
     "BLOCKS_SUMMARIES_INDEXING": __MEMPOOL_BLOCKS_SUMMARIES_INDEXING__,
-    "AUTOMATIC_BLOCK_REINDEXING": __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__
+    "AUTOMATIC_BLOCK_REINDEXING": __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__,
+    "ADVANCED_GBT_AUDIT": __MEMPOOL_ADVANCED_GBT_AUDIT__,
+    "ADVANCED_GBT_MEMPOOL": __MEMPOOL_ADVANCED_GBT_MEMPOOL__,
+    "CPFP_INDEXING": __MEMPOOL_CPFP_INDEXING__,
+    "RBF_DUAL_NODE": __RBF_DUAL_NODE__
   },
   "CORE_RPC": {
     "HOST": "__CORE_RPC_HOST__",

docker/backend/start.sh

@@ -3,6 +3,7 @@
 # MEMPOOL
 __MEMPOOL_NETWORK__=${MEMPOOL_NETWORK:=mainnet}
 __MEMPOOL_BACKEND__=${MEMPOOL_BACKEND:=electrum}
+__MEMPOOL_ENABLED__=${MEMPOOL_ENABLED:=true}
 __MEMPOOL_HTTP_PORT__=${BACKEND_HTTP_PORT:=8999}
 __MEMPOOL_SPAWN_CLUSTER_PROCS__=${MEMPOOL_SPAWN_CLUSTER_PROCS:=0}
 __MEMPOOL_API_URL_PREFIX__=${MEMPOOL_API_URL_PREFIX:=/api/v1/}
@@ -26,6 +27,11 @@ __MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=false}
 __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__=${MEMPOOL_AUTOMATIC_BLOCK_REINDEXING:=false}
 __MEMPOOL_POOLS_JSON_URL__=${MEMPOOL_POOLS_JSON_URL:=https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json}
 __MEMPOOL_POOLS_JSON_TREE_URL__=${MEMPOOL_POOLS_JSON_TREE_URL:=https://api.github.com/repos/mempool/mining-pools/git/trees/master}
+__MEMPOOL_ADVANCED_GBT_AUDIT__=${MEMPOOL_ADVANCED_GBT_AUDIT:=false}
+__MEMPOOL_ADVANCED_GBT_MEMPOOL__=${MEMPOOL_ADVANCED_GBT_MEMPOOL:=false}
+__MEMPOOL_CPFP_INDEXING__=${MEMPOOL_CPFP_INDEXING:=false}
+__MEMPOOL_RBF_DUAL_NODE__=${MEMPOOL_RBF_DUAL_NODE:=false}
+
 
 # CORE_RPC
 __CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1}
@@ -111,6 +117,7 @@ mkdir -p "${__MEMPOOL_CACHE_DIR__}"
 
 sed -i "s/__MEMPOOL_NETWORK__/${__MEMPOOL_NETWORK__}/g" mempool-config.json
 sed -i "s/__MEMPOOL_BACKEND__/${__MEMPOOL_BACKEND__}/g" mempool-config.json
+sed -i "s/__MEMPOOL_ENABLED__/${__MEMPOOL_ENABLED__}/g" mempool-config.json
 sed -i "s/__MEMPOOL_HTTP_PORT__/${__MEMPOOL_HTTP_PORT__}/g" mempool-config.json
 sed -i "s/__MEMPOOL_SPAWN_CLUSTER_PROCS__/${__MEMPOOL_SPAWN_CLUSTER_PROCS__}/g" mempool-config.json
 sed -i "s!__MEMPOOL_API_URL_PREFIX__!${__MEMPOOL_API_URL_PREFIX__}!g" mempool-config.json
@@ -134,6 +141,10 @@ sed -i "s/__MEMPOOL_INDEXING_BLOCKS_AMOUNT__/${__MEMPOOL_INDEXING_BLOCKS_AMOUNT_
 sed -i "s/__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__/${__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__}/g" mempool-config.json
 sed -i "s!__MEMPOOL_POOLS_JSON_URL__!${__MEMPOOL_POOLS_JSON_URL__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_POOLS_JSON_TREE_URL__!${__MEMPOOL_POOLS_JSON_TREE_URL__}!g" mempool-config.json
+sed -i "s!__MEMPOOL_ADVANCED_GBT_MEMPOOL__!${__MEMPOOL_ADVANCED_GBT_MEMPOOL__}!g" mempool-config.json
+sed -i "s!__MEMPOOL_ADVANCED_GBT_AUDIT__!${__MEMPOOL_ADVANCED_GBT_AUDIT__}!g" mempool-config.json
+sed -i "s!__MEMPOOL_CPFP_INDEXING__!${__MEMPOOL_CPFP_INDEXING__}!g" mempool-config.json
+sed -i "s/__MEMPOOL_RBF_DUAL_NODE__/${__MEMPOOL_RBF_DUAL_NODE__}/g" mempool-config.json
 
 sed -i "s/__CORE_RPC_HOST__/${__CORE_RPC_HOST__}/g" mempool-config.json
 sed -i "s/__CORE_RPC_PORT__/${__CORE_RPC_PORT__}/g" mempool-config.json

docker/frontend/Dockerfile

@@ -8,7 +8,9 @@ WORKDIR /build
 COPY . .
 RUN apt-get update
 RUN apt-get install -y build-essential rsync
+RUN cp mempool-frontend-config.sample.json mempool-frontend-config.json
 RUN npm install --omit=dev --omit=optional
+
 RUN npm run build
 
 FROM nginx:1.17.8-alpine
@@ -28,7 +30,9 @@ RUN chown -R 1000:1000 /patch && chmod -R 755 /patch && \
         chown -R 1000:1000 /var/cache/nginx && \
         chown -R 1000:1000 /var/log/nginx && \
         chown -R 1000:1000 /etc/nginx/nginx.conf && \
-        chown -R 1000:1000 /etc/nginx/conf.d
+        chown -R 1000:1000 /etc/nginx/conf.d && \
+        chown -R 1000:1000 /var/www/mempool
+
 RUN touch /var/run/nginx.pid && \
         chown -R 1000:1000 /var/run/nginx.pid
 

docker/frontend/entrypoint.sh

@@ -10,4 +10,57 @@ cp /etc/nginx/nginx.conf /patch/nginx.conf
 sed -i "s/__MEMPOOL_FRONTEND_HTTP_PORT__/${__MEMPOOL_FRONTEND_HTTP_PORT__}/g" /patch/nginx.conf
 cat /patch/nginx.conf > /etc/nginx/nginx.conf
 
+# Runtime overrides - read env vars defined in docker compose
+
+__TESTNET_ENABLED__=${TESTNET_ENABLED:=false}
+__SIGNET_ENABLED__=${SIGNET_ENABLED:=false}
+__LIQUID_ENABLED__=${LIQUID_EANBLED:=false}
+__LIQUID_TESTNET_ENABLED__=${LIQUID_TESTNET_ENABLED:=false}
+__BISQ_ENABLED__=${BISQ_ENABLED:=false}
+__BISQ_SEPARATE_BACKEND__=${BISQ_SEPARATE_BACKEND:=false}
+__ITEMS_PER_PAGE__=${ITEMS_PER_PAGE:=10}
+__KEEP_BLOCKS_AMOUNT__=${KEEP_BLOCKS_AMOUNT:=8}
+__NGINX_PROTOCOL__=${NGINX_PROTOCOL:=http}
+__NGINX_HOSTNAME__=${NGINX_HOSTNAME:=localhost}
+__NGINX_PORT__=${NGINX_PORT:=8999}
+__BLOCK_WEIGHT_UNITS__=${BLOCK_WEIGHT_UNITS:=4000000}
+__MEMPOOL_BLOCKS_AMOUNT__=${MEMPOOL_BLOCKS_AMOUNT:=8}
+__BASE_MODULE__=${BASE_MODULE:=mempool}
+__MEMPOOL_WEBSITE_URL__=${MEMPOOL_WEBSITE_URL:=https://mempool.space}
+__LIQUID_WEBSITE_URL__=${LIQUID_WEBSITE_URL:=https://liquid.network}
+__BISQ_WEBSITE_URL__=${BISQ_WEBSITE_URL:=https://bisq.markets}
+__MINING_DASHBOARD__=${MINING_DASHBOARD:=true}
+__LIGHTNING__=${LIGHTNING:=false}
+__MAINNET_BLOCK_AUDIT_START_HEIGHT__=${MAINNET_BLOCK_AUDIT_START_HEIGHT:=0}
+__TESTNET_BLOCK_AUDIT_START_HEIGHT__=${TESTNET_BLOCK_AUDIT_START_HEIGHT:=0}
+__SIGNET_BLOCK_AUDIT_START_HEIGHT__=${SIGNET_BLOCK_AUDIT_START_HEIGHT:=0}
+
+# Export as environment variables to be used by envsubst
+export __TESTNET_ENABLED__
+export __SIGNET_ENABLED__
+export __LIQUID_ENABLED__
+export __LIQUID_TESTNET_ENABLED__
+export __BISQ_ENABLED__
+export __BISQ_SEPARATE_BACKEND__
+export __ITEMS_PER_PAGE__
+export __KEEP_BLOCKS_AMOUNT__
+export __NGINX_PROTOCOL__
+export __NGINX_HOSTNAME__
+export __NGINX_PORT__
+export __BLOCK_WEIGHT_UNITS__
+export __MEMPOOL_BLOCKS_AMOUNT__
+export __BASE_MODULE__
+export __MEMPOOL_WEBSITE_URL__
+export __LIQUID_WEBSITE_URL__
+export __BISQ_WEBSITE_URL__
+export __MINING_DASHBOARD__
+export __LIGHTNING__
+export __MAINNET_BLOCK_AUDIT_START_HEIGHT__
+export __TESTNET_BLOCK_AUDIT_START_HEIGHT__
+export __SIGNET_BLOCK_AUDIT_START_HEIGHT__
+
+folder=$(find /var/www/mempool -name "config.js" | xargs dirname)
+echo ${folder}
+envsubst < ${folder}/config.template.js > ${folder}/config.js
+
 exec "$@"

frontend/README.md

@@ -113,7 +113,7 @@ https://www.transifex.com/mempool/mempool/dashboard/
 * French @Bayernatoor
 * Korean @kcalvinalvinn
 * Italian @HodlBits
-* Hebrew @Sh0ham
+* Hebrew @rapidlab309
 * Georgian @wyd_idk
 * Hungarian @btcdragonlord
 * Dutch @m__btc
@@ -127,7 +127,7 @@ https://www.transifex.com/mempool/mempool/dashboard/
 * Thai @Gusb3ll
 * Turkish @stackmore
 * Ukrainian @volbil
-* Vietnamese @bitcoin_vietnam
+* Vietnamese @BitcoinvnNews
 * Chinese @wdljt
 * Russian @TonyCrusoe @Bitconan
 * Romanian @mirceavesa

frontend/angular.json

@@ -137,6 +137,10 @@
           "hi": {
             "translation": "src/locale/messages.hi.xlf",
             "baseHref": "/hi/"
+          },
+          "lt": {
+            "translation": "src/locale/messages.lt.xlf",
+            "baseHref": "/lt/"
           }
         }
       },
@@ -152,15 +156,14 @@
             "assets": [
               "src/favicon.ico",
               "src/resources",
-              "src/robots.txt"
+              "src/robots.txt",
+              "src/config.js",
+              "src/config.template.js"
             ],
             "styles": [
               "src/styles.scss",
               "node_modules/@fortawesome/fontawesome-svg-core/styles.css"
             ],
-            "scripts": [
-              "generated-config.js"
-            ],
             "vendorChunk": true,
             "extractLicenses": false,
             "buildOptimizer": false,
@@ -222,6 +225,10 @@
               "proxyConfig": "proxy.conf.local.js",
               "verbose": true
             },
+            "local-esplora": {
+              "proxyConfig": "proxy.conf.local-esplora.js",
+              "verbose": true
+            },
             "mixed": {
               "proxyConfig": "proxy.conf.mixed.js",
               "verbose": true
@@ -265,57 +272,6 @@
             }
           }
         },
-        "server": {
-          "builder": "@angular-devkit/build-angular:server",
-          "options": {
-            "outputPath": "dist/mempool/server",
-            "main": "server.ts",
-            "tsConfig": "tsconfig.server.json",
-            "sourceMap": true,
-            "optimization": false
-          },
-          "configurations": {
-            "production": {
-              "outputHashing": "media",
-              "fileReplacements": [
-                {
-                  "replace": "src/environments/environment.ts",
-                  "with": "src/environments/environment.prod.ts"
-                }
-              ],
-              "sourceMap": false,
-              "localize": true,
-              "optimization": true
-            }
-          },
-          "defaultConfiguration": ""
-        },
-        "serve-ssr": {
-          "builder": "@nguniversal/builders:ssr-dev-server",
-          "options": {
-            "browserTarget": "mempool:build",
-            "serverTarget": "mempool:server"
-          },
-          "configurations": {
-            "production": {
-              "browserTarget": "mempool:build:production",
-              "serverTarget": "mempool:server:production"
-            }
-          }
-        },
-        "prerender": {
-          "builder": "@nguniversal/builders:prerender",
-          "options": {
-            "browserTarget": "mempool:build:production",
-            "serverTarget": "mempool:server:production",
-            "routes": [
-              "/"
-            ]
-          },
-          "configurations": {
-            "production": {}
-          }
-        },
         "cypress-run": {
           "builder": "@cypress/schematic:cypress",
           "options": {
@@ -336,6 +292,5 @@
         }
       }
     }
-  },
+  }
-  "defaultProject": "mempool"
 }

frontend/cypress/e2e/liquid/liquid.spec.ts

@@ -7,7 +7,6 @@ describe('Liquid', () => {
     cy.intercept('/liquid/api/blocks/').as('blocks');
     cy.intercept('/liquid/api/tx/**/outspends').as('outspends');
     cy.intercept('/liquid/api/block/**/txs/**').as('block-txs');
-    cy.intercept('/resources/pools.json').as('pools');
 
     Cypress.Commands.add('waitForBlockData', () => {
       cy.wait('@socket');

frontend/cypress/e2e/liquidtestnet/liquidtestnet.spec.ts

@@ -7,7 +7,6 @@ describe('Liquid Testnet', () => {
     cy.intercept('/liquidtestnet/api/blocks/').as('blocks');
     cy.intercept('/liquidtestnet/api/tx/**/outspends').as('outspends');
     cy.intercept('/liquidtestnet/api/block/**/txs/**').as('block-txs');
-    cy.intercept('/resources/pools.json').as('pools');
 
     Cypress.Commands.add('waitForBlockData', () => {
       cy.wait('@socket');

frontend/cypress/e2e/mainnet/mainnet.spec.ts

@@ -41,7 +41,6 @@ describe('Mainnet', () => {
     // cy.intercept('/api/v1/block/*/summary').as('block-summary');
     // cy.intercept('/api/v1/outspends/*').as('outspends');
     // cy.intercept('/api/tx/*/outspends').as('tx-outspends');
-    // cy.intercept('/resources/pools.json').as('pools');
 
     // Search Auto Complete
     cy.intercept('/api/address-prefix/1wiz').as('search-1wiz');

frontend/generate-config.js

@@ -2,7 +2,8 @@ var fs = require('fs');
 const { spawnSync } = require('child_process');
 
 const CONFIG_FILE_NAME = 'mempool-frontend-config.json';
-const GENERATED_CONFIG_FILE_NAME = 'generated-config.js';
+const GENERATED_CONFIG_FILE_NAME = 'src/resources/config.js';
+const GENERATED_TEMPLATE_CONFIG_FILE_NAME = 'src/resources/config.template.js';
 
 let settings = [];
 let configContent = {};
@@ -67,10 +68,17 @@ if (process.env.DOCKER_COMMIT_HASH) {
 
 const newConfig = `(function (window) {
   window.__env = window.__env || {};${settings.reduce((str, obj) => `${str}
-    window.__env.${obj.key} = ${ typeof obj.value === 'string' ? `'${obj.value}'` : obj.value };`, '')}
+    window.__env.${obj.key} = ${typeof obj.value === 'string' ? `'${obj.value}'` : obj.value};`, '')}
     window.__env.GIT_COMMIT_HASH = '${gitCommitHash}';
     window.__env.PACKAGE_JSON_VERSION = '${packetJsonVersion}';
-  }(global || this));`;
+  }(this));`;
+
+const newConfigTemplate = `(function (window) {
+  window.__env = window.__env || {};${settings.reduce((str, obj) => `${str}
+    window.__env.${obj.key} = ${typeof obj.value === 'string' ? `'\${__${obj.key}__}'` : `\${__${obj.key}__}`};`, '')}
+    window.__env.GIT_COMMIT_HASH = '${gitCommitHash}';
+    window.__env.PACKAGE_JSON_VERSION = '${packetJsonVersion}';
+  }(this));`;
 
 function readConfig(path) {
   try {
@@ -89,6 +97,16 @@ function writeConfig(path, config) {
   }
 }
 
+function writeConfigTemplate(path, config) {
+  try {
+    fs.writeFileSync(path, config, 'utf8');
+  } catch (e) {
+    throw new Error(e);
+  }
+}
+
+writeConfigTemplate(GENERATED_TEMPLATE_CONFIG_FILE_NAME, newConfigTemplate);
+
 const currentConfig = readConfig(GENERATED_CONFIG_FILE_NAME);
 
 if (currentConfig && currentConfig === newConfig) {
@@ -106,4 +124,4 @@ if (currentConfig && currentConfig === newConfig) {
   console.log('NEW CONFIG: ', newConfig);
   writeConfig(GENERATED_CONFIG_FILE_NAME, newConfig);
   console.log(`${GENERATED_CONFIG_FILE_NAME} file updated`);
-};
+}

frontend/mempool-frontend-config.sample.json

@@ -17,5 +17,10 @@
   "LIQUID_WEBSITE_URL": "https://liquid.network",
   "BISQ_WEBSITE_URL": "https://bisq.markets",
   "MINING_DASHBOARD": true,
-  "LIGHTNING": false
+  "MAINNET_BLOCK_AUDIT_START_HEIGHT": 0,
+  "TESTNET_BLOCK_AUDIT_START_HEIGHT": 0,
+  "SIGNET_BLOCK_AUDIT_START_HEIGHT": 0,
+  "LIGHTNING": false,
+  "FULL_RBF_ENABLED": false,
+  "ALT_BACKEND_ENABLED": false
 }

frontend/package.json

@@ -22,13 +22,14 @@
   "scripts": {
     "ng": "./node_modules/@angular/cli/bin/ng.js",
     "tsc": "./node_modules/typescript/bin/tsc",
-    "i18n-extract-from-source": "./node_modules/@angular/cli/bin/ng extract-i18n --out-file ./src/locale/messages.xlf",
+    "i18n-extract-from-source": "npm run ng -- extract-i18n --out-file ./src/locale/messages.xlf",
     "i18n-pull-from-transifex": "tx pull -a --parallel --minimum-perc 1 --force",
     "serve": "npm run generate-config && npm run ng -- serve -c local",
     "serve:stg": "npm run generate-config && npm run ng -- serve -c staging",
     "serve:local-prod": "npm run generate-config && npm run ng -- serve -c local-prod",
     "serve:local-staging": "npm run generate-config && npm run ng -- serve -c local-staging",
     "start": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c local",
+    "start:local-esplora": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c local-esplora",
     "start:stg": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c staging",
     "start:local-prod": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c local-prod",
     "start:local-staging": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c local-staging",
@@ -50,9 +51,6 @@
     "config:defaults:mempool": "node update-config.js TESTNET_ENABLED=true SIGNET_ENABLED=true LIQUID_ENABLED=true LIQUID_TESTNET_ENABLED=true BISQ_ENABLED=true ITEMS_PER_PAGE=25 BASE_MODULE=mempool BLOCK_WEIGHT_UNITS=4000000 && npm run generate-config",
     "config:defaults:liquid": "node update-config.js TESTNET_ENABLED=true SIGNET_ENABLED=true LIQUID_ENABLED=true LIQUID_TESTNET_ENABLED=true BISQ_ENABLED=true ITEMS_PER_PAGE=25 BASE_MODULE=liquid BLOCK_WEIGHT_UNITS=300000 && npm run generate-config",
     "config:defaults:bisq": "node update-config.js TESTNET_ENABLED=true SIGNET_ENABLED=true LIQUID_ENABLED=true BISQ_ENABLED=true ITEMS_PER_PAGE=25 BASE_MODULE=bisq BLOCK_WEIGHT_UNITS=4000000 && npm run generate-config",
-    "dev:ssr": "npm run generate-config && npm run ng -- run mempool:serve-ssr",
-    "serve:ssr": "node server.run.js",
-    "build:ssr": "npm run build && npm run ng -- run mempool:server:production && npm run tsc -- server.run.ts",
     "prerender": "npm run ng -- run mempool:prerender",
     "cypress:open": "cypress open",
     "cypress:run": "cypress run",
@@ -63,63 +61,59 @@
     "cypress:run:ci:staging": "node update-config.js TESTNET_ENABLED=true SIGNET_ENABLED=true LIQUID_ENABLED=true BISQ_ENABLED=true ITEMS_PER_PAGE=25 && npm run generate-config && start-server-and-test serve:local-staging 4200 cypress:run:record"
   },
   "dependencies": {
-    "@angular-devkit/build-angular": "~13.3.7",
+    "@angular-devkit/build-angular": "^14.2.10",
-    "@angular/animations": "~13.3.10",
+    "@angular/animations": "^14.2.12",
-    "@angular/cli": "~13.3.7",
+    "@angular/cli": "^14.2.10",
-    "@angular/common": "~13.3.10",
+    "@angular/common": "^14.2.12",
-    "@angular/compiler": "~13.3.10",
+    "@angular/compiler": "^14.2.12",
-    "@angular/core": "~13.3.10",
+    "@angular/core": "^14.2.12",
-    "@angular/forms": "~13.3.10",
+    "@angular/forms": "^14.2.12",
-    "@angular/localize": "~13.3.10",
+    "@angular/localize": "^14.2.12",
-    "@angular/platform-browser": "~13.3.10",
+    "@angular/platform-browser": "^14.2.12",
-    "@angular/platform-browser-dynamic": "~13.3.10",
+    "@angular/platform-browser-dynamic": "^14.2.12",
-    "@angular/platform-server": "~13.3.10",
+    "@angular/platform-server": "^14.2.12",
-    "@angular/router": "~13.3.10",
+    "@angular/router": "^14.2.12",
-    "@fortawesome/angular-fontawesome": "~0.10.2",
+    "@fortawesome/angular-fontawesome": "~0.11.1",
-    "@fortawesome/fontawesome-common-types": "~6.1.1",
+    "@fortawesome/fontawesome-common-types": "~6.2.1",
-    "@fortawesome/fontawesome-svg-core": "~6.1.1",
+    "@fortawesome/fontawesome-svg-core": "~6.2.1",
-    "@fortawesome/free-solid-svg-icons": "~6.1.1",
+    "@fortawesome/free-solid-svg-icons": "~6.2.1",
     "@mempool/mempool.js": "2.3.0",
-    "@ng-bootstrap/ng-bootstrap": "^11.0.0",
+    "@ng-bootstrap/ng-bootstrap": "^13.1.1",
-    "@nguniversal/express-engine": "~13.1.1",
+    "@types/qrcode": "~1.5.0",
-    "@types/qrcode": "~1.4.2",
+    "bootstrap": "~4.6.1",
-    "bootstrap": "~4.5.0",
     "browserify": "^17.0.0",
-    "clipboard": "^2.0.10",
+    "clipboard": "^2.0.11",
     "domino": "^2.1.6",
-    "echarts": "~5.3.2",
+    "echarts": "~5.4.1",
     "echarts-gl": "^2.0.9",
-    "express": "^4.17.1",
     "lightweight-charts": "~3.8.0",
-    "ngx-echarts": "8.0.1",
+    "ngx-echarts": "~14.0.0",
-    "ngx-infinite-scroll": "^10.0.1",
+    "ngx-infinite-scroll": "^14.0.1",
-    "qrcode": "1.5.0",
+    "qrcode": "1.5.1",
-    "rxjs": "~7.5.5",
+    "rxjs": "~7.8.0",
-    "tinyify": "^3.0.0",
+    "tinyify": "^3.1.0",
     "tlite": "^0.1.9",
-    "tslib": "~2.4.0",
+    "tslib": "~2.4.1",
-    "zone.js": "~0.11.5"
+    "zone.js": "~0.12.0"
   },
   "devDependencies": {
-    "@angular/compiler-cli": "~13.3.10",
+    "@angular/compiler-cli": "^14.2.12",
-    "@angular/language-service": "~13.3.10",
+    "@angular/language-service": "^14.2.12",
-    "@nguniversal/builders": "~13.1.1",
+    "@types/node": "^18.11.9",
-    "@types/express": "^4.17.0",
+    "@typescript-eslint/eslint-plugin": "^5.48.1",
-    "@types/node": "^12.11.1",
+    "@typescript-eslint/parser": "^5.48.1",
-    "@typescript-eslint/eslint-plugin": "^5.30.5",
+    "eslint": "^8.31.0",
-    "@typescript-eslint/parser": "^5.30.5",
-    "eslint": "^8.19.0",
     "http-proxy-middleware": "~2.0.6",
-    "prettier": "^2.7.1",
+    "prettier": "^2.8.2",
-    "ts-node": "~10.8.1",
+    "ts-node": "~10.9.1",
     "typescript": "~4.6.4"
   },
   "optionalDependencies": {
-    "@cypress/schematic": "~2.0.0",
+    "@cypress/schematic": "^2.4.0",
-    "cypress": "^10.3.0",
+    "cypress": "^12.3.0",
-    "cypress-fail-on-console-error": "~3.0.0",
+    "cypress-fail-on-console-error": "~4.0.2",
     "cypress-wait-until": "^1.7.2",
-    "mock-socket": "~9.1.4",
+    "mock-socket": "~9.1.5",
     "start-server-and-test": "~1.14.0"
   },
   "scarfSettings": {

frontend/proxy.conf.js

@@ -76,7 +76,7 @@ PROXY_CONFIG = [
 
 if (configContent && configContent.BASE_MODULE == "liquid") {
     PROXY_CONFIG.push({
-        context: ['/resources/pools.json',
+        context: [
             '/resources/assets.json', '/resources/assets.minimal.json',
             '/resources/assets-testnet.json', '/resources/assets-testnet.minimal.json'],
         target: "https://liquid.network",
@@ -85,7 +85,7 @@ if (configContent && configContent.BASE_MODULE == "liquid") {
     });
 } else {
     PROXY_CONFIG.push({
-        context: ['/resources/pools.json', '/resources/assets.json', '/resources/assets.minimal.json', '/resources/worldmap.json'],
+        context: ['/resources/assets.json', '/resources/assets.minimal.json', '/resources/worldmap.json'],
         target: "https://mempool.space",
         secure: false,
         changeOrigin: true,

frontend/proxy.conf.local-esplora.js

@@ -0,0 +1,137 @@
+const fs = require('fs');
+
+const FRONTEND_CONFIG_FILE_NAME = 'mempool-frontend-config.json';
+
+let configContent;
+
+// Read frontend config 
+try {
+    const rawConfig = fs.readFileSync(FRONTEND_CONFIG_FILE_NAME);
+    configContent = JSON.parse(rawConfig);
+    console.log(`${FRONTEND_CONFIG_FILE_NAME} file found, using provided config`);
+} catch (e) {
+    console.log(e);
+    if (e.code !== 'ENOENT') {
+      throw new Error(e);
+  } else {
+      console.log(`${FRONTEND_CONFIG_FILE_NAME} file not found, using default config`);
+  }
+}
+
+let PROXY_CONFIG = [];
+
+if (configContent && configContent.BASE_MODULE === 'liquid') {
+  PROXY_CONFIG.push(...[
+    {
+      context: ['/liquid/api/v1/**'],
+      target: `http://127.0.0.1:8999`,
+      secure: false,
+      ws: true,
+      changeOrigin: true,
+      proxyTimeout: 30000,
+      pathRewrite: {
+          "^/liquid": ""
+      },
+    },
+    {
+      context: ['/liquid/api/**'],
+      target: `http://127.0.0.1:3000`,
+      secure: false,
+      changeOrigin: true,
+      proxyTimeout: 30000,
+      pathRewrite: {
+          "^/liquid/api/": ""
+      },
+    },
+    {
+      context: ['/liquidtestnet/api/v1/**'],
+      target: `http://127.0.0.1:8999`,
+      secure: false,
+      ws: true,
+      changeOrigin: true,
+      proxyTimeout: 30000,
+      pathRewrite: {
+          "^/liquidtestnet": ""
+      },
+    },
+    {
+      context: ['/liquidtestnet/api/**'],
+      target: `http://127.0.0.1:3000`,
+      secure: false,
+      changeOrigin: true,
+      proxyTimeout: 30000,
+      pathRewrite: {
+          "^/liquidtestnet/api/": "/"
+      },
+    },
+  ]);
+}
+
+
+if (configContent && configContent.BASE_MODULE === 'bisq') {
+  PROXY_CONFIG.push(...[
+    {
+      context: ['/bisq/api/v1/ws'],
+      target: `http://127.0.0.1:8999`,
+      secure: false,
+      ws: true,
+      changeOrigin: true,
+      proxyTimeout: 30000,
+      pathRewrite: {
+          "^/bisq": ""
+      },
+    },
+    {
+      context: ['/bisq/api/v1/**'],
+      target: `http://127.0.0.1:8999`,
+      secure: false,
+      changeOrigin: true,
+      proxyTimeout: 30000,
+    },
+    {
+      context: ['/bisq/api/**'],
+      target: `http://127.0.0.1:8999`,
+      secure: false,
+      changeOrigin: true,
+      proxyTimeout: 30000,
+      pathRewrite: {
+          "^/bisq/api/": "/api/v1/bisq/"
+      },
+    }
+  ]);
+}
+
+PROXY_CONFIG.push(...[
+  {
+    context: ['/testnet/api/v1/lightning/**'],
+    target: `http://127.0.0.1:8999`,
+    secure: false,
+    changeOrigin: true,
+    proxyTimeout: 30000,
+    pathRewrite: {
+        "^/testnet": ""
+    },
+  },
+  {
+    context: ['/api/v1/**'],
+    target: `http://127.0.0.1:8999`,
+    secure: false,
+    ws: true,
+    changeOrigin: true,
+    proxyTimeout: 30000,
+  },
+  {
+    context: ['/api/**'],
+    target: `http://127.0.0.1:3000`,
+    secure: false,
+    changeOrigin: true,
+    proxyTimeout: 30000,
+    pathRewrite: {
+        "^/api": ""
+    },
+  }
+]);
+
+console.log(PROXY_CONFIG);
+
+module.exports = PROXY_CONFIG;

frontend/proxy.conf.staging.js

@@ -3,9 +3,9 @@ const fs = require('fs');
 let PROXY_CONFIG = require('./proxy.conf');
 
 PROXY_CONFIG.forEach(entry => {
-  entry.target = entry.target.replace("mempool.space", "mempool.ninja");
+  entry.target = entry.target.replace("mempool.space", "mempool-staging.tk7.mempool.space");
-  entry.target = entry.target.replace("liquid.network", "liquid.place");
+  entry.target = entry.target.replace("liquid.network", "liquid-staging.tk7.mempool.space");
-  entry.target = entry.target.replace("bisq.markets", "bisq.ninja");
+  entry.target = entry.target.replace("bisq.markets", "bisq-staging.fra.mempool.space");
 });
 
 module.exports = PROXY_CONFIG;

frontend/server.run.ts

@@ -1,96 +0,0 @@
-import 'zone.js/node';
-import './generated-config';
-
-import * as domino from 'domino';
-import * as express from 'express';
-import * as fs from 'fs';
-import * as path from 'path';
-
-const {readFileSync, existsSync} = require('fs');
-const {createProxyMiddleware} = require('http-proxy-middleware');
-
-const template = fs.readFileSync(path.join(process.cwd(), 'dist/mempool/browser/en-US/', 'index.html')).toString();
-const win = domino.createWindow(template);
-
-// @ts-ignore
-win.__env = global.__env;
-
-// @ts-ignore
-win.matchMedia = () => {
-  return {
-    matches: true
-  };
-};
-
-// @ts-ignore
-win.setTimeout = (fn) => { fn(); };
-win.document.body.scrollTo = (() => {});
-// @ts-ignore
-global['window'] = win;
-global['document'] = win.document;
-// @ts-ignore
-global['history'] = { state: { } };
-
-global['localStorage'] = {
-  getItem: () => '',
-  setItem: () => {},
-  removeItem: () => {},
-  clear: () => {},
-  length: 0,
-  key: () => '',
-};
-
-/**
- * Return the list of supported and actually active locales
- */
-function getActiveLocales() {
-  const angularConfig = JSON.parse(readFileSync('angular.json', 'utf8'));
-
-  const supportedLocales = [
-    angularConfig.projects.mempool.i18n.sourceLocale,
-    ...Object.keys(angularConfig.projects.mempool.i18n.locales),
-  ];
-
-  return supportedLocales.filter(locale => existsSync(`./dist/mempool/server/${locale}`));
-}
-
-function app() {
-  const server = express();
-
-  // proxy API to nginx
-  server.get('/api/**', createProxyMiddleware({
-    // @ts-ignore
-    target: win.__env.NGINX_PROTOCOL + '://' + win.__env.NGINX_HOSTNAME + ':' + win.__env.NGINX_PORT,
-    changeOrigin: true,
-  }));
-
-  // map / and /en to en-US
-  const defaultLocale = 'en-US';
-  console.log(`serving default locale: ${defaultLocale}`);
-  const appServerModule = require(`./dist/mempool/server/${defaultLocale}/main.js`);
-  server.use('/', appServerModule.app(defaultLocale));
-  server.use('/en', appServerModule.app(defaultLocale));
-
-  // map each locale to its localized main.js
-  getActiveLocales().forEach(locale => {
-    console.log('serving locale:', locale);
-    const appServerModule = require(`./dist/mempool/server/${locale}/main.js`);
-
-    // map everything to itself
-    server.use(`/${locale}`, appServerModule.app(locale));
-
-  });
-
-  return server;
-}
-
-function run() {
-  const port = process.env.PORT || 4000;
-
-  // Start up the Node server
-  app().listen(port, () => {
-    console.log(`Node Express server listening on port ${port}`);
-  });
-}
-
-run();

frontend/server.ts

@@ -1,160 +0,0 @@
-import 'zone.js/node';
-import './generated-config';
-
-import { ngExpressEngine } from '@nguniversal/express-engine';
-import * as express from 'express';
-import * as fs from 'fs';
-import * as path from 'path';
-import * as domino from 'domino';
-
-import { join } from 'path';
-import { AppServerModule } from './src/main.server';
-import { APP_BASE_HREF } from '@angular/common';
-import { existsSync } from 'fs';
-
-const template = fs.readFileSync(path.join(process.cwd(), 'dist/mempool/browser/en-US/', 'index.html')).toString();
-const win = domino.createWindow(template);
-
-// @ts-ignore
-win.__env = global.__env;
-
-// @ts-ignore
-win.matchMedia = () => {
-  return {
-    matches: true
-  };
-};
-
-// @ts-ignore
-win.setTimeout = (fn) => { fn(); };
-win.document.body.scrollTo = (() => {});
-// @ts-ignore
-global['window'] = win;
-global['document'] = win.document;
-// @ts-ignore
-global['history'] = { state: { } };
-
-global['localStorage'] = {
-  getItem: () => '',
-  setItem: () => {},
-  removeItem: () => {},
-  clear: () => {},
-  length: 0,
-  key: () => '',
-};
-
-// The Express app is exported so that it can be used by serverless Functions.
-export function app(locale: string): express.Express {
-  const server = express();
-  const distFolder = join(process.cwd(), `dist/mempool/browser/${locale}`);
-  const indexHtml = existsSync(join(distFolder, 'index.original.html')) ? 'index.original.html' : 'index';
-
-  // Our Universal express-engine (found @ https://github.com/angular/universal/tree/master/modules/express-engine)
-  server.engine('html', ngExpressEngine({
-    bootstrap: AppServerModule,
-  }));
-
-  server.set('view engine', 'html');
-  server.set('views', distFolder);
-
-  // only handle URLs that actually exist
-  //server.get(locale, getLocalizedSSR(indexHtml));
-  server.get('/', getLocalizedSSR(indexHtml));
-  server.get('/tx/*', getLocalizedSSR(indexHtml));
-  server.get('/block/*', getLocalizedSSR(indexHtml));
-  server.get('/mempool-block/*', getLocalizedSSR(indexHtml));
-  server.get('/address/*', getLocalizedSSR(indexHtml));
-  server.get('/blocks', getLocalizedSSR(indexHtml));
-  server.get('/mining/pools', getLocalizedSSR(indexHtml));
-  server.get('/mining/pool/*', getLocalizedSSR(indexHtml));
-  server.get('/graphs', getLocalizedSSR(indexHtml));
-  server.get('/liquid', getLocalizedSSR(indexHtml));
-  server.get('/liquid/tx/*', getLocalizedSSR(indexHtml));
-  server.get('/liquid/block/*', getLocalizedSSR(indexHtml));
-  server.get('/liquid/mempool-block/*', getLocalizedSSR(indexHtml));
-  server.get('/liquid/address/*', getLocalizedSSR(indexHtml));
-  server.get('/liquid/asset/*', getLocalizedSSR(indexHtml));
-  server.get('/liquid/blocks', getLocalizedSSR(indexHtml));
-  server.get('/liquid/graphs', getLocalizedSSR(indexHtml));
-  server.get('/liquid/assets', getLocalizedSSR(indexHtml));
-  server.get('/liquid/api', getLocalizedSSR(indexHtml));
-  server.get('/liquid/tv', getLocalizedSSR(indexHtml));
-  server.get('/liquid/status', getLocalizedSSR(indexHtml));
-  server.get('/liquid/about', getLocalizedSSR(indexHtml));
-  server.get('/testnet', getLocalizedSSR(indexHtml));
-  server.get('/testnet/tx/*', getLocalizedSSR(indexHtml));
-  server.get('/testnet/block/*', getLocalizedSSR(indexHtml));
-  server.get('/testnet/mempool-block/*', getLocalizedSSR(indexHtml));
-  server.get('/testnet/address/*', getLocalizedSSR(indexHtml));
-  server.get('/testnet/blocks', getLocalizedSSR(indexHtml));
-  server.get('/testnet/mining/pools', getLocalizedSSR(indexHtml));
-  server.get('/testnet/graphs', getLocalizedSSR(indexHtml));
-  server.get('/testnet/api', getLocalizedSSR(indexHtml));
-  server.get('/testnet/tv', getLocalizedSSR(indexHtml));
-  server.get('/testnet/status', getLocalizedSSR(indexHtml));
-  server.get('/testnet/about', getLocalizedSSR(indexHtml));
-  server.get('/signet', getLocalizedSSR(indexHtml));
-  server.get('/signet/tx/*', getLocalizedSSR(indexHtml));
-  server.get('/signet/block/*', getLocalizedSSR(indexHtml));
-  server.get('/signet/mempool-block/*', getLocalizedSSR(indexHtml));
-  server.get('/signet/address/*', getLocalizedSSR(indexHtml));
-  server.get('/signet/blocks', getLocalizedSSR(indexHtml));
-  server.get('/signet/mining/pools', getLocalizedSSR(indexHtml));
-  server.get('/signet/graphs', getLocalizedSSR(indexHtml));
-  server.get('/signet/api', getLocalizedSSR(indexHtml));
-  server.get('/signet/tv', getLocalizedSSR(indexHtml));
-  server.get('/signet/status', getLocalizedSSR(indexHtml));
-  server.get('/signet/about', getLocalizedSSR(indexHtml));
-  server.get('/bisq', getLocalizedSSR(indexHtml));
-  server.get('/bisq/tx/*', getLocalizedSSR(indexHtml));
-  server.get('/bisq/blocks', getLocalizedSSR(indexHtml));
-  server.get('/bisq/block/*', getLocalizedSSR(indexHtml));
-  server.get('/bisq/address/*', getLocalizedSSR(indexHtml));
-  server.get('/bisq/stats', getLocalizedSSR(indexHtml));
-  server.get('/bisq/about', getLocalizedSSR(indexHtml));
-  server.get('/bisq/api', getLocalizedSSR(indexHtml));
-  server.get('/about', getLocalizedSSR(indexHtml));
-  server.get('/api', getLocalizedSSR(indexHtml));
-  server.get('/tv', getLocalizedSSR(indexHtml));
-  server.get('/status', getLocalizedSSR(indexHtml));
-  server.get('/terms-of-service', getLocalizedSSR(indexHtml));
-
-  // fallback to static file handler so we send HTTP 404 to nginx
-  server.get('/**', express.static(distFolder, { maxAge: '1y' }));
-
-  return server;
-}
-
-function getLocalizedSSR(indexHtml) {
-  return (req, res) => {
-    res.render(indexHtml, {
-      req,
-      providers: [
-        { provide: APP_BASE_HREF, useValue: req.baseUrl }
-      ]
-    });
-  }
-}
-
-// only used for development mode
-function run(): void {
-  const port = process.env.PORT || 4000;
-
-  // Start up the Node server
-  const server = app('en-US');
-  server.listen(port, () => {
-    console.log(`Node Express server listening on port ${port}`);
-  });
-}
-
-// Webpack will replace 'require' with '__webpack_require__'
-// '__non_webpack_require__' is a proxy to Node 'require'
-// The below code is to ensure that the server is run only when not requiring the bundle.
-declare const __non_webpack_require__: NodeRequire;
-const mainModule = __non_webpack_require__.main;
-const moduleFilename = mainModule && mainModule.filename || '';
-if (moduleFilename === __filename || moduleFilename.includes('iisnode')) {
-  run();
-}
-
-export * from './src/main.server';

frontend/src/app/app-routing.module.ts

@@ -4,7 +4,6 @@ import { AppPreloadingStrategy } from './app.preloading-strategy'
 import { StartComponent } from './components/start/start.component';
 import { TransactionComponent } from './components/transaction/transaction.component';
 import { BlockComponent } from './components/block/block.component';
-import { BlockAuditComponent } from './components/block-audit/block-audit.component';
 import { AddressComponent } from './components/address/address.component';
 import { MasterPageComponent } from './components/master-page/master-page.component';
 import { AboutComponent } from './components/about/about.component';
@@ -74,12 +73,14 @@ let routes: Routes = [
             children: [],
             component: AddressComponent,
             data: {
-              ogImage: true
+              ogImage: true,
+              networkSpecific: true,
             }
           },
           {
             path: 'tx',
             component: StartComponent,
+            data: { networkSpecific: true },
             children: [
               {
                 path: ':id',
@@ -90,6 +91,7 @@ let routes: Routes = [
           {
             path: 'block',
             component: StartComponent,
+            data: { networkSpecific: true },
               children: [
               {
                 path: ':id',
@@ -100,15 +102,6 @@ let routes: Routes = [
               },
             ],
           },
-          {
-            path: 'block-audit',
-            children: [
-              {
-                path: ':id',
-                component: BlockAuditComponent,
-              },
-            ],
-          },
           {
             path: 'docs',
             loadChildren: () => import('./docs/docs.module').then(m => m.DocsModule),
@@ -121,12 +114,13 @@ let routes: Routes = [
           {
             path: 'lightning',
             loadChildren: () => import('./lightning/lightning.module').then(m => m.LightningModule),
-            data: { preload: browserWindowEnv && browserWindowEnv.LIGHTNING === true },
+            data: { preload: browserWindowEnv && browserWindowEnv.LIGHTNING === true, networks: ['bitcoin'] },
           },
         ],
       },
       {
         path: 'status',
+        data: { networks: ['bitcoin', 'liquid'] },
         component: StatusViewComponent
       },
       {
@@ -185,11 +179,13 @@ let routes: Routes = [
             children: [],
             component: AddressComponent,
             data: {
-              ogImage: true
+              ogImage: true,
+              networkSpecific: true,
             }
           },
           {
             path: 'tx',
+            data: { networkSpecific: true },
             component: StartComponent,
             children: [
               {
@@ -200,6 +196,7 @@ let routes: Routes = [
           },
           {
             path: 'block',
+            data: { networkSpecific: true },
             component: StartComponent,
             children: [
               {
@@ -211,15 +208,6 @@ let routes: Routes = [
               },
             ],
           },
-          {
-            path: 'block-audit',
-            children: [
-              {
-                path: ':id',
-                component: BlockAuditComponent,
-              },
-            ],
-          },
           {
             path: 'docs',
             loadChildren: () => import('./docs/docs.module').then(m => m.DocsModule)
@@ -230,12 +218,14 @@ let routes: Routes = [
           },
           {
             path: 'lightning',
+            data: { networks: ['bitcoin'] },
             loadChildren: () => import('./lightning/lightning.module').then(m => m.LightningModule)
           },
         ],
       },
       {
         path: 'status',
+        data: { networks: ['bitcoin', 'liquid'] },
         component: StatusViewComponent
       },
       {
@@ -291,11 +281,13 @@ let routes: Routes = [
         children: [],
         component: AddressComponent,
         data: {
-          ogImage: true
+          ogImage: true,
+          networkSpecific: true,
         }
       },
       {
         path: 'tx',
+        data: { networkSpecific: true },
         component: StartComponent,
         children: [
           {
@@ -306,6 +298,7 @@ let routes: Routes = [
       },
       {
         path: 'block',
+        data: { networkSpecific: true },
         component: StartComponent,
         children: [
           {
@@ -317,15 +310,6 @@ let routes: Routes = [
           },
         ],
       },
-      {
-        path: 'block-audit',
-        children: [
-          {
-            path: ':id',
-            component: BlockAuditComponent
-          },
-        ],
-      },
       {
         path: 'docs',
         loadChildren: () => import('./docs/docs.module').then(m => m.DocsModule)
@@ -336,6 +320,7 @@ let routes: Routes = [
       },
       {
         path: 'lightning',
+        data: { networks: ['bitcoin'] },
         loadChildren: () => import('./lightning/lightning.module').then(m => m.LightningModule)
       },
     ],
@@ -359,6 +344,7 @@ let routes: Routes = [
   },
   {
     path: 'status',
+    data: { networks: ['bitcoin', 'liquid'] },
     component: StatusViewComponent
   },
   {
@@ -422,11 +408,13 @@ if (browserWindowEnv && browserWindowEnv.BASE_MODULE === 'liquid') {
               children: [],
               component: AddressComponent,
               data: {
-                ogImage: true
+                ogImage: true,
+                networkSpecific: true,
               }
             },
             {
               path: 'tx',
+              data: { networkSpecific: true },
               component: StartComponent,
               children: [
                 {
@@ -437,6 +425,7 @@ if (browserWindowEnv && browserWindowEnv.BASE_MODULE === 'liquid') {
             },
             {
               path: 'block',
+              data: { networkSpecific: true },
               component: StartComponent,
               children: [
                 {
@@ -450,18 +439,22 @@ if (browserWindowEnv && browserWindowEnv.BASE_MODULE === 'liquid') {
             },
             {
               path: 'assets',
+              data: { networks: ['liquid'] },
               component: AssetsNavComponent,
               children: [
                 {
                   path: 'all',
+                  data: { networks: ['liquid'] },
                   component: AssetsComponent,
                 },
                 {
                   path: 'asset/:id',
+                  data: { networkSpecific: true },
                   component: AssetComponent
                 },
                 {
                   path: 'group/:id',
+                  data: { networkSpecific: true },
                   component: AssetGroupComponent
                 },
                 {
@@ -482,6 +475,7 @@ if (browserWindowEnv && browserWindowEnv.BASE_MODULE === 'liquid') {
         },
         {
           path: 'status',
+          data: { networks: ['bitcoin', 'liquid'] },
           component: StatusViewComponent
         },
         {
@@ -532,11 +526,13 @@ if (browserWindowEnv && browserWindowEnv.BASE_MODULE === 'liquid') {
           children: [],
           component: AddressComponent,
           data: {
-            ogImage: true
+            ogImage: true,
+            networkSpecific: true,
           }
         },
         {
           path: 'tx',
+          data: { networkSpecific: true },
           component: StartComponent,
           children: [
             {
@@ -547,6 +543,7 @@ if (browserWindowEnv && browserWindowEnv.BASE_MODULE === 'liquid') {
         },
         {
           path: 'block',
+          data: { networkSpecific: true },
           component: StartComponent,
           children: [
             {
@@ -560,22 +557,27 @@ if (browserWindowEnv && browserWindowEnv.BASE_MODULE === 'liquid') {
         },
         {
           path: 'assets',
+          data: { networks: ['liquid'] },
           component: AssetsNavComponent,
           children: [
             {
               path: 'featured',
+              data: { networkSpecific: true },
               component: AssetsFeaturedComponent,
             },
             {
               path: 'all',
+              data: { networks: ['liquid'] },
               component: AssetsComponent,
             },
             {
               path: 'asset/:id',
+              data: { networkSpecific: true },
               component: AssetComponent
             },
             {
               path: 'group/:id',
+              data: { networkSpecific: true },
               component: AssetGroupComponent
             },
             {
@@ -609,6 +611,7 @@ if (browserWindowEnv && browserWindowEnv.BASE_MODULE === 'liquid') {
     },
     {
       path: 'status',
+      data: { networks: ['bitcoin', 'liquid']},
       component: StatusViewComponent
     },
     {
@@ -624,7 +627,7 @@ if (browserWindowEnv && browserWindowEnv.BASE_MODULE === 'liquid') {
 
 @NgModule({
   imports: [RouterModule.forRoot(routes, {
-    initialNavigation: 'enabled',
+    initialNavigation: 'enabledBlocking',
     scrollPositionRestoration: 'enabled',
     anchorScrolling: 'enabled',
     preloadingStrategy: AppPreloadingStrategy

frontend/src/app/app.constants.ts

@@ -79,7 +79,7 @@ export const poolsColor = {
    'binancepool': '#1E88E5',
    'viabtc': '#039BE5',
    'btccom': '#00897B',
-   'slushpool': '#00ACC1',
+   'braiinspool': '#00ACC1',
    'sbicrypto': '#43A047',
    'marapool': '#7CB342',
    'luxor': '#C0CA33',
@@ -120,7 +120,7 @@ export const languages: Language[] = [
    { code: 'he', name: 'עברית' },           // Hebrew
    { code: 'ka', name: 'ქართული' },         // Georgian
 // { code: 'lv', name: 'Latviešu' },        // Latvian
-// { code: 'lt', name: 'Lietuvių' },        // Lithuanian
+   { code: 'lt', name: 'Lietuvių' },        // Lithuanian
    { code: 'hu', name: 'Magyar' },          // Hungarian
    { code: 'mk', name: 'Македонски' },      // Macedonian
 // { code: 'ms', name: 'Bahasa Melayu' },   // Malay

frontend/src/app/app.module.ts

@@ -6,6 +6,7 @@ import { AppRoutingModule } from './app-routing.module';
 import { AppComponent } from './components/app/app.component';
 import { ElectrsApiService } from './services/electrs-api.service';
 import { StateService } from './services/state.service';
+import { CacheService } from './services/cache.service';
 import { EnterpriseService } from './services/enterprise.service';
 import { WebsocketService } from './services/websocket.service';
 import { AudioService } from './services/audio.service';
@@ -23,6 +24,7 @@ import { AppPreloadingStrategy } from './app.preloading-strategy';
 const providers = [
   ElectrsApiService,
   StateService,
+  CacheService,
   WebsocketService,
   AudioService,
   SeoService,

frontend/src/app/bisq/bisq-address/bisq-address.component.html

@@ -1,11 +1,9 @@
 <div class="container-xl">
   <h1 i18n="shared.address">Address</h1>
   <span class="address-link">
-    <a  [routerLink]="['/address/' | relativeUrl, addressString]">
+    <app-truncate [text]="addressString" [lastChars]="8" [link]="['/address/' | relativeUrl, addressString]">
-      <span class="d-inline d-lg-none">{{ addressString | shortenString : 24 }}</span>
+      <app-clipboard [text]="addressString"></app-clipboard>
-      <span class="d-none d-lg-inline">{{ addressString }}</span>
+    </app-truncate>
-    </a>
-    <app-clipboard [text]="addressString"></app-clipboard>
   </span>
   <br>
 

frontend/src/app/bisq/bisq-market/bisq-market.component.html

@@ -10,27 +10,27 @@
       </div>
   
       <form [formGroup]="radioGroupForm" class="mb-3 radio-form">
-        <div class="btn-group btn-group-toggle" ngbRadioGroup name="radioBasic" formControlName="interval">
+        <div class="btn-group btn-group-toggle" name="radioBasic">
-          <label ngbButtonLabel class="btn-primary btn-sm">
+          <label class="btn btn-primary btn-sm" [class.active]="radioGroupForm.get('interval').value === 'half_hour'">
-            <input ngbButton type="radio" [value]="'half_hour'" (click)="setFragment('half_hour')"> 30M
+            <input type="radio" [value]="'half_hour'" (click)="setFragment('half_hour')" formControlName="interval"> 30M
           </label>
-          <label ngbButtonLabel class="btn-primary btn-sm">
+          <label class="btn btn-primary btn-sm" [class.active]="radioGroupForm.get('interval').value === 'hour'">
-            <input ngbButton type="radio" [value]="'hour'" (click)="setFragment('hour')"> 1H
+            <input type="radio" [value]="'hour'" (click)="setFragment('hour')" formControlName="interval"> 1H
           </label>
-          <label ngbButtonLabel class="btn-primary btn-sm">
+          <label class="btn btn-primary btn-sm" [class.active]="radioGroupForm.get('interval').value === 'half_day'">
-            <input ngbButton type="radio" [value]="'half_day'" (click)="setFragment('half_day')"> 12H
+            <input type="radio" [value]="'half_day'" (click)="setFragment('half_day')" formControlName="interval"> 12H
           </label>
-          <label ngbButtonLabel class="btn-primary btn-sm">
+          <label class="btn btn-primary btn-sm" [class.active]="radioGroupForm.get('interval').value === 'day'">
-            <input ngbButton type="radio" [value]="'day'" (click)="setFragment('day')"> 1D
+            <input type="radio" [value]="'day'" (click)="setFragment('day')" formControlName="interval"> 1D
           </label>
-          <label ngbButtonLabel class="btn-primary btn-sm">
+          <label class="btn btn-primary btn-sm" [class.active]="radioGroupForm.get('interval').value === 'week'">
-            <input ngbButton type="radio" [value]="'week'" (click)="setFragment('week')"> 1W
+            <input type="radio" [value]="'week'" (click)="setFragment('week')" formControlName="interval"> 1W
           </label>
-          <label ngbButtonLabel class="btn-primary btn-sm">
+          <label class="btn btn-primary btn-sm" [class.active]="radioGroupForm.get('interval').value === 'month'">
-            <input ngbButton type="radio" [value]="'month'" (click)="setFragment('month')"> 1M
+            <input type="radio" [value]="'month'" (click)="setFragment('month')" formControlName="interval"> 1M
           </label>
-          <label ngbButtonLabel class="btn-primary btn-sm">
+          <label class="btn btn-primary btn-sm" [class.active]="radioGroupForm.get('interval').value === 'year'">
-            <input ngbButton type="radio" [value]="'year'" (click)="setFragment('year')"> 1Y
+            <input type="radio" [value]="'year'" (click)="setFragment('year')" formControlName="interval"> 1Y
           </label>
         </div>
       </form>

frontend/src/app/bisq/bisq-market/bisq-market.component.ts

@@ -1,5 +1,5 @@
 import { ChangeDetectionStrategy, Component, OnDestroy, OnInit } from '@angular/core';
-import { FormBuilder, FormGroup } from '@angular/forms';
+import { UntypedFormBuilder, UntypedFormGroup } from '@angular/forms';
 import { ActivatedRoute, Router } from '@angular/router';
 import { combineLatest, merge, Observable, of } from 'rxjs';
 import { map, switchMap } from 'rxjs/operators';
@@ -19,7 +19,7 @@ export class BisqMarketComponent implements OnInit, OnDestroy {
   currency$: Observable<any>;
   offers$: Observable<OffersMarket>;
   trades$: Observable<Trade[]>;
-  radioGroupForm: FormGroup;
+  radioGroupForm: UntypedFormGroup;
   defaultInterval = 'day';
 
   isLoadingGraph = false;
@@ -28,7 +28,7 @@ export class BisqMarketComponent implements OnInit, OnDestroy {
     private websocketService: WebsocketService,
     private route: ActivatedRoute,
     private bisqApiService: BisqApiService,
-    private formBuilder: FormBuilder,
+    private formBuilder: UntypedFormBuilder,
     private seoService: SeoService,
     private router: Router,
   ) { }

frontend/src/app/bisq/bisq-transaction/bisq-transaction.component.html

@@ -6,12 +6,12 @@
         <h1 i18n="shared.transaction">Transaction</h1>
       </div>
 
-      <span class="tx-link float-left">
+      <span class="tx-link">
-        <a [routerLink]="['/tx' | relativeUrl, bisqTx.id]">
+        <span class="txid">
-          <span class="d-inline d-lg-none">{{ bisqTx.id | shortenString : 24 }}</span>
+          <app-truncate [text]="bisqTx.id" [lastChars]="12" [link]="['/tx/' | relativeUrl, bisqTx.id]">
-          <span class="d-none d-lg-inline">{{ bisqTx.id }}</span>
+            <app-clipboard [text]="bisqTx.id"></app-clipboard>
-        </a>
+          </app-truncate>
-        <app-clipboard [text]="bisqTx.id"></app-clipboard>
+        </span>
       </span>
       <span class="grow"></span>
       <div class="container-buttons">

frontend/src/app/bisq/bisq-transactions/bisq-transactions.component.ts

@@ -5,7 +5,7 @@ import { Observable, Subscription } from 'rxjs';
 import { switchMap, map, tap } from 'rxjs/operators';
 import { BisqApiService } from '../bisq-api.service';
 import { SeoService } from '../../services/seo.service';
-import { FormGroup, FormBuilder } from '@angular/forms';
+import { UntypedFormGroup, UntypedFormBuilder } from '@angular/forms';
 import { Router, ActivatedRoute } from '@angular/router';
 import { IMultiSelectOption, IMultiSelectSettings, IMultiSelectTexts } from '../../components/ngx-bootstrap-multiselect/types'
 import { WebsocketService } from '../../services/websocket.service';
@@ -23,7 +23,7 @@ export class BisqTransactionsComponent implements OnInit, OnDestroy {
   fiveItemsPxSize = 250;
   isLoading = true;
   loadingItems: number[];
-  radioGroupForm: FormGroup;
+  radioGroupForm: UntypedFormGroup;
   types: string[] = [];
   radioGroupSubscription: Subscription;
 
@@ -70,7 +70,7 @@ export class BisqTransactionsComponent implements OnInit, OnDestroy {
     private websocketService: WebsocketService,
     private bisqApiService: BisqApiService,
     private seoService: SeoService,
-    private formBuilder: FormBuilder,
+    private formBuilder: UntypedFormBuilder,
     private route: ActivatedRoute,
     private router: Router,
     private cd: ChangeDetectorRef,

frontend/src/app/bisq/bisq.routing.module.ts

@@ -20,14 +20,17 @@ const routes: Routes = [
     },
     {
       path: 'markets',
+      data: { networks: ['bisq'] },
       component: BisqDashboardComponent,
     },
     {
       path: 'transactions',
+      data: { networks: ['bisq'] },
       component: BisqTransactionsComponent
     },
     {
       path: 'market/:pair',
+      data: { networkSpecific: true },
       component: BisqMarketComponent,
     },
     {
@@ -36,6 +39,7 @@ const routes: Routes = [
     },
     {
       path: 'tx/:id',
+      data: { networkSpecific: true },
       component: BisqTransactionComponent
     },
     {
@@ -45,14 +49,17 @@ const routes: Routes = [
     },
     {
       path: 'block/:id',
+      data: { networkSpecific: true },
       component: BisqBlockComponent,
     },
     {
       path: 'address/:id',
+      data: { networkSpecific: true },
       component: BisqAddressComponent,
     },
     {
       path: 'stats',
+      data: { networks: ['bisq'] },
       component: BisqStatsComponent,
     },
     {

frontend/src/app/components/about/about.component.html

@@ -274,6 +274,10 @@
         <img class="image" src="/resources/profile/schildbach.svg" />
         <span>Schildbach</span>
       </a>
+      <a href="https://github.com/nunchuk-io" target="_blank" title="Nunchuck">
+        <img class="image" src="/resources/profile/nunchuk.svg" />
+        <span>Nunchuk</span>
+      </a>
     </div>
   </div>
 

frontend/src/app/components/about/about.component.scss

@@ -191,6 +191,6 @@
 }
 
 .community-integrations-sponsor {
-  max-width: 970px;
+  max-width: 965px;
   margin: auto;
 }

frontend/src/app/components/address-labels/address-labels.component.ts

@@ -36,7 +36,9 @@ export class AddressLabelsComponent implements OnChanges {
 
   handleChannel() {
     const type = this.vout ? 'open' : 'close';
-    this.label = `Channel ${type}: ${this.channel.node_left.alias} <> ${this.channel.node_right.alias}`;
+    const leftNodeName = this.channel.node_left.alias || this.channel.node_left.public_key.substring(0, 10);
+    const rightNodeName = this.channel.node_right.alias || this.channel.node_right.public_key.substring(0, 10);
+    this.label = `Channel ${type}: ${leftNodeName} <> ${rightNodeName}`;
   }
 
   handleVin() {

frontend/src/app/components/address/address-preview.component.html

@@ -6,17 +6,16 @@
     <div class="col-md">
       <div class="row d-flex justify-content-between">
         <div class="title-wrapper">
-          <h1 class="title truncated"><span class="first">{{addressString.slice(0,-4)}}</span><span class="last-four">{{addressString.slice(-4)}}</span></h1>
+          <h1 class="title"><app-truncate [text]="addressString"></app-truncate></h1>
         </div>
       </div>
       <table class="table table-borderless table-striped">
         <tbody>
           <tr *ngIf="addressInfo && addressInfo.unconfidential">
             <td i18n="address.unconfidential">Unconfidential</td>
-            <td><a [routerLink]="['/address/' | relativeUrl, addressInfo.unconfidential]">
+            <td>
-              <span class="d-inline d-lg-none">{{ addressInfo.unconfidential | shortenString : 14 }}</span>
+              <app-truncate [text]="addressInfo.unconfidential" [lastChars]="7" [link]="['/address/' | relativeUrl, addressInfo.unconfidential]"></app-truncate>
-              <span class="d-none d-lg-inline">{{ addressInfo.unconfidential }}</span>
+            </td>
-            </a> <app-clipboard [text]="addressInfo.unconfidential"></app-clipboard></td>
           </tr>
           <ng-template [ngIf]="!address.electrum">
             <tr>

frontend/src/app/components/address/address.component.html

@@ -2,11 +2,9 @@
   <div class="title-address">
     <h1 i18n="shared.address">Address</h1>
     <div class="tx-link">
-      <a [routerLink]="['/address/' | relativeUrl, addressString]" >
+      <app-truncate [text]="addressString" [lastChars]="8" [link]="['/address/' | relativeUrl, addressString]">
-        <span class="d-inline d-lg-none">{{ addressString | shortenString : 18 }}</span>
+        <app-clipboard [text]="addressString"></app-clipboard>
-        <span class="d-none d-lg-inline">{{ addressString }}</span>
+      </app-truncate>
-      </a>
-      <app-clipboard [text]="addressString"></app-clipboard>
     </div>
   </div>
 
@@ -21,10 +19,11 @@
             <tbody>
               <tr *ngIf="addressInfo && addressInfo.unconfidential">
                 <td i18n="address.unconfidential">Unconfidential</td>
-                <td><a [routerLink]="['/address/' | relativeUrl, addressInfo.unconfidential]">
+                <td>
-                  <span class="d-inline d-lg-none">{{ addressInfo.unconfidential | shortenString : 14 }}</span>
+                  <app-truncate [text]="addressInfo.unconfidential" [lastChars]="8" [link]="['/address/' | relativeUrl, addressInfo.unconfidential]">
-                  <span class="d-none d-lg-inline">{{ addressInfo.unconfidential }}</span>
+                    <app-clipboard [text]="addressInfo.unconfidential"></app-clipboard>
-                </a> <app-clipboard [text]="addressInfo.unconfidential"></app-clipboard></td>
+                  </app-truncate>
+                </td>
               </tr>
               <ng-template [ngIf]="!address.electrum">
                 <tr>

frontend/src/app/components/app/app.component.ts

@@ -25,6 +25,8 @@ export class AppComponent implements OnInit {
     if (this.locale.startsWith('ar') || this.locale.startsWith('fa') || this.locale.startsWith('he')) {
       this.dir = 'rtl';
       this.class = 'rtl-layout';
+    } else {
+      this.class = 'ltr-layout';
     }
 
     tooltipConfig.animation = false;
@@ -40,6 +42,10 @@ export class AppComponent implements OnInit {
     if (event.target instanceof HTMLInputElement) {
       return;
     }
+    // prevent arrow key horizontal scrolling
+    if(["ArrowLeft","ArrowRight"].indexOf(event.code) > -1) {
+      event.preventDefault();
+    }
     this.stateService.keyNavigation$.next(event);
   }
 

frontend/src/app/components/asset-circulation/asset-circulation.component.ts

@@ -4,7 +4,7 @@ import { map } from 'rxjs/operators';
 import { moveDec } from '../../bitcoin.utils';
 import { AssetsService } from '../../services/assets.service';
 import { ElectrsApiService } from '../../services/electrs-api.service';
-import { environment } from 'src/environments/environment';
+import { environment } from '../../../environments/environment';
 
 @Component({
   selector: 'app-asset-circulation',

frontend/src/app/components/asset/asset.component.html

@@ -2,11 +2,9 @@
   <div class="title-asset">
     <h1 i18n="asset|Liquid Asset page title">Asset</h1>
     <div class="tx-link">
-      <a [routerLink]="['/assets/asset/' | relativeUrl, assetString]">
+      <app-truncate [text]="assetString" [lastChars]="8" [link]="['/assets/asset/' | relativeUrl, assetString]">
-        <span class="d-inline d-lg-none">{{ assetString | shortenString : 24 }}</span>
+        <app-clipboard [text]="assetString"></app-clipboard>
-        <span class="d-none d-lg-inline">{{ assetString }}</span>
+      </app-truncate>
-      </a>
-      <app-clipboard [text]="assetString"></app-clipboard>
     </div>
   </div>
 

frontend/src/app/components/asset/asset.component.ts

@@ -9,7 +9,7 @@ import { AudioService } from '../../services/audio.service';
 import { ApiService } from '../../services/api.service';
 import { of, merge, Subscription, combineLatest } from 'rxjs';
 import { SeoService } from '../../services/seo.service';
-import { environment } from 'src/environments/environment';
+import { environment } from '../../../environments/environment';
 import { AssetsService } from '../../services/assets.service';
 import { moveDec } from '../../bitcoin.utils';
 

frontend/src/app/components/assets/assets-nav/assets-nav.component.ts

@@ -1,5 +1,5 @@
 import { Component, OnInit, ViewChild } from '@angular/core';
-import { FormBuilder, FormGroup, Validators } from '@angular/forms';
+import { UntypedFormBuilder, UntypedFormGroup, Validators } from '@angular/forms';
 import { Router } from '@angular/router';
 import { NgbTypeahead } from '@ng-bootstrap/ng-bootstrap';
 import { merge, Observable, of, Subject } from 'rxjs';
@@ -9,7 +9,7 @@ import { AssetsService } from '../../../services/assets.service';
 import { SeoService } from '../../../services/seo.service';
 import { StateService } from '../../../services/state.service';
 import { RelativeUrlPipe } from '../../../shared/pipes/relative-url/relative-url.pipe';
-import { environment } from 'src/environments/environment';
+import { environment } from '../../../../environments/environment';
 
 @Component({
   selector: 'app-assets-nav',
@@ -19,7 +19,7 @@ import { environment } from 'src/environments/environment';
 export class AssetsNavComponent implements OnInit {
   @ViewChild('instance', {static: true}) instance: NgbTypeahead;
   nativeAssetId = this.stateService.network === 'liquidtestnet' ? environment.nativeTestAssetId : environment.nativeAssetId;
-  searchForm: FormGroup;
+  searchForm: UntypedFormGroup;
   assetsCache: AssetExtended[];
 
   typeaheadSearchFn: ((text: Observable<string>) => Observable<readonly any[]>);
@@ -30,7 +30,7 @@ export class AssetsNavComponent implements OnInit {
   itemsPerPage = 15;
 
   constructor(
-    private formBuilder: FormBuilder,
+    private formBuilder: UntypedFormBuilder,
     private seoService: SeoService,
     private router: Router,
     private assetsService: AssetsService,

frontend/src/app/components/assets/assets.component.ts

@@ -1,7 +1,7 @@
 import { Component, OnInit, ChangeDetectionStrategy } from '@angular/core';
 import { AssetsService } from '../../services/assets.service';
-import { environment } from 'src/environments/environment';
+import { environment } from '../../../environments/environment';
-import { FormGroup } from '@angular/forms';
+import { UntypedFormGroup } from '@angular/forms';
 import { filter, map, switchMap, take } from 'rxjs/operators';
 import { ActivatedRoute, Router } from '@angular/router';
 import { combineLatest, Observable } from 'rxjs';
@@ -22,7 +22,7 @@ export class AssetsComponent implements OnInit {
 
   assets: AssetExtended[];
   assetsCache: AssetExtended[];
-  searchForm: FormGroup;
+  searchForm: UntypedFormGroup;
   assets$: Observable<AssetExtended[]>;
 
   page = 1;

frontend/src/app/components/bisq-master-page/bisq-master-page.component.html

@@ -44,13 +44,13 @@
       <app-svg-images name="bisq" width="20" height="20" viewBox="0 0 75 75" style="width: 25px; height: 25px;" class="mainnet mr-1"></app-svg-images>
     </button>
     <div ngbDropdownMenu [ngClass]="{'dropdown-menu-right' : isMobile}">
-      <a [href]="env.MEMPOOL_WEBSITE_URL + urlLanguage" ngbDropdownItem class="mainnet"><app-svg-images name="bitcoin" width="22" height="22" viewBox="0 0 65 65" style="width: 25px; height: 25px;" class="mainnet mr-1"></app-svg-images> Mainnet</a>
+      <a [href]="env.MEMPOOL_WEBSITE_URL + urlLanguage + (networkPaths['mainnet'] || '/')" ngbDropdownItem class="mainnet"><app-svg-images name="bitcoin" width="22" height="22" viewBox="0 0 65 65" style="width: 25px; height: 25px;" class="mainnet mr-1"></app-svg-images> Mainnet</a>
-      <a [href]="env.MEMPOOL_WEBSITE_URL + urlLanguage + '/signet'" ngbDropdownItem *ngIf="env.SIGNET_ENABLED" class="signet"><app-svg-images name="signet" width="22" height="22" viewBox="0 0 65 65" style="width: 25px; height: 25px;" class="mainnet mr-1"></app-svg-images> Signet</a>
+      <a [href]="env.MEMPOOL_WEBSITE_URL + urlLanguage + (networkPaths['signet'] || '/signet')" ngbDropdownItem *ngIf="env.SIGNET_ENABLED" class="signet"><app-svg-images name="signet" width="22" height="22" viewBox="0 0 65 65" style="width: 25px; height: 25px;" class="mainnet mr-1"></app-svg-images> Signet</a>
-      <a [href]="env.MEMPOOL_WEBSITE_URL + urlLanguage + '/testnet'" ngbDropdownItem *ngIf="env.TESTNET_ENABLED" class="testnet"><app-svg-images name="testnet" width="22" height="22" viewBox="0 0 65 65" style="width: 25px; height: 25px;" class="mainnet mr-1"></app-svg-images> Testnet</a>
+      <a [href]="env.MEMPOOL_WEBSITE_URL + urlLanguage + (networkPaths['testnet'] || '/testnet')" ngbDropdownItem *ngIf="env.TESTNET_ENABLED" class="testnet"><app-svg-images name="testnet" width="22" height="22" viewBox="0 0 65 65" style="width: 25px; height: 25px;" class="mainnet mr-1"></app-svg-images> Testnet</a>
       <h6 class="dropdown-header" i18n="master-page.layer2-networks-header">Layer 2 Networks</h6>
-      <a ngbDropdownItem class="mainnet active" routerLink="/"><app-svg-images name="bisq" width="20" height="20" viewBox="0 0 75 75" style="width: 25px; height: 25px;" class="mainnet mr-1"></app-svg-images> Bisq</a>
+      <a ngbDropdownItem class="mainnet active" [routerLink]="networkPaths['bisq'] || '/'"><app-svg-images name="bisq" width="20" height="20" viewBox="0 0 75 75" style="width: 25px; height: 25px;" class="mainnet mr-1"></app-svg-images> Bisq</a>
-      <a [href]="env.LIQUID_WEBSITE_URL + urlLanguage" ngbDropdownItem *ngIf="env.LIQUID_ENABLED" class="liquid"><app-svg-images name="liquid" width="22" height="22" viewBox="0 0 125 125" style="width: 25px; height: 25px;" class="mainnet mr-1"></app-svg-images> Liquid</a>
+      <a [href]="env.LIQUID_WEBSITE_URL + urlLanguage + (networkPaths['liquid'] || '/')" ngbDropdownItem *ngIf="env.LIQUID_ENABLED" class="liquid"><app-svg-images name="liquid" width="22" height="22" viewBox="0 0 125 125" style="width: 25px; height: 25px;" class="mainnet mr-1"></app-svg-images> Liquid</a>
-      <a [href]="env.LIQUID_WEBSITE_URL + urlLanguage + '/testnet'" ngbDropdownItem *ngIf="env.LIQUID_TESTNET_ENABLED" class="liquidtestnet"><app-svg-images name="liquidtestnet" width="22" height="22" viewBox="0 0 125 125" style="width: 25px; height: 25px;" class="mainnet mr-1"></app-svg-images> Liquid Testnet</a>
+      <a [href]="env.LIQUID_WEBSITE_URL + urlLanguage + (networkPaths['liquidtestnet'] || '/testnet')" ngbDropdownItem *ngIf="env.LIQUID_TESTNET_ENABLED" class="liquidtestnet"><app-svg-images name="liquidtestnet" width="22" height="22" viewBox="0 0 125 125" style="width: 25px; height: 25px;" class="mainnet mr-1"></app-svg-images> Liquid Testnet</a>
     </div>
   </div>
 

frontend/src/app/components/bisq-master-page/bisq-master-page.component.ts

@@ -3,6 +3,7 @@ import { Env, StateService } from '../../services/state.service';
 import { Observable } from 'rxjs';
 import { LanguageService } from '../../services/language.service';
 import { EnterpriseService } from '../../services/enterprise.service';
+import { NavigationService } from '../../services/navigation.service';
 
 @Component({
   selector: 'app-bisq-master-page',
@@ -15,17 +16,22 @@ export class BisqMasterPageComponent implements OnInit {
   env: Env;
   isMobile = window.innerWidth <= 767.98;
   urlLanguage: string;
+  networkPaths: { [network: string]: string };
 
   constructor(
     private stateService: StateService,
     private languageService: LanguageService,
     private enterpriseService: EnterpriseService,
+    private navigationService: NavigationService,
   ) { }
 
   ngOnInit() {
     this.env = this.stateService.env;
     this.connectionState$ = this.stateService.connectionState$;
     this.urlLanguage = this.languageService.getLanguageForUrl();
+    this.navigationService.subnetPaths.subscribe((paths) => {
+      this.networkPaths = paths;
+    });
   }
 
   collapse(): void {

frontend/src/app/components/block-audit/block-audit.component.html

@@ -1,111 +0,0 @@
-<div class="container-xl" (window:resize)="onResize($event)">
-
-  <div *ngIf="(auditObservable$ | async) as blockAudit; else skeleton">
-    <div class="title-block" id="block">
-      <h1>
-        <span class="next-previous-blocks">
-          <span i18n="shared.block-title">Block </span>
-          &nbsp;
-          <a [routerLink]="['/block/' | relativeUrl, blockAudit.id]">{{ blockAudit.height }}</a>
-          &nbsp;
-          <span i18n="shared.template-vs-mined">Template vs Mined</span>
-        </span>
-      </h1>
-
-      <div class="grow"></div>
-
-      <button [routerLink]="['/' | relativeUrl]" class="btn btn-sm">&#10005;</button>
-    </div>
-
-    <!-- OVERVIEW -->
-    <div class="box mb-3">
-      <div class="row">
-        <!-- LEFT COLUMN -->
-        <div class="col-sm">
-          <table class="table table-borderless table-striped">
-            <tbody>
-              <tr>
-                <td class="td-width" i18n="block.hash">Hash</td>
-                <td><a [routerLink]="['/block/' | relativeUrl, blockAudit.id]" title="{{ blockAudit.id }}">{{ blockAudit.id | shortenString : 13 }}</a>
-                  <app-clipboard class="d-none d-sm-inline-block" [text]="blockAudit.id"></app-clipboard>
-                </td>
-              </tr>
-              <tr>
-                <td i18n="blockAudit.timestamp">Timestamp</td>
-                <td>
-                  &lrm;{{ blockAudit.timestamp * 1000 | date:'yyyy-MM-dd HH:mm' }}
-                  <div class="lg-inline">
-                    <i class="symbol">(<app-time-since [time]="blockAudit.timestamp" [fastRender]="true">
-                      </app-time-since>)</i>
-                  </div>
-                </td>
-              </tr>
-              <tr>
-                <td i18n="blockAudit.size">Size</td>
-                <td [innerHTML]="'&lrm;' + (blockAudit.size | bytes: 2)"></td>
-              </tr>
-              <tr>
-                <td i18n="block.weight">Weight</td>
-                <td [innerHTML]="'&lrm;' + (blockAudit.weight | wuBytes: 2)"></td>
-              </tr>
-            </tbody>
-          </table>
-        </div>
-
-        <!-- RIGHT COLUMN -->
-        <div class="col-sm" *ngIf="blockAudit">
-          <table class="table table-borderless table-striped">
-            <tbody>
-              <tr>
-                <td class="td-width" i18n="shared.transaction-count">Transactions</td>
-                <td>{{ blockAudit.tx_count }}</td>
-              </tr>
-              <tr>
-                <td i18n="block.match-rate">Match rate</td>
-                <td>{{ blockAudit.matchRate }}%</td>
-              </tr>
-              <tr>
-                <td i18n="block.missing-txs">Missing txs</td>
-                <td>{{ blockAudit.missingTxs.length }}</td>
-              </tr>
-              <tr>
-                <td i18n="block.added-txs">Added txs</td>
-                <td>{{ blockAudit.addedTxs.length }}</td>
-              </tr>
-            </tbody>
-          </table>
-        </div>
-      </div> <!-- row -->
-    </div> <!-- box -->
-
-    <!-- ADDED vs MISSING button -->
-    <div class="d-flex justify-content-center menu mt-3" *ngIf="isMobile">
-      <a routerLinkActive="active" class="btn btn-primary w-50 mr-1 ml-1 menu-button" i18n="block.missing-txs"
-        fragment="missing" (click)="changeMode('missing')">Missing</a>
-      <a routerLinkActive="active" class="btn btn-primary w-50 mr-1 ml-1 menu-button" i18n="block.added-txs"
-        fragment="added" (click)="changeMode('added')">Added</a>
-    </div>
-  </div>
-
-  <!-- VISUALIZATIONS -->
-  <div class="box">
-    <div class="row">
-      <!-- MISSING TX RENDERING -->
-      <div class="col-sm" *ngIf="webGlEnabled">
-        <app-block-overview-graph #blockGraphTemplate [isLoading]="isLoading" [resolution]="75"
-          [blockLimit]="stateService.blockVSize" [orientation]="'top'" [flip]="false"
-          (txClickEvent)="onTxClick($event)"></app-block-overview-graph>
-      </div>
-
-      <!-- ADDED TX RENDERING -->
-      <div class="col-sm" *ngIf="webGlEnabled && !isMobile">
-        <app-block-overview-graph #blockGraphMined [isLoading]="isLoading" [resolution]="75"
-          [blockLimit]="stateService.blockVSize" [orientation]="'top'" [flip]="false"
-          (txClickEvent)="onTxClick($event)"></app-block-overview-graph>
-      </div>
-    </div> <!-- row -->
-  </div> <!-- box -->
-
-  <ng-template #skeleton></ng-template>
-
-</div>