add high contrast theme

.github/CODEOWNERS

@@ -1 +0,0 @@
-backend/src/api/database-migration.ts	@wiz @softsimon

.github/dependabot.yml

@@ -8,9 +8,6 @@ updates:
     ignore:
       - dependency-name: "*"
         update-types: ["version-update:semver-major"]
-    allow:
-      - dependency-type: "production"
-
   - package-ecosystem: npm
     directory: "/frontend"
     schedule:
@@ -19,29 +16,17 @@ updates:
     ignore:
       - dependency-name: "*"
         update-types: ["version-update:semver-major"]
-    allow:
-      - dependency-type: "production"
-
   - package-ecosystem: docker
     directory: "/docker/backend"
     schedule:
-      interval: weekly
+      interval: daily
     ignore:
       - dependency-name: "*"
         update-types: ["version-update:semver-major"]
-
-  - package-ecosystem: docker
-    directory: "/docker/frontend"
-    schedule:
-      interval: weekly
-    ignore:
-      - dependency-name: "*"
-        update-types: ["version-update:semver-major"]
-
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
-      interval: weekly
+      interval: daily
     ignore:
       - dependency-name: "*"
         update-types: ["version-update:semver-major"]

.github/workflows/ci.yml

@@ -9,7 +9,7 @@ jobs:
     if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')"
     strategy:
       matrix:
-        node: ["16.16.0", "18.14.1"]
+        node: ["16.16.0", "18.5.0"]
         flavor: ["dev", "prod"]
       fail-fast: false
     runs-on: "ubuntu-latest"
@@ -55,7 +55,7 @@ jobs:
     if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')"
     strategy:
       matrix:
-        node: ["16.16.0", "18.14.1"]
+        node: ["16.15.0", "18.5.0"]
         flavor: ["dev", "prod"]
       fail-fast: false
     runs-on: "ubuntu-latest"

.github/workflows/cypress.yml

@@ -1,11 +1,8 @@
 name: Cypress Tests
 
 on:
-  push:
-    branches: [master]
   pull_request:
-    types: [opened, synchronize]
-
+    types: [opened, review_requested, synchronize]
 jobs:
   cypress:
     if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')"

.github/workflows/get_image_digest.yml

@@ -1,26 +0,0 @@
-name: 'Print images digest'
-
-on:
-  workflow_dispatch:
-    inputs:
-      version:
-        description: 'Image Version'
-        required: false
-        default: 'latest'
-        type: string 
-jobs:
-  print-images-sha:
-    runs-on: 'ubuntu-latest'
-    name: Print digest for images
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          path: digest
-
-      - name: Run script
-        working-directory: digest
-        run: |
-          sh ./docker/scripts/get_image_digest.sh $VERSION
-        env: 
-          VERSION: ${{ github.event.inputs.version }}

.github/workflows/on-tag.yml

@@ -31,7 +31,7 @@ jobs:
         run: |
           sudo swapoff /mnt/swapfile
           sudo rm -v /mnt/swapfile
-          sudo fallocate -l 13G /mnt/swapfile
+          sudo fallocate -l 10G /mnt/swapfile
           sudo chmod 600 /mnt/swapfile
           sudo mkswap /mnt/swapfile
           sudo swapon /mnt/swapfile
@@ -68,24 +68,24 @@ jobs:
         run: echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin
 
       - name: Checkout project
-        uses: actions/checkout@v3
+        uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v2.5.0
 
       - name: Init repo for Dockerization
         run: docker/init.sh "$TAG"
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v2.1.0
         id: qemu
 
       - name: Setup Docker buildx action
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@8c0edbc76e98fa90f69d9a2c020dcb50019dc325 # v2.2.1
         id: buildx
 
       - name: Available platforms
         run: echo ${{ steps.buildx.outputs.platforms }}
 
       - name: Cache Docker layers
-        uses: actions/cache@v3
+        uses: actions/cache@9b0c1fce7a93df8e3bb8926b0d6e9d89e92f20a7 # v3.0.11
         id: cache
         with:
           path: /tmp/.buildx-cache

.vscode/settings.json

@@ -1,5 +1,4 @@
 {
   "editor.tabSize": 2,
-  "typescript.preferences.importModuleSpecifier": "relative",
   "typescript.tsdk": "./backend/node_modules/typescript/lib"
 }

LICENSE

@@ -1,5 +1,5 @@
 The Mempool Open Source Project
-Copyright (c) 2019-2023 The Mempool Open Source Project Developers
+Copyright (c) 2019-2022 The Mempool Open Source Project Developers
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of (at your option) either:

README.md

@@ -1,13 +1,11 @@
 # The Mempool Open Source Project™ [![mempool](https://img.shields.io/endpoint?url=https://dashboard.cypress.io/badge/simple/ry4br7/master&style=flat-square)](https://dashboard.cypress.io/projects/ry4br7/runs)
 
-https://user-images.githubusercontent.com/93150691/226236121-375ea64f-b4a1-4cc0-8fad-a6fb33226840.mp4
-
-<br>
-
 Mempool is the fully-featured mempool visualizer, explorer, and API service running at [mempool.space](https://mempool.space/). 
 
 It is an open-source project developed and operated for the benefit of the Bitcoin community, with a focus on the emerging transaction fee market that is evolving Bitcoin into a multi-layer ecosystem.
 
+![mempool](https://mempool.space/resources/screenshots/v2.4.0-dashboard.png)
+
 # Installation Methods
 
 Mempool can be self-hosted on a wide variety of your own hardware, ranging from a simple one-click installation on a Raspberry Pi full-node distro all the way to a robust production instance on a powerful FreeBSD server. 

backend/README.md

@@ -160,7 +160,7 @@ npm install -g ts-node nodemon
 Then, run the watcher:
 
 ```
-nodemon src/index.ts --ignore cache/
+nodemon src/index.ts --ignore cache/ --ignore pools.json
 ```
 
 `nodemon` should be in npm's global binary folder. If needed, you can determine where that is with `npm -g bin`.
@@ -171,84 +171,50 @@ Helpful link: https://gist.github.com/System-Glitch/cb4e87bf1ae3fec9925725bb3ebe
 
 Run bitcoind on regtest:
    ```
-   bitcoind -regtest
+   bitcoind -regtest -rpcport=8332
    ```
 
 Create a new wallet, if needed:
    ```
-   bitcoin-cli -regtest createwallet test
+   bitcoin-cli -regtest -rpcport=8332 createwallet test
    ```
 
 Load wallet (this command may take a while if you have lot of UTXOs):
    ```
-   bitcoin-cli -regtest loadwallet test
+   bitcoin-cli -regtest -rpcport=8332 loadwallet test
    ```
 
 Get a new address:
    ```
-   address=$(bitcoin-cli -regtest getnewaddress)
+   address=$(./src/bitcoin-cli -regtest -rpcport=8332 getnewaddress)
    ```
 
 Mine blocks to the previously generated address. You need at least 101 blocks before you can spend. This will take some time to execute (~1 min):
    ```
-   bitcoin-cli -regtest generatetoaddress 101 $address
+   bitcoin-cli -regtest -rpcport=8332 generatetoaddress 101 $address
    ```
 
 Send 0.1 BTC at 5 sat/vB to another address:
    ```
-   bitcoin-cli -named -regtest sendtoaddress address=$(bitcoin-cli -regtest getnewaddress) amount=0.1 fee_rate=5
+   ./src/bitcoin-cli -named -regtest -rpcport=8332 sendtoaddress address=$(./src/bitcoin-cli -regtest -rpcport=8332 getnewaddress) amount=0.1 fee_rate=5
    ```
 
 See more example of `sendtoaddress`:
    ```
-   bitcoin-cli sendtoaddress # will print the help
+   ./src/bitcoin-cli sendtoaddress # will print the help
    ```
 
-Mini script to generate random network activity (random TX count with random tx fee-rate). It's slow so don't expect to use this to test mempool spam, except if you let it run for a long time, or maybe with multiple regtest nodes connected to each other.
+Mini script to generate transactions with random TX fee-rate (between 1 to 100 sat/vB). It's slow so don't expect to use this to test mempool spam, except if you let it run for a long time, or maybe with multiple regtest nodes connected to each other.
    ```
    #!/bin/bash
-   address=$(bitcoin-cli -regtest getnewaddress)
-   bitcoin-cli -regtest generatetoaddress 101 $address
+   address=$(./src/bitcoin-cli -regtest -rpcport=8332 getnewaddress)
    for i in {1..1000000}
    do
-      for y in $(seq 1 "$(jot -r 1 1 1000)")
-      do
-         bitcoin-cli -regtest -named sendtoaddress address=$address amount=0.01 fee_rate=$(jot -r 1 1 100)
-      done
-      bitcoin-cli -regtest generatetoaddress 1 $address
-      sleep 5
+     ./src/bitcoin-cli -regtest -rpcport=8332 -named sendtoaddress address=$address amount=0.01 fee_rate=$(jot -r 1  1 100)
    done
    ```
 
 Generate block at regular interval (every 10 seconds in this example):
    ```
-   watch -n 10 "bitcoin-cli -regtest generatetoaddress 1 $address"
+   watch -n 10 "./src/bitcoin-cli -regtest -rpcport=8332 generatetoaddress 1 $address"
    ```
-
-### Mining pools update
-
-By default, mining pools will be not automatically updated regularly (`config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING` is set to `false`). 
-
-To manually update your mining pools, you can use the `--update-pools` command line flag when you run the nodejs backend. For example `npm run start --update-pools`. This will trigger the mining pools update and automatically re-index appropriate blocks.
-
-You can enabled the automatic mining pools update by settings `config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING` to `true` in your `mempool-config.json`.
-
-When a `coinbase tag` or `coinbase address` change is detected, all blocks tagged to the `unknown` mining pools (starting from height 130635) will be deleted from the `blocks` table. Additionaly, all blocks which were tagged to the pool which has been updated will also be deleted from the `blocks` table. Of course, those blocks will be automatically reindexed.
-
-### Re-index tables
-
-You can manually force the nodejs backend to drop all data from a specified set of tables for future re-index. This is mostly useful for the mining dashboard and the lightning explorer.
-
-Use the `--reindex` command to specify a list of comma separated table which will be truncated at start. Note that a 5 seconds delay will be observed before truncating tables in order to give you a chance to cancel (CTRL+C) in case of misuse of the command.
-
-Usage:
-```
-npm run start --reindex=blocks,hashrates
-```
-Example output:
-```
-Feb 13 14:55:27 [63246] WARN: <lightning> Indexed data for "hashrates" tables will be erased in 5 seconds (using '--reindex')
-Feb 13 14:55:32 [63246] NOTICE: <lightning> Table hashrates has been truncated
-```
-
-Reference: https://github.com/mempool/mempool/pull/1269

backend/mempool-config.sample.json

@@ -15,6 +15,7 @@
     "MEMPOOL_BLOCKS_AMOUNT": 8,
     "INDEXING_BLOCKS_AMOUNT": 11000,
     "BLOCKS_SUMMARIES_INDEXING": false,
+    "PRICE_FEED_UPDATE_INTERVAL": 600,
     "USE_SECOND_NODE_FOR_MINFEE": false,
     "EXTERNAL_ASSETS": [],
     "EXTERNAL_MAX_RETRY": 1,
@@ -22,20 +23,17 @@
     "USER_AGENT": "mempool",
     "STDOUT_LOG_MIN_PRIORITY": "debug",
     "AUTOMATIC_BLOCK_REINDEXING": false,
-    "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json",
+    "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json",
     "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
-    "AUDIT": false,
     "ADVANCED_GBT_AUDIT": false,
     "ADVANCED_GBT_MEMPOOL": false,
-    "CPFP_INDEXING": false,
-    "DISK_CACHE_BLOCK_INTERVAL": 6
+    "TRANSACTION_INDEXING": false
   },
   "CORE_RPC": {
     "HOST": "127.0.0.1",
     "PORT": 8332,
     "USERNAME": "mempool",
-    "PASSWORD": "mempool",
-    "TIMEOUT": 60000
+    "PASSWORD": "mempool"
   },
   "ELECTRUM": {
     "HOST": "127.0.0.1",
@@ -49,8 +47,7 @@
     "HOST": "127.0.0.1",
     "PORT": 8332,
     "USERNAME": "mempool",
-    "PASSWORD": "mempool",
-    "TIMEOUT": 60000
+    "PASSWORD": "mempool"
   },
   "DATABASE": {
     "ENABLED": true,
@@ -94,8 +91,7 @@
   "LND": {
     "TLS_CERT_PATH": "tls.cert",
     "MACAROON_PATH": "readonly.macaroon",
-    "REST_API_URL": "https://localhost:8080",
-    "TIMEOUT": 10000
+    "REST_API_URL": "https://localhost:8080"
   },
   "CLIGHTNING": {
     "SOCKET": "lightning-rpc"

backend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mempool-backend",
-  "version": "2.5.1",
+  "version": "2.5.0-dev",
   "description": "Bitcoin mempool visualizer and blockchain explorer backend",
   "license": "GNU Affero General Public License v3.0",
   "homepage": "https://mempool.space",
@@ -27,18 +27,18 @@
     "package": "npm run build && rm -rf package && mv dist package && mv node_modules package && npm run package-rm-build-deps",
     "package-rm-build-deps": "(cd package/node_modules; rm -r typescript @typescript-eslint)",
     "start": "node --max-old-space-size=2048 dist/index.js",
-    "start-production": "node --max-old-space-size=16384 dist/index.js",
+    "start-production": "node --max-old-space-size=4096 dist/index.js",
     "test": "./node_modules/.bin/jest --coverage",
     "lint": "./node_modules/.bin/eslint . --ext .ts",
     "lint:fix": "./node_modules/.bin/eslint . --ext .ts --fix",
     "prettier": "./node_modules/.bin/prettier --write \"src/**/*.{js,ts}\""
   },
   "dependencies": {
-    "@babel/core": "^7.20.12",
-    "@mempool/electrum-client": "1.1.9",
-    "@types/node": "^16.18.11",
+    "@babel/core": "^7.20.5",
+    "@mempool/electrum-client": "^1.1.7",
+    "@types/node": "^16.11.41",
     "axios": "~0.27.2",
-    "bitcoinjs-lib": "~6.1.0",
+    "bitcoinjs-lib": "~6.0.2",
     "crypto-js": "~4.1.1",
     "express": "~4.18.2",
     "maxmind": "~4.3.8",
@@ -49,19 +49,19 @@
     "ws": "~8.11.0"
   },
   "devDependencies": {
-    "@babel/core": "^7.20.7",
+    "@babel/core": "^7.20.5",
     "@babel/code-frame": "^7.18.6",
     "@types/compression": "^1.7.2",
     "@types/crypto-js": "^4.1.1",
-    "@types/express": "^4.17.15",
-    "@types/jest": "^29.2.5",
-    "@types/ws": "~8.5.4",
-    "@typescript-eslint/eslint-plugin": "^5.48.1",
-    "@typescript-eslint/parser": "^5.48.1",
-    "eslint": "^8.31.0",
+    "@types/express": "^4.17.14",
+    "@types/jest": "^29.2.3",
+    "@types/ws": "~8.5.3",
+    "@typescript-eslint/eslint-plugin": "^5.45.0",
+    "@typescript-eslint/parser": "^5.45.0",
+    "eslint": "^8.28.0",
     "eslint-config-prettier": "^8.5.0",
     "jest": "^29.3.1",
-    "prettier": "^2.8.2",
+    "prettier": "^2.8.0",
     "ts-jest": "^29.0.3",
     "ts-node": "^10.9.1"
   }

backend/src/__fixtures__/mempool-config.template.json

@@ -3,11 +3,12 @@
     "ENABLED": true,
     "NETWORK": "__MEMPOOL_NETWORK__",
     "BACKEND": "__MEMPOOL_BACKEND__",
+    "ENABLED": true,
     "BLOCKS_SUMMARIES_INDEXING": true,
     "HTTP_PORT": 1,
     "SPAWN_CLUSTER_PROCS": 2,
     "API_URL_PREFIX": "__MEMPOOL_API_URL_PREFIX__",
-    "AUTOMATIC_BLOCK_REINDEXING": false,
+    "AUTOMATIC_BLOCK_REINDEXING": true,
     "POLL_RATE_MS": 3,
     "CACHE_DIR": "__MEMPOOL_CACHE_DIR__",
     "CLEAR_PROTECTION_MINUTES": 4,
@@ -15,6 +16,7 @@
     "BLOCK_WEIGHT_UNITS": 6,
     "INITIAL_BLOCKS_AMOUNT": 7,
     "MEMPOOL_BLOCKS_AMOUNT": 8,
+    "PRICE_FEED_UPDATE_INTERVAL": 9,
     "USE_SECOND_NODE_FOR_MINFEE": 10,
     "EXTERNAL_ASSETS": 11,
     "EXTERNAL_MAX_RETRY": 12,
@@ -24,19 +26,15 @@
     "INDEXING_BLOCKS_AMOUNT": 14,
     "POOLS_JSON_TREE_URL": "__POOLS_JSON_TREE_URL__",
     "POOLS_JSON_URL": "__POOLS_JSON_URL__",
-    "AUDIT": "__MEMPOOL_AUDIT__",
-    "ADVANCED_GBT_AUDIT": "__MEMPOOL_ADVANCED_GBT_AUDIT__",
-    "ADVANCED_GBT_MEMPOOL": "__MEMPOOL_ADVANCED_GBT_MEMPOOL__",
-    "CPFP_INDEXING": "__MEMPOOL_CPFP_INDEXING__",
-    "MAX_BLOCKS_BULK_QUERY": "__MEMPOOL_MAX_BLOCKS_BULK_QUERY__",
-    "DISK_CACHE_BLOCK_INTERVAL": "__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__"
+    "ADVANCED_GBT_AUDIT": "__ADVANCED_GBT_AUDIT__",
+    "ADVANCED_GBT_MEMPOOL": "__ADVANCED_GBT_MEMPOOL__",
+    "TRANSACTION_INDEXING": "__TRANSACTION_INDEXING__"
   },
   "CORE_RPC": {
     "HOST": "__CORE_RPC_HOST__",
     "PORT": 15,
     "USERNAME": "__CORE_RPC_USERNAME__",
-    "PASSWORD": "__CORE_RPC_PASSWORD__",
-    "TIMEOUT": "__CORE_RPC_TIMEOUT__"
+    "PASSWORD": "__CORE_RPC_PASSWORD__"
   },
   "ELECTRUM": {
     "HOST": "__ELECTRUM_HOST__",
@@ -50,8 +48,7 @@
     "HOST": "__SECOND_CORE_RPC_HOST__",
     "PORT": 17,
     "USERNAME": "__SECOND_CORE_RPC_USERNAME__",
-    "PASSWORD": "__SECOND_CORE_RPC_PASSWORD__",
-    "TIMEOUT": "__SECOND_CORE_RPC_TIMEOUT__"
+    "PASSWORD": "__SECOND_CORE_RPC_PASSWORD__"
   },
   "DATABASE": {
     "ENABLED": false,
@@ -110,8 +107,7 @@
   "LND": {
     "TLS_CERT_PATH": "",
     "MACAROON_PATH": "",
-    "REST_API_URL": "https://localhost:8080",
-    "TIMEOUT": 10000
+    "REST_API_URL": "https://localhost:8080"
   },
   "CLIGHTNING": {
     "SOCKET": "__CLIGHTNING_SOCKET__"

backend/src/__tests__/api/difficulty-adjustment.test.ts

@@ -23,11 +23,9 @@ describe('Mempool Difficulty Adjustment', () => {
           remainingBlocks: 1834,
           remainingTime: 977591692,
           previousRetarget: 0.6280047707459726,
-          previousTime: 1660820820,
           nextRetargetHeight: 751968,
           timeAvg: 533038,
           timeOffset: 0,
-          expectedBlocks: 161.68833333333333,
         },
       ],
       [ // Vector 2 (testnet)
@@ -45,13 +43,11 @@ describe('Mempool Difficulty Adjustment', () => {
           estimatedRetargetDate: 1661895424692,
           remainingBlocks: 1834,
           remainingTime: 977591692,
-          previousTime: 1660820820,
           previousRetarget: 0.6280047707459726,
           nextRetargetHeight: 751968,
           timeAvg: 533038,
           timeOffset: -667000, // 11 min 7 seconds since last block (testnet only)
           // If we add time avg to abs(timeOffset) it makes exactly 1200000 ms, or 20 minutes
-          expectedBlocks: 161.68833333333333,
         },
       ],
     ] as [[number, number, number, number, string, number], DifficultyAdjustment][];

backend/src/__tests__/config.test.ts

@@ -29,6 +29,7 @@ describe('Mempool Backend Config', () => {
         INITIAL_BLOCKS_AMOUNT: 8,
         MEMPOOL_BLOCKS_AMOUNT: 8,
         INDEXING_BLOCKS_AMOUNT: 11000,
+        PRICE_FEED_UPDATE_INTERVAL: 600,
         USE_SECOND_NODE_FOR_MINFEE: false,
         EXTERNAL_ASSETS: [],
         EXTERNAL_MAX_RETRY: 1,
@@ -36,13 +37,10 @@ describe('Mempool Backend Config', () => {
         USER_AGENT: 'mempool',
         STDOUT_LOG_MIN_PRIORITY: 'debug',
         POOLS_JSON_TREE_URL: 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
-        POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json',
-        AUDIT: false,
+        POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json',
         ADVANCED_GBT_AUDIT: false,
         ADVANCED_GBT_MEMPOOL: false,
-        CPFP_INDEXING: false,
-        MAX_BLOCKS_BULK_QUERY: 0,
-        DISK_CACHE_BLOCK_INTERVAL: 6,
+        TRANSACTION_INDEXING: false,
       });
 
       expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true });
@@ -53,16 +51,14 @@ describe('Mempool Backend Config', () => {
         HOST: '127.0.0.1',
         PORT: 8332,
         USERNAME: 'mempool',
-        PASSWORD: 'mempool',
-        TIMEOUT: 60000
+        PASSWORD: 'mempool'
       });
 
       expect(config.SECOND_CORE_RPC).toStrictEqual({
         HOST: '127.0.0.1',
         PORT: 8332,
         USERNAME: 'mempool',
-        PASSWORD: 'mempool',
-        TIMEOUT: 60000
+        PASSWORD: 'mempool'
       });
 
       expect(config.DATABASE).toStrictEqual({
@@ -109,13 +105,6 @@ describe('Mempool Backend Config', () => {
         BISQ_URL: 'https://bisq.markets/api',
         BISQ_ONION: 'http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api'
       });
-
-      expect(config.MAXMIND).toStrictEqual({
-        ENABLED: false,
-        GEOLITE2_CITY: '/usr/local/share/GeoIP/GeoLite2-City.mmdb',
-        GEOLITE2_ASN: '/usr/local/share/GeoIP/GeoLite2-ASN.mmdb',
-        GEOIP2_ISP: '/usr/local/share/GeoIP/GeoIP2-ISP.mmdb'
-      });
     });
   });
 

backend/src/api/audit.ts

@@ -1,14 +1,18 @@
 import config from '../config';
-import logger from '../logger';
-import { TransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces';
+import bitcoinApi from './bitcoin/bitcoin-api-factory';
+import { Common } from './common';
+import { TransactionExtended, MempoolBlockWithTransactions, AuditScore } from '../mempool.interfaces';
+import blocksRepository from '../repositories/BlocksRepository';
+import blocksAuditsRepository from '../repositories/BlocksAuditsRepository';
+import blocks from '../api/blocks';
 
 const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners
 
 class Audit {
   auditBlock(transactions: TransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: TransactionExtended })
-   : { censored: string[], added: string[], fresh: string[], score: number, similarity: number } {
+   : { censored: string[], added: string[], fresh: string[], score: number } {
     if (!projectedBlocks?.[0]?.transactionIds || !mempool) {
-      return { censored: [], added: [], fresh: [], score: 0, similarity: 1 };
+      return { censored: [], added: [], fresh: [], score: 0 };
     }
 
     const matches: string[] = []; // present in both mined block and template
@@ -17,8 +21,6 @@ class Audit {
     const isCensored = {}; // missing, without excuse
     const isDisplaced = {};
     let displacedWeight = 0;
-    let matchedWeight = 0;
-    let projectedWeight = 0;
 
     const inBlock = {};
     const inTemplate = {};
@@ -40,19 +42,12 @@ class Audit {
         } else {
           isCensored[txid] = true;
         }
-        displacedWeight += mempool[txid]?.weight || 0;
-      } else {
-        matchedWeight += mempool[txid]?.weight || 0;
+        displacedWeight += mempool[txid].weight;
       }
-      projectedWeight += mempool[txid]?.weight || 0;
       inTemplate[txid] = true;
     }
 
-    if (transactions[0]) {
-      displacedWeight += (4000 - transactions[0].weight);
-      projectedWeight += transactions[0].weight;
-      matchedWeight += transactions[0].weight;
-    }
+    displacedWeight += (4000 - transactions[0].weight);
 
     // we can expect an honest miner to include 'displaced' transactions in place of recent arrivals and censored txs
     // these displaced transactions should occupy the first N weight units of the next projected block
@@ -62,24 +57,19 @@ class Audit {
     let failures = 0;
     while (projectedBlocks[1] && index < projectedBlocks[1].transactionIds.length && failures < 500) {
       const txid = projectedBlocks[1].transactionIds[index];
-      const tx = mempool[txid];
-      if (tx) {
-        const fits = (tx.weight - displacedWeightRemaining) < 4000;
-        const feeMatches = tx.effectiveFeePerVsize >= lastFeeRate;
-        if (fits || feeMatches) {
-          isDisplaced[txid] = true;
-          if (fits) {
-            lastFeeRate = Math.min(lastFeeRate, tx.effectiveFeePerVsize);
-          }
-          if (tx.firstSeen == null || (now - (tx?.firstSeen || 0)) > PROPAGATION_MARGIN) {
-            displacedWeightRemaining -= tx.weight;
-          }
-          failures = 0;
-        } else {
-          failures++;
+      const fits = (mempool[txid].weight - displacedWeightRemaining) < 4000;
+      const feeMatches = mempool[txid].effectiveFeePerVsize >= lastFeeRate;
+      if (fits || feeMatches) {
+        isDisplaced[txid] = true;
+        if (fits) {
+          lastFeeRate = Math.min(lastFeeRate, mempool[txid].effectiveFeePerVsize);
         }
+        if (mempool[txid].firstSeen == null || (now - (mempool[txid]?.firstSeen || 0)) > PROPAGATION_MARGIN) {
+          displacedWeightRemaining -= mempool[txid].weight;
+        }
+        failures = 0;
       } else {
-        logger.warn('projected transaction missing from mempool cache');
+        failures++;
       }
       index++;
     }
@@ -116,39 +106,31 @@ class Audit {
     index = projectedBlocks[0].transactionIds.length - 1;
     while (index >= 0) {
       const txid = projectedBlocks[0].transactionIds[index];
-      const tx = mempool[txid];
-      if (tx) {
-        if (overflowWeightRemaining > 0) {
-          if (isCensored[txid]) {
-            delete isCensored[txid];
-          }
-          if (tx.effectiveFeePerVsize > maxOverflowRate) {
-            maxOverflowRate = tx.effectiveFeePerVsize;
-            rateThreshold = (Math.ceil(maxOverflowRate * 100) / 100) + 0.005;
-          }
-        } else if (tx.effectiveFeePerVsize <= rateThreshold) { // tolerance of 0.01 sat/vb + rounding
-          if (isCensored[txid]) {
-            delete isCensored[txid];
-          }
+      if (overflowWeightRemaining > 0) {
+        if (isCensored[txid]) {
+          delete isCensored[txid];
+        }
+        if (mempool[txid].effectiveFeePerVsize > maxOverflowRate) {
+          maxOverflowRate = mempool[txid].effectiveFeePerVsize;
+          rateThreshold = (Math.ceil(maxOverflowRate * 100) / 100) + 0.005;
+        }
+      } else if (mempool[txid].effectiveFeePerVsize <= rateThreshold) { // tolerance of 0.01 sat/vb + rounding
+        if (isCensored[txid]) {
+          delete isCensored[txid];
         }
-        overflowWeightRemaining -= (mempool[txid]?.weight || 0);
-      } else {
-        logger.warn('projected transaction missing from mempool cache');
       }
+      overflowWeightRemaining -= (mempool[txid]?.weight || 0);
       index--;
     }
 
     const numCensored = Object.keys(isCensored).length;
-    const numMatches = matches.length - 1; // adjust for coinbase tx
-    const score = numMatches > 0 ? (numMatches / (numMatches + numCensored)) : 0;
-    const similarity = projectedWeight ? matchedWeight / projectedWeight : 1;
+    const score = matches.length > 0 ? (matches.length / (matches.length + numCensored)) : 0;
 
     return {
       censored: Object.keys(isCensored),
       added,
       fresh,
-      score,
-      similarity,
+      score
     };
   }
 }

backend/src/api/backend-info.ts

@@ -2,7 +2,6 @@ import fs from 'fs';
 import path from 'path';
 import os from 'os';
 import { IBackendInfo } from '../mempool.interfaces';
-import config from '../config';
 
 class BackendInfo {
   private backendInfo: IBackendInfo;
@@ -23,8 +22,7 @@ class BackendInfo {
     this.backendInfo = {
       hostname: os.hostname(),
       version: versionInfo.version,
-      gitCommit: versionInfo.gitCommit,
-      lightning: config.LIGHTNING.ENABLED
+      gitCommit: versionInfo.gitCommit
     };
   }
 

backend/src/api/bitcoin/bitcoin-api-factory.ts

@@ -17,6 +17,4 @@ function bitcoinApiFactory(): AbstractBitcoinApi {
   }
 }
 
-export const bitcoinCoreApi = new BitcoinApi(bitcoinClient);
-
 export default bitcoinApiFactory();

backend/src/api/bitcoin/bitcoin-api.interface.ts

@@ -172,35 +172,4 @@ export namespace IBitcoinApi {
     }
   }
 
-  export interface BlockStats {
-    "avgfee": number;
-    "avgfeerate": number;
-    "avgtxsize": number;
-    "blockhash": string;
-    "feerate_percentiles": [number, number, number, number, number];
-    "height": number;
-    "ins": number;
-    "maxfee": number;
-    "maxfeerate": number;
-    "maxtxsize": number;
-    "medianfee": number;
-    "mediantime": number;
-    "mediantxsize": number;
-    "minfee": number;
-    "minfeerate": number;
-    "mintxsize": number;
-    "outs": number;
-    "subsidy": number;
-    "swtotal_size": number;
-    "swtotal_weight": number;
-    "swtxs": number;
-    "time": number;
-    "total_out": number;
-    "total_size": number;
-    "total_weight": number;
-    "totalfee": number;
-    "txs": number;
-    "utxo_increase": number;
-    "utxo_size_inc": number;
-  }
 }

backend/src/api/bitcoin/bitcoin-api.ts

@@ -28,7 +28,6 @@ class BitcoinApi implements AbstractBitcoinApi {
       size: block.size,
       weight: block.weight,
       previousblockhash: block.previousblockhash,
-      mediantime: block.mediantime,
     };
   }
 

backend/src/api/bitcoin/bitcoin-client.ts

@@ -7,7 +7,7 @@ const nodeRpcCredentials: BitcoinRpcCredentials = {
   port: config.CORE_RPC.PORT,
   user: config.CORE_RPC.USERNAME,
   pass: config.CORE_RPC.PASSWORD,
-  timeout: config.CORE_RPC.TIMEOUT,
+  timeout: 60000,
 };
 
 export default new bitcoin.Client(nodeRpcCredentials);

backend/src/api/bitcoin/bitcoin-second-client.ts

@@ -7,7 +7,7 @@ const nodeRpcCredentials: BitcoinRpcCredentials = {
   port: config.SECOND_CORE_RPC.PORT,
   user: config.SECOND_CORE_RPC.USERNAME,
   pass: config.SECOND_CORE_RPC.PASSWORD,
-  timeout: config.SECOND_CORE_RPC.TIMEOUT,
+  timeout: 60000,
 };
 
 export default new bitcoin.Client(nodeRpcCredentials);

backend/src/api/bitcoin/bitcoin.routes.ts

@@ -6,7 +6,7 @@ import websocketHandler from '../websocket-handler';
 import mempool from '../mempool';
 import feeApi from '../fee-api';
 import mempoolBlocks from '../mempool-blocks';
-import bitcoinApi, { bitcoinCoreApi } from './bitcoin-api-factory';
+import bitcoinApi from './bitcoin-api-factory';
 import { Common } from '../common';
 import backendInfo from '../backend-info';
 import transactionUtils from '../transaction-utils';
@@ -18,7 +18,6 @@ import blocks from '../blocks';
 import bitcoinClient from './bitcoin-client';
 import difficultyAdjustment from '../difficulty-adjustment';
 import transactionRepository from '../../repositories/TransactionRepository';
-import rbfCache from '../rbf-cache';
 
 class BitcoinRoutes {
   public initRoutes(app: Application) {
@@ -32,8 +31,6 @@ class BitcoinRoutes {
       .get(config.MEMPOOL.API_URL_PREFIX + 'backend-info', this.getBackendInfo)
       .get(config.MEMPOOL.API_URL_PREFIX + 'init-data', this.getInitData)
       .get(config.MEMPOOL.API_URL_PREFIX + 'validate-address/:address', this.validateAddress)
-      .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/replaces', this.getRbfHistory)
-      .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/cached', this.getCachedTx)
       .post(config.MEMPOOL.API_URL_PREFIX + 'tx/push', this.$postTransactionForm)
       .get(config.MEMPOOL.API_URL_PREFIX + 'donations', async (req, res) => {
         try {
@@ -95,8 +92,6 @@ class BitcoinRoutes {
       .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/summary', this.getStrippedBlockTransactions)
       .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/audit-summary', this.getBlockAuditSummary)
       .post(config.MEMPOOL.API_URL_PREFIX + 'psbt/addparents', this.postPsbtCompletion)
-      .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from', this.getBlocksByBulk.bind(this))
-      .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from/:to', this.getBlocksByBulk.bind(this))
       ;
 
       if (config.MEMPOOL.BACKEND !== 'esplora') {
@@ -217,20 +212,13 @@ class BitcoinRoutes {
       res.json(cpfpInfo);
       return;
     } else {
-      let cpfpInfo;
-      if (config.DATABASE.ENABLED) {
-        cpfpInfo = await transactionRepository.$getCpfpInfo(req.params.txId);
-      }
+      const cpfpInfo = await transactionRepository.$getCpfpInfo(req.params.txId);
       if (cpfpInfo) {
         res.json(cpfpInfo);
         return;
-      } else {
-        res.json({
-          ancestors: []
-        });
-        return;
       }
     }
+    res.status(404).send(`Transaction has no CPFP info available.`);
   }
 
   private getBackendInfo(req: Request, res: Response) {
@@ -411,46 +399,10 @@ class BitcoinRoutes {
     }
   }
 
-  private async getBlocksByBulk(req: Request, res: Response) {
-    try {
-      if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { // Liquid, Bisq - Not implemented
-        return res.status(404).send(`This API is only available for Bitcoin networks`);
-      }
-      if (config.MEMPOOL.MAX_BLOCKS_BULK_QUERY <= 0) {
-        return res.status(404).send(`This API is disabled. Set config.MEMPOOL.MAX_BLOCKS_BULK_QUERY to a positive number to enable it.`);
-      }
-      if (!Common.indexingEnabled()) {
-        return res.status(404).send(`Indexing is required for this API`);
-      }
-
-      const from = parseInt(req.params.from, 10);
-      if (!req.params.from || from < 0) {
-        return res.status(400).send(`Parameter 'from' must be a block height (integer)`);
-      }
-      const to = req.params.to === undefined ? await bitcoinApi.$getBlockHeightTip() : parseInt(req.params.to, 10);
-      if (to < 0) {
-        return res.status(400).send(`Parameter 'to' must be a block height (integer)`);
-      }
-      if (from > to) {
-        return res.status(400).send(`Parameter 'to' must be a higher block height than 'from'`);
-      }
-      if ((to - from + 1) > config.MEMPOOL.MAX_BLOCKS_BULK_QUERY) {
-        return res.status(400).send(`You can only query ${config.MEMPOOL.MAX_BLOCKS_BULK_QUERY} blocks at once.`);
-      }
-
-      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
-      res.json(await blocks.$getBlocksBetweenHeight(from, to));
-
-    } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
-    }
-  }
-
   private async getLegacyBlocks(req: Request, res: Response) {
     try {
       const returnBlocks: IEsploraApi.Block[] = [];
-      const tip = blocks.getCurrentBlockHeight();
-      const fromHeight = Math.min(parseInt(req.params.height, 10) || tip, tip);
+      const fromHeight = parseInt(req.params.height, 10) || blocks.getCurrentBlockHeight();
 
       // Check if block height exist in local cache to skip the hash lookup
       const blockByHeight = blocks.getBlocks().find((b) => b.height === fromHeight);
@@ -468,7 +420,7 @@ class BitcoinRoutes {
           returnBlocks.push(localBlock);
           nextHash = localBlock.previousblockhash;
         } else {
-          const block = await bitcoinCoreApi.$getBlock(nextHash);
+          const block = await bitcoinApi.$getBlock(nextHash);
           returnBlocks.push(block);
           nextHash = block.previousblockhash;
         }
@@ -636,28 +588,6 @@ class BitcoinRoutes {
     }
   }
 
-  private async getRbfHistory(req: Request, res: Response) {
-    try {
-      const result = rbfCache.getReplaces(req.params.txId);
-      res.json(result || []);
-    } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
-    }
-  }
-
-  private async getCachedTx(req: Request, res: Response) {
-    try {
-      const result = rbfCache.getTx(req.params.txId);
-      if (result) {
-        res.json(result);
-      } else {
-        res.status(204).send();
-      }
-    } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
-    }
-  }
-
   private async getTransactionOutspends(req: Request, res: Response) {
     try {
       const result = await bitcoinApi.$getOutspends(req.params.txId);

backend/src/api/bitcoin/electrum-api.ts

@@ -16,7 +16,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
     super(bitcoinClient);
 
     const electrumConfig = { client: 'mempool-v2', version: '1.4' };
-    const electrumPersistencePolicy = { retryPeriod: 1000, maxRetry: Number.MAX_SAFE_INTEGER, callback: null };
+    const electrumPersistencePolicy = { retryPeriod: 10000, maxRetry: 1000, callback: null };
 
     const electrumCallbacks = {
       onConnect: (client, versionInfo) => { logger.info(`Connected to Electrum Server at ${config.ELECTRUM.HOST}:${config.ELECTRUM.PORT} (${JSON.stringify(versionInfo)})`); },

backend/src/api/bitcoin/esplora-api.interface.ts

@@ -88,7 +88,6 @@ export namespace IEsploraApi {
     size: number;
     weight: number;
     previousblockhash: string;
-    mediantime: number;
   }
 
   export interface Address {

backend/src/api/bitcoin/esplora-api.ts

@@ -1,13 +1,8 @@
 import config from '../../config';
 import axios, { AxiosRequestConfig } from 'axios';
-import http from 'http';
 import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
 import { IEsploraApi } from './esplora-api.interface';
 
-const axiosConnection = axios.create({
-  httpAgent: new http.Agent({ keepAlive: true })
-});
-
 class ElectrsApi implements AbstractBitcoinApi {
   axiosConfig: AxiosRequestConfig = {
     timeout: 10000,
@@ -16,52 +11,52 @@ class ElectrsApi implements AbstractBitcoinApi {
   constructor() { }
 
   $getRawMempool(): Promise<IEsploraApi.Transaction['txid'][]> {
-    return axiosConnection.get<IEsploraApi.Transaction['txid'][]>(config.ESPLORA.REST_API_URL + '/mempool/txids', this.axiosConfig)
+    return axios.get<IEsploraApi.Transaction['txid'][]>(config.ESPLORA.REST_API_URL + '/mempool/txids', this.axiosConfig)
       .then((response) => response.data);
   }
 
   $getRawTransaction(txId: string): Promise<IEsploraApi.Transaction> {
-    return axiosConnection.get<IEsploraApi.Transaction>(config.ESPLORA.REST_API_URL + '/tx/' + txId, this.axiosConfig)
+    return axios.get<IEsploraApi.Transaction>(config.ESPLORA.REST_API_URL + '/tx/' + txId, this.axiosConfig)
       .then((response) => response.data);
   }
 
   $getTransactionHex(txId: string): Promise<string> {
-    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/hex', this.axiosConfig)
+    return axios.get<string>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/hex', this.axiosConfig)
       .then((response) => response.data);
   }
 
   $getBlockHeightTip(): Promise<number> {
-    return axiosConnection.get<number>(config.ESPLORA.REST_API_URL + '/blocks/tip/height', this.axiosConfig)
+    return axios.get<number>(config.ESPLORA.REST_API_URL + '/blocks/tip/height', this.axiosConfig)
       .then((response) => response.data);
   }
 
   $getBlockHashTip(): Promise<string> {
-    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/blocks/tip/hash', this.axiosConfig)
+    return axios.get<string>(config.ESPLORA.REST_API_URL + '/blocks/tip/hash', this.axiosConfig)
       .then((response) => response.data);
   }
 
   $getTxIdsForBlock(hash: string): Promise<string[]> {
-    return axiosConnection.get<string[]>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/txids', this.axiosConfig)
+    return axios.get<string[]>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/txids', this.axiosConfig)
       .then((response) => response.data);
   }
 
   $getBlockHash(height: number): Promise<string> {
-    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/block-height/' + height, this.axiosConfig)
+    return axios.get<string>(config.ESPLORA.REST_API_URL + '/block-height/' + height, this.axiosConfig)
       .then((response) => response.data);
   }
 
   $getBlockHeader(hash: string): Promise<string> {
-    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/header', this.axiosConfig)
+    return axios.get<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/header', this.axiosConfig)
       .then((response) => response.data);
   }
 
   $getBlock(hash: string): Promise<IEsploraApi.Block> {
-    return axiosConnection.get<IEsploraApi.Block>(config.ESPLORA.REST_API_URL + '/block/' + hash, this.axiosConfig)
+    return axios.get<IEsploraApi.Block>(config.ESPLORA.REST_API_URL + '/block/' + hash, this.axiosConfig)
       .then((response) => response.data);
   }
 
   $getRawBlock(hash: string): Promise<Buffer> {
-    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + "/raw", { ...this.axiosConfig, responseType: 'arraybuffer' })
+    return axios.get<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + "/raw", { ...this.axiosConfig, responseType: 'arraybuffer' })
       .then((response) => { return Buffer.from(response.data); });
   }
 
@@ -82,12 +77,12 @@ class ElectrsApi implements AbstractBitcoinApi {
   }
 
   $getOutspend(txId: string, vout: number): Promise<IEsploraApi.Outspend> {
-    return axiosConnection.get<IEsploraApi.Outspend>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspend/' + vout, this.axiosConfig)
+    return axios.get<IEsploraApi.Outspend>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspend/' + vout, this.axiosConfig)
       .then((response) => response.data);
   }
 
   $getOutspends(txId: string): Promise<IEsploraApi.Outspend[]> {
-    return axiosConnection.get<IEsploraApi.Outspend[]>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspends', this.axiosConfig)
+    return axios.get<IEsploraApi.Outspend[]>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspends', this.axiosConfig)
       .then((response) => response.data);
   }
 

backend/src/api/blocks.ts

@@ -1,8 +1,8 @@
 import config from '../config';
-import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory';
+import bitcoinApi from './bitcoin/bitcoin-api-factory';
 import logger from '../logger';
 import memPool from './mempool';
-import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionStripped, TransactionMinerInfo } from '../mempool.interfaces';
+import { BlockExtended, BlockSummary, PoolTag, TransactionExtended, TransactionStripped, TransactionMinerInfo } from '../mempool.interfaces';
 import { Common } from './common';
 import diskCache from './disk-cache';
 import transactionUtils from './transaction-utils';
@@ -13,18 +13,21 @@ import poolsRepository from '../repositories/PoolsRepository';
 import blocksRepository from '../repositories/BlocksRepository';
 import loadingIndicators from './loading-indicators';
 import BitcoinApi from './bitcoin/bitcoin-api';
+import { prepareBlock } from '../utils/blocks-utils';
 import BlocksRepository from '../repositories/BlocksRepository';
 import HashratesRepository from '../repositories/HashratesRepository';
 import indexer from '../indexer';
+import fiatConversion from './fiat-conversion';
 import poolsParser from './pools-parser';
 import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
 import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository';
 import cpfpRepository from '../repositories/CpfpRepository';
+import transactionRepository from '../repositories/TransactionRepository';
 import mining from './mining/mining';
 import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository';
 import PricesRepository from '../repositories/PricesRepository';
 import priceUpdater from '../tasks/price-updater';
-import chainTips from './chain-tips';
+import { Block } from 'bitcoinjs-lib';
 
 class Blocks {
   private blocks: BlockExtended[] = [];
@@ -98,23 +101,12 @@ class Blocks {
           transactions.push(tx);
           transactionsFetched++;
         } catch (e) {
-          try {
-            if (config.MEMPOOL.BACKEND === 'esplora') {
-              // Try again with core
-              const tx = await transactionUtils.$getTransactionExtended(txIds[i], false, false, true);
-              transactions.push(tx);
-              transactionsFetched++;
-            } else {
-              throw e;
-            }
-          } catch (e) {
-            if (i === 0) {
-              const msg = `Cannot fetch coinbase tx ${txIds[i]}. Reason: ` + (e instanceof Error ? e.message : e); 
-              logger.err(msg);
-              throw new Error(msg);
-            } else {
-              logger.err(`Cannot fetch tx ${txIds[i]}. Reason: ` + (e instanceof Error ? e.message : e));
-            }
+          if (i === 0) {
+            const msg = `Cannot fetch coinbase tx ${txIds[i]}. Reason: ` + (e instanceof Error ? e.message : e); 
+            logger.err(msg);
+            throw new Error(msg);
+          } else {
+            logger.err(`Cannot fetch tx ${txIds[i]}. Reason: ` + (e instanceof Error ? e.message : e));
           }
         }
       }
@@ -142,11 +134,8 @@ class Blocks {
    * @param block
    * @returns BlockSummary
    */
-  public summarizeBlock(block: IBitcoinApi.VerboseBlock): BlockSummary {
-    if (Common.isLiquid()) {
-      block = this.convertLiquidFees(block);
-    }
-    const stripped = block.tx.map((tx: IBitcoinApi.VerboseTransaction) => {
+  private summarizeBlock(block: IBitcoinApi.VerboseBlock): BlockSummary {
+    const stripped = block.tx.map((tx) => {
       return {
         txid: tx.txid,
         vsize: tx.weight / 4,
@@ -161,13 +150,6 @@ class Blocks {
     };
   }
 
-  private convertLiquidFees(block: IBitcoinApi.VerboseBlock): IBitcoinApi.VerboseBlock {
-    block.tx.forEach(tx => {
-      tx.fee = Object.values(tx.fee || {}).reduce((total, output) => total + output, 0);
-    });
-    return block;
-  }
-
   /**
    * Return a block with additional data (reward, coinbase, fees...)
    * @param block
@@ -175,81 +157,33 @@ class Blocks {
    * @returns BlockExtended
    */
   private async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise<BlockExtended> {
-    const coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
-    
-    const blk: Partial<BlockExtended> = Object.assign({}, block);
-    const extras: Partial<BlockExtension> = {};
-
-    extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
-    extras.coinbaseRaw = coinbaseTx.vin[0].scriptsig;
-    extras.orphans = chainTips.getOrphanedBlocksAtHeight(blk.height);
+    const blockExtended: BlockExtended = Object.assign({ extras: {} }, block);
+    blockExtended.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
+    blockExtended.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
+    blockExtended.extras.coinbaseRaw = blockExtended.extras.coinbaseTx.vin[0].scriptsig;
+    blockExtended.extras.usd = fiatConversion.getConversionRates().USD;
 
     if (block.height === 0) {
-      extras.medianFee = 0; // 50th percentiles
-      extras.feeRange = [0, 0, 0, 0, 0, 0, 0];
-      extras.totalFees = 0;
-      extras.avgFee = 0;
-      extras.avgFeeRate = 0;
-      extras.utxoSetChange = 0;
-      extras.avgTxSize = 0;
-      extras.totalInputs = 0;
-      extras.totalOutputs = 1;
-      extras.totalOutputAmt = 0;
-      extras.segwitTotalTxs = 0;
-      extras.segwitTotalSize = 0;
-      extras.segwitTotalWeight = 0;
+      blockExtended.extras.medianFee = 0; // 50th percentiles
+      blockExtended.extras.feeRange = [0, 0, 0, 0, 0, 0, 0];
+      blockExtended.extras.totalFees = 0;
+      blockExtended.extras.avgFee = 0;
+      blockExtended.extras.avgFeeRate = 0;
     } else {
-      const stats: IBitcoinApi.BlockStats = await bitcoinClient.getBlockStats(block.id);
-      extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles
-      extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat();
-      extras.totalFees = stats.totalfee;
-      extras.avgFee = stats.avgfee;
-      extras.avgFeeRate = stats.avgfeerate;
-      extras.utxoSetChange = stats.utxo_increase;
-      extras.avgTxSize = Math.round(stats.total_size / stats.txs * 100) * 0.01;
-      extras.totalInputs = stats.ins;
-      extras.totalOutputs = stats.outs;
-      extras.totalOutputAmt = stats.total_out;
-      extras.segwitTotalTxs = stats.swtxs;
-      extras.segwitTotalSize = stats.swtotal_size;
-      extras.segwitTotalWeight = stats.swtotal_weight;
-    }
-
-    if (Common.blocksSummariesIndexingEnabled()) {
-      extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(block.id);
-      if (extras.feePercentiles !== null) {
-        extras.medianFeeAmt = extras.feePercentiles[3];
-      }
-    }
-  
-    extras.virtualSize = block.weight / 4.0;
-    if (coinbaseTx?.vout.length > 0) {
-      extras.coinbaseAddress = coinbaseTx.vout[0].scriptpubkey_address ?? null;
-      extras.coinbaseSignature = coinbaseTx.vout[0].scriptpubkey_asm ?? null;
-      extras.coinbaseSignatureAscii = transactionUtils.hex2ascii(coinbaseTx.vin[0].scriptsig) ?? null;
-    } else {
-      extras.coinbaseAddress = null;
-      extras.coinbaseSignature = null;
-      extras.coinbaseSignatureAscii = null;
-    }
-
-    const header = await bitcoinClient.getBlockHeader(block.id, false);
-    extras.header = header;
-
-    const coinStatsIndex = indexer.isCoreIndexReady('coinstatsindex');
-    if (coinStatsIndex !== null && coinStatsIndex.best_block_height >= block.height) {
-      const txoutset = await bitcoinClient.getTxoutSetinfo('none', block.height);
-      extras.utxoSetSize = txoutset.txouts,
-      extras.totalInputAmt = Math.round(txoutset.block_info.prevout_spent * 100000000);
-    } else {
-      extras.utxoSetSize = null;
-      extras.totalInputAmt = null;
+      const stats = await bitcoinClient.getBlockStats(block.id, [
+        'feerate_percentiles', 'minfeerate', 'maxfeerate', 'totalfee', 'avgfee', 'avgfeerate'
+      ]);
+      blockExtended.extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles
+      blockExtended.extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat();
+      blockExtended.extras.totalFees = stats.totalfee;
+      blockExtended.extras.avgFee = stats.avgfee;
+      blockExtended.extras.avgFeeRate = stats.avgfeerate;
     }
 
     if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
       let pool: PoolTag;
-      if (coinbaseTx !== undefined) {
-        pool = await this.$findBlockMiner(coinbaseTx);
+      if (blockExtended.extras?.coinbaseTx !== undefined) {
+        pool = await this.$findBlockMiner(blockExtended.extras?.coinbaseTx);
       } else {
         if (config.DATABASE.ENABLED === true) {
           pool = await poolsRepository.$getUnknownPool();
@@ -259,27 +193,23 @@ class Blocks {
       }
 
       if (!pool) { // We should never have this situation in practise
-        logger.warn(`Cannot assign pool to block ${blk.height} and 'unknown' pool does not exist. ` +
+        logger.warn(`Cannot assign pool to block ${blockExtended.height} and 'unknown' pool does not exist. ` +
           `Check your "pools" table entries`);
       } else {
-        extras.pool = {
-          id: pool.uniqueId,
+        blockExtended.extras.pool = {
+          id: pool.id,
           name: pool.name,
           slug: pool.slug,
         };
       }
 
-      extras.matchRate = null;
-      if (config.MEMPOOL.AUDIT) {
-        const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(block.id);
-        if (auditScore != null) {
-          extras.matchRate = auditScore.matchRate;
-        }
+      const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(block.id);
+      if (auditScore != null) {
+        blockExtended.extras.matchRate = auditScore.matchRate;
       }
     }
 
-    blk.extras = <BlockExtension>extras;
-    return <BlockExtended>blk;
+    return blockExtended;
   }
 
   /**
@@ -305,18 +235,15 @@ class Blocks {
     } else {
       pools = poolsParser.miningPools;
     }
-
     for (let i = 0; i < pools.length; ++i) {
       if (address !== undefined) {
-        const addresses: string[] = typeof pools[i].addresses === 'string' ?
-          JSON.parse(pools[i].addresses) : pools[i].addresses;
+        const addresses: string[] = JSON.parse(pools[i].addresses);
         if (addresses.indexOf(address) !== -1) {
           return pools[i];
         }
       }
 
-      const regexes: string[] = typeof pools[i].regexes === 'string' ?
-        JSON.parse(pools[i].regexes) : pools[i].regexes;
+      const regexes: string[] = JSON.parse(pools[i].regexes);
       for (let y = 0; y < regexes.length; ++y) {
         const regex = new RegExp(regexes[y], 'i');
         const match = asciiScriptSig.match(regex);
@@ -369,7 +296,7 @@ class Blocks {
           const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
           const blockPerSeconds = Math.max(1, indexedThisRun / elapsedSeconds);
           const progress = Math.round(totalIndexed / indexedBlocks.length * 10000) / 100;
-          logger.debug(`Indexing block summary for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexedBlocks.length} (${progress}%) | elapsed: ${runningFor} seconds`, logger.tags.mining);
+          logger.debug(`Indexing block summary for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexedBlocks.length} (${progress}%) | elapsed: ${runningFor} seconds`);
           timer = new Date().getTime() / 1000;
           indexedThisRun = 0;
         }
@@ -382,12 +309,12 @@ class Blocks {
         newlyIndexed++;
       }
       if (newlyIndexed > 0) {
-        logger.notice(`Blocks summaries indexing completed: indexed ${newlyIndexed} blocks`, logger.tags.mining);
+        logger.notice(`Blocks summaries indexing completed: indexed ${newlyIndexed} blocks`);
       } else {
-        logger.debug(`Blocks summaries indexing completed: indexed ${newlyIndexed} blocks`, logger.tags.mining);
+        logger.debug(`Blocks summaries indexing completed: indexed ${newlyIndexed} blocks`);
       }
     } catch (e) {
-      logger.err(`Blocks summaries indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`, logger.tags.mining);
+      logger.err(`Blocks summaries indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`);
       throw e;
     }
   }
@@ -402,10 +329,9 @@ class Blocks {
 
     try {
       // Get all indexed block hash
-      const unindexedBlockHeights = await blocksRepository.$getCPFPUnindexedBlocks();
-      logger.info(`Indexing cpfp data for ${unindexedBlockHeights.length} blocks`);
+      const unindexedBlocks = await blocksRepository.$getCPFPUnindexedBlocks();
 
-      if (!unindexedBlockHeights?.length) {
+      if (!unindexedBlocks?.length) {
         return;
       }
 
@@ -414,26 +340,30 @@ class Blocks {
       let countThisRun = 0;
       let timer = new Date().getTime() / 1000;
       const startedAt = new Date().getTime() / 1000;
-      for (const height of unindexedBlockHeights) {
+
+      for (const block of unindexedBlocks) {
         // Logging
-        const hash = await bitcoinApi.$getBlockHash(height);
         const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer);
         if (elapsedSeconds > 5) {
           const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
-          const blockPerSeconds = (countThisRun / elapsedSeconds);
-          const progress = Math.round(count / unindexedBlockHeights.length * 10000) / 100;
-          logger.debug(`Indexing cpfp clusters for #${height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${count}/${unindexedBlockHeights.length} (${progress}%) | elapsed: ${runningFor} seconds`);
+          const blockPerSeconds = Math.max(1, countThisRun / elapsedSeconds);
+          const progress = Math.round(count / unindexedBlocks.length * 10000) / 100;
+          logger.debug(`Indexing cpfp clusters for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${count}/${unindexedBlocks.length} (${progress}%) | elapsed: ${runningFor} seconds`);
           timer = new Date().getTime() / 1000;
           countThisRun = 0;
         }
 
-        await this.$indexCPFP(hash, height); // Calculate and save CPFP data for transactions in this block
+        await this.$indexCPFP(block.hash, block.height); // Calculate and save CPFP data for transactions in this block
 
         // Logging
         count++;
         countThisRun++;
       }
-      logger.notice(`CPFP indexing completed: indexed ${count} blocks`);
+      if (count > 0) {
+        logger.notice(`CPFP indexing completed: indexed ${count} blocks`);
+      } else {
+        logger.debug(`CPFP indexing completed: indexed ${count} blocks`);
+      }
     } catch (e) {
       logger.err(`CPFP indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`);
       throw e;
@@ -455,7 +385,7 @@ class Blocks {
 
       const lastBlockToIndex = Math.max(0, currentBlockHeight - indexingBlockAmount + 1);
 
-      logger.debug(`Indexing blocks from #${currentBlockHeight} to #${lastBlockToIndex}`, logger.tags.mining);
+      logger.debug(`Indexing blocks from #${currentBlockHeight} to #${lastBlockToIndex}`);
       loadingIndicators.setProgress('block-indexing', 0);
 
       const chunkSize = 10000;
@@ -475,7 +405,7 @@ class Blocks {
           continue;
         }
 
-        logger.info(`Indexing ${missingBlockHeights.length} blocks from #${currentBlockHeight} to #${endBlock}`, logger.tags.mining);
+        logger.info(`Indexing ${missingBlockHeights.length} blocks from #${currentBlockHeight} to #${endBlock}`);
 
         for (const blockHeight of missingBlockHeights) {
           if (blockHeight < lastBlockToIndex) {
@@ -488,13 +418,13 @@ class Blocks {
             const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
             const blockPerSeconds = Math.max(1, indexedThisRun / elapsedSeconds);
             const progress = Math.round(totalIndexed / indexingBlockAmount * 10000) / 100;
-            logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${runningFor} seconds`, logger.tags.mining);
+            logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${runningFor} seconds`);
             timer = new Date().getTime() / 1000;
             indexedThisRun = 0;
             loadingIndicators.setProgress('block-indexing', progress, false);
           }
           const blockHash = await bitcoinApi.$getBlockHash(blockHeight);
-          const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash);
+          const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash));
           const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, true);
           const blockExtended = await this.$getBlockExtended(block, transactions);
 
@@ -505,13 +435,13 @@ class Blocks {
         currentBlockHeight -= chunkSize;
       }
       if (newlyIndexed > 0) {
-        logger.notice(`Block indexing completed: indexed ${newlyIndexed} blocks`, logger.tags.mining);
+        logger.notice(`Block indexing completed: indexed ${newlyIndexed} blocks`);
       } else {
-        logger.debug(`Block indexing completed: indexed ${newlyIndexed} blocks`, logger.tags.mining);
+        logger.debug(`Block indexing completed: indexed ${newlyIndexed} blocks`);
       }
       loadingIndicators.setProgress('block-indexing', 100);
     } catch (e) {
-      logger.err('Block indexing failed. Trying again in 10 seconds. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
+      logger.err('Block indexing failed. Trying again in 10 seconds. Reason: ' + (e instanceof Error ? e.message : e));
       loadingIndicators.setProgress('block-indexing', 100);
       throw e;
     }
@@ -542,13 +472,13 @@ class Blocks {
       if (blockchainInfo.blocks === blockchainInfo.headers) {
         const heightDiff = blockHeightTip % 2016;
         const blockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff);
-        const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash);
+        const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash));
         this.lastDifficultyAdjustmentTime = block.timestamp;
         this.currentDifficulty = block.difficulty;
 
         if (blockHeightTip >= 2016) {
           const previousPeriodBlockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff - 2016);
-          const previousPeriodBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(previousPeriodBlockHash);
+          const previousPeriodBlock = await bitcoinClient.getBlock(previousPeriodBlockHash)
           this.previousDifficultyRetarget = (block.difficulty - previousPeriodBlock.difficulty) / previousPeriodBlock.difficulty * 100;
           logger.debug(`Initial difficulty adjustment data set.`);
         }
@@ -558,12 +488,11 @@ class Blocks {
     }
 
     while (this.currentBlockHeight < blockHeightTip) {
-      if (this.currentBlockHeight === 0) {
+      if (this.currentBlockHeight < blockHeightTip - config.MEMPOOL.INITIAL_BLOCKS_AMOUNT) {
         this.currentBlockHeight = blockHeightTip;
       } else {
         this.currentBlockHeight++;
         logger.debug(`New block found (#${this.currentBlockHeight})!`);
-        await chainTips.updateOrphanedBlocks();
       }
 
       const blockHash = await bitcoinApi.$getBlockHash(this.currentBlockHeight);
@@ -580,18 +509,18 @@ class Blocks {
       if (Common.indexingEnabled()) {
         if (!fastForwarded) {
           const lastBlock = await blocksRepository.$getBlockByHeight(blockExtended.height - 1);
-          if (lastBlock !== null && blockExtended.previousblockhash !== lastBlock.id) {
-            logger.warn(`Chain divergence detected at block ${lastBlock.height}, re-indexing most recent data`);
+          if (lastBlock !== null && blockExtended.previousblockhash !== lastBlock['hash']) {
+            logger.warn(`Chain divergence detected at block ${lastBlock['height']}, re-indexing most recent data`);
             // We assume there won't be a reorg with more than 10 block depth
-            await BlocksRepository.$deleteBlocksFrom(lastBlock.height - 10);
+            await BlocksRepository.$deleteBlocksFrom(lastBlock['height'] - 10);
             await HashratesRepository.$deleteLastEntries();
-            await BlocksSummariesRepository.$deleteBlocksFrom(lastBlock.height - 10);
-            await cpfpRepository.$deleteClustersFrom(lastBlock.height - 10);
+            await BlocksSummariesRepository.$deleteBlocksFrom(lastBlock['height'] - 10);
+            await cpfpRepository.$deleteClustersFrom(lastBlock['height'] - 10);
             for (let i = 10; i >= 0; --i) {
-              const newBlock = await this.$indexBlock(lastBlock.height - i);
+              const newBlock = await this.$indexBlock(lastBlock['height'] - i);
               await this.$getStrippedBlockTransactions(newBlock.id, true, true);
-              if (config.MEMPOOL.CPFP_INDEXING) {
-                await this.$indexCPFP(newBlock.id, lastBlock.height - i);
+              if (config.MEMPOOL.TRANSACTION_INDEXING) {
+                await this.$indexCPFP(newBlock.id, lastBlock['height'] - i);
               }
             }
             await mining.$indexDifficultyAdjustments();
@@ -608,7 +537,7 @@ class Blocks {
               priceId: lastestPriceId,
             }]);
           } else {
-            logger.info(`Cannot save block price for ${blockExtended.height} because the price updater hasnt completed yet. Trying again in 10 seconds.`, logger.tags.mining);
+            logger.info(`Cannot save block price for ${blockExtended.height} because the price updater hasnt completed yet. Trying again in 10 seconds.`)
             setTimeout(() => {
               indexer.runSingleTask('blocksPrices');
             }, 10000);
@@ -618,7 +547,7 @@ class Blocks {
           if (Common.blocksSummariesIndexingEnabled() === true) {
             await this.$getStrippedBlockTransactions(blockExtended.id, true);
           }
-          if (config.MEMPOOL.CPFP_INDEXING) {
+          if (config.MEMPOOL.TRANSACTION_INDEXING) {
             this.$indexCPFP(blockExtended.id, this.currentBlockHeight);
           }
         }
@@ -651,7 +580,7 @@ class Blocks {
       if (this.newBlockCallbacks.length) {
         this.newBlockCallbacks.forEach((cb) => cb(blockExtended, txIds, transactions));
       }
-      if (!memPool.hasPriority() && (block.height % config.MEMPOOL.DISK_CACHE_BLOCK_INTERVAL === 0)) {
+      if (!memPool.hasPriority()) {
         diskCache.$saveCacheToDisk();
       }
 
@@ -664,27 +593,23 @@ class Blocks {
    * Index a block if it's missing from the database. Returns the block after indexing
    */
   public async $indexBlock(height: number): Promise<BlockExtended> {
-    if (Common.indexingEnabled()) {
-      const dbBlock = await blocksRepository.$getBlockByHeight(height);
-      if (dbBlock !== null) {
-        return dbBlock;
-      }
+    const dbBlock = await blocksRepository.$getBlockByHeight(height);
+    if (dbBlock != null) {
+      return prepareBlock(dbBlock);
     }
 
     const blockHash = await bitcoinApi.$getBlockHash(height);
-    const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash);
+    const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash));
     const transactions = await this.$getTransactionsExtended(blockHash, block.height, true);
     const blockExtended = await this.$getBlockExtended(block, transactions);
 
-    if (Common.indexingEnabled()) {
-      await blocksRepository.$saveBlockInDatabase(blockExtended);
-    }
+    await blocksRepository.$saveBlockInDatabase(blockExtended);
 
-    return blockExtended;
+    return prepareBlock(blockExtended);
   }
 
   /**
-   * Get one block by its hash
+   * Index a block by hash if it's missing from the database. Returns the block after indexing
    */
   public async $getBlock(hash: string): Promise<BlockExtended | IEsploraApi.Block> {
     // Check the memory cache
@@ -693,14 +618,31 @@ class Blocks {
       return blockByHash;
     }
 
-    // Not Bitcoin network, return the block as it from the bitcoin backend
-    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
-      return await bitcoinCoreApi.$getBlock(hash);
+    // Block has already been indexed
+    if (Common.indexingEnabled()) {
+      const dbBlock = await blocksRepository.$getBlockByHash(hash);
+      if (dbBlock != null) {
+        return prepareBlock(dbBlock);
+      }
     }
 
+    // Not Bitcoin network, return the block as it
+    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
+      return await bitcoinApi.$getBlock(hash);
+    }
+
+    let block = await bitcoinClient.getBlock(hash);
+    block = prepareBlock(block);
+
     // Bitcoin network, add our custom data on top
-    const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(hash);
-    return await this.$indexBlock(block.height);
+    const transactions = await this.$getTransactionsExtended(hash, block.height, true);
+    const blockExtended = await this.$getBlockExtended(block, transactions);
+    if (Common.indexingEnabled()) {
+      delete(blockExtended['coinbaseTx']);
+      await blocksRepository.$saveBlockInDatabase(blockExtended);
+    }
+
+    return blockExtended;
   }
 
   public async $getStrippedBlockTransactions(hash: string, skipMemoryCache = false,
@@ -734,39 +676,35 @@ class Blocks {
     return summary.transactions;
   }
 
-  /**
-   * Get 15 blocks
-   * 
-   * Internally this function uses two methods to get the blocks, and
-   * the method is automatically selected:
-   *  - Using previous block hash links
-   *  - Using block height
-   * 
-   * @param fromHeight 
-   * @param limit 
-   * @returns 
-   */
   public async $getBlocks(fromHeight?: number, limit: number = 15): Promise<BlockExtended[]> {
-    let currentHeight = fromHeight !== undefined ? fromHeight : this.currentBlockHeight;
-    if (currentHeight > this.currentBlockHeight) {
-      limit -= currentHeight - this.currentBlockHeight;
-      currentHeight = this.currentBlockHeight;
-    }
+    let currentHeight = fromHeight !== undefined ? fromHeight : await blocksRepository.$mostRecentBlockHeight();
     const returnBlocks: BlockExtended[] = [];
 
     if (currentHeight < 0) {
       return returnBlocks;
     }
 
+    // Check if block height exist in local cache to skip the hash lookup
+    const blockByHeight = this.getBlocks().find((b) => b.height === currentHeight);
+    let startFromHash: string | null = null;
+    if (blockByHeight) {
+      startFromHash = blockByHeight.id;
+    } else if (!Common.indexingEnabled()) {
+      startFromHash = await bitcoinApi.$getBlockHash(currentHeight);
+    }
+
+    let nextHash = startFromHash;
     for (let i = 0; i < limit && currentHeight >= 0; i++) {
       let block = this.getBlocks().find((b) => b.height === currentHeight);
       if (block) {
-        // Using the memory cache (find by height)
         returnBlocks.push(block);
-      } else {
-        // Using indexing (find by height, index on the fly, save in database)
+      } else if (Common.indexingEnabled()) {
         block = await this.$indexBlock(currentHeight);
         returnBlocks.push(block);
+      } else if (nextHash != null) {
+        block = prepareBlock(await bitcoinClient.getBlock(nextHash));
+        nextHash = block.previousblockhash;
+        returnBlocks.push(block);
       }
       currentHeight--;
     }
@@ -774,114 +712,6 @@ class Blocks {
     return returnBlocks;
   }
 
-  /**
-   * Used for bulk block data query
-   * 
-   * @param fromHeight 
-   * @param toHeight 
-   */
-  public async $getBlocksBetweenHeight(fromHeight: number, toHeight: number): Promise<any> {
-    if (!Common.indexingEnabled()) {
-      return [];
-    }
-
-    const blocks: any[] = [];
-
-    while (fromHeight <= toHeight) {
-      let block: BlockExtended | null = await blocksRepository.$getBlockByHeight(fromHeight);
-      if (!block) {
-        await this.$indexBlock(fromHeight);
-        block = await blocksRepository.$getBlockByHeight(fromHeight);
-        if (!block) {
-          continue;
-        }
-      }
-
-      // Cleanup fields before sending the response
-      const cleanBlock: any = {
-        height: block.height ?? null,
-        hash: block.id ?? null,
-        timestamp: block.timestamp ?? null,
-        median_timestamp: block.mediantime ?? null,
-        previous_block_hash: block.previousblockhash ?? null,
-        difficulty: block.difficulty ?? null,
-        header: block.extras.header ?? null,
-        version: block.version ?? null,
-        bits: block.bits ?? null,
-        nonce: block.nonce ?? null,
-        size: block.size ?? null,
-        weight: block.weight ?? null,
-        tx_count: block.tx_count ?? null,
-        merkle_root: block.merkle_root ?? null,
-        reward: block.extras.reward ?? null,
-        total_fee_amt: block.extras.totalFees ?? null,
-        avg_fee_amt: block.extras.avgFee ?? null,
-        median_fee_amt: block.extras.medianFeeAmt ?? null,
-        fee_amt_percentiles: block.extras.feePercentiles ?? null,
-        avg_fee_rate: block.extras.avgFeeRate ?? null,
-        median_fee_rate: block.extras.medianFee ?? null,
-        fee_rate_percentiles: block.extras.feeRange ?? null,
-        total_inputs: block.extras.totalInputs ?? null,
-        total_input_amt: block.extras.totalInputAmt ?? null,
-        total_outputs: block.extras.totalOutputs ?? null,
-        total_output_amt: block.extras.totalOutputAmt ?? null,
-        segwit_total_txs: block.extras.segwitTotalTxs ?? null,
-        segwit_total_size: block.extras.segwitTotalSize ?? null,
-        segwit_total_weight: block.extras.segwitTotalWeight ?? null,
-        avg_tx_size: block.extras.avgTxSize ?? null,
-        utxoset_change: block.extras.utxoSetChange ?? null,
-        utxoset_size: block.extras.utxoSetSize ?? null,
-        coinbase_raw: block.extras.coinbaseRaw ?? null,
-        coinbase_address: block.extras.coinbaseAddress ?? null,
-        coinbase_signature: block.extras.coinbaseSignature ?? null,
-        coinbase_signature_ascii: block.extras.coinbaseSignatureAscii ?? null,
-        pool_slug: block.extras.pool.slug ?? null,
-        pool_id: block.extras.pool.id ?? null,
-      };
-
-      if (Common.blocksSummariesIndexingEnabled() && cleanBlock.fee_amt_percentiles === null) {
-        cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash);
-        if (cleanBlock.fee_amt_percentiles === null) {
-          const block = await bitcoinClient.getBlock(cleanBlock.hash, 2);
-          const summary = this.summarizeBlock(block);
-          await BlocksSummariesRepository.$saveSummary({ height: block.height, mined: summary });
-          cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash);
-        }
-        if (cleanBlock.fee_amt_percentiles !== null) {
-          cleanBlock.median_fee_amt = cleanBlock.fee_amt_percentiles[3];
-        }
-      }
-
-      cleanBlock.fee_amt_percentiles = {
-        'min': cleanBlock.fee_amt_percentiles[0],
-        'perc_10': cleanBlock.fee_amt_percentiles[1],
-        'perc_25': cleanBlock.fee_amt_percentiles[2],
-        'perc_50': cleanBlock.fee_amt_percentiles[3],
-        'perc_75': cleanBlock.fee_amt_percentiles[4],
-        'perc_90': cleanBlock.fee_amt_percentiles[5],
-        'max': cleanBlock.fee_amt_percentiles[6],
-      };
-      cleanBlock.fee_rate_percentiles = {
-        'min': cleanBlock.fee_rate_percentiles[0],
-        'perc_10': cleanBlock.fee_rate_percentiles[1],
-        'perc_25': cleanBlock.fee_rate_percentiles[2],
-        'perc_50': cleanBlock.fee_rate_percentiles[3],
-        'perc_75': cleanBlock.fee_rate_percentiles[4],
-        'perc_90': cleanBlock.fee_rate_percentiles[5],
-        'max': cleanBlock.fee_rate_percentiles[6],
-      };
-
-      // Re-org can happen after indexing so we need to always get the
-      // latest state from core
-      cleanBlock.orphans = chainTips.getOrphanedBlocksAtHeight(cleanBlock.height);
-
-      blocks.push(cleanBlock);
-      fromHeight++;
-    }
-
-    return blocks;
-  }
-
   public async $getBlockAuditSummary(hash: string): Promise<any> {
     let summary;
     if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
@@ -911,15 +741,34 @@ class Blocks {
   }
 
   public async $indexCPFP(hash: string, height: number): Promise<void> {
-    const block = await bitcoinClient.getBlock(hash, 2);
-    const transactions = block.tx.map(tx => {
-      tx.vsize = tx.weight / 4;
-      tx.fee *= 100_000_000;
-      return tx;
-    });
-
-    const clusters: any[] = [];
-
+    let transactions;
+    if (Common.blocksSummariesIndexingEnabled()) {
+      transactions = await this.$getStrippedBlockTransactions(hash);
+      const rawBlock = await bitcoinApi.$getRawBlock(hash);
+      const block = Block.fromBuffer(rawBlock);
+      const txMap = {};
+      for (const tx of block.transactions || []) {
+        txMap[tx.getId()] = tx;
+      }
+      for (const tx of transactions) {
+        // convert from bitcoinjs to esplora vin format
+        if (txMap[tx.txid]?.ins) {
+          tx.vin = txMap[tx.txid].ins.map(vin => {
+            return {
+              txid: vin.hash.slice().reverse().toString('hex')
+            };
+          });
+        }
+      }
+    } else {
+      const block = await bitcoinClient.getBlock(hash, 2);
+      transactions = block.tx.map(tx => {
+        tx.vsize = tx.weight / 4;
+        tx.fee *= 100_000_000;
+        return tx;
+      });
+    }
+ 
     let cluster: TransactionStripped[] = [];
     let ancestors: { [txid: string]: boolean } = {};
     for (let i = transactions.length - 1; i >= 0; i--) {
@@ -933,12 +782,10 @@ class Blocks {
         });
         const effectiveFeePerVsize = totalFee / totalVSize;
         if (cluster.length > 1) {
-          clusters.push({
-            root: cluster[0].txid,
-            height,
-            txs: cluster.map(tx => { return { txid: tx.txid, weight: tx.vsize * 4, fee: tx.fee || 0 }; }),
-            effectiveFeePerVsize,
-          });
+          await cpfpRepository.$saveCluster(height, cluster.map(tx => { return { txid: tx.txid, weight: tx.vsize * 4, fee: tx.fee || 0 }; }), effectiveFeePerVsize);
+          for (const tx of cluster) {
+            await transactionRepository.$setCluster(tx.txid, cluster[0].txid);
+          }
         }
         cluster = [];
         ancestors = {};
@@ -948,10 +795,7 @@ class Blocks {
         ancestors[vin.txid] = true;
       });
     }
-    const result = await cpfpRepository.$batchSaveClusters(clusters);
-    if (!result) {
-      await cpfpRepository.$insertProgressMarker(height);
-    }
+    await blocksRepository.$setCPFPIndexed(hash);
   }
 }
 

backend/src/api/chain-tips.ts

@@ -1,61 +0,0 @@
-import logger from '../logger';
-import bitcoinClient from './bitcoin/bitcoin-client';
-
-export interface ChainTip {
-  height: number;
-  hash: string;
-  branchlen: number;
-  status: 'invalid' | 'active' | 'valid-fork' | 'valid-headers' | 'headers-only';
-};
-
-export interface OrphanedBlock {
-  height: number;
-  hash: string;
-  status: 'valid-fork' | 'valid-headers' | 'headers-only';
-}
-
-class ChainTips {
-  private chainTips: ChainTip[] = [];
-  private orphanedBlocks: OrphanedBlock[] = [];
-
-  public async updateOrphanedBlocks(): Promise<void> {
-    try {
-      this.chainTips = await bitcoinClient.getChainTips();
-      this.orphanedBlocks = [];
-
-      for (const chain of this.chainTips) {
-        if (chain.status === 'valid-fork' || chain.status === 'valid-headers') {
-          let block = await bitcoinClient.getBlock(chain.hash);
-          while (block && block.confirmations === -1) {
-            this.orphanedBlocks.push({
-              height: block.height,
-              hash: block.hash,
-              status: chain.status
-            });
-            block = await bitcoinClient.getBlock(block.previousblockhash);
-          }
-        }
-      }
-
-      logger.debug(`Updated orphaned blocks cache. Found ${this.orphanedBlocks.length} orphaned blocks`);
-    } catch (e) {
-      logger.err(`Cannot get fetch orphaned blocks. Reason: ${e instanceof Error ? e.message : e}`);
-    }
-  }
-
-  public getOrphanedBlocksAtHeight(height: number | undefined): OrphanedBlock[] {
-    if (height === undefined) {
-      return [];
-    }
-
-    const orphans: OrphanedBlock[] = [];
-    for (const block of this.orphanedBlocks) {
-      if (block.height === height) {
-        orphans.push(block);
-      }
-    }
-    return orphans;
-  }
-}
-
-export default new ChainTips();

backend/src/api/common.ts

@@ -1,4 +1,4 @@
-import { CpfpInfo, MempoolBlockWithTransactions, TransactionExtended, TransactionStripped } from '../mempool.interfaces';
+import { CpfpInfo, TransactionExtended, TransactionStripped } from '../mempool.interfaces';
 import config from '../config';
 import { NodeSocket } from '../repositories/NodesSocketsRepository';
 import { isIP } from 'net';
@@ -35,31 +35,24 @@ export class Common {
   }
 
   static getFeesInRange(transactions: TransactionExtended[], rangeLength: number) {
-    const filtered: TransactionExtended[] = [];
-    let lastValidRate = Infinity;
-    // filter out anomalous fee rates to ensure monotonic range
-    for (const tx of transactions) {
-      if (tx.effectiveFeePerVsize <= lastValidRate) {
-        filtered.push(tx);
-        lastValidRate = tx.effectiveFeePerVsize;
-      }
-    }
-    const arr = [filtered[filtered.length - 1].effectiveFeePerVsize];
+    const arr = [transactions[transactions.length - 1].effectiveFeePerVsize];
     const chunk = 1 / (rangeLength - 1);
     let itemsToAdd = rangeLength - 2;
 
     while (itemsToAdd > 0) {
-      arr.push(filtered[Math.floor(filtered.length * chunk * itemsToAdd)].effectiveFeePerVsize);
+      arr.push(transactions[Math.floor(transactions.length * chunk * itemsToAdd)].effectiveFeePerVsize);
       itemsToAdd--;
     }
 
-    arr.push(filtered[0].effectiveFeePerVsize);
+    arr.push(transactions[0].effectiveFeePerVsize);
     return arr;
   }
 
   static findRbfTransactions(added: TransactionExtended[], deleted: TransactionExtended[]): { [txid: string]: TransactionExtended } {
     const matches: { [txid: string]: TransactionExtended } = {};
     deleted
+      // The replaced tx must have at least one input with nSequence < maxint-1 (That’s the opt-in)
+      .filter((tx) => tx.vin.some((vin) => vin.sequence < 0xfffffffe))
       .forEach((deletedTx) => {
         const foundMatches = added.find((addedTx) => {
           // The new tx must, absolutely speaking, pay at least as much fee as the replaced tx.
@@ -68,7 +61,7 @@ export class Common {
             && addedTx.feePerVsize > deletedTx.feePerVsize
             // Spends one or more of the same inputs
             && deletedTx.vin.some((deletedVin) =>
-              addedTx.vin.some((vin) => vin.txid === deletedVin.txid && vin.vout === deletedVin.vout));
+              addedTx.vin.some((vin) => vin.txid === deletedVin.txid));
             });
         if (foundMatches) {
           matches[deletedTx.txid] = foundMatches;
@@ -164,30 +157,6 @@ export class Common {
     return parents;
   }
 
-  // calculates the ratio of matched transactions to projected transactions by weight
-  static getSimilarity(projectedBlock: MempoolBlockWithTransactions, transactions: TransactionExtended[]): number {
-    let matchedWeight = 0;
-    let projectedWeight = 0;
-    const inBlock = {};
-
-    for (const tx of transactions) {
-      inBlock[tx.txid] = tx;
-    }
-
-    // look for transactions that were expected in the template, but missing from the mined block
-    for (const tx of projectedBlock.transactions) {
-      if (inBlock[tx.txid]) {
-        matchedWeight += tx.vsize * 4;
-      }
-      projectedWeight += tx.vsize * 4;
-    }
-
-    projectedWeight += transactions[0].weight;
-    matchedWeight += transactions[0].weight;
-
-    return projectedWeight ? matchedWeight / projectedWeight : 1;
-  }
-
   static getSqlInterval(interval: string | null): string | null {
     switch (interval) {
       case '24h': return '1 DAY';
@@ -199,7 +168,6 @@ export class Common {
       case '1y': return '1 YEAR';
       case '2y': return '2 YEAR';
       case '3y': return '3 YEAR';
-      case '4y': return '4 YEAR';
       default: return null;
     }
   }
@@ -222,7 +190,7 @@ export class Common {
   static cpfpIndexingEnabled(): boolean {
     return (
       Common.indexingEnabled() &&
-      config.MEMPOOL.CPFP_INDEXING === true
+      config.MEMPOOL.TRANSACTION_INDEXING === true
     );
   }
 
@@ -262,21 +230,14 @@ export class Common {
     ].join('x');
   }
 
-  static utcDateToMysql(date?: number | null): string | null {
-    if (date === null) {
-      return null;
-    }
+  static utcDateToMysql(date?: number): string {
     const d = new Date((date || 0) * 1000);
     return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0];
   }
 
   static findSocketNetwork(addr: string): {network: string | null, url: string} {
     let network: string | null = null;
-    let url: string = addr;
-
-    if (config.LIGHTNING.BACKEND === 'cln') {
-      url = addr.split('://')[1];
-    }
+    let url = addr.split('://')[1];
 
     if (!url) {
       return {
@@ -293,7 +254,7 @@ export class Common {
       }
     } else if (addr.indexOf('i2p') !== -1) {
       network = 'i2p';
-    } else if (addr.indexOf('ipv4') !== -1 || (config.LIGHTNING.BACKEND === 'lnd' && isIP(url.split(':')[0]) === 4)) {
+    } else if (addr.indexOf('ipv4') !== -1) {
       const ipv = isIP(url.split(':')[0]);
       if (ipv === 4) {
         network = 'ipv4';
@@ -303,7 +264,7 @@ export class Common {
           url: addr,
         };
       }
-    } else if (addr.indexOf('ipv6') !== -1 || (config.LIGHTNING.BACKEND === 'lnd' && url.indexOf(']:'))) {
+    } else if (addr.indexOf('ipv6') !== -1) {
       url = url.split('[')[1].split(']')[0];
       const ipv = isIP(url);
       if (ipv === 6) {

backend/src/api/database-migration.ts

@@ -2,12 +2,9 @@ import config from '../config';
 import DB from '../database';
 import logger from '../logger';
 import { Common } from './common';
-import blocksRepository from '../repositories/BlocksRepository';
-import cpfpRepository from '../repositories/CpfpRepository';
-import { RowDataPacket } from 'mysql2';
 
 class DatabaseMigration {
-  private static currentVersion = 59;
+  private static currentVersion = 49;
   private queryTimeout = 3600_000;
   private statisticsAddedIndexed = false;
   private uniqueLogs: string[] = [];
@@ -62,8 +59,8 @@ class DatabaseMigration {
 
     if (databaseSchemaVersion <= 2) {
       // Disable some spam logs when they're not relevant
-      this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
-      this.uniqueLog(logger.notice, this.hashratesTruncatedMessage);
+      this.uniqueLogs.push(this.blocksTruncatedMessage);
+      this.uniqueLogs.push(this.hashratesTruncatedMessage);
     }
 
     logger.debug('MIGRATIONS: Current state.schema_version ' + databaseSchemaVersion);
@@ -86,7 +83,7 @@ class DatabaseMigration {
       try {
         await this.$migrateTableSchemaFromVersion(databaseSchemaVersion);
         if (databaseSchemaVersion === 0) {
-          logger.notice(`MIGRATIONS: OK. Database schema has been properly initialized to version ${DatabaseMigration.currentVersion} (latest version)`);
+          logger.notice(`MIGRATIONS: OK. Database schema has been properly initialized to version ${DatabaseMigration.currentVersion} (latest version)`);          
         } else {
           logger.notice(`MIGRATIONS: OK. Database schema have been migrated from version ${databaseSchemaVersion} to ${DatabaseMigration.currentVersion} (latest version)`);
         }
@@ -300,7 +297,7 @@ class DatabaseMigration {
       await this.$executeQuery('ALTER TABLE `lightning_stats` ADD med_base_fee_mtokens bigint(20) unsigned NOT NULL DEFAULT "0"');
       await this.updateToSchemaVersion(27);
     }
-
+    
     if (databaseSchemaVersion < 28 && isBitcoin === true) {
       if (config.LIGHTNING.ENABLED) {
         this.uniqueLog(logger.notice, `'lightning_stats' and 'node_stats' tables have been truncated.`);
@@ -445,77 +442,6 @@ class DatabaseMigration {
       await this.$executeQuery('TRUNCATE TABLE `blocks_audits`');
       await this.updateToSchemaVersion(49);
     }
-
-    if (databaseSchemaVersion < 50) {
-      await this.$executeQuery('ALTER TABLE `blocks` DROP COLUMN `cpfp_indexed`');
-      await this.updateToSchemaVersion(50);
-    }
-
-    if (databaseSchemaVersion < 51) {
-      await this.$executeQuery('ALTER TABLE `cpfp_clusters` ADD INDEX `height` (`height`)');
-      await this.updateToSchemaVersion(51);
-    }
-
-    if (databaseSchemaVersion < 52) {
-      await this.$executeQuery(this.getCreateCompactCPFPTableQuery(), await this.$checkIfTableExists('compact_cpfp_clusters'));
-      await this.$executeQuery(this.getCreateCompactTransactionsTableQuery(), await this.$checkIfTableExists('compact_transactions'));
-      try {
-        await this.$convertCompactCpfpTables();
-        await this.$executeQuery('DROP TABLE IF EXISTS `transactions`');
-        await this.$executeQuery('DROP TABLE IF EXISTS `cpfp_clusters`');
-        await this.updateToSchemaVersion(52);
-      } catch (e) {
-        logger.warn('' + (e instanceof Error ? e.message : e));
-      }
-    }
-
-    if (databaseSchemaVersion < 53) {
-      await this.$executeQuery('ALTER TABLE statistics MODIFY mempool_byte_weight bigint(20) UNSIGNED NOT NULL');
-      await this.updateToSchemaVersion(53);
-    }
-
-    if (databaseSchemaVersion < 54) {
-      this.uniqueLog(logger.notice, `'prices' table has been truncated`);
-      await this.$executeQuery(`TRUNCATE prices`);
-      if (isBitcoin === true) {
-        this.uniqueLog(logger.notice, `'blocks_prices' table has been truncated`);
-        await this.$executeQuery(`TRUNCATE blocks_prices`);
-      }
-      await this.updateToSchemaVersion(54);
-    }
-
-    if (databaseSchemaVersion < 55) {
-      await this.$executeQuery(this.getAdditionalBlocksDataQuery());
-      this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
-      await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index
-      await this.updateToSchemaVersion(55);
-    }
-
-    if (databaseSchemaVersion < 56) {
-      await this.$executeQuery('ALTER TABLE pools ADD unique_id int NOT NULL DEFAULT -1');
-      await this.$executeQuery('TRUNCATE TABLE `blocks`');
-      this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
-      await this.$executeQuery('DELETE FROM `pools`');
-      await this.$executeQuery('ALTER TABLE pools AUTO_INCREMENT = 1');
-      await this.$executeQuery(`UPDATE state SET string = NULL WHERE name = 'pools_json_sha'`);
-      this.uniqueLog(logger.notice, '`pools` table has been truncated`');
-      await this.updateToSchemaVersion(56);
-    }
-
-    if (databaseSchemaVersion < 57 && isBitcoin === true) {
-      await this.$executeQuery(`ALTER TABLE nodes MODIFY updated_at datetime NULL`);
-      await this.updateToSchemaVersion(57);
-    }
-
-    if (databaseSchemaVersion < 58) {
-      // We only run some migration queries for this version
-      await this.updateToSchemaVersion(58);
-    }
-
-    if (databaseSchemaVersion < 59 && (config.MEMPOOL.NETWORK === 'signet' || config.MEMPOOL.NETWORK === 'testnet')) {
-      // https://github.com/mempool/mempool/issues/3360
-      await this.$executeQuery(`TRUNCATE prices`);
-    }
   }
 
   /**
@@ -639,15 +565,10 @@ class DatabaseMigration {
       queries.push(`INSERT INTO state(name, number, string) VALUES ('last_hashrates_indexing', 0, NULL)`);
     }
 
-    if (version < 9 && isBitcoin === true) {
+    if (version < 9  && isBitcoin === true) {
       queries.push(`INSERT INTO state(name, number, string) VALUES ('last_weekly_hashrates_indexing', 0, NULL)`);
     }
 
-    if (version < 58) {
-      queries.push(`DELETE FROM state WHERE name = 'last_hashrates_indexing'`);
-      queries.push(`DELETE FROM state WHERE name = 'last_weekly_hashrates_indexing'`);
-    }
-
     return queries;
   }
 
@@ -794,28 +715,6 @@ class DatabaseMigration {
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
 
-  private getAdditionalBlocksDataQuery(): string {
-    return `ALTER TABLE blocks
-      ADD median_timestamp timestamp NOT NULL,
-      ADD coinbase_address varchar(100) NULL,
-      ADD coinbase_signature varchar(500) NULL,
-      ADD coinbase_signature_ascii varchar(500) NULL,
-      ADD avg_tx_size double unsigned NOT NULL,
-      ADD total_inputs int unsigned NOT NULL,
-      ADD total_outputs int unsigned NOT NULL,
-      ADD total_output_amt bigint unsigned NOT NULL,
-      ADD fee_percentiles longtext NULL,
-      ADD median_fee_amt int unsigned NULL,
-      ADD segwit_total_txs int unsigned NOT NULL,
-      ADD segwit_total_size int unsigned NOT NULL,
-      ADD segwit_total_weight int unsigned NOT NULL,
-      ADD header varchar(160) NOT NULL,
-      ADD utxoset_change int NOT NULL,
-      ADD utxoset_size int unsigned NULL,
-      ADD total_input_amt bigint unsigned NULL
-    `;
-  }
-
   private getCreateDailyStatsTableQuery(): string {
     return `CREATE TABLE IF NOT EXISTS hashrates (
       hashrate_timestamp timestamp NOT NULL,
@@ -1014,77 +913,24 @@ class DatabaseMigration {
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
 
-  private getCreateCompactCPFPTableQuery(): string {
-    return `CREATE TABLE IF NOT EXISTS compact_cpfp_clusters (
-      root binary(32) NOT NULL,
-      height int(10) NOT NULL,
-      txs BLOB DEFAULT NULL,
-      fee_rate float unsigned,
-      PRIMARY KEY (root),
-      INDEX (height)
-    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
-  }
+  public async $truncateIndexedData(tables: string[]) {
+    const allowedTables = ['blocks', 'hashrates', 'prices'];
 
-  private getCreateCompactTransactionsTableQuery(): string {
-    return `CREATE TABLE IF NOT EXISTS compact_transactions (
-      txid binary(32) NOT NULL,
-      cluster binary(32) DEFAULT NULL,
-      PRIMARY KEY (txid)
-    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
-  }
-
-  public async $blocksReindexingTruncate(): Promise<void> {
-    logger.warn(`Truncating pools, blocks and hashrates for re-indexing (using '--reindex-blocks'). You can cancel this command within 5 seconds`);
-    await Common.sleep$(5000);
-
-    await this.$executeQuery(`TRUNCATE blocks`);
-    await this.$executeQuery(`TRUNCATE hashrates`);
-    await this.$executeQuery(`TRUNCATE difficulty_adjustments`);
-    await this.$executeQuery('DELETE FROM `pools`');
-    await this.$executeQuery('ALTER TABLE pools AUTO_INCREMENT = 1');
-    await this.$executeQuery(`UPDATE state SET string = NULL WHERE name = 'pools_json_sha'`);
-  }
-
-  private async $convertCompactCpfpTables(): Promise<void> {
     try {
-      const batchSize = 250;
-      const maxHeight = await blocksRepository.$mostRecentBlockHeight() || 0;
-      const [minHeightRows]: any = await DB.query(`SELECT MIN(height) AS minHeight from cpfp_clusters`);
-      const minHeight = (minHeightRows.length && minHeightRows[0].minHeight != null) ? minHeightRows[0].minHeight : maxHeight;
-      let height = maxHeight;
-
-      // Logging
-      let timer = new Date().getTime() / 1000;
-      const startedAt = new Date().getTime() / 1000;
-
-      while (height > minHeight) {
-        const [rows] = await DB.query(
-          `
-            SELECT * from cpfp_clusters
-            WHERE height <= ? AND height > ?
-            ORDER BY height
-          `,
-          [height, height - batchSize]
-        ) as RowDataPacket[][];
-        if (rows?.length) {
-          await cpfpRepository.$batchSaveClusters(rows.map(row => {
-            return {
-              root: row.root,
-              height: row.height,
-              txs: JSON.parse(row.txs),
-              effectiveFeePerVsize: row.fee_rate,
-            };
-          }));
+      for (const table of tables) {
+        if (!allowedTables.includes(table)) {
+          logger.debug(`Table ${table} cannot to be re-indexed (not allowed)`);
+          continue;
         }
 
-        const elapsed = new Date().getTime() / 1000 - timer;
-        const runningFor = new Date().getTime() / 1000 - startedAt;
-        logger.debug(`Migrated cpfp data from block ${height} to ${height - batchSize} in ${elapsed.toFixed(2)} seconds | total elapsed: ${runningFor.toFixed(2)} seconds`);
-        timer = new Date().getTime() / 1000;
-        height -= batchSize;
+        await this.$executeQuery(`TRUNCATE ${table}`, true);
+        if (table === 'hashrates') {
+          await this.$executeQuery('UPDATE state set number = 0 where name = "last_hashrates_indexing"', true);
+        }
+        logger.notice(`Table ${table} has been truncated`);
       }
     } catch (e) {
-      logger.warn(`Failed to migrate cpfp transaction data`);
+      logger.warn(`Unable to erase indexed data`);
     }
   }
 }

backend/src/api/difficulty-adjustment.ts

@@ -9,11 +9,9 @@ export interface DifficultyAdjustment {
   remainingBlocks: number;       // Block count
   remainingTime: number;         // Duration of time in ms
   previousRetarget: number;      // Percent: -75 to 300
-  previousTime: number;          // Unix time in ms
   nextRetargetHeight: number;    // Block Height
   timeAvg: number;               // Duration of time in ms
   timeOffset: number;            // (Testnet) Time since last block (cap @ 20min) in ms
-  expectedBlocks: number;         // Block count
 }
 
 export function calcDifficultyAdjustment(
@@ -24,28 +22,31 @@ export function calcDifficultyAdjustment(
   network: string,
   latestBlockTimestamp: number,
 ): DifficultyAdjustment {
+  const ESTIMATE_LAG_BLOCKS = 146; // For first 7.2% of epoch, don't estimate.
   const EPOCH_BLOCK_LENGTH = 2016; // Bitcoin mainnet
   const BLOCK_SECONDS_TARGET = 600; // Bitcoin mainnet
   const TESTNET_MAX_BLOCK_SECONDS = 1200; // Bitcoin testnet
 
-  const diffSeconds = Math.max(0, nowSeconds - DATime);
+  const diffSeconds = nowSeconds - DATime;
   const blocksInEpoch = (blockHeight >= 0) ? blockHeight % EPOCH_BLOCK_LENGTH : 0;
   const progressPercent = (blockHeight >= 0) ? blocksInEpoch / EPOCH_BLOCK_LENGTH * 100 : 100;
   const remainingBlocks = EPOCH_BLOCK_LENGTH - blocksInEpoch;
   const nextRetargetHeight = (blockHeight >= 0) ? blockHeight + remainingBlocks : 0;
-  const expectedBlocks = diffSeconds / BLOCK_SECONDS_TARGET;
 
   let difficultyChange = 0;
-  let timeAvgSecs = blocksInEpoch ? diffSeconds / blocksInEpoch : BLOCK_SECONDS_TARGET;
-
-  difficultyChange = (BLOCK_SECONDS_TARGET / timeAvgSecs - 1) * 100;
-  // Max increase is x4 (+300%)
-  if (difficultyChange > 300) {
-    difficultyChange = 300;
-  }
-  // Max decrease is /4 (-75%)
-  if (difficultyChange < -75) {
-    difficultyChange = -75;
+  let timeAvgSecs = BLOCK_SECONDS_TARGET;
+  // Only calculate the estimate once we have 7.2% of blocks in current epoch
+  if (blocksInEpoch >= ESTIMATE_LAG_BLOCKS) {
+    timeAvgSecs = diffSeconds / blocksInEpoch;
+    difficultyChange = (BLOCK_SECONDS_TARGET / timeAvgSecs - 1) * 100;
+    // Max increase is x4 (+300%)
+    if (difficultyChange > 300) {
+      difficultyChange = 300;
+    }
+    // Max decrease is /4 (-75%)
+    if (difficultyChange < -75) {
+      difficultyChange = -75;
+    }
   }
 
   // Testnet difficulty is set to 1 after 20 minutes of no blocks,
@@ -73,11 +74,9 @@ export function calcDifficultyAdjustment(
     remainingBlocks,
     remainingTime,
     previousRetarget,
-    previousTime: DATime,
     nextRetargetHeight,
     timeAvg,
     timeOffset,
-    expectedBlocks,
   };
 }
 

backend/src/api/disk-cache.ts

@@ -9,91 +9,50 @@ import { TransactionExtended } from '../mempool.interfaces';
 import { Common } from './common';
 
 class DiskCache {
-  private cacheSchemaVersion = 3;
+  private cacheSchemaVersion = 1;
 
-  private static TMP_FILE_NAME = config.MEMPOOL.CACHE_DIR + '/tmp-cache.json';
-  private static TMP_FILE_NAMES = config.MEMPOOL.CACHE_DIR + '/tmp-cache{number}.json';
   private static FILE_NAME = config.MEMPOOL.CACHE_DIR + '/cache.json';
   private static FILE_NAMES = config.MEMPOOL.CACHE_DIR + '/cache{number}.json';
   private static CHUNK_FILES = 25;
   private isWritingCache = false;
 
-  constructor() {
-    if (!cluster.isPrimary) {
-      return;
-    }
-    process.on('SIGINT', (e) => {
-      this.$saveCacheToDisk(true);
-      process.exit(0);
-    });
-  }
+  constructor() { }
 
-  async $saveCacheToDisk(sync: boolean = false): Promise<void> {
+  async $saveCacheToDisk(): Promise<void> {
     if (!cluster.isPrimary) {
       return;
     }
     if (this.isWritingCache) {
-      logger.debug('Saving cache already in progress. Skipping.');
+      logger.debug('Saving cache already in progress. Skipping.')
       return;
     }
     try {
-      logger.debug(`Writing mempool and blocks data to disk cache (${ sync ? 'sync' : 'async' })...`);
+      logger.debug('Writing mempool and blocks data to disk cache (async)...');
       this.isWritingCache = true;
 
       const mempool = memPool.getMempool();
       const mempoolArray: TransactionExtended[] = [];
       for (const tx in mempool) {
-        if (mempool[tx] && !mempool[tx].deleteAfter) {
-          mempoolArray.push(mempool[tx]);
-        }
+        mempoolArray.push(mempool[tx]);
       }
 
       Common.shuffleArray(mempoolArray);
 
       const chunkSize = Math.floor(mempoolArray.length / DiskCache.CHUNK_FILES);
 
-      if (sync) {
-        fs.writeFileSync(DiskCache.TMP_FILE_NAME, JSON.stringify({
-          network: config.MEMPOOL.NETWORK,
-          cacheSchemaVersion: this.cacheSchemaVersion,
-          blocks: blocks.getBlocks(),
-          blockSummaries: blocks.getBlockSummaries(),
+      await fsPromises.writeFile(DiskCache.FILE_NAME, JSON.stringify({
+        cacheSchemaVersion: this.cacheSchemaVersion,
+        blocks: blocks.getBlocks(),
+        blockSummaries: blocks.getBlockSummaries(),
+        mempool: {},
+        mempoolArray: mempoolArray.splice(0, chunkSize),
+      }), { flag: 'w' });
+      for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
+        await fsPromises.writeFile(DiskCache.FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
           mempool: {},
           mempoolArray: mempoolArray.splice(0, chunkSize),
         }), { flag: 'w' });
-        for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
-          fs.writeFileSync(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
-            mempool: {},
-            mempoolArray: mempoolArray.splice(0, chunkSize),
-          }), { flag: 'w' });
-        }
-
-        fs.renameSync(DiskCache.TMP_FILE_NAME, DiskCache.FILE_NAME);
-        for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
-          fs.renameSync(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), DiskCache.FILE_NAMES.replace('{number}', i.toString()));
-        }
-      } else {
-        await fsPromises.writeFile(DiskCache.TMP_FILE_NAME, JSON.stringify({
-          network: config.MEMPOOL.NETWORK,
-          cacheSchemaVersion: this.cacheSchemaVersion,
-          blocks: blocks.getBlocks(),
-          blockSummaries: blocks.getBlockSummaries(),
-          mempool: {},
-          mempoolArray: mempoolArray.splice(0, chunkSize),
-        }), { flag: 'w' });
-        for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
-          await fsPromises.writeFile(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
-            mempool: {},
-            mempoolArray: mempoolArray.splice(0, chunkSize),
-          }), { flag: 'w' });
-        }
-
-        await fsPromises.rename(DiskCache.TMP_FILE_NAME, DiskCache.FILE_NAME);
-        for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
-          await fsPromises.rename(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), DiskCache.FILE_NAMES.replace('{number}', i.toString()));
-        }
       }
-
       logger.debug('Mempool and blocks data saved to disk cache');
       this.isWritingCache = false;
     } catch (e) {
@@ -102,29 +61,14 @@ class DiskCache {
     }
   }
 
-  wipeCache(): void {
-    logger.notice(`Wiping nodejs backend cache/cache*.json files`);
-    try {
-      fs.unlinkSync(DiskCache.FILE_NAME);
-    } catch (e: any) {
-      if (e?.code !== 'ENOENT') {
-        logger.err(`Cannot wipe cache file ${DiskCache.FILE_NAME}. Exception ${JSON.stringify(e)}`);
-      }
-    }
-
+  wipeCache() {
+    fs.unlinkSync(DiskCache.FILE_NAME);
     for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
-      const filename = DiskCache.FILE_NAMES.replace('{number}', i.toString());
-      try {
-        fs.unlinkSync(filename);
-      } catch (e: any) {
-        if (e?.code !== 'ENOENT') {
-          logger.err(`Cannot wipe cache file ${filename}. Exception ${JSON.stringify(e)}`);
-        }
-      }
+      fs.unlinkSync(DiskCache.FILE_NAMES.replace('{number}', i.toString()));
     }
   }
 
-  loadMempoolCache(): void {
+  loadMempoolCache() {
     if (!fs.existsSync(DiskCache.FILE_NAME)) {
       return;
     }
@@ -138,10 +82,6 @@ class DiskCache {
           logger.notice('Disk cache contains an outdated schema version. Clearing it and skipping the cache loading.');
           return this.wipeCache();
         }
-        if (data.network && data.network !== config.MEMPOOL.NETWORK) {
-          logger.notice('Disk cache contains data from a different network. Clearing it and skipping the cache loading.');
-          return this.wipeCache();
-        }
 
         if (data.mempoolArray) {
           for (const tx of data.mempoolArray) {

backend/src/api/explorer/channels.api.ts

@@ -559,17 +559,6 @@ class ChannelsApi {
     const policy1: Partial<ILightningApi.RoutingPolicy> = channel.node1_policy || {};
     const policy2: Partial<ILightningApi.RoutingPolicy> = channel.node2_policy || {};
 
-    // https://github.com/mempool/mempool/issues/3006
-    if ((channel.last_update ?? 0) < 1514736061) { // January 1st 2018
-      channel.last_update = null;
-    }
-    if ((policy1.last_update ?? 0) < 1514736061) { // January 1st 2018
-      policy1.last_update = null;
-    }
-    if ((policy2.last_update ?? 0) < 1514736061) { // January 1st 2018
-      policy2.last_update = null;
-    }
-
     const query = `INSERT INTO channels
       (
         id,
@@ -681,7 +670,9 @@ class ChannelsApi {
         AND status != 2
       `);
       if (result[0].changedRows ?? 0 > 0) {
-        logger.debug(`Marked ${result[0].changedRows} channels as inactive because they are not in the graph`, logger.tags.ln);
+        logger.info(`Marked ${result[0].changedRows} channels as inactive because they are not in the graph`);
+      } else {
+        logger.debug(`Marked ${result[0].changedRows} channels as inactive because they are not in the graph`);
       }
     } catch (e) {
       logger.err('$setChannelsInactive() error: ' + (e instanceof Error ? e.message : e));

backend/src/api/explorer/nodes.api.ts

@@ -228,7 +228,7 @@ class NodesApi {
             nodes.capacity
           FROM nodes
           ORDER BY capacity DESC
-          LIMIT 6
+          LIMIT 100
         `;
 
         [rows] = await DB.query(query);
@@ -269,26 +269,14 @@ class NodesApi {
       let query: string;
       if (full === false) {
         query = `
-          SELECT
-            nodes.public_key as publicKey,
-            IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
-            nodes.channels,
-            geo_names_city.names as city, geo_names_country.names as country,
-            geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision
+          SELECT nodes.public_key as publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
+            nodes.channels
           FROM nodes
-          LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
-          LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
-          LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
-          LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
           ORDER BY channels DESC
-          LIMIT 6;
+          LIMIT 100;
         `;
 
         [rows] = await DB.query(query);
-        for (let i = 0; i < rows.length; ++i) {
-          rows[i].country = JSON.parse(rows[i].country);
-          rows[i].city = JSON.parse(rows[i].city);
-        }
       } else {
         query = `
           SELECT nodes.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
@@ -374,13 +362,7 @@ class NodesApi {
   public async $searchNodeByPublicKeyOrAlias(search: string) {
     try {
       const publicKeySearch = search.replace('%', '') + '%';
-      const aliasSearch = search
-        .replace(/[-_.]/g, ' ') // Replace all -_. characters with empty space. Eg: "ln.nicehash" becomes "ln nicehash".  
-        .replace(/[^a-zA-Z0-9 ]/g, '') // Remove all special characters and keep just A to Z, 0 to 9.
-        .split(' ')
-        .filter(key => key.length)
-        .map((search) => '+' + search + '*').join(' ');
-      // %keyword% is wildcard search and can't be indexed so it's slower as the node database grow. keyword% can be indexed but then you can't search for "Nicehash" and get result for ln.nicehash.com. So we use fulltext index for words "ln, nicehash, com" and nicehash* will find it instantly.
+      const aliasSearch = search.replace(/[-_.]/g, ' ').replace(/[^a-zA-Z0-9 ]/g, '').split(' ').map((search) => '+' + search + '*').join(' ');
       const query = `SELECT public_key, alias, capacity, channels, status FROM nodes WHERE public_key LIKE ? OR MATCH alias_search AGAINST (? IN BOOLEAN MODE) ORDER BY capacity DESC LIMIT 10`;
       const [rows]: any = await DB.query(query, [publicKeySearch, aliasSearch]);
       return rows;
@@ -417,24 +399,24 @@ class NodesApi {
 
         if (!ispList[isp1]) {
           ispList[isp1] = {
-            ids: [channel.isp1ID],
+            id: channel.isp1ID.toString(),
             capacity: 0,
             channels: 0,
             nodes: {},
           };
-        } else if (ispList[isp1].ids.includes(channel.isp1ID) === false) {
-          ispList[isp1].ids.push(channel.isp1ID);
+        } else if (ispList[isp1].id.indexOf(channel.isp1ID) === -1) {
+          ispList[isp1].id += ',' + channel.isp1ID.toString();
         }
 
         if (!ispList[isp2]) {
           ispList[isp2] = {
-            ids: [channel.isp2ID],
+            id: channel.isp2ID.toString(),
             capacity: 0,
             channels: 0,
             nodes: {},
           };
-        } else if (ispList[isp2].ids.includes(channel.isp2ID) === false) {
-          ispList[isp2].ids.push(channel.isp2ID);
+        } else if (ispList[isp2].id.indexOf(channel.isp2ID) === -1) {
+          ispList[isp2].id += ',' + channel.isp2ID.toString();
         }
         
         ispList[isp1].capacity += channel.capacity;
@@ -444,11 +426,11 @@ class NodesApi {
         ispList[isp2].channels += 1;
         ispList[isp2].nodes[channel.node2PublicKey] = true;
       }
-      
+
       const ispRanking: any[] = [];
       for (const isp of Object.keys(ispList)) {
         ispRanking.push([
-          ispList[isp].ids.sort((a, b) => a - b).join(','),
+          ispList[isp].id,
           isp,
           ispList[isp].capacity,
           ispList[isp].channels,
@@ -642,11 +624,6 @@ class NodesApi {
    */
   public async $saveNode(node: ILightningApi.Node): Promise<void> {
     try {
-      // https://github.com/mempool/mempool/issues/3006
-      if ((node.last_update ?? 0) < 1514736061) { // January 1st 2018
-        node.last_update = null;
-      }
-  
       const sockets = (node.addresses?.map(a => a.addr).join(',')) ?? '';
       const query = `INSERT INTO nodes(
           public_key,
@@ -708,7 +685,9 @@ class NodesApi {
         )
       `);
       if (result[0].changedRows ?? 0 > 0) {
-        logger.debug(`Marked ${result[0].changedRows} nodes as inactive because they are not in the graph`, logger.tags.ln);
+        logger.info(`Marked ${result[0].changedRows} nodes as inactive because they are not in the graph`);
+      } else {
+        logger.debug(`Marked ${result[0].changedRows} nodes as inactive because they are not in the graph`);
       }
     } catch (e) {
       logger.err('$setNodesInactive() error: ' + (e instanceof Error ? e.message : e));

backend/src/api/explorer/nodes.routes.ts

@@ -41,70 +41,13 @@ class NodesRoutes {
       let nodes: any[] = [];
       switch (config.MEMPOOL.NETWORK) {
         case 'testnet':
-          nodesList = [
-            '032c7c7819276c4f706a04df1a0f1e10a5495994a7be4c1d3d28ca766e5a2b957b',
-            '025a7e38c2834dd843591a4d23d5f09cdeb77ddca85f673c2d944a14220ff14cf7',
-            '0395e2731a1673ef21d7a16a727c4fc4d4c35a861c428ce2c819c53d2b81c8bd55',
-            '032ab2028c0b614c6d87824e2373529652fd7e4221b4c70cc4da7c7005c49afcf0',
-            '029001b22fe70b48bee12d014df91982eb85ff1bd404ec772d5c83c4ee3e88d2c3',
-            '0212e2848d79f928411da5f2ff0a8c95ec6ccb5a09d2031b6f71e91309dcde63af',
-            '03e871a2229523d34f76e6311ff197cfe7f26c2fbec13554b93a46f4e710c47dab',
-            '032202ec98d976b0e928bd1d91924e8bd3eab07231fc39feb3737b010071073df8',
-            '02fa7c5a948d03d563a9f36940c2205a814e594d17c0042ced242c71a857d72605',
-            '039c14fdec2d958e3d14cebf657451bbd9e039196615785e82c917f274e3fb2205',
-            '033589bbcb233ffc416cefd5437c7f37e9d7cb7942d405e39e72c4c846d9b37f18',
-            '029293110441c6e2eacb57e1255bf6ef05c41a6a676fe474922d33c19f98a7d584',
-            '0235ad0b56ed8c42c4354444c24e971c05e769ec0b5fb0ccea42880095dc02ea2c',
-            '029700819a37afea630f80e6cc461f3fd3c4ace2598a21cfbbe64d1c78d0ee69a5',
-            '02c2d8b2dbf87c7894af2f1d321290e2fe6db5446cd35323987cee98f06e2e0075',
-            '030b0ca1ea7b1075716d2a555630e6fd47ef11bc7391fe68963ec06cf370a5e382',
-            '031adb9eb2d66693f85fa31a4adca0319ba68219f3ad5f9a2ef9b34a6b40755fa1',
-            '02ccd07faa47eda810ecf5591ccf5ca50f6c1034d0d175052898d32a00b9bae24f',
-          ];
+          nodesList = ['032c7c7819276c4f706a04df1a0f1e10a5495994a7be4c1d3d28ca766e5a2b957b', '025a7e38c2834dd843591a4d23d5f09cdeb77ddca85f673c2d944a14220ff14cf7', '0395e2731a1673ef21d7a16a727c4fc4d4c35a861c428ce2c819c53d2b81c8bd55', '032ab2028c0b614c6d87824e2373529652fd7e4221b4c70cc4da7c7005c49afcf0', '029001b22fe70b48bee12d014df91982eb85ff1bd404ec772d5c83c4ee3e88d2c3', '0212e2848d79f928411da5f2ff0a8c95ec6ccb5a09d2031b6f71e91309dcde63af', '03e871a2229523d34f76e6311ff197cfe7f26c2fbec13554b93a46f4e710c47dab', '032202ec98d976b0e928bd1d91924e8bd3eab07231fc39feb3737b010071073df8', '02fa7c5a948d03d563a9f36940c2205a814e594d17c0042ced242c71a857d72605', '039c14fdec2d958e3d14cebf657451bbd9e039196615785e82c917f274e3fb2205', '033589bbcb233ffc416cefd5437c7f37e9d7cb7942d405e39e72c4c846d9b37f18', '029293110441c6e2eacb57e1255bf6ef05c41a6a676fe474922d33c19f98a7d584'];
           break;
         case 'signet':
-          nodesList = [
-            '03ddab321b760433cbf561b615ef62ac7d318630c5f51d523aaf5395b90b751956',
-            '033d92c7bfd213ef1b34c90e985fb5dc77f9ec2409d391492484e57a44c4aca1de',
-            '02ad010dda54253c1eb9efe38b0760657a3b43ecad62198c359c051c9d99d45781',
-            '025196512905b8a3f1597428b867bec63ec9a95e5089eb7dc7e63e2d2691669029',
-            '027c625aa1fbe3768db68ebcb05b53b6dc0ce68b7b54b8900d326d167363e684fe',
-            '03f1629af3101fcc56b7aac2667016be84e3defbf3d0c8719f836c9b41c9a57a43',
-            '02dfb81e2f7a3c4c9e8a51b70ef82b4a24549cc2fab1f5b2fd636501774a918991',
-            '02d01ccf832944c68f10d39006093769c5b8bda886d561b128534e313d729fdb34',
-            '02499ed23027d4698a6904ff4ec1b6085a61f10b9a6937f90438f9947e38e8ea86',
-            '038310e3a786340f2bd7770704c7ccfe560fd163d9a1c99d67894597419d12cbf7',
-            '03e5e9d879b72c7d67ecd483bae023bd33e695bb32b981a4021260f7b9d62bc761',
-            '028d16e1a0ace4c0c0a421536d8d32ce484dfe6e2f726b7b0e7c30f12a195f8cc7',
-            '02ff690d06c187ab994bf83c5a2114fe5bf50112c2c817af0f788f736be9fa2070',
-            '02a9f570c51a2526a5ee85802e88f9281bed771eb66a0c8a7d898430dd5d0eae45',
-            '038c3de773255d3bd7a50e31e58d423baac5c90826a74d75e64b74c95475de1097',
-            '0242c7f7d315095f37ad1421ae0a2fc967d4cbe65b61b079c5395a769436959853',
-            '02a909e70eb03742f12666ebb1f56ac42a5fbaab0c0e8b5b1df4aa9f10f8a09240',
-            '03a26efa12489803c07f3ac2f1dba63812e38f0f6e866ce3ebb34df7de1f458cd2',
-          ];
+          nodesList = ['03ddab321b760433cbf561b615ef62ac7d318630c5f51d523aaf5395b90b751956', '033d92c7bfd213ef1b34c90e985fb5dc77f9ec2409d391492484e57a44c4aca1de', '02ad010dda54253c1eb9efe38b0760657a3b43ecad62198c359c051c9d99d45781', '025196512905b8a3f1597428b867bec63ec9a95e5089eb7dc7e63e2d2691669029', '027c625aa1fbe3768db68ebcb05b53b6dc0ce68b7b54b8900d326d167363e684fe', '03f1629af3101fcc56b7aac2667016be84e3defbf3d0c8719f836c9b41c9a57a43', '02dfb81e2f7a3c4c9e8a51b70ef82b4a24549cc2fab1f5b2fd636501774a918991', '02d01ccf832944c68f10d39006093769c5b8bda886d561b128534e313d729fdb34', '02499ed23027d4698a6904ff4ec1b6085a61f10b9a6937f90438f9947e38e8ea86', '038310e3a786340f2bd7770704c7ccfe560fd163d9a1c99d67894597419d12cbf7', '03e5e9d879b72c7d67ecd483bae023bd33e695bb32b981a4021260f7b9d62bc761', '028d16e1a0ace4c0c0a421536d8d32ce484dfe6e2f726b7b0e7c30f12a195f8cc7'];
           break;
         default:
-          nodesList = [
-            '03fbc17549ec667bccf397ababbcb4cdc0e3394345e4773079ab2774612ec9be61',
-            '03da9a8623241ccf95f19cd645c6cecd4019ac91570e976eb0a128bebbc4d8a437',
-            '03ca5340cf85cb2e7cf076e489f785410838de174e40be62723e8a60972ad75144',
-            '0238bd27f02d67d6c51e269692bc8c9a32357a00e7777cba7f4f1f18a2a700b108',
-            '03f983dcabed6baa1eab5b56c8b2e8fdc846ab3fd931155377897335e85a9fa57c',
-            '03e399589533581e48796e29a825839a010036a61b20744fda929d6709fcbffcc5',
-            '021f5288b5f72c42cd0d8801086af7ce09a816d8ee9a4c47a4b436399b26cb601a',
-            '032b01b7585f781420cd4148841a82831ba37fa952342052cec16750852d4f2dd9',
-            '02848036488d4b8fb1f1c4064261ec36151f43b085f0b51bd239ade3ddfc940c34',
-            '02b6b1640fe029e304c216951af9fbefdb23b0bdc9baaf327540d31b6107841fdf',
-            '03694289827203a5b3156d753071ddd5bf92e371f5a462943f9555eef6d2d6606c',
-            '0283d850db7c3e8ea7cc9c4abc7afaab12bbdf72b677dcba1d608350d2537d7d43',
-            '02521287789f851268a39c9eccc9d6180d2c614315b583c9e6ae0addbd6d79df06',
-            '0258c2a7b7f8af2585b4411b1ec945f70988f30412bb1df179de941f14d0b1bc3e',
-            '03c3389ff1a896f84d921ed01a19fc99c6724ce8dc4b960cd3b7b2362b62cd60d7',
-            '038d118996b3eaa15dcd317b32a539c9ecfdd7698f204acf8a087336af655a9192',
-            '02a928903d93d78877dacc3642b696128a3636e9566dd42d2d132325b2c8891c09',
-            '0328cd17f3a9d3d90b532ade0d1a67e05eb8a51835b3dce0a2e38eac04b5a62a57',
-          ];
+          nodesList = ['03fbc17549ec667bccf397ababbcb4cdc0e3394345e4773079ab2774612ec9be61', '03da9a8623241ccf95f19cd645c6cecd4019ac91570e976eb0a128bebbc4d8a437', '03ca5340cf85cb2e7cf076e489f785410838de174e40be62723e8a60972ad75144', '0238bd27f02d67d6c51e269692bc8c9a32357a00e7777cba7f4f1f18a2a700b108', '03f983dcabed6baa1eab5b56c8b2e8fdc846ab3fd931155377897335e85a9fa57c', '03e399589533581e48796e29a825839a010036a61b20744fda929d6709fcbffcc5', '021f5288b5f72c42cd0d8801086af7ce09a816d8ee9a4c47a4b436399b26cb601a', '032b01b7585f781420cd4148841a82831ba37fa952342052cec16750852d4f2dd9', '02848036488d4b8fb1f1c4064261ec36151f43b085f0b51bd239ade3ddfc940c34', '02b6b1640fe029e304c216951af9fbefdb23b0bdc9baaf327540d31b6107841fdf', '03694289827203a5b3156d753071ddd5bf92e371f5a462943f9555eef6d2d6606c', '0283d850db7c3e8ea7cc9c4abc7afaab12bbdf72b677dcba1d608350d2537d7d43'];
       }
 
       for (let pubKey of nodesList) {

backend/src/api/fiat-conversion.ts

@@ -0,0 +1,123 @@
+import logger from '../logger';
+import * as http from 'http';
+import * as https from 'https';
+import axios, { AxiosResponse } from 'axios';
+import { IConversionRates } from '../mempool.interfaces';
+import config from '../config';
+import backendInfo from './backend-info';
+import { SocksProxyAgent } from 'socks-proxy-agent';
+
+class FiatConversion {
+  private debasingFiatCurrencies = ['AED', 'AUD', 'BDT', 'BHD', 'BMD', 'BRL', 'CAD', 'CHF', 'CLP',
+    'CNY', 'CZK', 'DKK', 'EUR', 'GBP', 'HKD', 'HUF', 'IDR', 'ILS', 'INR', 'JPY', 'KRW', 'KWD',
+    'LKR', 'MMK', 'MXN', 'MYR', 'NGN', 'NOK', 'NZD', 'PHP', 'PKR', 'PLN', 'RUB', 'SAR', 'SEK',
+    'SGD', 'THB', 'TRY', 'TWD', 'UAH', 'USD', 'VND', 'ZAR'];
+  private conversionRates: IConversionRates = {};
+  private ratesChangedCallback: ((rates: IConversionRates) => void) | undefined;
+  public ratesInitialized = false; // If true, it means rates are ready for use
+
+  constructor() {
+    for (const fiat of this.debasingFiatCurrencies) {
+      this.conversionRates[fiat] = 0;
+    }
+  }
+
+  public setProgressChangedCallback(fn: (rates: IConversionRates) => void) {
+    this.ratesChangedCallback = fn;
+  }
+
+  public startService() {
+    const fiatConversionUrl = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.PRICE_DATA_SERVER.TOR_URL : config.PRICE_DATA_SERVER.CLEARNET_URL;
+    logger.info('Starting currency rates service');
+    if (config.SOCKS5PROXY.ENABLED) {
+      logger.info(`Currency rates service will be queried over the Tor network using ${fiatConversionUrl}`);
+    } else {
+      logger.info(`Currency rates service will be queried over clearnet using ${config.PRICE_DATA_SERVER.CLEARNET_URL}`);
+    }
+    setInterval(this.updateCurrency.bind(this), 1000 * config.MEMPOOL.PRICE_FEED_UPDATE_INTERVAL);
+    this.updateCurrency();
+  }
+
+  public getConversionRates() {
+    return this.conversionRates;
+  }
+
+  private async updateCurrency(): Promise<void> {
+    type axiosOptions = {
+      headers: {
+        'User-Agent': string
+      };
+      timeout: number;
+      httpAgent?: http.Agent;
+      httpsAgent?: https.Agent;
+    }
+    const setDelay = (secs: number = 1): Promise<void> => new Promise(resolve => setTimeout(() => resolve(), secs * 1000));
+    const fiatConversionUrl = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.PRICE_DATA_SERVER.TOR_URL : config.PRICE_DATA_SERVER.CLEARNET_URL;
+    const isHTTP = (new URL(fiatConversionUrl).protocol.split(':')[0] === 'http') ? true : false;
+    const axiosOptions: axiosOptions = {
+      headers: {
+        'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
+      },
+      timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000
+    };
+
+    let retry = 0;
+
+    while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
+      try {
+        if (config.SOCKS5PROXY.ENABLED) {
+          let socksOptions: any = {
+            agentOptions: {
+              keepAlive: true,
+            },
+            hostname: config.SOCKS5PROXY.HOST,
+            port: config.SOCKS5PROXY.PORT
+          };
+
+          if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
+            socksOptions.username = config.SOCKS5PROXY.USERNAME;
+            socksOptions.password = config.SOCKS5PROXY.PASSWORD;
+          } else {
+            // Retry with different tor circuits https://stackoverflow.com/a/64960234
+            socksOptions.username = `circuit${retry}`;
+          }
+
+          // Handle proxy agent for onion addresses
+          if (isHTTP) {
+            axiosOptions.httpAgent = new SocksProxyAgent(socksOptions);
+          } else {
+            axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions);
+          }
+        }
+        
+        logger.debug('Querying currency rates service...');
+
+        const response: AxiosResponse = await axios.get(`${fiatConversionUrl}`, axiosOptions);
+
+        if (response.statusText === 'error' || !response.data) {
+          throw new Error(`Could not fetch data from ${fiatConversionUrl}, Error: ${response.status}`);
+        }
+
+        for (const rate of response.data.data) {
+          if (this.debasingFiatCurrencies.includes(rate.currencyCode) && rate.provider === 'Bisq-Aggregate') {
+            this.conversionRates[rate.currencyCode] = Math.round(100 * rate.price) / 100;
+          }
+        }
+
+        this.ratesInitialized = true;
+        logger.debug(`USD Conversion Rate: ${this.conversionRates.USD}`);
+
+        if (this.ratesChangedCallback) {
+          this.ratesChangedCallback(this.conversionRates);
+        }
+        break;
+      } catch (e) {
+        logger.err('Error updating fiat conversion rates: '  + (e instanceof Error ? e.message : e));
+        await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL);
+        retry++;
+      }
+    }
+  }
+}
+
+export default new FiatConversion();

backend/src/api/lightning/clightning/clightning-client.ts

@@ -141,13 +141,13 @@ export default class CLightningClient extends EventEmitter implements AbstractLi
       // main data directory provided, default to using the bitcoin mainnet subdirectory
       // to be removed in v0.2.0
       else if (fExists(rpcPath, 'bitcoin', 'lightning-rpc')) {
-        logger.warn(`${rpcPath}/lightning-rpc is missing, using the bitcoin mainnet subdirectory at ${rpcPath}/bitcoin instead.`, logger.tags.ln)
-        logger.warn(`specifying the main lightning data directory is deprecated, please specify the network directory explicitly.\n`, logger.tags.ln)
+        logger.warn(`[CLightningClient] ${rpcPath}/lightning-rpc is missing, using the bitcoin mainnet subdirectory at ${rpcPath}/bitcoin instead.`)
+        logger.warn(`[CLightningClient] specifying the main lightning data directory is deprecated, please specify the network directory explicitly.\n`)
         rpcPath = path.join(rpcPath, 'bitcoin', 'lightning-rpc')
       }
     }
 
-    logger.debug(`Connecting to ${rpcPath}`, logger.tags.ln);
+    logger.debug(`[CLightningClient] Connecting to ${rpcPath}`);
 
     super();
     this.rpcPath = rpcPath;
@@ -172,19 +172,19 @@ export default class CLightningClient extends EventEmitter implements AbstractLi
 
     this.clientConnectionPromise = new Promise<void>(resolve => {
       _self.client.on('connect', () => {
-        logger.info(`CLightning client connected`, logger.tags.ln);
+        logger.info(`[CLightningClient] Lightning client connected`);
         _self.reconnectWait = 1;
         resolve();
       });
 
       _self.client.on('end', () => {
-        logger.err(`CLightning client connection closed, reconnecting`, logger.tags.ln);
+        logger.err('[CLightningClient] Lightning client connection closed, reconnecting');
         _self.increaseWaitTime();
         _self.reconnect();
       });
 
       _self.client.on('error', error => {
-        logger.err(`CLightning client connection error: ${error}`, logger.tags.ln);
+        logger.err(`[CLightningClient] Lightning client connection error: ${error}`);
         _self.increaseWaitTime();
         _self.reconnect();
       });
@@ -196,6 +196,7 @@ export default class CLightningClient extends EventEmitter implements AbstractLi
         return;
       }
       const data = JSON.parse(line);
+      // logger.debug(`[CLightningClient] #${data.id} <-- ${JSON.stringify(data.error || data.result)}`);
       _self.emit('res:' + data.id, data);
     });
   }
@@ -216,7 +217,7 @@ export default class CLightningClient extends EventEmitter implements AbstractLi
     }
 
     this.reconnectTimeout = setTimeout(() => {
-      logger.debug(`Trying to reconnect...`, logger.tags.ln);
+      logger.debug('[CLightningClient] Trying to reconnect...');
 
       _self.client.connect(_self.rpcPath);
       _self.reconnectTimeout = null;
@@ -234,6 +235,7 @@ export default class CLightningClient extends EventEmitter implements AbstractLi
       id: '' + callInt
     };
 
+    // logger.debug(`[CLightningClient] #${callInt} --> ${method} ${args}`);
 
     // Wait for the client to connect
     return this.clientConnectionPromise

backend/src/api/lightning/clightning/clightning-convert.ts

@@ -2,7 +2,6 @@ import { ILightningApi } from '../lightning-api.interface';
 import FundingTxFetcher from '../../../tasks/lightning/sync-tasks/funding-tx-fetcher';
 import logger from '../../../logger';
 import { Common } from '../../common';
-import config from '../../../config';
 
 /**
  * Convert a clightning "listnode" entry to a lnd node entry
@@ -41,7 +40,7 @@ export function convertNode(clNode: any): ILightningApi.Node {
  * Convert clightning "listchannels" response to lnd "describegraph.edges" format
  */
 export async function convertAndmergeBidirectionalChannels(clChannels: any[]): Promise<ILightningApi.Channel[]> {
-  logger.debug(`Converting clightning nodes and channels to lnd graph format`, logger.tags.ln);
+  logger.info('Converting clightning nodes and channels to lnd graph format');
 
   let loggerTimer = new Date().getTime() / 1000;
   let channelProcessed = 0;
@@ -55,17 +54,16 @@ export async function convertAndmergeBidirectionalChannels(clChannels: any[]): P
       clChannelsDict[clChannel.short_channel_id] = clChannel;
       clChannelsDictCount[clChannel.short_channel_id] = 1;
     } else {
-      const fullChannel = await buildFullChannel(clChannel, clChannelsDict[clChannel.short_channel_id]);
-      if (fullChannel !== null) {
-        consolidatedChannelList.push(fullChannel);
-        delete clChannelsDict[clChannel.short_channel_id];
-        clChannelsDictCount[clChannel.short_channel_id]++;
-      }
+      consolidatedChannelList.push(
+        await buildFullChannel(clChannel, clChannelsDict[clChannel.short_channel_id])
+      );
+      delete clChannelsDict[clChannel.short_channel_id];
+      clChannelsDictCount[clChannel.short_channel_id]++;
     }
 
     const elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer);
-    if (elapsedSeconds > config.LIGHTNING.LOGGER_UPDATE_INTERVAL) {
-      logger.info(`Building complete channels from clightning output. Channels processed: ${channelProcessed + 1} of ${clChannels.length}`, logger.tags.ln);
+    if (elapsedSeconds > 10) {
+      logger.info(`Building complete channels from clightning output. Channels processed: ${channelProcessed + 1} of ${clChannels.length}`);
       loggerTimer = new Date().getTime() / 1000;
     }
 
@@ -75,13 +73,10 @@ export async function convertAndmergeBidirectionalChannels(clChannels: any[]): P
   channelProcessed = 0;
   const keys = Object.keys(clChannelsDict);
   for (const short_channel_id of keys) {
-    const incompleteChannel = await buildIncompleteChannel(clChannelsDict[short_channel_id]);
-    if (incompleteChannel !== null) {
-      consolidatedChannelList.push(incompleteChannel);
-    }
+    consolidatedChannelList.push(await buildIncompleteChannel(clChannelsDict[short_channel_id]));
 
     const elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer);
-    if (elapsedSeconds > config.LIGHTNING.LOGGER_UPDATE_INTERVAL) {
+    if (elapsedSeconds > 10) {
       logger.info(`Building partial channels from clightning output. Channels processed: ${channelProcessed + 1} of ${keys.length}`);
       loggerTimer = new Date().getTime() / 1000;
     }
@@ -96,19 +91,16 @@ export async function convertAndmergeBidirectionalChannels(clChannels: any[]): P
  * Convert two clightning "getchannels" entries into a full a lnd "describegraph.edges" format
  * In this case, clightning knows the channel policy for both nodes
  */
-async function buildFullChannel(clChannelA: any, clChannelB: any): Promise<ILightningApi.Channel | null> {
+async function buildFullChannel(clChannelA: any, clChannelB: any): Promise<ILightningApi.Channel> {
   const lastUpdate = Math.max(clChannelA.last_update ?? 0, clChannelB.last_update ?? 0);
 
   const tx = await FundingTxFetcher.$fetchChannelOpenTx(clChannelA.short_channel_id);
-  if (!tx) {
-    return null;
-  }
   const parts = clChannelA.short_channel_id.split('x');
   const outputIdx = parts[2];
 
   return {
     channel_id: Common.channelShortIdToIntegerId(clChannelA.short_channel_id),
-    capacity: (clChannelA.amount_msat / 1000).toString(),
+    capacity: clChannelA.satoshis,
     last_update: lastUpdate,
     node1_policy: convertPolicy(clChannelA),
     node2_policy: convertPolicy(clChannelB),
@@ -122,17 +114,14 @@ async function buildFullChannel(clChannelA: any, clChannelB: any): Promise<ILigh
  * Convert one clightning "getchannels" entry into a full a lnd "describegraph.edges" format
  * In this case, clightning knows the channel policy of only one node
  */
-async function buildIncompleteChannel(clChannel: any): Promise<ILightningApi.Channel | null> {
+async function buildIncompleteChannel(clChannel: any): Promise<ILightningApi.Channel> {
   const tx = await FundingTxFetcher.$fetchChannelOpenTx(clChannel.short_channel_id);
-  if (!tx) {
-    return null;
-  }
   const parts = clChannel.short_channel_id.split('x');
   const outputIdx = parts[2];
 
   return {
     channel_id: Common.channelShortIdToIntegerId(clChannel.short_channel_id),
-    capacity: (clChannel.amount_msat / 1000).toString(),
+    capacity: clChannel.satoshis,
     last_update: clChannel.last_update ?? 0,
     node1_policy: convertPolicy(clChannel),
     node2_policy: null,
@@ -148,8 +137,8 @@ async function buildIncompleteChannel(clChannel: any): Promise<ILightningApi.Cha
 function convertPolicy(clChannel: any): ILightningApi.RoutingPolicy {
   return {
     time_lock_delta: clChannel.delay,
-    min_htlc: clChannel.htlc_minimum_msat.toString(),
-    max_htlc_msat: clChannel.htlc_maximum_msat.toString(),
+    min_htlc: clChannel.htlc_minimum_msat.slice(0, -4),
+    max_htlc_msat: clChannel.htlc_maximum_msat.slice(0, -4),
     fee_base_msat: clChannel.base_fee_millisatoshi,
     fee_rate_milli_msat: clChannel.fee_per_millionth,
     disabled: !clChannel.active,

backend/src/api/lightning/lightning-api.interface.ts

@@ -21,7 +21,7 @@ export namespace ILightningApi {
   export interface Channel {
     channel_id: string;
     chan_point: string;
-    last_update: number | null;
+    last_update: number;
     node1_pub: string;
     node2_pub: string;
     capacity: string;
@@ -36,11 +36,11 @@ export namespace ILightningApi {
     fee_rate_milli_msat: string;
     disabled: boolean;
     max_htlc_msat: string;
-    last_update: number | null;
+    last_update: number;
   }
 
   export interface Node {
-    last_update: number | null;
+    last_update: number;
     pub_key: string;
     alias: string;
     addresses: {

backend/src/api/lightning/lnd/lnd-api.ts

@@ -4,29 +4,21 @@ import * as fs from 'fs';
 import { AbstractLightningApi } from '../lightning-api-abstract-factory';
 import { ILightningApi } from '../lightning-api.interface';
 import config from '../../../config';
-import logger from '../../../logger';
 
 class LndApi implements AbstractLightningApi {
   axiosConfig: AxiosRequestConfig = {};
 
   constructor() {
-    if (!config.LIGHTNING.ENABLED) {
-      return;
-    }
-    try {
+    if (config.LIGHTNING.ENABLED) {
       this.axiosConfig = {
         headers: {
-          'Grpc-Metadata-macaroon': fs.readFileSync(config.LND.MACAROON_PATH).toString('hex'),
+          'Grpc-Metadata-macaroon': fs.readFileSync(config.LND.MACAROON_PATH).toString('hex')
         },
         httpsAgent: new Agent({
           ca: fs.readFileSync(config.LND.TLS_CERT_PATH)
         }),
-        timeout: config.LND.TIMEOUT
+        timeout: 10000
       };
-    } catch (e) {
-      config.LIGHTNING.ENABLED = false;
-      logger.updateNetwork();
-      logger.err(`Could not initialize LND Macaroon/TLS Cert. Disabling LIGHTNING. ` + (e instanceof Error ? e.message : e));
     }
   }
 

backend/src/api/mempool-blocks.ts

@@ -33,7 +33,7 @@ class MempoolBlocks {
     return this.mempoolBlockDeltas;
   }
 
-  public updateMempoolBlocks(memPool: { [txid: string]: TransactionExtended }, saveResults: boolean = false): MempoolBlockWithTransactions[] {
+  public updateMempoolBlocks(memPool: { [txid: string]: TransactionExtended }): void {
     const latestMempool = memPool;
     const memPoolArray: TransactionExtended[] = [];
     for (const i in latestMempool) {
@@ -75,14 +75,10 @@ class MempoolBlocks {
     logger.debug('Mempool blocks calculated in ' + time / 1000 + ' seconds');
 
     const blocks = this.calculateMempoolBlocks(memPoolArray, this.mempoolBlocks);
+    const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, blocks);
 
-    if (saveResults) {
-      const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, blocks);
-      this.mempoolBlocks = blocks;
-      this.mempoolBlockDeltas = deltas;
-    }
-
-    return blocks;
+    this.mempoolBlocks = blocks;
+    this.mempoolBlockDeltas = deltas;
   }
 
   private calculateMempoolBlocks(transactionsSorted: TransactionExtended[], prevBlocks: MempoolBlockWithTransactions[]): MempoolBlockWithTransactions[] {
@@ -97,14 +93,14 @@ class MempoolBlocks {
         blockSize += tx.size;
         transactions.push(tx);
       } else {
-        mempoolBlocks.push(this.dataToMempoolBlocks(transactions, mempoolBlocks.length));
+        mempoolBlocks.push(this.dataToMempoolBlocks(transactions, blockSize, blockWeight, mempoolBlocks.length));
         blockWeight = tx.weight;
         blockSize = tx.size;
         transactions = [tx];
       }
     });
     if (transactions.length) {
-      mempoolBlocks.push(this.dataToMempoolBlocks(transactions, mempoolBlocks.length));
+      mempoolBlocks.push(this.dataToMempoolBlocks(transactions, blockSize, blockWeight, mempoolBlocks.length));
     }
 
     return mempoolBlocks;
@@ -147,11 +143,11 @@ class MempoolBlocks {
     return mempoolBlockDeltas;
   }
 
-  public async makeBlockTemplates(newMempool: { [txid: string]: TransactionExtended }, saveResults: boolean = false): Promise<MempoolBlockWithTransactions[]> {
+  public async makeBlockTemplates(newMempool: { [txid: string]: TransactionExtended }): Promise<void> {
     // prepare a stripped down version of the mempool with only the minimum necessary data
     // to reduce the overhead of passing this data to the worker thread
     const strippedMempool: { [txid: string]: ThreadTransaction } = {};
-    Object.values(newMempool).filter(tx => !tx.deleteAfter).forEach(entry => {
+    Object.values(newMempool).forEach(entry => {
       strippedMempool[entry.txid] = {
         txid: entry.txid,
         fee: entry.fee,
@@ -186,30 +182,21 @@ class MempoolBlocks {
         this.txSelectionWorker?.once('error', reject);
       });
       this.txSelectionWorker.postMessage({ type: 'set', mempool: strippedMempool });
-      let { blocks, clusters } = await workerResultPromise;
-      // filter out stale transactions
-      const unfilteredCount = blocks.reduce((total, block) => { return total + block.length; }, 0);
-      blocks = blocks.map(block => block.filter(tx => (tx.txid && tx.txid in newMempool)));
-      const filteredCount = blocks.reduce((total, block) => { return total + block.length; }, 0);
-      if (filteredCount < unfilteredCount) {
-        logger.warn(`tx selection worker thread returned ${unfilteredCount - filteredCount} stale transactions from makeBlockTemplates`);
-      }
+      const { blocks, clusters } = await workerResultPromise;
+
+      this.processBlockTemplates(newMempool, blocks, clusters);
 
       // clean up thread error listener
       this.txSelectionWorker?.removeListener('error', threadErrorListener);
-
-      return this.processBlockTemplates(newMempool, blocks, clusters, saveResults);
     } catch (e) {
       logger.err('makeBlockTemplates failed. ' + (e instanceof Error ? e.message : e));
     }
-    return this.mempoolBlocks;
   }
 
-  public async updateBlockTemplates(newMempool: { [txid: string]: TransactionExtended }, added: TransactionExtended[], removed: string[], saveResults: boolean = false): Promise<void> {
+  public async updateBlockTemplates(newMempool: { [txid: string]: TransactionExtended }, added: TransactionExtended[], removed: string[]): Promise<void> {
     if (!this.txSelectionWorker) {
       // need to reset the worker
-      this.makeBlockTemplates(newMempool, saveResults);
-      return;
+      return this.makeBlockTemplates(newMempool);
     }
     // prepare a stripped down version of the mempool with only the minimum necessary data
     // to reduce the overhead of passing this data to the worker thread
@@ -235,29 +222,22 @@ class MempoolBlocks {
         this.txSelectionWorker?.once('error', reject);
       });
       this.txSelectionWorker.postMessage({ type: 'update', added: addedStripped, removed });
-      let { blocks, clusters } = await workerResultPromise;
-      // filter out stale transactions
-      const unfilteredCount = blocks.reduce((total, block) => { return total + block.length; }, 0);
-      blocks = blocks.map(block => block.filter(tx => (tx.txid && tx.txid in newMempool)));
-      const filteredCount = blocks.reduce((total, block) => { return total + block.length; }, 0);
-      if (filteredCount < unfilteredCount) {
-        logger.warn(`tx selection worker thread returned ${unfilteredCount - filteredCount} stale transactions from updateBlockTemplates`);
-      }
+      const { blocks, clusters } = await workerResultPromise;
+
+      this.processBlockTemplates(newMempool, blocks, clusters);
 
       // clean up thread error listener
       this.txSelectionWorker?.removeListener('error', threadErrorListener);
-
-      this.processBlockTemplates(newMempool, blocks, clusters, saveResults);
     } catch (e) {
       logger.err('updateBlockTemplates failed. ' + (e instanceof Error ? e.message : e));
     }
   }
 
-  private processBlockTemplates(mempool, blocks, clusters, saveResults): MempoolBlockWithTransactions[] {
+  private processBlockTemplates(mempool, blocks, clusters): void {
     // update this thread's mempool with the results
     blocks.forEach(block => {
       block.forEach(tx => {
-        if (tx.txid && tx.txid in mempool) {
+        if (tx.txid in mempool) {
           if (tx.effectiveFeePerVsize != null) {
             mempool[tx.txid].effectiveFeePerVsize = tx.effectiveFeePerVsize;
           }
@@ -267,10 +247,6 @@ class MempoolBlocks {
             const cluster = clusters[tx.cpfpRoot];
             let matched = false;
             cluster.forEach(txid => {
-              if (!txid || !mempool[txid]) {
-                logger.warn('projected transaction ancestor missing from mempool cache');
-                return;
-              }
               if (txid === tx.txid) {
                 matched = true;
               } else {
@@ -291,8 +267,6 @@ class MempoolBlocks {
             mempool[tx.txid].bestDescendant = null;
           }
           mempool[tx.txid].cpfpChecked = tx.cpfpChecked;
-        } else {
-          logger.warn('projected transaction missing from mempool cache');
         }
       });
     });
@@ -301,29 +275,27 @@ class MempoolBlocks {
     const mempoolBlocks = blocks.map((transactions, blockIndex) => {
       return this.dataToMempoolBlocks(transactions.map(tx => {
         return mempool[tx.txid] || null;
-      }).filter(tx => !!tx), blockIndex);
+      }).filter(tx => !!tx), undefined, undefined, blockIndex);
     });
 
-    if (saveResults) {
-      const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, mempoolBlocks);
-      this.mempoolBlocks = mempoolBlocks;
-      this.mempoolBlockDeltas = deltas;
-    }
+    const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, mempoolBlocks);
 
-    return mempoolBlocks;
+    this.mempoolBlocks = mempoolBlocks;
+    this.mempoolBlockDeltas = deltas;
   }
 
-  private dataToMempoolBlocks(transactions: TransactionExtended[], blocksIndex: number): MempoolBlockWithTransactions {
-    let totalSize = 0;
-    let totalWeight = 0;
-    const fitTransactions: TransactionExtended[] = [];
-    transactions.forEach(tx => {
-      totalSize += tx.size;
-      totalWeight += tx.weight;
-      if ((totalWeight + tx.weight) <= config.MEMPOOL.BLOCK_WEIGHT_UNITS * 1.2) {
-        fitTransactions.push(tx);
-      }
-    });
+  private dataToMempoolBlocks(transactions: TransactionExtended[],
+    blockSize: number | undefined, blockWeight: number | undefined, blocksIndex: number): MempoolBlockWithTransactions {
+    let totalSize = blockSize || 0;
+    let totalWeight = blockWeight || 0;
+    if (blockSize === undefined && blockWeight === undefined) {
+      totalSize = 0;
+      totalWeight = 0;
+      transactions.forEach(tx => {
+        totalSize += tx.size;
+        totalWeight += tx.weight;
+      });
+    }
     let rangeLength = 4;
     if (blocksIndex === 0) {
       rangeLength = 8;
@@ -341,7 +313,7 @@ class MempoolBlocks {
       medianFee: Common.percentile(transactions.map((tx) => tx.effectiveFeePerVsize), config.MEMPOOL.RECOMMENDED_FEE_PERCENTILE),
       feeRange: Common.getFeesInRange(transactions, rangeLength),
       transactionIds: transactions.map((tx) => tx.txid),
-      transactions: fitTransactions.map((tx) => Common.stripTransaction(tx)),
+      transactions: transactions.map((tx) => Common.stripTransaction(tx)),
     };
   }
 }

backend/src/api/mempool.ts

@@ -31,13 +31,9 @@ class Mempool {
   private mempoolProtection = 0;
   private latestTransactions: any[] = [];
 
-  private ESPLORA_MISSING_TX_WARNING_THRESHOLD = 100; 
-  private SAMPLE_TIME = 10000; // In ms
-  private timer = new Date().getTime();
-  private missingTxCount = 0;
-
   constructor() {
     setInterval(this.updateTxPerSecond.bind(this), 1000);
+    setInterval(this.deleteExpiredTransactions.bind(this), 20000);
   }
 
   /**
@@ -132,16 +128,6 @@ class Mempool {
       loadingIndicators.setProgress('mempool', Object.keys(this.mempoolCache).length / transactions.length * 100);
     }
 
-    // https://github.com/mempool/mempool/issues/3283
-    const logEsplora404 = (missingTxCount, threshold, time) => {
-      const log = `In the past ${time / 1000} seconds, esplora tx API replied ${missingTxCount} times with a 404 error code while updating nodejs backend mempool`;
-      if (missingTxCount >= threshold) {
-        logger.warn(log);
-      } else if (missingTxCount > 0) {
-        logger.debug(log);
-      }
-    };
-
     for (const txid of transactions) {
       if (!this.mempoolCache[txid]) {
         try {
@@ -156,10 +142,7 @@ class Mempool {
           }
           hasChange = true;
           newTransactions.push(transaction);
-        } catch (e: any) {
-          if (config.MEMPOOL.BACKEND === 'esplora' && e.response?.status === 404) {
-            this.missingTxCount++;
-          }
+        } catch (e) {
           logger.debug(`Error finding transaction '${txid}' in the mempool: ` + (e instanceof Error ? e.message : e));
         }
       }
@@ -169,14 +152,6 @@ class Mempool {
       }
     }
 
-    // Reset esplora 404 counter and log a warning if needed
-    const elapsedTime = new Date().getTime() - this.timer;
-    if (elapsedTime > this.SAMPLE_TIME) {
-      logEsplora404(this.missingTxCount, this.ESPLORA_MISSING_TX_WARNING_THRESHOLD, elapsedTime);
-      this.timer = new Date().getTime();
-      this.missingTxCount = 0;
-    }
-
     // Prevent mempool from clear on bitcoind restart by delaying the deletion
     if (this.mempoolProtection === 0
       && currentMempoolSize > 20000
@@ -235,7 +210,7 @@ class Mempool {
     for (const rbfTransaction in rbfTransactions) {
       if (this.mempoolCache[rbfTransaction]) {
         // Store replaced transactions
-        rbfCache.add(this.mempoolCache[rbfTransaction], rbfTransactions[rbfTransaction].txid);
+        rbfCache.add(rbfTransaction, rbfTransactions[rbfTransaction].txid);
         // Erase the replaced transactions from the local mempool
         delete this.mempoolCache[rbfTransaction];
       }
@@ -255,13 +230,12 @@ class Mempool {
     }
   }
 
-  public deleteExpiredTransactions() {
+  private deleteExpiredTransactions() {
     const now = new Date().getTime();
     for (const tx in this.mempoolCache) {
       const lazyDeleteAt = this.mempoolCache[tx].deleteAfter;
       if (lazyDeleteAt && lazyDeleteAt < now) {
         delete this.mempoolCache[tx];
-        rbfCache.evict(tx);
       }
     }
   }

backend/src/api/mining/mining-routes.ts

@@ -1,13 +1,13 @@
 import { Application, Request, Response } from 'express';
 import config from "../../config";
 import logger from '../../logger';
+import audits from '../audit';
 import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository';
 import BlocksRepository from '../../repositories/BlocksRepository';
 import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjustmentsRepository';
 import HashratesRepository from '../../repositories/HashratesRepository';
 import bitcoinClient from '../bitcoin/bitcoin-client';
 import mining from "./mining";
-import PricesRepository from '../../repositories/PricesRepository';
 
 class MiningRoutes {
   public initRoutes(app: Application) {
@@ -32,27 +32,9 @@ class MiningRoutes {
       .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/score/:hash', this.$getBlockAuditScore)
       .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/:hash', this.$getBlockAudit)
       .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/timestamp/:timestamp', this.$getHeightFromTimestamp)
-      .get(config.MEMPOOL.API_URL_PREFIX + 'historical-price', this.$getHistoricalPrice)
     ;
   }
 
-  private async $getHistoricalPrice(req: Request, res: Response): Promise<void> {
-    try {
-      res.header('Pragma', 'public');
-      res.header('Cache-control', 'public');
-      res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
-      if (req.query.timestamp) {
-        res.status(200).send(await PricesRepository.$getNearestHistoricalPrice(
-          parseInt(<string>req.query.timestamp ?? 0, 10)
-        ));
-      } else {
-        res.status(200).send(await PricesRepository.$getHistoricalPrices());
-      }
-    } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
-    }
-  }
-
   private async $getPool(req: Request, res: Response): Promise<void> {
     try {
       const stats = await mining.$getPoolStat(req.params.slug);
@@ -263,7 +245,7 @@ class MiningRoutes {
       const audit = await BlocksAuditsRepository.$getBlockAudit(req.params.hash);
 
       if (!audit) {
-        res.status(204).send(`This block has not been audited.`);
+        res.status(404).send(`This block has not been audited.`);
         return;
       }
 

backend/src/api/mining/mining.ts

@@ -11,14 +11,12 @@ import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjust
 import config from '../../config';
 import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository';
 import PricesRepository from '../../repositories/PricesRepository';
-import { bitcoinCoreApi } from '../bitcoin/bitcoin-api-factory';
-import { IEsploraApi } from '../bitcoin/esplora-api.interface';
-import database from '../../database';
 
 class Mining {
-  private blocksPriceIndexingRunning = false;
-  public lastHashrateIndexingDate: number | null = null;
-  public lastWeeklyHashrateIndexingDate: number | null = null;
+  blocksPriceIndexingRunning = false;
+
+  constructor() {
+  }
 
   /**
    * Get historical block predictions match rate
@@ -102,7 +100,6 @@ class Mining {
         rank: rank++,
         emptyBlocks: emptyBlocksCount.length > 0 ? emptyBlocksCount[0]['count'] : 0,
         slug: poolInfo.slug,
-        avgMatchRate: poolInfo.avgMatchRate !== null ? Math.round(100 * poolInfo.avgMatchRate) / 100 : null,
       };
       poolsStats.push(poolStat);
     });
@@ -118,7 +115,7 @@ class Mining {
       poolsStatistics['lastEstimatedHashrate'] = await bitcoinClient.getNetworkHashPs(totalBlock24h);
     } catch (e) {
       poolsStatistics['lastEstimatedHashrate'] = 0;
-      logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate', logger.tags.mining);
+      logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate');
     }
 
     return poolsStatistics;
@@ -142,14 +139,11 @@ class Mining {
     const blockCount1w: number = await BlocksRepository.$blockCount(pool.id, '1w');
     const totalBlock1w: number = await BlocksRepository.$blockCount(null, '1w');
 
-    const avgHealth = await BlocksRepository.$getAvgBlockHealthPerPoolId(pool.id);    
-    const totalReward = await BlocksRepository.$getTotalRewardForPoolId(pool.id);    
-
     let currentEstimatedHashrate = 0;
     try {
       currentEstimatedHashrate = await bitcoinClient.getNetworkHashPs(totalBlock24h);
     } catch (e) {
-      logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate', logger.tags.mining);
+      logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate');
     }
 
     return {
@@ -166,8 +160,6 @@ class Mining {
       },
       estimatedHashrate: currentEstimatedHashrate * (blockCount24h / totalBlock24h),
       reportedHashrate: null,
-      avgBlockHealth: avgHealth,
-      totalReward: totalReward,
     };
   }
 
@@ -179,26 +171,25 @@ class Mining {
   }
 
   /**
-   * Generate weekly mining pool hashrate history
+   * [INDEXING] Generate weekly mining pool hashrate history
    */
   public async $generatePoolHashrateHistory(): Promise<void> {
     const now = new Date();
+    const lastestRunDate = await HashratesRepository.$getLatestRun('last_weekly_hashrates_indexing');
 
     // Run only if:
-    // * this.lastWeeklyHashrateIndexingDate is set to null (node backend restart, reorg)
+    // * lastestRunDate is set to 0 (node backend restart, reorg)
     // * we started a new week (around Monday midnight)
-    const runIndexing = this.lastWeeklyHashrateIndexingDate === null ||
-      now.getUTCDay() === 1 && this.lastWeeklyHashrateIndexingDate !== now.getUTCDate();
+    const runIndexing = lastestRunDate === 0 || now.getUTCDay() === 1 && lastestRunDate !== now.getUTCDate();
     if (!runIndexing) {
-      logger.debug(`Pool hashrate history indexing is up to date, nothing to do`, logger.tags.mining);
       return;
     }
 
     try {
       const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp;
 
-      const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0));
-      const genesisTimestamp = genesisBlock.timestamp * 1000;
+      const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0));
+      const genesisTimestamp = genesisBlock.time * 1000;
 
       const indexedTimestamp = await HashratesRepository.$getWeeklyHashrateTimestamps();
       const hashrates: any[] = [];
@@ -214,7 +205,7 @@ class Mining {
       const startedAt = new Date().getTime() / 1000;
       let timer = new Date().getTime() / 1000;
 
-      logger.debug(`Indexing weekly mining pool hashrate`, logger.tags.mining);
+      logger.debug(`Indexing weekly mining pool hashrate`);
       loadingIndicators.setProgress('weekly-hashrate-indexing', 0);
 
       while (toTimestamp > genesisTimestamp && toTimestamp > oldestConsecutiveBlockTimestamp) {
@@ -251,7 +242,7 @@ class Mining {
             });
           }
 
-          newlyIndexed += hashrates.length / Math.max(1, pools.length);
+          newlyIndexed += hashrates.length;
           await HashratesRepository.$saveHashrates(hashrates);
           hashrates.length = 0;
         }
@@ -262,7 +253,7 @@ class Mining {
           const weeksPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds));
           const progress = Math.round(totalIndexed / totalWeekIndexed * 10000) / 100;
           const formattedDate = new Date(fromTimestamp).toUTCString();
-          logger.debug(`Getting weekly pool hashrate for ${formattedDate} | ~${weeksPerSeconds.toFixed(2)} weeks/sec | total: ~${totalIndexed}/${Math.round(totalWeekIndexed)} (${progress}%) | elapsed: ${runningFor} seconds`, logger.tags.mining);
+          logger.debug(`Getting weekly pool hashrate for ${formattedDate} | ~${weeksPerSeconds.toFixed(2)} weeks/sec | total: ~${totalIndexed}/${Math.round(totalWeekIndexed)} (${progress}%) | elapsed: ${runningFor} seconds`);
           timer = new Date().getTime() / 1000;
           indexedThisRun = 0;
           loadingIndicators.setProgress('weekly-hashrate-indexing', progress, false);
@@ -272,36 +263,36 @@ class Mining {
         ++indexedThisRun;
         ++totalIndexed;
       }
-      this.lastWeeklyHashrateIndexingDate = new Date().getUTCDate();
+      await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', new Date().getUTCDate());
       if (newlyIndexed > 0) {
-        logger.info(`Weekly mining pools hashrates indexing completed: indexed ${newlyIndexed} weeks`, logger.tags.mining);
+        logger.notice(`Weekly mining pools hashrates indexing completed: indexed ${newlyIndexed}`);
       } else {
-        logger.debug(`Weekly mining pools hashrates indexing completed: indexed ${newlyIndexed} weeks`, logger.tags.mining);
+        logger.debug(`Weekly mining pools hashrates indexing completed: indexed ${newlyIndexed}`);
       }
       loadingIndicators.setProgress('weekly-hashrate-indexing', 100);
     } catch (e) {
       loadingIndicators.setProgress('weekly-hashrate-indexing', 100);
-      logger.err(`Weekly mining pools hashrates indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`, logger.tags.mining);
+      logger.err(`Weekly mining pools hashrates indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`);
       throw e;
     }
   }
 
   /**
-   * Generate daily hashrate data
+   * [INDEXING] Generate daily hashrate data
    */
   public async $generateNetworkHashrateHistory(): Promise<void> {
     // We only run this once a day around midnight
-    const today = new Date().getUTCDate();
-    if (today === this.lastHashrateIndexingDate) {
-      logger.debug(`Network hashrate history indexing is up to date, nothing to do`, logger.tags.mining);
+    const latestRunDate = await HashratesRepository.$getLatestRun('last_hashrates_indexing');
+    const now = new Date().getUTCDate();
+    if (now === latestRunDate) {
       return;
     }
 
     const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp;
 
     try {
-      const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0));
-      const genesisTimestamp = genesisBlock.timestamp * 1000;
+      const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0));
+      const genesisTimestamp = genesisBlock.time * 1000;
       const indexedTimestamp = (await HashratesRepository.$getRawNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp);
       const lastMidnight = this.getDateMidnight(new Date());
       let toTimestamp = Math.round(lastMidnight.getTime());
@@ -314,7 +305,7 @@ class Mining {
       const startedAt = new Date().getTime() / 1000;
       let timer = new Date().getTime() / 1000;
 
-      logger.debug(`Indexing daily network hashrate`, logger.tags.mining);
+      logger.debug(`Indexing daily network hashrate`);
       loadingIndicators.setProgress('daily-hashrate-indexing', 0);
 
       while (toTimestamp > genesisTimestamp && toTimestamp > oldestConsecutiveBlockTimestamp) {
@@ -352,7 +343,7 @@ class Mining {
           const daysPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds));
           const progress = Math.round(totalIndexed / totalDayIndexed * 10000) / 100;
           const formattedDate = new Date(fromTimestamp).toUTCString();
-          logger.debug(`Getting network daily hashrate for ${formattedDate} | ~${daysPerSeconds.toFixed(2)} days/sec | total: ~${totalIndexed}/${Math.round(totalDayIndexed)} (${progress}%) | elapsed: ${runningFor} seconds`, logger.tags.mining);
+          logger.debug(`Getting network daily hashrate for ${formattedDate} | ~${daysPerSeconds.toFixed(2)} days/sec | total: ~${totalIndexed}/${Math.round(totalDayIndexed)} (${progress}%) | elapsed: ${runningFor} seconds`);
           timer = new Date().getTime() / 1000;
           indexedThisRun = 0;
           loadingIndicators.setProgress('daily-hashrate-indexing', progress);
@@ -377,16 +368,16 @@ class Mining {
       newlyIndexed += hashrates.length;
       await HashratesRepository.$saveHashrates(hashrates);
 
-      this.lastHashrateIndexingDate = new Date().getUTCDate();
+      await HashratesRepository.$setLatestRun('last_hashrates_indexing', new Date().getUTCDate());
       if (newlyIndexed > 0) {
-        logger.info(`Daily network hashrate indexing completed: indexed ${newlyIndexed} days`, logger.tags.mining);
+        logger.notice(`Daily network hashrate indexing completed: indexed ${newlyIndexed} days`);
       } else {
-        logger.debug(`Daily network hashrate indexing completed: indexed ${newlyIndexed} days`, logger.tags.mining);
+        logger.debug(`Daily network hashrate indexing completed: indexed ${newlyIndexed} days`);
       }
       loadingIndicators.setProgress('daily-hashrate-indexing', 100);
     } catch (e) {
       loadingIndicators.setProgress('daily-hashrate-indexing', 100);
-      logger.err(`Daily network hashrate indexing failed. Trying again later. Reason: ${(e instanceof Error ? e.message : e)}`, logger.tags.mining);
+      logger.err(`Daily network hashrate indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`);
       throw e;
     }
   }
@@ -402,13 +393,13 @@ class Mining {
     }
 
     const blocks: any = await BlocksRepository.$getBlocksDifficulty();
-    const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0));
+    const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0));
     let currentDifficulty = genesisBlock.difficulty;
     let totalIndexed = 0;
 
     if (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT === -1 && indexedHeights[0] !== true) {
       await DifficultyAdjustmentsRepository.$saveAdjustments({
-        time: genesisBlock.timestamp,
+        time: genesisBlock.time,
         height: 0,
         difficulty: currentDifficulty,
         adjustment: 0.0,
@@ -452,22 +443,22 @@ class Mining {
       const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
       if (elapsedSeconds > 5) {
         const progress = Math.round(totalBlockChecked / blocks.length * 100);
-        logger.info(`Indexing difficulty adjustment at block #${block.height} | Progress: ${progress}%`, logger.tags.mining);
+        logger.info(`Indexing difficulty adjustment at block #${block.height} | Progress: ${progress}%`);
         timer = new Date().getTime() / 1000;
       }
     }
 
     if (totalIndexed > 0) {
-      logger.info(`Indexed ${totalIndexed} difficulty adjustments`, logger.tags.mining);
+      logger.notice(`Indexed ${totalIndexed} difficulty adjustments`);
     } else {
-      logger.debug(`Indexed ${totalIndexed} difficulty adjustments`, logger.tags.mining);
+      logger.debug(`Indexed ${totalIndexed} difficulty adjustments`);
     }
   }
 
   /**
    * Create a link between blocks and the latest price at when they were mined
    */
-  public async $indexBlockPrices(): Promise<void> {
+  public async $indexBlockPrices() {
     if (this.blocksPriceIndexingRunning === true) {
       return;
     }
@@ -505,7 +496,7 @@ class Mining {
           if (blocksWithoutPrices.length > 200000) {
             logStr += ` | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`;
           }
-          logger.debug(logStr, logger.tags.mining);
+          logger.debug(logStr);
           await BlocksRepository.$saveBlockPrices(blocksPrices);
           blocksPrices.length = 0;
         }
@@ -517,7 +508,7 @@ class Mining {
         if (blocksWithoutPrices.length > 200000) {
           logStr += ` | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`;
         }
-        logger.debug(logStr, logger.tags.mining);
+        logger.debug(logStr);
         await BlocksRepository.$saveBlockPrices(blocksPrices);
       }
     } catch (e) {
@@ -528,41 +519,6 @@ class Mining {
     this.blocksPriceIndexingRunning = false;
   }
 
-  /**
-   * Index core coinstatsindex
-   */
-  public async $indexCoinStatsIndex(): Promise<void> {
-    let timer = new Date().getTime() / 1000;
-    let totalIndexed = 0;
-
-    const blockchainInfo = await bitcoinClient.getBlockchainInfo();
-    let currentBlockHeight = blockchainInfo.blocks;
-
-    while (currentBlockHeight > 0) {
-      const indexedBlocks = await BlocksRepository.$getBlocksMissingCoinStatsIndex(
-        currentBlockHeight, currentBlockHeight - 10000);
-        
-      for (const block of indexedBlocks) {
-        const txoutset = await bitcoinClient.getTxoutSetinfo('none', block.height);
-        await BlocksRepository.$updateCoinStatsIndexData(block.hash, txoutset.txouts,
-          Math.round(txoutset.block_info.prevout_spent * 100000000));        
-        ++totalIndexed;
-
-        const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer);
-        if (elapsedSeconds > 5) {
-          logger.info(`Indexing coinstatsindex data for block #${block.height}. Indexed ${totalIndexed} blocks.`, logger.tags.mining);
-          timer = new Date().getTime() / 1000;
-        }
-      }
-
-      currentBlockHeight -= 10000;
-    }
-
-    if (totalIndexed) {
-      logger.info(`Indexing missing coinstatsindex data completed`, logger.tags.mining);
-    }
-  }
-
   private getDateMidnight(date: Date): Date {
     date.setUTCHours(0);
     date.setUTCMinutes(0);
@@ -574,7 +530,6 @@ class Mining {
 
   private getTimeRange(interval: string | null, scale = 1): number {
     switch (interval) {
-      case '4y': return 43200 * scale; // 12h
       case '3y': return 43200 * scale; // 12h
       case '2y': return 28800 * scale; // 8h
       case '1y': return 28800 * scale; // 8h

backend/src/api/pools-parser.ts

@@ -1,161 +1,289 @@
 import DB from '../database';
 import logger from '../logger';
 import config from '../config';
-import PoolsRepository from '../repositories/PoolsRepository';
-import { PoolTag } from '../mempool.interfaces';
-import diskCache from './disk-cache';
+import BlocksRepository from '../repositories/BlocksRepository';
+
+interface Pool {
+  name: string;
+  link: string;
+  regexes: string[];
+  addresses: string[];
+  slug: string;
+}
 
 class PoolsParser {
   miningPools: any[] = [];
   unknownPool: any = {
-    'id': 0,
     'name': 'Unknown',
     'link': 'https://learnmeabitcoin.com/technical/coinbase-transaction',
     'regexes': '[]',
     'addresses': '[]',
     'slug': 'unknown'
   };
-  private uniqueLogs: string[] = [];
-
-  private uniqueLog(loggerFunction: any, msg: string): void {
-    if (this.uniqueLogs.includes(msg)) {
-      return;
-    }
-    this.uniqueLogs.push(msg);
-    loggerFunction(msg);
-  }
-
-  public setMiningPools(pools): void {
-    for (const pool of pools) {
-      pool.regexes = pool.tags;
-      pool.slug = pool.name.replace(/[^a-z0-9]/gi, '').toLowerCase();
-      delete(pool.tags);
-    }
-    this.miningPools = pools;
-  }
+  slugWarnFlag = false;
 
   /**
-   * Populate our db with updated mining pool definition
-   * @param pools 
+   * Parse the pools.json file, consolidate the data and dump it into the database
    */
-  public async migratePoolsJson(): Promise<void> {
-    // We also need to wipe the backend cache to make sure we don't serve blocks with
-    // the wrong mining pool (usually happen with unknown blocks)
-    diskCache.wipeCache();
+  public async migratePoolsJson(poolsJson: object): Promise<void> {
+    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
+      return;
+    }
 
-    await this.$insertUnknownPool();
+    // First we save every entries without paying attention to pool duplication
+    const poolsDuplicated: Pool[] = [];
 
-    for (const pool of this.miningPools) {
-      if (!pool.id) {
-        logger.info(`Mining pool ${pool.name} has no unique 'id' defined. Skipping.`);
-        continue;
+    const coinbaseTags = Object.entries(poolsJson['coinbase_tags']);
+    for (let i = 0; i < coinbaseTags.length; ++i) {
+      poolsDuplicated.push({
+        'name': (<Pool>coinbaseTags[i][1]).name,
+        'link': (<Pool>coinbaseTags[i][1]).link,
+        'regexes': [coinbaseTags[i][0]],
+        'addresses': [],
+        'slug': ''
+      });
+    }
+    const addressesTags = Object.entries(poolsJson['payout_addresses']);
+    for (let i = 0; i < addressesTags.length; ++i) {
+      poolsDuplicated.push({
+        'name': (<Pool>addressesTags[i][1]).name,
+        'link': (<Pool>addressesTags[i][1]).link,
+        'regexes': [],
+        'addresses': [addressesTags[i][0]],
+        'slug': ''
+      });
+    }
+
+    // Then, we find unique mining pool names
+    const poolNames: string[] = [];
+    for (let i = 0; i < poolsDuplicated.length; ++i) {
+      if (poolNames.indexOf(poolsDuplicated[i].name) === -1) {
+        poolNames.push(poolsDuplicated[i].name);
+      }
+    }
+    logger.debug(`Found ${poolNames.length} unique mining pools`);
+
+    // Get existing pools from the db
+    let existingPools;
+    try {
+      if (config.DATABASE.ENABLED === true) {
+        [existingPools] = await DB.query({ sql: 'SELECT * FROM pools;', timeout: 120000 });
+      } else {
+        existingPools = [];
+      }
+    } catch (e) {
+      logger.err('Cannot get existing pools from the database, skipping pools.json import');
+      return;
+    }
+
+    this.miningPools = [];
+
+    // Finally, we generate the final consolidated pools data
+    const finalPoolDataAdd: Pool[] = [];
+    const finalPoolDataUpdate: Pool[] = [];
+    const finalPoolDataRename: Pool[] = [];
+    for (let i = 0; i < poolNames.length; ++i) {
+      let allAddresses: string[] = [];
+      let allRegexes: string[] = [];
+      const match = poolsDuplicated.filter((pool: Pool) => pool.name === poolNames[i]);
+
+      for (let y = 0; y < match.length; ++y) {
+        allAddresses = allAddresses.concat(match[y].addresses);
+        allRegexes = allRegexes.concat(match[y].regexes);
       }
 
-      const poolDB = await PoolsRepository.$getPoolByUniqueId(pool.id, false);
-      if (!poolDB) {
-        // New mining pool
-        const slug = pool.name.replace(/[^a-z0-9]/gi, '').toLowerCase();
-        logger.debug(`Inserting new mining pool ${pool.name}`);
-        await PoolsRepository.$insertNewMiningPool(pool, slug);
-        await this.$deleteUnknownBlocks();
+      const finalPoolName = poolNames[i].replace(`'`, `''`); // To support single quote in names when doing db queries
+
+      let slug: string | undefined;
+      try {
+        slug = poolsJson['slugs'][poolNames[i]];
+      } catch (e) {
+        if (this.slugWarnFlag === false) {
+          logger.warn(`pools.json does not seem to contain the 'slugs' object`);
+          this.slugWarnFlag = true;
+        }
+      }
+
+      if (slug === undefined) {
+        // Only keep alphanumerical
+        slug = poolNames[i].replace(/[^a-z0-9]/gi, '').toLowerCase();
+        logger.warn(`No slug found for '${poolNames[i]}', generating it => '${slug}'`);
+      }
+
+      const poolObj = {
+        'name': finalPoolName,
+        'link': match[0].link,
+        'regexes': allRegexes,
+        'addresses': allAddresses,
+        'slug': slug
+      };
+
+      const existingPool = existingPools.find((pool) => pool.name === poolNames[i]);
+      if (existingPool !== undefined) {
+        // Check if any data was actually updated
+        const equals = (a, b) =>
+          a.length === b.length &&
+          a.every((v, i) => v === b[i]);
+        if (!equals(JSON.parse(existingPool.addresses), poolObj.addresses) || !equals(JSON.parse(existingPool.regexes), poolObj.regexes)) {
+          finalPoolDataUpdate.push(poolObj);
+        }
       } else {
-        if (poolDB.name !== pool.name) {
-          // Pool has been renamed
-          const newSlug = pool.name.replace(/[^a-z0-9]/gi, '').toLowerCase();
-          logger.warn(`Renaming ${poolDB.name} mining pool to ${pool.name}. Slug has been updated. Maybe you want to make a redirection from 'https://mempool.space/mining/pool/${poolDB.slug}' to 'https://mempool.space/mining/pool/${newSlug}`);
-          await PoolsRepository.$renameMiningPool(poolDB.id, newSlug, pool.name);
+        // Double check that if we're not just renaming a pool (same address same regex)
+        const [poolToRename]: any[] = await DB.query(`
+          SELECT * FROM pools
+          WHERE addresses = ? OR regexes = ?`,
+          [JSON.stringify(poolObj.addresses), JSON.stringify(poolObj.regexes)]
+        );
+        if (poolToRename && poolToRename.length > 0) {
+          // We're actually renaming an existing pool
+          finalPoolDataRename.push({
+            'name': poolObj.name,
+            'link': poolObj.link,
+            'regexes': allRegexes,
+            'addresses': allAddresses,
+            'slug': slug
+          });
+          logger.debug(`Rename '${poolToRename[0].name}' mining pool to ${poolObj.name}`);
+        } else {
+          logger.debug(`Add '${finalPoolName}' mining pool`);
+          finalPoolDataAdd.push(poolObj);
         }
-        if (poolDB.link !== pool.link) {
-          // Pool link has changed
-          logger.debug(`Updating link for ${pool.name} mining pool`);
-          await PoolsRepository.$updateMiningPoolLink(poolDB.id, pool.link);
+      }
+
+      this.miningPools.push({
+        'name': finalPoolName,
+        'link': match[0].link,
+        'regexes': JSON.stringify(allRegexes),
+        'addresses': JSON.stringify(allAddresses),
+        'slug': slug
+      });
+    }
+
+    if (config.DATABASE.ENABLED === false) { // Don't run db operations
+      logger.info('Mining pools.json import completed (no database)');
+      return;
+    }
+
+    if (finalPoolDataAdd.length > 0 || finalPoolDataUpdate.length > 0 ||
+      finalPoolDataRename.length > 0
+    ) {    
+      logger.debug(`Update pools table now`);
+
+      // Add new mining pools into the database
+      let queryAdd: string = 'INSERT INTO pools(name, link, regexes, addresses, slug) VALUES ';
+      for (let i = 0; i < finalPoolDataAdd.length; ++i) {
+        queryAdd += `('${finalPoolDataAdd[i].name}', '${finalPoolDataAdd[i].link}',
+        '${JSON.stringify(finalPoolDataAdd[i].regexes)}', '${JSON.stringify(finalPoolDataAdd[i].addresses)}',
+        ${JSON.stringify(finalPoolDataAdd[i].slug)}),`;
+      }
+      queryAdd = queryAdd.slice(0, -1) + ';';
+
+      // Updated existing mining pools in the database
+      const updateQueries: string[] = [];
+      for (let i = 0; i < finalPoolDataUpdate.length; ++i) {
+        updateQueries.push(`
+          UPDATE pools
+          SET name='${finalPoolDataUpdate[i].name}', link='${finalPoolDataUpdate[i].link}',
+          regexes='${JSON.stringify(finalPoolDataUpdate[i].regexes)}', addresses='${JSON.stringify(finalPoolDataUpdate[i].addresses)}',
+          slug='${finalPoolDataUpdate[i].slug}'
+          WHERE name='${finalPoolDataUpdate[i].name}'
+        ;`);
+      }
+
+      // Rename mining pools
+      const renameQueries: string[] = [];
+      for (let i = 0; i < finalPoolDataRename.length; ++i) {
+        renameQueries.push(`
+          UPDATE pools
+          SET name='${finalPoolDataRename[i].name}', link='${finalPoolDataRename[i].link}',
+            slug='${finalPoolDataRename[i].slug}'
+          WHERE regexes='${JSON.stringify(finalPoolDataRename[i].regexes)}'
+            AND addresses='${JSON.stringify(finalPoolDataRename[i].addresses)}'
+        ;`);
+      }
+
+      try {
+        if (finalPoolDataAdd.length > 0 || updateQueries.length > 0) {
+          await this.$deleteBlocskToReindex(finalPoolDataUpdate);
         }
-        if (JSON.stringify(pool.addresses) !== poolDB.addresses ||
-          JSON.stringify(pool.regexes) !== poolDB.regexes) {
-          // Pool addresses changed or coinbase tags changed
-          logger.notice(`Updating addresses and/or coinbase tags for ${pool.name} mining pool. If 'AUTOMATIC_BLOCK_REINDEXING' is enabled, we will re-index its blocks and 'unknown' blocks`);
-          await PoolsRepository.$updateMiningPoolTags(poolDB.id, pool.addresses, pool.regexes);
-          await this.$deleteBlocksForPool(poolDB);
+
+        if (finalPoolDataAdd.length > 0) {
+          await DB.query({ sql: queryAdd, timeout: 120000 });
         }
+        for (const query of updateQueries) {
+          await DB.query({ sql: query, timeout: 120000 });
+        }
+        for (const query of renameQueries) {
+          await DB.query({ sql: query, timeout: 120000 });
+        }
+        await this.insertUnknownPool();
+        logger.info('Mining pools.json import completed');
+      } catch (e) {
+        logger.err(`Cannot import pools in the database`);
+        throw e;
       }
     }
 
-    logger.info('Mining pools-v2.json import completed');
+    try {
+      await this.insertUnknownPool();
+    } catch (e) {
+      logger.err(`Cannot insert unknown pool in the database`);
+      throw e;
+    }
   }
 
   /**
    * Manually add the 'unknown pool'
    */
-  public async $insertUnknownPool(): Promise<void> {
-    if (!config.DATABASE.ENABLED) {
-      return;
-    }
-
+  private async insertUnknownPool() {
     try {
       const [rows]: any[] = await DB.query({ sql: 'SELECT name from pools where name="Unknown"', timeout: 120000 });
       if (rows.length === 0) {
         await DB.query({
-          sql: `INSERT INTO pools(name, link, regexes, addresses, slug, unique_id)
-          VALUES("${this.unknownPool.name}", "${this.unknownPool.link}", "[]", "[]", "${this.unknownPool.slug}", 0);
+          sql: `INSERT INTO pools(name, link, regexes, addresses, slug)
+          VALUES("Unknown", "https://learnmeabitcoin.com/technical/coinbase-transaction", "[]", "[]", "unknown");
         `});
       } else {
         await DB.query(`UPDATE pools
-          SET name='${this.unknownPool.name}', link='${this.unknownPool.link}',
+          SET name='Unknown', link='https://learnmeabitcoin.com/technical/coinbase-transaction',
           regexes='[]', addresses='[]',
-          slug='${this.unknownPool.slug}',
-          unique_id=0
-          WHERE slug='${this.unknownPool.slug}'
+          slug='unknown'
+          WHERE name='Unknown'
         `);
       }
     } catch (e) {
-      logger.err(`Unable to insert or update "Unknown" mining pool. Reason: ${e instanceof Error ? e.message : e}`);
+      logger.err('Unable to insert "Unknown" mining pool');
     }
   }
 
   /**
-   * Delete indexed blocks for an updated mining pool
-   * 
-   * @param pool 
+   * Delete blocks which needs to be reindexed
    */
-  private async $deleteBlocksForPool(pool: PoolTag): Promise<void> {
+   private async $deleteBlocskToReindex(finalPoolDataUpdate: any[]) {
     if (config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING === false) {
       return;
     }
 
-    // Get oldest blocks mined by the pool and assume pools-v2.json updates only concern most recent years
-    // Ignore early days of Bitcoin as there were no mining pool yet
-    const [oldestPoolBlock]: any[] = await DB.query(`
-      SELECT height
-      FROM blocks
-      WHERE pool_id = ?
-      ORDER BY height
-      LIMIT 1`,
-      [pool.id]
-    );
-    const oldestBlockHeight = oldestPoolBlock.length ?? 0 > 0 ? oldestPoolBlock[0].height : 130635;
-    const [unknownPool] = await DB.query(`SELECT id from pools where slug = "unknown"`);
-    this.uniqueLog(logger.notice, `Deleting blocks with unknown mining pool from height ${oldestBlockHeight} for re-indexing`);
-    await DB.query(`
-      DELETE FROM blocks
-      WHERE pool_id = ? AND height >= ${oldestBlockHeight}`,
-      [unknownPool[0].id]
-    );
-    logger.notice(`Deleting blocks from ${pool.name} mining pool for re-indexing`);
-    await DB.query(`
-      DELETE FROM blocks
-      WHERE pool_id = ?`,
-      [pool.id]
-    );
-  }
+    const blockCount = await BlocksRepository.$blockCount(null, null);
+    if (blockCount === 0) {
+      return;
+    }
 
-  private async $deleteUnknownBlocks(): Promise<void> {
+    for (const updatedPool of finalPoolDataUpdate) {
+      const [pool]: any[] = await DB.query(`SELECT id, name from pools where slug = "${updatedPool.slug}"`);
+      if (pool.length > 0) {
+        logger.notice(`Deleting blocks from ${pool[0].name} mining pool for future re-indexing`);
+        await DB.query(`DELETE FROM blocks WHERE pool_id = ${pool[0].id}`);
+      }
+    }
+
+    // Ignore early days of Bitcoin as there were not mining pool yet
+    logger.notice('Deleting blocks with unknown mining pool from height 130635 for future re-indexing');
     const [unknownPool] = await DB.query(`SELECT id from pools where slug = "unknown"`);
-    this.uniqueLog(logger.notice, `Deleting blocks with unknown mining pool from height 130635 for re-indexing`);
-    await DB.query(`
-      DELETE FROM blocks
-      WHERE pool_id = ? AND height >= 130635`,
-      [unknownPool[0].id]
-    );
+    await DB.query(`DELETE FROM blocks WHERE pool_id = ${unknownPool[0].id} AND height > 130635`);
+
+    logger.notice('Truncating hashrates for future re-indexing');
+    await DB.query(`DELETE FROM hashrates`);
   }
 }
 

backend/src/api/rbf-cache.ts

@@ -1,62 +1,31 @@
-import { TransactionExtended } from "../mempool.interfaces";
+export interface CachedRbf {
+  txid: string;
+  expires: Date;
+}
 
 class RbfCache {
-  private replacedBy: { [txid: string]: string; } = {};
-  private replaces: { [txid: string]: string[] } = {};
-  private txs: { [txid: string]: TransactionExtended } = {};
-  private expiring: { [txid: string]: Date } = {};
+  private cache: { [txid: string]: CachedRbf; } = {};
 
   constructor() {
     setInterval(this.cleanup.bind(this), 1000 * 60 * 60);
   }
 
-  public add(replacedTx: TransactionExtended, newTxId: string): void {
-    this.replacedBy[replacedTx.txid] = newTxId;
-    this.txs[replacedTx.txid] = replacedTx;
-    if (!this.replaces[newTxId]) {
-      this.replaces[newTxId] = [];
-    }
-    this.replaces[newTxId].push(replacedTx.txid);
+  public add(replacedTxId: string, newTxId: string): void {
+    this.cache[replacedTxId] = {
+      expires: new Date(Date.now() + 1000 * 604800), // 1 week
+      txid: newTxId,
+    };
   }
 
-  public getReplacedBy(txId: string): string | undefined {
-    return this.replacedBy[txId];
-  }
-
-  public getReplaces(txId: string): string[] | undefined {
-    return this.replaces[txId];
-  }
-
-  public getTx(txId: string): TransactionExtended | undefined {
-    return this.txs[txId];
-  }
-
-  // flag a transaction as removed from the mempool
-  public evict(txid): void {
-    this.expiring[txid] = new Date(Date.now() + 1000 * 86400); // 24 hours
+  public get(txId: string): CachedRbf | undefined {
+    return this.cache[txId];
   }
 
   private cleanup(): void {
     const currentDate = new Date();
-    for (const txid in this.expiring) {
-      if (this.expiring[txid] < currentDate) {
-        delete this.expiring[txid];
-        this.remove(txid);
-      }
-    }
-  }
-
-  // remove a transaction & all previous versions from the cache
-  private remove(txid): void {
-    // don't remove a transaction while a newer version remains in the mempool
-    if (this.replaces[txid] && !this.replacedBy[txid]) {
-      const replaces = this.replaces[txid];
-      delete this.replaces[txid];
-      for (const tx of replaces) {
-        // recursively remove prior versions from the cache
-        delete this.replacedBy[tx];
-        delete this.txs[tx];
-        this.remove(tx);
+    for (const c in this.cache) {
+      if (this.cache[c].expires < currentDate) {
+        delete this.cache[c];
       }
     }
   }

backend/src/api/statistics/statistics-api.ts

@@ -375,17 +375,6 @@ class StatisticsApi {
     }
   }
 
-  public async $list4Y(): Promise<OptimizedStatistic[]> {
-    try {
-      const query = this.getQueryForDays(43200, '4 YEAR'); // 12h interval
-      const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
-      return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
-    } catch (e) {
-      logger.err('$list4Y() error' + (e instanceof Error ? e.message : e));
-      return [];
-    }
-  }
-
   private mapStatisticToOptimizedStatistic(statistic: Statistic[]): OptimizedStatistic[] {
     return statistic.map((s) => {
       return {

backend/src/api/statistics/statistics.routes.ts

@@ -14,11 +14,10 @@ class StatisticsRoutes {
       .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/1y', this.$getStatisticsByTime.bind(this, '1y'))
       .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/2y', this.$getStatisticsByTime.bind(this, '2y'))
       .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/3y', this.$getStatisticsByTime.bind(this, '3y'))
-      .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/4y', this.$getStatisticsByTime.bind(this, '4y'))
     ;
   }
 
-  private async $getStatisticsByTime(time: '2h' | '24h' | '1w' | '1m' | '3m' | '6m' | '1y' | '2y' | '3y' | '4y', req: Request, res: Response) {
+  private async $getStatisticsByTime(time: '2h' | '24h' | '1w' | '1m' | '3m' | '6m' | '1y' | '2y' | '3y', req: Request, res: Response) {
     res.header('Pragma', 'public');
     res.header('Cache-control', 'public');
     res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
@@ -55,9 +54,6 @@ class StatisticsRoutes {
         case '3y':
           result = await statisticsApi.$list3Y();
           break;
-        case '4y':
-          result = await statisticsApi.$list4Y();
-          break;
         default:
           result = await statisticsApi.$list2H();
       }

backend/src/api/transaction-utils.ts

@@ -1,7 +1,8 @@
+import bitcoinApi from './bitcoin/bitcoin-api-factory';
 import { TransactionExtended, TransactionMinerInfo } from '../mempool.interfaces';
 import { IEsploraApi } from './bitcoin/esplora-api.interface';
+import config from '../config';
 import { Common } from './common';
-import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory';
 
 class TransactionUtils {
   constructor() { }
@@ -14,26 +15,14 @@ class TransactionUtils {
       vout: tx.vout
         .map((vout) => ({
           scriptpubkey_address: vout.scriptpubkey_address,
-          scriptpubkey_asm: vout.scriptpubkey_asm,
           value: vout.value
         }))
         .filter((vout) => vout.value)
     };
   }
 
-  /**
-   * @param txId 
-   * @param addPrevouts 
-   * @param lazyPrevouts 
-   * @param forceCore - See https://github.com/mempool/mempool/issues/2904
-   */
-  public async $getTransactionExtended(txId: string, addPrevouts = false, lazyPrevouts = false, forceCore = false): Promise<TransactionExtended> {
-    let transaction: IEsploraApi.Transaction;
-    if (forceCore === true) {
-      transaction  = await bitcoinCoreApi.$getRawTransaction(txId, true);
-    } else {
-      transaction  = await bitcoinApi.$getRawTransaction(txId, false, addPrevouts, lazyPrevouts);
-    }
+  public async $getTransactionExtended(txId: string, addPrevouts = false, lazyPrevouts = false): Promise<TransactionExtended> {
+    const transaction: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(txId, false, addPrevouts, lazyPrevouts);
     return this.extendTransaction(transaction);
   }
 

backend/src/api/websocket-handler.ts

@@ -1,13 +1,14 @@
 import logger from '../logger';
 import * as WebSocket from 'ws';
 import {
-  BlockExtended, TransactionExtended, WebsocketResponse,
-  OptimizedStatistic, ILoadingIndicators
+  BlockExtended, TransactionExtended, WebsocketResponse, MempoolBlock, MempoolBlockDelta,
+  OptimizedStatistic, ILoadingIndicators, IConversionRates
 } from '../mempool.interfaces';
 import blocks from './blocks';
 import memPool from './mempool';
 import backendInfo from './backend-info';
 import mempoolBlocks from './mempool-blocks';
+import fiatConversion from './fiat-conversion';
 import { Common } from './common';
 import loadingIndicators from './loading-indicators';
 import config from '../config';
@@ -18,9 +19,6 @@ import feeApi from './fee-api';
 import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository';
 import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
 import Audit from './audit';
-import { deepClone } from '../utils/clone';
-import priceUpdater from '../tasks/price-updater';
-import { ApiPrice } from '../repositories/PricesRepository';
 
 class WebsocketHandler {
   private wss: WebSocket.Server | undefined;
@@ -60,10 +58,10 @@ class WebsocketHandler {
               client['track-tx'] = parsedMessage['track-tx'];
               // Client is telling the transaction wasn't found
               if (parsedMessage['watch-mempool']) {
-                const rbfCacheTxid = rbfCache.getReplacedBy(client['track-tx']);
-                if (rbfCacheTxid) {
+                const rbfCacheTx = rbfCache.get(client['track-tx']);
+                if (rbfCacheTx) {
                   response['txReplaced'] = {
-                    txid: rbfCacheTxid,
+                    txid: rbfCacheTx.txid,
                   };
                   client['track-tx'] = null;
                 } else {
@@ -194,7 +192,7 @@ class WebsocketHandler {
     });
   }
 
-  handleNewConversionRates(conversionRates: ApiPrice) {
+  handleNewConversionRates(conversionRates: IConversionRates) {
     if (!this.wss) {
       throw new Error('WebSocket.Server is not set');
     }
@@ -211,17 +209,16 @@ class WebsocketHandler {
     if (!_blocks) {
       _blocks = blocks.getBlocks().slice(-config.MEMPOOL.INITIAL_BLOCKS_AMOUNT);
     }
-    const da = difficultyAdjustment.getDifficultyAdjustment();
     return {
       'mempoolInfo': memPool.getMempoolInfo(),
       'vBytesPerSecond': memPool.getVBytesPerSecond(),
       'blocks': _blocks,
-      'conversions': priceUpdater.getLatestPrices(),
+      'conversions': fiatConversion.getConversionRates(),
       'mempool-blocks': mempoolBlocks.getMempoolBlocks(),
       'transactions': memPool.getLatestTransactions(),
       'backendInfo': backendInfo.getBackendInfo(),
       'loadingIndicators': loadingIndicators.getLoadingIndicators(),
-      'da': da?.previousTime ? da : undefined,
+      'da': difficultyAdjustment.getDifficultyAdjustment(),
       'fees': feeApi.getRecommendedFee(),
       ...this.extraInitProperties
     };
@@ -254,9 +251,9 @@ class WebsocketHandler {
     }
 
     if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) {
-      await mempoolBlocks.updateBlockTemplates(newMempool, newTransactions, deletedTransactions.map(tx => tx.txid), true);
+      await mempoolBlocks.updateBlockTemplates(newMempool, newTransactions, deletedTransactions.map(tx => tx.txid));
     } else {
-      mempoolBlocks.updateMempoolBlocks(newMempool, true);
+      mempoolBlocks.updateMempoolBlocks(newMempool);
     }
 
     const mBlocks = mempoolBlocks.getMempoolBlocks();
@@ -279,9 +276,7 @@ class WebsocketHandler {
         response['mempoolInfo'] = mempoolInfo;
         response['vBytesPerSecond'] = vBytesPerSecond;
         response['transactions'] = newTransactions.slice(0, 6).map((tx) => Common.stripTransaction(tx));
-        if (da?.previousTime) {
-          response['da'] = da;
-        }
+        response['da'] = da;
         response['fees'] = recommendedFees;
       }
 
@@ -423,57 +418,47 @@ class WebsocketHandler {
 
     const _memPool = memPool.getMempool();
 
-    if (config.MEMPOOL.AUDIT) {
-      let projectedBlocks;
-      // template calculation functions have mempool side effects, so calculate audits using
-      // a cloned copy of the mempool if we're running a different algorithm for mempool updates
-      const auditMempool = (config.MEMPOOL.ADVANCED_GBT_AUDIT === config.MEMPOOL.ADVANCED_GBT_MEMPOOL) ? _memPool : deepClone(_memPool);
-      if (config.MEMPOOL.ADVANCED_GBT_AUDIT) {
-        projectedBlocks = await mempoolBlocks.makeBlockTemplates(auditMempool, false);
-      } else {
-        projectedBlocks = mempoolBlocks.updateMempoolBlocks(auditMempool, false);
-      }
+    if (config.MEMPOOL.ADVANCED_GBT_AUDIT) {
+      await mempoolBlocks.makeBlockTemplates(_memPool);
+    } else {
+      mempoolBlocks.updateMempoolBlocks(_memPool);
+    }
 
-      if (Common.indexingEnabled() && memPool.isInSync()) {
-        const { censored, added, fresh, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool);
-        const matchRate = Math.round(score * 100 * 100) / 100;
+    if (Common.indexingEnabled() && memPool.isInSync()) {
+      const projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
 
-        const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions.map((tx) => {
-          return {
-            txid: tx.txid,
-            vsize: tx.vsize,
-            fee: tx.fee ? Math.round(tx.fee) : 0,
-            value: tx.value,
-          };
-        }) : [];
+      const { censored, added, fresh, score } = Audit.auditBlock(transactions, projectedBlocks, _memPool);
+      const matchRate = Math.round(score * 100 * 100) / 100;
 
-        BlocksSummariesRepository.$saveTemplate({
-          height: block.height,
-          template: {
-            id: block.id,
-            transactions: stripped
-          }
-        });
+      const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions.map((tx) => {
+        return {
+          txid: tx.txid,
+          vsize: tx.vsize,
+          fee: tx.fee ? Math.round(tx.fee) : 0,
+          value: tx.value,
+        };
+      }) : [];
 
-        BlocksAuditsRepository.$saveAudit({
-          time: block.timestamp,
-          height: block.height,
-          hash: block.id,
-          addedTxs: added,
-          missingTxs: censored,
-          freshTxs: fresh,
-          matchRate: matchRate,
-        });
-
-        if (block.extras) {
-          block.extras.matchRate = matchRate;
-          block.extras.similarity = similarity;
+      BlocksSummariesRepository.$saveTemplate({
+        height: block.height,
+        template: {
+          id: block.id,
+          transactions: stripped
         }
-      }
-    } else if (block.extras) {
-      const mBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
-      if (mBlocks?.length && mBlocks[0].transactions) {
-        block.extras.similarity = Common.getSimilarity(mBlocks[0], transactions);
+      });
+
+      BlocksAuditsRepository.$saveAudit({
+        time: block.timestamp,
+        height: block.height,
+        hash: block.id,
+        addedTxs: added,
+        missingTxs: censored,
+        freshTxs: fresh,
+        matchRate: matchRate,
+      });
+
+      if (block.extras) {
+        block.extras.matchRate = matchRate;
       }
     }
 
@@ -482,13 +467,12 @@ class WebsocketHandler {
     for (const txId of txIds) {
       delete _memPool[txId];
       removed.push(txId);
-      rbfCache.evict(txId);
     }
 
     if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) {
-      await mempoolBlocks.updateBlockTemplates(_memPool, [], removed, true);
+      await mempoolBlocks.updateBlockTemplates(_memPool, [], removed);
     } else {
-      mempoolBlocks.updateMempoolBlocks(_memPool, true);
+      mempoolBlocks.updateMempoolBlocks(_memPool);
     }
     const mBlocks = mempoolBlocks.getMempoolBlocks();
     const mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();
@@ -508,7 +492,7 @@ class WebsocketHandler {
       const response = {
         'block': block,
         'mempoolInfo': memPool.getMempoolInfo(),
-        'da': da?.previousTime ? da : undefined,
+        'da': da,
         'fees': fees,
       };
 

backend/src/config.ts

@@ -19,6 +19,7 @@ interface IConfig {
     MEMPOOL_BLOCKS_AMOUNT: number;
     INDEXING_BLOCKS_AMOUNT: number;
     BLOCKS_SUMMARIES_INDEXING: boolean;
+    PRICE_FEED_UPDATE_INTERVAL: number;
     USE_SECOND_NODE_FOR_MINFEE: boolean;
     EXTERNAL_ASSETS: string[];
     EXTERNAL_MAX_RETRY: number;
@@ -28,12 +29,9 @@ interface IConfig {
     AUTOMATIC_BLOCK_REINDEXING: boolean;
     POOLS_JSON_URL: string,
     POOLS_JSON_TREE_URL: string,
-    AUDIT: boolean;
     ADVANCED_GBT_AUDIT: boolean;
     ADVANCED_GBT_MEMPOOL: boolean;
-    CPFP_INDEXING: boolean;
-    MAX_BLOCKS_BULK_QUERY: number;
-    DISK_CACHE_BLOCK_INTERVAL: number;
+    TRANSACTION_INDEXING: boolean;
   };
   ESPLORA: {
     REST_API_URL: string;
@@ -52,7 +50,6 @@ interface IConfig {
     TLS_CERT_PATH: string;
     MACAROON_PATH: string;
     REST_API_URL: string;
-    TIMEOUT: number;
   };
   CLIGHTNING: {
     SOCKET: string;
@@ -67,14 +64,12 @@ interface IConfig {
     PORT: number;
     USERNAME: string;
     PASSWORD: string;
-    TIMEOUT: number;
   };
   SECOND_CORE_RPC: {
     HOST: string;
     PORT: number;
     USERNAME: string;
     PASSWORD: string;
-    TIMEOUT: number;
   };
   DATABASE: {
     ENABLED: boolean;
@@ -145,6 +140,7 @@ const defaults: IConfig = {
     'MEMPOOL_BLOCKS_AMOUNT': 8,
     'INDEXING_BLOCKS_AMOUNT': 11000, // 0 = disable indexing, -1 = index all blocks
     'BLOCKS_SUMMARIES_INDEXING': false,
+    'PRICE_FEED_UPDATE_INTERVAL': 600,
     'USE_SECOND_NODE_FOR_MINFEE': false,
     'EXTERNAL_ASSETS': [],
     'EXTERNAL_MAX_RETRY': 1,
@@ -152,14 +148,11 @@ const defaults: IConfig = {
     'USER_AGENT': 'mempool',
     'STDOUT_LOG_MIN_PRIORITY': 'debug',
     'AUTOMATIC_BLOCK_REINDEXING': false,
-    'POOLS_JSON_URL': 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json',
+    'POOLS_JSON_URL': 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json',
     'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
-    'AUDIT': false,
     'ADVANCED_GBT_AUDIT': false,
     'ADVANCED_GBT_MEMPOOL': false,
-    'CPFP_INDEXING': false,
-    'MAX_BLOCKS_BULK_QUERY': 0,
-    'DISK_CACHE_BLOCK_INTERVAL': 6,
+    'TRANSACTION_INDEXING': false,
   },
   'ESPLORA': {
     'REST_API_URL': 'http://127.0.0.1:3000',
@@ -173,15 +166,13 @@ const defaults: IConfig = {
     'HOST': '127.0.0.1',
     'PORT': 8332,
     'USERNAME': 'mempool',
-    'PASSWORD': 'mempool',
-    'TIMEOUT': 60000,
+    'PASSWORD': 'mempool'
   },
   'SECOND_CORE_RPC': {
     'HOST': '127.0.0.1',
     'PORT': 8332,
     'USERNAME': 'mempool',
-    'PASSWORD': 'mempool',
-    'TIMEOUT': 60000,
+    'PASSWORD': 'mempool'
   },
   'DATABASE': {
     'ENABLED': true,
@@ -221,7 +212,6 @@ const defaults: IConfig = {
     'TLS_CERT_PATH': '',
     'MACAROON_PATH': '',
     'REST_API_URL': 'https://localhost:8080',
-    'TIMEOUT': 10000,
   },
   'CLIGHTNING': {
     'SOCKET': '',

backend/src/database.ts

@@ -24,8 +24,7 @@ import { FieldPacket, OkPacket, PoolOptions, ResultSetHeader, RowDataPacket } fr
 
   private checkDBFlag() {
     if (config.DATABASE.ENABLED === false) {
-      const stack = new Error().stack;
-      logger.err(`Trying to use DB feature but config.DATABASE.ENABLED is set to false, please open an issue.\nStack trace: ${stack}}`);
+      logger.err('Trying to use DB feature but config.DATABASE.ENABLED is set to false, please open an issue');
     }
   }
 

backend/src/index.ts

@@ -10,6 +10,7 @@ import memPool from './api/mempool';
 import diskCache from './api/disk-cache';
 import statistics from './api/statistics/statistics';
 import websocketHandler from './api/websocket-handler';
+import fiatConversion from './api/fiat-conversion';
 import bisq from './api/bisq/bisq';
 import bisqMarkets from './api/bisq/markets';
 import logger from './logger';
@@ -35,11 +36,6 @@ import liquidRoutes from './api/liquid/liquid.routes';
 import bitcoinRoutes from './api/bitcoin/bitcoin.routes';
 import fundingTxFetcher from './tasks/lightning/sync-tasks/funding-tx-fetcher';
 import forensicsService from './tasks/lightning/forensics.service';
-import priceUpdater from './tasks/price-updater';
-import chainTips from './api/chain-tips';
-import { AxiosError } from 'axios';
-import v8 from 'v8';
-import { formatBytes, getBytesUnit } from './utils/format';
 
 class Server {
   private wss: WebSocket.Server | undefined;
@@ -47,11 +43,6 @@ class Server {
   private app: Application;
   private currentBackendRetryInterval = 5;
 
-  private maxHeapSize: number = 0;
-  private heapLogInterval: number = 60;
-  private warnedHeapCritical: boolean = false;
-  private lastHeapLogTime: number | null = null;
-
   constructor() {
     this.app = express();
 
@@ -87,18 +78,6 @@ class Server {
   async startServer(worker = false): Promise<void> {
     logger.notice(`Starting Mempool Server${worker ? ' (worker)' : ''}... (${backendInfo.getShortCommitHash()})`);
 
-    if (config.DATABASE.ENABLED) {
-      await DB.checkDbConnection();
-      try {
-        if (process.env.npm_config_reindex_blocks === 'true') { // Re-index requests
-          await databaseMigration.$blocksReindexingTruncate();
-        }
-        await databaseMigration.$initializeOrMigrateDatabase();
-      } catch (e) {
-        throw new Error(e instanceof Error ? e.message : 'Error');
-      }
-    }
-
     this.app
       .use((req: Request, res: Response, next: NextFunction) => {
         res.setHeader('Access-Control-Allow-Origin', '*');
@@ -108,21 +87,34 @@ class Server {
       .use(express.text({ type: ['text/plain', 'application/base64'] }))
       ;
 
-    if (config.DATABASE.ENABLED) {
-      await priceUpdater.$initializeLatestPriceWithDb();
-    }
-
     this.server = http.createServer(this.app);
     this.wss = new WebSocket.Server({ server: this.server });
 
     this.setUpWebsocketHandling();
 
-    await poolsUpdater.updatePoolsJson(); // Needs to be done before loading the disk cache because we sometimes wipe it
     await syncAssets.syncAssets$();
     if (config.MEMPOOL.ENABLED) {
       diskCache.loadMempoolCache();
     }
 
+    if (config.DATABASE.ENABLED) {
+      await DB.checkDbConnection();
+      try {
+        if (process.env.npm_config_reindex !== undefined) { // Re-index requests
+          const tables = process.env.npm_config_reindex.split(',');
+          logger.warn(`Indexed data for "${process.env.npm_config_reindex}" tables will be erased in 5 seconds (using '--reindex')`);
+          await Common.sleep$(5000);
+          await databaseMigration.$truncateIndexedData(tables);
+        }
+        await databaseMigration.$initializeOrMigrateDatabase();
+        if (Common.indexingEnabled()) {
+          await indexer.$resetHashratesIndexingState();
+        }
+      } catch (e) {
+        throw new Error(e instanceof Error ? e.message : 'Error');
+      }
+    }
+
     if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED && cluster.isPrimary) {
       statistics.startStatistics();
     }
@@ -135,8 +127,7 @@ class Server {
       }
     }
 
-    priceUpdater.$run();
-    await chainTips.updateOrphanedBlocks();
+    fiatConversion.startService();
 
     this.setUpHttpApiRoutes();
 
@@ -144,8 +135,6 @@ class Server {
       this.runMainUpdateLoop();
     }
 
-    setInterval(() => { this.healthCheck(); }, 2500);
-
     if (config.BISQ.ENABLED) {
       bisq.startBisqService();
       bisq.setPriceCallbackFunction((price) => websocketHandler.setExtraInitProperties('bsq-price', price));
@@ -178,31 +167,22 @@ class Server {
           logger.debug(msg);
         }
       }
-      memPool.deleteExpiredTransactions();
+      await poolsUpdater.updatePoolsJson();
       await blocks.$updateBlocks();
       await memPool.$updateMempool();
       indexer.$run();
 
       setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS);
       this.currentBackendRetryInterval = 5;
-    } catch (e: any) {
-      let loggerMsg = `Exception in runMainUpdateLoop(). Retrying in ${this.currentBackendRetryInterval} sec.`;
-      loggerMsg += ` Reason: ${(e instanceof Error ? e.message : e)}.`;
-      if (e?.stack) {
-        loggerMsg += ` Stack trace: ${e.stack}`;
-      }
-      // When we get a first Exception, only `logger.debug` it and retry after 5 seconds
-      // From the second Exception, `logger.warn` the Exception and increase the retry delay
-      // Maximum retry delay is 60 seconds
+    } catch (e) {
+      const loggerMsg = `runMainLoop error: ${(e instanceof Error ? e.message : e)}. Retrying in ${this.currentBackendRetryInterval} sec.`;
       if (this.currentBackendRetryInterval > 5) {
         logger.warn(loggerMsg);
         mempool.setOutOfSync();
       } else {
         logger.debug(loggerMsg);
       }
-      if (e instanceof AxiosError) {
-        logger.debug(`AxiosError: ${e?.message}`);
-      }
+      logger.debug(JSON.stringify(e));
       setTimeout(this.runMainUpdateLoop.bind(this), 1000 * this.currentBackendRetryInterval);
       this.currentBackendRetryInterval *= 2;
       this.currentBackendRetryInterval = Math.min(this.currentBackendRetryInterval, 60);
@@ -213,14 +193,14 @@ class Server {
     try {
       await fundingTxFetcher.$init();
       await networkSyncService.$startService();
-      await lightningStatsUpdater.$startService();
       await forensicsService.$startService();
+      await lightningStatsUpdater.$startService();
     } catch(e) {
-      logger.err(`Exception in $runLightningBackend. Restarting in 1 minute. Reason: ${(e instanceof Error ? e.message : e)}`);
+      logger.err(`Nodejs lightning backend crashed. Restarting in 1 minute. Reason: ${(e instanceof Error ? e.message : e)}`);
       await Common.sleep$(1000 * 60);
       this.$runLightningBackend();
     };
-  }
+}
 
   setUpWebsocketHandling(): void {
     if (this.wss) {
@@ -241,7 +221,7 @@ class Server {
       memPool.setAsyncMempoolChangedCallback(websocketHandler.handleMempoolChange.bind(websocketHandler));
       blocks.setNewAsyncBlockCallback(websocketHandler.handleNewBlock.bind(websocketHandler));
     }
-    priceUpdater.setRatesChangedCallback(websocketHandler.handleNewConversionRates.bind(websocketHandler));
+    fiatConversion.setProgressChangedCallback(websocketHandler.handleNewConversionRates.bind(websocketHandler));
     loadingIndicators.setProgressChangedCallback(websocketHandler.handleLoadingChanged.bind(websocketHandler));
   }
   
@@ -265,26 +245,6 @@ class Server {
       channelsRoutes.initRoutes(this.app);
     }
   }
-
-  healthCheck(): void {
-    const now = Date.now();
-    const stats = v8.getHeapStatistics();
-    this.maxHeapSize = Math.max(stats.used_heap_size, this.maxHeapSize);
-    const warnThreshold = 0.8 * stats.heap_size_limit;
-
-    const byteUnits = getBytesUnit(Math.max(this.maxHeapSize, stats.heap_size_limit));
-
-    if (!this.warnedHeapCritical && this.maxHeapSize > warnThreshold) {
-      this.warnedHeapCritical = true;
-      logger.warn(`Used ${(this.maxHeapSize / stats.heap_size_limit).toFixed(2)}% of heap limit (${formatBytes(this.maxHeapSize, byteUnits, true)} / ${formatBytes(stats.heap_size_limit, byteUnits)})!`);
-    }
-    if (this.lastHeapLogTime === null || (now - this.lastHeapLogTime) > (this.heapLogInterval * 1000)) {
-      logger.debug(`Memory usage: ${formatBytes(this.maxHeapSize, byteUnits)} / ${formatBytes(stats.heap_size_limit, byteUnits)}`);
-      this.warnedHeapCritical = false;
-      this.maxHeapSize = 0;
-      this.lastHeapLogTime = now;
-    }
-  }
 }
 
 ((): Server => new Server())();

backend/src/indexer.ts

@@ -3,71 +3,23 @@ import blocks from './api/blocks';
 import mempool from './api/mempool';
 import mining from './api/mining/mining';
 import logger from './logger';
+import HashratesRepository from './repositories/HashratesRepository';
 import bitcoinClient from './api/bitcoin/bitcoin-client';
 import priceUpdater from './tasks/price-updater';
 import PricesRepository from './repositories/PricesRepository';
 
-export interface CoreIndex {
-  name: string;
-  synced: boolean;
-  best_block_height: number;
-}
-
 class Indexer {
   runIndexer = true;
   indexerRunning = false;
   tasksRunning: string[] = [];
-  coreIndexes: CoreIndex[] = [];
 
-  /**
-   * Check which core index is available for indexing
-   */
-  public async checkAvailableCoreIndexes(): Promise<void> {
-    const updatedCoreIndexes: CoreIndex[] = [];
-
-    const indexes: any = await bitcoinClient.getIndexInfo();
-    for (const indexName in indexes) {
-      const newState = {
-        name: indexName,
-        synced: indexes[indexName].synced,
-        best_block_height: indexes[indexName].best_block_height,
-      };
-      logger.info(`Core index '${indexName}' is ${indexes[indexName].synced ? 'synced' : 'not synced'}. Best block height is ${indexes[indexName].best_block_height}`);      
-      updatedCoreIndexes.push(newState);
-
-      if (indexName === 'coinstatsindex' && newState.synced === true) {
-        const previousState = this.isCoreIndexReady('coinstatsindex');
-        // if (!previousState || previousState.synced === false) {
-          this.runSingleTask('coinStatsIndex');
-        // }
-      }
-    }
-
-    this.coreIndexes = updatedCoreIndexes;
-  }
-
-  /**
-   * Return the best block height if a core index is available, or 0 if not
-   * 
-   * @param name 
-   * @returns 
-   */
-  public isCoreIndexReady(name: string): CoreIndex | null {
-    for (const index of this.coreIndexes) {
-      if (index.name === name && index.synced === true) {
-        return index;
-      }
-    }
-    return null;
-  }
-
-  public reindex(): void {
+  public reindex() {
     if (Common.indexingEnabled()) {
       this.runIndexer = true;
     }
   }
 
-  public async runSingleTask(task: 'blocksPrices' | 'coinStatsIndex'): Promise<void> {
+  public async runSingleTask(task: 'blocksPrices') {
     if (!Common.indexingEnabled()) {
       return;
     }
@@ -76,27 +28,20 @@ class Indexer {
       this.tasksRunning.push(task);
       const lastestPriceId = await PricesRepository.$getLatestPriceId();
       if (priceUpdater.historyInserted === false || lastestPriceId === null) {
-        logger.debug(`Blocks prices indexer is waiting for the price updater to complete`, logger.tags.mining);
+        logger.debug(`Blocks prices indexer is waiting for the price updater to complete`)
         setTimeout(() => {
-          this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task);
+          this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task)
           this.runSingleTask('blocksPrices');
         }, 10000);
       } else {
-        logger.debug(`Blocks prices indexer will run now`, logger.tags.mining);
+        logger.debug(`Blocks prices indexer will run now`)
         await mining.$indexBlockPrices();
-        this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task);
+        this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task)
       }
     }
-
-    if (task === 'coinStatsIndex' && !this.tasksRunning.includes(task)) {
-      this.tasksRunning.push(task);
-      logger.debug(`Indexing coinStatsIndex now`);
-      await mining.$indexCoinStatsIndex();
-      this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task);
-    }
   }
 
-  public async $run(): Promise<void> {
+  public async $run() {
     if (!Common.indexingEnabled() || this.runIndexer === false ||
       this.indexerRunning === true || mempool.hasPriority()
     ) {
@@ -114,15 +59,13 @@ class Indexer {
 
     logger.debug(`Running mining indexer`);
 
-    await this.checkAvailableCoreIndexes();
-
     try {
       await priceUpdater.$run();
 
       const chainValid = await blocks.$generateBlockDatabase();
       if (chainValid === false) {
         // Chain of block hash was invalid, so we need to reindex. Stop here and continue at the next iteration
-        logger.warn(`The chain of block hash is invalid, re-indexing invalid data in 10 seconds.`, logger.tags.mining);
+        logger.warn(`The chain of block hash is invalid, re-indexing invalid data in 10 seconds.`);
         setTimeout(() => this.reindex(), 10000);
         this.indexerRunning = false;
         return;
@@ -130,6 +73,7 @@ class Indexer {
 
       this.runSingleTask('blocksPrices');
       await mining.$indexDifficultyAdjustments();
+      await this.$resetHashratesIndexingState(); // TODO - Remove this as it's not efficient
       await mining.$generateNetworkHashrateHistory();
       await mining.$generatePoolHashrateHistory();
       await blocks.$generateBlocksSummariesDatabase();
@@ -148,6 +92,16 @@ class Indexer {
     logger.debug(`Indexing completed. Next run planned at ${new Date(new Date().getTime() + runEvery).toUTCString()}`);
     setTimeout(() => this.reindex(), runEvery);
   }
+
+  async $resetHashratesIndexingState() {
+    try {
+      await HashratesRepository.$setLatestRun('last_hashrates_indexing', 0);
+      await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', 0);
+    } catch (e) {
+      logger.err(`Cannot reset hashrate indexing timestamps. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
 }
 
 export default new Indexer();

backend/src/logger.ts

@@ -32,27 +32,22 @@ class Logger {
     local7: 23
   };
 
-  public tags = {
-    mining: 'Mining',
-    ln: 'Lightning',
-  };  
-
   // @ts-ignore
-  public emerg: ((msg: string, tag?: string) => void);
+  public emerg: ((msg: string) => void);
   // @ts-ignore
-  public alert: ((msg: string, tag?: string) => void);
+  public alert: ((msg: string) => void);
   // @ts-ignore
-  public crit: ((msg: string, tag?: string) => void);
+  public crit: ((msg: string) => void);
   // @ts-ignore
-  public err: ((msg: string, tag?: string) => void);
+  public err: ((msg: string) => void);
   // @ts-ignore
-  public warn: ((msg: string, tag?: string) => void);
+  public warn: ((msg: string) => void);
   // @ts-ignore
-  public notice: ((msg: string, tag?: string) => void);
+  public notice: ((msg: string) => void);
   // @ts-ignore
-  public info: ((msg: string, tag?: string) => void);
+  public info: ((msg: string) => void);
   // @ts-ignore
-  public debug: ((msg: string, tag?: string) => void);
+  public debug: ((msg: string) => void);
 
   private name = 'mempool';
   private client: dgram.Socket;
@@ -69,14 +64,10 @@ class Logger {
     this.network = this.getNetwork();
   }
 
-  public updateNetwork(): void {
-    this.network = this.getNetwork();
-  }
-
   private addprio(prio): void {
     this[prio] = (function(_this) {
-      return function(msg, tag?: string) {
-        return _this.msg(prio, msg, tag);
+      return function(msg) {
+        return _this.msg(prio, msg);
       };
     })(this);
   }
@@ -94,7 +85,7 @@ class Logger {
     return '';
   }
 
-  private msg(priority, msg, tag?: string) {
+  private msg(priority, msg) {
     let consolemsg, prionum, syslogmsg;
     if (typeof msg === 'string' && msg.length > 0) {
       while (msg[msg.length - 1].charCodeAt(0) === 10) {
@@ -103,10 +94,10 @@ class Logger {
     }
     const network = this.network ? ' <' + this.network + '>' : '';
     prionum = Logger.priorities[priority] || Logger.priorities.info;
-    consolemsg = `${this.ts()} [${process.pid}] ${priority.toUpperCase()}:${network} ${tag ? '[' + tag + '] ' : ''}${msg}`;
+    consolemsg = `${this.ts()} [${process.pid}] ${priority.toUpperCase()}:${network} ${msg}`;
 
     if (config.SYSLOG.ENABLED && Logger.priorities[priority] <= Logger.priorities[config.SYSLOG.MIN_PRIORITY]) {
-      syslogmsg = `<${(Logger.facilities[config.SYSLOG.FACILITY] * 8 + prionum)}> ${this.name}[${process.pid}]: ${priority.toUpperCase()}${network} ${tag ? '[' + tag + '] ' : ''}${msg}`;
+      syslogmsg = `<${(Logger.facilities[config.SYSLOG.FACILITY] * 8 + prionum)}> ${this.name}[${process.pid}]: ${priority.toUpperCase()}${network} ${msg}`;
       this.syslog(syslogmsg);
     }
     if (Logger.priorities[priority] > Logger.priorities[config.MEMPOOL.STDOUT_LOG_MIN_PRIORITY]) {

backend/src/mempool.interfaces.ts

@@ -1,10 +1,8 @@
 import { IEsploraApi } from './api/bitcoin/esplora-api.interface';
-import { OrphanedBlock } from './api/chain-tips';
-import { HeapNode } from './utils/pairing-heap';
+import { HeapNode } from "./utils/pairing-heap";
 
 export interface PoolTag {
-  id: number;
-  uniqueId: number;
+  id: number; // mysql row id
   name: string;
   link: string;
   regexes: string; // JSON array
@@ -18,7 +16,6 @@ export interface PoolInfo {
   link: string;
   blockCount: number;
   slug: string;
-  avgMatchRate: number | null;
 }
 
 export interface PoolStats extends PoolInfo {
@@ -66,7 +63,6 @@ interface VinStrippedToScriptsig {
 
 interface VoutStrippedToScriptPubkey {
   scriptpubkey_address: string | undefined;
-  scriptpubkey_asm: string | undefined;
   value: number;
 }
 
@@ -148,45 +144,23 @@ export interface TransactionStripped {
 }
 
 export interface BlockExtension {
-  totalFees: number;
-  medianFee: number; // median fee rate
-  feeRange: number[]; // fee rate percentiles
-  reward: number;
-  matchRate: number | null;
-  similarity?: number;
-  pool: {
-    id: number; // Note - This is the `unique_id`, not to mix with the auto increment `id`
+  totalFees?: number;
+  medianFee?: number;
+  feeRange?: number[];
+  reward?: number;
+  coinbaseTx?: TransactionMinerInfo;
+  matchRate?: number;
+  pool?: {
+    id: number;
     name: string;
     slug: string;
   };
-  avgFee: number;
-  avgFeeRate: number;
-  coinbaseRaw: string;
-  orphans: OrphanedBlock[] | null;
-  coinbaseAddress: string | null;
-  coinbaseSignature: string | null;
-  coinbaseSignatureAscii: string | null;
-  virtualSize: number;
-  avgTxSize: number;
-  totalInputs: number;
-  totalOutputs: number;
-  totalOutputAmt: number;
-  medianFeeAmt: number | null; // median fee in sats
-  feePercentiles: number[] | null, // fee percentiles in sats
-  segwitTotalTxs: number;
-  segwitTotalSize: number;
-  segwitTotalWeight: number;
-  header: string;
-  utxoSetChange: number;
-  // Requires coinstatsindex, will be set to NULL otherwise
-  utxoSetSize: number | null;
-  totalInputAmt: number | null;
+  avgFee?: number;
+  avgFeeRate?: number;
+  coinbaseRaw?: string;
+  usd?: number | null;
 }
 
-/**
- * Note: Everything that is added in here will be automatically returned through
- * /api/v1/block and /api/v1/blocks APIs
- */
 export interface BlockExtended extends IEsploraApi.Block {
   extras: BlockExtension;
 }
@@ -294,12 +268,12 @@ interface RequiredParams {
 }
 
 export interface ILoadingIndicators { [name: string]: number; }
+export interface IConversionRates { [currency: string]: number; }
 
 export interface IBackendInfo {
   hostname: string;
   gitCommit: string;
   version: string;
-  lightning: boolean;
 }
 
 export interface IDifficultyAdjustment {
@@ -309,11 +283,9 @@ export interface IDifficultyAdjustment {
   remainingBlocks: number;
   remainingTime: number;
   previousRetarget: number;
-  previousTime: number;
   nextRetargetHeight: number;
   timeAvg: number;
   timeOffset: number;
-  expectedBlocks: number;
 }
 
 export interface IndexedDifficultyAdjustment {
@@ -365,4 +337,4 @@ export interface IOldestNodes {
   updatedAt?: number,
   city?: any,
   country?: any,
-}
+}

backend/src/repositories/BlocksRepository.ts

@@ -1,98 +1,34 @@
-import { BlockExtended, BlockExtension, BlockPrice } from '../mempool.interfaces';
+import { BlockExtended, BlockPrice } from '../mempool.interfaces';
 import DB from '../database';
 import logger from '../logger';
 import { Common } from '../api/common';
+import { prepareBlock } from '../utils/blocks-utils';
 import PoolsRepository from './PoolsRepository';
 import HashratesRepository from './HashratesRepository';
 import { escape } from 'mysql2';
 import BlocksSummariesRepository from './BlocksSummariesRepository';
 import DifficultyAdjustmentsRepository from './DifficultyAdjustmentsRepository';
-import bitcoinClient from '../api/bitcoin/bitcoin-client';
-import config from '../config';
-import chainTips from '../api/chain-tips';
-import blocks from '../api/blocks';
-import BlocksAuditsRepository from './BlocksAuditsRepository';
-
-const BLOCK_DB_FIELDS = `
-  blocks.hash AS id,
-  blocks.height,
-  blocks.version,
-  UNIX_TIMESTAMP(blocks.blockTimestamp) AS timestamp,
-  blocks.bits,
-  blocks.nonce,
-  blocks.difficulty,
-  blocks.merkle_root,
-  blocks.tx_count,
-  blocks.size,
-  blocks.weight,
-  blocks.previous_block_hash AS previousblockhash,
-  UNIX_TIMESTAMP(blocks.median_timestamp) AS mediantime,
-  blocks.fees AS totalFees,
-  blocks.median_fee AS medianFee,
-  blocks.fee_span AS feeRange,
-  blocks.reward,
-  pools.unique_id AS poolId,
-  pools.name AS poolName,
-  pools.slug AS poolSlug,
-  blocks.avg_fee AS avgFee,
-  blocks.avg_fee_rate AS avgFeeRate,
-  blocks.coinbase_raw AS coinbaseRaw,
-  blocks.coinbase_address AS coinbaseAddress,
-  blocks.coinbase_signature AS coinbaseSignature,
-  blocks.coinbase_signature_ascii AS coinbaseSignatureAscii,
-  blocks.avg_tx_size AS avgTxSize,
-  blocks.total_inputs AS totalInputs,
-  blocks.total_outputs AS totalOutputs,
-  blocks.total_output_amt AS totalOutputAmt,
-  blocks.median_fee_amt AS medianFeeAmt,
-  blocks.fee_percentiles AS feePercentiles,
-  blocks.segwit_total_txs AS segwitTotalTxs,
-  blocks.segwit_total_size AS segwitTotalSize,
-  blocks.segwit_total_weight AS segwitTotalWeight,
-  blocks.header,
-  blocks.utxoset_change AS utxoSetChange,
-  blocks.utxoset_size AS utxoSetSize,
-  blocks.total_input_amt AS totalInputAmts
-`;
 
 class BlocksRepository {
   /**
    * Save indexed block data in the database
    */
   public async $saveBlockInDatabase(block: BlockExtended) {
-    const truncatedCoinbaseSignature = block?.extras?.coinbaseSignature?.substring(0, 500);
-    const truncatedCoinbaseSignatureAscii = block?.extras?.coinbaseSignatureAscii?.substring(0, 500);
-
     try {
       const query = `INSERT INTO blocks(
-        height,             hash,                blockTimestamp,    size,
-        weight,             tx_count,            coinbase_raw,      difficulty,
-        pool_id,            fees,                fee_span,          median_fee,
-        reward,             version,             bits,              nonce,
-        merkle_root,        previous_block_hash, avg_fee,           avg_fee_rate,
-        median_timestamp,   header,              coinbase_address,
-        coinbase_signature, utxoset_size,        utxoset_change,    avg_tx_size,
-        total_inputs,       total_outputs,       total_input_amt,   total_output_amt,
-        fee_percentiles,    segwit_total_txs,    segwit_total_size, segwit_total_weight,
-        median_fee_amt,     coinbase_signature_ascii
+        height,           hash,                blockTimestamp, size,
+        weight,           tx_count,            coinbase_raw,   difficulty,
+        pool_id,          fees,                fee_span,       median_fee,
+        reward,           version,             bits,           nonce,
+        merkle_root,      previous_block_hash, avg_fee,        avg_fee_rate
       ) VALUE (
         ?, ?, FROM_UNIXTIME(?), ?,
         ?, ?, ?, ?,
         ?, ?, ?, ?,
         ?, ?, ?, ?,
-        ?, ?, ?, ?,
-        FROM_UNIXTIME(?), ?, ?,
-        ?, ?, ?, ?,
-        ?, ?, ?, ?,
-        ?, ?, ?, ?,
-        ?, ?
+        ?, ?, ?, ?
       )`;
 
-      const poolDbId = await PoolsRepository.$getPoolByUniqueId(block.extras.pool.id);
-      if (!poolDbId) {
-        throw Error(`Could not find a mining pool with the unique_id = ${block.extras.pool.id}. This error should never be printed.`);
-      }
-
       const params: any[] = [
         block.height,
         block.id,
@@ -102,7 +38,7 @@ class BlocksRepository {
         block.tx_count,
         block.extras.coinbaseRaw,
         block.difficulty,
-        poolDbId.id,
+        block.extras.pool?.id, // Should always be set to something
         block.extras.totalFees,
         JSON.stringify(block.extras.feeRange),
         block.extras.medianFee,
@@ -114,63 +50,19 @@ class BlocksRepository {
         block.previousblockhash,
         block.extras.avgFee,
         block.extras.avgFeeRate,
-        block.mediantime,
-        block.extras.header,
-        block.extras.coinbaseAddress,
-        truncatedCoinbaseSignature,
-        block.extras.utxoSetSize,
-        block.extras.utxoSetChange,
-        block.extras.avgTxSize,
-        block.extras.totalInputs,
-        block.extras.totalOutputs,
-        block.extras.totalInputAmt,
-        block.extras.totalOutputAmt,
-        block.extras.feePercentiles ? JSON.stringify(block.extras.feePercentiles) : null,
-        block.extras.segwitTotalTxs,
-        block.extras.segwitTotalSize,
-        block.extras.segwitTotalWeight,
-        block.extras.medianFeeAmt,
-        truncatedCoinbaseSignatureAscii,
       ];
 
       await DB.query(query, params);
     } catch (e: any) {
       if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
-        logger.debug(`$saveBlockInDatabase() - Block ${block.height} has already been indexed, ignoring`, logger.tags.mining);
+        logger.debug(`$saveBlockInDatabase() - Block ${block.height} has already been indexed, ignoring`);
       } else {
-        logger.err('Cannot save indexed block into db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
+        logger.err('Cannot save indexed block into db. Reason: ' + (e instanceof Error ? e.message : e));
         throw e;
       }
     }
   }
 
-  /**
-   * Save newly indexed data from core coinstatsindex
-   * 
-   * @param utxoSetSize 
-   * @param totalInputAmt 
-   */
-  public async $updateCoinStatsIndexData(blockHash: string, utxoSetSize: number,
-    totalInputAmt: number
-  ) : Promise<void> {
-    try {
-      const query = `
-        UPDATE blocks
-        SET utxoset_size = ?, total_input_amt = ?
-        WHERE hash = ?
-      `;
-      const params: any[] = [
-        utxoSetSize,
-        totalInputAmt,
-        blockHash
-      ];
-      await DB.query(query, params);
-    } catch (e: any) {
-      logger.err('Cannot update indexed block coinstatsindex. Reason: ' + (e instanceof Error ? e.message : e));
-      throw e;
-    }
-  }
-
   /**
    * Get all block height that have not been indexed between [startHeight, endHeight]
    */
@@ -330,55 +222,6 @@ class BlocksRepository {
     }
   }
 
-  /**
-   * Get average block health for all blocks for a single pool
-   */
-  public async $getAvgBlockHealthPerPoolId(poolId: number): Promise<number> {
-    const params: any[] = [];
-    const query = `
-      SELECT AVG(blocks_audits.match_rate) AS avg_match_rate
-      FROM blocks
-      JOIN blocks_audits ON blocks.height = blocks_audits.height
-      WHERE blocks.pool_id = ?
-    `;
-    params.push(poolId);
-
-    try {
-      const [rows] = await DB.query(query, params);
-      if (!rows[0] || !rows[0].avg_match_rate) {
-        return 0;
-      }
-      return Math.round(rows[0].avg_match_rate * 100) / 100;
-    } catch (e) {
-      logger.err(`Cannot get average block health for pool id ${poolId}. Reason: ` + (e instanceof Error ? e.message : e));
-      throw e;
-    }
-  }
-
-  /**
-   * Get average block health for all blocks for a single pool
-   */
-  public async $getTotalRewardForPoolId(poolId: number): Promise<number> {
-    const params: any[] = [];
-    const query = `
-      SELECT sum(reward) as total_reward
-      FROM blocks
-      WHERE blocks.pool_id = ?
-    `;
-    params.push(poolId);
-
-    try {
-      const [rows] = await DB.query(query, params);
-      if (!rows[0] || !rows[0].total_reward) {
-        return 0;
-      }
-      return rows[0].total_reward;
-    } catch (e) {
-      logger.err(`Cannot get total reward for pool id ${poolId}. Reason: ` + (e instanceof Error ? e.message : e));
-      throw e;
-    }
-  }
-
   /**
    * Get the oldest indexed block
    */
@@ -405,17 +248,34 @@ class BlocksRepository {
   /**
    * Get blocks mined by a specific mining pool
    */
-  public async $getBlocksByPool(slug: string, startHeight?: number): Promise<BlockExtended[]> {
+  public async $getBlocksByPool(slug: string, startHeight?: number): Promise<object[]> {
     const pool = await PoolsRepository.$getPool(slug);
     if (!pool) {
       throw new Error('This mining pool does not exist ' + escape(slug));
     }
 
     const params: any[] = [];
-    let query = `
-      SELECT ${BLOCK_DB_FIELDS}
+    let query = ` SELECT
+      blocks.height,
+      hash as id,
+      UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp,
+      size,
+      weight,
+      tx_count,
+      coinbase_raw,
+      difficulty,
+      fees,
+      fee_span,
+      median_fee,
+      reward,
+      version,
+      bits,
+      nonce,
+      merkle_root,
+      previous_block_hash as previousblockhash,
+      avg_fee,
+      avg_fee_rate
       FROM blocks
-      JOIN pools ON blocks.pool_id = pools.id
       WHERE pool_id = ?`;
     params.push(pool.id);
 
@@ -428,11 +288,11 @@ class BlocksRepository {
       LIMIT 10`;
 
     try {
-      const [rows]: any[] = await DB.query(query, params);
+      const [rows] = await DB.query(query, params);
 
       const blocks: BlockExtended[] = [];
-      for (const block of rows) {
-        blocks.push(await this.formatDbBlockIntoExtendedBlock(block));
+      for (const block of <object[]>rows) {
+        blocks.push(prepareBlock(block));
       }
 
       return blocks;
@@ -445,21 +305,46 @@ class BlocksRepository {
   /**
    * Get one block by height
    */
-  public async $getBlockByHeight(height: number): Promise<BlockExtended | null> {
+  public async $getBlockByHeight(height: number): Promise<object | null> {
     try {
-      const [rows]: any[] = await DB.query(`
-        SELECT ${BLOCK_DB_FIELDS}
+      const [rows]: any[] = await DB.query(`SELECT
+        blocks.height,
+        hash,
+        hash as id,
+        UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp,
+        size,
+        weight,
+        tx_count,
+        coinbase_raw,
+        difficulty,
+        pools.id as pool_id,
+        pools.name as pool_name,
+        pools.link as pool_link,
+        pools.slug as pool_slug,
+        pools.addresses as pool_addresses,
+        pools.regexes as pool_regexes,
+        fees,
+        fee_span,
+        median_fee,
+        reward,
+        version,
+        bits,
+        nonce,
+        merkle_root,
+        previous_block_hash as previousblockhash,
+        avg_fee,
+        avg_fee_rate
         FROM blocks
         JOIN pools ON blocks.pool_id = pools.id
-        WHERE blocks.height = ?`,
-        [height]
-      );
+        WHERE blocks.height = ${height}
+      `);
 
       if (rows.length <= 0) {
         return null;
       }
 
-      return await this.formatDbBlockIntoExtendedBlock(rows[0]);  
+      rows[0].fee_span = JSON.parse(rows[0].fee_span);
+      return rows[0];
     } catch (e) {
       logger.err(`Cannot get indexed block ${height}. Reason: ` + (e instanceof Error ? e.message : e));
       throw e;
@@ -472,7 +357,10 @@ class BlocksRepository {
   public async $getBlockByHash(hash: string): Promise<object | null> {
     try {
       const query = `
-        SELECT ${BLOCK_DB_FIELDS}
+        SELECT *, blocks.height, UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp, hash as id,
+        pools.id as pool_id, pools.name as pool_name, pools.link as pool_link, pools.slug as pool_slug,
+        pools.addresses as pool_addresses, pools.regexes as pool_regexes,
+        previous_block_hash as previousblockhash
         FROM blocks
         JOIN pools ON blocks.pool_id = pools.id
         WHERE hash = ?;
@@ -482,8 +370,9 @@ class BlocksRepository {
       if (rows.length <= 0) {
         return null;
       }
- 
-      return await this.formatDbBlockIntoExtendedBlock(rows[0]);
+
+      rows[0].fee_span = JSON.parse(rows[0].fee_span);
+      return rows[0];
     } catch (e) {
       logger.err(`Cannot get indexed block ${hash}. Reason: ` + (e instanceof Error ? e.message : e));
       throw e;
@@ -574,15 +463,8 @@ class BlocksRepository {
   public async $validateChain(): Promise<boolean> {
     try {
       const start = new Date().getTime();
-      const [blocks]: any[] = await DB.query(`
-        SELECT
-          height,
-          hash,
-          previous_block_hash,
-          UNIX_TIMESTAMP(blockTimestamp) AS timestamp
-        FROM blocks
-        ORDER BY height
-      `);
+      const [blocks]: any[] = await DB.query(`SELECT height, hash, previous_block_hash,
+        UNIX_TIMESTAMP(blockTimestamp) as timestamp FROM blocks ORDER BY height`);
 
       let partialMsg = false;
       let idx = 1;
@@ -785,34 +667,18 @@ class BlocksRepository {
    */
    public async $getCPFPUnindexedBlocks(): Promise<any[]> {
     try {
-      const blockchainInfo = await bitcoinClient.getBlockchainInfo();
-      const currentBlockHeight = blockchainInfo.blocks;
-      let indexingBlockAmount = Math.min(config.MEMPOOL.INDEXING_BLOCKS_AMOUNT, currentBlockHeight);
-      if (indexingBlockAmount <= -1) {
-        indexingBlockAmount = currentBlockHeight + 1;
-      }
-      const minHeight = Math.max(0, currentBlockHeight - indexingBlockAmount + 1);
-
-      const [rows]: any[] = await DB.query(`
-        SELECT height
-        FROM compact_cpfp_clusters
-        WHERE height <= ? AND height >= ?
-        GROUP BY height
-        ORDER BY height DESC;
-      `, [currentBlockHeight, minHeight]);
-
-      const indexedHeights = {};
-      rows.forEach((row) => { indexedHeights[row.height] = true; });
-      const allHeights: number[] = Array.from(Array(currentBlockHeight - minHeight + 1).keys(), n => n + minHeight).reverse();
-      const unindexedHeights = allHeights.filter(x => !indexedHeights[x]);
-
-      return unindexedHeights;
+      const [rows]: any = await DB.query(`SELECT height, hash FROM blocks WHERE cpfp_indexed = 0 ORDER BY height DESC`);
+      return rows;
     } catch (e) {
       logger.err('Cannot fetch CPFP unindexed blocks. Reason: ' + (e instanceof Error ? e.message : e));
       throw e;
     }
   }
 
+  public async $setCPFPIndexed(hash: string): Promise<void> {
+    await DB.query(`UPDATE blocks SET cpfp_indexed = 1 WHERE hash = ?`, [hash]);
+  }
+
   /**
    * Return the oldest block  from a consecutive chain of block from the most recent one
    */
@@ -857,7 +723,7 @@ class BlocksRepository {
     try {
       let query = `INSERT INTO blocks_prices(height, price_id) VALUES`;
       for (const price of blockPrices) {
-        query += ` (${price.height}, ${price.priceId}),`;
+        query += ` (${price.height}, ${price.priceId}),`
       }
       query = query.slice(0, -1);
       await DB.query(query);
@@ -870,132 +736,6 @@ class BlocksRepository {
       }
     }
   }
-
-  /**
-   * Get all indexed blocsk with missing coinstatsindex data
-   */
-  public async $getBlocksMissingCoinStatsIndex(maxHeight: number, minHeight: number): Promise<any> {
-    try {
-      const [blocks] = await DB.query(`
-        SELECT height, hash
-        FROM blocks
-        WHERE height >= ${minHeight} AND height <= ${maxHeight} AND
-          (utxoset_size IS NULL OR total_input_amt IS NULL)
-      `);
-      return blocks;
-    } catch (e) {
-      logger.err(`Cannot get blocks with missing coinstatsindex. Reason: ` + (e instanceof Error ? e.message : e));
-      throw e;
-    }
-  }
-
-  /**
-   * Save indexed median fee to avoid recomputing it later
-   * 
-   * @param id 
-   * @param feePercentiles 
-   */
-  public async $saveFeePercentilesForBlockId(id: string, feePercentiles: number[]): Promise<void> {
-    try {
-      await DB.query(`
-        UPDATE blocks SET fee_percentiles = ?, median_fee_amt = ?
-        WHERE hash = ?`,
-        [JSON.stringify(feePercentiles), feePercentiles[3], id]
-      );
-    } catch (e) {
-      logger.err(`Cannot update block fee_percentiles. Reason: ` + (e instanceof Error ? e.message : e));
-      throw e;
-    }
-  }
-
-  /**
-   * Convert a mysql row block into a BlockExtended. Note that you
-   * must provide the correct field into dbBlk object param
-   * 
-   * @param dbBlk 
-   */
-  private async formatDbBlockIntoExtendedBlock(dbBlk: any): Promise<BlockExtended> {
-    const blk: Partial<BlockExtended> = {};
-    const extras: Partial<BlockExtension> = {};
-
-    // IEsploraApi.Block
-    blk.id = dbBlk.id;
-    blk.height = dbBlk.height;
-    blk.version = dbBlk.version;
-    blk.timestamp = dbBlk.timestamp;
-    blk.bits = dbBlk.bits;
-    blk.nonce = dbBlk.nonce;
-    blk.difficulty = dbBlk.difficulty;
-    blk.merkle_root = dbBlk.merkle_root;
-    blk.tx_count = dbBlk.tx_count;
-    blk.size = dbBlk.size;
-    blk.weight = dbBlk.weight;
-    blk.previousblockhash = dbBlk.previousblockhash;
-    blk.mediantime = dbBlk.mediantime;
-    
-    // BlockExtension
-    extras.totalFees = dbBlk.totalFees;
-    extras.medianFee = dbBlk.medianFee;
-    extras.feeRange = JSON.parse(dbBlk.feeRange);
-    extras.reward = dbBlk.reward;
-    extras.pool = {
-      id: dbBlk.poolId,
-      name: dbBlk.poolName,
-      slug: dbBlk.poolSlug,
-    };
-    extras.avgFee = dbBlk.avgFee;
-    extras.avgFeeRate = dbBlk.avgFeeRate;
-    extras.coinbaseRaw = dbBlk.coinbaseRaw;
-    extras.coinbaseAddress = dbBlk.coinbaseAddress;
-    extras.coinbaseSignature = dbBlk.coinbaseSignature;
-    extras.coinbaseSignatureAscii = dbBlk.coinbaseSignatureAscii;
-    extras.avgTxSize = dbBlk.avgTxSize;
-    extras.totalInputs = dbBlk.totalInputs;
-    extras.totalOutputs = dbBlk.totalOutputs;
-    extras.totalOutputAmt = dbBlk.totalOutputAmt;
-    extras.medianFeeAmt = dbBlk.medianFeeAmt;
-    extras.feePercentiles = JSON.parse(dbBlk.feePercentiles);
-    extras.segwitTotalTxs = dbBlk.segwitTotalTxs;
-    extras.segwitTotalSize = dbBlk.segwitTotalSize;
-    extras.segwitTotalWeight = dbBlk.segwitTotalWeight;
-    extras.header = dbBlk.header,
-    extras.utxoSetChange = dbBlk.utxoSetChange;
-    extras.utxoSetSize = dbBlk.utxoSetSize;
-    extras.totalInputAmt = dbBlk.totalInputAmt;
-    extras.virtualSize = dbBlk.weight / 4.0;
-
-    // Re-org can happen after indexing so we need to always get the
-    // latest state from core
-    extras.orphans = chainTips.getOrphanedBlocksAtHeight(dbBlk.height);
-
-    // Match rate is not part of the blocks table, but it is part of APIs so we must include it
-    extras.matchRate = null;
-    if (config.MEMPOOL.AUDIT) {
-      const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(dbBlk.id);
-      if (auditScore != null) {
-        extras.matchRate = auditScore.matchRate;
-      }
-    }
-
-    // If we're missing block summary related field, check if we can populate them on the fly now
-    if (Common.blocksSummariesIndexingEnabled() &&
-      (extras.medianFeeAmt === null || extras.feePercentiles === null))
-    {
-      extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id);
-      if (extras.feePercentiles === null) {
-        const block = await bitcoinClient.getBlock(dbBlk.id, 2);
-        const summary = blocks.summarizeBlock(block);
-        await BlocksSummariesRepository.$saveSummary({ height: block.height, mined: summary });
-        extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id);
-      }
-      if (extras.feePercentiles !== null) {
-        extras.medianFeeAmt = extras.feePercentiles[3];
-      }
-    }
-
-    blk.extras = <BlockExtension>extras;
-    return <BlockExtended>blk;
-  }
 }
 
 export default new BlocksRepository();

backend/src/repositories/BlocksSummariesRepository.ts

@@ -80,48 +80,6 @@ class BlocksSummariesRepository {
       logger.err('Cannot delete indexed blocks summaries. Reason: ' + (e instanceof Error ? e.message : e));
     }
   }
-
-  /**
-   * Get the fee percentiles if the block has already been indexed, [] otherwise
-   * 
-   * @param id 
-   */
-  public async $getFeePercentilesByBlockId(id: string): Promise<number[] | null> {
-    try {
-      const [rows]: any[] = await DB.query(`
-        SELECT transactions
-        FROM blocks_summaries
-        WHERE id = ?`,
-        [id]
-      );
-      if (rows === null || rows.length === 0) {
-        return null;
-      }
-
-      const transactions = JSON.parse(rows[0].transactions);
-      if (transactions === null) {
-        return null;
-      }
-
-      transactions.shift(); // Ignore coinbase
-      transactions.sort((a: any, b: any) => a.fee - b.fee);
-      const fees = transactions.map((t: any) => t.fee);
-
-      return [
-        fees[0] ?? 0, // min
-        fees[Math.max(0, Math.floor(fees.length * 0.1) - 1)] ?? 0, // 10th
-        fees[Math.max(0, Math.floor(fees.length * 0.25) - 1)] ?? 0, // 25th
-        fees[Math.max(0, Math.floor(fees.length * 0.5) - 1)] ?? 0, // median
-        fees[Math.max(0, Math.floor(fees.length * 0.75) - 1)] ?? 0, // 75th
-        fees[Math.max(0, Math.floor(fees.length * 0.9) - 1)] ?? 0, // 90th
-        fees[fees.length - 1] ?? 0, // max
-      ];
-
-    } catch (e) {
-      logger.err(`Cannot get block summaries transactions. Reason: ` + (e instanceof Error ? e.message : e));
-      return null;
-    }
-  }
 }
 
 export default new BlocksSummariesRepository();

backend/src/repositories/CpfpRepository.ts

@@ -1,154 +1,34 @@
-import cluster, { Cluster } from 'cluster';
-import { RowDataPacket } from 'mysql2';
 import DB from '../database';
 import logger from '../logger';
 import { Ancestor } from '../mempool.interfaces';
-import transactionRepository from '../repositories/TransactionRepository';
 
 class CpfpRepository {
-  public async $saveCluster(clusterRoot: string, height: number, txs: Ancestor[], effectiveFeePerVsize: number): Promise<boolean> {
-    if (!txs[0]) {
-      return false;
-    }
-    // skip clusters of transactions with the same fees
-    const roundedEffectiveFee = Math.round(effectiveFeePerVsize * 100) / 100;
-    const equalFee = txs.reduce((acc, tx) => {
-      return (acc && Math.round(((tx.fee || 0) / (tx.weight / 4)) * 100) / 100 === roundedEffectiveFee);
-    }, true);
-    if (equalFee) {
-      return false;
-    }
-
+  public async $saveCluster(height: number, txs: Ancestor[], effectiveFeePerVsize: number): Promise<void> {
     try {
-      const packedTxs = Buffer.from(this.pack(txs));
+      const txsJson = JSON.stringify(txs);
       await DB.query(
         `
-          INSERT INTO compact_cpfp_clusters(root, height, txs, fee_rate)
-          VALUE (UNHEX(?), ?, ?, ?)
+          INSERT INTO cpfp_clusters(root, height, txs, fee_rate)
+          VALUE (?, ?, ?, ?)
           ON DUPLICATE KEY UPDATE
             height = ?,
             txs = ?,
             fee_rate = ?
         `,
-        [clusterRoot, height, packedTxs, effectiveFeePerVsize, height, packedTxs, effectiveFeePerVsize]
+        [txs[0].txid, height, txsJson, effectiveFeePerVsize, height, txsJson, effectiveFeePerVsize, height]
       );
-      const maxChunk = 10;
-      let chunkIndex = 0;
-      while (chunkIndex < txs.length) {
-        const chunk = txs.slice(chunkIndex, chunkIndex + maxChunk).map(tx => {
-          return { txid: tx.txid, cluster: clusterRoot };
-        });
-        await transactionRepository.$batchSetCluster(chunk);
-        chunkIndex += maxChunk;
-      }
-      return true;
     } catch (e: any) {
       logger.err(`Cannot save cpfp cluster into db. Reason: ` + (e instanceof Error ? e.message : e));
       throw e;
     }
   }
 
-  public async $batchSaveClusters(clusters: { root: string, height: number, txs: any, effectiveFeePerVsize: number}[]): Promise<boolean> {
-    try {
-      const clusterValues: any[] = [];
-      const txs: any[] = [];
-
-      for (const cluster of clusters) {
-        if (cluster.txs?.length > 1) {
-          const roundedEffectiveFee = Math.round(cluster.effectiveFeePerVsize * 100) / 100;
-          const equalFee = cluster.txs.reduce((acc, tx) => {
-            return (acc && Math.round(((tx.fee || 0) / (tx.weight / 4)) * 100) / 100 === roundedEffectiveFee);
-          }, true);
-          if (!equalFee) {
-            clusterValues.push([
-              cluster.root,
-              cluster.height,
-              Buffer.from(this.pack(cluster.txs)),
-              cluster.effectiveFeePerVsize
-            ]);
-            for (const tx of cluster.txs) {
-              txs.push({ txid: tx.txid, cluster: cluster.root });
-            }
-          }
-        }
-      }
-
-      if (!clusterValues.length) {
-        return false;
-      }
-
-      const maxChunk = 100;
-      let chunkIndex = 0;
-      // insert transactions in batches of up to 100 rows
-      while (chunkIndex < txs.length) {
-        const chunk = txs.slice(chunkIndex, chunkIndex + maxChunk);
-        await transactionRepository.$batchSetCluster(chunk);
-        chunkIndex += maxChunk;
-      }
-
-      chunkIndex = 0;
-      // insert clusters in batches of up to 100 rows
-      while (chunkIndex < clusterValues.length) {
-        const chunk = clusterValues.slice(chunkIndex, chunkIndex + maxChunk);
-        let query = `
-            INSERT IGNORE INTO compact_cpfp_clusters(root, height, txs, fee_rate)
-            VALUES
-        `;
-        query += chunk.map(chunk => {
-          return (' (UNHEX(?), ?, ?, ?)');
-        }) + ';';
-        const values = chunk.flat();
-        await DB.query(
-          query,
-          values
-        );
-        chunkIndex += maxChunk;
-      }
-      return true;
-    } catch (e: any) {
-      logger.err(`Cannot save cpfp clusters into db. Reason: ` + (e instanceof Error ? e.message : e));
-      throw e;
-    }
-  }
-
-  public async $getCluster(clusterRoot: string): Promise<Cluster | void> {
-    const [clusterRows]: any = await DB.query(
-      `
-        SELECT *
-        FROM compact_cpfp_clusters
-        WHERE root = UNHEX(?)
-      `,
-      [clusterRoot]
-    );
-    const cluster = clusterRows[0];
-    if (cluster?.txs) {
-      cluster.txs = this.unpack(cluster.txs);
-      return cluster;
-    }
-    return;
-  }
-
   public async $deleteClustersFrom(height: number): Promise<void> {
     logger.info(`Delete newer cpfp clusters from height ${height} from the database`);
     try {
-      const [rows] = await DB.query(
-        `
-          SELECT txs, height, root from compact_cpfp_clusters
-          WHERE height >= ?
-        `,
-        [height]
-      ) as RowDataPacket[][];
-      if (rows?.length) {
-        for (const clusterToDelete of rows) {
-          const txs = this.unpack(clusterToDelete?.txs);
-          for (const tx of txs) {
-            await transactionRepository.$removeTransaction(tx.txid);
-          }
-        }
-      }
       await DB.query(
         `
-          DELETE from compact_cpfp_clusters
+          DELETE from cpfp_clusters
           WHERE height >= ?
         `,
         [height]
@@ -158,75 +38,6 @@ class CpfpRepository {
       throw e;
     }
   }
-
-  // insert a dummy row to mark that we've indexed as far as this block
-  public async $insertProgressMarker(height: number): Promise<void> {
-    try {
-      const [rows]: any = await DB.query(
-        `
-          SELECT root
-          FROM compact_cpfp_clusters
-          WHERE height = ?
-        `,
-        [height]
-      );
-      if (!rows?.length) {
-        const rootBuffer = Buffer.alloc(32);
-        rootBuffer.writeInt32LE(height);
-        await DB.query(
-          `
-            INSERT INTO compact_cpfp_clusters(root, height, fee_rate)
-            VALUE (?, ?, ?)
-          `,
-          [rootBuffer, height, 0]
-        );
-      }
-    } catch (e: any) {
-      logger.err(`Cannot insert cpfp progress marker. Reason: ` + (e instanceof Error ? e.message : e));
-      throw e;
-    }
-  }
-
-  public pack(txs: Ancestor[]): ArrayBuffer {
-    const buf = new ArrayBuffer(44 * txs.length);
-    const view = new DataView(buf);
-    txs.forEach((tx, i) => {
-      const offset = i * 44;
-      for (let x = 0; x < 32; x++) {
-        // store txid in little-endian
-        view.setUint8(offset + (31 - x), parseInt(tx.txid.slice(x * 2, (x * 2) + 2), 16));
-      }
-      view.setUint32(offset + 32, tx.weight);
-      view.setBigUint64(offset + 36, BigInt(Math.round(tx.fee)));
-    });
-    return buf;
-  }
-
-  public unpack(buf: Buffer): Ancestor[] {
-    if (!buf) {
-      return [];
-    }
-
-    try {
-      const arrayBuffer = buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength);
-      const txs: Ancestor[] = [];
-      const view = new DataView(arrayBuffer);
-      for (let offset = 0; offset < arrayBuffer.byteLength; offset += 44) {
-        const txid = Array.from(new Uint8Array(arrayBuffer, offset, 32)).reverse().map(b => b.toString(16).padStart(2, '0')).join('');
-        const weight = view.getUint32(offset + 32);
-        const fee = Number(view.getBigUint64(offset + 36));
-        txs.push({
-          txid,
-          weight,
-          fee
-        });
-      }
-      return txs;
-    } catch (e) {
-      logger.warn(`Failed to unpack CPFP cluster. Reason: ` + (e instanceof Error ? e.message : e));
-      return [];
-    }
-  }
 }
 
 export default new CpfpRepository();

backend/src/repositories/DifficultyAdjustmentsRepository.ts

@@ -1,4 +1,5 @@
 import { Common } from '../api/common';
+import config from '../config';
 import DB from '../database';
 import logger from '../logger';
 import { IndexedDifficultyAdjustment } from '../mempool.interfaces';
@@ -20,9 +21,9 @@ class DifficultyAdjustmentsRepository {
       await DB.query(query, params);
     } catch (e: any) {
       if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
-        logger.debug(`Cannot save difficulty adjustment at block ${adjustment.height}, already indexed, ignoring`, logger.tags.mining);
+        logger.debug(`Cannot save difficulty adjustment at block ${adjustment.height}, already indexed, ignoring`);
       } else {
-        logger.err(`Cannot save difficulty adjustment at block ${adjustment.height}. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.mining);
+        logger.err(`Cannot save difficulty adjustment at block ${adjustment.height}. Reason: ${e instanceof Error ? e.message : e}`);
         throw e;
       }
     }
@@ -54,7 +55,7 @@ class DifficultyAdjustmentsRepository {
       const [rows] = await DB.query(query);
       return rows as IndexedDifficultyAdjustment[];
     } catch (e) {
-      logger.err(`Cannot get difficulty adjustments from the database. Reason: ` + (e instanceof Error ? e.message : e), logger.tags.mining);
+      logger.err(`Cannot get difficulty adjustments from the database. Reason: ` + (e instanceof Error ? e.message : e));
       throw e;
     }
   }
@@ -83,7 +84,7 @@ class DifficultyAdjustmentsRepository {
       const [rows] = await DB.query(query);
       return rows as IndexedDifficultyAdjustment[];
     } catch (e) {
-      logger.err(`Cannot get difficulty adjustments from the database. Reason: ` + (e instanceof Error ? e.message : e), logger.tags.mining);
+      logger.err(`Cannot get difficulty adjustments from the database. Reason: ` + (e instanceof Error ? e.message : e));
       throw e;
     }
   }
@@ -93,27 +94,27 @@ class DifficultyAdjustmentsRepository {
       const [rows]: any[] = await DB.query(`SELECT height FROM difficulty_adjustments`);
       return rows.map(block => block.height);
     } catch (e: any) {
-      logger.err(`Cannot get difficulty adjustment block heights. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.mining);
+      logger.err(`Cannot get difficulty adjustment block heights. Reason: ${e instanceof Error ? e.message : e}`);
       throw e;
     }
   }
 
   public async $deleteAdjustementsFromHeight(height: number): Promise<void> {
     try {
-      logger.info(`Delete newer difficulty adjustments from height ${height} from the database`, logger.tags.mining);
+      logger.info(`Delete newer difficulty adjustments from height ${height} from the database`);
       await DB.query(`DELETE FROM difficulty_adjustments WHERE height >= ?`, [height]);
     } catch (e: any) {
-      logger.err(`Cannot delete difficulty adjustments from the database. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.mining);
+      logger.err(`Cannot delete difficulty adjustments from the database. Reason: ${e instanceof Error ? e.message : e}`);
       throw e;
     }
   }
 
   public async $deleteLastAdjustment(): Promise<void> {
     try {
-      logger.info(`Delete last difficulty adjustment from the database`, logger.tags.mining);
+      logger.info(`Delete last difficulty adjustment from the database`);
       await DB.query(`DELETE FROM difficulty_adjustments ORDER BY time LIMIT 1`);
     } catch (e: any) {
-      logger.err(`Cannot delete last difficulty adjustment from the database. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.mining);
+      logger.err(`Cannot delete last difficulty adjustment from the database. Reason: ${e instanceof Error ? e.message : e}`);
       throw e;
     }
   }

backend/src/repositories/HashratesRepository.ts

@@ -1,6 +1,5 @@
 import { escape } from 'mysql2';
 import { Common } from '../api/common';
-import mining from '../api/mining/mining';
 import DB from '../database';
 import logger from '../logger';
 import PoolsRepository from './PoolsRepository';
@@ -25,7 +24,7 @@ class HashratesRepository {
     try {
       await DB.query(query);
     } catch (e: any) {
-      logger.err('Cannot save indexed hashrate into db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
+      logger.err('Cannot save indexed hashrate into db. Reason: ' + (e instanceof Error ? e.message : e));
       throw e;
     }
   }
@@ -51,7 +50,7 @@ class HashratesRepository {
       const [rows]: any[] = await DB.query(query);
       return rows;
     } catch (e) {
-      logger.err('Cannot fetch network hashrate history. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
+      logger.err('Cannot fetch network hashrate history. Reason: ' + (e instanceof Error ? e.message : e));
       throw e;
     }
   }
@@ -78,7 +77,7 @@ class HashratesRepository {
       const [rows]: any[] = await DB.query(query);
       return rows;
     } catch (e) {
-      logger.err('Cannot fetch network hashrate history. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
+      logger.err('Cannot fetch network hashrate history. Reason: ' + (e instanceof Error ? e.message : e));
       throw e;
     }
   }
@@ -93,7 +92,7 @@ class HashratesRepository {
       const [rows]: any[] = await DB.query(query);
       return rows.map(row => row.timestamp);
     } catch (e) {
-      logger.err('Cannot retreive indexed weekly hashrate timestamps. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
+      logger.err('Cannot retreive indexed weekly hashrate timestamps. Reason: ' + (e instanceof Error ? e.message : e));
       throw e;
     }
   }
@@ -128,7 +127,7 @@ class HashratesRepository {
       const [rows]: any[] = await DB.query(query);
       return rows;
     } catch (e) {
-      logger.err('Cannot fetch weekly pools hashrate history. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
+      logger.err('Cannot fetch weekly pools hashrate history. Reason: ' + (e instanceof Error ? e.message : e));
       throw e;
     }
   }
@@ -158,7 +157,7 @@ class HashratesRepository {
       const [rows]: any[] = await DB.query(query, [pool.id]);
       boundaries = rows[0];
     } catch (e) {
-      logger.err('Cannot fetch hashrate start/end timestamps for this pool. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
+      logger.err('Cannot fetch hashrate start/end timestamps for this pool. Reason: ' + (e instanceof Error ? e.message : e));
     }
 
     // Get hashrates entries between boundaries
@@ -173,7 +172,21 @@ class HashratesRepository {
       const [rows]: any[] = await DB.query(query, [boundaries.firstTimestamp, boundaries.lastTimestamp, pool.id]);
       return rows;
     } catch (e) {
-      logger.err('Cannot fetch pool hashrate history for this pool. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
+      logger.err('Cannot fetch pool hashrate history for this pool. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Set latest run timestamp
+   */
+  public async $setLatestRun(key: string, val: number) {
+    const query = `UPDATE state SET number = ? WHERE name = ?`;
+
+    try {
+      await DB.query(query, [val, key]);
+    } catch (e) {
+      logger.err(`Cannot set last indexing run for ${key}. Reason: ` + (e instanceof Error ? e.message : e));
       throw e;
     }
   }
@@ -192,7 +205,7 @@ class HashratesRepository {
       }
       return rows[0]['number'];
     } catch (e) {
-      logger.err(`Cannot retrieve last indexing run for ${key}. Reason: ` + (e instanceof Error ? e.message : e), logger.tags.mining);
+      logger.err(`Cannot retrieve last indexing run for ${key}. Reason: ` + (e instanceof Error ? e.message : e));
       throw e;
     }
   }
@@ -201,7 +214,7 @@ class HashratesRepository {
    * Delete most recent data points for re-indexing
    */
   public async $deleteLastEntries() {
-    logger.info(`Delete latest hashrates data points from the database`, logger.tags.mining);
+    logger.info(`Delete latest hashrates data points from the database`);
 
     try {
       const [rows]: any[] = await DB.query(`SELECT MAX(hashrate_timestamp) as timestamp FROM hashrates GROUP BY type`);
@@ -209,10 +222,10 @@ class HashratesRepository {
         await DB.query(`DELETE FROM hashrates WHERE hashrate_timestamp = ?`, [row.timestamp]);
       }
       // Re-run the hashrate indexing to fill up missing data
-      mining.lastHashrateIndexingDate = null;
-      mining.lastWeeklyHashrateIndexingDate = null;
+      await this.$setLatestRun('last_hashrates_indexing', 0);
+      await this.$setLatestRun('last_weekly_hashrates_indexing', 0);
     } catch (e) {
-      logger.err('Cannot delete latest hashrates data points. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
+      logger.err('Cannot delete latest hashrates data points. Reason: ' + (e instanceof Error ? e.message : e));
     }
   }
   
@@ -225,10 +238,10 @@ class HashratesRepository {
     try {
       await DB.query(`DELETE FROM hashrates WHERE hashrate_timestamp >= FROM_UNIXTIME(?)`, [timestamp]);
       // Re-run the hashrate indexing to fill up missing data
-      mining.lastHashrateIndexingDate = null;
-      mining.lastWeeklyHashrateIndexingDate = null;
+      await this.$setLatestRun('last_hashrates_indexing', 0);
+      await this.$setLatestRun('last_weekly_hashrates_indexing', 0);
     } catch (e) {
-      logger.err('Cannot delete latest hashrates data points. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
+      logger.err('Cannot delete latest hashrates data points. Reason: ' + (e instanceof Error ? e.message : e));
     }
   }
 }

backend/src/repositories/PoolsRepository.ts

@@ -1,5 +1,4 @@
 import { Common } from '../api/common';
-import poolsParser from '../api/pools-parser';
 import config from '../config';
 import DB from '../database';
 import logger from '../logger';
@@ -10,7 +9,7 @@ class PoolsRepository {
    * Get all pools tagging info
    */
   public async $getPools(): Promise<PoolTag[]> {
-    const [rows] = await DB.query('SELECT id, unique_id as uniqueId, name, addresses, regexes, slug FROM pools');
+    const [rows] = await DB.query('SELECT id, name, addresses, regexes, slug FROM pools;');
     return <PoolTag[]>rows;
   }
 
@@ -18,11 +17,7 @@ class PoolsRepository {
    * Get unknown pool tagging info
    */
   public async $getUnknownPool(): Promise<PoolTag> {
-    let [rows]: any[] = await DB.query('SELECT id, unique_id as uniqueId, name, slug FROM pools where name = "Unknown"');
-    if (rows && rows.length === 0 && config.DATABASE.ENABLED) {
-      await poolsParser.$insertUnknownPool();
-      [rows] = await DB.query('SELECT id, unique_id as uniqueId, name, slug FROM pools where name = "Unknown"');
-    }
+    const [rows] = await DB.query('SELECT id, name, slug FROM pools where name = "Unknown"');
     return <PoolTag>rows[0];
   }
 
@@ -32,25 +27,16 @@ class PoolsRepository {
   public async $getPoolsInfo(interval: string | null = null): Promise<PoolInfo[]> {
     interval = Common.getSqlInterval(interval);
 
-    let query = `
-      SELECT
-        COUNT(blocks.height) As blockCount,
-          pool_id AS poolId,
-          pools.name AS name,
-          pools.link AS link,
-          slug,
-          AVG(blocks_audits.match_rate) AS avgMatchRate
+    let query = `SELECT COUNT(height) as blockCount, pool_id as poolId, pools.name as name, pools.link as link, slug
       FROM blocks
-      JOIN pools on pools.id = pool_id
-      LEFT JOIN blocks_audits ON blocks_audits.height = blocks.height
-    `;
+      JOIN pools on pools.id = pool_id`;
 
     if (interval) {
       query += ` WHERE blocks.blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
     }
 
     query += ` GROUP BY pool_id
-      ORDER BY COUNT(blocks.height) DESC`;
+      ORDER BY COUNT(height) DESC`;
 
     try {
       const [rows] = await DB.query(query);
@@ -64,7 +50,7 @@ class PoolsRepository {
   /**
    * Get basic pool info and block count between two timestamp
    */
-  public async $getPoolsInfoBetween(from: number, to: number): Promise<PoolInfo[]> {
+   public async $getPoolsInfoBetween(from: number, to: number): Promise<PoolInfo[]> {
     const query = `SELECT COUNT(height) as blockCount, pools.id as poolId, pools.name as poolName
       FROM pools
       LEFT JOIN blocks on pools.id = blocks.pool_id AND blocks.blockTimestamp BETWEEN FROM_UNIXTIME(?) AND FROM_UNIXTIME(?)
@@ -80,9 +66,9 @@ class PoolsRepository {
   }
 
   /**
-   * Get a mining pool info
+   * Get mining pool statistics for one pool
    */
-  public async $getPool(slug: string, parse: boolean = true): Promise<PoolTag | null> {
+   public async $getPool(slug: string): Promise<PoolTag | null> {
     const query = `
       SELECT *
       FROM pools
@@ -95,44 +81,10 @@ class PoolsRepository {
         return null;
       }
 
-      if (parse) {
-        rows[0].regexes = JSON.parse(rows[0].regexes);
-      }
-      if (['testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
-        rows[0].addresses = []; // pools-v2.json only contains mainnet addresses
-      } else if (parse) {
-        rows[0].addresses = JSON.parse(rows[0].addresses);
-      }
-
-      return rows[0];
-    } catch (e) {
-      logger.err('Cannot get pool from db. Reason: ' + (e instanceof Error ? e.message : e));
-      throw e;
-    }
-  }
-
-  /**
-   * Get a mining pool info by its unique id
-   */
-  public async $getPoolByUniqueId(id: number, parse: boolean = true): Promise<PoolTag | null> {
-    const query = `
-      SELECT *
-      FROM pools
-      WHERE pools.unique_id = ?`;
-
-    try {
-      const [rows]: any[] = await DB.query(query, [id]);
-
-      if (rows.length < 1) {
-        return null;
-      }
-
-      if (parse) {
-        rows[0].regexes = JSON.parse(rows[0].regexes);
-      }
+      rows[0].regexes = JSON.parse(rows[0].regexes);
       if (['testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
         rows[0].addresses = []; // pools.json only contains mainnet addresses
-      } else if (parse) {
+      } else {
         rows[0].addresses = JSON.parse(rows[0].addresses);
       }
 
@@ -142,84 +94,6 @@ class PoolsRepository {
       throw e;
     }
   }
-
-  /**
-   * Insert a new mining pool in the database
-   * 
-   * @param pool 
-   */
-  public async $insertNewMiningPool(pool: any, slug: string): Promise<void> {
-    try {
-      await DB.query(`
-        INSERT INTO pools
-        SET name = ?, link = ?, addresses = ?, regexes = ?, slug = ?, unique_id = ?`,
-        [pool.name, pool.link, JSON.stringify(pool.addresses), JSON.stringify(pool.regexes), slug, pool.id]
-      );
-    } catch (e: any) {
-      logger.err(`Cannot insert new mining pool into db. Reason: ` + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  /**
-   * Rename an existing mining pool
-   * 
-   * @param dbId
-   * @param newSlug
-   * @param newName 
-   */
-  public async $renameMiningPool(dbId: number, newSlug: string, newName: string): Promise<void> {
-    try {
-      await DB.query(`
-        UPDATE pools
-        SET slug = ?, name = ?
-        WHERE id = ?`,
-        [newSlug, newName, dbId]
-      );
-    } catch (e: any) {
-      logger.err(`Cannot rename mining pool id ${dbId}. Reason: ` + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  /**
-   * Update an exisiting mining pool link
-   * 
-   * @param dbId 
-   * @param newLink 
-   */
-  public async $updateMiningPoolLink(dbId: number, newLink: string): Promise<void> {
-    try {
-      await DB.query(`
-        UPDATE pools
-        SET link = ?
-        WHERE id = ?`,
-        [newLink, dbId]
-      );
-    } catch (e: any) {
-      logger.err(`Cannot update link for mining pool id ${dbId}. Reason: ` + (e instanceof Error ? e.message : e));
-    }
-
-  }
-
-  /**
-   * Update an existing mining pool addresses or coinbase tags
-   * 
-   * @param dbId 
-   * @param addresses 
-   * @param regexes 
-   */
-  public async $updateMiningPoolTags(dbId: number, addresses: string, regexes: string): Promise<void> {
-    try {
-      await DB.query(`
-        UPDATE pools
-        SET addresses = ?, regexes = ?
-        WHERE id = ?`,
-        [JSON.stringify(addresses), JSON.stringify(regexes), dbId]
-      );
-    } catch (e: any) {
-      logger.err(`Cannot update mining pool id ${dbId}. Reason: ` + (e instanceof Error ? e.message : e));
-    }
-  }
-
 }
 
 export default new PoolsRepository();

backend/src/repositories/PricesRepository.ts

@@ -1,228 +1,50 @@
 import DB from '../database';
 import logger from '../logger';
-import priceUpdater from '../tasks/price-updater';
-
-export interface ApiPrice {
-  time?: number,
-  USD: number,
-  EUR: number,
-  GBP: number,
-  CAD: number,
-  CHF: number,
-  AUD: number,
-  JPY: number,
-}
-const ApiPriceFields = `
-  UNIX_TIMESTAMP(time) as time,
-  USD,
-  EUR,
-  GBP,
-  CAD,
-  CHF,
-  AUD,
-  JPY
-`;
-
-export interface ExchangeRates {
-  USDEUR: number,
-  USDGBP: number,
-  USDCAD: number,
-  USDCHF: number,
-  USDAUD: number,
-  USDJPY: number,
-}
-
-export interface Conversion {
-  prices: ApiPrice[],
-  exchangeRates: ExchangeRates;
-}
-
-export const MAX_PRICES = {
-  USD: 100000000,
-  EUR: 100000000,
-  GBP: 100000000,
-  CAD: 100000000,
-  CHF: 100000000,
-  AUD: 100000000,
-  JPY: 10000000000,
-};
+import { Prices } from '../tasks/price-updater';
 
 class PricesRepository {
-  public async $savePrices(time: number, prices: ApiPrice): Promise<void> {
+  public async $savePrices(time: number, prices: Prices): Promise<void> {
     if (prices.USD === -1) {
-      // Some historical price entries have no USD prices, so we just ignore them to avoid future UX issues
-      // As of today there are only 4 (on 2013-09-05, 2013-0909, 2013-09-12 and 2013-09-26) so that's fine
+      // Some historical price entries have not USD prices, so we just ignore them to avoid future UX issues
+      // As of today there are only 4 (on 2013-09-05, 2013-09-19, 2013-09-12 and 2013-09-26) so that's fine
       return;
     }
 
-    // Sanity check
-    for (const currency of Object.keys(prices)) {
-      if (prices[currency] < -1 || prices[currency] > MAX_PRICES[currency]) { // We use -1 to mark a "missing data, so it's a valid entry"
-        logger.info(`Ignore BTC${currency} price of ${prices[currency]}`);
-        prices[currency] = 0;
-      }
-    }
-    
     try {
       await DB.query(`
         INSERT INTO prices(time,             USD, EUR, GBP, CAD, CHF, AUD, JPY)
         VALUE             (FROM_UNIXTIME(?), ?,   ?,   ?,   ?,   ?,   ?,   ?  )`,
         [time, prices.USD, prices.EUR, prices.GBP, prices.CAD, prices.CHF, prices.AUD, prices.JPY]
       );
-    } catch (e) {
+    } catch (e: any) {
       logger.err(`Cannot save exchange rate into db. Reason: ` + (e instanceof Error ? e.message : e));
       throw e;
     }
   }
 
   public async $getOldestPriceTime(): Promise<number> {
-    const [oldestRow] = await DB.query(`
-      SELECT UNIX_TIMESTAMP(time) AS time
-      FROM prices
-      ORDER BY time
-      LIMIT 1
-    `);
+    const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1 ORDER BY time LIMIT 1`);
     return oldestRow[0] ? oldestRow[0].time : 0;
   }
 
   public async $getLatestPriceId(): Promise<number | null> {
-    const [oldestRow] = await DB.query(`
-      SELECT id
-      FROM prices
-      ORDER BY time DESC
-      LIMIT 1`
-    );
+    const [oldestRow] = await DB.query(`SELECT id from prices WHERE USD != -1 ORDER BY time DESC LIMIT 1`);
     return oldestRow[0] ? oldestRow[0].id : null;
   }
 
   public async $getLatestPriceTime(): Promise<number> {
-    const [oldestRow] = await DB.query(`
-      SELECT UNIX_TIMESTAMP(time) AS time
-      FROM prices
-      ORDER BY time DESC
-      LIMIT 1`
-    );
+    const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1 ORDER BY time DESC LIMIT 1`);
     return oldestRow[0] ? oldestRow[0].time : 0;
   }
 
   public async $getPricesTimes(): Promise<number[]> {
-    const [times] = await DB.query(`
-      SELECT UNIX_TIMESTAMP(time) AS time
-      FROM prices
-      WHERE USD != -1
-      ORDER BY time
-    `);
-    if (!Array.isArray(times)) {
-      return [];
-    }
+    const [times]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1 ORDER BY time`);
     return times.map(time => time.time);
   }
 
-  public async $getPricesTimesAndId(): Promise<{time: number, id: number, USD: number}[]> {
-    const [times] = await DB.query(`
-      SELECT
-        UNIX_TIMESTAMP(time) AS time,
-        id,
-        USD
-      FROM prices
-      ORDER BY time
-    `);
-    return times as {time: number, id: number, USD: number}[];
-  }
-
-  public async $getLatestConversionRates(): Promise<ApiPrice> {
-    const [rates] = await DB.query(`
-      SELECT ${ApiPriceFields}
-      FROM prices
-      ORDER BY time DESC
-      LIMIT 1`
-    );
-
-    if (!Array.isArray(rates) || rates.length === 0) {
-      return priceUpdater.getEmptyPricesObj();
-    }
-    return rates[0] as ApiPrice;
-  }
-
-  public async $getNearestHistoricalPrice(timestamp: number | undefined): Promise<Conversion | null> {
-    try {
-      const [rates] = await DB.query(`
-        SELECT ${ApiPriceFields}
-        FROM prices
-        WHERE UNIX_TIMESTAMP(time) < ?
-        ORDER BY time DESC
-        LIMIT 1`,
-        [timestamp]
-      );
-      if (!Array.isArray(rates)) {
-        throw Error(`Cannot get single historical price from the database`);
-      }
-
-      // Compute fiat exchange rates
-      let latestPrice = rates[0] as ApiPrice;
-      if (latestPrice.USD === -1) {
-        latestPrice = priceUpdater.getEmptyPricesObj();
-      }
-
-      const computeFx = (usd: number, other: number): number =>
-        Math.round(Math.max(other, 0) / Math.max(usd, 1) * 100) / 100;
-      
-      const exchangeRates: ExchangeRates = {
-        USDEUR: computeFx(latestPrice.USD, latestPrice.EUR),
-        USDGBP: computeFx(latestPrice.USD, latestPrice.GBP),
-        USDCAD: computeFx(latestPrice.USD, latestPrice.CAD),
-        USDCHF: computeFx(latestPrice.USD, latestPrice.CHF),
-        USDAUD: computeFx(latestPrice.USD, latestPrice.AUD),
-        USDJPY: computeFx(latestPrice.USD, latestPrice.JPY),
-      };
-
-      return {
-        prices: rates as ApiPrice[],
-        exchangeRates: exchangeRates
-      };
-    } catch (e) {
-      logger.err(`Cannot fetch single historical prices from the db. Reason ${e instanceof Error ? e.message : e}`);
-      return null;
-    }
-  }
-
-  public async $getHistoricalPrices(): Promise<Conversion | null> {
-    try {
-      const [rates] = await DB.query(`
-        SELECT ${ApiPriceFields}
-        FROM prices
-        ORDER BY time DESC
-      `);
-      if (!Array.isArray(rates)) {
-        throw Error(`Cannot get average historical price from the database`);
-      }
-
-      // Compute fiat exchange rates
-      let latestPrice = rates[0] as ApiPrice;
-      if (latestPrice.USD === -1) {
-        latestPrice = priceUpdater.getEmptyPricesObj();
-      }
-
-      const computeFx = (usd: number, other: number): number =>
-        Math.round(Math.max(other, 0) / Math.max(usd, 1) * 100) / 100;
-      
-      const exchangeRates: ExchangeRates = {
-        USDEUR: computeFx(latestPrice.USD, latestPrice.EUR),
-        USDGBP: computeFx(latestPrice.USD, latestPrice.GBP),
-        USDCAD: computeFx(latestPrice.USD, latestPrice.CAD),
-        USDCHF: computeFx(latestPrice.USD, latestPrice.CHF),
-        USDAUD: computeFx(latestPrice.USD, latestPrice.AUD),
-        USDJPY: computeFx(latestPrice.USD, latestPrice.JPY),
-      };
-
-      return {
-        prices: rates as ApiPrice[],
-        exchangeRates: exchangeRates
-      };
-    } catch (e) {
-      logger.err(`Cannot fetch historical prices from the db. Reason ${e instanceof Error ? e.message : e}`);
-      return null;
-    }
+  public async $getPricesTimesAndId(): Promise<number[]> {
+    const [times]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time, id, USD from prices ORDER BY time`);
+    return times;
   }
 }
 

backend/src/repositories/TransactionRepository.ts

@@ -1,23 +1,31 @@
 import DB from '../database';
 import logger from '../logger';
 import { Ancestor, CpfpInfo } from '../mempool.interfaces';
-import cpfpRepository from './CpfpRepository';
+
+interface CpfpSummary {
+  txid: string;
+  cluster: string;
+  root: string;
+  txs: Ancestor[];
+  height: number;
+  fee_rate: number;
+}
 
 class TransactionRepository {
-  public async $setCluster(txid: string, clusterRoot: string): Promise<void> {
+  public async $setCluster(txid: string, cluster: string): Promise<void> {
     try {
       await DB.query(
         `
-          INSERT INTO compact_transactions
+          INSERT INTO transactions
           (
             txid,
             cluster
           )
-          VALUE (UNHEX(?), UNHEX(?))
+          VALUE (?, ?)
           ON DUPLICATE KEY UPDATE
-            cluster = UNHEX(?)
+            cluster = ?
         ;`,
-        [txid, clusterRoot, clusterRoot]
+        [txid, cluster, cluster]
       );
     } catch (e: any) {
       logger.err(`Cannot save transaction cpfp cluster into db. Reason: ` + (e instanceof Error ? e.message : e));
@@ -25,46 +33,19 @@ class TransactionRepository {
     }
   }
 
-  public async $batchSetCluster(txs): Promise<void> {
-    try {
-      let query = `
-          INSERT IGNORE INTO compact_transactions
-          (
-            txid,
-            cluster
-          )
-          VALUES
-      `;
-      query += txs.map(tx => {
-        return (' (UNHEX(?), UNHEX(?))');
-      }) + ';';
-      const values = txs.map(tx => [tx.txid, tx.cluster]).flat();
-      await DB.query(
-        query,
-        values
-      );
-    } catch (e: any) {
-      logger.err(`Cannot save cpfp transactions into db. Reason: ` + (e instanceof Error ? e.message : e));
-      throw e;
-    }
-  }
-
   public async $getCpfpInfo(txid: string): Promise<CpfpInfo | void> {
     try {
-      const [txRows]: any = await DB.query(
-        `
-          SELECT HEX(txid) as id, HEX(cluster) as root
-          FROM compact_transactions
-          WHERE txid = UNHEX(?)
-        `,
-        [txid]
-      );
-      if (txRows.length && txRows[0].root != null) {
-        const txid = txRows[0].id.toLowerCase();
-        const clusterId = txRows[0].root.toLowerCase();
-        const cluster = await cpfpRepository.$getCluster(clusterId);
-        if (cluster) {
-          return this.convertCpfp(txid, cluster);
+      let query = `
+        SELECT *
+        FROM transactions
+        LEFT JOIN cpfp_clusters AS cluster ON cluster.root = transactions.cluster
+        WHERE transactions.txid = ?
+      `;
+      const [rows]: any = await DB.query(query, [txid]);
+      if (rows.length) {
+        rows[0].txs = JSON.parse(rows[0].txs) as Ancestor[];
+        if (rows[0]?.txs?.length) {
+          return this.convertCpfp(rows[0]);
         }
       }
     } catch (e) {
@@ -73,28 +54,12 @@ class TransactionRepository {
     }
   }
 
-  public async $removeTransaction(txid: string): Promise<void> {
-    try {
-      await DB.query(
-        `
-          DELETE FROM compact_transactions
-          WHERE txid = UNHEX(?)
-        `,
-        [txid]
-      );
-    } catch (e) {
-      logger.warn('Cannot delete transaction cpfp info from db. Reason: ' + (e instanceof Error ? e.message : e));
-      throw e;
-    }
-  }
-
-  private convertCpfp(txid, cluster): CpfpInfo {
+  private convertCpfp(cpfp: CpfpSummary): CpfpInfo {
     const descendants: Ancestor[] = [];
     const ancestors: Ancestor[] = [];
     let matched = false;
-
-    for (const tx of (cluster?.txs || [])) {
-      if (tx.txid === txid) {
+    for (const tx of cpfp.txs) {
+      if (tx.txid === cpfp.txid) {
         matched = true;
       } else if (!matched) {
         descendants.push(tx);
@@ -105,6 +70,7 @@ class TransactionRepository {
     return {
       descendants,
       ancestors,
+      effectiveFeePerVsize: cpfp.fee_rate
     };
   }
 }

backend/src/rpc-api/commands.ts

@@ -88,7 +88,5 @@ module.exports = {
   verifyTxOutProof: 'verifytxoutproof', // bitcoind v0.11.0+
   walletLock: 'walletlock',
   walletPassphrase: 'walletpassphrase',
-  walletPassphraseChange: 'walletpassphrasechange',
-  getTxoutSetinfo: 'gettxoutsetinfo',
-  getIndexInfo: 'getindexinfo',
-};
+  walletPassphraseChange: 'walletpassphrasechange'
+}

backend/src/tasks/lightning/network-sync.service.ts

@@ -23,7 +23,7 @@ class NetworkSyncService {
   constructor() {}
 
   public async $startService(): Promise<void> {
-    logger.info(`Starting lightning network sync service`, logger.tags.ln);
+    logger.info('Starting lightning network sync service');
 
     this.loggerTimer = new Date().getTime() / 1000;
 
@@ -33,11 +33,11 @@ class NetworkSyncService {
   private async $runTasks(): Promise<void> {
     const taskStartTime = Date.now();
     try {
-      logger.debug(`Updating nodes and channels`, logger.tags.ln);
+      logger.info(`Updating nodes and channels`);
 
       const networkGraph = await lightningApi.$getNetworkGraph();
       if (networkGraph.nodes.length === 0 || networkGraph.edges.length === 0) {
-        logger.info(`LN Network graph is empty, retrying in 10 seconds`, logger.tags.ln);
+        logger.info(`LN Network graph is empty, retrying in 10 seconds`);
         setTimeout(() => { this.$runTasks(); }, 10000);
         return;
       }
@@ -55,7 +55,7 @@ class NetworkSyncService {
       }
 
     } catch (e) {
-      logger.err(`$runTasks() error: ${e instanceof Error ? e.message : e}`, logger.tags.ln);
+      logger.err('$runTasks() error: ' + (e instanceof Error ? e.message : e));
     }
 
     setTimeout(() => { this.$runTasks(); }, Math.max(1, (1000 * config.LIGHTNING.GRAPH_REFRESH_INTERVAL) - (Date.now() - taskStartTime)));
@@ -72,15 +72,15 @@ class NetworkSyncService {
     const graphNodesPubkeys: string[] = [];
     for (const node of nodes) {
       const latestUpdated = await channelsApi.$getLatestChannelUpdateForNode(node.pub_key);
-      node.last_update = Math.max(node.last_update ?? 0, latestUpdated);
+      node.last_update = Math.max(node.last_update, latestUpdated);
 
       await nodesApi.$saveNode(node);
       graphNodesPubkeys.push(node.pub_key);
       ++progress;
 
       const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
-      if (elapsedSeconds > config.LIGHTNING.LOGGER_UPDATE_INTERVAL) {
-        logger.debug(`Updating node ${progress}/${nodes.length}`, logger.tags.ln);
+      if (elapsedSeconds > 10) {
+        logger.info(`Updating node ${progress}/${nodes.length}`);
         this.loggerTimer = new Date().getTime() / 1000;
       }
 
@@ -106,7 +106,7 @@ class NetworkSyncService {
         deletedRecords += await NodeRecordsRepository.$deleteUnusedRecords(node.pub_key, customRecordTypes);
       }
     }
-    logger.debug(`${progress} nodes updated. ${deletedSockets} sockets deleted. ${deletedRecords} custom records deleted.`);
+    logger.info(`${progress} nodes updated. ${deletedSockets} sockets deleted. ${deletedRecords} custom records deleted.`);
 
     // If a channel if not present in the graph, mark it as inactive
     await nodesApi.$setNodesInactive(graphNodesPubkeys);
@@ -138,18 +138,18 @@ class NetworkSyncService {
         ++progress;
 
         const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
-        if (elapsedSeconds > config.LIGHTNING.LOGGER_UPDATE_INTERVAL) {
-          logger.debug(`Updating channel ${progress}/${channels.length}`, logger.tags.ln);
+        if (elapsedSeconds > 10) {
+          logger.info(`Updating channel ${progress}/${channels.length}`);
           this.loggerTimer = new Date().getTime() / 1000;
         }
       }
 
-      logger.debug(`${progress} channels updated`, logger.tags.ln);
+      logger.info(`${progress} channels updated`);
 
       // If a channel if not present in the graph, mark it as inactive
       await channelsApi.$setChannelsInactive(graphChannelsIds);
     } catch (e) {
-      logger.err(` Cannot update channel list. Reason: ${(e instanceof Error ? e.message : e)}`, logger.tags.ln);
+      logger.err(`Cannot update channel list. Reason: ${(e instanceof Error ? e.message : e)}`);
     }
   }
 
@@ -184,52 +184,44 @@ class NetworkSyncService {
         if (lowest < node.first_seen) {
           const query = `UPDATE nodes SET first_seen = FROM_UNIXTIME(?) WHERE public_key = ?`;
           const params = [lowest, node.public_key];
-          ++updated;
           await DB.query(query, params);
         }
         ++progress;
         const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
-        if (elapsedSeconds > config.LIGHTNING.LOGGER_UPDATE_INTERVAL) {
-          logger.debug(`Updating node first seen date ${progress}/${nodes.length}`, logger.tags.ln);
+        if (elapsedSeconds > 10) {
+          logger.info(`Updating node first seen date ${progress}/${nodes.length}`);
           this.loggerTimer = new Date().getTime() / 1000;
+          ++updated;
         }
       }
-      if (updated > 0) {
-        logger.debug(`Updated ${updated} node first seen dates`, logger.tags.ln);
-      }
+      logger.info(`Updated ${updated} node first seen dates`);
     } catch (e) {
-      logger.err(`$updateNodeFirstSeen() error: ${e instanceof Error ? e.message : e}`, logger.tags.ln);
+      logger.err('$updateNodeFirstSeen() error: ' + (e instanceof Error ? e.message : e));
     }
   }
 
   private async $lookUpCreationDateFromChain(): Promise<void> {
     let progress = 0;
 
-    logger.debug(`Running channel creation date lookup`, logger.tags.ln);
+    logger.info(`Running channel creation date lookup`);
     try {
       const channels = await channelsApi.$getChannelsWithoutCreatedDate();
       for (const channel of channels) {
         const transaction = await fundingTxFetcher.$fetchChannelOpenTx(channel.short_id);
-        if (!transaction) {
-          continue;
-        }
         await DB.query(`
           UPDATE channels SET created = FROM_UNIXTIME(?) WHERE channels.id = ?`,
           [transaction.timestamp, channel.id]
         );
         ++progress;
         const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
-        if (elapsedSeconds > config.LIGHTNING.LOGGER_UPDATE_INTERVAL) {
-          logger.debug(`Updating channel creation date ${progress}/${channels.length}`, logger.tags.ln);
+        if (elapsedSeconds > 10) {
+          logger.info(`Updating channel creation date ${progress}/${channels.length}`);
           this.loggerTimer = new Date().getTime() / 1000;
         }
       }
-
-      if (channels.length > 0) {
-        logger.debug(`Updated ${channels.length} channels' creation date`, logger.tags.ln);
-      }      
+      logger.info(`Updated ${channels.length} channels' creation date`);
     } catch (e) {
-      logger.err(`$lookUpCreationDateFromChain() error: ${e instanceof Error ? e.message : e}`, logger.tags.ln);
+      logger.err('$lookUpCreationDateFromChain() error: ' + (e instanceof Error ? e.message : e));
     }
   }
 
@@ -238,7 +230,7 @@ class NetworkSyncService {
    * mark that channel as inactive
    */
   private async $deactivateChannelsWithoutActiveNodes(): Promise<void> {
-    logger.debug(`Find channels which nodes are offline`, logger.tags.ln);
+    logger.info(`Find channels which nodes are offline`);
 
     try {
       const result = await DB.query<ResultSetHeader>(`
@@ -261,10 +253,12 @@ class NetworkSyncService {
         `);
 
       if (result[0].changedRows ?? 0 > 0) {
-        logger.debug(`Marked ${result[0].changedRows} channels as inactive because they are not linked to any active node`, logger.tags.ln);
+        logger.info(`Marked ${result[0].changedRows} channels as inactive because they are not linked to any active node`);
+      } else {
+        logger.debug(`Marked ${result[0].changedRows} channels as inactive because they are not linked to any active node`);
       }
     } catch (e) {
-      logger.err(`$deactivateChannelsWithoutActiveNodes() error: ${e instanceof Error ? e.message : e}`, logger.tags.ln);
+      logger.err('$deactivateChannelsWithoutActiveNodes() error: ' + (e instanceof Error ? e.message : e));
     }
   }
 
@@ -283,13 +277,13 @@ class NetworkSyncService {
       } else {
         log += ` for the first time`;
       }
-      logger.info(`${log}`, logger.tags.ln);
+      logger.info(log);
 
       const channels = await channelsApi.$getChannelsByStatus([0, 1]);
       for (const channel of channels) {
         const spendingTx = await bitcoinApi.$getOutspend(channel.transaction_id, channel.transaction_vout);
         if (spendingTx.spent === true && spendingTx.status?.confirmed === true) {
-          logger.debug(`Marking channel: ${channel.id} as closed.`, logger.tags.ln);
+          logger.debug('Marking channel: ' + channel.id + ' as closed.');
           await DB.query(`UPDATE channels SET status = 2, closing_date = FROM_UNIXTIME(?) WHERE id = ?`,
             [spendingTx.status.block_time, channel.id]);
           if (spendingTx.txid && !channel.closing_transaction_id) {
@@ -299,16 +293,16 @@ class NetworkSyncService {
 
         ++progress;
         const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
-        if (elapsedSeconds > config.LIGHTNING.LOGGER_UPDATE_INTERVAL) {
-          logger.info(`Checking if channel has been closed ${progress}/${channels.length}`, logger.tags.ln);
+        if (elapsedSeconds > 10) {
+          logger.info(`Checking if channel has been closed ${progress}/${channels.length}`);
           this.loggerTimer = new Date().getTime() / 1000;
         }
       }
 
       this.closedChannelsScanBlock = blocks.getCurrentBlockHeight();
-      logger.debug(`Closed channels scan completed at block ${this.closedChannelsScanBlock}`, logger.tags.ln);
+      logger.info(`Closed channels scan completed at block ${this.closedChannelsScanBlock}`);
     } catch (e) {
-      logger.err(`$scanForClosedChannels() error: ${e instanceof Error ? e.message : e}`, logger.tags.ln);
+      logger.err('$scanForClosedChannels() error: ' + (e instanceof Error ? e.message : e));
     }
   }
 }

backend/src/tasks/lightning/stats-updater.service.ts

@@ -6,7 +6,7 @@ import { Common } from '../../api/common';
 
 class LightningStatsUpdater {
   public async $startService(): Promise<void> {
-    logger.info(`Starting Lightning Stats service`, logger.tags.ln);
+    logger.info('Starting Lightning Stats service');
 
     await this.$runTasks();
     LightningStatsImporter.$run();
@@ -22,15 +22,12 @@ class LightningStatsUpdater {
    * Update the latest entry for each node every config.LIGHTNING.STATS_REFRESH_INTERVAL seconds
    */
   private async $logStatsDaily(): Promise<void> {
-    try {
-      const date = new Date();
-      Common.setDateMidnight(date);
-      const networkGraph = await lightningApi.$getNetworkGraph();
-      await LightningStatsImporter.computeNetworkStats(date.getTime() / 1000, networkGraph);
-      logger.debug(`Updated latest network stats`, logger.tags.ln);
-    } catch (e) {
-      logger.err(`Exception in $logStatsDaily. Reason: ${(e instanceof Error ? e.message : e)}`);
-    }
+    const date = new Date();
+    Common.setDateMidnight(date);
+    const networkGraph = await lightningApi.$getNetworkGraph();
+    await LightningStatsImporter.computeNetworkStats(date.getTime() / 1000, networkGraph);
+    
+    logger.info(`Updated latest network stats`);
   }
 }
 

backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts

@@ -21,10 +21,10 @@ class FundingTxFetcher {
       try {
         this.fundingTxCache = JSON.parse(await fsPromises.readFile(CACHE_FILE_NAME, 'utf-8'));
       } catch (e) {
-        logger.err(`Unable to parse channels funding txs disk cache. Starting from scratch`, logger.tags.ln);
+        logger.err(`Unable to parse channels funding txs disk cache. Starting from scratch`);
         this.fundingTxCache = {};
       }
-      logger.debug(`Imported ${Object.keys(this.fundingTxCache).length} funding tx amount from the disk cache`, logger.tags.ln);
+      logger.debug(`Imported ${Object.keys(this.fundingTxCache).length} funding tx amount from the disk cache`);
     }
   }
 
@@ -44,34 +44,33 @@ class FundingTxFetcher {
       ++channelProcessed;
 
       let elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer);
-      if (elapsedSeconds > config.LIGHTNING.LOGGER_UPDATE_INTERVAL) {
+      if (elapsedSeconds > 10) {
         elapsedSeconds = Math.round((new Date().getTime() / 1000) - globalTimer);
         logger.info(`Indexing channels funding tx ${channelProcessed + 1} of ${channelIds.length} ` +
           `(${Math.floor(channelProcessed / channelIds.length * 10000) / 100}%) | ` +
-          `elapsed: ${elapsedSeconds} seconds`,
-          logger.tags.ln
+          `elapsed: ${elapsedSeconds} seconds`
         );
         loggerTimer = new Date().getTime() / 1000;
       }
 
       elapsedSeconds = Math.round((new Date().getTime() / 1000) - cacheTimer);
       if (elapsedSeconds > 60) {
-        logger.debug(`Saving ${Object.keys(this.fundingTxCache).length} funding txs cache into disk`, logger.tags.ln);
+        logger.debug(`Saving ${Object.keys(this.fundingTxCache).length} funding txs cache into disk`);
         fsPromises.writeFile(CACHE_FILE_NAME, JSON.stringify(this.fundingTxCache));
         cacheTimer = new Date().getTime() / 1000;
       }
     }
 
     if (this.channelNewlyProcessed > 0) {
-      logger.info(`Indexed ${this.channelNewlyProcessed} additional channels funding tx`, logger.tags.ln);
-      logger.debug(`Saving ${Object.keys(this.fundingTxCache).length} funding txs cache into disk`, logger.tags.ln);
+      logger.info(`Indexed ${this.channelNewlyProcessed} additional channels funding tx`);
+      logger.debug(`Saving ${Object.keys(this.fundingTxCache).length} funding txs cache into disk`);
       fsPromises.writeFile(CACHE_FILE_NAME, JSON.stringify(this.fundingTxCache));
     }
 
     this.running = false;
   }
   
-  public async $fetchChannelOpenTx(channelId: string): Promise<{timestamp: number, txid: string, value: number} | null> {
+  public async $fetchChannelOpenTx(channelId: string): Promise<{timestamp: number, txid: string, value: number}> {
     channelId = Common.channelIntegerIdToShortId(channelId);
 
     if (this.fundingTxCache[channelId]) {
@@ -102,11 +101,6 @@ class FundingTxFetcher {
     const rawTx = await bitcoinClient.getRawTransaction(txid);
     const tx = await bitcoinClient.decodeRawTransaction(rawTx);
 
-    if (!tx || !tx.vout || tx.vout.length < parseInt(outputIdx, 10) + 1 || tx.vout[outputIdx].value === undefined) {
-      logger.err(`Cannot find blockchain funding tx for channel id ${channelId}. Possible reasons are: bitcoin backend timeout or the channel shortId is not valid`);
-      return null;
-    }
-
     this.fundingTxCache[channelId] = {
       timestamp: block.time,
       txid: txid,

backend/src/tasks/lightning/sync-tasks/node-locations.ts

@@ -14,7 +14,7 @@ export async function $lookupNodeLocation(): Promise<void> {
   let nodesUpdated = 0;
   let geoNamesInserted = 0;
 
-  logger.debug(`Running node location updater using Maxmind`, logger.tags.ln);
+  logger.info(`Running node location updater using Maxmind`);
   try {
     const nodes = await nodesApi.$getAllNodes();
     const lookupCity = await maxmind.open<CityResponse>(config.MAXMIND.GEOLITE2_CITY);
@@ -152,8 +152,8 @@ export async function $lookupNodeLocation(): Promise<void> {
 
           ++progress;
           const elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer);
-          if (elapsedSeconds > config.LIGHTNING.LOGGER_UPDATE_INTERVAL) {
-            logger.debug(`Updating node location data ${progress}/${nodes.length}`);
+          if (elapsedSeconds > 10) {
+            logger.info(`Updating node location data ${progress}/${nodes.length}`);
             loggerTimer = new Date().getTime() / 1000;
           }
         }
@@ -161,7 +161,9 @@ export async function $lookupNodeLocation(): Promise<void> {
     }
 
     if (nodesUpdated > 0) {
-      logger.debug(`${nodesUpdated} nodes maxmind data updated, ${geoNamesInserted} geo names inserted`, logger.tags.ln);
+      logger.info(`${nodesUpdated} nodes maxmind data updated, ${geoNamesInserted} geo names inserted`);
+    } else {
+      logger.debug(`${nodesUpdated} nodes maxmind data updated, ${geoNamesInserted} geo names inserted`);
     }
   } catch (e) {
     logger.err('$lookupNodeLocation() error: ' + (e instanceof Error ? e.message : e));

backend/src/tasks/lightning/sync-tasks/stats-importer.ts

@@ -8,6 +8,7 @@ import { isIP } from 'net';
 import { Common } from '../../../api/common';
 import channelsApi from '../../../api/explorer/channels.api';
 import nodesApi from '../../../api/explorer/nodes.api';
+import { ResultSetHeader } from 'mysql2';
 
 const fsPromises = promises;
 
@@ -16,7 +17,7 @@ class LightningStatsImporter {
 
   async $run(): Promise<void> {
     const [channels]: any[] = await DB.query('SELECT short_id from channels;');
-    logger.info(`Caching funding txs for currently existing channels`, logger.tags.ln);
+    logger.info('Caching funding txs for currently existing channels');
     await fundingTxFetcher.$fetchChannelsFundingTxs(channels.map(channel => channel.short_id));
 
     if (config.MEMPOOL.NETWORK !== 'mainnet' || config.DATABASE.ENABLED === false) {
@@ -107,7 +108,7 @@ class LightningStatsImporter {
 
       const tx = await fundingTxFetcher.$fetchChannelOpenTx(short_id);
       if (!tx) {
-        logger.err(`Unable to fetch funding tx for channel ${short_id}. Capacity and creation date is unknown. Skipping channel.`, logger.tags.ln);
+        logger.err(`Unable to fetch funding tx for channel ${short_id}. Capacity and creation date is unknown. Skipping channel.`);
         continue;
       }
 
@@ -320,7 +321,7 @@ class LightningStatsImporter {
       try {
         fileList = await fsPromises.readdir(this.topologiesFolder);
       } catch (e) {
-        logger.err(`Unable to open topology folder at ${this.topologiesFolder}`, logger.tags.ln);
+        logger.err(`Unable to open topology folder at ${this.topologiesFolder}`);
         throw e;
       }
       // Insert history from the most recent to the oldest
@@ -358,7 +359,7 @@ class LightningStatsImporter {
           continue;
         }
 
-        logger.debug(`Reading ${this.topologiesFolder}/${filename}`, logger.tags.ln);
+        logger.debug(`Reading ${this.topologiesFolder}/${filename}`);
         let fileContent = '';
         try {
           fileContent = await fsPromises.readFile(`${this.topologiesFolder}/${filename}`, 'utf8');
@@ -367,7 +368,7 @@ class LightningStatsImporter {
             totalProcessed++;
             continue;
           }
-          logger.err(`Unable to open ${this.topologiesFolder}/${filename}`, logger.tags.ln);
+          logger.err(`Unable to open ${this.topologiesFolder}/${filename}`);
           totalProcessed++;
           continue;
         }
@@ -377,7 +378,7 @@ class LightningStatsImporter {
           graph = JSON.parse(fileContent);
           graph = await this.cleanupTopology(graph);
         } catch (e) {
-          logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.ln);
+          logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content. Reason: ${e instanceof Error ? e.message : e}`);
           totalProcessed++;
           continue;
         }
@@ -389,20 +390,20 @@ class LightningStatsImporter {
         }
 
         if (!logStarted) {
-          logger.info(`Founds a topology file that we did not import. Importing historical lightning stats now.`, logger.tags.ln);
+          logger.info(`Founds a topology file that we did not import. Importing historical lightning stats now.`);
           logStarted = true;
         }
         
         const datestr = `${new Date(timestamp * 1000).toUTCString()} (${timestamp})`;
-        logger.debug(`${datestr}: Found ${graph.nodes.length} nodes and ${graph.edges.length} channels`, logger.tags.ln);
+        logger.debug(`${datestr}: Found ${graph.nodes.length} nodes and ${graph.edges.length} channels`);
 
         totalProcessed++;
 
         if (processed > 10) {
-          logger.info(`Generating LN network stats for ${datestr}. Processed ${totalProcessed}/${fileList.length} files`, logger.tags.ln);
+          logger.info(`Generating LN network stats for ${datestr}. Processed ${totalProcessed}/${fileList.length} files`);
           processed = 0;
         } else {
-          logger.debug(`Generating LN network stats for ${datestr}. Processed ${totalProcessed}/${fileList.length} files`, logger.tags.ln);
+          logger.debug(`Generating LN network stats for ${datestr}. Processed ${totalProcessed}/${fileList.length} files`);
         }
         await fundingTxFetcher.$fetchChannelsFundingTxs(graph.edges.map(channel => channel.channel_id.slice(0, -2)));
         const stat = await this.computeNetworkStats(timestamp, graph, true);
@@ -411,10 +412,10 @@ class LightningStatsImporter {
       }
 
       if (totalProcessed > 0) {
-        logger.info(`Lightning network stats historical import completed`, logger.tags.ln);
+        logger.info(`Lightning network stats historical import completed`);
       }
     } catch (e) {
-      logger.err(`Lightning network stats historical failed. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.ln);
+      logger.err(`Lightning network stats historical failed. Reason: ${e instanceof Error ? e.message : e}`);
     }
   }
 

backend/src/tasks/pools-updater.ts

@@ -8,18 +8,16 @@ import { SocksProxyAgent } from 'socks-proxy-agent';
 import * as https from 'https';
 
 /**
- * Maintain the most recent version of pools-v2.json
+ * Maintain the most recent version of pools.json
  */
 class PoolsUpdater {
   lastRun: number = 0;
-  currentSha: string | null = null;
+  currentSha: string | undefined = undefined;
   poolsUrl: string = config.MEMPOOL.POOLS_JSON_URL;
   treeUrl: string = config.MEMPOOL.POOLS_JSON_TREE_URL;
 
   public async updatePoolsJson(): Promise<void> {
-    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false ||
-      config.MEMPOOL.ENABLED === false
-    ) {
+    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
       return;
     }
 
@@ -33,9 +31,15 @@ class PoolsUpdater {
 
     this.lastRun = now;
 
+    if (config.SOCKS5PROXY.ENABLED) {
+      logger.info(`Updating latest mining pools from ${this.poolsUrl} over the Tor network`);
+    } else {
+      logger.info(`Updating latest mining pools from ${this.poolsUrl} over clearnet`);
+    }
+
     try {
-      const githubSha = await this.fetchPoolsSha(); // Fetch pools-v2.json sha from github
-      if (githubSha === null) {
+      const githubSha = await this.fetchPoolsSha(); // Fetch pools.json sha from github
+      if (githubSha === undefined) {
         return;
       }
 
@@ -43,57 +47,32 @@ class PoolsUpdater {
         this.currentSha = await this.getShaFromDb();
       }
 
-      logger.debug(`pools-v2.json sha | Current: ${this.currentSha} | Github: ${githubSha}`);
-      if (this.currentSha !== null && this.currentSha === githubSha) {
+      logger.debug(`Pools.json sha | Current: ${this.currentSha} | Github: ${githubSha}`);
+      if (this.currentSha !== undefined && this.currentSha === githubSha) {
         return;
       }
 
-      // See backend README for more details about the mining pools update process
-      if (this.currentSha !== null && // If we don't have any mining pool, download it at least once
-        config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING !== true && // Automatic pools update is disabled
-        !process.env.npm_config_update_pools // We're not manually updating mining pool
-      ) {
-        logger.warn(`Updated mining pools data is available (${githubSha}) but AUTOMATIC_BLOCK_REINDEXING is disabled`);
-        logger.info(`You can update your mining pools using the --update-pools command flag. You may want to clear your nginx cache as well if applicable`);
-        return;
-      }
-
-      const network = config.SOCKS5PROXY.ENABLED ? 'tor' : 'clearnet';
-      if (this.currentSha === null) {
-        logger.info(`Downloading pools-v2.json for the first time from ${this.poolsUrl} over ${network}`, logger.tags.mining);
+      if (this.currentSha === undefined) {
+        logger.info(`Downloading pools.json for the first time from ${this.poolsUrl}`);
       } else {
-        logger.warn(`pools-v2.json is outdated, fetch latest from ${this.poolsUrl} over ${network}`, logger.tags.mining);
+        logger.warn(`Pools.json is outdated, fetch latest from ${this.poolsUrl}`);
       }
       const poolsJson = await this.query(this.poolsUrl);
       if (poolsJson === undefined) {
         return;
       }
-      poolsParser.setMiningPools(poolsJson);
-
-      if (config.DATABASE.ENABLED === false) { // Don't run db operations
-        logger.info('Mining pools-v2.json import completed (no database)');
-        return;
-      }
-
-      try {
-        await DB.query('START TRANSACTION;');
-        await poolsParser.migratePoolsJson();
-        await this.updateDBSha(githubSha);
-        await DB.query('COMMIT;');
-      } catch (e) {
-        logger.err(`Could not migrate mining pools, rolling back. Exception: ${JSON.stringify(e)}`, logger.tags.mining);
-        await DB.query('ROLLBACK;');
-      }
-      logger.info('PoolsUpdater completed');
+      await poolsParser.migratePoolsJson(poolsJson);
+      await this.updateDBSha(githubSha);
+      logger.notice('PoolsUpdater completed');
 
     } catch (e) {
       this.lastRun = now - (oneWeek - oneDay); // Try again in 24h instead of waiting next week
-      logger.err(`PoolsUpdater failed. Will try again in 24h. Exception: ${JSON.stringify(e)}`, logger.tags.mining);
+      logger.err('PoolsUpdater failed. Will try again in 24h. Reason: ' + (e instanceof Error ? e.message : e));
     }
   }
 
   /**
-   * Fetch our latest pools-v2.json sha from the db
+   * Fetch our latest pools.json sha from the db
    */
   private async updateDBSha(githubSha: string): Promise<void> {
     this.currentSha = githubSha;
@@ -102,46 +81,46 @@ class PoolsUpdater {
         await DB.query('DELETE FROM state where name="pools_json_sha"');
         await DB.query(`INSERT INTO state VALUES('pools_json_sha', NULL, '${githubSha}')`);
       } catch (e) {
-        logger.err('Cannot save github pools-v2.json sha into the db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
+        logger.err('Cannot save github pools.json sha into the db. Reason: ' + (e instanceof Error ? e.message : e));
       }
     }
   }
 
   /**
-   * Fetch our latest pools-v2.json sha from the db
+   * Fetch our latest pools.json sha from the db
    */
-  private async getShaFromDb(): Promise<string | null> {
+  private async getShaFromDb(): Promise<string | undefined> {
     try {
       const [rows]: any[] = await DB.query('SELECT string FROM state WHERE name="pools_json_sha"');
-      return (rows.length > 0 ? rows[0].string : null);
+      return (rows.length > 0 ? rows[0].string : undefined);
     } catch (e) {
-      logger.err('Cannot fetch pools-v2.json sha from db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
-      return null;
+      logger.err('Cannot fetch pools.json sha from db. Reason: ' + (e instanceof Error ? e.message : e));
+      return undefined;
     }
   }
 
   /**
-   * Fetch our latest pools-v2.json sha from github
+   * Fetch our latest pools.json sha from github
    */
-  private async fetchPoolsSha(): Promise<string | null> {
+  private async fetchPoolsSha(): Promise<string | undefined> {
     const response = await this.query(this.treeUrl);
 
     if (response !== undefined) {
       for (const file of response['tree']) {
-        if (file['path'] === 'pools-v2.json') {
+        if (file['path'] === 'pools.json') {
           return file['sha'];
         }
       }
     }
 
-    logger.err(`Cannot find "pools-v2.json" in git tree (${this.treeUrl})`, logger.tags.mining);
-    return null;
+    logger.err(`Cannot find "pools.json" in git tree (${this.treeUrl})`);
+    return undefined;
   }
 
   /**
    * Http request wrapper
    */
-  private async query(path): Promise<any[] | undefined> {
+  private async query(path): Promise<object | undefined> {
     type axiosOptions = {
       headers: {
         'User-Agent': string

backend/src/tasks/price-feeds/bitfinex-api.ts

@@ -8,13 +8,12 @@ class BitfinexApi implements PriceFeed {
   public url: string = 'https://api.bitfinex.com/v1/pubticker/BTC';
   public urlHist: string = 'https://api-pub.bitfinex.com/v2/candles/trade:{GRANULARITY}:tBTC{CURRENCY}/hist';
 
+  constructor() {
+  }
+
   public async $fetchPrice(currency): Promise<number> {
     const response = await query(this.url + currency);
-    if (response && response['last_price']) {
-      return parseInt(response['last_price'], 10);
-    } else {
-      return -1;
-    }
+    return response ? parseInt(response['last_price'], 10) : -1;
   }
 
   public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {

backend/src/tasks/price-feeds/bitflyer-api.ts

@@ -13,11 +13,7 @@ class BitflyerApi implements PriceFeed {
 
   public async $fetchPrice(currency): Promise<number> {
     const response = await query(this.url + currency);
-    if (response && response['ltp']) {
-      return parseInt(response['ltp'], 10);
-    } else {
-      return -1;
-    }
+    return response ? parseInt(response['ltp'], 10) : -1;
   }
 
   public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {

backend/src/tasks/price-feeds/coinbase-api.ts

@@ -13,11 +13,7 @@ class CoinbaseApi implements PriceFeed {
 
   public async $fetchPrice(currency): Promise<number> {
     const response = await query(this.url + currency);
-    if (response && response['data'] && response['data']['amount']) {
-      return parseInt(response['data']['amount'], 10);
-    } else {
-      return -1;
-    }
+    return response ? parseInt(response['data']['amount'], 10) : -1;
   }
 
   public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {

backend/src/tasks/price-feeds/gemini-api.ts

@@ -13,11 +13,7 @@ class GeminiApi implements PriceFeed {
 
   public async $fetchPrice(currency): Promise<number> {
     const response = await query(this.url + currency);
-    if (response && response['last']) {
-      return parseInt(response['last'], 10);
-    } else {
-      return -1;
-    }
+    return response ? parseInt(response['last'], 10) : -1;
   }
 
   public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {

backend/src/tasks/price-feeds/kraken-api.ts

@@ -23,14 +23,7 @@ class KrakenApi implements PriceFeed {
 
   public async $fetchPrice(currency): Promise<number> {
     const response = await query(this.url + currency);
-    const ticker = this.getTicker(currency);
-    if (response && response['result'] && response['result'][ticker] &&
-      response['result'][ticker]['c'] && response['result'][ticker]['c'].length > 0
-    ) {
-      return parseInt(response['result'][ticker]['c'][0], 10);
-    } else {
-      return -1;
-    }
+    return response ? parseInt(response['result'][this.getTicker(currency)]['c'][0], 10) : -1;
   }
 
   public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {
@@ -98,7 +91,7 @@ class KrakenApi implements PriceFeed {
     }
 
     if (Object.keys(priceHistory).length > 0) {
-      logger.info(`Inserted ${Object.keys(priceHistory).length} Kraken EUR, USD, GBP, JPY, CAD, CHF and AUD weekly price history into db`, logger.tags.mining);
+      logger.notice(`Inserted ${Object.keys(priceHistory).length} Kraken EUR, USD, GBP, JPY, CAD, CHF and AUD weekly price history into db`);
     }
   }
 }

backend/src/tasks/price-updater.ts

@@ -1,8 +1,8 @@
 import * as fs from 'fs';
-import path from 'path';
+import path from "path";
 import config from '../config';
 import logger from '../logger';
-import PricesRepository, { ApiPrice, MAX_PRICES } from '../repositories/PricesRepository';
+import PricesRepository from '../repositories/PricesRepository';
 import BitfinexApi from './price-feeds/bitfinex-api';
 import BitflyerApi from './price-feeds/bitflyer-api';
 import CoinbaseApi from './price-feeds/coinbase-api';
@@ -20,18 +20,27 @@ export interface PriceFeed {
 }
 
 export interface PriceHistory {
-  [timestamp: number]: ApiPrice;
+  [timestamp: number]: Prices;
+}
+
+export interface Prices {
+  USD: number;
+  EUR: number;
+  GBP: number;
+  CAD: number;
+  CHF: number;
+  AUD: number;
+  JPY: number;
 }
 
 class PriceUpdater {
   public historyInserted = false;
-  private lastRun = 0;
-  private lastHistoricalRun = 0;
-  private running = false;
-  private feeds: PriceFeed[] = [];
-  private currencies: string[] = ['USD', 'EUR', 'GBP', 'CAD', 'CHF', 'AUD', 'JPY'];
-  private latestPrices: ApiPrice;
-  private ratesChangedCallback: ((rates: ApiPrice) => void) | undefined;
+  lastRun = 0;
+  lastHistoricalRun = 0;
+  running = false;
+  feeds: PriceFeed[] = [];
+  currencies: string[] = ['USD', 'EUR', 'GBP', 'CAD', 'CHF', 'AUD', 'JPY'];
+  latestPrices: Prices;
 
   constructor() {
     this.latestPrices = this.getEmptyPricesObj();
@@ -43,13 +52,8 @@ class PriceUpdater {
     this.feeds.push(new GeminiApi());
   }
 
-  public getLatestPrices(): ApiPrice {
-    return this.latestPrices;
-  }
-
-  public getEmptyPricesObj(): ApiPrice {
+  public getEmptyPricesObj(): Prices {
     return {
-      time: 0,
       USD: -1,
       EUR: -1,
       GBP: -1,
@@ -60,24 +64,7 @@ class PriceUpdater {
     };
   }
 
-  public setRatesChangedCallback(fn: (rates: ApiPrice) => void): void {
-    this.ratesChangedCallback = fn;
-  }
-
-  /**
-   * We execute this function before the websocket initialization since
-   * the websocket init is not done asyncronously
-   */
-  public async $initializeLatestPriceWithDb(): Promise<void> {
-    this.latestPrices = await PricesRepository.$getLatestConversionRates();
-  }
-
   public async $run(): Promise<void> {
-    if (config.MEMPOOL.NETWORK === 'signet' || config.MEMPOOL.NETWORK === 'testnet') {
-      // Coins have no value on testnet/signet, so we want to always show 0
-      return;
-    }
-
     if (this.running === true) {
       return;
     }
@@ -89,12 +76,13 @@ class PriceUpdater {
     }
 
     try {
-      await this.$updatePrice();
       if (this.historyInserted === false && config.DATABASE.ENABLED === true) {
         await this.$insertHistoricalPrices();
+      } else {
+        await this.$updatePrice();
       }
-    } catch (e: any) {
-      logger.err(`Cannot save BTC prices in db. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.mining);
+    } catch (e) {
+      logger.err(`Cannot save BTC prices in db. Reason: ${e instanceof Error ? e.message : e}`);
     }
 
     this.running = false;
@@ -124,26 +112,22 @@ class PriceUpdater {
         if (feed.currencies.includes(currency)) {
           try {
             const price = await feed.$fetchPrice(currency);
-            if (price > -1 && price < MAX_PRICES[currency]) {
+            if (price > 0) {
               prices.push(price);
             }
-            logger.debug(`${feed.name} BTC/${currency} price: ${price}`, logger.tags.mining);
+            logger.debug(`${feed.name} BTC/${currency} price: ${price}`);
           } catch (e) {
-            logger.debug(`Could not fetch BTC/${currency} price at ${feed.name}. Reason: ${(e instanceof Error ? e.message : e)}`, logger.tags.mining);
+            logger.debug(`Could not fetch BTC/${currency} price at ${feed.name}. Reason: ${(e instanceof Error ? e.message : e)}`);
           }
         }
       }
       if (prices.length === 1) {
-        logger.debug(`Only ${prices.length} feed available for BTC/${currency} price`, logger.tags.mining);
+        logger.debug(`Only ${prices.length} feed available for BTC/${currency} price`);
       }
 
       // Compute average price, non weighted
       prices = prices.filter(price => price > 0);
-      if (prices.length === 0) {
-        this.latestPrices[currency] = -1;
-      } else {
-        this.latestPrices[currency] = Math.round((prices.reduce((partialSum, a) => partialSum + a, 0)) / prices.length);
-      }
+      this.latestPrices[currency] = Math.round((prices.reduce((partialSum, a) => partialSum + a, 0)) / prices.length);
     }
 
     logger.info(`Latest BTC fiat averaged price: ${JSON.stringify(this.latestPrices)}`);
@@ -160,15 +144,7 @@ class PriceUpdater {
       }
     }
 
-    if (this.ratesChangedCallback) {
-      this.ratesChangedCallback(this.latestPrices);
-    }
-
     this.lastRun = new Date().getTime() / 1000;
-
-    if (this.latestPrices.USD === -1) {
-      this.latestPrices = await PricesRepository.$getLatestConversionRates();
-    }
   }
 
   /**
@@ -199,9 +175,9 @@ class PriceUpdater {
       ++insertedCount;
     }
     if (insertedCount > 0) {
-      logger.notice(`Inserted ${insertedCount} MtGox USD weekly price history into db`, logger.tags.mining);
+      logger.notice(`Inserted ${insertedCount} MtGox USD weekly price history into db`);
     } else {
-      logger.debug(`Inserted ${insertedCount} MtGox USD weekly price history into db`, logger.tags.mining);
+      logger.debug(`Inserted ${insertedCount} MtGox USD weekly price history into db`);
     }
 
     // Insert Kraken weekly prices
@@ -222,7 +198,7 @@ class PriceUpdater {
   private async $insertMissingRecentPrices(type: 'hour' | 'day'): Promise<void> {
     const existingPriceTimes = await PricesRepository.$getPricesTimes();
 
-    logger.info(`Fetching ${type === 'day' ? 'dai' : 'hour'}ly price history from exchanges and saving missing ones into the database`, logger.tags.mining);
+    logger.info(`Fetching ${type === 'day' ? 'dai' : 'hour'}ly price history from exchanges and saving missing ones into the database, this may take a while`);
 
     const historicalPrices: PriceHistory[] = [];
 
@@ -231,13 +207,13 @@ class PriceUpdater {
       try {
         historicalPrices.push(await feed.$fetchRecentPrice(this.currencies, type));
       } catch (e) {
-        logger.err(`Cannot fetch hourly historical price from ${feed.name}. Ignoring this feed. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.mining);
+        logger.err(`Cannot fetch hourly historical price from ${feed.name}. Ignoring this feed. Reason: ${e instanceof Error ? e.message : e}`);
       }
     }
 
     // Group them by timestamp and currency, for example
     // grouped[123456789]['USD'] = [1, 2, 3, 4];
-    const grouped = {};
+    const grouped: Object = {};
     for (const historicalEntry of historicalPrices) {
       for (const time in historicalEntry) {
         if (existingPriceTimes.includes(parseInt(time, 10))) {
@@ -252,8 +228,8 @@ class PriceUpdater {
 
         for (const currency of this.currencies) {
           const price = historicalEntry[time][currency];
-          if (price > -1 && price < MAX_PRICES[currency]) {
-            grouped[time][currency].push(typeof price === 'string' ? parseInt(price, 10) : price);
+          if (price > 0) {
+            grouped[time][currency].push(parseInt(price, 10));
           }
         }
       }
@@ -262,7 +238,7 @@ class PriceUpdater {
     // Average prices and insert everything into the db
     let totalInserted = 0;
     for (const time in grouped) {
-      const prices: ApiPrice = this.getEmptyPricesObj();
+      const prices: Prices = this.getEmptyPricesObj();
       for (const currency in grouped[time]) {
         if (grouped[time][currency].length === 0) {
           continue;
@@ -276,9 +252,9 @@ class PriceUpdater {
     }
 
     if (totalInserted > 0) {
-      logger.notice(`Inserted ${totalInserted} ${type === 'day' ? 'dai' : 'hour'}ly historical prices into the db`, logger.tags.mining);
+      logger.notice(`Inserted ${totalInserted} ${type === 'day' ? 'dai' : 'hour'}ly historical prices into the db`);
     } else {
-      logger.debug(`Inserted ${totalInserted} ${type === 'day' ? 'dai' : 'hour'}ly historical prices into the db`, logger.tags.mining);
+      logger.debug(`Inserted ${totalInserted} ${type === 'day' ? 'dai' : 'hour'}ly historical prices into the db`);
     }
   }
 }

backend/src/utils/blocks-utils.ts

@@ -0,0 +1,33 @@
+import { BlockExtended } from '../mempool.interfaces';
+
+export function prepareBlock(block: any): BlockExtended {
+  return <BlockExtended>{
+    id: block.id ?? block.hash, // hash for indexed block
+    timestamp: block.timestamp ?? block.time ?? block.blockTimestamp, // blockTimestamp for indexed block
+    height: block.height,
+    version: block.version,
+    bits: (typeof block.bits === 'string' ? parseInt(block.bits, 16): block.bits),
+    nonce: block.nonce,
+    difficulty: block.difficulty,
+    merkle_root: block.merkle_root ?? block.merkleroot,
+    tx_count: block.tx_count ?? block.nTx,
+    size: block.size,
+    weight: block.weight,
+    previousblockhash: block.previousblockhash,
+    extras: {
+      coinbaseRaw: block.coinbase_raw ?? block.extras?.coinbaseRaw,
+      medianFee: block.medianFee ?? block.median_fee ?? block.extras?.medianFee,
+      feeRange: block.feeRange ?? block.fee_span,
+      reward: block.reward ?? block?.extras?.reward,
+      totalFees: block.totalFees ?? block?.fees ?? block?.extras?.totalFees,
+      avgFee: block?.extras?.avgFee ?? block.avg_fee,
+      avgFeeRate: block?.avgFeeRate ?? block.avg_fee_rate,
+      pool: block?.extras?.pool ?? (block?.pool_id ? {
+        id: block.pool_id,
+        name: block.pool_name,
+        slug: block.pool_slug,
+      } : undefined),
+      usd: block?.extras?.usd ?? block.usd ?? null,
+    }
+  };
+}

backend/src/utils/clone.ts

@@ -1,14 +0,0 @@
-// simple recursive deep clone for literal-type objects
-// does not preserve Dates, Maps, Sets etc
-// does not support recursive objects
-// properties deeper than maxDepth will be shallow cloned
-export function deepClone(obj: any, maxDepth: number = 50, depth: number = 0): any {
-  let cloned = obj;
-  if (depth < maxDepth && typeof obj === 'object') {
-    cloned = Array.isArray(obj) ? [] : {};
-    for (const key in obj) {
-      cloned[key] = deepClone(obj[key], maxDepth, depth + 1);
-    }
-  }
-  return cloned;
-}

backend/src/utils/format.ts

@@ -1,29 +0,0 @@
-const byteUnits = ['B', 'kB', 'MB', 'GB', 'TB'];
-
-export function getBytesUnit(bytes: number): string {
-  if (isNaN(bytes) || !isFinite(bytes)) {
-    return 'B';
-  }
-  
-  let unitIndex = 0;
-  while (unitIndex < byteUnits.length && bytes > 1024) {
-    unitIndex++;
-    bytes /= 1024;
-  }
-
-  return byteUnits[unitIndex];
-}
-
-export function formatBytes(bytes: number, toUnit: string, skipUnit = false): string {
-  if (isNaN(bytes) || !isFinite(bytes)) {
-    return `${bytes}`;
-  }
-  
-  let unitIndex = 0;
-  while (unitIndex < byteUnits.length && (toUnit && byteUnits[unitIndex] !== toUnit || (!toUnit && bytes > 1024))) {
-    unitIndex++;
-    bytes /= 1024;
-  }
-
-  return `${bytes.toFixed(2)}${skipUnit ? '' : ' ' + byteUnits[unitIndex]}`;
-}

contributors/AlexLloyd0.txt

@@ -1,3 +0,0 @@
-I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of January 25, 2022.
-
-Signed: AlexLloyd0

contributors/Arooba-git.txt

@@ -1,3 +0,0 @@
-I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of January 25, 2022.
-
-Signed: Arooba-git

contributors/piterden.txt

@@ -1,3 +0,0 @@
-I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of December 17, 2022.
-
-Signed: piterden

docker/README.md

@@ -17,7 +17,7 @@ _Note: address lookups require an Electrum Server and will not work with this co
 
 The default Docker configuration assumes you have the following configuration in your `bitcoin.conf` file:
 
-```ini
+```
 txindex=1
 server=1
 rpcuser=mempool
@@ -26,7 +26,7 @@ rpcpassword=mempool
 
 If you want to use different credentials, specify them in the `docker-compose.yml` file:
 
-```yaml
+```
   api:
     environment:
       MEMPOOL_BACKEND: "none"
@@ -34,7 +34,6 @@ If you want to use different credentials, specify them in the `docker-compose.ym
       CORE_RPC_PORT: "8332"
       CORE_RPC_USERNAME: "customuser"
       CORE_RPC_PASSWORD: "custompassword"
-      CORE_RPC_TIMEOUT: "60000"
 ```
 
 The IP address in the example above refers to Docker's default gateway IP address so that the container can hit the `bitcoind` instance running on the host machine. If your setup is different, update it accordingly.
@@ -55,7 +54,7 @@ First, configure `bitcoind` as specified above, and make sure your Electrum Serv
 
 Then, set the following variables in `docker-compose.yml` so Mempool can connect to your Electrum Server:
 
-```yaml
+```
   api:
     environment:
       MEMPOOL_BACKEND: "electrum"
@@ -86,7 +85,7 @@ Below we list all settings from `mempool-config.json` and the corresponding over
 <br/>
 
 `mempool-config.json`:
-```json
+```
   "MEMPOOL": {
     "NETWORK": "mainnet",
     "BACKEND": "electrum",
@@ -101,24 +100,17 @@ Below we list all settings from `mempool-config.json` and the corresponding over
     "BLOCK_WEIGHT_UNITS": 4000000,
     "INITIAL_BLOCKS_AMOUNT": 8,
     "MEMPOOL_BLOCKS_AMOUNT": 8,
-    "BLOCKS_SUMMARIES_INDEXING": false,
+    "PRICE_FEED_UPDATE_INTERVAL": 600,
     "USE_SECOND_NODE_FOR_MINFEE": false,
-    "EXTERNAL_ASSETS": [],
+    "EXTERNAL_ASSETS": ["https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json"],
     "STDOUT_LOG_MIN_PRIORITY": "info",
-    "INDEXING_BLOCKS_AMOUNT": false,
-    "AUTOMATIC_BLOCK_REINDEXING": false,
-    "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json",
-    "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
-    "ADVANCED_GBT_AUDIT": false,
-    "ADVANCED_GBT_MEMPOOL": false,
-    "CPFP_INDEXING": false,
-    "MAX_BLOCKS_BULK_QUERY": 0,
-    "DISK_CACHE_BLOCK_INTERVAL": 6
+    "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json",
+    "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master"
   },
 ```
 
 Corresponding `docker-compose.yml` overrides:
-```yaml
+```
   api:
     environment:
       MEMPOOL_NETWORK: ""
@@ -133,55 +125,42 @@ Corresponding `docker-compose.yml` overrides:
       MEMPOOL_BLOCK_WEIGHT_UNITS: ""
       MEMPOOL_INITIAL_BLOCKS_AMOUNT: ""
       MEMPOOL_MEMPOOL_BLOCKS_AMOUNT: ""
-      MEMPOOL_BLOCKS_SUMMARIES_INDEXING: ""
+      MEMPOOL_PRICE_FEED_UPDATE_INTERVAL: ""
       MEMPOOL_USE_SECOND_NODE_FOR_MINFEE: ""
       MEMPOOL_EXTERNAL_ASSETS: ""
       MEMPOOL_STDOUT_LOG_MIN_PRIORITY: ""
-      MEMPOOL_INDEXING_BLOCKS_AMOUNT: ""
-      MEMPOOL_AUTOMATIC_BLOCK_REINDEXING: ""
       MEMPOOL_POOLS_JSON_URL: ""
       MEMPOOL_POOLS_JSON_TREE_URL: ""
-      MEMPOOL_ADVANCED_GBT_AUDIT: ""
-      MEMPOOL_ADVANCED_GBT_MEMPOOL: ""
-      MEMPOOL_CPFP_INDEXING: ""
-      MAX_BLOCKS_BULK_QUERY: ""
-      DISK_CACHE_BLOCK_INTERVAL: ""
       ...
 ```
 
-`ADVANCED_GBT_AUDIT` AND `ADVANCED_GBT_MEMPOOL` enable a more accurate (but slower) block prediction algorithm for the block audit feature and the projected mempool-blocks respectively.
-
-`CPFP_INDEXING` enables indexing CPFP (Child Pays For Parent) information for the last `INDEXING_BLOCKS_AMOUNT` blocks.
-
 <br/>
 
 `mempool-config.json`:
-```json
-  "CORE_RPC": {
+```
+"CORE_RPC": {
     "HOST": "127.0.0.1",
     "PORT": 8332,
     "USERNAME": "mempool",
-    "PASSWORD": "mempool",
-    "TIMEOUT": 60000
+    "PASSWORD": "mempool"
   },
 ```
 
 Corresponding `docker-compose.yml` overrides:
-```yaml
+```
   api:
     environment:
       CORE_RPC_HOST: ""
       CORE_RPC_PORT: ""
       CORE_RPC_USERNAME: ""
       CORE_RPC_PASSWORD: ""
-      CORE_RPC_TIMEOUT: 60000
       ...
 ```
 
 <br/>
 
 `mempool-config.json`:
-```json
+```
   "ELECTRUM": {
     "HOST": "127.0.0.1",
     "PORT": 50002,
@@ -190,7 +169,7 @@ Corresponding `docker-compose.yml` overrides:
 ```
 
 Corresponding `docker-compose.yml` overrides:
-```yaml
+```
   api:
     environment:
       ELECTRUM_HOST: ""
@@ -202,14 +181,14 @@ Corresponding `docker-compose.yml` overrides:
 <br/>
 
 `mempool-config.json`:
-```json
+```
   "ESPLORA": {
     "REST_API_URL": "http://127.0.0.1:3000"
   },
 ```
 
 Corresponding `docker-compose.yml` overrides:
-```yaml
+```
   api:
     environment:
       ESPLORA_REST_API_URL: ""
@@ -219,32 +198,30 @@ Corresponding `docker-compose.yml` overrides:
 <br/>
 
 `mempool-config.json`:
-```json
+```
   "SECOND_CORE_RPC": {
     "HOST": "127.0.0.1",
     "PORT": 8332,
     "USERNAME": "mempool",
-    "PASSWORD": "mempool",
-    "TIMEOUT": 60000
+    "PASSWORD": "mempool"
   },
 ```
 
 Corresponding `docker-compose.yml` overrides:
-```yaml
+```
   api:
     environment:
       SECOND_CORE_RPC_HOST: ""
       SECOND_CORE_RPC_PORT: ""
       SECOND_CORE_RPC_USERNAME: ""
       SECOND_CORE_RPC_PASSWORD: ""
-      SECOND_CORE_RPC_TIMEOUT: ""
       ...
 ```
 
 <br/>
 
 `mempool-config.json`:
-```json
+```
   "DATABASE": {
     "ENABLED": true,
     "HOST": "127.0.0.1",
@@ -256,7 +233,7 @@ Corresponding `docker-compose.yml` overrides:
 ```
 
 Corresponding `docker-compose.yml` overrides:
-```yaml
+```
   api:
     environment:
       DATABASE_ENABLED: ""
@@ -271,7 +248,7 @@ Corresponding `docker-compose.yml` overrides:
 <br/>
 
 `mempool-config.json`:
-```json
+```
   "SYSLOG": {
     "ENABLED": true,
     "HOST": "127.0.0.1",
@@ -282,7 +259,7 @@ Corresponding `docker-compose.yml` overrides:
 ```
 
 Corresponding `docker-compose.yml` overrides:
-```yaml
+```
   api:
     environment:
       SYSLOG_ENABLED: ""
@@ -296,7 +273,7 @@ Corresponding `docker-compose.yml` overrides:
 <br/>
 
 `mempool-config.json`:
-```json
+```
   "STATISTICS": {
     "ENABLED": true,
     "TX_PER_SECOND_SAMPLE_PERIOD": 150
@@ -304,7 +281,7 @@ Corresponding `docker-compose.yml` overrides:
 ```
 
 Corresponding `docker-compose.yml` overrides:
-```yaml
+```
   api:
     environment:
       STATISTICS_ENABLED: ""
@@ -315,7 +292,7 @@ Corresponding `docker-compose.yml` overrides:
 <br/>
 
 `mempool-config.json`:
-```json
+```
   "BISQ": {
     "ENABLED": false,
     "DATA_PATH": "/bisq/statsnode-data/btc_mainnet/db"
@@ -323,7 +300,7 @@ Corresponding `docker-compose.yml` overrides:
 ```
 
 Corresponding `docker-compose.yml` overrides:
-```yaml
+```
   api:
     environment:
       BISQ_ENABLED: ""
@@ -334,7 +311,7 @@ Corresponding `docker-compose.yml` overrides:
 <br/>
 
 `mempool-config.json`:
-```json
+```
   "SOCKS5PROXY": {
     "ENABLED": false,
     "HOST": "127.0.0.1",
@@ -345,7 +322,7 @@ Corresponding `docker-compose.yml` overrides:
 ```
 
 Corresponding `docker-compose.yml` overrides:
-```yaml
+```
   api:
     environment:
       SOCKS5PROXY_ENABLED: ""
@@ -359,7 +336,7 @@ Corresponding `docker-compose.yml` overrides:
 <br/>
 
 `mempool-config.json`:
-```json
+```
   "PRICE_DATA_SERVER": {
     "TOR_URL": "http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices",
     "CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices"
@@ -367,7 +344,7 @@ Corresponding `docker-compose.yml` overrides:
 ```
 
 Corresponding `docker-compose.yml` overrides:
-```yaml
+```
   api:
     environment:
       PRICE_DATA_SERVER_TOR_URL: ""
@@ -378,7 +355,7 @@ Corresponding `docker-compose.yml` overrides:
 <br/>
 
 `mempool-config.json`:
-```json
+```
   "LIGHTNING": {
     "ENABLED": false
     "BACKEND": "lnd"
@@ -390,7 +367,7 @@ Corresponding `docker-compose.yml` overrides:
 ```
 
 Corresponding `docker-compose.yml` overrides:
-```yaml
+```
   api:
     environment:
       LIGHTNING_ENABLED: false
@@ -405,62 +382,37 @@ Corresponding `docker-compose.yml` overrides:
 <br/>
 
 `mempool-config.json`:
-```json
+```
   "LND": {
     "TLS_CERT_PATH": ""
     "MACAROON_PATH": ""
     "REST_API_URL": "https://localhost:8080"
-    "TIMEOUT": 10000
   }
 ```
 
 Corresponding `docker-compose.yml` overrides:
-```yaml
+```
   api:
     environment:
       LND_TLS_CERT_PATH: ""
       LND_MACAROON_PATH: ""
       LND_REST_API_URL: "https://localhost:8080"
-      LND_TIMEOUT: 10000
       ...
 ```
 
 <br/>
 
 `mempool-config.json`:
-```json
+```
   "CLIGHTNING": {
     "SOCKET": ""
   }
 ```
 
 Corresponding `docker-compose.yml` overrides:
-```yaml
+```
   api:
     environment:
       CLIGHTNING_SOCKET: ""
       ...
 ```
-
-<br/>
-
-`mempool-config.json`:
-```json
-  "MAXMIND": {
-    "ENABLED": true,
-    "GEOLITE2_CITY": "/usr/local/share/GeoIP/GeoLite2-City.mmdb",
-    "GEOLITE2_ASN": "/usr/local/share/GeoIP/GeoLite2-ASN.mmdb",
-    "GEOIP2_ISP": "/usr/local/share/GeoIP/GeoIP2-ISP.mmdb"
-  }
-```
-
-Corresponding `docker-compose.yml` overrides:
-```yaml
-  api:
-    environment:
-      MAXMIND_ENABLED: true,
-      MAXMIND_GEOLITE2_CITY: "/backend/GeoIP/GeoLite2-City.mmdb",
-      MAXMIND_GEOLITE2_ASN": "/backend/GeoIP/GeoLite2-ASN.mmdb",
-      MAXMIND_GEOIP2_ISP": "/backend/GeoIP/GeoIP2-ISP.mmdb"
-      ...
-```

docker/backend/Dockerfile

@@ -17,7 +17,6 @@ WORKDIR /backend
 
 RUN chown 1000:1000 ./
 COPY --from=builder --chown=1000:1000 /build/package ./package/
-COPY --from=builder --chown=1000:1000 /build/GeoIP ./GeoIP/
 COPY --from=builder --chown=1000:1000 /build/mempool-config.json /build/start.sh /build/wait-for-it.sh ./
 
 USER 1000

docker/backend/mempool-config.json

@@ -13,6 +13,7 @@
     "BLOCK_WEIGHT_UNITS": __MEMPOOL_BLOCK_WEIGHT_UNITS__,
     "INITIAL_BLOCKS_AMOUNT": __MEMPOOL_INITIAL_BLOCKS_AMOUNT__,
     "MEMPOOL_BLOCKS_AMOUNT": __MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__,
+    "PRICE_FEED_UPDATE_INTERVAL": __MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__,
     "USE_SECOND_NODE_FOR_MINFEE": __MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__,
     "EXTERNAL_ASSETS": __MEMPOOL_EXTERNAL_ASSETS__,
     "EXTERNAL_MAX_RETRY": __MEMPOOL_EXTERNAL_MAX_RETRY__,
@@ -21,20 +22,13 @@
     "STDOUT_LOG_MIN_PRIORITY": "__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__",
     "INDEXING_BLOCKS_AMOUNT": __MEMPOOL_INDEXING_BLOCKS_AMOUNT__,
     "BLOCKS_SUMMARIES_INDEXING": __MEMPOOL_BLOCKS_SUMMARIES_INDEXING__,
-    "AUTOMATIC_BLOCK_REINDEXING": __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__,
-    "AUDIT": __MEMPOOL_AUDIT__,
-    "ADVANCED_GBT_AUDIT": __MEMPOOL_ADVANCED_GBT_AUDIT__,
-    "ADVANCED_GBT_MEMPOOL": __MEMPOOL_ADVANCED_GBT_MEMPOOL__,
-    "CPFP_INDEXING": __MEMPOOL_CPFP_INDEXING__,
-    "MAX_BLOCKS_BULK_QUERY": __MEMPOOL_MAX_BLOCKS_BULK_QUERY__,
-    "DISK_CACHE_BLOCK_INTERVAL": __MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__
+    "AUTOMATIC_BLOCK_REINDEXING": __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__
   },
   "CORE_RPC": {
     "HOST": "__CORE_RPC_HOST__",
     "PORT": __CORE_RPC_PORT__,
     "USERNAME": "__CORE_RPC_USERNAME__",
-    "PASSWORD": "__CORE_RPC_PASSWORD__",
-    "TIMEOUT": __CORE_RPC_TIMEOUT__
+    "PASSWORD": "__CORE_RPC_PASSWORD__"
   },
   "ELECTRUM": {
     "HOST": "__ELECTRUM_HOST__",
@@ -48,8 +42,7 @@
     "HOST": "__SECOND_CORE_RPC_HOST__",
     "PORT": __SECOND_CORE_RPC_PORT__,
     "USERNAME": "__SECOND_CORE_RPC_USERNAME__",
-    "PASSWORD": "__SECOND_CORE_RPC_PASSWORD__",
-    "TIMEOUT": __SECOND_CORE_RPC_TIMEOUT__
+    "PASSWORD": "__SECOND_CORE_RPC_PASSWORD__"
   },
   "DATABASE": {
     "ENABLED": __DATABASE_ENABLED__,
@@ -86,8 +79,7 @@
   "LND": {
     "TLS_CERT_PATH": "__LND_TLS_CERT_PATH__",
     "MACAROON_PATH": "__LND_MACAROON_PATH__",
-    "REST_API_URL": "__LND_REST_API_URL__",
-    "TIMEOUT": "__LND_TIMEOUT__"
+    "REST_API_URL": "__LND_REST_API_URL__"
   },
   "CLIGHTNING": {
     "SOCKET": "__CLIGHTNING_SOCKET__"
@@ -111,11 +103,5 @@
     "LIQUID_ONION": "__EXTERNAL_DATA_SERVER_LIQUID_ONION__",
     "BISQ_URL": "__EXTERNAL_DATA_SERVER_BISQ_URL__",
     "BISQ_ONION": "__EXTERNAL_DATA_SERVER_BISQ_ONION__"
-  },
-  "MAXMIND": {
-    "ENABLED": __MAXMIND_ENABLED__,
-    "GEOLITE2_CITY": "__MAXMIND_GEOLITE2_CITY__",
-    "GEOLITE2_ASN": "__MAXMIND_GEOLITE2_ASN__",
-    "GEOIP2_ISP": "__MAXMIND_GEOIP2_ISP__"
   }
 }

docker/backend/start.sh

@@ -16,28 +16,23 @@ __MEMPOOL_INITIAL_BLOCKS_AMOUNT__=${MEMPOOL_INITIAL_BLOCKS_AMOUNT:=8}
 __MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__=${MEMPOOL_MEMPOOL_BLOCKS_AMOUNT:=8}
 __MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=11000}
 __MEMPOOL_BLOCKS_SUMMARIES_INDEXING__=${MEMPOOL_BLOCKS_SUMMARIES_INDEXING:=false}
+__MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__=${MEMPOOL_PRICE_FEED_UPDATE_INTERVAL:=600}
 __MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__=${MEMPOOL_USE_SECOND_NODE_FOR_MINFEE:=false}
 __MEMPOOL_EXTERNAL_ASSETS__=${MEMPOOL_EXTERNAL_ASSETS:=[]}
 __MEMPOOL_EXTERNAL_MAX_RETRY__=${MEMPOOL_EXTERNAL_MAX_RETRY:=1}
 __MEMPOOL_EXTERNAL_RETRY_INTERVAL__=${MEMPOOL_EXTERNAL_RETRY_INTERVAL:=0}
 __MEMPOOL_USER_AGENT__=${MEMPOOL_USER_AGENT:=mempool}
 __MEMPOOL_STDOUT_LOG_MIN_PRIORITY__=${MEMPOOL_STDOUT_LOG_MIN_PRIORITY:=info}
+__MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=false}
 __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__=${MEMPOOL_AUTOMATIC_BLOCK_REINDEXING:=false}
-__MEMPOOL_POOLS_JSON_URL__=${MEMPOOL_POOLS_JSON_URL:=https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json}
+__MEMPOOL_POOLS_JSON_URL__=${MEMPOOL_POOLS_JSON_URL:=https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json}
 __MEMPOOL_POOLS_JSON_TREE_URL__=${MEMPOOL_POOLS_JSON_TREE_URL:=https://api.github.com/repos/mempool/mining-pools/git/trees/master}
-__MEMPOOL_AUDIT__=${MEMPOOL_AUDIT:=false}
-__MEMPOOL_ADVANCED_GBT_AUDIT__=${MEMPOOL_ADVANCED_GBT_AUDIT:=false}
-__MEMPOOL_ADVANCED_GBT_MEMPOOL__=${MEMPOOL_ADVANCED_GBT_MEMPOOL:=false}
-__MEMPOOL_CPFP_INDEXING__=${MEMPOOL_CPFP_INDEXING:=false}
-__MEMPOOL_MAX_BLOCKS_BULK_QUERY__=${MEMPOOL_MAX_BLOCKS_BULK_QUERY:=0}
-__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__=${MEMPOOL_DISK_CACHE_BLOCK_INTERVAL:=6}
 
 # CORE_RPC
 __CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1}
 __CORE_RPC_PORT__=${CORE_RPC_PORT:=8332}
 __CORE_RPC_USERNAME__=${CORE_RPC_USERNAME:=mempool}
 __CORE_RPC_PASSWORD__=${CORE_RPC_PASSWORD:=mempool}
-__CORE_RPC_TIMEOUT__=${CORE_RPC_TIMEOUT:=60000}
 
 # ELECTRUM
 __ELECTRUM_HOST__=${ELECTRUM_HOST:=127.0.0.1}
@@ -52,7 +47,6 @@ __SECOND_CORE_RPC_HOST__=${SECOND_CORE_RPC_HOST:=127.0.0.1}
 __SECOND_CORE_RPC_PORT__=${SECOND_CORE_RPC_PORT:=8332}
 __SECOND_CORE_RPC_USERNAME__=${SECOND_CORE_RPC_USERNAME:=mempool}
 __SECOND_CORE_RPC_PASSWORD__=${SECOND_CORE_RPC_PASSWORD:=mempool}
-__SECOND_CORE_RPC_TIMEOUT__=${SECOND_CORE_RPC_TIMEOUT:=60000}
 
 # DATABASE
 __DATABASE_ENABLED__=${DATABASE_ENABLED:=true}
@@ -110,18 +104,10 @@ __LIGHTNING_LOGGER_UPDATE_INTERVAL__=${LIGHTNING_LOGGER_UPDATE_INTERVAL:=30}
 __LND_TLS_CERT_PATH__=${LND_TLS_CERT_PATH:=""}
 __LND_MACAROON_PATH__=${LND_MACAROON_PATH:=""}
 __LND_REST_API_URL__=${LND_REST_API_URL:="https://localhost:8080"}
-__LND_TIMEOUT__=${LND_TIMEOUT:=10000}
 
 # CLN
 __CLIGHTNING_SOCKET__=${CLIGHTNING_SOCKET:=""}
 
-# MAXMIND
-__MAXMIND_ENABLED__=${MAXMIND_ENABLED:=true}
-__MAXMIND_GEOLITE2_CITY__=${MAXMIND_GEOLITE2_CITY:="/backend/GeoIP/GeoLite2-City.mmdb"}
-__MAXMIND_GEOLITE2_ASN__=${MAXMIND_GEOLITE2_ASN:="/backend/GeoIP/GeoLite2-ASN.mmdb"}
-__MAXMIND_GEOIP2_ISP__=${MAXMIND_GEOIP2_ISP:=""}
-
-
 mkdir -p "${__MEMPOOL_CACHE_DIR__}"
 
 sed -i "s/__MEMPOOL_NETWORK__/${__MEMPOOL_NETWORK__}/g" mempool-config.json
@@ -139,27 +125,22 @@ sed -i "s/__MEMPOOL_INITIAL_BLOCKS_AMOUNT__/${__MEMPOOL_INITIAL_BLOCKS_AMOUNT__}
 sed -i "s/__MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__/${__MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__}/g" mempool-config.json
 sed -i "s/__MEMPOOL_INDEXING_BLOCKS_AMOUNT__/${__MEMPOOL_INDEXING_BLOCKS_AMOUNT__}/g" mempool-config.json
 sed -i "s/__MEMPOOL_BLOCKS_SUMMARIES_INDEXING__/${__MEMPOOL_BLOCKS_SUMMARIES_INDEXING__}/g" mempool-config.json
+sed -i "s/__MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__/${__MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__}/g" mempool-config.json
 sed -i "s/__MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__/${__MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__}/g" mempool-config.json
 sed -i "s!__MEMPOOL_EXTERNAL_ASSETS__!${__MEMPOOL_EXTERNAL_ASSETS__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_EXTERNAL_MAX_RETRY__!${__MEMPOOL_EXTERNAL_MAX_RETRY__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_EXTERNAL_RETRY_INTERVAL__!${__MEMPOOL_EXTERNAL_RETRY_INTERVAL__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_USER_AGENT__!${__MEMPOOL_USER_AGENT__}!g" mempool-config.json
 sed -i "s/__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__/${__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__}/g" mempool-config.json
+sed -i "s/__MEMPOOL_INDEXING_BLOCKS_AMOUNT__/${__MEMPOOL_INDEXING_BLOCKS_AMOUNT__}/g" mempool-config.json
 sed -i "s/__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__/${__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__}/g" mempool-config.json
 sed -i "s!__MEMPOOL_POOLS_JSON_URL__!${__MEMPOOL_POOLS_JSON_URL__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_POOLS_JSON_TREE_URL__!${__MEMPOOL_POOLS_JSON_TREE_URL__}!g" mempool-config.json
-sed -i "s!__MEMPOOL_AUDIT__!${__MEMPOOL_AUDIT__}!g" mempool-config.json
-sed -i "s!__MEMPOOL_ADVANCED_GBT_MEMPOOL__!${__MEMPOOL_ADVANCED_GBT_MEMPOOL__}!g" mempool-config.json
-sed -i "s!__MEMPOOL_ADVANCED_GBT_AUDIT__!${__MEMPOOL_ADVANCED_GBT_AUDIT__}!g" mempool-config.json
-sed -i "s!__MEMPOOL_CPFP_INDEXING__!${__MEMPOOL_CPFP_INDEXING__}!g" mempool-config.json
-sed -i "s!__MEMPOOL_MAX_BLOCKS_BULK_QUERY__!${__MEMPOOL_MAX_BLOCKS_BULK_QUERY__}!g" mempool-config.json
-sed -i "s!__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__!${__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__}!g" mempool-config.json
 
 sed -i "s/__CORE_RPC_HOST__/${__CORE_RPC_HOST__}/g" mempool-config.json
 sed -i "s/__CORE_RPC_PORT__/${__CORE_RPC_PORT__}/g" mempool-config.json
 sed -i "s/__CORE_RPC_USERNAME__/${__CORE_RPC_USERNAME__}/g" mempool-config.json
 sed -i "s/__CORE_RPC_PASSWORD__/${__CORE_RPC_PASSWORD__}/g" mempool-config.json
-sed -i "s/__CORE_RPC_TIMEOUT__/${__CORE_RPC_TIMEOUT__}/g" mempool-config.json
 
 sed -i "s/__ELECTRUM_HOST__/${__ELECTRUM_HOST__}/g" mempool-config.json
 sed -i "s/__ELECTRUM_PORT__/${__ELECTRUM_PORT__}/g" mempool-config.json
@@ -171,7 +152,6 @@ sed -i "s/__SECOND_CORE_RPC_HOST__/${__SECOND_CORE_RPC_HOST__}/g" mempool-config
 sed -i "s/__SECOND_CORE_RPC_PORT__/${__SECOND_CORE_RPC_PORT__}/g" mempool-config.json
 sed -i "s/__SECOND_CORE_RPC_USERNAME__/${__SECOND_CORE_RPC_USERNAME__}/g" mempool-config.json
 sed -i "s/__SECOND_CORE_RPC_PASSWORD__/${__SECOND_CORE_RPC_PASSWORD__}/g" mempool-config.json
-sed -i "s/__SECOND_CORE_RPC_TIMEOUT__/${__SECOND_CORE_RPC_TIMEOUT__}/g" mempool-config.json
 
 sed -i "s/__DATABASE_ENABLED__/${__DATABASE_ENABLED__}/g" mempool-config.json
 sed -i "s/__DATABASE_HOST__/${__DATABASE_HOST__}/g" mempool-config.json
@@ -223,16 +203,8 @@ sed -i "s!__LIGHTNING_LOGGER_UPDATE_INTERVAL__!${__LIGHTNING_LOGGER_UPDATE_INTER
 sed -i "s!__LND_TLS_CERT_PATH__!${__LND_TLS_CERT_PATH__}!g" mempool-config.json
 sed -i "s!__LND_MACAROON_PATH__!${__LND_MACAROON_PATH__}!g" mempool-config.json
 sed -i "s!__LND_REST_API_URL__!${__LND_REST_API_URL__}!g" mempool-config.json
-sed -i "s!__LND_TIMEOUT__!${__LND_TIMEOUT__}!g" mempool-config.json
 
 # CLN
 sed -i "s!__CLIGHTNING_SOCKET__!${__CLIGHTNING_SOCKET__}!g" mempool-config.json
 
-# MAXMIND
-sed -i "s!__MAXMIND_ENABLED__!${__MAXMIND_ENABLED__}!g" mempool-config.json
-sed -i "s!__MAXMIND_GEOLITE2_CITY__!${__MAXMIND_GEOLITE2_CITY__}!g" mempool-config.json
-sed -i "s!__MAXMIND_GEOLITE2_ASN__!${__MAXMIND_GEOLITE2_ASN__}!g" mempool-config.json
-sed -i "s!__MAXMIND_GEOIP2_ISP__!${__MAXMIND_GEOIP2_ISP__}!g" mempool-config.json
-
-
 node /backend/package/index.js

docker/frontend/entrypoint.sh

@@ -10,10 +10,6 @@ cp /etc/nginx/nginx.conf /patch/nginx.conf
 sed -i "s/__MEMPOOL_FRONTEND_HTTP_PORT__/${__MEMPOOL_FRONTEND_HTTP_PORT__}/g" /patch/nginx.conf
 cat /patch/nginx.conf > /etc/nginx/nginx.conf
 
-if [ "${LIGHTNING_DETECTED_PORT}" != "" ];then
-  export LIGHTNING=true
-fi
-
 # Runtime overrides - read env vars defined in docker compose
 
 __TESTNET_ENABLED__=${TESTNET_ENABLED:=false}
@@ -35,11 +31,6 @@ __LIQUID_WEBSITE_URL__=${LIQUID_WEBSITE_URL:=https://liquid.network}
 __BISQ_WEBSITE_URL__=${BISQ_WEBSITE_URL:=https://bisq.markets}
 __MINING_DASHBOARD__=${MINING_DASHBOARD:=true}
 __LIGHTNING__=${LIGHTNING:=false}
-__AUDIT__=${AUDIT:=false}
-__MAINNET_BLOCK_AUDIT_START_HEIGHT__=${MAINNET_BLOCK_AUDIT_START_HEIGHT:=0}
-__TESTNET_BLOCK_AUDIT_START_HEIGHT__=${TESTNET_BLOCK_AUDIT_START_HEIGHT:=0}
-__SIGNET_BLOCK_AUDIT_START_HEIGHT__=${SIGNET_BLOCK_AUDIT_START_HEIGHT:=0}
-__HISTORICAL_PRICE__=${HISTORICAL_PRICE:=true}
 
 # Export as environment variables to be used by envsubst
 export __TESTNET_ENABLED__
@@ -61,11 +52,6 @@ export __LIQUID_WEBSITE_URL__
 export __BISQ_WEBSITE_URL__
 export __MINING_DASHBOARD__
 export __LIGHTNING__
-export __AUDIT__
-export __MAINNET_BLOCK_AUDIT_START_HEIGHT__
-export __TESTNET_BLOCK_AUDIT_START_HEIGHT__
-export __SIGNET_BLOCK_AUDIT_START_HEIGHT__
-export __HISTORICAL_PRICE__
 
 folder=$(find /var/www/mempool -name "config.js" | xargs dirname)
 echo ${folder}

docker/init.sh

@@ -3,11 +3,6 @@
 #backend
 cp ./docker/backend/* ./backend/
 
-#geoip-data
-mkdir -p ./backend/GeoIP/
-wget -O ./backend/GeoIP/GeoLite2-City.mmdb https://raw.githubusercontent.com/mempool/geoip-data/master/GeoLite2-City.mmdb
-wget -O ./backend/GeoIP/GeoLite2-ASN.mmdb https://raw.githubusercontent.com/mempool/geoip-data/master/GeoLite2-ASN.mmdb
-
 #frontend
 localhostIP="127.0.0.1"
 cp ./docker/frontend/* ./frontend

frontend/.gitignore

@@ -54,8 +54,6 @@ src/resources/assets-testnet.json
 src/resources/assets-testnet.minimal.json
 src/resources/pools.json
 src/resources/mining-pools/*
-src/resources/**/*.mp4
-src/resources/**/*.vtt
 
 # environment config
 mempool-frontend-config.json

frontend/.tx/config

@@ -1,9 +1,7 @@
 [main]
 host = https://www.transifex.com
 
-[o:mempool:p:mempool:r:frontend-src-locale-messages-xlf--master]
+[mempool.frontend-src-locale-messages-xlf--master]
 file_filter = frontend/src/locale/messages.<lang>.xlf
-source_file = frontend/src/locale/messages.en-US.xlf
 source_lang = en-US
-type        = XLIFF
-
+type = XLIFF

frontend/README.md

@@ -106,15 +106,13 @@ https://www.transifex.com/mempool/mempool/dashboard/
 
 * Arabic @baro0k
 * Czech @pixelmade2
-* Danish @pierrevendelboe
 * German @Emzy
 * English (default)
 * Spanish @maxhodler @bisqes
 * Persian @techmix
 * French @Bayernatoor
-* Korean @kcalvinalvinn @sogoagain
+* Korean @kcalvinalvinn
 * Italian @HodlBits
-* Lithuanian @eimze21
 * Hebrew @rapidlab309
 * Georgian @wyd_idk
 * Hungarian @btcdragonlord
@@ -134,4 +132,3 @@ https://www.transifex.com/mempool/mempool/dashboard/
 * Russian @TonyCrusoe @Bitconan
 * Romanian @mirceavesa
 * Macedonian @SkechBoy
-* Nepalese @kebinm

frontend/angular.json

@@ -38,10 +38,6 @@
             "translation": "src/locale/messages.de.xlf",
             "baseHref": "/de/"
           },
-          "da": {
-            "translation": "src/locale/messages.da.xlf",
-            "baseHref": "/da/"
-          },
           "es": {
             "translation": "src/locale/messages.es.xlf",
             "baseHref": "/es/"
@@ -142,10 +138,6 @@
             "translation": "src/locale/messages.hi.xlf",
             "baseHref": "/hi/"
           },
-          "ne": {
-            "translation": "src/locale/messages.ne.xlf",
-            "baseHref": "/ne/"
-          },
           "lt": {
             "translation": "src/locale/messages.lt.xlf",
             "baseHref": "/lt/"
@@ -170,6 +162,11 @@
             ],
             "styles": [
               "src/styles.scss",
+              {
+                "input": "src/theme-contrast.scss",
+                "bundleName": "contrast",
+                "inject": false
+              },
               "node_modules/@fortawesome/fontawesome-svg-core/styles.css"
             ],
             "vendorChunk": true,

frontend/cypress.config.ts

@@ -1,4 +1,4 @@
-import { defineConfig } from 'cypress';
+import { defineConfig } from 'cypress'
 
 export default defineConfig({
   projectId: 'ry4br7',
@@ -12,18 +12,12 @@ export default defineConfig({
   },
   chromeWebSecurity: false,
   e2e: {
-    setupNodeEvents(on: any, config: any) {
-      const fs = require('fs');
-      const CONFIG_FILE = 'mempool-frontend-config.json';
-      if (fs.existsSync(CONFIG_FILE)) {
-        let contents = JSON.parse(fs.readFileSync(CONFIG_FILE, 'utf8'));
-        config.env.BASE_MODULE = contents.BASE_MODULE ? contents.BASE_MODULE : 'mempool';
-      } else {
-        config.env.BASE_MODULE = 'mempool';
-      }
-      return config;
+    // We've imported your old cypress plugins here.
+    // You may want to clean this up later by importing these.
+    setupNodeEvents(on, config) {
+      return require('./cypress/plugins/index.js')(on, config)
     },
     baseUrl: 'http://localhost:4200',
     specPattern: 'cypress/e2e/**/*.{js,jsx,ts,tsx}',
   },
-});
+})

frontend/cypress/e2e/bisq/bisq.spec.ts

@@ -1,5 +1,5 @@
 describe('Bisq', () => {
-  const baseModule = Cypress.env('BASE_MODULE');
+  const baseModule = Cypress.env("BASE_MODULE");
   const basePath = '';
 
   beforeEach(() => {
@@ -20,7 +20,7 @@ describe('Bisq', () => {
       cy.waitForSkeletonGone();
     });
 
-    describe('transactions', () => {
+    describe("transactions", () => {
       it('loads the transactions screen', () => {
         cy.visit(`${basePath}`);
         cy.waitForSkeletonGone();
@@ -30,9 +30,9 @@ describe('Bisq', () => {
       });
 
       const filters = [
-        'Asset listing fee', 'Blind vote', 'Compensation request',
-        'Genesis', 'Irregular', 'Lockup', 'Pay trade fee', 'Proof of burn',
-        'Proposal', 'Reimbursement request', 'Transfer BSQ', 'Unlock', 'Vote reveal'
+        "Asset listing fee", "Blind vote", "Compensation request",
+        "Genesis", "Irregular", "Lockup", "Pay trade fee", "Proof of burn",
+        "Proposal", "Reimbursement request", "Transfer BSQ", "Unlock", "Vote reveal"
       ];
       filters.forEach((filter) => {
         it.only(`filters the transaction screen by ${filter}`, () => {
@@ -49,7 +49,7 @@ describe('Bisq', () => {
         });
       });
 
-      it('filters using multiple criteria', () => {
+      it("filters using multiple criteria", () => {
         const filters = ['Proposal', 'Lockup', 'Unlock'];
         cy.visit(`${basePath}/transactions`);
         cy.waitForSkeletonGone();

frontend/cypress/e2e/liquid/liquid.spec.ts

@@ -1,5 +1,5 @@
 describe('Liquid', () => {
-  const baseModule = Cypress.env('BASE_MODULE');
+  const baseModule = Cypress.env("BASE_MODULE");
   const basePath = '';
 
   beforeEach(() => {
@@ -7,6 +7,7 @@ describe('Liquid', () => {
     cy.intercept('/liquid/api/blocks/').as('blocks');
     cy.intercept('/liquid/api/tx/**/outspends').as('outspends');
     cy.intercept('/liquid/api/block/**/txs/**').as('block-txs');
+    cy.intercept('/resources/pools.json').as('pools');
 
     Cypress.Commands.add('waitForBlockData', () => {
       cy.wait('@socket');
@@ -158,10 +159,10 @@ describe('Liquid', () => {
       it('show empty unblinded TX', () => {
         cy.visit(`${basePath}/tx/f2f41c0850e8e7e3f1af233161fd596662e67c11ef10ed15943884186fbb7f46#blinded=`);
         cy.waitForSkeletonGone();
-        cy.get('.table-tx-vin tr:nth-child(1)').should('have.class', 'ng-star-inserted');
+        cy.get('.table-tx-vin tr:nth-child(1)').should('have.class', '');
         cy.get('.table-tx-vin tr:nth-child(1) .amount').should('contain.text', 'Confidential');
-        cy.get('.table-tx-vout tr:nth-child(1)').should('have.class', 'ng-star-inserted');
-        cy.get('.table-tx-vout tr:nth-child(2)').should('have.class', 'ng-star-inserted');
+        cy.get('.table-tx-vout tr:nth-child(1)').should('have.class', '');
+        cy.get('.table-tx-vout tr:nth-child(2)').should('have.class', '');
         cy.get('.table-tx-vout tr:nth-child(1) .amount').should('contain.text', 'Confidential');
         cy.get('.table-tx-vout tr:nth-child(2) .amount').should('contain.text', 'Confidential');
       });
@@ -169,8 +170,8 @@ describe('Liquid', () => {
       it('show invalid unblinded TX hex', () => {
         cy.visit(`${basePath}/tx/f2f41c0850e8e7e3f1af233161fd596662e67c11ef10ed15943884186fbb7f46#blinded=123`);
         cy.waitForSkeletonGone();
-        cy.get('.table-tx-vin tr').should('have.class', 'ng-star-inserted');
-        cy.get('.table-tx-vout tr').should('have.class', 'ng-star-inserted');
+        cy.get('.table-tx-vin tr').should('have.class', '');
+        cy.get('.table-tx-vout tr').should('have.class', '');
         cy.get('.error-unblinded').contains('Error: Invalid blinding data (invalid hex)');
       });
 

frontend/cypress/e2e/liquidtestnet/liquidtestnet.spec.ts

@@ -1,5 +1,5 @@
 describe('Liquid Testnet', () => {
-  const baseModule = Cypress.env('BASE_MODULE');
+  const baseModule = Cypress.env("BASE_MODULE");
   const basePath = '/testnet';
 
   beforeEach(() => {
@@ -7,6 +7,7 @@ describe('Liquid Testnet', () => {
     cy.intercept('/liquidtestnet/api/blocks/').as('blocks');
     cy.intercept('/liquidtestnet/api/tx/**/outspends').as('outspends');
     cy.intercept('/liquidtestnet/api/block/**/txs/**').as('block-txs');
+    cy.intercept('/resources/pools.json').as('pools');
 
     Cypress.Commands.add('waitForBlockData', () => {
       cy.wait('@socket');
@@ -109,10 +110,10 @@ describe('Liquid Testnet', () => {
       it('show empty unblinded TX', () => {
         cy.visit(`${basePath}/tx/c3d908ab77891e4c569b0df71aae90f4720b157019ebb20db176f4f9c4d626b8#blinded=`);
         cy.waitForSkeletonGone();
-        cy.get('.table-tx-vin tr:nth-child(1)').should('have.class', 'ng-star-inserted');
+        cy.get('.table-tx-vin tr:nth-child(1)').should('have.class', '');
         cy.get('.table-tx-vin tr:nth-child(1) .amount').should('contain.text', 'Confidential');
-        cy.get('.table-tx-vout tr:nth-child(1)').should('have.class', 'ng-star-inserted');
-        cy.get('.table-tx-vout tr:nth-child(2)').should('have.class', 'ng-star-inserted');
+        cy.get('.table-tx-vout tr:nth-child(1)').should('have.class', '');
+        cy.get('.table-tx-vout tr:nth-child(2)').should('have.class', '');
         cy.get('.table-tx-vout tr:nth-child(1) .amount').should('contain.text', 'Confidential');
         cy.get('.table-tx-vout tr:nth-child(2) .amount').should('contain.text', 'Confidential');
       });
@@ -120,8 +121,8 @@ describe('Liquid Testnet', () => {
       it('show invalid unblinded TX hex', () => {
         cy.visit(`${basePath}/tx/2477f220eef1d03f8ffa4a2861c275d155c3562adf0d79523aeeb0c59ee611ba#blinded=5000`);
         cy.waitForSkeletonGone();
-        cy.get('.table-tx-vin tr').should('have.class', 'ng-star-inserted');
-        cy.get('.table-tx-vout tr').should('have.class', 'ng-star-inserted');
+        cy.get('.table-tx-vin tr').should('have.class', '');
+        cy.get('.table-tx-vout tr').should('have.class', '');
         cy.get('.error-unblinded').contains('Error: Invalid blinding data (invalid hex)');
       });
 

frontend/cypress/e2e/mainnet/mainnet.spec.ts

@@ -1,6 +1,6 @@
-import { emitMempoolInfo, dropWebSocket } from '../../support/websocket';
+import { emitMempoolInfo, dropWebSocket } from "../../support/websocket";
 
-const baseModule = Cypress.env('BASE_MODULE');
+const baseModule = Cypress.env("BASE_MODULE");
 
 
 //Credit: https://github.com/bahmutov/cypress-examples/blob/6cedb17f83a3bb03ded13cf1d6a3f0656ca2cdf5/docs/recipes/overlapping-elements.md
@@ -41,6 +41,7 @@ describe('Mainnet', () => {
     // cy.intercept('/api/v1/block/*/summary').as('block-summary');
     // cy.intercept('/api/v1/outspends/*').as('outspends');
     // cy.intercept('/api/tx/*/outspends').as('tx-outspends');
+    // cy.intercept('/resources/pools.json').as('pools');
 
     // Search Auto Complete
     cy.intercept('/api/address-prefix/1wiz').as('search-1wiz');
@@ -64,7 +65,7 @@ describe('Mainnet', () => {
     it('loads the status screen', () => {
       cy.visit('/status');
       cy.get('#mempool-block-0').should('be.visible');
-      cy.get('[id^="bitcoin-block-"]').should('have.length', 22);
+      cy.get('[id^="bitcoin-block-"]').should('have.length', 8);
       cy.get('.footer').should('be.visible');
       cy.get('.row > :nth-child(1)').invoke('text').then((text) => {
         expect(text).to.match(/Incoming transactions.* vB\/s/);
@@ -219,11 +220,11 @@ describe('Mainnet', () => {
     describe('blocks navigation', () => {
 
       describe('keyboard events', () => {
-        it('loads first blockchain block visible and keypress arrow right', () => {
+        it('loads first blockchain blocks visible and keypress arrow right', () => {
           cy.viewport('macbook-16');
           cy.visit('/');
           cy.waitForSkeletonGone();
-          cy.get('[data-cy="bitcoin-block-offset-0-index-0"]').click().then(() => {
+          cy.get('.blockchain-blocks-0 > a').click().then(() => {
             cy.get('[ngbtooltip="Next Block"] > .ng-fa-icon > .svg-inline--fa').should('not.exist');
             cy.get('[ngbtooltip="Previous Block"] > .ng-fa-icon > .svg-inline--fa').should('be.visible');
             cy.waitForPageIdle();
@@ -233,11 +234,11 @@ describe('Mainnet', () => {
           });
         });
 
-        it('loads first blockchain block visible and keypress arrow left', () => {
+        it('loads first blockchain blocks visible and keypress arrow left', () => {
           cy.viewport('macbook-16');
           cy.visit('/');
           cy.waitForSkeletonGone();
-          cy.get('[data-cy="bitcoin-block-offset-0-index-0"]').click().then(() => {
+          cy.get('.blockchain-blocks-0 > a').click().then(() => {
             cy.waitForPageIdle();
             cy.get('[ngbtooltip="Next Block"] > .ng-fa-icon > .svg-inline--fa').should('not.exist');
             cy.get('[ngbtooltip="Previous Block"] > .ng-fa-icon > .svg-inline--fa').should('be.visible');
@@ -246,11 +247,11 @@ describe('Mainnet', () => {
           });
         });
 
-        it.skip('loads last blockchain block and keypress arrow right', () => { //Skip for now as "last" doesn't really work with infinite scrolling
+        it('loads last blockchain blocks and keypress arrow right', () => {
           cy.viewport('macbook-16');
           cy.visit('/');
           cy.waitForSkeletonGone();
-          cy.get('bitcoin-block-offset-0-index-7').click().then(() => {
+          cy.get('.blockchain-blocks-4 > a').click().then(() => {
             cy.waitForPageIdle();
 
             // block 6
@@ -309,7 +310,7 @@ describe('Mainnet', () => {
           cy.viewport('macbook-16');
           cy.visit('/');
           cy.waitForSkeletonGone();
-          cy.get('[data-cy="bitcoin-block-offset-0-index-0"]').click().then(() => {
+          cy.get('.blockchain-blocks-0 > a').click().then(() => {
             cy.waitForPageIdle();
             cy.get('[ngbtooltip="Next Block"] > .ng-fa-icon > .svg-inline--fa').should('not.exist');
             cy.get('[ngbtooltip="Previous Block"] > .ng-fa-icon > .svg-inline--fa').should('be.visible');
@@ -339,14 +340,14 @@ describe('Mainnet', () => {
       cy.visit('/');
       cy.waitForSkeletonGone();
 
-      cy.changeNetwork('testnet');
-      cy.changeNetwork('signet');
-      cy.changeNetwork('mainnet');
+      cy.changeNetwork("testnet");
+      cy.changeNetwork("signet");
+      cy.changeNetwork("mainnet");
     });
 
     it.skip('loads the dashboard with the skeleton blocks', () => {
       cy.mockMempoolSocket();
-      cy.visit('/');
+      cy.visit("/");
       cy.get(':nth-child(1) > #bitcoin-block-0').should('be.visible');
       cy.get(':nth-child(2) > #bitcoin-block-0').should('be.visible');
       cy.get(':nth-child(3) > #bitcoin-block-0').should('be.visible');

frontend/cypress/e2e/mainnet/mining.spec.ts

@@ -1,4 +1,4 @@
-const baseModule = Cypress.env('BASE_MODULE');
+const baseModule = Cypress.env("BASE_MODULE");
 
 describe('Mainnet - Mining Features', () => {
   beforeEach(() => {