Merge pull request #3548 from mempool/simon/updating-node-map-loading

Update channels map indexing indicator

.github/CODEOWNERS

@@ -0,0 +1 @@
+backend/src/api/database-migration.ts	@wiz @softsimon

.github/FUNDING.yml

@@ -1,12 +0,0 @@
-# These are supported funding model platforms
-
-github: ['mempool'] # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
-patreon: # Replace with a single Patreon username
-open_collective: # Replace with a single Open Collective username
-ko_fi: # Replace with a single Ko-fi username
-tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
-community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
-liberapay: # Replace with a single Liberapay username
-issuehunt: # Replace with a single IssueHunt username
-otechie: # Replace with a single Otechie username
-custom: ['https://mempool.space/about'] # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.github/ISSUE_TEMPLATE.md

@@ -1,37 +0,0 @@
-<!--
-   SUPPORT REQUESTS: This is for reporting bugs in Mempool.
-   If you have a support request, please join our Keybase group:
-   https://keybase.io/team/mempool
--->
-
-### Description
-
-<!-- brief description of the bug -->
-
-#### Version
-
-<!-- commit id or version number -->
-
-### Steps to reproduce
-
-<!--if you can reliably reproduce the bug, list the steps here  -->
-
-### Expected behaviour
-
-<!--description of the expected behavior -->
-
-### Actual behaviour
-
-<!-- explain what happened instead of the expected behaviour -->
-
-### Screenshots
-
-<!--Screenshots if gui related, drag and drop to add to the issue -->
-
-#### Device or machine
-
-<!-- device/machine used, operating system -->
-
-#### Additional info
-
-<!-- Additional information useful for debugging (e.g. logs) -->

.github/ISSUE_TEMPLATE/00-bug-issue.md

@@ -0,0 +1,43 @@
+---
+name: 🐛 Bug Report
+about: Report bugs (no support requests, please)
+---
+
+<!--
+   SUPPORT REQUESTS:
+   This is for reporting bugs in Mempool, not for support requests. 
+   If you have a support request, please reach out on Matrix:
+   https://matrix.to/#/#mempool.support:bitcoin.kyoto
+-->
+
+### Description
+
+<!-- brief description of the bug -->
+
+#### Version
+
+<!-- commit id or version number -->
+
+### Steps to reproduce
+
+<!-- if you can reliably reproduce the bug, list the steps here -->
+
+### Expected behaviour
+
+<!-- description of the expected behavior -->
+
+### Actual behaviour
+
+<!-- explain what happened instead of the expected behaviour -->
+
+### Screenshots
+
+<!-- Screenshots if gui related, drag and drop to add to the issue -->
+
+#### Device or machine
+
+<!-- device/machine used, operating system -->
+
+#### Additional info
+
+<!-- Additional information useful for debugging (e.g. logs) -->

.github/ISSUE_TEMPLATE/30-feature-request.md

@@ -0,0 +1,27 @@
+---
+name: ✨ Feature Request
+about: Request a feature or suggest other enhancements
+---
+
+<!--
+   SUPPORT REQUESTS:
+   This is for requesting features in Mempool, not for support requests. 
+   If you have a support request, please reach out on Matrix:
+   https://matrix.to/#/#mempool.support:bitcoin.kyoto
+-->
+
+### Description
+
+<!-- brief description of the feature request -->
+
+### Problem to be solved
+
+<!-- description of the the problem you're having -->
+
+### Proposed solution
+
+<!-- explain how you think we should solve the problem -->
+
+#### Additional info
+
+<!-- Additional information useful for implementing (e.g. docs, links, etc.) -->

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,8 @@
+blank_issues_enabled: false
+contact_links:
+  - name: 🙋 Need help? Chat with us on Matrix
+    url: https://matrix.to/#/#mempool.support:bitcoin.kyoto
+    about: For support requests or general questions
+  - name: 🌐 Want to help with translations? Use Transifex
+    url: https://www.transifex.com/mempool/mempool
+    about: All translations work is done on Transifex

.github/dependabot.yml

@@ -0,0 +1,47 @@
+version: 2
+updates:
+  - package-ecosystem: npm
+    directory: "/backend"
+    schedule:
+      interval: daily
+    open-pull-requests-limit: 10
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-major"]
+    allow:
+      - dependency-type: "production"
+
+  - package-ecosystem: npm
+    directory: "/frontend"
+    schedule:
+      interval: daily
+    open-pull-requests-limit: 10
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-major"]
+    allow:
+      - dependency-type: "production"
+
+  - package-ecosystem: docker
+    directory: "/docker/backend"
+    schedule:
+      interval: weekly
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-major"]
+
+  - package-ecosystem: docker
+    directory: "/docker/frontend"
+    schedule:
+      interval: weekly
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-major"]
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: weekly
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-major"]

.github/pull_request_template.md

@@ -0,0 +1,6 @@
+<!--
+Please do not open pull requests for translations.
+
+All translations work is done on Transifex:
+https://www.transifex.com/mempool/mempool
+-->

.github/workflows/ci.yml

@@ -0,0 +1,96 @@
+name: CI Pipeline for the Backend and Frontend
+
+on:
+  pull_request:
+    types: [opened, review_requested, synchronize]
+
+jobs:
+  backend:
+    if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')"
+    strategy:
+      matrix:
+        node: ["16.16.0", "18.14.1"]
+        flavor: ["dev", "prod"]
+      fail-fast: false
+    runs-on: "ubuntu-latest"
+
+    name: Backend (${{ matrix.flavor }}) - node ${{ matrix.node }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          path: ${{ matrix.node }}/${{ matrix.flavor }}
+
+      - name: Setup Node
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node }}
+          registry-url: "https://registry.npmjs.org"
+
+      - name: Install
+        if: ${{ matrix.flavor == 'dev'}}
+        run: npm ci
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/backend
+
+      - name: Install (Prod dependencies only)
+        if: ${{ matrix.flavor == 'prod'}}
+        run: npm ci --omit=dev --omit=optional
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/backend
+
+      - name: Lint
+        if: ${{ matrix.flavor == 'dev'}}
+        run: npm run lint
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/backend
+
+      - name: Unit Tests
+        if: ${{ matrix.flavor == 'dev'}}
+        run: npm run test
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/backend
+
+      - name: Build
+        run: npm run build
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/backend
+
+  frontend:
+    if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')"
+    strategy:
+      matrix:
+        node: ["16.16.0", "18.14.1"]
+        flavor: ["dev", "prod"]
+      fail-fast: false
+    runs-on: "ubuntu-latest"
+
+    name: Frontend (${{ matrix.flavor }}) - node ${{ matrix.node }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          path: ${{ matrix.node }}/${{ matrix.flavor }}
+
+      - name: Setup Node
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node }}
+          registry-url: "https://registry.npmjs.org"
+
+      - name: Install (Prod dependencies only)
+        run: npm ci --omit=dev --omit=optional
+        if: ${{ matrix.flavor == 'prod'}}
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/frontend
+
+      - name: Install
+        if: ${{ matrix.flavor == 'dev'}}
+        run: npm ci
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/frontend
+
+      - name: Lint
+        if: ${{ matrix.flavor == 'dev'}}
+        run: npm run lint
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/frontend
+
+      # - name: Test
+      #   run: npm run test
+
+      - name: Build
+        run: npm run build
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/frontend

.github/workflows/cypress.yml

@@ -0,0 +1,64 @@
+name: Cypress Tests
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    types: [opened, synchronize]
+
+jobs:
+  cypress:
+    if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')"
+    runs-on: "ubuntu-latest"
+    strategy:
+      fail-fast: false
+      matrix:
+        module: ["mempool", "liquid", "bisq"]
+        include:
+          - module: "mempool"
+            spec: |
+              cypress/e2e/mainnet/*.spec.ts
+              cypress/e2e/signet/*.spec.ts
+              cypress/e2e/testnet/*.spec.ts
+          - module: "liquid"
+            spec: |
+              cypress/e2e/liquid/liquid.spec.ts
+              cypress/e2e/liquidtestnet/liquidtestnet.spec.ts
+          - module: "bisq"
+            spec: |
+              cypress/e2e/bisq/bisq.spec.ts
+
+    name: E2E tests for ${{ matrix.module }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          path: ${{ matrix.module }}
+
+      - name: Setup node
+        uses: actions/setup-node@v3
+        with:
+          node-version: 16.15.0
+          cache: "npm"
+          cache-dependency-path: ${{ matrix.module }}/frontend/package-lock.json
+
+      - name: Chrome browser tests (${{ matrix.module }})
+        uses: cypress-io/github-action@v5
+        with:
+          tag: ${{ github.event_name }}
+          working-directory: ${{ matrix.module }}/frontend
+          build: npm run config:defaults:${{ matrix.module }}
+          start: npm run start:local-staging
+          wait-on: "http://localhost:4200"
+          wait-on-timeout: 120
+          record: true
+          parallel: true
+          spec: ${{ matrix.spec }}
+          group: Tests on Chrome (${{ matrix.module }})
+          browser: "chrome"
+          ci-build-id: "${{ github.sha }}-${{ github.workflow }}-${{ github.event_name }}"
+        env:
+          COMMIT_INFO_MESSAGE: ${{ github.event.pull_request.title }}
+          CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}

.github/workflows/get_image_digest.yml

@@ -0,0 +1,26 @@
+name: 'Print images digest'
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Image Version'
+        required: false
+        default: 'latest'
+        type: string 
+jobs:
+  print-images-sha:
+    runs-on: 'ubuntu-latest'
+    name: Print digest for images
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          path: digest
+
+      - name: Run script
+        working-directory: digest
+        run: |
+          sh ./docker/scripts/get_image_digest.sh $VERSION
+        env: 
+          VERSION: ${{ github.event.inputs.version }}

.github/workflows/on-tag.yml

@@ -1,7 +1,7 @@
 name: Docker build on tag
 env:
   DOCKER_CLI_EXPERIMENTAL: enabled
-  TAG_FMT: '^refs/tags/(((.?[0-9]+){3,4}))$'
+  TAG_FMT: "^refs/tags/(((.?[0-9]+){3,4}))$"
   DOCKER_BUILDKIT: 0
   COMPOSE_DOCKER_CLI_BUILD: 0
 
@@ -11,6 +11,9 @@ on:
       - v[0-9]+.[0-9]+.[0-9]+
       - v[0-9]+.[0-9]+.[0-9]+-*
 
+permissions:
+  contents: read
+
 jobs:
   build:
     strategy:
@@ -18,16 +21,46 @@ jobs:
         service:
           - frontend
           - backend
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
     name: Build and push to DockerHub
     steps:
+      # Workaround based on JonasAlfredsson/docker-on-tmpfs@v1.0.1
+      - name: Replace the current swap file
+        shell: bash
+        run: |
+          sudo swapoff /mnt/swapfile
+          sudo rm -v /mnt/swapfile
+          sudo fallocate -l 13G /mnt/swapfile
+          sudo chmod 600 /mnt/swapfile
+          sudo mkswap /mnt/swapfile
+          sudo swapon /mnt/swapfile
+
+      - name: Show current memory and swap status
+        shell: bash
+        run: |
+          sudo free -h
+          echo
+          sudo swapon --show
+
+      - name: Mount a tmpfs over /var/lib/docker
+        shell: bash
+        run: |
+          if [ ! -d "/var/lib/docker" ]; then
+            echo "Directory '/var/lib/docker' not found"
+            exit 1
+          fi
+          sudo mount -t tmpfs -o size=10G tmpfs /var/lib/docker
+          sudo systemctl restart docker
+          sudo df -h | grep docker
+
       - name: Set env variables
         run: echo "TAG=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV
 
       - name: Show set environment variables
         run: |
           printf "      TAG: %s\n"  "$TAG"
-      
+
       - name: Add SHORT_SHA env property with commit short sha
         run: echo "SHORT_SHA=`echo ${GITHUB_SHA} | cut -c1-8`" >> $GITHUB_ENV
 
@@ -35,24 +68,24 @@ jobs:
         run: echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin
 
       - name: Checkout project
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Init repo for Dockerization
         run: docker/init.sh "$TAG"
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v2
         id: qemu
 
       - name: Setup Docker buildx action
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
         id: buildx
 
       - name: Available platforms
         run: echo ${{ steps.buildx.outputs.platforms }}
 
       - name: Cache Docker layers
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         id: cache
         with:
           path: /tmp/.buildx-cache

.gitignore

@@ -2,3 +2,6 @@ sitemap
 data
 docker-compose.yml
 backend/mempool-config.json
+*.swp
+frontend/src/resources/config.template.js
+frontend/src/resources/config.js

.nvmrc

@@ -0,0 +1 @@
+v16.16.0

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+  "editor.tabSize": 2,
+  "typescript.preferences.importModuleSpecifier": "relative",
+  "typescript.tsdk": "./backend/node_modules/typescript/lib"
+}

CONTRIBUTING.md

@@ -0,0 +1,51 @@
+# Contributing to The Mempool Open Source Project
+
+Thank you for contributing to The Mempool Open Source Project managed by Mempool Space K.K. (“Mempool”).
+
+In order to clarify the intellectual property license granted with Contributions from any person or entity, Mempool must have a statement on file from each Contributor indicating their agreement to the Contributor License Agreement (“Agreement”). This license is for your protection as a Contributor as well as the protection of Mempool and its other contributors and users; it does not change your rights to use your own Contributions for any other purpose.
+
+When submitting a pull request for the first time, please create a file with a name like `/contributors/{github_username}.txt`, and in the content of that file indicate your agreement to the Contributor License Agreement terms below. An example of what that file should contain can be seen in wiz's agreement file. (This method of CLA "signing" is borrowed from Medium's open source project.)
+
+Also, please GPG-sign all your commits (`git config commit.gpgsign true`).
+
+# Contributor License Agreement
+
+Last Updated: January 25, 2022
+
+By accepting this Agreement, You agree to the following terms and conditions for Your present and future Contributions submitted to Mempool. Except for the license granted herein to Mempool and recipients of software distributed by Mempool, You reserve all right, title, and interest in and to Your Contributions.
+
+### 1. Definitions
+
+“You” (or “Your”) shall mean the copyright owner or legal entity authorized by the copyright owner that is making this Agreement with Mempool. For legal entities, the entity making a Contribution and all other entities that control, are controlled by, or are under common control with that entity are considered to be a single Contributor. For the purposes of this definition, “control” means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity.
+
+“Contribution” shall mean any original work of authorship, including any modifications or additions to an existing work, that is intentionally submitted by You to Mempool for inclusion in, or documentation of, any of the products owned or managed by Mempool (“Work”). For the purposes of this definition, “submitted” means any form of electronic, verbal, or written communication sent to Mempool or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, Mempool for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by You as “Not a Contribution.”
+
+### 2. Grant of Copyright License
+
+Subject to the terms and conditions of this Agreement, You hereby grant to Mempool and to recipients of software distributed by Mempool a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare derivative works of, publicly display, publicly perform, sublicense, and distribute Your Contributions and such derivative works.
+
+### 3. Grant of Patent License
+
+Subject to the terms and conditions of this Agreement, You hereby grant to Mempool and to recipients of software distributed by Mempool a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by You that are necessarily infringed by Your Contribution(s) alone or by combination of Your Contribution(s) with the Work to which such Contribution(s) was submitted. If any entity institutes patent litigation against You or any other entity (including a cross-claim or counterclaim in a lawsuit) alleging that your Contribution, or the Work to which you have contributed, constitutes direct or contributory patent infringement, then any patent licenses granted to that entity under this Agreement for that Contribution or Work shall terminate as of the date such litigation is filed.
+
+### 4. Authority
+
+You represent that you are legally entitled to grant the above license. If your employer(s) has rights to intellectual property that you create that includes your Contributions, you represent that you have received permission to make Contributions on behalf of that employer, that your employer has waived such rights for your Contributions to Mempool, or that your employer has executed a separate Corporate Contributor License Agreement with Mempool.
+
+### 5. Originality
+
+You represent that each of Your Contributions is Your original creation (see section 7 for submissions on behalf of others). You represent that Your Contribution submissions include complete details of any third-party license or other restriction (including, but not limited to, related patents and trademarks) of which you are personally aware, and which are associated with any part of Your Contributions.
+
+### 6. Support
+
+You are not expected to provide support for Your Contributions, except to the extent You desire to provide support. You may provide support for free, for a fee, or not at all. Unless required by applicable law or agreed to in writing, You provide Your Contributions on an “AS IS” BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON- INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE.
+
+### 7. Third Party Contributions
+
+Should You wish to submit work that is not Your original creation, You may submit it to Mempool separately from any Contribution, identifying the complete details of its source and of any license or other restriction (including, but not limited to, related patents, trademarks, and license agreements) of which you are personally aware, and conspicuously marking the work as “Submitted on behalf of a third-party: [named here]”.
+
+### 8. Notifications
+
+You agree to notify Mempool of any facts or circumstances of which you become aware that would make these representations inaccurate in any respect.
+
+EOF

GNUmakefile

@@ -25,8 +25,7 @@ help:
 .PHONY: init
 init:
 	@echo ''
-	mkdir -p   $(DATA) $(DATA)/mysql $(DATA)/mysql/db-scripts $(DATA)/mysql/data 
-	install -v mariadb-structure.sql $(DATA)/mysql/db-scripts
+	mkdir -p   $(DATA) $(DATA)/mysql $(DATA)/mysql/data 
 	#REF: https://github.com/mempool/mempool/blob/master/docker/README.md
 	cat docker/docker-compose.yml > docker-compose.yml
 	cat backend/mempool-config.sample.json > backend/mempool-config.json

LICENSE

@@ -1,5 +1,5 @@
 The Mempool Open Source Project
-Copyright (c) 2019-2021 The Mempool Open Source Project Developers
+Copyright (c) 2019-2023 The Mempool Open Source Project Developers
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of (at your option) either:
@@ -12,8 +12,13 @@ the terms of (at your option) either:
  Foundation, either version 3 of the License or any later version approved by a
  proxy statement published on <https://mempool.space/about>.
 
-However, these licenses do not grant you any rights to use the "mempool.space"
-trademarks or logos, or any other trademarks of Mempool Space K.K.
+However, this copyright license does not include an implied right or license to
+use our trademarks: The Mempool Open Source Project™, mempool.space™, the
+mempool Logo™, the mempool.space Vertical Logo™, the mempool.space Horizontal
+Logo™, the mempool Square Logo™, and the mempool Blocks logo™ are registered
+trademarks or trademarks of Mempool Space K.K in Japan, the United States,
+and/or other countries. See our full Trademark Policy and Guidelines for more
+details, published on <https://mempool.space/trademark-policy>.
 
 This program is distributed in the hope that it will be useful, but WITHOUT ANY
 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A

README.md

@@ -1,192 +1,35 @@
-# The Mempool Open Source Project
+# The Mempool Open Source Project™ [![mempool](https://img.shields.io/endpoint?url=https://dashboard.cypress.io/badge/simple/ry4br7/master&style=flat-square)](https://dashboard.cypress.io/projects/ry4br7/runs)
 
-Mempool is the fully featured visualizer, explorer, and API service running on [mempool.space](https://mempool.space/), an open source project developed and operated for the benefit of the Bitcoin community, with a focus on the emerging transaction fee market to help our transition into a multi-layer ecosystem.
+https://user-images.githubusercontent.com/93150691/226236121-375ea64f-b4a1-4cc0-8fad-a6fb33226840.mp4
 
-![mempool](https://mempool.space/resources/screenshots/v2.2.0-dashboard.png)
+<br>
 
-## Installation Methods
+Mempool is the fully-featured mempool visualizer, explorer, and API service running at [mempool.space](https://mempool.space/). 
 
-Mempool can be self-hosted on a wide variety of your own hardware, ranging from a simple one-click installation on a Raspberry Pi distro, all the way to an advanced high availability cluster of powerful servers for a production instance. We support the following installation methods, ranked in order from simple to advanced:
+It is an open-source project developed and operated for the benefit of the Bitcoin community, with a focus on the emerging transaction fee market that is evolving Bitcoin into a multi-layer ecosystem.
 
-1) One-click installation on: [Umbrel](https://github.com/getumbrel/umbrel), [RaspiBlitz](https://github.com/rootzoll/raspiblitz), [RoninDojo](https://code.samourai.io/ronindojo/RoninDojo), or [MyNode](https://github.com/mynodebtc/mynode).
-2) [Docker installation on Linux using docker-compose](https://github.com/mempool/mempool/tree/master/docker)
-3) [Manual installation on Linux or FreeBSD](https://github.com/mempool/mempool#manual-installation)
-4) [Production installation on a powerful FreeBSD server](https://github.com/mempool/mempool/tree/master/production)
-5) [High Availability cluster using powerful FreeBSD servers](https://github.com/mempool/mempool/tree/master/production#high-availability)
+# Installation Methods
 
-# Manual Installation
+Mempool can be self-hosted on a wide variety of your own hardware, ranging from a simple one-click installation on a Raspberry Pi full-node distro all the way to a robust production instance on a powerful FreeBSD server. 
 
-The following instructions are for a manual installation on Linux or FreeBSD. The file and directory paths may need to be changed to match your OS.
+**Most people should use a one-click install method.** Other install methods are meant for developers and others with experience managing servers. 
 
-## Dependencies
+<a id="one-click-installation"></a>
+## One-Click Installation
 
-* Bitcoin Core (no pruning, txindex=1)
-* Electrum Server (romanz/electrs)
-* NodeJS (official stable LTS)
-* MariaDB (default config)
-* Nginx (use supplied nginx.conf and nginx-mempool.conf)
+Mempool can be conveniently installed on the following full-node distros: 
+- [Umbrel](https://github.com/getumbrel/umbrel)
+- [RaspiBlitz](https://github.com/rootzoll/raspiblitz)
+- [RoninDojo](https://code.samourai.io/ronindojo/RoninDojo)
+- [myNode](https://github.com/mynodebtc/mynode)
+- [Start9](https://github.com/Start9Labs/embassy-os)
 
-## Mempool
+**We highly recommend you deploy your own Mempool instance this way.** No matter which option you pick, you'll be able to get your own fully-sovereign instance of Mempool up quickly without needing to fiddle with any settings.
 
-Clone the mempool repo, and checkout the latest release tag:
-```bash
-  git clone https://github.com/mempool/mempool
-  cd mempool
-  latestrelease=$(curl -s https://api.github.com/repos/mempool/mempool/releases/latest|grep tag_name|head -1|cut -d '"' -f4)
-  git checkout $latestrelease
-```
+## Advanced Installation Methods
 
-## Bitcoin Core (bitcoind)
+Mempool can be installed in other ways too, but we only recommend doing so if you're a developer, have experience managing servers, or otherwise know what you're doing.
 
-Enable RPC and txindex in `bitcoin.conf`:
-```bash
-  rpcuser=mempool
-  rpcpassword=71b61986da5b03a5694d7c7d5165ece5
-  txindex=1
-```
-
-## MySQL
-
-Install MariaDB from OS package manager:
-```bash
-  # Linux
-  apt-get install mariadb-server mariadb-client
-
-  # macOS
-  brew install mariadb
-  brew services start mariadb
-```
-
-Create database and grant privileges:
-```bash
-  MariaDB [(none)]> drop database mempool;
-  Query OK, 0 rows affected (0.00 sec)
-
-  MariaDB [(none)]> create database mempool;
-  Query OK, 1 row affected (0.00 sec)
-
-  MariaDB [(none)]> grant all privileges on mempool.* to 'mempool'@'%' identified by 'mempool';
-  Query OK, 0 rows affected (0.00 sec)
-```
-
-From the mempool repo's top-level folder, import the database structure:
-```bash
-  mysql -u mempool -p mempool < mariadb-structure.sql
-```
-
-## Mempool Backend
-Install mempool dependencies from npm and build the backend:
-
-```bash
-  # backend
-  cd backend
-  npm install
-  npm run build
-```
-
-In the `backend` folder, make a copy of the sample config and modify it to fit your settings.
-
-```bash
-  cp mempool-config.sample.json mempool-config.json
-```
-
-Edit `mempool-config.json` to add your Bitcoin Core node RPC credentials:
-```bash
-{
-  "MEMPOOL": {
-    "NETWORK": "mainnet",
-    "BACKEND": "electrum",
-    "HTTP_PORT": 8999,
-    "API_URL_PREFIX": "/api/v1/",
-    "POLL_RATE_MS": 2000
-  },
-  "CORE_RPC": {
-    "USERNAME": "mempool",
-    "PASSWORD": "71b61986da5b03a5694d7c7d5165ece5"
-  },
-  "ELECTRUM": {
-    "HOST": "127.0.0.1",
-    "PORT": 50002,
-    "TLS_ENABLED": true,
-  },
-  "DATABASE": {
-    "ENABLED": true,
-    "HOST": "127.0.0.1",
-    "PORT": 3306,
-    "USERNAME": "mempool",
-    "PASSWORD": "mempool",
-    "DATABASE": "mempool"
-  },
-  "STATISTICS": {
-    "ENABLED": true,
-    "TX_PER_SECOND_SAMPLE_PERIOD": 150
-  }
-}
-```
-
-Start the backend:
-
-```bash
-  npm run start
-```
-
-When it's running you should see output like this:
-
-```bash
-  Mempool updated in 0.189 seconds
-  Updating mempool
-  Mempool updated in 0.096 seconds
-  Updating mempool
-  Mempool updated in 0.099 seconds
-  Updating mempool
-  Calculated fee for transaction 1 / 10
-  Calculated fee for transaction 2 / 10
-  Calculated fee for transaction 3 / 10
-  Calculated fee for transaction 4 / 10
-  Calculated fee for transaction 5 / 10
-  Calculated fee for transaction 6 / 10
-  Calculated fee for transaction 7 / 10
-  Calculated fee for transaction 8 / 10
-  Calculated fee for transaction 9 / 10
-  Calculated fee for transaction 10 / 10
-  Mempool updated in 0.243 seconds
-  Updating mempool
-```
-
-## Mempool Frontend
-
-Install mempool dependencies from npm and build the frontend static HTML/CSS/JS:
-
-```bash
-  # frontend
-  cd frontend
-  npm install
-  npm run build
-```
-
-Install the output into nginx webroot folder:
-
-```bash
-  sudo rsync -av --delete dist/mempool/ /var/www/html/
-```
-
-## nginx + certbot
-
-Install the supplied nginx.conf and nginx-mempool.conf in /etc/nginx
-
-```bash
-  # install nginx and certbot
-  apt-get install -y nginx python-certbot-nginx
-
-  # install the mempool configuration for nginx
-  cp nginx.conf nginx-mempool.conf /etc/nginx/nginx.conf
-
-  # replace example.com with your domain name
-  certbot --nginx -d example.com
-
-```
-
-If everything went okay you should see the beautiful mempool :grin:
-
-If you get stuck on "loading blocks", this means the websocket can't connect.
-Check your nginx proxy setup, firewalls, etc. and open an issue if you need help.
+- See the [`docker/`](./docker/) directory for instructions on deploying Mempool with Docker.
+- See the [`backend/`](./backend/) and [`frontend/`](./frontend/) directories for manual install instructions oriented for developers.
+- See the [`production/`](./production/) directory for guidance on setting up a more serious Mempool instance designed for high performance at scale.

backend/.editorconfig

@@ -0,0 +1,17 @@
+# Editor configuration, see https://editorconfig.org
+root = true
+
+[*]
+charset = utf-8
+indent_style = space
+indent_size = 2
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[*.ts]
+quote_type = single
+
+[*.md]
+max_line_length = off
+trim_trailing_whitespace = false
+

backend/.eslintignore

@@ -0,0 +1,2 @@
+node_modules
+dist

backend/.eslintrc

@@ -0,0 +1,37 @@
+{
+  "root": true,
+  "parser": "@typescript-eslint/parser",
+  "plugins": [
+    "@typescript-eslint"
+  ],
+  "extends": [
+    "eslint:recommended",
+    "plugin:@typescript-eslint/eslint-recommended",
+    "plugin:@typescript-eslint/recommended",
+    "prettier"
+  ],
+  "rules": {
+    "@typescript-eslint/ban-ts-comment": 1,
+    "@typescript-eslint/ban-types": 1,
+    "@typescript-eslint/no-empty-function": 1,
+    "@typescript-eslint/no-explicit-any": 1,
+    "@typescript-eslint/no-inferrable-types": 0,
+    "@typescript-eslint/no-namespace": 1,
+    "@typescript-eslint/no-this-alias": 1,
+    "@typescript-eslint/no-var-requires": 1,
+    "@typescript-eslint/explicit-function-return-type": 1,
+    "no-console": 1,
+    "no-constant-condition": 1,
+    "no-dupe-else-if": 1,
+    "no-empty": 1,
+    "no-prototype-builtins": 1,
+    "no-self-assign": 1,
+    "no-useless-catch": 1,
+    "no-var": 1,
+    "prefer-const": 1,
+    "prefer-rest-params": 1,
+    "quotes": [1, "single", { "allowTemplateLiterals": true }],
+    "semi": 1,
+    "eqeqeq": 1
+  }
+}

backend/.gitignore

@@ -1,7 +1,11 @@
 # See http://help.github.com/ignore-files/ for more about ignoring files.
 
-# production config
+# production config and external assets
+!mempool-config.template.json
+!mempool-config.sample.json
 mempool-config.json
+pools.json
+icons.json
 
 # compiled output
 /dist

backend/.prettierignore

@@ -0,0 +1,2 @@
+node_modules
+package-lock.json

backend/.prettierrc

@@ -0,0 +1,6 @@
+{
+  "endOfLine": "lf",
+  "printWidth": 80,
+  "tabWidth": 2,
+  "trailingComma": "es5"
+}

backend/.vscode/settings.json

@@ -0,0 +1,4 @@
+{
+  "editor.tabSize": 2,
+  "typescript.tsdk": "../backend/node_modules/typescript/lib"
+}

backend/README.md

@@ -0,0 +1,254 @@
+# Mempool Backend
+
+These instructions are mostly intended for developers. 
+
+If you choose to use these instructions for a production setup, be aware that you will still probably need to do additional configuration for your specific OS, environment, use-case, etc. We do our best here to provide a good starting point, but only proceed if you know what you're doing. Mempool does not provide support for custom setups.
+
+See other ways to set up Mempool on [the main README](/../../#installation-methods).
+
+Jump to a section in this doc:
+- [Set Up the Backend](#setup)
+- [Development Tips](#development-tips)
+
+## Setup
+
+### 1. Clone Mempool Repository
+
+Get the latest Mempool code:
+
+```
+git clone https://github.com/mempool/mempool
+cd mempool
+```
+
+Check out the latest release:
+
+```
+latestrelease=$(curl -s https://api.github.com/repos/mempool/mempool/releases/latest|grep tag_name|head -1|cut -d '"' -f4)
+git checkout $latestrelease
+```
+
+### 2. Configure Bitcoin Core
+
+Turn on `txindex`, enable RPC, and set RPC credentials in `bitcoin.conf`:
+
+```
+txindex=1
+server=1
+rpcuser=mempool
+rpcpassword=mempool
+```
+
+### 3. Configure Electrum Server
+
+[Pick an Electrum Server implementation](https://mempool.space/docs/faq#address-lookup-issues), configure it, and make sure it's synced.
+
+**This step is optional.** You can run Mempool without configuring an Electrum Server for it, but address lookups will be disabled.
+
+### 4. Configure MariaDB
+
+_Mempool needs MariaDB v10.5 or later. If you already have MySQL installed, make sure to migrate any existing databases **before** installing MariaDB._
+
+Get MariaDB from your operating system's package manager:
+
+```
+# Debian, Ubuntu, etc.
+apt-get install mariadb-server mariadb-client
+
+# macOS
+brew install mariadb
+mysql.server start
+```
+
+Create a database and grant privileges:
+
+```
+MariaDB [(none)]> drop database mempool;
+Query OK, 0 rows affected (0.00 sec)
+
+MariaDB [(none)]> create database mempool;
+Query OK, 1 row affected (0.00 sec)
+
+MariaDB [(none)]> grant all privileges on mempool.* to 'mempool'@'%' identified by 'mempool';
+Query OK, 0 rows affected (0.00 sec)
+```
+
+### 5. Prepare Mempool Backend
+
+#### Build
+
+_Make sure to use Node.js 16.10 and npm 7._
+
+Install dependencies with `npm` and build the backend:
+
+```
+cd backend
+npm install
+npm run build
+```
+
+#### Configure
+
+In the backend folder, make a copy of the sample config file:
+
+```
+cp mempool-config.sample.json mempool-config.json
+```
+
+Edit `mempool-config.json` as needed. 
+
+In particular, make sure:
+- the correct Bitcoin Core RPC credentials are specified in `CORE_RPC`
+- the correct `BACKEND` is specified in `MEMPOOL`:
+  - "electrum" if you're using [romanz/electrs](https://github.com/romanz/electrs) or [cculianu/Fulcrum](https://github.com/cculianu/Fulcrum)
+  - "esplora" if you're using [Blockstream/electrs](https://github.com/Blockstream/electrs)
+  - "none" if you're not using any Electrum Server
+
+### 6. Run Mempool Backend
+
+Run the Mempool backend:
+
+```
+npm run start
+
+```
+You can also set env var `MEMPOOL_CONFIG_FILE` to specify a custom config file location:
+```
+MEMPOOL_CONFIG_FILE=/path/to/mempool-config.json npm run start
+```
+
+When it's running, you should see output like this:
+
+```
+Mempool updated in 0.189 seconds
+Updating mempool
+Mempool updated in 0.096 seconds
+Updating mempool
+Mempool updated in 0.099 seconds
+Updating mempool
+Calculated fee for transaction 1 / 10
+Calculated fee for transaction 2 / 10
+Calculated fee for transaction 3 / 10
+Calculated fee for transaction 4 / 10
+Calculated fee for transaction 5 / 10
+Calculated fee for transaction 6 / 10
+Calculated fee for transaction 7 / 10
+Calculated fee for transaction 8 / 10
+Calculated fee for transaction 9 / 10
+Calculated fee for transaction 10 / 10
+Mempool updated in 0.243 seconds
+Updating mempool
+```
+
+### 7. Set Up Mempool Frontend
+With the backend configured and running, proceed to set up the [Mempool frontend](../frontend#manual-setup).
+
+## Development Tips
+
+### Set Up Backend Watchers
+
+The Mempool backend is static. TypeScript scripts are compiled into the `dist` folder and served through a Node.js web server. 
+
+As a result, for development purposes, you may find it helpful to set up backend watchers to avoid the manual shutdown/recompile/restart command-line cycle.
+
+First, install `nodemon` and `ts-node`:
+
+```
+npm install -g ts-node nodemon
+```
+
+Then, run the watcher:
+
+```
+nodemon src/index.ts --ignore cache/
+```
+
+`nodemon` should be in npm's global binary folder. If needed, you can determine where that is with `npm -g bin`.
+
+### Useful Regtest Commands
+
+Helpful link: https://gist.github.com/System-Glitch/cb4e87bf1ae3fec9925725bb3ebe223a
+
+Run bitcoind on regtest:
+   ```
+   bitcoind -regtest
+   ```
+
+Create a new wallet, if needed:
+   ```
+   bitcoin-cli -regtest createwallet test
+   ```
+
+Load wallet (this command may take a while if you have lot of UTXOs):
+   ```
+   bitcoin-cli -regtest loadwallet test
+   ```
+
+Get a new address:
+   ```
+   address=$(bitcoin-cli -regtest getnewaddress)
+   ```
+
+Mine blocks to the previously generated address. You need at least 101 blocks before you can spend. This will take some time to execute (~1 min):
+   ```
+   bitcoin-cli -regtest generatetoaddress 101 $address
+   ```
+
+Send 0.1 BTC at 5 sat/vB to another address:
+   ```
+   bitcoin-cli -named -regtest sendtoaddress address=$(bitcoin-cli -regtest getnewaddress) amount=0.1 fee_rate=5
+   ```
+
+See more example of `sendtoaddress`:
+   ```
+   bitcoin-cli sendtoaddress # will print the help
+   ```
+
+Mini script to generate random network activity (random TX count with random tx fee-rate). It's slow so don't expect to use this to test mempool spam, except if you let it run for a long time, or maybe with multiple regtest nodes connected to each other.
+   ```
+   #!/bin/bash
+   address=$(bitcoin-cli -regtest getnewaddress)
+   bitcoin-cli -regtest generatetoaddress 101 $address
+   for i in {1..1000000}
+   do
+      for y in $(seq 1 "$(jot -r 1 1 1000)")
+      do
+         bitcoin-cli -regtest -named sendtoaddress address=$address amount=0.01 fee_rate=$(jot -r 1 1 100)
+      done
+      bitcoin-cli -regtest generatetoaddress 1 $address
+      sleep 5
+   done
+   ```
+
+Generate block at regular interval (every 10 seconds in this example):
+   ```
+   watch -n 10 "bitcoin-cli -regtest generatetoaddress 1 $address"
+   ```
+
+### Mining pools update
+
+By default, mining pools will be not automatically updated regularly (`config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING` is set to `false`). 
+
+To manually update your mining pools, you can use the `--update-pools` command line flag when you run the nodejs backend. For example `npm run start --update-pools`. This will trigger the mining pools update and automatically re-index appropriate blocks.
+
+You can enabled the automatic mining pools update by settings `config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING` to `true` in your `mempool-config.json`.
+
+When a `coinbase tag` or `coinbase address` change is detected, all blocks tagged to the `unknown` mining pools (starting from height 130635) will be deleted from the `blocks` table. Additionaly, all blocks which were tagged to the pool which has been updated will also be deleted from the `blocks` table. Of course, those blocks will be automatically reindexed.
+
+### Re-index tables
+
+You can manually force the nodejs backend to drop all data from a specified set of tables for future re-index. This is mostly useful for the mining dashboard and the lightning explorer.
+
+Use the `--reindex` command to specify a list of comma separated table which will be truncated at start. Note that a 5 seconds delay will be observed before truncating tables in order to give you a chance to cancel (CTRL+C) in case of misuse of the command.
+
+Usage:
+```
+npm run start --reindex=blocks,hashrates
+```
+Example output:
+```
+Feb 13 14:55:27 [63246] WARN: <lightning> Indexed data for "hashrates" tables will be erased in 5 seconds (using '--reindex')
+Feb 13 14:55:32 [63246] NOTICE: <lightning> Table hashrates has been truncated
+```
+
+Reference: https://github.com/mempool/mempool/pull/1269

backend/jest.config.ts

@@ -0,0 +1,20 @@
+import type { Config } from "@jest/types"
+
+const config: Config.InitialOptions = {
+  preset: "ts-jest",
+  testEnvironment: "node",
+  verbose: true,
+  automock: false,
+  collectCoverage: true,
+  collectCoverageFrom: ["./src/**/**.ts"],
+  coverageProvider: "babel",
+  coverageThreshold: {
+    global: {
+      lines: 1
+    }
+  },
+  setupFiles: [
+    "./testSetup.ts",
+  ],
+}
+export default config;

backend/mempool-config.sample.json

@@ -2,19 +2,40 @@
   "MEMPOOL": {
     "NETWORK": "mainnet",
     "BACKEND": "electrum",
+    "ENABLED": true,
     "HTTP_PORT": 8999,
     "SPAWN_CLUSTER_PROCS": 0,
     "API_URL_PREFIX": "/api/v1/",
     "POLL_RATE_MS": 2000,
     "CACHE_DIR": "./cache",
     "CLEAR_PROTECTION_MINUTES": 20,
-    "RECOMMENDED_FEE_PERCENTILE": 50
+    "RECOMMENDED_FEE_PERCENTILE": 50,
+    "BLOCK_WEIGHT_UNITS": 4000000,
+    "INITIAL_BLOCKS_AMOUNT": 8,
+    "MEMPOOL_BLOCKS_AMOUNT": 8,
+    "INDEXING_BLOCKS_AMOUNT": 11000,
+    "BLOCKS_SUMMARIES_INDEXING": false,
+    "USE_SECOND_NODE_FOR_MINFEE": false,
+    "EXTERNAL_ASSETS": [],
+    "EXTERNAL_MAX_RETRY": 1,
+    "EXTERNAL_RETRY_INTERVAL": 0,
+    "USER_AGENT": "mempool",
+    "STDOUT_LOG_MIN_PRIORITY": "debug",
+    "AUTOMATIC_BLOCK_REINDEXING": false,
+    "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json",
+    "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
+    "AUDIT": false,
+    "ADVANCED_GBT_AUDIT": false,
+    "ADVANCED_GBT_MEMPOOL": false,
+    "CPFP_INDEXING": false,
+    "DISK_CACHE_BLOCK_INTERVAL": 6
   },
   "CORE_RPC": {
     "HOST": "127.0.0.1",
     "PORT": 8332,
     "USERNAME": "mempool",
-    "PASSWORD": "mempool"
+    "PASSWORD": "mempool",
+    "TIMEOUT": 60000
   },
   "ELECTRUM": {
     "HOST": "127.0.0.1",
@@ -24,17 +45,18 @@
   "ESPLORA": {
     "REST_API_URL": "http://127.0.0.1:3000"
   },
-  "CORE_RPC_MINFEE": {
-    "ENABLED": false,
+  "SECOND_CORE_RPC": {
     "HOST": "127.0.0.1",
     "PORT": 8332,
     "USERNAME": "mempool",
-    "PASSWORD": "mempool"
+    "PASSWORD": "mempool",
+    "TIMEOUT": 60000
   },
   "DATABASE": {
     "ENABLED": true,
     "HOST": "127.0.0.1",
     "PORT": 3306,
+    "SOCKET": "/var/run/mysql/mysql.sock",
     "DATABASE": "mempool",
     "USERNAME": "mempool",
     "PASSWORD": "mempool"
@@ -50,8 +72,52 @@
     "ENABLED": true,
     "TX_PER_SECOND_SAMPLE_PERIOD": 150
   },
+  "MAXMIND": {
+    "ENABLED": false,
+    "GEOLITE2_CITY": "/usr/local/share/GeoIP/GeoLite2-City.mmdb",
+    "GEOLITE2_ASN": "/usr/local/share/GeoIP/GeoLite2-ASN.mmdb",
+    "GEOIP2_ISP": "/usr/local/share/GeoIP/GeoIP2-ISP.mmdb"
+  },
   "BISQ": {
     "ENABLED": false,
     "DATA_PATH": "/bisq/statsnode-data/btc_mainnet/db"
+  },
+  "LIGHTNING": {
+    "ENABLED": false,
+    "BACKEND": "lnd",
+    "STATS_REFRESH_INTERVAL": 600,
+    "GRAPH_REFRESH_INTERVAL": 600,
+    "LOGGER_UPDATE_INTERVAL": 30,
+    "FORENSICS_INTERVAL": 43200,
+    "FORENSICS_RATE_LIMIT": 20
+  },
+  "LND": {
+    "TLS_CERT_PATH": "tls.cert",
+    "MACAROON_PATH": "readonly.macaroon",
+    "REST_API_URL": "https://localhost:8080",
+    "TIMEOUT": 10000
+  },
+  "CLIGHTNING": {
+    "SOCKET": "lightning-rpc"
+  },
+  "SOCKS5PROXY": {
+    "ENABLED": false,
+    "USE_ONION": true,
+    "HOST": "127.0.0.1",
+    "PORT": 9050,
+    "USERNAME": "",
+    "PASSWORD": ""
+  },
+  "PRICE_DATA_SERVER": {
+    "TOR_URL": "http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices",
+    "CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices"
+  },
+  "EXTERNAL_DATA_SERVER": {
+    "MEMPOOL_API": "https://mempool.space/api/v1",
+    "MEMPOOL_ONION": "http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1",
+    "LIQUID_API": "https://liquid.network/api/v1",
+    "LIQUID_ONION": "http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1",
+    "BISQ_URL": "https://bisq.markets/api",
+    "BISQ_ONION": "http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api"
   }
 }

backend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mempool-backend",
-  "version": "2.2.0",
+  "version": "2.5.0-dev",
   "description": "Bitcoin mempool visualizer and blockchain explorer backend",
   "license": "GNU Affero General Public License v3.0",
   "homepage": "https://mempool.space",
@@ -16,35 +16,53 @@
     "mempool",
     "blockchain",
     "explorer",
-    "liquid"
+    "liquid",
+    "lightning"
   ],
   "main": "index.ts",
   "scripts": {
-    "ng": "./node_modules/@angular/cli/bin/ng",
-    "tsc": "./node_modules/typescript/bin/tsc",
-    "build": "npm run tsc",
+    "tsc": "./node_modules/typescript/bin/tsc -p tsconfig.build.json",
+    "build": "npm run tsc && npm run create-resources",
+    "create-resources": "cp ./src/tasks/price-feeds/mtgox-weekly.json ./dist/tasks && node dist/api/fetch-version.js",
+    "package": "npm run build && rm -rf package && mv dist package && mv node_modules package && npm run package-rm-build-deps",
+    "package-rm-build-deps": "(cd package/node_modules; rm -r typescript @typescript-eslint)",
     "start": "node --max-old-space-size=2048 dist/index.js",
-    "start-production": "node --max-old-space-size=4096 dist/index.js",
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "start-production": "node --max-old-space-size=16384 dist/index.js",
+    "test": "./node_modules/.bin/jest --coverage",
+    "lint": "./node_modules/.bin/eslint . --ext .ts",
+    "lint:fix": "./node_modules/.bin/eslint . --ext .ts --fix",
+    "prettier": "./node_modules/.bin/prettier --write \"src/**/*.{js,ts}\""
   },
   "dependencies": {
-    "@mempool/bitcoin": "^3.0.2",
+    "@babel/core": "^7.20.12",
     "@mempool/electrum-client": "^1.1.7",
-    "axios": "^0.21.1",
-    "bitcoinjs-lib": "^5.2.0",
-    "crypto-js": "^4.0.0",
-    "express": "^4.17.1",
-    "locutus": "^2.0.12",
-    "mysql2": "2.2.5",
-    "node-worker-threads-pool": "^1.4.3",
-    "ws": "^7.4.6"
+    "@types/node": "^16.18.11",
+    "axios": "~0.27.2",
+    "bitcoinjs-lib": "~6.1.0",
+    "crypto-js": "~4.1.1",
+    "express": "~4.18.2",
+    "maxmind": "~4.3.8",
+    "mysql2": "~2.3.3",
+    "node-worker-threads-pool": "~1.5.1",
+    "socks-proxy-agent": "~7.0.0",
+    "typescript": "~4.7.4",
+    "ws": "~8.11.0"
   },
   "devDependencies": {
-    "@types/compression": "^1.0.1",
-    "@types/express": "^4.17.2",
-    "@types/locutus": "^0.0.6",
-    "@types/ws": "^7.4.4",
-    "tslint": "^6.1.0",
-    "typescript": "^4.1.5"
+    "@babel/core": "^7.20.7",
+    "@babel/code-frame": "^7.18.6",
+    "@types/compression": "^1.7.2",
+    "@types/crypto-js": "^4.1.1",
+    "@types/express": "^4.17.15",
+    "@types/jest": "^29.2.5",
+    "@types/ws": "~8.5.4",
+    "@typescript-eslint/eslint-plugin": "^5.48.1",
+    "@typescript-eslint/parser": "^5.48.1",
+    "eslint": "^8.31.0",
+    "eslint-config-prettier": "^8.5.0",
+    "jest": "^29.3.1",
+    "prettier": "^2.8.2",
+    "ts-jest": "^29.0.3",
+    "ts-node": "^10.9.1"
   }
 }

backend/src/__fixtures__/mempool-config.template.json

@@ -0,0 +1,119 @@
+{
+  "MEMPOOL": {
+    "ENABLED": true,
+    "NETWORK": "__MEMPOOL_NETWORK__",
+    "BACKEND": "__MEMPOOL_BACKEND__",
+    "BLOCKS_SUMMARIES_INDEXING": true,
+    "HTTP_PORT": 1,
+    "SPAWN_CLUSTER_PROCS": 2,
+    "API_URL_PREFIX": "__MEMPOOL_API_URL_PREFIX__",
+    "AUTOMATIC_BLOCK_REINDEXING": false,
+    "POLL_RATE_MS": 3,
+    "CACHE_DIR": "__MEMPOOL_CACHE_DIR__",
+    "CLEAR_PROTECTION_MINUTES": 4,
+    "RECOMMENDED_FEE_PERCENTILE": 5,
+    "BLOCK_WEIGHT_UNITS": 6,
+    "INITIAL_BLOCKS_AMOUNT": 7,
+    "MEMPOOL_BLOCKS_AMOUNT": 8,
+    "USE_SECOND_NODE_FOR_MINFEE": 10,
+    "EXTERNAL_ASSETS": 11,
+    "EXTERNAL_MAX_RETRY": 12,
+    "EXTERNAL_RETRY_INTERVAL": 13,
+    "USER_AGENT": "__MEMPOOL_USER_AGENT__",
+    "STDOUT_LOG_MIN_PRIORITY": "__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__",
+    "INDEXING_BLOCKS_AMOUNT": 14,
+    "POOLS_JSON_TREE_URL": "__POOLS_JSON_TREE_URL__",
+    "POOLS_JSON_URL": "__POOLS_JSON_URL__",
+    "AUDIT": "__MEMPOOL_AUDIT__",
+    "ADVANCED_GBT_AUDIT": "__MEMPOOL_ADVANCED_GBT_AUDIT__",
+    "ADVANCED_GBT_MEMPOOL": "__MEMPOOL_ADVANCED_GBT_MEMPOOL__",
+    "CPFP_INDEXING": "__MEMPOOL_CPFP_INDEXING__",
+    "MAX_BLOCKS_BULK_QUERY": "__MEMPOOL_MAX_BLOCKS_BULK_QUERY__",
+    "DISK_CACHE_BLOCK_INTERVAL": "__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__"
+  },
+  "CORE_RPC": {
+    "HOST": "__CORE_RPC_HOST__",
+    "PORT": 15,
+    "USERNAME": "__CORE_RPC_USERNAME__",
+    "PASSWORD": "__CORE_RPC_PASSWORD__",
+    "TIMEOUT": "__CORE_RPC_TIMEOUT__"
+  },
+  "ELECTRUM": {
+    "HOST": "__ELECTRUM_HOST__",
+    "PORT": 16,
+    "TLS_ENABLED": true
+  },
+  "ESPLORA": {
+    "REST_API_URL": "__ESPLORA_REST_API_URL__"
+  },
+  "SECOND_CORE_RPC": {
+    "HOST": "__SECOND_CORE_RPC_HOST__",
+    "PORT": 17,
+    "USERNAME": "__SECOND_CORE_RPC_USERNAME__",
+    "PASSWORD": "__SECOND_CORE_RPC_PASSWORD__",
+    "TIMEOUT": "__SECOND_CORE_RPC_TIMEOUT__"
+  },
+  "DATABASE": {
+    "ENABLED": false,
+    "HOST": "__DATABASE_HOST__",
+    "SOCKET": "__DATABASE_SOCKET__",
+    "PORT": 18,
+    "DATABASE": "__DATABASE_DATABASE__",
+    "USERNAME": "__DATABASE_USERNAME__",
+    "PASSWORD": "__DATABASE_PASSWORD__"
+  },
+  "SYSLOG": {
+    "ENABLED": false,
+    "HOST": "__SYSLOG_HOST__",
+    "PORT": 19,
+    "MIN_PRIORITY": "__SYSLOG_MIN_PRIORITY__",
+    "FACILITY": "__SYSLOG_FACILITY__"
+  },
+  "STATISTICS": {
+    "ENABLED": false,
+    "TX_PER_SECOND_SAMPLE_PERIOD": 20
+  },
+  "BISQ": {
+    "ENABLED": true,
+    "DATA_PATH": "__BISQ_DATA_PATH__"
+  },
+  "SOCKS5PROXY": {
+    "ENABLED": true,
+    "USE_ONION": true,
+    "HOST": "__SOCKS5PROXY_HOST__",
+    "PORT": "__SOCKS5PROXY_PORT__",
+    "USERNAME": "__SOCKS5PROXY_USERNAME__",
+    "PASSWORD": "__SOCKS5PROXY_PASSWORD__"
+  },
+  "PRICE_DATA_SERVER": {
+    "TOR_URL": "__PRICE_DATA_SERVER_TOR_URL__",
+    "CLEARNET_URL": "__PRICE_DATA_SERVER_CLEARNET_URL__"
+  },
+  "EXTERNAL_DATA_SERVER": {
+    "MEMPOOL_API": "__EXTERNAL_DATA_SERVER_MEMPOOL_API__",
+    "MEMPOOL_ONION": "__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__",
+    "LIQUID_API": "__EXTERNAL_DATA_SERVER_LIQUID_API__",
+    "LIQUID_ONION": "__EXTERNAL_DATA_SERVER_LIQUID_ONION__",
+    "BISQ_URL": "__EXTERNAL_DATA_SERVER_BISQ_URL__",
+    "BISQ_ONION": "__EXTERNAL_DATA_SERVER_BISQ_ONION__"
+  },
+  "LIGHTNING": {
+    "ENABLED": "__LIGHTNING_ENABLED__",
+    "BACKEND": "__LIGHTNING_BACKEND__",
+    "TOPOLOGY_FOLDER": "__LIGHTNING_TOPOLOGY_FOLDER__",
+    "STATS_REFRESH_INTERVAL": 600,
+    "GRAPH_REFRESH_INTERVAL": 600,
+    "LOGGER_UPDATE_INTERVAL": 30,
+    "FORENSICS_INTERVAL": 43200,
+    "FORENSICS_RATE_LIMIT": "__FORENSICS_RATE_LIMIT__"
+  },
+  "LND": {
+    "TLS_CERT_PATH": "",
+    "MACAROON_PATH": "",
+    "REST_API_URL": "https://localhost:8080",
+    "TIMEOUT": 10000
+  },
+  "CLIGHTNING": {
+    "SOCKET": "__CLIGHTNING_SOCKET__"
+  }
+}

backend/src/__tests__/api/difficulty-adjustment.test.ts

@@ -0,0 +1,66 @@
+import { calcDifficultyAdjustment, DifficultyAdjustment } from '../../api/difficulty-adjustment';
+
+describe('Mempool Difficulty Adjustment', () => {
+  test('should calculate Difficulty Adjustments properly', () => {
+    const dt = (dtString) => {
+      return Math.floor(new Date(dtString).getTime() / 1000);
+    };
+
+    const vectors = [
+      [ // Vector 1
+        [ // Inputs
+          dt('2022-08-18T11:07:00.000Z'), // Last DA time (in seconds)
+          dt('2022-08-19T14:03:53.000Z'), // Current time (now) (in seconds)
+          750134,                         // Current block height
+          0.6280047707459726,             // Previous retarget % (Passed through)
+          'mainnet',                      // Network (if testnet, next value is non-zero)
+          0,                              // If not testnet, not used
+        ],
+        { // Expected Result
+          progressPercent: 9.027777777777777,
+          difficultyChange: 12.562233927411782,
+          estimatedRetargetDate: 1661895424692,
+          remainingBlocks: 1834,
+          remainingTime: 977591692,
+          previousRetarget: 0.6280047707459726,
+          previousTime: 1660820820,
+          nextRetargetHeight: 751968,
+          timeAvg: 533038,
+          timeOffset: 0,
+          expectedBlocks: 161.68833333333333,
+        },
+      ],
+      [ // Vector 2 (testnet)
+        [ // Inputs
+          dt('2022-08-18T11:07:00.000Z'), // Last DA time (in seconds)
+          dt('2022-08-19T14:03:53.000Z'), // Current time (now) (in seconds)
+          750134,                         // Current block height
+          0.6280047707459726,             // Previous retarget % (Passed through)
+          'testnet',                      // Network
+          dt('2022-08-19T13:52:46.000Z'), // Latest block timestamp in seconds
+        ],
+        { // Expected Result is same other than timeOffset
+          progressPercent: 9.027777777777777,
+          difficultyChange: 12.562233927411782,
+          estimatedRetargetDate: 1661895424692,
+          remainingBlocks: 1834,
+          remainingTime: 977591692,
+          previousTime: 1660820820,
+          previousRetarget: 0.6280047707459726,
+          nextRetargetHeight: 751968,
+          timeAvg: 533038,
+          timeOffset: -667000, // 11 min 7 seconds since last block (testnet only)
+          // If we add time avg to abs(timeOffset) it makes exactly 1200000 ms, or 20 minutes
+          expectedBlocks: 161.68833333333333,
+        },
+      ],
+    ] as [[number, number, number, number, string, number], DifficultyAdjustment][];
+
+    for (const vector of vectors) {
+      const result = calcDifficultyAdjustment(...vector[0]);
+      // previousRetarget is passed through untouched
+      expect(result.previousRetarget).toStrictEqual(vector[0][3]);
+      expect(result).toStrictEqual(vector[1]);
+    }
+  });
+});

backend/src/__tests__/config.test.ts

@@ -0,0 +1,154 @@
+import * as fs from 'fs';
+
+describe('Mempool Backend Config', () => {
+  beforeEach(() => {
+    jest.resetAllMocks();
+    jest.resetModules();
+  });
+
+  test('should return defaults when no file is present', () => {
+    jest.isolateModules(() => {
+      jest.mock('../../mempool-config.json', () => ({}), { virtual: true });
+
+      const config = jest.requireActual('../config').default;
+
+      expect(config.MEMPOOL).toStrictEqual({
+        ENABLED: true,
+        NETWORK: 'mainnet',
+        BACKEND: 'none',
+        BLOCKS_SUMMARIES_INDEXING: false,
+        HTTP_PORT: 8999,
+        SPAWN_CLUSTER_PROCS: 0,
+        API_URL_PREFIX: '/api/v1/',
+        AUTOMATIC_BLOCK_REINDEXING: false,
+        POLL_RATE_MS: 2000,
+        CACHE_DIR: './cache',
+        CLEAR_PROTECTION_MINUTES: 20,
+        RECOMMENDED_FEE_PERCENTILE: 50,
+        BLOCK_WEIGHT_UNITS: 4000000,
+        INITIAL_BLOCKS_AMOUNT: 8,
+        MEMPOOL_BLOCKS_AMOUNT: 8,
+        INDEXING_BLOCKS_AMOUNT: 11000,
+        USE_SECOND_NODE_FOR_MINFEE: false,
+        EXTERNAL_ASSETS: [],
+        EXTERNAL_MAX_RETRY: 1,
+        EXTERNAL_RETRY_INTERVAL: 0,
+        USER_AGENT: 'mempool',
+        STDOUT_LOG_MIN_PRIORITY: 'debug',
+        POOLS_JSON_TREE_URL: 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
+        POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json',
+        AUDIT: false,
+        ADVANCED_GBT_AUDIT: false,
+        ADVANCED_GBT_MEMPOOL: false,
+        CPFP_INDEXING: false,
+        MAX_BLOCKS_BULK_QUERY: 0,
+        DISK_CACHE_BLOCK_INTERVAL: 6,
+      });
+
+      expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true });
+
+      expect(config.ESPLORA).toStrictEqual({ REST_API_URL: 'http://127.0.0.1:3000' });
+
+      expect(config.CORE_RPC).toStrictEqual({
+        HOST: '127.0.0.1',
+        PORT: 8332,
+        USERNAME: 'mempool',
+        PASSWORD: 'mempool',
+        TIMEOUT: 60000
+      });
+
+      expect(config.SECOND_CORE_RPC).toStrictEqual({
+        HOST: '127.0.0.1',
+        PORT: 8332,
+        USERNAME: 'mempool',
+        PASSWORD: 'mempool',
+        TIMEOUT: 60000
+      });
+
+      expect(config.DATABASE).toStrictEqual({
+        ENABLED: true,
+        HOST: '127.0.0.1',
+        SOCKET: '',
+        PORT: 3306,
+        DATABASE: 'mempool',
+        USERNAME: 'mempool',
+        PASSWORD: 'mempool'
+      });
+
+      expect(config.SYSLOG).toStrictEqual({
+        ENABLED: true,
+        HOST: '127.0.0.1',
+        PORT: 514,
+        MIN_PRIORITY: 'info',
+        FACILITY: 'local7'
+      });
+
+      expect(config.STATISTICS).toStrictEqual({ ENABLED: true, TX_PER_SECOND_SAMPLE_PERIOD: 150 });
+
+      expect(config.BISQ).toStrictEqual({ ENABLED: false, DATA_PATH: '/bisq/statsnode-data/btc_mainnet/db' });
+
+      expect(config.SOCKS5PROXY).toStrictEqual({
+        ENABLED: false,
+        USE_ONION: true,
+        HOST: '127.0.0.1',
+        PORT: 9050,
+        USERNAME: '',
+        PASSWORD: ''
+      });
+
+      expect(config.PRICE_DATA_SERVER).toStrictEqual({
+        TOR_URL: 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices',
+        CLEARNET_URL: 'https://price.bisq.wiz.biz/getAllMarketPrices'
+      });
+
+      expect(config.EXTERNAL_DATA_SERVER).toStrictEqual({
+        MEMPOOL_API: 'https://mempool.space/api/v1',
+        MEMPOOL_ONION: 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1',
+        LIQUID_API: 'https://liquid.network/api/v1',
+        LIQUID_ONION: 'http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1',
+        BISQ_URL: 'https://bisq.markets/api',
+        BISQ_ONION: 'http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api'
+      });
+
+      expect(config.MAXMIND).toStrictEqual({
+        ENABLED: false,
+        GEOLITE2_CITY: '/usr/local/share/GeoIP/GeoLite2-City.mmdb',
+        GEOLITE2_ASN: '/usr/local/share/GeoIP/GeoLite2-ASN.mmdb',
+        GEOIP2_ISP: '/usr/local/share/GeoIP/GeoIP2-ISP.mmdb'
+      });
+    });
+  });
+
+  test('should override the default values with the passed values', () => {
+    jest.isolateModules(() => {
+      const fixture = JSON.parse(fs.readFileSync(`${__dirname}/../__fixtures__/mempool-config.template.json`, 'utf8'));
+      jest.mock('../../mempool-config.json', () => (fixture), { virtual: true });
+
+      const config = jest.requireActual('../config').default;
+
+      expect(config.MEMPOOL).toStrictEqual(fixture.MEMPOOL);
+
+      expect(config.ELECTRUM).toStrictEqual(fixture.ELECTRUM);
+
+      expect(config.ESPLORA).toStrictEqual(fixture.ESPLORA);
+
+      expect(config.CORE_RPC).toStrictEqual(fixture.CORE_RPC);
+
+      expect(config.SECOND_CORE_RPC).toStrictEqual(fixture.SECOND_CORE_RPC);
+
+      expect(config.DATABASE).toStrictEqual(fixture.DATABASE);
+
+      expect(config.SYSLOG).toStrictEqual(fixture.SYSLOG);
+
+      expect(config.STATISTICS).toStrictEqual(fixture.STATISTICS);
+
+      expect(config.BISQ).toStrictEqual(fixture.BISQ);
+
+      expect(config.SOCKS5PROXY).toStrictEqual(fixture.SOCKS5PROXY);
+
+      expect(config.PRICE_DATA_SERVER).toStrictEqual(fixture.PRICE_DATA_SERVER);
+
+      expect(config.EXTERNAL_DATA_SERVER).toStrictEqual(fixture.EXTERNAL_DATA_SERVER);
+    });
+  });
+});

backend/src/api/audit.ts

@@ -0,0 +1,143 @@
+import config from '../config';
+import { TransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces';
+
+const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners
+
+class Audit {
+  auditBlock(transactions: TransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: TransactionExtended })
+   : { censored: string[], added: string[], fresh: string[], score: number, similarity: number } {
+    if (!projectedBlocks?.[0]?.transactionIds || !mempool) {
+      return { censored: [], added: [], fresh: [], score: 0, similarity: 1 };
+    }
+
+    const matches: string[] = []; // present in both mined block and template
+    const added: string[] = []; // present in mined block, not in template
+    const fresh: string[] = []; // missing, but firstSeen within PROPAGATION_MARGIN
+    const isCensored = {}; // missing, without excuse
+    const isDisplaced = {};
+    let displacedWeight = 0;
+    let matchedWeight = 0;
+    let projectedWeight = 0;
+
+    const inBlock = {};
+    const inTemplate = {};
+
+    const now = Math.round((Date.now() / 1000));
+    for (const tx of transactions) {
+      inBlock[tx.txid] = tx;
+    }
+    // coinbase is always expected
+    if (transactions[0]) {
+      inTemplate[transactions[0].txid] = true;
+    }
+    // look for transactions that were expected in the template, but missing from the mined block
+    for (const txid of projectedBlocks[0].transactionIds) {
+      if (!inBlock[txid]) {
+        // tx is recent, may have reached the miner too late for inclusion
+        if (mempool[txid]?.firstSeen != null && (now - (mempool[txid]?.firstSeen || 0)) <= PROPAGATION_MARGIN) {
+          fresh.push(txid);
+        } else {
+          isCensored[txid] = true;
+        }
+        displacedWeight += mempool[txid].weight;
+      } else {
+        matchedWeight += mempool[txid].weight;
+      }
+      projectedWeight += mempool[txid].weight;
+      inTemplate[txid] = true;
+    }
+
+    displacedWeight += (4000 - transactions[0].weight);
+    projectedWeight += transactions[0].weight;
+    matchedWeight += transactions[0].weight;
+
+    // we can expect an honest miner to include 'displaced' transactions in place of recent arrivals and censored txs
+    // these displaced transactions should occupy the first N weight units of the next projected block
+    let displacedWeightRemaining = displacedWeight;
+    let index = 0;
+    let lastFeeRate = Infinity;
+    let failures = 0;
+    while (projectedBlocks[1] && index < projectedBlocks[1].transactionIds.length && failures < 500) {
+      const txid = projectedBlocks[1].transactionIds[index];
+      const fits = (mempool[txid].weight - displacedWeightRemaining) < 4000;
+      const feeMatches = mempool[txid].effectiveFeePerVsize >= lastFeeRate;
+      if (fits || feeMatches) {
+        isDisplaced[txid] = true;
+        if (fits) {
+          lastFeeRate = Math.min(lastFeeRate, mempool[txid].effectiveFeePerVsize);
+        }
+        if (mempool[txid].firstSeen == null || (now - (mempool[txid]?.firstSeen || 0)) > PROPAGATION_MARGIN) {
+          displacedWeightRemaining -= mempool[txid].weight;
+        }
+        failures = 0;
+      } else {
+        failures++;
+      }
+      index++;
+    }
+
+    // mark unexpected transactions in the mined block as 'added'
+    let overflowWeight = 0;
+    let totalWeight = 0;
+    for (const tx of transactions) {
+      if (inTemplate[tx.txid]) {
+        matches.push(tx.txid);
+      } else {
+        if (!isDisplaced[tx.txid]) {
+          added.push(tx.txid);
+        } else {
+        }
+        let blockIndex = -1;
+        let index = -1;
+        projectedBlocks.forEach((block, bi) => {
+          const i = block.transactionIds.indexOf(tx.txid);
+          if (i >= 0) {
+            blockIndex = bi;
+            index = i;
+          }
+        });
+        overflowWeight += tx.weight;
+      }
+      totalWeight += tx.weight;
+    }
+
+    // transactions missing from near the end of our template are probably not being censored
+    let overflowWeightRemaining = overflowWeight - (config.MEMPOOL.BLOCK_WEIGHT_UNITS - totalWeight);
+    let maxOverflowRate = 0;
+    let rateThreshold = 0;
+    index = projectedBlocks[0].transactionIds.length - 1;
+    while (index >= 0) {
+      const txid = projectedBlocks[0].transactionIds[index];
+      if (overflowWeightRemaining > 0) {
+        if (isCensored[txid]) {
+          delete isCensored[txid];
+        }
+        if (mempool[txid].effectiveFeePerVsize > maxOverflowRate) {
+          maxOverflowRate = mempool[txid].effectiveFeePerVsize;
+          rateThreshold = (Math.ceil(maxOverflowRate * 100) / 100) + 0.005;
+        }
+      } else if (mempool[txid].effectiveFeePerVsize <= rateThreshold) { // tolerance of 0.01 sat/vb + rounding
+        if (isCensored[txid]) {
+          delete isCensored[txid];
+        }
+      }
+      overflowWeightRemaining -= (mempool[txid]?.weight || 0);
+      index--;
+    }
+
+    const numCensored = Object.keys(isCensored).length;
+    const numMatches = matches.length - 1; // adjust for coinbase tx
+    const score = numMatches > 0 ? (numMatches / (numMatches + numCensored)) : 0;
+    const similarity = projectedWeight ? matchedWeight / projectedWeight : 1;
+
+    return {
+      censored: Object.keys(isCensored),
+      added,
+      fresh,
+      score,
+      similarity,
+    };
+  }
+}
+
+export default new Audit();

backend/src/api/backend-info.ts

@@ -1,46 +1,39 @@
-import * as fs from 'fs';
-import * as os from 'os';
-import logger from '../logger';
+import fs from 'fs';
+import path from 'path';
+import os from 'os';
 import { IBackendInfo } from '../mempool.interfaces';
+import config from '../config';
 
 class BackendInfo {
-  private gitCommitHash = '';
-  private hostname = '';
-  private version = '';
+  private backendInfo: IBackendInfo;
 
   constructor() {
-    this.setLatestCommitHash();
-    this.setVersion();
-    this.hostname = os.hostname();
-  }
-
-  public getBackendInfo(): IBackendInfo {
-    return {
-      hostname: this.hostname,
-      gitCommit: this.gitCommitHash,
-      version: this.version,
+    // This file is created by ./fetch-version.ts during building
+    const versionFile = path.join(__dirname, 'version.json')
+    var versionInfo;
+    if (fs.existsSync(versionFile)) {
+      versionInfo = JSON.parse(fs.readFileSync(versionFile).toString());
+    } else {
+      // Use dummy values if `versionFile` doesn't exist (e.g., during testing)
+      versionInfo = {
+        version: '?',
+        gitCommit: '?'
+      };
+    }
+    this.backendInfo = {
+      hostname: os.hostname(),
+      version: versionInfo.version,
+      gitCommit: versionInfo.gitCommit,
+      lightning: config.LIGHTNING.ENABLED
     };
   }
 
+  public getBackendInfo(): IBackendInfo {
+    return this.backendInfo;
+  }
+
   public getShortCommitHash() {
-    return this.gitCommitHash.slice(0, 7);
-  }
-
-  private setLatestCommitHash(): void {
-    try {
-      this.gitCommitHash = fs.readFileSync('../.git/refs/heads/master').toString().trim();
-    } catch (e) {
-      logger.err('Could not load git commit info: ' + e.message || e);
-    }
-  }
-
-  private setVersion(): void {
-    try {
-      const packageJson = fs.readFileSync('package.json').toString();
-      this.version = JSON.parse(packageJson).version;
-    } catch (e) {
-      throw new Error(e);
-    }
+    return this.backendInfo.gitCommit.slice(0, 7);
   }
 }
 

backend/src/api/bisq/bisq.routes.ts

@@ -0,0 +1,381 @@
+import { Application, Request, Response } from 'express';
+import config from '../../config';
+import { RequiredSpec } from '../../mempool.interfaces';
+import bisq from './bisq';
+import { MarketsApiError } from './interfaces';
+import marketsApi from './markets-api';
+
+class BisqRoutes {
+  public initRoutes(app: Application) {
+    app
+      .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/stats', this.getBisqStats)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/tx/:txId', this.getBisqTransaction)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/block/:hash', this.getBisqBlock)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/blocks/tip/height', this.getBisqTip)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/blocks/:index/:length', this.getBisqBlocks)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/address/:address', this.getBisqAddress)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/txs/:index/:length', this.getBisqTransactions)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/currencies', this.getBisqMarketCurrencies.bind(this))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/depth', this.getBisqMarketDepth.bind(this))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/hloc', this.getBisqMarketHloc.bind(this))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/markets', this.getBisqMarketMarkets.bind(this))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/offers', this.getBisqMarketOffers.bind(this))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/ticker', this.getBisqMarketTicker.bind(this))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/trades', this.getBisqMarketTrades.bind(this))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/volumes', this.getBisqMarketVolumes.bind(this))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/volumes/7d', this.getBisqMarketVolumes7d.bind(this))
+    ;
+  }
+
+
+  private getBisqStats(req: Request, res: Response) {
+    const result = bisq.getStats();
+    res.json(result);
+  }
+
+  private getBisqTip(req: Request, res: Response) {
+    const result = bisq.getLatestBlockHeight();
+    res.type('text/plain');
+    res.send(result.toString());
+  }
+
+  private getBisqTransaction(req: Request, res: Response) {
+    const result = bisq.getTransaction(req.params.txId);
+    if (result) {
+      res.json(result);
+    } else {
+      res.status(404).send('Bisq transaction not found');
+    }
+  }
+
+  private getBisqTransactions(req: Request, res: Response) {
+    const types: string[] = [];
+    req.query.types = req.query.types || [];
+    if (!Array.isArray(req.query.types)) {
+      res.status(500).send('Types is not an array');
+      return;
+    }
+
+    for (const _type in req.query.types) {
+      if (typeof req.query.types[_type] === 'string') {
+        types.push(req.query.types[_type].toString());
+      }
+    }
+
+    const index = parseInt(req.params.index, 10) || 0;
+    const length = parseInt(req.params.length, 10) > 100 ? 100 : parseInt(req.params.length, 10) || 25;
+    const [transactions, count] = bisq.getTransactions(index, length, types);
+    res.header('X-Total-Count', count.toString());
+    res.json(transactions);
+  }
+
+  private getBisqBlock(req: Request, res: Response) {
+    const result = bisq.getBlock(req.params.hash);
+    if (result) {
+      res.json(result);
+    } else {
+      res.status(404).send('Bisq block not found');
+    }
+  }
+
+  private getBisqBlocks(req: Request, res: Response) {
+    const index = parseInt(req.params.index, 10) || 0;
+    const length = parseInt(req.params.length, 10) > 100 ? 100 : parseInt(req.params.length, 10) || 25;
+    const [transactions, count] = bisq.getBlocks(index, length);
+    res.header('X-Total-Count', count.toString());
+    res.json(transactions);
+  }
+
+  private getBisqAddress(req: Request, res: Response) {
+    const result = bisq.getAddress(req.params.address.substr(1));
+    if (result) {
+      res.json(result);
+    } else {
+      res.status(404).send('Bisq address not found');
+    }
+  }
+
+  private getBisqMarketCurrencies(req: Request, res: Response) {
+    const constraints: RequiredSpec = {
+      'type': {
+        required: false,
+        types: ['crypto', 'fiat', 'all']
+      },
+    };
+
+    const p = this.parseRequestParameters(req.query, constraints);
+    if (p.error) {
+      res.status(400).json(this.getBisqMarketErrorResponse(p.error));
+      return;
+    }
+
+    const result = marketsApi.getCurrencies(p.type);
+    if (result) {
+      res.json(result);
+    } else {
+      res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketCurrencies error'));
+    }
+  }
+
+  private getBisqMarketDepth(req: Request, res: Response) {
+    const constraints: RequiredSpec = {
+      'market': {
+        required: true,
+        types: ['@string']
+      },
+    };
+
+    const p = this.parseRequestParameters(req.query, constraints);
+    if (p.error) {
+      res.status(400).json(this.getBisqMarketErrorResponse(p.error));
+      return;
+    }
+
+    const result = marketsApi.getDepth(p.market);
+    if (result) {
+      res.json(result);
+    } else {
+      res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketDepth error'));
+    }
+  }
+
+  private getBisqMarketMarkets(req: Request, res: Response) {
+    const result = marketsApi.getMarkets();
+    if (result) {
+      res.json(result);
+    } else {
+      res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketMarkets error'));
+    }
+  }
+
+  private getBisqMarketTrades(req: Request, res: Response) {
+    const constraints: RequiredSpec = {
+      'market': {
+        required: true,
+        types: ['@string']
+      },
+      'timestamp_from': {
+        required: false,
+        types: ['@number']
+      },
+      'timestamp_to': {
+        required: false,
+        types: ['@number']
+      },
+      'trade_id_to': {
+        required: false,
+        types: ['@string']
+      },
+      'trade_id_from': {
+        required: false,
+        types: ['@string']
+      },
+      'direction': {
+        required: false,
+        types: ['buy', 'sell']
+      },
+      'limit': {
+        required: false,
+        types: ['@number']
+      },
+      'sort': {
+        required: false,
+        types: ['asc', 'desc']
+      }
+    };
+
+    const p = this.parseRequestParameters(req.query, constraints);
+    if (p.error) {
+      res.status(400).json(this.getBisqMarketErrorResponse(p.error));
+      return;
+    }
+
+    const result = marketsApi.getTrades(p.market, p.timestamp_from,
+      p.timestamp_to, p.trade_id_from, p.trade_id_to, p.direction, p.limit, p.sort);
+    if (result) {
+      res.json(result);
+    } else {
+      res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketTrades error'));
+    }
+  }
+
+  private getBisqMarketOffers(req: Request, res: Response) {
+    const constraints: RequiredSpec = {
+      'market': {
+        required: true,
+        types: ['@string']
+      },
+      'direction': {
+        required: false,
+        types: ['buy', 'sell']
+      },
+    };
+
+    const p = this.parseRequestParameters(req.query, constraints);
+    if (p.error) {
+      res.status(400).json(this.getBisqMarketErrorResponse(p.error));
+      return;
+    }
+
+    const result = marketsApi.getOffers(p.market, p.direction);
+    if (result) {
+      res.json(result);
+    } else {
+      res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketOffers error'));
+    }
+  }
+
+  private getBisqMarketVolumes(req: Request, res: Response) {
+    const constraints: RequiredSpec = {
+      'market': {
+        required: false,
+        types: ['@string']
+      },
+      'interval': {
+        required: false,
+        types: ['minute', 'half_hour', 'hour', 'half_day', 'day', 'week', 'month', 'year', 'auto']
+      },
+      'timestamp_from': {
+        required: false,
+        types: ['@number']
+      },
+      'timestamp_to': {
+        required: false,
+        types: ['@number']
+      },
+      'milliseconds': {
+        required: false,
+        types: ['@boolean']
+      },
+      'timestamp': {
+        required: false,
+        types: ['no', 'yes']
+      },
+    };
+
+    const p = this.parseRequestParameters(req.query, constraints);
+    if (p.error) {
+      res.status(400).json(this.getBisqMarketErrorResponse(p.error));
+      return;
+    }
+
+    const result = marketsApi.getVolumes(p.market, p.timestamp_from, p.timestamp_to, p.interval, p.milliseconds, p.timestamp);
+    if (result) {
+      res.json(result);
+    } else {
+      res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketVolumes error'));
+    }
+  }
+
+  private getBisqMarketHloc(req: Request, res: Response) {
+    const constraints: RequiredSpec = {
+      'market': {
+        required: true,
+        types: ['@string']
+      },
+      'interval': {
+        required: false,
+        types: ['minute', 'half_hour', 'hour', 'half_day', 'day', 'week', 'month', 'year', 'auto']
+      },
+      'timestamp_from': {
+        required: false,
+        types: ['@number']
+      },
+      'timestamp_to': {
+        required: false,
+        types: ['@number']
+      },
+      'milliseconds': {
+        required: false,
+        types: ['@boolean']
+      },
+      'timestamp': {
+        required: false,
+        types: ['no', 'yes']
+      },
+    };
+
+    const p = this.parseRequestParameters(req.query, constraints);
+    if (p.error) {
+      res.status(400).json(this.getBisqMarketErrorResponse(p.error));
+      return;
+    }
+
+    const result = marketsApi.getHloc(p.market, p.interval, p.timestamp_from, p.timestamp_to, p.milliseconds, p.timestamp);
+    if (result) {
+      res.json(result);
+    } else {
+      res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketHloc error'));
+    }
+  }
+
+  private getBisqMarketTicker(req: Request, res: Response) {
+    const constraints: RequiredSpec = {
+      'market': {
+        required: false,
+        types: ['@string']
+      },
+    };
+
+    const p = this.parseRequestParameters(req.query, constraints);
+    if (p.error) {
+      res.status(400).json(this.getBisqMarketErrorResponse(p.error));
+      return;
+    }
+
+    const result = marketsApi.getTicker(p.market);
+    if (result) {
+      res.json(result);
+    } else {
+      res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketTicker error'));
+    }
+  }
+
+  private getBisqMarketVolumes7d(req: Request, res: Response) {
+    const result = marketsApi.getVolumesByTime(604800);
+    if (result) {
+      res.json(result);
+    } else {
+      res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketVolumes7d error'));
+    }
+  }
+
+  private parseRequestParameters(requestParams: object, params: RequiredSpec): { [name: string]: any; } {
+    const final = {};
+    for (const i in params) {
+      if (params.hasOwnProperty(i)) {
+        if (params[i].required && requestParams[i] === undefined) {
+          return { error: i + ' parameter missing'};
+        }
+        if (typeof requestParams[i] === 'string') {
+          const str = (requestParams[i] || '').toString().toLowerCase();
+          if (params[i].types.indexOf('@number') > -1) {
+            const number = parseInt((str).toString(), 10);
+            final[i] = number;
+          } else if (params[i].types.indexOf('@string') > -1) {
+            final[i] = str;
+          } else if (params[i].types.indexOf('@boolean') > -1) {
+            final[i] = str === 'true' || str === 'yes';
+          } else if (params[i].types.indexOf(str) > -1) {
+            final[i] = str;
+          } else {
+            return { error: i + ' parameter invalid'};
+          }
+        } else if (typeof requestParams[i] === 'number') {
+          final[i] = requestParams[i];
+        }
+      }
+    }
+    return final;
+  }
+
+  private getBisqMarketErrorResponse(message: string): MarketsApiError {
+    return {
+      'success': 0,
+      'error': message
+    };
+  }
+
+}
+
+export default new BisqRoutes;

backend/src/api/bisq/bisq.ts

@@ -1,10 +1,14 @@
 import config from '../../config';
 import * as fs from 'fs';
-import axios from 'axios';
+import axios, { AxiosResponse } from 'axios';
+import * as http from 'http';
+import * as https from 'https';
+import { SocksProxyAgent } from 'socks-proxy-agent';
 import { BisqBlocks, BisqBlock, BisqTransaction, BisqStats, BisqTrade } from './interfaces';
 import { Common } from '../common';
 import { BlockExtended } from '../../mempool.interfaces';
 import { StaticPool } from 'node-worker-threads-pool';
+import backendInfo from '../backend-info';
 import logger from '../../logger';
 
 class Bisq {
@@ -35,7 +39,13 @@ class Bisq {
   constructor() {}
 
   startBisqService(): void {
-    this.checkForBisqDataFolder();
+    try {
+      this.checkForBisqDataFolder();
+    } catch (e) {
+      logger.info('Retrying to start bisq service in 3 minutes');
+      setTimeout(this.startBisqService.bind(this), 180000);
+      return;
+    }
     this.loadBisqDumpFile();
     setInterval(this.updatePrice.bind(this), 1000 * 60 * 60);
     this.updatePrice();
@@ -90,7 +100,7 @@ class Bisq {
   private checkForBisqDataFolder() {
     if (!fs.existsSync(Bisq.BLOCKS_JSON_FILE_PATH)) {
       logger.warn(Bisq.BLOCKS_JSON_FILE_PATH + ` doesn't exist. Make sure Bisq is running and the config is correct before starting the server.`);
-      return process.exit(1);
+      throw new Error(`Cannot load BISQ ${Bisq.BLOCKS_JSON_FILE_PATH} file`);
     }
   }
 
@@ -137,12 +147,59 @@ class Bisq {
       }, 2000);
     });
   }
+  private async updatePrice() {
+    type axiosOptions = {
+      headers: {
+        'User-Agent': string
+      };
+      timeout: number;
+      httpAgent?: http.Agent;
+      httpsAgent?: https.Agent;
+    }
+    const setDelay = (secs: number = 1): Promise<void> => new Promise(resolve => setTimeout(() => resolve(), secs * 1000));
+    const BISQ_URL = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.EXTERNAL_DATA_SERVER.BISQ_ONION : config.EXTERNAL_DATA_SERVER.BISQ_URL;
+    const isHTTP = (new URL(BISQ_URL).protocol.split(':')[0] === 'http') ? true : false;
+    const axiosOptions: axiosOptions = {
+      headers: {
+        'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
+      },
+      timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000
+    };
+    let retry = 0;
 
-  private updatePrice() {
-    axios.get<BisqTrade[]>('https://bisq.markets/api/trades/?market=bsq_btc', { timeout: 10000 })
-      .then((response) => {
+    while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
+      try {
+        if (config.SOCKS5PROXY.ENABLED) {
+          const socksOptions: any = {
+            agentOptions: {
+              keepAlive: true,
+            },
+            hostname: config.SOCKS5PROXY.HOST,
+            port: config.SOCKS5PROXY.PORT
+          };
+
+          if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
+            socksOptions.username = config.SOCKS5PROXY.USERNAME;
+            socksOptions.password = config.SOCKS5PROXY.PASSWORD;
+          } else {
+            // Retry with different tor circuits https://stackoverflow.com/a/64960234
+            socksOptions.username = `circuit${retry}`;
+          }
+
+          // Handle proxy agent for onion addresses
+          if (isHTTP) {
+            axiosOptions.httpAgent = new SocksProxyAgent(socksOptions);
+          } else {
+            axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions);
+          }
+        }
+        
+        const data: AxiosResponse = await axios.get(`${BISQ_URL}/trades/?market=bsq_btc`, axiosOptions);
+        if (data.statusText === 'error' || !data.data) {
+          throw new Error(`Could not fetch data from Bisq market, Error: ${data.status}`);
+        }
         const prices: number[] = [];
-        response.data.forEach((trade) => {
+        data.data.forEach((trade) => {
           prices.push(parseFloat(trade.price) * 100000000);
         });
         prices.sort((a, b) => a - b);
@@ -150,19 +207,24 @@ class Bisq {
         if (this.priceUpdateCallbackFunction) {
           this.priceUpdateCallbackFunction(this.price);
         }
-    }).catch((err) => {
-      logger.err('Error updating Bisq market price: ' + err);
-    });
+        logger.debug('Successfully updated Bisq market price');
+        break;
+      } catch (e) {
+        logger.err('Error updating Bisq market price: '  + (e instanceof Error ? e.message : e));
+        await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL);
+        retry++;
+      }
+    }
   }
 
   private async loadBisqDumpFile(): Promise<void> {
+    this.allBlocks = [];
     try {
-      const data = await this.loadData();
-      await this.loadBisqBlocksDump(data);
+      await this.loadData();
       this.buildIndex();
       this.calculateStats();
     } catch (e) {
-      logger.err('loadBisqDumpFile() error.' + e.message || e);
+      logger.info('Cannot load bisq dump file because: ' + (e instanceof Error ? e.message : e));
     }
   }
 
@@ -241,36 +303,61 @@ class Bisq {
     };
   }
 
-  private async loadBisqBlocksDump(cacheData: string): Promise<void> {
-    const start = new Date().getTime();
-    if (cacheData && cacheData.length !== 0) {
-      logger.debug('Processing Bisq data dump...');
-      const data: BisqBlocks = await this.jsonParsePool.exec(cacheData);
-      if (data.blocks && data.blocks.length !== this.allBlocks.length) {
-        this.allBlocks = data.blocks;
-        this.allBlocks.reverse();
-        this.blocks = this.allBlocks.filter((block) => block.txs.length > 0);
-        this.latestBlockHeight = data.chainHeight;
-        const time = new Date().getTime() - start;
-        logger.debug('Bisq dump processed in ' + time + ' ms (worker thread)');
-      } else {
-        throw new Error(`Bisq dump didn't contain any blocks`);
-      }
+  private async loadData(): Promise<any> {
+    if (!fs.existsSync(Bisq.BLOCKS_JSON_FILE_PATH)) {
+      throw new Error(Bisq.BLOCKS_JSON_FILE_PATH + ` doesn't exist`);
     }
-  }
 
-  private loadData(): Promise<string> {
-    return new Promise((resolve, reject) => {
-      if (!fs.existsSync(Bisq.BLOCKS_JSON_FILE_PATH)) {
-        return reject(Bisq.BLOCKS_JSON_FILE_PATH + ` doesn't exist`);
-      }
-      fs.readFile(Bisq.BLOCKS_JSON_FILE_PATH, 'utf8', (err, data) => {
-        if (err) {
-          reject(err);
-        }
-        resolve(data);
-      });
+    const readline = require('readline');
+    const events = require('events');
+
+    const rl = readline.createInterface({
+      input: fs.createReadStream(Bisq.BLOCKS_JSON_FILE_PATH),
+      crlfDelay: Infinity
     });
+
+    let blockBuffer = '';
+    let readingBlock = false;
+    let lineCount = 1;
+    const start = new Date().getTime();
+
+    logger.debug('Processing Bisq data dump...');
+
+    rl.on('line', (line) => {
+      if (lineCount === 2) {
+        line = line.replace('  "chainHeight": ', '');
+        this.latestBlockHeight = parseInt(line, 10);
+      }
+
+      if (line === '    {') {
+        readingBlock = true;
+      } else if (line === '    },') {
+        blockBuffer += '}';
+        try {
+          const block: BisqBlock =  JSON.parse(blockBuffer);
+          this.allBlocks.push(block);
+          readingBlock = false;
+          blockBuffer = '';
+        } catch (e) {
+          logger.debug(blockBuffer);
+          throw Error(`Unable to parse Bisq data dump at line ${lineCount}` + (e instanceof Error ? e.message : e));
+        }
+      }
+
+      if (readingBlock === true) {
+        blockBuffer += line;
+      }
+
+      ++lineCount;
+    });
+
+    await events.once(rl, 'close');
+
+    this.allBlocks.reverse();
+    this.blocks = this.allBlocks.filter((block) => block.txs.length > 0);
+
+    const time = new Date().getTime() - start;
+    logger.debug('Bisq dump processed in ' + time + ' ms');
   }
 }
 

backend/src/api/bisq/interfaces.ts

@@ -71,7 +71,7 @@ interface BisqScriptPubKey {
   addresses: string[];
   asm: string;
   hex: string;
-  reqSigs: number;
+  reqSigs?: number;
   type: string;
 }
 

backend/src/api/bisq/markets-api.ts

@@ -1,7 +1,7 @@
 import { Currencies, OffersData, TradesData, Depth, Currency, Interval, HighLowOpenClose,
   Markets, Offers, Offer, BisqTrade, MarketVolume, Tickers, Ticker, SummarizedIntervals, SummarizedInterval } from './interfaces';
 
-import * as datetime from 'locutus/php/datetime';
+const strtotime = require('./strtotime');
 
 class BisqMarketsApi {
   private cryptoCurrencyData: Currency[] = [];
@@ -312,7 +312,7 @@ class BisqMarketsApi {
 
   getTickerFromMarket(market: string): Ticker | null {
     let ticker: Ticker;
-    const timestamp_from = datetime.strtotime('-24 hour');
+    const timestamp_from = strtotime('-24 hour');
     const timestamp_to = new Date().getTime() / 1000;
     const trades = this.getTradesByCriteria(market, timestamp_to, timestamp_from,
       undefined, undefined, undefined, 'asc', Number.MAX_SAFE_INTEGER);
@@ -638,13 +638,13 @@ class BisqMarketsApi {
         case 'half_day':
             return (ts - (ts % (3600 * 12)));
         case 'day':
-            return datetime.strtotime('midnight today', ts);
+            return strtotime('midnight today', ts);
         case 'week':
-            return datetime.strtotime('midnight sunday last week', ts);
+            return strtotime('midnight sunday last week', ts);
         case 'month':
-            return datetime.strtotime('midnight first day of this month', ts);
+            return strtotime('midnight first day of this month', ts);
         case 'year':
-            return datetime.strtotime('midnight first day of january', ts);
+            return strtotime('midnight first day of january', ts);
         default:
             throw new Error('Unsupported interval: ' + interval);
     }

backend/src/api/bisq/markets.ts

@@ -26,7 +26,13 @@ class Bisq {
   constructor() {}
 
   startBisqService(): void {
-    this.checkForBisqDataFolder();
+    try {
+      this.checkForBisqDataFolder();
+    } catch (e) {
+      logger.info('Retrying to start bisq service (markets) in 3 minutes');
+      setTimeout(this.startBisqService.bind(this), 180000);
+      return;
+    }
     this.loadBisqDumpFile();
     this.startBisqDirectoryWatcher();
   }
@@ -34,7 +40,7 @@ class Bisq {
   private checkForBisqDataFolder() {
     if (!fs.existsSync(Bisq.MARKET_JSON_PATH + Bisq.MARKET_JSON_FILE_PATHS.cryptoCurrency)) {
       logger.err(Bisq.MARKET_JSON_PATH + Bisq.MARKET_JSON_FILE_PATHS.cryptoCurrency + ` doesn't exist. Make sure Bisq is running and the config is correct before starting the server.`);
-      return process.exit(1);
+      throw new Error(`Cannot load BISQ ${Bisq.MARKET_JSON_FILE_PATHS.cryptoCurrency} file`);
     }
   }
 
@@ -102,7 +108,7 @@ class Bisq {
         logger.debug('Bisq market data updated in ' + time + ' ms');
       }
     } catch (e) {
-      logger.err('loadBisqMarketDataDumpFile() error.' + e.message || e);
+      logger.err('loadBisqMarketDataDumpFile() error.' + (e instanceof Error ? e.message : e));
     }
   }
 

backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts

@@ -2,12 +2,27 @@ import { IEsploraApi } from './esplora-api.interface';
 
 export interface AbstractBitcoinApi {
   $getRawMempool(): Promise<IEsploraApi.Transaction['txid'][]>;
-  $getRawTransaction(txId: string, skipConversion?: boolean, addPrevout?: boolean): Promise<IEsploraApi.Transaction>;
+  $getRawTransaction(txId: string, skipConversion?: boolean, addPrevout?: boolean, lazyPrevouts?: boolean): Promise<IEsploraApi.Transaction>;
+  $getTransactionHex(txId: string): Promise<string>;
   $getBlockHeightTip(): Promise<number>;
+  $getBlockHashTip(): Promise<string>;
   $getTxIdsForBlock(hash: string): Promise<string[]>;
   $getBlockHash(height: number): Promise<string>;
+  $getBlockHeader(hash: string): Promise<string>;
   $getBlock(hash: string): Promise<IEsploraApi.Block>;
+  $getRawBlock(hash: string): Promise<Buffer>;
   $getAddress(address: string): Promise<IEsploraApi.Address>;
   $getAddressTransactions(address: string, lastSeenTxId: string): Promise<IEsploraApi.Transaction[]>;
   $getAddressPrefix(prefix: string): string[];
+  $sendRawTransaction(rawTransaction: string): Promise<string>;
+  $getOutspend(txId: string, vout: number): Promise<IEsploraApi.Outspend>;
+  $getOutspends(txId: string): Promise<IEsploraApi.Outspend[]>;
+  $getBatchedOutspends(txId: string[]): Promise<IEsploraApi.Outspend[][]>;
+}
+export interface BitcoinRpcCredentials {
+  host: string;
+  port: number;
+  user: string;
+  pass: string;
+  timeout: number;
 }

backend/src/api/bitcoin/bitcoin-api-factory.ts

@@ -3,17 +3,20 @@ import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
 import EsploraApi from './esplora-api';
 import BitcoinApi from './bitcoin-api';
 import ElectrumApi from './electrum-api';
+import bitcoinClient from './bitcoin-client';
 
 function bitcoinApiFactory(): AbstractBitcoinApi {
   switch (config.MEMPOOL.BACKEND) {
     case 'esplora':
       return new EsploraApi();
     case 'electrum':
-      return new ElectrumApi();
+      return new ElectrumApi(bitcoinClient);
     case 'none':
     default:
-      return new BitcoinApi();
+      return new BitcoinApi(bitcoinClient);
   }
 }
 
+export const bitcoinCoreApi = new BitcoinApi(bitcoinClient);
+
 export default bitcoinApiFactory();

backend/src/api/bitcoin/bitcoin-api.interface.ts

@@ -4,6 +4,7 @@ export namespace IBitcoinApi {
     size: number;                    //  (numeric) Current tx count
     bytes: number;                   //  (numeric) Sum of all virtual transaction sizes as defined in BIP 141.
     usage: number;                   //  (numeric) Total memory usage for the mempool
+    total_fee: number;               //  (numeric) Total fees of transactions in the mempool
     maxmempool: number;              //  (numeric) Maximum memory usage for the mempool
     mempoolminfee: number;           //  (numeric) Minimum fee rate in BTC/kB for tx to be accepted.
     minrelaytxfee: number;           //  (numeric) Current minimum relay fee for transactions
@@ -72,7 +73,15 @@ export namespace IBitcoinApi {
     time: number;                    //  (numeric) Same as blocktime
   }
 
-  interface Vin {
+  export interface VerboseBlock extends Block {
+    tx: VerboseTransaction[];        // The transactions in the format of the getrawtransaction RPC. Different from verbosity = 1 "tx" result
+  }
+
+  export interface VerboseTransaction extends Transaction {
+    fee?: number;                   //  (numeric) The transaction fee in BTC, omitted if block undo data is not available
+  }
+
+  export interface Vin {
     txid?: string;                   //  (string) The transaction id
     vout?: number;                   //  (string)
     scriptSig?: {                    //  (json object) The script
@@ -82,28 +91,36 @@ export namespace IBitcoinApi {
     sequence: number;                //  (numeric) The script sequence number
     txinwitness?: string[];          //  (string) hex-encoded witness data
     coinbase?: string;
+    is_pegin?: boolean;              //  (boolean) Elements peg-in
   }
 
-  interface Vout {
+  export interface Vout {
     value: number;                   //  (numeric) The value in BTC
     n: number;                       //  (numeric) index
+    asset?: string;                  //  (string) Elements asset id
     scriptPubKey: {                  //  (json object)
       asm: string;                   //  (string) the asm
       hex: string;                   //  (string) the hex
-      reqSigs: number;               //  (numeric) The required sigs
+      reqSigs?: number;              //  (numeric) The required sigs
       type: string;                  //  (string) The type, eg 'pubkeyhash'
-      addresses: string[]            //  (string) bitcoin address
+      address?: string;              //  (string) bitcoin address
+      addresses?: string[];           //  (string) bitcoin addresses
+      pegout_chain?: string;         //  (string) Elements peg-out chain
+      pegout_addresses?: string[];   //  (string) Elements peg-out addresses
     };
   }
 
   export interface AddressInformation {
     isvalid: boolean;                //  (boolean) If the address is valid or not. If not, this is the only property returned.
+    isvalid_parent?: boolean;        //  (boolean) Elements only
     address: string;                 //  (string) The bitcoin address validated
     scriptPubKey: string;            //  (string) The hex-encoded scriptPubKey generated by the address
     isscript: boolean;               //  (boolean) If the key is a script
     iswitness: boolean;              //  (boolean) If the address is a witness
-    witness_version?: boolean;       //  (numeric, optional) The version number of the witness program
+    witness_version?: number;        //  (numeric, optional) The version number of the witness program
     witness_program: string;         //  (string, optional) The hex value of the witness program
+    confidential_key?: string;       //  (string) Elements only
+    unconfidential?: string;         //  (string) Elements only
   }
 
   export interface ChainTips {
@@ -113,4 +130,77 @@ export namespace IBitcoinApi {
     status: 'invalid' | 'headers-only' | 'valid-headers' | 'valid-fork' | 'active';
   }
 
+  export interface BlockchainInfo {
+    chain: number;                   // (string) current network name as defined in BIP70 (main, test, regtest)
+    blocks: number;                  // (numeric) the current number of blocks processed in the server
+    headers: number;                 // (numeric) the current number of headers we have validated
+    bestblockhash: string,           // (string) the hash of the currently best block
+    difficulty: number;              // (numeric) the current difficulty
+    mediantime: number;              // (numeric) median time for the current best block
+    verificationprogress: number;    // (numeric) estimate of verification progress [0..1]
+    initialblockdownload: boolean;   // (bool) (debug information) estimate of whether this node is in Initial Block Download mode.
+    chainwork: string                // (string) total amount of work in active chain, in hexadecimal
+    size_on_disk: number;            // (numeric) the estimated size of the block and undo files on disk
+    pruned: number;                  // (boolean) if the blocks are subject to pruning
+    pruneheight: number;             // (numeric) lowest-height complete block stored (only present if pruning is enabled)
+    automatic_pruning: number;       // (boolean) whether automatic pruning is enabled (only present if pruning is enabled)
+    prune_target_size: number;       // (numeric) the target size used by pruning (only present if automatic pruning is enabled)
+    softforks: SoftFork[];           // (array) status of softforks in progress
+    bip9_softforks: { [name: string]: Bip9SoftForks[] } // (object) status of BIP9 softforks in progress
+    warnings: string;                // (string) any network and blockchain warnings.
+  }
+
+  interface SoftFork {
+    id: string;                      // (string) name of softfork
+    version: number;                 // (numeric) block version
+    reject: {                        // (object) progress toward rejecting pre-softfork blocks
+      status: boolean;               // (boolean) true if threshold reached
+    },
+  }
+  interface Bip9SoftForks {
+    status: number;                  // (string) one of defined, started, locked_in, active, failed
+    bit: number;                     // (numeric) the bit (0-28) in the block version field used to signal this softfork (only for started status)
+    startTime: number;               // (numeric) the minimum median time past of a block at which the bit gains its meaning
+    timeout: number;                 // (numeric) the median time past of a block at which the deployment is considered failed if not yet locked in
+    since: number;                   // (numeric) height of the first block to which the status applies
+    statistics: {                    // (object) numeric statistics about BIP9 signalling for a softfork (only for started status)
+      period: number;                // (numeric) the length in blocks of the BIP9 signalling period 
+      threshold: number;             // (numeric) the number of blocks with the version bit set required to activate the feature 
+      elapsed: number;               // (numeric) the number of blocks elapsed since the beginning of the current period 
+      count: number;                 // (numeric) the number of blocks with the version bit set in the current period 
+      possible: boolean;             // (boolean) returns false if there are not enough blocks left in this period to pass activation threshold 
+    }
+  }
+
+  export interface BlockStats {
+    "avgfee": number;
+    "avgfeerate": number;
+    "avgtxsize": number;
+    "blockhash": string;
+    "feerate_percentiles": [number, number, number, number, number];
+    "height": number;
+    "ins": number;
+    "maxfee": number;
+    "maxfeerate": number;
+    "maxtxsize": number;
+    "medianfee": number;
+    "mediantime": number;
+    "mediantxsize": number;
+    "minfee": number;
+    "minfeerate": number;
+    "mintxsize": number;
+    "outs": number;
+    "subsidy": number;
+    "swtotal_size": number;
+    "swtotal_weight": number;
+    "swtxs": number;
+    "time": number;
+    "total_out": number;
+    "total_size": number;
+    "total_weight": number;
+    "totalfee": number;
+    "txs": number;
+    "utxo_increase": number;
+    "utxo_size_inc": number;
+  }
 }

backend/src/api/bitcoin/bitcoin-api.ts

@@ -1,5 +1,3 @@
-import config from '../../config';
-import * as bitcoin from '@mempool/bitcoin';
 import * as bitcoinjs from 'bitcoinjs-lib';
 import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
 import { IBitcoinApi } from './bitcoin-api.interface';
@@ -10,45 +8,73 @@ import { TransactionExtended } from '../../mempool.interfaces';
 
 class BitcoinApi implements AbstractBitcoinApi {
   private rawMempoolCache: IBitcoinApi.RawMempool | null = null;
-  private bitcoindClient: any;
+  protected bitcoindClient: any;
 
-  constructor() {
-    this.bitcoindClient = new bitcoin.Client({
-      host: config.CORE_RPC.HOST,
-      port: config.CORE_RPC.PORT,
-      user: config.CORE_RPC.USERNAME,
-      pass: config.CORE_RPC.PASSWORD,
-      timeout: 60000,
-    });
+  constructor(bitcoinClient: any) {
+    this.bitcoindClient = bitcoinClient;
   }
 
-  $getRawTransaction(txId: string, skipConversion = false, addPrevout = false): Promise<IEsploraApi.Transaction> {
+  static convertBlock(block: IBitcoinApi.Block): IEsploraApi.Block {
+    return {
+      id: block.hash,
+      height: block.height,
+      version: block.version,
+      timestamp: block.time,
+      bits: parseInt(block.bits, 16),
+      nonce: block.nonce,
+      difficulty: block.difficulty,
+      merkle_root: block.merkleroot,
+      tx_count: block.nTx,
+      size: block.size,
+      weight: block.weight,
+      previousblockhash: block.previousblockhash,
+      mediantime: block.mediantime,
+    };
+  }
+
+
+  $getRawTransaction(txId: string, skipConversion = false, addPrevout = false, lazyPrevouts = false): Promise<IEsploraApi.Transaction> {
     // If the transaction is in the mempool we already converted and fetched the fee. Only prevouts are missing
     const txInMempool = mempool.getMempool()[txId];
     if (txInMempool && addPrevout) {
       return this.$addPrevouts(txInMempool);
     }
 
-    // Special case to fetch the Coinbase transaction
-    if (txId === '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b') {
-      return this.$returnCoinbaseTransaction();
-    }
-
     return this.bitcoindClient.getRawTransaction(txId, true)
       .then((transaction: IBitcoinApi.Transaction) => {
         if (skipConversion) {
           transaction.vout.forEach((vout) => {
-            vout.value = vout.value * 100000000;
+            vout.value = Math.round(vout.value * 100000000);
           });
           return transaction;
         }
-        return this.$convertTransaction(transaction, addPrevout);
+        return this.$convertTransaction(transaction, addPrevout, lazyPrevouts);
+      })
+      .catch((e: Error) => {
+        if (e.message.startsWith('The genesis block coinbase')) {
+          return this.$returnCoinbaseTransaction();
+        }
+        throw e;
       });
   }
 
+  $getTransactionHex(txId: string): Promise<string> {
+    return this.$getRawTransaction(txId, true)
+      .then((tx) => tx.hex || '');
+  }
+
   $getBlockHeightTip(): Promise<number> {
     return this.bitcoindClient.getChainTips()
-      .then((result: IBitcoinApi.ChainTips[]) => result[0].height);
+      .then((result: IBitcoinApi.ChainTips[]) => {
+        return result.find(tip => tip.status === 'active')!.height;
+      });
+  }
+
+  $getBlockHashTip(): Promise<string> {
+    return this.bitcoindClient.getChainTips()
+      .then((result: IBitcoinApi.ChainTips[]) => {
+        return result.find(tip => tip.status === 'active')!.hash;
+      });
   }
 
   $getTxIdsForBlock(hash: string): Promise<string[]> {
@@ -56,10 +82,19 @@ class BitcoinApi implements AbstractBitcoinApi {
       .then((rpcBlock: IBitcoinApi.Block) => rpcBlock.tx);
   }
 
+  $getRawBlock(hash: string): Promise<Buffer> {
+    return this.bitcoindClient.getBlock(hash, 0)
+      .then((raw: string) => Buffer.from(raw, "hex"));
+  }
+
   $getBlockHash(height: number): Promise<string> {
     return this.bitcoindClient.getBlockHash(height);
   }
 
+  $getBlockHeader(hash: string): Promise<string> {
+    return this.bitcoindClient.getBlockHeader(hash, false);
+  }
+
   async $getBlock(hash: string): Promise<IEsploraApi.Block> {
     const foundBlock = blocks.getBlocks().find((block) => block.id === hash);
     if (foundBlock) {
@@ -67,7 +102,7 @@ class BitcoinApi implements AbstractBitcoinApi {
     }
 
     return this.bitcoindClient.getBlock(hash)
-      .then((block: IBitcoinApi.Block) => this.convertBlock(block));
+      .then((block: IBitcoinApi.Block) => BitcoinApi.convertBlock(block));
   }
 
   $getAddress(address: string): Promise<IEsploraApi.Address> {
@@ -83,22 +118,68 @@ class BitcoinApi implements AbstractBitcoinApi {
   }
 
   $getAddressPrefix(prefix: string): string[] {
-    const found: string[] = [];
+    const found: { [address: string]: string } = {};
     const mp = mempool.getMempool();
     for (const tx in mp) {
       for (const vout of mp[tx].vout) {
         if (vout.scriptpubkey_address.indexOf(prefix) === 0) {
-          found.push(vout.scriptpubkey_address);
-          if (found.length >= 10) {
-            return found;
+          found[vout.scriptpubkey_address] = '';
+          if (Object.keys(found).length >= 10) {
+            return Object.keys(found);
           }
         }
       }
     }
-    return found;
+    return Object.keys(found);
   }
 
-  protected async $convertTransaction(transaction: IBitcoinApi.Transaction, addPrevout: boolean): Promise<IEsploraApi.Transaction> {
+  $sendRawTransaction(rawTransaction: string): Promise<string> {
+    return this.bitcoindClient.sendRawTransaction(rawTransaction);
+  }
+
+  async $getOutspend(txId: string, vout: number): Promise<IEsploraApi.Outspend> {
+    const txOut = await this.bitcoindClient.getTxOut(txId, vout, false);
+    return {
+      spent: txOut === null,
+      status: {
+        confirmed: true,
+      }
+    };
+  }
+
+  async $getOutspends(txId: string): Promise<IEsploraApi.Outspend[]> {
+    const outSpends: IEsploraApi.Outspend[] = [];
+    const tx = await this.$getRawTransaction(txId, true, false);
+    for (let i = 0; i < tx.vout.length; i++) {
+      if (tx.status && tx.status.block_height === 0) {
+        outSpends.push({
+          spent: false
+        });
+      } else {
+        const txOut = await this.bitcoindClient.getTxOut(txId, i);
+        outSpends.push({
+          spent: txOut === null,
+        });
+      }
+    }
+    return outSpends;
+  }
+
+  async $getBatchedOutspends(txId: string[]): Promise<IEsploraApi.Outspend[][]> {
+    const outspends: IEsploraApi.Outspend[][] = [];
+    for (const tx of txId) {
+      const outspend = await this.$getOutspends(tx);
+      outspends.push(outspend);
+    }
+    return outspends;
+  }
+
+  $getEstimatedHashrate(blockHeight: number): Promise<number> {
+    // 120 is the default block span in Core
+    return this.bitcoindClient.getNetworkHashPs(120, blockHeight);
+  }
+
+  protected async $convertTransaction(transaction: IBitcoinApi.Transaction, addPrevout: boolean, lazyPrevouts = false): Promise<IEsploraApi.Transaction> {
     let esploraTransaction: IEsploraApi.Transaction = {
       txid: transaction.txid,
       version: transaction.version,
@@ -113,10 +194,11 @@ class BitcoinApi implements AbstractBitcoinApi {
 
     esploraTransaction.vout = transaction.vout.map((vout) => {
       return {
-        value: vout.value * 100000000,
+        value: Math.round(vout.value * 100000000),
         scriptpubkey: vout.scriptPubKey.hex,
-        scriptpubkey_address: vout.scriptPubKey && vout.scriptPubKey.addresses ? vout.scriptPubKey.addresses[0] : '',
-        scriptpubkey_asm: vout.scriptPubKey.asm ? this.convertScriptSigAsm(vout.scriptPubKey.asm) : '',
+        scriptpubkey_address: vout.scriptPubKey && vout.scriptPubKey.address ? vout.scriptPubKey.address
+          : vout.scriptPubKey.addresses ? vout.scriptPubKey.addresses[0] : '',
+        scriptpubkey_asm: vout.scriptPubKey.asm ? this.convertScriptSigAsm(vout.scriptPubKey.hex) : '',
         scriptpubkey_type: this.translateScriptPubKeyType(vout.scriptPubKey.type),
       };
     });
@@ -126,11 +208,13 @@ class BitcoinApi implements AbstractBitcoinApi {
         is_coinbase: !!vin.coinbase,
         prevout: null,
         scriptsig: vin.scriptSig && vin.scriptSig.hex || vin.coinbase || '',
-        scriptsig_asm: vin.scriptSig && this.convertScriptSigAsm(vin.scriptSig.asm) || '',
+        scriptsig_asm: vin.scriptSig && this.convertScriptSigAsm(vin.scriptSig.hex) || '',
         sequence: vin.sequence,
         txid: vin.txid || '',
         vout: vin.vout || 0,
-        witness: vin.txinwitness,
+        witness: vin.txinwitness || [],
+        inner_redeemscript_asm: '',
+        inner_witnessscript_asm: '',
       };
     });
 
@@ -143,35 +227,15 @@ class BitcoinApi implements AbstractBitcoinApi {
       };
     }
 
-    if (transaction.confirmations) {
-      esploraTransaction = await this.$calculateFeeFromInputs(esploraTransaction, addPrevout);
-    } else {
+    if (addPrevout) {
+      esploraTransaction = await this.$calculateFeeFromInputs(esploraTransaction, false, lazyPrevouts);
+    } else if (!transaction.confirmations) {
       esploraTransaction = await this.$appendMempoolFeeData(esploraTransaction);
-      if (addPrevout) {
-        esploraTransaction = await this.$calculateFeeFromInputs(esploraTransaction, addPrevout);
-      }
     }
 
     return esploraTransaction;
   }
 
-  private convertBlock(block: IBitcoinApi.Block): IEsploraApi.Block {
-    return {
-      id: block.hash,
-      height: block.height,
-      version: block.version,
-      timestamp: block.time,
-      bits: parseInt(block.bits, 16),
-      nonce: block.nonce,
-      difficulty: block.difficulty,
-      merkle_root: block.merkleroot,
-      tx_count: block.nTx,
-      size: block.size,
-      weight: block.weight,
-      previousblockhash: block.previousblockhash,
-    };
-  }
-
   private translateScriptPubKeyType(outputType: string): string {
     const map = {
       'pubkey': 'p2pk',
@@ -181,13 +245,14 @@ class BitcoinApi implements AbstractBitcoinApi {
       'witness_v0_scripthash': 'v0_p2wsh',
       'witness_v1_taproot': 'v1_p2tr',
       'nonstandard': 'nonstandard',
+      'multisig': 'multisig',
       'nulldata': 'op_return'
     };
 
     if (map[outputType]) {
       return map[outputType];
     } else {
-      return '';
+      return 'unknown';
     }
   }
 
@@ -204,7 +269,7 @@ class BitcoinApi implements AbstractBitcoinApi {
     } else {
       mempoolEntry = await this.$getMempoolEntry(transaction.txid);
     }
-    transaction.fee = mempoolEntry.fees.base * 100000000;
+    transaction.fee = Math.round(mempoolEntry.fees.base * 100000000);
     return transaction;
   }
 
@@ -213,7 +278,7 @@ class BitcoinApi implements AbstractBitcoinApi {
       if (vin.prevout) {
         continue;
       }
-      const innerTx = await this.$getRawTransaction(vin.txid, false);
+      const innerTx = await this.$getRawTransaction(vin.txid, false, false);
       vin.prevout = innerTx.vout[vin.vout];
       this.addInnerScriptsToVin(vin);
     }
@@ -221,16 +286,14 @@ class BitcoinApi implements AbstractBitcoinApi {
   }
 
   protected $returnCoinbaseTransaction(): Promise<IEsploraApi.Transaction> {
-    return this.bitcoindClient.getBlock('000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f', 2)
-      .then((block: IBitcoinApi.Block) => {
-        return this.$convertTransaction(Object.assign(block.tx[0], {
-          confirmations: blocks.getCurrentBlockHeight() + 1,
-          blocktime: 1231006505 }), false);
-      });
-  }
-
-  protected $validateAddress(address: string): Promise<IBitcoinApi.AddressInformation> {
-    return this.bitcoindClient.validateAddress(address);
+    return this.bitcoindClient.getBlockHash(0).then((hash: string) =>
+      this.bitcoindClient.getBlock(hash, 2)
+        .then((block: IBitcoinApi.Block) => {
+          return this.$convertTransaction(Object.assign(block.tx[0], {
+            confirmations: blocks.getCurrentBlockHeight() + 1,
+            blocktime: block.time }), false);
+        })
+    );
   }
 
   private $getMempoolEntry(txid: string): Promise<IBitcoinApi.MempoolEntry> {
@@ -241,42 +304,95 @@ class BitcoinApi implements AbstractBitcoinApi {
     return this.bitcoindClient.getRawMemPool(true);
   }
 
-  private async $calculateFeeFromInputs(transaction: IEsploraApi.Transaction, addPrevout: boolean): Promise<IEsploraApi.Transaction> {
+
+  private async $calculateFeeFromInputs(transaction: IEsploraApi.Transaction, addPrevout: boolean, lazyPrevouts: boolean): Promise<IEsploraApi.Transaction> {
     if (transaction.vin[0].is_coinbase) {
       transaction.fee = 0;
       return transaction;
     }
     let totalIn = 0;
-    for (const vin of transaction.vin) {
-      const innerTx = await this.$getRawTransaction(vin.txid, !addPrevout);
-      if (addPrevout) {
-        vin.prevout = innerTx.vout[vin.vout];
-        this.addInnerScriptsToVin(vin);
+
+    for (let i = 0; i < transaction.vin.length; i++) {
+      if (lazyPrevouts && i > 12) {
+        transaction.vin[i].lazy = true;
+        continue;
       }
-      totalIn += innerTx.vout[vin.vout].value;
+      const innerTx = await this.$getRawTransaction(transaction.vin[i].txid, false, false);
+      transaction.vin[i].prevout = innerTx.vout[transaction.vin[i].vout];
+      this.addInnerScriptsToVin(transaction.vin[i]);
+      totalIn += innerTx.vout[transaction.vin[i].vout].value;
+    }
+    if (lazyPrevouts && transaction.vin.length > 12) {
+      transaction.fee = -1;
+    } else {
+      const totalOut = transaction.vout.reduce((p, output) => p + output.value, 0);
+      transaction.fee = parseFloat((totalIn - totalOut).toFixed(8));
     }
-    const totalOut = transaction.vout.reduce((p, output) => p + output.value, 0);
-    transaction.fee = parseFloat((totalIn - totalOut).toFixed(8));
     return transaction;
   }
 
-  private convertScriptSigAsm(str: string): string {
-    const a = str.split(' ');
+  private convertScriptSigAsm(hex: string): string {
+    const buf = Buffer.from(hex, 'hex');
+
     const b: string[] = [];
-    a.forEach((chunk) => {
-      if (chunk.substr(0, 3) === 'OP_') {
-        chunk = chunk.replace(/^OP_(\d+)/, 'OP_PUSHNUM_$1');
-        chunk = chunk.replace('OP_CHECKSEQUENCEVERIFY', 'OP_CSV');
-        b.push(chunk);
-      } else {
-        chunk = chunk.replace('[ALL]', '01');
-        if (chunk === '0') {
-          b.push('OP_0');
+
+    let i = 0;
+    while (i < buf.length) {
+      const op = buf[i];
+      if (op >= 0x01 && op <= 0x4e) {
+        i++;
+        let push: number;
+        if (op === 0x4c) {
+          push = buf.readUInt8(i);
+          b.push('OP_PUSHDATA1');
+          i += 1;
+        } else if (op === 0x4d) {
+          push = buf.readUInt16LE(i);
+          b.push('OP_PUSHDATA2');
+          i += 2;
+        } else if (op === 0x4e) {
+          push = buf.readUInt32LE(i);
+          b.push('OP_PUSHDATA4');
+          i += 4;
         } else {
-          b.push('OP_PUSHBYTES_' + Math.round(chunk.length / 2) + ' ' + chunk);
+          push = op;
+          b.push('OP_PUSHBYTES_' + push);
         }
+
+        const data = buf.slice(i, i + push);
+        if (data.length !== push) {
+          break;
+        }
+
+        b.push(data.toString('hex'));
+        i += data.length;
+      } else {
+        if (op === 0x00) {
+          b.push('OP_0');
+        } else if (op === 0x4f) {
+          b.push('OP_PUSHNUM_NEG1');
+        } else if (op === 0xb1) {
+          b.push('OP_CLTV');
+        } else if (op === 0xb2) {
+          b.push('OP_CSV');
+        } else if (op === 0xba) {
+          b.push('OP_CHECKSIGADD');
+        } else {
+          const opcode = bitcoinjs.script.toASM([ op ]);
+          if (opcode && op < 0xfd) {
+            if (/^OP_(\d+)$/.test(opcode)) {
+              b.push(opcode.replace(/^OP_(\d+)$/, 'OP_PUSHNUM_$1'));
+            } else {
+              b.push(opcode);
+            }
+          } else {
+            b.push('OP_RETURN_' + op);
+          }
+        }
+        i += 1;
       }
-    });
+    }
+
     return b.join(' ');
   }
 
@@ -287,16 +403,21 @@ class BitcoinApi implements AbstractBitcoinApi {
 
     if (vin.prevout.scriptpubkey_type === 'p2sh') {
       const redeemScript = vin.scriptsig_asm.split(' ').reverse()[0];
-      vin.inner_redeemscript_asm = this.convertScriptSigAsm(bitcoinjs.script.toASM(Buffer.from(redeemScript, 'hex')));
+      vin.inner_redeemscript_asm = this.convertScriptSigAsm(redeemScript);
       if (vin.witness && vin.witness.length > 2) {
         const witnessScript = vin.witness[vin.witness.length - 1];
-        vin.inner_witnessscript_asm = this.convertScriptSigAsm(bitcoinjs.script.toASM(Buffer.from(witnessScript, 'hex')));
+        vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript);
       }
     }
 
     if (vin.prevout.scriptpubkey_type === 'v0_p2wsh' && vin.witness) {
       const witnessScript = vin.witness[vin.witness.length - 1];
-      vin.inner_witnessscript_asm = this.convertScriptSigAsm(bitcoinjs.script.toASM(Buffer.from(witnessScript, 'hex')));
+      vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript);
+    }
+
+    if (vin.prevout.scriptpubkey_type === 'v1_p2tr' && vin.witness && vin.witness.length > 1) {
+      const witnessScript = vin.witness[vin.witness.length - 2];
+      vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript);
     }
   }
 

backend/src/api/bitcoin/bitcoin-base.api.ts

@@ -1,45 +0,0 @@
-import config from '../../config';
-import * as bitcoin from '@mempool/bitcoin';
-import { IBitcoinApi } from './bitcoin-api.interface';
-
-class BitcoinBaseApi {
-  bitcoindClient: any;
-  bitcoindClientMempoolInfo: any;
-
-  constructor() {
-    this.bitcoindClient = new bitcoin.Client({
-      host: config.CORE_RPC.HOST,
-      port: config.CORE_RPC.PORT,
-      user: config.CORE_RPC.USERNAME,
-      pass: config.CORE_RPC.PASSWORD,
-      timeout: 60000,
-    });
-
-    if (config.CORE_RPC_MINFEE.ENABLED) {
-      this.bitcoindClientMempoolInfo = new bitcoin.Client({
-        host: config.CORE_RPC_MINFEE.HOST,
-        port: config.CORE_RPC_MINFEE.PORT,
-        user: config.CORE_RPC_MINFEE.USERNAME,
-        pass: config.CORE_RPC_MINFEE.PASSWORD,
-        timeout: 60000,
-      });
-    }
-  }
-
-  $getMempoolInfo(): Promise<IBitcoinApi.MempoolInfo> {
-    if (config.CORE_RPC_MINFEE.ENABLED) {
-      return Promise.all([
-        this.bitcoindClient.getMempoolInfo(),
-        this.bitcoindClientMempoolInfo.getMempoolInfo()
-      ]).then(([mempoolInfo, secondMempoolInfo]) => {
-        mempoolInfo.maxmempool = secondMempoolInfo.maxmempool;
-        mempoolInfo.mempoolminfee = secondMempoolInfo.mempoolminfee;
-        mempoolInfo.minrelaytxfee = secondMempoolInfo.minrelaytxfee;
-        return mempoolInfo;
-      });
-    }
-    return this.bitcoindClient.getMempoolInfo();
-  }
-}
-
-export default new BitcoinBaseApi();

backend/src/api/bitcoin/bitcoin-client.ts

@@ -0,0 +1,13 @@
+import config from '../../config';
+const bitcoin = require('../../rpc-api/index');
+import { BitcoinRpcCredentials } from './bitcoin-api-abstract-factory';
+
+const nodeRpcCredentials: BitcoinRpcCredentials = {
+  host: config.CORE_RPC.HOST,
+  port: config.CORE_RPC.PORT,
+  user: config.CORE_RPC.USERNAME,
+  pass: config.CORE_RPC.PASSWORD,
+  timeout: config.CORE_RPC.TIMEOUT,
+};
+
+export default new bitcoin.Client(nodeRpcCredentials);

backend/src/api/bitcoin/bitcoin-second-client.ts

@@ -0,0 +1,13 @@
+import config from '../../config';
+const bitcoin = require('../../rpc-api/index');
+import { BitcoinRpcCredentials } from './bitcoin-api-abstract-factory';
+
+const nodeRpcCredentials: BitcoinRpcCredentials = {
+  host: config.SECOND_CORE_RPC.HOST,
+  port: config.SECOND_CORE_RPC.PORT,
+  user: config.SECOND_CORE_RPC.USERNAME,
+  pass: config.SECOND_CORE_RPC.PASSWORD,
+  timeout: config.SECOND_CORE_RPC.TIMEOUT,
+};
+
+export default new bitcoin.Client(nodeRpcCredentials);

backend/src/api/bitcoin/bitcoin.routes.ts

@@ -0,0 +1,718 @@
+import { Application, Request, Response } from 'express';
+import axios from 'axios';
+import * as bitcoinjs from 'bitcoinjs-lib';
+import config from '../../config';
+import websocketHandler from '../websocket-handler';
+import mempool from '../mempool';
+import feeApi from '../fee-api';
+import mempoolBlocks from '../mempool-blocks';
+import bitcoinApi, { bitcoinCoreApi } from './bitcoin-api-factory';
+import { Common } from '../common';
+import backendInfo from '../backend-info';
+import transactionUtils from '../transaction-utils';
+import { IEsploraApi } from './esplora-api.interface';
+import loadingIndicators from '../loading-indicators';
+import { TransactionExtended } from '../../mempool.interfaces';
+import logger from '../../logger';
+import blocks from '../blocks';
+import bitcoinClient from './bitcoin-client';
+import difficultyAdjustment from '../difficulty-adjustment';
+import transactionRepository from '../../repositories/TransactionRepository';
+import rbfCache from '../rbf-cache';
+
+class BitcoinRoutes {
+  public initRoutes(app: Application) {
+    app
+      .get(config.MEMPOOL.API_URL_PREFIX + 'transaction-times', this.getTransactionTimes)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'outspends', this.$getBatchedOutspends)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'cpfp/:txId', this.$getCpfpInfo)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'difficulty-adjustment', this.getDifficultyChange)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'fees/recommended', this.getRecommendedFees)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'fees/mempool-blocks', this.getMempoolBlocks)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'backend-info', this.getBackendInfo)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'init-data', this.getInitData)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'validate-address/:address', this.validateAddress)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/replaces', this.getRbfHistory)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/cached', this.getCachedTx)
+      .post(config.MEMPOOL.API_URL_PREFIX + 'tx/push', this.$postTransactionForm)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'donations', async (req, res) => {
+        try {
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/donations`, { responseType: 'stream', timeout: 10000 });
+          response.data.pipe(res);
+        } catch (e) {
+          res.status(500).end();
+        }
+      })
+      .get(config.MEMPOOL.API_URL_PREFIX + 'donations/images/:id', async (req, res) => {
+        try {
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/donations/images/${req.params.id}`, {
+            responseType: 'stream', timeout: 10000
+          });
+          response.data.pipe(res);
+        } catch (e) {
+          res.status(500).end();
+        }
+      })
+      .get(config.MEMPOOL.API_URL_PREFIX + 'contributors', async (req, res) => {
+        try {
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/contributors`, { responseType: 'stream', timeout: 10000 });
+          response.data.pipe(res);
+        } catch (e) {
+          res.status(500).end();
+        }
+      })
+      .get(config.MEMPOOL.API_URL_PREFIX + 'contributors/images/:id', async (req, res) => {
+        try {
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/contributors/images/${req.params.id}`, {
+            responseType: 'stream', timeout: 10000
+          });
+          response.data.pipe(res);
+        } catch (e) {
+          res.status(500).end();
+        }
+      })
+      .get(config.MEMPOOL.API_URL_PREFIX + 'translators', async (req, res) => {
+        try {
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/translators`, { responseType: 'stream', timeout: 10000 });
+          response.data.pipe(res);
+        } catch (e) {
+          res.status(500).end();
+        }
+      })
+      .get(config.MEMPOOL.API_URL_PREFIX + 'translators/images/:id', async (req, res) => {
+        try {
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/translators/images/${req.params.id}`, {
+            responseType: 'stream', timeout: 10000
+          });
+          response.data.pipe(res);
+        } catch (e) {
+          res.status(500).end();
+        }
+      })
+      .get(config.MEMPOOL.API_URL_PREFIX + 'blocks', this.getBlocks.bind(this))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/:height', this.getBlocks.bind(this))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash', this.getBlock)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/summary', this.getStrippedBlockTransactions)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/audit-summary', this.getBlockAuditSummary)
+      .post(config.MEMPOOL.API_URL_PREFIX + 'psbt/addparents', this.postPsbtCompletion)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from', this.getBlocksByBulk.bind(this))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from/:to', this.getBlocksByBulk.bind(this))
+      ;
+
+      if (config.MEMPOOL.BACKEND !== 'esplora') {
+        app
+          .get(config.MEMPOOL.API_URL_PREFIX + 'mempool', this.getMempool)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'mempool/txids', this.getMempoolTxIds)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'mempool/recent', this.getRecentMempoolTransactions)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId', this.getTransaction)
+          .post(config.MEMPOOL.API_URL_PREFIX + 'tx', this.$postTransaction)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/hex', this.getRawTransaction)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/status', this.getTransactionStatus)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/outspends', this.getTransactionOutspends)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/header', this.getBlockHeader)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/height', this.getBlockTipHeight)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/hash', this.getBlockTipHash)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/raw', this.getRawBlock)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txids', this.getTxIdsForBlock)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txs', this.getBlockTransactions)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txs/:index', this.getBlockTransactions)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'block-height/:height', this.getBlockHeight)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address', this.getAddress)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs', this.getAddressTransactions)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs/chain/:txId', this.getAddressTransactions)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'address-prefix/:prefix', this.getAddressPrefix)
+          ;
+      }
+  }
+
+
+  private getInitData(req: Request, res: Response) {
+    try {
+      const result = websocketHandler.getInitData();
+      res.json(result);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private getRecommendedFees(req: Request, res: Response) {
+    if (!mempool.isInSync()) {
+      res.statusCode = 503;
+      res.send('Service Unavailable');
+      return;
+    }
+    const result = feeApi.getRecommendedFee();
+    res.json(result);
+  }
+
+  private getMempoolBlocks(req: Request, res: Response) {
+    try {
+      const result = mempoolBlocks.getMempoolBlocks();
+      res.json(result);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private getTransactionTimes(req: Request, res: Response) {
+    if (!Array.isArray(req.query.txId)) {
+      res.status(500).send('Not an array');
+      return;
+    }
+    const txIds: string[] = [];
+    for (const _txId in req.query.txId) {
+      if (typeof req.query.txId[_txId] === 'string') {
+        txIds.push(req.query.txId[_txId].toString());
+      }
+    }
+
+    const times = mempool.getFirstSeenForTransactions(txIds);
+    res.json(times);
+  }
+
+  private async $getBatchedOutspends(req: Request, res: Response) {
+    if (!Array.isArray(req.query.txId)) {
+      res.status(500).send('Not an array');
+      return;
+    }
+    if (req.query.txId.length > 50) {
+      res.status(400).send('Too many txids requested');
+      return;
+    }
+    const txIds: string[] = [];
+    for (const _txId in req.query.txId) {
+      if (typeof req.query.txId[_txId] === 'string') {
+        txIds.push(req.query.txId[_txId].toString());
+      }
+    }
+
+    try {
+      const batchedOutspends = await bitcoinApi.$getBatchedOutspends(txIds);
+      res.json(batchedOutspends);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getCpfpInfo(req: Request, res: Response) {
+    if (!/^[a-fA-F0-9]{64}$/.test(req.params.txId)) {
+      res.status(501).send(`Invalid transaction ID.`);
+      return;
+    }
+
+    const tx = mempool.getMempool()[req.params.txId];
+    if (tx) {
+      if (tx?.cpfpChecked) {
+        res.json({
+          ancestors: tx.ancestors,
+          bestDescendant: tx.bestDescendant || null,
+          descendants: tx.descendants || null,
+          effectiveFeePerVsize: tx.effectiveFeePerVsize || null,
+        });
+        return;
+      }
+
+      const cpfpInfo = Common.setRelativesAndGetCpfpInfo(tx, mempool.getMempool());
+
+      res.json(cpfpInfo);
+      return;
+    } else {
+      let cpfpInfo;
+      if (config.DATABASE.ENABLED) {
+        cpfpInfo = await transactionRepository.$getCpfpInfo(req.params.txId);
+      }
+      if (cpfpInfo) {
+        res.json(cpfpInfo);
+        return;
+      } else {
+        res.json({
+          ancestors: []
+        });
+        return;
+      }
+    }
+  }
+
+  private getBackendInfo(req: Request, res: Response) {
+    res.json(backendInfo.getBackendInfo());
+  }
+
+  private async getTransaction(req: Request, res: Response) {
+    try {
+      const transaction = await transactionUtils.$getTransactionExtended(req.params.txId, true);
+      res.json(transaction);
+    } catch (e) {
+      let statusCode = 500;
+      if (e instanceof Error && e instanceof Error && e.message && e.message.indexOf('No such mempool or blockchain transaction') > -1) {
+        statusCode = 404;
+      }
+      res.status(statusCode).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getRawTransaction(req: Request, res: Response) {
+    try {
+      const transaction: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(req.params.txId, true);
+      res.setHeader('content-type', 'text/plain');
+      res.send(transaction.hex);
+    } catch (e) {
+      let statusCode = 500;
+      if (e instanceof Error && e.message && e.message.indexOf('No such mempool or blockchain transaction') > -1) {
+        statusCode = 404;
+      }
+      res.status(statusCode).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  /**
+   * Takes the PSBT as text/plain body, parses it, and adds the full
+   * parent transaction to each input that doesn't already have it.
+   * This is used for BTCPayServer / Trezor users which need access to
+   * the full parent transaction even with segwit inputs.
+   * It will respond with a text/plain PSBT in the same format (hex|base64).
+   */
+  private async postPsbtCompletion(req: Request, res: Response): Promise<void> {
+    res.setHeader('content-type', 'text/plain');
+    const notFoundError = `Couldn't get transaction hex for parent of input`;
+    try {
+      let psbt: bitcoinjs.Psbt;
+      let format: 'hex' | 'base64';
+      let isModified = false;
+      try {
+        psbt = bitcoinjs.Psbt.fromBase64(req.body);
+        format = 'base64';
+      } catch (e1) {
+        try {
+          psbt = bitcoinjs.Psbt.fromHex(req.body);
+          format = 'hex';
+        } catch (e2) {
+          throw new Error(`Unable to parse PSBT`);
+        }
+      }
+      for (const [index, input] of psbt.data.inputs.entries()) {
+        if (!input.nonWitnessUtxo) {
+          // Buffer.from ensures it won't be modified in place by reverse()
+          const txid = Buffer.from(psbt.txInputs[index].hash)
+            .reverse()
+            .toString('hex');
+
+          let transactionHex: string;
+          // If missing transaction, return 404 status error
+          try {
+            transactionHex = await bitcoinApi.$getTransactionHex(txid);
+            if (!transactionHex) {
+              throw new Error('');
+            }
+          } catch (err) {
+            throw new Error(`${notFoundError} #${index} @ ${txid}`);
+          }
+
+          psbt.updateInput(index, {
+            nonWitnessUtxo: Buffer.from(transactionHex, 'hex'),
+          });
+          if (!isModified) {
+            isModified = true;
+          }
+        }
+      }
+      if (isModified) {
+        res.send(format === 'hex' ? psbt.toHex() : psbt.toBase64());
+      } else {
+        // Not modified
+        // 422 Unprocessable Entity
+        // https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/422
+        res.status(422).send(`Psbt had no missing nonWitnessUtxos.`);
+      }
+    } catch (e: any) {
+      if (e instanceof Error && new RegExp(notFoundError).test(e.message)) {
+        res.status(404).send(e.message);
+      } else {
+        res.status(500).send(e instanceof Error ? e.message : e);
+      }
+    }
+  }
+
+  private async getTransactionStatus(req: Request, res: Response) {
+    try {
+      const transaction = await transactionUtils.$getTransactionExtended(req.params.txId, true);
+      res.json(transaction.status);
+    } catch (e) {
+      let statusCode = 500;
+      if (e instanceof Error && e.message && e.message.indexOf('No such mempool or blockchain transaction') > -1) {
+        statusCode = 404;
+      }
+      res.status(statusCode).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getStrippedBlockTransactions(req: Request, res: Response) {
+    try {
+      const transactions = await blocks.$getStrippedBlockTransactions(req.params.hash);
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24 * 30).toUTCString());
+      res.json(transactions);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getBlock(req: Request, res: Response) {
+    try {
+      const block = await blocks.$getBlock(req.params.hash);
+
+      const blockAge = new Date().getTime() / 1000 - block.timestamp;
+      const day = 24 * 3600;
+      let cacheDuration;
+      if (blockAge > 365 * day) {
+        cacheDuration = 30 * day;
+      } else if (blockAge > 30 * day) {
+        cacheDuration = 10 * day;
+      } else {
+        cacheDuration = 600
+      }
+
+      res.setHeader('Expires', new Date(Date.now() + 1000 * cacheDuration).toUTCString());
+      res.json(block);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getBlockHeader(req: Request, res: Response) {
+    try {
+      const blockHeader = await bitcoinApi.$getBlockHeader(req.params.hash);
+      res.setHeader('content-type', 'text/plain');
+      res.send(blockHeader);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getBlockAuditSummary(req: Request, res: Response) {
+    try {
+      const transactions = await blocks.$getBlockAuditSummary(req.params.hash);
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24 * 30).toUTCString());
+      res.json(transactions);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getBlocks(req: Request, res: Response) {
+    try {
+      if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) { // Bitcoin
+        const height = req.params.height === undefined ? undefined : parseInt(req.params.height, 10);
+        res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+        res.json(await blocks.$getBlocks(height, 15));
+      } else { // Liquid, Bisq
+        return await this.getLegacyBlocks(req, res);
+      }
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getBlocksByBulk(req: Request, res: Response) {
+    try {
+      if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { // Liquid, Bisq - Not implemented
+        return res.status(404).send(`This API is only available for Bitcoin networks`);
+      }
+      if (config.MEMPOOL.MAX_BLOCKS_BULK_QUERY <= 0) {
+        return res.status(404).send(`This API is disabled. Set config.MEMPOOL.MAX_BLOCKS_BULK_QUERY to a positive number to enable it.`);
+      }
+      if (!Common.indexingEnabled()) {
+        return res.status(404).send(`Indexing is required for this API`);
+      }
+
+      const from = parseInt(req.params.from, 10);
+      if (!req.params.from || from < 0) {
+        return res.status(400).send(`Parameter 'from' must be a block height (integer)`);
+      }
+      const to = req.params.to === undefined ? await bitcoinApi.$getBlockHeightTip() : parseInt(req.params.to, 10);
+      if (to < 0) {
+        return res.status(400).send(`Parameter 'to' must be a block height (integer)`);
+      }
+      if (from > to) {
+        return res.status(400).send(`Parameter 'to' must be a higher block height than 'from'`);
+      }
+      if ((to - from + 1) > config.MEMPOOL.MAX_BLOCKS_BULK_QUERY) {
+        return res.status(400).send(`You can only query ${config.MEMPOOL.MAX_BLOCKS_BULK_QUERY} blocks at once.`);
+      }
+
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(await blocks.$getBlocksBetweenHeight(from, to));
+
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getLegacyBlocks(req: Request, res: Response) {
+    try {
+      const returnBlocks: IEsploraApi.Block[] = [];
+      const tip = blocks.getCurrentBlockHeight();
+      const fromHeight = Math.min(parseInt(req.params.height, 10) || tip, tip);
+
+      // Check if block height exist in local cache to skip the hash lookup
+      const blockByHeight = blocks.getBlocks().find((b) => b.height === fromHeight);
+      let startFromHash: string | null = null;
+      if (blockByHeight) {
+        startFromHash = blockByHeight.id;
+      } else {
+        startFromHash = await bitcoinApi.$getBlockHash(fromHeight);
+      }
+
+      let nextHash = startFromHash;
+      for (let i = 0; i < 10 && nextHash; i++) {
+        const localBlock = blocks.getBlocks().find((b) => b.id === nextHash);
+        if (localBlock) {
+          returnBlocks.push(localBlock);
+          nextHash = localBlock.previousblockhash;
+        } else {
+          const block = await bitcoinCoreApi.$getBlock(nextHash);
+          returnBlocks.push(block);
+          nextHash = block.previousblockhash;
+        }
+      }
+
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(returnBlocks);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+  
+  private async getBlockTransactions(req: Request, res: Response) {
+    try {
+      loadingIndicators.setProgress('blocktxs-' + req.params.hash, 0);
+
+      const txIds = await bitcoinApi.$getTxIdsForBlock(req.params.hash);
+      const transactions: TransactionExtended[] = [];
+      const startingIndex = Math.max(0, parseInt(req.params.index || '0', 10));
+
+      const endIndex = Math.min(startingIndex + 10, txIds.length);
+      for (let i = startingIndex; i < endIndex; i++) {
+        try {
+          const transaction = await transactionUtils.$getTransactionExtended(txIds[i], true, true);
+          transactions.push(transaction);
+          loadingIndicators.setProgress('blocktxs-' + req.params.hash, (i - startingIndex + 1) / (endIndex - startingIndex) * 100);
+        } catch (e) {
+          logger.debug('getBlockTransactions error: ' + (e instanceof Error ? e.message : e));
+        }
+      }
+      res.json(transactions);
+    } catch (e) {
+      loadingIndicators.setProgress('blocktxs-' + req.params.hash, 100);
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getBlockHeight(req: Request, res: Response) {
+    try {
+      const blockHash = await bitcoinApi.$getBlockHash(parseInt(req.params.height, 10));
+      res.send(blockHash);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getAddress(req: Request, res: Response) {
+    if (config.MEMPOOL.BACKEND === 'none') {
+      res.status(405).send('Address lookups cannot be used with bitcoind as backend.');
+      return;
+    }
+
+    try {
+      const addressData = await bitcoinApi.$getAddress(req.params.address);
+      res.json(addressData);
+    } catch (e) {
+      if (e instanceof Error && e.message && (e.message.indexOf('too long') > 0 || e.message.indexOf('confirmed status') > 0)) {
+        return res.status(413).send(e instanceof Error ? e.message : e);
+      }
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getAddressTransactions(req: Request, res: Response) {
+    if (config.MEMPOOL.BACKEND === 'none') {
+      res.status(405).send('Address lookups cannot be used with bitcoind as backend.');
+      return;
+    }
+
+    try {
+      const transactions = await bitcoinApi.$getAddressTransactions(req.params.address, req.params.txId);
+      res.json(transactions);
+    } catch (e) {
+      if (e instanceof Error && e.message && (e.message.indexOf('too long') > 0 || e.message.indexOf('confirmed status') > 0)) {
+        return res.status(413).send(e instanceof Error ? e.message : e);
+      }
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getAdressTxChain(req: Request, res: Response) {
+    res.status(501).send('Not implemented');
+  }
+
+  private async getAddressPrefix(req: Request, res: Response) {
+    try {
+      const blockHash = await bitcoinApi.$getAddressPrefix(req.params.prefix);
+      res.send(blockHash);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getRecentMempoolTransactions(req: Request, res: Response) {
+    const latestTransactions = Object.entries(mempool.getMempool())
+      .sort((a, b) => (b[1].firstSeen || 0) - (a[1].firstSeen || 0))
+      .slice(0, 10).map((tx) => Common.stripTransaction(tx[1]));
+
+    res.json(latestTransactions);
+  }
+
+  private async getMempool(req: Request, res: Response) {
+    const info = mempool.getMempoolInfo();
+    res.json({
+      count: info.size,
+      vsize: info.bytes,
+      total_fee: info.total_fee * 1e8,
+      fee_histogram: []
+    });
+  }
+
+  private async getMempoolTxIds(req: Request, res: Response) {
+    try {
+      const rawMempool = await bitcoinApi.$getRawMempool();
+      res.send(rawMempool);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getBlockTipHeight(req: Request, res: Response) {
+    try {
+      const result = await bitcoinApi.$getBlockHeightTip();
+      res.json(result);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getBlockTipHash(req: Request, res: Response) {
+    try {
+      const result = await bitcoinApi.$getBlockHashTip();
+      res.setHeader('content-type', 'text/plain');
+      res.send(result);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getRawBlock(req: Request, res: Response) {
+    try {
+      const result = await bitcoinApi.$getRawBlock(req.params.hash);
+      res.setHeader('content-type', 'application/octet-stream');
+      res.send(result);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getTxIdsForBlock(req: Request, res: Response) {
+    try {
+      const result = await bitcoinApi.$getTxIdsForBlock(req.params.hash);
+      res.json(result);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async validateAddress(req: Request, res: Response) {
+    try {
+      const result = await bitcoinClient.validateAddress(req.params.address);
+      res.json(result);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getRbfHistory(req: Request, res: Response) {
+    try {
+      const result = rbfCache.getReplaces(req.params.txId);
+      res.json(result || []);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getCachedTx(req: Request, res: Response) {
+    try {
+      const result = rbfCache.getTx(req.params.txId);
+      if (result) {
+        res.json(result);
+      } else {
+        res.status(204).send();
+      }
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getTransactionOutspends(req: Request, res: Response) {
+    try {
+      const result = await bitcoinApi.$getOutspends(req.params.txId);
+      res.json(result);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private getDifficultyChange(req: Request, res: Response) {
+    try {
+      const da = difficultyAdjustment.getDifficultyAdjustment();
+      if (da) {
+        res.json(da);
+      } else {
+        res.status(503).send(`Service Temporarily Unavailable`);
+      }
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $postTransaction(req: Request, res: Response) {
+    res.setHeader('content-type', 'text/plain');
+    try {
+      let rawTx;
+      if (typeof req.body === 'object') {
+        rawTx = Object.keys(req.body)[0];
+      } else {
+        rawTx = req.body;
+      }
+      const txIdResult = await bitcoinApi.$sendRawTransaction(rawTx);
+      res.send(txIdResult);
+    } catch (e: any) {
+      res.status(400).send(e.message && e.code ? 'sendrawtransaction RPC error: ' + JSON.stringify({ code: e.code, message: e.message })
+        : (e.message || 'Error'));
+    }
+  }
+
+  private async $postTransactionForm(req: Request, res: Response) {
+    res.setHeader('content-type', 'text/plain');
+    const matches = /tx=([a-z0-9]+)/.exec(req.body);
+    let txHex = '';
+    if (matches && matches[1]) {
+      txHex = matches[1];
+    }
+    try {
+      const txIdResult = await bitcoinClient.sendRawTransaction(txHex);
+      res.send(txIdResult);
+    } catch (e: any) {
+      res.status(400).send(e.message && e.code ? 'sendrawtransaction RPC error: ' + JSON.stringify({ code: e.code, message: e.message })
+        : (e.message || 'Error'));
+    }
+  }
+
+}
+
+export default new BitcoinRoutes();

backend/src/api/bitcoin/electrum-api.ts

@@ -1,22 +1,19 @@
 import config from '../../config';
+import Client from '@mempool/electrum-client';
 import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
-import { IBitcoinApi } from './bitcoin-api.interface';
 import { IEsploraApi } from './esplora-api.interface';
 import { IElectrumApi } from './electrum-api.interface';
 import BitcoinApi from './bitcoin-api';
-import mempool from '../mempool';
 import logger from '../../logger';
-import * as ElectrumClient from '@mempool/electrum-client';
-import * as sha256 from 'crypto-js/sha256';
-import * as hexEnc from 'crypto-js/enc-hex';
+import crypto from "crypto-js";
 import loadingIndicators from '../loading-indicators';
 import memoryCache from '../memory-cache';
 
 class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
   private electrumClient: any;
 
-  constructor() {
-    super();
+  constructor(bitcoinClient: any) {
+    super(bitcoinClient);
 
     const electrumConfig = { client: 'mempool-v2', version: '1.4' };
     const electrumPersistencePolicy = { retryPeriod: 10000, maxRetry: 1000, callback: null };
@@ -28,7 +25,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
       onLog: (str) => { logger.debug(str); },
     };
 
-    this.electrumClient = new ElectrumClient(
+    this.electrumClient = new Client(
       config.ELECTRUM.PORT,
       config.ELECTRUM.HOST,
       config.ELECTRUM.TLS_ENABLED ? 'tls' : 'tcp',
@@ -37,14 +34,14 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
     );
 
     this.electrumClient.initElectrum(electrumConfig, electrumPersistencePolicy)
-      .then(() => {})
+      .then(() => { })
       .catch((err) => {
         logger.err(`Error connecting to Electrum Server at ${config.ELECTRUM.HOST}:${config.ELECTRUM.PORT}`);
       });
   }
 
   async $getAddress(address: string): Promise<IEsploraApi.Address> {
-    const addressInfo = await this.$validateAddress(address);
+    const addressInfo = await this.bitcoindClient.validateAddress(address);
     if (!addressInfo || !addressInfo.isvalid) {
       return ({
         'address': address,
@@ -89,18 +86,15 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
         },
         'electrum': true,
       };
-    } catch (e) {
-      if (e === 'failed to get confirmed status') {
-        e = 'The number of transactions on this address exceeds the Electrum server limit';
-      }
-      throw new Error(e);
+    } catch (e: any) {
+      throw new Error(typeof e === 'string' ? e : e && e.message || e);
     }
   }
 
   async $getAddressTransactions(address: string, lastSeenTxId: string): Promise<IEsploraApi.Transaction[]> {
-    const addressInfo = await this.$validateAddress(address);
+    const addressInfo = await this.bitcoindClient.validateAddress(address);
     if (!addressInfo || !addressInfo.isvalid) {
-     return [];
+      return [];
     }
 
     try {
@@ -126,12 +120,9 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
       }
 
       return transactions;
-    } catch (e) {
+    } catch (e: any) {
       loadingIndicators.setProgress('address-' + address, 100);
-      if (e === 'failed to get confirmed status') {
-        e = 'The number of transactions on this address exceeds the Electrum server limit';
-      }
-      throw new Error(e);
+      throw new Error(typeof e === 'string' ? e : e && e.message || e);
     }
   }
 
@@ -152,8 +143,8 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
   }
 
   private encodeScriptHash(scriptPubKey: string): string {
-    const addrScripthash = hexEnc.stringify(sha256(hexEnc.parse(scriptPubKey)));
-    return addrScripthash.match(/.{2}/g).reverse().join('');
+    const addrScripthash = crypto.enc.Hex.stringify(crypto.SHA256(crypto.enc.Hex.parse(scriptPubKey)));
+    return addrScripthash!.match(/.{2}/g)!.reverse().join('');
   }
 
 }

backend/src/api/bitcoin/esplora-api.interface.ts

@@ -9,6 +9,7 @@ export namespace IEsploraApi {
     vin: Vin[];
     vout: Vout[];
     status: Status;
+    hex?: string;
   }
 
   export interface Recent {
@@ -24,14 +25,16 @@ export namespace IEsploraApi {
     is_coinbase: boolean;
     scriptsig: string;
     scriptsig_asm: string;
-    inner_redeemscript_asm?: string;
-    inner_witnessscript_asm?: string;
+    inner_redeemscript_asm: string;
+    inner_witnessscript_asm: string;
     sequence: any;
-    witness?: string[];
+    witness: string[];
     prevout: Vout | null;
     // Elements
     is_pegin?: boolean;
     issuance?: Issuance;
+    // Custom
+    lazy?: boolean;
   }
 
   interface Issuance {
@@ -85,6 +88,7 @@ export namespace IEsploraApi {
     size: number;
     weight: number;
     previousblockhash: string;
+    mediantime: number;
   }
 
   export interface Address {
@@ -112,9 +116,9 @@ export namespace IEsploraApi {
 
   export interface Outspend {
     spent: boolean;
-    txid: string;
-    vin: number;
-    status: Status;
+    txid?: string;
+    vin?: number;
+    status?: Status;
   }
 
   export interface Asset {

backend/src/api/bitcoin/esplora-api.ts

@@ -1,8 +1,13 @@
 import config from '../../config';
 import axios, { AxiosRequestConfig } from 'axios';
+import http from 'http';
 import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
 import { IEsploraApi } from './esplora-api.interface';
 
+const axiosConnection = axios.create({
+  httpAgent: new http.Agent({ keepAlive: true })
+});
+
 class ElectrsApi implements AbstractBitcoinApi {
   axiosConfig: AxiosRequestConfig = {
     timeout: 10000,
@@ -11,35 +16,55 @@ class ElectrsApi implements AbstractBitcoinApi {
   constructor() { }
 
   $getRawMempool(): Promise<IEsploraApi.Transaction['txid'][]> {
-    return axios.get<IEsploraApi.Transaction['txid'][]>(config.ESPLORA.REST_API_URL + '/mempool/txids', this.axiosConfig)
+    return axiosConnection.get<IEsploraApi.Transaction['txid'][]>(config.ESPLORA.REST_API_URL + '/mempool/txids', this.axiosConfig)
       .then((response) => response.data);
   }
 
   $getRawTransaction(txId: string): Promise<IEsploraApi.Transaction> {
-    return axios.get<IEsploraApi.Transaction>(config.ESPLORA.REST_API_URL + '/tx/' + txId, this.axiosConfig)
+    return axiosConnection.get<IEsploraApi.Transaction>(config.ESPLORA.REST_API_URL + '/tx/' + txId, this.axiosConfig)
+      .then((response) => response.data);
+  }
+
+  $getTransactionHex(txId: string): Promise<string> {
+    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/hex', this.axiosConfig)
       .then((response) => response.data);
   }
 
   $getBlockHeightTip(): Promise<number> {
-    return axios.get<number>(config.ESPLORA.REST_API_URL + '/blocks/tip/height', this.axiosConfig)
+    return axiosConnection.get<number>(config.ESPLORA.REST_API_URL + '/blocks/tip/height', this.axiosConfig)
+      .then((response) => response.data);
+  }
+
+  $getBlockHashTip(): Promise<string> {
+    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/blocks/tip/hash', this.axiosConfig)
       .then((response) => response.data);
   }
 
   $getTxIdsForBlock(hash: string): Promise<string[]> {
-    return axios.get<string[]>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/txids', this.axiosConfig)
+    return axiosConnection.get<string[]>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/txids', this.axiosConfig)
       .then((response) => response.data);
   }
 
   $getBlockHash(height: number): Promise<string> {
-    return axios.get<string>(config.ESPLORA.REST_API_URL + '/block-height/' + height, this.axiosConfig)
+    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/block-height/' + height, this.axiosConfig)
+      .then((response) => response.data);
+  }
+
+  $getBlockHeader(hash: string): Promise<string> {
+    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/header', this.axiosConfig)
       .then((response) => response.data);
   }
 
   $getBlock(hash: string): Promise<IEsploraApi.Block> {
-    return axios.get<IEsploraApi.Block>(config.ESPLORA.REST_API_URL + '/block/' + hash, this.axiosConfig)
+    return axiosConnection.get<IEsploraApi.Block>(config.ESPLORA.REST_API_URL + '/block/' + hash, this.axiosConfig)
       .then((response) => response.data);
   }
 
+  $getRawBlock(hash: string): Promise<Buffer> {
+    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + "/raw", { ...this.axiosConfig, responseType: 'arraybuffer' })
+      .then((response) => { return Buffer.from(response.data); });
+  }
+
   $getAddress(address: string): Promise<IEsploraApi.Address> {
     throw new Error('Method getAddress not implemented.');
   }
@@ -51,6 +76,29 @@ class ElectrsApi implements AbstractBitcoinApi {
   $getAddressPrefix(prefix: string): string[] {
     throw new Error('Method not implemented.');
   }
+
+  $sendRawTransaction(rawTransaction: string): Promise<string> {
+    throw new Error('Method not implemented.');
+  }
+
+  $getOutspend(txId: string, vout: number): Promise<IEsploraApi.Outspend> {
+    return axiosConnection.get<IEsploraApi.Outspend>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspend/' + vout, this.axiosConfig)
+      .then((response) => response.data);
+  }
+
+  $getOutspends(txId: string): Promise<IEsploraApi.Outspend[]> {
+    return axiosConnection.get<IEsploraApi.Outspend[]>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspends', this.axiosConfig)
+      .then((response) => response.data);
+  }
+
+  async $getBatchedOutspends(txId: string[]): Promise<IEsploraApi.Outspend[][]> {
+    const outspends: IEsploraApi.Outspend[][] = [];
+    for (const tx of txId) {
+      const outspend = await this.$getOutspends(tx);
+      outspends.push(outspend);
+    }
+    return outspends;
+  }
 }
 
 export default ElectrsApi;

backend/src/api/chain-tips.ts

@@ -0,0 +1,61 @@
+import logger from '../logger';
+import bitcoinClient from './bitcoin/bitcoin-client';
+
+export interface ChainTip {
+  height: number;
+  hash: string;
+  branchlen: number;
+  status: 'invalid' | 'active' | 'valid-fork' | 'valid-headers' | 'headers-only';
+};
+
+export interface OrphanedBlock {
+  height: number;
+  hash: string;
+  status: 'valid-fork' | 'valid-headers' | 'headers-only';
+}
+
+class ChainTips {
+  private chainTips: ChainTip[] = [];
+  private orphanedBlocks: OrphanedBlock[] = [];
+
+  public async updateOrphanedBlocks(): Promise<void> {
+    try {
+      this.chainTips = await bitcoinClient.getChainTips();
+      this.orphanedBlocks = [];
+
+      for (const chain of this.chainTips) {
+        if (chain.status === 'valid-fork' || chain.status === 'valid-headers') {
+          let block = await bitcoinClient.getBlock(chain.hash);
+          while (block && block.confirmations === -1) {
+            this.orphanedBlocks.push({
+              height: block.height,
+              hash: block.hash,
+              status: chain.status
+            });
+            block = await bitcoinClient.getBlock(block.previousblockhash);
+          }
+        }
+      }
+
+      logger.debug(`Updated orphaned blocks cache. Found ${this.orphanedBlocks.length} orphaned blocks`);
+    } catch (e) {
+      logger.err(`Cannot get fetch orphaned blocks. Reason: ${e instanceof Error ? e.message : e}`);
+    }
+  }
+
+  public getOrphanedBlocksAtHeight(height: number | undefined): OrphanedBlock[] {
+    if (height === undefined) {
+      return [];
+    }
+
+    const orphans: OrphanedBlock[] = [];
+    for (const block of this.orphanedBlocks) {
+      if (block.height === height) {
+        orphans.push(block);
+      }
+    }
+    return orphans;
+  }
+}
+
+export default new ChainTips();

backend/src/api/common.ts

@@ -1,6 +1,17 @@
-import { CpfpInfo, TransactionExtended, TransactionStripped } from '../mempool.interfaces';
-
+import { CpfpInfo, MempoolBlockWithTransactions, TransactionExtended, TransactionStripped } from '../mempool.interfaces';
+import config from '../config';
+import { NodeSocket } from '../repositories/NodesSocketsRepository';
+import { isIP } from 'net';
 export class Common {
+  static nativeAssetId = config.MEMPOOL.NETWORK === 'liquidtestnet' ?
+    '144c654344aa716d6f3abcc1ca90e5641e4e2a7f633bc09fe3baf64585819a49'
+  : '6f0279e9ed041c3d710a9f57d0c02928416460c4b722ae3457a11eec381c526d';
+  static _isLiquid = config.MEMPOOL.NETWORK === 'liquid' || config.MEMPOOL.NETWORK === 'liquidtestnet';
+
+  static isLiquid(): boolean {
+    return this._isLiquid;
+  }
+
   static median(numbers: number[]) {
     let medianNr = 0;
     const numsLen = numbers.length;
@@ -24,24 +35,31 @@ export class Common {
   }
 
   static getFeesInRange(transactions: TransactionExtended[], rangeLength: number) {
-    const arr = [transactions[transactions.length - 1].effectiveFeePerVsize];
+    const filtered: TransactionExtended[] = [];
+    let lastValidRate = Infinity;
+    // filter out anomalous fee rates to ensure monotonic range
+    for (const tx of transactions) {
+      if (tx.effectiveFeePerVsize <= lastValidRate) {
+        filtered.push(tx);
+        lastValidRate = tx.effectiveFeePerVsize;
+      }
+    }
+    const arr = [filtered[filtered.length - 1].effectiveFeePerVsize];
     const chunk = 1 / (rangeLength - 1);
     let itemsToAdd = rangeLength - 2;
 
     while (itemsToAdd > 0) {
-      arr.push(transactions[Math.floor(transactions.length * chunk * itemsToAdd)].effectiveFeePerVsize);
+      arr.push(filtered[Math.floor(filtered.length * chunk * itemsToAdd)].effectiveFeePerVsize);
       itemsToAdd--;
     }
 
-    arr.push(transactions[0].effectiveFeePerVsize);
+    arr.push(filtered[0].effectiveFeePerVsize);
     return arr;
   }
 
   static findRbfTransactions(added: TransactionExtended[], deleted: TransactionExtended[]): { [txid: string]: TransactionExtended } {
     const matches: { [txid: string]: TransactionExtended } = {};
     deleted
-      // The replaced tx must have at least one input with nSequence < maxint-1 (That’s the opt-in)
-      .filter((tx) => tx.vin.some((vin) => vin.sequence < 0xfffffffe))
       .forEach((deletedTx) => {
         const foundMatches = added.find((addedTx) => {
           // The new tx must, absolutely speaking, pay at least as much fee as the replaced tx.
@@ -50,7 +68,7 @@ export class Common {
             && addedTx.feePerVsize > deletedTx.feePerVsize
             // Spends one or more of the same inputs
             && deletedTx.vin.some((deletedVin) =>
-              addedTx.vin.some((vin) => vin.txid === deletedVin.txid));
+              addedTx.vin.some((vin) => vin.txid === deletedVin.txid && vin.vout === deletedVin.vout));
             });
         if (foundMatches) {
           matches[deletedTx.txid] = foundMatches;
@@ -68,7 +86,7 @@ export class Common {
     };
   }
 
-  static sleep(ms: number): Promise<void> {
+  static sleep$(ms: number): Promise<void> {
     return new Promise((resolve) => {
        setTimeout(() => {
          resolve();
@@ -105,7 +123,7 @@ export class Common {
       totalFees += tx.bestDescendant.fee;
     }
 
-    tx.effectiveFeePerVsize = totalFees / (totalWeight / 4);
+    tx.effectiveFeePerVsize = Math.max(0, totalFees / (totalWeight / 4));
     tx.cpfpChecked = true;
 
     return {
@@ -145,4 +163,186 @@ export class Common {
     });
     return parents;
   }
+
+  // calculates the ratio of matched transactions to projected transactions by weight
+  static getSimilarity(projectedBlock: MempoolBlockWithTransactions, transactions: TransactionExtended[]): number {
+    let matchedWeight = 0;
+    let projectedWeight = 0;
+    const inBlock = {};
+
+    for (const tx of transactions) {
+      inBlock[tx.txid] = tx;
+    }
+
+    // look for transactions that were expected in the template, but missing from the mined block
+    for (const tx of projectedBlock.transactions) {
+      if (inBlock[tx.txid]) {
+        matchedWeight += tx.vsize * 4;
+      }
+      projectedWeight += tx.vsize * 4;
+    }
+
+    projectedWeight += transactions[0].weight;
+    matchedWeight += transactions[0].weight;
+
+    return projectedWeight ? matchedWeight / projectedWeight : 1;
+  }
+
+  static getSqlInterval(interval: string | null): string | null {
+    switch (interval) {
+      case '24h': return '1 DAY';
+      case '3d': return '3 DAY';
+      case '1w': return '1 WEEK';
+      case '1m': return '1 MONTH';
+      case '3m': return '3 MONTH';
+      case '6m': return '6 MONTH';
+      case '1y': return '1 YEAR';
+      case '2y': return '2 YEAR';
+      case '3y': return '3 YEAR';
+      case '4y': return '4 YEAR';
+      default: return null;
+    }
+  }
+
+  static indexingEnabled(): boolean {
+    return (
+      ['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) &&
+      config.DATABASE.ENABLED === true &&
+      config.MEMPOOL.INDEXING_BLOCKS_AMOUNT !== 0
+    );
+  }
+
+  static blocksSummariesIndexingEnabled(): boolean {
+    return (
+      Common.indexingEnabled() &&
+      config.MEMPOOL.BLOCKS_SUMMARIES_INDEXING === true
+    );
+  }
+
+  static cpfpIndexingEnabled(): boolean {
+    return (
+      Common.indexingEnabled() &&
+      config.MEMPOOL.CPFP_INDEXING === true
+    );
+  }
+
+  static setDateMidnight(date: Date): void {
+    date.setUTCHours(0);
+    date.setUTCMinutes(0);
+    date.setUTCSeconds(0);
+    date.setUTCMilliseconds(0);
+  }
+
+  static channelShortIdToIntegerId(channelId: string): string {
+    if (channelId.indexOf('x') === -1) { // Already an integer id
+      return channelId;
+    }
+    if (channelId.indexOf('/') !== -1) { // Topology import
+      channelId = channelId.slice(0, -2);
+    }
+    const s = channelId.split('x').map(part => BigInt(part));
+    return ((s[0] << 40n) | (s[1] << 16n) | s[2]).toString();
+  }
+
+  /** Decodes a channel id returned by lnd as uint64 to a short channel id */
+  static channelIntegerIdToShortId(id: string): string {
+    if (id.indexOf('/') !== -1) {
+      id = id.slice(0, -2);
+    }
+    
+    if (id.indexOf('x') !== -1) { // Already a short id
+      return id;
+    }
+
+    const n = BigInt(id);
+    return [
+      n >> 40n, // nth block
+      (n >> 16n) & 0xffffffn, // nth tx of the block
+      n & 0xffffn // nth output of the tx
+    ].join('x');
+  }
+
+  static utcDateToMysql(date?: number | null): string | null {
+    if (date === null) {
+      return null;
+    }
+    const d = new Date((date || 0) * 1000);
+    return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0];
+  }
+
+  static findSocketNetwork(addr: string): {network: string | null, url: string} {
+    let network: string | null = null;
+    let url: string = addr;
+
+    if (config.LIGHTNING.BACKEND === 'cln') {
+      url = addr.split('://')[1];
+    }
+
+    if (!url) {
+      return {
+        network: null,
+        url: addr,
+      };
+    }
+
+    if (addr.indexOf('onion') !== -1) {
+      if (url.split('.')[0].length >= 56) {
+        network = 'torv3';
+      } else {
+        network = 'torv2';
+      }
+    } else if (addr.indexOf('i2p') !== -1) {
+      network = 'i2p';
+    } else if (addr.indexOf('ipv4') !== -1 || (config.LIGHTNING.BACKEND === 'lnd' && isIP(url.split(':')[0]) === 4)) {
+      const ipv = isIP(url.split(':')[0]);
+      if (ipv === 4) {
+        network = 'ipv4';
+      } else {
+        return {
+          network: null,
+          url: addr,
+        };
+      }
+    } else if (addr.indexOf('ipv6') !== -1 || (config.LIGHTNING.BACKEND === 'lnd' && url.indexOf(']:'))) {
+      url = url.split('[')[1].split(']')[0];
+      const ipv = isIP(url);
+      if (ipv === 6) {
+        const parts = addr.split(':');
+        network = 'ipv6';
+        url = `[${url}]:${parts[parts.length - 1]}`;
+      } else {
+        return {
+          network: null,
+          url: addr,
+        };
+      }
+    } else {
+      return {
+        network: null,
+        url: addr,
+      };
+    }
+
+    return {
+      network: network,
+      url: url,
+    };
+  }
+
+  static formatSocket(publicKey: string, socket: {network: string, addr: string}): NodeSocket {
+    if (config.LIGHTNING.BACKEND === 'cln') {
+      return {
+        publicKey: publicKey,
+        network: socket.network,
+        addr: socket.addr,
+      };
+    } else /* if (config.LIGHTNING.BACKEND === 'lnd') */ {
+      const formatted = this.findSocketNetwork(socket.addr);
+      return {
+        publicKey: publicKey,
+        network: formatted.network,
+        addr: formatted.url,
+      };
+    }
+  }
 }

backend/src/api/difficulty-adjustment.ts

@@ -0,0 +1,106 @@
+import config from '../config';
+import { IDifficultyAdjustment } from '../mempool.interfaces';
+import blocks from './blocks';
+
+export interface DifficultyAdjustment {
+  progressPercent: number;       // Percent: 0 to 100
+  difficultyChange: number;      // Percent: -75 to 300
+  estimatedRetargetDate: number; // Unix time in ms
+  remainingBlocks: number;       // Block count
+  remainingTime: number;         // Duration of time in ms
+  previousRetarget: number;      // Percent: -75 to 300
+  previousTime: number;          // Unix time in ms
+  nextRetargetHeight: number;    // Block Height
+  timeAvg: number;               // Duration of time in ms
+  timeOffset: number;            // (Testnet) Time since last block (cap @ 20min) in ms
+  expectedBlocks: number;         // Block count
+}
+
+export function calcDifficultyAdjustment(
+  DATime: number,
+  nowSeconds: number,
+  blockHeight: number,
+  previousRetarget: number,
+  network: string,
+  latestBlockTimestamp: number,
+): DifficultyAdjustment {
+  const ESTIMATE_LAG_BLOCKS = 146; // For first 7.2% of epoch, don't estimate.
+  const EPOCH_BLOCK_LENGTH = 2016; // Bitcoin mainnet
+  const BLOCK_SECONDS_TARGET = 600; // Bitcoin mainnet
+  const TESTNET_MAX_BLOCK_SECONDS = 1200; // Bitcoin testnet
+
+  const diffSeconds = nowSeconds - DATime;
+  const blocksInEpoch = (blockHeight >= 0) ? blockHeight % EPOCH_BLOCK_LENGTH : 0;
+  const progressPercent = (blockHeight >= 0) ? blocksInEpoch / EPOCH_BLOCK_LENGTH * 100 : 100;
+  const remainingBlocks = EPOCH_BLOCK_LENGTH - blocksInEpoch;
+  const nextRetargetHeight = (blockHeight >= 0) ? blockHeight + remainingBlocks : 0;
+  const expectedBlocks = diffSeconds / BLOCK_SECONDS_TARGET;
+
+  let difficultyChange = 0;
+  let timeAvgSecs = diffSeconds / blocksInEpoch;
+  // Only calculate the estimate once we have 7.2% of blocks in current epoch
+  if (blocksInEpoch >= ESTIMATE_LAG_BLOCKS) {
+    difficultyChange = (BLOCK_SECONDS_TARGET / timeAvgSecs - 1) * 100;
+    // Max increase is x4 (+300%)
+    if (difficultyChange > 300) {
+      difficultyChange = 300;
+    }
+    // Max decrease is /4 (-75%)
+    if (difficultyChange < -75) {
+      difficultyChange = -75;
+    }
+  }
+
+  // Testnet difficulty is set to 1 after 20 minutes of no blocks,
+  // therefore the time between blocks will always be below 20 minutes (1200s).
+  let timeOffset = 0;
+  if (network === 'testnet') {
+    if (timeAvgSecs > TESTNET_MAX_BLOCK_SECONDS) {
+      timeAvgSecs = TESTNET_MAX_BLOCK_SECONDS;
+    }
+
+    const secondsSinceLastBlock = nowSeconds - latestBlockTimestamp;
+    if (secondsSinceLastBlock + timeAvgSecs > TESTNET_MAX_BLOCK_SECONDS) {
+      timeOffset = -Math.min(secondsSinceLastBlock, TESTNET_MAX_BLOCK_SECONDS) * 1000;
+    }
+  }
+
+  const timeAvg = Math.floor(timeAvgSecs * 1000);
+  const remainingTime = remainingBlocks * timeAvg;
+  const estimatedRetargetDate = remainingTime + nowSeconds * 1000;
+
+  return {
+    progressPercent,
+    difficultyChange,
+    estimatedRetargetDate,
+    remainingBlocks,
+    remainingTime,
+    previousRetarget,
+    previousTime: DATime,
+    nextRetargetHeight,
+    timeAvg,
+    timeOffset,
+    expectedBlocks,
+  };
+}
+
+class DifficultyAdjustmentApi {
+  public getDifficultyAdjustment(): IDifficultyAdjustment | null {
+    const DATime = blocks.getLastDifficultyAdjustmentTime();
+    const previousRetarget = blocks.getPreviousDifficultyRetarget();
+    const blockHeight = blocks.getCurrentBlockHeight();
+    const blocksCache = blocks.getBlocks();
+    const latestBlock = blocksCache[blocksCache.length - 1];
+    if (!latestBlock) {
+      return null;
+    }
+    const nowSeconds = Math.floor(new Date().getTime() / 1000);
+
+    return calcDifficultyAdjustment(
+      DATime, nowSeconds, blockHeight, previousRetarget,
+      config.MEMPOOL.NETWORK, latestBlock.timestamp
+    );
+  }
+}
+
+export default new DifficultyAdjustmentApi();

backend/src/api/disk-cache.ts

@@ -1,6 +1,6 @@
 import * as fs from 'fs';
 const fsPromises = fs.promises;
-import * as cluster from 'cluster';
+import cluster from 'cluster';
 import memPool from './mempool';
 import blocks from './blocks';
 import logger from '../logger';
@@ -9,23 +9,35 @@ import { TransactionExtended } from '../mempool.interfaces';
 import { Common } from './common';
 
 class DiskCache {
+  private cacheSchemaVersion = 3;
+
+  private static TMP_FILE_NAME = config.MEMPOOL.CACHE_DIR + '/tmp-cache.json';
+  private static TMP_FILE_NAMES = config.MEMPOOL.CACHE_DIR + '/tmp-cache{number}.json';
   private static FILE_NAME = config.MEMPOOL.CACHE_DIR + '/cache.json';
   private static FILE_NAMES = config.MEMPOOL.CACHE_DIR + '/cache{number}.json';
   private static CHUNK_FILES = 25;
   private isWritingCache = false;
 
-  constructor() { }
+  constructor() {
+    if (!cluster.isPrimary) {
+      return;
+    }
+    process.on('SIGINT', (e) => {
+      this.$saveCacheToDisk(true);
+      process.exit(0);
+    });
+  }
 
-  async $saveCacheToDisk(): Promise<void> {
-    if (!cluster.isMaster) {
+  async $saveCacheToDisk(sync: boolean = false): Promise<void> {
+    if (!cluster.isPrimary) {
       return;
     }
     if (this.isWritingCache) {
-      logger.debug('Saving cache already in progress. Skipping.')
+      logger.debug('Saving cache already in progress. Skipping.');
       return;
     }
     try {
-      logger.debug('Writing mempool and blocks data to disk cache (async)...');
+      logger.debug(`Writing mempool and blocks data to disk cache (${ sync ? 'sync' : 'async' })...`);
       this.isWritingCache = true;
 
       const mempool = memPool.getMempool();
@@ -38,26 +50,79 @@ class DiskCache {
 
       const chunkSize = Math.floor(mempoolArray.length / DiskCache.CHUNK_FILES);
 
-      await fsPromises.writeFile(DiskCache.FILE_NAME, JSON.stringify({
-        blocks: blocks.getBlocks(),
-        mempool: {},
-        mempoolArray: mempoolArray.splice(0, chunkSize),
-      }), {flag: 'w'});
-      for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
-        await fsPromises.writeFile(DiskCache.FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
+      if (sync) {
+        fs.writeFileSync(DiskCache.TMP_FILE_NAME, JSON.stringify({
+          network: config.MEMPOOL.NETWORK,
+          cacheSchemaVersion: this.cacheSchemaVersion,
+          blocks: blocks.getBlocks(),
+          blockSummaries: blocks.getBlockSummaries(),
           mempool: {},
           mempoolArray: mempoolArray.splice(0, chunkSize),
-        }), {flag: 'w'});
+        }), { flag: 'w' });
+        for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
+          fs.writeFileSync(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
+            mempool: {},
+            mempoolArray: mempoolArray.splice(0, chunkSize),
+          }), { flag: 'w' });
+        }
+
+        fs.renameSync(DiskCache.TMP_FILE_NAME, DiskCache.FILE_NAME);
+        for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
+          fs.renameSync(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), DiskCache.FILE_NAMES.replace('{number}', i.toString()));
+        }
+      } else {
+        await fsPromises.writeFile(DiskCache.TMP_FILE_NAME, JSON.stringify({
+          network: config.MEMPOOL.NETWORK,
+          cacheSchemaVersion: this.cacheSchemaVersion,
+          blocks: blocks.getBlocks(),
+          blockSummaries: blocks.getBlockSummaries(),
+          mempool: {},
+          mempoolArray: mempoolArray.splice(0, chunkSize),
+        }), { flag: 'w' });
+        for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
+          await fsPromises.writeFile(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
+            mempool: {},
+            mempoolArray: mempoolArray.splice(0, chunkSize),
+          }), { flag: 'w' });
+        }
+
+        await fsPromises.rename(DiskCache.TMP_FILE_NAME, DiskCache.FILE_NAME);
+        for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
+          await fsPromises.rename(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), DiskCache.FILE_NAMES.replace('{number}', i.toString()));
+        }
       }
+
       logger.debug('Mempool and blocks data saved to disk cache');
       this.isWritingCache = false;
     } catch (e) {
-      logger.warn('Error writing to cache file: ' + e.message || e);
+      logger.warn('Error writing to cache file: ' + (e instanceof Error ? e.message : e));
       this.isWritingCache = false;
     }
   }
 
-  loadMempoolCache() {
+  wipeCache(): void {
+    logger.notice(`Wiping nodejs backend cache/cache*.json files`);
+    try {
+      fs.unlinkSync(DiskCache.FILE_NAME);
+    } catch (e: any) {
+      if (e?.code !== 'ENOENT') {
+        logger.err(`Cannot wipe cache file ${DiskCache.FILE_NAME}. Exception ${JSON.stringify(e)}`);
+      }
+    }
+
+    for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
+      const filename = DiskCache.FILE_NAMES.replace('{number}', i.toString());
+      try {
+        fs.unlinkSync(filename);
+      } catch (e: any) {
+        if (e?.code !== 'ENOENT') {
+          logger.err(`Cannot wipe cache file ${filename}. Exception ${JSON.stringify(e)}`);
+        }
+      }
+    }
+  }
+
+  loadMempoolCache(): void {
     if (!fs.existsSync(DiskCache.FILE_NAME)) {
       return;
     }
@@ -67,6 +132,15 @@ class DiskCache {
       if (cacheData) {
         logger.info('Restoring mempool and blocks data from disk cache');
         data = JSON.parse(cacheData);
+        if (data.cacheSchemaVersion === undefined || data.cacheSchemaVersion !== this.cacheSchemaVersion) {
+          logger.notice('Disk cache contains an outdated schema version. Clearing it and skipping the cache loading.');
+          return this.wipeCache();
+        }
+        if (data.network && data.network !== config.MEMPOOL.NETWORK) {
+          logger.notice('Disk cache contains data from a different network. Clearing it and skipping the cache loading.');
+          return this.wipeCache();
+        }
+
         if (data.mempoolArray) {
           for (const tx of data.mempoolArray) {
             data.mempool[tx.txid] = tx;
@@ -88,14 +162,15 @@ class DiskCache {
             }
           }
         } catch (e) {
-          logger.debug('Error parsing ' + fileName + '. Skipping.');
+          logger.info('Error parsing ' + fileName + '. Skipping. Reason: ' + (e instanceof Error ? e.message : e));
         }
       }
 
       memPool.setMempool(data.mempool);
       blocks.setBlocks(data.blocks);
+      blocks.setBlockSummaries(data.blockSummaries || []);
     } catch (e) {
-      logger.warn('Failed to parse mempoool and blocks cache. Skipping.');
+      logger.warn('Failed to parse mempoool and blocks cache. Skipping. Reason: ' + (e instanceof Error ? e.message : e));
     }
   }
 }

backend/src/api/explorer/channels.api.ts

@@ -0,0 +1,709 @@
+import logger from '../../logger';
+import DB from '../../database';
+import nodesApi from './nodes.api';
+import { ResultSetHeader } from 'mysql2';
+import { ILightningApi } from '../lightning/lightning-api.interface';
+import { Common } from '../common';
+
+class ChannelsApi {
+  public async $getAllChannels(): Promise<any[]> {
+    try {
+      const query = `SELECT * FROM channels`;
+      const [rows]: any = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('$getAllChannels error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getAllChannelsGeo(publicKey?: string, style?: string): Promise<any[]> {
+    try {
+      let select: string;
+      if (style === 'widget') {
+        select = `
+          nodes_1.latitude AS node1_latitude, nodes_1.longitude AS node1_longitude,
+          nodes_2.latitude AS node2_latitude, nodes_2.longitude AS node2_longitude
+        `;
+      } else {
+        select = `
+          nodes_1.public_key as node1_public_key, nodes_1.alias AS node1_alias,
+          nodes_1.latitude AS node1_latitude, nodes_1.longitude AS node1_longitude,
+          nodes_2.public_key as node2_public_key, nodes_2.alias AS node2_alias,
+          nodes_2.latitude AS node2_latitude, nodes_2.longitude AS node2_longitude
+        `;
+      }
+
+      const params: string[] = [];
+      let query = `SELECT ${select}
+        FROM channels
+        JOIN nodes AS nodes_1 on nodes_1.public_key = channels.node1_public_key
+        JOIN nodes AS nodes_2 on nodes_2.public_key = channels.node2_public_key
+        WHERE channels.status = 1
+          AND nodes_1.latitude IS NOT NULL AND nodes_1.longitude IS NOT NULL
+          AND nodes_2.latitude IS NOT NULL AND nodes_2.longitude IS NOT NULL
+      `;
+
+      if (publicKey !== undefined) {
+        query += ' AND (nodes_1.public_key = ? OR nodes_2.public_key = ?)';
+        params.push(publicKey);
+        params.push(publicKey);
+      } else {
+        query += ` AND channels.capacity > 1000000
+          GROUP BY nodes_1.public_key, nodes_2.public_key
+          ORDER BY channels.capacity DESC
+          LIMIT 10000
+        `;        
+      }
+
+      const [rows]: any = await DB.query(query, params);
+      return rows.map((row) => {
+        if (style === 'widget') {
+          return [
+            row.node1_longitude, row.node1_latitude,
+            row.node2_longitude, row.node2_latitude,
+          ];
+        } else {
+          return [
+            row.node1_public_key, row.node1_alias,
+            row.node1_longitude, row.node1_latitude,
+            row.node2_public_key, row.node2_alias,
+            row.node2_longitude, row.node2_latitude,
+          ];
+        }
+      });
+    } catch (e) {
+      logger.err('$getAllChannelsGeo error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $searchChannelsById(search: string): Promise<any[]> {
+    try {
+      const searchStripped = search.replace('%', '') + '%';
+      const query = `SELECT id, short_id, capacity, status FROM channels WHERE id LIKE ? OR short_id LIKE ? LIMIT 10`;
+      const [rows]: any = await DB.query(query, [searchStripped, searchStripped]);
+      return rows;
+    } catch (e) {
+      logger.err('$searchChannelsById error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getChannelsByStatus(status: number | number[]): Promise<any[]> {
+    try {
+      let query: string;
+      if (Array.isArray(status)) {
+        query = `SELECT * FROM channels WHERE status IN (${status.join(',')})`;
+      } else {
+        query = `SELECT * FROM channels WHERE status = ?`;
+      }
+      const [rows]: any = await DB.query(query, [status]);
+      return rows;
+    } catch (e) {
+      logger.err('$getChannelsByStatus error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getClosedChannelsWithoutReason(): Promise<any[]> {
+    try {
+      const query = `SELECT * FROM channels WHERE status = 2 AND closing_reason IS NULL AND closing_transaction_id != ''`;
+      const [rows]: any = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('$getClosedChannelsWithoutReason error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getUnresolvedClosedChannels(): Promise<any[]> {
+    try {
+      const query = `SELECT * FROM channels WHERE status = 2 AND closing_reason = 2 AND closing_resolved = 0 AND closing_transaction_id != ''`;
+      const [rows]: any = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('$getUnresolvedClosedChannels error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getChannelsWithoutSourceChecked(): Promise<any[]> {
+    try {
+      const query = `
+        SELECT channels.*
+        FROM channels
+        WHERE channels.source_checked != 1
+      `;
+      const [rows]: any = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('$getUnresolvedClosedChannels error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getChannelsWithoutCreatedDate(): Promise<any[]> {
+    try {
+      const query = `SELECT * FROM channels WHERE created IS NULL`;
+      const [rows]: any = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('$getChannelsWithoutCreatedDate error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getChannel(id: string): Promise<any> {
+    try {
+      const query = `
+        SELECT n1.alias AS alias_left, n1.longitude as node1_longitude, n1.latitude as node1_latitude,
+          n2.alias AS alias_right, n2.longitude as node2_longitude, n2.latitude as node2_latitude,
+          channels.*,
+          ns1.channels AS channels_left, ns1.capacity AS capacity_left, ns2.channels AS channels_right, ns2.capacity AS capacity_right
+        FROM channels
+        LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key
+        LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key
+        LEFT JOIN node_stats AS ns1 ON ns1.public_key = channels.node1_public_key
+        LEFT JOIN node_stats AS ns2 ON ns2.public_key = channels.node2_public_key
+        WHERE (
+          ns1.id = (
+            SELECT MAX(id)
+            FROM node_stats
+            WHERE public_key = channels.node1_public_key
+          )
+          AND ns2.id = (
+            SELECT MAX(id)
+            FROM node_stats
+            WHERE public_key = channels.node2_public_key
+          )
+        )
+        AND channels.id = ?
+      `;
+
+      const [rows]: any = await DB.query(query, [id]);
+      if (rows[0]) {
+        return this.convertChannel(rows[0]);
+      }
+    } catch (e) {
+      logger.err('$getChannel error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getChannelsStats(): Promise<any> {
+    try {
+      // Feedback from zerofeerouting:
+      // "I would argue > 5000ppm can be ignored. Channels charging more than .5% fee are ignored by CLN for example."
+      const ignoredFeeRateThreshold = 5000;
+      const ignoredBaseFeeThreshold = 5000;
+
+      // Capacity
+      let query = `SELECT AVG(capacity) AS avgCapacity FROM channels WHERE status = 1 ORDER BY capacity`;
+      const [avgCapacity]: any = await DB.query(query);
+
+      query = `SELECT capacity FROM channels WHERE status = 1 ORDER BY capacity`;
+      let [capacity]: any = await DB.query(query);
+      capacity = capacity.map(capacity => capacity.capacity);
+      const medianCapacity = capacity[Math.floor(capacity.length / 2)];
+
+      // Fee rates
+      query = `SELECT node1_fee_rate FROM channels WHERE node1_fee_rate < ${ignoredFeeRateThreshold} AND status = 1`;
+      let [feeRates1]: any = await DB.query(query);
+      feeRates1 = feeRates1.map(rate => rate.node1_fee_rate);
+      query = `SELECT node2_fee_rate FROM channels WHERE node2_fee_rate < ${ignoredFeeRateThreshold} AND status = 1`;
+      let [feeRates2]: any = await DB.query(query);
+      feeRates2 = feeRates2.map(rate => rate.node2_fee_rate);
+
+      let feeRates = (feeRates1.concat(feeRates2)).sort((a, b) => a - b);
+      let avgFeeRate = 0;
+      for (const rate of feeRates) {
+        avgFeeRate += rate; 
+      }
+      avgFeeRate /= feeRates.length;
+      const medianFeeRate = feeRates[Math.floor(feeRates.length / 2)];
+
+      // Base fees
+      query = `SELECT node1_base_fee_mtokens FROM channels WHERE node1_base_fee_mtokens < ${ignoredBaseFeeThreshold} AND status = 1`;
+      let [baseFees1]: any = await DB.query(query);
+      baseFees1 = baseFees1.map(rate => rate.node1_base_fee_mtokens);
+      query = `SELECT node2_base_fee_mtokens FROM channels WHERE node2_base_fee_mtokens < ${ignoredBaseFeeThreshold} AND status = 1`;
+      let [baseFees2]: any = await DB.query(query);
+      baseFees2 = baseFees2.map(rate => rate.node2_base_fee_mtokens);
+
+      let baseFees = (baseFees1.concat(baseFees2)).sort((a, b) => a - b);
+      let avgBaseFee = 0;
+      for (const fee of baseFees) {
+        avgBaseFee += fee; 
+      }
+      avgBaseFee /= baseFees.length;
+      const medianBaseFee = feeRates[Math.floor(baseFees.length / 2)];
+      
+      return {
+        avgCapacity: parseInt(avgCapacity[0].avgCapacity, 10),
+        avgFeeRate: avgFeeRate,
+        avgBaseFee: avgBaseFee,
+        medianCapacity: medianCapacity,
+        medianFeeRate: medianFeeRate,
+        medianBaseFee: medianBaseFee,
+      }
+
+    } catch (e) {
+      logger.err(`Cannot calculate channels statistics. Reason: ${e instanceof Error ? e.message : e}`);
+      throw e;
+    }
+  }
+
+  public async $getChannelsByTransactionId(transactionIds: string[]): Promise<any[]> {
+    try {
+      const query = `
+        SELECT n1.alias AS alias_left, n2.alias AS alias_right, channels.*
+        FROM channels
+        LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key
+        LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key
+        WHERE channels.transaction_id IN ? OR channels.closing_transaction_id IN ?
+      `;
+      const [rows]: any = await DB.query(query, [[transactionIds], [transactionIds]]);
+      const channels = rows.map((row) => this.convertChannel(row));
+      return channels;
+    } catch (e) {
+      logger.err('$getChannelByTransactionId error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getChannelByClosingId(transactionId: string): Promise<any> {
+    try {
+      const query = `
+        SELECT
+          channels.*
+        FROM channels
+        WHERE channels.closing_transaction_id = ?
+      `;
+      const [rows]: any = await DB.query(query, [transactionId]);
+      if (rows.length > 0) {
+        rows[0].outputs = JSON.parse(rows[0].outputs);
+        return rows[0];
+      }
+    } catch (e) {
+      logger.err('$getChannelByClosingId error: ' + (e instanceof Error ? e.message : e));
+      // don't throw - this data isn't essential
+    }
+  }
+
+  public async $getChannelsByOpeningId(transactionId: string): Promise<any> {
+    try {
+      const query = `
+        SELECT
+          channels.*
+        FROM channels
+        WHERE channels.transaction_id = ?
+      `;
+      const [rows]: any = await DB.query(query, [transactionId]);
+      if (rows.length > 0) {
+        return rows.map(row => {
+          row.outputs = JSON.parse(row.outputs);
+          return row;
+        });
+      }
+    } catch (e) {
+      logger.err('$getChannelsByOpeningId error: ' + (e instanceof Error ? e.message : e));
+      // don't throw - this data isn't essential
+    }
+  }
+
+  public async $updateClosingInfo(channelInfo: { id: string, node1_closing_balance: number, node2_closing_balance: number, closed_by: string | null, closing_fee: number, outputs: ILightningApi.ForensicOutput[]}): Promise<void> {
+    try {
+      const query = `
+        UPDATE channels SET
+          node1_closing_balance = ?,
+          node2_closing_balance = ?,
+          closed_by = ?,
+          closing_fee = ?,
+          outputs = ?
+        WHERE channels.id = ?
+      `;
+      await DB.query<ResultSetHeader>(query, [
+        channelInfo.node1_closing_balance || 0,
+        channelInfo.node2_closing_balance || 0,
+        channelInfo.closed_by,
+        channelInfo.closing_fee || 0,
+        JSON.stringify(channelInfo.outputs),
+        channelInfo.id,
+      ]);
+    } catch (e) {
+      logger.err('$updateClosingInfo error: ' + (e instanceof Error ? e.message : e));
+      // don't throw - this data isn't essential
+    }
+  }
+
+  public async $updateOpeningInfo(channelInfo: { id: string, node1_funding_balance: number, node2_funding_balance: number, funding_ratio: number, single_funded: boolean | void }): Promise<void> {
+    try {
+      const query = `
+        UPDATE channels SET
+          node1_funding_balance = ?,
+          node2_funding_balance = ?,
+          funding_ratio = ?,
+          single_funded = ?
+        WHERE channels.id = ?
+      `;
+      await DB.query<ResultSetHeader>(query, [
+        channelInfo.node1_funding_balance || 0,
+        channelInfo.node2_funding_balance || 0,
+        channelInfo.funding_ratio,
+        channelInfo.single_funded ? 1 : 0,
+        channelInfo.id,
+      ]);
+    } catch (e) {
+      logger.err('$updateOpeningInfo error: ' + (e instanceof Error ? e.message : e));
+      // don't throw - this data isn't essential
+    }
+  }
+
+  public async $markChannelSourceChecked(id: string): Promise<void> {
+    try {
+      const query = `
+        UPDATE channels
+        SET source_checked = 1
+        WHERE id = ?
+      `;
+      await DB.query<ResultSetHeader>(query, [id]);
+    } catch (e) {
+      logger.err('$markChannelSourceChecked error: ' + (e instanceof Error ? e.message : e));
+      // don't throw - this data isn't essential
+    }
+  }
+
+  public async $getChannelsForNode(public_key: string, index: number, length: number, status: string): Promise<any[]> {
+    try {
+      let channelStatusFilter;
+      if (status === 'open') {
+        channelStatusFilter = '< 2';
+      } else if (status === 'active') {
+        channelStatusFilter = '= 1';
+      } else if (status === 'closed') {
+        channelStatusFilter = '= 2';
+      } else {
+        throw new Error('getChannelsForNode: Invalid status requested');
+      }
+
+      // Channels originating from node
+      let query = `
+        SELECT COALESCE(node2.alias, SUBSTRING(node2_public_key, 0, 20)) AS alias, COALESCE(node2.public_key, node2_public_key) AS public_key,
+          channels.status, channels.node1_fee_rate,
+          channels.capacity, channels.short_id, channels.id, channels.closing_reason,
+          UNIX_TIMESTAMP(closing_date) as closing_date, UNIX_TIMESTAMP(channels.updated_at) as updated_at
+        FROM channels
+        LEFT JOIN nodes AS node2 ON node2.public_key = channels.node2_public_key
+        WHERE node1_public_key = ? AND channels.status ${channelStatusFilter}
+      `;
+      const [channelsFromNode]: any = await DB.query(query, [public_key]);
+
+      // Channels incoming to node
+      query = `
+        SELECT COALESCE(node1.alias, SUBSTRING(node1_public_key, 0, 20)) AS alias, COALESCE(node1.public_key, node1_public_key) AS public_key,
+          channels.status, channels.node2_fee_rate,
+          channels.capacity, channels.short_id, channels.id, channels.closing_reason,
+          UNIX_TIMESTAMP(closing_date) as closing_date, UNIX_TIMESTAMP(channels.updated_at) as updated_at
+        FROM channels
+        LEFT JOIN nodes AS node1 ON node1.public_key = channels.node1_public_key
+        WHERE node2_public_key = ? AND channels.status ${channelStatusFilter}
+      `;
+      const [channelsToNode]: any = await DB.query(query, [public_key]);
+
+      let allChannels = channelsFromNode.concat(channelsToNode);
+      allChannels.sort((a, b) => {
+        if (status === 'closed') {
+          if (!b.closing_date && !a.closing_date) {
+            return (b.updated_at ?? 0) - (a.updated_at ?? 0);
+          } else {
+            return (b.closing_date ?? 0) - (a.closing_date ?? 0);
+          }
+        } else {
+          return b.capacity - a.capacity;
+        }
+      });
+
+      if (index >= 0) {
+        allChannels = allChannels.slice(index, index + length);
+      } else if (index === -1) { // Node channels tree chart
+        allChannels = allChannels.slice(0, 1000);
+      }
+
+      const channels: any[] = []
+      for (const row of allChannels) {
+        let channel;
+        if (index >= 0) {
+          const activeChannelsStats: any = await nodesApi.$getActiveChannelsStats(row.public_key);
+          channel = {
+            status: row.status,
+            closing_reason: row.closing_reason,
+            closing_date: row.closing_date,
+            capacity: row.capacity ?? 0,
+            short_id: row.short_id,
+            id: row.id,
+            fee_rate: row.node1_fee_rate ?? row.node2_fee_rate ?? 0,
+            node: {
+              alias: row.alias.length > 0 ? row.alias : row.public_key.slice(0, 20),
+              public_key: row.public_key,
+              channels: activeChannelsStats.active_channel_count ?? 0,
+              capacity: activeChannelsStats.capacity ?? 0,
+            }
+          };
+        } else if (index === -1) {
+          channel = {
+            capacity: row.capacity ?? 0,
+            short_id: row.short_id,
+            id: row.id,
+            node: {
+              alias: row.alias.length > 0 ? row.alias : row.public_key.slice(0, 20),
+              public_key: row.public_key,
+            }
+          };
+        }
+
+        channels.push(channel);
+      }
+
+      return channels;
+    } catch (e) {
+      logger.err('$getChannelsForNode error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getChannelsCountForNode(public_key: string, status: string): Promise<any> {
+    try {
+      // Default active and inactive channels
+      let statusQuery = '< 2';
+      // Closed channels only
+      if (status === 'closed') {
+        statusQuery = '= 2';
+      }
+      const query = `
+        SELECT COUNT(*) AS count
+        FROM channels
+        WHERE (node1_public_key = ? OR node2_public_key = ?)
+        AND status ${statusQuery}
+      `;
+      const [rows]: any = await DB.query(query, [public_key, public_key]);
+      return rows[0]['count'];
+    } catch (e) {
+      logger.err('$getChannelsForNode error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  private convertChannel(channel: any): any {
+    return {
+      'id': channel.id,
+      'short_id': channel.short_id,
+      'capacity': channel.capacity,
+      'transaction_id': channel.transaction_id,
+      'transaction_vout': channel.transaction_vout,
+      'closing_transaction_id': channel.closing_transaction_id,
+      'closing_fee': channel.closing_fee,
+      'closing_reason': channel.closing_reason,
+      'closing_date': channel.closing_date,
+      'updated_at': channel.updated_at,
+      'created': channel.created,
+      'status': channel.status,
+      'funding_ratio': channel.funding_ratio,
+      'closed_by': channel.closed_by,
+      'single_funded': !!channel.single_funded,
+      'node_left': {
+        'alias': channel.alias_left,
+        'public_key': channel.node1_public_key,
+        'channels': channel.channels_left,
+        'capacity': channel.capacity_left,
+        'base_fee_mtokens': channel.node1_base_fee_mtokens,
+        'cltv_delta': channel.node1_cltv_delta,
+        'fee_rate': channel.node1_fee_rate,
+        'is_disabled': channel.node1_is_disabled,
+        'max_htlc_mtokens': channel.node1_max_htlc_mtokens,
+        'min_htlc_mtokens': channel.node1_min_htlc_mtokens,
+        'updated_at': channel.node1_updated_at,
+        'longitude': channel.node1_longitude,
+        'latitude': channel.node1_latitude,
+        'funding_balance': channel.node1_funding_balance,
+        'closing_balance': channel.node1_closing_balance,
+        'initiated_close': channel.closed_by === channel.node1_public_key ? true : undefined,
+      },
+      'node_right': {
+        'alias': channel.alias_right,
+        'public_key': channel.node2_public_key,
+        'channels': channel.channels_right,
+        'capacity': channel.capacity_right,
+        'base_fee_mtokens': channel.node2_base_fee_mtokens,
+        'cltv_delta': channel.node2_cltv_delta,
+        'fee_rate': channel.node2_fee_rate,
+        'is_disabled': channel.node2_is_disabled,
+        'max_htlc_mtokens': channel.node2_max_htlc_mtokens,
+        'min_htlc_mtokens': channel.node2_min_htlc_mtokens,
+        'updated_at': channel.node2_updated_at,
+        'longitude': channel.node2_longitude,
+        'latitude': channel.node2_latitude,
+        'funding_balance': channel.node2_funding_balance,
+        'closing_balance': channel.node2_closing_balance,
+        'initiated_close': channel.closed_by === channel.node2_public_key ? true : undefined,
+      },
+    };
+  }
+
+  /**
+   * Save or update a channel present in the graph
+   */
+  public async $saveChannel(channel: ILightningApi.Channel, status = 1): Promise<void> {
+    const [ txid, vout ] = channel.chan_point.split(':');
+
+    const policy1: Partial<ILightningApi.RoutingPolicy> = channel.node1_policy || {};
+    const policy2: Partial<ILightningApi.RoutingPolicy> = channel.node2_policy || {};
+
+    // https://github.com/mempool/mempool/issues/3006
+    if ((channel.last_update ?? 0) < 1514736061) { // January 1st 2018
+      channel.last_update = null;
+    }
+    if ((policy1.last_update ?? 0) < 1514736061) { // January 1st 2018
+      policy1.last_update = null;
+    }
+    if ((policy2.last_update ?? 0) < 1514736061) { // January 1st 2018
+      policy2.last_update = null;
+    }
+
+    const query = `INSERT INTO channels
+      (
+        id,
+        short_id,
+        capacity,
+        transaction_id,
+        transaction_vout,
+        updated_at,
+        status,
+        node1_public_key,
+        node1_base_fee_mtokens,
+        node1_cltv_delta,
+        node1_fee_rate,
+        node1_is_disabled,
+        node1_max_htlc_mtokens,
+        node1_min_htlc_mtokens,
+        node1_updated_at,
+        node2_public_key,
+        node2_base_fee_mtokens,
+        node2_cltv_delta,
+        node2_fee_rate,
+        node2_is_disabled,
+        node2_max_htlc_mtokens,
+        node2_min_htlc_mtokens,
+        node2_updated_at
+      )
+      VALUES (?, ?, ?, ?, ?, ?, ${status}, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      ON DUPLICATE KEY UPDATE
+        capacity = ?,
+        updated_at = ?,
+        status = ${status},
+        node1_public_key = ?,
+        node1_base_fee_mtokens = ?,
+        node1_cltv_delta = ?,
+        node1_fee_rate = ?,
+        node1_is_disabled = ?,
+        node1_max_htlc_mtokens = ?,
+        node1_min_htlc_mtokens = ?,
+        node1_updated_at = ?,
+        node2_public_key = ?,
+        node2_base_fee_mtokens = ?,
+        node2_cltv_delta = ?,
+        node2_fee_rate = ?,
+        node2_is_disabled = ?,
+        node2_max_htlc_mtokens = ?,
+        node2_min_htlc_mtokens = ?,
+        node2_updated_at = ?
+      ;`;
+
+    await DB.query(query, [
+      Common.channelShortIdToIntegerId(channel.channel_id),
+      Common.channelIntegerIdToShortId(channel.channel_id),
+      channel.capacity,
+      txid,
+      vout,
+      Common.utcDateToMysql(channel.last_update),
+      channel.node1_pub,
+      policy1.fee_base_msat,
+      policy1.time_lock_delta,
+      policy1.fee_rate_milli_msat,
+      policy1.disabled,
+      policy1.max_htlc_msat,
+      policy1.min_htlc,
+      Common.utcDateToMysql(policy1.last_update),
+      channel.node2_pub,
+      policy2.fee_base_msat,
+      policy2.time_lock_delta,
+      policy2.fee_rate_milli_msat,
+      policy2.disabled,
+      policy2.max_htlc_msat,
+      policy2.min_htlc,
+      Common.utcDateToMysql(policy2.last_update),
+      channel.capacity,
+      Common.utcDateToMysql(channel.last_update),
+      channel.node1_pub,
+      policy1.fee_base_msat,
+      policy1.time_lock_delta,
+      policy1.fee_rate_milli_msat,
+      policy1.disabled,
+      policy1.max_htlc_msat,
+      policy1.min_htlc,
+      Common.utcDateToMysql(policy1.last_update),
+      channel.node2_pub,
+      policy2.fee_base_msat,
+      policy2.time_lock_delta,
+      policy2.fee_rate_milli_msat,
+      policy2.disabled,
+      policy2.max_htlc_msat,
+      policy2.min_htlc,
+      Common.utcDateToMysql(policy2.last_update)
+    ]);
+  }
+
+  /**
+   * Set all channels not in `graphChannelsIds` as inactive (status = 0)
+   */
+  public async $setChannelsInactive(graphChannelsIds: string[]): Promise<void> {
+    if (graphChannelsIds.length === 0) {
+      return;
+    }
+
+    try {
+      const result = await DB.query<ResultSetHeader>(`
+        UPDATE channels
+        SET status = 0
+        WHERE id NOT IN (
+          ${graphChannelsIds.map(id => `"${id}"`).join(',')}
+        )
+        AND status != 2
+      `);
+      if (result[0].changedRows ?? 0 > 0) {
+        logger.debug(`Marked ${result[0].changedRows} channels as inactive because they are not in the graph`, logger.tags.ln);
+      }
+    } catch (e) {
+      logger.err('$setChannelsInactive() error: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  public async $getLatestChannelUpdateForNode(publicKey: string): Promise<number> {
+    try {
+      const query = `
+        SELECT MAX(UNIX_TIMESTAMP(updated_at)) as updated_at
+        FROM channels
+        WHERE node1_public_key = ?
+      `;
+      const [rows]: any[] = await DB.query(query, [publicKey]);
+      if (rows.length > 0) {
+        return rows[0].updated_at;
+      }
+    } catch (e) {
+      logger.err(`Can't getLatestChannelUpdateForNode for ${publicKey}. Reason ${e instanceof Error ? e.message : e}`);
+    }
+    return 0;
+  }
+}
+
+export default new ChannelsApi();

backend/src/api/explorer/channels.routes.ts

@@ -0,0 +1,126 @@
+import config from '../../config';
+import { Application, Request, Response } from 'express';
+import channelsApi from './channels.api';
+
+class ChannelsRoutes {
+  constructor() { }
+
+  public initRoutes(app: Application) {
+    app
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels/txids', this.$getChannelsByTransactionIds)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels/search/:search', this.$searchChannelsById)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels/:short_id', this.$getChannel)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels', this.$getChannelsForNode)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels-geo', this.$getAllChannelsGeo)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels-geo/:publicKey', this.$getAllChannelsGeo)
+    ;
+  }
+
+  private async $searchChannelsById(req: Request, res: Response) {
+    try {
+      const channels = await channelsApi.$searchChannelsById(req.params.search);
+      res.json(channels);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getChannel(req: Request, res: Response) {
+    try {
+      const channel = await channelsApi.$getChannel(req.params.short_id);
+      if (!channel) {
+        res.status(404).send('Channel not found');
+        return;
+      }
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(channel);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getChannelsForNode(req: Request, res: Response) {
+    try {
+      if (typeof req.query.public_key !== 'string') {
+        res.status(400).send('Missing parameter: public_key');
+        return;
+      }
+
+      const index = parseInt(typeof req.query.index === 'string' ? req.query.index : '0', 10) || 0;
+      const status: string = typeof req.query.status === 'string' ? req.query.status : '';
+
+      if (index < -1) {
+        res.status(400).send('Invalid index');
+      }
+      if (['open', 'active', 'closed'].includes(status) === false) {
+        res.status(400).send('Invalid status');
+      }
+
+      const channels = await channelsApi.$getChannelsForNode(req.query.public_key, index, 10, status);
+      const channelsCount = await channelsApi.$getChannelsCountForNode(req.query.public_key, status);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.header('X-Total-Count', channelsCount.toString());
+      res.json(channels);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getChannelsByTransactionIds(req: Request, res: Response): Promise<void> {
+    try {
+      if (!Array.isArray(req.query.txId)) {
+        res.status(400).send('Not an array');
+        return;
+      }
+      const txIds: string[] = [];
+      for (const _txId in req.query.txId) {
+        if (typeof req.query.txId[_txId] === 'string') {
+          txIds.push(req.query.txId[_txId].toString());
+        }
+      }
+      const channels = await channelsApi.$getChannelsByTransactionId(txIds);
+      const result: any[] = [];
+      for (const txid of txIds) {
+        const inputs: any = {};
+        const outputs: any = {};
+        // Assuming that we only have one lightning close input in each transaction. This may not be true in the future
+        const foundChannelsFromInput = channels.find((channel) => channel.closing_transaction_id === txid);
+        if (foundChannelsFromInput) {
+          inputs[0] = foundChannelsFromInput;
+        }
+        const foundChannelsFromOutputs = channels.filter((channel) => channel.transaction_id === txid);
+        for (const output of foundChannelsFromOutputs) {
+          outputs[output.transaction_vout] = output;
+        }
+        result.push({
+          inputs,
+          outputs,
+        });
+      }
+
+      res.json(result);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getAllChannelsGeo(req: Request, res: Response) {
+    try {
+      const style: string = typeof req.query.style === 'string' ? req.query.style : '';
+      const channels = await channelsApi.$getAllChannelsGeo(req.params?.publicKey, style);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(channels);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+}
+
+export default new ChannelsRoutes();

backend/src/api/explorer/general.routes.ts

@@ -0,0 +1,58 @@
+import config from '../../config';
+import { Application, Request, Response } from 'express';
+import nodesApi from './nodes.api';
+import channelsApi from './channels.api';
+import statisticsApi from './statistics.api';
+class GeneralLightningRoutes {
+  constructor() { }
+
+  public initRoutes(app: Application) {
+    app
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/search', this.$searchNodesAndChannels)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/statistics/latest', this.$getGeneralStats)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/statistics/:interval', this.$getStatistics)
+    ;
+  }
+
+  private async $searchNodesAndChannels(req: Request, res: Response) {
+    if (typeof req.query.searchText !== 'string') {
+      res.status(400).send('Missing parameter: searchText');
+      return;
+    }
+    try {
+      const nodes = await nodesApi.$searchNodeByPublicKeyOrAlias(req.query.searchText);
+      const channels = await channelsApi.$searchChannelsById(req.query.searchText);
+      res.json({
+        nodes: nodes,
+        channels: channels,
+      });
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getStatistics(req: Request, res: Response) {
+    try {
+      const statistics = await statisticsApi.$getStatistics(req.params.interval);
+      const statisticsCount = await statisticsApi.$getStatisticsCount();
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.header('X-total-count', statisticsCount.toString());
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(statistics);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getGeneralStats(req: Request, res: Response) {
+    try {
+      const statistics = await statisticsApi.$getLatestStatistics();
+      res.json(statistics);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+}
+
+export default new GeneralLightningRoutes();

backend/src/api/explorer/nodes.api.ts

@@ -0,0 +1,723 @@
+import logger from '../../logger';
+import DB from '../../database';
+import { ResultSetHeader } from 'mysql2';
+import { ILightningApi } from '../lightning/lightning-api.interface';
+import { ITopNodesPerCapacity, ITopNodesPerChannels } from '../../mempool.interfaces';
+
+class NodesApi {
+  public async $getWorldNodes(): Promise<any> {
+    try {
+      let query = `
+        SELECT nodes.public_key as publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
+        CAST(COALESCE(nodes.capacity, 0) as INT) as capacity,
+        CAST(COALESCE(nodes.channels, 0) as INT) as channels,
+        nodes.longitude, nodes.latitude,
+        geo_names_country.names as country, geo_names_iso.names as isoCode
+        FROM nodes
+        JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
+        JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+        WHERE status = 1 AND nodes.as_number IS NOT NULL
+        ORDER BY capacity
+      `;
+
+      const [nodes]: any[] = await DB.query(query);
+
+      for (let i = 0; i < nodes.length; ++i) {
+        nodes[i].country = JSON.parse(nodes[i].country);
+      }
+
+      query = `
+        SELECT MAX(nodes.capacity) as maxLiquidity, MAX(nodes.channels) as maxChannels
+        FROM nodes
+        WHERE status = 1 AND nodes.as_number IS NOT NULL
+      `;
+
+      const [maximums]: any[] = await DB.query(query);
+      
+      return {
+        maxLiquidity: maximums[0].maxLiquidity,
+        maxChannels: maximums[0].maxChannels,
+        nodes: nodes.map(node => [
+          node.longitude, node.latitude,
+          node.publicKey, node.alias, node.capacity, node.channels,
+          node.country, node.isoCode
+        ])
+      };
+    } catch (e) {
+      logger.err(`Can't get world nodes list. Reason: ${e instanceof Error ? e.message : e}`);
+    }
+  }
+
+  public async $getNode(public_key: string): Promise<any> {
+    try {
+      // General info
+      let query = `
+        SELECT public_key, alias, UNIX_TIMESTAMP(first_seen) AS first_seen,
+          UNIX_TIMESTAMP(updated_at) AS updated_at, color, sockets as sockets,
+          as_number, city_id, country_id, subdivision_id, longitude, latitude,
+          geo_names_iso.names as iso_code, geo_names_as.names as as_organization, geo_names_city.names as city,
+          geo_names_country.names as country, geo_names_subdivision.names as subdivision
+        FROM nodes
+        LEFT JOIN geo_names geo_names_as on geo_names_as.id = as_number
+        LEFT JOIN geo_names geo_names_city on geo_names_city.id = city_id
+        LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = subdivision_id
+        LEFT JOIN geo_names geo_names_country on geo_names_country.id = country_id
+        LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+        WHERE public_key = ?
+      `;
+      let [rows]: any[] = await DB.query(query, [public_key]);
+      if (rows.length === 0) {
+        throw new Error(`This node does not exist, or our node is not seeing it yet`);
+      }
+
+      const node = rows[0];
+      node.as_organization = JSON.parse(node.as_organization);
+      node.subdivision = JSON.parse(node.subdivision);
+      node.city = JSON.parse(node.city);
+      node.country = JSON.parse(node.country);
+
+      // Active channels and capacity
+      const activeChannelsStats: any = await this.$getActiveChannelsStats(public_key);
+      node.active_channel_count = activeChannelsStats.active_channel_count ?? 0;
+      node.capacity = activeChannelsStats.capacity ?? 0;
+
+      // Opened channels count
+      query = `
+        SELECT count(short_id) as opened_channel_count
+        FROM channels
+        WHERE status != 2 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)
+      `;
+      [rows] = await DB.query(query, [public_key, public_key]);
+      node.opened_channel_count = 0;
+      if (rows.length > 0) {
+        node.opened_channel_count = rows[0].opened_channel_count;
+      }
+
+      // Closed channels count
+      query = `
+        SELECT count(short_id) as closed_channel_count
+        FROM channels
+        WHERE status = 2 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)
+      `;
+      [rows] = await DB.query(query, [public_key, public_key]);
+      node.closed_channel_count = 0;
+      if (rows.length > 0) {
+        node.closed_channel_count = rows[0].closed_channel_count;
+      }
+
+      // Custom records
+      query = `
+        SELECT type, payload
+        FROM nodes_records
+        WHERE public_key = ?
+      `;
+      [rows] = await DB.query(query, [public_key]);
+      node.custom_records = {};
+      for (const record of rows) {
+        node.custom_records[record.type] = Buffer.from(record.payload, 'binary').toString('hex');
+      }
+
+      return node;
+    } catch (e) {
+      logger.err(`Cannot get node information for ${public_key}. Reason: ${(e instanceof Error ? e.message : e)}`);
+      throw e;
+    }
+  }
+
+  public async $getActiveChannelsStats(node_public_key: string): Promise<unknown> {
+    const query = `
+      SELECT count(short_id) as active_channel_count, sum(capacity) as capacity
+      FROM channels
+      WHERE status = 1 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)
+    `;
+    const [rows]: any[] = await DB.query(query, [node_public_key, node_public_key]);
+    if (rows.length > 0) {
+      return {
+        active_channel_count: rows[0].active_channel_count,
+        capacity: rows[0].capacity
+      };
+    } else {
+      return null;
+    }
+  }
+
+  public async $getFeeHistogram(node_public_key: string): Promise<unknown> {
+    try {
+      const inQuery = `
+        SELECT CASE WHEN fee_rate <= 10.0 THEN CEIL(fee_rate)
+                    WHEN (fee_rate > 10.0 and fee_rate <= 100.0) THEN CEIL(fee_rate / 10.0) * 10.0
+                    WHEN (fee_rate > 100.0 and fee_rate <= 1000.0) THEN CEIL(fee_rate / 100.0) * 100.0
+                    WHEN fee_rate > 1000.0 THEN CEIL(fee_rate / 1000.0) * 1000.0
+               END as bucket,
+               count(short_id) as count,
+               sum(capacity) as capacity
+        FROM (
+          SELECT CASE WHEN node1_public_key = ? THEN node2_fee_rate WHEN node2_public_key = ? THEN node1_fee_rate END as fee_rate,
+                 short_id as short_id,
+                 capacity as capacity
+          FROM channels
+          WHERE status = 1 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)
+        ) as fee_rate_table
+        GROUP BY bucket;
+      `;
+      const [inRows]: any[] = await DB.query(inQuery, [node_public_key, node_public_key, node_public_key, node_public_key]);
+
+      const outQuery = `
+        SELECT CASE WHEN fee_rate <= 10.0 THEN CEIL(fee_rate)
+                    WHEN (fee_rate > 10.0 and fee_rate <= 100.0) THEN CEIL(fee_rate / 10.0) * 10.0
+                    WHEN (fee_rate > 100.0 and fee_rate <= 1000.0) THEN CEIL(fee_rate / 100.0) * 100.0
+                    WHEN fee_rate > 1000.0 THEN CEIL(fee_rate / 1000.0) * 1000.0
+               END as bucket,
+               count(short_id) as count,
+               sum(capacity) as capacity
+        FROM (
+          SELECT CASE WHEN node1_public_key = ? THEN node1_fee_rate WHEN node2_public_key = ? THEN node2_fee_rate END as fee_rate,
+                 short_id as short_id,
+                 capacity as capacity
+          FROM channels
+          WHERE status = 1 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)
+        ) as fee_rate_table
+        GROUP BY bucket;
+      `;
+      const [outRows]: any[] = await DB.query(outQuery, [node_public_key, node_public_key, node_public_key, node_public_key]);
+
+      return {
+        incoming: inRows.length > 0 ? inRows : [],
+        outgoing: outRows.length > 0 ? outRows : [],
+      };
+    } catch (e) {
+      logger.err(`Cannot get node fee distribution for ${node_public_key}. Reason: ${(e instanceof Error ? e.message : e)}`);
+      throw e;
+    }
+  }
+
+  public async $getAllNodes(): Promise<any> {
+    try {
+      const query = `SELECT * FROM nodes`;
+      const [rows]: any = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('$getAllNodes error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getNodeStats(public_key: string): Promise<any> {
+    try {
+      const query = `
+        SELECT UNIX_TIMESTAMP(added) AS added, capacity, channels
+        FROM node_stats
+        WHERE public_key = ?
+        ORDER BY added DESC
+      `;
+      const [rows]: any = await DB.query(query, [public_key]);
+      return rows;
+    } catch (e) {
+      logger.err('$getNodeStats error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getTopCapacityNodes(full: boolean): Promise<ITopNodesPerCapacity[]> {
+    try {
+      let rows: any;
+      let query: string;
+      if (full === false) {
+        query = `
+          SELECT nodes.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
+            nodes.capacity
+          FROM nodes
+          ORDER BY capacity DESC
+          LIMIT 6
+        `;
+
+        [rows] = await DB.query(query);
+      } else {
+        query = `
+          SELECT nodes.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
+            CAST(COALESCE(nodes.capacity, 0) as INT) as capacity,
+            CAST(COALESCE(nodes.channels, 0) as INT) as channels,
+            UNIX_TIMESTAMP(nodes.first_seen) as firstSeen, UNIX_TIMESTAMP(nodes.updated_at) as updatedAt,
+            geo_names_city.names as city, geo_names_country.names as country,
+            geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision
+          FROM nodes
+          LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
+          LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
+          LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+          LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
+          ORDER BY capacity DESC
+          LIMIT 100
+        `;
+
+        [rows] = await DB.query(query);
+        for (let i = 0; i < rows.length; ++i) {
+          rows[i].country = JSON.parse(rows[i].country);
+          rows[i].city = JSON.parse(rows[i].city);
+        }
+      }
+
+      return rows;
+    } catch (e) {
+      logger.err('$getTopCapacityNodes error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getTopChannelsNodes(full: boolean): Promise<ITopNodesPerChannels[]> {
+    try {
+      let rows: any;
+      let query: string;
+      if (full === false) {
+        query = `
+          SELECT
+            nodes.public_key as publicKey,
+            IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
+            nodes.channels,
+            geo_names_city.names as city, geo_names_country.names as country,
+            geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision
+          FROM nodes
+          LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
+          LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
+          LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+          LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
+          ORDER BY channels DESC
+          LIMIT 6;
+        `;
+
+        [rows] = await DB.query(query);
+        for (let i = 0; i < rows.length; ++i) {
+          rows[i].country = JSON.parse(rows[i].country);
+          rows[i].city = JSON.parse(rows[i].city);
+        }
+      } else {
+        query = `
+          SELECT nodes.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
+            CAST(COALESCE(nodes.channels, 0) as INT) as channels,
+            CAST(COALESCE(nodes.capacity, 0) as INT) as capacity,
+            UNIX_TIMESTAMP(nodes.first_seen) as firstSeen, UNIX_TIMESTAMP(nodes.updated_at) as updatedAt,
+            geo_names_city.names as city, geo_names_country.names as country,
+            geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision
+          FROM nodes
+          LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
+          LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
+          LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+          LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
+          ORDER BY channels DESC
+          LIMIT 100
+        `;
+
+        [rows] = await DB.query(query);
+        for (let i = 0; i < rows.length; ++i) {
+          rows[i].country = JSON.parse(rows[i].country);
+          rows[i].city = JSON.parse(rows[i].city);
+        }
+      }
+
+      return rows;
+    } catch (e) {
+      logger.err('$getTopChannelsNodes error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getOldestNodes(full: boolean): Promise<ITopNodesPerChannels[]> {
+    try {
+      let [rows]: any[] = await DB.query('SELECT UNIX_TIMESTAMP(MAX(added)) as maxAdded FROM node_stats');
+      const latestDate = rows[0].maxAdded;
+
+      let query: string;
+      if (full === false) {
+        query = `
+          SELECT nodes.public_key, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
+            node_stats.channels
+          FROM node_stats
+          JOIN nodes ON nodes.public_key = node_stats.public_key
+          WHERE added = FROM_UNIXTIME(${latestDate})
+          ORDER BY first_seen
+          LIMIT 100;
+        `;
+
+        [rows] = await DB.query(query);
+      } else {
+        query = `
+          SELECT node_stats.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(node_stats.public_key, 1, 20), alias) as alias,
+            CAST(COALESCE(node_stats.channels, 0) as INT) as channels,
+            CAST(COALESCE(node_stats.capacity, 0) as INT) as capacity,
+            UNIX_TIMESTAMP(nodes.first_seen) as firstSeen, UNIX_TIMESTAMP(nodes.updated_at) as updatedAt,
+            geo_names_city.names as city, geo_names_country.names as country,
+            geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision
+          FROM node_stats
+          RIGHT JOIN nodes ON nodes.public_key = node_stats.public_key
+          LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
+          LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
+          LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+          LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
+          WHERE added = FROM_UNIXTIME(${latestDate})
+          ORDER BY first_seen
+          LIMIT 100
+        `;
+
+        [rows] = await DB.query(query);
+        for (let i = 0; i < rows.length; ++i) {
+          rows[i].country = JSON.parse(rows[i].country);
+          rows[i].city = JSON.parse(rows[i].city);
+        }
+      }
+
+      return rows;
+    } catch (e) {
+      logger.err('$getTopChannelsNodes error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $searchNodeByPublicKeyOrAlias(search: string) {
+    try {
+      const publicKeySearch = search.replace('%', '') + '%';
+      const aliasSearch = search
+        .replace(/[-_.]/g, ' ') // Replace all -_. characters with empty space. Eg: "ln.nicehash" becomes "ln nicehash".  
+        .replace(/[^a-zA-Z0-9 ]/g, '') // Remove all special characters and keep just A to Z, 0 to 9.
+        .split(' ')
+        .filter(key => key.length)
+        .map((search) => '+' + search + '*').join(' ');
+      // %keyword% is wildcard search and can't be indexed so it's slower as the node database grow. keyword% can be indexed but then you can't search for "Nicehash" and get result for ln.nicehash.com. So we use fulltext index for words "ln, nicehash, com" and nicehash* will find it instantly.
+      const query = `SELECT public_key, alias, capacity, channels, status FROM nodes WHERE public_key LIKE ? OR MATCH alias_search AGAINST (? IN BOOLEAN MODE) ORDER BY capacity DESC LIMIT 10`;
+      const [rows]: any = await DB.query(query, [publicKeySearch, aliasSearch]);
+      return rows;
+    } catch (e) {
+      logger.err('$searchNodeByPublicKeyOrAlias error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getNodesISPRanking() {
+    try {
+      let query = '';
+
+      // List all channels and the two linked ISP
+      query = `
+        SELECT short_id, channels.capacity,
+          channels.node1_public_key AS node1PublicKey, isp1.names AS isp1, isp1.id as isp1ID,
+          channels.node2_public_key AS node2PublicKey, isp2.names AS isp2, isp2.id as isp2ID
+        FROM channels
+        JOIN nodes node1 ON node1.public_key = channels.node1_public_key
+        JOIN nodes node2 ON node2.public_key = channels.node2_public_key
+        JOIN geo_names isp1 ON isp1.id = node1.as_number
+        JOIN geo_names isp2 ON isp2.id = node2.as_number
+        WHERE channels.status = 1
+        ORDER BY short_id DESC
+      `;
+      const [channelsIsp]: any = await DB.query(query);
+
+      // Sum channels capacity and node count per ISP
+      const ispList = {};
+      for (const channel of channelsIsp) {
+        const isp1 = JSON.parse(channel.isp1);
+        const isp2 = JSON.parse(channel.isp2);
+
+        if (!ispList[isp1]) {
+          ispList[isp1] = {
+            ids: [channel.isp1ID],
+            capacity: 0,
+            channels: 0,
+            nodes: {},
+          };
+        } else if (ispList[isp1].ids.includes(channel.isp1ID) === false) {
+          ispList[isp1].ids.push(channel.isp1ID);
+        }
+
+        if (!ispList[isp2]) {
+          ispList[isp2] = {
+            ids: [channel.isp2ID],
+            capacity: 0,
+            channels: 0,
+            nodes: {},
+          };
+        } else if (ispList[isp2].ids.includes(channel.isp2ID) === false) {
+          ispList[isp2].ids.push(channel.isp2ID);
+        }
+        
+        ispList[isp1].capacity += channel.capacity;
+        ispList[isp1].channels += 1;
+        ispList[isp1].nodes[channel.node1PublicKey] = true;
+        ispList[isp2].capacity += channel.capacity;
+        ispList[isp2].channels += 1;
+        ispList[isp2].nodes[channel.node2PublicKey] = true;
+      }
+      
+      const ispRanking: any[] = [];
+      for (const isp of Object.keys(ispList)) {
+        ispRanking.push([
+          ispList[isp].ids.sort((a, b) => a - b).join(','),
+          isp,
+          ispList[isp].capacity,
+          ispList[isp].channels,
+          Object.keys(ispList[isp].nodes).length,
+        ]);
+      }
+
+      // Total active channels capacity
+      query = `SELECT SUM(capacity) AS capacity FROM channels WHERE status = 1`;
+      const [totalCapacity]: any = await DB.query(query);
+
+      // Get the total capacity of all channels which have at least one node on clearnet
+      query = `
+        SELECT SUM(capacity) as capacity
+        FROM (
+          SELECT capacity, GROUP_CONCAT(socket1.type, socket2.type) as networks
+          FROM channels
+          JOIN nodes_sockets socket1 ON node1_public_key = socket1.public_key
+          JOIN nodes_sockets socket2 ON node2_public_key = socket2.public_key
+          AND channels.status = 1
+          GROUP BY short_id
+        ) channels_tmp
+        WHERE channels_tmp.networks LIKE '%ipv%'
+      `;
+      const [clearnetCapacity]: any = await DB.query(query);
+
+      // Get the total capacity of all channels which have both nodes on Tor 
+      query = `
+        SELECT SUM(capacity) as capacity
+        FROM (
+          SELECT capacity, GROUP_CONCAT(socket1.type, socket2.type) as networks
+          FROM channels
+          JOIN nodes_sockets socket1 ON node1_public_key = socket1.public_key
+          JOIN nodes_sockets socket2 ON node2_public_key = socket2.public_key
+          AND channels.status = 1
+          GROUP BY short_id
+        ) channels_tmp
+        WHERE channels_tmp.networks NOT LIKE '%ipv%' AND
+          channels_tmp.networks NOT LIKE '%dns%' AND
+          channels_tmp.networks NOT LIKE '%websocket%'
+      `;
+      const [torCapacity]: any = await DB.query(query);
+
+      const clearnetCapacityValue = parseInt(clearnetCapacity[0].capacity, 10);
+      const torCapacityValue = parseInt(torCapacity[0].capacity, 10);
+      const unknownCapacityValue = parseInt(totalCapacity[0].capacity) - clearnetCapacityValue - torCapacityValue;
+
+      return {
+        clearnetCapacity: clearnetCapacityValue,
+        torCapacity: torCapacityValue,
+        unknownCapacity: unknownCapacityValue,
+        ispRanking: ispRanking,
+      };
+    } catch (e) {
+      logger.err(`Cannot get LN ISP ranking. Reason: ${e instanceof Error ? e.message : e}`);
+      throw e;
+    }
+  }
+
+  public async $getNodesPerCountry(countryId: string) {
+    try {
+      const query = `
+        SELECT nodes.public_key, CAST(COALESCE(nodes.capacity, 0) as INT) as capacity, CAST(COALESCE(nodes.channels, 0) as INT) as channels,
+          nodes.alias, UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at,
+          geo_names_city.names as city, geo_names_country.names as country,
+          geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision,
+          nodes.longitude, nodes.latitude, nodes.as_number, geo_names_isp.names as isp
+        FROM nodes
+        LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
+        LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
+        LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+        LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
+        LEFT JOIN geo_names geo_names_isp on geo_names_isp.id = nodes.as_number AND geo_names_isp.type = 'as_organization'
+        WHERE geo_names_country.id = ?
+        ORDER BY capacity DESC
+      `;
+
+      const [rows]: any = await DB.query(query, [countryId]);
+      for (let i = 0; i < rows.length; ++i) {
+        rows[i].country = JSON.parse(rows[i].country);
+        rows[i].city = JSON.parse(rows[i].city);
+        rows[i].subdivision = JSON.parse(rows[i].subdivision);
+        rows[i].isp = JSON.parse(rows[i].isp);
+      }
+      return rows;
+    } catch (e) {
+      logger.err(`Cannot get nodes for country id ${countryId}. Reason: ${e instanceof Error ? e.message : e}`);
+      throw e;
+    }
+  }
+
+  public async $getNodesPerISP(ISPId: string) {
+    try {
+      let query = `
+        SELECT channels.node1_public_key AS node1PublicKey, isp1.id as isp1ID,
+          channels.node2_public_key AS node2PublicKey, isp2.id as isp2ID
+        FROM channels
+        JOIN nodes node1 ON node1.public_key = channels.node1_public_key
+        JOIN nodes node2 ON node2.public_key = channels.node2_public_key
+        JOIN geo_names isp1 ON isp1.id = node1.as_number
+        JOIN geo_names isp2 ON isp2.id = node2.as_number
+        WHERE channels.status = 1 AND (node1.as_number IN (?) OR node2.as_number IN (?))
+        ORDER BY short_id DESC
+      `;
+
+      const IPSIds = ISPId.split(',');
+      const [rows]: any = await DB.query(query, [IPSIds, IPSIds]);
+      if (!rows || rows.length === 0) {
+        return [];
+      }
+
+      const nodes = {};
+
+      const intISPIds: number[] = [];
+      for (const ispId of IPSIds) {
+        intISPIds.push(parseInt(ispId, 10));
+      }
+
+      for (const channel of rows) {
+        if (intISPIds.includes(channel.isp1ID)) {
+          nodes[channel.node1PublicKey] = true;
+        }
+        if (intISPIds.includes(channel.isp2ID)) {
+          nodes[channel.node2PublicKey] = true;
+        }
+      }
+
+      query = `
+        SELECT nodes.public_key, CAST(COALESCE(nodes.capacity, 0) as INT) as capacity, CAST(COALESCE(nodes.channels, 0) as INT) as channels,
+          nodes.alias, UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at,
+          geo_names_city.names as city, geo_names_country.names as country,
+          geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision,
+          nodes.longitude, nodes.latitude
+        FROM nodes
+        LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
+        LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
+        LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+        LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
+        WHERE nodes.public_key IN (?)
+        ORDER BY capacity DESC
+      `;
+
+      const [rows2]: any = await DB.query(query, [Object.keys(nodes)]);
+      for (let i = 0; i < rows2.length; ++i) {
+        rows2[i].country = JSON.parse(rows2[i].country);
+        rows2[i].city = JSON.parse(rows2[i].city);
+        rows2[i].subdivision = JSON.parse(rows2[i].subdivision);
+      }
+      return rows2;
+
+    } catch (e) {
+      logger.err(`Cannot get nodes for ISP id ${ISPId}. Reason: ${e instanceof Error ? e.message : e}`);
+      throw e;
+    }
+  }
+
+  public async $getNodesCountries() {
+    try {
+      let query = `SELECT geo_names.names as names, geo_names_iso.names as iso_code, COUNT(DISTINCT nodes.public_key) as nodesCount, SUM(capacity) as capacity
+        FROM nodes
+        JOIN geo_names ON geo_names.id = nodes.country_id AND geo_names.type = 'country'
+        JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+        GROUP BY country_id
+        ORDER BY COUNT(DISTINCT nodes.public_key) DESC
+      `;
+      const [nodesCountPerCountry]: any = await DB.query(query);
+
+      query = `SELECT COUNT(*) as total FROM nodes WHERE country_id IS NOT NULL`;
+      const [nodesWithAS]: any = await DB.query(query);
+
+      const nodesPerCountry: any[] = [];
+      for (const country of nodesCountPerCountry) {
+        nodesPerCountry.push({
+          name: JSON.parse(country.names),
+          iso: country.iso_code, 
+          count: country.nodesCount,
+          share: Math.floor(country.nodesCount / nodesWithAS[0].total * 10000) / 100,
+          capacity: country.capacity,
+        })
+      }
+
+      return nodesPerCountry;
+    } catch (e) {
+      logger.err(`Cannot get nodes grouped by AS. Reason: ${e instanceof Error ? e.message : e}`);
+      throw e;
+    }
+  }
+
+  /**
+   * Save or update a node present in the graph
+   */
+  public async $saveNode(node: ILightningApi.Node): Promise<void> {
+    try {
+      // https://github.com/mempool/mempool/issues/3006
+      if ((node.last_update ?? 0) < 1514736061) { // January 1st 2018
+        node.last_update = null;
+      }
+  
+      const sockets = (node.addresses?.map(a => a.addr).join(',')) ?? '';
+      const query = `INSERT INTO nodes(
+          public_key,
+          first_seen,
+          updated_at,
+          alias,
+          alias_search,
+          color,
+          sockets,
+          status
+        )
+        VALUES (?, NOW(), FROM_UNIXTIME(?), ?, ?, ?, ?, 1)
+        ON DUPLICATE KEY UPDATE updated_at = FROM_UNIXTIME(?), alias = ?, alias_search = ?, color = ?, sockets = ?, status = 1`;
+
+      await DB.query(query, [
+        node.pub_key,
+        node.last_update,
+        node.alias,
+        this.aliasToSearchText(node.alias),
+        node.color,
+        sockets,
+        node.last_update,
+        node.alias,
+        this.aliasToSearchText(node.alias),
+        node.color,
+        sockets,
+      ]);
+    } catch (e) {
+      logger.err('$saveNode() error: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  /**
+   * Update node sockets
+   */
+  public async $updateNodeSockets(publicKey: string, sockets: {network: string; addr: string}[]): Promise<void> {
+    const formattedSockets = (sockets.map(a => a.addr).join(',')) ?? '';
+    try {
+      await DB.query(`UPDATE nodes SET sockets = ? WHERE public_key = ?`, [formattedSockets, publicKey]);
+    } catch (e) {
+      logger.err(`Cannot update node sockets for ${publicKey}. Reason: ${e instanceof Error ? e.message : e}`);
+    }
+  }
+
+  /**
+   * Set all nodes not in `nodesPubkeys` as inactive (status = 0)
+   */
+   public async $setNodesInactive(graphNodesPubkeys: string[]): Promise<void> {
+    if (graphNodesPubkeys.length === 0) {
+      return;
+    }
+
+    try {
+      const result = await DB.query<ResultSetHeader>(`
+        UPDATE nodes
+        SET status = 0
+        WHERE public_key NOT IN (
+          ${graphNodesPubkeys.map(pubkey => `"${pubkey}"`).join(',')}
+        )
+      `);
+      if (result[0].changedRows ?? 0 > 0) {
+        logger.debug(`Marked ${result[0].changedRows} nodes as inactive because they are not in the graph`, logger.tags.ln);
+      }
+    } catch (e) {
+      logger.err('$setNodesInactive() error: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  private aliasToSearchText(str: string): string {
+    return str.replace(/[-_.]/g, ' ').replace(/[^a-zA-Z0-9 ]/g, '');
+  }
+}
+
+export default new NodesApi();

backend/src/api/explorer/nodes.routes.ts

@@ -0,0 +1,316 @@
+import config from '../../config';
+import { Application, Request, Response } from 'express';
+import nodesApi from './nodes.api';
+import DB from '../../database';
+import { INodesRanking } from '../../mempool.interfaces';
+
+class NodesRoutes {
+  constructor() { }
+
+  public initRoutes(app: Application) {
+    app
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/world', this.$getWorldNodes)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/country/:country', this.$getNodesPerCountry)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/search/:search', this.$searchNode)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/isp-ranking', this.$getISPRanking)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/isp/:isp', this.$getNodesPerISP)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/countries', this.$getNodesCountries)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings', this.$getNodesRanking)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings/liquidity', this.$getTopNodesByCapacity)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings/connectivity', this.$getTopNodesByChannels)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings/age', this.$getOldestNodes)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/:public_key/statistics', this.$getHistoricalNodeStats)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/:public_key/fees/histogram', this.$getFeeHistogram)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/:public_key', this.$getNode)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/group/:name', this.$getNodeGroup)
+    ;
+  }
+
+  private async $searchNode(req: Request, res: Response) {
+    try {
+      const nodes = await nodesApi.$searchNodeByPublicKeyOrAlias(req.params.search);
+      res.json(nodes);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getNodeGroup(req: Request, res: Response) {
+    try {
+      let nodesList;
+      let nodes: any[] = [];
+      switch (config.MEMPOOL.NETWORK) {
+        case 'testnet':
+          nodesList = [
+            '032c7c7819276c4f706a04df1a0f1e10a5495994a7be4c1d3d28ca766e5a2b957b',
+            '025a7e38c2834dd843591a4d23d5f09cdeb77ddca85f673c2d944a14220ff14cf7',
+            '0395e2731a1673ef21d7a16a727c4fc4d4c35a861c428ce2c819c53d2b81c8bd55',
+            '032ab2028c0b614c6d87824e2373529652fd7e4221b4c70cc4da7c7005c49afcf0',
+            '029001b22fe70b48bee12d014df91982eb85ff1bd404ec772d5c83c4ee3e88d2c3',
+            '0212e2848d79f928411da5f2ff0a8c95ec6ccb5a09d2031b6f71e91309dcde63af',
+            '03e871a2229523d34f76e6311ff197cfe7f26c2fbec13554b93a46f4e710c47dab',
+            '032202ec98d976b0e928bd1d91924e8bd3eab07231fc39feb3737b010071073df8',
+            '02fa7c5a948d03d563a9f36940c2205a814e594d17c0042ced242c71a857d72605',
+            '039c14fdec2d958e3d14cebf657451bbd9e039196615785e82c917f274e3fb2205',
+            '033589bbcb233ffc416cefd5437c7f37e9d7cb7942d405e39e72c4c846d9b37f18',
+            '029293110441c6e2eacb57e1255bf6ef05c41a6a676fe474922d33c19f98a7d584',
+            '0235ad0b56ed8c42c4354444c24e971c05e769ec0b5fb0ccea42880095dc02ea2c',
+            '029700819a37afea630f80e6cc461f3fd3c4ace2598a21cfbbe64d1c78d0ee69a5',
+            '02c2d8b2dbf87c7894af2f1d321290e2fe6db5446cd35323987cee98f06e2e0075',
+            '030b0ca1ea7b1075716d2a555630e6fd47ef11bc7391fe68963ec06cf370a5e382',
+            '031adb9eb2d66693f85fa31a4adca0319ba68219f3ad5f9a2ef9b34a6b40755fa1',
+            '02ccd07faa47eda810ecf5591ccf5ca50f6c1034d0d175052898d32a00b9bae24f',
+          ];
+          break;
+        case 'signet':
+          nodesList = [
+            '03ddab321b760433cbf561b615ef62ac7d318630c5f51d523aaf5395b90b751956',
+            '033d92c7bfd213ef1b34c90e985fb5dc77f9ec2409d391492484e57a44c4aca1de',
+            '02ad010dda54253c1eb9efe38b0760657a3b43ecad62198c359c051c9d99d45781',
+            '025196512905b8a3f1597428b867bec63ec9a95e5089eb7dc7e63e2d2691669029',
+            '027c625aa1fbe3768db68ebcb05b53b6dc0ce68b7b54b8900d326d167363e684fe',
+            '03f1629af3101fcc56b7aac2667016be84e3defbf3d0c8719f836c9b41c9a57a43',
+            '02dfb81e2f7a3c4c9e8a51b70ef82b4a24549cc2fab1f5b2fd636501774a918991',
+            '02d01ccf832944c68f10d39006093769c5b8bda886d561b128534e313d729fdb34',
+            '02499ed23027d4698a6904ff4ec1b6085a61f10b9a6937f90438f9947e38e8ea86',
+            '038310e3a786340f2bd7770704c7ccfe560fd163d9a1c99d67894597419d12cbf7',
+            '03e5e9d879b72c7d67ecd483bae023bd33e695bb32b981a4021260f7b9d62bc761',
+            '028d16e1a0ace4c0c0a421536d8d32ce484dfe6e2f726b7b0e7c30f12a195f8cc7',
+            '02ff690d06c187ab994bf83c5a2114fe5bf50112c2c817af0f788f736be9fa2070',
+            '02a9f570c51a2526a5ee85802e88f9281bed771eb66a0c8a7d898430dd5d0eae45',
+            '038c3de773255d3bd7a50e31e58d423baac5c90826a74d75e64b74c95475de1097',
+            '0242c7f7d315095f37ad1421ae0a2fc967d4cbe65b61b079c5395a769436959853',
+            '02a909e70eb03742f12666ebb1f56ac42a5fbaab0c0e8b5b1df4aa9f10f8a09240',
+            '03a26efa12489803c07f3ac2f1dba63812e38f0f6e866ce3ebb34df7de1f458cd2',
+          ];
+          break;
+        default:
+          nodesList = [
+            '03fbc17549ec667bccf397ababbcb4cdc0e3394345e4773079ab2774612ec9be61',
+            '03da9a8623241ccf95f19cd645c6cecd4019ac91570e976eb0a128bebbc4d8a437',
+            '03ca5340cf85cb2e7cf076e489f785410838de174e40be62723e8a60972ad75144',
+            '0238bd27f02d67d6c51e269692bc8c9a32357a00e7777cba7f4f1f18a2a700b108',
+            '03f983dcabed6baa1eab5b56c8b2e8fdc846ab3fd931155377897335e85a9fa57c',
+            '03e399589533581e48796e29a825839a010036a61b20744fda929d6709fcbffcc5',
+            '021f5288b5f72c42cd0d8801086af7ce09a816d8ee9a4c47a4b436399b26cb601a',
+            '032b01b7585f781420cd4148841a82831ba37fa952342052cec16750852d4f2dd9',
+            '02848036488d4b8fb1f1c4064261ec36151f43b085f0b51bd239ade3ddfc940c34',
+            '02b6b1640fe029e304c216951af9fbefdb23b0bdc9baaf327540d31b6107841fdf',
+            '03694289827203a5b3156d753071ddd5bf92e371f5a462943f9555eef6d2d6606c',
+            '0283d850db7c3e8ea7cc9c4abc7afaab12bbdf72b677dcba1d608350d2537d7d43',
+            '02521287789f851268a39c9eccc9d6180d2c614315b583c9e6ae0addbd6d79df06',
+            '0258c2a7b7f8af2585b4411b1ec945f70988f30412bb1df179de941f14d0b1bc3e',
+            '03c3389ff1a896f84d921ed01a19fc99c6724ce8dc4b960cd3b7b2362b62cd60d7',
+            '038d118996b3eaa15dcd317b32a539c9ecfdd7698f204acf8a087336af655a9192',
+            '02a928903d93d78877dacc3642b696128a3636e9566dd42d2d132325b2c8891c09',
+            '0328cd17f3a9d3d90b532ade0d1a67e05eb8a51835b3dce0a2e38eac04b5a62a57',
+          ];
+      }
+
+      for (let pubKey of nodesList) {
+        try {
+          const node = await nodesApi.$getNode(pubKey);
+          if (node) {
+            nodes.push(node);
+          }
+        } catch (e) {}
+      }
+      
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(nodes);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getNode(req: Request, res: Response) {
+    try {
+      const node = await nodesApi.$getNode(req.params.public_key);
+      if (!node) {
+        res.status(404).send('Node not found');
+        return;
+      }
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(node);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getHistoricalNodeStats(req: Request, res: Response) {
+    try {
+      const statistics = await nodesApi.$getNodeStats(req.params.public_key);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(statistics);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getFeeHistogram(req: Request, res: Response) {
+    try {
+      const node = await nodesApi.$getFeeHistogram(req.params.public_key);
+      if (!node) {
+        res.status(404).send('Node not found');
+        return;
+      }
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(node);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getNodesRanking(req: Request, res: Response): Promise<void> {
+    try {
+      const topCapacityNodes = await nodesApi.$getTopCapacityNodes(false);
+      const topChannelsNodes = await nodesApi.$getTopChannelsNodes(false);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(<INodesRanking>{
+        topByCapacity: topCapacityNodes,
+        topByChannels: topChannelsNodes,
+      });
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getTopNodesByCapacity(req: Request, res: Response): Promise<void> {
+    try {
+      const topCapacityNodes = await nodesApi.$getTopCapacityNodes(true);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(topCapacityNodes);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getTopNodesByChannels(req: Request, res: Response): Promise<void> {
+    try {
+      const topCapacityNodes = await nodesApi.$getTopChannelsNodes(true);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(topCapacityNodes);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getOldestNodes(req: Request, res: Response): Promise<void> {
+    try {
+      const topCapacityNodes = await nodesApi.$getOldestNodes(true);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(topCapacityNodes);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getISPRanking(req: Request, res: Response): Promise<void> {
+    try {
+      const nodesPerAs = await nodesApi.$getNodesISPRanking();
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 600).toUTCString());
+      res.json(nodesPerAs);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getWorldNodes(req: Request, res: Response) {
+    try {
+      const worldNodes = await nodesApi.$getWorldNodes();
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 600).toUTCString());
+      res.json(worldNodes);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getNodesPerCountry(req: Request, res: Response) {
+    try {
+      const [country]: any[] = await DB.query(
+        `SELECT geo_names.id, geo_names_country.names as country_names
+        FROM geo_names
+        JOIN geo_names geo_names_country on geo_names.id = geo_names_country.id AND geo_names_country.type = 'country'
+        WHERE geo_names.type = 'country_iso_code' AND geo_names.names = ?`,
+        [req.params.country]
+      );
+
+      if (country.length === 0) {
+        res.status(404).send(`This country does not exist or does not host any lightning nodes on clearnet`);
+        return;
+      }
+
+      const nodes = await nodesApi.$getNodesPerCountry(country[0].id);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json({
+        country: JSON.parse(country[0].country_names),
+        nodes: nodes,
+      });
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getNodesPerISP(req: Request, res: Response) {
+    try {
+      const [isp]: any[] = await DB.query(
+        `SELECT geo_names.names as isp_name
+        FROM geo_names
+        WHERE geo_names.type = 'as_organization' AND geo_names.id = ?`,
+        [req.params.isp]
+      );
+
+      if (isp.length === 0) {
+        res.status(404).send(`This ISP does not exist or does not host any lightning nodes on clearnet`);
+        return;
+      }
+
+      const nodes = await nodesApi.$getNodesPerISP(req.params.isp);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json({
+        isp: JSON.parse(isp[0].isp_name),
+        nodes: nodes,
+      });
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getNodesCountries(req: Request, res: Response) {
+    try {
+      const nodesPerAs = await nodesApi.$getNodesCountries();
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 600).toUTCString());
+      res.json(nodesPerAs);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+}
+
+export default new NodesRoutes();

backend/src/api/explorer/statistics.api.ts

@@ -0,0 +1,53 @@
+import logger from '../../logger';
+import DB from '../../database';
+import { Common } from '../common';
+
+class StatisticsApi {
+  public async $getStatistics(interval: string | null = null): Promise<any> {
+    interval = Common.getSqlInterval(interval);
+
+    let query = `SELECT UNIX_TIMESTAMP(added) AS added, channel_count, total_capacity,
+      tor_nodes, clearnet_nodes, unannounced_nodes, clearnet_tor_nodes
+      FROM lightning_stats`;
+
+    if (interval) {
+      query += ` WHERE added BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
+    }
+
+    query += ` ORDER BY added DESC`;
+
+    try {
+      const [rows]: any = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('$getStatistics error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getLatestStatistics(): Promise<any> {
+    try {
+      const [rows]: any = await DB.query(`SELECT * FROM lightning_stats ORDER BY added DESC LIMIT 1`);
+      const [rows2]: any = await DB.query(`SELECT * FROM lightning_stats WHERE DATE(added) = DATE(NOW() - INTERVAL 7 DAY)`);
+      return {
+        latest: rows[0],
+        previous: rows2[0],
+      };
+    } catch (e) {
+      logger.err('$getLatestStatistics error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getStatisticsCount(): Promise<number> {
+    try {
+      const [rows]: any = await DB.query(`SELECT count(*) as count FROM lightning_stats`);
+      return rows[0].count;
+    } catch (e) {
+      logger.err('$getLatestStatistics error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+}
+
+export default new StatisticsApi();

backend/src/api/fee-api.ts

@@ -1,12 +1,12 @@
-import config from '../config';
 import { MempoolBlock } from '../mempool.interfaces';
+import { Common } from './common';
 import mempool from './mempool';
 import projectedBlocks from './mempool-blocks';
 
 class FeeApi {
   constructor() { }
 
-  defaultFee = config.MEMPOOL.NETWORK === 'liquid' ? 0.1 : 1;
+  defaultFee = Common.isLiquid() ? 0.1 : 1;
 
   public getRecommendedFee() {
     const pBlocks = projectedBlocks.getMempoolBlocks();
@@ -18,6 +18,7 @@ class FeeApi {
         'fastestFee': this.defaultFee,
         'halfHourFee': this.defaultFee,
         'hourFee': this.defaultFee,
+        'economyFee': minimumFee,
         'minimumFee': minimumFee,
       };
     }
@@ -30,6 +31,7 @@ class FeeApi {
       'fastestFee': firstMedianFee,
       'halfHourFee': secondMedianFee,
       'hourFee': thirdMedianFee,
+      'economyFee': Math.min(2 * minimumFee, thirdMedianFee),
       'minimumFee': minimumFee,
     };
   }

backend/src/api/fetch-version.ts

@@ -0,0 +1,37 @@
+import fs from 'fs';
+import path from "path";
+const { spawnSync } = require('child_process');
+
+function getVersion(): string {
+  const packageJson = fs.readFileSync('package.json').toString();
+  return JSON.parse(packageJson).version;
+}
+
+function getGitCommit(): string {
+  if (process.env.MEMPOOL_COMMIT_HASH) {
+    return process.env.MEMPOOL_COMMIT_HASH;
+  } else {
+    const gitRevParse = spawnSync('git', ['rev-parse', '--short', 'HEAD']);
+    if (!gitRevParse.error) {
+      const output = gitRevParse.stdout.toString('utf-8').replace(/[\n\r\s]+$/, '');
+      if (output) {
+        return output;
+      } else {
+        console.log('Could not fetch git commit: No repo available');
+      }
+    } else if (gitRevParse.error.code === 'ENOENT') {
+      console.log('Could not fetch git commit: Command `git` is unavailable');
+    }
+  }
+  return '?';
+}
+
+const versionInfo = {
+  version: getVersion(),
+  gitCommit: getGitCommit()
+}
+
+fs.writeFileSync(
+  path.join(__dirname, 'version.json'),
+  JSON.stringify(versionInfo, null, 2) + "\n"
+);

backend/src/api/fiat-conversion.ts

@@ -1,43 +0,0 @@
-import logger from '../logger';
-import axios from 'axios';
-import { IConversionRates } from '../mempool.interfaces';
-
-class FiatConversion {
-  private conversionRates: IConversionRates = {
-    'USD': 0
-  };
-  private ratesChangedCallback: ((rates: IConversionRates) => void) | undefined;
-
-  constructor() { }
-
-  public setProgressChangedCallback(fn: (rates: IConversionRates) => void) {
-    this.ratesChangedCallback = fn;
-  }
-
-  public startService() {
-    logger.info('Starting currency rates service');
-    setInterval(this.updateCurrency.bind(this), 1000 * 60);
-    this.updateCurrency();
-  }
-
-  public getConversionRates() {
-    return this.conversionRates;
-  }
-
-  private async updateCurrency(): Promise<void> {
-    try {
-      const response = await axios.get('https://price.bisq.wiz.biz/getAllMarketPrices', { timeout: 10000 });
-      const usd = response.data.data.find((item: any) => item.currencyCode === 'USD');
-      this.conversionRates = {
-        'USD': usd.price,
-      };
-      if (this.ratesChangedCallback) {
-        this.ratesChangedCallback(this.conversionRates);
-      }
-    } catch (e) {
-      logger.err('Error updating fiat conversion rates: ' + e);
-    }
-  }
-}
-
-export default new FiatConversion();

backend/src/api/lightning/clightning/clightning-client.ts

@@ -0,0 +1,270 @@
+// Imported from https://github.com/shesek/lightning-client-js
+
+'use strict';
+
+const methods = [
+  'addgossip',
+  'autocleaninvoice',
+  'check',
+  'checkmessage',
+  'close',
+  'connect',
+  'createinvoice',
+  'createinvoicerequest',
+  'createoffer',
+  'createonion',
+  'decode',
+  'decodepay',
+  'delexpiredinvoice',
+  'delinvoice',
+  'delpay',
+  'dev-listaddrs',
+  'dev-rescan-outputs',
+  'disableoffer',
+  'disconnect',
+  'estimatefees',
+  'feerates',
+  'fetchinvoice',
+  'fundchannel',
+  'fundchannel_cancel',
+  'fundchannel_complete',
+  'fundchannel_start',
+  'fundpsbt',
+  'getchaininfo',
+  'getinfo',
+  'getlog',
+  'getrawblockbyheight',
+  'getroute',
+  'getsharedsecret',
+  'getutxout',
+  'help',
+  'invoice',
+  'keysend',
+  'legacypay',
+  'listchannels',
+  'listconfigs',
+  'listforwards',
+  'listfunds',
+  'listinvoices',
+  'listnodes',
+  'listoffers',
+  'listpays',
+  'listpeers',
+  'listsendpays',
+  'listtransactions',
+  'multifundchannel',
+  'multiwithdraw',
+  'newaddr',
+  'notifications',
+  'offer',
+  'offerout',
+  'openchannel_abort',
+  'openchannel_bump',
+  'openchannel_init',
+  'openchannel_signed',
+  'openchannel_update',
+  'pay',
+  'payersign',
+  'paystatus',
+  'ping',
+  'plugin',
+  'reserveinputs',
+  'sendinvoice',
+  'sendonion',
+  'sendonionmessage',
+  'sendpay',
+  'sendpsbt',
+  'sendrawtransaction',
+  'setchannelfee',
+  'signmessage',
+  'signpsbt',
+  'stop',
+  'txdiscard',
+  'txprepare',
+  'txsend',
+  'unreserveinputs',
+  'utxopsbt',
+  'waitanyinvoice',
+  'waitblockheight',
+  'waitinvoice',
+  'waitsendpay',
+  'withdraw'
+];
+
+
+import EventEmitter from 'events';
+import { existsSync, statSync } from 'fs';
+import { createConnection, Socket } from 'net';
+import { homedir } from 'os';
+import path from 'path';
+import { createInterface, Interface } from 'readline';
+import logger from '../../../logger';
+import { AbstractLightningApi } from '../lightning-api-abstract-factory';
+import { ILightningApi } from '../lightning-api.interface';
+import { convertAndmergeBidirectionalChannels, convertNode } from './clightning-convert';
+
+class LightningError extends Error {
+  type: string = 'lightning';
+  message: string = 'lightning-client error';
+
+  constructor(error) {
+    super();
+    this.type = error.type;
+    this.message = error.message;
+  }
+}
+
+const defaultRpcPath = path.join(homedir(), '.lightning')
+  , fStat = (...p) => statSync(path.join(...p))
+  , fExists = (...p) => existsSync(path.join(...p))
+
+export default class CLightningClient extends EventEmitter implements AbstractLightningApi {
+  private rpcPath: string;
+  private reconnectWait: number;
+  private reconnectTimeout;
+  private reqcount: number;
+  private client: Socket;
+  private rl: Interface;
+  private clientConnectionPromise: Promise<unknown>;
+
+  constructor(rpcPath = defaultRpcPath) {
+    if (!path.isAbsolute(rpcPath)) {
+      throw new Error('The rpcPath must be an absolute path');
+    }
+
+    if (!fExists(rpcPath) || !fStat(rpcPath).isSocket()) {
+      // network directory provided, use the lightning-rpc within in
+      if (fExists(rpcPath, 'lightning-rpc')) {
+        rpcPath = path.join(rpcPath, 'lightning-rpc');
+      }
+
+      // main data directory provided, default to using the bitcoin mainnet subdirectory
+      // to be removed in v0.2.0
+      else if (fExists(rpcPath, 'bitcoin', 'lightning-rpc')) {
+        logger.warn(`${rpcPath}/lightning-rpc is missing, using the bitcoin mainnet subdirectory at ${rpcPath}/bitcoin instead.`, logger.tags.ln)
+        logger.warn(`specifying the main lightning data directory is deprecated, please specify the network directory explicitly.\n`, logger.tags.ln)
+        rpcPath = path.join(rpcPath, 'bitcoin', 'lightning-rpc')
+      }
+    }
+
+    logger.debug(`Connecting to ${rpcPath}`, logger.tags.ln);
+
+    super();
+    this.rpcPath = rpcPath;
+    this.reconnectWait = 0.5;
+    this.reconnectTimeout = null;
+    this.reqcount = 0;
+
+    const _self = this;
+
+    this.client = createConnection(rpcPath).on(
+      'error', () => {
+        _self.increaseWaitTime();
+        _self.reconnect();
+      }
+    );
+    this.rl = createInterface({ input: this.client }).on(
+      'error', () => {
+        _self.increaseWaitTime();
+        _self.reconnect();
+      }
+    );
+
+    this.clientConnectionPromise = new Promise<void>(resolve => {
+      _self.client.on('connect', () => {
+        logger.info(`CLightning client connected`, logger.tags.ln);
+        _self.reconnectWait = 1;
+        resolve();
+      });
+
+      _self.client.on('end', () => {
+        logger.err(`CLightning client connection closed, reconnecting`, logger.tags.ln);
+        _self.increaseWaitTime();
+        _self.reconnect();
+      });
+
+      _self.client.on('error', error => {
+        logger.err(`CLightning client connection error: ${error}`, logger.tags.ln);
+        _self.increaseWaitTime();
+        _self.reconnect();
+      });
+    });
+
+    this.rl.on('line', line => {
+      line = line.trim();
+      if (!line) {
+        return;
+      }
+      const data = JSON.parse(line);
+      _self.emit('res:' + data.id, data);
+    });
+  }
+
+  increaseWaitTime(): void {
+    if (this.reconnectWait >= 16) {
+      this.reconnectWait = 16;
+    } else {
+      this.reconnectWait *= 2;
+    }
+  }
+
+  reconnect(): void {
+    const _self = this;
+
+    if (this.reconnectTimeout) {
+      return;
+    }
+
+    this.reconnectTimeout = setTimeout(() => {
+      logger.debug(`Trying to reconnect...`, logger.tags.ln);
+
+      _self.client.connect(_self.rpcPath);
+      _self.reconnectTimeout = null;
+    }, this.reconnectWait * 1000);
+  }
+
+  call(method, args = []): Promise<any> {
+    const _self = this;
+
+    const callInt = ++this.reqcount;
+    const sendObj = {
+      jsonrpc: '2.0',
+      method,
+      params: args,
+      id: '' + callInt
+    };
+
+
+    // Wait for the client to connect
+    return this.clientConnectionPromise
+      .then(() => new Promise((resolve, reject) => {
+        // Wait for a response
+        this.once('res:' + callInt, res => res.error == null
+          ? resolve(res.result)
+          : reject(new LightningError(res.error))
+        );
+
+        // Send the command
+        _self.client.write(JSON.stringify(sendObj));
+      }));
+  }
+
+  async $getNetworkGraph(): Promise<ILightningApi.NetworkGraph> {
+    const listnodes: any[] = await this.call('listnodes');
+    const listchannels: any[] = await this.call('listchannels');
+    const channelsList = await convertAndmergeBidirectionalChannels(listchannels['channels']);
+
+    return {
+      nodes: listnodes['nodes'].map(node => convertNode(node)),
+      edges: channelsList,
+    };
+  }
+}
+
+const protify = s => s.replace(/-([a-z])/g, m => m[1].toUpperCase());
+
+methods.forEach(k => {
+  CLightningClient.prototype[protify(k)] = function (...args: any) {
+    return this.call(k, args);
+  };
+});

backend/src/api/lightning/clightning/clightning-convert.ts

@@ -0,0 +1,158 @@
+import { ILightningApi } from '../lightning-api.interface';
+import FundingTxFetcher from '../../../tasks/lightning/sync-tasks/funding-tx-fetcher';
+import logger from '../../../logger';
+import { Common } from '../../common';
+import config from '../../../config';
+
+/**
+ * Convert a clightning "listnode" entry to a lnd node entry
+ */
+export function convertNode(clNode: any): ILightningApi.Node {
+  let custom_records: { [type: number]: string } | undefined = undefined;
+  if (clNode.option_will_fund) {
+    try {
+      custom_records = { '1': Buffer.from(clNode.option_will_fund.compact_lease || '', 'hex').toString('base64') };
+    } catch (e) {
+      logger.err(`Cannot decode option_will_fund compact_lease for ${clNode.nodeid}). Reason: ` + (e instanceof Error ? e.message : e));
+      custom_records = undefined;
+    }
+  }
+  return {
+    alias: clNode.alias ?? '',
+    color: `#${clNode.color ?? ''}`,
+    features: [], // TODO parse and return clNode.feature
+    pub_key: clNode.nodeid,
+    addresses: clNode.addresses?.map((addr) => {
+      let address = addr.address;
+      if (addr.type === 'ipv6') {
+        address = `[${address}]`;
+      }
+      return {
+        network: addr.type,
+        addr: `${address}:${addr.port}`
+      };
+    }) ?? [],
+    last_update: clNode?.last_timestamp ?? 0,
+    custom_records
+  };
+}
+
+/**
+ * Convert clightning "listchannels" response to lnd "describegraph.edges" format
+ */
+export async function convertAndmergeBidirectionalChannels(clChannels: any[]): Promise<ILightningApi.Channel[]> {
+  logger.debug(`Converting clightning nodes and channels to lnd graph format`, logger.tags.ln);
+
+  let loggerTimer = new Date().getTime() / 1000;
+  let channelProcessed = 0;
+
+  const consolidatedChannelList: ILightningApi.Channel[] = [];
+  const clChannelsDict = {};
+  const clChannelsDictCount = {};
+
+  for (const clChannel of clChannels) {
+    if (!clChannelsDict[clChannel.short_channel_id]) {
+      clChannelsDict[clChannel.short_channel_id] = clChannel;
+      clChannelsDictCount[clChannel.short_channel_id] = 1;
+    } else {
+      const fullChannel = await buildFullChannel(clChannel, clChannelsDict[clChannel.short_channel_id]);
+      if (fullChannel !== null) {
+        consolidatedChannelList.push(fullChannel);
+        delete clChannelsDict[clChannel.short_channel_id];
+        clChannelsDictCount[clChannel.short_channel_id]++;
+      }
+    }
+
+    const elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer);
+    if (elapsedSeconds > config.LIGHTNING.LOGGER_UPDATE_INTERVAL) {
+      logger.info(`Building complete channels from clightning output. Channels processed: ${channelProcessed + 1} of ${clChannels.length}`, logger.tags.ln);
+      loggerTimer = new Date().getTime() / 1000;
+    }
+
+    ++channelProcessed;
+  }
+
+  channelProcessed = 0;
+  const keys = Object.keys(clChannelsDict);
+  for (const short_channel_id of keys) {
+    const incompleteChannel = await buildIncompleteChannel(clChannelsDict[short_channel_id]);
+    if (incompleteChannel !== null) {
+      consolidatedChannelList.push(incompleteChannel);
+    }
+
+    const elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer);
+    if (elapsedSeconds > config.LIGHTNING.LOGGER_UPDATE_INTERVAL) {
+      logger.info(`Building partial channels from clightning output. Channels processed: ${channelProcessed + 1} of ${keys.length}`);
+      loggerTimer = new Date().getTime() / 1000;
+    }
+
+    channelProcessed++;
+  }
+
+  return consolidatedChannelList;
+}
+
+/**
+ * Convert two clightning "getchannels" entries into a full a lnd "describegraph.edges" format
+ * In this case, clightning knows the channel policy for both nodes
+ */
+async function buildFullChannel(clChannelA: any, clChannelB: any): Promise<ILightningApi.Channel | null> {
+  const lastUpdate = Math.max(clChannelA.last_update ?? 0, clChannelB.last_update ?? 0);
+
+  const tx = await FundingTxFetcher.$fetchChannelOpenTx(clChannelA.short_channel_id);
+  if (!tx) {
+    return null;
+  }
+  const parts = clChannelA.short_channel_id.split('x');
+  const outputIdx = parts[2];
+
+  return {
+    channel_id: Common.channelShortIdToIntegerId(clChannelA.short_channel_id),
+    capacity: clChannelA.satoshis,
+    last_update: lastUpdate,
+    node1_policy: convertPolicy(clChannelA),
+    node2_policy: convertPolicy(clChannelB),
+    chan_point: `${tx.txid}:${outputIdx}`,
+    node1_pub: clChannelA.source,
+    node2_pub: clChannelB.source,
+  };
+}
+
+/**
+ * Convert one clightning "getchannels" entry into a full a lnd "describegraph.edges" format
+ * In this case, clightning knows the channel policy of only one node
+ */
+async function buildIncompleteChannel(clChannel: any): Promise<ILightningApi.Channel | null> {
+  const tx = await FundingTxFetcher.$fetchChannelOpenTx(clChannel.short_channel_id);
+  if (!tx) {
+    return null;
+  }
+  const parts = clChannel.short_channel_id.split('x');
+  const outputIdx = parts[2];
+
+  return {
+    channel_id: Common.channelShortIdToIntegerId(clChannel.short_channel_id),
+    capacity: clChannel.satoshis,
+    last_update: clChannel.last_update ?? 0,
+    node1_policy: convertPolicy(clChannel),
+    node2_policy: null,
+    chan_point: `${tx.txid}:${outputIdx}`,
+    node1_pub: clChannel.source,
+    node2_pub: clChannel.destination,
+  };
+}
+
+/**
+ * Convert a clightning "listnode" response to a lnd channel policy format
+ */
+function convertPolicy(clChannel: any): ILightningApi.RoutingPolicy {
+  return {
+    time_lock_delta: clChannel.delay,
+    min_htlc: clChannel.htlc_minimum_msat.slice(0, -4),
+    max_htlc_msat: clChannel.htlc_maximum_msat.slice(0, -4),
+    fee_base_msat: clChannel.base_fee_millisatoshi,
+    fee_rate_milli_msat: clChannel.fee_per_millionth,
+    disabled: !clChannel.active,
+    last_update: clChannel.last_update ?? 0,
+  };
+}

backend/src/api/lightning/lightning-api-abstract-factory.ts

@@ -0,0 +1,5 @@
+import { ILightningApi } from './lightning-api.interface';
+
+export interface AbstractLightningApi {
+  $getNetworkGraph(): Promise<ILightningApi.NetworkGraph>;
+}

backend/src/api/lightning/lightning-api-factory.ts

@@ -0,0 +1,16 @@
+import config from '../../config';
+import CLightningClient from './clightning/clightning-client';
+import { AbstractLightningApi } from './lightning-api-abstract-factory';
+import LndApi from './lnd/lnd-api';
+
+function lightningApiFactory(): AbstractLightningApi {
+  switch (config.LIGHTNING.ENABLED === true && config.LIGHTNING.BACKEND) {
+    case 'cln':
+      return new CLightningClient(config.CLIGHTNING.SOCKET);
+    case 'lnd':
+    default:
+      return new LndApi();
+  }
+}
+
+export default lightningApiFactory();

backend/src/api/lightning/lightning-api.interface.ts

@@ -0,0 +1,92 @@
+export namespace ILightningApi {
+  export interface NetworkInfo {
+    graph_diameter: number;
+    avg_out_degree: number;
+    max_out_degree: number;
+    num_nodes: number;
+    num_channels: number;
+    total_network_capacity: string;
+    avg_channel_size: number;
+    min_channel_size: string;
+    max_channel_size: string;
+    median_channel_size_sat: string;
+    num_zombie_chans: string;
+  }
+
+  export interface NetworkGraph {
+    nodes: Node[];
+    edges: Channel[];
+  }
+
+  export interface Channel {
+    channel_id: string;
+    chan_point: string;
+    last_update: number | null;
+    node1_pub: string;
+    node2_pub: string;
+    capacity: string;
+    node1_policy: RoutingPolicy | null;
+    node2_policy: RoutingPolicy | null;
+  }
+
+  export interface RoutingPolicy {
+    time_lock_delta: number;
+    min_htlc: string;
+    fee_base_msat: string;
+    fee_rate_milli_msat: string;
+    disabled: boolean;
+    max_htlc_msat: string;
+    last_update: number | null;
+  }
+
+  export interface Node {
+    last_update: number | null;
+    pub_key: string;
+    alias: string;
+    addresses: {
+      network: string;
+      addr: string;
+    }[];
+    color: string;
+    features: { [key: number]: Feature };
+    custom_records?: { [type: number]: string };
+  }
+
+  export interface Info {
+    identity_pubkey: string;
+    alias: string;
+    num_pending_channels: number;
+    num_active_channels: number;
+    num_peers: number;
+    block_height: number;
+    block_hash: string;
+    synced_to_chain: boolean;
+    testnet: boolean;
+    uris: string[];
+    best_header_timestamp: string;
+    version: string;
+    num_inactive_channels: number;
+    chains: {
+      chain: string;
+      network: string;
+    }[];
+    color: string;
+    synced_to_graph: boolean;
+    features: { [key: number]: Feature };
+    commit_hash: string;
+    /** Available on LND since v0.15.0-beta */
+    require_htlc_interceptor?: boolean;
+  }
+
+  export interface Feature {
+    name: string;
+    is_required: boolean;
+    is_known: boolean;
+  }
+
+  export interface ForensicOutput {
+    node?: 1 | 2;
+    type: number;
+    value: number;
+  }
+}

backend/src/api/lightning/lnd/lnd-api.ts

@@ -0,0 +1,49 @@
+import axios, { AxiosRequestConfig } from 'axios';
+import { Agent } from 'https';
+import * as fs from 'fs';
+import { AbstractLightningApi } from '../lightning-api-abstract-factory';
+import { ILightningApi } from '../lightning-api.interface';
+import config from '../../../config';
+import logger from '../../../logger';
+
+class LndApi implements AbstractLightningApi {
+  axiosConfig: AxiosRequestConfig = {};
+
+  constructor() {
+    if (!config.LIGHTNING.ENABLED) {
+      return;
+    }
+    try {
+      this.axiosConfig = {
+        headers: {
+          'Grpc-Metadata-macaroon': fs.readFileSync(config.LND.MACAROON_PATH).toString('hex'),
+        },
+        httpsAgent: new Agent({
+          ca: fs.readFileSync(config.LND.TLS_CERT_PATH)
+        }),
+        timeout: config.LND.TIMEOUT
+      };
+    } catch (e) {
+      config.LIGHTNING.ENABLED = false;
+      logger.updateNetwork();
+      logger.err(`Could not initialize LND Macaroon/TLS Cert. Disabling LIGHTNING. ` + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  async $getNetworkInfo(): Promise<ILightningApi.NetworkInfo> {
+    return axios.get<ILightningApi.NetworkInfo>(config.LND.REST_API_URL + '/v1/graph/info', this.axiosConfig)
+      .then((response) => response.data);
+  }
+
+  async $getInfo(): Promise<ILightningApi.Info> {
+    return axios.get<ILightningApi.Info>(config.LND.REST_API_URL + '/v1/getinfo', this.axiosConfig)
+      .then((response) => response.data);
+  }
+
+  async $getNetworkGraph(): Promise<ILightningApi.NetworkGraph> {
+    return axios.get<ILightningApi.NetworkGraph>(config.LND.REST_API_URL + '/v1/graph', this.axiosConfig)
+      .then((response) => response.data);
+  }
+}
+
+export default LndApi;

backend/src/api/liquid/elements-parser.ts

@@ -0,0 +1,103 @@
+import { IBitcoinApi } from '../bitcoin/bitcoin-api.interface';
+import bitcoinClient from '../bitcoin/bitcoin-client';
+import bitcoinSecondClient from '../bitcoin/bitcoin-second-client';
+import { Common } from '../common';
+import DB from '../../database';
+import logger from '../../logger';
+
+class ElementsParser {
+  private isRunning = false;
+
+  constructor() { }
+
+  public async $parse() {
+    if (this.isRunning) {
+      return;
+    }
+    try {
+      this.isRunning = true;
+      const result = await bitcoinClient.getChainTips();
+      const tip = result[0].height;
+      const latestBlockHeight = await this.$getLatestBlockHeightFromDatabase();
+      for (let height = latestBlockHeight + 1; height <= tip; height++) {
+        const blockHash: IBitcoinApi.ChainTips = await bitcoinClient.getBlockHash(height);
+        const block: IBitcoinApi.Block = await bitcoinClient.getBlock(blockHash, 2);
+        await this.$parseBlock(block);
+        await this.$saveLatestBlockToDatabase(block.height);
+      }
+      this.isRunning = false;
+    } catch (e) {
+      this.isRunning = false;
+      throw new Error(e instanceof Error ? e.message : 'Error');
+    }
+  }
+
+  public async $getPegDataByMonth(): Promise<any> {
+    const query = `SELECT SUM(amount) AS amount, DATE_FORMAT(FROM_UNIXTIME(datetime), '%Y-%m-01') AS date FROM elements_pegs GROUP BY DATE_FORMAT(FROM_UNIXTIME(datetime), '%Y%m')`;
+    const [rows] = await DB.query(query);
+    return rows;
+  }
+
+  protected async $parseBlock(block: IBitcoinApi.Block) {
+    for (const tx of block.tx) {
+      await this.$parseInputs(tx, block);
+      await this.$parseOutputs(tx, block);
+    }
+  }
+
+  protected async $parseInputs(tx: IBitcoinApi.Transaction, block: IBitcoinApi.Block) {
+    for (const [index, input] of tx.vin.entries()) {
+      if (input.is_pegin) {
+        await this.$parsePegIn(input, index, tx.txid, block);
+      }
+    }
+  }
+
+  protected async $parsePegIn(input: IBitcoinApi.Vin, vindex: number, txid: string, block: IBitcoinApi.Block) {
+    const bitcoinTx: IBitcoinApi.Transaction = await bitcoinSecondClient.getRawTransaction(input.txid, true);
+    const prevout = bitcoinTx.vout[input.vout || 0];
+    const outputAddress = prevout.scriptPubKey.address || (prevout.scriptPubKey.addresses && prevout.scriptPubKey.addresses[0]) || '';
+    await this.$savePegToDatabase(block.height, block.time, prevout.value * 100000000, txid, vindex,
+      outputAddress, bitcoinTx.txid, prevout.n, 1);
+  }
+
+  protected async $parseOutputs(tx: IBitcoinApi.Transaction, block: IBitcoinApi.Block) {
+    for (const output of tx.vout) {
+      if (output.scriptPubKey.pegout_chain) {
+        await this.$savePegToDatabase(block.height, block.time, 0 - output.value * 100000000, tx.txid, output.n,
+          (output.scriptPubKey.pegout_addresses && output.scriptPubKey.pegout_addresses[0] || ''), '', 0, 0);
+      }
+      if (!output.scriptPubKey.pegout_chain && output.scriptPubKey.type === 'nulldata'
+        && output.value && output.value > 0 && output.asset && output.asset === Common.nativeAssetId) {
+        await this.$savePegToDatabase(block.height, block.time, 0 - output.value * 100000000, tx.txid, output.n,
+          (output.scriptPubKey.pegout_addresses && output.scriptPubKey.pegout_addresses[0] || ''), '', 0, 1);
+      }
+    }
+  }
+
+  protected async $savePegToDatabase(height: number, blockTime: number, amount: number, txid: string,
+    txindex: number, bitcoinaddress: string, bitcointxid: string, bitcoinindex: number, final_tx: number): Promise<void> {
+    const query = `INSERT INTO elements_pegs(
+        block, datetime, amount, txid, txindex, bitcoinaddress, bitcointxid, bitcoinindex, final_tx
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`;
+
+    const params: (string | number)[] = [
+      height, blockTime, amount, txid, txindex, bitcoinaddress, bitcointxid, bitcoinindex, final_tx
+    ];
+    await DB.query(query, params);
+    logger.debug(`Saved L-BTC peg from block height #${height} with TXID ${txid}.`);
+  }
+
+  protected async $getLatestBlockHeightFromDatabase(): Promise<number> {
+    const query = `SELECT number FROM state WHERE name = 'last_elements_block'`;
+    const [rows] = await DB.query(query);
+    return rows[0]['number'];
+  }
+
+  protected async $saveLatestBlockToDatabase(blockHeight: number) {
+    const query = `UPDATE state SET number = ? WHERE name = 'last_elements_block'`;
+    await DB.query(query, [blockHeight]);
+  }
+}
+
+export default new ElementsParser();

backend/src/api/liquid/icons.ts

@@ -0,0 +1,38 @@
+import * as fs from 'fs';
+import logger from '../../logger';
+
+class Icons {
+  private static FILE_NAME = './icons.json';
+  private iconIds: string[] = [];
+  private icons: { [assetId: string]: string; } = {};
+
+  constructor() {}
+
+  public loadIcons() {
+    if (!fs.existsSync(Icons.FILE_NAME)) {
+      logger.warn(`${Icons.FILE_NAME} does not exist. No Liquid icons loaded.`);
+      return;
+    }
+    const cacheData = fs.readFileSync(Icons.FILE_NAME, 'utf8');
+    this.icons = JSON.parse(cacheData);
+
+    for (const i in this.icons) {
+      this.iconIds.push(i);
+    }
+    logger.debug(`Liquid icons has been loaded.`);
+  }
+
+  public getIconByAssetId(assetId: string): Buffer | undefined {
+    const icon = this.icons[assetId];
+    if (icon) {
+      return Buffer.from(icon, 'base64');
+    }
+  }
+
+  public getAllIconIds() {
+    return this.iconIds;
+  }
+
+}
+
+export default new Icons();

backend/src/api/liquid/liquid.routes.ts

@@ -0,0 +1,73 @@
+import axios from 'axios';
+import { Application, Request, Response } from 'express';
+import config from '../../config';
+import elementsParser from './elements-parser';
+import icons from './icons';
+
+class LiquidRoutes {
+  public initRoutes(app: Application) {
+    app
+      .get(config.MEMPOOL.API_URL_PREFIX + 'assets/icons', this.getAllLiquidIcon)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'assets/featured', this.$getAllFeaturedLiquidAssets)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'asset/:assetId/icon', this.getLiquidIcon)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'assets/group/:id', this.$getAssetGroup)
+      ;
+    
+    if (config.DATABASE.ENABLED) {
+      app
+        .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/pegs/month', this.$getElementsPegsByMonth)
+        ;
+    }
+  }
+
+
+  private getLiquidIcon(req: Request, res: Response) {
+    const result = icons.getIconByAssetId(req.params.assetId);
+    if (result) {
+      res.setHeader('content-type', 'image/png');
+      res.setHeader('content-length', result.length);
+      res.send(result);
+    } else {
+      res.status(404).send('Asset icon not found');
+    }
+  }
+
+  private getAllLiquidIcon(req: Request, res: Response) {
+    const result = icons.getAllIconIds();
+    if (result) {
+      res.json(result);
+    } else {
+      res.status(404).send('Asset icons not found');
+    }
+  }
+
+  private async $getAllFeaturedLiquidAssets(req: Request, res: Response) {
+    try {
+      const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.LIQUID_API}/assets/featured`, { responseType: 'stream', timeout: 10000 });
+      response.data.pipe(res);
+    } catch (e) {
+      res.status(500).end();
+    }
+  }
+
+  private async $getAssetGroup(req: Request, res: Response) {
+    try {
+      const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.LIQUID_API}/assets/group/${parseInt(req.params.id, 10)}`,
+        { responseType: 'stream', timeout: 10000 });
+      response.data.pipe(res);
+    } catch (e) {
+      res.status(500).end();
+    }
+  }
+
+  private async $getElementsPegsByMonth(req: Request, res: Response) {
+    try {
+      const pegs = await elementsParser.$getPegDataByMonth();
+      res.json(pegs);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+}
+
+export default new LiquidRoutes();

backend/src/api/loading-indicators.ts

@@ -12,8 +12,8 @@ class LoadingIndicators {
     this.progressChangedCallback = fn;
   }
 
-  public setProgress(name: string, progressPercent: number) {
-    const newProgress = Math.round(progressPercent);
+  public setProgress(name: string, progressPercent: number, rounded: boolean = true) {
+    const newProgress = rounded === true ? Math.round(progressPercent) : progressPercent;
     if (newProgress >= 100) {
       delete this.loadingIndicators[name];
     } else {

backend/src/api/mempool-blocks.ts

@@ -1,11 +1,14 @@
 import logger from '../logger';
-import { MempoolBlock, TransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces';
+import { MempoolBlock, TransactionExtended, ThreadTransaction, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor } from '../mempool.interfaces';
 import { Common } from './common';
 import config from '../config';
+import { Worker } from 'worker_threads';
+import path from 'path';
 
 class MempoolBlocks {
-  private static DEFAULT_PROJECTED_BLOCKS_AMOUNT = 8;
   private mempoolBlocks: MempoolBlockWithTransactions[] = [];
+  private mempoolBlockDeltas: MempoolBlockDelta[] = [];
+  private txSelectionWorker: Worker | null = null;
 
   constructor() {}
 
@@ -26,7 +29,11 @@ class MempoolBlocks {
     return this.mempoolBlocks;
   }
 
-  public updateMempoolBlocks(memPool: { [txid: string]: TransactionExtended }): void {
+  public getMempoolBlockDeltas(): MempoolBlockDelta[] {
+    return this.mempoolBlockDeltas;
+  }
+
+  public updateMempoolBlocks(memPool: { [txid: string]: TransactionExtended }, saveResults: boolean = false): MempoolBlockWithTransactions[] {
     const latestMempool = memPool;
     const memPoolArray: TransactionExtended[] = [];
     for (const i in latestMempool) {
@@ -67,34 +74,236 @@ class MempoolBlocks {
     const time = end - start;
     logger.debug('Mempool blocks calculated in ' + time / 1000 + ' seconds');
 
-    this.mempoolBlocks = this.calculateMempoolBlocks(memPoolArray);
+    const blocks = this.calculateMempoolBlocks(memPoolArray, this.mempoolBlocks);
+
+    if (saveResults) {
+      const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, blocks);
+      this.mempoolBlocks = blocks;
+      this.mempoolBlockDeltas = deltas;
+    }
+
+    return blocks;
   }
 
-  private calculateMempoolBlocks(transactionsSorted: TransactionExtended[]): MempoolBlockWithTransactions[] {
+  private calculateMempoolBlocks(transactionsSorted: TransactionExtended[], prevBlocks: MempoolBlockWithTransactions[]): MempoolBlockWithTransactions[] {
     const mempoolBlocks: MempoolBlockWithTransactions[] = [];
-    let blockVSize = 0;
+    let blockWeight = 0;
     let blockSize = 0;
     let transactions: TransactionExtended[] = [];
     transactionsSorted.forEach((tx) => {
-      if (blockVSize + tx.vsize <= 1000000 || mempoolBlocks.length === MempoolBlocks.DEFAULT_PROJECTED_BLOCKS_AMOUNT - 1) {
-        blockVSize += tx.vsize;
+      if (blockWeight + tx.weight <= config.MEMPOOL.BLOCK_WEIGHT_UNITS
+        || mempoolBlocks.length === config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT - 1) {
+        blockWeight += tx.weight;
         blockSize += tx.size;
         transactions.push(tx);
       } else {
-        mempoolBlocks.push(this.dataToMempoolBlocks(transactions, blockSize, blockVSize, mempoolBlocks.length));
-        blockVSize = tx.vsize;
+        mempoolBlocks.push(this.dataToMempoolBlocks(transactions, mempoolBlocks.length));
+        blockWeight = tx.weight;
         blockSize = tx.size;
         transactions = [tx];
       }
     });
     if (transactions.length) {
-      mempoolBlocks.push(this.dataToMempoolBlocks(transactions, blockSize, blockVSize, mempoolBlocks.length));
+      mempoolBlocks.push(this.dataToMempoolBlocks(transactions, mempoolBlocks.length));
     }
+
     return mempoolBlocks;
   }
 
-  private dataToMempoolBlocks(transactions: TransactionExtended[],
-    blockSize: number, blockVSize: number, blocksIndex: number): MempoolBlockWithTransactions {
+  private calculateMempoolDeltas(prevBlocks: MempoolBlockWithTransactions[], mempoolBlocks: MempoolBlockWithTransactions[]): MempoolBlockDelta[] {
+    const mempoolBlockDeltas: MempoolBlockDelta[] = [];
+    for (let i = 0; i < Math.max(mempoolBlocks.length, prevBlocks.length); i++) {
+      let added: TransactionStripped[] = [];
+      let removed: string[] = [];
+      if (mempoolBlocks[i] && !prevBlocks[i]) {
+        added = mempoolBlocks[i].transactions;
+      } else if (!mempoolBlocks[i] && prevBlocks[i]) {
+        removed = prevBlocks[i].transactions.map(tx => tx.txid);
+      } else if (mempoolBlocks[i] && prevBlocks[i]) {
+        const prevIds = {};
+        const newIds = {};
+        prevBlocks[i].transactions.forEach(tx => {
+          prevIds[tx.txid] = true;
+        });
+        mempoolBlocks[i].transactions.forEach(tx => {
+          newIds[tx.txid] = true;
+        });
+        prevBlocks[i].transactions.forEach(tx => {
+          if (!newIds[tx.txid]) {
+            removed.push(tx.txid);
+          }
+        });
+        mempoolBlocks[i].transactions.forEach(tx => {
+          if (!prevIds[tx.txid]) {
+            added.push(tx);
+          }
+        });
+      }
+      mempoolBlockDeltas.push({
+        added,
+        removed
+      });
+    }
+    return mempoolBlockDeltas;
+  }
+
+  public async makeBlockTemplates(newMempool: { [txid: string]: TransactionExtended }, saveResults: boolean = false): Promise<MempoolBlockWithTransactions[]> {
+    // prepare a stripped down version of the mempool with only the minimum necessary data
+    // to reduce the overhead of passing this data to the worker thread
+    const strippedMempool: { [txid: string]: ThreadTransaction } = {};
+    Object.values(newMempool).forEach(entry => {
+      strippedMempool[entry.txid] = {
+        txid: entry.txid,
+        fee: entry.fee,
+        weight: entry.weight,
+        feePerVsize: entry.fee / (entry.weight / 4),
+        effectiveFeePerVsize: entry.fee / (entry.weight / 4),
+        vin: entry.vin.map(v => v.txid),
+      };
+    });
+
+    // (re)initialize tx selection worker thread
+    if (!this.txSelectionWorker) {
+      this.txSelectionWorker = new Worker(path.resolve(__dirname, './tx-selection-worker.js'));
+      // if the thread throws an unexpected error, or exits for any other reason,
+      // reset worker state so that it will be re-initialized on the next run
+      this.txSelectionWorker.once('error', () => {
+        this.txSelectionWorker = null;
+      });
+      this.txSelectionWorker.once('exit', () => {
+        this.txSelectionWorker = null;
+      });
+    }
+
+    // run the block construction algorithm in a separate thread, and wait for a result
+    let threadErrorListener;
+    try {
+      const workerResultPromise = new Promise<{ blocks: ThreadTransaction[][], clusters: { [root: string]: string[] } }>((resolve, reject) => {
+        threadErrorListener = reject;
+        this.txSelectionWorker?.once('message', (result): void => {
+          resolve(result);
+        });
+        this.txSelectionWorker?.once('error', reject);
+      });
+      this.txSelectionWorker.postMessage({ type: 'set', mempool: strippedMempool });
+      const { blocks, clusters } = await workerResultPromise;
+
+      // clean up thread error listener
+      this.txSelectionWorker?.removeListener('error', threadErrorListener);
+
+      return this.processBlockTemplates(newMempool, blocks, clusters, saveResults);
+    } catch (e) {
+      logger.err('makeBlockTemplates failed. ' + (e instanceof Error ? e.message : e));
+    }
+    return this.mempoolBlocks;
+  }
+
+  public async updateBlockTemplates(newMempool: { [txid: string]: TransactionExtended }, added: TransactionExtended[], removed: string[], saveResults: boolean = false): Promise<void> {
+    if (!this.txSelectionWorker) {
+      // need to reset the worker
+      this.makeBlockTemplates(newMempool, saveResults);
+      return;
+    }
+    // prepare a stripped down version of the mempool with only the minimum necessary data
+    // to reduce the overhead of passing this data to the worker thread
+    const addedStripped: ThreadTransaction[] = added.map(entry => {
+      return {
+        txid: entry.txid,
+        fee: entry.fee,
+        weight: entry.weight,
+        feePerVsize: entry.fee / (entry.weight / 4),
+        effectiveFeePerVsize: entry.fee / (entry.weight / 4),
+        vin: entry.vin.map(v => v.txid),
+      };
+    });
+
+    // run the block construction algorithm in a separate thread, and wait for a result
+    let threadErrorListener;
+    try {
+      const workerResultPromise = new Promise<{ blocks: ThreadTransaction[][], clusters: { [root: string]: string[] } }>((resolve, reject) => {
+        threadErrorListener = reject;
+        this.txSelectionWorker?.once('message', (result): void => {
+          resolve(result);
+        });
+        this.txSelectionWorker?.once('error', reject);
+      });
+      this.txSelectionWorker.postMessage({ type: 'update', added: addedStripped, removed });
+      const { blocks, clusters } = await workerResultPromise;
+
+      // clean up thread error listener
+      this.txSelectionWorker?.removeListener('error', threadErrorListener);
+
+      this.processBlockTemplates(newMempool, blocks, clusters, saveResults);
+    } catch (e) {
+      logger.err('updateBlockTemplates failed. ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  private processBlockTemplates(mempool, blocks, clusters, saveResults): MempoolBlockWithTransactions[] {
+    // update this thread's mempool with the results
+    blocks.forEach(block => {
+      block.forEach(tx => {
+        if (tx.txid in mempool) {
+          if (tx.effectiveFeePerVsize != null) {
+            mempool[tx.txid].effectiveFeePerVsize = tx.effectiveFeePerVsize;
+          }
+          if (tx.cpfpRoot && tx.cpfpRoot in clusters) {
+            const ancestors: Ancestor[] = [];
+            const descendants: Ancestor[] = [];
+            const cluster = clusters[tx.cpfpRoot];
+            let matched = false;
+            cluster.forEach(txid => {
+              if (txid === tx.txid) {
+                matched = true;
+              } else {
+                const relative = {
+                  txid: txid,
+                  fee: mempool[txid].fee,
+                  weight: mempool[txid].weight,
+                };
+                if (matched) {
+                  descendants.push(relative);
+                } else {
+                  ancestors.push(relative);
+                }
+              }
+            });
+            mempool[tx.txid].ancestors = ancestors;
+            mempool[tx.txid].descendants = descendants;
+            mempool[tx.txid].bestDescendant = null;
+          }
+          mempool[tx.txid].cpfpChecked = tx.cpfpChecked;
+        }
+      });
+    });
+
+    // unpack the condensed blocks into proper mempool blocks
+    const mempoolBlocks = blocks.map((transactions, blockIndex) => {
+      return this.dataToMempoolBlocks(transactions.map(tx => {
+        return mempool[tx.txid] || null;
+      }).filter(tx => !!tx), blockIndex);
+    });
+
+    if (saveResults) {
+      const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, mempoolBlocks);
+      this.mempoolBlocks = mempoolBlocks;
+      this.mempoolBlockDeltas = deltas;
+    }
+
+    return mempoolBlocks;
+  }
+
+  private dataToMempoolBlocks(transactions: TransactionExtended[], blocksIndex: number): MempoolBlockWithTransactions {
+    let totalSize = 0;
+    let totalWeight = 0;
+    const fitTransactions: TransactionExtended[] = [];
+    transactions.forEach(tx => {
+      totalSize += tx.size;
+      totalWeight += tx.weight;
+      if ((totalWeight + tx.weight) <= config.MEMPOOL.BLOCK_WEIGHT_UNITS * 1.2) {
+        fitTransactions.push(tx);
+      }
+    });
     let rangeLength = 4;
     if (blocksIndex === 0) {
       rangeLength = 8;
@@ -105,13 +314,14 @@ class MempoolBlocks {
       rangeLength = 8;
     }
     return {
-      blockSize: blockSize,
-      blockVSize: blockVSize,
+      blockSize: totalSize,
+      blockVSize: totalWeight / 4,
       nTx: transactions.length,
       totalFees: transactions.reduce((acc, cur) => acc + cur.fee, 0),
       medianFee: Common.percentile(transactions.map((tx) => tx.effectiveFeePerVsize), config.MEMPOOL.RECOMMENDED_FEE_PERCENTILE),
       feeRange: Common.getFeesInRange(transactions, rangeLength),
       transactionIds: transactions.map((tx) => tx.txid),
+      transactions: fitTransactions.map((tx) => Common.stripTransaction(tx)),
     };
   }
 }

backend/src/api/mempool.ts

@@ -5,18 +5,23 @@ import logger from '../logger';
 import { Common } from './common';
 import transactionUtils from './transaction-utils';
 import { IBitcoinApi } from './bitcoin/bitcoin-api.interface';
-import bitcoinBaseApi from './bitcoin/bitcoin-base.api';
 import loadingIndicators from './loading-indicators';
+import bitcoinClient from './bitcoin/bitcoin-client';
+import bitcoinSecondClient from './bitcoin/bitcoin-second-client';
+import rbfCache from './rbf-cache';
 
 class Mempool {
   private static WEBSOCKET_REFRESH_RATE_MS = 10000;
   private static LAZY_DELETE_AFTER_SECONDS = 30;
   private inSync: boolean = false;
+  private mempoolCacheDelta: number = -1;
   private mempoolCache: { [txId: string]: TransactionExtended } = {};
-  private mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0,
+  private mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0, total_fee: 0,
                                                     maxmempool: 300000000, mempoolminfee: 0.00001000, minrelaytxfee: 0.00001000 };
   private mempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[],
     deletedTransactions: TransactionExtended[]) => void) | undefined;
+  private asyncMempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[],
+    deletedTransactions: TransactionExtended[]) => Promise<void>) | undefined;
 
   private txPerSecondArray: number[] = [];
   private txPerSecond: number = 0;
@@ -26,11 +31,27 @@ class Mempool {
   private mempoolProtection = 0;
   private latestTransactions: any[] = [];
 
+  private ESPLORA_MISSING_TX_WARNING_THRESHOLD = 100; 
+  private SAMPLE_TIME = 10000; // In ms
+  private timer = new Date().getTime();
+  private missingTxCount = 0;
+
   constructor() {
     setInterval(this.updateTxPerSecond.bind(this), 1000);
     setInterval(this.deleteExpiredTransactions.bind(this), 20000);
   }
 
+  /**
+   * Return true if we should leave resources available for mempool tx caching
+   */
+  public hasPriority(): boolean {
+    if (this.inSync) {
+      return false;
+    } else {
+      return this.mempoolCacheDelta == -1 || this.mempoolCacheDelta > 25;
+    }
+  }
+
   public isInSync(): boolean {
     return this.inSync;
   }
@@ -49,6 +70,11 @@ class Mempool {
     this.mempoolChangedCallback = fn;
   }
 
+  public setAsyncMempoolChangedCallback(fn: (newMempool: { [txId: string]: TransactionExtended; },
+    newTransactions: TransactionExtended[], deletedTransactions: TransactionExtended[]) => Promise<void>) {
+    this.asyncMempoolChangedCallback = fn;
+  }
+
   public getMempool(): { [txid: string]: TransactionExtended } {
     return this.mempoolCache;
   }
@@ -58,10 +84,13 @@ class Mempool {
     if (this.mempoolChangedCallback) {
       this.mempoolChangedCallback(this.mempoolCache, [], []);
     }
+    if (this.asyncMempoolChangedCallback) {
+      this.asyncMempoolChangedCallback(this.mempoolCache, [], []);
+    }
   }
 
   public async $updateMemPoolInfo() {
-    this.mempoolInfo = await bitcoinBaseApi.$getMempoolInfo();
+    this.mempoolInfo = await this.$getMempoolInfo();
   }
 
   public getMempoolInfo(): IBitcoinApi.MempoolInfo {
@@ -89,26 +118,36 @@ class Mempool {
     return txTimes;
   }
 
-  public async $updateMempool() {
-    logger.debug('Updating mempool');
+  public async $updateMempool(): Promise<void> {
+    logger.debug(`Updating mempool...`);
     const start = new Date().getTime();
     let hasChange: boolean = false;
     const currentMempoolSize = Object.keys(this.mempoolCache).length;
-    let txCount = 0;
     const transactions = await bitcoinApi.$getRawMempool();
     const diff = transactions.length - currentMempoolSize;
     const newTransactions: TransactionExtended[] = [];
 
+    this.mempoolCacheDelta = Math.abs(diff);
+
     if (!this.inSync) {
       loadingIndicators.setProgress('mempool', Object.keys(this.mempoolCache).length / transactions.length * 100);
     }
 
+    // https://github.com/mempool/mempool/issues/3283
+    const logEsplora404 = (missingTxCount, threshold, time) => {
+      const log = `In the past ${time / 1000} seconds, esplora tx API replied ${missingTxCount} times with a 404 error code while updating nodejs backend mempool`;
+      if (missingTxCount >= threshold) {
+        logger.warn(log);
+      } else if (missingTxCount > 0) {
+        logger.debug(log);
+      }
+    };
+
     for (const txid of transactions) {
       if (!this.mempoolCache[txid]) {
         try {
           const transaction = await transactionUtils.$getTransactionExtended(txid);
           this.mempoolCache[txid] = transaction;
-          txCount++;
           if (this.inSync) {
             this.txPerSecondArray.push(new Date().getTime());
             this.vBytesPerSecondArray.push({
@@ -117,14 +156,12 @@ class Mempool {
             });
           }
           hasChange = true;
-          if (diff > 0) {
-            logger.debug('Fetched transaction ' + txCount + ' / ' + diff);
-          } else {
-            logger.debug('Fetched transaction ' + txCount);
-          }
           newTransactions.push(transaction);
-        } catch (e) {
-          logger.debug('Error finding transaction in mempool: ' + e.message || e);
+        } catch (e: any) {
+          if (config.MEMPOOL.BACKEND === 'esplora' && e.response?.status === 404) {
+            this.missingTxCount++;
+          }
+          logger.debug(`Error finding transaction '${txid}' in the mempool: ` + (e instanceof Error ? e.message : e));
         }
       }
 
@@ -133,6 +170,14 @@ class Mempool {
       }
     }
 
+    // Reset esplora 404 counter and log a warning if needed
+    const elapsedTime = new Date().getTime() - this.timer;
+    if (elapsedTime > this.SAMPLE_TIME) {
+      logEsplora404(this.missingTxCount, this.ESPLORA_MISSING_TX_WARNING_THRESHOLD, elapsedTime);
+      this.timer = new Date().getTime();
+      this.missingTxCount = 0;
+    }
+
     // Prevent mempool from clear on bitcoind restart by delaying the deletion
     if (this.mempoolProtection === 0
       && currentMempoolSize > 20000
@@ -173,14 +218,29 @@ class Mempool {
       loadingIndicators.setProgress('mempool', 100);
     }
 
+    this.mempoolCacheDelta = Math.abs(transactions.length - Object.keys(this.mempoolCache).length);
+
     if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) {
       this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions);
     }
+    if (this.asyncMempoolChangedCallback && (hasChange || deletedTransactions.length)) {
+      await this.asyncMempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions);
+    }
 
     const end = new Date().getTime();
     const time = end - start;
-    logger.debug(`New mempool size: ${Object.keys(this.mempoolCache).length} Change: ${diff}`);
-    logger.debug('Mempool updated in ' + time / 1000 + ' seconds');
+    logger.debug(`Mempool updated in ${time / 1000} seconds. New size: ${Object.keys(this.mempoolCache).length} (${diff > 0 ? '+' + diff : diff})`);
+  }
+
+  public handleRbfTransactions(rbfTransactions: { [txid: string]: TransactionExtended; }) {
+    for (const rbfTransaction in rbfTransactions) {
+      if (this.mempoolCache[rbfTransaction]) {
+        // Store replaced transactions
+        rbfCache.add(this.mempoolCache[rbfTransaction], rbfTransactions[rbfTransaction].txid);
+        // Erase the replaced transactions from the local mempool
+        delete this.mempoolCache[rbfTransaction];
+      }
+    }
   }
 
   private updateTxPerSecond() {
@@ -202,9 +262,25 @@ class Mempool {
       const lazyDeleteAt = this.mempoolCache[tx].deleteAfter;
       if (lazyDeleteAt && lazyDeleteAt < now) {
         delete this.mempoolCache[tx];
+        rbfCache.evict(tx);
       }
     }
   }
+
+  private $getMempoolInfo() {
+    if (config.MEMPOOL.USE_SECOND_NODE_FOR_MINFEE) {
+      return Promise.all([
+        bitcoinClient.getMempoolInfo(),
+        bitcoinSecondClient.getMempoolInfo()
+      ]).then(([mempoolInfo, secondMempoolInfo]) => {
+        mempoolInfo.maxmempool = secondMempoolInfo.maxmempool;
+        mempoolInfo.mempoolminfee = secondMempoolInfo.mempoolminfee;
+        mempoolInfo.minrelaytxfee = secondMempoolInfo.minrelaytxfee;
+        return mempoolInfo;
+      });
+    }
+    return bitcoinClient.getMempoolInfo();
+  }
 }
 
 export default new Mempool();

backend/src/api/mining/mining-routes.ts

@@ -0,0 +1,329 @@
+import { Application, Request, Response } from 'express';
+import config from "../../config";
+import logger from '../../logger';
+import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository';
+import BlocksRepository from '../../repositories/BlocksRepository';
+import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjustmentsRepository';
+import HashratesRepository from '../../repositories/HashratesRepository';
+import bitcoinClient from '../bitcoin/bitcoin-client';
+import mining from "./mining";
+import PricesRepository from '../../repositories/PricesRepository';
+
+class MiningRoutes {
+  public initRoutes(app: Application) {
+    app
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/:interval', this.$getPools)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug/hashrate', this.$getPoolHistoricalHashrate)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug/blocks', this.$getPoolBlocks)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug/blocks/:height', this.$getPoolBlocks)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug', this.$getPool)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/hashrate/pools/:interval', this.$getPoolsHistoricalHashrate)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/hashrate/:interval', this.$getHistoricalHashrate)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/difficulty-adjustments', this.$getDifficultyAdjustments)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/reward-stats/:blockCount', this.$getRewardStats)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/fees/:interval', this.$getHistoricalBlockFees)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/rewards/:interval', this.$getHistoricalBlockRewards)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/fee-rates/:interval', this.$getHistoricalBlockFeeRates)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/sizes-weights/:interval', this.$getHistoricalBlockSizeAndWeight)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/difficulty-adjustments/:interval', this.$getDifficultyAdjustments)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/predictions/:interval', this.$getHistoricalBlockPrediction)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/scores', this.$getBlockAuditScores)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/scores/:height', this.$getBlockAuditScores)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/score/:hash', this.$getBlockAuditScore)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/:hash', this.$getBlockAudit)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/timestamp/:timestamp', this.$getHeightFromTimestamp)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'historical-price', this.$getHistoricalPrice)
+    ;
+  }
+
+  private async $getHistoricalPrice(req: Request, res: Response): Promise<void> {
+    try {
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
+      if (req.query.timestamp) {
+        res.status(200).send(await PricesRepository.$getNearestHistoricalPrice(
+          parseInt(<string>req.query.timestamp ?? 0, 10)
+        ));
+      } else {
+        res.status(200).send(await PricesRepository.$getHistoricalPrices());
+      }
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getPool(req: Request, res: Response): Promise<void> {
+    try {
+      const stats = await mining.$getPoolStat(req.params.slug);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(stats);
+    } catch (e) {
+      if (e instanceof Error && e.message.indexOf('This mining pool does not exist') > -1) {
+        res.status(404).send(e.message);
+      } else {
+        res.status(500).send(e instanceof Error ? e.message : e);
+      }
+    }
+  }
+
+  private async $getPoolBlocks(req: Request, res: Response) {
+    try {
+      const poolBlocks = await BlocksRepository.$getBlocksByPool(
+        req.params.slug,
+        req.params.height === undefined ? undefined : parseInt(req.params.height, 10),
+      );
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(poolBlocks);
+    } catch (e) {
+      if (e instanceof Error && e.message.indexOf('This mining pool does not exist') > -1) {
+        res.status(404).send(e.message);
+      } else {
+        res.status(500).send(e instanceof Error ? e.message : e);
+      }
+    }
+  }
+
+  private async $getPools(req: Request, res: Response) {
+    try {
+      const stats = await mining.$getPoolsStats(req.params.interval);
+      const blockCount = await BlocksRepository.$blockCount(null, null);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.header('X-total-count', blockCount.toString());
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(stats);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getPoolsHistoricalHashrate(req: Request, res: Response) {
+    try {
+      const hashrates = await HashratesRepository.$getPoolsWeeklyHashrate(req.params.interval);
+      const blockCount = await BlocksRepository.$blockCount(null, null);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.header('X-total-count', blockCount.toString());
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
+      res.json(hashrates);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getPoolHistoricalHashrate(req: Request, res: Response) {
+    try {
+      const hashrates = await HashratesRepository.$getPoolWeeklyHashrate(req.params.slug);
+      const blockCount = await BlocksRepository.$blockCount(null, null);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.header('X-total-count', blockCount.toString());
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
+      res.json(hashrates);
+    } catch (e) {
+      if (e instanceof Error && e.message.indexOf('This mining pool does not exist') > -1) {
+        res.status(404).send(e.message);
+      } else {
+        res.status(500).send(e instanceof Error ? e.message : e);
+      }
+    }
+  }
+
+  private async $getHistoricalHashrate(req: Request, res: Response) {
+    let currentHashrate = 0, currentDifficulty = 0;
+    try {
+      currentHashrate = await bitcoinClient.getNetworkHashPs();
+      currentDifficulty = await bitcoinClient.getDifficulty();
+    } catch (e) {
+      logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate and difficulty');
+    }
+
+    try {
+      const hashrates = await HashratesRepository.$getNetworkDailyHashrate(req.params.interval);
+      const difficulty = await DifficultyAdjustmentsRepository.$getAdjustments(req.params.interval, false);
+      const blockCount = await BlocksRepository.$blockCount(null, null);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.header('X-total-count', blockCount.toString());
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
+      res.json({
+        hashrates: hashrates,
+        difficulty: difficulty,
+        currentHashrate: currentHashrate,
+        currentDifficulty: currentDifficulty,
+      });
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getHistoricalBlockFees(req: Request, res: Response) {
+    try {
+      const blockFees = await mining.$getHistoricalBlockFees(req.params.interval);
+      const blockCount = await BlocksRepository.$blockCount(null, null);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.header('X-total-count', blockCount.toString());
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(blockFees);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getHistoricalBlockRewards(req: Request, res: Response) {
+    try {
+      const blockRewards = await mining.$getHistoricalBlockRewards(req.params.interval);
+      const blockCount = await BlocksRepository.$blockCount(null, null);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.header('X-total-count', blockCount.toString());
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(blockRewards);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getHistoricalBlockFeeRates(req: Request, res: Response) {
+    try {
+      const blockFeeRates = await mining.$getHistoricalBlockFeeRates(req.params.interval);
+      const blockCount = await BlocksRepository.$blockCount(null, null);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.header('X-total-count', blockCount.toString());
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(blockFeeRates);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getHistoricalBlockSizeAndWeight(req: Request, res: Response) {
+    try {
+      const blockSizes = await mining.$getHistoricalBlockSizes(req.params.interval);
+      const blockWeights = await mining.$getHistoricalBlockWeights(req.params.interval);
+      const blockCount = await BlocksRepository.$blockCount(null, null);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.header('X-total-count', blockCount.toString());
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json({
+        sizes: blockSizes,
+        weights: blockWeights
+      });
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getDifficultyAdjustments(req: Request, res: Response) {
+    try {
+      const difficulty = await DifficultyAdjustmentsRepository.$getRawAdjustments(req.params.interval, true);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
+      res.json(difficulty.map(adj => [adj.time, adj.height, adj.difficulty, adj.adjustment]));
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getRewardStats(req: Request, res: Response) {
+    try {
+      const response = await mining.$getRewardStats(parseInt(req.params.blockCount, 10));
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(response);
+    } catch (e) {
+      res.status(500).end();
+    }
+  }
+
+  private async $getHistoricalBlockPrediction(req: Request, res: Response) {
+    try {
+      const blockPredictions = await mining.$getBlockPredictionsHistory(req.params.interval);
+      const blockCount = await BlocksAuditsRepository.$getPredictionsCount();
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.header('X-total-count', blockCount.toString());
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(blockPredictions.map(prediction => [prediction.time, prediction.height, prediction.match_rate]));
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  public async $getBlockAudit(req: Request, res: Response) {
+    try {
+      const audit = await BlocksAuditsRepository.$getBlockAudit(req.params.hash);
+
+      if (!audit) {
+        res.status(204).send(`This block has not been audited.`);
+        return;
+      }
+
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24).toUTCString());
+      res.json(audit);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getHeightFromTimestamp(req: Request, res: Response) {
+    try {
+      const timestamp = parseInt(req.params.timestamp, 10);
+      // This will prevent people from entering milliseconds etc.
+      // Block timestamps are allowed to be up to 2 hours off, so 24 hours
+      // will never put the maximum value before the most recent block
+      const nowPlus1day = Math.floor(Date.now() / 1000) + 60 * 60 * 24;
+      // Prevent non-integers that are not seconds
+      if (!/^[1-9][0-9]*$/.test(req.params.timestamp) || timestamp > nowPlus1day) {
+        throw new Error(`Invalid timestamp, value must be Unix seconds`);
+      }
+      const result = await BlocksRepository.$getBlockHeightFromTimestamp(
+        timestamp,
+      );
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
+      res.json(result);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getBlockAuditScores(req: Request, res: Response) {
+    try {
+      let height = req.params.height === undefined ? undefined : parseInt(req.params.height, 10);
+      if (height == null) {
+        height = await BlocksRepository.$mostRecentBlockHeight();
+      }
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(await BlocksAuditsRepository.$getBlockAuditScores(height, height - 15));
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  public async $getBlockAuditScore(req: Request, res: Response) {
+    try {
+      const audit = await BlocksAuditsRepository.$getBlockAuditScore(req.params.hash);
+
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24).toUTCString());
+      res.json(audit || 'null');
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+}
+
+export default new MiningRoutes();

backend/src/api/mining/mining.ts

@@ -0,0 +1,592 @@
+import { BlockPrice, PoolInfo, PoolStats, RewardStats } from '../../mempool.interfaces';
+import BlocksRepository from '../../repositories/BlocksRepository';
+import PoolsRepository from '../../repositories/PoolsRepository';
+import HashratesRepository from '../../repositories/HashratesRepository';
+import bitcoinClient from '../bitcoin/bitcoin-client';
+import logger from '../../logger';
+import { Common } from '../common';
+import loadingIndicators from '../loading-indicators';
+import { escape } from 'mysql2';
+import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjustmentsRepository';
+import config from '../../config';
+import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository';
+import PricesRepository from '../../repositories/PricesRepository';
+import { bitcoinCoreApi } from '../bitcoin/bitcoin-api-factory';
+import { IEsploraApi } from '../bitcoin/esplora-api.interface';
+import database from '../../database';
+
+class Mining {
+  private blocksPriceIndexingRunning = false;
+  public lastHashrateIndexingDate: number | null = null;
+  public lastWeeklyHashrateIndexingDate: number | null = null;
+
+  /**
+   * Get historical block predictions match rate
+   */
+   public async $getBlockPredictionsHistory(interval: string | null = null): Promise<any> {
+    return await BlocksAuditsRepository.$getBlockPredictionsHistory(
+      this.getTimeRange(interval),
+      Common.getSqlInterval(interval)
+    );
+  }
+
+  /**
+   * Get historical block total fee
+   */
+  public async $getHistoricalBlockFees(interval: string | null = null): Promise<any> {
+    return await BlocksRepository.$getHistoricalBlockFees(
+      this.getTimeRange(interval, 5),
+      Common.getSqlInterval(interval)
+    );
+  }
+
+  /**
+   * Get historical block rewards
+   */
+  public async $getHistoricalBlockRewards(interval: string | null = null): Promise<any> {
+    return await BlocksRepository.$getHistoricalBlockRewards(
+      this.getTimeRange(interval),
+      Common.getSqlInterval(interval)
+    );
+  }
+
+  /**
+   * Get historical block fee rates percentiles
+   */
+   public async $getHistoricalBlockFeeRates(interval: string | null = null): Promise<any> {
+    return await BlocksRepository.$getHistoricalBlockFeeRates(
+      this.getTimeRange(interval),
+      Common.getSqlInterval(interval)
+    );
+  }
+
+  /**
+   * Get historical block sizes
+   */
+   public async $getHistoricalBlockSizes(interval: string | null = null): Promise<any> {
+    return await BlocksRepository.$getHistoricalBlockSizes(
+      this.getTimeRange(interval),
+      Common.getSqlInterval(interval)
+    );
+  }
+
+  /**
+   * Get historical block weights
+   */
+   public async $getHistoricalBlockWeights(interval: string | null = null): Promise<any> {
+    return await BlocksRepository.$getHistoricalBlockWeights(
+      this.getTimeRange(interval),
+      Common.getSqlInterval(interval)
+    );
+  }
+
+  /**
+   * Generate high level overview of the pool ranks and general stats
+   */
+  public async $getPoolsStats(interval: string | null): Promise<object> {
+    const poolsStatistics = {};
+
+    const poolsInfo: PoolInfo[] = await PoolsRepository.$getPoolsInfo(interval);
+    const emptyBlocks: any[] = await BlocksRepository.$countEmptyBlocks(null, interval);
+
+    const poolsStats: PoolStats[] = [];
+    let rank = 1;
+
+    poolsInfo.forEach((poolInfo: PoolInfo) => {
+      const emptyBlocksCount = emptyBlocks.filter((emptyCount) => emptyCount.poolId === poolInfo.poolId);
+      const poolStat: PoolStats = {
+        poolId: poolInfo.poolId, // mysql row id
+        name: poolInfo.name,
+        link: poolInfo.link,
+        blockCount: poolInfo.blockCount,
+        rank: rank++,
+        emptyBlocks: emptyBlocksCount.length > 0 ? emptyBlocksCount[0]['count'] : 0,
+        slug: poolInfo.slug,
+        avgMatchRate: poolInfo.avgMatchRate !== null ? Math.round(100 * poolInfo.avgMatchRate) / 100 : null,
+      };
+      poolsStats.push(poolStat);
+    });
+
+    poolsStatistics['pools'] = poolsStats;
+
+    const blockCount: number = await BlocksRepository.$blockCount(null, interval);
+    poolsStatistics['blockCount'] = blockCount;
+
+    const totalBlock24h: number = await BlocksRepository.$blockCount(null, '24h');
+
+    try {
+      poolsStatistics['lastEstimatedHashrate'] = await bitcoinClient.getNetworkHashPs(totalBlock24h);
+    } catch (e) {
+      poolsStatistics['lastEstimatedHashrate'] = 0;
+      logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate', logger.tags.mining);
+    }
+
+    return poolsStatistics;
+  }
+
+  /**
+   * Get all mining pool stats for a pool
+   */
+  public async $getPoolStat(slug: string): Promise<object> {
+    const pool = await PoolsRepository.$getPool(slug);
+    if (!pool) {
+      throw new Error('This mining pool does not exist ' + escape(slug));
+    }
+
+    const blockCount: number = await BlocksRepository.$blockCount(pool.id);
+    const totalBlock: number = await BlocksRepository.$blockCount(null, null);
+
+    const blockCount24h: number = await BlocksRepository.$blockCount(pool.id, '24h');
+    const totalBlock24h: number = await BlocksRepository.$blockCount(null, '24h');
+
+    const blockCount1w: number = await BlocksRepository.$blockCount(pool.id, '1w');
+    const totalBlock1w: number = await BlocksRepository.$blockCount(null, '1w');
+
+    const avgHealth = await BlocksRepository.$getAvgBlockHealthPerPoolId(pool.id);    
+    const totalReward = await BlocksRepository.$getTotalRewardForPoolId(pool.id);    
+
+    let currentEstimatedHashrate = 0;
+    try {
+      currentEstimatedHashrate = await bitcoinClient.getNetworkHashPs(totalBlock24h);
+    } catch (e) {
+      logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate', logger.tags.mining);
+    }
+
+    return {
+      pool: pool,
+      blockCount: {
+        'all': blockCount,
+        '24h': blockCount24h,
+        '1w': blockCount1w,
+      },
+      blockShare: {
+        'all': blockCount / totalBlock,
+        '24h': blockCount24h / totalBlock24h,
+        '1w': blockCount1w / totalBlock1w,
+      },
+      estimatedHashrate: currentEstimatedHashrate * (blockCount24h / totalBlock24h),
+      reportedHashrate: null,
+      avgBlockHealth: avgHealth,
+      totalReward: totalReward,
+    };
+  }
+
+  /**
+   * Get miner reward stats
+   */
+  public async $getRewardStats(blockCount: number): Promise<RewardStats> {
+    return await BlocksRepository.$getBlockStats(blockCount);
+  }
+
+  /**
+   * Generate weekly mining pool hashrate history
+   */
+  public async $generatePoolHashrateHistory(): Promise<void> {
+    const now = new Date();
+
+    // Run only if:
+    // * this.lastWeeklyHashrateIndexingDate is set to null (node backend restart, reorg)
+    // * we started a new week (around Monday midnight)
+    const runIndexing = this.lastWeeklyHashrateIndexingDate === null ||
+      now.getUTCDay() === 1 && this.lastWeeklyHashrateIndexingDate !== now.getUTCDate();
+    if (!runIndexing) {
+      logger.debug(`Pool hashrate history indexing is up to date, nothing to do`, logger.tags.mining);
+      return;
+    }
+
+    try {
+      const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp;
+
+      const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0));
+      const genesisTimestamp = genesisBlock.timestamp * 1000;
+
+      const indexedTimestamp = await HashratesRepository.$getWeeklyHashrateTimestamps();
+      const hashrates: any[] = [];
+ 
+      const lastMonday = new Date(now.setDate(now.getDate() - (now.getDay() + 6) % 7));
+      const lastMondayMidnight = this.getDateMidnight(lastMonday);
+      let toTimestamp = lastMondayMidnight.getTime();
+
+      const totalWeekIndexed = (await BlocksRepository.$blockCount(null, null)) / 1008;
+      let indexedThisRun = 0;
+      let totalIndexed = 0;
+      let newlyIndexed = 0;
+      const startedAt = new Date().getTime() / 1000;
+      let timer = new Date().getTime() / 1000;
+
+      logger.debug(`Indexing weekly mining pool hashrate`, logger.tags.mining);
+      loadingIndicators.setProgress('weekly-hashrate-indexing', 0);
+
+      while (toTimestamp > genesisTimestamp && toTimestamp > oldestConsecutiveBlockTimestamp) {
+        const fromTimestamp = toTimestamp - 604800000;
+
+        // Skip already indexed weeks
+        if (indexedTimestamp.includes(toTimestamp / 1000)) {
+          toTimestamp -= 604800000;
+          ++totalIndexed;
+          continue;
+        }
+
+        const blockStats: any = await BlocksRepository.$blockCountBetweenTimestamp(
+          null, fromTimestamp / 1000, toTimestamp / 1000);
+        const lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount,
+          blockStats.lastBlockHeight);
+
+        let pools = await PoolsRepository.$getPoolsInfoBetween(fromTimestamp / 1000, toTimestamp / 1000);
+        const totalBlocks = pools.reduce((acc, pool) => acc + pool.blockCount, 0);
+        if (totalBlocks > 0) {
+          pools = pools.map((pool: any) => {
+            pool.hashrate = (pool.blockCount / totalBlocks) * lastBlockHashrate;
+            pool.share = (pool.blockCount / totalBlocks);
+            return pool;
+          });
+
+          for (const pool of pools) {
+            hashrates.push({
+              hashrateTimestamp: toTimestamp / 1000,
+              avgHashrate: pool['hashrate'] ,
+              poolId: pool.poolId,
+              share: pool['share'],
+              type: 'weekly',
+            });
+          }
+
+          newlyIndexed += hashrates.length / Math.max(1, pools.length);
+          await HashratesRepository.$saveHashrates(hashrates);
+          hashrates.length = 0;
+        }
+
+        const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
+        if (elapsedSeconds > 1) {
+          const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
+          const weeksPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds));
+          const progress = Math.round(totalIndexed / totalWeekIndexed * 10000) / 100;
+          const formattedDate = new Date(fromTimestamp).toUTCString();
+          logger.debug(`Getting weekly pool hashrate for ${formattedDate} | ~${weeksPerSeconds.toFixed(2)} weeks/sec | total: ~${totalIndexed}/${Math.round(totalWeekIndexed)} (${progress}%) | elapsed: ${runningFor} seconds`, logger.tags.mining);
+          timer = new Date().getTime() / 1000;
+          indexedThisRun = 0;
+          loadingIndicators.setProgress('weekly-hashrate-indexing', progress, false);
+        }
+
+        toTimestamp -= 604800000;
+        ++indexedThisRun;
+        ++totalIndexed;
+      }
+      this.lastWeeklyHashrateIndexingDate = new Date().getUTCDate();
+      if (newlyIndexed > 0) {
+        logger.info(`Weekly mining pools hashrates indexing completed: indexed ${newlyIndexed} weeks`, logger.tags.mining);
+      } else {
+        logger.debug(`Weekly mining pools hashrates indexing completed: indexed ${newlyIndexed} weeks`, logger.tags.mining);
+      }
+      loadingIndicators.setProgress('weekly-hashrate-indexing', 100);
+    } catch (e) {
+      loadingIndicators.setProgress('weekly-hashrate-indexing', 100);
+      logger.err(`Weekly mining pools hashrates indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`, logger.tags.mining);
+      throw e;
+    }
+  }
+
+  /**
+   * Generate daily hashrate data
+   */
+  public async $generateNetworkHashrateHistory(): Promise<void> {
+    // We only run this once a day around midnight
+    const today = new Date().getUTCDate();
+    if (today === this.lastHashrateIndexingDate) {
+      logger.debug(`Network hashrate history indexing is up to date, nothing to do`, logger.tags.mining);
+      return;
+    }
+
+    const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp;
+
+    try {
+      const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0));
+      const genesisTimestamp = genesisBlock.timestamp * 1000;
+      const indexedTimestamp = (await HashratesRepository.$getRawNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp);
+      const lastMidnight = this.getDateMidnight(new Date());
+      let toTimestamp = Math.round(lastMidnight.getTime());
+      const hashrates: any[] = [];
+
+      const totalDayIndexed = (await BlocksRepository.$blockCount(null, null)) / 144;
+      let indexedThisRun = 0;
+      let totalIndexed = 0;
+      let newlyIndexed = 0;
+      const startedAt = new Date().getTime() / 1000;
+      let timer = new Date().getTime() / 1000;
+
+      logger.debug(`Indexing daily network hashrate`, logger.tags.mining);
+      loadingIndicators.setProgress('daily-hashrate-indexing', 0);
+
+      while (toTimestamp > genesisTimestamp && toTimestamp > oldestConsecutiveBlockTimestamp) {
+        const fromTimestamp = toTimestamp - 86400000;
+
+        // Skip already indexed days
+        if (indexedTimestamp.includes(toTimestamp / 1000)) {
+          toTimestamp -= 86400000;
+          ++totalIndexed;
+          continue;
+        }
+
+        const blockStats: any = await BlocksRepository.$blockCountBetweenTimestamp(
+          null, fromTimestamp / 1000, toTimestamp / 1000);
+        const lastBlockHashrate = blockStats.blockCount === 0 ? 0 : await bitcoinClient.getNetworkHashPs(blockStats.blockCount,
+          blockStats.lastBlockHeight);
+
+        hashrates.push({
+          hashrateTimestamp: toTimestamp / 1000,
+          avgHashrate: lastBlockHashrate,
+          poolId: 0,
+          share: 1,
+          type: 'daily',
+        });
+
+        if (hashrates.length > 10) {
+          newlyIndexed += hashrates.length;
+          await HashratesRepository.$saveHashrates(hashrates);
+          hashrates.length = 0;
+        }
+
+        const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
+        if (elapsedSeconds > 1) {
+          const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
+          const daysPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds));
+          const progress = Math.round(totalIndexed / totalDayIndexed * 10000) / 100;
+          const formattedDate = new Date(fromTimestamp).toUTCString();
+          logger.debug(`Getting network daily hashrate for ${formattedDate} | ~${daysPerSeconds.toFixed(2)} days/sec | total: ~${totalIndexed}/${Math.round(totalDayIndexed)} (${progress}%) | elapsed: ${runningFor} seconds`, logger.tags.mining);
+          timer = new Date().getTime() / 1000;
+          indexedThisRun = 0;
+          loadingIndicators.setProgress('daily-hashrate-indexing', progress);
+        }
+
+        toTimestamp -= 86400000;
+        ++indexedThisRun;
+        ++totalIndexed;
+      }
+
+      // Add genesis block manually
+      if (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT === -1 && !indexedTimestamp.includes(genesisTimestamp / 1000)) {
+        hashrates.push({
+          hashrateTimestamp: genesisTimestamp / 1000,
+          avgHashrate: await bitcoinClient.getNetworkHashPs(1, 1),
+          poolId: 0,
+          share: 1,
+          type: 'daily',
+        });
+      }
+
+      newlyIndexed += hashrates.length;
+      await HashratesRepository.$saveHashrates(hashrates);
+
+      this.lastHashrateIndexingDate = new Date().getUTCDate();
+      if (newlyIndexed > 0) {
+        logger.info(`Daily network hashrate indexing completed: indexed ${newlyIndexed} days`, logger.tags.mining);
+      } else {
+        logger.debug(`Daily network hashrate indexing completed: indexed ${newlyIndexed} days`, logger.tags.mining);
+      }
+      loadingIndicators.setProgress('daily-hashrate-indexing', 100);
+    } catch (e) {
+      loadingIndicators.setProgress('daily-hashrate-indexing', 100);
+      logger.err(`Daily network hashrate indexing failed. Trying again later. Reason: ${(e instanceof Error ? e.message : e)}`, logger.tags.mining);
+      throw e;
+    }
+  }
+
+  /**
+   * Index difficulty adjustments
+   */
+  public async $indexDifficultyAdjustments(): Promise<void> {
+    const indexedHeightsArray = await DifficultyAdjustmentsRepository.$getAdjustmentsHeights();
+    const indexedHeights = {};
+    for (const height of indexedHeightsArray) {
+      indexedHeights[height] = true;
+    }
+
+    const blocks: any = await BlocksRepository.$getBlocksDifficulty();
+    const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0));
+    let currentDifficulty = genesisBlock.difficulty;
+    let totalIndexed = 0;
+
+    if (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT === -1 && indexedHeights[0] !== true) {
+      await DifficultyAdjustmentsRepository.$saveAdjustments({
+        time: genesisBlock.timestamp,
+        height: 0,
+        difficulty: currentDifficulty,
+        adjustment: 0.0,
+      });
+    }
+
+    const oldestConsecutiveBlock = await BlocksRepository.$getOldestConsecutiveBlock();
+    if (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT !== -1) {
+      currentDifficulty = oldestConsecutiveBlock.difficulty;
+    }
+
+    let totalBlockChecked = 0;
+    let timer = new Date().getTime() / 1000;
+
+    for (const block of blocks) {
+      if (block.difficulty !== currentDifficulty) {
+        if (indexedHeights[block.height] === true) { // Already indexed
+          if (block.height >= oldestConsecutiveBlock.height) {
+            currentDifficulty = block.difficulty;
+          }
+          continue;          
+        }
+
+        let adjustment = block.difficulty / currentDifficulty;
+        adjustment = Math.round(adjustment * 1000000) / 1000000; // Remove float point noise
+
+        await DifficultyAdjustmentsRepository.$saveAdjustments({
+          time: block.time,
+          height: block.height,
+          difficulty: block.difficulty,
+          adjustment: adjustment,
+        });
+
+        totalIndexed++;
+        if (block.height >= oldestConsecutiveBlock.height) {
+          currentDifficulty = block.difficulty;
+        }
+    }
+
+      totalBlockChecked++;
+      const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
+      if (elapsedSeconds > 5) {
+        const progress = Math.round(totalBlockChecked / blocks.length * 100);
+        logger.info(`Indexing difficulty adjustment at block #${block.height} | Progress: ${progress}%`, logger.tags.mining);
+        timer = new Date().getTime() / 1000;
+      }
+    }
+
+    if (totalIndexed > 0) {
+      logger.info(`Indexed ${totalIndexed} difficulty adjustments`, logger.tags.mining);
+    } else {
+      logger.debug(`Indexed ${totalIndexed} difficulty adjustments`, logger.tags.mining);
+    }
+  }
+
+  /**
+   * Create a link between blocks and the latest price at when they were mined
+   */
+  public async $indexBlockPrices(): Promise<void> {
+    if (this.blocksPriceIndexingRunning === true) {
+      return;
+    }
+    this.blocksPriceIndexingRunning = true;
+
+    try {
+      const prices: any[] = await PricesRepository.$getPricesTimesAndId();    
+      const blocksWithoutPrices: any[] = await BlocksRepository.$getBlocksWithoutPrice();
+
+      let totalInserted = 0;
+      const blocksPrices: BlockPrice[] = [];
+
+      for (const block of blocksWithoutPrices) {
+        // Quick optimisation, out mtgox feed only goes back to 2010-07-19 02:00:00, so skip the first 68951 blocks
+        if (['mainnet', 'testnet'].includes(config.MEMPOOL.NETWORK) && block.height < 68951) {
+          blocksPrices.push({
+            height: block.height,
+            priceId: prices[0].id,
+          });
+          continue;
+        }
+        for (const price of prices) {
+          if (block.timestamp < price.time) {
+            blocksPrices.push({
+              height: block.height,
+              priceId: price.id,
+            });
+            break;
+          };
+        }
+
+        if (blocksPrices.length >= 100000) {
+          totalInserted += blocksPrices.length;
+          let logStr = `Linking ${blocksPrices.length} blocks to their closest price`;
+          if (blocksWithoutPrices.length > 200000) {
+            logStr += ` | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`;
+          }
+          logger.debug(logStr, logger.tags.mining);
+          await BlocksRepository.$saveBlockPrices(blocksPrices);
+          blocksPrices.length = 0;
+        }
+      }
+
+      if (blocksPrices.length > 0) {
+        totalInserted += blocksPrices.length;
+        let logStr = `Linking ${blocksPrices.length} blocks to their closest price`;
+        if (blocksWithoutPrices.length > 200000) {
+          logStr += ` | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`;
+        }
+        logger.debug(logStr, logger.tags.mining);
+        await BlocksRepository.$saveBlockPrices(blocksPrices);
+      }
+    } catch (e) {
+      this.blocksPriceIndexingRunning = false;
+      throw e;
+    }
+
+    this.blocksPriceIndexingRunning = false;
+  }
+
+  /**
+   * Index core coinstatsindex
+   */
+  public async $indexCoinStatsIndex(): Promise<void> {
+    let timer = new Date().getTime() / 1000;
+    let totalIndexed = 0;
+
+    const blockchainInfo = await bitcoinClient.getBlockchainInfo();
+    let currentBlockHeight = blockchainInfo.blocks;
+
+    while (currentBlockHeight > 0) {
+      const indexedBlocks = await BlocksRepository.$getBlocksMissingCoinStatsIndex(
+        currentBlockHeight, currentBlockHeight - 10000);
+        
+      for (const block of indexedBlocks) {
+        const txoutset = await bitcoinClient.getTxoutSetinfo('none', block.height);
+        await BlocksRepository.$updateCoinStatsIndexData(block.hash, txoutset.txouts,
+          Math.round(txoutset.block_info.prevout_spent * 100000000));        
+        ++totalIndexed;
+
+        const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer);
+        if (elapsedSeconds > 5) {
+          logger.info(`Indexing coinstatsindex data for block #${block.height}. Indexed ${totalIndexed} blocks.`, logger.tags.mining);
+          timer = new Date().getTime() / 1000;
+        }
+      }
+
+      currentBlockHeight -= 10000;
+    }
+
+    if (totalIndexed) {
+      logger.info(`Indexing missing coinstatsindex data completed`, logger.tags.mining);
+    }
+  }
+
+  private getDateMidnight(date: Date): Date {
+    date.setUTCHours(0);
+    date.setUTCMinutes(0);
+    date.setUTCSeconds(0);
+    date.setUTCMilliseconds(0);
+
+    return date;
+  }
+
+  private getTimeRange(interval: string | null, scale = 1): number {
+    switch (interval) {
+      case '4y': return 43200 * scale; // 12h
+      case '3y': return 43200 * scale; // 12h
+      case '2y': return 28800 * scale; // 8h
+      case '1y': return 28800 * scale; // 8h
+      case '6m': return 10800 * scale; // 3h
+      case '3m': return 7200 * scale; // 2h
+      case '1m': return 1800 * scale; // 30min
+      case '1w': return 300 * scale; // 5min
+      case '3d': return 1 * scale;
+      case '24h': return 1 * scale;
+      default: return 86400 * scale;
+    }
+  }
+}
+
+export default new Mining();

backend/src/api/pools-parser.ts

@@ -0,0 +1,162 @@
+import DB from '../database';
+import logger from '../logger';
+import config from '../config';
+import PoolsRepository from '../repositories/PoolsRepository';
+import { PoolTag } from '../mempool.interfaces';
+import diskCache from './disk-cache';
+
+class PoolsParser {
+  miningPools: any[] = [];
+  unknownPool: any = {
+    'id': 0,
+    'name': 'Unknown',
+    'link': 'https://learnmeabitcoin.com/technical/coinbase-transaction',
+    'regexes': '[]',
+    'addresses': '[]',
+    'slug': 'unknown'
+  };
+  private uniqueLogs: string[] = [];
+
+  private uniqueLog(loggerFunction: any, msg: string): void {
+    if (this.uniqueLogs.includes(msg)) {
+      return;
+    }
+    this.uniqueLogs.push(msg);
+    loggerFunction(msg);
+  }
+
+  public setMiningPools(pools): void {
+    for (const pool of pools) {
+      pool.regexes = pool.tags;
+      pool.slug = pool.name.replace(/[^a-z0-9]/gi, '').toLowerCase();
+      delete(pool.tags);
+    }
+    this.miningPools = pools;
+  }
+
+  /**
+   * Populate our db with updated mining pool definition
+   * @param pools 
+   */
+  public async migratePoolsJson(): Promise<void> {
+    // We also need to wipe the backend cache to make sure we don't serve blocks with
+    // the wrong mining pool (usually happen with unknown blocks)
+    diskCache.wipeCache();
+
+    await this.$insertUnknownPool();
+
+    for (const pool of this.miningPools) {
+      if (!pool.id) {
+        logger.info(`Mining pool ${pool.name} has no unique 'id' defined. Skipping.`);
+        continue;
+      }
+
+      const poolDB = await PoolsRepository.$getPoolByUniqueId(pool.id, false);
+      if (!poolDB) {
+        // New mining pool
+        const slug = pool.name.replace(/[^a-z0-9]/gi, '').toLowerCase();
+        logger.debug(`Inserting new mining pool ${pool.name}`);
+        await PoolsRepository.$insertNewMiningPool(pool, slug);
+        await this.$deleteUnknownBlocks();
+      } else {
+        if (poolDB.name !== pool.name) {
+          // Pool has been renamed
+          const newSlug = pool.name.replace(/[^a-z0-9]/gi, '').toLowerCase();
+          logger.warn(`Renaming ${poolDB.name} mining pool to ${pool.name}. Slug has been updated. Maybe you want to make a redirection from 'https://mempool.space/mining/pool/${poolDB.slug}' to 'https://mempool.space/mining/pool/${newSlug}`);
+          await PoolsRepository.$renameMiningPool(poolDB.id, newSlug, pool.name);
+        }
+        if (poolDB.link !== pool.link) {
+          // Pool link has changed
+          logger.debug(`Updating link for ${pool.name} mining pool`);
+          await PoolsRepository.$updateMiningPoolLink(poolDB.id, pool.link);
+        }
+        if (JSON.stringify(pool.addresses) !== poolDB.addresses ||
+          JSON.stringify(pool.regexes) !== poolDB.regexes) {
+          // Pool addresses changed or coinbase tags changed
+          logger.notice(`Updating addresses and/or coinbase tags for ${pool.name} mining pool. If 'AUTOMATIC_BLOCK_REINDEXING' is enabled, we will re-index its blocks and 'unknown' blocks`);
+          await PoolsRepository.$updateMiningPoolTags(poolDB.id, pool.addresses, pool.regexes);
+          await this.$deleteBlocksForPool(poolDB);
+        }
+      }
+    }
+
+    logger.info('Mining pools-v2.json import completed');
+  }
+
+  /**
+   * Manually add the 'unknown pool'
+   */
+  public async $insertUnknownPool(): Promise<void> {
+    if (!config.DATABASE.ENABLED) {
+      return;
+    }
+
+    try {
+      const [rows]: any[] = await DB.query({ sql: 'SELECT name from pools where name="Unknown"', timeout: 120000 });
+      if (rows.length === 0) {
+        await DB.query({
+          sql: `INSERT INTO pools(name, link, regexes, addresses, slug, unique_id)
+          VALUES("${this.unknownPool.name}", "${this.unknownPool.link}", "[]", "[]", "${this.unknownPool.slug}", 0);
+        `});
+      } else {
+        await DB.query(`UPDATE pools
+          SET name='${this.unknownPool.name}', link='${this.unknownPool.link}',
+          regexes='[]', addresses='[]',
+          slug='${this.unknownPool.slug}',
+          unique_id=0
+          WHERE slug='${this.unknownPool.slug}'
+        `);
+      }
+    } catch (e) {
+      logger.err(`Unable to insert or update "Unknown" mining pool. Reason: ${e instanceof Error ? e.message : e}`);
+    }
+  }
+
+  /**
+   * Delete indexed blocks for an updated mining pool
+   * 
+   * @param pool 
+   */
+  private async $deleteBlocksForPool(pool: PoolTag): Promise<void> {
+    if (config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING === false) {
+      return;
+    }
+
+    // Get oldest blocks mined by the pool and assume pools-v2.json updates only concern most recent years
+    // Ignore early days of Bitcoin as there were no mining pool yet
+    const [oldestPoolBlock]: any[] = await DB.query(`
+      SELECT height
+      FROM blocks
+      WHERE pool_id = ?
+      ORDER BY height
+      LIMIT 1`,
+      [pool.id]
+    );
+    const oldestBlockHeight = oldestPoolBlock.length ?? 0 > 0 ? oldestPoolBlock[0].height : 130635;
+    const [unknownPool] = await DB.query(`SELECT id from pools where slug = "unknown"`);
+    this.uniqueLog(logger.notice, `Deleting blocks with unknown mining pool from height ${oldestBlockHeight} for re-indexing`);
+    await DB.query(`
+      DELETE FROM blocks
+      WHERE pool_id = ? AND height >= ${oldestBlockHeight}`,
+      [unknownPool[0].id]
+    );
+    logger.notice(`Deleting blocks from ${pool.name} mining pool for re-indexing`);
+    await DB.query(`
+      DELETE FROM blocks
+      WHERE pool_id = ?`,
+      [pool.id]
+    );
+  }
+
+  private async $deleteUnknownBlocks(): Promise<void> {
+    const [unknownPool] = await DB.query(`SELECT id from pools where slug = "unknown"`);
+    this.uniqueLog(logger.notice, `Deleting blocks with unknown mining pool from height 130635 for re-indexing`);
+    await DB.query(`
+      DELETE FROM blocks
+      WHERE pool_id = ? AND height >= 130635`,
+      [unknownPool[0].id]
+    );
+  }
+}
+
+export default new PoolsParser();

backend/src/api/rbf-cache.ts

@@ -0,0 +1,65 @@
+import { TransactionExtended } from "../mempool.interfaces";
+
+class RbfCache {
+  private replacedBy: { [txid: string]: string; } = {};
+  private replaces: { [txid: string]: string[] } = {};
+  private txs: { [txid: string]: TransactionExtended } = {};
+  private expiring: { [txid: string]: Date } = {};
+
+  constructor() {
+    setInterval(this.cleanup.bind(this), 1000 * 60 * 60);
+  }
+
+  public add(replacedTx: TransactionExtended, newTxId: string): void {
+    this.replacedBy[replacedTx.txid] = newTxId;
+    this.txs[replacedTx.txid] = replacedTx;
+    if (!this.replaces[newTxId]) {
+      this.replaces[newTxId] = [];
+    }
+    this.replaces[newTxId].push(replacedTx.txid);
+  }
+
+  public getReplacedBy(txId: string): string | undefined {
+    return this.replacedBy[txId];
+  }
+
+  public getReplaces(txId: string): string[] | undefined {
+    return this.replaces[txId];
+  }
+
+  public getTx(txId: string): TransactionExtended | undefined {
+    return this.txs[txId];
+  }
+
+  // flag a transaction as removed from the mempool
+  public evict(txid): void {
+    this.expiring[txid] = new Date(Date.now() + 1000 * 86400); // 24 hours
+  }
+
+  private cleanup(): void {
+    const currentDate = new Date();
+    for (const txid in this.expiring) {
+      if (this.expiring[txid] < currentDate) {
+        delete this.expiring[txid];
+        this.remove(txid);
+      }
+    }
+  }
+
+  // remove a transaction & all previous versions from the cache
+  private remove(txid): void {
+    // don't remove a transaction while a newer version remains in the mempool
+    if (this.replaces[txid] && !this.replacedBy[txid]) {
+      const replaces = this.replaces[txid];
+      delete this.replaces[txid];
+      for (const tx of replaces) {
+        // recursively remove prior versions from the cache
+        delete this.replacedBy[tx];
+        delete this.txs[tx];
+        this.remove(tx);
+      }
+    }
+  }
+}
+
+export default new RbfCache();

backend/src/api/statistics.ts

@@ -1,466 +0,0 @@
-import memPool from './mempool';
-import { DB } from '../database';
-import logger from '../logger';
-
-import { Statistic, TransactionExtended, OptimizedStatistic } from '../mempool.interfaces';
-
-class Statistics {
-  protected intervalTimer: NodeJS.Timer | undefined;
-  protected newStatisticsEntryCallback: ((stats: OptimizedStatistic) => void) | undefined;
-  protected queryTimeout = 120000;
-  protected cache: { [date: string]: OptimizedStatistic[] } = {
-    '24h': [], '1w': [], '1m': [], '3m': [], '6m': [], '1y': [],
-  };
-
-  public setNewStatisticsEntryCallback(fn: (stats: OptimizedStatistic) => void) {
-    this.newStatisticsEntryCallback = fn;
-  }
-
-  constructor() { }
-
-  public startStatistics(): void {
-    logger.info('Starting statistics service');
-
-    const now = new Date();
-    const nextInterval = new Date(now.getFullYear(), now.getMonth(), now.getDate(), now.getHours(),
-      Math.floor(now.getMinutes() / 1) * 1 + 1, 0, 0);
-    const difference = nextInterval.getTime() - now.getTime();
-
-    setTimeout(() => {
-      this.runStatistics();
-      this.intervalTimer = setInterval(() => {
-        this.runStatistics();
-      }, 1 * 60 * 1000);
-    }, difference);
-
-    this.createCache();
-    setInterval(this.createCache.bind(this), 600000);
-  }
-
-  public getCache() {
-    return this.cache;
-  }
-
-  private async createCache() {
-    this.cache['24h'] = await this.$list24H();
-    this.cache['1w'] = await this.$list1W();
-    this.cache['1m'] = await this.$list1M();
-    this.cache['3m'] = await this.$list3M();
-    this.cache['6m'] = await this.$list6M();
-    this.cache['1y'] = await this.$list1Y();
-    logger.debug('Statistics cache created');
-  }
-
-  private async runStatistics(): Promise<void> {
-    if (!memPool.isInSync()) {
-      return;
-    }
-    const currentMempool = memPool.getMempool();
-    const txPerSecond = memPool.getTxPerSecond();
-    const vBytesPerSecond = memPool.getVBytesPerSecond();
-
-    logger.debug('Running statistics');
-
-    let memPoolArray: TransactionExtended[] = [];
-    for (const i in currentMempool) {
-      if (currentMempool.hasOwnProperty(i)) {
-        memPoolArray.push(currentMempool[i]);
-      }
-    }
-    // Remove 0 and undefined
-    memPoolArray = memPoolArray.filter((tx) => tx.effectiveFeePerVsize);
-
-    if (!memPoolArray.length) {
-      return;
-    }
-
-    memPoolArray.sort((a, b) => a.effectiveFeePerVsize - b.effectiveFeePerVsize);
-    const totalWeight = memPoolArray.map((tx) => tx.vsize).reduce((acc, curr) => acc + curr) * 4;
-    const totalFee = memPoolArray.map((tx) => tx.fee).reduce((acc, curr) => acc + curr);
-
-    const logFees = [1, 2, 3, 4, 5, 6, 8, 10, 12, 15, 20, 30, 40, 50, 60, 70, 80, 90, 100, 125, 150, 175, 200,
-      250, 300, 350, 400, 500, 600, 700, 800, 900, 1000, 1200, 1400, 1600, 1800, 2000];
-
-    const weightVsizeFees: { [feePerWU: number]: number } = {};
-
-    memPoolArray.forEach((transaction) => {
-      for (let i = 0; i < logFees.length; i++) {
-        if ((logFees[i] === 2000 && transaction.effectiveFeePerVsize >= 2000) || transaction.effectiveFeePerVsize <= logFees[i]) {
-          if (weightVsizeFees[logFees[i]]) {
-            weightVsizeFees[logFees[i]] += transaction.vsize;
-          } else {
-            weightVsizeFees[logFees[i]] = transaction.vsize;
-          }
-          break;
-        }
-      }
-    });
-
-    const insertId = await this.$create({
-      added: 'NOW()',
-      unconfirmed_transactions: memPoolArray.length,
-      tx_per_second: txPerSecond,
-      vbytes_per_second: Math.round(vBytesPerSecond),
-      mempool_byte_weight: totalWeight,
-      total_fee: totalFee,
-      fee_data: '',
-      vsize_1: weightVsizeFees['1'] || 0,
-      vsize_2: weightVsizeFees['2'] || 0,
-      vsize_3: weightVsizeFees['3'] || 0,
-      vsize_4: weightVsizeFees['4'] || 0,
-      vsize_5: weightVsizeFees['5'] || 0,
-      vsize_6: weightVsizeFees['6'] || 0,
-      vsize_8: weightVsizeFees['8'] || 0,
-      vsize_10: weightVsizeFees['10'] || 0,
-      vsize_12: weightVsizeFees['12'] || 0,
-      vsize_15: weightVsizeFees['15'] || 0,
-      vsize_20: weightVsizeFees['20'] || 0,
-      vsize_30: weightVsizeFees['30'] || 0,
-      vsize_40: weightVsizeFees['40'] || 0,
-      vsize_50: weightVsizeFees['50'] || 0,
-      vsize_60: weightVsizeFees['60'] || 0,
-      vsize_70: weightVsizeFees['70'] || 0,
-      vsize_80: weightVsizeFees['80'] || 0,
-      vsize_90: weightVsizeFees['90'] || 0,
-      vsize_100: weightVsizeFees['100'] || 0,
-      vsize_125: weightVsizeFees['125'] || 0,
-      vsize_150: weightVsizeFees['150'] || 0,
-      vsize_175: weightVsizeFees['175'] || 0,
-      vsize_200: weightVsizeFees['200'] || 0,
-      vsize_250: weightVsizeFees['250'] || 0,
-      vsize_300: weightVsizeFees['300'] || 0,
-      vsize_350: weightVsizeFees['350'] || 0,
-      vsize_400: weightVsizeFees['400'] || 0,
-      vsize_500: weightVsizeFees['500'] || 0,
-      vsize_600: weightVsizeFees['600'] || 0,
-      vsize_700: weightVsizeFees['700'] || 0,
-      vsize_800: weightVsizeFees['800'] || 0,
-      vsize_900: weightVsizeFees['900'] || 0,
-      vsize_1000: weightVsizeFees['1000'] || 0,
-      vsize_1200: weightVsizeFees['1200'] || 0,
-      vsize_1400: weightVsizeFees['1400'] || 0,
-      vsize_1600: weightVsizeFees['1600'] || 0,
-      vsize_1800: weightVsizeFees['1800'] || 0,
-      vsize_2000: weightVsizeFees['2000'] || 0,
-    });
-
-    if (this.newStatisticsEntryCallback && insertId) {
-      const newStats = await this.$get(insertId);
-      if (newStats) {
-        this.newStatisticsEntryCallback(newStats);
-      }
-    }
-  }
-
-  private async $create(statistics: Statistic): Promise<number | undefined> {
-    try {
-      const connection = await DB.pool.getConnection();
-      const query = `INSERT INTO statistics(
-              added,
-              unconfirmed_transactions,
-              tx_per_second,
-              vbytes_per_second,
-              mempool_byte_weight,
-              fee_data,
-              total_fee,
-              vsize_1,
-              vsize_2,
-              vsize_3,
-              vsize_4,
-              vsize_5,
-              vsize_6,
-              vsize_8,
-              vsize_10,
-              vsize_12,
-              vsize_15,
-              vsize_20,
-              vsize_30,
-              vsize_40,
-              vsize_50,
-              vsize_60,
-              vsize_70,
-              vsize_80,
-              vsize_90,
-              vsize_100,
-              vsize_125,
-              vsize_150,
-              vsize_175,
-              vsize_200,
-              vsize_250,
-              vsize_300,
-              vsize_350,
-              vsize_400,
-              vsize_500,
-              vsize_600,
-              vsize_700,
-              vsize_800,
-              vsize_900,
-              vsize_1000,
-              vsize_1200,
-              vsize_1400,
-              vsize_1600,
-              vsize_1800,
-              vsize_2000
-            )
-            VALUES (${statistics.added}, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
-               ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
-
-      const params: (string | number)[] = [
-        statistics.unconfirmed_transactions,
-        statistics.tx_per_second,
-        statistics.vbytes_per_second,
-        statistics.mempool_byte_weight,
-        statistics.fee_data,
-        statistics.total_fee,
-        statistics.vsize_1,
-        statistics.vsize_2,
-        statistics.vsize_3,
-        statistics.vsize_4,
-        statistics.vsize_5,
-        statistics.vsize_6,
-        statistics.vsize_8,
-        statistics.vsize_10,
-        statistics.vsize_12,
-        statistics.vsize_15,
-        statistics.vsize_20,
-        statistics.vsize_30,
-        statistics.vsize_40,
-        statistics.vsize_50,
-        statistics.vsize_60,
-        statistics.vsize_70,
-        statistics.vsize_80,
-        statistics.vsize_90,
-        statistics.vsize_100,
-        statistics.vsize_125,
-        statistics.vsize_150,
-        statistics.vsize_175,
-        statistics.vsize_200,
-        statistics.vsize_250,
-        statistics.vsize_300,
-        statistics.vsize_350,
-        statistics.vsize_400,
-        statistics.vsize_500,
-        statistics.vsize_600,
-        statistics.vsize_700,
-        statistics.vsize_800,
-        statistics.vsize_900,
-        statistics.vsize_1000,
-        statistics.vsize_1200,
-        statistics.vsize_1400,
-        statistics.vsize_1600,
-        statistics.vsize_1800,
-        statistics.vsize_2000,
-      ];
-      const [result]: any = await connection.query(query, params);
-      connection.release();
-      return result.insertId;
-    } catch (e) {
-      logger.err('$create() error' + e.message || e);
-    }
-  }
-
-  private getQueryForDays(div: number) {
-    return `SELECT id, added, unconfirmed_transactions,
-      tx_per_second,
-      vbytes_per_second,
-      vsize_1,
-      vsize_2,
-      vsize_3,
-      vsize_4,
-      vsize_5,
-      vsize_6,
-      vsize_8,
-      vsize_10,
-      vsize_12,
-      vsize_15,
-      vsize_20,
-      vsize_30,
-      vsize_40,
-      vsize_50,
-      vsize_60,
-      vsize_70,
-      vsize_80,
-      vsize_90,
-      vsize_100,
-      vsize_125,
-      vsize_150,
-      vsize_175,
-      vsize_200,
-      vsize_250,
-      vsize_300,
-      vsize_350,
-      vsize_400,
-      vsize_500,
-      vsize_600,
-      vsize_700,
-      vsize_800,
-      vsize_900,
-      vsize_1000,
-      vsize_1200,
-      vsize_1400,
-      vsize_1600,
-      vsize_1800,
-      vsize_2000 FROM statistics GROUP BY UNIX_TIMESTAMP(added) DIV ${div} ORDER BY id DESC LIMIT 480`;
-  }
-
-  public async $get(id: number): Promise<OptimizedStatistic | undefined> {
-    try {
-      const connection = await DB.pool.getConnection();
-      const query = `SELECT * FROM statistics WHERE id = ?`;
-      const [rows] = await connection.query<any>(query, [id]);
-      connection.release();
-      if (rows[0]) {
-        return this.mapStatisticToOptimizedStatistic([rows[0]])[0];
-      }
-    } catch (e) {
-      logger.err('$list2H() error' + e.message || e);
-    }
-  }
-
-  public async $list2H(): Promise<OptimizedStatistic[]> {
-    try {
-      const connection = await DB.pool.getConnection();
-      const query = `SELECT * FROM statistics ORDER BY id DESC LIMIT 120`;
-      const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
-      connection.release();
-      return this.mapStatisticToOptimizedStatistic(rows);
-    } catch (e) {
-      logger.err('$list2H() error' + e.message || e);
-      return [];
-    }
-  }
-
-  public async $list24H(): Promise<OptimizedStatistic[]> {
-    try {
-      const connection = await DB.pool.getConnection();
-      const query = this.getQueryForDays(180);
-      const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
-      connection.release();
-      return this.mapStatisticToOptimizedStatistic(rows);
-    } catch (e) {
-      logger.err('$list24h() error' + e.message || e);
-      return [];
-    }
-  }
-
-  public async $list1W(): Promise<OptimizedStatistic[]> {
-    try {
-      const connection = await DB.pool.getConnection();
-      const query = this.getQueryForDays(1260);
-      const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
-      connection.release();
-      return this.mapStatisticToOptimizedStatistic(rows);
-    } catch (e) {
-      logger.err('$list1W() error' + e);
-      return [];
-    }
-  }
-
-  public async $list1M(): Promise<OptimizedStatistic[]> {
-    try {
-      const connection = await DB.pool.getConnection();
-      const query = this.getQueryForDays(5040);
-      const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
-      connection.release();
-      return this.mapStatisticToOptimizedStatistic(rows);
-    } catch (e) {
-      logger.err('$list1M() error' + e);
-      return [];
-    }
-  }
-
-  public async $list3M(): Promise<OptimizedStatistic[]> {
-    try {
-      const connection = await DB.pool.getConnection();
-      const query = this.getQueryForDays(15120);
-      const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
-      connection.release();
-      return this.mapStatisticToOptimizedStatistic(rows);
-    } catch (e) {
-      logger.err('$list3M() error' + e);
-      return [];
-    }
-  }
-
-  public async $list6M(): Promise<OptimizedStatistic[]> {
-    try {
-      const connection = await DB.pool.getConnection();
-      const query = this.getQueryForDays(30240);
-      const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
-      connection.release();
-      return this.mapStatisticToOptimizedStatistic(rows);
-    } catch (e) {
-      logger.err('$list6M() error' + e);
-      return [];
-    }
-  }
-
-  public async $list1Y(): Promise<OptimizedStatistic[]> {
-    try {
-      const connection = await DB.pool.getConnection();
-      const query = this.getQueryForDays(60480);
-      const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
-      connection.release();
-      return this.mapStatisticToOptimizedStatistic(rows);
-    } catch (e) {
-      logger.err('$list6M() error' + e);
-      return [];
-    }
-  }
-  private mapStatisticToOptimizedStatistic(statistic: Statistic[]): OptimizedStatistic[] {
-    return statistic.map((s) => {
-      return {
-        id: s.id || 0,
-        added: s.added,
-        unconfirmed_transactions: s.unconfirmed_transactions,
-        tx_per_second: s.tx_per_second,
-        vbytes_per_second: s.vbytes_per_second,
-        mempool_byte_weight: s.mempool_byte_weight,
-        total_fee: s.total_fee,
-        vsizes: [
-          s.vsize_1,
-          s.vsize_2,
-          s.vsize_3,
-          s.vsize_4,
-          s.vsize_5,
-          s.vsize_6,
-          s.vsize_8,
-          s.vsize_10,
-          s.vsize_12,
-          s.vsize_15,
-          s.vsize_20,
-          s.vsize_30,
-          s.vsize_40,
-          s.vsize_50,
-          s.vsize_60,
-          s.vsize_70,
-          s.vsize_80,
-          s.vsize_90,
-          s.vsize_100,
-          s.vsize_125,
-          s.vsize_150,
-          s.vsize_175,
-          s.vsize_200,
-          s.vsize_250,
-          s.vsize_300,
-          s.vsize_350,
-          s.vsize_400,
-          s.vsize_500,
-          s.vsize_600,
-          s.vsize_700,
-          s.vsize_800,
-          s.vsize_900,
-          s.vsize_1000,
-          s.vsize_1200,
-          s.vsize_1400,
-          s.vsize_1600,
-          s.vsize_1800,
-          s.vsize_2000,
-        ]
-      };
-    });
-  }
-
-}
-
-export default new Statistics();

backend/src/api/statistics/statistics-api.ts

@@ -0,0 +1,441 @@
+import DB from '../../database';
+import logger from '../../logger';
+import { Statistic, OptimizedStatistic } from '../../mempool.interfaces';
+
+class StatisticsApi {
+  protected queryTimeout = 120000;
+
+  public async $createZeroedStatistic(): Promise<number | undefined> {
+    try {
+      const query = `INSERT INTO statistics(
+              added,
+              unconfirmed_transactions,
+              tx_per_second,
+              vbytes_per_second,
+              mempool_byte_weight,
+              fee_data,
+              total_fee,
+              vsize_1,
+              vsize_2,
+              vsize_3,
+              vsize_4,
+              vsize_5,
+              vsize_6,
+              vsize_8,
+              vsize_10,
+              vsize_12,
+              vsize_15,
+              vsize_20,
+              vsize_30,
+              vsize_40,
+              vsize_50,
+              vsize_60,
+              vsize_70,
+              vsize_80,
+              vsize_90,
+              vsize_100,
+              vsize_125,
+              vsize_150,
+              vsize_175,
+              vsize_200,
+              vsize_250,
+              vsize_300,
+              vsize_350,
+              vsize_400,
+              vsize_500,
+              vsize_600,
+              vsize_700,
+              vsize_800,
+              vsize_900,
+              vsize_1000,
+              vsize_1200,
+              vsize_1400,
+              vsize_1600,
+              vsize_1800,
+              vsize_2000
+            )
+            VALUES (NOW(), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+               0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)`;
+      const [result]: any = await DB.query(query);
+      return result.insertId;
+    } catch (e) {
+      logger.err('$create() error' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  public async $create(statistics: Statistic): Promise<number | undefined> {
+    try {
+      const query = `INSERT INTO statistics(
+              added,
+              unconfirmed_transactions,
+              tx_per_second,
+              vbytes_per_second,
+              mempool_byte_weight,
+              fee_data,
+              total_fee,
+              vsize_1,
+              vsize_2,
+              vsize_3,
+              vsize_4,
+              vsize_5,
+              vsize_6,
+              vsize_8,
+              vsize_10,
+              vsize_12,
+              vsize_15,
+              vsize_20,
+              vsize_30,
+              vsize_40,
+              vsize_50,
+              vsize_60,
+              vsize_70,
+              vsize_80,
+              vsize_90,
+              vsize_100,
+              vsize_125,
+              vsize_150,
+              vsize_175,
+              vsize_200,
+              vsize_250,
+              vsize_300,
+              vsize_350,
+              vsize_400,
+              vsize_500,
+              vsize_600,
+              vsize_700,
+              vsize_800,
+              vsize_900,
+              vsize_1000,
+              vsize_1200,
+              vsize_1400,
+              vsize_1600,
+              vsize_1800,
+              vsize_2000
+            )
+            VALUES (${statistics.added}, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
+               ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
+
+      const params: (string | number)[] = [
+        statistics.unconfirmed_transactions,
+        statistics.tx_per_second,
+        statistics.vbytes_per_second,
+        statistics.mempool_byte_weight,
+        statistics.fee_data,
+        statistics.total_fee,
+        statistics.vsize_1,
+        statistics.vsize_2,
+        statistics.vsize_3,
+        statistics.vsize_4,
+        statistics.vsize_5,
+        statistics.vsize_6,
+        statistics.vsize_8,
+        statistics.vsize_10,
+        statistics.vsize_12,
+        statistics.vsize_15,
+        statistics.vsize_20,
+        statistics.vsize_30,
+        statistics.vsize_40,
+        statistics.vsize_50,
+        statistics.vsize_60,
+        statistics.vsize_70,
+        statistics.vsize_80,
+        statistics.vsize_90,
+        statistics.vsize_100,
+        statistics.vsize_125,
+        statistics.vsize_150,
+        statistics.vsize_175,
+        statistics.vsize_200,
+        statistics.vsize_250,
+        statistics.vsize_300,
+        statistics.vsize_350,
+        statistics.vsize_400,
+        statistics.vsize_500,
+        statistics.vsize_600,
+        statistics.vsize_700,
+        statistics.vsize_800,
+        statistics.vsize_900,
+        statistics.vsize_1000,
+        statistics.vsize_1200,
+        statistics.vsize_1400,
+        statistics.vsize_1600,
+        statistics.vsize_1800,
+        statistics.vsize_2000,
+      ];
+      const [result]: any = await DB.query(query, params);
+      return result.insertId;
+    } catch (e) {
+      logger.err('$create() error' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  private getQueryForDaysAvg(div: number, interval: string) {
+    return `SELECT
+      UNIX_TIMESTAMP(added) as added,
+      CAST(avg(vbytes_per_second) as DOUBLE) as vbytes_per_second,
+      CAST(avg(vsize_1) as DOUBLE) as vsize_1,
+      CAST(avg(vsize_2) as DOUBLE) as vsize_2,
+      CAST(avg(vsize_3) as DOUBLE) as vsize_3,
+      CAST(avg(vsize_4) as DOUBLE) as vsize_4,
+      CAST(avg(vsize_5) as DOUBLE) as vsize_5,
+      CAST(avg(vsize_6) as DOUBLE) as vsize_6,
+      CAST(avg(vsize_8) as DOUBLE) as vsize_8,
+      CAST(avg(vsize_10) as DOUBLE) as vsize_10,
+      CAST(avg(vsize_12) as DOUBLE) as vsize_12,
+      CAST(avg(vsize_15) as DOUBLE) as vsize_15,
+      CAST(avg(vsize_20) as DOUBLE) as vsize_20,
+      CAST(avg(vsize_30) as DOUBLE) as vsize_30,
+      CAST(avg(vsize_40) as DOUBLE) as vsize_40,
+      CAST(avg(vsize_50) as DOUBLE) as vsize_50,
+      CAST(avg(vsize_60) as DOUBLE) as vsize_60,
+      CAST(avg(vsize_70) as DOUBLE) as vsize_70,
+      CAST(avg(vsize_80) as DOUBLE) as vsize_80,
+      CAST(avg(vsize_90) as DOUBLE) as vsize_90,
+      CAST(avg(vsize_100) as DOUBLE) as vsize_100,
+      CAST(avg(vsize_125) as DOUBLE) as vsize_125,
+      CAST(avg(vsize_150) as DOUBLE) as vsize_150,
+      CAST(avg(vsize_175) as DOUBLE) as vsize_175,
+      CAST(avg(vsize_200) as DOUBLE) as vsize_200,
+      CAST(avg(vsize_250) as DOUBLE) as vsize_250,
+      CAST(avg(vsize_300) as DOUBLE) as vsize_300,
+      CAST(avg(vsize_350) as DOUBLE) as vsize_350,
+      CAST(avg(vsize_400) as DOUBLE) as vsize_400,
+      CAST(avg(vsize_500) as DOUBLE) as vsize_500,
+      CAST(avg(vsize_600) as DOUBLE) as vsize_600,
+      CAST(avg(vsize_700) as DOUBLE) as vsize_700,
+      CAST(avg(vsize_800) as DOUBLE) as vsize_800,
+      CAST(avg(vsize_900) as DOUBLE) as vsize_900,
+      CAST(avg(vsize_1000) as DOUBLE) as vsize_1000,
+      CAST(avg(vsize_1200) as DOUBLE) as vsize_1200,
+      CAST(avg(vsize_1400) as DOUBLE) as vsize_1400,
+      CAST(avg(vsize_1600) as DOUBLE) as vsize_1600,
+      CAST(avg(vsize_1800) as DOUBLE) as vsize_1800,
+      CAST(avg(vsize_2000) as DOUBLE) as vsize_2000 \
+      FROM statistics \
+      WHERE added BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW() \
+      GROUP BY UNIX_TIMESTAMP(added) DIV ${div} \
+      ORDER BY statistics.added DESC;`;
+  }
+
+  private getQueryForDays(div: number, interval: string) {
+    return `SELECT
+      UNIX_TIMESTAMP(added) as added,
+      CAST(avg(vbytes_per_second) as DOUBLE) as vbytes_per_second,
+      vsize_1,
+      vsize_2,
+      vsize_3,
+      vsize_4,
+      vsize_5,
+      vsize_6,
+      vsize_8,
+      vsize_10,
+      vsize_12,
+      vsize_15,
+      vsize_20,
+      vsize_30,
+      vsize_40,
+      vsize_50,
+      vsize_60,
+      vsize_70,
+      vsize_80,
+      vsize_90,
+      vsize_100,
+      vsize_125,
+      vsize_150,
+      vsize_175,
+      vsize_200,
+      vsize_250,
+      vsize_300,
+      vsize_350,
+      vsize_400,
+      vsize_500,
+      vsize_600,
+      vsize_700,
+      vsize_800,
+      vsize_900,
+      vsize_1000,
+      vsize_1200,
+      vsize_1400,
+      vsize_1600,
+      vsize_1800,
+      vsize_2000 \
+      FROM statistics \
+      WHERE added BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW() \
+      GROUP BY UNIX_TIMESTAMP(added) DIV ${div} \
+      ORDER BY statistics.added DESC;`;
+  }
+
+  public async $get(id: number): Promise<OptimizedStatistic | undefined> {
+    try {
+      const query = `SELECT *, UNIX_TIMESTAMP(added) as added FROM statistics WHERE id = ?`;
+      const [rows] = await DB.query(query, [id]);
+      if (rows[0]) {
+        return this.mapStatisticToOptimizedStatistic([rows[0]])[0];
+      }
+    } catch (e) {
+      logger.err('$list2H() error' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  public async $list2H(): Promise<OptimizedStatistic[]> {
+    try {
+      const query = `SELECT *, UNIX_TIMESTAMP(added) as added FROM statistics ORDER BY statistics.added DESC LIMIT 120`;
+      const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
+      return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
+    } catch (e) {
+      logger.err('$list2H() error' + (e instanceof Error ? e.message : e));
+      return [];
+    }
+  }
+
+  public async $list24H(): Promise<OptimizedStatistic[]> {
+    try {
+      const query = `SELECT *, UNIX_TIMESTAMP(added) as added FROM statistics ORDER BY statistics.added DESC LIMIT 1440`;
+      const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
+      return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
+    } catch (e) {
+      logger.err('$list24h() error' + (e instanceof Error ? e.message : e));
+      return [];
+    }
+  }
+
+  public async $list1W(): Promise<OptimizedStatistic[]> {
+    try {
+      const query = this.getQueryForDaysAvg(300, '1 WEEK'); // 5m interval
+      const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
+      return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
+    } catch (e) {
+      logger.err('$list1W() error' + (e instanceof Error ? e.message : e));
+      return [];
+    }
+  }
+
+  public async $list1M(): Promise<OptimizedStatistic[]> {
+    try {
+      const query = this.getQueryForDaysAvg(1800, '1 MONTH'); // 30m interval
+      const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
+      return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
+    } catch (e) {
+      logger.err('$list1M() error' + (e instanceof Error ? e.message : e));
+      return [];
+    }
+  }
+
+  public async $list3M(): Promise<OptimizedStatistic[]> {
+    try {
+      const query = this.getQueryForDaysAvg(7200, '3 MONTH'); // 2h interval
+      const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
+      return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
+    } catch (e) {
+      logger.err('$list3M() error' + (e instanceof Error ? e.message : e));
+      return [];
+    }
+  }
+
+  public async $list6M(): Promise<OptimizedStatistic[]> {
+    try {
+      const query = this.getQueryForDaysAvg(10800, '6 MONTH'); // 3h interval
+      const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
+      return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
+    } catch (e) {
+      logger.err('$list6M() error' + (e instanceof Error ? e.message : e));
+      return [];
+    }
+  }
+
+  public async $list1Y(): Promise<OptimizedStatistic[]> {
+    try {
+      const query = this.getQueryForDays(28800, '1 YEAR'); // 8h interval
+      const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
+      return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
+    } catch (e) {
+      logger.err('$list1Y() error' + (e instanceof Error ? e.message : e));
+      return [];
+    }
+  }
+
+  public async $list2Y(): Promise<OptimizedStatistic[]> {
+    try {
+      const query = this.getQueryForDays(28800, '2 YEAR'); // 8h interval
+      const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
+      return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
+    } catch (e) {
+      logger.err('$list2Y() error' + (e instanceof Error ? e.message : e));
+      return [];
+    }
+  }
+
+  public async $list3Y(): Promise<OptimizedStatistic[]> {
+    try {
+      const query = this.getQueryForDays(43200, '3 YEAR'); // 12h interval
+      const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
+      return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
+    } catch (e) {
+      logger.err('$list3Y() error' + (e instanceof Error ? e.message : e));
+      return [];
+    }
+  }
+
+  public async $list4Y(): Promise<OptimizedStatistic[]> {
+    try {
+      const query = this.getQueryForDays(43200, '4 YEAR'); // 12h interval
+      const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
+      return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
+    } catch (e) {
+      logger.err('$list4Y() error' + (e instanceof Error ? e.message : e));
+      return [];
+    }
+  }
+
+  private mapStatisticToOptimizedStatistic(statistic: Statistic[]): OptimizedStatistic[] {
+    return statistic.map((s) => {
+      return {
+        added: s.added,
+        vbytes_per_second: s.vbytes_per_second,
+        mempool_byte_weight: s.mempool_byte_weight,
+        total_fee: s.total_fee,
+        vsizes: [
+          s.vsize_1,
+          s.vsize_2,
+          s.vsize_3,
+          s.vsize_4,
+          s.vsize_5,
+          s.vsize_6,
+          s.vsize_8,
+          s.vsize_10,
+          s.vsize_12,
+          s.vsize_15,
+          s.vsize_20,
+          s.vsize_30,
+          s.vsize_40,
+          s.vsize_50,
+          s.vsize_60,
+          s.vsize_70,
+          s.vsize_80,
+          s.vsize_90,
+          s.vsize_100,
+          s.vsize_125,
+          s.vsize_150,
+          s.vsize_175,
+          s.vsize_200,
+          s.vsize_250,
+          s.vsize_300,
+          s.vsize_350,
+          s.vsize_400,
+          s.vsize_500,
+          s.vsize_600,
+          s.vsize_700,
+          s.vsize_800,
+          s.vsize_900,
+          s.vsize_1000,
+          s.vsize_1200,
+          s.vsize_1400,
+          s.vsize_1600,
+          s.vsize_1800,
+          s.vsize_2000,
+        ]
+      };
+    });
+  }
+}
+
+export default new StatisticsApi();

backend/src/api/statistics/statistics.routes.ts

@@ -0,0 +1,71 @@
+import { Application, Request, Response } from 'express';
+import config from '../../config';
+import statisticsApi from './statistics-api';
+
+class StatisticsRoutes {
+  public initRoutes(app: Application) {
+    app
+      .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/2h', this.$getStatisticsByTime.bind(this, '2h'))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/24h', this.$getStatisticsByTime.bind(this, '24h'))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/1w', this.$getStatisticsByTime.bind(this, '1w'))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/1m', this.$getStatisticsByTime.bind(this, '1m'))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/3m', this.$getStatisticsByTime.bind(this, '3m'))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/6m', this.$getStatisticsByTime.bind(this, '6m'))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/1y', this.$getStatisticsByTime.bind(this, '1y'))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/2y', this.$getStatisticsByTime.bind(this, '2y'))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/3y', this.$getStatisticsByTime.bind(this, '3y'))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/4y', this.$getStatisticsByTime.bind(this, '4y'))
+    ;
+  }
+
+  private async $getStatisticsByTime(time: '2h' | '24h' | '1w' | '1m' | '3m' | '6m' | '1y' | '2y' | '3y' | '4y', req: Request, res: Response) {
+    res.header('Pragma', 'public');
+    res.header('Cache-control', 'public');
+    res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
+
+    try {
+      let result;
+      switch (time as string) {
+        case '2h':
+          result = await statisticsApi.$list2H();
+          res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
+          break;
+        case '24h':
+          result = await statisticsApi.$list24H();
+          res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+          break;
+        case '1w':
+          result = await statisticsApi.$list1W();
+          break;
+        case '1m':
+          result = await statisticsApi.$list1M();
+          break;
+        case '3m':
+          result = await statisticsApi.$list3M();
+          break;
+        case '6m':
+          result = await statisticsApi.$list6M();
+          break;
+        case '1y':
+          result = await statisticsApi.$list1Y();
+          break;
+        case '2y':
+          result = await statisticsApi.$list2Y();
+          break;
+        case '3y':
+          result = await statisticsApi.$list3Y();
+          break;
+        case '4y':
+          result = await statisticsApi.$list4Y();
+          break;
+        default:
+          result = await statisticsApi.$list2H();
+      }
+      res.json(result);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+}
+
+export default new StatisticsRoutes();

backend/src/api/statistics/statistics.ts

@@ -0,0 +1,153 @@
+import memPool from '../mempool';
+import logger from '../../logger';
+import { TransactionExtended, OptimizedStatistic } from '../../mempool.interfaces';
+import { Common } from '../common';
+import statisticsApi from './statistics-api';
+
+class Statistics {
+  protected intervalTimer: NodeJS.Timer | undefined;
+  protected newStatisticsEntryCallback: ((stats: OptimizedStatistic) => void) | undefined;
+
+  public setNewStatisticsEntryCallback(fn: (stats: OptimizedStatistic) => void) {
+    this.newStatisticsEntryCallback = fn;
+  }
+
+  public startStatistics(): void {
+    logger.info('Starting statistics service');
+
+    const now = new Date();
+    const nextInterval = new Date(now.getFullYear(), now.getMonth(), now.getDate(), now.getHours(),
+      Math.floor(now.getMinutes() / 1) * 1 + 1, 0, 0);
+    const difference = nextInterval.getTime() - now.getTime();
+
+    setTimeout(() => {
+      this.runStatistics();
+      this.intervalTimer = setInterval(() => {
+        this.runStatistics();
+      }, 1 * 60 * 1000);
+    }, difference);
+  }
+
+  private async runStatistics(): Promise<void> {
+    if (!memPool.isInSync()) {
+      return;
+    }
+    const currentMempool = memPool.getMempool();
+    const txPerSecond = memPool.getTxPerSecond();
+    const vBytesPerSecond = memPool.getVBytesPerSecond();
+
+    logger.debug('Running statistics');
+
+    let memPoolArray: TransactionExtended[] = [];
+    for (const i in currentMempool) {
+      if (currentMempool.hasOwnProperty(i)) {
+        memPoolArray.push(currentMempool[i]);
+      }
+    }
+    // Remove 0 and undefined
+    memPoolArray = memPoolArray.filter((tx) => tx.effectiveFeePerVsize);
+
+    if (!memPoolArray.length) {
+      try {
+        const insertIdZeroed = await statisticsApi.$createZeroedStatistic();
+        if (this.newStatisticsEntryCallback && insertIdZeroed) {
+          const newStats = await statisticsApi.$get(insertIdZeroed);
+          if (newStats) {
+            this.newStatisticsEntryCallback(newStats);
+          }
+        }
+      } catch (e) {
+        logger.err('Unable to insert zeroed statistics. ' + e);
+      }
+      return;
+    }
+
+    memPoolArray.sort((a, b) => a.effectiveFeePerVsize - b.effectiveFeePerVsize);
+    const totalWeight = memPoolArray.map((tx) => tx.vsize).reduce((acc, curr) => acc + curr) * 4;
+    const totalFee = memPoolArray.map((tx) => tx.fee).reduce((acc, curr) => acc + curr);
+
+    const logFees = [1, 2, 3, 4, 5, 6, 8, 10, 12, 15, 20, 30, 40, 50, 60, 70, 80, 90, 100, 125, 150, 175, 200,
+      250, 300, 350, 400, 500, 600, 700, 800, 900, 1000, 1200, 1400, 1600, 1800, 2000];
+
+    const weightVsizeFees: { [feePerWU: number]: number } = {};
+    const lastItem = logFees.length - 1;
+
+    memPoolArray.forEach((transaction) => {
+      for (let i = 0; i < logFees.length; i++) {
+        if (
+          (Common.isLiquid() && (i === lastItem || transaction.effectiveFeePerVsize * 10 < logFees[i + 1]))
+          ||
+          (!Common.isLiquid() && (i === lastItem || transaction.effectiveFeePerVsize < logFees[i + 1]))
+        ) {
+          if (weightVsizeFees[logFees[i]]) {
+            weightVsizeFees[logFees[i]] += transaction.vsize;
+          } else {
+            weightVsizeFees[logFees[i]] = transaction.vsize;
+          }
+          break;
+        }
+      }
+    });
+
+    try {
+      const insertId = await statisticsApi.$create({
+        added: 'NOW()',
+        unconfirmed_transactions: memPoolArray.length,
+        tx_per_second: txPerSecond,
+        vbytes_per_second: Math.round(vBytesPerSecond),
+        mempool_byte_weight: totalWeight,
+        total_fee: totalFee,
+        fee_data: '',
+        vsize_1: weightVsizeFees['1'] || 0,
+        vsize_2: weightVsizeFees['2'] || 0,
+        vsize_3: weightVsizeFees['3'] || 0,
+        vsize_4: weightVsizeFees['4'] || 0,
+        vsize_5: weightVsizeFees['5'] || 0,
+        vsize_6: weightVsizeFees['6'] || 0,
+        vsize_8: weightVsizeFees['8'] || 0,
+        vsize_10: weightVsizeFees['10'] || 0,
+        vsize_12: weightVsizeFees['12'] || 0,
+        vsize_15: weightVsizeFees['15'] || 0,
+        vsize_20: weightVsizeFees['20'] || 0,
+        vsize_30: weightVsizeFees['30'] || 0,
+        vsize_40: weightVsizeFees['40'] || 0,
+        vsize_50: weightVsizeFees['50'] || 0,
+        vsize_60: weightVsizeFees['60'] || 0,
+        vsize_70: weightVsizeFees['70'] || 0,
+        vsize_80: weightVsizeFees['80'] || 0,
+        vsize_90: weightVsizeFees['90'] || 0,
+        vsize_100: weightVsizeFees['100'] || 0,
+        vsize_125: weightVsizeFees['125'] || 0,
+        vsize_150: weightVsizeFees['150'] || 0,
+        vsize_175: weightVsizeFees['175'] || 0,
+        vsize_200: weightVsizeFees['200'] || 0,
+        vsize_250: weightVsizeFees['250'] || 0,
+        vsize_300: weightVsizeFees['300'] || 0,
+        vsize_350: weightVsizeFees['350'] || 0,
+        vsize_400: weightVsizeFees['400'] || 0,
+        vsize_500: weightVsizeFees['500'] || 0,
+        vsize_600: weightVsizeFees['600'] || 0,
+        vsize_700: weightVsizeFees['700'] || 0,
+        vsize_800: weightVsizeFees['800'] || 0,
+        vsize_900: weightVsizeFees['900'] || 0,
+        vsize_1000: weightVsizeFees['1000'] || 0,
+        vsize_1200: weightVsizeFees['1200'] || 0,
+        vsize_1400: weightVsizeFees['1400'] || 0,
+        vsize_1600: weightVsizeFees['1600'] || 0,
+        vsize_1800: weightVsizeFees['1800'] || 0,
+        vsize_2000: weightVsizeFees['2000'] || 0,
+      });
+
+      if (this.newStatisticsEntryCallback && insertId) {
+        const newStats = await statisticsApi.$get(insertId);
+        if (newStats) {
+          this.newStatisticsEntryCallback(newStats);
+        }
+      }
+    } catch (e) {
+      logger.err('Unable to insert statistics. ' + e);
+    }
+  }
+}
+
+export default new Statistics();

backend/src/api/transaction-utils.ts

@@ -1,7 +1,7 @@
-import bitcoinApi from './bitcoin/bitcoin-api-factory';
-import logger from '../logger';
 import { TransactionExtended, TransactionMinerInfo } from '../mempool.interfaces';
 import { IEsploraApi } from './bitcoin/esplora-api.interface';
+import { Common } from './common';
+import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory';
 
 class TransactionUtils {
   constructor() { }
@@ -14,14 +14,26 @@ class TransactionUtils {
       vout: tx.vout
         .map((vout) => ({
           scriptpubkey_address: vout.scriptpubkey_address,
+          scriptpubkey_asm: vout.scriptpubkey_asm,
           value: vout.value
         }))
         .filter((vout) => vout.value)
     };
   }
 
-  public async $getTransactionExtended(txId: string, addPrevouts = false): Promise<TransactionExtended> {
-    const transaction: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(txId, false, addPrevouts);
+  /**
+   * @param txId 
+   * @param addPrevouts 
+   * @param lazyPrevouts 
+   * @param forceCore - See https://github.com/mempool/mempool/issues/2904
+   */
+  public async $getTransactionExtended(txId: string, addPrevouts = false, lazyPrevouts = false, forceCore = false): Promise<TransactionExtended> {
+    let transaction: IEsploraApi.Transaction;
+    if (forceCore === true) {
+      transaction  = await bitcoinCoreApi.$getRawTransaction(txId, true);
+    } else {
+      transaction  = await bitcoinApi.$getRawTransaction(txId, false, addPrevouts, lazyPrevouts);
+    }
     return this.extendTransaction(transaction);
   }
 
@@ -31,7 +43,8 @@ class TransactionUtils {
       // @ts-ignore
       return transaction;
     }
-    const feePerVbytes = Math.max(1, (transaction.fee || 0) / (transaction.weight / 4));
+    const feePerVbytes = Math.max(Common.isLiquid() ? 0.1 : 1,
+      (transaction.fee || 0) / (transaction.weight / 4));
     const transactionExtended: TransactionExtended = Object.assign({
       vsize: Math.round(transaction.weight / 4),
       feePerVsize: feePerVbytes,
@@ -42,6 +55,14 @@ class TransactionUtils {
     }
     return transactionExtended;
   }
+
+  public hex2ascii(hex: string) {
+    let str = '';
+    for (let i = 0; i < hex.length; i += 2) {
+      str += String.fromCharCode(parseInt(hex.substr(i, 2), 16));
+    }
+    return str;
+  }
 }
 
 export default new TransactionUtils();

backend/src/api/tx-selection-worker.ts

@@ -0,0 +1,294 @@
+import config from '../config';
+import logger from '../logger';
+import { ThreadTransaction, MempoolBlockWithTransactions, AuditTransaction } from '../mempool.interfaces';
+import { PairingHeap } from '../utils/pairing-heap';
+import { Common } from './common';
+import { parentPort } from 'worker_threads';
+
+let mempool: { [txid: string]: ThreadTransaction } = {};
+
+if (parentPort) {
+  parentPort.on('message', (params) => {
+    if (params.type === 'set') {
+      mempool = params.mempool;
+    } else if (params.type === 'update') {
+      params.added.forEach(tx => {
+        mempool[tx.txid] = tx;
+      });
+      params.removed.forEach(txid => {
+        delete mempool[txid];
+      });
+    }
+    
+    const { blocks, clusters } = makeBlockTemplates(mempool);
+
+    // return the result to main thread.
+    if (parentPort) {
+     parentPort.postMessage({ blocks, clusters });
+    }
+  });
+}
+
+/*
+* Build projected mempool blocks using an approximation of the transaction selection algorithm from Bitcoin Core
+* (see BlockAssembler in https://github.com/bitcoin/bitcoin/blob/master/src/node/miner.cpp)
+*/
+function makeBlockTemplates(mempool: { [txid: string]: ThreadTransaction })
+  : { blocks: ThreadTransaction[][], clusters: { [root: string]: string[] } } {
+  const start = Date.now();
+  const auditPool: { [txid: string]: AuditTransaction } = {};
+  const mempoolArray: AuditTransaction[] = [];
+  const restOfArray: ThreadTransaction[] = [];
+  const cpfpClusters: { [root: string]: string[] } = {};
+  
+  // grab the top feerate txs up to maxWeight
+  Object.values(mempool).sort((a, b) => b.feePerVsize - a.feePerVsize).forEach(tx => {
+    // initializing everything up front helps V8 optimize property access later
+    auditPool[tx.txid] = {
+      txid: tx.txid,
+      fee: tx.fee,
+      weight: tx.weight,
+      feePerVsize: tx.feePerVsize,
+      effectiveFeePerVsize: tx.feePerVsize,
+      vin: tx.vin,
+      relativesSet: false,
+      ancestorMap: new Map<string, AuditTransaction>(),
+      children: new Set<AuditTransaction>(),
+      ancestorFee: 0,
+      ancestorWeight: 0,
+      score: 0,
+      used: false,
+      modified: false,
+      modifiedNode: null,
+    };
+    mempoolArray.push(auditPool[tx.txid]);
+  });
+
+  // Build relatives graph & calculate ancestor scores
+  for (const tx of mempoolArray) {
+    if (!tx.relativesSet) {
+      setRelatives(tx, auditPool);
+    }
+  }
+
+  // Sort by descending ancestor score
+  mempoolArray.sort((a, b) => (b.score || 0) - (a.score || 0));
+
+  // Build blocks by greedily choosing the highest feerate package
+  // (i.e. the package rooted in the transaction with the best ancestor score)
+  const blocks: ThreadTransaction[][] = [];
+  let blockWeight = 4000;
+  let blockSize = 0;
+  let transactions: AuditTransaction[] = [];
+  const modified: PairingHeap<AuditTransaction> = new PairingHeap((a, b): boolean => (a.score || 0) > (b.score || 0));
+  let overflow: AuditTransaction[] = [];
+  let failures = 0;
+  let top = 0;
+  while ((top < mempoolArray.length || !modified.isEmpty())) {
+    // skip invalid transactions
+    while (top < mempoolArray.length && (mempoolArray[top].used || mempoolArray[top].modified)) {
+      top++;
+    }
+
+    // Select best next package
+    let nextTx;
+    const nextPoolTx = mempoolArray[top];
+    const nextModifiedTx = modified.peek();
+    if (nextPoolTx && (!nextModifiedTx || (nextPoolTx.score || 0) > (nextModifiedTx.score || 0))) {
+      nextTx = nextPoolTx;
+      top++;
+    } else {
+      modified.pop();
+      if (nextModifiedTx) {
+        nextTx = nextModifiedTx;
+        nextTx.modifiedNode = undefined;
+      }
+    }
+
+    if (nextTx && !nextTx?.used) {
+      // Check if the package fits into this block
+      if (blockWeight + nextTx.ancestorWeight < config.MEMPOOL.BLOCK_WEIGHT_UNITS) {
+        const ancestors: AuditTransaction[] = Array.from(nextTx.ancestorMap.values());
+        // sort ancestors by dependency graph (equivalent to sorting by ascending ancestor count)
+        const sortedTxSet = [...ancestors.sort((a, b) => { return (a.ancestorMap.size || 0) - (b.ancestorMap.size || 0); }), nextTx];
+        let isCluster = false;
+        if (sortedTxSet.length > 1) {
+          cpfpClusters[nextTx.txid] = sortedTxSet.map(tx => tx.txid);
+          isCluster = true;
+        }
+        const effectiveFeeRate = nextTx.ancestorFee / (nextTx.ancestorWeight / 4);
+        const used: AuditTransaction[] = [];
+        while (sortedTxSet.length) {
+          const ancestor = sortedTxSet.pop();
+          const mempoolTx = mempool[ancestor.txid];
+          ancestor.used = true;
+          ancestor.usedBy = nextTx.txid;
+          // update original copy of this tx with effective fee rate & relatives data
+          mempoolTx.effectiveFeePerVsize = effectiveFeeRate;
+          if (isCluster) {
+            mempoolTx.cpfpRoot = nextTx.txid;
+          }
+          mempoolTx.cpfpChecked = true;
+          transactions.push(ancestor);
+          blockSize += ancestor.size;
+          blockWeight += ancestor.weight;
+          used.push(ancestor);
+        }
+
+        // remove these as valid package ancestors for any descendants remaining in the mempool
+        if (used.length) {
+          used.forEach(tx => {
+            updateDescendants(tx, auditPool, modified);
+          });
+        }
+
+        failures = 0;
+      } else {
+        // hold this package in an overflow list while we check for smaller options
+        overflow.push(nextTx);
+        failures++;
+      }
+    }
+
+    // this block is full
+    const exceededPackageTries = failures > 1000 && blockWeight > (config.MEMPOOL.BLOCK_WEIGHT_UNITS - 4000);
+    const queueEmpty = top >= mempoolArray.length && modified.isEmpty();
+    if ((exceededPackageTries || queueEmpty) && blocks.length < 7) {
+      // construct this block
+      if (transactions.length) {
+        blocks.push(transactions.map(t => mempool[t.txid]));
+      }
+      // reset for the next block
+      transactions = [];
+      blockSize = 0;
+      blockWeight = 4000;
+
+      // 'overflow' packages didn't fit in this block, but are valid candidates for the next
+      for (const overflowTx of overflow.reverse()) {
+        if (overflowTx.modified) {
+          overflowTx.modifiedNode = modified.add(overflowTx);
+        } else {
+          top--;
+          mempoolArray[top] = overflowTx;
+        }
+      }
+      overflow = [];
+    }
+  }
+  // pack any leftover transactions into the last block
+  for (const tx of overflow) {
+    if (!tx || tx?.used) {
+      continue;
+    }
+    blockWeight += tx.weight;
+    const mempoolTx = mempool[tx.txid];
+    // update original copy of this tx with effective fee rate & relatives data
+    mempoolTx.effectiveFeePerVsize = tx.score;
+    if (tx.ancestorMap.size > 0) {
+      cpfpClusters[tx.txid] = Array.from(tx.ancestorMap?.values()).map(a => a.txid);
+      mempoolTx.cpfpRoot = tx.txid;
+    }
+    mempoolTx.cpfpChecked = true;
+    transactions.push(tx);
+    tx.used = true;
+  }
+  const blockTransactions = transactions.map(t => mempool[t.txid]);
+  restOfArray.forEach(tx => {
+    blockWeight += tx.weight;
+    tx.effectiveFeePerVsize = tx.feePerVsize;
+    tx.cpfpChecked = false;
+    blockTransactions.push(tx);
+  });
+  if (blockTransactions.length) {
+    blocks.push(blockTransactions);
+  }
+  transactions = [];
+
+  const end = Date.now();
+  const time = end - start;
+  logger.debug('Mempool templates calculated in ' + time / 1000 + ' seconds');
+
+  return { blocks, clusters: cpfpClusters };
+}
+
+// traverse in-mempool ancestors
+// recursion unavoidable, but should be limited to depth < 25 by mempool policy
+function setRelatives(
+  tx: AuditTransaction,
+  mempool: { [txid: string]: AuditTransaction },
+): void {
+  for (const parent of tx.vin) {
+    const parentTx = mempool[parent];
+    if (parentTx && !tx.ancestorMap?.has(parent)) {
+      tx.ancestorMap.set(parent, parentTx);
+      parentTx.children.add(tx);
+      // visit each node only once
+      if (!parentTx.relativesSet) {
+        setRelatives(parentTx, mempool);
+      }
+      parentTx.ancestorMap.forEach((ancestor) => {
+        tx.ancestorMap.set(ancestor.txid, ancestor);
+      });
+    }
+  };
+  tx.ancestorFee = tx.fee || 0;
+  tx.ancestorWeight = tx.weight || 0;
+  tx.ancestorMap.forEach((ancestor) => {
+    tx.ancestorFee += ancestor.fee;
+    tx.ancestorWeight += ancestor.weight;
+  });
+  tx.score = tx.ancestorFee / ((tx.ancestorWeight / 4) || 1);
+  tx.relativesSet = true;
+}
+
+// iterate over remaining descendants, removing the root as a valid ancestor & updating the ancestor score
+// avoids recursion to limit call stack depth
+function updateDescendants(
+  rootTx: AuditTransaction,
+  mempool: { [txid: string]: AuditTransaction },
+  modified: PairingHeap<AuditTransaction>,
+): void {
+  const descendantSet: Set<AuditTransaction> = new Set();
+  // stack of nodes left to visit
+  const descendants: AuditTransaction[] = [];
+  let descendantTx;
+  let tmpScore;
+  rootTx.children.forEach(childTx => {
+    if (!descendantSet.has(childTx)) {
+      descendants.push(childTx);
+      descendantSet.add(childTx);
+    }
+  });
+  while (descendants.length) {
+    descendantTx = descendants.pop();
+    if (descendantTx && descendantTx.ancestorMap && descendantTx.ancestorMap.has(rootTx.txid)) {
+      // remove tx as ancestor
+      descendantTx.ancestorMap.delete(rootTx.txid);
+      descendantTx.ancestorFee -= rootTx.fee;
+      descendantTx.ancestorWeight -= rootTx.weight;
+      tmpScore = descendantTx.score;
+      descendantTx.score = descendantTx.ancestorFee / (descendantTx.ancestorWeight / 4);
+
+      if (!descendantTx.modifiedNode) {
+        descendantTx.modified = true;
+        descendantTx.modifiedNode = modified.add(descendantTx);
+      } else {
+        // rebalance modified heap if score has changed
+        if (descendantTx.score < tmpScore) {
+          modified.decreasePriority(descendantTx.modifiedNode);
+        } else if (descendantTx.score > tmpScore) {
+          modified.increasePriority(descendantTx.modifiedNode);
+        }
+      }
+
+      // add this node's children to the stack
+      descendantTx.children.forEach(childTx => {
+        // visit each node only once
+        if (!descendantSet.has(childTx)) {
+          descendants.push(childTx);
+          descendantSet.add(childTx);
+        }
+      });
+    }
+  }
+}

backend/src/api/websocket-handler.ts

@@ -1,20 +1,29 @@
 import logger from '../logger';
 import * as WebSocket from 'ws';
-import { BlockExtended, TransactionExtended, WebsocketResponse, MempoolBlock,
-  OptimizedStatistic, ILoadingIndicators, IConversionRates } from '../mempool.interfaces';
+import {
+  BlockExtended, TransactionExtended, WebsocketResponse,
+  OptimizedStatistic, ILoadingIndicators
+} from '../mempool.interfaces';
 import blocks from './blocks';
 import memPool from './mempool';
 import backendInfo from './backend-info';
 import mempoolBlocks from './mempool-blocks';
-import fiatConversion from './fiat-conversion';
 import { Common } from './common';
 import loadingIndicators from './loading-indicators';
 import config from '../config';
 import transactionUtils from './transaction-utils';
+import rbfCache from './rbf-cache';
+import difficultyAdjustment from './difficulty-adjustment';
+import feeApi from './fee-api';
+import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository';
+import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
+import Audit from './audit';
+import { deepClone } from '../utils/clone';
+import priceUpdater from '../tasks/price-updater';
+import { ApiPrice } from '../repositories/PricesRepository';
 
 class WebsocketHandler {
   private wss: WebSocket.Server | undefined;
-  private nativeAssetId = '6f0279e9ed041c3d710a9f57d0c02928416460c4b722ae3457a11eec381c526d';
   private extraInitProperties = {};
 
   constructor() { }
@@ -49,22 +58,38 @@ class WebsocketHandler {
           if (parsedMessage && parsedMessage['track-tx']) {
             if (/^[a-fA-F0-9]{64}$/.test(parsedMessage['track-tx'])) {
               client['track-tx'] = parsedMessage['track-tx'];
-              // Client is telling the transaction wasn't found but it might have appeared before we had the time to start watching for it
+              // Client is telling the transaction wasn't found
               if (parsedMessage['watch-mempool']) {
-                const tx = memPool.getMempool()[client['track-tx']];
-                if (tx) {
-                  if (config.MEMPOOL.BACKEND !== 'esplora') {
-                    try {
-                      const fullTx = await transactionUtils.$getTransactionExtended(tx.txid, true);
-                      response['tx'] = fullTx;
-                    } catch (e) {
-                      logger.debug('Error finding transaction in mempool: ' + e.message || e);
+                const rbfCacheTxid = rbfCache.getReplacedBy(client['track-tx']);
+                if (rbfCacheTxid) {
+                  response['txReplaced'] = {
+                    txid: rbfCacheTxid,
+                  };
+                  client['track-tx'] = null;
+                } else {
+                  // It might have appeared before we had the time to start watching for it
+                  const tx = memPool.getMempool()[client['track-tx']];
+                  if (tx) {
+                    if (config.MEMPOOL.BACKEND === 'esplora') {
+                      response['tx'] = tx;
+                    } else {
+                      // tx.prevout is missing from transactions when in bitcoind mode
+                      try {
+                        const fullTx = await transactionUtils.$getTransactionExtended(tx.txid, true);
+                        response['tx'] = fullTx;
+                      } catch (e) {
+                        logger.debug('Error finding transaction: ' + (e instanceof Error ? e.message : e));
+                      }
                     }
                   } else {
-                    response['tx'] = tx;
+                    try {
+                      const fullTx = await transactionUtils.$getTransactionExtended(client['track-tx'], true);
+                      response['tx'] = fullTx;
+                    } catch (e) {
+                      logger.debug('Error finding transaction. ' + (e instanceof Error ? e.message : e));
+                      client['track-mempool-tx'] = parsedMessage['track-tx'];
+                    }
                   }
-                } else {
-                  client['track-mempool-tx'] = parsedMessage['track-tx'];
                 }
               }
             } else {
@@ -73,9 +98,13 @@ class WebsocketHandler {
           }
 
           if (parsedMessage && parsedMessage['track-address']) {
-            if (/^([a-km-zA-HJ-NP-Z1-9]{26,35}|[a-km-zA-HJ-NP-Z1-9]{80}|[a-z]{2,5}1[ac-hj-np-z02-9]{8,87})$/
+            if (/^([a-km-zA-HJ-NP-Z1-9]{26,35}|[a-km-zA-HJ-NP-Z1-9]{80}|[a-z]{2,5}1[ac-hj-np-z02-9]{8,100}|[A-Z]{2,5}1[AC-HJ-NP-Z02-9]{8,100})$/
               .test(parsedMessage['track-address'])) {
-              client['track-address'] = parsedMessage['track-address'];
+              let matchedAddress = parsedMessage['track-address'];
+              if (/^[A-Z]{2,5}1[AC-HJ-NP-Z02-9]{8,100}$/.test(parsedMessage['track-address'])) {
+                matchedAddress = matchedAddress.toLowerCase();
+              }
+              client['track-address'] = matchedAddress;
             } else {
               client['track-address'] = null;
             }
@@ -89,8 +118,22 @@ class WebsocketHandler {
             }
           }
 
+          if (parsedMessage && parsedMessage['track-mempool-block'] !== undefined) {
+            if (Number.isInteger(parsedMessage['track-mempool-block']) && parsedMessage['track-mempool-block'] >= 0) {
+              const index = parsedMessage['track-mempool-block'];
+              client['track-mempool-block'] = index;
+              const mBlocksWithTransactions = mempoolBlocks.getMempoolBlocksWithTransactions();
+              response['projected-block-transactions'] = {
+                index: index,
+                blockTransactions: mBlocksWithTransactions[index]?.transactions || [],
+              };
+            } else {
+              client['track-mempool-block'] = null;
+            }
+          }
+
           if (parsedMessage.action === 'init') {
-            const _blocks = blocks.getBlocks().slice(-8);
+            const _blocks = blocks.getBlocks().slice(-config.MEMPOOL.INITIAL_BLOCKS_AMOUNT);
             if (!_blocks) {
               return;
             }
@@ -117,7 +160,7 @@ class WebsocketHandler {
             client.send(JSON.stringify(response));
           }
         } catch (e) {
-          logger.debug('Error parsing websocket message: ' + e.message || e);
+          logger.debug('Error parsing websocket message: ' + (e instanceof Error ? e.message : e));
         }
       });
     });
@@ -128,7 +171,7 @@ class WebsocketHandler {
       throw new Error('WebSocket.Server is not set');
     }
 
-    this.wss.clients.forEach((client: WebSocket) => {
+    this.wss.clients.forEach((client) => {
       if (client.readyState !== WebSocket.OPEN) {
         return;
       }
@@ -143,7 +186,7 @@ class WebsocketHandler {
       throw new Error('WebSocket.Server is not set');
     }
 
-    this.wss.clients.forEach((client: WebSocket) => {
+    this.wss.clients.forEach((client) => {
       if (client.readyState !== WebSocket.OPEN) {
         return;
       }
@@ -151,12 +194,12 @@ class WebsocketHandler {
     });
   }
 
-  handleNewConversionRates(conversionRates: IConversionRates) {
+  handleNewConversionRates(conversionRates: ApiPrice) {
     if (!this.wss) {
       throw new Error('WebSocket.Server is not set');
     }
 
-    this.wss.clients.forEach((client: WebSocket) => {
+    this.wss.clients.forEach((client) => {
       if (client.readyState !== WebSocket.OPEN) {
         return;
       }
@@ -166,18 +209,19 @@ class WebsocketHandler {
 
   getInitData(_blocks?: BlockExtended[]) {
     if (!_blocks) {
-      _blocks = blocks.getBlocks().slice(-8);
+      _blocks = blocks.getBlocks().slice(-config.MEMPOOL.INITIAL_BLOCKS_AMOUNT);
     }
     return {
       'mempoolInfo': memPool.getMempoolInfo(),
       'vBytesPerSecond': memPool.getVBytesPerSecond(),
-      'lastDifficultyAdjustment': blocks.getLastDifficultyAdjustmentTime(),
       'blocks': _blocks,
-      'conversions': fiatConversion.getConversionRates(),
+      'conversions': priceUpdater.getLatestPrices(),
       'mempool-blocks': mempoolBlocks.getMempoolBlocks(),
       'transactions': memPool.getLatestTransactions(),
       'backendInfo': backendInfo.getBackendInfo(),
       'loadingIndicators': loadingIndicators.getLoadingIndicators(),
+      'da': difficultyAdjustment.getDifficultyAdjustment(),
+      'fees': feeApi.getRecommendedFee(),
       ...this.extraInitProperties
     };
   }
@@ -187,7 +231,7 @@ class WebsocketHandler {
       throw new Error('WebSocket.Server is not set');
     }
 
-    this.wss.clients.forEach((client: WebSocket) => {
+    this.wss.clients.forEach((client) => {
       if (client.readyState !== WebSocket.OPEN) {
         return;
       }
@@ -202,24 +246,28 @@ class WebsocketHandler {
     });
   }
 
-  handleMempoolChange(newMempool: { [txid: string]: TransactionExtended },
-    newTransactions: TransactionExtended[], deletedTransactions: TransactionExtended[]) {
+  async handleMempoolChange(newMempool: { [txid: string]: TransactionExtended },
+    newTransactions: TransactionExtended[], deletedTransactions: TransactionExtended[]): Promise<void> {
     if (!this.wss) {
       throw new Error('WebSocket.Server is not set');
     }
 
-    mempoolBlocks.updateMempoolBlocks(newMempool);
+    if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) {
+      await mempoolBlocks.updateBlockTemplates(newMempool, newTransactions, deletedTransactions.map(tx => tx.txid), true);
+    } else {
+      mempoolBlocks.updateMempoolBlocks(newMempool, true);
+    }
+
     const mBlocks = mempoolBlocks.getMempoolBlocks();
-    const mempool = memPool.getMempool();
+    const mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();
     const mempoolInfo = memPool.getMempoolInfo();
     const vBytesPerSecond = memPool.getVBytesPerSecond();
     const rbfTransactions = Common.findRbfTransactions(newTransactions, deletedTransactions);
+    const da = difficultyAdjustment.getDifficultyAdjustment();
+    memPool.handleRbfTransactions(rbfTransactions);
+    const recommendedFees = feeApi.getRecommendedFee();
 
-    for (const rbfTransaction in rbfTransactions) {
-      delete mempool[rbfTransaction];
-    }
-
-    this.wss.clients.forEach(async (client: WebSocket) => {
+    this.wss.clients.forEach(async (client) => {
       if (client.readyState !== WebSocket.OPEN) {
         return;
       }
@@ -230,6 +278,8 @@ class WebsocketHandler {
         response['mempoolInfo'] = mempoolInfo;
         response['vBytesPerSecond'] = vBytesPerSecond;
         response['transactions'] = newTransactions.slice(0, 6).map((tx) => Common.stripTransaction(tx));
+        response['da'] = da;
+        response['fees'] = recommendedFees;
       }
 
       if (client['want-mempool-blocks']) {
@@ -244,7 +294,7 @@ class WebsocketHandler {
               const fullTx = await transactionUtils.$getTransactionExtended(tx.txid, true);
               response['tx'] = fullTx;
             } catch (e) {
-              logger.debug('Error finding transaction in mempool: ' + e.message || e);
+              logger.debug('Error finding transaction in mempool: ' + (e instanceof Error ? e.message : e));
             }
           } else {
             response['tx'] = tx;
@@ -264,7 +314,7 @@ class WebsocketHandler {
                 const fullTx = await transactionUtils.$getTransactionExtended(tx.txid, true);
                 foundTransactions.push(fullTx);
               } catch (e) {
-                logger.debug('Error finding transaction in mempool: ' + e.message || e);
+                logger.debug('Error finding transaction in mempool: ' + (e instanceof Error ? e.message : e));
               }
             } else {
               foundTransactions.push(tx);
@@ -278,7 +328,7 @@ class WebsocketHandler {
                 const fullTx = await transactionUtils.$getTransactionExtended(tx.txid, true);
                 foundTransactions.push(fullTx);
               } catch (e) {
-                logger.debug('Error finding transaction in mempool: ' + e.message || e);
+                logger.debug('Error finding transaction in mempool: ' + (e instanceof Error ? e.message : e));
               }
             } else {
               foundTransactions.push(tx);
@@ -296,7 +346,7 @@ class WebsocketHandler {
 
         newTransactions.forEach((tx) => {
 
-          if (client['track-asset'] === this.nativeAssetId) {
+          if (client['track-asset'] === Common.nativeAssetId) {
             if (tx.vin.some((vin) => !!vin.is_pegin)) {
               foundTransactions.push(tx);
               return;
@@ -320,22 +370,40 @@ class WebsocketHandler {
         }
       }
 
-      if (client['track-tx'] && rbfTransactions[client['track-tx']]) {
-        for (const rbfTransaction in rbfTransactions) {
-          if (client['track-tx'] === rbfTransaction) {
-            const rbfTx = rbfTransactions[rbfTransaction];
-            if (config.MEMPOOL.BACKEND !== 'esplora') {
-              try {
-                const fullTx = await transactionUtils.$getTransactionExtended(rbfTransaction, true);
-                response['rbfTransaction'] = fullTx;
-              } catch (e) {
-                logger.debug('Error finding transaction in mempool: ' + e.message || e);
-              }
-            } else {
-              response['rbfTransaction'] = rbfTx;
-            }
-            break;
+      if (client['track-tx']) {
+        const outspends: object = {};
+        newTransactions.forEach((tx) => tx.vin.forEach((vin, i) => {
+          if (vin.txid === client['track-tx']) {
+            outspends[vin.vout] = {
+              vin: i,
+              txid: tx.txid,
+            };
           }
+        }));
+
+        if (Object.keys(outspends).length) {
+          response['utxoSpent'] = outspends;
+        }
+
+        if (rbfTransactions[client['track-tx']]) {
+          for (const rbfTransaction in rbfTransactions) {
+            if (client['track-tx'] === rbfTransaction) {
+              response['rbfTransaction'] = {
+                txid: rbfTransactions[rbfTransaction].txid,
+              };
+              break;
+            }
+          }
+        }
+      }
+
+      if (client['track-mempool-block'] >= 0) {
+        const index = client['track-mempool-block'];
+        if (mBlockDeltas[index]) {
+          response['projected-block-transactions'] = {
+            index: index,
+            delta: mBlockDeltas[index],
+          };
         }
       }
 
@@ -344,32 +412,86 @@ class WebsocketHandler {
       }
     });
   }
-
-  handleNewBlock(block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) {
+ 
+  async handleNewBlock(block: BlockExtended, txIds: string[], transactions: TransactionExtended[]): Promise<void> {
     if (!this.wss) {
       throw new Error('WebSocket.Server is not set');
     }
 
-    let mBlocks: undefined | MempoolBlock[];
-    let matchRate = 0;
     const _memPool = memPool.getMempool();
-    const _mempoolBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
 
-    if (_mempoolBlocks[0]) {
-      const matches: string[] = [];
-      for (const txId of txIds) {
-        if (_mempoolBlocks[0].transactionIds.indexOf(txId) > -1) {
-          matches.push(txId);
-        }
-        delete _memPool[txId];
+    if (config.MEMPOOL.AUDIT) {
+      let projectedBlocks;
+      // template calculation functions have mempool side effects, so calculate audits using
+      // a cloned copy of the mempool if we're running a different algorithm for mempool updates
+      const auditMempool = (config.MEMPOOL.ADVANCED_GBT_AUDIT === config.MEMPOOL.ADVANCED_GBT_MEMPOOL) ? _memPool : deepClone(_memPool);
+      if (config.MEMPOOL.ADVANCED_GBT_AUDIT) {
+        projectedBlocks = await mempoolBlocks.makeBlockTemplates(auditMempool, false);
+      } else {
+        projectedBlocks = mempoolBlocks.updateMempoolBlocks(auditMempool, false);
       }
 
-      matchRate = Math.round((matches.length / (txIds.length - 1)) * 100);
-      mempoolBlocks.updateMempoolBlocks(_memPool);
-      mBlocks = mempoolBlocks.getMempoolBlocks();
+      if (Common.indexingEnabled() && memPool.isInSync()) {
+        const { censored, added, fresh, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool);
+        const matchRate = Math.round(score * 100 * 100) / 100;
+
+        const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions.map((tx) => {
+          return {
+            txid: tx.txid,
+            vsize: tx.vsize,
+            fee: tx.fee ? Math.round(tx.fee) : 0,
+            value: tx.value,
+          };
+        }) : [];
+
+        BlocksSummariesRepository.$saveTemplate({
+          height: block.height,
+          template: {
+            id: block.id,
+            transactions: stripped
+          }
+        });
+
+        BlocksAuditsRepository.$saveAudit({
+          time: block.timestamp,
+          height: block.height,
+          hash: block.id,
+          addedTxs: added,
+          missingTxs: censored,
+          freshTxs: fresh,
+          matchRate: matchRate,
+        });
+
+        if (block.extras) {
+          block.extras.matchRate = matchRate;
+          block.extras.similarity = similarity;
+        }
+      }
+    } else if (block.extras) {
+      const mBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
+      if (mBlocks?.length && mBlocks[0].transactions) {
+        block.extras.similarity = Common.getSimilarity(mBlocks[0], transactions);
+      }
     }
 
-    block.matchRate = matchRate;
+    const removed: string[] = [];
+    // Update mempool to remove transactions included in the new block
+    for (const txId of txIds) {
+      delete _memPool[txId];
+      removed.push(txId);
+      rbfCache.evict(txId);
+    }
+
+    if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) {
+      await mempoolBlocks.updateBlockTemplates(_memPool, [], removed, true);
+    } else {
+      mempoolBlocks.updateMempoolBlocks(_memPool, true);
+    }
+    const mBlocks = mempoolBlocks.getMempoolBlocks();
+    const mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();
+
+    const da = difficultyAdjustment.getDifficultyAdjustment();
+    const fees = feeApi.getRecommendedFee();
 
     this.wss.clients.forEach((client) => {
       if (client.readyState !== WebSocket.OPEN) {
@@ -383,7 +505,8 @@ class WebsocketHandler {
       const response = {
         'block': block,
         'mempoolInfo': memPool.getMempoolInfo(),
-        'lastDifficultyAdjustment': blocks.getLastDifficultyAdjustmentTime(),
+        'da': da,
+        'fees': fees,
       };
 
       if (mBlocks && client['want-mempool-blocks']) {
@@ -391,7 +514,6 @@ class WebsocketHandler {
       }
 
       if (client['track-tx'] && txIds.indexOf(client['track-tx']) > -1) {
-        client['track-tx'] = null;
         response['txConfirmed'] = true;
       }
 
@@ -426,7 +548,7 @@ class WebsocketHandler {
         const foundTransactions: TransactionExtended[] = [];
 
         transactions.forEach((tx) => {
-          if (client['track-asset'] === this.nativeAssetId) {
+          if (client['track-asset'] === Common.nativeAssetId) {
             if (tx.vin && tx.vin.some((vin) => !!vin.is_pegin)) {
               foundTransactions.push(tx);
               return;
@@ -459,6 +581,16 @@ class WebsocketHandler {
         }
       }
 
+      if (client['track-mempool-block'] >= 0) {
+        const index = client['track-mempool-block'];
+        if (mBlockDeltas && mBlockDeltas[index]) {
+          response['projected-block-transactions'] = {
+            index: index,
+            delta: mBlockDeltas[index],
+          };
+        }
+      }
+
       client.send(JSON.stringify(response));
     });
   }

backend/src/config.ts

@@ -1,8 +1,11 @@
-const configFile = require('../mempool-config.json');
+const configFromFile = require(
+    process.env.MEMPOOL_CONFIG_FILE ? process.env.MEMPOOL_CONFIG_FILE : '../mempool-config.json'
+);
 
 interface IConfig {
   MEMPOOL: {
-    NETWORK: 'mainnet' | 'testnet' | 'signet' | 'liquid';
+    ENABLED: boolean;
+    NETWORK: 'mainnet' | 'testnet' | 'signet' | 'liquid' | 'liquidtestnet';
     BACKEND: 'esplora' | 'electrum' | 'none';
     HTTP_PORT: number;
     SPAWN_CLUSTER_PROCS: number;
@@ -11,10 +14,49 @@ interface IConfig {
     CACHE_DIR: string;
     CLEAR_PROTECTION_MINUTES: number;
     RECOMMENDED_FEE_PERCENTILE: number;
+    BLOCK_WEIGHT_UNITS: number;
+    INITIAL_BLOCKS_AMOUNT: number;
+    MEMPOOL_BLOCKS_AMOUNT: number;
+    INDEXING_BLOCKS_AMOUNT: number;
+    BLOCKS_SUMMARIES_INDEXING: boolean;
+    USE_SECOND_NODE_FOR_MINFEE: boolean;
+    EXTERNAL_ASSETS: string[];
+    EXTERNAL_MAX_RETRY: number;
+    EXTERNAL_RETRY_INTERVAL: number;
+    USER_AGENT: string;
+    STDOUT_LOG_MIN_PRIORITY: 'emerg' | 'alert' | 'crit' | 'err' | 'warn' | 'notice' | 'info' | 'debug';
+    AUTOMATIC_BLOCK_REINDEXING: boolean;
+    POOLS_JSON_URL: string,
+    POOLS_JSON_TREE_URL: string,
+    AUDIT: boolean;
+    ADVANCED_GBT_AUDIT: boolean;
+    ADVANCED_GBT_MEMPOOL: boolean;
+    CPFP_INDEXING: boolean;
+    MAX_BLOCKS_BULK_QUERY: number;
+    DISK_CACHE_BLOCK_INTERVAL: number;
   };
   ESPLORA: {
     REST_API_URL: string;
   };
+  LIGHTNING: {
+    ENABLED: boolean;
+    BACKEND: 'lnd' | 'cln' | 'ldk';
+    TOPOLOGY_FOLDER: string;
+    STATS_REFRESH_INTERVAL: number;
+    GRAPH_REFRESH_INTERVAL: number;
+    LOGGER_UPDATE_INTERVAL: number;
+    FORENSICS_INTERVAL: number;
+    FORENSICS_RATE_LIMIT: number;
+  };
+  LND: {
+    TLS_CERT_PATH: string;
+    MACAROON_PATH: string;
+    REST_API_URL: string;
+    TIMEOUT: number;
+  };
+  CLIGHTNING: {
+    SOCKET: string;
+  };
   ELECTRUM: {
     HOST: string;
     PORT: number;
@@ -25,17 +67,19 @@ interface IConfig {
     PORT: number;
     USERNAME: string;
     PASSWORD: string;
+    TIMEOUT: number;
   };
-  CORE_RPC_MINFEE: {
-    ENABLED: boolean;
+  SECOND_CORE_RPC: {
     HOST: string;
     PORT: number;
     USERNAME: string;
     PASSWORD: string;
+    TIMEOUT: number;
   };
   DATABASE: {
     ENABLED: boolean;
     HOST: string,
+    SOCKET: string,
     PORT: number;
     DATABASE: string;
     USERNAME: string;
@@ -45,7 +89,7 @@ interface IConfig {
     ENABLED: boolean;
     HOST: string;
     PORT: number;
-    MIN_PRIORITY: 'emerg' | 'alert' | 'crit' | 'err' |'warn' | 'notice' | 'info' | 'debug';
+    MIN_PRIORITY: 'emerg' | 'alert' | 'crit' | 'err' | 'warn' | 'notice' | 'info' | 'debug';
     FACILITY: string;
   };
   STATISTICS: {
@@ -56,10 +100,37 @@ interface IConfig {
     ENABLED: boolean;
     DATA_PATH: string;
   };
+  SOCKS5PROXY: {
+    ENABLED: boolean;
+    USE_ONION: boolean;
+    HOST: string;
+    PORT: number;
+    USERNAME: string;
+    PASSWORD: string;
+  };
+  PRICE_DATA_SERVER: {
+    TOR_URL: string;
+    CLEARNET_URL: string;
+  };
+  EXTERNAL_DATA_SERVER: {
+    MEMPOOL_API: string;
+    MEMPOOL_ONION: string;
+    LIQUID_API: string;
+    LIQUID_ONION: string;
+    BISQ_URL: string;
+    BISQ_ONION: string;
+  };
+  MAXMIND: {
+    ENABLED: boolean;
+    GEOLITE2_CITY: string;
+    GEOLITE2_ASN: string;
+    GEOIP2_ISP: string;
+  },
 }
 
 const defaults: IConfig = {
   'MEMPOOL': {
+    'ENABLED': true,
     'NETWORK': 'mainnet',
     'BACKEND': 'none',
     'HTTP_PORT': 8999,
@@ -69,6 +140,26 @@ const defaults: IConfig = {
     'CACHE_DIR': './cache',
     'CLEAR_PROTECTION_MINUTES': 20,
     'RECOMMENDED_FEE_PERCENTILE': 50,
+    'BLOCK_WEIGHT_UNITS': 4000000,
+    'INITIAL_BLOCKS_AMOUNT': 8,
+    'MEMPOOL_BLOCKS_AMOUNT': 8,
+    'INDEXING_BLOCKS_AMOUNT': 11000, // 0 = disable indexing, -1 = index all blocks
+    'BLOCKS_SUMMARIES_INDEXING': false,
+    'USE_SECOND_NODE_FOR_MINFEE': false,
+    'EXTERNAL_ASSETS': [],
+    'EXTERNAL_MAX_RETRY': 1,
+    'EXTERNAL_RETRY_INTERVAL': 0,
+    'USER_AGENT': 'mempool',
+    'STDOUT_LOG_MIN_PRIORITY': 'debug',
+    'AUTOMATIC_BLOCK_REINDEXING': false,
+    'POOLS_JSON_URL': 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json',
+    'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
+    'AUDIT': false,
+    'ADVANCED_GBT_AUDIT': false,
+    'ADVANCED_GBT_MEMPOOL': false,
+    'CPFP_INDEXING': false,
+    'MAX_BLOCKS_BULK_QUERY': 0,
+    'DISK_CACHE_BLOCK_INTERVAL': 6,
   },
   'ESPLORA': {
     'REST_API_URL': 'http://127.0.0.1:3000',
@@ -82,18 +173,20 @@ const defaults: IConfig = {
     'HOST': '127.0.0.1',
     'PORT': 8332,
     'USERNAME': 'mempool',
-    'PASSWORD': 'mempool'
+    'PASSWORD': 'mempool',
+    'TIMEOUT': 60000,
   },
-  'CORE_RPC_MINFEE': {
-    'ENABLED': false,
+  'SECOND_CORE_RPC': {
     'HOST': '127.0.0.1',
     'PORT': 8332,
     'USERNAME': 'mempool',
-    'PASSWORD': 'mempool'
+    'PASSWORD': 'mempool',
+    'TIMEOUT': 60000,
   },
   'DATABASE': {
     'ENABLED': true,
     'HOST': '127.0.0.1',
+    'SOCKET': '',
     'PORT': 3306,
     'DATABASE': 'mempool',
     'USERNAME': 'mempool',
@@ -114,6 +207,51 @@ const defaults: IConfig = {
     'ENABLED': false,
     'DATA_PATH': '/bisq/statsnode-data/btc_mainnet/db'
   },
+  'LIGHTNING': {
+    'ENABLED': false,
+    'BACKEND': 'lnd',
+    'TOPOLOGY_FOLDER': '',
+    'STATS_REFRESH_INTERVAL': 600,
+    'GRAPH_REFRESH_INTERVAL': 600,
+    'LOGGER_UPDATE_INTERVAL': 30,
+    'FORENSICS_INTERVAL': 43200,
+    'FORENSICS_RATE_LIMIT': 20,
+  },
+  'LND': {
+    'TLS_CERT_PATH': '',
+    'MACAROON_PATH': '',
+    'REST_API_URL': 'https://localhost:8080',
+    'TIMEOUT': 10000,
+  },
+  'CLIGHTNING': {
+    'SOCKET': '',
+  },
+  'SOCKS5PROXY': {
+    'ENABLED': false,
+    'USE_ONION': true,
+    'HOST': '127.0.0.1',
+    'PORT': 9050,
+    'USERNAME': '',
+    'PASSWORD': ''
+  },
+  'PRICE_DATA_SERVER': {
+    'TOR_URL': 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices',
+    'CLEARNET_URL': 'https://price.bisq.wiz.biz/getAllMarketPrices'
+  },
+  'EXTERNAL_DATA_SERVER': {
+    'MEMPOOL_API': 'https://mempool.space/api/v1',
+    'MEMPOOL_ONION': 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1',
+    'LIQUID_API': 'https://liquid.network/api/v1',
+    'LIQUID_ONION': 'http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1',
+    'BISQ_URL': 'https://bisq.markets/api',
+    'BISQ_ONION': 'http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api'
+  },
+  'MAXMIND': {
+    'ENABLED': false,
+    'GEOLITE2_CITY': '/usr/local/share/GeoIP/GeoLite2-City.mmdb',
+    'GEOLITE2_ASN': '/usr/local/share/GeoIP/GeoLite2-ASN.mmdb',
+    'GEOIP2_ISP': '/usr/local/share/GeoIP/GeoIP2-ISP.mmdb'
+  },
 };
 
 class Config implements IConfig {
@@ -121,23 +259,37 @@ class Config implements IConfig {
   ESPLORA: IConfig['ESPLORA'];
   ELECTRUM: IConfig['ELECTRUM'];
   CORE_RPC: IConfig['CORE_RPC'];
-  CORE_RPC_MINFEE: IConfig['CORE_RPC_MINFEE'];
+  SECOND_CORE_RPC: IConfig['SECOND_CORE_RPC'];
   DATABASE: IConfig['DATABASE'];
   SYSLOG: IConfig['SYSLOG'];
   STATISTICS: IConfig['STATISTICS'];
   BISQ: IConfig['BISQ'];
+  LIGHTNING: IConfig['LIGHTNING'];
+  LND: IConfig['LND'];
+  CLIGHTNING: IConfig['CLIGHTNING'];
+  SOCKS5PROXY: IConfig['SOCKS5PROXY'];
+  PRICE_DATA_SERVER: IConfig['PRICE_DATA_SERVER'];
+  EXTERNAL_DATA_SERVER: IConfig['EXTERNAL_DATA_SERVER'];
+  MAXMIND: IConfig['MAXMIND'];
 
   constructor() {
-    const configs = this.merge(configFile, defaults);
+    const configs = this.merge(configFromFile, defaults);
     this.MEMPOOL = configs.MEMPOOL;
     this.ESPLORA = configs.ESPLORA;
     this.ELECTRUM = configs.ELECTRUM;
     this.CORE_RPC = configs.CORE_RPC;
-    this.CORE_RPC_MINFEE = configs.CORE_RPC_MINFEE;
+    this.SECOND_CORE_RPC = configs.SECOND_CORE_RPC;
     this.DATABASE = configs.DATABASE;
     this.SYSLOG = configs.SYSLOG;
     this.STATISTICS = configs.STATISTICS;
     this.BISQ = configs.BISQ;
+    this.LIGHTNING = configs.LIGHTNING;
+    this.LND = configs.LND;
+    this.CLIGHTNING = configs.CLIGHTNING;
+    this.SOCKS5PROXY = configs.SOCKS5PROXY;
+    this.PRICE_DATA_SERVER = configs.PRICE_DATA_SERVER;
+    this.EXTERNAL_DATA_SERVER = configs.EXTERNAL_DATA_SERVER;
+    this.MAXMIND = configs.MAXMIND;
   }
 
   merge = (...objects: object[]): IConfig => {

backend/src/database.ts

@@ -1,26 +1,62 @@
 import config from './config';
-import { createPool } from 'mysql2/promise';
+import { createPool, Pool, PoolConnection } from 'mysql2/promise';
 import logger from './logger';
+import { FieldPacket, OkPacket, PoolOptions, ResultSetHeader, RowDataPacket } from 'mysql2/typings/mysql';
 
-export class DB {
-  static pool = createPool({
-    host: config.DATABASE.HOST,
+ class DB {
+  constructor() {
+    if (config.DATABASE.SOCKET !== '') {
+      this.poolConfig.socketPath = config.DATABASE.SOCKET;
+    } else {
+      this.poolConfig.host = config.DATABASE.HOST;
+    }
+  }
+  private pool: Pool | null = null;
+  private poolConfig: PoolOptions = {
     port: config.DATABASE.PORT,
     database: config.DATABASE.DATABASE,
     user: config.DATABASE.USERNAME,
     password: config.DATABASE.PASSWORD,
     connectionLimit: 10,
     supportBigNumbers: true,
-  });
-}
+    timezone: '+00:00',
+  };
 
-export async function checkDbConnection() {
-  try {
-    const connection = await DB.pool.getConnection();
-    logger.info('Database connection established.');
-    connection.release();
-  } catch (e) {
-    logger.err('Could not connect to database: ' + e.message || e);
-    process.exit(1);
+  private checkDBFlag() {
+    if (config.DATABASE.ENABLED === false) {
+      const stack = new Error().stack;
+      logger.err(`Trying to use DB feature but config.DATABASE.ENABLED is set to false, please open an issue.\nStack trace: ${stack}}`);
+    }
+  }
+
+  public async query<T extends RowDataPacket[][] | RowDataPacket[] | OkPacket |
+    OkPacket[] | ResultSetHeader>(query, params?): Promise<[T, FieldPacket[]]>
+  {
+    this.checkDBFlag();
+    const pool = await this.getPool();
+    return pool.query(query, params);
+  }
+
+  public async checkDbConnection() {
+    this.checkDBFlag();
+    try {
+      await this.query('SELECT ?', [1]);
+      logger.info('Database connection established.');
+    } catch (e) {
+      logger.err('Could not connect to database: ' + (e instanceof Error ? e.message : e));
+      process.exit(1);
+    }
+  }
+
+  private async getPool(): Promise<Pool> {
+    if (this.pool === null) {
+      this.pool = createPool(this.poolConfig);
+      this.pool.on('connection', function (newConnection: PoolConnection) {
+        newConnection.query(`SET time_zone='+00:00'`);
+      });
+    }
+    return this.pool;
   }
 }
+
+export default new DB();

backend/src/index.ts

@@ -1,32 +1,57 @@
-import { Express, Request, Response, NextFunction } from 'express';
-import * as express from 'express';
+import express from 'express';
+import { Application, Request, Response, NextFunction } from 'express';
 import * as http from 'http';
 import * as WebSocket from 'ws';
-import * as cluster from 'cluster';
-import axios from 'axios';
-
-import { checkDbConnection } from './database';
+import cluster from 'cluster';
+import DB from './database';
 import config from './config';
-import routes from './routes';
 import blocks from './api/blocks';
 import memPool from './api/mempool';
 import diskCache from './api/disk-cache';
-import statistics from './api/statistics';
+import statistics from './api/statistics/statistics';
 import websocketHandler from './api/websocket-handler';
-import fiatConversion from './api/fiat-conversion';
 import bisq from './api/bisq/bisq';
 import bisqMarkets from './api/bisq/markets';
 import logger from './logger';
 import backendInfo from './api/backend-info';
 import loadingIndicators from './api/loading-indicators';
 import mempool from './api/mempool';
+import elementsParser from './api/liquid/elements-parser';
+import databaseMigration from './api/database-migration';
+import syncAssets from './sync-assets';
+import icons from './api/liquid/icons';
+import { Common } from './api/common';
+import poolsUpdater from './tasks/pools-updater';
+import indexer from './indexer';
+import nodesRoutes from './api/explorer/nodes.routes';
+import channelsRoutes from './api/explorer/channels.routes';
+import generalLightningRoutes from './api/explorer/general.routes';
+import lightningStatsUpdater from './tasks/lightning/stats-updater.service';
+import networkSyncService from './tasks/lightning/network-sync.service';
+import statisticsRoutes from './api/statistics/statistics.routes';
+import miningRoutes from './api/mining/mining-routes';
+import bisqRoutes from './api/bisq/bisq.routes';
+import liquidRoutes from './api/liquid/liquid.routes';
+import bitcoinRoutes from './api/bitcoin/bitcoin.routes';
+import fundingTxFetcher from './tasks/lightning/sync-tasks/funding-tx-fetcher';
+import forensicsService from './tasks/lightning/forensics.service';
+import priceUpdater from './tasks/price-updater';
+import chainTips from './api/chain-tips';
+import { AxiosError } from 'axios';
+import v8 from 'v8';
+import { formatBytes, getBytesUnit } from './utils/format';
 
 class Server {
   private wss: WebSocket.Server | undefined;
   private server: http.Server | undefined;
-  private app: Express;
+  private app: Application;
   private currentBackendRetryInterval = 5;
 
+  private maxHeapSize: number = 0;
+  private heapLogInterval: number = 60;
+  private warnedHeapCritical: boolean = false;
+  private lastHeapLogTime: number | null = null;
+
   constructor() {
     this.app = express();
 
@@ -35,7 +60,7 @@ class Server {
       return;
     }
 
-    if (cluster.isMaster) {
+    if (cluster.isPrimary) {
       logger.notice(`Mempool Server (Master) is running on port ${config.MEMPOOL.HTTP_PORT} (${backendInfo.getShortCommitHash()})`);
 
       const numCPUs = config.MEMPOOL.SPAWN_CLUSTER_PROCS;
@@ -59,8 +84,20 @@ class Server {
     }
   }
 
-  async startServer(worker = false) {
-    logger.debug(`Starting Mempool Server${worker ? ' (worker)' : ''}... (${backendInfo.getShortCommitHash()})`);
+  async startServer(worker = false): Promise<void> {
+    logger.notice(`Starting Mempool Server${worker ? ' (worker)' : ''}... (${backendInfo.getShortCommitHash()})`);
+
+    if (config.DATABASE.ENABLED) {
+      await DB.checkDbConnection();
+      try {
+        if (process.env.npm_config_reindex_blocks === 'true') { // Re-index requests
+          await databaseMigration.$blocksReindexingTruncate();
+        }
+        await databaseMigration.$initializeOrMigrateDatabase();
+      } catch (e) {
+        throw new Error(e instanceof Error ? e.message : 'Error');
+      }
+    }
 
     this.app
       .use((req: Request, res: Response, next: NextFunction) => {
@@ -68,27 +105,46 @@ class Server {
         next();
       })
       .use(express.urlencoded({ extended: true }))
-      .use(express.json());
+      .use(express.text({ type: ['text/plain', 'application/base64'] }))
+      ;
+
+    if (config.DATABASE.ENABLED) {
+      await priceUpdater.$initializeLatestPriceWithDb();
+    }
 
     this.server = http.createServer(this.app);
     this.wss = new WebSocket.Server({ server: this.server });
 
     this.setUpWebsocketHandling();
 
-    diskCache.loadMempoolCache();
-
-    if (config.DATABASE.ENABLED) {
-      await checkDbConnection();
+    await poolsUpdater.updatePoolsJson(); // Needs to be done before loading the disk cache because we sometimes wipe it
+    await syncAssets.syncAssets$();
+    if (config.MEMPOOL.ENABLED) {
+      diskCache.loadMempoolCache();
     }
 
-    if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED && cluster.isMaster) {
+    if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED && cluster.isPrimary) {
       statistics.startStatistics();
     }
 
-    fiatConversion.startService();
+    if (Common.isLiquid()) {
+      try {
+        icons.loadIcons();
+      } catch (e) {
+        logger.err('Cannot load liquid icons. Ignoring. Reason: ' + (e instanceof Error ? e.message : e));
+      }
+    }
+
+    priceUpdater.$run();
+    await chainTips.updateOrphanedBlocks();
 
     this.setUpHttpApiRoutes();
-    this.runMainUpdateLoop();
+
+    if (config.MEMPOOL.ENABLED) {
+      this.runMainUpdateLoop();
+    }
+
+    setInterval(() => { this.healthCheck(); }, 2500);
 
     if (config.BISQ.ENABLED) {
       bisq.startBisqService();
@@ -97,6 +153,10 @@ class Server {
       bisqMarkets.startBisqService();
     }
 
+    if (config.LIGHTNING.ENABLED) {
+      this.$runLightningBackend();
+    }
+
     this.server.listen(config.MEMPOOL.HTTP_PORT, () => {
       if (worker) {
         logger.info(`Mempool Server worker #${process.pid} started`);
@@ -106,13 +166,13 @@ class Server {
     });
   }
 
-  async runMainUpdateLoop() {
+  async runMainUpdateLoop(): Promise<void> {
     try {
       try {
         await memPool.$updateMemPoolInfo();
       } catch (e) {
-        const msg = `updateMempoolInfo: ${(e.message || e)}`;
-        if (config.CORE_RPC_MINFEE.ENABLED) {
+        const msg = `updateMempoolInfo: ${(e instanceof Error ? e.message : e)}`;
+        if (config.MEMPOOL.USE_SECOND_NODE_FOR_MINFEE) {
           logger.warn(msg);
         } else {
           logger.debug(msg);
@@ -120,137 +180,110 @@ class Server {
       }
       await blocks.$updateBlocks();
       await memPool.$updateMempool();
+      indexer.$run();
+
       setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS);
       this.currentBackendRetryInterval = 5;
-    } catch (e) {
-      const loggerMsg = `runMainLoop error: ${(e.message || e)}. Retrying in ${this.currentBackendRetryInterval} sec.`;
+    } catch (e: any) {
+      let loggerMsg = `Exception in runMainUpdateLoop(). Retrying in ${this.currentBackendRetryInterval} sec.`;
+      loggerMsg += ` Reason: ${(e instanceof Error ? e.message : e)}.`;
+      if (e?.stack) {
+        loggerMsg += ` Stack trace: ${e.stack}`;
+      }
+      // When we get a first Exception, only `logger.debug` it and retry after 5 seconds
+      // From the second Exception, `logger.warn` the Exception and increase the retry delay
+      // Maximum retry delay is 60 seconds
       if (this.currentBackendRetryInterval > 5) {
         logger.warn(loggerMsg);
         mempool.setOutOfSync();
       } else {
         logger.debug(loggerMsg);
       }
-      logger.debug(JSON.stringify(e));
+      if (e instanceof AxiosError) {
+        logger.debug(`AxiosError: ${e?.message}`);
+      }
       setTimeout(this.runMainUpdateLoop.bind(this), 1000 * this.currentBackendRetryInterval);
       this.currentBackendRetryInterval *= 2;
       this.currentBackendRetryInterval = Math.min(this.currentBackendRetryInterval, 60);
     }
   }
 
-  setUpWebsocketHandling() {
+  async $runLightningBackend(): Promise<void> {
+    try {
+      await fundingTxFetcher.$init();
+      await networkSyncService.$startService();
+      await lightningStatsUpdater.$startService();
+      await forensicsService.$startService();
+    } catch(e) {
+      logger.err(`Exception in $runLightningBackend. Restarting in 1 minute. Reason: ${(e instanceof Error ? e.message : e)}`);
+      await Common.sleep$(1000 * 60);
+      this.$runLightningBackend();
+    };
+  }
+
+  setUpWebsocketHandling(): void {
     if (this.wss) {
       websocketHandler.setWebsocketServer(this.wss);
     }
+    if (Common.isLiquid() && config.DATABASE.ENABLED) {
+      blocks.setNewBlockCallback(async () => {
+        try {
+          await elementsParser.$parse();
+        } catch (e) {
+          logger.warn('Elements parsing error: ' + (e instanceof Error ? e.message : e));
+        }
+      });
+    }
     websocketHandler.setupConnectionHandling();
-    statistics.setNewStatisticsEntryCallback(websocketHandler.handleNewStatistic.bind(websocketHandler));
-    blocks.setNewBlockCallback(websocketHandler.handleNewBlock.bind(websocketHandler));
-    memPool.setMempoolChangedCallback(websocketHandler.handleMempoolChange.bind(websocketHandler));
-    fiatConversion.setProgressChangedCallback(websocketHandler.handleNewConversionRates.bind(websocketHandler));
+    if (config.MEMPOOL.ENABLED) {
+      statistics.setNewStatisticsEntryCallback(websocketHandler.handleNewStatistic.bind(websocketHandler));
+      memPool.setAsyncMempoolChangedCallback(websocketHandler.handleMempoolChange.bind(websocketHandler));
+      blocks.setNewAsyncBlockCallback(websocketHandler.handleNewBlock.bind(websocketHandler));
+    }
+    priceUpdater.setRatesChangedCallback(websocketHandler.handleNewConversionRates.bind(websocketHandler));
     loadingIndicators.setProgressChangedCallback(websocketHandler.handleLoadingChanged.bind(websocketHandler));
   }
-
-  setUpHttpApiRoutes() {
-    this.app
-      .get(config.MEMPOOL.API_URL_PREFIX + 'transaction-times', routes.getTransactionTimes)
-      .get(config.MEMPOOL.API_URL_PREFIX + 'cpfp/:txId', routes.getCpfpInfo)
-      .get(config.MEMPOOL.API_URL_PREFIX + 'fees/recommended', routes.getRecommendedFees)
-      .get(config.MEMPOOL.API_URL_PREFIX + 'fees/mempool-blocks', routes.getMempoolBlocks)
-      .get(config.MEMPOOL.API_URL_PREFIX + 'backend-info', routes.getBackendInfo)
-      .get(config.MEMPOOL.API_URL_PREFIX + 'init-data', routes.getInitData)
-      .get(config.MEMPOOL.API_URL_PREFIX + 'donations', async (req, res) => {
-        try {
-          const response = await axios.get('https://mempool.space/api/v1/donations', { responseType: 'stream', timeout: 10000 });
-          response.data.pipe(res);
-        } catch (e) {
-          res.status(500).end();
-        }
-      })
-      .get(config.MEMPOOL.API_URL_PREFIX + 'donations/images/:id', async (req, res) => {
-        try {
-          const response = await axios.get('https://mempool.space/api/v1/donations/images/' + req.params.id, {
-            responseType: 'stream', timeout: 10000
-          });
-          response.data.pipe(res);
-        } catch (e) {
-          res.status(500).end();
-        }
-      })
-      .get(config.MEMPOOL.API_URL_PREFIX + 'contributors', async (req, res) => {
-        try {
-          const response = await axios.get('https://mempool.space/api/v1/contributors', { responseType: 'stream', timeout: 10000 });
-          response.data.pipe(res);
-        } catch (e) {
-          res.status(500).end();
-        }
-      })
-      .get(config.MEMPOOL.API_URL_PREFIX + 'contributors/images/:id', async (req, res) => {
-        try {
-          const response = await axios.get('https://mempool.space/api/v1/contributors/images/' + req.params.id, {
-            responseType: 'stream', timeout: 10000
-          });
-          response.data.pipe(res);
-        } catch (e) {
-          res.status(500).end();
-        }
-      })
-    ;
-
-    if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED) {
-      this.app
-        .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/2h', routes.get2HStatistics)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/24h', routes.get24HStatistics.bind(routes))
-        .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/1w', routes.get1WHStatistics.bind(routes))
-        .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/1m', routes.get1MStatistics.bind(routes))
-        .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/3m', routes.get3MStatistics.bind(routes))
-        .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/6m', routes.get6MStatistics.bind(routes))
-        .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/1y', routes.get1YStatistics.bind(routes))
-        ;
+  
+  setUpHttpApiRoutes(): void {
+    bitcoinRoutes.initRoutes(this.app);
+    if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED && config.MEMPOOL.ENABLED) {
+      statisticsRoutes.initRoutes(this.app);
+    }
+    if (Common.indexingEnabled() && config.MEMPOOL.ENABLED) {
+      miningRoutes.initRoutes(this.app);
     }
-
     if (config.BISQ.ENABLED) {
-      this.app
-        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/stats', routes.getBisqStats)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/tx/:txId', routes.getBisqTransaction)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/block/:hash', routes.getBisqBlock)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/blocks/tip/height', routes.getBisqTip)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/blocks/:index/:length', routes.getBisqBlocks)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/address/:address', routes.getBisqAddress)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/txs/:index/:length', routes.getBisqTransactions)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/currencies', routes.getBisqMarketCurrencies.bind(routes))
-        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/depth', routes.getBisqMarketDepth.bind(routes))
-        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/hloc', routes.getBisqMarketHloc.bind(routes))
-        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/markets', routes.getBisqMarketMarkets.bind(routes))
-        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/offers', routes.getBisqMarketOffers.bind(routes))
-        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/ticker', routes.getBisqMarketTicker.bind(routes))
-        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/trades', routes.getBisqMarketTrades.bind(routes))
-        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/volumes', routes.getBisqMarketVolumes.bind(routes))
-        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/volumes/7d', routes.getBisqMarketVolumes7d.bind(routes))
-        ;
+      bisqRoutes.initRoutes(this.app);
     }
+    if (Common.isLiquid()) {
+      liquidRoutes.initRoutes(this.app);
+    }
+    if (config.LIGHTNING.ENABLED) {
+      generalLightningRoutes.initRoutes(this.app);
+      nodesRoutes.initRoutes(this.app);
+      channelsRoutes.initRoutes(this.app);
+    }
+  }
 
-    if (config.MEMPOOL.BACKEND !== 'esplora') {
-      this.app
-        .get(config.MEMPOOL.API_URL_PREFIX + 'mempool', routes.getMempool)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'mempool/txids', routes.getMempoolTxIds)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'mempool/recent', routes.getRecentMempoolTransactions)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId', routes.getTransaction)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/status', routes.getTransactionStatus)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/outspends', routes.getTransactionOutspends)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash', routes.getBlock)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'blocks', routes.getBlocks)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/:height', routes.getBlocks)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/height', routes.getBlockTipHeight)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txs', routes.getBlockTransactions)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txs/:index', routes.getBlockTransactions)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txids', routes.getTxIdsForBlock)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'block-height/:height', routes.getBlockHeight)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address', routes.getAddress)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs', routes.getAddressTransactions)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs/chain/:txId', routes.getAddressTransactions)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'address-prefix/:prefix', routes.getAddressPrefix)
-      ;
+  healthCheck(): void {
+    const now = Date.now();
+    const stats = v8.getHeapStatistics();
+    this.maxHeapSize = Math.max(stats.used_heap_size, this.maxHeapSize);
+    const warnThreshold = 0.8 * stats.heap_size_limit;
+
+    const byteUnits = getBytesUnit(Math.max(this.maxHeapSize, stats.heap_size_limit));
+
+    if (!this.warnedHeapCritical && this.maxHeapSize > warnThreshold) {
+      this.warnedHeapCritical = true;
+      logger.warn(`Used ${(this.maxHeapSize / stats.heap_size_limit).toFixed(2)}% of heap limit (${formatBytes(this.maxHeapSize, byteUnits, true)} / ${formatBytes(stats.heap_size_limit, byteUnits)})!`);
+    }
+    if (this.lastHeapLogTime === null || (now - this.lastHeapLogTime) > (this.heapLogInterval * 1000)) {
+      logger.debug(`Memory usage: ${formatBytes(this.maxHeapSize, byteUnits)} / ${formatBytes(stats.heap_size_limit, byteUnits)}`);
+      this.warnedHeapCritical = false;
+      this.maxHeapSize = 0;
+      this.lastHeapLogTime = now;
     }
   }
 }
 
-const server = new Server();
+((): Server => new Server())();

backend/src/indexer.ts

@@ -0,0 +1,153 @@
+import { Common } from './api/common';
+import blocks from './api/blocks';
+import mempool from './api/mempool';
+import mining from './api/mining/mining';
+import logger from './logger';
+import bitcoinClient from './api/bitcoin/bitcoin-client';
+import priceUpdater from './tasks/price-updater';
+import PricesRepository from './repositories/PricesRepository';
+
+export interface CoreIndex {
+  name: string;
+  synced: boolean;
+  best_block_height: number;
+}
+
+class Indexer {
+  runIndexer = true;
+  indexerRunning = false;
+  tasksRunning: string[] = [];
+  coreIndexes: CoreIndex[] = [];
+
+  /**
+   * Check which core index is available for indexing
+   */
+  public async checkAvailableCoreIndexes(): Promise<void> {
+    const updatedCoreIndexes: CoreIndex[] = [];
+
+    const indexes: any = await bitcoinClient.getIndexInfo();
+    for (const indexName in indexes) {
+      const newState = {
+        name: indexName,
+        synced: indexes[indexName].synced,
+        best_block_height: indexes[indexName].best_block_height,
+      };
+      logger.info(`Core index '${indexName}' is ${indexes[indexName].synced ? 'synced' : 'not synced'}. Best block height is ${indexes[indexName].best_block_height}`);      
+      updatedCoreIndexes.push(newState);
+
+      if (indexName === 'coinstatsindex' && newState.synced === true) {
+        const previousState = this.isCoreIndexReady('coinstatsindex');
+        // if (!previousState || previousState.synced === false) {
+          this.runSingleTask('coinStatsIndex');
+        // }
+      }
+    }
+
+    this.coreIndexes = updatedCoreIndexes;
+  }
+
+  /**
+   * Return the best block height if a core index is available, or 0 if not
+   * 
+   * @param name 
+   * @returns 
+   */
+  public isCoreIndexReady(name: string): CoreIndex | null {
+    for (const index of this.coreIndexes) {
+      if (index.name === name && index.synced === true) {
+        return index;
+      }
+    }
+    return null;
+  }
+
+  public reindex(): void {
+    if (Common.indexingEnabled()) {
+      this.runIndexer = true;
+    }
+  }
+
+  public async runSingleTask(task: 'blocksPrices' | 'coinStatsIndex'): Promise<void> {
+    if (!Common.indexingEnabled()) {
+      return;
+    }
+
+    if (task === 'blocksPrices' && !this.tasksRunning.includes(task)) {
+      this.tasksRunning.push(task);
+      const lastestPriceId = await PricesRepository.$getLatestPriceId();
+      if (priceUpdater.historyInserted === false || lastestPriceId === null) {
+        logger.debug(`Blocks prices indexer is waiting for the price updater to complete`, logger.tags.mining);
+        setTimeout(() => {
+          this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task);
+          this.runSingleTask('blocksPrices');
+        }, 10000);
+      } else {
+        logger.debug(`Blocks prices indexer will run now`, logger.tags.mining);
+        await mining.$indexBlockPrices();
+        this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task);
+      }
+    }
+
+    if (task === 'coinStatsIndex' && !this.tasksRunning.includes(task)) {
+      this.tasksRunning.push(task);
+      logger.debug(`Indexing coinStatsIndex now`);
+      await mining.$indexCoinStatsIndex();
+      this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task);
+    }
+  }
+
+  public async $run(): Promise<void> {
+    if (!Common.indexingEnabled() || this.runIndexer === false ||
+      this.indexerRunning === true || mempool.hasPriority()
+    ) {
+      return;
+    }
+
+    // Do not attempt to index anything unless Bitcoin Core is fully synced
+    const blockchainInfo = await bitcoinClient.getBlockchainInfo();
+    if (blockchainInfo.blocks !== blockchainInfo.headers) {
+      return;
+    }
+
+    this.runIndexer = false;
+    this.indexerRunning = true;
+
+    logger.debug(`Running mining indexer`);
+
+    await this.checkAvailableCoreIndexes();
+
+    try {
+      await priceUpdater.$run();
+
+      const chainValid = await blocks.$generateBlockDatabase();
+      if (chainValid === false) {
+        // Chain of block hash was invalid, so we need to reindex. Stop here and continue at the next iteration
+        logger.warn(`The chain of block hash is invalid, re-indexing invalid data in 10 seconds.`, logger.tags.mining);
+        setTimeout(() => this.reindex(), 10000);
+        this.indexerRunning = false;
+        return;
+      }
+
+      this.runSingleTask('blocksPrices');
+      await mining.$indexDifficultyAdjustments();
+      await mining.$generateNetworkHashrateHistory();
+      await mining.$generatePoolHashrateHistory();
+      await blocks.$generateBlocksSummariesDatabase();
+      await blocks.$generateCPFPDatabase();
+    } catch (e) {
+      this.indexerRunning = false;
+      logger.err(`Indexer failed, trying again in 10 seconds. Reason: ` + (e instanceof Error ? e.message : e));
+      setTimeout(() => this.reindex(), 10000);
+      this.indexerRunning = false;
+      return;
+    }
+
+    this.indexerRunning = false;
+
+    const runEvery = 1000 * 3600; // 1 hour
+    logger.debug(`Indexing completed. Next run planned at ${new Date(new Date().getTime() + runEvery).toUTCString()}`);
+    setTimeout(() => this.reindex(), runEvery);
+  }
+}
+
+export default new Indexer();

backend/src/logger.ts

@@ -32,22 +32,27 @@ class Logger {
     local7: 23
   };
 
+  public tags = {
+    mining: 'Mining',
+    ln: 'Lightning',
+  };  
+
   // @ts-ignore
-  public emerg: ((msg: string) => void);
+  public emerg: ((msg: string, tag?: string) => void);
   // @ts-ignore
-  public alert: ((msg: string) => void);
+  public alert: ((msg: string, tag?: string) => void);
   // @ts-ignore
-  public crit: ((msg: string) => void);
+  public crit: ((msg: string, tag?: string) => void);
   // @ts-ignore
-  public err: ((msg: string) => void);
+  public err: ((msg: string, tag?: string) => void);
   // @ts-ignore
-  public warn: ((msg: string) => void);
+  public warn: ((msg: string, tag?: string) => void);
   // @ts-ignore
-  public notice: ((msg: string) => void);
+  public notice: ((msg: string, tag?: string) => void);
   // @ts-ignore
-  public info: ((msg: string) => void);
+  public info: ((msg: string, tag?: string) => void);
   // @ts-ignore
-  public debug: ((msg: string) => void);
+  public debug: ((msg: string, tag?: string) => void);
 
   private name = 'mempool';
   private client: dgram.Socket;
@@ -64,15 +69,22 @@ class Logger {
     this.network = this.getNetwork();
   }
 
+  public updateNetwork(): void {
+    this.network = this.getNetwork();
+  }
+
   private addprio(prio): void {
     this[prio] = (function(_this) {
-      return function(msg) {
-        return _this.msg(prio, msg);
+      return function(msg, tag?: string) {
+        return _this.msg(prio, msg, tag);
       };
     })(this);
   }
 
   private getNetwork(): string {
+    if (config.LIGHTNING.ENABLED) {
+      return config.MEMPOOL.NETWORK === 'mainnet' ? 'lightning' : `${config.MEMPOOL.NETWORK}-lightning`; 
+    }
     if (config.BISQ.ENABLED) {
       return 'bisq';
     }
@@ -82,7 +94,7 @@ class Logger {
     return '';
   }
 
-  private msg(priority, msg) {
+  private msg(priority, msg, tag?: string) {
     let consolemsg, prionum, syslogmsg;
     if (typeof msg === 'string' && msg.length > 0) {
       while (msg[msg.length - 1].charCodeAt(0) === 10) {
@@ -91,12 +103,15 @@ class Logger {
     }
     const network = this.network ? ' <' + this.network + '>' : '';
     prionum = Logger.priorities[priority] || Logger.priorities.info;
-    consolemsg = `${this.ts()} [${process.pid}] ${priority.toUpperCase()}:${network} ${msg}`;
+    consolemsg = `${this.ts()} [${process.pid}] ${priority.toUpperCase()}:${network} ${tag ? '[' + tag + '] ' : ''}${msg}`;
 
     if (config.SYSLOG.ENABLED && Logger.priorities[priority] <= Logger.priorities[config.SYSLOG.MIN_PRIORITY]) {
-      syslogmsg = `<${(Logger.facilities[config.SYSLOG.FACILITY] * 8 + prionum)}> ${this.name}[${process.pid}]: ${priority.toUpperCase()}${network} ${msg}`;
+      syslogmsg = `<${(Logger.facilities[config.SYSLOG.FACILITY] * 8 + prionum)}> ${this.name}[${process.pid}]: ${priority.toUpperCase()}${network} ${tag ? '[' + tag + '] ' : ''}${msg}`;
       this.syslog(syslogmsg);
     }
+    if (Logger.priorities[priority] > Logger.priorities[config.MEMPOOL.STDOUT_LOG_MIN_PRIORITY]) {
+      return;
+    }
     if (priority === 'warning') {
       priority = 'warn';
     }

backend/src/mempool.interfaces.ts

@@ -1,4 +1,45 @@
 import { IEsploraApi } from './api/bitcoin/esplora-api.interface';
+import { OrphanedBlock } from './api/chain-tips';
+import { HeapNode } from './utils/pairing-heap';
+
+export interface PoolTag {
+  id: number;
+  uniqueId: number;
+  name: string;
+  link: string;
+  regexes: string; // JSON array
+  addresses: string; // JSON array
+  slug: string;
+}
+
+export interface PoolInfo {
+  poolId: number; // mysql row id
+  name: string;
+  link: string;
+  blockCount: number;
+  slug: string;
+  avgMatchRate: number | null;
+}
+
+export interface PoolStats extends PoolInfo {
+  rank: number;
+  emptyBlocks: number;
+}
+
+export interface BlockAudit {
+  time: number,
+  height: number,
+  hash: string,
+  missingTxs: string[],
+  freshTxs: string[],
+  addedTxs: string[],
+  matchRate: number,
+}
+
+export interface AuditScore {
+  hash: string,
+  matchRate?: number,
+}
 
 export interface MempoolBlock {
   blockSize: number;
@@ -11,6 +52,12 @@ export interface MempoolBlock {
 
 export interface MempoolBlockWithTransactions extends MempoolBlock {
   transactionIds: string[];
+  transactions: TransactionStripped[];
+}
+
+export interface MempoolBlockDelta {
+  added: TransactionStripped[];
+  removed: string[];
 }
 
 interface VinStrippedToScriptsig {
@@ -19,6 +66,7 @@ interface VinStrippedToScriptsig {
 
 interface VoutStrippedToScriptPubkey {
   scriptpubkey_address: string | undefined;
+  scriptpubkey_asm: string | undefined;
   value: number;
 }
 
@@ -28,17 +76,57 @@ export interface TransactionExtended extends IEsploraApi.Transaction {
   firstSeen?: number;
   effectiveFeePerVsize: number;
   ancestors?: Ancestor[];
+  descendants?: Ancestor[];
   bestDescendant?: BestDescendant | null;
   cpfpChecked?: boolean;
   deleteAfter?: number;
 }
 
-interface Ancestor {
+export interface AuditTransaction {
+  txid: string;
+  fee: number;
+  weight: number;
+  feePerVsize: number;
+  effectiveFeePerVsize: number;
+  vin: string[];
+  relativesSet: boolean;
+  ancestorMap: Map<string, AuditTransaction>;
+  children: Set<AuditTransaction>;
+  ancestorFee: number;
+  ancestorWeight: number;
+  score: number;
+  used: boolean;
+  modified: boolean;
+  modifiedNode: HeapNode<AuditTransaction>;
+}
+
+export interface ThreadTransaction {
+  txid: string;
+  fee: number;
+  weight: number;
+  feePerVsize: number;
+  effectiveFeePerVsize?: number;
+  vin: string[];
+  cpfpRoot?: string;
+  cpfpChecked?: boolean;
+}
+
+export interface Ancestor {
   txid: string;
   weight: number;
   fee: number;
 }
 
+export interface TransactionSet {
+  fee: number;
+  weight: number;
+  score: number;
+  children?: Set<string>;
+  available?: boolean;
+  modified?: boolean;
+  modifiedNode?: HeapNode<string>;
+}
+
 interface BestDescendant {
   txid: string;
   weight: number;
@@ -47,7 +135,9 @@ interface BestDescendant {
 
 export interface CpfpInfo {
   ancestors: Ancestor[];
-  bestDescendant: BestDescendant | null;
+  bestDescendant?: BestDescendant | null;
+  descendants?: Ancestor[];
+  effectiveFeePerVsize?: number;
 }
 
 export interface TransactionStripped {
@@ -56,12 +146,59 @@ export interface TransactionStripped {
   vsize: number;
   value: number;
 }
+
+export interface BlockExtension {
+  totalFees: number;
+  medianFee: number; // median fee rate
+  feeRange: number[]; // fee rate percentiles
+  reward: number;
+  matchRate: number | null;
+  similarity?: number;
+  pool: {
+    id: number; // Note - This is the `unique_id`, not to mix with the auto increment `id`
+    name: string;
+    slug: string;
+  };
+  avgFee: number;
+  avgFeeRate: number;
+  coinbaseRaw: string;
+  orphans: OrphanedBlock[] | null;
+  coinbaseAddress: string | null;
+  coinbaseSignature: string | null;
+  coinbaseSignatureAscii: string | null;
+  virtualSize: number;
+  avgTxSize: number;
+  totalInputs: number;
+  totalOutputs: number;
+  totalOutputAmt: number;
+  medianFeeAmt: number | null; // median fee in sats
+  feePercentiles: number[] | null, // fee percentiles in sats
+  segwitTotalTxs: number;
+  segwitTotalSize: number;
+  segwitTotalWeight: number;
+  header: string;
+  utxoSetChange: number;
+  // Requires coinstatsindex, will be set to NULL otherwise
+  utxoSetSize: number | null;
+  totalInputAmt: number | null;
+}
+
+/**
+ * Note: Everything that is added in here will be automatically returned through
+ * /api/v1/block and /api/v1/blocks APIs
+ */
 export interface BlockExtended extends IEsploraApi.Block {
-  medianFee?: number;
-  feeRange?: number[];
-  reward?: number;
-  coinbaseTx?: TransactionMinerInfo;
-  matchRate?: number;
+  extras: BlockExtension;
+}
+
+export interface BlockSummary {
+  id: string;
+  transactions: TransactionStripped[];
+}
+
+export interface BlockPrice {
+  height: number;
+  priceId: number;
 }
 
 export interface TransactionMinerInfo {
@@ -128,10 +265,7 @@ export interface Statistic {
 }
 
 export interface OptimizedStatistic {
-  id: number;
   added: string;
-  unconfirmed_transactions: number;
-  tx_per_second: number;
   vbytes_per_second: number;
   total_fee: number;
   mempool_byte_weight: number;
@@ -160,10 +294,73 @@ interface RequiredParams {
 }
 
 export interface ILoadingIndicators { [name: string]: number; }
-export interface IConversionRates { [currency: string]: number; }
 
 export interface IBackendInfo {
   hostname: string;
   gitCommit: string;
   version: string;
+  lightning: boolean;
+}
+
+export interface IDifficultyAdjustment {
+  progressPercent: number;
+  difficultyChange: number;
+  estimatedRetargetDate: number;
+  remainingBlocks: number;
+  remainingTime: number;
+  previousRetarget: number;
+  nextRetargetHeight: number;
+  timeAvg: number;
+  timeOffset: number;
+}
+
+export interface IndexedDifficultyAdjustment {
+  time: number; // UNIX timestamp
+  height: number; // Block height
+  difficulty: number;
+  adjustment: number;
+}
+
+export interface RewardStats {
+  totalReward: number;
+  totalFee: number;
+  totalTx: number;
+}
+
+export interface ITopNodesPerChannels {
+  publicKey: string,
+  alias: string,
+  channels?: number,
+  capacity: number,
+  firstSeen?: number,
+  updatedAt?: number,
+  city?: any,
+  country?: any,
+}
+
+export interface ITopNodesPerCapacity {
+  publicKey: string,
+  alias: string,
+  capacity: number,
+  channels?: number,
+  firstSeen?: number,
+  updatedAt?: number,
+  city?: any,
+  country?: any,
+}
+
+export interface INodesRanking {
+  topByCapacity: ITopNodesPerCapacity[];
+  topByChannels: ITopNodesPerChannels[];
+}
+
+export interface IOldestNodes {
+  publicKey: string,
+  alias: string,
+  firstSeen: number,
+  channels?: number,
+  capacity: number,
+  updatedAt?: number,
+  city?: any,
+  country?: any,
 }

backend/src/repositories/BlocksAuditsRepository.ts

@@ -0,0 +1,110 @@
+import blocks from '../api/blocks';
+import DB from '../database';
+import logger from '../logger';
+import { BlockAudit, AuditScore } from '../mempool.interfaces';
+
+class BlocksAuditRepositories {
+  public async $saveAudit(audit: BlockAudit): Promise<void> {
+    try {
+      await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, fresh_txs, match_rate)
+        VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs),
+          JSON.stringify(audit.addedTxs), JSON.stringify(audit.freshTxs), audit.matchRate]);
+    } catch (e: any) {
+      if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
+        logger.debug(`Cannot save block audit for block ${audit.hash} because it has already been indexed, ignoring`);
+      } else {
+        logger.err(`Cannot save block audit into db. Reason: ` + (e instanceof Error ? e.message : e));
+        throw e;
+      }
+    }
+  }
+
+  public async $getBlockPredictionsHistory(div: number, interval: string | null): Promise<any> {
+    try {
+      let query = `SELECT UNIX_TIMESTAMP(time) as time, height, match_rate FROM blocks_audits`;
+
+      if (interval !== null) {
+        query += ` WHERE time BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
+      }
+
+      query += ` GROUP BY UNIX_TIMESTAMP(time) DIV ${div} ORDER BY height`;
+
+      const [rows] = await DB.query(query);
+      return rows;
+    } catch (e: any) {
+      logger.err(`Cannot fetch block prediction history. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getPredictionsCount(): Promise<number> {
+    try {
+      const [rows] = await DB.query(`SELECT count(hash) as count FROM blocks_audits`);
+      return rows[0].count;
+    } catch (e: any) {
+      logger.err(`Cannot fetch block prediction history. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getBlockAudit(hash: string): Promise<any> {
+    try {
+      const [rows]: any[] = await DB.query(
+        `SELECT blocks.height, blocks.hash as id, UNIX_TIMESTAMP(blocks.blockTimestamp) as timestamp, blocks.size,
+        blocks.weight, blocks.tx_count,
+        transactions, template, missing_txs as missingTxs, added_txs as addedTxs, fresh_txs as freshTxs, match_rate as matchRate
+        FROM blocks_audits
+        JOIN blocks ON blocks.hash = blocks_audits.hash
+        JOIN blocks_summaries ON blocks_summaries.id = blocks_audits.hash
+        WHERE blocks_audits.hash = "${hash}"
+      `);
+      
+      if (rows.length) {
+        rows[0].missingTxs = JSON.parse(rows[0].missingTxs);
+        rows[0].addedTxs = JSON.parse(rows[0].addedTxs);
+        rows[0].freshTxs = JSON.parse(rows[0].freshTxs);
+        rows[0].transactions = JSON.parse(rows[0].transactions);
+        rows[0].template = JSON.parse(rows[0].template);
+
+        if (rows[0].transactions.length) {
+          return rows[0];
+        }
+      }
+      return null;
+    } catch (e: any) {
+      logger.err(`Cannot fetch block audit from db. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getBlockAuditScore(hash: string): Promise<AuditScore> {
+    try {
+      const [rows]: any[] = await DB.query(
+        `SELECT hash, match_rate as matchRate
+        FROM blocks_audits
+        WHERE blocks_audits.hash = "${hash}"
+      `);
+      return rows[0];
+    } catch (e: any) {
+      logger.err(`Cannot fetch block audit from db. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getBlockAuditScores(maxHeight: number, minHeight: number): Promise<AuditScore[]> {
+    try {
+      const [rows]: any[] = await DB.query(
+        `SELECT hash, match_rate as matchRate
+        FROM blocks_audits
+        WHERE blocks_audits.height BETWEEN ? AND ?
+      `, [minHeight, maxHeight]);
+      return rows;
+    } catch (e: any) {
+      logger.err(`Cannot fetch block audit from db. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+}
+
+export default new BlocksAuditRepositories();
+

backend/src/repositories/BlocksSummariesRepository.ts

@@ -0,0 +1,128 @@
+import DB from '../database';
+import logger from '../logger';
+import { BlockSummary } from '../mempool.interfaces';
+
+class BlocksSummariesRepository {
+  public async $getByBlockId(id: string): Promise<BlockSummary | undefined> {
+    try {
+      const [summary]: any[] = await DB.query(`SELECT * from blocks_summaries WHERE id = ?`, [id]);
+      if (summary.length > 0) {
+        summary[0].transactions = JSON.parse(summary[0].transactions);
+        return summary[0];
+      }
+    } catch (e) {
+      logger.err(`Cannot get block summary for block id ${id}. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+
+    return undefined;
+  }
+
+  public async $saveSummary(params: { height: number, mined?: BlockSummary}) {
+    const blockId = params.mined?.id;
+    try {
+      const transactions = JSON.stringify(params.mined?.transactions || []);
+      await DB.query(`
+        INSERT INTO blocks_summaries (height, id, transactions, template)
+        VALUE (?, ?, ?, ?)
+        ON DUPLICATE KEY UPDATE
+          transactions = ?
+      `, [params.height, blockId, transactions, '[]', transactions]);
+    } catch (e: any) {
+      if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
+        logger.debug(`Cannot save block summary for ${blockId} because it has already been indexed, ignoring`);
+      } else {
+        logger.debug(`Cannot save block summary for ${blockId}. Reason: ${e instanceof Error ? e.message : e}`);
+        throw e;
+      }
+    }
+  }
+
+  public async $saveTemplate(params: { height: number, template: BlockSummary}) {
+    const blockId = params.template?.id;
+    try {
+      const transactions = JSON.stringify(params.template?.transactions || []);
+      await DB.query(`
+        INSERT INTO blocks_summaries (height, id, transactions, template)
+        VALUE (?, ?, ?, ?)
+        ON DUPLICATE KEY UPDATE
+          template = ?
+      `, [params.height, blockId, '[]', transactions, transactions]);
+    } catch (e: any) {
+      if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
+        logger.debug(`Cannot save block template for ${blockId} because it has already been indexed, ignoring`);
+      } else {
+        logger.debug(`Cannot save block template for ${blockId}. Reason: ${e instanceof Error ? e.message : e}`);
+        throw e;
+      }
+    }
+  }
+
+  public async $getIndexedSummariesId(): Promise<string[]> {
+    try {
+      const [rows]: any[] = await DB.query(`SELECT id from blocks_summaries`);
+      return rows.map(row => row.id);
+    } catch (e) {
+      logger.err(`Cannot get block summaries id list. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+
+    return [];
+  }
+
+  /**
+   * Delete blocks from the database from blockHeight
+   */
+  public async $deleteBlocksFrom(blockHeight: number) {
+    logger.info(`Delete newer blocks summary from height ${blockHeight} from the database`);
+
+    try {
+      await DB.query(`DELETE FROM blocks_summaries where height >= ${blockHeight}`);
+    } catch (e) {
+      logger.err('Cannot delete indexed blocks summaries. Reason: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  /**
+   * Get the fee percentiles if the block has already been indexed, [] otherwise
+   * 
+   * @param id 
+   */
+  public async $getFeePercentilesByBlockId(id: string): Promise<number[] | null> {
+    try {
+      const [rows]: any[] = await DB.query(`
+        SELECT transactions
+        FROM blocks_summaries
+        WHERE id = ?`,
+        [id]
+      );
+      if (rows === null || rows.length === 0) {
+        return null;
+      }
+
+      const transactions = JSON.parse(rows[0].transactions);
+      if (transactions === null) {
+        return null;
+      }
+
+      transactions.shift(); // Ignore coinbase
+      transactions.sort((a: any, b: any) => a.fee - b.fee);
+      const fees = transactions.map((t: any) => t.fee);
+
+      return [
+        fees[0] ?? 0, // min
+        fees[Math.max(0, Math.floor(fees.length * 0.1) - 1)] ?? 0, // 10th
+        fees[Math.max(0, Math.floor(fees.length * 0.25) - 1)] ?? 0, // 25th
+        fees[Math.max(0, Math.floor(fees.length * 0.5) - 1)] ?? 0, // median
+        fees[Math.max(0, Math.floor(fees.length * 0.75) - 1)] ?? 0, // 75th
+        fees[Math.max(0, Math.floor(fees.length * 0.9) - 1)] ?? 0, // 90th
+        fees[fees.length - 1] ?? 0, // max
+      ];
+
+    } catch (e) {
+      logger.err(`Cannot get block summaries transactions. Reason: ` + (e instanceof Error ? e.message : e));
+      return null;
+    }
+  }
+}
+
+export default new BlocksSummariesRepository();
+