add sp widget to custom dashboard

remove unused lightweight-charts

.github/CODEOWNERS

@@ -0,0 +1 @@
+backend/src/api/database-migration.ts	@wiz @softsimon

.github/FUNDING.yml

@@ -1,12 +0,0 @@
-# These are supported funding model platforms
-
-github: ['mempool'] # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
-patreon: # Replace with a single Patreon username
-open_collective: # Replace with a single Open Collective username
-ko_fi: # Replace with a single Ko-fi username
-tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
-community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
-liberapay: # Replace with a single Liberapay username
-issuehunt: # Replace with a single IssueHunt username
-otechie: # Replace with a single Otechie username
-custom: ['https://mempool.space/sponsor'] # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.github/ISSUE_TEMPLATE.md

@@ -1,37 +0,0 @@
-<!--
-   SUPPORT REQUESTS: This is for reporting bugs in Mempool.
-   If you have a support request, please join our Keybase group:
-   https://keybase.io/team/mempool
--->
-
-### Description
-
-<!-- brief description of the bug -->
-
-#### Version
-
-<!-- commit id or version number -->
-
-### Steps to reproduce
-
-<!--if you can reliably reproduce the bug, list the steps here  -->
-
-### Expected behaviour
-
-<!--description of the expected behavior -->
-
-### Actual behaviour
-
-<!-- explain what happened instead of the expected behaviour -->
-
-### Screenshots
-
-<!--Screenshots if gui related, drag and drop to add to the issue -->
-
-#### Device or machine
-
-<!-- device/machine used, operating system -->
-
-#### Additional info
-
-<!-- Additional information useful for debugging (e.g. logs) -->

.github/ISSUE_TEMPLATE/00-bug-issue.md

@@ -0,0 +1,43 @@
+---
+name: 🐛 Bug Report
+about: Report bugs (no support requests, please)
+---
+
+<!--
+   SUPPORT REQUESTS:
+   This is for reporting bugs in Mempool, not for support requests. 
+   If you have a support request, please reach out on Matrix:
+   https://matrix.to/#/#mempool.support:bitcoin.kyoto
+-->
+
+### Description
+
+<!-- brief description of the bug -->
+
+#### Version
+
+<!-- commit id or version number -->
+
+### Steps to reproduce
+
+<!-- if you can reliably reproduce the bug, list the steps here -->
+
+### Expected behaviour
+
+<!-- description of the expected behavior -->
+
+### Actual behaviour
+
+<!-- explain what happened instead of the expected behaviour -->
+
+### Screenshots
+
+<!-- Screenshots if gui related, drag and drop to add to the issue -->
+
+#### Device or machine
+
+<!-- device/machine used, operating system -->
+
+#### Additional info
+
+<!-- Additional information useful for debugging (e.g. logs) -->

.github/ISSUE_TEMPLATE/30-feature-request.md

@@ -0,0 +1,27 @@
+---
+name: ✨ Feature Request
+about: Request a feature or suggest other enhancements
+---
+
+<!--
+   SUPPORT REQUESTS:
+   This is for requesting features in Mempool, not for support requests. 
+   If you have a support request, please reach out on Matrix:
+   https://matrix.to/#/#mempool.support:bitcoin.kyoto
+-->
+
+### Description
+
+<!-- brief description of the feature request -->
+
+### Problem to be solved
+
+<!-- description of the the problem you're having -->
+
+### Proposed solution
+
+<!-- explain how you think we should solve the problem -->
+
+#### Additional info
+
+<!-- Additional information useful for implementing (e.g. docs, links, etc.) -->

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,8 @@
+blank_issues_enabled: false
+contact_links:
+  - name: 🙋 Need help? Chat with us on Matrix
+    url: https://matrix.to/#/#mempool.support:bitcoin.kyoto
+    about: For support requests or general questions
+  - name: 🌐 Want to help with translations? Use Transifex
+    url: https://www.transifex.com/mempool/mempool
+    about: All translations work is done on Transifex

.github/dependabot.yml

@@ -0,0 +1,68 @@
+version: 2
+updates:
+  - package-ecosystem: npm
+    versioning-strategy: increase
+    directory: "/backend"
+    schedule:
+      interval: daily
+    open-pull-requests-limit: 10
+    ignore:
+      - dependency-name: "*"
+        update-types:
+          ["version-update:semver-major", "version-update:semver-patch"]
+    allow:
+      - dependency-type: "production"
+
+  - package-ecosystem: npm
+    directory: "/frontend"
+    versioning-strategy: increase
+    groups:
+      frontend-angular-dependencies:
+        patterns:
+          - "@angular*"
+          - "@ng-*"
+          - "ngx-*"
+      frontend-jest-dependencies:
+        patterns:
+          - "@types/jest"
+          - "jest"
+      frontend-eslint-dependencies:
+        patterns:
+          - "@typescript-eslint*"
+          - "eslint"
+    schedule:
+      interval: daily
+    open-pull-requests-limit: 10
+    ignore:
+      - dependency-name: "*"
+        update-types:
+          ["version-update:semver-major", "version-update:semver-patch"]
+    allow:
+      - dependency-type: "production"
+
+  - package-ecosystem: docker
+    directory: "/docker/backend"
+    schedule:
+      interval: weekly
+    ignore:
+      - dependency-name: "*"
+        update-types:
+          ["version-update:semver-major", "version-update:semver-patch"]
+
+  - package-ecosystem: docker
+    directory: "/docker/frontend"
+    schedule:
+      interval: weekly
+    ignore:
+      - dependency-name: "*"
+        update-types:
+          ["version-update:semver-major", "version-update:semver-patch"]
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: weekly
+    ignore:
+      - dependency-name: "*"
+        update-types:
+          ["version-update:semver-major", "version-update:semver-patch"]

.github/pull_request_template.md

@@ -0,0 +1,6 @@
+<!--
+Please do not open pull requests for translations.
+
+All translations work is done on Transifex:
+https://www.transifex.com/mempool/mempool
+-->

.github/workflows/ci.yml

@@ -0,0 +1,362 @@
+name: CI Pipeline for the Backend and Frontend
+
+on:
+  pull_request:
+    types: [opened, review_requested, synchronize]
+
+jobs:
+  backend:
+    if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')"
+    strategy:
+      matrix:
+        node: ["20", "21"]
+        flavor: ["dev", "prod"]
+      fail-fast: false
+    runs-on: "ubuntu-latest"
+
+    name: Backend (${{ matrix.flavor }}) - node ${{ matrix.node }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          path: ${{ matrix.node }}/${{ matrix.flavor }}
+
+      - name: Setup Node
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node }}
+          registry-url: "https://registry.npmjs.org"
+
+      - name: Read rust-toolchain file from repository
+        id: gettoolchain
+        run: echo "::set-output name=toolchain::$(cat ./rust/gbt/rust-toolchain)"
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}
+
+      - name: Install ${{ steps.gettoolchain.outputs.toolchain }} Rust toolchain
+        # Latest version available on this commit is 1.71.1
+        # Commit date is Aug 3, 2023
+        uses: dtolnay/rust-toolchain@d8352f6b1d2e870bc5716e7a6d9b65c4cc244a1a
+        with:
+          toolchain: ${{ steps.gettoolchain.outputs.toolchain }}
+
+      - name: Install
+        if: ${{ matrix.flavor == 'dev'}}
+        run: npm ci
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/backend
+
+      - name: Install (Prod dependencies only)
+        if: ${{ matrix.flavor == 'prod'}}
+        run: npm ci --omit=dev --omit=optional
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/backend
+
+      - name: Lint
+        if: ${{ matrix.flavor == 'dev'}}
+        run: npm run lint
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/backend
+
+      - name: Unit Tests
+        if: ${{ matrix.flavor == 'dev'}}
+        run: npm run test:ci
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/backend
+
+      - name: Build
+        run: npm run build
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/backend
+
+
+  cache:
+    name: "Cache assets for builds"
+    runs-on: "ubuntu-latest"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          path: assets
+
+      - name: Setup Node
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node }}
+          registry-url: "https://registry.npmjs.org"
+
+      - name: Install (Prod dependencies only)
+        run: npm ci --omit=dev --omit=optional
+        working-directory: assets/frontend
+
+      - name: Restore cached mining pool assets
+        continue-on-error: true
+        id: cache-mining-pool-restore
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            mining-pool-assets.zip
+          key: mining-pool-assets-cache
+
+      - name: Restore promo video assets
+        continue-on-error: true
+        id: cache-promo-video-restore
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            promo-video-assets.zip
+          key: promo-video-assets-cache
+
+      - name: Unzip assets before building (src/resources)
+        continue-on-error: true
+        run: unzip -o mining-pool-assets.zip -d assets/frontend/src/resources/mining-pools
+
+      - name: Unzip assets before building (src/resources)
+        continue-on-error: true
+        run: unzip -o promo-video-assets.zip -d assets/frontend/src/resources/promo-video
+
+      # - name: Unzip assets before building (dist)
+      #   continue-on-error: true
+      #   run: unzip assets.zip -d assets/frontend/dist/mempool/browser/resources
+
+      - name: Sync-assets
+        run: npm run sync-assets-dev
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          MEMPOOL_CDN: 1
+          VERBOSE: 1
+        working-directory: assets/frontend
+
+      - name: Zip mining-pool assets
+        run: zip -jrq mining-pool-assets.zip assets/frontend/src/resources/mining-pools/*
+
+      - name: Zip promo-video assets
+        run: zip -jrq promo-video-assets.zip assets/frontend/src/resources/promo-video/*
+
+      - name: Upload mining pool assets as artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: mining-pool-assets
+          path: mining-pool-assets.zip
+
+      - name: Upload promo video assets as artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: promo-video-assets
+          path: promo-video-assets.zip
+
+      - name: Save mining pool assets cache
+        id: cache-mining-pool-save
+        uses: actions/cache/save@v4
+        with:
+          path: |
+            mining-pool-assets.zip
+          key: mining-pool-assets-cache
+
+      - name: Save promo video assets cache
+        id: cache-promo-video-save
+        uses: actions/cache/save@v4
+        with:
+          path: |
+            promo-video-assets.zip
+          key: promo-video-assets-cache
+
+  frontend:
+    needs: cache
+    if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')"
+    strategy:
+      matrix:
+        node: ["20", "21"]
+        flavor: ["dev", "prod"]
+      fail-fast: false
+    runs-on: "ubuntu-latest"
+
+    name: Frontend (${{ matrix.flavor }}) - node ${{ matrix.node }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          path: ${{ matrix.node }}/${{ matrix.flavor }}
+
+      - name: Setup Node
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node }}
+          registry-url: "https://registry.npmjs.org"
+
+      - name: Install (Prod dependencies only)
+        run: npm ci --omit=dev --omit=optional
+        if: ${{ matrix.flavor == 'prod'}}
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/frontend
+
+      - name: Install
+        if: ${{ matrix.flavor == 'dev'}}
+        run: npm ci
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/frontend
+
+      - name: Lint
+        if: ${{ matrix.flavor == 'dev'}}
+        run: npm run lint
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/frontend
+
+      # - name: Test
+      #   run: npm run test
+
+      - name: Restore cached mining pool assets
+        continue-on-error: true
+        id: cache-mining-pool-restore
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            mining-pool-assets.zip
+          key: mining-pool-assets-cache
+
+      - name: Restore promo video assets
+        continue-on-error: true
+        id: cache-promo-video-restore
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            promo-video-assets.zip
+          key: promo-video-assets-cache
+
+      - name: Download artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: mining-pool-assets
+
+      - name: Unzip assets before building (src/resources)
+        run: unzip -o mining-pool-assets.zip -d ${{ matrix.node }}/${{ matrix.flavor }}/frontend/src/resources/mining-pools
+
+      - name: Download artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: promo-video-assets
+
+      - name: Unzip assets before building (src/resources)
+        run: unzip -o promo-video-assets.zip -d ${{ matrix.node }}/${{ matrix.flavor }}/frontend/src/resources/promo-video
+
+      # - name: Unzip assets before building (dist)
+      #   run: unzip assets.zip -d ${{ matrix.node }}/${{ matrix.flavor }}/frontend/dist/mempool/browser/resources
+
+      - name: Display resulting source tree
+        run: ls -R
+
+      - name: Build
+        run: npm run build
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/frontend
+        env: 
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          MEMPOOL_CDN: 1
+          VERBOSE: 1
+  
+  e2e:
+    if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')"
+    runs-on: "ubuntu-latest"
+    needs: frontend
+    strategy:
+      fail-fast: false
+      matrix:
+        module: ["mempool", "liquid"]
+        include:
+          - module: "mempool"
+            spec: |
+              cypress/e2e/mainnet/*.spec.ts
+              cypress/e2e/signet/*.spec.ts
+              cypress/e2e/testnet4/*.spec.ts
+          - module: "liquid"
+            spec: |
+              cypress/e2e/liquid/liquid.spec.ts
+              cypress/e2e/liquidtestnet/liquidtestnet.spec.ts
+
+    name: E2E tests for ${{ matrix.module }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          path: ${{ matrix.module }}
+
+      - name: Setup node
+        uses: actions/setup-node@v3
+        with:
+          node-version: 20
+          cache: "npm"
+          cache-dependency-path: ${{ matrix.module }}/frontend/package-lock.json
+
+      - name: Restore cached mining pool assets
+        continue-on-error: true
+        id: cache-mining-pool-restore
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            mining-pool-assets.zip
+          key: mining-pool-assets-cache
+  
+      - name: Restore cached promo video assets
+        continue-on-error: true
+        id: cache-promo-video-restore
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            promo-video-assets.zip
+          key: promo-video-assets-cache
+
+      - name: Download artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: mining-pool-assets
+
+      - name: Unzip assets before building (src/resources)
+        run: unzip -o mining-pool-assets.zip -d ${{ matrix.module }}/frontend/src/resources/mining-pools
+
+      - name: Download artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: promo-video-assets
+
+      - name: Unzip assets before building (src/resources)
+        run: unzip -o promo-video-assets.zip -d ${{ matrix.module }}/frontend/src/resources/promo-video
+      
+      - name: Chrome browser tests (${{ matrix.module }})
+        uses: cypress-io/github-action@v5
+        with:
+          tag: ${{ github.event_name }}
+          working-directory: ${{ matrix.module }}/frontend
+          build: npm run config:defaults:${{ matrix.module }}
+          start: npm run start:local-staging
+          wait-on: "http://localhost:4200"
+          wait-on-timeout: 120
+          record: true
+          parallel: true
+          spec: ${{ matrix.spec }}
+          group: Tests on Chrome (${{ matrix.module }})
+          browser: "chrome"
+          ci-build-id: "${{ github.sha }}-${{ github.workflow }}-${{ github.event_name }}"
+        env:
+          COMMIT_INFO_MESSAGE: ${{ github.event.pull_request.title }}
+          CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}
+
+  validate_docker_json:
+    if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')"
+    runs-on: "ubuntu-latest"
+    name: Validate generated backend Docker JSON
+
+    steps: 
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          path: docker
+      
+      - name: Install jq
+        run: sudo apt-get install jq -y
+
+      - name: Create new start script to run on CI
+        run: |
+          sed '$d' start.sh > start_ci.sh
+        working-directory: docker/docker/backend
+
+      - name: Run the script to generate the sample JSON
+        run: |
+          sh start_ci.sh
+        working-directory: docker/docker/backend
+
+      - name: Validate JSON syntax
+        run: |
+          cat mempool-config.json | jq
+        working-directory: docker/docker/backend

.github/workflows/cypress.yml

@@ -1,80 +0,0 @@
-name: Cypress Tests
-
-on: [push, pull_request]
-
-jobs:
-  cypress:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        containers: [1, 2, 3, 4, 5]
-        os: ["ubuntu-latest"]
-        browser: [chrome]
-    name: E2E tests on ${{ matrix.browser }} - ${{ matrix.os }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-      - name: Setup node
-        uses: actions/setup-node@v2
-        with:
-          node-version: 16.10.0
-          cache: 'npm'
-          cache-dependency-path: frontend/package-lock.json
-      - name: ${{ matrix.browser }} browser tests (Mempool)
-        uses: cypress-io/github-action@v2
-        with:
-          working-directory: frontend
-          build: npm run config:defaults:mempool
-          start: npm run start:local-prod
-          wait-on: 'http://localhost:4200'
-          wait-on-timeout: 120
-          record: true
-          parallel: true
-          group: Tests on ${{ matrix.browser }} (Mempool)
-          browser: ${{ matrix.browser }}
-          ci-build-id: '${{ github.sha }}-${{ github.workflow }}-${{ github.event_name }}'
-        env:
-          CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}
-
-      - name: ${{ matrix.browser }} browser tests (Liquid)
-        uses: cypress-io/github-action@v2
-        if: always()
-        with:
-          working-directory: frontend
-          build: npm run config:defaults:liquid
-          start: npm run start:local-prod
-          wait-on: 'http://localhost:4200'
-          wait-on-timeout: 120
-          record: true
-          parallel: true
-          spec: cypress/integration/liquid/liquid.spec.ts
-          group: Tests on ${{ matrix.browser }} (Liquid)
-          browser: ${{ matrix.browser }}
-          ci-build-id: '${{ github.sha }}-${{ github.workflow }}-${{ github.event_name }}'
-        env:
-          CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}
-
-      - name: ${{ matrix.browser }} browser tests (Bisq)
-        uses: cypress-io/github-action@v2
-        if: always()
-        with:
-          working-directory: frontend
-          build: npm run config:defaults:bisq
-          start: npm run start:local-prod
-          wait-on: 'http://localhost:4200'
-          wait-on-timeout: 120
-          record: true
-          parallel: true
-          spec: cypress/integration/bisq/bisq.spec.ts
-          group: Tests on ${{ matrix.browser }} (Bisq)
-          browser: ${{ matrix.browser }}
-          ci-build-id: '${{ github.sha }}-${{ github.workflow }}-${{ github.event_name }}'
-        env:
-          CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}

.github/workflows/get_backend_block_height.yml

@@ -0,0 +1,19 @@
+name: 'Check if servers are in sync'
+
+on: [workflow_dispatch]
+
+jobs:
+  print-backend-sha:
+    runs-on: 'ubuntu-latest'
+    name: Get block height
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          path: repo
+
+      - name: Run script
+        working-directory: repo
+        run: |
+          chmod +x ./scripts/get_block_tip_height.sh
+          sh ./scripts/get_block_tip_height.sh

.github/workflows/get_backend_hash.yml

@@ -0,0 +1,19 @@
+name: 'Print backend hashes'
+
+on: [workflow_dispatch]
+
+jobs:
+  print-backend-sha:
+    runs-on: 'ubuntu-latest'
+    name: Print backend hashes
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          path: repo
+
+      - name: Run script
+        working-directory: repo
+        run: |
+          chmod +x ./scripts/get_backend_hash.sh
+          sh ./scripts/get_backend_hash.sh

.github/workflows/get_image_digest.yml

@@ -0,0 +1,26 @@
+name: 'Print images digest'
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Image Version'
+        required: false
+        default: 'latest'
+        type: string 
+jobs:
+  print-images-sha:
+    runs-on: 'ubuntu-latest'
+    name: Print digest for images
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          path: digest
+
+      - name: Run script
+        working-directory: digest
+        run: |
+          sh ./docker/scripts/get_image_digest.sh $VERSION
+        env: 
+          VERSION: ${{ github.event.inputs.version }}

.github/workflows/on-tag.yml

@@ -1,7 +1,7 @@
 name: Docker build on tag
 env:
   DOCKER_CLI_EXPERIMENTAL: enabled
-  TAG_FMT: '^refs/tags/(((.?[0-9]+){3,4}))$'
+  TAG_FMT: "^refs/tags/(((.?[0-9]+){3,4}))$"
   DOCKER_BUILDKIT: 0
   COMPOSE_DOCKER_CLI_BUILD: 0
 
@@ -11,6 +11,9 @@ on:
       - v[0-9]+.[0-9]+.[0-9]+
       - v[0-9]+.[0-9]+.[0-9]+-*
 
+permissions:
+  contents: read
+
 jobs:
   build:
     strategy:
@@ -18,16 +21,46 @@ jobs:
         service:
           - frontend
           - backend
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
     name: Build and push to DockerHub
     steps:
+      # Workaround based on JonasAlfredsson/docker-on-tmpfs@v1.0.1
+      - name: Replace the current swap file
+        shell: bash
+        run: |
+          sudo swapoff /mnt/swapfile
+          sudo rm -v /mnt/swapfile
+          sudo fallocate -l 13G /mnt/swapfile
+          sudo chmod 600 /mnt/swapfile
+          sudo mkswap /mnt/swapfile
+          sudo swapon /mnt/swapfile
+
+      - name: Show current memory and swap status
+        shell: bash
+        run: |
+          sudo free -h
+          echo
+          sudo swapon --show
+
+      - name: Mount a tmpfs over /var/lib/docker
+        shell: bash
+        run: |
+          if [ ! -d "/var/lib/docker" ]; then
+            echo "Directory '/var/lib/docker' not found"
+            exit 1
+          fi
+          sudo mount -t tmpfs -o size=10G tmpfs /var/lib/docker
+          sudo systemctl restart docker
+          sudo df -h | grep docker
+
       - name: Set env variables
         run: echo "TAG=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV
 
       - name: Show set environment variables
         run: |
           printf "      TAG: %s\n"  "$TAG"
-      
+
       - name: Add SHORT_SHA env property with commit short sha
         run: echo "SHORT_SHA=`echo ${GITHUB_SHA} | cut -c1-8`" >> $GITHUB_ENV
 
@@ -35,24 +68,24 @@ jobs:
         run: echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin
 
       - name: Checkout project
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Init repo for Dockerization
         run: docker/init.sh "$TAG"
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
         id: qemu
 
       - name: Setup Docker buildx action
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
         id: buildx
 
       - name: Available platforms
         run: echo ${{ steps.buildx.outputs.platforms }}
 
       - name: Cache Docker layers
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         id: cache
         with:
           path: /tmp/.buildx-cache
@@ -65,8 +98,10 @@ jobs:
           docker buildx build \
           --cache-from "type=local,src=/tmp/.buildx-cache" \
           --cache-to "type=local,dest=/tmp/.buildx-cache" \
-          --platform linux/amd64,linux/arm64,linux/arm/v7 \
+          --platform linux/amd64,linux/arm64 \
           --tag ${{ secrets.DOCKER_HUB_USER }}/${{ matrix.service }}:$TAG \
           --tag ${{ secrets.DOCKER_HUB_USER }}/${{ matrix.service }}:latest \
+          --build-context rustgbt=./rust \
+          --build-context backend=./backend \
           --output "type=registry" ./${{ matrix.service }}/ \
           --build-arg commitHash=$SHORT_SHA

.gitignore

@@ -2,3 +2,8 @@ sitemap
 data
 docker-compose.yml
 backend/mempool-config.json
+*.swp
+frontend/src/resources/config.template.js
+frontend/src/resources/config.js
+target
+docker/backend/start_ci.sh

.nvmrc

@@ -1 +1 @@
-v16.10.0
+v20.8.0

.vscode/settings.json

@@ -1,4 +1,6 @@
 {
   "editor.tabSize": 2,
-  "typescript.tsdk": "./backend/node_modules/typescript/lib"
+  "typescript.preferences.importModuleSpecifier": "relative",
+  "typescript.tsdk": "./backend/node_modules/typescript/lib",
+  "rust-analyzer.procMacro.ignored": { "napi-derive": ["napi"] }
 }

CONTRIBUTING.md

@@ -0,0 +1,51 @@
+# Contributing to The Mempool Open Source Project
+
+Thank you for contributing to The Mempool Open Source Project managed by Mempool Space K.K. (“Mempool”).
+
+In order to clarify the intellectual property license granted with Contributions from any person or entity, Mempool must have a statement on file from each Contributor indicating their agreement to the Contributor License Agreement (“Agreement”). This license is for your protection as a Contributor as well as the protection of Mempool and its other contributors and users; it does not change your rights to use your own Contributions for any other purpose.
+
+When submitting a pull request for the first time, please create a file with a name like `/contributors/{github_username}.txt`, and in the content of that file indicate your agreement to the Contributor License Agreement terms below. An example of what that file should contain can be seen in wiz's agreement file. (This method of CLA "signing" is borrowed from Medium's open source project.)
+
+Also, please GPG-sign all your commits (`git config commit.gpgsign true`).
+
+# Contributor License Agreement
+
+Last Updated: January 25, 2022
+
+By accepting this Agreement, You agree to the following terms and conditions for Your present and future Contributions submitted to Mempool. Except for the license granted herein to Mempool and recipients of software distributed by Mempool, You reserve all right, title, and interest in and to Your Contributions.
+
+### 1. Definitions
+
+“You” (or “Your”) shall mean the copyright owner or legal entity authorized by the copyright owner that is making this Agreement with Mempool. For legal entities, the entity making a Contribution and all other entities that control, are controlled by, or are under common control with that entity are considered to be a single Contributor. For the purposes of this definition, “control” means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity.
+
+“Contribution” shall mean any original work of authorship, including any modifications or additions to an existing work, that is intentionally submitted by You to Mempool for inclusion in, or documentation of, any of the products owned or managed by Mempool (“Work”). For the purposes of this definition, “submitted” means any form of electronic, verbal, or written communication sent to Mempool or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, Mempool for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by You as “Not a Contribution.”
+
+### 2. Grant of Copyright License
+
+Subject to the terms and conditions of this Agreement, You hereby grant to Mempool and to recipients of software distributed by Mempool a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare derivative works of, publicly display, publicly perform, sublicense, and distribute Your Contributions and such derivative works.
+
+### 3. Grant of Patent License
+
+Subject to the terms and conditions of this Agreement, You hereby grant to Mempool and to recipients of software distributed by Mempool a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by You that are necessarily infringed by Your Contribution(s) alone or by combination of Your Contribution(s) with the Work to which such Contribution(s) was submitted. If any entity institutes patent litigation against You or any other entity (including a cross-claim or counterclaim in a lawsuit) alleging that your Contribution, or the Work to which you have contributed, constitutes direct or contributory patent infringement, then any patent licenses granted to that entity under this Agreement for that Contribution or Work shall terminate as of the date such litigation is filed.
+
+### 4. Authority
+
+You represent that you are legally entitled to grant the above license. If your employer(s) has rights to intellectual property that you create that includes your Contributions, you represent that you have received permission to make Contributions on behalf of that employer, that your employer has waived such rights for your Contributions to Mempool, or that your employer has executed a separate Corporate Contributor License Agreement with Mempool.
+
+### 5. Originality
+
+You represent that each of Your Contributions is Your original creation (see section 7 for submissions on behalf of others). You represent that Your Contribution submissions include complete details of any third-party license or other restriction (including, but not limited to, related patents and trademarks) of which you are personally aware, and which are associated with any part of Your Contributions.
+
+### 6. Support
+
+You are not expected to provide support for Your Contributions, except to the extent You desire to provide support. You may provide support for free, for a fee, or not at all. Unless required by applicable law or agreed to in writing, You provide Your Contributions on an “AS IS” BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON- INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE.
+
+### 7. Third Party Contributions
+
+Should You wish to submit work that is not Your original creation, You may submit it to Mempool separately from any Contribution, identifying the complete details of its source and of any license or other restriction (including, but not limited to, related patents, trademarks, and license agreements) of which you are personally aware, and conspicuously marking the work as “Submitted on behalf of a third-party: [named here]”.
+
+### 8. Notifications
+
+You agree to notify Mempool of any facts or circumstances of which you become aware that would make these representations inaccurate in any respect.
+
+EOF

GNUmakefile

@@ -1,48 +0,0 @@
-# If you see pwd_unknown showing up check permissions
-PWD ?= pwd_unknown
-
-# DATABASE DEPLOY FOLDER CONFIG - default ./data
-ifeq ($(data),)
-DATA := data
-export DATA
-else
-DATA := $(data)
-export DATA
-endif
-
-.PHONY: help
-help:
-	@echo ''
-	@echo ''
-	@echo '	Usage: make [COMMAND]'
-	@echo ''
-	@echo '		make all		# build init mempool and electrs'
-	@echo '		make init		# setup some useful configs'
-	@echo '		make mempool		# build q dockerized mempool.space'
-	@echo '		make electrs		# build a docker electrs image'
-	@echo ''
-
-.PHONY: init
-init:
-	@echo ''
-	mkdir -p   $(DATA) $(DATA)/mysql $(DATA)/mysql/db-scripts $(DATA)/mysql/data 
-	install -v mariadb-structure.sql $(DATA)/mysql/db-scripts
-	#REF: https://github.com/mempool/mempool/blob/master/docker/README.md
-	cat docker/docker-compose.yml > docker-compose.yml
-	cat backend/mempool-config.sample.json > backend/mempool-config.json
-.PHONY: mempool
-mempool: init
-	@echo ''
-	docker-compose up --force-recreate --always-recreate-deps
-	@echo ''
-.PHONY: electrs
-electrum:
-	#REF: https://hub.docker.com/r/beli/electrum
-	@echo ''
-	docker build -f docker/electrum/Dockerfile .
-	@echo ''
-.PHONY: all
-all: init
-	make mempool
-#######################
--include Makefile

LICENSE

@@ -1,29 +1,30 @@
-The Mempool Open Source Project
-Copyright (c) 2019-2021 The Mempool Open Source Project Developers
+The Mempool Open Source Project®
+Copyright (c) 2019-2024 Mempool Space K.K. and other shadowy super-coders
 
 This program is free software; you can redistribute it and/or modify it under
-the terms of (at your option) either:
+the terms of the GNU Affero General Public License as published by the Free
+Software Foundation, either version 3 of the License or any later version
+approved by a proxy statement published on <https://mempool.space/about>.
 
- 1) the GNU Affero General Public License as published by the Free Software
- Foundation, either version 3 of the License or any later version approved by a
- proxy statement published on <https://mempool.space/about>; or
+However, this copyright license does not include an implied right or license
+to use any trademarks, service marks, logos, or trade names of Mempool Space K.K.
+or any other contributor to The Mempool Open Source Project.
 
- 2) the GNU General Public License as published by the Free Software
- Foundation, either version 3 of the License or any later version approved by a
- proxy statement published on <https://mempool.space/about>.
+The Mempool Open Source Project®, Mempool Accelerator™, Mempool Enterprise®, 
+Mempool Liquidity™, mempool.space®, Be your own explorer™, Explore the full 
+Bitcoin ecosystem™, Mempool Goggles™, the mempool Logo, the mempool Square Logo, 
+the mempool block visualization Logo, the mempool Blocks Logo, the mempool 
+transaction Logo, the mempool Blocks 3 | 2 Logo, the mempool research Logo, 
+the mempool.space Vertical Logo, and the mempool.space Horizontal Logo are 
+registered trademarks or trademarks of Mempool Space K.K in Japan, 
+the United States, and/or other countries.
 
-However, this copyright license does not include an implied right or license to
-use our trademarks: The Mempool Open Source Project™, mempool.space™, the
-mempool Logo™, the mempool.space Vertical Logo™, the mempool.space Horizontal
-Logo™, the mempool Square Logo™, and the mempool Blocks logo™ are registered
-trademarks or trademarks of Mempool Space K.K in Japan, the United States,
-and/or other countries. See our full Trademark Policy and Guidelines for more
-details, published on <https://mempool.space/trademark-policy>.
+See our full Trademark Policy and Guidelines for more details, published on 
+<https://mempool.space/trademark-policy>.
 
 This program is distributed in the hope that it will be useful, but WITHOUT ANY
 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
 PARTICULAR PURPOSE. See the full license terms for more details.
 
 You should have received a copy of both the GNU Affero General Public License
-and the GNU General Public License along with this program.  If not, see
-<http://www.gnu.org/licenses/>.
+along with this program. If not, see <http://www.gnu.org/licenses/>.

LICENSE.GPL-3.md

@@ -1,675 +0,0 @@
-### GNU GENERAL PUBLIC LICENSE
-
-Version 3, 29 June 2007
-
-Copyright (C) 2007 Free Software Foundation, Inc.
-<https://fsf.org/>
-
-Everyone is permitted to copy and distribute verbatim copies of this
-license document, but changing it is not allowed.
-
-### Preamble
-
-The GNU General Public License is a free, copyleft license for
-software and other kinds of works.
-
-The licenses for most software and other practical works are designed
-to take away your freedom to share and change the works. By contrast,
-the GNU General Public License is intended to guarantee your freedom
-to share and change all versions of a program--to make sure it remains
-free software for all its users. We, the Free Software Foundation, use
-the GNU General Public License for most of our software; it applies
-also to any other work released this way by its authors. You can apply
-it to your programs, too.
-
-When we speak of free software, we are referring to freedom, not
-price. Our General Public Licenses are designed to make sure that you
-have the freedom to distribute copies of free software (and charge for
-them if you wish), that you receive source code or can get it if you
-want it, that you can change the software or use pieces of it in new
-free programs, and that you know you can do these things.
-
-To protect your rights, we need to prevent others from denying you
-these rights or asking you to surrender the rights. Therefore, you
-have certain responsibilities if you distribute copies of the
-software, or if you modify it: responsibilities to respect the freedom
-of others.
-
-For example, if you distribute copies of such a program, whether
-gratis or for a fee, you must pass on to the recipients the same
-freedoms that you received. You must make sure that they, too, receive
-or can get the source code. And you must show them these terms so they
-know their rights.
-
-Developers that use the GNU GPL protect your rights with two steps:
-(1) assert copyright on the software, and (2) offer you this License
-giving you legal permission to copy, distribute and/or modify it.
-
-For the developers' and authors' protection, the GPL clearly explains
-that there is no warranty for this free software. For both users' and
-authors' sake, the GPL requires that modified versions be marked as
-changed, so that their problems will not be attributed erroneously to
-authors of previous versions.
-
-Some devices are designed to deny users access to install or run
-modified versions of the software inside them, although the
-manufacturer can do so. This is fundamentally incompatible with the
-aim of protecting users' freedom to change the software. The
-systematic pattern of such abuse occurs in the area of products for
-individuals to use, which is precisely where it is most unacceptable.
-Therefore, we have designed this version of the GPL to prohibit the
-practice for those products. If such problems arise substantially in
-other domains, we stand ready to extend this provision to those
-domains in future versions of the GPL, as needed to protect the
-freedom of users.
-
-Finally, every program is threatened constantly by software patents.
-States should not allow patents to restrict development and use of
-software on general-purpose computers, but in those that do, we wish
-to avoid the special danger that patents applied to a free program
-could make it effectively proprietary. To prevent this, the GPL
-assures that patents cannot be used to render the program non-free.
-
-The precise terms and conditions for copying, distribution and
-modification follow.
-
-### TERMS AND CONDITIONS
-
-#### 0. Definitions.
-
-"This License" refers to version 3 of the GNU General Public License.
-
-"Copyright" also means copyright-like laws that apply to other kinds
-of works, such as semiconductor masks.
-
-"The Program" refers to any copyrightable work licensed under this
-License. Each licensee is addressed as "you". "Licensees" and
-"recipients" may be individuals or organizations.
-
-To "modify" a work means to copy from or adapt all or part of the work
-in a fashion requiring copyright permission, other than the making of
-an exact copy. The resulting work is called a "modified version" of
-the earlier work or a work "based on" the earlier work.
-
-A "covered work" means either the unmodified Program or a work based
-on the Program.
-
-To "propagate" a work means to do anything with it that, without
-permission, would make you directly or secondarily liable for
-infringement under applicable copyright law, except executing it on a
-computer or modifying a private copy. Propagation includes copying,
-distribution (with or without modification), making available to the
-public, and in some countries other activities as well.
-
-To "convey" a work means any kind of propagation that enables other
-parties to make or receive copies. Mere interaction with a user
-through a computer network, with no transfer of a copy, is not
-conveying.
-
-An interactive user interface displays "Appropriate Legal Notices" to
-the extent that it includes a convenient and prominently visible
-feature that (1) displays an appropriate copyright notice, and (2)
-tells the user that there is no warranty for the work (except to the
-extent that warranties are provided), that licensees may convey the
-work under this License, and how to view a copy of this License. If
-the interface presents a list of user commands or options, such as a
-menu, a prominent item in the list meets this criterion.
-
-#### 1. Source Code.
-
-The "source code" for a work means the preferred form of the work for
-making modifications to it. "Object code" means any non-source form of
-a work.
-
-A "Standard Interface" means an interface that either is an official
-standard defined by a recognized standards body, or, in the case of
-interfaces specified for a particular programming language, one that
-is widely used among developers working in that language.
-
-The "System Libraries" of an executable work include anything, other
-than the work as a whole, that (a) is included in the normal form of
-packaging a Major Component, but which is not part of that Major
-Component, and (b) serves only to enable use of the work with that
-Major Component, or to implement a Standard Interface for which an
-implementation is available to the public in source code form. A
-"Major Component", in this context, means a major essential component
-(kernel, window system, and so on) of the specific operating system
-(if any) on which the executable work runs, or a compiler used to
-produce the work, or an object code interpreter used to run it.
-
-The "Corresponding Source" for a work in object code form means all
-the source code needed to generate, install, and (for an executable
-work) run the object code and to modify the work, including scripts to
-control those activities. However, it does not include the work's
-System Libraries, or general-purpose tools or generally available free
-programs which are used unmodified in performing those activities but
-which are not part of the work. For example, Corresponding Source
-includes interface definition files associated with source files for
-the work, and the source code for shared libraries and dynamically
-linked subprograms that the work is specifically designed to require,
-such as by intimate data communication or control flow between those
-subprograms and other parts of the work.
-
-The Corresponding Source need not include anything that users can
-regenerate automatically from other parts of the Corresponding Source.
-
-The Corresponding Source for a work in source code form is that same
-work.
-
-#### 2. Basic Permissions.
-
-All rights granted under this License are granted for the term of
-copyright on the Program, and are irrevocable provided the stated
-conditions are met. This License explicitly affirms your unlimited
-permission to run the unmodified Program. The output from running a
-covered work is covered by this License only if the output, given its
-content, constitutes a covered work. This License acknowledges your
-rights of fair use or other equivalent, as provided by copyright law.
-
-You may make, run and propagate covered works that you do not convey,
-without conditions so long as your license otherwise remains in force.
-You may convey covered works to others for the sole purpose of having
-them make modifications exclusively for you, or provide you with
-facilities for running those works, provided that you comply with the
-terms of this License in conveying all material for which you do not
-control copyright. Those thus making or running the covered works for
-you must do so exclusively on your behalf, under your direction and
-control, on terms that prohibit them from making any copies of your
-copyrighted material outside their relationship with you.
-
-Conveying under any other circumstances is permitted solely under the
-conditions stated below. Sublicensing is not allowed; section 10 makes
-it unnecessary.
-
-#### 3. Protecting Users' Legal Rights From Anti-Circumvention Law.
-
-No covered work shall be deemed part of an effective technological
-measure under any applicable law fulfilling obligations under article
-11 of the WIPO copyright treaty adopted on 20 December 1996, or
-similar laws prohibiting or restricting circumvention of such
-measures.
-
-When you convey a covered work, you waive any legal power to forbid
-circumvention of technological measures to the extent such
-circumvention is effected by exercising rights under this License with
-respect to the covered work, and you disclaim any intention to limit
-operation or modification of the work as a means of enforcing, against
-the work's users, your or third parties' legal rights to forbid
-circumvention of technological measures.
-
-#### 4. Conveying Verbatim Copies.
-
-You may convey verbatim copies of the Program's source code as you
-receive it, in any medium, provided that you conspicuously and
-appropriately publish on each copy an appropriate copyright notice;
-keep intact all notices stating that this License and any
-non-permissive terms added in accord with section 7 apply to the code;
-keep intact all notices of the absence of any warranty; and give all
-recipients a copy of this License along with the Program.
-
-You may charge any price or no price for each copy that you convey,
-and you may offer support or warranty protection for a fee.
-
-#### 5. Conveying Modified Source Versions.
-
-You may convey a work based on the Program, or the modifications to
-produce it from the Program, in the form of source code under the
-terms of section 4, provided that you also meet all of these
-conditions:
-
--   a) The work must carry prominent notices stating that you modified
-    it, and giving a relevant date.
--   b) The work must carry prominent notices stating that it is
-    released under this License and any conditions added under
-    section 7. This requirement modifies the requirement in section 4
-    to "keep intact all notices".
--   c) You must license the entire work, as a whole, under this
-    License to anyone who comes into possession of a copy. This
-    License will therefore apply, along with any applicable section 7
-    additional terms, to the whole of the work, and all its parts,
-    regardless of how they are packaged. This License gives no
-    permission to license the work in any other way, but it does not
-    invalidate such permission if you have separately received it.
--   d) If the work has interactive user interfaces, each must display
-    Appropriate Legal Notices; however, if the Program has interactive
-    interfaces that do not display Appropriate Legal Notices, your
-    work need not make them do so.
-
-A compilation of a covered work with other separate and independent
-works, which are not by their nature extensions of the covered work,
-and which are not combined with it such as to form a larger program,
-in or on a volume of a storage or distribution medium, is called an
-"aggregate" if the compilation and its resulting copyright are not
-used to limit the access or legal rights of the compilation's users
-beyond what the individual works permit. Inclusion of a covered work
-in an aggregate does not cause this License to apply to the other
-parts of the aggregate.
-
-#### 6. Conveying Non-Source Forms.
-
-You may convey a covered work in object code form under the terms of
-sections 4 and 5, provided that you also convey the machine-readable
-Corresponding Source under the terms of this License, in one of these
-ways:
-
--   a) Convey the object code in, or embodied in, a physical product
-    (including a physical distribution medium), accompanied by the
-    Corresponding Source fixed on a durable physical medium
-    customarily used for software interchange.
--   b) Convey the object code in, or embodied in, a physical product
-    (including a physical distribution medium), accompanied by a
-    written offer, valid for at least three years and valid for as
-    long as you offer spare parts or customer support for that product
-    model, to give anyone who possesses the object code either (1) a
-    copy of the Corresponding Source for all the software in the
-    product that is covered by this License, on a durable physical
-    medium customarily used for software interchange, for a price no
-    more than your reasonable cost of physically performing this
-    conveying of source, or (2) access to copy the Corresponding
-    Source from a network server at no charge.
--   c) Convey individual copies of the object code with a copy of the
-    written offer to provide the Corresponding Source. This
-    alternative is allowed only occasionally and noncommercially, and
-    only if you received the object code with such an offer, in accord
-    with subsection 6b.
--   d) Convey the object code by offering access from a designated
-    place (gratis or for a charge), and offer equivalent access to the
-    Corresponding Source in the same way through the same place at no
-    further charge. You need not require recipients to copy the
-    Corresponding Source along with the object code. If the place to
-    copy the object code is a network server, the Corresponding Source
-    may be on a different server (operated by you or a third party)
-    that supports equivalent copying facilities, provided you maintain
-    clear directions next to the object code saying where to find the
-    Corresponding Source. Regardless of what server hosts the
-    Corresponding Source, you remain obligated to ensure that it is
-    available for as long as needed to satisfy these requirements.
--   e) Convey the object code using peer-to-peer transmission,
-    provided you inform other peers where the object code and
-    Corresponding Source of the work are being offered to the general
-    public at no charge under subsection 6d.
-
-A separable portion of the object code, whose source code is excluded
-from the Corresponding Source as a System Library, need not be
-included in conveying the object code work.
-
-A "User Product" is either (1) a "consumer product", which means any
-tangible personal property which is normally used for personal,
-family, or household purposes, or (2) anything designed or sold for
-incorporation into a dwelling. In determining whether a product is a
-consumer product, doubtful cases shall be resolved in favor of
-coverage. For a particular product received by a particular user,
-"normally used" refers to a typical or common use of that class of
-product, regardless of the status of the particular user or of the way
-in which the particular user actually uses, or expects or is expected
-to use, the product. A product is a consumer product regardless of
-whether the product has substantial commercial, industrial or
-non-consumer uses, unless such uses represent the only significant
-mode of use of the product.
-
-"Installation Information" for a User Product means any methods,
-procedures, authorization keys, or other information required to
-install and execute modified versions of a covered work in that User
-Product from a modified version of its Corresponding Source. The
-information must suffice to ensure that the continued functioning of
-the modified object code is in no case prevented or interfered with
-solely because modification has been made.
-
-If you convey an object code work under this section in, or with, or
-specifically for use in, a User Product, and the conveying occurs as
-part of a transaction in which the right of possession and use of the
-User Product is transferred to the recipient in perpetuity or for a
-fixed term (regardless of how the transaction is characterized), the
-Corresponding Source conveyed under this section must be accompanied
-by the Installation Information. But this requirement does not apply
-if neither you nor any third party retains the ability to install
-modified object code on the User Product (for example, the work has
-been installed in ROM).
-
-The requirement to provide Installation Information does not include a
-requirement to continue to provide support service, warranty, or
-updates for a work that has been modified or installed by the
-recipient, or for the User Product in which it has been modified or
-installed. Access to a network may be denied when the modification
-itself materially and adversely affects the operation of the network
-or violates the rules and protocols for communication across the
-network.
-
-Corresponding Source conveyed, and Installation Information provided,
-in accord with this section must be in a format that is publicly
-documented (and with an implementation available to the public in
-source code form), and must require no special password or key for
-unpacking, reading or copying.
-
-#### 7. Additional Terms.
-
-"Additional permissions" are terms that supplement the terms of this
-License by making exceptions from one or more of its conditions.
-Additional permissions that are applicable to the entire Program shall
-be treated as though they were included in this License, to the extent
-that they are valid under applicable law. If additional permissions
-apply only to part of the Program, that part may be used separately
-under those permissions, but the entire Program remains governed by
-this License without regard to the additional permissions.
-
-When you convey a copy of a covered work, you may at your option
-remove any additional permissions from that copy, or from any part of
-it. (Additional permissions may be written to require their own
-removal in certain cases when you modify the work.) You may place
-additional permissions on material, added by you to a covered work,
-for which you have or can give appropriate copyright permission.
-
-Notwithstanding any other provision of this License, for material you
-add to a covered work, you may (if authorized by the copyright holders
-of that material) supplement the terms of this License with terms:
-
--   a) Disclaiming warranty or limiting liability differently from the
-    terms of sections 15 and 16 of this License; or
--   b) Requiring preservation of specified reasonable legal notices or
-    author attributions in that material or in the Appropriate Legal
-    Notices displayed by works containing it; or
--   c) Prohibiting misrepresentation of the origin of that material,
-    or requiring that modified versions of such material be marked in
-    reasonable ways as different from the original version; or
--   d) Limiting the use for publicity purposes of names of licensors
-    or authors of the material; or
--   e) Declining to grant rights under trademark law for use of some
-    trade names, trademarks, or service marks; or
--   f) Requiring indemnification of licensors and authors of that
-    material by anyone who conveys the material (or modified versions
-    of it) with contractual assumptions of liability to the recipient,
-    for any liability that these contractual assumptions directly
-    impose on those licensors and authors.
-
-All other non-permissive additional terms are considered "further
-restrictions" within the meaning of section 10. If the Program as you
-received it, or any part of it, contains a notice stating that it is
-governed by this License along with a term that is a further
-restriction, you may remove that term. If a license document contains
-a further restriction but permits relicensing or conveying under this
-License, you may add to a covered work material governed by the terms
-of that license document, provided that the further restriction does
-not survive such relicensing or conveying.
-
-If you add terms to a covered work in accord with this section, you
-must place, in the relevant source files, a statement of the
-additional terms that apply to those files, or a notice indicating
-where to find the applicable terms.
-
-Additional terms, permissive or non-permissive, may be stated in the
-form of a separately written license, or stated as exceptions; the
-above requirements apply either way.
-
-#### 8. Termination.
-
-You may not propagate or modify a covered work except as expressly
-provided under this License. Any attempt otherwise to propagate or
-modify it is void, and will automatically terminate your rights under
-this License (including any patent licenses granted under the third
-paragraph of section 11).
-
-However, if you cease all violation of this License, then your license
-from a particular copyright holder is reinstated (a) provisionally,
-unless and until the copyright holder explicitly and finally
-terminates your license, and (b) permanently, if the copyright holder
-fails to notify you of the violation by some reasonable means prior to
-60 days after the cessation.
-
-Moreover, your license from a particular copyright holder is
-reinstated permanently if the copyright holder notifies you of the
-violation by some reasonable means, this is the first time you have
-received notice of violation of this License (for any work) from that
-copyright holder, and you cure the violation prior to 30 days after
-your receipt of the notice.
-
-Termination of your rights under this section does not terminate the
-licenses of parties who have received copies or rights from you under
-this License. If your rights have been terminated and not permanently
-reinstated, you do not qualify to receive new licenses for the same
-material under section 10.
-
-#### 9. Acceptance Not Required for Having Copies.
-
-You are not required to accept this License in order to receive or run
-a copy of the Program. Ancillary propagation of a covered work
-occurring solely as a consequence of using peer-to-peer transmission
-to receive a copy likewise does not require acceptance. However,
-nothing other than this License grants you permission to propagate or
-modify any covered work. These actions infringe copyright if you do
-not accept this License. Therefore, by modifying or propagating a
-covered work, you indicate your acceptance of this License to do so.
-
-#### 10. Automatic Licensing of Downstream Recipients.
-
-Each time you convey a covered work, the recipient automatically
-receives a license from the original licensors, to run, modify and
-propagate that work, subject to this License. You are not responsible
-for enforcing compliance by third parties with this License.
-
-An "entity transaction" is a transaction transferring control of an
-organization, or substantially all assets of one, or subdividing an
-organization, or merging organizations. If propagation of a covered
-work results from an entity transaction, each party to that
-transaction who receives a copy of the work also receives whatever
-licenses to the work the party's predecessor in interest had or could
-give under the previous paragraph, plus a right to possession of the
-Corresponding Source of the work from the predecessor in interest, if
-the predecessor has it or can get it with reasonable efforts.
-
-You may not impose any further restrictions on the exercise of the
-rights granted or affirmed under this License. For example, you may
-not impose a license fee, royalty, or other charge for exercise of
-rights granted under this License, and you may not initiate litigation
-(including a cross-claim or counterclaim in a lawsuit) alleging that
-any patent claim is infringed by making, using, selling, offering for
-sale, or importing the Program or any portion of it.
-
-#### 11. Patents.
-
-A "contributor" is a copyright holder who authorizes use under this
-License of the Program or a work on which the Program is based. The
-work thus licensed is called the contributor's "contributor version".
-
-A contributor's "essential patent claims" are all patent claims owned
-or controlled by the contributor, whether already acquired or
-hereafter acquired, that would be infringed by some manner, permitted
-by this License, of making, using, or selling its contributor version,
-but do not include claims that would be infringed only as a
-consequence of further modification of the contributor version. For
-purposes of this definition, "control" includes the right to grant
-patent sublicenses in a manner consistent with the requirements of
-this License.
-
-Each contributor grants you a non-exclusive, worldwide, royalty-free
-patent license under the contributor's essential patent claims, to
-make, use, sell, offer for sale, import and otherwise run, modify and
-propagate the contents of its contributor version.
-
-In the following three paragraphs, a "patent license" is any express
-agreement or commitment, however denominated, not to enforce a patent
-(such as an express permission to practice a patent or covenant not to
-sue for patent infringement). To "grant" such a patent license to a
-party means to make such an agreement or commitment not to enforce a
-patent against the party.
-
-If you convey a covered work, knowingly relying on a patent license,
-and the Corresponding Source of the work is not available for anyone
-to copy, free of charge and under the terms of this License, through a
-publicly available network server or other readily accessible means,
-then you must either (1) cause the Corresponding Source to be so
-available, or (2) arrange to deprive yourself of the benefit of the
-patent license for this particular work, or (3) arrange, in a manner
-consistent with the requirements of this License, to extend the patent
-license to downstream recipients. "Knowingly relying" means you have
-actual knowledge that, but for the patent license, your conveying the
-covered work in a country, or your recipient's use of the covered work
-in a country, would infringe one or more identifiable patents in that
-country that you have reason to believe are valid.
-
-If, pursuant to or in connection with a single transaction or
-arrangement, you convey, or propagate by procuring conveyance of, a
-covered work, and grant a patent license to some of the parties
-receiving the covered work authorizing them to use, propagate, modify
-or convey a specific copy of the covered work, then the patent license
-you grant is automatically extended to all recipients of the covered
-work and works based on it.
-
-A patent license is "discriminatory" if it does not include within the
-scope of its coverage, prohibits the exercise of, or is conditioned on
-the non-exercise of one or more of the rights that are specifically
-granted under this License. You may not convey a covered work if you
-are a party to an arrangement with a third party that is in the
-business of distributing software, under which you make payment to the
-third party based on the extent of your activity of conveying the
-work, and under which the third party grants, to any of the parties
-who would receive the covered work from you, a discriminatory patent
-license (a) in connection with copies of the covered work conveyed by
-you (or copies made from those copies), or (b) primarily for and in
-connection with specific products or compilations that contain the
-covered work, unless you entered into that arrangement, or that patent
-license was granted, prior to 28 March 2007.
-
-Nothing in this License shall be construed as excluding or limiting
-any implied license or other defenses to infringement that may
-otherwise be available to you under applicable patent law.
-
-#### 12. No Surrender of Others' Freedom.
-
-If conditions are imposed on you (whether by court order, agreement or
-otherwise) that contradict the conditions of this License, they do not
-excuse you from the conditions of this License. If you cannot convey a
-covered work so as to satisfy simultaneously your obligations under
-this License and any other pertinent obligations, then as a
-consequence you may not convey it at all. For example, if you agree to
-terms that obligate you to collect a royalty for further conveying
-from those to whom you convey the Program, the only way you could
-satisfy both those terms and this License would be to refrain entirely
-from conveying the Program.
-
-#### 13. Use with the GNU Affero General Public License.
-
-Notwithstanding any other provision of this License, you have
-permission to link or combine any covered work with a work licensed
-under version 3 of the GNU Affero General Public License into a single
-combined work, and to convey the resulting work. The terms of this
-License will continue to apply to the part which is the covered work,
-but the special requirements of the GNU Affero General Public License,
-section 13, concerning interaction through a network will apply to the
-combination as such.
-
-#### 14. Revised Versions of this License.
-
-The Free Software Foundation may publish revised and/or new versions
-of the GNU General Public License from time to time. Such new versions
-will be similar in spirit to the present version, but may differ in
-detail to address new problems or concerns.
-
-Each version is given a distinguishing version number. If the Program
-specifies that a certain numbered version of the GNU General Public
-License "or any later version" applies to it, you have the option of
-following the terms and conditions either of that numbered version or
-of any later version published by the Free Software Foundation. If the
-Program does not specify a version number of the GNU General Public
-License, you may choose any version ever published by the Free
-Software Foundation.
-
-If the Program specifies that a proxy can decide which future versions
-of the GNU General Public License can be used, that proxy's public
-statement of acceptance of a version permanently authorizes you to
-choose that version for the Program.
-
-Later license versions may give you additional or different
-permissions. However, no additional obligations are imposed on any
-author or copyright holder as a result of your choosing to follow a
-later version.
-
-#### 15. Disclaimer of Warranty.
-
-THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
-APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
-HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT
-WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND
-PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE
-DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR
-CORRECTION.
-
-#### 16. Limitation of Liability.
-
-IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
-WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR
-CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
-INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES
-ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT
-NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR
-LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM
-TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER
-PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
-
-#### 17. Interpretation of Sections 15 and 16.
-
-If the disclaimer of warranty and limitation of liability provided
-above cannot be given local legal effect according to their terms,
-reviewing courts shall apply local law that most closely approximates
-an absolute waiver of all civil liability in connection with the
-Program, unless a warranty or assumption of liability accompanies a
-copy of the Program in return for a fee.
-
-END OF TERMS AND CONDITIONS
-
-### How to Apply These Terms to Your New Programs
-
-If you develop a new program, and you want it to be of the greatest
-possible use to the public, the best way to achieve this is to make it
-free software which everyone can redistribute and change under these
-terms.
-
-To do so, attach the following notices to the program. It is safest to
-attach them to the start of each source file to most effectively state
-the exclusion of warranty; and each file should have at least the
-"copyright" line and a pointer to where the full notice is found.
-
-        <one line to give the program's name and a brief idea of what it does.>
-        Copyright (C) <year>  <name of author>
-
-        This program is free software: you can redistribute it and/or modify
-        it under the terms of the GNU General Public License as published by
-        the Free Software Foundation, either version 3 of the License, or
-        (at your option) any later version.
-
-        This program is distributed in the hope that it will be useful,
-        but WITHOUT ANY WARRANTY; without even the implied warranty of
-        MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-        GNU General Public License for more details.
-
-        You should have received a copy of the GNU General Public License
-        along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-Also add information on how to contact you by electronic and paper
-mail.
-
-If the program does terminal interaction, make it output a short
-notice like this when it starts in an interactive mode:
-
-        <program>  Copyright (C) <year>  <name of author>
-        This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
-        This is free software, and you are welcome to redistribute it
-        under certain conditions; type `show c' for details.
-
-The hypothetical commands \`show w' and \`show c' should show the
-appropriate parts of the General Public License. Of course, your
-program's commands might be different; for a GUI interface, you would
-use an "about box".
-
-You should also get your employer (if you work as a programmer) or
-school, if any, to sign a "copyright disclaimer" for the program, if
-necessary. For more information on this, and how to apply and follow
-the GNU GPL, see <https://www.gnu.org/licenses/>.
-
-The GNU General Public License does not permit incorporating your
-program into proprietary programs. If your program is a subroutine
-library, you may consider it more useful to permit linking proprietary
-applications with the library. If this is what you want to do, use the
-GNU Lesser General Public License instead of this License. But first,
-please read <https://www.gnu.org/licenses/why-not-lgpl.html>.

Makefile

@@ -1 +0,0 @@
-# For additional configs/scripting

README.md

@@ -1,188 +1,38 @@
-# The Mempool Open Source Project™
+# The Mempool Open Source Project® [![mempool](https://img.shields.io/endpoint?url=https://dashboard.cypress.io/badge/simple/ry4br7/master&style=flat-square)](https://dashboard.cypress.io/projects/ry4br7/runs)
 
-Mempool is the fully featured visualizer, explorer, and API service running on [mempool.space](https://mempool.space/), an open source project developed and operated for the benefit of the Bitcoin community, with a focus on the emerging transaction fee market to help our transition into a multi-layer ecosystem.
+https://user-images.githubusercontent.com/93150691/226236121-375ea64f-b4a1-4cc0-8fad-a6fb33226840.mp4
 
-![mempool](https://mempool.space/resources/screenshots/v2.3.0-dashboard.png)
+<br>
 
-## Installation Methods
+Mempool is the fully-featured mempool visualizer, explorer, and API service running at [mempool.space](https://mempool.space/). 
 
-Mempool can be self-hosted on a wide variety of your own hardware, ranging from a simple one-click installation on a Raspberry Pi distro, all the way to an advanced high availability cluster of powerful servers for a production instance. We support the following installation methods, ranked in order from simple to advanced:
+It is an open-source project developed and operated for the benefit of the Bitcoin community, with a focus on the emerging transaction fee market that is evolving Bitcoin into a multi-layer ecosystem.
 
-1) One-click installation on: [Umbrel](https://github.com/getumbrel/umbrel), [RaspiBlitz](https://github.com/rootzoll/raspiblitz), [RoninDojo](https://code.samourai.io/ronindojo/RoninDojo), or [MyNode](https://github.com/mynodebtc/mynode).
-2) [Docker installation on Linux using docker-compose](https://github.com/mempool/mempool/tree/master/docker)
-3) [Manual installation on Linux or FreeBSD](https://github.com/mempool/mempool#manual-installation)
-4) [Production installation on a powerful FreeBSD server](https://github.com/mempool/mempool/tree/master/production)
-5) [High Availability cluster using powerful FreeBSD servers](https://github.com/mempool/mempool/tree/master/production#high-availability)
+# Installation Methods
 
-# Manual Installation
+Mempool can be self-hosted on a wide variety of your own hardware, ranging from a simple one-click installation on a Raspberry Pi full-node distro all the way to a robust production instance on a powerful FreeBSD server. 
 
-The following instructions are for a manual installation on Linux or FreeBSD. The file and directory paths may need to be changed to match your OS.
+Most people should use a <a href="#one-click-installation">one-click install method</a>.
 
-## Dependencies
+Other install methods are meant for developers and others with experience managing servers. If you want support for your own production instance of Mempool, or if you'd like to have your own instance of Mempool run by the mempool.space team on their own global ISP infrastructure—check out <a href="https://mempool.space/enterprise" target="_blank">Mempool Enterprise®</a>.
 
-* [Bitcoin](https://github.com/bitcoin/bitcoin)
-* [Electrum](https://github.com/romanz/electrs)
-* [NodeJS](https://github.com/nodejs/node)
-* [MariaDB](https://github.com/mariadb/server)
-* [Nginx](https://github.com/nginx/nginx)
+<a id="one-click-installation"></a>
+## One-Click Installation
 
-## Mempool
+Mempool can be conveniently installed on the following full-node distros: 
+- [Umbrel](https://github.com/getumbrel/umbrel)
+- [RaspiBlitz](https://github.com/rootzoll/raspiblitz)
+- [RoninDojo](https://code.samourai.io/ronindojo/RoninDojo)
+- [myNode](https://github.com/mynodebtc/mynode)
+- [StartOS](https://github.com/Start9Labs/start-os)
+- [nix-bitcoin](https://github.com/fort-nix/nix-bitcoin/blob/a1eacce6768ca4894f365af8f79be5bbd594e1c3/examples/configuration.nix#L129)
 
-Clone the mempool repo, and checkout the latest release tag:
-```bash
-  git clone https://github.com/mempool/mempool
-  cd mempool
-  latestrelease=$(curl -s https://api.github.com/repos/mempool/mempool/releases/latest|grep tag_name|head -1|cut -d '"' -f4)
-  git checkout $latestrelease
-```
+**We highly recommend you deploy your own Mempool instance this way.** No matter which option you pick, you'll be able to get your own fully-sovereign instance of Mempool up quickly without needing to fiddle with any settings.
 
-## Bitcoin Core (bitcoind)
+## Advanced Installation Methods
 
-Enable RPC and txindex in `bitcoin.conf`:
-```bash
-  rpcuser=mempool
-  rpcpassword=mempool
-  txindex=1
-```
+Mempool can be installed in other ways too, but we only recommend doing so if you're a developer, have experience managing servers, or otherwise know what you're doing.
 
-## MySQL
-
-Install MariaDB from OS package manager:
-```bash
-  # Linux
-  apt-get install mariadb-server mariadb-client
-
-  # macOS
-  brew install mariadb
-  mysql.server start
-```
-
-Create database and grant privileges:
-```bash
-  MariaDB [(none)]> drop database mempool;
-  Query OK, 0 rows affected (0.00 sec)
-
-  MariaDB [(none)]> create database mempool;
-  Query OK, 1 row affected (0.00 sec)
-
-  MariaDB [(none)]> grant all privileges on mempool.* to 'mempool'@'%' identified by 'mempool';
-  Query OK, 0 rows affected (0.00 sec)
-```
-
-From the mempool repo's top-level folder, import the database structure:
-```bash
-  mysql -u mempool -p mempool < mariadb-structure.sql
-```
-
-## Mempool Backend
-Install mempool dependencies from npm and build the backend:
-
-```bash
-  # backend
-  cd backend
-  npm install --prod
-  npm run build
-```
-
-In the `backend` folder, make a copy of the sample config and modify it to fit your settings.
-
-```bash
-  cp mempool-config.sample.json mempool-config.json
-```
-
-Edit `mempool-config.json` to add your Bitcoin Core node RPC credentials:
-```bash
-{
-  "MEMPOOL": {
-    "NETWORK": "mainnet",
-    "BACKEND": "electrum",
-    "HTTP_PORT": 8999
-  },
-  "CORE_RPC": {
-    "HOST": "127.0.0.1",
-    "PORT": 8332,
-    "USERNAME": "mempool",
-    "PASSWORD": "mempool"
-  },
-  "ELECTRUM": {
-    "HOST": "127.0.0.1",
-    "PORT": 50002,
-    "TLS_ENABLED": true
-  },
-  "DATABASE": {
-    "ENABLED": true,
-    "HOST": "127.0.0.1",
-    "PORT": 3306,
-    "USERNAME": "mempool",
-    "PASSWORD": "mempool",
-    "DATABASE": "mempool"
-  }
-}
-```
-
-Start the backend:
-
-```bash
-  npm run start
-```
-
-When it's running you should see output like this:
-
-```bash
-  Mempool updated in 0.189 seconds
-  Updating mempool
-  Mempool updated in 0.096 seconds
-  Updating mempool
-  Mempool updated in 0.099 seconds
-  Updating mempool
-  Calculated fee for transaction 1 / 10
-  Calculated fee for transaction 2 / 10
-  Calculated fee for transaction 3 / 10
-  Calculated fee for transaction 4 / 10
-  Calculated fee for transaction 5 / 10
-  Calculated fee for transaction 6 / 10
-  Calculated fee for transaction 7 / 10
-  Calculated fee for transaction 8 / 10
-  Calculated fee for transaction 9 / 10
-  Calculated fee for transaction 10 / 10
-  Mempool updated in 0.243 seconds
-  Updating mempool
-```
-
-## Mempool Frontend
-
-Install mempool dependencies from npm and build the frontend static HTML/CSS/JS:
-
-```bash
-  # frontend
-  cd frontend
-  npm install --prod
-  npm run build
-```
-
-Install the output into nginx webroot folder:
-
-```bash
-  sudo rsync -av --delete dist/mempool/ /var/www/
-```
-
-## nginx + certbot
-
-Install the supplied nginx.conf and nginx-mempool.conf in /etc/nginx
-
-```bash
-  # install nginx and certbot
-  apt-get install -y nginx python3-certbot-nginx
-
-  # install the mempool configuration for nginx
-  cp nginx.conf nginx-mempool.conf /etc/nginx/
-
-  # replace example.com with your domain name
-  certbot --nginx -d example.com
-
-```
-
-If everything went okay you should see the beautiful mempool :grin:
-
-If you get stuck on "loading blocks", this means the websocket can't connect.
-Check your nginx proxy setup, firewalls, etc. and open an issue if you need help.
+- See the [`docker/`](./docker/) directory for instructions on deploying Mempool with Docker.
+- See the [`backend/`](./backend/) and [`frontend/`](./frontend/) directories for manual install instructions oriented for developers.
+- See the [`production/`](./production/) directory for guidance on setting up a more serious Mempool instance designed for high performance at scale.

backend/.dockerignore

@@ -0,0 +1 @@
+Dockerfile

backend/.editorconfig

@@ -0,0 +1,17 @@
+# Editor configuration, see https://editorconfig.org
+root = true
+
+[*]
+charset = utf-8
+indent_style = space
+indent_size = 2
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[*.ts]
+quote_type = single
+
+[*.md]
+max_line_length = off
+trim_trailing_whitespace = false
+

backend/.eslintignore

@@ -0,0 +1,2 @@
+node_modules
+dist

backend/.eslintrc

@@ -0,0 +1,40 @@
+{
+  "root": true,
+  "parser": "@typescript-eslint/parser",
+  "plugins": [
+    "@typescript-eslint"
+  ],
+  "extends": [
+    "eslint:recommended",
+    "plugin:@typescript-eslint/eslint-recommended",
+    "plugin:@typescript-eslint/recommended",
+    "prettier"
+  ],
+  "rules": {
+    "@typescript-eslint/ban-ts-comment": 1,
+    "@typescript-eslint/ban-types": 1,
+    "@typescript-eslint/no-empty-function": 1,
+    "@typescript-eslint/no-explicit-any": 1,
+    "@typescript-eslint/no-inferrable-types": 0,
+    "@typescript-eslint/no-namespace": 1,
+    "@typescript-eslint/no-this-alias": 1,
+    "@typescript-eslint/no-var-requires": 1,
+    "@typescript-eslint/explicit-function-return-type": 1,
+    "@typescript-eslint/no-unused-vars": 1,
+    "no-console": 1,
+    "no-constant-condition": 1,
+    "no-dupe-else-if": 1,
+    "no-empty": 1,
+    "no-prototype-builtins": 1,
+    "no-self-assign": 1,
+    "no-useless-catch": 1,
+    "no-var": 1,
+    "prefer-const": 1,
+    "prefer-rest-params": 1,
+    "quotes": [1, "single", { "allowTemplateLiterals": true }],
+    "semi": 1,
+    "curly": [1, "all"],
+    "eqeqeq": 1,
+    "no-trailing-spaces": 1
+  }
+}

backend/.gitignore

@@ -1,7 +1,17 @@
 # See http://help.github.com/ignore-files/ for more about ignoring files.
 
-# production config
+# production config and external assets
+!mempool-config.template.json
+!mempool-config.sample.json
 mempool-config.json
+pools.json
+icons.json
+
+# docker
+Dockerfile
+GeoIP
+start.sh
+wait-for-it.sh
 
 # compiled output
 /dist
@@ -41,3 +51,9 @@ testem.log
 #System Files
 .DS_Store
 Thumbs.db
+
+# package folder (npm run package output)
+/package
+
+# Rust GBT folder (We build externally first)
+/rust-gbt

backend/.prettierignore

@@ -0,0 +1,2 @@
+node_modules
+package-lock.json

backend/.prettierrc

@@ -0,0 +1,6 @@
+{
+  "endOfLine": "lf",
+  "printWidth": 80,
+  "tabWidth": 2,
+  "trailingComma": "es5"
+}

backend/README.md

@@ -0,0 +1,256 @@
+# Mempool Backend
+
+These instructions are mostly intended for developers. 
+
+If you choose to use these instructions for a production setup, be aware that you will still probably need to do additional configuration for your specific OS, environment, use-case, etc. We do our best here to provide a good starting point, but only proceed if you know what you're doing. Mempool only provides support for custom setups to project sponsors through [Mempool Enterprise®](https://mempool.space/enterprise).
+
+See other ways to set up Mempool on [the main README](/../../#installation-methods).
+
+Jump to a section in this doc:
+- [Set Up the Backend](#setup)
+- [Development Tips](#development-tips)
+
+## Setup
+
+### 1. Clone Mempool Repository
+
+Get the latest Mempool code:
+
+```
+git clone https://github.com/mempool/mempool
+cd mempool
+```
+
+Check out the latest release:
+
+```
+latestrelease=$(curl -s https://api.github.com/repos/mempool/mempool/releases/latest|grep tag_name|head -1|cut -d '"' -f4)
+git checkout $latestrelease
+```
+
+### 2. Configure Bitcoin Core
+
+Turn on `txindex`, enable RPC, and set RPC credentials in `bitcoin.conf`:
+
+```
+txindex=1
+server=1
+rpcuser=mempool
+rpcpassword=mempool
+```
+
+### 3. Configure Electrum Server
+
+[Pick an Electrum Server implementation](https://mempool.space/docs/faq#address-lookup-issues), configure it, and make sure it's synced.
+
+**This step is optional.** You can run Mempool without configuring an Electrum Server for it, but address lookups will be disabled.
+
+### 4. Configure MariaDB
+
+_Mempool needs MariaDB v10.5 or later. If you already have MySQL installed, make sure to migrate any existing databases **before** installing MariaDB._
+
+Get MariaDB from your operating system's package manager:
+
+```
+# Debian, Ubuntu, etc.
+apt-get install mariadb-server mariadb-client
+
+# macOS
+brew install mariadb
+mysql.server start
+```
+
+Create a database and grant privileges:
+
+```
+MariaDB [(none)]> drop database mempool;
+Query OK, 0 rows affected (0.00 sec)
+
+MariaDB [(none)]> create database mempool;
+Query OK, 1 row affected (0.00 sec)
+
+MariaDB [(none)]> grant all privileges on mempool.* to 'mempool'@'%' identified by 'mempool';
+Query OK, 0 rows affected (0.00 sec)
+```
+
+### 5. Prepare Mempool Backend
+
+#### Build
+
+_Make sure to use Node.js 20.x and npm 9.x or newer_
+
+_The build process requires [Rust](https://www.rust-lang.org/tools/install) to be installed._
+
+Install dependencies with `npm` and build the backend:
+
+```
+cd backend
+npm install --no-install-links # npm@9.4.2 and later can omit the --no-install-links
+npm run build
+```
+
+#### Configure
+
+In the backend folder, make a copy of the sample config file:
+
+```
+cp mempool-config.sample.json mempool-config.json
+```
+
+Edit `mempool-config.json` as needed. 
+
+In particular, make sure:
+- the correct Bitcoin Core RPC credentials are specified in `CORE_RPC`
+- the correct `BACKEND` is specified in `MEMPOOL`:
+  - "electrum" if you're using [romanz/electrs](https://github.com/romanz/electrs) or [cculianu/Fulcrum](https://github.com/cculianu/Fulcrum)
+  - "esplora" if you're using [mempool/electrs](https://github.com/mempool/electrs)
+  - "none" if you're not using any Electrum Server
+
+### 6. Run Mempool Backend
+
+Run the Mempool backend:
+
+```
+npm run start
+
+```
+You can also set env var `MEMPOOL_CONFIG_FILE` to specify a custom config file location:
+```
+MEMPOOL_CONFIG_FILE=/path/to/mempool-config.json npm run start
+```
+
+When it's running, you should see output like this:
+
+```
+Mempool updated in 0.189 seconds
+Updating mempool
+Mempool updated in 0.096 seconds
+Updating mempool
+Mempool updated in 0.099 seconds
+Updating mempool
+Calculated fee for transaction 1 / 10
+Calculated fee for transaction 2 / 10
+Calculated fee for transaction 3 / 10
+Calculated fee for transaction 4 / 10
+Calculated fee for transaction 5 / 10
+Calculated fee for transaction 6 / 10
+Calculated fee for transaction 7 / 10
+Calculated fee for transaction 8 / 10
+Calculated fee for transaction 9 / 10
+Calculated fee for transaction 10 / 10
+Mempool updated in 0.243 seconds
+Updating mempool
+```
+
+### 7. Set Up Mempool Frontend
+With the backend configured and running, proceed to set up the [Mempool frontend](../frontend#manual-setup).
+
+## Development Tips
+
+### Set Up Backend Watchers
+
+The Mempool backend is static. TypeScript scripts are compiled into the `dist` folder and served through a Node.js web server. 
+
+As a result, for development purposes, you may find it helpful to set up backend watchers to avoid the manual shutdown/recompile/restart command-line cycle.
+
+First, install `nodemon` and `ts-node`:
+
+```
+npm install -g ts-node nodemon
+```
+
+Then, run the watcher:
+
+```
+nodemon src/index.ts --ignore cache/
+```
+
+`nodemon` should be in npm's global binary folder. If needed, you can determine where that is with `npm -g bin`.
+
+### Useful Regtest Commands
+
+Helpful link: https://gist.github.com/System-Glitch/cb4e87bf1ae3fec9925725bb3ebe223a
+
+Run bitcoind on regtest:
+   ```
+   bitcoind -regtest
+   ```
+
+Create a new wallet, if needed:
+   ```
+   bitcoin-cli -regtest createwallet test
+   ```
+
+Load wallet (this command may take a while if you have a lot of UTXOs):
+   ```
+   bitcoin-cli -regtest loadwallet test
+   ```
+
+Get a new address:
+   ```
+   address=$(bitcoin-cli -regtest getnewaddress)
+   ```
+
+Mine blocks to the previously generated address. You need at least 101 blocks before you can spend. This will take some time to execute (~1 min):
+   ```
+   bitcoin-cli -regtest generatetoaddress 101 $address
+   ```
+
+Send 0.1 BTC at 5 sat/vB to another address:
+   ```
+   bitcoin-cli -named -regtest sendtoaddress address=$(bitcoin-cli -regtest getnewaddress) amount=0.1 fee_rate=5
+   ```
+
+See more example of `sendtoaddress`:
+   ```
+   bitcoin-cli sendtoaddress # will print the help
+   ```
+
+Mini script to generate random network activity (random TX count with random tx fee-rate). It's slow so don't expect to use this to test mempool spam, except if you let it run for a long time, or maybe with multiple regtest nodes connected to each other.
+   ```
+   #!/bin/bash
+   address=$(bitcoin-cli -regtest getnewaddress)
+   bitcoin-cli -regtest generatetoaddress 101 $address
+   for i in {1..1000000}
+   do
+      for y in $(seq 1 "$(jot -r 1 1 1000)")
+      do
+         bitcoin-cli -regtest -named sendtoaddress address=$address amount=0.01 fee_rate=$(jot -r 1 1 100)
+      done
+      bitcoin-cli -regtest generatetoaddress 1 $address
+      sleep 5
+   done
+   ```
+
+Generate block at regular interval (every 10 seconds in this example):
+   ```
+   watch -n 10 "bitcoin-cli -regtest generatetoaddress 1 $address"
+   ```
+
+### Mining pools update
+
+By default, mining pools will be not automatically updated regularly (`config.MEMPOOL.AUTOMATIC_POOLS_UPDATE` is set to `false`). 
+
+To manually update your mining pools, you can use the `--update-pools` command line flag when you run the nodejs backend. For example `npm run start --update-pools`. This will trigger the mining pools update and automatically re-index appropriate blocks.
+
+You can enable the automatic mining pools update by settings `config.MEMPOOL.AUTOMATIC_POOLS_UPDATE` to `true` in your `mempool-config.json`.
+
+When a `coinbase tag` or `coinbase address` change is detected, pool assignments for all relevant blocks (tagged to that pool or the `unknown` mining pool, starting from height 130635) are updated using the new criteria.
+
+### Re-index tables
+
+You can manually force the nodejs backend to drop all data from a specified set of tables for future re-index. This is mostly useful for the mining dashboard and the lightning explorer.
+
+Use the `--reindex` command to specify a list of comma separated table which will be truncated at start. Note that a 5 seconds delay will be observed before truncating tables in order to give you a chance to cancel (CTRL+C) in case of misuse of the command.
+
+Usage:
+```
+npm run start --reindex=blocks,hashrates
+```
+Example output:
+```
+Feb 13 14:55:27 [63246] WARN: <lightning> Indexed data for "hashrates" tables will be erased in 5 seconds (using '--reindex')
+Feb 13 14:55:32 [63246] NOTICE: <lightning> Table hashrates has been truncated
+```
+
+Reference: https://github.com/mempool/mempool/pull/1269

backend/jest.config.ts

@@ -0,0 +1,20 @@
+import type { Config } from "@jest/types"
+
+const config: Config.InitialOptions = {
+  preset: "ts-jest",
+  testEnvironment: "node",
+  verbose: true,
+  automock: false,
+  collectCoverage: true,
+  collectCoverageFrom: ["./src/**/**.ts"],
+  coverageProvider: "babel",
+  coverageThreshold: {
+    global: {
+      lines: 1
+    }
+  },
+  setupFiles: [
+    "./testSetup.ts",
+  ],
+}
+export default config;

backend/mempool-config.sample.json

@@ -1,25 +1,53 @@
 {
   "MEMPOOL": {
+    "OFFICIAL": false,
     "NETWORK": "mainnet",
     "BACKEND": "electrum",
+    "ENABLED": true,
     "HTTP_PORT": 8999,
     "SPAWN_CLUSTER_PROCS": 0,
     "API_URL_PREFIX": "/api/v1/",
     "POLL_RATE_MS": 2000,
     "CACHE_DIR": "./cache",
+    "CACHE_ENABLED": true,
     "CLEAR_PROTECTION_MINUTES": 20,
     "RECOMMENDED_FEE_PERCENTILE": 50,
     "BLOCK_WEIGHT_UNITS": 4000000,
     "INITIAL_BLOCKS_AMOUNT": 8,
     "MEMPOOL_BLOCKS_AMOUNT": 8,
-    "PRICE_FEED_UPDATE_INTERVAL": 3600,
-    "USE_SECOND_NODE_FOR_MINFEE": false
+    "INDEXING_BLOCKS_AMOUNT": 11000,
+    "BLOCKS_SUMMARIES_INDEXING": false,
+    "GOGGLES_INDEXING": false,
+    "USE_SECOND_NODE_FOR_MINFEE": false,
+    "EXTERNAL_ASSETS": [],
+    "EXTERNAL_MAX_RETRY": 1,
+    "EXTERNAL_RETRY_INTERVAL": 0,
+    "USER_AGENT": "mempool",
+    "STDOUT_LOG_MIN_PRIORITY": "debug",
+    "AUTOMATIC_POOLS_UPDATE": false,
+    "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json",
+    "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
+    "POOLS_UPDATE_DELAY": 604800,
+    "AUDIT": false,
+    "RUST_GBT": true,
+    "LIMIT_GBT": false,
+    "CPFP_INDEXING": false,
+    "DISK_CACHE_BLOCK_INTERVAL": 6,
+    "MAX_PUSH_TX_SIZE_WEIGHT": 4000000,
+    "ALLOW_UNREACHABLE": true,
+    "PRICE_UPDATES_PER_HOUR": 1,
+    "MAX_TRACKED_ADDRESSES": 100,
+    "UNIX_SOCKET_PATH": ""
   },
   "CORE_RPC": {
     "HOST": "127.0.0.1",
     "PORT": 8332,
     "USERNAME": "mempool",
-    "PASSWORD": "mempool"
+    "PASSWORD": "mempool",
+    "TIMEOUT": 60000,
+    "COOKIE": false,
+    "COOKIE_PATH": "/path/to/bitcoin/.cookie",
+    "DEBUG_LOG_PATH": "/path/to/bitcoin/debug.log"
   },
   "ELECTRUM": {
     "HOST": "127.0.0.1",
@@ -27,21 +55,34 @@
     "TLS_ENABLED": true
   },
   "ESPLORA": {
-    "REST_API_URL": "http://127.0.0.1:3000"
+    "REST_API_URL": "http://127.0.0.1:3000",
+    "UNIX_SOCKET_PATH": "/tmp/esplora-bitcoin-mainnet",
+    "BATCH_QUERY_BASE_SIZE": 1000,
+    "RETRY_UNIX_SOCKET_AFTER": 30000,
+    "REQUEST_TIMEOUT": 10000,
+    "FALLBACK_TIMEOUT": 5000,
+    "FALLBACK": [],
+    "MAX_BEHIND_TIP": 2
   },
   "SECOND_CORE_RPC": {
     "HOST": "127.0.0.1",
     "PORT": 8332,
     "USERNAME": "mempool",
-    "PASSWORD": "mempool"
+    "PASSWORD": "mempool",
+    "TIMEOUT": 60000,
+    "COOKIE": false,
+    "COOKIE_PATH": "/path/to/bitcoin/.cookie"
   },
   "DATABASE": {
     "ENABLED": true,
     "HOST": "127.0.0.1",
     "PORT": 3306,
+    "SOCKET": "/var/run/mysql/mysql.sock",
     "DATABASE": "mempool",
     "USERNAME": "mempool",
-    "PASSWORD": "mempool"
+    "PASSWORD": "mempool",
+    "TIMEOUT": 180000,
+    "PID_DIR": ""
   },
   "SYSLOG": {
     "ENABLED": true,
@@ -54,8 +95,69 @@
     "ENABLED": true,
     "TX_PER_SECOND_SAMPLE_PERIOD": 150
   },
-  "BISQ": {
+  "MAXMIND": {
     "ENABLED": false,
-    "DATA_PATH": "/bisq/statsnode-data/btc_mainnet/db"
+    "GEOLITE2_CITY": "/usr/local/share/GeoIP/GeoLite2-City.mmdb",
+    "GEOLITE2_ASN": "/usr/local/share/GeoIP/GeoLite2-ASN.mmdb",
+    "GEOIP2_ISP": "/usr/local/share/GeoIP/GeoIP2-ISP.mmdb"
+  },
+  "LIGHTNING": {
+    "ENABLED": false,
+    "BACKEND": "lnd",
+    "STATS_REFRESH_INTERVAL": 600,
+    "GRAPH_REFRESH_INTERVAL": 600,
+    "LOGGER_UPDATE_INTERVAL": 30,
+    "FORENSICS_INTERVAL": 43200,
+    "FORENSICS_RATE_LIMIT": 20
+  },
+  "LND": {
+    "TLS_CERT_PATH": "tls.cert",
+    "MACAROON_PATH": "readonly.macaroon",
+    "REST_API_URL": "https://localhost:8080",
+    "TIMEOUT": 10000
+  },
+  "CLIGHTNING": {
+    "SOCKET": "lightning-rpc"
+  },
+  "SOCKS5PROXY": {
+    "ENABLED": false,
+    "USE_ONION": true,
+    "HOST": "127.0.0.1",
+    "PORT": 9050,
+    "USERNAME": "",
+    "PASSWORD": ""
+  },
+  "EXTERNAL_DATA_SERVER": {
+    "MEMPOOL_API": "https://mempool.space/api/v1",
+    "MEMPOOL_ONION": "http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1",
+    "LIQUID_API": "https://liquid.network/api/v1",
+    "LIQUID_ONION": "http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1"
+  },
+  "REDIS": {
+    "ENABLED": false,
+    "UNIX_SOCKET_PATH": "/tmp/redis.sock",
+    "BATCH_QUERY_BASE_SIZE": 5000
+  },
+  "REPLICATION": {
+    "ENABLED": false,
+    "AUDIT": false,
+    "AUDIT_START_HEIGHT": 774000,
+    "STATISTICS": false,
+    "STATISTICS_START_TIME": 1481932800,
+    "SERVERS": [
+      "list",
+      "of",
+      "trusted",
+      "servers"
+    ]
+  },
+  "MEMPOOL_SERVICES": {
+    "API": "https://mempool.space/api/v1/services",
+    "ACCELERATIONS": false
+  },
+  "FIAT_PRICE": {
+    "ENABLED": true,
+    "PAID": false,
+    "API_KEY": "your-api-key-from-freecurrencyapi.com"
   }
 }

backend/npm_package.sh

@@ -0,0 +1,17 @@
+#/bin/sh
+set -e
+
+# Remove previous dist folder
+rm -rf dist
+# Build new dist folder
+npm run build
+# Remove previous package folder
+rm -rf package
+# Move JS and deps
+mv dist package
+cp -R node_modules package
+# Remove symlink for rust-gbt and insert real folder
+rm package/node_modules/rust-gbt
+cp -R rust-gbt package/node_modules
+# Clean up deps
+npm run package-rm-build-deps

backend/npm_package_rm_build_deps.sh

@@ -0,0 +1,9 @@
+#/bin/sh
+set -e
+
+# Cleaning up inside the node_modules folder
+cd package/node_modules
+rm -rf \
+  typescript \
+  @typescript-eslint \
+  @napi-rs

backend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mempool-backend",
-  "version": "2.3.0-dev",
+  "version": "3.1.0-dev",
   "description": "Bitcoin mempool visualizer and blockchain explorer backend",
   "license": "GNU Affero General Public License v3.0",
   "homepage": "https://mempool.space",
@@ -16,35 +16,59 @@
     "mempool",
     "blockchain",
     "explorer",
-    "liquid"
+    "liquid",
+    "lightning"
   ],
   "main": "index.ts",
   "scripts": {
-    "ng": "./node_modules/@angular/cli/bin/ng",
-    "tsc": "./node_modules/typescript/bin/tsc",
-    "build": "npm run tsc",
+    "tsc": "./node_modules/typescript/bin/tsc -p tsconfig.build.json",
+    "build": "npm run tsc && npm run create-resources",
+    "clean": "rm -rf ./dist/ ./node_modules/ ./package/ ./rust-gbt/",
+    "create-resources": "cp ./src/tasks/price-feeds/mtgox-weekly.json ./dist/tasks && node dist/api/fetch-version.js",
+    "package": "./npm_package.sh",
+    "package-rm-build-deps": "./npm_package_rm_build_deps.sh",
+    "preinstall": "cd ../rust/gbt && npm run build-release && npm run to-backend",
     "start": "node --max-old-space-size=2048 dist/index.js",
-    "start-production": "node --max-old-space-size=4096 dist/index.js",
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "start-production": "node --max-old-space-size=16384 dist/index.js",
+    "reindex-updated-pools": "npm run start-production --update-pools",
+    "reindex-all-blocks": "npm run start-production --update-pools --reindex-blocks",
+    "test": "./node_modules/.bin/jest --coverage",
+    "test:ci": "CI=true ./node_modules/.bin/jest --coverage",
+    "lint": "./node_modules/.bin/eslint . --ext .ts",
+    "lint:fix": "./node_modules/.bin/eslint . --ext .ts --fix",
+    "prettier": "./node_modules/.bin/prettier --write \"src/**/*.{js,ts}\""
   },
   "dependencies": {
-    "@mempool/bitcoin": "^3.0.3",
-    "@mempool/electrum-client": "^1.1.7",
-    "@types/ws": "8.2.2",
-    "axios": "0.24.0",
-    "bitcoinjs-lib": "6.0.1",
-    "crypto-js": "^4.0.0",
-    "express": "^4.17.1",
-    "locutus": "^2.0.12",
-    "mysql2": "2.3.3",
-    "node-worker-threads-pool": "^1.4.3",
-    "typescript": "4.4.4",
-    "ws": "8.3.0"
+    "@babel/core": "^7.25.2",
+    "@mempool/electrum-client": "1.1.9",
+    "@types/node": "^18.15.3",
+    "axios": "1.7.2",
+    "bitcoinjs-lib": "~6.1.3",
+    "crypto-js": "~4.2.0",
+    "express": "~4.21.1",
+    "maxmind": "~4.3.11",
+    "mysql2": "~3.11.0",
+    "rust-gbt": "file:./rust-gbt",
+    "redis": "^4.7.0",
+    "socks-proxy-agent": "~7.0.0",
+    "typescript": "~4.9.3",
+    "ws": "~8.18.0"
   },
   "devDependencies": {
-    "@types/compression": "^1.0.1",
-    "@types/express": "^4.17.2",
-    "@types/locutus": "^0.0.6",
-    "tslint": "^6.1.0"
+    "@babel/code-frame": "^7.18.6",
+    "@babel/core": "^7.25.2",
+    "@types/compression": "^1.7.2",
+    "@types/crypto-js": "^4.1.1",
+    "@types/express": "^4.17.17",
+    "@types/jest": "^29.5.0",
+    "@types/ws": "~8.5.10",
+    "@typescript-eslint/eslint-plugin": "^5.55.0",
+    "@typescript-eslint/parser": "^5.55.0",
+    "eslint": "^8.36.0",
+    "eslint-config-prettier": "^8.8.0",
+    "jest": "^29.5.0",
+    "prettier": "^3.0.0",
+    "ts-jest": "^29.1.1",
+    "ts-node": "^10.9.1"
   }
 }

backend/src/__fixtures__/mempool-config.template.json

@@ -0,0 +1,155 @@
+{
+  "MEMPOOL": {
+    "ENABLED": true,
+    "OFFICIAL": false,
+    "NETWORK": "__MEMPOOL_NETWORK__",
+    "BACKEND": "__MEMPOOL_BACKEND__",
+    "BLOCKS_SUMMARIES_INDEXING": true,
+    "GOGGLES_INDEXING": false,
+    "HTTP_PORT": 1,
+    "UNIX_SOCKET_PATH": "/mempool/socket/mempool-bitcoin-mainnet",
+    "SPAWN_CLUSTER_PROCS": 2,
+    "API_URL_PREFIX": "__MEMPOOL_API_URL_PREFIX__",
+    "AUTOMATIC_POOLS_UPDATE": false,
+    "POLL_RATE_MS": 3,
+    "CACHE_DIR": "__MEMPOOL_CACHE_DIR__",
+    "CACHE_ENABLED": true,
+    "CLEAR_PROTECTION_MINUTES": 4,
+    "RECOMMENDED_FEE_PERCENTILE": 5,
+    "BLOCK_WEIGHT_UNITS": 6,
+    "INITIAL_BLOCKS_AMOUNT": 7,
+    "MEMPOOL_BLOCKS_AMOUNT": 8,
+    "USE_SECOND_NODE_FOR_MINFEE": 10,
+    "EXTERNAL_ASSETS": [],
+    "EXTERNAL_MAX_RETRY": 12,
+    "EXTERNAL_RETRY_INTERVAL": 13,
+    "USER_AGENT": "__MEMPOOL_USER_AGENT__",
+    "STDOUT_LOG_MIN_PRIORITY": "__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__",
+    "INDEXING_BLOCKS_AMOUNT": 14,
+    "POOLS_JSON_TREE_URL": "__MEMPOOL_POOLS_JSON_TREE_URL__",
+    "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__",
+    "POOLS_UPDATE_DELAY": 604800,
+    "AUDIT": true,
+    "RUST_GBT": false,
+    "LIMIT_GBT": false,
+    "CPFP_INDEXING": true,
+    "MAX_BLOCKS_BULK_QUERY": 999,
+    "DISK_CACHE_BLOCK_INTERVAL": 999,
+    "MAX_PUSH_TX_SIZE_WEIGHT": 4000000,
+    "ALLOW_UNREACHABLE": true,
+    "PRICE_UPDATES_PER_HOUR": 1,
+    "MAX_TRACKED_ADDRESSES": 1
+  },
+  "CORE_RPC": {
+    "HOST": "__CORE_RPC_HOST__",
+    "PORT": 15,
+    "USERNAME": "__CORE_RPC_USERNAME__",
+    "PASSWORD": "__CORE_RPC_PASSWORD__",
+    "TIMEOUT": 1000,
+    "COOKIE": false,
+    "COOKIE_PATH": "__CORE_RPC_COOKIE_PATH__",
+    "DEBUG_LOG_PATH": "__CORE_RPC_DEBUG_LOG_PATH__"
+  },
+  "ELECTRUM": {
+    "HOST": "__ELECTRUM_HOST__",
+    "PORT": 16,
+    "TLS_ENABLED": true
+  },
+  "ESPLORA": {
+    "REST_API_URL": "__ESPLORA_REST_API_URL__",
+    "UNIX_SOCKET_PATH": "__ESPLORA_UNIX_SOCKET_PATH__",
+    "BATCH_QUERY_BASE_SIZE": 1000,
+    "RETRY_UNIX_SOCKET_AFTER": 888,
+    "REQUEST_TIMEOUT": 10000,
+    "FALLBACK_TIMEOUT": 5000,
+    "FALLBACK": [],
+    "MAX_BEHIND_TIP": 2
+  },
+  "SECOND_CORE_RPC": {
+    "HOST": "__SECOND_CORE_RPC_HOST__",
+    "PORT": 17,
+    "USERNAME": "__SECOND_CORE_RPC_USERNAME__",
+    "PASSWORD": "__SECOND_CORE_RPC_PASSWORD__",
+    "TIMEOUT": 2000,
+    "COOKIE": false,
+    "COOKIE_PATH": "__SECOND_CORE_RPC_COOKIE_PATH__"
+  },
+  "DATABASE": {
+    "ENABLED": false,
+    "HOST": "__DATABASE_HOST__",
+    "SOCKET": "__DATABASE_SOCKET__",
+    "PORT": 18,
+    "DATABASE": "__DATABASE_DATABASE__",
+    "USERNAME": "__DATABASE_USERNAME__",
+    "PASSWORD": "__DATABASE_PASSWORD__",
+    "PID_DIR": "__DATABASE_PID_FILE__",
+    "TIMEOUT": 3000,
+    "POOL_SIZE": 100
+  },
+  "SYSLOG": {
+    "ENABLED": false,
+    "HOST": "__SYSLOG_HOST__",
+    "PORT": 19,
+    "MIN_PRIORITY": "__SYSLOG_MIN_PRIORITY__",
+    "FACILITY": "__SYSLOG_FACILITY__"
+  },
+  "STATISTICS": {
+    "ENABLED": false,
+    "TX_PER_SECOND_SAMPLE_PERIOD": 20
+  },
+  "SOCKS5PROXY": {
+    "ENABLED": true,
+    "USE_ONION": true,
+    "HOST": "__SOCKS5PROXY_HOST__",
+    "PORT": "__SOCKS5PROXY_PORT__",
+    "USERNAME": "__SOCKS5PROXY_USERNAME__",
+    "PASSWORD": "__SOCKS5PROXY_PASSWORD__"
+  },
+  "EXTERNAL_DATA_SERVER": {
+    "MEMPOOL_API": "__EXTERNAL_DATA_SERVER_MEMPOOL_API__",
+    "MEMPOOL_ONION": "__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__",
+    "LIQUID_API": "__EXTERNAL_DATA_SERVER_LIQUID_API__",
+    "LIQUID_ONION": "__EXTERNAL_DATA_SERVER_LIQUID_ONION__"
+  },
+  "LIGHTNING": {
+    "ENABLED": true,
+    "BACKEND": "__LIGHTNING_BACKEND__",
+    "TOPOLOGY_FOLDER": "__LIGHTNING_TOPOLOGY_FOLDER__",
+    "STATS_REFRESH_INTERVAL": 600,
+    "GRAPH_REFRESH_INTERVAL": 600,
+    "LOGGER_UPDATE_INTERVAL": 30,
+    "FORENSICS_INTERVAL": 43200,
+    "FORENSICS_RATE_LIMIT": 1234
+  },
+  "LND": {
+    "TLS_CERT_PATH": "",
+    "MACAROON_PATH": "",
+    "REST_API_URL": "https://localhost:8080",
+    "TIMEOUT": 10000
+  },
+  "CLIGHTNING": {
+    "SOCKET": "__CLIGHTNING_SOCKET__"
+  },
+  "REPLICATION": {
+    "ENABLED": false,
+    "AUDIT": false,
+    "AUDIT_START_HEIGHT": 774000,
+    "STATISTICS": false,
+    "STATISTICS_START_TIME": 1481932800,
+    "SERVERS": []
+  },
+  "MEMPOOL_SERVICES": {
+    "API": "",
+    "ACCELERATIONS": false
+  },
+  "REDIS": {
+    "ENABLED": false,
+    "UNIX_SOCKET_PATH": "/tmp/redis.sock",
+    "BATCH_QUERY_BASE_SIZE": 5000
+  },
+  "FIAT_PRICE": {
+    "ENABLED": true,
+    "PAID": false,
+    "API_KEY": "__MEMPOOL_CURRENCY_API_KEY__"
+  }
+}

backend/src/__tests__/api/common.ts

@@ -0,0 +1,40 @@
+import { Common } from '../../api/common';
+import { MempoolTransactionExtended, TransactionExtended } from '../../mempool.interfaces';
+
+const randomTransactions = require('./test-data/transactions-random.json');
+const replacedTransactions = require('./test-data/transactions-replaced.json');
+const rbfTransactions = require('./test-data/transactions-rbfs.json');
+const nonStandardTransactions = require('./test-data/non-standard-txs.json');
+
+describe('Common', () => {
+  describe('RBF', () => {
+    const newTransactions = rbfTransactions.concat(randomTransactions);
+    test('should detect RBF transactions with fast method', () => {
+      const result: { [txid: string]: { replaced: MempoolTransactionExtended[], replacedBy: TransactionExtended }} = Common.findRbfTransactions(newTransactions, replacedTransactions);
+      expect(Object.values(result).length).toEqual(2);
+      expect(result).toHaveProperty('7219d95161f3718335991ac6d967d24eedec370908c9879bb1e192e6d797d0a6');
+      expect(result).toHaveProperty('5387881d695d4564d397026dc5f740f816f8390b4b2c5ec8c20309122712a875');
+    });
+
+    test('should detect RBF transactions with scalable method', () => {
+      const result: { [txid: string]: { replaced: MempoolTransactionExtended[], replacedBy: TransactionExtended }} = Common.findRbfTransactions(newTransactions, replacedTransactions, true);
+      expect(Object.values(result).length).toEqual(2);
+      expect(result).toHaveProperty('7219d95161f3718335991ac6d967d24eedec370908c9879bb1e192e6d797d0a6');
+      expect(result).toHaveProperty('5387881d695d4564d397026dc5f740f816f8390b4b2c5ec8c20309122712a875');
+    });
+  });
+
+  describe('Mempool Goggles', () => {
+    test('should detect nonstandard transactions', () => {
+      nonStandardTransactions.forEach((tx) => {
+        expect(Common.isNonStandard(tx)).toEqual(true);
+      });
+    });
+  
+    test('should not misclassify as nonstandard transactions', () => {
+      randomTransactions.forEach((tx) => {
+        expect(Common.isNonStandard(tx)).toEqual(false);
+      });
+    });
+  });
+});

backend/src/__tests__/api/difficulty-adjustment.test.ts

@@ -0,0 +1,166 @@
+import {
+  calcBitsDifference,
+  calcDifficultyAdjustment,
+  DifficultyAdjustment,
+} from '../../api/difficulty-adjustment';
+
+describe('Mempool Difficulty Adjustment', () => {
+  test('should calculate Difficulty Adjustments properly', () => {
+    const dt = (dtString) => {
+      return Math.floor(new Date(dtString).getTime() / 1000);
+    };
+
+    const vectors = [
+      [ // Vector 1 (normal adjustment)
+        [ // Inputs
+          dt('2024-02-02T15:42:06.000Z'), // Last DA time (in seconds)
+          dt('2024-02-08T14:43:05.000Z'), // timestamp of 504 blocks ago (in seconds)
+          dt('2024-02-11T22:43:01.000Z'), // Current time (now) (in seconds)
+          830027,                         // Current block height
+          7.333505241141637,             // Previous retarget % (Passed through)
+          'mainnet',                      // Network (if testnet, next value is non-zero)
+          0,                              // Latest block timestamp in seconds (only used if difficulty already locked in)
+        ],
+        { // Expected Result
+          progressPercent: 71.97420634920636,
+          difficultyChange: 8.512745140778843,
+          estimatedRetargetDate: 1708004001715,
+          remainingBlocks: 565,
+          remainingTime: 312620715,
+          previousRetarget: 7.333505241141637,
+          previousTime: 1706888526,
+          nextRetargetHeight: 830592,
+          timeAvg: 553311,
+          adjustedTimeAvg: 553311,
+          timeOffset: 0,
+          expectedBlocks: 1338.0916666666667,
+        },
+      ],
+      [ // Vector 2 (within quarter-epoch overlap)
+        [ // Inputs
+          dt('2022-08-18T11:07:00.000Z'), // Last DA time (in seconds)
+          dt('2022-08-16T03:16:54.000Z'), // timestamp of 504 blocks ago (in seconds)
+          dt('2022-08-19T14:03:53.000Z'), // Current time (now) (in seconds)
+          750134,                         // Current block height
+          0.6280047707459726,             // Previous retarget % (Passed through)
+          'mainnet',                      // Network (if testnet, next value is non-zero)
+          0,                              // Latest block timestamp in seconds (only used if difficulty already locked in)
+        ],
+        { // Expected Result
+          progressPercent: 9.027777777777777,
+          difficultyChange: 1.0420538959004633,
+          estimatedRetargetDate: 1662009048328,
+          remainingBlocks: 1834,
+          remainingTime: 1091215328,
+          previousRetarget: 0.6280047707459726,
+          previousTime: 1660820820,
+          nextRetargetHeight: 751968,
+          timeAvg: 533038,
+          adjustedTimeAvg: 594992,
+          timeOffset: 0,
+          expectedBlocks: 161.68833333333333,
+        },
+      ],
+      [ // Vector 3 (testnet)
+        [ // Inputs
+          dt('2022-08-18T11:07:00.000Z'), // Last DA time (in seconds)
+          dt('2022-08-16T03:16:54.000Z'), // timestamp of 504 blocks ago (in seconds)
+          dt('2022-08-19T14:03:53.000Z'), // Current time (now) (in seconds)
+          750134,                         // Current block height
+          0.6280047707459726,             // Previous retarget % (Passed through)
+          'testnet',                      // Network
+          dt('2022-08-19T13:52:46.000Z'), // Latest block timestamp in seconds
+        ],
+        { // Expected Result is same other than timeOffset
+          progressPercent: 9.027777777777777,
+          difficultyChange: 1.0420538959004633,
+          estimatedRetargetDate: 1662009048328,
+          remainingBlocks: 1834,
+          remainingTime: 1091215328,
+          previousTime: 1660820820,
+          previousRetarget: 0.6280047707459726,
+          nextRetargetHeight: 751968,
+          timeAvg: 533038,
+          adjustedTimeAvg: 594992,
+          timeOffset: -667000, // 11 min 7 seconds since last block (testnet only)
+          // If we add time avg to abs(timeOffset) it makes exactly 1200000 ms, or 20 minutes
+          expectedBlocks: 161.68833333333333,
+        },
+      ],
+      [ // Vector 4 (mainnet lock-in (epoch ending 788255))
+        [ // Inputs
+          dt('2023-04-20T09:57:33.000Z'), // Last DA time (in seconds)
+          dt('2022-08-16T03:16:54.000Z'), // timestamp of 504 blocks ago (in seconds)
+          dt('2023-05-04T14:54:09.000Z'), // Current time (now) (in seconds)
+          788255,                         // Current block height
+          1.7220298879531821,             // Previous retarget % (Passed through)
+          'mainnet',                      // Network (if testnet, next value is non-zero)
+          dt('2023-05-04T14:54:26.000Z'), // Latest block timestamp in seconds
+        ],
+        { // Expected Result
+          progressPercent: 99.95039682539682,
+          difficultyChange: -1.4512637555574193,
+          estimatedRetargetDate: 1683212658129,
+          remainingBlocks: 1,
+          remainingTime: 609129,
+          previousRetarget: 1.7220298879531821,
+          previousTime: 1681984653,
+          nextRetargetHeight: 788256,
+          timeAvg: 609129,
+          adjustedTimeAvg: 609129,
+          timeOffset: 0,
+          expectedBlocks: 2045.66,
+        },
+      ],
+    ] as [[number, number, number, number, number, string, number], DifficultyAdjustment][];
+
+    for (const vector of vectors) {
+      const result = calcDifficultyAdjustment(...vector[0]);
+      // previousRetarget is passed through untouched
+      expect(result.previousRetarget).toStrictEqual(vector[0][4]);
+      expect(result).toStrictEqual(vector[1]);
+    }
+  });
+
+  test('should calculate Difficulty change from bits fields of two blocks', () => {
+    // Check same exponent + check min max for output
+    expect(calcBitsDifference(0x1d000200, 0x1d000100)).toEqual(100);
+    expect(calcBitsDifference(0x1d000400, 0x1d000100)).toEqual(300);
+    expect(calcBitsDifference(0x1d000800, 0x1d000100)).toEqual(300); // Actually 700
+    expect(calcBitsDifference(0x1d000100, 0x1d000200)).toEqual(-50);
+    expect(calcBitsDifference(0x1d000100, 0x1d000400)).toEqual(-75);
+    expect(calcBitsDifference(0x1d000100, 0x1d000800)).toEqual(-75); // Actually -87.5
+    // Check new higher exponent
+    expect(calcBitsDifference(0x1c000200, 0x1d000001)).toEqual(100);
+    expect(calcBitsDifference(0x1c000400, 0x1d000001)).toEqual(300);
+    expect(calcBitsDifference(0x1c000800, 0x1d000001)).toEqual(300);
+    expect(calcBitsDifference(0x1c000100, 0x1d000002)).toEqual(-50);
+    expect(calcBitsDifference(0x1c000100, 0x1d000004)).toEqual(-75);
+    expect(calcBitsDifference(0x1c000100, 0x1d000008)).toEqual(-75);
+    // Check new lower exponent
+    expect(calcBitsDifference(0x1d000002, 0x1c000100)).toEqual(100);
+    expect(calcBitsDifference(0x1d000004, 0x1c000100)).toEqual(300);
+    expect(calcBitsDifference(0x1d000008, 0x1c000100)).toEqual(300);
+    expect(calcBitsDifference(0x1d000001, 0x1c000200)).toEqual(-50);
+    expect(calcBitsDifference(0x1d000001, 0x1c000400)).toEqual(-75);
+    expect(calcBitsDifference(0x1d000001, 0x1c000800)).toEqual(-75);
+    // Check error when exponents are too far apart
+    expect(() => calcBitsDifference(0x1d000001, 0x1a000800)).toThrow(
+      /Impossible exponent difference/
+    );
+    // Check invalid inputs
+    expect(() => calcBitsDifference(0x7f000001, 0x1a000800)).toThrow(
+      /Invalid bits/
+    );
+    expect(() => calcBitsDifference(0, 0x1a000800)).toThrow(/Invalid bits/);
+    expect(() => calcBitsDifference(100.2783, 0x1a000800)).toThrow(
+      /Invalid bits/
+    );
+    expect(() => calcBitsDifference(0x00800000, 0x1a000800)).toThrow(
+      /Invalid bits/
+    );
+    expect(() => calcBitsDifference(0x1c000000, 0x1a000800)).toThrow(
+      /Invalid bits/
+    );
+  });
+});

backend/src/__tests__/api/test-data/non-standard-txs.json

@@ -0,0 +1,52 @@
+[
+  {
+    "txid": "50136231cb7eeeffb17fc41d1cca213426abe5bf3760e3d6421cad0c0edad367",
+    "version": 1,
+    "locktime": 0,
+    "vin": [
+        {
+            "txid": "c7f86fb7b830124057475b282809f3474ef3565daa3de0b599980fb9e84ab019",
+            "vout": 4217,
+            "prevout": {
+                "scriptpubkey": "001466197b5eadd8067ec194a457e1044b6d1fbdd3b3",
+                "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 66197b5eadd8067ec194a457e1044b6d1fbdd3b3",
+                "scriptpubkey_type": "v0_p2wpkh",
+                "scriptpubkey_address": "bc1qvcvhkh4dmqr8asv553t7zpztd50mm5ang4na33",
+                "value": 106
+            },
+            "scriptsig": "",
+            "scriptsig_asm": "",
+            "witness": [
+                "3043021f2af6060a142c6cfd7428adad6a50745d2424813d7ced5c0bbcca85e70de1be022021440ca1c8c3ed49ecd1b64dca6911adcd430c5d3dd60d77ffe0072953999f5b01",
+                "02ead5c34e3d2c506574b562f857576e11380b6ba15d9f0ad7b7303fdaa9c1513d"
+            ],
+            "is_coinbase": false,
+            "sequence": 4294967295
+        }
+    ],
+    "vout": [
+        {
+            "scriptpubkey": "6a023a29",
+            "scriptpubkey_asm": "OP_RETURN OP_PUSHBYTES_2 3a29",
+            "scriptpubkey_type": "op_return",
+            "value": 0
+        },
+        {
+            "scriptpubkey": "6a036d7648",
+            "scriptpubkey_asm": "OP_RETURN OP_PUSHBYTES_3 6d7648",
+            "scriptpubkey_type": "op_return",
+            "value": 0
+        }
+    ],
+    "size": 186,
+    "weight": 420,
+    "sigops": 1,
+    "fee": 106,
+    "status": {
+        "confirmed": true,
+        "block_height": 836361,
+        "block_hash": "0000000000000000000341cc26cda4af82cd25f7063c448772228cbf2836915b",
+        "block_time": 1711448028
+    }
+  }
+]

backend/src/__tests__/api/test-data/transactions-random.json

@@ -0,0 +1,600 @@
+[
+    {
+        "txid": "13f007241d78e8b0b4e57d2ae3fd37bcfe3226534d7cadeba5a549860d960db0",
+        "version": 2,
+        "locktime": 0,
+        "vin": [
+            {
+                "txid": "cb8f206f4e88bec97107089f3e9e61d50cde53d4541992ae19759b71103cf75c",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "0014fd6d15ff832c12f1ff04a5ccd5039f7227b260bd",
+                    "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 fd6d15ff832c12f1ff04a5ccd5039f7227b260bd",
+                    "scriptpubkey_type": "v0_p2wpkh",
+                    "scriptpubkey_address": "bc1ql4k3tlur9sf0rlcy5hxd2qulwgnmyc9akehvth",
+                    "value": 610677
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "304302205c430b36ebd2bb327951d83440af1f58f127871b2baada4c4dde2bc0b6721f56021f3445099f1a40e35baeda32e8e3727b505ffba0d882b11f498c7762f4184e9901",
+                    "0236b5edd4fbbcfb045960e42ec8a9968944084785932e32940e8cd2583b37da67"
+                ],
+                "is_coinbase": false,
+                "sequence": 2147483648
+            }
+        ],
+        "vout": [
+            {
+                "scriptpubkey": "76a9149d32ef812385f3811634e0c0117dd153a5de10a488ac",
+                "scriptpubkey_asm": "OP_DUP OP_HASH160 OP_PUSHBYTES_20 9d32ef812385f3811634e0c0117dd153a5de10a4 OP_EQUALVERIFY OP_CHECKSIG",
+                "scriptpubkey_type": "p2pkh",
+                "scriptpubkey_address": "1FLC7Bag7okAkKPCyZbgZZg3Hh1EuGZ5Rd",
+                "value": 344697
+            },
+            {
+                "scriptpubkey": "00147dee8a7a38abbfb00dbfba365c8d6712934cc491",
+                "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 7dee8a7a38abbfb00dbfba365c8d6712934cc491",
+                "scriptpubkey_type": "v0_p2wpkh",
+                "scriptpubkey_address": "bc1q0hhg573c4wlmqrdlhgm9ert8z2f5e3y3lf9hvx",
+                "value": 265396
+            }
+        ],
+        "size": 224,
+        "weight": 572,
+        "fee": 584,
+        "status": {
+            "confirmed": false
+        },
+        "order": 2953680397,
+        "vsize": 143,
+        "adjustedVsize": 143,
+        "sigops": 5,
+        "feePerVsize": 4.083916083916084,
+        "adjustedFeePerVsize": 4.083916083916084,
+        "effectiveFeePerVsize": 4.083916083916084,
+        "firstSeen": 1691222538,
+        "uid": 526973,
+        "inputs": [
+            526728
+        ],
+        "position": {
+            "block": 7,
+            "vsize": 21429708.5
+        },
+        "bestDescendant": null,
+        "cpfpChecked": true
+    },
+    {
+        "txid": "8e89b20f8a7fadb0e4cdbe57a00eee224f5076bac5387fc276916724e7c4a16a",
+        "version": 2,
+        "locktime": 800571,
+        "vin": [
+            {
+                "txid": "35e16762459539f3a8e52c5dee6a9ccaa9e9268efed33aa2c6e1b7805e849f24",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "0014d4f16ef275b3e1c4a4ecbef55a164933e0f6460f",
+                    "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 d4f16ef275b3e1c4a4ecbef55a164933e0f6460f",
+                    "scriptpubkey_type": "v0_p2wpkh",
+                    "scriptpubkey_address": "bc1q6nckaun4k0suff8vhm6459jfx0s0v3s0ff4ukl",
+                    "value": 1528924
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "3044022019008b26e885bb43da25a11ffac147a057722072eedb68411f114f6e7eb82ebc02201b618264bb97756b88fc3bbc365b73044ac18b33b1067e31cfd5bcd0f50ed2c701",
+                    "039b71145070bd3e8af28e27fa577f2e12ab6bb4e212d3eeaef08b4bc39e8cbc13"
+                ],
+                "is_coinbase": false,
+                "sequence": 4294967293
+            },
+            {
+                "txid": "67c27ed0f767526234bcd5f795a31fab8ec4d0251bf12c68f2746951f4110d90",
+                "vout": 3,
+                "prevout": {
+                    "scriptpubkey": "0014a7c3d613b321375054b2ac9b6114367bc034ad6f",
+                    "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 a7c3d613b321375054b2ac9b6114367bc034ad6f",
+                    "scriptpubkey_type": "v0_p2wpkh",
+                    "scriptpubkey_address": "bc1q5lpavyanyym4q49j4jdkz9pk00qrftt0yqzvk3",
+                    "value": 436523
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "304402204e67285fc656bc45ed082499b076d5dba2fa21d0d7e64a0ae52b19d69a11760002200f037d81ee540b74397844513b72b08ed92b06db76bd20b08f7a0a3b36ab13d501",
+                    "02a3ebae85f0225b6fbb5ff060afce683a4683507a57544605a29ee7d287e591b4"
+                ],
+                "is_coinbase": false,
+                "sequence": 4294967293
+            },
+            {
+                "txid": "21c38fb9a2521e438c614f53b19ddd7a5594bcc4b77480e762fd4b702fad3374",
+                "vout": 1,
+                "prevout": {
+                    "scriptpubkey": "00149660e34ef88106536c816c037b5b28dd64a812e2",
+                    "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 9660e34ef88106536c816c037b5b28dd64a812e2",
+                    "scriptpubkey_type": "v0_p2wpkh",
+                    "scriptpubkey_address": "bc1qjeswxnhcsyr9xmypdsphkkegm4j2syhztgzxv4",
+                    "value": 758149
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "3044022021b556f0aa99329076bcc435338aceaf534963efcab306931b1b2b0461e16e0c02203a78942a3745c4da656bddfd8cf16b85dc04d652904e88682127cdd9ca63339001",
+                    "0298963be4a8f66aca9fcf1c6dc95547aeaa82347543190c91e094c2321142b9f0"
+                ],
+                "is_coinbase": false,
+                "sequence": 4294967293
+            },
+            {
+                "txid": "aa998dbae65240a7386bf7d468459551d99c3de8e2f9057ff5f2d38e17daf788",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "00147bb7413a39943b21ded98ad5e6ad7a222d273e17",
+                    "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 7bb7413a39943b21ded98ad5e6ad7a222d273e17",
+                    "scriptpubkey_type": "v0_p2wpkh",
+                    "scriptpubkey_address": "bc1q0wm5zw3ejsajrhke3t27dtt6ygkjw0sh9lltg6",
+                    "value": 1067200
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "304402205e2269f7d4ee0513b34354c38e920aef2dabac6f4350afb2dd105ff3ee43ae7b02202870322f2cb85cb0b2b0e38152f018bfff271dc3ec5aed0515854d0b259aaf3d01",
+                    "03b87320cf3263a644a0d3f89c1b4a7304d9dfda9eb8c891560716abcb73e88b99"
+                ],
+                "is_coinbase": false,
+                "sequence": 4294967293
+            },
+            {
+                "txid": "230253d195d779d4688ba16993985cd27b2e7a687d8b889b3bc63f19ece36f20",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "001439647bd997819d12dfc72b0fb9ff9ffcb84946f8",
+                    "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 39647bd997819d12dfc72b0fb9ff9ffcb84946f8",
+                    "scriptpubkey_type": "v0_p2wpkh",
+                    "scriptpubkey_address": "bc1q89j8hkvhsxw39h789v8mnlulljuyj3hc9zve97",
+                    "value": 361950
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "304402204f7ca868bb9b92a07fecdc6b9dd56e4e1d007ca1405952f98ed6bc416345b5f2022055320a97791417abf6628fcf6513ac5785b06c630f854d8595e96ea06c3841d301",
+                    "03a3ffe8e3ef2eea129b227e9658164bae0a6d21c17da6de9973ba34d9e04b21a0"
+                ],
+                "is_coinbase": false,
+                "sequence": 4294967293
+            },
+            {
+                "txid": "670771e265a0b62dbd3c1fec2b865177eaf0bafd0ae49dd40a1c9fcd9a847a81",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "0014d45d1b0022c7387e42c5452ced561bdb8fd4b521",
+                    "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 d45d1b0022c7387e42c5452ced561bdb8fd4b521",
+                    "scriptpubkey_type": "v0_p2wpkh",
+                    "scriptpubkey_address": "bc1q63w3kqpzcuu8usk9g5kw64smmw8afdfpxmc2m0",
+                    "value": 453275
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "3044022071312921800441903b2099e723add8702dd0f92ec11526ff87acf6967ec64cbd02203deabe7ed56d5daaa9a95c5a607b1ab705ff1c46bc6984a6dca120e63a91768601",
+                    "0257302ac8d9c4c8f9b1744f19bb432359326b9cc7bdddeeab9202749a6d92be58"
+                ],
+                "is_coinbase": false,
+                "sequence": 4294967293
+            },
+            {
+                "txid": "0af82159eee2b69242f2ff032636e410b67ec1ace52e55fb0d20ed814cd64803",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "001459e4d6bfefc6b45f955a69c4aeca26348e9d54ed",
+                    "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 59e4d6bfefc6b45f955a69c4aeca26348e9d54ed",
+                    "scriptpubkey_type": "v0_p2wpkh",
+                    "scriptpubkey_address": "bc1qt8jdd0l0c669l926d8z2aj3xxj8f648dtyn7tc",
+                    "value": 439961
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "3044022027540322e92c23c5513aa2587e7feb56a8ce82f879269d6b3cbd425634b44f8e022045572dee7262b02130bfe32d8aa8abbfaa64e101abfc819bba5380c78876692d01",
+                    "03fe02262d87f4a5289d3dd66e3d9a74cd49fa1cad0249284a7451896a827249a5"
+                ],
+                "is_coinbase": false,
+                "sequence": 4294967293
+            },
+            {
+                "txid": "68cf9c784870a4f888f044755f7ce318557f652461db8ef887d279672f186018",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "001454822b2d5d52597a78b630921cf439a41e32f2f9",
+                    "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 54822b2d5d52597a78b630921cf439a41e32f2f9",
+                    "scriptpubkey_type": "v0_p2wpkh",
+                    "scriptpubkey_address": "bc1q2jpzkt2a2fvh579kxzfpeape5s0r9uhewhl5n4",
+                    "value": 227639
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "304402203ad511d6a8730748b8828bc38897d360451adf620ebdc1d229c08c097c80bef202202f50c793d95b5200cf2258e03896a3be7720df0eb3b8c810c86db74341a7e83e01",
+                    "0294992e9f4546e6e119741f908411ae531e9d1ff732d69b4dff8172aaf2a4b216"
+                ],
+                "is_coinbase": false,
+                "sequence": 4294967293
+            },
+            {
+                "txid": "793f01dfdb19bf41f958fd917c16d9c4dd5d5e1a5c0434bfdb367212659d1b5b",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "0014f54edf8ae647b5300e2674523254e923d93d169f",
+                    "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 f54edf8ae647b5300e2674523254e923d93d169f",
+                    "scriptpubkey_type": "v0_p2wpkh",
+                    "scriptpubkey_address": "bc1q748dlzhxg76nqr3xw3fry48fy0vn695lvhlkxv",
+                    "value": 227070
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "304402206e807ab616f4f2887ba703ae744d856142d9aca8128698419bbb67fb4fad8177022060fc65c7cd66baa88ad1e1d317a6edd5f6cb52fe8bff6e5405ffa1acf9d945d901",
+                    "02a0ad0167c6e9edf62677404d74d3b80ea276e47e758ffaa6ca17bd65ac79f7aa"
+                ],
+                "is_coinbase": false,
+                "sequence": 4294967293
+            }
+        ],
+        "vout": [
+            {
+                "scriptpubkey": "00148a5c45ccfc29d209940d94525e2edb7743a1ad8a",
+                "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 8a5c45ccfc29d209940d94525e2edb7743a1ad8a",
+                "scriptpubkey_type": "v0_p2wpkh",
+                "scriptpubkey_address": "bc1q3fwytn8u98fqn9qdj3f9utkmwap6rtv2ym33zm",
+                "value": 5500000
+            }
+        ],
+        "size": 1375,
+        "weight": 2605,
+        "fee": 691,
+        "status": {
+            "confirmed": false
+        },
+        "order": 1788986599,
+        "vsize": 651,
+        "adjustedVsize": 651.25,
+        "sigops": 9,
+        "feePerVsize": 1.0610364683301343,
+        "adjustedFeePerVsize": 1.0610364683301343,
+        "effectiveFeePerVsize": 1.0610364683301343,
+        "firstSeen": 1691163298,
+        "uid": 120494,
+        "inputs": [],
+        "position": {
+            "block": 7,
+            "vsize": 93780091.5
+        },
+        "bestDescendant": null,
+        "cpfpChecked": true
+    },
+    {
+        "txid": "20b984492b5264162a4c92c9a34bc7fa08b67d669de7b4c5982ad3cb28aaecf6",
+        "version": 2,
+        "locktime": 0,
+        "vin": [
+            {
+                "txid": "3adda6afd547193793c248e667c2b7dbf26d705003de65e3a25e5be698286aef",
+                "vout": 2,
+                "prevout": {
+                    "scriptpubkey": "0014989cf12774fc705609610c7b9419f2d1c4807644",
+                    "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 989cf12774fc705609610c7b9419f2d1c4807644",
+                    "scriptpubkey_type": "v0_p2wpkh",
+                    "scriptpubkey_address": "bc1qnzw0zfm5l3c9vztpp3aegx0j68zgqajyffr2r6",
+                    "value": 27619
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "304402205d7f1e0d928982645c2bcc4c730c4545c382d6520c2a14eebc71594702cd06b302200511d452ce51c79017536f50acb115eefe7c04506ad12b9307d2b5d56b999beb01",
+                    "03716cb4f0430fe69c596a12c6680c55803150645989b406772838d548cde7cca5"
+                ],
+                "is_coinbase": false,
+                "sequence": 4294967295
+            }
+        ],
+        "vout": [
+            {
+                "scriptpubkey": "6a5d0614c0a2331441",
+                "scriptpubkey_asm": "OP_RETURN OP_PUSHNUM_13 OP_PUSHBYTES_6 14c0a2331441",
+                "scriptpubkey_type": "op_return",
+                "value": 0
+            },
+            {
+                "scriptpubkey": "5114d71c6c3ea7ba7e6ee477a0bfd82c20c78997882c",
+                "scriptpubkey_asm": "OP_PUSHNUM_1 OP_PUSHBYTES_20 d71c6c3ea7ba7e6ee477a0bfd82c20c78997882c",
+                "scriptpubkey_type": "unknown",
+                "scriptpubkey_address": "bc1p6uwxc048hflxaerh5zlastpqc7ye0zpvq7gq2a",
+                "value": 546
+            },
+            {
+                "scriptpubkey": "0014989cf12774fc705609610c7b9419f2d1c4807644",
+                "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 989cf12774fc705609610c7b9419f2d1c4807644",
+                "scriptpubkey_type": "v0_p2wpkh",
+                "scriptpubkey_address": "bc1qnzw0zfm5l3c9vztpp3aegx0j68zgqajyffr2r6",
+                "value": 23073
+            }
+        ],
+        "size": 240,
+        "weight": 633,
+        "sigops": 1,
+        "fee": 4000,
+        "status": {
+            "confirmed": true,
+            "block_height": 848136,
+            "block_hash": "00000000000000000002c69c7a3010fcd596c0c7451c23e7cd1f5e19ebf8ee6d",
+            "block_time": 1718517071
+        }
+    },
+    {
+        "txid": "b10c0000004da5a9d1d9b4ae32e09f0b3e62d21a5cce5428d4ad714fb444eb5d",
+        "version": 1,
+        "locktime": 1231006505,
+        "vin": [
+            {
+                "txid": "d46a24962c1d7bd6e87d80570c6a53413eaf30d7fde7f52347f13645ae53969b",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "41049434a2dd7c5b82df88f578f8d7fd14e8d36513aaa9c003eb5bd6cb56065e44b7e0227139e8a8e68e7de0a4ed32b8c90edc9673b8a7ea541b52f2a22196f7b8cfac",
+                    "scriptpubkey_asm": "OP_PUSHBYTES_65 049434a2dd7c5b82df88f578f8d7fd14e8d36513aaa9c003eb5bd6cb56065e44b7e0227139e8a8e68e7de0a4ed32b8c90edc9673b8a7ea541b52f2a22196f7b8cf OP_CHECKSIG",
+                    "scriptpubkey_type": "p2pk",
+                    "value": 6102
+                },
+                "scriptsig": "473044022004f027ae0b19bb7a7aa8fcdf135f1da769d087342020359ef4099a9f0f0ba4ec02206a83a9b78df3fed89a3b6052e69963e1fb08d8f6d17d945e43b51b5214aa41e601",
+                "scriptsig_asm": "OP_PUSHBYTES_71 3044022004f027ae0b19bb7a7aa8fcdf135f1da769d087342020359ef4099a9f0f0ba4ec02206a83a9b78df3fed89a3b6052e69963e1fb08d8f6d17d945e43b51b5214aa41e601",
+                "is_coinbase": false,
+                "sequence": 20090103
+            },
+            {
+                "txid": "cb9b47ac04023b29fb633a8ef04af351ac9fd74c57c9a2163f683516274767e3",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "76a914bbb1f7d0f7e15ac088af9bafe25aaac1a59832d088ac",
+                    "scriptpubkey_asm": "OP_DUP OP_HASH160 OP_PUSHBYTES_20 bbb1f7d0f7e15ac088af9bafe25aaac1a59832d0 OP_EQUALVERIFY OP_CHECKSIG",
+                    "scriptpubkey_type": "p2pkh",
+                    "scriptpubkey_address": "1J7SZJry7CX4zWdH3P8E8UJjZrhcLEjJ39",
+                    "value": 1913
+                },
+                "scriptsig": "46304302204dc2939be89ab6626457fff40aec2cc4e6213e64bcb4d2c43bf6b49358ff638c021f33d2f8fdf6d54a2c82bb7cddc62becc2cbbaca6fd7f3ec927ea975f29ad8510221028b98707adfd6f468d56c1a6067a6f0c7fef43afbacad45384017f8be93a18d40",
+                "scriptsig_asm": "OP_PUSHBYTES_70 304302204dc2939be89ab6626457fff40aec2cc4e6213e64bcb4d2c43bf6b49358ff638c021f33d2f8fdf6d54a2c82bb7cddc62becc2cbbaca6fd7f3ec927ea975f29ad85102 OP_PUSHBYTES_33 028b98707adfd6f468d56c1a6067a6f0c7fef43afbacad45384017f8be93a18d40",
+                "is_coinbase": false,
+                "sequence": 20081031
+            },
+            {
+                "txid": "cb9b47ac04023b29fb633a8ef04af351ac9fd74c57c9a2163f683516274767e3",
+                "vout": 1,
+                "prevout": {
+                    "scriptpubkey": "52210304e708d258a632ffb128a62ecf5eebd1904e505497d031619513afc8bca7858f2102b9dc03f1133e7cbc7eb311631acc2dbda908fb0f0fae095da2f4dd427f51308a4104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5f53ae",
+                    "scriptpubkey_asm": "OP_PUSHNUM_2 OP_PUSHBYTES_33 0304e708d258a632ffb128a62ecf5eebd1904e505497d031619513afc8bca7858f OP_PUSHBYTES_33 02b9dc03f1133e7cbc7eb311631acc2dbda908fb0f0fae095da2f4dd427f51308a OP_PUSHBYTES_65 04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5f OP_PUSHNUM_3 OP_CHECKMULTISIG",
+                    "scriptpubkey_type": "multisig",
+                    "value": 1971
+                },
+                "scriptsig": "00453042021e4f6ff73d7b304a5cbf3bb7738abb5f81a4af6335962134ce27a1cc45fec702201b95e3acb7db93257b20651cdcb79af66bf0bb86a8ae5b4e0a5df4e3f86787e2033b303802153b78ce563f89a0ed9414f5aa28ad0d96d6795f9c63021f34793e2878497561e7616291ebdda3024b681cdacc8b863b5b0804cd30c2a481",
+                "scriptsig_asm": "OP_0 OP_PUSHBYTES_69 3042021e4f6ff73d7b304a5cbf3bb7738abb5f81a4af6335962134ce27a1cc45fec702201b95e3acb7db93257b20651cdcb79af66bf0bb86a8ae5b4e0a5df4e3f86787e203 OP_PUSHBYTES_59 303802153b78ce563f89a0ed9414f5aa28ad0d96d6795f9c63021f34793e2878497561e7616291ebdda3024b681cdacc8b863b5b0804cd30c2a481",
+                "is_coinbase": false,
+                "sequence": 19750504
+            },
+            {
+                "txid": "45e1cb33599acb071810ccc801b71bd7610865f5b899492946ab1bfbcb61cad6",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "a91419f0b86f61606c6eb51b217698ca7e8bff1e398b87",
+                    "scriptpubkey_asm": "OP_HASH160 OP_PUSHBYTES_20 19f0b86f61606c6eb51b217698ca7e8bff1e398b OP_EQUAL",
+                    "scriptpubkey_type": "p2sh",
+                    "scriptpubkey_address": "344BBtYkhaCXgA7oYSXASUfh4bFieiponG",
+                    "value": 2140
+                },
+                "scriptsig": "00443041021d1313459a48bd1d0628eec635495f793e970729684394f9b814d2b24012022050be6d9918444e283da0136884f8311ec465d0fed2f8d24b75a8485ebdc13aea013a303702153b78ce563f89a0ed9414f5aa28ad0d96d6795f9c63021e78644ba72eab69fefb5fe50700671bfb91dda699f72ffbb325edc6a3c4ef8239303602153b78ce563f89a0ed9414f5aa28ad0d96d6795f9c63021d2c2db104e70720c39af43b6ba3edd930c26e0818aa59ff9c886281d8ba834ced532103e0a220d36f6f7ed5f3f58c279d055707c454135baf18fd00d798fec3cb52dfbc2103cf689db9313b9f7fc0b984dd9cac750be76041b392919b06f6bf94813da34cd421027f8af2eb6e904deddaa60d5af393d430575eb35e4dfd942a8a5882734b078906410411db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5cb2e0eaddfb84ccf9744464f82e160bfa9b8b64f9d4c03f999b8643f656b412a34104ae1a62fe09c5f51b13905f07f06b99a2f7159b2225f374cd378d71302fa28414e7aab37397f554a7df5f142c21c1b7303b8a0626f1baded5c72a704f7e6cd84c55ae",
+                "scriptsig_asm": "OP_0 OP_PUSHBYTES_68 3041021d1313459a48bd1d0628eec635495f793e970729684394f9b814d2b24012022050be6d9918444e283da0136884f8311ec465d0fed2f8d24b75a8485ebdc13aea01 OP_PUSHBYTES_58 303702153b78ce563f89a0ed9414f5aa28ad0d96d6795f9c63021e78644ba72eab69fefb5fe50700671bfb91dda699f72ffbb325edc6a3c4ef82 OP_PUSHBYTES_57 303602153b78ce563f89a0ed9414f5aa28ad0d96d6795f9c63021d2c2db104e70720c39af43b6ba3edd930c26e0818aa59ff9c886281d8ba83 OP_PUSHDATA1 532103e0a220d36f6f7ed5f3f58c279d055707c454135baf18fd00d798fec3cb52dfbc2103cf689db9313b9f7fc0b984dd9cac750be76041b392919b06f6bf94813da34cd421027f8af2eb6e904deddaa60d5af393d430575eb35e4dfd942a8a5882734b078906410411db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5cb2e0eaddfb84ccf9744464f82e160bfa9b8b64f9d4c03f999b8643f656b412a34104ae1a62fe09c5f51b13905f07f06b99a2f7159b2225f374cd378d71302fa28414e7aab37397f554a7df5f142c21c1b7303b8a0626f1baded5c72a704f7e6cd84c55ae",
+                "is_coinbase": false,
+                "sequence": 16,
+                "inner_redeemscript_asm": "OP_PUSHNUM_3 OP_PUSHBYTES_33 03e0a220d36f6f7ed5f3f58c279d055707c454135baf18fd00d798fec3cb52dfbc OP_PUSHBYTES_33 03cf689db9313b9f7fc0b984dd9cac750be76041b392919b06f6bf94813da34cd4 OP_PUSHBYTES_33 027f8af2eb6e904deddaa60d5af393d430575eb35e4dfd942a8a5882734b078906 OP_PUSHBYTES_65 0411db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5cb2e0eaddfb84ccf9744464f82e160bfa9b8b64f9d4c03f999b8643f656b412a3 OP_PUSHBYTES_65 04ae1a62fe09c5f51b13905f07f06b99a2f7159b2225f374cd378d71302fa28414e7aab37397f554a7df5f142c21c1b7303b8a0626f1baded5c72a704f7e6cd84c OP_PUSHNUM_5 OP_CHECKMULTISIG"
+            },
+            {
+                "txid": "cb9b47ac04023b29fb633a8ef04af351ac9fd74c57c9a2163f683516274767e3",
+                "vout": 2,
+                "prevout": {
+                    "scriptpubkey": "a9143b13a1f71c20c799d86bb624b3898c826d6c82da87",
+                    "scriptpubkey_asm": "OP_HASH160 OP_PUSHBYTES_20 3b13a1f71c20c799d86bb624b3898c826d6c82da OP_EQUAL",
+                    "scriptpubkey_type": "p2sh",
+                    "scriptpubkey_address": "375PJxsKRtAq4WoS6u82jvgZW94R8Wx3iH",
+                    "value": 5139
+                },
+                "scriptsig": "1600149b27f072e4b972927c445d1946162a550b0914d8",
+                "scriptsig_asm": "OP_PUSHBYTES_22 00149b27f072e4b972927c445d1946162a550b0914d8",
+                "witness": [
+                    "3040021c23902a01d4c5cff2c33c8bdb778a5aadea78a9a0d6d4db60aaa0fba1022069237d9dbf2db8cff9c260ba71250493682d01a746f4a45c5c7ea386e56d2bc902",
+                    "0240187acd3e2fd3d8e1acffefa85907b6550730c24f78dfd3301c829fc4daf3cc"
+                ],
+                "is_coinbase": false,
+                "sequence": 141,
+                "inner_redeemscript_asm": "OP_0 OP_PUSHBYTES_20 9b27f072e4b972927c445d1946162a550b0914d8"
+            },
+            {
+                "txid": "cb9b47ac04023b29fb633a8ef04af351ac9fd74c57c9a2163f683516274767e3",
+                "vout": 3,
+                "prevout": {
+                    "scriptpubkey": "a914a3c0698f2300c7b2e8107d4c9c988e642110039087",
+                    "scriptpubkey_asm": "OP_HASH160 OP_PUSHBYTES_20 a3c0698f2300c7b2e8107d4c9c988e6421100390 OP_EQUAL",
+                    "scriptpubkey_type": "p2sh",
+                    "scriptpubkey_address": "3GcrZrbUuvE4UtUdSbKTXcRnTqmfMdyMAC",
+                    "value": 3220
+                },
+                "scriptsig": "220020a18160de7291554f349c7d5cbee4ab97fb542e94cf302ce8d7e9747e4188ca75",
+                "scriptsig_asm": "OP_PUSHBYTES_34 0020a18160de7291554f349c7d5cbee4ab97fb542e94cf302ce8d7e9747e4188ca75",
+                "witness": [
+                    "303f021c65aee6696e80be6e14545cfd64b44f17b0514c150eefdb090c0f0bd9021f3fef4aa95c252a225622aba99e4d5af5a6fe40d177acd593e64cf2f8557ccc03",
+                    "03b55c6f0749e0f3e2caeca05f68e3699f1b3c62a550730f704985a6a9aae437a1",
+                    "76a914db865fd920959506111079995f1e4017b489bfe38763ac6721024d560f7f5d28aae5e1a8aa2b7ba615d7fc48e4ea27e5d27336e6a8f5fa0f5c8c7c820120876475527c2103443e8834fa7d79d7b5e95e0e9d0847f6b03ac3ea977979858b4104947fca87ca52ae67a91446c3747322b220fdb925c9802f0e949c1feab99988ac6868"
+                ],
+                "is_coinbase": false,
+                "sequence": 3735928559,
+                "inner_redeemscript_asm": "OP_0 OP_PUSHBYTES_32 a18160de7291554f349c7d5cbee4ab97fb542e94cf302ce8d7e9747e4188ca75",
+                "inner_witnessscript_asm": "OP_DUP OP_HASH160 OP_PUSHBYTES_20 db865fd920959506111079995f1e4017b489bfe3 OP_EQUAL OP_IF OP_CHECKSIG OP_ELSE OP_PUSHBYTES_33 024d560f7f5d28aae5e1a8aa2b7ba615d7fc48e4ea27e5d27336e6a8f5fa0f5c8c OP_SWAP OP_SIZE OP_PUSHBYTES_1 20 OP_EQUAL OP_NOTIF OP_DROP OP_PUSHNUM_2 OP_SWAP OP_PUSHBYTES_33 03443e8834fa7d79d7b5e95e0e9d0847f6b03ac3ea977979858b4104947fca87ca OP_PUSHNUM_2 OP_CHECKMULTISIG OP_ELSE OP_HASH160 OP_PUSHBYTES_20 46c3747322b220fdb925c9802f0e949c1feab999 OP_EQUALVERIFY OP_CHECKSIG OP_ENDIF OP_ENDIF"
+            },
+            {
+                "txid": "cb9b47ac04023b29fb633a8ef04af351ac9fd74c57c9a2163f683516274767e3",
+                "vout": 4,
+                "prevout": {
+                    "scriptpubkey": "0014c0ca6e754e65d3ba59112d7abc33e500c00ecfa7",
+                    "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 c0ca6e754e65d3ba59112d7abc33e500c00ecfa7",
+                    "scriptpubkey_type": "v0_p2wpkh",
+                    "scriptpubkey_address": "bc1qcr9xua2wvhfm5kg394atcvl9qrqqana8rrmy8h",
+                    "value": 17144
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "303e021c11f60486afd0f5d6573603fb2076ef2f676455b92ada257d2f25558a021e317719c946f951d49bf4df4285a618629cd9e554fcbf787c319a0c4dd22601",
+                    "032467f24cc31664f0cf34ff8d5cbb590888ddc1dcfec724a32ae3dd5338b8508e"
+                ],
+                "is_coinbase": false,
+                "sequence": 21000000
+            },
+            {
+                "txid": "637db3928a8fb1b22b81f92dc738ee7637e5b172d650363d0b327429578bd001",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "0020a9530a167fcada672c142ee636dcd171796e69ef8e37aa1f77f35c58edd7a357",
+                    "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_32 a9530a167fcada672c142ee636dcd171796e69ef8e37aa1f77f35c58edd7a357",
+                    "scriptpubkey_type": "v0_p2wsh",
+                    "scriptpubkey_address": "bc1q49fs59nletdxwtq59mnrdhx3w9uku6003cm658mh7dw93mwh5dts2w2kht",
+                    "value": 8149
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "303d021c32f9454db85cb1a4ca63a9883d4347c5e13f3654e884ae44e9efa3c8021d62f07fe452c06b084bc3e09afd3aac4039136549a465533bc1ca66967902",
+                    "01",
+                    "632102fd6db4de50399b2aa086edb23f8e140bbc823d6651e024a0eb871288068789cd67012ab27521034134a2bb35c3f83dab2489d96160741888b8b5589bb694dea6e7bc24486e9c6f68ac"
+                ],
+                "is_coinbase": false,
+                "sequence": 4190024921,
+                "inner_witnessscript_asm": "OP_IF OP_PUSHBYTES_33 02fd6db4de50399b2aa086edb23f8e140bbc823d6651e024a0eb871288068789cd OP_ELSE OP_PUSHBYTES_1 2a OP_CSV OP_DROP OP_PUSHBYTES_33 034134a2bb35c3f83dab2489d96160741888b8b5589bb694dea6e7bc24486e9c6f OP_ENDIF OP_CHECKSIG"
+            },
+            {
+                "txid": "0020db02df125062ebae5bacd189ebff22577b2817c1872be79a0d3ba3982c41",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "512071212ded0ff4c9b1b0c505d8012772e2dbe98a3cae7168377b950fb6b866a849",
+                    "scriptpubkey_asm": "OP_PUSHNUM_1 OP_PUSHBYTES_32 71212ded0ff4c9b1b0c505d8012772e2dbe98a3cae7168377b950fb6b866a849",
+                    "scriptpubkey_type": "v1_p2tr",
+                    "scriptpubkey_address": "bc1pwysjmmg07nymrvx9qhvqzfmjutd7nz3u4ecksdmmj58mdwrx4pysq6m68g",
+                    "value": 9001
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "d822f203827852998cad370232e8c57294540a5da51107fa26cf466bdd2b8b0b3d161999cc80aed8de7386a2bd5d5313aea159a231cc26fa53aaa702b7fa21ed"
+                ],
+                "is_coinbase": false,
+                "sequence": 341
+            },
+            {
+                "txid": "795741ecf9c431b14b1c8d2dd017d3978fd4f6452e91edf416f31ef9971206b4",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "512089ac120a490eee88db5588112f95f88093284c814f07c3ad943a7faefba2271a",
+                    "scriptpubkey_asm": "OP_PUSHNUM_1 OP_PUSHBYTES_32 89ac120a490eee88db5588112f95f88093284c814f07c3ad943a7faefba2271a",
+                    "scriptpubkey_type": "v1_p2tr",
+                    "scriptpubkey_address": "bc1p3xkpyzjfpmhg3k643qgjl90cszfjsnypfuru8tv58fl6a7azyudqkcu66k",
+                    "value": 19953
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "fe6eb715dceffefc067fdc787d250a9a9116682d216f6356ea38fc1f112bd74995faa90315e81981d2c2260b7eaca3c41a16b280362980f0d8faf4c05ebb82c5",
+                    "e34ad0ad33885a473831f8ba8d9339123cb19d0e642e156d8e0d6e2ab2691aedb30e55a35637a806927225e1aa72223d41e59f92c6579b819e7d331a7ada9d2e01",
+                    "2a4861fb4cb951c791bf6c93859ef65abccd90034f91b9b77abb918e13b6fce75d5fa3e2d2f6eeeae105315178c2cb9db2ef238fe89b282f691c06db43bc71ca02",
+                    "fc97bb2be673c3bf388aaf58178ef14d354caf83c92aca8ef1831d619b8511e928f4f5fdea3962067b11e7cecfe094cd0f66a4ea9af9ec836d70d18f2b37df0281",
+                    "a5781a0adaa80ab7f7f164172dd1a1cb127e523daa0d6949aba074a15c589f12dfb8183182afec9230cb7947b7422a4abc1bb78173550d66274ea19f6c9dd92c82",
+                    "",
+                    "",
+                    "205f4237bd7dae576b34abc8a9c6fa4f0e4787c04234ca963e9e96c8f9b67b56d1ac205f4237bd7f93c69403a30c6b641f27ccf5201090152fcf1596474221307831c3ba205ac8ff25ce63564963d1148b84627f614af1f3c77d7caa23adc61264fa5e4996ba20b210c83e6f5b3f866837112d023d9ae8da2a6412168d54968ab87860ab970690ba20d3ee3b7a8b8149122b3c886330b3241538ba4b935c4040f4a73ddab917241bc5ba20cdfabb9d0e5c8f09a83f19e36e100d8f5e882f1b60aa60dacd9e6d072c117bc0ba20aab038c238e95fb54cdd0a6705dc1b1f8d135a9e9b20ab9c7ff96eef0e9bf545ba559c",
+                    "c0b1674191a88ec5cdd733e4240a81803105dc412d6c6708d53ab94fc248f4f5534a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33bf4184fc596403b9d638783cf57adfe4c75c605f6356fbc91338530e9831e9e166f7cf9580f1c2dfb3c4d5d043cdbb128c640e3f20161245aa7372e9666168516a1075db55d416d3ca199f55b6084e2115b9345e16c5cf302fc80e9d5fbf5d48dd5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb46829a3efd3ef04f9153d47a990bd7b048a4b2d213daaa5fb8ed670fb85f13bdbcf54e48e5f5c656b26c3bca14a8c95aa583d07ebe84dde3b7dd4a78f4e4186e713d29c9c0e8e4d2a9790922af73f0b8d51f0bd4bb19940d9cf910ead8fbe85bc9bbb41a757f405890fb0f5856228e23b715702d714d59bf2b1feb70d8b2b4e3e089fdbcf0ef9d8d00f66e47917f67cc5d78aec1ac786e2abb8d2facb4e4790aad6cc455ae816e6cdafdb58d54e35d4f46d860047458eacf1c7405dc634631c570d8d31992805518fd62daa3bdd2a5c4fd2cd3054c9b3dca1d78055e9528cff6adc8f907925d2ebe48765103e6845c06f1f2bb77c6adc1cc002865865eb5cfd5c1cb10c007c60e14f9d087e0291d4d0c7869697c6681d979c6639dbd960792b4d4133e794d097969002ee05d336686fc03c9e15a597c1b9827669460fac9879903637777defed8717c581b4c0509329550e344bdc14ac38f71fc050096887e535c8fd456524104a6674693c29946543f8a0befccce5a352bda55ec8559fc630f5f37393096d97bfee8660f4100ffd61874d62f9a65de9fb6acf740c4c386990ef7373be398c4bdc43709db7398106609eea2a7841aaf3a4fa2000dc18184faa2a7eb5a2af5845a8d3796308ff9840e567b14cf6bb158ff26c999e6f9a1f5448f9aa"
+                ],
+                "is_coinbase": false,
+                "sequence": 342,
+                "inner_witnessscript_asm": "OP_PUSHBYTES_32 5f4237bd7dae576b34abc8a9c6fa4f0e4787c04234ca963e9e96c8f9b67b56d1 OP_CHECKSIG OP_PUSHBYTES_32 5f4237bd7f93c69403a30c6b641f27ccf5201090152fcf1596474221307831c3 OP_CHECKSIGADD OP_PUSHBYTES_32 5ac8ff25ce63564963d1148b84627f614af1f3c77d7caa23adc61264fa5e4996 OP_CHECKSIGADD OP_PUSHBYTES_32 b210c83e6f5b3f866837112d023d9ae8da2a6412168d54968ab87860ab970690 OP_CHECKSIGADD OP_PUSHBYTES_32 d3ee3b7a8b8149122b3c886330b3241538ba4b935c4040f4a73ddab917241bc5 OP_CHECKSIGADD OP_PUSHBYTES_32 cdfabb9d0e5c8f09a83f19e36e100d8f5e882f1b60aa60dacd9e6d072c117bc0 OP_CHECKSIGADD OP_PUSHBYTES_32 aab038c238e95fb54cdd0a6705dc1b1f8d135a9e9b20ab9c7ff96eef0e9bf545 OP_CHECKSIGADD OP_PUSHNUM_5 OP_NUMEQUAL"
+            }
+        ],
+        "vout": [
+            {
+                "scriptpubkey": "210261542eb020b36c1da48e2e607b90a8c1f2ccdbd06eaf5fb4bb0d7cc34293d32aac",
+                "scriptpubkey_asm": "OP_PUSHBYTES_33 0261542eb020b36c1da48e2e607b90a8c1f2ccdbd06eaf5fb4bb0d7cc34293d32a OP_CHECKSIG",
+                "scriptpubkey_type": "p2pk",
+                "value": 576
+            },
+            {
+                "scriptpubkey": "76a9140240539af6c68431e4ce9cc5ef464f12c1741b3c88ac",
+                "scriptpubkey_asm": "OP_DUP OP_HASH160 OP_PUSHBYTES_20 0240539af6c68431e4ce9cc5ef464f12c1741b3c OP_EQUALVERIFY OP_CHECKSIG",
+                "scriptpubkey_type": "p2pkh",
+                "scriptpubkey_address": "1CuQsdrcgcmPvugo3NqEwh1kDcpeEnuFC",
+                "value": 546
+            },
+            {
+                "scriptpubkey": "5121028b45a50f795be0413680036665d17a3eca099648ea80637bc3a70a7d2b52ae2851ae",
+                "scriptpubkey_asm": "OP_PUSHNUM_1 OP_PUSHBYTES_33 028b45a50f795be0413680036665d17a3eca099648ea80637bc3a70a7d2b52ae28 OP_PUSHNUM_1 OP_CHECKMULTISIG",
+                "scriptpubkey_type": "multisig",
+                "value": 582
+            },
+            {
+                "scriptpubkey": "a91449ed2c96e33b6134408af8484508bcc3248c8dbd87",
+                "scriptpubkey_asm": "OP_HASH160 OP_PUSHBYTES_20 49ed2c96e33b6134408af8484508bcc3248c8dbd OP_EQUAL",
+                "scriptpubkey_type": "p2sh",
+                "scriptpubkey_address": "38RuNhSiZiftB6WVnStu5aUz6jXtCDXQZk",
+                "value": 540
+            },
+            {
+                "scriptpubkey": "0014c8e51cf6891c0a2101aecea8cd5ce9bbbfaf7bba",
+                "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 c8e51cf6891c0a2101aecea8cd5ce9bbbfaf7bba",
+                "scriptpubkey_type": "v0_p2wpkh",
+                "scriptpubkey_address": "bc1qerj3ea5frs9zzqdwe65v6h8fhwl677a6s0hxhf",
+                "value": 294
+            },
+            {
+                "scriptpubkey": "0020c485bbb80c4be276e77eac3a983a391cc8b1a1b5f160995a36c3dff18296385a",
+                "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_32 c485bbb80c4be276e77eac3a983a391cc8b1a1b5f160995a36c3dff18296385a",
+                "scriptpubkey_type": "v0_p2wsh",
+                "scriptpubkey_address": "bc1qcjzmhwqvf038dem74safsw3ernytrgd479sfjk3kc00lrq5k8pdqczl83q",
+                "value": 330
+            },
+            {
+                "scriptpubkey": "5120a7a42b268957a06c9de4d7260f1df392ce4d6e7b743f5adc27415ce2afceb3b9",
+                "scriptpubkey_asm": "OP_PUSHNUM_1 OP_PUSHBYTES_32 a7a42b268957a06c9de4d7260f1df392ce4d6e7b743f5adc27415ce2afceb3b9",
+                "scriptpubkey_type": "v1_p2tr",
+                "scriptpubkey_address": "bc1p57jzkf5f27sxe80y6unq780njt8y6mnmwsl44hp8g9ww9t7wkwusv7av76",
+                "value": 330
+            },
+            {
+                "scriptpubkey": "51024e73",
+                "scriptpubkey_asm": "OP_PUSHNUM_1 OP_PUSHBYTES_2 4e73",
+                "scriptpubkey_type": "unknown",
+                "scriptpubkey_address": "bc1pfeessrawgf",
+                "value": 240
+            },
+            {
+                "scriptpubkey": "6a224e6f7420796f757220696e707574732c206e6f7420796f7572206f7574707574732e005152535455565758595a5b5c5d5e5f60",
+                "scriptpubkey_asm": "OP_RETURN OP_PUSHBYTES_34 4e6f7420796f757220696e707574732c206e6f7420796f7572206f7574707574732e OP_0 OP_PUSHNUM_1 OP_PUSHNUM_2 OP_PUSHNUM_3 OP_PUSHNUM_4 OP_PUSHNUM_5 OP_PUSHNUM_6 OP_PUSHNUM_7 OP_PUSHNUM_8 OP_PUSHNUM_9 OP_PUSHNUM_10 OP_PUSHNUM_11 OP_PUSHNUM_12 OP_PUSHNUM_13 OP_PUSHNUM_14 OP_PUSHNUM_15 OP_PUSHNUM_16",
+                "scriptpubkey_type": "op_return",
+                "value": 0
+            }
+        ],
+        "size": 3500,
+        "weight": 8186,
+        "sigops": 115,
+        "fee": 71294,
+        "status": {
+            "confirmed": true,
+            "block_height": 850000,
+            "block_hash": "00000000000000000002a0b5db2a7f8d9087464c2586b546be7bce8eb53b8187",
+            "block_time": 1719689674
+        }
+    }
+]

backend/src/__tests__/api/test-data/transactions-rbfs.json

@@ -0,0 +1,121 @@
+[
+    {
+        "txid": "7219d95161f3718335991ac6d967d24eedec370908c9879bb1e192e6d797d0a6",
+        "version": 1,
+        "locktime": 0,
+        "vin": [
+            {
+                "txid": "d863deb706de5a611028f7547e16ea81d7819e44beb640fb30a9ba30c585140f",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "76a914cd5b6566b455d043558829f6932edaae5d8f0ad388ac",
+                    "scriptpubkey_asm": "OP_DUP OP_HASH160 OP_PUSHBYTES_20 cd5b6566b455d043558829f6932edaae5d8f0ad3 OP_EQUALVERIFY OP_CHECKSIG",
+                    "scriptpubkey_type": "p2pkh",
+                    "scriptpubkey_address": "1Kiq1dyVBzYLWGrBPWjChvKyzB2H95x5RJ",
+                    "value": 799995000
+                },
+                "scriptsig": "483045022100aeeddfb9785c5a4b70e90d0445785c68b7a44e28853441134a70ddc4da39527602203dfe1ec1a377aaacb64ae65c7c944caf1398d2dc063f712251b4cf696d44d3cb01210314338e3e191aea3ac9e9292611faeedf0379bbe62c30fd76c7450722a1ac47c6",
+                "scriptsig_asm": "OP_PUSHBYTES_72 3045022100aeeddfb9785c5a4b70e90d0445785c68b7a44e28853441134a70ddc4da39527602203dfe1ec1a377aaacb64ae65c7c944caf1398d2dc063f712251b4cf696d44d3cb01 OP_PUSHBYTES_33 0314338e3e191aea3ac9e9292611faeedf0379bbe62c30fd76c7450722a1ac47c6",
+                "is_coinbase": false,
+                "sequence": 4294967293
+            }
+        ],
+        "vout": [
+            {
+                "scriptpubkey": "6a4c5058325b8669baa9259e082f064005bc92274b559337ac317798f5d76f2d0577ed5a96042fce8c33d841b6c47a99f9597000ab04a10b34cd419fc19784d9e36f1a33fd7b000c3bce00b6000c1d1e00614b",
+                "scriptpubkey_asm": "OP_RETURN OP_PUSHDATA1 58325b8669baa9259e082f064005bc92274b559337ac317798f5d76f2d0577ed5a96042fce8c33d841b6c47a99f9597000ab04a10b34cd419fc19784d9e36f1a33fd7b000c3bce00b6000c1d1e00614b",
+                "scriptpubkey_type": "op_return",
+                "value": 0
+            },
+            {
+                "scriptpubkey": "a9144890aae025c84cb72a9730b49ca12595d6f6088d87",
+                "scriptpubkey_asm": "OP_HASH160 OP_PUSHBYTES_20 4890aae025c84cb72a9730b49ca12595d6f6088d OP_EQUAL",
+                "scriptpubkey_type": "p2sh",
+                "scriptpubkey_address": "38Jht2bzmJL4EwoFvvyFzejhfEb4J7KxLb",
+                "value": 155000
+            },
+            {
+                "scriptpubkey": "76a91486e7dad6617303942a448b7f8afe9653e5624a5e88ac",
+                "scriptpubkey_asm": "OP_DUP OP_HASH160 OP_PUSHBYTES_20 86e7dad6617303942a448b7f8afe9653e5624a5e OP_EQUALVERIFY OP_CHECKSIG",
+                "scriptpubkey_type": "p2pkh",
+                "scriptpubkey_address": "1DJKJGApgX4W8BSQ8FRPLqX78UaCskT4r2",
+                "value": 155000
+            },
+            {
+                "scriptpubkey": "76a914cd5b6566b455d043558829f6932edaae5d8f0ad388ac",
+                "scriptpubkey_asm": "OP_DUP OP_HASH160 OP_PUSHBYTES_20 cd5b6566b455d043558829f6932edaae5d8f0ad3 OP_EQUALVERIFY OP_CHECKSIG",
+                "scriptpubkey_type": "p2pkh",
+                "scriptpubkey_address": "1Kiq1dyVBzYLWGrBPWjChvKyzB2H95x5RJ",
+                "value": 799675549
+            }
+        ],
+        "size": 350,
+        "weight": 1400,
+        "fee": 9451,
+        "status": {
+            "confirmed": false
+        },
+        "order": 2798688215,
+        "vsize": 350,
+        "adjustedVsize": 350,
+        "sigops": 8,
+        "feePerVsize": 27.002857142857142,
+        "adjustedFeePerVsize": 27.002857142857142,
+        "effectiveFeePerVsize": 27.002857142857142,
+        "firstSeen": 1691218536,
+        "uid": 513598,
+        "inputs": [],
+        "position": {
+            "block": 0,
+            "vsize": 22166
+        },
+        "bestDescendant": null,
+        "cpfpChecked": true
+    },
+    {
+        "txid": "5387881d695d4564d397026dc5f740f816f8390b4b2c5ec8c20309122712a875",
+        "version": 2,
+        "locktime": 0,
+        "vin": [
+            {
+                "txid": "b50225a04a1d6fbbfa7a2122bc0580396f614027b3957f476229633576f06130",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "0014a24f913f8a9c30a4c302c2c78f2fd7addb08fd07",
+                    "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 a24f913f8a9c30a4c302c2c78f2fd7addb08fd07",
+                    "scriptpubkey_type": "v0_p2wpkh",
+                    "scriptpubkey_address": "bc1q5f8ez0u2nsc2fsczctrc7t7h4hds3lg82ewqhz",
+                    "value": 612917
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "3045022100a0c23953ace5d022b7a6d45d1ae1730bf20a4d594bb5d4fa7aa80e4881b44d320220008f9b144805bb91995fc0f452a56e09f4ad16fa149d71ae9b5d57c742e8e2cc01",
+                    "03dc2c7b687019b40a68d713322675206cc266e34e5340ec982c13ff0222c3b2b6"
+                ],
+                "is_coinbase": false,
+                "sequence": 2147483649
+            }
+        ],
+        "vout": [
+            {
+                "scriptpubkey": "0014199a98f9589364ffe5ef5bbae45ce5dfcbb873bd",
+                "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 199a98f9589364ffe5ef5bbae45ce5dfcbb873bd",
+                "scriptpubkey_type": "v0_p2wpkh",
+                "scriptpubkey_address": "bc1qrxdf372cjdj0le00twawgh89ml9msuaau62gk4",
+                "value": 611909
+            }
+        ],
+        "size": 192,
+        "weight": 438,
+        "fee": 1008,
+        "status": {
+            "confirmed": false
+        },
+        "bestDescendant": null,
+        "descendants": null,
+        "adjustedFeePerVsize": 10.2283,
+        "sigops": 1,
+        "adjustedVsize": 109.5
+    }
+]

backend/src/__tests__/api/test-data/transactions-replaced.json

@@ -0,0 +1,139 @@
+[
+    {
+        "txid": "008592364e21c1e3d62ba9538ac78a81779897b52100af5707ab063df98964f2",
+        "version": 1,
+        "locktime": 0,
+        "vin": [
+            {
+                "txid": "d863deb706de5a611028f7547e16ea81d7819e44beb640fb30a9ba30c585140f",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "76a914cd5b6566b455d043558829f6932edaae5d8f0ad388ac",
+                    "scriptpubkey_asm": "OP_DUP OP_HASH160 OP_PUSHBYTES_20 cd5b6566b455d043558829f6932edaae5d8f0ad3 OP_EQUALVERIFY OP_CHECKSIG",
+                    "scriptpubkey_type": "p2pkh",
+                    "scriptpubkey_address": "1Kiq1dyVBzYLWGrBPWjChvKyzB2H95x5RJ",
+                    "value": 799995000
+                },
+                "scriptsig": "483045022100c1fb331d155a7d299a0451d14fa1122b328e0e239afc9ba8dc2aff449ddc5a3a02201c1e19030d1efa432f5069cd369d7ad09a67f68501345e4db35f7b799605f55601210314338e3e191aea3ac9e9292611faeedf0379bbe62c30fd76c7450722a1ac47c6",
+                "scriptsig_asm": "OP_PUSHBYTES_72 3045022100c1fb331d155a7d299a0451d14fa1122b328e0e239afc9ba8dc2aff449ddc5a3a02201c1e19030d1efa432f5069cd369d7ad09a67f68501345e4db35f7b799605f55601 OP_PUSHBYTES_33 0314338e3e191aea3ac9e9292611faeedf0379bbe62c30fd76c7450722a1ac47c6",
+                "is_coinbase": false,
+                "sequence": 4294967293
+            }
+        ],
+        "vout": [
+            {
+                "scriptpubkey": "6a4c5058325b78064160b631b5a15d9078d99c0db066449fb4c59bbfa4d987ba906e2990088b2fce8c33d841b6c47a99f9597000ab04a10b34cd419fc19784d9e36f1a33fd7b000c3bce00b6000c1d1e00614b",
+                "scriptpubkey_asm": "OP_RETURN OP_PUSHDATA1 58325b78064160b631b5a15d9078d99c0db066449fb4c59bbfa4d987ba906e2990088b2fce8c33d841b6c47a99f9597000ab04a10b34cd419fc19784d9e36f1a33fd7b000c3bce00b6000c1d1e00614b",
+                "scriptpubkey_type": "op_return",
+                "value": 0
+            },
+            {
+                "scriptpubkey": "a9144890aae025c84cb72a9730b49ca12595d6f6088d87",
+                "scriptpubkey_asm": "OP_HASH160 OP_PUSHBYTES_20 4890aae025c84cb72a9730b49ca12595d6f6088d OP_EQUAL",
+                "scriptpubkey_type": "p2sh",
+                "scriptpubkey_address": "38Jht2bzmJL4EwoFvvyFzejhfEb4J7KxLb",
+                "value": 155000
+            },
+            {
+                "scriptpubkey": "76a91486e7dad6617303942a448b7f8afe9653e5624a5e88ac",
+                "scriptpubkey_asm": "OP_DUP OP_HASH160 OP_PUSHBYTES_20 86e7dad6617303942a448b7f8afe9653e5624a5e OP_EQUALVERIFY OP_CHECKSIG",
+                "scriptpubkey_type": "p2pkh",
+                "scriptpubkey_address": "1DJKJGApgX4W8BSQ8FRPLqX78UaCskT4r2",
+                "value": 155000
+            },
+            {
+                "scriptpubkey": "76a914cd5b6566b455d043558829f6932edaae5d8f0ad388ac",
+                "scriptpubkey_asm": "OP_DUP OP_HASH160 OP_PUSHBYTES_20 cd5b6566b455d043558829f6932edaae5d8f0ad3 OP_EQUALVERIFY OP_CHECKSIG",
+                "scriptpubkey_type": "p2pkh",
+                "scriptpubkey_address": "1Kiq1dyVBzYLWGrBPWjChvKyzB2H95x5RJ",
+                "value": 799676250
+            }
+        ],
+        "size": 350,
+        "weight": 1400,
+        "fee": 8750,
+        "status": {
+            "confirmed": false
+        },
+        "order": 4066675193,
+        "vsize": 350,
+        "adjustedVsize": 350,
+        "sigops": 8,
+        "feePerVsize": 25,
+        "adjustedFeePerVsize": 25,
+        "effectiveFeePerVsize": 25,
+        "firstSeen": 1691218516,
+        "uid": 512584,
+        "inputs": [],
+        "position": {
+            "block": 0,
+            "vsize": 13846
+        },
+        "bestDescendant": null,
+        "cpfpChecked": true
+    },
+    {
+        "txid": "b7981a624e4261c11f1246314d41e74be56af82eb557bcd054a5e0f94c023668",
+        "version": 2,
+        "locktime": 0,
+        "vin": [
+            {
+                "txid": "b50225a04a1d6fbbfa7a2122bc0580396f614027b3957f476229633576f06130",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "0014a24f913f8a9c30a4c302c2c78f2fd7addb08fd07",
+                    "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 a24f913f8a9c30a4c302c2c78f2fd7addb08fd07",
+                    "scriptpubkey_type": "v0_p2wpkh",
+                    "scriptpubkey_address": "bc1q5f8ez0u2nsc2fsczctrc7t7h4hds3lg82ewqhz",
+                    "value": 612917
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "304402204dd10f14afa41bc76d8278140ff1ec3d3f87f2c207bbb5418cc76dab30d7f6a402207877cc9c6a2c724b6ea7a1c24ac00022469f194fd1a4bd8030bbca1787d3f5f301",
+                    "03dc2c7b687019b40a68d713322675206cc266e34e5340ec982c13ff0222c3b2b6"
+                ],
+                "is_coinbase": false,
+                "sequence": 2147483648
+            }
+        ],
+        "vout": [
+            {
+                "scriptpubkey": "76a9149d32ef812385f3811634e0c0117dd153a5de10a488ac",
+                "scriptpubkey_asm": "OP_DUP OP_HASH160 OP_PUSHBYTES_20 9d32ef812385f3811634e0c0117dd153a5de10a4 OP_EQUALVERIFY OP_CHECKSIG",
+                "scriptpubkey_type": "p2pkh",
+                "scriptpubkey_address": "1FLC7Bag7okAkKPCyZbgZZg3Hh1EuGZ5Rd",
+                "value": 344697
+            },
+            {
+                "scriptpubkey": "00144c2671336ca8761863b4c68d64d4672491fec1b9",
+                "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 4c2671336ca8761863b4c68d64d4672491fec1b9",
+                "scriptpubkey_type": "v0_p2wpkh",
+                "scriptpubkey_address": "bc1qfsn8zvmv4pmpsca5c6xkf4r8yjglasdesrawcx",
+                "value": 267636
+            }
+        ],
+        "size": 225,
+        "weight": 573,
+        "fee": 584,
+        "status": {
+            "confirmed": false
+        },
+        "order": 1748369996,
+        "vsize": 143,
+        "adjustedVsize": 143.25,
+        "sigops": 5,
+        "feePerVsize": 4.076788830715532,
+        "adjustedFeePerVsize": 4.076788830715532,
+        "effectiveFeePerVsize": 4.076788830715532,
+        "firstSeen": 1691222376,
+        "uid": 526515,
+        "inputs": [],
+        "position": {
+            "block": 7,
+            "vsize": 22021095.5
+        },
+        "bestDescendant": null,
+        "cpfpChecked": true
+    }
+]

backend/src/__tests__/config.test.ts

@@ -0,0 +1,296 @@
+import * as fs from 'fs';
+
+describe('Mempool Backend Config', () => {
+  beforeEach(() => {
+    jest.resetAllMocks();
+    jest.resetModules();
+  });
+
+  test('should return defaults when no file is present', () => {
+    jest.isolateModules(() => {
+      jest.mock('../../mempool-config.json', () => ({}), { virtual: true });
+
+      const config = jest.requireActual('../config').default;
+
+      expect(config.MEMPOOL).toStrictEqual({
+        ENABLED: true,
+        OFFICIAL: false,
+        NETWORK: 'mainnet',
+        BACKEND: 'none',
+        BLOCKS_SUMMARIES_INDEXING: false,
+        GOGGLES_INDEXING: false,
+        HTTP_PORT: 8999,
+        UNIX_SOCKET_PATH: '',
+        SPAWN_CLUSTER_PROCS: 0,
+        API_URL_PREFIX: '/api/v1/',
+        AUTOMATIC_POOLS_UPDATE: false,
+        POLL_RATE_MS: 2000,
+        CACHE_DIR: './cache',
+        CACHE_ENABLED: true,
+        CLEAR_PROTECTION_MINUTES: 20,
+        RECOMMENDED_FEE_PERCENTILE: 50,
+        BLOCK_WEIGHT_UNITS: 4000000,
+        INITIAL_BLOCKS_AMOUNT: 8,
+        MEMPOOL_BLOCKS_AMOUNT: 8,
+        INDEXING_BLOCKS_AMOUNT: 11000,
+        USE_SECOND_NODE_FOR_MINFEE: false,
+        EXTERNAL_ASSETS: [],
+        EXTERNAL_MAX_RETRY: 1,
+        EXTERNAL_RETRY_INTERVAL: 0,
+        USER_AGENT: 'mempool',
+        STDOUT_LOG_MIN_PRIORITY: 'debug',
+        POOLS_JSON_TREE_URL: 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
+        POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json',
+        POOLS_UPDATE_DELAY: 604800,
+        AUDIT: false,
+        RUST_GBT: true,
+        LIMIT_GBT: false,
+        CPFP_INDEXING: false,
+        MAX_BLOCKS_BULK_QUERY: 0,
+        DISK_CACHE_BLOCK_INTERVAL: 6,
+        MAX_PUSH_TX_SIZE_WEIGHT: 400000,
+        ALLOW_UNREACHABLE: true,
+        PRICE_UPDATES_PER_HOUR: 1,
+        MAX_TRACKED_ADDRESSES: 1,
+      });
+
+      expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true });
+
+      expect(config.ESPLORA).toStrictEqual({
+        REST_API_URL: 'http://127.0.0.1:3000',
+        UNIX_SOCKET_PATH: null,
+        BATCH_QUERY_BASE_SIZE: 1000,
+        RETRY_UNIX_SOCKET_AFTER: 30000,
+        REQUEST_TIMEOUT: 10000,
+        FALLBACK_TIMEOUT: 5000,
+        FALLBACK: [],
+        MAX_BEHIND_TIP: 2,
+       });
+
+      expect(config.CORE_RPC).toStrictEqual({
+        HOST: '127.0.0.1',
+        PORT: 8332,
+        USERNAME: 'mempool',
+        PASSWORD: 'mempool',
+        TIMEOUT: 60000,
+        COOKIE: false,
+        COOKIE_PATH: '/bitcoin/.cookie',
+        DEBUG_LOG_PATH: '',
+      });
+
+      expect(config.SECOND_CORE_RPC).toStrictEqual({
+        HOST: '127.0.0.1',
+        PORT: 8332,
+        USERNAME: 'mempool',
+        PASSWORD: 'mempool',
+        TIMEOUT: 60000,
+        COOKIE: false,
+        COOKIE_PATH: '/bitcoin/.cookie'
+      });
+
+      expect(config.DATABASE).toStrictEqual({
+        ENABLED: true,
+        HOST: '127.0.0.1',
+        SOCKET: '',
+        PORT: 3306,
+        DATABASE: 'mempool',
+        USERNAME: 'mempool',
+        PASSWORD: 'mempool',
+        TIMEOUT: 180000,
+        PID_DIR: '',
+        POOL_SIZE: 100,
+      });
+
+      expect(config.SYSLOG).toStrictEqual({
+        ENABLED: true,
+        HOST: '127.0.0.1',
+        PORT: 514,
+        MIN_PRIORITY: 'info',
+        FACILITY: 'local7'
+      });
+
+      expect(config.STATISTICS).toStrictEqual({ ENABLED: true, TX_PER_SECOND_SAMPLE_PERIOD: 150 });
+
+      expect(config.SOCKS5PROXY).toStrictEqual({
+        ENABLED: false,
+        USE_ONION: true,
+        HOST: '127.0.0.1',
+        PORT: 9050,
+        USERNAME: '',
+        PASSWORD: ''
+      });
+
+      expect(config.EXTERNAL_DATA_SERVER).toStrictEqual({
+        MEMPOOL_API: 'https://mempool.space/api/v1',
+        MEMPOOL_ONION: 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1',
+        LIQUID_API: 'https://liquid.network/api/v1',
+        LIQUID_ONION: 'http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1'
+      });
+
+      expect(config.MAXMIND).toStrictEqual({
+        ENABLED: false,
+        GEOLITE2_CITY: '/usr/local/share/GeoIP/GeoLite2-City.mmdb',
+        GEOLITE2_ASN: '/usr/local/share/GeoIP/GeoLite2-ASN.mmdb',
+        GEOIP2_ISP: '/usr/local/share/GeoIP/GeoIP2-ISP.mmdb'
+      });
+
+      expect(config.REPLICATION).toStrictEqual({
+        ENABLED: false,
+        AUDIT: false,
+        AUDIT_START_HEIGHT: 774000,
+        STATISTICS: false,
+        STATISTICS_START_TIME: 1481932800,
+        SERVERS: []
+      });
+
+      expect(config.MEMPOOL_SERVICES).toStrictEqual({
+        API: "",
+        ACCELERATIONS: false,
+      });
+
+      expect(config.REDIS).toStrictEqual({
+        ENABLED: false,
+        UNIX_SOCKET_PATH: '',
+        BATCH_QUERY_BASE_SIZE: 5000,
+      });
+
+      expect(config.FIAT_PRICE).toStrictEqual({
+        ENABLED: true,
+        PAID: false,
+        API_KEY: '',
+      });
+    });
+  });
+
+  test('should override the default values with the passed values', () => {
+    jest.isolateModules(() => {
+      const fixture = JSON.parse(fs.readFileSync(`${__dirname}/../__fixtures__/mempool-config.template.json`, 'utf8'));
+      jest.mock('../../mempool-config.json', () => (fixture), { virtual: true });
+
+      const config = jest.requireActual('../config').default;
+
+      expect(config.MEMPOOL).toStrictEqual(fixture.MEMPOOL);
+
+      expect(config.ELECTRUM).toStrictEqual(fixture.ELECTRUM);
+
+      expect(config.ESPLORA).toStrictEqual(fixture.ESPLORA);
+
+      expect(config.CORE_RPC).toStrictEqual(fixture.CORE_RPC);
+
+      expect(config.SECOND_CORE_RPC).toStrictEqual(fixture.SECOND_CORE_RPC);
+
+      expect(config.DATABASE).toStrictEqual(fixture.DATABASE);
+
+      expect(config.SYSLOG).toStrictEqual(fixture.SYSLOG);
+
+      expect(config.STATISTICS).toStrictEqual(fixture.STATISTICS);
+
+      expect(config.SOCKS5PROXY).toStrictEqual(fixture.SOCKS5PROXY);
+
+      expect(config.EXTERNAL_DATA_SERVER).toStrictEqual(fixture.EXTERNAL_DATA_SERVER);
+
+      expect(config.MEMPOOL_SERVICES).toStrictEqual(fixture.MEMPOOL_SERVICES);
+
+      expect(config.REDIS).toStrictEqual(fixture.REDIS);
+    });
+  });
+
+  test('should ensure the docker start.sh script has default values', () => {
+    jest.isolateModules(() => {
+      const startSh = fs.readFileSync(`${__dirname}/../../../docker/backend/start.sh`, 'utf-8');
+      const fixture = JSON.parse(fs.readFileSync(`${__dirname}/../__fixtures__/mempool-config.template.json`, 'utf8'));
+
+      function parseJson(jsonObj, root?) {
+        for (const [key, value] of Object.entries(jsonObj)) {
+          // We have a few cases where we can't follow the pattern
+          if (root === 'MEMPOOL' && key === 'HTTP_PORT') {
+            if (process.env.CI) {
+              console.log('skipping check for MEMPOOL_HTTP_PORT');
+            }
+            continue;
+          }
+
+          if (root) {
+              //The flattened string, i.e, __MEMPOOL_ENABLED__
+              const replaceStr = `${root ? '__' + root + '_' : '__'}${key}__`;
+
+              //The string used as the environment variable, i.e, MEMPOOL_ENABLED
+              const envVarStr = `${root ? root : ''}_${key}`;
+
+              let defaultEntry;
+              //The string used as the default value, to be checked as a regex, i.e, __MEMPOOL_ENABLED__=${MEMPOOL_ENABLED:=(.*)}
+              if (Array.isArray(value)) {
+                defaultEntry = `${replaceStr}=\${${envVarStr}:=[]}`;
+                if (process.env.CI) {
+                  console.log(`looking for ${defaultEntry} in the start.sh script`);
+                }
+                //Regex matching does not work with the array values
+                expect(startSh).toContain(defaultEntry);
+              } else {
+                 defaultEntry = replaceStr + '=' + '\\${' + envVarStr + ':=(.*)' + '}';
+                 if (process.env.CI) {
+                  console.log(`looking for ${defaultEntry} in the start.sh script`);
+                }
+                const re = new RegExp(defaultEntry);
+                expect(startSh).toMatch(re);
+              }
+
+              //The string that actually replaces the values in the config file
+              const sedStr = 'sed -i "s!' + replaceStr + '!${' + replaceStr + '}!g" mempool-config.json';
+              if (process.env.CI) {
+                console.log(`looking for ${sedStr} in the start.sh script`);
+              }
+              expect(startSh).toContain(sedStr);
+            }
+          else {
+            parseJson(value, key);
+          }
+        }
+      }
+
+      parseJson(fixture);
+    });
+  });
+
+  test('should ensure that the mempool-config.json Docker template has all the keys', () => {
+    jest.isolateModules(() => {
+      const fixture = JSON.parse(fs.readFileSync(`${__dirname}/../__fixtures__/mempool-config.template.json`, 'utf8'));
+      const dockerJson = fs.readFileSync(`${__dirname}/../../../docker/backend/mempool-config.json`, 'utf-8');
+
+      function parseJson(jsonObj, root?) {
+        for (const [key, value] of Object.entries(jsonObj)) {
+          switch (typeof value) {
+            case 'object': {
+              if (Array.isArray(value)) {
+                // numbers, arrays and booleans won't be enclosed by quotes
+                const replaceStr = `${root ? '__' + root + '_' : '__'}${key}__`;
+                expect(dockerJson).toContain(`"${key}": ${replaceStr}`);
+                break;
+              } else {
+                //Check for top level config keys
+                expect(dockerJson).toContain(`"${key}"`);
+                parseJson(value, key);
+                break;
+              }
+            }
+            case 'string': {
+              // strings should be enclosed by quotes
+              const replaceStr = `${root ? '__' + root + '_' : '__'}${key}__`;
+              expect(dockerJson).toContain(`"${key}": "${replaceStr}"`);
+              break;
+            }
+            default: {
+              // numbers, arrays and booleans won't be enclosed by quotes
+              const replaceStr = `${root ? '__' + root + '_' : '__'}${key}__`;
+              expect(dockerJson).toContain(`"${key}": ${replaceStr}`);
+              break;
+            }
+          }
+        };
+      }
+      parseJson(fixture);
+    });
+  });
+
+
+});

backend/src/__tests__/gbt/gbt-tests.ts

@@ -0,0 +1,68 @@
+import fs from 'fs';
+import { GbtGenerator, ThreadTransaction } from 'rust-gbt';
+import path from 'path';
+
+const baseline = require('./test-data/target-template.json');
+const testVector = require('./test-data/test-data-ids.json');
+const vectorUidMap: Map<number, string> = new Map(testVector.map(x => [x[0], x[1]]));
+const vectorTxidMap: Map<string, number>  = new Map(testVector.map(x => [x[1], x[0]]));
+// Note that this test buffer is specially constructed
+// such that uids are assigned in numerical txid order
+// so that ties break the same way as in Core's implementation
+const vectorBuffer: Buffer = fs.readFileSync(path.join(__dirname, './', './test-data/test-buffer.bin'));
+
+describe('Rust GBT', () => {
+  test('should produce the same template as getBlockTemplate from Bitcoin Core', async () => {
+    const rustGbt = new GbtGenerator(4_000_000, 8);
+    const { mempool, maxUid } = mempoolFromArrayBuffer(vectorBuffer.buffer);
+    const result = await rustGbt.make(mempool, [], maxUid);
+
+    const blocks: [string, number][][] = result.blocks.map(block => {
+      return block.map(uid => [vectorUidMap.get(uid) || 'missing', uid]);
+    });
+    const template = baseline.map(tx => [tx.txid, vectorTxidMap.get(tx.txid)]);
+
+    expect(blocks[0].length).toEqual(baseline.length);
+    expect(blocks[0]).toEqual(template);
+  });
+});
+
+function mempoolFromArrayBuffer(buf: ArrayBuffer): { mempool: ThreadTransaction[], maxUid: number } {
+  let maxUid = 0;
+  const view = new DataView(buf);
+  const count = view.getUint32(0, false);
+  const txs: ThreadTransaction[] = [];
+  let offset = 4;
+  for (let i = 0; i < count; i++) {
+    const uid = view.getUint32(offset, false);
+    maxUid = Math.max(maxUid, uid);
+    const tx: ThreadTransaction = {
+      uid,
+      order: txidToOrdering(vectorUidMap.get(uid) as string),
+      fee: view.getFloat64(offset + 4, false),
+      weight: view.getUint32(offset + 12, false),
+      sigops: view.getUint32(offset + 16, false),
+      // feePerVsize: view.getFloat64(offset + 20, false),
+      effectiveFeePerVsize: view.getFloat64(offset + 28, false),
+      inputs: [],
+    };
+    const numInputs = view.getUint32(offset + 36, false);
+    offset += 40;
+    for (let j = 0; j < numInputs; j++) {
+      tx.inputs.push(view.getUint32(offset, false));
+      offset += 4;
+    }
+    txs.push(tx);
+  }
+  return { mempool: txs, maxUid };
+}
+
+function txidToOrdering(txid: string): number {
+  return parseInt(
+    txid.substr(62, 2) +
+      txid.substr(60, 2) +
+      txid.substr(58, 2) +
+      txid.substr(56, 2),
+    16
+  );
+}

backend/src/api/about.routes.ts

@@ -0,0 +1,87 @@
+import { Application } from "express";
+import config from "../config";
+import axios from "axios";
+import logger from "../logger";
+
+class AboutRoutes {
+  public initRoutes(app: Application) {
+    app
+      .get(config.MEMPOOL.API_URL_PREFIX + 'donations', async (req, res) => {
+        try {
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/donations`, { responseType: 'stream', timeout: 10000 });
+          response.data.pipe(res);
+        } catch (e) {
+          res.status(500).end();
+        }
+      })
+      .get(config.MEMPOOL.API_URL_PREFIX + 'donations/images/:id', async (req, res) => {
+        try {
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/donations/images/${req.params.id}`, {
+            responseType: 'stream', timeout: 10000
+          });
+          response.data.pipe(res);
+        } catch (e) {
+          res.status(500).end();
+        }
+      })
+      .get(config.MEMPOOL.API_URL_PREFIX + 'contributors', async (req, res) => {
+        try {
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/contributors`, { responseType: 'stream', timeout: 10000 });
+          response.data.pipe(res);
+        } catch (e) {
+          res.status(500).end();
+        }
+      })
+      .get(config.MEMPOOL.API_URL_PREFIX + 'contributors/images/:id', async (req, res) => {
+        try {
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/contributors/images/${req.params.id}`, {
+            responseType: 'stream', timeout: 10000
+          });
+          response.data.pipe(res);
+        } catch (e) {
+          res.status(500).end();
+        }
+      })
+      .get(config.MEMPOOL.API_URL_PREFIX + 'translators', async (req, res) => {
+        try {
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/translators`, { responseType: 'stream', timeout: 10000 });
+          response.data.pipe(res);
+        } catch (e) {
+          res.status(500).end();
+        }
+      })
+      .get(config.MEMPOOL.API_URL_PREFIX + 'translators/images/:id', async (req, res) => {
+        try {
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/translators/images/${req.params.id}`, {
+            responseType: 'stream', timeout: 10000
+          });
+          response.data.pipe(res);
+        } catch (e) {
+          res.status(500).end();
+        }
+      })
+      .get(config.MEMPOOL.API_URL_PREFIX + 'services/sponsors', async (req, res) => {
+        const url = `${config.MEMPOOL_SERVICES.API}/${req.originalUrl.replace('/api/v1/services/', '')}`;
+        try {
+          const response = await axios.get(url, { responseType: 'stream', timeout: 10000 });
+          response.data.pipe(res);
+        } catch (e) {
+          logger.err(`Unable to fetch sponsors from ${url}. ${e}`, 'About Page');
+          res.status(500).end();
+        }
+      })
+      .get(config.MEMPOOL.API_URL_PREFIX + 'services/account/images/:username/:md5', async (req, res) => {
+        const url = `${config.MEMPOOL_SERVICES.API}/${req.originalUrl.replace('/api/v1/services/', '')}`;
+        try {
+          const response = await axios.get(url, { responseType: 'stream', timeout: 10000 });
+          response.data.pipe(res);
+        } catch (e) {
+          logger.err(`Unable to fetch sponsor profile image from ${url}. ${e}`, 'About Page');
+          res.status(500).end();
+        }
+      })
+    ;
+  }
+}
+
+export default new AboutRoutes();

backend/src/api/acceleration/acceleration.routes.ts

@@ -0,0 +1,84 @@
+import { Application, Request, Response } from 'express';
+import config from '../../config';
+import axios from 'axios';
+import logger from '../../logger';
+import mempool from '../mempool';
+import AccelerationRepository from '../../repositories/AccelerationRepository';
+
+class AccelerationRoutes {
+  private tag = 'Accelerator';
+
+  public initRoutes(app: Application): void {
+    app
+      .get(config.MEMPOOL.API_URL_PREFIX + 'services/accelerator/accelerations', this.$getAcceleratorAccelerations.bind(this))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'services/accelerator/accelerations/history', this.$getAcceleratorAccelerationsHistory.bind(this))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'services/accelerator/accelerations/history/aggregated', this.$getAcceleratorAccelerationsHistoryAggregated.bind(this))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'services/accelerator/accelerations/stats', this.$getAcceleratorAccelerationsStats.bind(this))
+      .post(config.MEMPOOL.API_URL_PREFIX + 'services/accelerator/estimate', this.$getAcceleratorEstimate.bind(this))
+    ;
+  }
+
+  private async $getAcceleratorAccelerations(req: Request, res: Response): Promise<void> {
+    const accelerations = mempool.getAccelerations();
+    res.status(200).send(Object.values(accelerations));
+  }
+
+  private async $getAcceleratorAccelerationsHistory(req: Request, res: Response): Promise<void> {
+    const history = await AccelerationRepository.$getAccelerationInfo(null, req.query.blockHeight ? parseInt(req.query.blockHeight as string, 10) : null);
+    res.status(200).send(history.map(accel => ({
+      txid: accel.txid,
+      added: accel.added,
+      status: 'completed',
+      effectiveFee: accel.effective_fee,
+      effectiveVsize: accel.effective_vsize,
+      boostRate: accel.boost_rate,
+      boostCost: accel.boost_cost,
+      blockHeight: accel.height,
+      pools: [accel.pool],
+    })));
+  }
+
+  private async $getAcceleratorAccelerationsHistoryAggregated(req: Request, res: Response): Promise<void> {
+    const url = `${config.MEMPOOL_SERVICES.API}/${req.originalUrl.replace('/api/v1/services/', '')}`;
+    try {
+      const response = await axios.get(url, { responseType: 'stream', timeout: 10000 });
+      for (const key in response.headers) {
+        res.setHeader(key, response.headers[key]);
+      }
+      response.data.pipe(res);
+    } catch (e) {
+      logger.err(`Unable to get aggregated acceleration history from ${url} in $getAcceleratorAccelerationsHistoryAggregated(), ${e}`, this.tag);
+      res.status(500).end();
+    }
+  }
+
+  private async $getAcceleratorAccelerationsStats(req: Request, res: Response): Promise<void> {
+    const url = `${config.MEMPOOL_SERVICES.API}/${req.originalUrl.replace('/api/v1/services/', '')}`;
+    try {
+      const response = await axios.get(url, { responseType: 'stream', timeout: 10000 });
+      for (const key in response.headers) {
+        res.setHeader(key, response.headers[key]);
+      }
+      response.data.pipe(res);
+    } catch (e) {
+      logger.err(`Unable to get acceleration stats from ${url} in $getAcceleratorAccelerationsStats(), ${e}`, this.tag);
+      res.status(500).end();
+    }
+  }
+
+  private async $getAcceleratorEstimate(req: Request, res: Response): Promise<void> {
+    const url = `${config.MEMPOOL_SERVICES.API}/${req.originalUrl.replace('/api/v1/services/', '')}`;
+    try {
+      const response = await axios.post(url, req.body, { responseType: 'stream', timeout: 10000 });
+      for (const key in response.headers) {
+        res.setHeader(key, response.headers[key]);
+      }
+      response.data.pipe(res);
+    } catch (e) {
+      logger.err(`Unable to get acceleration estimate from ${url} in $getAcceleratorEstimate(), ${e}`, this.tag);
+      res.status(500).end();
+    }
+  }
+}
+
+export default new AccelerationRoutes();

backend/src/api/acceleration/acceleration.ts

@@ -0,0 +1,244 @@
+import logger from '../../logger';
+import { MempoolTransactionExtended } from '../../mempool.interfaces';
+import { GraphTx, getSameBlockRelatives, initializeRelatives, makeBlockTemplate, mempoolComparator, removeAncestors, setAncestorScores } from '../mini-miner';
+
+const BLOCK_WEIGHT_UNITS = 4_000_000;
+const MAX_RELATIVE_GRAPH_SIZE = 200;
+const BID_BOOST_WINDOW = 40_000;
+const BID_BOOST_MIN_OFFSET = 10_000;
+const BID_BOOST_MAX_OFFSET = 400_000;
+
+export type Acceleration = {
+  txid: string;
+  max_bid: number;
+};
+
+interface TxSummary {
+  txid: string; // txid of the current transaction
+  effectiveVsize: number; // Total vsize of the dependency tree
+  effectiveFee: number;  // Total fee of the dependency tree in sats
+  ancestorCount: number; // Number of ancestors
+}
+
+export interface AccelerationInfo {
+  txSummary: TxSummary;
+  targetFeeRate: number; // target fee rate (recommended next block fee, or median fee for mined block)
+  nextBlockFee: number; // fee in sats required to be in the next block (using recommended next block fee, or median fee for mined block)
+  cost: number; // additional cost to accelerate ((cost + txSummary.effectiveFee) / txSummary.effectiveVsize) >= targetFeeRate
+}
+
+class AccelerationCosts {
+  /**
+   * Takes a list of accelerations and verbose block data
+   * Returns the "fair" boost rate to charge accelerations
+   *
+   * @param accelerationsx
+   * @param verboseBlock
+   */
+  public calculateBoostRate(accelerations: Acceleration[], blockTxs: MempoolTransactionExtended[]): number {
+    // Run GBT ourselves to calculate accurate effective fee rates
+    // the list of transactions comes from a mined block, so we already know everything fits within consensus limits
+    const template = makeBlockTemplate(blockTxs, accelerations, 1, Infinity, Infinity);
+
+    // initialize working maps for fast tx lookups
+    const accMap = {};
+    const txMap = {};
+    for (const acceleration of accelerations) {
+      accMap[acceleration.txid] = acceleration;
+    }
+    for (const tx of template) {
+      txMap[tx.txid] = tx;
+    }
+
+    // Identify and exclude accelerated and otherwise prioritized transactions
+    const excludeMap = {};
+    let totalWeight = 0;
+    let minAcceleratedPackage = Infinity;
+    let lastEffectiveRate = 0;
+    // Iterate over the mined template from bottom to top.
+    // Transactions should appear in ascending order of mining priority.
+    for (const blockTx of [...blockTxs].reverse()) {
+      const txid = blockTx.txid;
+      const tx = txMap[txid];
+      totalWeight += tx.weight;
+      const isAccelerated = accMap[txid] != null;
+      // If a cluster has a in-band effective fee rate than the previous cluster,
+      // it must have been prioritized out-of-band (in order to have a higher mining priority)
+      // so exclude from the analysis.
+      const isPrioritized = tx.effectiveFeePerVsize < lastEffectiveRate;
+      if (isPrioritized || isAccelerated) {
+        let packageWeight = 0;
+        // exclude this whole CPFP cluster
+        for (const clusterTxid of tx.cluster) {
+          packageWeight += txMap[clusterTxid].weight;
+          if (!excludeMap[clusterTxid]) {
+            excludeMap[clusterTxid] = true;
+          }
+        }
+        // keep track of the smallest accelerated CPFP cluster for later
+        if (isAccelerated) {
+          minAcceleratedPackage = Math.min(minAcceleratedPackage, packageWeight);
+        }
+      }
+      if (!isPrioritized) {
+        if (!isAccelerated) {
+          lastEffectiveRate = tx.effectiveFeePerVsize;
+        }
+      }
+    }
+
+    // The Bid Boost Rate is calculated by disregarding the bottom X weight units of the block,
+    // where X is the larger of BID_BOOST_MIN_OFFSET or the smallest accelerated package weight (the "offset"),
+    // then taking the average fee rate of the following BID_BOOST_WINDOW weight units
+    // (ignoring accelerated transactions and their ancestors).
+    //
+    // Transactions within the offset might pay less than the fair rate due to bin-packing effects
+    // But the average rate paid by the next chunk of non-accelerated transactions provides a good
+    // upper bound on the "next best rate" of alternatives to including the accelerated transactions
+    // (since, if there were any better options, they would have been included instead)
+    const spareWeight = BLOCK_WEIGHT_UNITS - totalWeight;
+    const windowOffset = Math.min(Math.max(minAcceleratedPackage, BID_BOOST_MIN_OFFSET, spareWeight), BID_BOOST_MAX_OFFSET);
+    const leftBound = windowOffset;
+    const rightBound = windowOffset + BID_BOOST_WINDOW;
+    let totalFeeInWindow = 0;
+    let totalWeightInWindow = Math.max(0, spareWeight - leftBound);
+    let txIndex = blockTxs.length - 1;
+    for (let offset = spareWeight; offset < BLOCK_WEIGHT_UNITS && txIndex >= 0; txIndex--) {
+      const txid = blockTxs[txIndex].txid;
+      const tx = txMap[txid];
+      if (excludeMap[txid]) {
+        // skip prioritized transactions and their ancestors
+        continue;
+      }
+
+      const left = offset;
+      const right = offset + tx.weight;
+      offset += tx.weight;
+      if (right < leftBound) {
+        // not within window yet
+        continue;
+      }
+      if (left > rightBound) {
+        // past window
+        break;
+      }
+      // count fees for weight units within the window
+      const overlapLeft = Math.max(leftBound, left);
+      const overlapRight = Math.min(rightBound, right);
+      const overlapUnits = overlapRight - overlapLeft;
+      totalFeeInWindow += (tx.effectiveFeePerVsize * (overlapUnits / 4));
+      totalWeightInWindow += overlapUnits;
+    }
+
+    if (totalWeightInWindow < BID_BOOST_WINDOW) {
+      // not enough un-prioritized transactions to calculate a fair rate
+      // just charge everyone their max bids
+      return Infinity;
+    }
+    // Divide the total fee by the size of the BID_BOOST_WINDOW in vbytes
+    const averageRate = totalFeeInWindow / (BID_BOOST_WINDOW / 4);
+    return averageRate;
+  }
+
+
+  /**
+   * Takes an accelerated mined txid and a target rate
+   * Returns the total vsize, fees and acceleration cost (in sats) of the tx and all same-block ancestors
+   *
+   * @param txid
+   * @param medianFeeRate
+   */
+  public getAccelerationInfo(tx: MempoolTransactionExtended, targetFeeRate: number, transactions: MempoolTransactionExtended[]): AccelerationInfo {
+    // Get same-block transaction ancestors
+    const allRelatives = getSameBlockRelatives(tx, transactions);
+    const relativesMap = initializeRelatives(allRelatives);
+    const rootTx = relativesMap.get(tx.txid) as GraphTx;
+
+    // Calculate cost to boost
+    return this.calculateAccelerationAncestors(rootTx, relativesMap, targetFeeRate);
+  }
+
+  /**
+   * Given a root transaction, a list of in-mempool ancestors, and a target fee rate,
+   * Calculate the minimum set of transactions to fee-bump, their total vsize + fees
+   *
+   * @param tx
+   * @param ancestors
+   */
+  private calculateAccelerationAncestors(tx: GraphTx, relatives: Map<string, GraphTx>, targetFeeRate: number): AccelerationInfo {
+    // add root tx to the ancestor map
+    relatives.set(tx.txid, tx);
+
+    // Check for high-sigop transactions (not supported)
+    relatives.forEach(entry => {
+      if (entry.vsize > Math.ceil(entry.weight / 4)) {
+        throw new Error(`high_sigop_tx`);
+      }
+    });
+
+    // Initialize individual & ancestor fee rates
+    relatives.forEach(entry => setAncestorScores(entry));
+
+    // Sort by descending ancestor score
+    let sortedRelatives = Array.from(relatives.values()).sort(mempoolComparator);
+
+    let includedInCluster: Map<string, GraphTx> | null = null;
+
+    // While highest score >= targetFeeRate
+    let maxIterations = MAX_RELATIVE_GRAPH_SIZE;
+    while (sortedRelatives.length && sortedRelatives[0].score && sortedRelatives[0].score >= targetFeeRate && maxIterations > 0) {
+      maxIterations--;
+      // Grab the highest scoring entry
+      const best = sortedRelatives.shift();
+      if (best) {
+        const cluster = new Map<string, GraphTx>(best.ancestors?.entries() || []);
+        if (best.ancestors.has(tx.txid)) {
+          includedInCluster = cluster;
+        }
+        cluster.set(best.txid, best);
+        // Remove this cluster (it already pays over the target rate, so doesn't need to be boosted)
+        // and update scores, ancestor totals and dependencies for the survivors
+        removeAncestors(cluster, relatives);
+
+        // re-sort
+        sortedRelatives = Array.from(relatives.values()).sort(mempoolComparator);
+      }
+    }
+
+    // sanity check for infinite loops / too many ancestors (should never happen)
+    if (maxIterations <= 0) {
+      logger.warn(`acceleration dependency calculation failed: calculateAccelerationAncestors loop exceeded ${MAX_RELATIVE_GRAPH_SIZE} iterations, unable to proceed`);
+      throw new Error('invalid_tx_dependencies');
+    }
+
+    let totalFee = tx.fees.ancestor;
+
+    // transaction is already CPFP-d above the target rate by some descendant
+    if (includedInCluster) {
+      let clusterSize = 0;
+      let clusterFee = 0;
+      includedInCluster.forEach(entry => {
+        clusterSize += entry.vsize;
+        clusterFee += entry.fees.base;
+      });
+      const clusterRate = clusterFee / clusterSize;
+      totalFee = Math.ceil(tx.ancestorsize * clusterRate);
+    }
+
+    // Whatever remains in the accelerated tx's dependencies needs to be boosted to the targetFeeRate
+    // Cost = (totalVsize * targetFeeRate) - totalFee
+    return {
+      txSummary: {
+        txid: tx.txid,
+        effectiveVsize: tx.ancestorsize,
+        effectiveFee: totalFee,
+        ancestorCount: tx.ancestorcount,
+      },
+      cost: Math.max(0, Math.ceil(tx.ancestorsize * targetFeeRate) - totalFee),
+      targetFeeRate,
+      nextBlockFee: Math.ceil(tx.ancestorsize * targetFeeRate),
+    };
+  }
+}
+
+export default new AccelerationCosts;

backend/src/api/audit.ts

@@ -0,0 +1,194 @@
+import config from '../config';
+import logger from '../logger';
+import { MempoolTransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces';
+import rbfCache from './rbf-cache';
+import transactionUtils from './transaction-utils';
+
+const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners
+
+class Audit {
+  auditBlock(height: number, transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended })
+   : { unseen: string[], censored: string[], added: string[], prioritized: string[], fresh: string[], sigop: string[], fullrbf: string[], accelerated: string[], score: number, similarity: number } {
+    if (!projectedBlocks?.[0]?.transactionIds || !mempool) {
+      return { unseen: [], censored: [], added: [], prioritized: [], fresh: [], sigop: [], fullrbf: [], accelerated: [], score: 1, similarity: 1 };
+    }
+
+    const matches: string[] = []; // present in both mined block and template
+    const added: string[] = []; // present in mined block, not in template
+    const unseen: string[] = []; // present in the mined block, not in our mempool
+    let prioritized: string[] = []; // higher in the block than would be expected by in-band feerate alone
+    let deprioritized: string[] = []; // lower in the block than would be expected by in-band feerate alone
+    const fresh: string[] = []; // missing, but firstSeen or lastBoosted within PROPAGATION_MARGIN
+    const rbf: string[] = []; // either missing or present, and either part of a full-rbf replacement, or a conflict with the mined block
+    const accelerated: string[] = []; // prioritized by the mempool accelerator
+    const isCensored = {}; // missing, without excuse
+    const isDisplaced = {};
+    const isAccelerated = {};
+    let displacedWeight = 0;
+    let matchedWeight = 0;
+    let projectedWeight = 0;
+
+    const inBlock = {};
+    const inTemplate = {};
+
+    const now = Math.round((Date.now() / 1000));
+    for (const tx of transactions) {
+      inBlock[tx.txid] = tx;
+      if (mempool[tx.txid] && mempool[tx.txid].acceleration) {
+        accelerated.push(tx.txid);
+        isAccelerated[tx.txid] = true;
+      }
+    }
+    // coinbase is always expected
+    if (transactions[0]) {
+      inTemplate[transactions[0].txid] = true;
+    }
+    // look for transactions that were expected in the template, but missing from the mined block
+    for (const txid of projectedBlocks[0].transactionIds) {
+      if (!inBlock[txid]) {
+        // allow missing transactions which either belong to a full rbf tree, or conflict with any transaction in the mined block
+        if (rbfCache.has(txid) && (rbfCache.isFullRbf(txid) || rbfCache.anyInSameTree(txid, (tx) => inBlock[tx.txid]))) {
+          rbf.push(txid);
+        } else if (mempool[txid]?.firstSeen != null && (now - (mempool[txid]?.firstSeen || 0)) <= PROPAGATION_MARGIN) {
+          // tx is recent, may have reached the miner too late for inclusion
+          fresh.push(txid);
+        } else if (mempool[txid]?.lastBoosted != null && (now - (mempool[txid]?.lastBoosted || 0)) <= PROPAGATION_MARGIN) {
+          // tx was recently cpfp'd, miner may not have the latest effective rate
+          fresh.push(txid);
+        } else {
+          isCensored[txid] = true;
+        }
+        displacedWeight += mempool[txid]?.weight || 0;
+      } else {
+        matchedWeight += mempool[txid]?.weight || 0;
+      }
+      projectedWeight += mempool[txid]?.weight || 0;
+      inTemplate[txid] = true;
+    }
+
+    if (transactions[0]) {
+      displacedWeight += (4000 - transactions[0].weight);
+      projectedWeight += transactions[0].weight;
+      matchedWeight += transactions[0].weight;
+    }
+
+    // we can expect an honest miner to include 'displaced' transactions in place of recent arrivals and censored txs
+    // these displaced transactions should occupy the first N weight units of the next projected block
+    let displacedWeightRemaining = displacedWeight + 4000;
+    let index = 0;
+    let lastFeeRate = Infinity;
+    let failures = 0;
+    let blockIndex = 1;
+    while (projectedBlocks[blockIndex] && failures < 500) {
+      const txid = projectedBlocks[blockIndex].transactionIds[index];
+      const tx = mempool[txid];
+      if (tx) {
+        const fits = (tx.weight - displacedWeightRemaining) < 4000;
+        // 0.005 margin of error for any remaining vsize rounding issues
+        const feeMatches = tx.effectiveFeePerVsize >= (lastFeeRate - 0.005);
+        if (fits || feeMatches) {
+          isDisplaced[txid] = true;
+          if (fits) {
+            // (tx.effectiveFeePerVsize * tx.vsize) / Math.ceil(tx.vsize) attempts to correct for vsize rounding in the simple non-CPFP case
+            lastFeeRate = Math.min(lastFeeRate, (tx.effectiveFeePerVsize * tx.vsize) / Math.ceil(tx.vsize));
+          }
+          if (tx.firstSeen == null || (now - (tx?.firstSeen || 0)) > PROPAGATION_MARGIN) {
+            displacedWeightRemaining -= tx.weight;
+          }
+          failures = 0;
+        } else {
+          failures++;
+        }
+      } else {
+        logger.warn('projected transaction missing from mempool cache');
+      }
+      index++;
+      if (index >= projectedBlocks[blockIndex].transactionIds.length) {
+        index = 0;
+        blockIndex++;
+      }
+    }
+
+    // mark unexpected transactions in the mined block as 'added'
+    let overflowWeight = 0;
+    let totalWeight = 0;
+    for (const tx of transactions) {
+      if (inTemplate[tx.txid]) {
+        matches.push(tx.txid);
+      } else {
+        if (rbfCache.has(tx.txid)) {
+          rbf.push(tx.txid);
+          if (!mempool[tx.txid] && !rbfCache.getReplacedBy(tx.txid)) {
+            unseen.push(tx.txid);
+          }
+        } else {
+          if (mempool[tx.txid]) {
+            if (isDisplaced[tx.txid]) {
+              added.push(tx.txid);
+            }
+          } else {
+            unseen.push(tx.txid);
+          }
+        }
+        overflowWeight += tx.weight;
+      }
+      totalWeight += tx.weight;
+    }
+
+    ({ prioritized, deprioritized } = transactionUtils.identifyPrioritizedTransactions(transactions, 'effectiveFeePerVsize'));
+
+    // transactions missing from near the end of our template are probably not being censored
+    let overflowWeightRemaining = overflowWeight - (config.MEMPOOL.BLOCK_WEIGHT_UNITS - totalWeight);
+    let maxOverflowRate = 0;
+    let rateThreshold = 0;
+    index = projectedBlocks[0].transactionIds.length - 1;
+    while (index >= 0) {
+      const txid = projectedBlocks[0].transactionIds[index];
+      const tx = mempool[txid];
+      if (tx) {
+        if (overflowWeightRemaining > 0) {
+          if (isCensored[txid]) {
+            delete isCensored[txid];
+          }
+          if (tx.effectiveFeePerVsize > maxOverflowRate) {
+            maxOverflowRate = tx.effectiveFeePerVsize;
+            rateThreshold = (Math.ceil(maxOverflowRate * 100) / 100) + 0.005;
+          }
+        } else if (tx.effectiveFeePerVsize <= rateThreshold) { // tolerance of 0.01 sat/vb + rounding
+          if (isCensored[txid]) {
+            delete isCensored[txid];
+          }
+        }
+        overflowWeightRemaining -= (mempool[txid]?.weight || 0);
+      } else {
+        logger.warn('projected transaction missing from mempool cache');
+      }
+      index--;
+    }
+
+    const numCensored = Object.keys(isCensored).length;
+    const numMatches = matches.length - 1; // adjust for coinbase tx
+    let score = 0;
+    if (numMatches <= 0 && numCensored <= 0) {
+      score = 1;
+    } else if (numMatches > 0) {
+      score = (numMatches / (numMatches + numCensored));
+    }
+    const similarity = projectedWeight ? matchedWeight / projectedWeight : 1;
+
+    return {
+      unseen,
+      censored: Object.keys(isCensored),
+      added,
+      prioritized,
+      fresh,
+      sigop: [],
+      fullrbf: rbf,
+      accelerated,
+      score,
+      similarity,
+    };
+  }
+}
+
+export default new Audit();

backend/src/api/backend-info.ts

@@ -1,46 +1,40 @@
-import * as fs from 'fs';
-import * as os from 'os';
-import logger from '../logger';
+import fs from 'fs';
+import path from 'path';
+import os from 'os';
 import { IBackendInfo } from '../mempool.interfaces';
+import config from '../config';
 
 class BackendInfo {
-  private gitCommitHash = '';
-  private hostname = '';
-  private version = '';
+  private backendInfo: IBackendInfo;
 
   constructor() {
-    this.setLatestCommitHash();
-    this.setVersion();
-    this.hostname = os.hostname();
-  }
-
-  public getBackendInfo(): IBackendInfo {
-    return {
-      hostname: this.hostname,
-      gitCommit: this.gitCommitHash,
-      version: this.version,
+    // This file is created by ./fetch-version.ts during building
+    const versionFile = path.join(__dirname, 'version.json');
+    let versionInfo;
+    if (fs.existsSync(versionFile)) {
+      versionInfo = JSON.parse(fs.readFileSync(versionFile).toString());
+    } else {
+      // Use dummy values if `versionFile` doesn't exist (e.g., during testing)
+      versionInfo = {
+        version: '?',
+        gitCommit: '?'
+      };
+    }
+    this.backendInfo = {
+      hostname: os.hostname(),
+      version: versionInfo.version,
+      gitCommit: versionInfo.gitCommit,
+      lightning: config.LIGHTNING.ENABLED,
+      backend: config.MEMPOOL.BACKEND,
     };
   }
 
-  public getShortCommitHash() {
-    return this.gitCommitHash.slice(0, 7);
+  public getBackendInfo(): IBackendInfo {
+    return this.backendInfo;
   }
 
-  private setLatestCommitHash(): void {
-    try {
-      this.gitCommitHash = fs.readFileSync('../.git/refs/heads/master').toString().trim();
-    } catch (e) {
-      logger.err('Could not load git commit info: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  private setVersion(): void {
-    try {
-      const packageJson = fs.readFileSync('package.json').toString();
-      this.version = JSON.parse(packageJson).version;
-    } catch (e) {
-      throw new Error(e instanceof Error ? e.message : 'Error');
-    }
+  public getShortCommitHash(): string {
+    return this.backendInfo.gitCommit.slice(0, 7);
   }
 }
 

backend/src/api/bisq/bisq.ts

@@ -1,277 +0,0 @@
-import config from '../../config';
-import * as fs from 'fs';
-import axios from 'axios';
-import { BisqBlocks, BisqBlock, BisqTransaction, BisqStats, BisqTrade } from './interfaces';
-import { Common } from '../common';
-import { BlockExtended } from '../../mempool.interfaces';
-import { StaticPool } from 'node-worker-threads-pool';
-import logger from '../../logger';
-
-class Bisq {
-  private static BLOCKS_JSON_FILE_PATH = config.BISQ.DATA_PATH + '/json/all/blocks.json';
-  private latestBlockHeight = 0;
-  private blocks: BisqBlock[] = [];
-  private allBlocks: BisqBlock[] = [];
-  private transactions: BisqTransaction[] = [];
-  private transactionIndex: { [txId: string]: BisqTransaction } = {};
-  private blockIndex: { [hash: string]: BisqBlock } = {};
-  private addressIndex: { [address: string]: BisqTransaction[] } = {};
-  private stats: BisqStats = {
-    minted: 0,
-    burnt: 0,
-    addresses: 0,
-    unspent_txos: 0,
-    spent_txos: 0,
-  };
-  private price: number = 0;
-  private priceUpdateCallbackFunction: ((price: number) => void) | undefined;
-  private topDirectoryWatcher: fs.FSWatcher | undefined;
-  private subdirectoryWatcher: fs.FSWatcher | undefined;
-  private jsonParsePool = new StaticPool({
-    size: 4,
-    task: (blob: string) => JSON.parse(blob),
-  });
-
-  constructor() {}
-
-  startBisqService(): void {
-    this.checkForBisqDataFolder();
-    this.loadBisqDumpFile();
-    setInterval(this.updatePrice.bind(this), 1000 * 60 * 60);
-    this.updatePrice();
-    this.startTopDirectoryWatcher();
-    this.startSubDirectoryWatcher();
-  }
-
-  handleNewBitcoinBlock(block: BlockExtended): void {
-    if (block.height - 10 > this.latestBlockHeight && this.latestBlockHeight !== 0) {
-      logger.warn(`Bitcoin block height (#${block.height}) has diverged from the latest Bisq block height (#${this.latestBlockHeight}). Restarting watchers...`);
-      this.startTopDirectoryWatcher();
-      this.startSubDirectoryWatcher();
-    }
-  }
-
-  getTransaction(txId: string): BisqTransaction | undefined {
-    return this.transactionIndex[txId];
-  }
-
-  getTransactions(start: number, length: number, types: string[]): [BisqTransaction[], number] {
-    let transactions = this.transactions;
-    if (types.length) {
-      transactions = transactions.filter((tx) => types.indexOf(tx.txType) > -1);
-    }
-    return [transactions.slice(start, length + start), transactions.length];
-  }
-
-  getBlock(hash: string): BisqBlock | undefined {
-    return this.blockIndex[hash];
-  }
-
-  getAddress(hash: string): BisqTransaction[] {
-    return this.addressIndex[hash];
-  }
-
-  getBlocks(start: number, length: number): [BisqBlock[], number] {
-    return [this.blocks.slice(start, length + start), this.blocks.length];
-  }
-
-  getStats(): BisqStats {
-    return this.stats;
-  }
-
-  setPriceCallbackFunction(fn: (price: number) => void) {
-    this.priceUpdateCallbackFunction = fn;
-  }
-
-  getLatestBlockHeight(): number {
-    return this.latestBlockHeight;
-  }
-
-  private checkForBisqDataFolder() {
-    if (!fs.existsSync(Bisq.BLOCKS_JSON_FILE_PATH)) {
-      logger.warn(Bisq.BLOCKS_JSON_FILE_PATH + ` doesn't exist. Make sure Bisq is running and the config is correct before starting the server.`);
-      return process.exit(1);
-    }
-  }
-
-  private startTopDirectoryWatcher() {
-    if (this.topDirectoryWatcher) {
-      this.topDirectoryWatcher.close();
-    }
-    let fsWait: NodeJS.Timeout | null = null;
-    this.topDirectoryWatcher = fs.watch(config.BISQ.DATA_PATH + '/json', () => {
-      if (fsWait) {
-        clearTimeout(fsWait);
-      }
-      if (this.subdirectoryWatcher) {
-        this.subdirectoryWatcher.close();
-      }
-      fsWait = setTimeout(() => {
-        logger.debug(`Bisq restart detected. Resetting both watchers in 3 minutes.`);
-        setTimeout(() => {
-          this.startTopDirectoryWatcher();
-          this.startSubDirectoryWatcher();
-          this.loadBisqDumpFile();
-        }, 180000);
-      }, 15000);
-    });
-  }
-
-  private startSubDirectoryWatcher() {
-    if (this.subdirectoryWatcher) {
-      this.subdirectoryWatcher.close();
-    }
-    if (!fs.existsSync(Bisq.BLOCKS_JSON_FILE_PATH)) {
-      logger.warn(Bisq.BLOCKS_JSON_FILE_PATH + ` doesn't exist. Trying to restart sub directory watcher again in 3 minutes.`);
-      setTimeout(() => this.startSubDirectoryWatcher(), 180000);
-      return;
-    }
-    let fsWait: NodeJS.Timeout | null = null;
-    this.subdirectoryWatcher = fs.watch(config.BISQ.DATA_PATH + '/json/all', () => {
-      if (fsWait) {
-        clearTimeout(fsWait);
-      }
-      fsWait = setTimeout(() => {
-        logger.debug(`Change detected in the Bisq data folder.`);
-        this.loadBisqDumpFile();
-      }, 2000);
-    });
-  }
-
-  private updatePrice() {
-    axios.get<BisqTrade[]>('https://bisq.markets/api/trades/?market=bsq_btc', { timeout: 10000 })
-      .then((response) => {
-        const prices: number[] = [];
-        response.data.forEach((trade) => {
-          prices.push(parseFloat(trade.price) * 100000000);
-        });
-        prices.sort((a, b) => a - b);
-        this.price = Common.median(prices);
-        if (this.priceUpdateCallbackFunction) {
-          this.priceUpdateCallbackFunction(this.price);
-        }
-    }).catch((err) => {
-      logger.err('Error updating Bisq market price: ' + err);
-    });
-  }
-
-  private async loadBisqDumpFile(): Promise<void> {
-    try {
-      const data = await this.loadData();
-      await this.loadBisqBlocksDump(data);
-      this.buildIndex();
-      this.calculateStats();
-    } catch (e) {
-      logger.info('loadBisqDumpFile() error.' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  private buildIndex() {
-    const start = new Date().getTime();
-    this.transactions = [];
-    this.transactionIndex = {};
-    this.addressIndex = {};
-
-    this.allBlocks.forEach((block) => {
-      /* Build block index */
-      if (!this.blockIndex[block.hash]) {
-        this.blockIndex[block.hash] = block;
-      }
-
-      /* Build transactions index */
-      block.txs.forEach((tx) => {
-        this.transactions.push(tx);
-        this.transactionIndex[tx.id] = tx;
-      });
-    });
-
-    /* Build address index */
-    this.transactions.forEach((tx) => {
-      tx.inputs.forEach((input) => {
-        if (!this.addressIndex[input.address]) {
-          this.addressIndex[input.address] = [];
-        }
-        if (this.addressIndex[input.address].indexOf(tx) === -1) {
-          this.addressIndex[input.address].push(tx);
-        }
-      });
-      tx.outputs.forEach((output) => {
-        if (!this.addressIndex[output.address]) {
-          this.addressIndex[output.address] = [];
-        }
-        if (this.addressIndex[output.address].indexOf(tx) === -1) {
-          this.addressIndex[output.address].push(tx);
-        }
-      });
-    });
-
-    const time = new Date().getTime() - start;
-    logger.debug('Bisq data index rebuilt in ' + time + ' ms');
-  }
-
-  private calculateStats() {
-    let minted = 0;
-    let burned = 0;
-    let unspent = 0;
-    let spent = 0;
-
-    this.transactions.forEach((tx) => {
-      tx.outputs.forEach((output) => {
-        if (output.opReturn) {
-          return;
-        }
-        if (output.txOutputType === 'GENESIS_OUTPUT' || output.txOutputType === 'ISSUANCE_CANDIDATE_OUTPUT' && output.isVerified) {
-          minted += output.bsqAmount;
-        }
-        if (output.isUnspent) {
-          unspent++;
-        } else {
-          spent++;
-        }
-      });
-      burned += tx['burntFee'];
-    });
-
-    this.stats = {
-      addresses: Object.keys(this.addressIndex).length,
-      minted: minted / 100,
-      burnt: burned / 100,
-      spent_txos: spent,
-      unspent_txos: unspent,
-    };
-  }
-
-  private async loadBisqBlocksDump(cacheData: string): Promise<void> {
-    const start = new Date().getTime();
-    if (cacheData && cacheData.length !== 0) {
-      logger.debug('Processing Bisq data dump...');
-      const data: BisqBlocks = await this.jsonParsePool.exec(cacheData);
-      if (data.blocks && data.blocks.length !== this.allBlocks.length) {
-        this.allBlocks = data.blocks;
-        this.allBlocks.reverse();
-        this.blocks = this.allBlocks.filter((block) => block.txs.length > 0);
-        this.latestBlockHeight = data.chainHeight;
-        const time = new Date().getTime() - start;
-        logger.debug('Bisq dump processed in ' + time + ' ms (worker thread)');
-      } else {
-        throw new Error(`Bisq dump didn't contain any blocks`);
-      }
-    }
-  }
-
-  private loadData(): Promise<string> {
-    return new Promise((resolve, reject) => {
-      if (!fs.existsSync(Bisq.BLOCKS_JSON_FILE_PATH)) {
-        return reject(Bisq.BLOCKS_JSON_FILE_PATH + ` doesn't exist`);
-      }
-      fs.readFile(Bisq.BLOCKS_JSON_FILE_PATH, 'utf8', (err, data) => {
-        if (err) {
-          reject(err);
-        }
-        resolve(data);
-      });
-    });
-  }
-}
-
-export default new Bisq();

backend/src/api/bisq/interfaces.ts

@@ -1,258 +0,0 @@
-
-export interface BisqBlocks {
-  chainHeight: number;
-  blocks: BisqBlock[];
-}
-
-export interface BisqBlock {
-  height: number;
-  time: number;
-  hash: string;
-  previousBlockHash: string;
-  txs: BisqTransaction[];
-}
-
-export interface BisqTransaction {
-  txVersion: string;
-  id: string;
-  blockHeight: number;
-  blockHash: string;
-  time: number;
-  inputs: BisqInput[];
-  outputs: BisqOutput[];
-  txType: string;
-  txTypeDisplayString: string;
-  burntFee: number;
-  invalidatedBsq: number;
-  unlockBlockHeight: number;
-}
-
-export interface BisqStats {
-  minted: number;
-  burnt: number;
-  addresses: number;
-  unspent_txos: number;
-  spent_txos: number;
-}
-
-interface BisqInput {
-  spendingTxOutputIndex: number;
-  spendingTxId: string;
-  bsqAmount: number;
-  isVerified: boolean;
-  address: string;
-  time: number;
-}
-
-interface BisqOutput {
-  txVersion: string;
-  txId: string;
-  index: number;
-  bsqAmount: number;
-  btcAmount: number;
-  height: number;
-  isVerified: boolean;
-  burntFee: number;
-  invalidatedBsq: number;
-  address: string;
-  scriptPubKey: BisqScriptPubKey;
-  time: any;
-  txType: string;
-  txTypeDisplayString: string;
-  txOutputType: string;
-  txOutputTypeDisplayString: string;
-  lockTime: number;
-  isUnspent: boolean;
-  spentInfo: SpentInfo;
-  opReturn?: string;
-}
-
-interface BisqScriptPubKey {
-  addresses: string[];
-  asm: string;
-  hex: string;
-  reqSigs?: number;
-  type: string;
-}
-
-interface SpentInfo {
-  height: number;
-  inputIndex: number;
-  txId: string;
-}
-
-export interface BisqTrade {
-  direction: string;
-  price: string;
-  amount: string;
-  volume: string;
-  payment_method: string;
-  trade_id: string;
-  trade_date: number;
-  market?: string;
-}
-
-export interface Currencies { [txid: string]: Currency; }
-
-export interface Currency {
-  code: string;
-  name: string;
-  precision: number;
-
-  _type: string;
-}
-
-export interface Depth { [market: string]: Market; }
-
-interface Market {
-  'buys': string[];
-  'sells': string[];
-}
-
-export interface HighLowOpenClose {
-  period_start: number | string;
-  open: string;
-  high: string;
-  low: string;
-  close: string;
-  volume_left: string;
-  volume_right: string;
-  avg: string;
-}
-
-export interface Markets { [txid: string]: Pair; }
-
-interface Pair {
-  pair: string;
-  lname: string;
-  rname: string;
-  lsymbol: string;
-  rsymbol: string;
-  lprecision: number;
-  rprecision: number;
-  ltype: string;
-  rtype: string;
-  name: string;
-}
-
-export interface Offers { [market: string]: OffersMarket; }
-
-interface OffersMarket {
-  buys: Offer[] | null;
-  sells: Offer[] | null;
-}
-
-export interface OffersData {
-  direction: string;
-  currencyCode: string;
-  minAmount: number;
-  amount: number;
-  price: number;
-  date: number;
-  useMarketBasedPrice: boolean;
-  marketPriceMargin: number;
-  paymentMethod: string;
-  id: string;
-  currencyPair: string;
-  primaryMarketDirection: string;
-  priceDisplayString: string;
-  primaryMarketAmountDisplayString: string;
-  primaryMarketMinAmountDisplayString: string;
-  primaryMarketVolumeDisplayString: string;
-  primaryMarketMinVolumeDisplayString: string;
-  primaryMarketPrice: number;
-  primaryMarketAmount: number;
-  primaryMarketMinAmount: number;
-  primaryMarketVolume: number;
-  primaryMarketMinVolume: number;
-}
-
-export interface Offer {
-  offer_id: string;
-  offer_date: number;
-  direction: string;
-  min_amount: string;
-  amount: string;
-  price: string;
-  volume: string;
-  payment_method: string;
-  offer_fee_txid: any;
-}
-
-export interface Tickers { [market: string]: Ticker | null; }
-
-export interface Ticker {
-  last: string;
-  high: string;
-  low: string;
-  volume_left: string;
-  volume_right: string;
-  buy: string | null;
-  sell: string | null;
-}
-
-export interface Trade {
-  direction: string;
-  price: string;
-  amount: string;
-  volume: string;
-  payment_method: string;
-  trade_id: string;
-  trade_date: number;
-}
-
-export interface TradesData {
-  currency: string;
-  direction: string;
-  tradePrice: number;
-  tradeAmount: number;
-  tradeDate: number;
-  paymentMethod: string;
-  offerDate: number;
-  useMarketBasedPrice: boolean;
-  marketPriceMargin: number;
-  offerAmount: number;
-  offerMinAmount: number;
-  offerId: string;
-  depositTxId?: string;
-  currencyPair: string;
-  primaryMarketDirection: string;
-  primaryMarketTradePrice: number;
-  primaryMarketTradeAmount: number;
-  primaryMarketTradeVolume: number;
-
-  _market: string;
-  _tradePriceStr: string;
-  _tradeAmountStr: string;
-  _tradeVolumeStr: string;
-  _offerAmountStr: string;
-  _tradePrice: number;
-  _tradeAmount: number;
-  _tradeVolume: number;
-  _offerAmount: number;
-}
-
-export interface MarketVolume {
-  period_start: number;
-  num_trades: number;
-  volume: string;
-}
-
-export interface MarketsApiError {
-  success: number;
-  error: string;
-}
-
-export type Interval = 'minute' | 'half_hour' | 'hour' | 'half_day' | 'day' | 'week' | 'month' | 'year' | 'auto';
-
-export interface SummarizedIntervals { [market: string]: SummarizedInterval; }
-export interface SummarizedInterval {
-  'period_start': number;
-  'open': number;
-  'close': number;
-  'high': number;
-  'low': number;
-  'avg': number;
-  'volume_right': number;
-  'volume_left': number;
-}

backend/src/api/bisq/markets-api.ts

@@ -1,679 +0,0 @@
-import { Currencies, OffersData, TradesData, Depth, Currency, Interval, HighLowOpenClose,
-  Markets, Offers, Offer, BisqTrade, MarketVolume, Tickers, Ticker, SummarizedIntervals, SummarizedInterval } from './interfaces';
-
-import * as datetime from 'locutus/php/datetime';
-
-class BisqMarketsApi {
-  private cryptoCurrencyData: Currency[] = [];
-  private fiatCurrencyData: Currency[] = [];
-  private activeCryptoCurrencyData: Currency[] = [];
-  private activeFiatCurrencyData: Currency[] = [];
-  private offersData: OffersData[] = [];
-  private tradesData: TradesData[] = [];
-  private fiatCurrenciesIndexed: { [code: string]: true } = {};
-  private allCurrenciesIndexed: { [code: string]: Currency } = {};
-  private tradeDataByMarket: { [market: string]: TradesData[] } = {};
-  private tickersCache: Ticker | Tickers | null = null;
-
-  constructor() { }
-
-  setOffersData(offers: OffersData[]) {
-    this.offersData = offers;
-  }
-
-  setTradesData(trades: TradesData[]) {
-    this.tradesData = trades;
-    this.tradeDataByMarket = {};
-
-    this.tradesData.forEach((trade) => {
-      trade._market = trade.currencyPair.toLowerCase().replace('/', '_');
-      if (!this.tradeDataByMarket[trade._market]) {
-        this.tradeDataByMarket[trade._market] = [];
-      }
-      this.tradeDataByMarket[trade._market].push(trade);
-    });
-  }
-
-  setCurrencyData(cryptoCurrency: Currency[], fiatCurrency: Currency[], activeCryptoCurrency: Currency[], activeFiatCurrency: Currency[]) {
-    this.cryptoCurrencyData = cryptoCurrency,
-    this.fiatCurrencyData = fiatCurrency,
-    this.activeCryptoCurrencyData = activeCryptoCurrency,
-    this.activeFiatCurrencyData = activeFiatCurrency;
-
-    this.fiatCurrenciesIndexed = {};
-    this.allCurrenciesIndexed = {};
-
-    this.fiatCurrencyData.forEach((currency) => {
-      currency._type = 'fiat';
-      this.fiatCurrenciesIndexed[currency.code] = true;
-      this.allCurrenciesIndexed[currency.code] = currency;
-    });
-    this.cryptoCurrencyData.forEach((currency) => {
-       currency._type = 'crypto';
-       this.allCurrenciesIndexed[currency.code] = currency;
-    });
-  }
-
-  updateCache() {
-    this.tickersCache = null;
-    this.tickersCache = this.getTicker();
-  }
-
-  getCurrencies(
-    type: 'crypto' | 'fiat' | 'active' | 'all' = 'all',
-  ): Currencies {
-    let currencies: Currency[];
-
-    switch (type) {
-      case 'fiat':
-        currencies = this.fiatCurrencyData;
-        break;
-      case 'crypto':
-        currencies = this.cryptoCurrencyData;
-        break;
-      case 'active':
-        currencies = this.activeCryptoCurrencyData.concat(this.activeFiatCurrencyData);
-        break;
-      case 'all':
-      default:
-        currencies = this.cryptoCurrencyData.concat(this.fiatCurrencyData);
-    }
-    const result = {};
-    currencies.forEach((currency) => {
-      result[currency.code] = currency;
-    });
-    return result;
-  }
-
-  getDepth(
-    market: string,
-  ): Depth {
-    const currencyPair = market.replace('_', '/').toUpperCase();
-
-    const buys = this.offersData
-      .filter((offer) => offer.currencyPair === currencyPair && offer.primaryMarketDirection === 'BUY')
-      .map((offer) => offer.price)
-      .sort((a, b) => b - a)
-      .map((price) => this.intToBtc(price));
-
-    const sells = this.offersData
-      .filter((offer) => offer.currencyPair === currencyPair && offer.primaryMarketDirection === 'SELL')
-      .map((offer) => offer.price)
-      .sort((a, b) => a - b)
-      .map((price) => this.intToBtc(price));
-
-    const result = {};
-    result[market] = {
-      'buys': buys,
-      'sells': sells,
-    };
-    return result;
-  }
-
-  getOffers(
-    market: string,
-    direction?: 'buy' | 'sell',
-  ): Offers {
-    const currencyPair = market.replace('_', '/').toUpperCase();
-
-    let buys: Offer[] | null = null;
-    let sells: Offer[] | null = null;
-
-    if (!direction || direction === 'buy') {
-      buys = this.offersData
-        .filter((offer) => offer.currencyPair === currencyPair && offer.primaryMarketDirection === 'BUY')
-        .sort((a, b) => b.price - a.price)
-        .map((offer) => this.offerDataToOffer(offer, market));
-    }
-
-    if (!direction || direction === 'sell') {
-      sells = this.offersData
-        .filter((offer) => offer.currencyPair === currencyPair && offer.primaryMarketDirection === 'SELL')
-        .sort((a, b) => a.price - b.price)
-        .map((offer) => this.offerDataToOffer(offer, market));
-    }
-
-    const result: Offers = {};
-    result[market] = {
-      'buys': buys,
-      'sells': sells,
-    };
-    return result;
-  }
-
-  getMarkets(): Markets {
-    const allCurrencies = this.getCurrencies();
-    const activeCurrencies = this.getCurrencies('active');
-    const markets = {};
-
-    for (const currency of Object.keys(activeCurrencies)) {
-      if (allCurrencies[currency].code === 'BTC') {
-        continue;
-      }
-
-      const isFiat = allCurrencies[currency]._type === 'fiat';
-      const pmarketname = allCurrencies['BTC']['name'];
-
-      const lsymbol = isFiat ? 'BTC' : currency;
-      const rsymbol = isFiat ? currency : 'BTC';
-      const lname = isFiat ? pmarketname : allCurrencies[currency].name;
-      const rname = isFiat ? allCurrencies[currency].name : pmarketname;
-      const ltype = isFiat ? 'crypto' : allCurrencies[currency]._type;
-      const rtype = isFiat ? 'fiat' : 'crypto';
-      const lprecision = 8;
-      const rprecision = isFiat ? 2 : 8;
-      const pair = lsymbol.toLowerCase() + '_' + rsymbol.toLowerCase();
-
-      markets[pair] = {
-        'pair': pair,
-        'lname': lname,
-        'rname': rname,
-        'lsymbol': lsymbol,
-        'rsymbol': rsymbol,
-        'lprecision': lprecision,
-        'rprecision': rprecision,
-        'ltype': ltype,
-        'rtype': rtype,
-        'name': lname + '/' + rname,
-      };
-    }
-
-    return markets;
-  }
-
-  getTrades(
-    market: string,
-    timestamp_from?: number,
-    timestamp_to?: number,
-    trade_id_from?: string,
-    trade_id_to?: string,
-    direction?: 'buy' | 'sell',
-    limit: number = 100,
-    sort: 'asc' | 'desc' = 'desc',
-  ): BisqTrade[] {
-      limit = Math.min(limit, 2000);
-      const _market = market === 'all' ? undefined : market;
-
-      if (!timestamp_from) {
-        timestamp_from = new Date('2016-01-01').getTime() / 1000;
-      }
-      if (!timestamp_to) {
-        timestamp_to = new Date().getTime() / 1000;
-      }
-
-      const matches = this.getTradesByCriteria(_market, timestamp_to, timestamp_from,
-        trade_id_to, trade_id_from, direction, sort, limit, false);
-
-      if (sort === 'asc') {
-        matches.sort((a, b) => a.tradeDate - b.tradeDate);
-      } else {
-        matches.sort((a, b) => b.tradeDate - a.tradeDate);
-      }
-
-      return matches.map((trade) => {
-        const bsqTrade: BisqTrade = {
-          direction: trade.primaryMarketDirection,
-          price: trade._tradePriceStr,
-          amount: trade._tradeAmountStr,
-          volume: trade._tradeVolumeStr,
-          payment_method: trade.paymentMethod,
-          trade_id: trade.offerId,
-          trade_date: trade.tradeDate,
-        };
-        if (market === 'all') {
-          bsqTrade.market = trade._market;
-        }
-        return bsqTrade;
-      });
-  }
-
-  getVolumes(
-    market?: string,
-    timestamp_from?: number,
-    timestamp_to?: number,
-    interval: Interval = 'auto',
-    milliseconds?: boolean,
-    timestamp: 'no' | 'yes' = 'yes',
-  ): MarketVolume[] {
-    if (milliseconds) {
-      timestamp_from = timestamp_from ? timestamp_from / 1000 : timestamp_from;
-      timestamp_to = timestamp_to ? timestamp_to / 1000 : timestamp_to;
-    }
-    if (!timestamp_from) {
-      timestamp_from = new Date('2016-01-01').getTime() / 1000;
-    }
-    if (!timestamp_to) {
-      timestamp_to = new Date().getTime() / 1000;
-    }
-
-    const trades = this.getTradesByCriteria(market, timestamp_to, timestamp_from,
-      undefined, undefined, undefined, 'asc', Number.MAX_SAFE_INTEGER);
-
-    if (interval === 'auto') {
-      const range = timestamp_to - timestamp_from;
-      interval = this.getIntervalFromRange(range);
-    }
-
-    const intervals: any = {};
-    const marketVolumes: MarketVolume[] = [];
-
-    for (const trade of trades) {
-      const traded_at = trade['tradeDate'] / 1000;
-      const interval_start = this.intervalStart(traded_at, interval);
-
-      if (!intervals[interval_start]) {
-        intervals[interval_start] = {
-          'volume': 0,
-          'num_trades': 0,
-        };
-      }
-
-      const period = intervals[interval_start];
-      period['period_start'] = interval_start;
-      period['volume'] += this.fiatCurrenciesIndexed[trade.currency] ? trade._tradeAmount : trade._tradeVolume;
-      period['num_trades']++;
-    }
-
-    for (const p in intervals) {
-      if (intervals.hasOwnProperty(p)) {
-        const period = intervals[p];
-        marketVolumes.push({
-          period_start: timestamp === 'no' ? new Date(period['period_start'] * 1000).toISOString() : period['period_start'],
-          num_trades: period['num_trades'],
-          volume: this.intToBtc(period['volume']),
-        });
-      }
-    }
-
-    return marketVolumes;
-  }
-
-  getTicker(
-    market?: string,
-  ): Tickers | Ticker | null {
-    if (market) {
-      return this.getTickerFromMarket(market);
-    }
-
-    if (this.tickersCache) {
-      return this.tickersCache;
-    }
-
-    const allMarkets = this.getMarkets();
-    const tickers = {};
-    for (const m in allMarkets) {
-      if (allMarkets.hasOwnProperty(m)) {
-        tickers[allMarkets[m].pair] = this.getTickerFromMarket(allMarkets[m].pair);
-      }
-    }
-
-    return tickers;
-  }
-
-  getTickerFromMarket(market: string): Ticker | null {
-    let ticker: Ticker;
-    const timestamp_from = datetime.strtotime('-24 hour');
-    const timestamp_to = new Date().getTime() / 1000;
-    const trades = this.getTradesByCriteria(market, timestamp_to, timestamp_from,
-      undefined, undefined, undefined, 'asc', Number.MAX_SAFE_INTEGER);
-
-    const periods: SummarizedInterval[] = Object.values(this.getTradesSummarized(trades, timestamp_from));
-
-    const allCurrencies = this.getCurrencies();
-    const currencyRight = allCurrencies[market.split('_')[1].toUpperCase()];
-
-    if (periods[0]) {
-      ticker = {
-        'last': this.intToBtc(periods[0].close),
-        'high': this.intToBtc(periods[0].high),
-        'low': this.intToBtc(periods[0].low),
-        'volume_left': this.intToBtc(periods[0].volume_left),
-        'volume_right': this.intToBtc(periods[0].volume_right),
-        'buy': null,
-        'sell': null,
-      };
-    } else {
-      const lastTrade = this.tradeDataByMarket[market];
-      if (!lastTrade) {
-        return null;
-      }
-      const tradePrice = lastTrade[0].primaryMarketTradePrice * Math.pow(10, 8 - currencyRight.precision);
-
-      const lastTradePrice = this.intToBtc(tradePrice);
-      ticker = {
-        'last': lastTradePrice,
-        'high': lastTradePrice,
-        'low': lastTradePrice,
-        'volume_left': '0',
-        'volume_right': '0',
-        'buy': null,
-        'sell': null,
-      };
-    }
-
-    const timestampFromMilli = timestamp_from * 1000;
-    const timestampToMilli = timestamp_to * 1000;
-
-    const currencyPair = market.replace('_', '/').toUpperCase();
-    const offersData = this.offersData.slice().sort((a, b) => a.price - b.price);
-
-    const buy = offersData.find((offer) => offer.currencyPair === currencyPair
-                                          && offer.primaryMarketDirection === 'BUY'
-                                          && offer.date >= timestampFromMilli
-                                          && offer.date <= timestampToMilli
-                                        );
-    const sell = offersData.find((offer) => offer.currencyPair === currencyPair
-                                            && offer.primaryMarketDirection === 'SELL'
-                                            && offer.date >= timestampFromMilli
-                                            && offer.date <= timestampToMilli
-                                          );
-
-    if (buy) {
-      ticker.buy = this.intToBtc(buy.primaryMarketPrice * Math.pow(10, 8 - currencyRight.precision));
-    }
-    if (sell) {
-      ticker.sell = this.intToBtc(sell.primaryMarketPrice * Math.pow(10, 8 - currencyRight.precision));
-    }
-
-    return ticker;
-  }
-
-  getHloc(
-    market: string,
-    interval: Interval = 'auto',
-    timestamp_from?: number,
-    timestamp_to?: number,
-    milliseconds?: boolean,
-    timestamp: 'no' | 'yes' = 'yes',
-  ): HighLowOpenClose[] {
-    if (milliseconds) {
-      timestamp_from = timestamp_from ? timestamp_from / 1000 : timestamp_from;
-      timestamp_to = timestamp_to ? timestamp_to / 1000 : timestamp_to;
-    }
-    if (!timestamp_from) {
-      timestamp_from = new Date('2016-01-01').getTime() / 1000;
-    }
-    if (!timestamp_to) {
-      timestamp_to = new Date().getTime() / 1000;
-    }
-
-    const trades = this.getTradesByCriteria(market, timestamp_to, timestamp_from,
-      undefined, undefined, undefined, 'asc', Number.MAX_SAFE_INTEGER);
-
-    if (interval === 'auto') {
-      const range = timestamp_to - timestamp_from;
-      interval = this.getIntervalFromRange(range);
-    }
-
-    const intervals = this.getTradesSummarized(trades, timestamp_from, interval);
-
-    const hloc: HighLowOpenClose[] = [];
-
-    for (const p in intervals) {
-      if (intervals.hasOwnProperty(p)) {
-        const period = intervals[p];
-        hloc.push({
-          period_start: timestamp === 'no' ? new Date(period['period_start'] * 1000).toISOString() : period['period_start'],
-          open: this.intToBtc(period['open']),
-          close: this.intToBtc(period['close']),
-          high: this.intToBtc(period['high']),
-          low: this.intToBtc(period['low']),
-          avg: this.intToBtc(period['avg']),
-          volume_right: this.intToBtc(period['volume_right']),
-          volume_left: this.intToBtc(period['volume_left']),
-        });
-      }
-    }
-
-    return hloc;
-  }
-
-  private getIntervalFromRange(range: number): Interval {
-    // two days range loads minute data
-    if (range <= 3600) {
-      // up to one hour range loads minutely data
-      return 'minute';
-    } else if (range <= 1 * 24 * 3600) {
-      // up to one day range loads half-hourly data
-      return 'half_hour';
-    } else if (range <= 3 * 24 * 3600) {
-      // up to 3 day range loads hourly data
-      return 'hour';
-    } else if (range <= 7 * 24 * 3600) {
-      // up to 7 day range loads half-daily data
-      return 'half_day';
-    } else if (range <= 60 * 24 * 3600) {
-      // up to 2 month range loads daily data
-      return 'day';
-    } else if (range <= 12 * 31 * 24 * 3600) {
-      // up to one year range loads weekly data
-      return 'week';
-    } else if (range <= 12 * 31 * 24 * 3600) {
-      // up to 5 year range loads monthly data
-      return 'month';
-    } else {
-      // greater range loads yearly data
-      return 'year';
-    }
-  }
-
-  getVolumesByTime(time: number): MarketVolume[] {
-    const timestamp_from = new Date().getTime() / 1000 - time;
-    const timestamp_to = new Date().getTime() / 1000;
-
-    const trades = this.getTradesByCriteria(undefined, timestamp_to, timestamp_from,
-      undefined, undefined, undefined, 'asc', Number.MAX_SAFE_INTEGER);
-
-    const markets: any = {};
-
-    for (const trade of trades) {
-      if (!markets[trade._market]) {
-        markets[trade._market] = {
-          'volume': 0,
-          'num_trades': 0,
-        };
-      }
-
-      markets[trade._market]['volume'] += this.fiatCurrenciesIndexed[trade.currency] ? trade._tradeAmount : trade._tradeVolume;
-      markets[trade._market]['num_trades']++;
-    }
-
-    return markets;
-  }
-
-  private getTradesSummarized(trades: TradesData[], timestamp_from: number, interval?: string): SummarizedIntervals {
-    const intervals: any = {};
-    const intervals_prices: any = {};
-
-    for (const trade of trades) {
-      const traded_at = trade.tradeDate / 1000;
-      const interval_start = !interval ? timestamp_from : this.intervalStart(traded_at, interval);
-
-      if (!intervals[interval_start]) {
-          intervals[interval_start] = {
-            'open': 0,
-            'close': 0,
-            'high': 0,
-            'low': 0,
-            'avg': 0,
-            'volume_right': 0,
-            'volume_left': 0,
-        };
-        intervals_prices[interval_start] = [];
-      }
-      const period = intervals[interval_start];
-      const price = trade._tradePrice;
-
-      if (!intervals_prices[interval_start]['leftvol']) {
-        intervals_prices[interval_start]['leftvol'] = [];
-      }
-      if (!intervals_prices[interval_start]['rightvol']) {
-        intervals_prices[interval_start]['rightvol'] = [];
-      }
-
-      intervals_prices[interval_start]['leftvol'].push(trade._tradeAmount);
-      intervals_prices[interval_start]['rightvol'].push(trade._tradeVolume);
-
-      if (price) {
-          const plow = period['low'];
-          period['period_start'] = interval_start;
-          period['open'] = period['open'] || price;
-          period['close'] = price;
-          period['high'] = price > period['high'] ? price : period['high'];
-          period['low'] = (plow && price > plow) ? period['low'] : price;
-          period['avg'] = intervals_prices[interval_start]['rightvol'].reduce((p: number, c: number) => c + p, 0)
-            / intervals_prices[interval_start]['leftvol'].reduce((c: number, p: number) => c + p, 0) * 100000000;
-          period['volume_left'] += trade._tradeAmount;
-          period['volume_right'] += trade._tradeVolume;
-      }
-    }
-    return intervals;
-  }
-
-  private getTradesByCriteria(
-    market: string | undefined,
-    timestamp_to: number,
-    timestamp_from: number,
-    trade_id_to: string | undefined,
-    trade_id_from: string | undefined,
-    direction: 'buy' | 'sell' | undefined,
-    sort: string,
-    limit: number,
-    integerAmounts: boolean = true,
- ): TradesData[] {
-    let trade_id_from_ts: number | null = null;
-    let trade_id_to_ts: number | null = null;
-    const allCurrencies = this.getCurrencies();
-
-    const timestampFromMilli = timestamp_from * 1000;
-    const timestampToMilli = timestamp_to * 1000;
-
-    // note: the offer_id_from/to depends on iterating over trades in
-    // descending chronological order.
-    const tradesDataSorted = this.tradesData.slice();
-    if (sort === 'asc') {
-      tradesDataSorted.reverse();
-    }
-
-    let matches: TradesData[] = [];
-    for (const trade of tradesDataSorted) {
-      if (trade_id_from === trade.offerId) {
-        trade_id_from_ts = trade.tradeDate;
-      }
-      if (trade_id_to === trade.offerId) {
-        trade_id_to_ts = trade.tradeDate;
-      }
-      if (trade_id_to && trade_id_to_ts === null) {
-        continue;
-      }
-      if (trade_id_from && trade_id_from_ts != null && trade_id_from_ts !== trade.tradeDate) {
-        continue;
-      }
-      if (market && market !== trade._market) {
-        continue;
-      }
-      if (timestampFromMilli && timestampFromMilli > trade.tradeDate) {
-        continue;
-      }
-      if (timestampToMilli && timestampToMilli < trade.tradeDate) {
-        continue;
-      }
-      if (direction && direction !== trade.direction.toLowerCase()) {
-        continue;
-      }
-
-      // Filter out bogus trades with BTC/BTC or XXX/XXX market.
-      // See github issue: https://github.com/bitsquare/bitsquare/issues/883
-      const currencyPairs = trade.currencyPair.split('/');
-      if (currencyPairs[0] === currencyPairs[1]) {
-        continue;
-      }
-
-      const currencyLeft = allCurrencies[currencyPairs[0]];
-      const currencyRight = allCurrencies[currencyPairs[1]];
-
-      if (!currencyLeft || !currencyRight) {
-        continue;
-      }
-
-      const tradePrice = trade.primaryMarketTradePrice * Math.pow(10, 8 - currencyRight.precision);
-      const tradeAmount = trade.primaryMarketTradeAmount * Math.pow(10, 8 - currencyLeft.precision);
-      const tradeVolume = trade.primaryMarketTradeVolume * Math.pow(10, 8 - currencyRight.precision);
-
-      if (integerAmounts) {
-        trade._tradePrice = tradePrice;
-        trade._tradeAmount = tradeAmount;
-        trade._tradeVolume = tradeVolume;
-        trade._offerAmount = trade.offerAmount;
-      } else {
-        trade._tradePriceStr = this.intToBtc(tradePrice);
-        trade._tradeAmountStr = this.intToBtc(tradeAmount);
-        trade._tradeVolumeStr = this.intToBtc(tradeVolume);
-        trade._offerAmountStr = this.intToBtc(trade.offerAmount);
-      }
-
-      matches.push(trade);
-
-      if (matches.length >= limit) {
-        break;
-      }
-    }
-
-    if ((trade_id_from && !trade_id_from_ts) || (trade_id_to && !trade_id_to_ts)) {
-      matches = [];
-    }
-    return matches;
-  }
-
-  private intervalStart(ts: number, interval: string): number {
-    switch (interval) {
-        case 'minute':
-            return (ts - (ts % 60));
-        case '10_minute':
-            return (ts - (ts % 600));
-        case 'half_hour':
-            return (ts - (ts % 1800));
-        case 'hour':
-            return (ts - (ts % 3600));
-        case 'half_day':
-            return (ts - (ts % (3600 * 12)));
-        case 'day':
-            return datetime.strtotime('midnight today', ts);
-        case 'week':
-            return datetime.strtotime('midnight sunday last week', ts);
-        case 'month':
-            return datetime.strtotime('midnight first day of this month', ts);
-        case 'year':
-            return datetime.strtotime('midnight first day of january', ts);
-        default:
-            throw new Error('Unsupported interval: ' + interval);
-    }
-}
-
-  private offerDataToOffer(offer: OffersData, market: string): Offer {
-    const currencyPairs = market.split('_');
-    const currencyRight = this.allCurrenciesIndexed[currencyPairs[1].toUpperCase()];
-    const currencyLeft = this.allCurrenciesIndexed[currencyPairs[0].toUpperCase()];
-    const price = offer['primaryMarketPrice'] * Math.pow( 10, 8 - currencyRight['precision']);
-    const amount = offer['primaryMarketAmount'] * Math.pow( 10, 8 - currencyLeft['precision']);
-    const volume = offer['primaryMarketVolume'] * Math.pow( 10, 8 - currencyRight['precision']);
-
-    return {
-      offer_id: offer.id,
-      offer_date: offer.date,
-      direction: offer.primaryMarketDirection,
-      min_amount: this.intToBtc(offer.minAmount),
-      amount: this.intToBtc(amount),
-      price: this.intToBtc(price),
-      volume: this.intToBtc(volume),
-      payment_method: offer.paymentMethod,
-      offer_fee_txid: null,
-    };
-  }
-
-  private intToBtc(val: number): string {
-    return (val / 100000000).toFixed(8);
-  }
-}
-
-export default new BisqMarketsApi();

backend/src/api/bisq/markets.ts

@@ -1,131 +0,0 @@
-import config from '../../config';
-import * as fs from 'fs';
-import { OffersData as OffersData, TradesData, Currency } from './interfaces';
-import bisqMarket from './markets-api';
-import logger from '../../logger';
-
-class Bisq {
-  private static FOLDER_WATCH_CHANGE_DETECTION_DEBOUNCE = 4000;
-  private static MARKET_JSON_PATH = config.BISQ.DATA_PATH;
-  private static MARKET_JSON_FILE_PATHS = {
-    activeCryptoCurrency: '/active_crypto_currency_list.json',
-    activeFiatCurrency: '/active_fiat_currency_list.json',
-    cryptoCurrency: '/crypto_currency_list.json',
-    fiatCurrency: '/fiat_currency_list.json',
-    offers: '/offers_statistics.json',
-    trades: '/trade_statistics.json',
-  };
-
-  private cryptoCurrencyLastMtime = new Date('2016-01-01');
-  private fiatCurrencyLastMtime = new Date('2016-01-01');
-  private offersLastMtime = new Date('2016-01-01');
-  private tradesLastMtime = new Date('2016-01-01');
-
-  private subdirectoryWatcher: fs.FSWatcher | undefined;
-
-  constructor() {}
-
-  startBisqService(): void {
-    this.checkForBisqDataFolder();
-    this.loadBisqDumpFile();
-    this.startBisqDirectoryWatcher();
-  }
-
-  private checkForBisqDataFolder() {
-    if (!fs.existsSync(Bisq.MARKET_JSON_PATH + Bisq.MARKET_JSON_FILE_PATHS.cryptoCurrency)) {
-      logger.err(Bisq.MARKET_JSON_PATH + Bisq.MARKET_JSON_FILE_PATHS.cryptoCurrency + ` doesn't exist. Make sure Bisq is running and the config is correct before starting the server.`);
-      return process.exit(1);
-    }
-  }
-
-  private startBisqDirectoryWatcher() {
-    if (this.subdirectoryWatcher) {
-      this.subdirectoryWatcher.close();
-    }
-    if (!fs.existsSync(Bisq.MARKET_JSON_PATH + Bisq.MARKET_JSON_FILE_PATHS.cryptoCurrency)) {
-      logger.warn(Bisq.MARKET_JSON_PATH + Bisq.MARKET_JSON_FILE_PATHS.cryptoCurrency + ` doesn't exist. Trying to restart sub directory watcher again in 3 minutes.`);
-      setTimeout(() => this.startBisqDirectoryWatcher(), 180000);
-      return;
-    }
-    let fsWait: NodeJS.Timeout | null = null;
-    this.subdirectoryWatcher = fs.watch(Bisq.MARKET_JSON_PATH, () => {
-      if (fsWait) {
-        clearTimeout(fsWait);
-      }
-      fsWait = setTimeout(() => {
-        logger.debug(`Change detected in the Bisq market data folder.`);
-        this.loadBisqDumpFile();
-      }, Bisq.FOLDER_WATCH_CHANGE_DETECTION_DEBOUNCE);
-    });
-  }
-
-  private async loadBisqDumpFile(): Promise<void> {
-    const start = new Date().getTime();
-    try {
-      let marketsDataUpdated = false;
-      const cryptoMtime = this.getFileMtime(Bisq.MARKET_JSON_FILE_PATHS.cryptoCurrency);
-      const fiatMtime = this.getFileMtime(Bisq.MARKET_JSON_FILE_PATHS.fiatCurrency);
-      if (cryptoMtime > this.cryptoCurrencyLastMtime || fiatMtime > this.fiatCurrencyLastMtime) {
-        const cryptoCurrencyData = await this.loadData<Currency[]>(Bisq.MARKET_JSON_FILE_PATHS.cryptoCurrency);
-        const fiatCurrencyData = await this.loadData<Currency[]>(Bisq.MARKET_JSON_FILE_PATHS.fiatCurrency);
-        const activeCryptoCurrencyData = await this.loadData<Currency[]>(Bisq.MARKET_JSON_FILE_PATHS.activeCryptoCurrency);
-        const activeFiatCurrencyData = await this.loadData<Currency[]>(Bisq.MARKET_JSON_FILE_PATHS.activeFiatCurrency);
-        logger.debug('Updating Bisq Market Currency Data');
-        bisqMarket.setCurrencyData(cryptoCurrencyData, fiatCurrencyData, activeCryptoCurrencyData, activeFiatCurrencyData);
-        if (cryptoMtime > this.cryptoCurrencyLastMtime) {
-          this.cryptoCurrencyLastMtime = cryptoMtime;
-        }
-        if (fiatMtime > this.fiatCurrencyLastMtime) {
-          this.fiatCurrencyLastMtime = fiatMtime;
-        }
-        marketsDataUpdated = true;
-      }
-      const offersMtime = this.getFileMtime(Bisq.MARKET_JSON_FILE_PATHS.offers);
-      if (offersMtime > this.offersLastMtime) {
-        const offersData = await this.loadData<OffersData[]>(Bisq.MARKET_JSON_FILE_PATHS.offers);
-        logger.debug('Updating Bisq Market Offers Data');
-        bisqMarket.setOffersData(offersData);
-        this.offersLastMtime = offersMtime;
-        marketsDataUpdated = true;
-      }
-      const tradesMtime = this.getFileMtime(Bisq.MARKET_JSON_FILE_PATHS.trades);
-      if (tradesMtime > this.tradesLastMtime) {
-        const tradesData = await this.loadData<TradesData[]>(Bisq.MARKET_JSON_FILE_PATHS.trades);
-        logger.debug('Updating Bisq Market Trades Data');
-        bisqMarket.setTradesData(tradesData);
-        this.tradesLastMtime = tradesMtime;
-        marketsDataUpdated = true;
-      }
-      if (marketsDataUpdated) {
-        bisqMarket.updateCache();
-        const time = new Date().getTime() - start;
-        logger.debug('Bisq market data updated in ' + time + ' ms');
-      }
-    } catch (e) {
-      logger.err('loadBisqMarketDataDumpFile() error.' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  private getFileMtime(path: string): Date {
-    const stats = fs.statSync(Bisq.MARKET_JSON_PATH + path);
-    return stats.mtime;
-  }
-
-  private loadData<T>(path: string): Promise<T> {
-    return new Promise((resolve, reject) => {
-      fs.readFile(Bisq.MARKET_JSON_PATH + path, 'utf8', (err, data) => {
-        if (err) {
-          reject(err);
-        }
-        try {
-          const parsedData = JSON.parse(data);
-          resolve(parsedData);
-        } catch (e) {
-          reject('JSON parse error (' + path + ')');
-        }
-      });
-    });
-  }
-}
-
-export default new Bisq();

backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts

@@ -1,17 +1,39 @@
+import { IBitcoinApi, SubmitPackageResult, TestMempoolAcceptResult } from './bitcoin-api.interface';
 import { IEsploraApi } from './esplora-api.interface';
 
 export interface AbstractBitcoinApi {
   $getRawMempool(): Promise<IEsploraApi.Transaction['txid'][]>;
-  $getRawTransaction(txId: string, skipConversion?: boolean, addPrevout?: boolean): Promise<IEsploraApi.Transaction>;
+  $getRawTransaction(txId: string, skipConversion?: boolean, addPrevout?: boolean, lazyPrevouts?: boolean): Promise<IEsploraApi.Transaction>;
+  $getRawTransactions(txids: string[]): Promise<IEsploraApi.Transaction[]>;
+  $getMempoolTransactions(txids: string[]): Promise<IEsploraApi.Transaction[]>;
+  $getAllMempoolTransactions(lastTxid?: string, max_txs?: number);
+  $getTransactionHex(txId: string): Promise<string>;
   $getBlockHeightTip(): Promise<number>;
+  $getBlockHashTip(): Promise<string>;
   $getTxIdsForBlock(hash: string): Promise<string[]>;
+  $getTxsForBlock(hash: string): Promise<IEsploraApi.Transaction[]>;
   $getBlockHash(height: number): Promise<string>;
   $getBlockHeader(hash: string): Promise<string>;
   $getBlock(hash: string): Promise<IEsploraApi.Block>;
+  $getRawBlock(hash: string): Promise<Buffer>;
   $getAddress(address: string): Promise<IEsploraApi.Address>;
   $getAddressTransactions(address: string, lastSeenTxId: string): Promise<IEsploraApi.Transaction[]>;
   $getAddressPrefix(prefix: string): string[];
+  $getScriptHash(scripthash: string): Promise<IEsploraApi.ScriptHash>;
+  $getScriptHashTransactions(address: string, lastSeenTxId: string): Promise<IEsploraApi.Transaction[]>;
   $sendRawTransaction(rawTransaction: string): Promise<string>;
+  $testMempoolAccept(rawTransactions: string[], maxfeerate?: number): Promise<TestMempoolAcceptResult[]>;
+  $submitPackage(rawTransactions: string[], maxfeerate?: number, maxburnamount?: number): Promise<SubmitPackageResult>;
+  $getOutspend(txId: string, vout: number): Promise<IEsploraApi.Outspend>;
+  $getOutspends(txId: string): Promise<IEsploraApi.Outspend[]>;
+  $getBatchedOutspends(txId: string[]): Promise<IEsploraApi.Outspend[][]>;
+  $getBatchedOutspendsInternal(txId: string[]): Promise<IEsploraApi.Outspend[][]>;
+  $getOutSpendsByOutpoint(outpoints: { txid: string, vout: number }[]): Promise<IEsploraApi.Outspend[]>;
+  $getCoinbaseTx(blockhash: string): Promise<IEsploraApi.Transaction>;
+  $getAddressTransactionSummary(address: string): Promise<IEsploraApi.AddressTxSummary[]>;
+
+  startHealthChecks(): void;
+  getHealthStatus(): HealthCheckHost[];
 }
 export interface BitcoinRpcCredentials {
   host: string;
@@ -19,4 +41,17 @@ export interface BitcoinRpcCredentials {
   user: string;
   pass: string;
   timeout: number;
+  cookie?: string;
+}
+
+export interface HealthCheckHost {
+  host: string;
+  active: boolean;
+  rtt: number;
+  latestHeight: number;
+  socket: boolean;
+  outOfSync: boolean;
+  unreachable: boolean;
+  checked: boolean;
+  lastChecked: number;
 }

backend/src/api/bitcoin/bitcoin-api-factory.ts

@@ -17,4 +17,6 @@ function bitcoinApiFactory(): AbstractBitcoinApi {
   }
 }
 
+export const bitcoinCoreApi = new BitcoinApi(bitcoinClient);
+
 export default bitcoinApiFactory();

backend/src/api/bitcoin/bitcoin-api.interface.ts

@@ -4,6 +4,7 @@ export namespace IBitcoinApi {
     size: number;                    //  (numeric) Current tx count
     bytes: number;                   //  (numeric) Sum of all virtual transaction sizes as defined in BIP 141.
     usage: number;                   //  (numeric) Total memory usage for the mempool
+    total_fee: number;               //  (numeric) Total fees of transactions in the mempool
     maxmempool: number;              //  (numeric) Maximum memory usage for the mempool
     mempoolminfee: number;           //  (numeric) Minimum fee rate in BTC/kB for tx to be accepted.
     minrelaytxfee: number;           //  (numeric) Current minimum relay fee for transactions
@@ -72,6 +73,14 @@ export namespace IBitcoinApi {
     time: number;                    //  (numeric) Same as blocktime
   }
 
+  export interface VerboseBlock extends Block {
+    tx: VerboseTransaction[];        // The transactions in the format of the getrawtransaction RPC. Different from verbosity = 1 "tx" result
+  }
+
+  export interface VerboseTransaction extends Transaction {
+    fee?: number;                   //  (numeric) The transaction fee in BTC, omitted if block undo data is not available
+  }
+
   export interface Vin {
     txid?: string;                   //  (string) The transaction id
     vout?: number;                   //  (string)
@@ -97,6 +106,7 @@ export namespace IBitcoinApi {
       address?: string;              //  (string) bitcoin address
       addresses?: string[];           //  (string) bitcoin addresses
       pegout_chain?: string;         //  (string) Elements peg-out chain
+      pegout_address?: string;       //  (string) Elements peg-out address
       pegout_addresses?: string[];   //  (string) Elements peg-out addresses
     };
   }
@@ -108,7 +118,7 @@ export namespace IBitcoinApi {
     scriptPubKey: string;            //  (string) The hex-encoded scriptPubKey generated by the address
     isscript: boolean;               //  (boolean) If the key is a script
     iswitness: boolean;              //  (boolean) If the address is a witness
-    witness_version?: boolean;       //  (numeric, optional) The version number of the witness program
+    witness_version?: number;        //  (numeric, optional) The version number of the witness program
     witness_program: string;         //  (string, optional) The hex value of the witness program
     confidential_key?: string;       //  (string) Elements only
     unconfidential?: string;         //  (string) Elements only
@@ -163,4 +173,66 @@ export namespace IBitcoinApi {
     }
   }
 
+  export interface BlockStats {
+    "avgfee": number;
+    "avgfeerate": number;
+    "avgtxsize": number;
+    "blockhash": string;
+    "feerate_percentiles": [number, number, number, number, number];
+    "height": number;
+    "ins": number;
+    "maxfee": number;
+    "maxfeerate": number;
+    "maxtxsize": number;
+    "medianfee": number;
+    "mediantime": number;
+    "mediantxsize": number;
+    "minfee": number;
+    "minfeerate": number;
+    "mintxsize": number;
+    "outs": number;
+    "subsidy": number;
+    "swtotal_size": number;
+    "swtotal_weight": number;
+    "swtxs": number;
+    "time": number;
+    "total_out": number;
+    "total_size": number;
+    "total_weight": number;
+    "totalfee": number;
+    "txs": number;
+    "utxo_increase": number;
+    "utxo_size_inc": number;
+  }
+}
+
+export interface TestMempoolAcceptResult {
+  txid: string,
+  wtxid: string,
+  allowed?: boolean,
+  vsize?: number,
+  fees?: {
+    base: number,
+    "effective-feerate": number,
+    "effective-includes": string[],
+  },
+  ['reject-reason']?: string,
+}
+
+export interface SubmitPackageResult {
+  package_msg: string;
+  "tx-results": { [wtxid: string]: TxResult };
+  "replaced-transactions"?: string[];
+}
+
+export interface TxResult {
+  txid: string;
+  "other-wtxid"?: string;
+  vsize?: number;
+  fees?: {
+    base: number;
+    "effective-feerate"?: number;
+    "effective-includes"?: string[];
+  };
+  error?: string;
 }

backend/src/api/bitcoin/bitcoin-api.ts

@@ -1,10 +1,11 @@
 import * as bitcoinjs from 'bitcoinjs-lib';
-import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
-import { IBitcoinApi } from './bitcoin-api.interface';
+import { AbstractBitcoinApi, HealthCheckHost } from './bitcoin-api-abstract-factory';
+import { IBitcoinApi, SubmitPackageResult, TestMempoolAcceptResult } from './bitcoin-api.interface';
 import { IEsploraApi } from './esplora-api.interface';
 import blocks from '../blocks';
 import mempool from '../mempool';
 import { TransactionExtended } from '../../mempool.interfaces';
+import transactionUtils from '../transaction-utils';
 
 class BitcoinApi implements AbstractBitcoinApi {
   private rawMempoolCache: IBitcoinApi.RawMempool | null = null;
@@ -14,33 +15,91 @@ class BitcoinApi implements AbstractBitcoinApi {
     this.bitcoindClient = bitcoinClient;
   }
 
-  $getRawTransaction(txId: string, skipConversion = false, addPrevout = false): Promise<IEsploraApi.Transaction> {
+  static convertBlock(block: IBitcoinApi.Block): IEsploraApi.Block {
+    return {
+      id: block.hash,
+      height: block.height,
+      version: block.version,
+      timestamp: block.time,
+      bits: parseInt(block.bits, 16),
+      nonce: block.nonce,
+      difficulty: block.difficulty,
+      merkle_root: block.merkleroot,
+      tx_count: block.nTx,
+      size: block.size,
+      weight: block.weight,
+      previousblockhash: block.previousblockhash,
+      mediantime: block.mediantime,
+      stale: block.confirmations === -1,
+    };
+  }
+
+
+  $getRawTransaction(txId: string, skipConversion = false, addPrevout = false, lazyPrevouts = false): Promise<IEsploraApi.Transaction> {
     // If the transaction is in the mempool we already converted and fetched the fee. Only prevouts are missing
     const txInMempool = mempool.getMempool()[txId];
     if (txInMempool && addPrevout) {
       return this.$addPrevouts(txInMempool);
     }
 
-    // Special case to fetch the Coinbase transaction
-    if (txId === '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b') {
-      return this.$returnCoinbaseTransaction();
-    }
-
     return this.bitcoindClient.getRawTransaction(txId, true)
       .then((transaction: IBitcoinApi.Transaction) => {
         if (skipConversion) {
           transaction.vout.forEach((vout) => {
-            vout.value = vout.value * 100000000;
+            vout.value = Math.round(vout.value * 100000000);
           });
           return transaction;
         }
-        return this.$convertTransaction(transaction, addPrevout);
+        return this.$convertTransaction(transaction, addPrevout, lazyPrevouts);
+      })
+      .catch((e: Error) => {
+        if (e.message.startsWith('The genesis block coinbase')) {
+          return this.$returnCoinbaseTransaction();
+        }
+        throw e;
+      });
+  }
+
+  async $getRawTransactions(txids: string[]): Promise<IEsploraApi.Transaction[]> {
+    const txs: IEsploraApi.Transaction[] = [];
+    for (const txid of txids) {
+      try {
+        const tx = await this.$getRawTransaction(txid, false, true);
+        txs.push(tx);
+      } catch (err) {
+        // skip failures
+      }
+    }
+    return txs;
+  }
+
+  $getMempoolTransactions(txids: string[]): Promise<IEsploraApi.Transaction[]> {
+    throw new Error('Method getMempoolTransactions not supported by the Bitcoin RPC API.');
+  }
+
+  $getAllMempoolTransactions(lastTxid?: string, max_txs?: number): Promise<IEsploraApi.Transaction[]> {
+    throw new Error('Method getAllMempoolTransactions not supported by the Bitcoin RPC API.');
+
+  }
+
+  async $getTransactionHex(txId: string): Promise<string> {
+    const txInMempool = mempool.getMempool()[txId];
+    if (txInMempool && txInMempool.hex) {
+      return txInMempool.hex;
+    }
+
+    return this.bitcoindClient.getRawTransaction(txId, true)
+      .then((transaction: IBitcoinApi.Transaction) => {
+        return transaction.hex;
       });
   }
 
   $getBlockHeightTip(): Promise<number> {
-    return this.bitcoindClient.getChainTips()
-      .then((result: IBitcoinApi.ChainTips[]) => result[0].height);
+    return this.bitcoindClient.getBlockCount();
+  }
+
+  $getBlockHashTip(): Promise<string> {
+    return this.bitcoindClient.getBestBlockHash();
   }
 
   $getTxIdsForBlock(hash: string): Promise<string[]> {
@@ -48,8 +107,19 @@ class BitcoinApi implements AbstractBitcoinApi {
       .then((rpcBlock: IBitcoinApi.Block) => rpcBlock.tx);
   }
 
-  $getRawBlock(hash: string): Promise<string> {
-    return this.bitcoindClient.getBlock(hash, 0);
+  async $getTxsForBlock(hash: string): Promise<IEsploraApi.Transaction[]> {
+    const verboseBlock: IBitcoinApi.VerboseBlock = await this.bitcoindClient.getBlock(hash, 2);
+    const transactions: IEsploraApi.Transaction[] = [];
+    for (const tx of verboseBlock.tx) {
+      const converted = await this.$convertTransaction(tx, true);
+      transactions.push(converted);
+    }
+    return transactions;
+  }
+
+  $getRawBlock(hash: string): Promise<Buffer> {
+    return this.bitcoindClient.getBlock(hash, 0)
+      .then((raw: string) => Buffer.from(raw, "hex"));
   }
 
   $getBlockHash(height: number): Promise<string> {
@@ -67,7 +137,7 @@ class BitcoinApi implements AbstractBitcoinApi {
     }
 
     return this.bitcoindClient.getBlock(hash)
-      .then((block: IBitcoinApi.Block) => this.convertBlock(block));
+      .then((block: IBitcoinApi.Block) => BitcoinApi.convertBlock(block));
   }
 
   $getAddress(address: string): Promise<IEsploraApi.Address> {
@@ -78,31 +148,123 @@ class BitcoinApi implements AbstractBitcoinApi {
     throw new Error('Method getAddressTransactions not supported by the Bitcoin RPC API.');
   }
 
+  $getScriptHash(scripthash: string): Promise<IEsploraApi.ScriptHash> {
+    throw new Error('Method getScriptHash not supported by the Bitcoin RPC API.');
+  }
+
+  $getScriptHashTransactions(scripthash: string, lastSeenTxId: string): Promise<IEsploraApi.Transaction[]> {
+    throw new Error('Method getScriptHashTransactions not supported by the Bitcoin RPC API.');
+  }
+
   $getRawMempool(): Promise<IEsploraApi.Transaction['txid'][]> {
     return this.bitcoindClient.getRawMemPool();
   }
 
   $getAddressPrefix(prefix: string): string[] {
-    const found: string[] = [];
+    const found: { [address: string]: string } = {};
     const mp = mempool.getMempool();
     for (const tx in mp) {
       for (const vout of mp[tx].vout) {
-        if (vout.scriptpubkey_address.indexOf(prefix) === 0) {
-          found.push(vout.scriptpubkey_address);
-          if (found.length >= 10) {
-            return found;
+        if (vout.scriptpubkey_address?.indexOf(prefix) === 0) {
+          found[vout.scriptpubkey_address] = '';
+          if (Object.keys(found).length >= 10) {
+            return Object.keys(found);
+          }
+        }
+      }
+      for (const vin of mp[tx].vin) {
+        if (vin.prevout?.scriptpubkey_address?.indexOf(prefix) === 0) {
+          found[vin.prevout?.scriptpubkey_address] = '';
+          if (Object.keys(found).length >= 10) {
+            return Object.keys(found);
           }
         }
       }
     }
-    return found;
+    return Object.keys(found);
   }
 
   $sendRawTransaction(rawTransaction: string): Promise<string> {
     return this.bitcoindClient.sendRawTransaction(rawTransaction);
   }
 
-  protected async $convertTransaction(transaction: IBitcoinApi.Transaction, addPrevout: boolean): Promise<IEsploraApi.Transaction> {
+  async $testMempoolAccept(rawTransactions: string[], maxfeerate?: number): Promise<TestMempoolAcceptResult[]> {
+    if (rawTransactions.length) {
+      return this.bitcoindClient.testMempoolAccept(rawTransactions, maxfeerate ?? undefined);
+    } else {
+      return [];
+    }
+  }
+
+  $submitPackage(rawTransactions: string[], maxfeerate?: number, maxburnamount?: number): Promise<SubmitPackageResult> {
+    return this.bitcoindClient.submitPackage(rawTransactions, maxfeerate ?? undefined, maxburnamount ?? undefined);
+  }
+
+  async $getOutspend(txId: string, vout: number): Promise<IEsploraApi.Outspend> {
+    const txOut = await this.bitcoindClient.getTxOut(txId, vout, false);
+    return {
+      spent: txOut === null,
+      status: {
+        confirmed: true,
+      }
+    };
+  }
+
+  async $getOutspends(txId: string): Promise<IEsploraApi.Outspend[]> {
+    const outSpends: IEsploraApi.Outspend[] = [];
+    const tx = await this.$getRawTransaction(txId, true, false);
+    for (let i = 0; i < tx.vout.length; i++) {
+      if (tx.status && tx.status.block_height === 0) {
+        outSpends.push({
+          spent: false
+        });
+      } else {
+        const txOut = await this.bitcoindClient.getTxOut(txId, i);
+        outSpends.push({
+          spent: txOut === null,
+        });
+      }
+    }
+    return outSpends;
+  }
+
+  async $getBatchedOutspends(txId: string[]): Promise<IEsploraApi.Outspend[][]> {
+    const outspends: IEsploraApi.Outspend[][] = [];
+    for (const tx of txId) {
+      const outspend = await this.$getOutspends(tx);
+      outspends.push(outspend);
+    }
+    return outspends;
+  }
+
+  async $getBatchedOutspendsInternal(txId: string[]): Promise<IEsploraApi.Outspend[][]> {
+    return this.$getBatchedOutspends(txId);
+  }
+
+  async $getOutSpendsByOutpoint(outpoints: { txid: string, vout: number }[]): Promise<IEsploraApi.Outspend[]> {
+    const outspends: IEsploraApi.Outspend[] = [];
+    for (const outpoint of outpoints) {
+      const outspend = await this.$getOutspend(outpoint.txid, outpoint.vout);
+      outspends.push(outspend);
+    }
+    return outspends;
+  }
+
+  async $getCoinbaseTx(blockhash: string): Promise<IEsploraApi.Transaction> {
+    const txids = await this.$getTxIdsForBlock(blockhash);
+    return this.$getRawTransaction(txids[0]);
+  }
+
+  async $getAddressTransactionSummary(address: string): Promise<IEsploraApi.AddressTxSummary[]> {
+    throw new Error('Method getAddressTransactionSummary not supported by the Bitcoin RPC API.');
+  }
+
+  $getEstimatedHashrate(blockHeight: number): Promise<number> {
+    // 120 is the default block span in Core
+    return this.bitcoindClient.getNetworkHashPs(120, blockHeight);
+  }
+
+  protected async $convertTransaction(transaction: IBitcoinApi.Transaction, addPrevout: boolean, lazyPrevouts = false): Promise<IEsploraApi.Transaction> {
     let esploraTransaction: IEsploraApi.Transaction = {
       txid: transaction.txid,
       version: transaction.version,
@@ -117,11 +279,11 @@ class BitcoinApi implements AbstractBitcoinApi {
 
     esploraTransaction.vout = transaction.vout.map((vout) => {
       return {
-        value: vout.value * 100000000,
+        value: Math.round(vout.value * 100000000),
         scriptpubkey: vout.scriptPubKey.hex,
         scriptpubkey_address: vout.scriptPubKey && vout.scriptPubKey.address ? vout.scriptPubKey.address
           : vout.scriptPubKey.addresses ? vout.scriptPubKey.addresses[0] : '',
-        scriptpubkey_asm: vout.scriptPubKey.asm ? this.convertScriptSigAsm(vout.scriptPubKey.asm) : '',
+        scriptpubkey_asm: vout.scriptPubKey.asm ? transactionUtils.convertScriptSigAsm(vout.scriptPubKey.hex) : '',
         scriptpubkey_type: this.translateScriptPubKeyType(vout.scriptPubKey.type),
       };
     });
@@ -131,11 +293,13 @@ class BitcoinApi implements AbstractBitcoinApi {
         is_coinbase: !!vin.coinbase,
         prevout: null,
         scriptsig: vin.scriptSig && vin.scriptSig.hex || vin.coinbase || '',
-        scriptsig_asm: vin.scriptSig && this.convertScriptSigAsm(vin.scriptSig.asm) || '',
+        scriptsig_asm: vin.scriptSig && transactionUtils.convertScriptSigAsm(vin.scriptSig.hex) || '',
         sequence: vin.sequence,
         txid: vin.txid || '',
         vout: vin.vout || 0,
-        witness: vin.txinwitness,
+        witness: vin.txinwitness || [],
+        inner_redeemscript_asm: '',
+        inner_witnessscript_asm: '',
       };
     });
 
@@ -148,35 +312,15 @@ class BitcoinApi implements AbstractBitcoinApi {
       };
     }
 
-    if (transaction.confirmations) {
-      esploraTransaction = await this.$calculateFeeFromInputs(esploraTransaction, addPrevout);
-    } else {
+    if (addPrevout) {
+      esploraTransaction = await this.$calculateFeeFromInputs(esploraTransaction, false, lazyPrevouts);
+    } else if (!transaction.confirmations) {
       esploraTransaction = await this.$appendMempoolFeeData(esploraTransaction);
-      if (addPrevout) {
-        esploraTransaction = await this.$calculateFeeFromInputs(esploraTransaction, addPrevout);
-      }
     }
 
     return esploraTransaction;
   }
 
-  private convertBlock(block: IBitcoinApi.Block): IEsploraApi.Block {
-    return {
-      id: block.hash,
-      height: block.height,
-      version: block.version,
-      timestamp: block.time,
-      bits: parseInt(block.bits, 16),
-      nonce: block.nonce,
-      difficulty: block.difficulty,
-      merkle_root: block.merkleroot,
-      tx_count: block.nTx,
-      size: block.size,
-      weight: block.weight,
-      previousblockhash: block.previousblockhash,
-    };
-  }
-
   private translateScriptPubKeyType(outputType: string): string {
     const map = {
       'pubkey': 'p2pk',
@@ -186,13 +330,15 @@ class BitcoinApi implements AbstractBitcoinApi {
       'witness_v0_scripthash': 'v0_p2wsh',
       'witness_v1_taproot': 'v1_p2tr',
       'nonstandard': 'nonstandard',
+      'multisig': 'multisig',
+      'anchor': 'anchor',
       'nulldata': 'op_return'
     };
 
     if (map[outputType]) {
       return map[outputType];
     } else {
-      return '';
+      return 'unknown';
     }
   }
 
@@ -209,7 +355,7 @@ class BitcoinApi implements AbstractBitcoinApi {
     } else {
       mempoolEntry = await this.$getMempoolEntry(transaction.txid);
     }
-    transaction.fee = mempoolEntry.fees.base * 100000000;
+    transaction.fee = Math.round(mempoolEntry.fees.base * 100000000);
     return transaction;
   }
 
@@ -218,20 +364,22 @@ class BitcoinApi implements AbstractBitcoinApi {
       if (vin.prevout) {
         continue;
       }
-      const innerTx = await this.$getRawTransaction(vin.txid, false);
+      const innerTx = await this.$getRawTransaction(vin.txid, false, false);
       vin.prevout = innerTx.vout[vin.vout];
-      this.addInnerScriptsToVin(vin);
+      transactionUtils.addInnerScriptsToVin(vin);
     }
     return transaction;
   }
 
   protected $returnCoinbaseTransaction(): Promise<IEsploraApi.Transaction> {
-    return this.bitcoindClient.getBlock('000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f', 2)
-      .then((block: IBitcoinApi.Block) => {
-        return this.$convertTransaction(Object.assign(block.tx[0], {
-          confirmations: blocks.getCurrentBlockHeight() + 1,
-          blocktime: 1231006505 }), false);
-      });
+    return this.bitcoindClient.getBlockHash(0).then((hash: string) =>
+      this.bitcoindClient.getBlock(hash, 2)
+        .then((block: IBitcoinApi.Block) => {
+          return this.$convertTransaction(Object.assign(block.tx[0], {
+            confirmations: blocks.getCurrentBlockHeight() + 1,
+            blocktime: block.time }), false);
+        })
+    );
   }
 
   private $getMempoolEntry(txid: string): Promise<IBitcoinApi.MempoolEntry> {
@@ -242,70 +390,38 @@ class BitcoinApi implements AbstractBitcoinApi {
     return this.bitcoindClient.getRawMemPool(true);
   }
 
-  private async $calculateFeeFromInputs(transaction: IEsploraApi.Transaction, addPrevout: boolean): Promise<IEsploraApi.Transaction> {
+
+  private async $calculateFeeFromInputs(transaction: IEsploraApi.Transaction, addPrevout: boolean, lazyPrevouts: boolean): Promise<IEsploraApi.Transaction> {
     if (transaction.vin[0].is_coinbase) {
       transaction.fee = 0;
       return transaction;
     }
     let totalIn = 0;
-    for (const vin of transaction.vin) {
-      const innerTx = await this.$getRawTransaction(vin.txid, !addPrevout);
-      if (addPrevout) {
-        vin.prevout = innerTx.vout[vin.vout];
-        this.addInnerScriptsToVin(vin);
+
+    for (let i = 0; i < transaction.vin.length; i++) {
+      if (lazyPrevouts && i > 12) {
+        transaction.vin[i].lazy = true;
+        continue;
       }
-      totalIn += innerTx.vout[vin.vout].value;
+      const innerTx = await this.$getRawTransaction(transaction.vin[i].txid, false, false);
+      transaction.vin[i].prevout = innerTx.vout[transaction.vin[i].vout];
+      transactionUtils.addInnerScriptsToVin(transaction.vin[i]);
+      totalIn += innerTx.vout[transaction.vin[i].vout].value;
+    }
+    if (lazyPrevouts && transaction.vin.length > 12) {
+      transaction.fee = -1;
+    } else {
+      const totalOut = transaction.vout.reduce((p, output) => p + output.value, 0);
+      transaction.fee = parseFloat((totalIn - totalOut).toFixed(8));
     }
-    const totalOut = transaction.vout.reduce((p, output) => p + output.value, 0);
-    transaction.fee = parseFloat((totalIn - totalOut).toFixed(8));
     return transaction;
   }
 
-  private convertScriptSigAsm(str: string): string {
-    const a = str.split(' ');
-    const b: string[] = [];
-    a.forEach((chunk) => {
-      if (chunk.substr(0, 3) === 'OP_') {
-        chunk = chunk.replace(/^OP_(\d+)/, 'OP_PUSHNUM_$1');
-        chunk = chunk.replace('OP_CHECKSEQUENCEVERIFY', 'OP_CSV');
-        b.push(chunk);
-      } else {
-        chunk = chunk.replace('[ALL]', '01');
-        if (chunk === '0') {
-          b.push('OP_0');
-        } else {
-          b.push('OP_PUSHBYTES_' + Math.round(chunk.length / 2) + ' ' + chunk);
-        }
-      }
-    });
-    return b.join(' ');
+  public startHealthChecks(): void {};
+
+  public getHealthStatus() {
+    return [];
   }
-
-  private addInnerScriptsToVin(vin: IEsploraApi.Vin): void {
-    if (!vin.prevout) {
-      return;
-    }
-
-    if (vin.prevout.scriptpubkey_type === 'p2sh') {
-      const redeemScript = vin.scriptsig_asm.split(' ').reverse()[0];
-      vin.inner_redeemscript_asm = this.convertScriptSigAsm(bitcoinjs.script.toASM(Buffer.from(redeemScript, 'hex')));
-      if (vin.witness && vin.witness.length > 2) {
-        const witnessScript = vin.witness[vin.witness.length - 1];
-        vin.inner_witnessscript_asm = this.convertScriptSigAsm(bitcoinjs.script.toASM(Buffer.from(witnessScript, 'hex')));
-      }
-    }
-
-    if (vin.prevout.scriptpubkey_type === 'v0_p2wsh' && vin.witness) {
-      const witnessScript = vin.witness[vin.witness.length - 1];
-      vin.inner_witnessscript_asm = this.convertScriptSigAsm(bitcoinjs.script.toASM(Buffer.from(witnessScript, 'hex')));
-    }
-
-    if (vin.prevout.scriptpubkey_type === 'v1_p2tr' && vin.witness && vin.witness.length > 1) {
-      const witnessScript = vin.witness[vin.witness.length - 2];
-      vin.inner_witnessscript_asm = this.convertScriptSigAsm(bitcoinjs.script.toASM(Buffer.from(witnessScript, 'hex')));
-    }
-  }
-
 }
 
 export default BitcoinApi;

backend/src/api/bitcoin/bitcoin-client.ts

@@ -1,5 +1,5 @@
 import config from '../../config';
-import * as bitcoin from '@mempool/bitcoin';
+const bitcoin = require('../../rpc-api/index');
 import { BitcoinRpcCredentials } from './bitcoin-api-abstract-factory';
 
 const nodeRpcCredentials: BitcoinRpcCredentials = {
@@ -7,7 +7,8 @@ const nodeRpcCredentials: BitcoinRpcCredentials = {
   port: config.CORE_RPC.PORT,
   user: config.CORE_RPC.USERNAME,
   pass: config.CORE_RPC.PASSWORD,
-  timeout: 60000,
+  timeout: config.CORE_RPC.TIMEOUT,
+  cookie: config.CORE_RPC.COOKIE ? config.CORE_RPC.COOKIE_PATH : undefined,
 };
 
 export default new bitcoin.Client(nodeRpcCredentials);

backend/src/api/bitcoin/bitcoin-core.routes.ts

@@ -0,0 +1,254 @@
+import { Application, NextFunction, Request, Response } from 'express';
+import logger from '../../logger';
+import bitcoinClient from './bitcoin-client';
+import config from '../../config';
+
+const BLOCKHASH_REGEX = /^[a-f0-9]{64}$/i;
+const TXID_REGEX = /^[a-f0-9]{64}$/i;
+const RAW_TX_REGEX = /^[a-f0-9]{2,}$/i;
+
+/**
+ * Define a set of routes used by the accelerator server
+ * Those routes are not designed to be public
+ */
+class BitcoinBackendRoutes {
+  private static tag = 'BitcoinBackendRoutes';
+
+  public initRoutes(app: Application): void {
+    app
+      .get(config.MEMPOOL.API_URL_PREFIX + 'internal/bitcoin-core/' + 'get-mempool-entry', this.disableCache, this.$getMempoolEntry)
+      .post(config.MEMPOOL.API_URL_PREFIX + 'internal/bitcoin-core/' + 'decode-raw-transaction', this.disableCache, this.$decodeRawTransaction)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'internal/bitcoin-core/' + 'get-raw-transaction', this.disableCache, this.$getRawTransaction)
+      .post(config.MEMPOOL.API_URL_PREFIX + 'internal/bitcoin-core/' + 'send-raw-transaction', this.disableCache, this.$sendRawTransaction)
+      .post(config.MEMPOOL.API_URL_PREFIX + 'internal/bitcoin-core/' + 'test-mempool-accept', this.disableCache, this.$testMempoolAccept)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'internal/bitcoin-core/' + 'get-mempool-ancestors', this.disableCache, this.$getMempoolAncestors)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'internal/bitcoin-core/' + 'get-block', this.disableCache, this.$getBlock)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'internal/bitcoin-core/' + 'get-block-hash', this.disableCache, this.$getBlockHash)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'internal/bitcoin-core/' + 'get-block-count', this.disableCache, this.$getBlockCount)
+    ;
+  }
+
+  /**
+   * Disable caching for bitcoin core routes
+   *
+   * @param req
+   * @param res
+   * @param next
+   */
+  private disableCache(req: Request, res: Response, next: NextFunction): void  {
+    res.setHeader('Pragma', 'no-cache');
+    res.setHeader('Cache-control', 'private, no-store, no-cache, must-revalidate, proxy-revalidate, max-age=0');
+    res.setHeader('expires', -1);
+    next();
+  }
+
+  /**
+   * Exeption handler to return proper details to the accelerator server
+   *
+   * @param e
+   * @param fnName
+   * @param res
+   */
+  private static handleException(e: any, fnName: string, res: Response): void {
+    if (typeof(e.code) === 'number') {
+      res.status(400).send(JSON.stringify(e, ['code']));
+    } else {
+      const err = `unknown exception in ${fnName}`;
+      logger.err(err, BitcoinBackendRoutes.tag);
+      res.status(500).send(err);
+    }
+  }
+
+  private async $getMempoolEntry(req: Request, res: Response): Promise<void> {
+    const txid = req.query.txid;
+    try {
+      if (typeof(txid) !== 'string' || txid.length !== 64 || !TXID_REGEX.test(txid)) {
+        res.status(400).send(`invalid param txid. must be 64 hexadecimal characters`);
+        return;
+      }
+      const mempoolEntry = await bitcoinClient.getMempoolEntry(txid);
+      if (!mempoolEntry) {
+        res.status(404).send();
+        return;
+      }
+      res.status(200).send(mempoolEntry);
+    } catch (e: any) {
+      BitcoinBackendRoutes.handleException(e, 'getMempoolEntry', res);
+    }
+  }
+
+  private async $decodeRawTransaction(req: Request, res: Response): Promise<void> {
+    const rawTx = req.body.rawTx;
+    try {
+      if (typeof(rawTx) !== 'string' || !RAW_TX_REGEX.test(rawTx)) {
+        res.status(400).send(`invalid param rawTx. must be a string of hexadecimal characters`);
+        return;
+      }
+      const decodedTx = await bitcoinClient.decodeRawTransaction(rawTx);
+      if (!decodedTx) {
+        res.status(400).send(`unable to decode rawTx`);
+        return;
+      }
+      res.status(200).send(decodedTx);
+    } catch (e: any) {
+      BitcoinBackendRoutes.handleException(e, 'decodeRawTransaction', res);
+    }
+  }
+
+  private async $getRawTransaction(req: Request, res: Response): Promise<void> {
+    const txid = req.query.txid;
+    const verbose = req.query.verbose;
+    try {
+      if (typeof(txid) !== 'string' || txid.length !== 64 || !TXID_REGEX.test(txid)) {
+        res.status(400).send(`invalid param txid. must be 64 hexadecimal characters`);
+        return;
+      }
+      if (typeof(verbose) !== 'string') {
+        res.status(400).send(`invalid param verbose. must be a string representing an integer`);
+        return;
+      }
+      const verboseNumber = parseInt(verbose, 10);
+      if (typeof(verboseNumber) !== 'number') {
+        res.status(400).send(`invalid param verbose. must be a valid integer`);
+        return;
+      }
+
+      const decodedTx = await bitcoinClient.getRawTransaction(txid, verboseNumber);
+      if (!decodedTx) {
+        res.status(400).send(`unable to get raw transaction`);
+        return;
+      }
+      res.status(200).send(decodedTx);
+    } catch (e: any) {
+      BitcoinBackendRoutes.handleException(e, 'decodeRawTransaction', res);
+    }
+  }
+
+  private async $sendRawTransaction(req: Request, res: Response): Promise<void> {
+    const rawTx = req.body.rawTx;
+    try {
+      if (typeof(rawTx) !== 'string' || !RAW_TX_REGEX.test(rawTx)) {
+        res.status(400).send(`invalid param rawTx. must be a string of hexadecimal characters`);
+        return;
+      }
+      const txHex = await bitcoinClient.sendRawTransaction(rawTx);
+      if (!txHex) {
+        res.status(400).send(`unable to send rawTx`);
+        return;
+      }
+      res.status(200).send(txHex);
+    } catch (e: any) {
+      BitcoinBackendRoutes.handleException(e, 'sendRawTransaction', res);
+    }
+  }
+
+  private async $testMempoolAccept(req: Request, res: Response): Promise<void> {
+    const rawTxs = req.body.rawTxs;
+    try {
+      if (typeof(rawTxs) !== 'object' || !Array.isArray(rawTxs) || rawTxs.some((tx) => typeof(tx) !== 'string' || !RAW_TX_REGEX.test(tx))) {
+        res.status(400).send(`invalid param rawTxs. must be an array of strings of hexadecimal characters`);
+        return;
+      }
+      const txHex = await bitcoinClient.testMempoolAccept(rawTxs);
+      if (typeof(txHex) !== 'object' || txHex.length === 0) {
+        res.status(400).send(`testmempoolaccept failed for raw txs, got an empty result`);
+        return;
+      }
+      res.status(200).send(txHex);
+    } catch (e: any) {
+      BitcoinBackendRoutes.handleException(e, 'testMempoolAccept', res);
+    }
+  }
+
+  private async $getMempoolAncestors(req: Request, res: Response): Promise<void> {
+    const txid = req.query.txid;
+    const verbose = req.query.verbose;
+    try {
+      if (typeof(txid) !== 'string' || txid.length !== 64 || !TXID_REGEX.test(txid)) {
+        res.status(400).send(`invalid param txid. must be 64 hexadecimal characters`);
+        return;
+      }
+      if (typeof(verbose) !== 'string' || (verbose !== 'true' && verbose !== 'false')) {
+        res.status(400).send(`invalid param verbose. must be a string ('true' | 'false')`);
+        return;
+      }
+
+      const ancestors = await bitcoinClient.getMempoolAncestors(txid, verbose === 'true' ? true : false);
+      if (!ancestors) {
+        res.status(400).send(`unable to get mempool ancestors`);
+        return;
+      }
+      res.status(200).send(ancestors);
+    } catch (e: any) {
+      BitcoinBackendRoutes.handleException(e, 'getMempoolAncestors', res);
+    }
+  }
+
+  private async $getBlock(req: Request, res: Response): Promise<void> {
+    const blockHash = req.query.hash;
+    const verbosity = req.query.verbosity;
+    try {
+      if (typeof(blockHash) !== 'string' || blockHash.length !== 64 || !BLOCKHASH_REGEX.test(blockHash)) {
+        res.status(400).send(`invalid param blockHash. must be 64 hexadecimal characters`);
+        return;
+      }
+      if (typeof(verbosity) !== 'string') {
+        res.status(400).send(`invalid param verbosity. must be a string representing an integer`);
+        return;
+      }
+      const verbosityNumber = parseInt(verbosity, 10);
+      if (typeof(verbosityNumber) !== 'number') {
+        res.status(400).send(`invalid param verbosity. must be a valid integer`);
+        return;
+      }
+
+      const block = await bitcoinClient.getBlock(blockHash, verbosityNumber);
+      if (!block) {
+        res.status(400).send(`unable to get block`);
+        return;
+      }
+      res.status(200).send(block);
+    } catch (e: any) {
+      BitcoinBackendRoutes.handleException(e, 'getBlock', res);
+    }
+  }
+
+  private async $getBlockHash(req: Request, res: Response): Promise<void> {
+    const blockHeight = req.query.height;
+    try {
+      if (typeof(blockHeight) !== 'string') {
+        res.status(400).send(`invalid param blockHeight, must be a string representing an integer`);
+        return;
+      }
+      const blockHeightNumber = parseInt(blockHeight, 10);
+      if (typeof(blockHeightNumber) !== 'number') {
+        res.status(400).send(`invalid param blockHeight. must be a valid integer`);
+        return;
+      }
+
+      const block = await bitcoinClient.getBlockHash(blockHeightNumber);
+      if (!block) {
+        res.status(400).send(`unable to get block hash`);
+        return;
+      }
+      res.status(200).send(block);
+    } catch (e: any) {
+      BitcoinBackendRoutes.handleException(e, 'getBlockHash', res);
+    }
+  }
+
+  private async $getBlockCount(req: Request, res: Response): Promise<void> {
+    try {
+      const count = await bitcoinClient.getBlockCount();
+      if (!count) {
+        res.status(400).send(`unable to get block count`);
+        return;
+      }
+      res.status(200).send(`${count}`);
+    } catch (e: any) {
+      BitcoinBackendRoutes.handleException(e, 'getBlockCount', res);
+    }
+  }
+}
+
+export default new BitcoinBackendRoutes;

backend/src/api/bitcoin/bitcoin-second-client.ts

@@ -1,5 +1,5 @@
 import config from '../../config';
-import * as bitcoin from '@mempool/bitcoin';
+const bitcoin = require('../../rpc-api/index');
 import { BitcoinRpcCredentials } from './bitcoin-api-abstract-factory';
 
 const nodeRpcCredentials: BitcoinRpcCredentials = {
@@ -7,7 +7,8 @@ const nodeRpcCredentials: BitcoinRpcCredentials = {
   port: config.SECOND_CORE_RPC.PORT,
   user: config.SECOND_CORE_RPC.USERNAME,
   pass: config.SECOND_CORE_RPC.PASSWORD,
-  timeout: 60000,
+  timeout: config.SECOND_CORE_RPC.TIMEOUT,
+  cookie: config.SECOND_CORE_RPC.COOKIE ? config.SECOND_CORE_RPC.COOKIE_PATH : undefined,
 };
 
 export default new bitcoin.Client(nodeRpcCredentials);

backend/src/api/bitcoin/bitcoin.routes.ts

@@ -0,0 +1,935 @@
+import { Application, Request, Response } from 'express';
+import axios from 'axios';
+import * as bitcoinjs from 'bitcoinjs-lib';
+import config from '../../config';
+import websocketHandler from '../websocket-handler';
+import mempool from '../mempool';
+import feeApi from '../fee-api';
+import mempoolBlocks from '../mempool-blocks';
+import bitcoinApi from './bitcoin-api-factory';
+import { Common } from '../common';
+import backendInfo from '../backend-info';
+import transactionUtils from '../transaction-utils';
+import { IEsploraApi } from './esplora-api.interface';
+import loadingIndicators from '../loading-indicators';
+import { TransactionExtended } from '../../mempool.interfaces';
+import logger from '../../logger';
+import blocks from '../blocks';
+import bitcoinClient from './bitcoin-client';
+import difficultyAdjustment from '../difficulty-adjustment';
+import transactionRepository from '../../repositories/TransactionRepository';
+import rbfCache from '../rbf-cache';
+import { calculateMempoolTxCpfp } from '../cpfp';
+import { handleError } from '../../utils/api';
+
+const TXID_REGEX = /^[a-f0-9]{64}$/i;
+const BLOCK_HASH_REGEX = /^[a-f0-9]{64}$/i;
+const ADDRESS_REGEX = /^[a-z0-9]{2,120}$/i;
+const SCRIPT_HASH_REGEX = /^([a-f0-9]{2})+$/i;
+
+class BitcoinRoutes {
+  public initRoutes(app: Application) {
+    app
+      .get(config.MEMPOOL.API_URL_PREFIX + 'transaction-times', this.getTransactionTimes)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'cpfp/:txId', this.$getCpfpInfo)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'difficulty-adjustment', this.getDifficultyChange)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'fees/recommended', this.getRecommendedFees)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'fees/mempool-blocks', this.getMempoolBlocks)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'backend-info', this.getBackendInfo)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'init-data', this.getInitData)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'validate-address/:address', this.validateAddress)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/rbf', this.getRbfHistory)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/cached', this.getCachedTx)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'replacements', this.getRbfReplacements)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'fullrbf/replacements', this.getFullRbfReplacements)
+      .post(config.MEMPOOL.API_URL_PREFIX + 'tx/push', this.$postTransactionForm)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'blocks', this.getBlocks.bind(this))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/:height', this.getBlocks.bind(this))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash', this.getBlock)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/summary', this.getStrippedBlockTransactions)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/tx/:txid/summary', this.getStrippedBlockTransaction)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/audit-summary', this.getBlockAuditSummary)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/tx/:txid/audit', this.$getBlockTxAuditSummary)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/height', this.getBlockTipHeight)
+      .post(config.MEMPOOL.API_URL_PREFIX + 'psbt/addparents', this.postPsbtCompletion)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from', this.getBlocksByBulk.bind(this))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from/:to', this.getBlocksByBulk.bind(this))
+      // Temporarily add txs/package endpoint for all backends until esplora supports it
+      .post(config.MEMPOOL.API_URL_PREFIX + 'txs/package', this.$submitPackage)
+      ;
+
+      if (config.MEMPOOL.BACKEND !== 'esplora') {
+        app
+          .get(config.MEMPOOL.API_URL_PREFIX + 'mempool', this.getMempool)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'mempool/txids', this.getMempoolTxIds)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'mempool/recent', this.getRecentMempoolTransactions)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId', this.getTransaction)
+          .post(config.MEMPOOL.API_URL_PREFIX + 'tx', this.$postTransaction)
+          .post(config.MEMPOOL.API_URL_PREFIX + 'txs/test', this.$testTransactions)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/hex', this.getRawTransaction)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/status', this.getTransactionStatus)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/outspends', this.getTransactionOutspends)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'txs/outspends', this.$getBatchedOutspends)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/header', this.getBlockHeader)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/hash', this.getBlockTipHash)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/raw', this.getRawBlock)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txids', this.getTxIdsForBlock)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txs', this.getBlockTransactions)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txs/:index', this.getBlockTransactions)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'block-height/:height', this.getBlockHeight)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address', this.getAddress)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs', this.getAddressTransactions)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs/summary', this.getAddressTransactionSummary)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'scripthash/:scripthash', this.getScriptHash)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'scripthash/:scripthash/txs', this.getScriptHashTransactions)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'scripthash/:scripthash/txs/summary', this.getScriptHashTransactionSummary)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'address-prefix/:prefix', this.getAddressPrefix)
+          ;
+      }
+  }
+
+
+  private getInitData(req: Request, res: Response) {
+    try {
+      const result = websocketHandler.getSerializedInitData();
+      res.set('Content-Type', 'application/json');
+      res.send(result);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get init data');
+    }
+  }
+
+  private getRecommendedFees(req: Request, res: Response) {
+    if (!mempool.isInSync()) {
+      res.statusCode = 503;
+      res.send('Service Unavailable');
+      return;
+    }
+    const result = feeApi.getRecommendedFee();
+    res.json(result);
+  }
+
+  private getMempoolBlocks(req: Request, res: Response) {
+    try {
+      const result = mempoolBlocks.getMempoolBlocks();
+      res.json(result);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get mempool blocks');
+    }
+  }
+
+  private getTransactionTimes(req: Request, res: Response) {
+    if (!Array.isArray(req.query.txId)) {
+      handleError(req, res, 500, 'Not an array');
+      return;
+    }
+    const txIds: string[] = [];
+    for (const _txId in req.query.txId) {
+      if (typeof req.query.txId[_txId] === 'string') {
+        const txid = req.query.txId[_txId].toString();
+        if (TXID_REGEX.test(txid)) {
+          txIds.push(txid);
+        }
+      }
+    }
+
+    const times = mempool.getFirstSeenForTransactions(txIds);
+    res.json(times);
+  }
+
+  private async $getBatchedOutspends(req: Request, res: Response): Promise<IEsploraApi.Outspend[][] | void> {
+    const txids_csv = req.query.txids;
+    if (!txids_csv || typeof txids_csv !== 'string') {
+      handleError(req, res, 500, 'Invalid txids format');
+      return;
+    }
+    const txids = txids_csv.split(',');
+    if (txids.length > 50) {
+      handleError(req, res, 400, 'Too many txids requested');
+      return;
+    }
+    if (txids.some((txid) => !TXID_REGEX.test(txid))) {
+      handleError(req, res, 400, 'Invalid txids format');
+      return;
+    }
+
+    try {
+      const batchedOutspends = await bitcoinApi.$getBatchedOutspends(txids);
+      res.json(batchedOutspends);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get batched outspends');
+    }
+  }
+
+  private async $getCpfpInfo(req: Request, res: Response) {
+    if (!TXID_REGEX.test(req.params.txId)) {
+      handleError(req, res, 501, `Invalid transaction ID`);
+      return;
+    }
+
+    const tx = mempool.getMempool()[req.params.txId];
+    if (tx) {
+      if (tx?.cpfpChecked) {
+        res.json({
+          ancestors: tx.ancestors,
+          bestDescendant: tx.bestDescendant || null,
+          descendants: tx.descendants || null,
+          effectiveFeePerVsize: tx.effectiveFeePerVsize || null,
+          sigops: tx.sigops,
+          fee: tx.fee,
+          adjustedVsize: tx.adjustedVsize,
+          acceleration: tx.acceleration,
+          acceleratedBy: tx.acceleratedBy || undefined,
+          acceleratedAt: tx.acceleratedAt || undefined,
+          feeDelta: tx.feeDelta || undefined,
+        });
+        return;
+      }
+
+      const cpfpInfo = calculateMempoolTxCpfp(tx, mempool.getMempool());
+
+      res.json(cpfpInfo);
+      return;
+    } else {
+      let cpfpInfo;
+      if (config.DATABASE.ENABLED) {
+        try {
+          cpfpInfo = await transactionRepository.$getCpfpInfo(req.params.txId);
+        } catch (e) {
+          handleError(req, res, 500, 'Failed to get CPFP info');
+          return;
+        }
+      }
+      if (cpfpInfo) {
+        res.json(cpfpInfo);
+        return;
+      } else {
+        res.json({
+          ancestors: []
+        });
+        return;
+      }
+    }
+  }
+
+  private getBackendInfo(req: Request, res: Response) {
+    res.json(backendInfo.getBackendInfo());
+  }
+
+  private async getTransaction(req: Request, res: Response) {
+    if (!TXID_REGEX.test(req.params.txId)) {
+      handleError(req, res, 501, `Invalid transaction ID`);
+      return;
+    }
+    try {
+      const transaction = await transactionUtils.$getTransactionExtended(req.params.txId, true, false, false, true);
+      res.json(transaction);
+    } catch (e) {
+      let statusCode = 500;
+      if (e instanceof Error && e instanceof Error && e.message && e.message.indexOf('No such mempool or blockchain transaction') > -1) {
+        statusCode = 404;
+        handleError(req, res, statusCode, 'No such mempool or blockchain transaction');
+        return;
+      }
+      handleError(req, res, statusCode, 'Failed to get transaction');
+    }
+  }
+
+  private async getRawTransaction(req: Request, res: Response) {
+    if (!TXID_REGEX.test(req.params.txId)) {
+      handleError(req, res, 501, `Invalid transaction ID`);
+      return;
+    }
+    try {
+      const transaction: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(req.params.txId, true);
+      res.setHeader('content-type', 'text/plain');
+      res.send(transaction.hex);
+    } catch (e) {
+      let statusCode = 500;
+      if (e instanceof Error && e.message && e.message.indexOf('No such mempool or blockchain transaction') > -1) {
+        statusCode = 404;
+        handleError(req, res, statusCode, 'No such mempool or blockchain transaction');
+        return;
+      }
+      handleError(req, res, statusCode, 'Failed to get raw transaction');
+    }
+  }
+
+  /**
+   * Takes the PSBT as text/plain body, parses it, and adds the full
+   * parent transaction to each input that doesn't already have it.
+   * This is used for BTCPayServer / Trezor users which need access to
+   * the full parent transaction even with segwit inputs.
+   * It will respond with a text/plain PSBT in the same format (hex|base64).
+   */
+  private async postPsbtCompletion(req: Request, res: Response): Promise<void> {
+    res.setHeader('content-type', 'text/plain');
+    const notFoundError = `Couldn't get transaction hex for parent of input`;
+    try {
+      let psbt: bitcoinjs.Psbt;
+      let format: 'hex' | 'base64';
+      let isModified = false;
+      try {
+        psbt = bitcoinjs.Psbt.fromBase64(req.body);
+        format = 'base64';
+      } catch (e1) {
+        try {
+          psbt = bitcoinjs.Psbt.fromHex(req.body);
+          format = 'hex';
+        } catch (e2) {
+          throw new Error(`Unable to parse PSBT`);
+        }
+      }
+      for (const [index, input] of psbt.data.inputs.entries()) {
+        if (!input.nonWitnessUtxo) {
+          // Buffer.from ensures it won't be modified in place by reverse()
+          const txid = Buffer.from(psbt.txInputs[index].hash)
+            .reverse()
+            .toString('hex');
+
+          let transactionHex: string;
+          // If missing transaction, return 404 status error
+          try {
+            transactionHex = await bitcoinApi.$getTransactionHex(txid);
+            if (!transactionHex) {
+              throw new Error('');
+            }
+          } catch (err) {
+            throw new Error(`${notFoundError} #${index} @ ${txid}`);
+          }
+
+          psbt.updateInput(index, {
+            nonWitnessUtxo: Buffer.from(transactionHex, 'hex'),
+          });
+          if (!isModified) {
+            isModified = true;
+          }
+        }
+      }
+      if (isModified) {
+        res.send(format === 'hex' ? psbt.toHex() : psbt.toBase64());
+      } else {
+        // Not modified
+        // 422 Unprocessable Entity
+        // https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/422
+        handleError(req, res, 422, `Psbt had no missing nonWitnessUtxos.`);
+      }
+    } catch (e: any) {
+      if (e instanceof Error && new RegExp(notFoundError).test(e.message)) {
+        handleError(req, res, 404, notFoundError);
+      } else {
+        handleError(req, res, 500, 'Failed to process PSBT');
+      }
+    }
+  }
+
+  private async getTransactionStatus(req: Request, res: Response) {
+    if (!TXID_REGEX.test(req.params.txId)) {
+      handleError(req, res, 501, `Invalid transaction ID`);
+      return;
+    }
+    try {
+      const transaction = await transactionUtils.$getTransactionExtended(req.params.txId, true);
+      res.json(transaction.status);
+    } catch (e) {
+      let statusCode = 500;
+      if (e instanceof Error && e.message && e.message.indexOf('No such mempool or blockchain transaction') > -1) {
+        statusCode = 404;
+        handleError(req, res, statusCode, 'No such mempool or blockchain transaction');
+        return;
+      }
+      handleError(req, res, statusCode, 'Failed to get transaction status');
+    }
+  }
+
+  private async getStrippedBlockTransactions(req: Request, res: Response) {
+    if (!BLOCK_HASH_REGEX.test(req.params.hash)) {
+      handleError(req, res, 501, `Invalid block hash`);
+      return;
+    }
+    try {
+      const transactions = await blocks.$getStrippedBlockTransactions(req.params.hash);
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24 * 30).toUTCString());
+      res.json(transactions);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get block summary');
+    }
+  }
+
+  private async getStrippedBlockTransaction(req: Request, res: Response) {
+    if (!BLOCK_HASH_REGEX.test(req.params.hash)) {
+      handleError(req, res, 501, `Invalid block hash`);
+      return;
+    }
+    if (!TXID_REGEX.test(req.params.txid)) {
+      handleError(req, res, 501, `Invalid transaction ID`);
+      return;
+    }
+    try {
+      const transaction = await blocks.$getSingleTxFromSummary(req.params.hash, req.params.txid);
+      if (!transaction) {
+        handleError(req, res, 404, `Transaction not found in summary`);
+        return;
+      }
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24 * 30).toUTCString());
+      res.json(transaction);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get transaction from summary');
+    }
+  }
+
+  private async getBlock(req: Request, res: Response) {
+    if (!BLOCK_HASH_REGEX.test(req.params.hash)) {
+      handleError(req, res, 501, `Invalid block hash`);
+      return;
+    }
+    try {
+      const block = await blocks.$getBlock(req.params.hash);
+
+      const blockAge = new Date().getTime() / 1000 - block.timestamp;
+      const day = 24 * 3600;
+      let cacheDuration;
+      if (blockAge > 365 * day) {
+        cacheDuration = 30 * day;
+      } else if (blockAge > 30 * day) {
+        cacheDuration = 10 * day;
+      } else {
+        cacheDuration = 600;
+      }
+
+      res.setHeader('Expires', new Date(Date.now() + 1000 * cacheDuration).toUTCString());
+      res.json(block);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get block');
+    }
+  }
+
+  private async getBlockHeader(req: Request, res: Response) {
+    if (!BLOCK_HASH_REGEX.test(req.params.hash)) {
+      handleError(req, res, 501, `Invalid block hash`);
+      return;
+    }
+    try {
+      const blockHeader = await bitcoinApi.$getBlockHeader(req.params.hash);
+      res.setHeader('content-type', 'text/plain');
+      res.send(blockHeader);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get block header');
+    }
+  }
+
+  private async getBlockAuditSummary(req: Request, res: Response) {
+    if (!BLOCK_HASH_REGEX.test(req.params.hash)) {
+      handleError(req, res, 501, `Invalid block hash`);
+      return;
+    }
+    try {
+      const auditSummary = await blocks.$getBlockAuditSummary(req.params.hash);
+      if (auditSummary) {
+        res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24 * 30).toUTCString());
+        res.json(auditSummary);
+      } else {
+        handleError(req, res, 404, `Audit not available`);
+        return;
+      }
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get block audit summary');
+    }
+  }
+
+  private async $getBlockTxAuditSummary(req: Request, res: Response) {
+    if (!BLOCK_HASH_REGEX.test(req.params.hash)) {
+      handleError(req, res, 501, `Invalid block hash`);
+      return;
+    }
+    if (!TXID_REGEX.test(req.params.txid)) {
+      handleError(req, res, 501, `Invalid transaction ID`);
+      return;
+    }
+    try {
+      const auditSummary = await blocks.$getBlockTxAuditSummary(req.params.hash, req.params.txid);
+      if (auditSummary) {
+        res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24 * 30).toUTCString());
+        res.json(auditSummary);
+      } else {
+        handleError(req, res, 404, `Transaction audit not available`);
+        return;
+      }
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get transaction audit summary');
+    }
+  }
+
+  private async getBlocks(req: Request, res: Response) {
+    try {
+      if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) { // Bitcoin
+        const height = req.params.height === undefined ? undefined : parseInt(req.params.height, 10);
+        res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+        res.json(await blocks.$getBlocks(height, 15));
+      } else { // Liquid
+        return await this.getLegacyBlocks(req, res);
+      }
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get blocks');
+    }
+  }
+
+  private async getBlocksByBulk(req: Request, res: Response) {
+    try {
+      if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { // Liquid - Not implemented
+        handleError(req, res, 404, `This API is only available for Bitcoin networks`);
+        return;
+      }
+      if (config.MEMPOOL.MAX_BLOCKS_BULK_QUERY <= 0) {
+        handleError(req, res, 404, `This API is disabled. Set config.MEMPOOL.MAX_BLOCKS_BULK_QUERY to a positive number to enable it.`);
+        return;
+      }
+      if (!Common.indexingEnabled()) {
+        handleError(req, res, 404, `Indexing is required for this API`);
+        return;
+      }
+
+      const from = parseInt(req.params.from, 10);
+      if (!req.params.from || from < 0) {
+        handleError(req, res, 400, `Parameter 'from' must be a block height (integer)`);
+        return;
+      }
+      const to = req.params.to === undefined ? await bitcoinApi.$getBlockHeightTip() : parseInt(req.params.to, 10);
+      if (to < 0) {
+        handleError(req, res, 400, `Parameter 'to' must be a block height (integer)`);
+        return;
+      }
+      if (from > to) {
+        handleError(req, res, 400, `Parameter 'to' must be a higher block height than 'from'`);
+        return;
+      }
+      if ((to - from + 1) > config.MEMPOOL.MAX_BLOCKS_BULK_QUERY) {
+        handleError(req, res, 400, `You can only query ${config.MEMPOOL.MAX_BLOCKS_BULK_QUERY} blocks at once.`);
+        return;
+      }
+
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(await blocks.$getBlocksBetweenHeight(from, to));
+
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get blocks');
+    }
+  }
+
+  private async getLegacyBlocks(req: Request, res: Response) {
+    try {
+      const returnBlocks: IEsploraApi.Block[] = [];
+      const tip = blocks.getCurrentBlockHeight();
+      const fromHeight = Math.min(parseInt(req.params.height, 10) || tip, tip);
+
+      // Check if block height exist in local cache to skip the hash lookup
+      const blockByHeight = blocks.getBlocks().find((b) => b.height === fromHeight);
+      let startFromHash: string | null = null;
+      if (blockByHeight) {
+        startFromHash = blockByHeight.id;
+      } else {
+        startFromHash = await bitcoinApi.$getBlockHash(fromHeight);
+      }
+
+      let nextHash = startFromHash;
+      for (let i = 0; i < 15 && nextHash; i++) {
+        const localBlock = blocks.getBlocks().find((b) => b.id === nextHash);
+        if (localBlock) {
+          returnBlocks.push(localBlock);
+          nextHash = localBlock.previousblockhash;
+        } else {
+          const block = await bitcoinApi.$getBlock(nextHash);
+          returnBlocks.push(block);
+          nextHash = block.previousblockhash;
+        }
+      }
+
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(returnBlocks);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get blocks');
+    }
+  }
+
+  private async getBlockTransactions(req: Request, res: Response) {
+    if (!BLOCK_HASH_REGEX.test(req.params.hash)) {
+      handleError(req, res, 501, `Invalid block hash`);
+      return;
+    }
+    try {
+      loadingIndicators.setProgress('blocktxs-' + req.params.hash, 0);
+
+      const txIds = await bitcoinApi.$getTxIdsForBlock(req.params.hash);
+      const transactions: TransactionExtended[] = [];
+      const startingIndex = Math.max(0, parseInt(req.params.index || '0', 10));
+
+      const endIndex = Math.min(startingIndex + 10, txIds.length);
+      for (let i = startingIndex; i < endIndex; i++) {
+        try {
+          const transaction = await transactionUtils.$getTransactionExtended(txIds[i], true, true);
+          transactions.push(transaction);
+          loadingIndicators.setProgress('blocktxs-' + req.params.hash, (i - startingIndex + 1) / (endIndex - startingIndex) * 100);
+        } catch (e) {
+          logger.debug('getBlockTransactions error: ' + (e instanceof Error ? e.message : e));
+        }
+      }
+      res.json(transactions);
+    } catch (e) {
+      loadingIndicators.setProgress('blocktxs-' + req.params.hash, 100);
+      handleError(req, res, 500, 'Failed to get block transactions');
+    }
+  }
+
+  private async getBlockHeight(req: Request, res: Response) {
+    try {
+      const blockHash = await bitcoinApi.$getBlockHash(parseInt(req.params.height, 10));
+      res.send(blockHash);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get block at height');
+    }
+  }
+
+  private async getAddress(req: Request, res: Response) {
+    if (config.MEMPOOL.BACKEND === 'none') {
+      handleError(req, res, 405, 'Address lookups cannot be used with bitcoind as backend.');
+      return;
+    }
+    if (!ADDRESS_REGEX.test(req.params.address)) {
+      handleError(req, res, 501, `Invalid address`);
+      return;
+    }
+
+    try {
+      const addressData = await bitcoinApi.$getAddress(req.params.address);
+      res.json(addressData);
+    } catch (e) {
+      if (e instanceof Error && e.message && (e.message.indexOf('too long') > 0 || e.message.indexOf('confirmed status') > 0)) {
+        handleError(req, res, 413, e.message);
+        return;
+      }
+      handleError(req, res, 500, 'Failed to get address');
+    }
+  }
+
+  private async getAddressTransactions(req: Request, res: Response): Promise<void> {
+    if (config.MEMPOOL.BACKEND === 'none') {
+      handleError(req, res, 405, 'Address lookups cannot be used with bitcoind as backend.');
+      return;
+    }
+    if (!ADDRESS_REGEX.test(req.params.address)) {
+      handleError(req, res, 501, `Invalid address`);
+      return;
+    }
+
+    try {
+      let lastTxId: string = '';
+      if (req.query.after_txid && typeof req.query.after_txid === 'string') {
+        lastTxId = req.query.after_txid;
+      }
+      const transactions = await bitcoinApi.$getAddressTransactions(req.params.address, lastTxId);
+      res.json(transactions);
+    } catch (e) {
+      if (e instanceof Error && e.message && (e.message.indexOf('too long') > 0 || e.message.indexOf('confirmed status') > 0)) {
+        handleError(req, res, 413, e.message);
+        return;
+      }
+      handleError(req, res, 500, 'Failed to get address transactions');
+    }
+  }
+
+  private async getAddressTransactionSummary(req: Request, res: Response): Promise<void> {
+    if (config.MEMPOOL.BACKEND !== 'esplora') {
+      handleError(req, res, 405, 'Address summary lookups require mempool/electrs backend.');
+      return;
+    }
+  }
+
+  private async getScriptHash(req: Request, res: Response) {
+    if (config.MEMPOOL.BACKEND === 'none') {
+      handleError(req, res, 405, 'Address lookups cannot be used with bitcoind as backend.');
+      return;
+    }
+    if (!SCRIPT_HASH_REGEX.test(req.params.scripthash)) {
+      handleError(req, res, 501, `Invalid scripthash`);
+      return;
+    }
+
+    try {
+      // electrum expects scripthashes in little-endian
+      const electrumScripthash = req.params.scripthash.match(/../g)?.reverse().join('') ?? '';
+      const addressData = await bitcoinApi.$getScriptHash(electrumScripthash);
+      res.json(addressData);
+    } catch (e) {
+      if (e instanceof Error && e.message && (e.message.indexOf('too long') > 0 || e.message.indexOf('confirmed status') > 0)) {
+        handleError(req, res, 413, e.message);
+        return;
+      }
+      handleError(req, res, 500, 'Failed to get script hash');
+    }
+  }
+
+  private async getScriptHashTransactions(req: Request, res: Response): Promise<void> {
+    if (config.MEMPOOL.BACKEND === 'none') {
+      handleError(req, res, 405, 'Address lookups cannot be used with bitcoind as backend.');
+      return;
+    }
+    if (!SCRIPT_HASH_REGEX.test(req.params.scripthash)) {
+      handleError(req, res, 501, `Invalid scripthash`);
+      return;
+    }
+
+    try {
+      // electrum expects scripthashes in little-endian
+      const electrumScripthash = req.params.scripthash.match(/../g)?.reverse().join('') ?? '';
+      let lastTxId: string = '';
+      if (req.query.after_txid && typeof req.query.after_txid === 'string') {
+        lastTxId = req.query.after_txid;
+      }
+      const transactions = await bitcoinApi.$getScriptHashTransactions(electrumScripthash, lastTxId);
+      res.json(transactions);
+    } catch (e) {
+      if (e instanceof Error && e.message && (e.message.indexOf('too long') > 0 || e.message.indexOf('confirmed status') > 0)) {
+        handleError(req, res, 413, e.message);
+        return;
+      }
+      handleError(req, res, 500, 'Failed to get script hash transactions');
+    }
+  }
+
+  private async getScriptHashTransactionSummary(req: Request, res: Response): Promise<void> {
+    if (config.MEMPOOL.BACKEND !== 'esplora') {
+      handleError(req, res, 405, 'Scripthash summary lookups require mempool/electrs backend.');
+      return;
+    }
+  }
+
+  private async getAddressPrefix(req: Request, res: Response) {
+    try {
+      const addressPrefix = await bitcoinApi.$getAddressPrefix(req.params.prefix);
+      res.send(addressPrefix);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get address prefix');
+    }
+  }
+
+  private async getRecentMempoolTransactions(req: Request, res: Response) {
+    const latestTransactions = Object.entries(mempool.getMempool())
+      .sort((a, b) => (b[1].firstSeen || 0) - (a[1].firstSeen || 0))
+      .slice(0, 10).map((tx) => Common.stripTransaction(tx[1]));
+
+    res.json(latestTransactions);
+  }
+
+  private async getMempool(req: Request, res: Response) {
+    const info = mempool.getMempoolInfo();
+    res.json({
+      count: info.size,
+      vsize: info.bytes,
+      total_fee: info.total_fee * 1e8,
+      fee_histogram: []
+    });
+  }
+
+  private async getMempoolTxIds(req: Request, res: Response) {
+    try {
+      const rawMempool = await bitcoinApi.$getRawMempool();
+      res.send(rawMempool);
+    } catch (e) {
+      handleError(req, res, 500, e instanceof Error ? e.message : e);
+    }
+  }
+
+  private getBlockTipHeight(req: Request, res: Response) {
+    try {
+      const result = blocks.getCurrentBlockHeight();
+      if (!result) {
+        handleError(req, res, 503, `Service Temporarily Unavailable`);
+        return;
+      }
+      res.setHeader('content-type', 'text/plain');
+      res.send(result.toString());
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get height at tip');
+    }
+  }
+
+  private async getBlockTipHash(req: Request, res: Response) {
+    try {
+      const result = await bitcoinApi.$getBlockHashTip();
+      res.setHeader('content-type', 'text/plain');
+      res.send(result);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get hash at tip');
+    }
+  }
+
+  private async getRawBlock(req: Request, res: Response) {
+    if (!BLOCK_HASH_REGEX.test(req.params.hash)) {
+      handleError(req, res, 501, `Invalid block hash`);
+      return;
+    }
+    try {
+      const result = await bitcoinApi.$getRawBlock(req.params.hash);
+      res.setHeader('content-type', 'application/octet-stream');
+      res.send(result);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get raw block');
+    }
+  }
+
+  private async getTxIdsForBlock(req: Request, res: Response) {
+    if (!BLOCK_HASH_REGEX.test(req.params.hash)) {
+      handleError(req, res, 501, `Invalid block hash`);
+      return;
+    }
+    try {
+      const result = await bitcoinApi.$getTxIdsForBlock(req.params.hash);
+      res.json(result);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get txids for block');
+    }
+  }
+
+  private async validateAddress(req: Request, res: Response) {
+    if (!ADDRESS_REGEX.test(req.params.address)) {
+      handleError(req, res, 501, `Invalid address`);
+      return;
+    }
+    try {
+      const result = await bitcoinClient.validateAddress(req.params.address);
+      res.json(result);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to validate address');
+    }
+  }
+
+  private async getRbfHistory(req: Request, res: Response) {
+    if (!TXID_REGEX.test(req.params.txId)) {
+      handleError(req, res, 501, `Invalid transaction ID`);
+      return;
+    }
+    try {
+      const replacements = rbfCache.getRbfTree(req.params.txId) || null;
+      const replaces = rbfCache.getReplaces(req.params.txId) || null;
+      res.json({
+        replacements,
+        replaces
+      });
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get rbf history');
+    }
+  }
+
+  private async getRbfReplacements(req: Request, res: Response) {
+    try {
+      const result = rbfCache.getRbfTrees(false);
+      res.json(result);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get rbf trees');
+    }
+  }
+
+  private async getFullRbfReplacements(req: Request, res: Response) {
+    try {
+      const result = rbfCache.getRbfTrees(true);
+      res.json(result);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get full rbf replacements');
+    }
+  }
+
+  private async getCachedTx(req: Request, res: Response) {
+    if (!TXID_REGEX.test(req.params.txId)) {
+      handleError(req, res, 501, `Invalid transaction ID`);
+      return;
+    }
+    try {
+      const result = rbfCache.getTx(req.params.txId);
+      if (result) {
+        res.json(result);
+      } else {
+        res.status(204).send();
+      }
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get cached tx');
+    }
+  }
+
+  private async getTransactionOutspends(req: Request, res: Response) {
+    if (!TXID_REGEX.test(req.params.txId)) {
+      handleError(req, res, 501, `Invalid transaction ID`);
+      return;
+    }
+    try {
+      const result = await bitcoinApi.$getOutspends(req.params.txId);
+      res.json(result);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get transaction outspends');
+    }
+  }
+
+  private getDifficultyChange(req: Request, res: Response) {
+    try {
+      const da = difficultyAdjustment.getDifficultyAdjustment();
+      if (da) {
+        res.json(da);
+      } else {
+        handleError(req, res, 503, `Service Temporarily Unavailable`);
+      }
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get difficulty change');
+    }
+  }
+
+  private async $postTransaction(req: Request, res: Response) {
+    res.setHeader('content-type', 'text/plain');
+    try {
+      const rawTx = Common.getTransactionFromRequest(req, false);
+      const txIdResult = await bitcoinApi.$sendRawTransaction(rawTx);
+      res.send(txIdResult);
+    } catch (e: any) {
+      handleError(req, res, 400, (e.message && e.code) ? 'sendrawtransaction RPC error: ' + JSON.stringify({ code: e.code })
+        : 'Failed to send raw transaction');
+    }
+  }
+
+  private async $postTransactionForm(req: Request, res: Response) {
+    res.setHeader('content-type', 'text/plain');
+    try {
+      const txHex = Common.getTransactionFromRequest(req, true);
+      const txIdResult = await bitcoinClient.sendRawTransaction(txHex);
+      res.send(txIdResult);
+    } catch (e: any) {
+      handleError(req, res, 400, (e.message && e.code) ? 'sendrawtransaction RPC error: ' + JSON.stringify({ code: e.code })
+        : 'Failed to send raw transaction');
+    }
+  }
+
+  private async $testTransactions(req: Request, res: Response) {
+    try {
+      const rawTxs = Common.getTransactionsFromRequest(req);
+      const maxfeerate = parseFloat(req.query.maxfeerate as string);
+      const result = await bitcoinApi.$testMempoolAccept(rawTxs, maxfeerate);
+      res.send(result);
+    } catch (e: any) {
+      handleError(req, res, 400, (e.message && e.code) ? 'testmempoolaccept RPC error: ' + JSON.stringify({ code: e.code })
+        : 'Failed to test transactions');
+    }
+  }
+
+  private async $submitPackage(req: Request, res: Response) {
+    try {
+      const rawTxs = Common.getTransactionsFromRequest(req);
+      const maxfeerate = parseFloat(req.query.maxfeerate as string);
+      const maxburnamount = parseFloat(req.query.maxburnamount as string);
+      const result = await bitcoinClient.submitPackage(rawTxs, maxfeerate ?? undefined, maxburnamount ?? undefined);
+      res.send(result);
+    } catch (e: any) {
+      handleError(req, res, 400, (e.message && e.code) ? 'submitpackage RPC error: ' + JSON.stringify({ code: e.code })
+        : 'Failed to submit package');
+    }
+  }
+
+}
+
+export default new BitcoinRoutes();

backend/src/api/bitcoin/electrum-api.ts

@@ -1,12 +1,11 @@
 import config from '../../config';
+import Client from '@mempool/electrum-client';
 import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
 import { IEsploraApi } from './esplora-api.interface';
 import { IElectrumApi } from './electrum-api.interface';
 import BitcoinApi from './bitcoin-api';
 import logger from '../../logger';
-import * as ElectrumClient from '@mempool/electrum-client';
-import * as sha256 from 'crypto-js/sha256';
-import * as hexEnc from 'crypto-js/enc-hex';
+import crypto from "crypto-js";
 import loadingIndicators from '../loading-indicators';
 import memoryCache from '../memory-cache';
 
@@ -17,7 +16,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
     super(bitcoinClient);
 
     const electrumConfig = { client: 'mempool-v2', version: '1.4' };
-    const electrumPersistencePolicy = { retryPeriod: 10000, maxRetry: 1000, callback: null };
+    const electrumPersistencePolicy = { retryPeriod: 1000, maxRetry: Number.MAX_SAFE_INTEGER, callback: null };
 
     const electrumCallbacks = {
       onConnect: (client, versionInfo) => { logger.info(`Connected to Electrum Server at ${config.ELECTRUM.HOST}:${config.ELECTRUM.PORT} (${JSON.stringify(versionInfo)})`); },
@@ -26,7 +25,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
       onLog: (str) => { logger.debug(str); },
     };
 
-    this.electrumClient = new ElectrumClient(
+    this.electrumClient = new Client(
       config.ELECTRUM.PORT,
       config.ELECTRUM.HOST,
       config.ELECTRUM.TLS_ENABLED ? 'tls' : 'tcp',
@@ -35,7 +34,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
     );
 
     this.electrumClient.initElectrum(electrumConfig, electrumPersistencePolicy)
-      .then(() => {})
+      .then(() => { })
       .catch((err) => {
         logger.err(`Error connecting to Electrum Server at ${config.ELECTRUM.HOST}:${config.ELECTRUM.PORT}`);
       });
@@ -95,7 +94,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
   async $getAddressTransactions(address: string, lastSeenTxId: string): Promise<IEsploraApi.Transaction[]> {
     const addressInfo = await this.bitcoindClient.validateAddress(address);
     if (!addressInfo || !addressInfo.isvalid) {
-     return [];
+      return [];
     }
 
     try {
@@ -127,6 +126,77 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
     }
   }
 
+  async $getScriptHash(scripthash: string): Promise<IEsploraApi.ScriptHash> {
+    try {
+      const balance = await this.electrumClient.blockchainScripthash_getBalance(scripthash);
+      let history = memoryCache.get<IElectrumApi.ScriptHashHistory[]>('Scripthash_getHistory', scripthash);
+      if (!history) {
+        history = await this.electrumClient.blockchainScripthash_getHistory(scripthash);
+        memoryCache.set('Scripthash_getHistory', scripthash, history, 2);
+      }
+
+      const unconfirmed = history ? history.filter((h) => h.fee).length : 0;
+
+      return {
+        'scripthash': scripthash,
+        'chain_stats': {
+          'funded_txo_count': 0,
+          'funded_txo_sum': balance.confirmed ? balance.confirmed : 0,
+          'spent_txo_count': 0,
+          'spent_txo_sum': balance.confirmed < 0 ? balance.confirmed : 0,
+          'tx_count': (history?.length || 0) - unconfirmed,
+        },
+        'mempool_stats': {
+          'funded_txo_count': 0,
+          'funded_txo_sum': balance.unconfirmed > 0 ? balance.unconfirmed : 0,
+          'spent_txo_count': 0,
+          'spent_txo_sum': balance.unconfirmed < 0 ? -balance.unconfirmed : 0,
+          'tx_count': unconfirmed,
+        },
+        'electrum': true,
+      };
+    } catch (e: any) {
+      throw new Error(typeof e === 'string' ? e : e && e.message || e);
+    }
+  }
+
+  async $getScriptHashTransactions(scripthash: string, lastSeenTxId?: string): Promise<IEsploraApi.Transaction[]> {
+    try {
+      loadingIndicators.setProgress('address-' + scripthash, 0);
+
+      const transactions: IEsploraApi.Transaction[] = [];
+      let history = memoryCache.get<IElectrumApi.ScriptHashHistory[]>('Scripthash_getHistory', scripthash);
+      if (!history) {
+        history = await this.electrumClient.blockchainScripthash_getHistory(scripthash);
+        memoryCache.set('Scripthash_getHistory', scripthash, history, 2);
+      }
+      if (!history) {
+        throw new Error('failed to get scripthash history');
+      }
+      history.sort((a, b) => (b.height || 9999999) - (a.height || 9999999));
+
+      let startingIndex = 0;
+      if (lastSeenTxId) {
+        const pos = history.findIndex((historicalTx) => historicalTx.tx_hash === lastSeenTxId);
+        if (pos) {
+          startingIndex = pos + 1;
+        }
+      }
+      const endIndex = Math.min(startingIndex + 10, history.length);
+
+      for (let i = startingIndex; i < endIndex; i++) {
+        const tx = await this.$getRawTransaction(history[i].tx_hash, false, true);
+        transactions.push(tx);
+        loadingIndicators.setProgress('address-' + scripthash, (i + 1) / endIndex * 100);
+      }
+
+      return transactions;
+    } catch (e: any) {
+      loadingIndicators.setProgress('address-' + scripthash, 100);
+      throw new Error(typeof e === 'string' ? e : e && e.message || e);
+    }
+  }
+
   private $getScriptHashBalance(scriptHash: string): Promise<IElectrumApi.ScriptHashBalance> {
     return this.electrumClient.blockchainScripthash_getBalance(this.encodeScriptHash(scriptHash));
   }
@@ -144,8 +214,8 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
   }
 
   private encodeScriptHash(scriptPubKey: string): string {
-    const addrScripthash = hexEnc.stringify(sha256(hexEnc.parse(scriptPubKey)));
-    return addrScripthash.match(/.{2}/g).reverse().join('');
+    const addrScripthash = crypto.enc.Hex.stringify(crypto.SHA256(crypto.enc.Hex.parse(scriptPubKey)));
+    return addrScripthash!.match(/.{2}/g)!.reverse().join('');
   }
 
 }

backend/src/api/bitcoin/esplora-api.interface.ts

@@ -6,6 +6,7 @@ export namespace IEsploraApi {
     size: number;
     weight: number;
     fee: number;
+    sigops?: number;
     vin: Vin[];
     vout: Vout[];
     status: Status;
@@ -25,14 +26,16 @@ export namespace IEsploraApi {
     is_coinbase: boolean;
     scriptsig: string;
     scriptsig_asm: string;
-    inner_redeemscript_asm?: string;
-    inner_witnessscript_asm?: string;
+    inner_redeemscript_asm: string;
+    inner_witnessscript_asm: string;
     sequence: any;
-    witness?: string[];
+    witness: string[];
     prevout: Vout | null;
     // Elements
     is_pegin?: boolean;
     issuance?: Issuance;
+    // Custom
+    lazy?: boolean;
   }
 
   interface Issuance {
@@ -51,7 +54,7 @@ export namespace IEsploraApi {
     scriptpubkey: string;
     scriptpubkey_asm: string;
     scriptpubkey_type: string;
-    scriptpubkey_address: string;
+    scriptpubkey_address?: string;
     value: number;
     // Elements
     valuecommitment?: number;
@@ -86,6 +89,8 @@ export namespace IEsploraApi {
     size: number;
     weight: number;
     previousblockhash: string;
+    mediantime: number;
+    stale: boolean;
   }
 
   export interface Address {
@@ -95,6 +100,13 @@ export namespace IEsploraApi {
     electrum?: boolean;
   }
 
+  export interface ScriptHash {
+    scripthash: string;
+    chain_stats: ChainStats;
+    mempool_stats: MempoolStats;
+    electrum?: boolean;
+  }
+
   export interface ChainStats {
     funded_txo_count: number;
     funded_txo_sum: number;
@@ -113,9 +125,9 @@ export namespace IEsploraApi {
 
   export interface Outspend {
     spent: boolean;
-    txid: string;
-    vin: number;
-    status: Status;
+    txid?: string;
+    vin?: number;
+    status?: Status;
   }
 
   export interface Asset {
@@ -167,4 +179,11 @@ export namespace IEsploraApi {
     burn_count: number;
   }
 
+  export interface AddressTxSummary {
+    txid: string;
+    value: number;
+    height: number;
+    time: number;
+    tx_position?: number;
+  }
 }

backend/src/api/bitcoin/esplora-api.ts

@@ -1,58 +1,406 @@
 import config from '../../config';
-import axios, { AxiosRequestConfig } from 'axios';
-import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
+import axios, { isAxiosError } from 'axios';
+import http from 'http';
+import { AbstractBitcoinApi, HealthCheckHost } from './bitcoin-api-abstract-factory';
 import { IEsploraApi } from './esplora-api.interface';
+import logger from '../../logger';
+import { Common } from '../common';
+import { SubmitPackageResult, TestMempoolAcceptResult } from './bitcoin-api.interface';
+import os from 'os';
+interface FailoverHost {
+  host: string,
+  rtts: number[],
+  rtt: number,
+  timedOut?: boolean,
+  failures: number,
+  latestHeight?: number,
+  socket?: boolean,
+  outOfSync?: boolean,
+  unreachable?: boolean,
+  preferred?: boolean,
+  checked: boolean,
+  lastChecked?: number,
+  publicDomain: string,
+  hashes: {
+    frontend?: string,
+    backend?: string,
+    electrs?: string,
+    lastUpdated: number,
+  }
+}
+
+class FailoverRouter {
+  activeHost: FailoverHost;
+  fallbackHost: FailoverHost;
+  maxSlippage: number = config.ESPLORA.MAX_BEHIND_TIP ?? 2;
+  maxHeight: number = 0;
+  hosts: FailoverHost[];
+  multihost: boolean;
+  gitHashInterval: number = 600000; // 10 minutes
+  pollInterval: number = 60000; // 1 minute
+  pollTimer: NodeJS.Timeout | null = null;
+  pollConnection = axios.create();
+  localHostname: string = 'localhost';
+  requestConnection = axios.create({
+    httpAgent: new http.Agent({ keepAlive: true })
+  });
+
+  constructor() {
+    try {
+      this.localHostname = os.hostname();
+    } catch (e) {
+      logger.warn('Failed to set local hostname, using "localhost"');
+    }
+    // setup list of hosts
+    this.hosts = (config.ESPLORA.FALLBACK || []).map(domain => {
+      return {
+        host: domain,
+        checked: false,
+        rtts: [],
+        rtt: Infinity,
+        failures: 0,
+        publicDomain: 'https://' + this.extractPublicDomain(domain),
+        hashes: {
+          lastUpdated: 0,
+        },
+      };
+    });
+    this.activeHost = {
+      host: config.ESPLORA.UNIX_SOCKET_PATH || config.ESPLORA.REST_API_URL,
+      rtts: [],
+      rtt: 0,
+      failures: 0,
+      socket: !!config.ESPLORA.UNIX_SOCKET_PATH,
+      preferred: true,
+      checked: false,
+      publicDomain: `http://${this.localHostname}`,
+      hashes: {
+        lastUpdated: 0,
+      },
+    };
+    this.fallbackHost = this.activeHost;
+    this.hosts.unshift(this.activeHost);
+    this.multihost = this.hosts.length > 1;
+  }
+
+  public startHealthChecks(): void {
+    // use axios interceptors to measure request rtt
+    this.pollConnection.interceptors.request.use((config) => {
+      config['meta'] = { startTime: Date.now() };
+      return config;
+    });
+    this.pollConnection.interceptors.response.use((response) => {
+      response.config['meta'].rtt = Date.now() - response.config['meta'].startTime;
+      return response;
+    });
+
+    if (this.multihost) {
+      this.pollHosts();
+    }
+  }
+
+  // start polling hosts to measure availability & rtt
+  private async pollHosts(): Promise<void> {
+    if (this.pollTimer) {
+      clearTimeout(this.pollTimer);
+    }
+
+    const start = Date.now();
+
+    // update rtts & sync status
+    for (const host of this.hosts) {
+      try {
+        const result = await (host.socket
+          ? this.pollConnection.get<number>('/blocks/tip/height', { socketPath: host.host, timeout: config.ESPLORA.FALLBACK_TIMEOUT })
+          : this.pollConnection.get<number>(host.host + '/blocks/tip/height', { timeout: config.ESPLORA.FALLBACK_TIMEOUT })
+        );
+        if (result) {
+          const height = result.data;
+          host.latestHeight = height;
+          this.maxHeight = Math.max(height || 0, ...this.hosts.map(h => (!(h.unreachable || h.timedOut || h.outOfSync) ? h.latestHeight || 0 : 0)));
+          const rtt = result.config['meta'].rtt;
+          host.rtts.unshift(rtt);
+          host.rtts.slice(0, 5);
+          host.rtt = host.rtts.reduce((acc, l) => acc + l, 0) / host.rtts.length;
+          if (height == null || isNaN(height) || (this.maxHeight - height > this.maxSlippage)) {
+            host.outOfSync = true;
+          } else {
+            host.outOfSync = false;
+          }
+          host.unreachable = false;
+
+          // update esplora git hash using the x-powered-by header from the height check
+          const poweredBy = result.headers['x-powered-by'];
+          if (poweredBy) {
+            const match = poweredBy.match(/([a-fA-F0-9]{5,40})/);
+            if (match && match[1]?.length) {
+              host.hashes.electrs = match[1];
+            }
+          }
+
+          // Check front and backend git hashes less often
+          if (Date.now() - host.hashes.lastUpdated > this.gitHashInterval) {
+            await Promise.all([
+              this.$updateFrontendGitHash(host),
+              this.$updateBackendGitHash(host)
+            ]);
+            host.hashes.lastUpdated = Date.now();
+          }
+        } else {
+          host.outOfSync = true;
+          host.unreachable = true;
+          host.rtts = [];
+          host.rtt = Infinity;
+        }
+        host.timedOut = false;
+      } catch (e) {
+        host.outOfSync = true;
+        host.unreachable = true;
+        host.rtts = [];
+        host.rtt = Infinity;
+        if (isAxiosError(e) && (e.code === 'ECONNABORTED' || e.code === 'ETIMEDOUT')) {
+          host.timedOut = true;
+        } else {
+          host.timedOut = false;
+        }
+      }
+      host.checked = true;
+      host.lastChecked = Date.now();
+
+      const rankOrder = this.sortHosts();
+      // switch if the current host is out of sync or significantly slower than the next best alternative
+      if (this.activeHost.outOfSync || this.activeHost.unreachable || (this.activeHost !== rankOrder[0] && rankOrder[0].preferred) || (!this.activeHost.preferred && this.activeHost.rtt > (rankOrder[0].rtt * 2) + 50)) {
+        if (this.activeHost.unreachable) {
+          logger.warn(`🚨🚨🚨 Unable to reach ${this.activeHost.host}, failing over to next best alternative 🚨🚨🚨`);
+        } else if (this.activeHost.outOfSync) {
+          logger.warn(`🚨🚨🚨 ${this.activeHost.host} has fallen behind, failing over to next best alternative 🚨🚨🚨`);
+        } else {
+          logger.debug(`🛠️ ${this.activeHost.host} is no longer the best esplora host 🛠️`);
+        }
+        this.electHost();
+      }
+      await Common.sleep$(50);
+    }
+
+    const rankOrder = this.updateFallback();
+    logger.debug(`Tomahawk ranking:\n${rankOrder.map((host, index) => this.formatRanking(index, host, this.activeHost, this.maxHeight)).join('\n')}`);
+
+    const elapsed = Date.now() - start;
+
+    this.pollTimer = setTimeout(() => { this.pollHosts(); }, Math.max(1, this.pollInterval - elapsed));
+  }
+
+  private formatRanking(index: number, host: FailoverHost, active: FailoverHost, maxHeight: number): string {
+    const heightStatus = !host.checked ? '⏳' : (host.outOfSync ? '🚫' : (host.latestHeight && host.latestHeight < maxHeight ? '🟧' : '✅'));
+    return `${host === active ? '⭐️' : '  '} ${host.rtt < Infinity ? Math.round(host.rtt).toString().padStart(5, ' ') + 'ms' : (host.timedOut ? '  ⌛️💥 ' : '    -  ')} ${!host.checked ? '⏳' : (host.unreachable ? '🔥' : '✅')} | block: ${host.latestHeight || '??????'} ${heightStatus} | ${host.host} ${host === active ? '⭐️' : '  '}`;
+  }
+
+  private updateFallback(): FailoverHost[] {
+    const rankOrder = this.sortHosts();
+    if (rankOrder.length > 1 && rankOrder[0] === this.activeHost) {
+      this.fallbackHost = rankOrder[1];
+    } else {
+      this.fallbackHost = rankOrder[0];
+    }
+    return rankOrder;
+  }
+
+  // sort hosts by connection quality, and update default fallback
+  public sortHosts(): FailoverHost[] {
+    // sort by connection quality
+    return this.hosts.slice().sort((a, b) => {
+      if ((a.unreachable || a.outOfSync) === (b.unreachable || b.outOfSync)) {
+        if  (a.preferred === b.preferred) {
+          // lower rtt is best
+          return a.rtt - b.rtt;
+        } else { // unless we have a preferred host
+          return a.preferred ? -1 : 1;
+        }
+      } else { // or the host is out of sync
+        return (a.unreachable || a.outOfSync) ? 1 : -1;
+      }
+    });
+  }
+
+  // depose the active host and choose the next best replacement
+  private electHost(): void {
+    this.activeHost.failures = 0;
+    const rankOrder = this.sortHosts();
+    this.activeHost = rankOrder[0];
+    logger.warn(`Switching esplora host to ${this.activeHost.host}`);
+  }
+
+  private addFailure(host: FailoverHost): FailoverHost {
+    host.failures++;
+    if (host.failures > 5 && this.multihost) {
+      logger.warn(`🚨🚨🚨 Too many esplora failures on ${this.activeHost.host}, falling back to next best alternative 🚨🚨🚨`);
+      this.activeHost.unreachable = true;
+      this.electHost();
+      return this.activeHost;
+    } else {
+      return this.fallbackHost;
+    }
+  }
+
+  // methods for retrieving git hashes by host
+  private async $updateFrontendGitHash(host: FailoverHost): Promise<void> {
+    try {
+      const url = `${host.publicDomain}/resources/config.js`;
+      const response = await this.pollConnection.get<string>(url, { timeout: config.ESPLORA.FALLBACK_TIMEOUT });
+      const match = response.data.match(/GIT_COMMIT_HASH\s*=\s*['"](.*?)['"]/);
+      if (match && match[1]?.length) {
+        host.hashes.frontend = match[1];
+      }
+    } catch (e) {
+      // failed to get frontend build hash - do nothing
+    }
+  }
+
+  private async $updateBackendGitHash(host: FailoverHost): Promise<void> {
+    try {
+      const url = `${host.publicDomain}/api/v1/backend-info`;
+      const response = await this.pollConnection.get<any>(url, { timeout: config.ESPLORA.FALLBACK_TIMEOUT });
+      if (response.data?.gitCommit) {
+        host.hashes.backend = response.data.gitCommit;
+      }
+    } catch (e) {
+      // failed to get backend build hash - do nothing
+    }
+  }
+
+  // returns the public mempool domain corresponding to an esplora server url
+  // (a bit of a hack to avoid manually specifying frontend & backend URLs for each esplora server)
+  private extractPublicDomain(url: string): string {
+    // force the url to start with a valid protocol
+    const urlWithProtocol = url.startsWith('http') ? url : `https://${url}`;
+    // parse as URL and extract the hostname
+    try {
+      const parsed = new URL(urlWithProtocol);
+      return parsed.hostname;
+    } catch (e) {
+      // fallback to the original url
+      return url;
+    }
+  }
+
+  private async $query<T>(method: 'get'| 'post', path, data: any, responseType = 'json', host = this.activeHost, retry: boolean = true): Promise<T> {
+    let axiosConfig;
+    let url;
+    if (host.socket) {
+      axiosConfig = { socketPath: host.host, timeout: config.ESPLORA.REQUEST_TIMEOUT, responseType };
+      url = path;
+    } else {
+      axiosConfig = { timeout: config.ESPLORA.REQUEST_TIMEOUT, responseType };
+      url = host.host + path;
+    }
+    if (data?.params) {
+      axiosConfig.params = data.params;
+    }
+    return (method === 'post'
+        ? this.requestConnection.post<T>(url, data, axiosConfig)
+        : this.requestConnection.get<T>(url, axiosConfig)
+    ).then((response) => { host.failures = Math.max(0, host.failures - 1); return response.data; })
+      .catch((e) => {
+        let fallbackHost = this.fallbackHost;
+        if (e?.response?.status !== 404) {
+          logger.warn(`esplora request failed ${e?.response?.status} ${host.host}${path}`);
+          logger.warn(e instanceof Error ? e.message : e);
+          fallbackHost = this.addFailure(host);
+        }
+        if (retry && e?.code === 'ECONNREFUSED' && this.multihost) {
+          // Retry immediately
+          return this.$query(method, path, data, responseType, fallbackHost, false);
+        } else {
+          throw e;
+        }
+      });
+  }
+
+  public async $get<T>(path, responseType = 'json', params: any = null): Promise<T> {
+    return this.$query<T>('get', path, params ? { params } : null, responseType);
+  }
+
+  public async $post<T>(path, data: any, responseType = 'json'): Promise<T> {
+    return this.$query<T>('post', path, data, responseType);
+  }
+}
 
 class ElectrsApi implements AbstractBitcoinApi {
-  axiosConfig: AxiosRequestConfig = {
-    timeout: 10000,
-  };
-
-  constructor() { }
+  private failoverRouter = new FailoverRouter();
 
   $getRawMempool(): Promise<IEsploraApi.Transaction['txid'][]> {
-    return axios.get<IEsploraApi.Transaction['txid'][]>(config.ESPLORA.REST_API_URL + '/mempool/txids', this.axiosConfig)
-      .then((response) => response.data);
+    return this.failoverRouter.$get<IEsploraApi.Transaction['txid'][]>('/mempool/txids');
   }
 
   $getRawTransaction(txId: string): Promise<IEsploraApi.Transaction> {
-    return axios.get<IEsploraApi.Transaction>(config.ESPLORA.REST_API_URL + '/tx/' + txId, this.axiosConfig)
-      .then((response) => response.data);
+    return this.failoverRouter.$get<IEsploraApi.Transaction>('/tx/' + txId);
+  }
+
+  async $getRawTransactions(txids: string[]): Promise<IEsploraApi.Transaction[]> {
+    return this.failoverRouter.$post<IEsploraApi.Transaction[]>('/internal/txs', txids, 'json');
+  }
+
+  async $getMempoolTransactions(txids: string[]): Promise<IEsploraApi.Transaction[]> {
+    return this.failoverRouter.$post<IEsploraApi.Transaction[]>('/internal/mempool/txs', txids, 'json');
+  }
+
+  async $getAllMempoolTransactions(lastSeenTxid?: string, max_txs?: number): Promise<IEsploraApi.Transaction[]> {
+    return this.failoverRouter.$get<IEsploraApi.Transaction[]>('/internal/mempool/txs' + (lastSeenTxid ? '/' + lastSeenTxid : ''), 'json', max_txs ? { max_txs } : null);
+  }
+
+  $getTransactionHex(txId: string): Promise<string> {
+    return this.failoverRouter.$get<string>('/tx/' + txId + '/hex');
   }
 
   $getBlockHeightTip(): Promise<number> {
-    return axios.get<number>(config.ESPLORA.REST_API_URL + '/blocks/tip/height', this.axiosConfig)
-      .then((response) => response.data);
+    return this.failoverRouter.$get<number>('/blocks/tip/height');
+  }
+
+  $getBlockHashTip(): Promise<string> {
+    return this.failoverRouter.$get<string>('/blocks/tip/hash');
   }
 
   $getTxIdsForBlock(hash: string): Promise<string[]> {
-    return axios.get<string[]>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/txids', this.axiosConfig)
-      .then((response) => response.data);
+    return this.failoverRouter.$get<string[]>('/block/' + hash + '/txids');
+  }
+
+  $getTxsForBlock(hash: string): Promise<IEsploraApi.Transaction[]> {
+    return this.failoverRouter.$get<IEsploraApi.Transaction[]>('/internal/block/' + hash + '/txs');
   }
 
   $getBlockHash(height: number): Promise<string> {
-    return axios.get<string>(config.ESPLORA.REST_API_URL + '/block-height/' + height, this.axiosConfig)
-      .then((response) => response.data);
+    return this.failoverRouter.$get<string>('/block-height/' + height);
   }
 
   $getBlockHeader(hash: string): Promise<string> {
-    return axios.get<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/header', this.axiosConfig)
-      .then((response) => response.data);
+    return this.failoverRouter.$get<string>('/block/' + hash + '/header');
   }
 
   $getBlock(hash: string): Promise<IEsploraApi.Block> {
-    return axios.get<IEsploraApi.Block>(config.ESPLORA.REST_API_URL + '/block/' + hash, this.axiosConfig)
-      .then((response) => response.data);
+    return this.failoverRouter.$get<IEsploraApi.Block>('/block/' + hash);
+  }
+
+  $getRawBlock(hash: string): Promise<Buffer> {
+    return this.failoverRouter.$get<any>('/block/' + hash + '/raw', 'arraybuffer')
+      .then((response) => { return Buffer.from(response.data); });
   }
 
   $getAddress(address: string): Promise<IEsploraApi.Address> {
-    throw new Error('Method getAddress not implemented.');
+    return this.failoverRouter.$get<IEsploraApi.Address>('/address/' + address);
   }
 
   $getAddressTransactions(address: string, txId?: string): Promise<IEsploraApi.Transaction[]> {
     throw new Error('Method getAddressTransactions not implemented.');
   }
 
+  $getScriptHash(scripthash: string): Promise<IEsploraApi.ScriptHash> {
+    throw new Error('Method getScriptHash not implemented.');
+  }
+
+  $getScriptHashTransactions(scripthash: string, txId?: string): Promise<IEsploraApi.Transaction[]> {
+    throw new Error('Method getScriptHashTransactions not implemented.');
+  }
+
   $getAddressPrefix(prefix: string): string[] {
     throw new Error('Method not implemented.');
   }
@@ -60,6 +408,66 @@ class ElectrsApi implements AbstractBitcoinApi {
   $sendRawTransaction(rawTransaction: string): Promise<string> {
     throw new Error('Method not implemented.');
   }
+
+  $testMempoolAccept(rawTransactions: string[], maxfeerate?: number): Promise<TestMempoolAcceptResult[]> {
+    throw new Error('Method not implemented.');
+  }
+
+  $submitPackage(rawTransactions: string[]): Promise<SubmitPackageResult> {
+    throw new Error('Method not implemented.');
+  }
+
+  $getOutspend(txId: string, vout: number): Promise<IEsploraApi.Outspend> {
+    return this.failoverRouter.$get<IEsploraApi.Outspend>('/tx/' + txId + '/outspend/' + vout);
+  }
+
+  $getOutspends(txId: string): Promise<IEsploraApi.Outspend[]> {
+    return this.failoverRouter.$get<IEsploraApi.Outspend[]>('/tx/' + txId + '/outspends');
+  }
+
+  async $getBatchedOutspends(txids: string[]): Promise<IEsploraApi.Outspend[][]> {
+    throw new Error('Method not implemented.');
+  }
+
+  async $getBatchedOutspendsInternal(txids: string[]): Promise<IEsploraApi.Outspend[][]> {
+    return this.failoverRouter.$post<IEsploraApi.Outspend[][]>('/internal/txs/outspends/by-txid', txids, 'json');
+  }
+
+  async $getOutSpendsByOutpoint(outpoints: { txid: string, vout: number }[]): Promise<IEsploraApi.Outspend[]> {
+    return this.failoverRouter.$post<IEsploraApi.Outspend[]>('/internal/txs/outspends/by-outpoint', outpoints.map(out => `${out.txid}:${out.vout}`), 'json');
+  }
+
+  async $getCoinbaseTx(blockhash: string): Promise<IEsploraApi.Transaction> {
+    const txid = await this.failoverRouter.$get<string>(`/block/${blockhash}/txid/0`);
+    return this.failoverRouter.$get<IEsploraApi.Transaction>('/tx/' + txid);
+  }
+
+  async $getAddressTransactionSummary(address: string): Promise<IEsploraApi.AddressTxSummary[]> {
+    return this.failoverRouter.$get<IEsploraApi.AddressTxSummary[]>('/address/' + address + '/txs/summary');
+  }
+
+  public startHealthChecks(): void {
+    this.failoverRouter.startHealthChecks();
+  }
+
+  public getHealthStatus(): HealthCheckHost[] {
+    if (config.MEMPOOL.OFFICIAL) {
+      return this.failoverRouter.sortHosts().map(host => ({
+        host: host.host,
+        active: host === this.failoverRouter.activeHost,
+        rtt: host.rtt,
+        latestHeight: host.latestHeight || 0,
+        socket: !!host.socket,
+        outOfSync: !!host.outOfSync,
+        unreachable: !!host.unreachable,
+        checked: !!host.checked,
+        lastChecked: host.lastChecked || 0,
+        hashes: host.hashes,
+      }));
+    } else {
+      return [];
+    }
+  }
 }
 
 export default ElectrsApi;

backend/src/api/chain-tips.ts

@@ -0,0 +1,91 @@
+import logger from '../logger';
+import bitcoinClient from './bitcoin/bitcoin-client';
+
+export interface ChainTip {
+  height: number;
+  hash: string;
+  branchlen: number;
+  status: 'invalid' | 'active' | 'valid-fork' | 'valid-headers' | 'headers-only';
+};
+
+export interface OrphanedBlock {
+  height: number;
+  hash: string;
+  status: 'valid-fork' | 'valid-headers' | 'headers-only';
+  prevhash: string;
+}
+
+class ChainTips {
+  private chainTips: ChainTip[] = [];
+  private orphanedBlocks: { [hash: string]: OrphanedBlock } = {};
+  private blockCache: { [hash: string]: OrphanedBlock } = {};
+  private orphansByHeight: { [height: number]: OrphanedBlock[] } = {};
+
+  public async updateOrphanedBlocks(): Promise<void> {
+    try {
+      this.chainTips = await bitcoinClient.getChainTips();
+
+      const start = Date.now();
+      const breakAt = start + 10000;
+      let newOrphans = 0;
+      this.orphanedBlocks = {};
+
+      for (const chain of this.chainTips) {
+        if (chain.status === 'valid-fork' || chain.status === 'valid-headers') {
+          const orphans: OrphanedBlock[] = [];
+          let hash = chain.hash;
+          do {
+            let orphan = this.blockCache[hash];
+            if (!orphan) {
+              const block = await bitcoinClient.getBlock(hash);
+              if (block && block.confirmations === -1) {
+                newOrphans++;
+                orphan = {
+                  height: block.height,
+                  hash: block.hash,
+                  status: chain.status,
+                  prevhash: block.previousblockhash,
+                };
+                this.blockCache[hash] = orphan;
+              }
+            }
+            if (orphan) {
+              orphans.push(orphan);
+            }
+            hash = orphan?.prevhash;
+          } while (hash && (Date.now() < breakAt));
+          for (const orphan of orphans) {
+            this.orphanedBlocks[orphan.hash] = orphan;
+          }
+        }
+        if (Date.now() >= breakAt) {
+          logger.debug(`Breaking orphaned blocks updater after 10s, will continue next block`);
+          break;
+        }
+      }
+
+      this.orphansByHeight = {};
+      const allOrphans = Object.values(this.orphanedBlocks);
+      for (const orphan of allOrphans) {
+        if (!this.orphansByHeight[orphan.height]) {
+          this.orphansByHeight[orphan.height] = [];
+        }
+        this.orphansByHeight[orphan.height].push(orphan);
+      }
+
+      logger.debug(`Updated orphaned blocks cache. Fetched ${newOrphans} new orphaned blocks. Total ${allOrphans.length}`);
+    } catch (e) {
+      logger.err(`Cannot get fetch orphaned blocks. Reason: ${e instanceof Error ? e.message : e}`);
+    }
+  }
+
+  public getOrphanedBlocksAtHeight(height: number | undefined): OrphanedBlock[] {
+    if (height === undefined) {
+      return [];
+    }
+
+    return this.orphansByHeight[height] || [];
+  }
+}
+
+export default new ChainTips();

backend/src/api/cpfp.ts

@@ -0,0 +1,272 @@
+import { Ancestor, CpfpCluster, CpfpInfo, CpfpSummary, MempoolTransactionExtended, TransactionExtended } from '../mempool.interfaces';
+import { GraphTx, convertToGraphTx, expandRelativesGraph, initializeRelatives, makeBlockTemplate, mempoolComparator, removeAncestors, setAncestorScores } from './mini-miner';
+import memPool from './mempool';
+import { Acceleration } from './acceleration/acceleration';
+
+const CPFP_UPDATE_INTERVAL = 60_000; // update CPFP info at most once per 60s per transaction
+const MAX_CLUSTER_ITERATIONS = 100;
+
+export function calculateFastBlockCpfp(height: number, transactions: MempoolTransactionExtended[], saveRelatives: boolean = false): CpfpSummary {
+  const clusters: CpfpCluster[] = []; // list of all cpfp clusters in this block
+  const clusterMap: { [txid: string]: CpfpCluster } = {}; // map transactions to their cpfp cluster
+  let clusterTxs: TransactionExtended[] = []; // working list of elements of the current cluster
+  let ancestors: { [txid: string]: boolean } = {}; // working set of ancestors of the current cluster root
+  const txMap: { [txid: string]: TransactionExtended } = {};
+  // initialize the txMap
+  for (const tx of transactions) {
+    txMap[tx.txid] = tx;
+  }
+  // reverse pass to identify CPFP clusters
+  for (let i = transactions.length - 1; i >= 0; i--) {
+    const tx = transactions[i];
+    if (!ancestors[tx.txid]) {
+      let totalFee = 0;
+      let totalVSize = 0;
+      clusterTxs.forEach(tx => {
+        totalFee += tx?.fee || 0;
+        totalVSize += (tx.weight / 4);
+      });
+      const effectiveFeePerVsize = totalFee / totalVSize;
+      let cluster: CpfpCluster;
+      if (clusterTxs.length > 1) {
+        cluster = {
+          root: clusterTxs[0].txid,
+          height,
+          txs: clusterTxs.map(tx => { return { txid: tx.txid, weight: tx.weight, fee: tx.fee || 0 }; }),
+          effectiveFeePerVsize,
+        };
+        clusters.push(cluster);
+      }
+      clusterTxs.forEach(tx => {
+        txMap[tx.txid].effectiveFeePerVsize = effectiveFeePerVsize;
+        if (cluster) {
+          clusterMap[tx.txid] = cluster;
+        }
+      });
+      // reset working vars
+      clusterTxs = [];
+      ancestors = {};
+    }
+    clusterTxs.push(tx);
+    tx.vin.forEach(vin => {
+      ancestors[vin.txid] = true;
+    });
+  }
+  // forward pass to enforce ancestor rate caps
+  for (const tx of transactions) {
+    let minAncestorRate = tx.effectiveFeePerVsize;
+    for (const vin of tx.vin) {
+      if (txMap[vin.txid]?.effectiveFeePerVsize) {
+        minAncestorRate = Math.min(minAncestorRate, txMap[vin.txid].effectiveFeePerVsize);
+      }
+    }
+    // check rounded values to skip cases with almost identical fees
+    const roundedMinAncestorRate = Math.ceil(minAncestorRate);
+    const roundedEffectiveFeeRate = Math.floor(tx.effectiveFeePerVsize);
+    if (roundedMinAncestorRate < roundedEffectiveFeeRate) {
+      tx.effectiveFeePerVsize = minAncestorRate;
+      if (!clusterMap[tx.txid]) {
+        // add a single-tx cluster to record the dependent rate
+        const cluster = {
+          root: tx.txid,
+          height,
+          txs: [{ txid: tx.txid, weight: tx.weight, fee: tx.fee || 0 }],
+          effectiveFeePerVsize: minAncestorRate,
+        };
+        clusterMap[tx.txid] = cluster;
+        clusters.push(cluster);
+      } else {
+        // update the existing cluster with the dependent rate
+        clusterMap[tx.txid].effectiveFeePerVsize = minAncestorRate;
+      }
+    }
+  }
+  if (saveRelatives) {
+    for (const cluster of clusters) {
+      cluster.txs.forEach((member, index) => {
+        txMap[member.txid].descendants = cluster.txs.slice(0, index).reverse();
+        txMap[member.txid].ancestors = cluster.txs.slice(index + 1).reverse();
+        txMap[member.txid].effectiveFeePerVsize = cluster.effectiveFeePerVsize;
+      });
+    }
+  }
+  return {
+    transactions,
+    clusters,
+    version: 1,
+  };
+}
+
+export function calculateGoodBlockCpfp(height: number, transactions: MempoolTransactionExtended[], accelerations: Acceleration[]): CpfpSummary {
+  const txMap: { [txid: string]: MempoolTransactionExtended } = {};
+  for (const tx of transactions) {
+    txMap[tx.txid] = tx;
+  }
+  const template = makeBlockTemplate(transactions, accelerations, 1, Infinity, Infinity);
+  const clusters = new Map<string, string[]>();
+  for (const tx of template) {
+    const cluster = tx.cluster || [];
+    const root = cluster.length ? cluster[cluster.length - 1] : null;
+    if (cluster.length > 1 && root && !clusters.has(root)) {
+      clusters.set(root, cluster);
+    }
+    txMap[tx.txid].effectiveFeePerVsize = tx.effectiveFeePerVsize;
+  }
+
+  const clusterArray: CpfpCluster[] = [];
+
+  for (const cluster of clusters.values()) {
+    for (const txid of cluster) {
+      const mempoolTx = txMap[txid];
+      if (mempoolTx) {
+        const ancestors: Ancestor[] = [];
+        const descendants: Ancestor[] = [];
+        let matched = false;
+        cluster.forEach(relativeTxid => {
+          if (relativeTxid === txid) {
+            matched = true;
+          } else {
+            const relative = {
+              txid: relativeTxid,
+              fee: txMap[relativeTxid].fee,
+              weight: (txMap[relativeTxid].adjustedVsize * 4) || txMap[relativeTxid].weight,
+            };
+            if (matched) {
+              descendants.push(relative);
+            } else {
+              ancestors.push(relative);
+            }
+          }
+        });
+        if (mempoolTx.ancestors?.length !== ancestors.length || mempoolTx.descendants?.length !== descendants.length) {
+          mempoolTx.cpfpDirty = true;
+        }
+        Object.assign(mempoolTx, { ancestors, descendants, bestDescendant: null, cpfpChecked: true });
+      }
+    }
+    const root = cluster[cluster.length - 1];
+    clusterArray.push({
+      root: root,
+      height,
+      txs: cluster.reverse().map(txid => ({
+        txid,
+        fee: txMap[txid].fee,
+        weight: (txMap[txid].adjustedVsize * 4) || txMap[txid].weight,
+      })),
+      effectiveFeePerVsize: txMap[root].effectiveFeePerVsize,
+    });
+  }
+
+  return {
+    transactions: transactions.map(tx => txMap[tx.txid]),
+    clusters: clusterArray,
+    version: 2,
+  };
+}
+
+/**
+ * Takes a mempool transaction and a copy of the current mempool, and calculates the CPFP data for
+ * that transaction (and all others in the same cluster)
+ */
+export function calculateMempoolTxCpfp(tx: MempoolTransactionExtended, mempool: { [txid: string]: MempoolTransactionExtended }): CpfpInfo {
+  if (tx.cpfpUpdated && Date.now() < (tx.cpfpUpdated + CPFP_UPDATE_INTERVAL)) {
+    tx.cpfpDirty = false;
+    return {
+      ancestors: tx.ancestors || [],
+      bestDescendant: tx.bestDescendant || null,
+      descendants: tx.descendants || [],
+      effectiveFeePerVsize: tx.effectiveFeePerVsize || tx.adjustedFeePerVsize || tx.feePerVsize,
+      sigops: tx.sigops,
+      fee: tx.fee,
+      adjustedVsize: tx.adjustedVsize,
+      acceleration: tx.acceleration
+    };
+  }
+
+  const ancestorMap = new Map<string, GraphTx>();
+  const graphTx = convertToGraphTx(tx, memPool.getSpendMap());
+  ancestorMap.set(tx.txid, graphTx);
+
+  const allRelatives = expandRelativesGraph(mempool, ancestorMap, memPool.getSpendMap());
+  const relativesMap = initializeRelatives(allRelatives);
+  const cluster = calculateCpfpCluster(tx.txid, relativesMap);
+
+  let totalVsize = 0;
+  let totalFee = 0;
+  for (const tx of cluster.values()) {
+    totalVsize += tx.vsize;
+    totalFee += tx.fees.base;
+  }
+  const effectiveFeePerVsize = totalFee / totalVsize;
+  for (const tx of cluster.values()) {
+    mempool[tx.txid].effectiveFeePerVsize = effectiveFeePerVsize;
+    mempool[tx.txid].ancestors = Array.from(tx.ancestors.values()).map(tx => ({ txid: tx.txid, weight: tx.weight, fee: tx.fees.base }));
+    mempool[tx.txid].descendants = Array.from(cluster.values()).filter(entry => entry.txid !== tx.txid && !tx.ancestors.has(entry.txid)).map(tx => ({ txid: tx.txid, weight: tx.weight, fee: tx.fees.base }));
+    mempool[tx.txid].bestDescendant = null;
+    mempool[tx.txid].cpfpChecked = true;
+    mempool[tx.txid].cpfpDirty = true;
+    mempool[tx.txid].cpfpUpdated = Date.now();
+  }
+
+  tx = mempool[tx.txid];
+
+  return {
+    ancestors: tx.ancestors || [],
+    bestDescendant: tx.bestDescendant || null,
+    descendants: tx.descendants || [],
+    effectiveFeePerVsize: tx.effectiveFeePerVsize || tx.adjustedFeePerVsize || tx.feePerVsize,
+    sigops: tx.sigops,
+    fee: tx.fee,
+    adjustedVsize: tx.adjustedVsize,
+    acceleration: tx.acceleration
+  };
+}
+
+/**
+   * Given a root transaction and a list of in-mempool ancestors,
+   * Calculate the CPFP cluster
+   * 
+   * @param tx
+   * @param ancestors
+   */
+function calculateCpfpCluster(txid: string, graph: Map<string, GraphTx>): Map<string, GraphTx> {
+  const tx = graph.get(txid);
+  if (!tx) {
+    return new Map<string, GraphTx>([]);
+  }
+
+  // Initialize individual & ancestor fee rates
+  graph.forEach(entry => setAncestorScores(entry));
+
+  // Sort by descending ancestor score
+  let sortedRelatives = Array.from(graph.values()).sort(mempoolComparator);
+
+  // Iterate until we reach a cluster that includes our target tx
+  let maxIterations = MAX_CLUSTER_ITERATIONS;
+  let best = sortedRelatives.shift();
+  let bestCluster = new Map<string, GraphTx>(best?.ancestors?.entries() || []);
+  while (sortedRelatives.length && best && (best.txid !== tx.txid && !best.ancestors.has(tx.txid)) && maxIterations > 0) {
+    maxIterations--;
+    if ((best && best.txid === tx.txid) || (bestCluster && bestCluster.has(tx.txid))) {
+      break;
+    } else {
+      // Remove this cluster (it doesn't include our target tx)
+      // and update scores, ancestor totals and dependencies for the survivors
+      removeAncestors(bestCluster, graph);
+
+      // re-sort
+      sortedRelatives = Array.from(graph.values()).sort(mempoolComparator);
+
+      // Grab the next highest scoring entry
+      best = sortedRelatives.shift();
+      if (best) {
+        bestCluster = new Map<string, GraphTx>(best?.ancestors?.entries() || []);
+        bestCluster.set(best?.txid, best);
+      }
+    }
+  }
+
+  bestCluster.set(tx.txid, tx);
+
+  return bestCluster;
+}

backend/src/api/difficulty-adjustment.ts

@@ -0,0 +1,183 @@
+import config from '../config';
+import { IDifficultyAdjustment } from '../mempool.interfaces';
+import blocks from './blocks';
+
+export interface DifficultyAdjustment {
+  progressPercent: number;       // Percent: 0 to 100
+  difficultyChange: number;      // Percent: -75 to 300
+  estimatedRetargetDate: number; // Unix time in ms
+  remainingBlocks: number;       // Block count
+  remainingTime: number;         // Duration of time in ms
+  previousRetarget: number;      // Percent: -75 to 300
+  previousTime: number;          // Unix time in ms
+  nextRetargetHeight: number;    // Block Height
+  timeAvg: number;               // Duration of time in ms
+  adjustedTimeAvg;               // Expected block interval with hashrate implied over last 504 blocks
+  timeOffset: number;            // (Testnet) Time since last block (cap @ 20min) in ms
+  expectedBlocks: number;         // Block count
+}
+
+/**
+ * Calculate the difficulty increase/decrease by using the `bits` integer contained in two
+ * block headers.
+ *
+ * Warning: Only compare `bits` from blocks in two adjacent difficulty periods. This code
+ * assumes the maximum difference is x4 or /4 (as per the protocol) and will throw an
+ * error if an exponent difference of 2 or more is seen.
+ *
+ * @param {number} oldBits The 32 bit `bits` integer from a block header.
+ * @param {number} newBits The 32 bit `bits` integer from a block header in the next difficulty period.
+ * @returns {number} A floating point decimal of the difficulty change from old to new.
+ *          (ie. 21.3 means 21.3% increase in difficulty, -21.3 is a 21.3% decrease in difficulty)
+ */
+export function calcBitsDifference(oldBits: number, newBits: number): number {
+  // Must be
+  // - integer
+  // - highest exponent is 0x20, so max value (as integer) is 0x207fffff
+  // - min value is 1 (exponent = 0)
+  // - highest bit of the number-part is +- sign, it must not be 1
+  const verifyBits = (bits: number): void => {
+    if (
+      Math.floor(bits) !== bits ||
+      bits > 0x207fffff ||
+      bits < 1 ||
+      (bits & 0x00800000) !== 0 ||
+      (bits & 0x007fffff) === 0
+    ) {
+      throw new Error('Invalid bits');
+    }
+  };
+  verifyBits(oldBits);
+  verifyBits(newBits);
+
+  // No need to mask exponents because we checked the bounds above
+  const oldExp = oldBits >> 24;
+  const newExp = newBits >> 24;
+  const oldNum = oldBits & 0x007fffff;
+  const newNum = newBits & 0x007fffff;
+  // The diff can only possibly be 1, 0, -1
+  // (because maximum difficulty change is x4 or /4 (2 bits up or down))
+  let result: number;
+  switch (newExp - oldExp) {
+    // New less than old, target lowered, difficulty increased
+    case -1:
+      result = ((oldNum << 8) * 100) / newNum - 100;
+      break;
+    // Same exponent, compare numbers as is.
+    case 0:
+      result = (oldNum * 100) / newNum - 100;
+      break;
+    // Old less than new, target raised, difficulty decreased
+    case 1:
+      result = (oldNum * 100) / (newNum << 8) - 100;
+      break;
+    default:
+      throw new Error('Impossible exponent difference');
+  }
+
+  // Min/Max values
+  return result > 300 ? 300 : result < -75 ? -75 : result;
+}
+
+export function calcDifficultyAdjustment(
+  DATime: number,
+  quarterEpochTime: number | null,
+  nowSeconds: number,
+  blockHeight: number,
+  previousRetarget: number,
+  network: string,
+  latestBlockTimestamp: number,
+): DifficultyAdjustment {
+  const EPOCH_BLOCK_LENGTH = 2016; // Bitcoin mainnet
+  const BLOCK_SECONDS_TARGET = 600; // Bitcoin mainnet
+  const TESTNET_MAX_BLOCK_SECONDS = 1200; // Bitcoin testnet
+
+  const diffSeconds = Math.max(0, nowSeconds - DATime);
+  const blocksInEpoch = (blockHeight >= 0) ? blockHeight % EPOCH_BLOCK_LENGTH : 0;
+  const progressPercent = (blockHeight >= 0) ? blocksInEpoch / EPOCH_BLOCK_LENGTH * 100 : 100;
+  const remainingBlocks = EPOCH_BLOCK_LENGTH - blocksInEpoch;
+  const nextRetargetHeight = (blockHeight >= 0) ? blockHeight + remainingBlocks : 0;
+  const expectedBlocks = diffSeconds / BLOCK_SECONDS_TARGET;
+  const actualTimespan = (blocksInEpoch === 2015 ? latestBlockTimestamp : nowSeconds) - DATime;
+
+  let difficultyChange = 0;
+  let timeAvgSecs = blocksInEpoch ? diffSeconds / blocksInEpoch : BLOCK_SECONDS_TARGET;
+  let adjustedTimeAvgSecs = timeAvgSecs;
+
+  // for the first 504 blocks of the epoch, calculate the expected avg block interval
+  // from a sliding window over the last 504 blocks
+  if (quarterEpochTime && blocksInEpoch < 503) {
+    const timeLastEpoch = DATime - quarterEpochTime;
+    const adjustedTimeLastEpoch = timeLastEpoch * (1 + (previousRetarget / 100));
+    const adjustedTimeSpan = diffSeconds + adjustedTimeLastEpoch;
+    adjustedTimeAvgSecs = adjustedTimeSpan / 503;
+    difficultyChange = (BLOCK_SECONDS_TARGET / (adjustedTimeSpan / 504) - 1) * 100;
+  } else {
+    difficultyChange = (BLOCK_SECONDS_TARGET / (actualTimespan / (blocksInEpoch + 1)) - 1) * 100;
+  }
+
+  // Max increase is x4 (+300%)
+  if (difficultyChange > 300) {
+    difficultyChange = 300;
+  }
+  // Max decrease is /4 (-75%)
+  if (difficultyChange < -75) {
+    difficultyChange = -75;
+  }
+
+  // Testnet difficulty is set to 1 after 20 minutes of no blocks,
+  // therefore the time between blocks will always be below 20 minutes (1200s).
+  let timeOffset = 0;
+  if (network === 'testnet') {
+    if (timeAvgSecs > TESTNET_MAX_BLOCK_SECONDS) {
+      timeAvgSecs = TESTNET_MAX_BLOCK_SECONDS;
+    }
+
+    const secondsSinceLastBlock = nowSeconds - latestBlockTimestamp;
+    if (secondsSinceLastBlock + timeAvgSecs > TESTNET_MAX_BLOCK_SECONDS) {
+      timeOffset = -Math.min(secondsSinceLastBlock, TESTNET_MAX_BLOCK_SECONDS) * 1000;
+    }
+  }
+
+  const timeAvg = Math.floor(timeAvgSecs * 1000);
+  const adjustedTimeAvg = Math.floor(adjustedTimeAvgSecs * 1000);
+  const remainingTime = remainingBlocks * adjustedTimeAvg;
+  const estimatedRetargetDate = remainingTime + nowSeconds * 1000;
+
+  return {
+    progressPercent,
+    difficultyChange,
+    estimatedRetargetDate,
+    remainingBlocks,
+    remainingTime,
+    previousRetarget,
+    previousTime: DATime,
+    nextRetargetHeight,
+    timeAvg,
+    adjustedTimeAvg,
+    timeOffset,
+    expectedBlocks,
+  };
+}
+
+class DifficultyAdjustmentApi {
+  public getDifficultyAdjustment(): IDifficultyAdjustment | null {
+    const DATime = blocks.getLastDifficultyAdjustmentTime();
+    const previousRetarget = blocks.getPreviousDifficultyRetarget();
+    const blockHeight = blocks.getCurrentBlockHeight();
+    const blocksCache = blocks.getBlocks();
+    const latestBlock = blocksCache[blocksCache.length - 1];
+    if (!latestBlock) {
+      return null;
+    }
+    const nowSeconds = Math.floor(new Date().getTime() / 1000);
+    const quarterEpochBlockTime = blocks.getQuarterEpochBlockTime();
+
+    return calcDifficultyAdjustment(
+      DATime, quarterEpochBlockTime, nowSeconds, blockHeight, previousRetarget,
+      config.MEMPOOL.NETWORK, latestBlock.timestamp
+    );
+  }
+}
+
+export default new DifficultyAdjustmentApi();

backend/src/api/disk-cache.ts

@@ -1,74 +1,202 @@
 import * as fs from 'fs';
 const fsPromises = fs.promises;
-import * as cluster from 'cluster';
+import cluster from 'cluster';
 import memPool from './mempool';
 import blocks from './blocks';
 import logger from '../logger';
 import config from '../config';
 import { TransactionExtended } from '../mempool.interfaces';
 import { Common } from './common';
+import rbfCache from './rbf-cache';
 
 class DiskCache {
+  private cacheSchemaVersion = 3;
+  private rbfCacheSchemaVersion = 1;
+
+  private static TMP_FILE_NAME = config.MEMPOOL.CACHE_DIR + '/tmp-cache.json';
+  private static TMP_FILE_NAMES = config.MEMPOOL.CACHE_DIR + '/tmp-cache{number}.json';
   private static FILE_NAME = config.MEMPOOL.CACHE_DIR + '/cache.json';
   private static FILE_NAMES = config.MEMPOOL.CACHE_DIR + '/cache{number}.json';
+  private static TMP_RBF_FILE_NAME = config.MEMPOOL.CACHE_DIR + '/tmp-rbfcache.json';
+  private static RBF_FILE_NAME = config.MEMPOOL.CACHE_DIR + '/rbfcache.json';
   private static CHUNK_FILES = 25;
   private isWritingCache = false;
+  private ignoreBlocksCache = false;
 
-  constructor() { }
+  private semaphore: { resume: (() => void)[], locks: number } = {
+    resume: [],
+    locks: 0,
+  };
 
-  async $saveCacheToDisk(): Promise<void> {
-    if (!cluster.isMaster) {
+  constructor() {
+    if (!cluster.isPrimary || !config.MEMPOOL.CACHE_ENABLED) {
+      return;
+    }
+    process.on('SIGINT', (e) => {
+      this.$saveCacheToDisk(true);
+      process.exit(0);
+    });
+  }
+
+  async $saveCacheToDisk(sync: boolean = false): Promise<void> {
+    if (!cluster.isPrimary || !config.MEMPOOL.CACHE_ENABLED) {
       return;
     }
     if (this.isWritingCache) {
-      logger.debug('Saving cache already in progress. Skipping.')
+      logger.debug('Saving cache already in progress. Skipping.');
       return;
     }
     try {
-      logger.debug('Writing mempool and blocks data to disk cache (async)...');
+      logger.debug(`Writing mempool and blocks data to disk cache (${ sync ? 'sync' : 'async' })...`);
       this.isWritingCache = true;
 
       const mempool = memPool.getMempool();
       const mempoolArray: TransactionExtended[] = [];
       for (const tx in mempool) {
-        mempoolArray.push(mempool[tx]);
+        if (mempool[tx]) {
+          mempoolArray.push(mempool[tx]);
+        }
       }
 
       Common.shuffleArray(mempoolArray);
 
       const chunkSize = Math.floor(mempoolArray.length / DiskCache.CHUNK_FILES);
 
-      await fsPromises.writeFile(DiskCache.FILE_NAME, JSON.stringify({
-        blocks: blocks.getBlocks(),
-        mempool: {},
-        mempoolArray: mempoolArray.splice(0, chunkSize),
-      }), {flag: 'w'});
-      for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
-        await fsPromises.writeFile(DiskCache.FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
+      if (sync) {
+        fs.writeFileSync(DiskCache.TMP_FILE_NAME, JSON.stringify({
+          network: config.MEMPOOL.NETWORK,
+          cacheSchemaVersion: this.cacheSchemaVersion,
+          blocks: blocks.getBlocks(),
+          blockSummaries: blocks.getBlockSummaries(),
           mempool: {},
           mempoolArray: mempoolArray.splice(0, chunkSize),
-        }), {flag: 'w'});
+        }), { flag: 'w' });
+        for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
+          fs.writeFileSync(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
+            mempool: {},
+            mempoolArray: mempoolArray.splice(0, chunkSize),
+          }), { flag: 'w' });
+        }
+
+        fs.renameSync(DiskCache.TMP_FILE_NAME, DiskCache.FILE_NAME);
+        for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
+          fs.renameSync(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), DiskCache.FILE_NAMES.replace('{number}', i.toString()));
+        }
+      } else {
+        await this.$yield();
+        await fsPromises.writeFile(DiskCache.TMP_FILE_NAME, JSON.stringify({
+          network: config.MEMPOOL.NETWORK,
+          cacheSchemaVersion: this.cacheSchemaVersion,
+          blocks: blocks.getBlocks(),
+          blockSummaries: blocks.getBlockSummaries(),
+          mempool: {},
+          mempoolArray: mempoolArray.splice(0, chunkSize),
+        }), { flag: 'w' });
+        for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
+          await this.$yield();
+          await fsPromises.writeFile(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
+            mempool: {},
+            mempoolArray: mempoolArray.splice(0, chunkSize),
+          }), { flag: 'w' });
+        }
+
+        await fsPromises.rename(DiskCache.TMP_FILE_NAME, DiskCache.FILE_NAME);
+        for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
+          await fsPromises.rename(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), DiskCache.FILE_NAMES.replace('{number}', i.toString()));
+        }
       }
+
       logger.debug('Mempool and blocks data saved to disk cache');
       this.isWritingCache = false;
     } catch (e) {
       logger.warn('Error writing to cache file: ' + (e instanceof Error ? e.message : e));
       this.isWritingCache = false;
     }
+
+    try {
+      logger.debug('Writing rbf data to disk cache (async)...');
+      this.isWritingCache = true;
+      const rbfData = rbfCache.dump();
+      if (sync) {
+        fs.writeFileSync(DiskCache.TMP_RBF_FILE_NAME, JSON.stringify({
+          network: config.MEMPOOL.NETWORK,
+          rbfCacheSchemaVersion: this.rbfCacheSchemaVersion,
+          rbf: rbfData,
+        }), { flag: 'w' });
+        fs.renameSync(DiskCache.TMP_RBF_FILE_NAME, DiskCache.RBF_FILE_NAME);
+      } else {
+        await fsPromises.writeFile(DiskCache.TMP_RBF_FILE_NAME, JSON.stringify({
+          network: config.MEMPOOL.NETWORK,
+          rbfCacheSchemaVersion: this.rbfCacheSchemaVersion,
+          rbf: rbfData,
+        }), { flag: 'w' });
+        await fsPromises.rename(DiskCache.TMP_RBF_FILE_NAME, DiskCache.RBF_FILE_NAME);
+      }
+      logger.debug('Rbf data saved to disk cache');
+      this.isWritingCache = false;
+    } catch (e) {
+      logger.warn('Error writing rbf data to cache file: ' + (e instanceof Error ? e.message : e));
+      this.isWritingCache = false;
+    }
   }
 
-  loadMempoolCache() {
-    if (!fs.existsSync(DiskCache.FILE_NAME)) {
+  wipeCache(): void {
+    logger.notice(`Wiping nodejs backend cache/cache*.json files`);
+    try {
+      fs.unlinkSync(DiskCache.FILE_NAME);
+    } catch (e: any) {
+      if (e?.code !== 'ENOENT') {
+        logger.err(`Cannot wipe cache file ${DiskCache.FILE_NAME}. Exception ${JSON.stringify(e)}`);
+      }
+    }
+
+    for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
+      const filename = DiskCache.FILE_NAMES.replace('{number}', i.toString());
+      try {
+        fs.unlinkSync(filename);
+      } catch (e: any) {
+        if (e?.code !== 'ENOENT') {
+          logger.err(`Cannot wipe cache file ${filename}. Exception ${JSON.stringify(e)}`);
+        }
+      }
+    }
+  }
+
+  wipeRbfCache() {
+    logger.notice(`Wipping nodejs backend cache/rbfcache.json file`);
+
+    try {
+      fs.unlinkSync(DiskCache.RBF_FILE_NAME);
+    } catch (e: any) {
+      if (e?.code !== 'ENOENT') {
+        logger.err(`Cannot wipe cache file ${DiskCache.RBF_FILE_NAME}. Exception ${JSON.stringify(e)}`);
+      }
+    }
+  }
+
+  async $loadMempoolCache(): Promise<void> {
+    if (!config.MEMPOOL.CACHE_ENABLED || !fs.existsSync(DiskCache.FILE_NAME)) {
       return;
     }
     try {
+      const start = Date.now();
       let data: any = {};
       const cacheData = fs.readFileSync(DiskCache.FILE_NAME, 'utf8');
       if (cacheData) {
         logger.info('Restoring mempool and blocks data from disk cache');
         data = JSON.parse(cacheData);
+        if (data.cacheSchemaVersion === undefined || data.cacheSchemaVersion !== this.cacheSchemaVersion) {
+          logger.notice('Disk cache contains an outdated schema version. Clearing it and skipping the cache loading.');
+          return this.wipeCache();
+        }
+        if (data.network && data.network !== config.MEMPOOL.NETWORK) {
+          logger.notice('Disk cache contains data from a different network. Clearing it and skipping the cache loading.');
+          return this.wipeCache();
+        }
+
         if (data.mempoolArray) {
           for (const tx of data.mempoolArray) {
+            delete tx.uid;
             data.mempool[tx.txid] = tx;
           }
         }
@@ -81,6 +209,7 @@ class DiskCache {
             const cacheData2 = JSON.parse(fs.readFileSync(fileName, 'utf8'));
             if (cacheData2.mempoolArray) {
               for (const tx of cacheData2.mempoolArray) {
+                delete tx.uid;
                 data.mempool[tx.txid] = tx;
               }
             } else {
@@ -88,15 +217,82 @@ class DiskCache {
             }
           }
         } catch (e) {
-          logger.debug('Error parsing ' + fileName + '. Skipping.');
+          logger.err('Error parsing ' + fileName + '. Skipping. Reason: ' + (e instanceof Error ? e.message : e));
         }
       }
 
-      memPool.setMempool(data.mempool);
-      blocks.setBlocks(data.blocks);
+      logger.info(`Loaded mempool from disk cache in ${Date.now() - start} ms`);
+
+      await memPool.$setMempool(data.mempool);
+      if (!this.ignoreBlocksCache) {
+        blocks.setBlocks(data.blocks);
+        blocks.setBlockSummaries(data.blockSummaries || []);
+      } else {
+        logger.info('Re-saving cache with empty recent blocks data');
+        await this.$saveCacheToDisk(true);
+      }
     } catch (e) {
-      logger.warn('Failed to parse mempoool and blocks cache. Skipping.');
+      logger.warn('Failed to parse mempoool and blocks cache. Skipping. Reason: ' + (e instanceof Error ? e.message : e));
     }
+
+    try {
+      let rbfData: any = {};
+      const rbfCacheData = fs.readFileSync(DiskCache.RBF_FILE_NAME, 'utf8');
+      if (rbfCacheData) {
+        logger.info('Restoring rbf data from disk cache');
+        rbfData = JSON.parse(rbfCacheData);
+        if (rbfData.rbfCacheSchemaVersion === undefined || rbfData.rbfCacheSchemaVersion !== this.rbfCacheSchemaVersion) {
+          logger.notice('Rbf disk cache contains an outdated schema version. Clearing it and skipping the cache loading.');
+          return this.wipeRbfCache();
+        }
+        if (rbfData.network && rbfData.network !== config.MEMPOOL.NETWORK) {
+          logger.notice('Rbf disk cache contains data from a different network. Clearing it and skipping the cache loading.');
+          return this.wipeRbfCache();
+        }
+      }
+
+      if (rbfData?.rbf) {
+        rbfCache.load({
+          txs: rbfData.rbf.txs.map(([txid, entry]) => ({ value: entry })),
+          trees: rbfData.rbf.trees,
+          expiring: rbfData.rbf.expiring.map(([txid, value]) => ({ key: txid, value })),
+          mempool: memPool.getMempool(),
+          spendMap: memPool.getSpendMap(),
+        });
+      }
+    } catch (e) {
+      logger.warn('Failed to parse rbf cache. Skipping. Reason: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  private $yield(): Promise<void> {
+    if (this.semaphore.locks) {
+      logger.debug('Pause writing mempool and blocks data to disk cache (async)');
+      return new Promise((resolve) => {
+        this.semaphore.resume.push(resolve);
+      });
+    } else {
+      return Promise.resolve();
+    }
+  }
+
+  public lock(): void {
+    this.semaphore.locks++;
+  }
+
+  public unlock(): void {
+    this.semaphore.locks = Math.max(0, this.semaphore.locks - 1);
+    if (!this.semaphore.locks && this.semaphore.resume.length) {
+      const nextResume = this.semaphore.resume.shift();
+      if (nextResume) {
+        logger.debug('Resume writing mempool and blocks data to disk cache (async)');
+        nextResume();
+      }
+    }
+  }
+
+  public setIgnoreBlocksCache(): void {
+    this.ignoreBlocksCache = true;
   }
 }
 

backend/src/api/explorer/channels.api.ts

@@ -0,0 +1,735 @@
+import logger from '../../logger';
+import DB from '../../database';
+import nodesApi from './nodes.api';
+import { ResultSetHeader } from 'mysql2';
+import { ILightningApi } from '../lightning/lightning-api.interface';
+import { Common } from '../common';
+
+class ChannelsApi {
+  public async $getAllChannels(): Promise<any[]> {
+    try {
+      const query = `SELECT * FROM channels`;
+      const [rows]: any = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('$getAllChannels error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getAllChannelsGeo(publicKey?: string, style?: string): Promise<any[]> {
+    try {
+      let select: string;
+      if (style === 'widget') {
+        select = `
+          nodes_1.latitude AS node1_latitude, nodes_1.longitude AS node1_longitude,
+          nodes_2.latitude AS node2_latitude, nodes_2.longitude AS node2_longitude
+        `;
+      } else {
+        select = `
+          nodes_1.public_key as node1_public_key, nodes_1.alias AS node1_alias,
+          nodes_1.latitude AS node1_latitude, nodes_1.longitude AS node1_longitude,
+          nodes_2.public_key as node2_public_key, nodes_2.alias AS node2_alias,
+          nodes_2.latitude AS node2_latitude, nodes_2.longitude AS node2_longitude
+        `;
+      }
+
+      const params: string[] = [];
+      let query = `SELECT ${select}
+        FROM channels
+        JOIN nodes AS nodes_1 on nodes_1.public_key = channels.node1_public_key
+        JOIN nodes AS nodes_2 on nodes_2.public_key = channels.node2_public_key
+        WHERE channels.status = 1
+          AND nodes_1.latitude IS NOT NULL AND nodes_1.longitude IS NOT NULL
+          AND nodes_2.latitude IS NOT NULL AND nodes_2.longitude IS NOT NULL
+      `;
+
+      if (publicKey !== undefined) {
+        query += ' AND (nodes_1.public_key = ? OR nodes_2.public_key = ?)';
+        params.push(publicKey);
+        params.push(publicKey);
+      } else {
+        query += ` AND channels.capacity > 1000000
+          GROUP BY nodes_1.public_key, nodes_2.public_key
+          ORDER BY channels.capacity DESC
+          LIMIT 10000
+        `;        
+      }
+
+      const [rows]: any = await DB.query(query, params);
+      return rows.map((row) => {
+        if (style === 'widget') {
+          return [
+            row.node1_longitude, row.node1_latitude,
+            row.node2_longitude, row.node2_latitude,
+          ];
+        } else {
+          return [
+            row.node1_public_key, row.node1_alias,
+            row.node1_longitude, row.node1_latitude,
+            row.node2_public_key, row.node2_alias,
+            row.node2_longitude, row.node2_latitude,
+          ];
+        }
+      });
+    } catch (e) {
+      logger.err('$getAllChannelsGeo error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $searchChannelsById(search: string): Promise<any[]> {
+    try {
+      // restrict search to valid id/short_id prefix formats
+      let searchStripped = search.match(/^[0-9]+[0-9x]*$/)?.[0] || '';
+      if (!searchStripped.length) {
+        return [];
+      }
+      // add wildcard to search by prefix
+      searchStripped += '%';
+      const query = `SELECT id, short_id, capacity, status FROM channels WHERE id LIKE ? OR short_id LIKE ? LIMIT 10`;
+      const [rows]: any = await DB.query(query, [searchStripped, searchStripped]);
+      return rows;
+    } catch (e) {
+      logger.err('$searchChannelsById error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getChannelsByStatus(status: number | number[]): Promise<any[]> {
+    try {
+      let query: string;
+      if (Array.isArray(status)) {
+        query = `SELECT * FROM channels WHERE status IN (${status.join(',')})`;
+      } else {
+        query = `SELECT * FROM channels WHERE status = ?`;
+      }
+      const [rows]: any = await DB.query(query, [status]);
+      return rows;
+    } catch (e) {
+      logger.err('$getChannelsByStatus error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getClosedChannelsWithoutReason(): Promise<any[]> {
+    try {
+      const query = `SELECT * FROM channels WHERE status = 2 AND closing_reason IS NULL AND closing_transaction_id != ''`;
+      const [rows]: any = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('$getClosedChannelsWithoutReason error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getPenaltyClosedChannels(): Promise<any[]> {
+    try {
+      const query = `
+        SELECT n1.alias AS alias_left,
+          n2.alias AS alias_right,
+          channels.*
+        FROM channels
+        LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key
+        LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key
+        WHERE channels.status = 2 AND channels.closing_reason = 3
+        ORDER BY closing_date DESC
+      `;
+      const [rows]: any = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('$getPenaltyClosedChannels error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getUnresolvedClosedChannels(): Promise<any[]> {
+    try {
+      const query = `SELECT * FROM channels WHERE status = 2 AND closing_reason = 2 AND closing_resolved = 0 AND closing_transaction_id != ''`;
+      const [rows]: any = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('$getUnresolvedClosedChannels error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getChannelsWithoutSourceChecked(): Promise<any[]> {
+    try {
+      const query = `
+        SELECT channels.*
+        FROM channels
+        WHERE channels.source_checked != 1
+      `;
+      const [rows]: any = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('$getUnresolvedClosedChannels error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getChannelsWithoutCreatedDate(): Promise<any[]> {
+    try {
+      const query = `SELECT * FROM channels WHERE created IS NULL`;
+      const [rows]: any = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('$getChannelsWithoutCreatedDate error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getChannel(id: string): Promise<any> {
+    try {
+      const query = `
+        SELECT n1.alias AS alias_left, n1.longitude as node1_longitude, n1.latitude as node1_latitude,
+          n2.alias AS alias_right, n2.longitude as node2_longitude, n2.latitude as node2_latitude,
+          channels.*,
+          ns1.channels AS channels_left, ns1.capacity AS capacity_left, ns2.channels AS channels_right, ns2.capacity AS capacity_right
+        FROM channels
+        LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key
+        LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key
+        LEFT JOIN node_stats AS ns1 ON ns1.public_key = channels.node1_public_key
+        LEFT JOIN node_stats AS ns2 ON ns2.public_key = channels.node2_public_key
+        WHERE (
+          ns1.id = (
+            SELECT MAX(id)
+            FROM node_stats
+            WHERE public_key = channels.node1_public_key
+          )
+          AND ns2.id = (
+            SELECT MAX(id)
+            FROM node_stats
+            WHERE public_key = channels.node2_public_key
+          )
+        )
+        AND channels.id = ?
+      `;
+
+      const [rows]: any = await DB.query(query, [id]);
+      if (rows[0]) {
+        return this.convertChannel(rows[0]);
+      }
+    } catch (e) {
+      logger.err('$getChannel error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getChannelsStats(): Promise<any> {
+    try {
+      // Feedback from zerofeerouting:
+      // "I would argue > 5000ppm can be ignored. Channels charging more than .5% fee are ignored by CLN for example."
+      const ignoredFeeRateThreshold = 5000;
+      const ignoredBaseFeeThreshold = 5000;
+
+      // Capacity
+      let query = `SELECT AVG(capacity) AS avgCapacity FROM channels WHERE status = 1 ORDER BY capacity`;
+      const [avgCapacity]: any = await DB.query(query);
+
+      query = `SELECT capacity FROM channels WHERE status = 1 ORDER BY capacity`;
+      let [capacity]: any = await DB.query(query);
+      capacity = capacity.map(capacity => capacity.capacity);
+      const medianCapacity = capacity[Math.floor(capacity.length / 2)];
+
+      // Fee rates
+      query = `SELECT node1_fee_rate FROM channels WHERE node1_fee_rate < ${ignoredFeeRateThreshold} AND status = 1`;
+      let [feeRates1]: any = await DB.query(query);
+      feeRates1 = feeRates1.map(rate => rate.node1_fee_rate);
+      query = `SELECT node2_fee_rate FROM channels WHERE node2_fee_rate < ${ignoredFeeRateThreshold} AND status = 1`;
+      let [feeRates2]: any = await DB.query(query);
+      feeRates2 = feeRates2.map(rate => rate.node2_fee_rate);
+
+      let feeRates = (feeRates1.concat(feeRates2)).sort((a, b) => a - b);
+      let avgFeeRate = 0;
+      for (const rate of feeRates) {
+        avgFeeRate += rate; 
+      }
+      avgFeeRate /= feeRates.length;
+      const medianFeeRate = feeRates[Math.floor(feeRates.length / 2)];
+
+      // Base fees
+      query = `SELECT node1_base_fee_mtokens FROM channels WHERE node1_base_fee_mtokens < ${ignoredBaseFeeThreshold} AND status = 1`;
+      let [baseFees1]: any = await DB.query(query);
+      baseFees1 = baseFees1.map(rate => rate.node1_base_fee_mtokens);
+      query = `SELECT node2_base_fee_mtokens FROM channels WHERE node2_base_fee_mtokens < ${ignoredBaseFeeThreshold} AND status = 1`;
+      let [baseFees2]: any = await DB.query(query);
+      baseFees2 = baseFees2.map(rate => rate.node2_base_fee_mtokens);
+
+      let baseFees = (baseFees1.concat(baseFees2)).sort((a, b) => a - b);
+      let avgBaseFee = 0;
+      for (const fee of baseFees) {
+        avgBaseFee += fee; 
+      }
+      avgBaseFee /= baseFees.length;
+      const medianBaseFee = feeRates[Math.floor(baseFees.length / 2)];
+      
+      return {
+        avgCapacity: parseInt(avgCapacity[0].avgCapacity, 10),
+        avgFeeRate: avgFeeRate,
+        avgBaseFee: avgBaseFee,
+        medianCapacity: medianCapacity,
+        medianFeeRate: medianFeeRate,
+        medianBaseFee: medianBaseFee,
+      }
+
+    } catch (e) {
+      logger.err(`Cannot calculate channels statistics. Reason: ${e instanceof Error ? e.message : e}`);
+      throw e;
+    }
+  }
+
+  public async $getChannelsByTransactionId(transactionIds: string[]): Promise<any[]> {
+    try {
+      const query = `
+        SELECT n1.alias AS alias_left, n2.alias AS alias_right, channels.*
+        FROM channels
+        LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key
+        LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key
+        WHERE channels.transaction_id IN ? OR channels.closing_transaction_id IN ?
+      `;
+      const [rows]: any = await DB.query(query, [[transactionIds], [transactionIds]]);
+      const channels = rows.map((row) => this.convertChannel(row));
+      return channels;
+    } catch (e) {
+      logger.err('$getChannelByTransactionId error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getChannelByClosingId(transactionId: string): Promise<any> {
+    try {
+      const query = `
+        SELECT
+          channels.*
+        FROM channels
+        WHERE channels.closing_transaction_id = ?
+      `;
+      const [rows]: any = await DB.query(query, [transactionId]);
+      if (rows.length > 0) {
+        rows[0].outputs = JSON.parse(rows[0].outputs);
+        return rows[0];
+      }
+    } catch (e) {
+      logger.err('$getChannelByClosingId error: ' + (e instanceof Error ? e.message : e));
+      // don't throw - this data isn't essential
+    }
+  }
+
+  public async $getChannelsByOpeningId(transactionId: string): Promise<any> {
+    try {
+      const query = `
+        SELECT
+          channels.*
+        FROM channels
+        WHERE channels.transaction_id = ?
+      `;
+      const [rows]: any = await DB.query(query, [transactionId]);
+      if (rows.length > 0) {
+        return rows.map(row => {
+          row.outputs = JSON.parse(row.outputs);
+          return row;
+        });
+      }
+    } catch (e) {
+      logger.err('$getChannelsByOpeningId error: ' + (e instanceof Error ? e.message : e));
+      // don't throw - this data isn't essential
+    }
+  }
+
+  public async $updateClosingInfo(channelInfo: { id: string, node1_closing_balance: number, node2_closing_balance: number, closed_by: string | null, closing_fee: number, outputs: ILightningApi.ForensicOutput[]}): Promise<void> {
+    try {
+      const query = `
+        UPDATE channels SET
+          node1_closing_balance = ?,
+          node2_closing_balance = ?,
+          closed_by = ?,
+          closing_fee = ?,
+          outputs = ?
+        WHERE channels.id = ?
+      `;
+      await DB.query<ResultSetHeader>(query, [
+        channelInfo.node1_closing_balance || 0,
+        channelInfo.node2_closing_balance || 0,
+        channelInfo.closed_by,
+        channelInfo.closing_fee || 0,
+        JSON.stringify(channelInfo.outputs),
+        channelInfo.id,
+      ]);
+    } catch (e) {
+      logger.err('$updateClosingInfo error: ' + (e instanceof Error ? e.message : e));
+      // don't throw - this data isn't essential
+    }
+  }
+
+  public async $updateOpeningInfo(channelInfo: { id: string, node1_funding_balance: number, node2_funding_balance: number, funding_ratio: number, single_funded: boolean | void }): Promise<void> {
+    try {
+      const query = `
+        UPDATE channels SET
+          node1_funding_balance = ?,
+          node2_funding_balance = ?,
+          funding_ratio = ?,
+          single_funded = ?
+        WHERE channels.id = ?
+      `;
+      await DB.query<ResultSetHeader>(query, [
+        channelInfo.node1_funding_balance || 0,
+        channelInfo.node2_funding_balance || 0,
+        channelInfo.funding_ratio,
+        channelInfo.single_funded ? 1 : 0,
+        channelInfo.id,
+      ]);
+    } catch (e) {
+      logger.err('$updateOpeningInfo error: ' + (e instanceof Error ? e.message : e));
+      // don't throw - this data isn't essential
+    }
+  }
+
+  public async $markChannelSourceChecked(id: string): Promise<void> {
+    try {
+      const query = `
+        UPDATE channels
+        SET source_checked = 1
+        WHERE id = ?
+      `;
+      await DB.query<ResultSetHeader>(query, [id]);
+    } catch (e) {
+      logger.err('$markChannelSourceChecked error: ' + (e instanceof Error ? e.message : e));
+      // don't throw - this data isn't essential
+    }
+  }
+
+  public async $getChannelsForNode(public_key: string, index: number, length: number, status: string): Promise<any[]> {
+    try {
+      let channelStatusFilter;
+      if (status === 'open') {
+        channelStatusFilter = '< 2';
+      } else if (status === 'active') {
+        channelStatusFilter = '= 1';
+      } else if (status === 'closed') {
+        channelStatusFilter = '= 2';
+      } else {
+        throw new Error('getChannelsForNode: Invalid status requested');
+      }
+
+      // Channels originating from node
+      let query = `
+        SELECT COALESCE(node2.alias, SUBSTRING(node2_public_key, 0, 20)) AS alias, COALESCE(node2.public_key, node2_public_key) AS public_key,
+          channels.status, channels.node1_fee_rate,
+          channels.capacity, channels.short_id, channels.id, channels.closing_reason,
+          UNIX_TIMESTAMP(closing_date) as closing_date, UNIX_TIMESTAMP(channels.updated_at) as updated_at
+        FROM channels
+        LEFT JOIN nodes AS node2 ON node2.public_key = channels.node2_public_key
+        WHERE node1_public_key = ? AND channels.status ${channelStatusFilter}
+      `;
+      const [channelsFromNode]: any = await DB.query(query, [public_key]);
+
+      // Channels incoming to node
+      query = `
+        SELECT COALESCE(node1.alias, SUBSTRING(node1_public_key, 0, 20)) AS alias, COALESCE(node1.public_key, node1_public_key) AS public_key,
+          channels.status, channels.node2_fee_rate,
+          channels.capacity, channels.short_id, channels.id, channels.closing_reason,
+          UNIX_TIMESTAMP(closing_date) as closing_date, UNIX_TIMESTAMP(channels.updated_at) as updated_at
+        FROM channels
+        LEFT JOIN nodes AS node1 ON node1.public_key = channels.node1_public_key
+        WHERE node2_public_key = ? AND channels.status ${channelStatusFilter}
+      `;
+      const [channelsToNode]: any = await DB.query(query, [public_key]);
+
+      let allChannels = channelsFromNode.concat(channelsToNode);
+      allChannels.sort((a, b) => {
+        if (status === 'closed') {
+          if (!b.closing_date && !a.closing_date) {
+            return (b.updated_at ?? 0) - (a.updated_at ?? 0);
+          } else {
+            return (b.closing_date ?? 0) - (a.closing_date ?? 0);
+          }
+        } else {
+          return b.capacity - a.capacity;
+        }
+      });
+
+      if (index >= 0) {
+        allChannels = allChannels.slice(index, index + length);
+      } else if (index === -1) { // Node channels tree chart
+        allChannels = allChannels.slice(0, 1000);
+      }
+
+      const channels: any[] = []
+      for (const row of allChannels) {
+        let channel;
+        if (index >= 0) {
+          const activeChannelsStats: any = await nodesApi.$getActiveChannelsStats(row.public_key);
+          channel = {
+            status: row.status,
+            closing_reason: row.closing_reason,
+            closing_date: row.closing_date,
+            capacity: row.capacity ?? 0,
+            short_id: row.short_id,
+            id: row.id,
+            fee_rate: row.node1_fee_rate ?? row.node2_fee_rate ?? 0,
+            node: {
+              alias: row.alias.length > 0 ? row.alias : row.public_key.slice(0, 20),
+              public_key: row.public_key,
+              channels: activeChannelsStats.active_channel_count ?? 0,
+              capacity: activeChannelsStats.capacity ?? 0,
+            }
+          };
+        } else if (index === -1) {
+          channel = {
+            capacity: row.capacity ?? 0,
+            short_id: row.short_id,
+            id: row.id,
+            node: {
+              alias: row.alias.length > 0 ? row.alias : row.public_key.slice(0, 20),
+              public_key: row.public_key,
+            }
+          };
+        }
+
+        channels.push(channel);
+      }
+
+      return channels;
+    } catch (e) {
+      logger.err('$getChannelsForNode error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getChannelsCountForNode(public_key: string, status: string): Promise<any> {
+    try {
+      // Default active and inactive channels
+      let statusQuery = '< 2';
+      // Closed channels only
+      if (status === 'closed') {
+        statusQuery = '= 2';
+      }
+      const query = `
+        SELECT COUNT(*) AS count
+        FROM channels
+        WHERE (node1_public_key = ? OR node2_public_key = ?)
+        AND status ${statusQuery}
+      `;
+      const [rows]: any = await DB.query(query, [public_key, public_key]);
+      return rows[0]['count'];
+    } catch (e) {
+      logger.err('$getChannelsForNode error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  private convertChannel(channel: any): any {
+    return {
+      'id': channel.id,
+      'short_id': channel.short_id,
+      'capacity': channel.capacity,
+      'transaction_id': channel.transaction_id,
+      'transaction_vout': channel.transaction_vout,
+      'closing_transaction_id': channel.closing_transaction_id,
+      'closing_fee': channel.closing_fee,
+      'closing_reason': channel.closing_reason,
+      'closing_date': channel.closing_date,
+      'updated_at': channel.updated_at,
+      'created': channel.created,
+      'status': channel.status,
+      'funding_ratio': channel.funding_ratio,
+      'closed_by': channel.closed_by,
+      'single_funded': !!channel.single_funded,
+      'node_left': {
+        'alias': channel.alias_left,
+        'public_key': channel.node1_public_key,
+        'channels': channel.channels_left,
+        'capacity': channel.capacity_left,
+        'base_fee_mtokens': channel.node1_base_fee_mtokens,
+        'cltv_delta': channel.node1_cltv_delta,
+        'fee_rate': channel.node1_fee_rate,
+        'is_disabled': channel.node1_is_disabled,
+        'max_htlc_mtokens': channel.node1_max_htlc_mtokens,
+        'min_htlc_mtokens': channel.node1_min_htlc_mtokens,
+        'updated_at': channel.node1_updated_at,
+        'longitude': channel.node1_longitude,
+        'latitude': channel.node1_latitude,
+        'funding_balance': channel.node1_funding_balance,
+        'closing_balance': channel.node1_closing_balance,
+        'initiated_close': channel.closed_by === channel.node1_public_key ? true : undefined,
+      },
+      'node_right': {
+        'alias': channel.alias_right,
+        'public_key': channel.node2_public_key,
+        'channels': channel.channels_right,
+        'capacity': channel.capacity_right,
+        'base_fee_mtokens': channel.node2_base_fee_mtokens,
+        'cltv_delta': channel.node2_cltv_delta,
+        'fee_rate': channel.node2_fee_rate,
+        'is_disabled': channel.node2_is_disabled,
+        'max_htlc_mtokens': channel.node2_max_htlc_mtokens,
+        'min_htlc_mtokens': channel.node2_min_htlc_mtokens,
+        'updated_at': channel.node2_updated_at,
+        'longitude': channel.node2_longitude,
+        'latitude': channel.node2_latitude,
+        'funding_balance': channel.node2_funding_balance,
+        'closing_balance': channel.node2_closing_balance,
+        'initiated_close': channel.closed_by === channel.node2_public_key ? true : undefined,
+      },
+    };
+  }
+
+  /**
+   * Save or update a channel present in the graph
+   */
+  public async $saveChannel(channel: ILightningApi.Channel, status = 1): Promise<void> {
+    const [ txid, vout ] = channel.chan_point.split(':');
+
+    const policy1: Partial<ILightningApi.RoutingPolicy> = channel.node1_policy || {};
+    const policy2: Partial<ILightningApi.RoutingPolicy> = channel.node2_policy || {};
+
+    // https://github.com/mempool/mempool/issues/3006
+    if ((channel.last_update ?? 0) < 1514736061) { // January 1st 2018
+      channel.last_update = null;
+    }
+    if ((policy1.last_update ?? 0) < 1514736061) { // January 1st 2018
+      policy1.last_update = null;
+    }
+    if ((policy2.last_update ?? 0) < 1514736061) { // January 1st 2018
+      policy2.last_update = null;
+    }
+
+    const query = `INSERT INTO channels
+      (
+        id,
+        short_id,
+        capacity,
+        transaction_id,
+        transaction_vout,
+        updated_at,
+        status,
+        node1_public_key,
+        node1_base_fee_mtokens,
+        node1_cltv_delta,
+        node1_fee_rate,
+        node1_is_disabled,
+        node1_max_htlc_mtokens,
+        node1_min_htlc_mtokens,
+        node1_updated_at,
+        node2_public_key,
+        node2_base_fee_mtokens,
+        node2_cltv_delta,
+        node2_fee_rate,
+        node2_is_disabled,
+        node2_max_htlc_mtokens,
+        node2_min_htlc_mtokens,
+        node2_updated_at
+      )
+      VALUES (?, ?, ?, ?, ?, ?, ${status}, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      ON DUPLICATE KEY UPDATE
+        capacity = ?,
+        updated_at = ?,
+        status = ${status},
+        node1_public_key = ?,
+        node1_base_fee_mtokens = ?,
+        node1_cltv_delta = ?,
+        node1_fee_rate = ?,
+        node1_is_disabled = ?,
+        node1_max_htlc_mtokens = ?,
+        node1_min_htlc_mtokens = ?,
+        node1_updated_at = ?,
+        node2_public_key = ?,
+        node2_base_fee_mtokens = ?,
+        node2_cltv_delta = ?,
+        node2_fee_rate = ?,
+        node2_is_disabled = ?,
+        node2_max_htlc_mtokens = ?,
+        node2_min_htlc_mtokens = ?,
+        node2_updated_at = ?
+      ;`;
+
+    await DB.query(query, [
+      Common.channelShortIdToIntegerId(channel.channel_id),
+      Common.channelIntegerIdToShortId(channel.channel_id),
+      channel.capacity,
+      txid,
+      vout,
+      Common.utcDateToMysql(channel.last_update),
+      channel.node1_pub,
+      policy1.fee_base_msat,
+      policy1.time_lock_delta,
+      policy1.fee_rate_milli_msat,
+      policy1.disabled,
+      policy1.max_htlc_msat,
+      policy1.min_htlc,
+      Common.utcDateToMysql(policy1.last_update),
+      channel.node2_pub,
+      policy2.fee_base_msat,
+      policy2.time_lock_delta,
+      policy2.fee_rate_milli_msat,
+      policy2.disabled,
+      policy2.max_htlc_msat,
+      policy2.min_htlc,
+      Common.utcDateToMysql(policy2.last_update),
+      channel.capacity,
+      Common.utcDateToMysql(channel.last_update),
+      channel.node1_pub,
+      policy1.fee_base_msat,
+      policy1.time_lock_delta,
+      policy1.fee_rate_milli_msat,
+      policy1.disabled,
+      policy1.max_htlc_msat,
+      policy1.min_htlc,
+      Common.utcDateToMysql(policy1.last_update),
+      channel.node2_pub,
+      policy2.fee_base_msat,
+      policy2.time_lock_delta,
+      policy2.fee_rate_milli_msat,
+      policy2.disabled,
+      policy2.max_htlc_msat,
+      policy2.min_htlc,
+      Common.utcDateToMysql(policy2.last_update)
+    ]);
+  }
+
+  /**
+   * Set all channels not in `graphChannelsIds` as inactive (status = 0)
+   */
+  public async $setChannelsInactive(graphChannelsIds: string[]): Promise<void> {
+    if (graphChannelsIds.length === 0) {
+      return;
+    }
+
+    try {
+      const result = await DB.query<ResultSetHeader>(`
+        UPDATE channels
+        SET status = 0
+        WHERE id NOT IN (
+          ${graphChannelsIds.map(id => `"${id}"`).join(',')}
+        )
+        AND status != 2
+      `);
+      if (result[0].changedRows ?? 0 > 0) {
+        logger.debug(`Marked ${result[0].changedRows} channels as inactive because they are not in the graph`, logger.tags.ln);
+      }
+    } catch (e) {
+      logger.err('$setChannelsInactive() error: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  public async $getLatestChannelUpdateForNode(publicKey: string): Promise<number> {
+    try {
+      const query = `
+        SELECT MAX(UNIX_TIMESTAMP(updated_at)) as updated_at
+        FROM channels
+        WHERE node1_public_key = ?
+      `;
+      const [rows]: any[] = await DB.query(query, [publicKey]);
+      if (rows.length > 0) {
+        return rows[0].updated_at;
+      }
+    } catch (e) {
+      logger.err(`Can't getLatestChannelUpdateForNode for ${publicKey}. Reason ${e instanceof Error ? e.message : e}`);
+    }
+    return 0;
+  }
+}
+
+export default new ChannelsApi();

backend/src/api/explorer/channels.routes.ts

@@ -0,0 +1,147 @@
+import config from '../../config';
+import { Application, Request, Response } from 'express';
+import channelsApi from './channels.api';
+import { handleError } from '../../utils/api';
+
+const TXID_REGEX = /^[a-f0-9]{64}$/i;
+
+class ChannelsRoutes {
+  constructor() { }
+
+  public initRoutes(app: Application) {
+    app
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels/txids', this.$getChannelsByTransactionIds)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels/search/:search', this.$searchChannelsById)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels/:short_id', this.$getChannel)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels', this.$getChannelsForNode)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/penalties', this.$getPenaltyClosedChannels)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels-geo', this.$getAllChannelsGeo)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels-geo/:publicKey', this.$getAllChannelsGeo)
+    ;
+  }
+
+  private async $searchChannelsById(req: Request, res: Response) {
+    try {
+      const channels = await channelsApi.$searchChannelsById(req.params.search);
+      res.json(channels);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to search channels by id');
+    }
+  }
+
+  private async $getChannel(req: Request, res: Response) {
+    try {
+      const channel = await channelsApi.$getChannel(req.params.short_id);
+      if (!channel) {
+        res.status(404).send('Channel not found');
+        return;
+      }
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(channel);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get channel');
+    }
+  }
+
+  private async $getChannelsForNode(req: Request, res: Response) {
+    try {
+      if (typeof req.query.public_key !== 'string') {
+        res.status(400).send('Missing parameter: public_key');
+        return;
+      }
+
+      const index = parseInt(typeof req.query.index === 'string' ? req.query.index : '0', 10) || 0;
+      const status: string = typeof req.query.status === 'string' ? req.query.status : '';
+
+      if (index < -1) {
+        handleError(req, res, 400, 'Invalid index');
+        return;
+      }
+      if (['open', 'active', 'closed'].includes(status) === false) {
+        handleError(req, res, 400, 'Invalid status');
+        return;
+      }
+
+      const channels = await channelsApi.$getChannelsForNode(req.query.public_key, index, 10, status);
+      const channelsCount = await channelsApi.$getChannelsCountForNode(req.query.public_key, status);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.header('X-Total-Count', channelsCount.toString());
+      res.json(channels);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get channels for node');
+    }
+  }
+
+  private async $getChannelsByTransactionIds(req: Request, res: Response): Promise<void> {
+    try {
+      if (!Array.isArray(req.query.txId)) {
+        handleError(req, res, 400, 'Not an array');
+        return;
+      }
+      const txIds: string[] = [];
+      for (const _txId in req.query.txId) {
+        if (typeof req.query.txId[_txId] === 'string') {
+          const txid = req.query.txId[_txId].toString();
+          if (TXID_REGEX.test(txid)) {
+            txIds.push(txid);
+          }
+        }
+      }
+      const channels = await channelsApi.$getChannelsByTransactionId(txIds);
+      const result: any[] = [];
+      for (const txid of txIds) {
+        const inputs: any = {};
+        const outputs: any = {};
+        // Assuming that we only have one lightning close input in each transaction. This may not be true in the future
+        const foundChannelsFromInput = channels.find((channel) => channel.closing_transaction_id === txid);
+        if (foundChannelsFromInput) {
+          inputs[0] = foundChannelsFromInput;
+        }
+        const foundChannelsFromOutputs = channels.filter((channel) => channel.transaction_id === txid);
+        for (const output of foundChannelsFromOutputs) {
+          outputs[output.transaction_vout] = output;
+        }
+        result.push({
+          inputs,
+          outputs,
+        });
+      }
+
+      res.json(result);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get channels by transaction ids');
+    }
+  }
+
+  private async $getPenaltyClosedChannels(req: Request, res: Response): Promise<void> {
+    try {
+      const channels = await channelsApi.$getPenaltyClosedChannels();
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(channels);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get penalty closed channels');
+    }
+  }
+
+  private async $getAllChannelsGeo(req: Request, res: Response) {
+    try {
+      const style: string = typeof req.query.style === 'string' ? req.query.style : '';
+      const channels = await channelsApi.$getAllChannelsGeo(req.params?.publicKey, style);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(channels);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get channel geodata');
+    }
+  }
+
+}
+
+export default new ChannelsRoutes();

backend/src/api/explorer/general.routes.ts

@@ -0,0 +1,60 @@
+import config from '../../config';
+import { Application, Request, Response } from 'express';
+import nodesApi from './nodes.api';
+import channelsApi from './channels.api';
+import statisticsApi from './statistics.api';
+import { handleError } from '../../utils/api';
+
+class GeneralLightningRoutes {
+  constructor() { }
+
+  public initRoutes(app: Application) {
+    app
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/search', this.$searchNodesAndChannels)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/statistics/latest', this.$getGeneralStats)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/statistics/:interval', this.$getStatistics)
+    ;
+  }
+
+  private async $searchNodesAndChannels(req: Request, res: Response) {
+    if (typeof req.query.searchText !== 'string') {
+      res.status(400).send('Missing parameter: searchText');
+      return;
+    }
+    try {
+      const nodes = await nodesApi.$searchNodeByPublicKeyOrAlias(req.query.searchText);
+      const channels = await channelsApi.$searchChannelsById(req.query.searchText);
+      res.json({
+        nodes: nodes,
+        channels: channels,
+      });
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to search for nodes and channels');
+    }
+  }
+
+  private async $getStatistics(req: Request, res: Response) {
+    try {
+      const statistics = await statisticsApi.$getStatistics(req.params.interval);
+      const statisticsCount = await statisticsApi.$getStatisticsCount();
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.header('X-total-count', statisticsCount.toString());
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(statistics);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get lightning statistics');
+    }
+  }
+
+  private async $getGeneralStats(req: Request, res: Response) {
+    try {
+      const statistics = await statisticsApi.$getLatestStatistics();
+      res.json(statistics);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get lightning statistics');
+    }
+  }
+}
+
+export default new GeneralLightningRoutes();

backend/src/api/explorer/nodes.api.ts

@@ -0,0 +1,757 @@
+import logger from '../../logger';
+import DB from '../../database';
+import { ResultSetHeader } from 'mysql2';
+import { ILightningApi } from '../lightning/lightning-api.interface';
+import { ITopNodesPerCapacity, ITopNodesPerChannels } from '../../mempool.interfaces';
+import { bin2hex } from '../../utils/format';
+
+class NodesApi {
+  public async $getWorldNodes(): Promise<any> {
+    try {
+      let query = `
+        SELECT nodes.public_key as publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
+        CAST(COALESCE(nodes.capacity, 0) as INT) as capacity,
+        CAST(COALESCE(nodes.channels, 0) as INT) as channels,
+        nodes.longitude, nodes.latitude,
+        geo_names_country.names as country, geo_names_iso.names as isoCode
+        FROM nodes
+        JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
+        JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+        WHERE status = 1 AND nodes.as_number IS NOT NULL
+        ORDER BY capacity
+      `;
+
+      const [nodes]: any[] = await DB.query(query);
+
+      for (let i = 0; i < nodes.length; ++i) {
+        nodes[i].country = JSON.parse(nodes[i].country);
+      }
+
+      query = `
+        SELECT MAX(nodes.capacity) as maxLiquidity, MAX(nodes.channels) as maxChannels
+        FROM nodes
+        WHERE status = 1 AND nodes.as_number IS NOT NULL
+      `;
+
+      const [maximums]: any[] = await DB.query(query);
+      
+      return {
+        maxLiquidity: maximums[0].maxLiquidity,
+        maxChannels: maximums[0].maxChannels,
+        nodes: nodes.map(node => [
+          node.longitude, node.latitude,
+          node.publicKey, node.alias, node.capacity, node.channels,
+          node.country, node.isoCode
+        ])
+      };
+    } catch (e) {
+      logger.err(`Can't get world nodes list. Reason: ${e instanceof Error ? e.message : e}`);
+    }
+  }
+
+  public async $getNode(public_key: string): Promise<any> {
+    try {
+      // General info
+      let query = `
+        SELECT public_key, alias, UNIX_TIMESTAMP(first_seen) AS first_seen,
+          UNIX_TIMESTAMP(updated_at) AS updated_at, color, sockets as sockets,
+          as_number, city_id, country_id, subdivision_id, longitude, latitude,
+          geo_names_iso.names as iso_code, geo_names_as.names as as_organization, geo_names_city.names as city,
+          geo_names_country.names as country, geo_names_subdivision.names as subdivision,
+          features
+        FROM nodes
+        LEFT JOIN geo_names geo_names_as on geo_names_as.id = as_number
+        LEFT JOIN geo_names geo_names_city on geo_names_city.id = city_id
+        LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = subdivision_id
+        LEFT JOIN geo_names geo_names_country on geo_names_country.id = country_id
+        LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+        WHERE public_key = ?
+      `;
+      let [rows]: any[] = await DB.query(query, [public_key]);
+      if (rows.length === 0) {
+        throw new Error(`This node does not exist, or our node is not seeing it yet`);
+      }
+
+      const node = rows[0];
+      node.as_organization = JSON.parse(node.as_organization);
+      node.subdivision = JSON.parse(node.subdivision);
+      node.city = JSON.parse(node.city);
+      node.country = JSON.parse(node.country);
+
+      // Features      
+      node.features = JSON.parse(node.features);
+      node.featuresBits = null;
+      if (node.features) {
+        let maxBit = 0;
+        for (const feature of node.features) {
+          maxBit = Math.max(maxBit, feature.bit);
+        }
+        maxBit = Math.ceil(maxBit / 4) * 4 - 1;
+        
+        node.featuresBits = new Array(maxBit + 1).fill(0);
+        for (const feature of node.features) {
+          node.featuresBits[feature.bit] = 1;
+        }
+        node.featuresBits = bin2hex(node.featuresBits.reverse().join(''));
+      }
+
+      // Active channels and capacity
+      const activeChannelsStats: any = await this.$getActiveChannelsStats(public_key);
+      node.active_channel_count = activeChannelsStats.active_channel_count ?? 0;
+      node.capacity = activeChannelsStats.capacity ?? 0;
+
+      // Opened channels count
+      query = `
+        SELECT count(short_id) as opened_channel_count
+        FROM channels
+        WHERE status != 2 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)
+      `;
+      [rows] = await DB.query(query, [public_key, public_key]);
+      node.opened_channel_count = 0;
+      if (rows.length > 0) {
+        node.opened_channel_count = rows[0].opened_channel_count;
+      }
+
+      // Closed channels count
+      query = `
+        SELECT count(short_id) as closed_channel_count
+        FROM channels
+        WHERE status = 2 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)
+      `;
+      [rows] = await DB.query(query, [public_key, public_key]);
+      node.closed_channel_count = 0;
+      if (rows.length > 0) {
+        node.closed_channel_count = rows[0].closed_channel_count;
+      }
+
+      // Custom records
+      query = `
+        SELECT type, payload
+        FROM nodes_records
+        WHERE public_key = ?
+      `;
+      [rows] = await DB.query(query, [public_key]);
+      node.custom_records = {};
+      for (const record of rows) {
+        node.custom_records[record.type] = Buffer.from(record.payload, 'binary').toString('hex');
+      }
+
+      return node;
+    } catch (e) {
+      logger.err(`Cannot get node information for ${public_key}. Reason: ${(e instanceof Error ? e.message : e)}`);
+      throw e;
+    }
+  }
+
+  public async $getActiveChannelsStats(node_public_key: string): Promise<unknown> {
+    const query = `
+      SELECT count(short_id) as active_channel_count, sum(capacity) as capacity
+      FROM channels
+      WHERE status = 1 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)
+    `;
+    const [rows]: any[] = await DB.query(query, [node_public_key, node_public_key]);
+    if (rows.length > 0) {
+      return {
+        active_channel_count: rows[0].active_channel_count,
+        capacity: rows[0].capacity
+      };
+    } else {
+      return null;
+    }
+  }
+
+  public async $getFeeHistogram(node_public_key: string): Promise<unknown> {
+    try {
+      const inQuery = `
+        SELECT CASE WHEN fee_rate <= 10.0 THEN CEIL(fee_rate)
+                    WHEN (fee_rate > 10.0 and fee_rate <= 100.0) THEN CEIL(fee_rate / 10.0) * 10.0
+                    WHEN (fee_rate > 100.0 and fee_rate <= 1000.0) THEN CEIL(fee_rate / 100.0) * 100.0
+                    WHEN fee_rate > 1000.0 THEN CEIL(fee_rate / 1000.0) * 1000.0
+               END as bucket,
+               count(short_id) as count,
+               sum(capacity) as capacity
+        FROM (
+          SELECT CASE WHEN node1_public_key = ? THEN node2_fee_rate WHEN node2_public_key = ? THEN node1_fee_rate END as fee_rate,
+                 short_id as short_id,
+                 capacity as capacity
+          FROM channels
+          WHERE status = 1 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)
+        ) as fee_rate_table
+        GROUP BY bucket;
+      `;
+      const [inRows]: any[] = await DB.query(inQuery, [node_public_key, node_public_key, node_public_key, node_public_key]);
+
+      const outQuery = `
+        SELECT CASE WHEN fee_rate <= 10.0 THEN CEIL(fee_rate)
+                    WHEN (fee_rate > 10.0 and fee_rate <= 100.0) THEN CEIL(fee_rate / 10.0) * 10.0
+                    WHEN (fee_rate > 100.0 and fee_rate <= 1000.0) THEN CEIL(fee_rate / 100.0) * 100.0
+                    WHEN fee_rate > 1000.0 THEN CEIL(fee_rate / 1000.0) * 1000.0
+               END as bucket,
+               count(short_id) as count,
+               sum(capacity) as capacity
+        FROM (
+          SELECT CASE WHEN node1_public_key = ? THEN node1_fee_rate WHEN node2_public_key = ? THEN node2_fee_rate END as fee_rate,
+                 short_id as short_id,
+                 capacity as capacity
+          FROM channels
+          WHERE status = 1 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)
+        ) as fee_rate_table
+        GROUP BY bucket;
+      `;
+      const [outRows]: any[] = await DB.query(outQuery, [node_public_key, node_public_key, node_public_key, node_public_key]);
+
+      return {
+        incoming: inRows.length > 0 ? inRows : [],
+        outgoing: outRows.length > 0 ? outRows : [],
+      };
+    } catch (e) {
+      logger.err(`Cannot get node fee distribution for ${node_public_key}. Reason: ${(e instanceof Error ? e.message : e)}`);
+      throw e;
+    }
+  }
+
+  public async $getAllNodes(): Promise<any> {
+    try {
+      const query = `SELECT * FROM nodes`;
+      const [rows]: any = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('$getAllNodes error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getNodeStats(public_key: string): Promise<any> {
+    try {
+      const query = `
+        SELECT UNIX_TIMESTAMP(added) AS added, capacity, channels
+        FROM node_stats
+        WHERE public_key = ?
+        ORDER BY added DESC
+      `;
+      const [rows]: any = await DB.query(query, [public_key]);
+      return rows;
+    } catch (e) {
+      logger.err('$getNodeStats error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getTopCapacityNodes(full: boolean): Promise<ITopNodesPerCapacity[]> {
+    try {
+      let rows: any;
+      let query: string;
+      if (full === false) {
+        query = `
+          SELECT nodes.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
+            nodes.capacity
+          FROM nodes
+          ORDER BY capacity DESC
+          LIMIT 6
+        `;
+
+        [rows] = await DB.query(query);
+      } else {
+        query = `
+          SELECT nodes.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
+            CAST(COALESCE(nodes.capacity, 0) as INT) as capacity,
+            CAST(COALESCE(nodes.channels, 0) as INT) as channels,
+            UNIX_TIMESTAMP(nodes.first_seen) as firstSeen, UNIX_TIMESTAMP(nodes.updated_at) as updatedAt,
+            geo_names_city.names as city, geo_names_country.names as country,
+            geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision
+          FROM nodes
+          LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
+          LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
+          LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+          LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
+          ORDER BY capacity DESC
+          LIMIT 100
+        `;
+
+        [rows] = await DB.query(query);
+        for (let i = 0; i < rows.length; ++i) {
+          rows[i].country = JSON.parse(rows[i].country);
+          rows[i].city = JSON.parse(rows[i].city);
+        }
+      }
+
+      return rows;
+    } catch (e) {
+      logger.err('$getTopCapacityNodes error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getTopChannelsNodes(full: boolean): Promise<ITopNodesPerChannels[]> {
+    try {
+      let rows: any;
+      let query: string;
+      if (full === false) {
+        query = `
+          SELECT
+            nodes.public_key as publicKey,
+            IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
+            nodes.channels,
+            geo_names_city.names as city, geo_names_country.names as country,
+            geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision
+          FROM nodes
+          LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
+          LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
+          LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+          LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
+          ORDER BY channels DESC
+          LIMIT 6;
+        `;
+
+        [rows] = await DB.query(query);
+        for (let i = 0; i < rows.length; ++i) {
+          rows[i].country = JSON.parse(rows[i].country);
+          rows[i].city = JSON.parse(rows[i].city);
+        }
+      } else {
+        query = `
+          SELECT nodes.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
+            CAST(COALESCE(nodes.channels, 0) as INT) as channels,
+            CAST(COALESCE(nodes.capacity, 0) as INT) as capacity,
+            UNIX_TIMESTAMP(nodes.first_seen) as firstSeen, UNIX_TIMESTAMP(nodes.updated_at) as updatedAt,
+            geo_names_city.names as city, geo_names_country.names as country,
+            geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision
+          FROM nodes
+          LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
+          LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
+          LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+          LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
+          ORDER BY channels DESC
+          LIMIT 100
+        `;
+
+        [rows] = await DB.query(query);
+        for (let i = 0; i < rows.length; ++i) {
+          rows[i].country = JSON.parse(rows[i].country);
+          rows[i].city = JSON.parse(rows[i].city);
+        }
+      }
+
+      return rows;
+    } catch (e) {
+      logger.err('$getTopChannelsNodes error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getOldestNodes(full: boolean): Promise<ITopNodesPerChannels[]> {
+    try {
+      let [rows]: any[] = await DB.query('SELECT UNIX_TIMESTAMP(MAX(added)) as maxAdded FROM node_stats');
+      const latestDate = rows[0].maxAdded;
+
+      let query: string;
+      if (full === false) {
+        query = `
+          SELECT nodes.public_key, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias,
+            node_stats.channels
+          FROM node_stats
+          JOIN nodes ON nodes.public_key = node_stats.public_key
+          WHERE added = FROM_UNIXTIME(${latestDate})
+          ORDER BY first_seen
+          LIMIT 100;
+        `;
+
+        [rows] = await DB.query(query);
+      } else {
+        query = `
+          SELECT node_stats.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(node_stats.public_key, 1, 20), alias) as alias,
+            CAST(COALESCE(node_stats.channels, 0) as INT) as channels,
+            CAST(COALESCE(node_stats.capacity, 0) as INT) as capacity,
+            UNIX_TIMESTAMP(nodes.first_seen) as firstSeen, UNIX_TIMESTAMP(nodes.updated_at) as updatedAt,
+            geo_names_city.names as city, geo_names_country.names as country,
+            geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision
+          FROM node_stats
+          RIGHT JOIN nodes ON nodes.public_key = node_stats.public_key
+          LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
+          LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
+          LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+          LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
+          WHERE added = FROM_UNIXTIME(${latestDate})
+          ORDER BY first_seen
+          LIMIT 100
+        `;
+
+        [rows] = await DB.query(query);
+        for (let i = 0; i < rows.length; ++i) {
+          rows[i].country = JSON.parse(rows[i].country);
+          rows[i].city = JSON.parse(rows[i].city);
+        }
+      }
+
+      return rows;
+    } catch (e) {
+      logger.err('$getTopChannelsNodes error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $searchNodeByPublicKeyOrAlias(search: string) {
+    try {
+      const publicKeySearch = search.replace(/[^a-zA-Z0-9]/g, '') + '%';
+      const aliasSearch = search
+        .replace(/[-_.]/g, ' ') // Replace all -_. characters with empty space. Eg: "ln.nicehash" becomes "ln nicehash".  
+        .replace(/[^a-zA-Z0-9 ]/g, '') // Remove all special characters and keep just A to Z, 0 to 9.
+        .split(' ')
+        .filter(key => key.length)
+        .map((search) => '+' + search + '*').join(' ');
+      // %keyword% is wildcard search and can't be indexed so it's slower as the node database grow. keyword% can be indexed but then you can't search for "Nicehash" and get result for ln.nicehash.com. So we use fulltext index for words "ln, nicehash, com" and nicehash* will find it instantly.
+      const query = `SELECT public_key, alias, capacity, channels, status FROM nodes WHERE public_key LIKE ? OR MATCH alias_search AGAINST (? IN BOOLEAN MODE) ORDER BY capacity DESC LIMIT 10`;
+      const [rows]: any = await DB.query(query, [publicKeySearch, aliasSearch]);
+      return rows;
+    } catch (e) {
+      logger.err('$searchNodeByPublicKeyOrAlias error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getNodesISPRanking() {
+    try {
+      let query = '';
+
+      // List all channels and the two linked ISP
+      query = `
+        SELECT short_id, channels.capacity,
+          channels.node1_public_key AS node1PublicKey, isp1.names AS isp1, isp1.id as isp1ID,
+          channels.node2_public_key AS node2PublicKey, isp2.names AS isp2, isp2.id as isp2ID
+        FROM channels
+        JOIN nodes node1 ON node1.public_key = channels.node1_public_key
+        JOIN nodes node2 ON node2.public_key = channels.node2_public_key
+        JOIN geo_names isp1 ON isp1.id = node1.as_number
+        JOIN geo_names isp2 ON isp2.id = node2.as_number
+        WHERE channels.status = 1
+        ORDER BY short_id DESC
+      `;
+      const [channelsIsp]: any = await DB.query(query);
+
+      // Sum channels capacity and node count per ISP
+      const ispList = {};
+      for (const channel of channelsIsp) {
+        const isp1 = JSON.parse(channel.isp1);
+        const isp2 = JSON.parse(channel.isp2);
+
+        if (!ispList[isp1]) {
+          ispList[isp1] = {
+            ids: [channel.isp1ID],
+            capacity: 0,
+            channels: 0,
+            nodes: {},
+          };
+        } else if (ispList[isp1].ids.includes(channel.isp1ID) === false) {
+          ispList[isp1].ids.push(channel.isp1ID);
+        }
+
+        if (!ispList[isp2]) {
+          ispList[isp2] = {
+            ids: [channel.isp2ID],
+            capacity: 0,
+            channels: 0,
+            nodes: {},
+          };
+        } else if (ispList[isp2].ids.includes(channel.isp2ID) === false) {
+          ispList[isp2].ids.push(channel.isp2ID);
+        }
+        
+        ispList[isp1].capacity += channel.capacity;
+        ispList[isp1].channels += 1;
+        ispList[isp1].nodes[channel.node1PublicKey] = true;
+        ispList[isp2].capacity += channel.capacity;
+        ispList[isp2].channels += 1;
+        ispList[isp2].nodes[channel.node2PublicKey] = true;
+      }
+      
+      const ispRanking: any[] = [];
+      for (const isp of Object.keys(ispList)) {
+        ispRanking.push([
+          ispList[isp].ids.sort((a, b) => a - b).join(','),
+          isp,
+          ispList[isp].capacity,
+          ispList[isp].channels,
+          Object.keys(ispList[isp].nodes).length,
+        ]);
+      }
+
+      // Total active channels capacity
+      query = `SELECT SUM(capacity) AS capacity FROM channels WHERE status = 1`;
+      const [totalCapacity]: any = await DB.query(query);
+
+      // Get the total capacity of all channels which have at least one node on clearnet
+      query = `
+        SELECT SUM(capacity) as capacity
+        FROM (
+          SELECT capacity, GROUP_CONCAT(socket1.type, socket2.type) as networks
+          FROM channels
+          JOIN nodes_sockets socket1 ON node1_public_key = socket1.public_key
+          JOIN nodes_sockets socket2 ON node2_public_key = socket2.public_key
+          AND channels.status = 1
+          GROUP BY short_id
+        ) channels_tmp
+        WHERE channels_tmp.networks LIKE '%ipv%'
+      `;
+      const [clearnetCapacity]: any = await DB.query(query);
+
+      // Get the total capacity of all channels which have both nodes on Tor 
+      query = `
+        SELECT SUM(capacity) as capacity
+        FROM (
+          SELECT capacity, GROUP_CONCAT(socket1.type, socket2.type) as networks
+          FROM channels
+          JOIN nodes_sockets socket1 ON node1_public_key = socket1.public_key
+          JOIN nodes_sockets socket2 ON node2_public_key = socket2.public_key
+          AND channels.status = 1
+          GROUP BY short_id
+        ) channels_tmp
+        WHERE channels_tmp.networks NOT LIKE '%ipv%' AND
+          channels_tmp.networks NOT LIKE '%dns%' AND
+          channels_tmp.networks NOT LIKE '%websocket%'
+      `;
+      const [torCapacity]: any = await DB.query(query);
+
+      const clearnetCapacityValue = parseInt(clearnetCapacity[0].capacity, 10);
+      const torCapacityValue = parseInt(torCapacity[0].capacity, 10);
+      const unknownCapacityValue = parseInt(totalCapacity[0].capacity) - clearnetCapacityValue - torCapacityValue;
+
+      return {
+        clearnetCapacity: clearnetCapacityValue,
+        torCapacity: torCapacityValue,
+        unknownCapacity: unknownCapacityValue,
+        ispRanking: ispRanking,
+      };
+    } catch (e) {
+      logger.err(`Cannot get LN ISP ranking. Reason: ${e instanceof Error ? e.message : e}`);
+      throw e;
+    }
+  }
+
+  public async $getNodesPerCountry(countryId: string) {
+    try {
+      const query = `
+        SELECT nodes.public_key, CAST(COALESCE(nodes.capacity, 0) as INT) as capacity, CAST(COALESCE(nodes.channels, 0) as INT) as channels,
+          nodes.alias, UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at,
+          geo_names_city.names as city, geo_names_country.names as country,
+          geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision,
+          nodes.longitude, nodes.latitude, nodes.as_number, geo_names_isp.names as isp
+        FROM nodes
+        LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
+        LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
+        LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+        LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
+        LEFT JOIN geo_names geo_names_isp on geo_names_isp.id = nodes.as_number AND geo_names_isp.type = 'as_organization'
+        WHERE geo_names_country.id = ?
+        ORDER BY capacity DESC
+      `;
+
+      const [rows]: any = await DB.query(query, [countryId]);
+      for (let i = 0; i < rows.length; ++i) {
+        rows[i].country = JSON.parse(rows[i].country);
+        rows[i].city = JSON.parse(rows[i].city);
+        rows[i].subdivision = JSON.parse(rows[i].subdivision);
+        rows[i].isp = JSON.parse(rows[i].isp);
+      }
+      return rows;
+    } catch (e) {
+      logger.err(`Cannot get nodes for country id ${countryId}. Reason: ${e instanceof Error ? e.message : e}`);
+      throw e;
+    }
+  }
+
+  public async $getNodesPerISP(ISPId: string) {
+    try {
+      let query = `
+        SELECT channels.node1_public_key AS node1PublicKey, isp1.id as isp1ID,
+          channels.node2_public_key AS node2PublicKey, isp2.id as isp2ID
+        FROM channels
+        JOIN nodes node1 ON node1.public_key = channels.node1_public_key
+        JOIN nodes node2 ON node2.public_key = channels.node2_public_key
+        JOIN geo_names isp1 ON isp1.id = node1.as_number
+        JOIN geo_names isp2 ON isp2.id = node2.as_number
+        WHERE channels.status = 1 AND (node1.as_number IN (?) OR node2.as_number IN (?))
+        ORDER BY short_id DESC
+      `;
+
+      const IPSIds = ISPId.split(',');
+      const [rows]: any = await DB.query(query, [IPSIds, IPSIds]);
+      if (!rows || rows.length === 0) {
+        return [];
+      }
+
+      const nodes = {};
+
+      const intISPIds: number[] = [];
+      for (const ispId of IPSIds) {
+        intISPIds.push(parseInt(ispId, 10));
+      }
+
+      for (const channel of rows) {
+        if (intISPIds.includes(channel.isp1ID)) {
+          nodes[channel.node1PublicKey] = true;
+        }
+        if (intISPIds.includes(channel.isp2ID)) {
+          nodes[channel.node2PublicKey] = true;
+        }
+      }
+
+      query = `
+        SELECT nodes.public_key, CAST(COALESCE(nodes.capacity, 0) as INT) as capacity, CAST(COALESCE(nodes.channels, 0) as INT) as channels,
+          nodes.alias, UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at,
+          geo_names_city.names as city, geo_names_country.names as country,
+          geo_names_iso.names as iso_code, geo_names_subdivision.names as subdivision,
+          nodes.longitude, nodes.latitude
+        FROM nodes
+        LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
+        LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
+        LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+        LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
+        WHERE nodes.public_key IN (?)
+        ORDER BY capacity DESC
+      `;
+
+      const [rows2]: any = await DB.query(query, [Object.keys(nodes)]);
+      for (let i = 0; i < rows2.length; ++i) {
+        rows2[i].country = JSON.parse(rows2[i].country);
+        rows2[i].city = JSON.parse(rows2[i].city);
+        rows2[i].subdivision = JSON.parse(rows2[i].subdivision);
+      }
+      return rows2;
+
+    } catch (e) {
+      logger.err(`Cannot get nodes for ISP id ${ISPId}. Reason: ${e instanceof Error ? e.message : e}`);
+      throw e;
+    }
+  }
+
+  public async $getNodesCountries() {
+    try {
+      let query = `SELECT geo_names.names as names, geo_names_iso.names as iso_code, COUNT(DISTINCT nodes.public_key) as nodesCount, SUM(capacity) as capacity
+        FROM nodes
+        JOIN geo_names ON geo_names.id = nodes.country_id AND geo_names.type = 'country'
+        JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
+        GROUP BY country_id
+        ORDER BY COUNT(DISTINCT nodes.public_key) DESC
+      `;
+      const [nodesCountPerCountry]: any = await DB.query(query);
+
+      query = `SELECT COUNT(*) as total FROM nodes WHERE country_id IS NOT NULL`;
+      const [nodesWithAS]: any = await DB.query(query);
+
+      const nodesPerCountry: any[] = [];
+      for (const country of nodesCountPerCountry) {
+        nodesPerCountry.push({
+          name: JSON.parse(country.names),
+          iso: country.iso_code, 
+          count: country.nodesCount,
+          share: Math.floor(country.nodesCount / nodesWithAS[0].total * 10000) / 100,
+          capacity: country.capacity,
+        })
+      }
+
+      return nodesPerCountry;
+    } catch (e) {
+      logger.err(`Cannot get nodes grouped by AS. Reason: ${e instanceof Error ? e.message : e}`);
+      throw e;
+    }
+  }
+
+  /**
+   * Save or update a node present in the graph
+   */
+  public async $saveNode(node: ILightningApi.Node): Promise<void> {
+    try {
+      // https://github.com/mempool/mempool/issues/3006
+      if ((node.last_update ?? 0) < 1514736061) { // January 1st 2018
+        node.last_update = null;
+      }
+  
+      const uniqueAddr = [...new Set(node.addresses?.map(a => a.addr))];
+      const formattedSockets = (uniqueAddr.join(',')) ?? '';
+
+      const query = `INSERT INTO nodes(
+          public_key,
+          first_seen,
+          updated_at,
+          alias,
+          alias_search,
+          color,
+          sockets,
+          status,
+          features
+        )
+        VALUES (?, NOW(), FROM_UNIXTIME(?), ?, ?, ?, ?, 1, ?)
+        ON DUPLICATE KEY UPDATE
+          updated_at = FROM_UNIXTIME(?),
+          alias = ?,
+          alias_search = ?,
+          color = ?,
+          sockets = ?,
+          status = 1,
+          features = ?
+      `;
+
+      await DB.query(query, [
+        node.pub_key,
+        node.last_update,
+        node.alias,
+        this.aliasToSearchText(node.alias),
+        node.color,
+        formattedSockets,
+        JSON.stringify(node.features),
+        node.last_update,
+        node.alias,
+        this.aliasToSearchText(node.alias),
+        node.color,
+        formattedSockets,
+        JSON.stringify(node.features),
+      ]);
+    } catch (e) {
+      logger.err('$saveNode() error: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  /**
+   * Update node sockets
+   */
+  public async $updateNodeSockets(publicKey: string, sockets: {network: string; addr: string}[]): Promise<void> {
+    const uniqueAddr = [...new Set(sockets.map(a => a.addr))];
+
+    const formattedSockets = (uniqueAddr.join(',')) ?? '';
+    try {
+      await DB.query(`UPDATE nodes SET sockets = ? WHERE public_key = ?`, [formattedSockets, publicKey]);
+    } catch (e) {
+      logger.err(`Cannot update node sockets for ${publicKey}. Reason: ${e instanceof Error ? e.message : e}`);
+    }
+  }
+
+  /**
+   * Set all nodes not in `nodesPubkeys` as inactive (status = 0)
+   */
+   public async $setNodesInactive(graphNodesPubkeys: string[]): Promise<void> {
+    if (graphNodesPubkeys.length === 0) {
+      return;
+    }
+
+    try {
+      const result = await DB.query<ResultSetHeader>(`
+        UPDATE nodes
+        SET status = 0
+        WHERE public_key NOT IN (
+          ${graphNodesPubkeys.map(pubkey => `"${pubkey}"`).join(',')}
+        )
+      `);
+      if (result[0].changedRows ?? 0 > 0) {
+        logger.debug(`Marked ${result[0].changedRows} nodes as inactive because they are not in the graph`, logger.tags.ln);
+      }
+    } catch (e) {
+      logger.err('$setNodesInactive() error: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  private aliasToSearchText(str: string): string {
+    return str.replace(/[-_.]/g, ' ').replace(/[^a-zA-Z0-9 ]/g, '');
+  }
+}
+
+export default new NodesApi();

backend/src/api/explorer/nodes.routes.ts

@@ -0,0 +1,383 @@
+import config from '../../config';
+import { Application, Request, Response } from 'express';
+import nodesApi from './nodes.api';
+import DB from '../../database';
+import { INodesRanking } from '../../mempool.interfaces';
+import { handleError } from '../../utils/api';
+
+class NodesRoutes {
+  constructor() { }
+
+  public initRoutes(app: Application) {
+    app
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/world', this.$getWorldNodes)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/country/:country', this.$getNodesPerCountry)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/search/:search', this.$searchNode)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/isp-ranking', this.$getISPRanking)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/isp/:isp', this.$getNodesPerISP)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/countries', this.$getNodesCountries)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings', this.$getNodesRanking)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings/liquidity', this.$getTopNodesByCapacity)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings/connectivity', this.$getTopNodesByChannels)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings/age', this.$getOldestNodes)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/:public_key/statistics', this.$getHistoricalNodeStats)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/:public_key/fees/histogram', this.$getFeeHistogram)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/:public_key', this.$getNode)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/group/:name', this.$getNodeGroup)
+    ;
+  }
+
+  private async $searchNode(req: Request, res: Response) {
+    try {
+      const nodes = await nodesApi.$searchNodeByPublicKeyOrAlias(req.params.search);
+      res.json(nodes);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to search for node');
+    }
+  }
+
+  private async $getNodeGroup(req: Request, res: Response) {
+    try {
+      let nodesList;
+      let nodes: any[] = [];
+      switch (config.MEMPOOL.NETWORK) {
+        case 'testnet':
+          nodesList = [
+            '0259db43b4e4ac0ff12a805f2d81e521253ba2317f6739bc611d8e2fa156d64256',
+            '0352b9944b9a52bd2116c91f1ba70c4ef851ac5ba27e1b20f1d92da3ade010dd10',
+            '03424f5a7601eaa47482cb17100b31a84a04d14fb44b83a57eeceffd8e299878e3',
+            '032850492ee61a5f7006a2fda6925e4b4ec3782f2b6de2ff0e439ef5a38c3b2470',
+            '022c80bace98831c44c32fb69755f2b353434e0ee9e7fbda29507f7ef8abea1421',
+            '02c3559c833e6f99f9ca05fe503e0b4e7524dea9121344edfd3e811101e0c28680',
+            '02b36a324fa2dd3af2a63ac65f241907882829bed5002b4e14171d25c219e0d470',
+            '0231b6e8f21f9f6c057f6bf8a812f79e396ee16a66ece91939a1576ce9fb9e87a5',
+            '034b6aac206bffcbd651b7ead1ab8a0991c945dfafe19ff27dcdeadc6843ebd15c',
+            '039c065f7e344acd969ebdd4a94550915b6f24e8782ae2be540bb96c8a4fcfb86b',
+            '03d9f9f4803fc75920f14dd13d83fbecc53229a65d4ee4cd2d86fdf211f7337576',
+            '0357fe48c4dece744f70865eda66e396aab5d05e09e1145cd3b7da83f11446d4cf',
+            '02bca4d642eda631f2c8659758e2a2868e518b93503f2bfcd767749c6530a10679',
+            '03f32c99c0bb9f62dae53671d1d300565773455248f34134cc02779b881561174e',
+            '032c7c7819276c4f706a04df1a0f1e10a5495994a7be4c1d3d28ca766e5a2b957b',
+            '025a7e38c2834dd843591a4d23d5f09cdeb77ddca85f673c2d944a14220ff14cf7',
+            '0395e2731a1673ef21d7a16a727c4fc4d4c35a861c428ce2c819c53d2b81c8bd55',
+            '032ab2028c0b614c6d87824e2373529652fd7e4221b4c70cc4da7c7005c49afcf0',
+            '029001b22fe70b48bee12d014df91982eb85ff1bd404ec772d5c83c4ee3e88d2c3',
+            '0212e2848d79f928411da5f2ff0a8c95ec6ccb5a09d2031b6f71e91309dcde63af',
+            '03e871a2229523d34f76e6311ff197cfe7f26c2fbec13554b93a46f4e710c47dab',
+            '032202ec98d976b0e928bd1d91924e8bd3eab07231fc39feb3737b010071073df8',
+            '02fa7c5a948d03d563a9f36940c2205a814e594d17c0042ced242c71a857d72605',
+            '039c14fdec2d958e3d14cebf657451bbd9e039196615785e82c917f274e3fb2205',
+            '033589bbcb233ffc416cefd5437c7f37e9d7cb7942d405e39e72c4c846d9b37f18',
+            '029293110441c6e2eacb57e1255bf6ef05c41a6a676fe474922d33c19f98a7d584',
+            '038eb09bed4532ff36d12acc1279f55cbe8d95212d19f809e057bb50de00051fba',
+            '027b7c0278366a0268e8bd0072b14539f6cb455a7bd588ae22d888bed541f65311',
+            '02f4dd78f6eda8838029b2cdbaaea6e875e2fa373cd348ee41a7c1bb177d3fca66',
+            '036b3fb692da214a3edaac5b67903b958f5ccd8712e09aa61b67ea7acfd94b40c2',
+            '023bc8915d308e0b65f8de6867f95960141372436fce3edad5cec3f364d6ac948f',
+            '0341690503ef21d0e203dddd9e62646380d0dfc32c499e055e7f698b9064d1c736',
+            '0355d573805c018a37a5b2288378d70e9b5b438f7394abd6f467cb9b47c90eeb93',
+            '0361aa68deb561a8b47b41165848edcccb98a1b56a5ea922d9d5b30a09bb7282ea',
+            '0235ad0b56ed8c42c4354444c24e971c05e769ec0b5fb0ccea42880095dc02ea2c',
+            '029700819a37afea630f80e6cc461f3fd3c4ace2598a21cfbbe64d1c78d0ee69a5',
+            '02c2d8b2dbf87c7894af2f1d321290e2fe6db5446cd35323987cee98f06e2e0075',
+            '030b0ca1ea7b1075716d2a555630e6fd47ef11bc7391fe68963ec06cf370a5e382',
+            '031adb9eb2d66693f85fa31a4adca0319ba68219f3ad5f9a2ef9b34a6b40755fa1',
+            '02ccd07faa47eda810ecf5591ccf5ca50f6c1034d0d175052898d32a00b9bae24f',
+          ];
+          break;
+        case 'signet':
+          nodesList = [
+            '029fe3621fc0c6e08056a14b868f8fb9acca1aa28a129512f6cea0f0d7654d9f92',
+            '02f60cd7a3a4f1c953dd9554a6ebd51a34f8b10b8124b7fc43a0b381139b55c883',
+            '03cbbf581774700865eebd1be42d022bc004ba30881274ab304e088a25d70e773d',
+            '0243348cb3741cfe2d8485fa8375c29c7bc7cbb67577c363cb6987a5e5fd0052cc',
+            '02cb73e631af44bee600d80f8488a9194c9dc5c7590e575c421a070d1be05bc8e9',
+            '0306f55ee631aa1e2cd4d9b2bfcbc14404faec5c541cef8b2e6f779061029d09c4',
+            '030bbbd8495561a894e301fe6ba5b22f8941fc661cc0e673e0206158231d8ac130',
+            '03ee1f08e516ed083475f39c6cae4fa1eec686d004d2f105218269e27d7f2da5a4',
+            '028c378b998f476ed22d6815c170dd2a3388a43fdf791a7cff70b9997349b8447a',
+            '036f19f044d19cb1b04f14d91b6e7e5443ce337217a8c14d43861f3e86dd07bd7f',
+            '03058d61869e8b88436493648b2e3e530627edf5a0b253c285cd565c1477a5c237',
+            '0279dfedc87b47a941f1797f2c422c03aa3108914ea6b519d76537d60860535a9a',
+            '0353486b8016761e58ec8aee7305ee58d5dc66b55ef5bd8cbaf49508f66d52d62e',
+            '03df5db8eccfabcae47ff15553cfdecb2d3f56979f43a0c3578f28d056b5e35104',
+            '03ddab321b760433cbf561b615ef62ac7d318630c5f51d523aaf5395b90b751956',
+            '033d92c7bfd213ef1b34c90e985fb5dc77f9ec2409d391492484e57a44c4aca1de',
+            '02ad010dda54253c1eb9efe38b0760657a3b43ecad62198c359c051c9d99d45781',
+            '025196512905b8a3f1597428b867bec63ec9a95e5089eb7dc7e63e2d2691669029',
+            '027c625aa1fbe3768db68ebcb05b53b6dc0ce68b7b54b8900d326d167363e684fe',
+            '03f1629af3101fcc56b7aac2667016be84e3defbf3d0c8719f836c9b41c9a57a43',
+            '02dfb81e2f7a3c4c9e8a51b70ef82b4a24549cc2fab1f5b2fd636501774a918991',
+            '02d01ccf832944c68f10d39006093769c5b8bda886d561b128534e313d729fdb34',
+            '02499ed23027d4698a6904ff4ec1b6085a61f10b9a6937f90438f9947e38e8ea86',
+            '038310e3a786340f2bd7770704c7ccfe560fd163d9a1c99d67894597419d12cbf7',
+            '03e5e9d879b72c7d67ecd483bae023bd33e695bb32b981a4021260f7b9d62bc761',
+            '028d16e1a0ace4c0c0a421536d8d32ce484dfe6e2f726b7b0e7c30f12a195f8cc7',
+            '0326cf9a4ca67a5b9cdffae57293dbd6f7c5113b93010dc6f6fe4af3afde1a1739',
+            '034867e16f62cebb8c2c2c22b91117c173bbece9c8a1e5bd001374a3699551cd8f',
+            '038dfb1f1b637a8c27e342ffc6f9feca20e0b47be3244e09ae78df4998e2ae83b9',
+            '03cb1cea3394d973355c11bc61c2f689f9d3e1c3db60d205f27770f5ad83200f77',
+            '03535447b592cbdb153189b3e06a455452b1011380cb3e6511a31090c15d8efc9f',
+            '028e90e9984d262ebfa3c23fb3f335a2ae061a0bdedee03f45f72b438d9e7d2ce3',
+            '03ee0176289dc4a6111fa5ef22eed5273758c420fbe58cc1d2d76def75dd7e640c',
+            '0370b2cd9f0eaf436d5c25c93fb39210d8cc06b31f688fc2f54418aabe394aed79',
+            '02ff690d06c187ab994bf83c5a2114fe5bf50112c2c817af0f788f736be9fa2070',
+            '02a9f570c51a2526a5ee85802e88f9281bed771eb66a0c8a7d898430dd5d0eae45',
+            '038c3de773255d3bd7a50e31e58d423baac5c90826a74d75e64b74c95475de1097',
+            '0242c7f7d315095f37ad1421ae0a2fc967d4cbe65b61b079c5395a769436959853',
+            '02a909e70eb03742f12666ebb1f56ac42a5fbaab0c0e8b5b1df4aa9f10f8a09240',
+            '03a26efa12489803c07f3ac2f1dba63812e38f0f6e866ce3ebb34df7de1f458cd2',
+          ];
+          break;
+        default:
+          nodesList = [
+            '02b12b889fe3c943cb05645921040ef13d6d397a2e7a4ad000e28500c505ff26d6',
+            '0302240ac9d71b39617cbde2764837ec3d6198bd6074b15b75d2ff33108e89d2e1',
+            '03364a8ace313376e5e4b68c954e287c6388e16df9e9fdbaf0363ecac41105cbf6',
+            '03229ab4b7f692753e094b93df90530150680f86b535b5183b0cffd75b3df583fc',
+            '03a696eb7acde991c1be97a58a9daef416659539ae462b897f5e9ae361f990228e',
+            '0248bf26cf3a63ab8870f34dc0ec9e6c8c6288cdba96ba3f026f34ec0f13ac4055',
+            '021b28ecdd782fd909705d6be354db268977b1a2ac5a5275186fc19e08bb8fca93',
+            '031bec1fbd8eb7fe94d2bda108c9c3cc8c22ecfc1c3a5c11d36f5881b01b4a81a6',
+            '03879c4f827a3188574d5757e002f574265a966d70aea942169785b31369b067d5',
+            '0228d4b5a4fd73a03967b76f8b8cb37b9d0b6e7039126a9397bb732c15bed78e9b',
+            '03f58dbb629f4427f5a1dbc02e6a7ec79345fdf13a0e4163d4f3b7aea2539cf095',
+            '021cdcb8123aa670cdfc9f43909dbb297363c093883409e9e7fc82e7267f7c72bd',
+            '02f2aa2c2b7b432a70dc4d0b04afa19d48715ed3b90594d49c1c8744f2e9ebb030',
+            '03709a02fb3ab4857689a8ea0bd489a6ab6f56f8a397be578bc6d5ad22efbe3756',
+            '03fbc17549ec667bccf397ababbcb4cdc0e3394345e4773079ab2774612ec9be61',
+            '03da9a8623241ccf95f19cd645c6cecd4019ac91570e976eb0a128bebbc4d8a437',
+            '03ca5340cf85cb2e7cf076e489f785410838de174e40be62723e8a60972ad75144',
+            '0238bd27f02d67d6c51e269692bc8c9a32357a00e7777cba7f4f1f18a2a700b108',
+            '03f983dcabed6baa1eab5b56c8b2e8fdc846ab3fd931155377897335e85a9fa57c',
+            '03e399589533581e48796e29a825839a010036a61b20744fda929d6709fcbffcc5',
+            '021f5288b5f72c42cd0d8801086af7ce09a816d8ee9a4c47a4b436399b26cb601a',
+            '032b01b7585f781420cd4148841a82831ba37fa952342052cec16750852d4f2dd9',
+            '02848036488d4b8fb1f1c4064261ec36151f43b085f0b51bd239ade3ddfc940c34',
+            '02b6b1640fe029e304c216951af9fbefdb23b0bdc9baaf327540d31b6107841fdf',
+            '03694289827203a5b3156d753071ddd5bf92e371f5a462943f9555eef6d2d6606c',
+            '0283d850db7c3e8ea7cc9c4abc7afaab12bbdf72b677dcba1d608350d2537d7d43',
+            '03b4dda7878d3b7b71ecd6d4738322c7f9a9c1fb583374d2724f4ccc4947f37570',
+            '0279a35f05b5acf159429549e56fd426685c4fec191431c58738968bbc77a39f25',
+            '03cb102d796ddcf08610cd03fae8b7a1df69ff48e9e8a152af315f9edf71762eb8',
+            '036b89526f4d5ac4c317f4fd23cb9f8e4ad844498bc7950a41114d060101d995d4',
+            '0313eade145959d7036db009fd5b0bf1947a739c7c3c790b491ec9161b94e6ad1e',
+            '02b670ca4c4bb2c5ea89c3b691da98a194cfc48fcd5c072df02a20290bddd60610',
+            '02a9196d5e08598211397a83cf013a5962b84bd61198abfdd204dff987e54f7a0d',
+            '036d015cd2f486fb38348182980b7e596e6c9733873102ea126fed7b4152be03b8',
+            '02521287789f851268a39c9eccc9d6180d2c614315b583c9e6ae0addbd6d79df06',
+            '0258c2a7b7f8af2585b4411b1ec945f70988f30412bb1df179de941f14d0b1bc3e',
+            '03c3389ff1a896f84d921ed01a19fc99c6724ce8dc4b960cd3b7b2362b62cd60d7',
+            '038d118996b3eaa15dcd317b32a539c9ecfdd7698f204acf8a087336af655a9192',
+            '02a928903d93d78877dacc3642b696128a3636e9566dd42d2d132325b2c8891c09',
+            '0328cd17f3a9d3d90b532ade0d1a67e05eb8a51835b3dce0a2e38eac04b5a62a57',
+          ];
+      }
+
+      for (let pubKey of nodesList) {
+        try {
+          const node = await nodesApi.$getNode(pubKey);
+          if (node) {
+            nodes.push(node);
+          }
+        } catch (e) {}
+      }
+
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(nodes);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get node group');
+    }
+  }
+
+  private async $getNode(req: Request, res: Response) {
+    try {
+      const node = await nodesApi.$getNode(req.params.public_key);
+      if (!node) {
+        handleError(req, res, 404, 'Node not found');
+        return;
+      }
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(node);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get node');
+    }
+  }
+
+  private async $getHistoricalNodeStats(req: Request, res: Response) {
+    try {
+      const statistics = await nodesApi.$getNodeStats(req.params.public_key);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(statistics);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get historical node stats');
+    }
+  }
+
+  private async $getFeeHistogram(req: Request, res: Response) {
+    try {
+      const node = await nodesApi.$getFeeHistogram(req.params.public_key);
+      if (!node) {
+        handleError(req, res, 404, 'Node not found');
+        return;
+      }
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(node);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get fee histogram');
+    }
+  }
+
+  private async $getNodesRanking(req: Request, res: Response): Promise<void> {
+    try {
+      const topCapacityNodes = await nodesApi.$getTopCapacityNodes(false);
+      const topChannelsNodes = await nodesApi.$getTopChannelsNodes(false);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(<INodesRanking>{
+        topByCapacity: topCapacityNodes,
+        topByChannels: topChannelsNodes,
+      });
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get nodes ranking');
+    }
+  }
+
+  private async $getTopNodesByCapacity(req: Request, res: Response): Promise<void> {
+    try {
+      const topCapacityNodes = await nodesApi.$getTopCapacityNodes(true);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(topCapacityNodes);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get top nodes by capacity');
+    }
+  }
+
+  private async $getTopNodesByChannels(req: Request, res: Response): Promise<void> {
+    try {
+      const topCapacityNodes = await nodesApi.$getTopChannelsNodes(true);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(topCapacityNodes);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get top nodes by channels');
+    }
+  }
+
+  private async $getOldestNodes(req: Request, res: Response): Promise<void> {
+    try {
+      const topCapacityNodes = await nodesApi.$getOldestNodes(true);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(topCapacityNodes);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get oldest nodes');
+    }
+  }
+
+  private async $getISPRanking(req: Request, res: Response): Promise<void> {
+    try {
+      const nodesPerAs = await nodesApi.$getNodesISPRanking();
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 600).toUTCString());
+      res.json(nodesPerAs);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get ISP ranking');
+    }
+  }
+
+  private async $getWorldNodes(req: Request, res: Response) {
+    try {
+      const worldNodes = await nodesApi.$getWorldNodes();
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 600).toUTCString());
+      res.json(worldNodes);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get world nodes');
+    }
+  }
+
+  private async $getNodesPerCountry(req: Request, res: Response) {
+    try {
+      const [country]: any[] = await DB.query(
+        `SELECT geo_names.id, geo_names_country.names as country_names
+        FROM geo_names
+        JOIN geo_names geo_names_country on geo_names.id = geo_names_country.id AND geo_names_country.type = 'country'
+        WHERE geo_names.type = 'country_iso_code' AND geo_names.names = ?`,
+        [req.params.country]
+      );
+
+      if (country.length === 0) {
+        handleError(req, res, 404, `This country does not exist or does not host any lightning nodes on clearnet`);
+        return;
+      }
+
+      const nodes = await nodesApi.$getNodesPerCountry(country[0].id);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json({
+        country: JSON.parse(country[0].country_names),
+        nodes: nodes,
+      });
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get nodes per country');
+    }
+  }
+
+  private async $getNodesPerISP(req: Request, res: Response) {
+    try {
+      const [isp]: any[] = await DB.query(
+        `SELECT geo_names.names as isp_name
+        FROM geo_names
+        WHERE geo_names.type = 'as_organization' AND geo_names.id = ?`,
+        [req.params.isp]
+      );
+
+      if (isp.length === 0) {
+        handleError(req, res, 404, `This ISP does not exist or does not host any lightning nodes on clearnet`);
+        return;
+      }
+
+      const nodes = await nodesApi.$getNodesPerISP(req.params.isp);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json({
+        isp: JSON.parse(isp[0].isp_name),
+        nodes: nodes,
+      });
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get nodes per ISP');
+    }
+  }
+
+  private async $getNodesCountries(req: Request, res: Response) {
+    try {
+      const nodesPerAs = await nodesApi.$getNodesCountries();
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 600).toUTCString());
+      res.json(nodesPerAs);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get nodes per country');
+    }
+  }
+}
+
+export default new NodesRoutes();

backend/src/api/explorer/statistics.api.ts

@@ -0,0 +1,53 @@
+import logger from '../../logger';
+import DB from '../../database';
+import { Common } from '../common';
+
+class StatisticsApi {
+  public async $getStatistics(interval: string | null = null): Promise<any> {
+    interval = Common.getSqlInterval(interval);
+
+    let query = `SELECT UNIX_TIMESTAMP(added) AS added, channel_count, total_capacity,
+      tor_nodes, clearnet_nodes, unannounced_nodes, clearnet_tor_nodes
+      FROM lightning_stats`;
+
+    if (interval) {
+      query += ` WHERE added BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
+    }
+
+    query += ` ORDER BY added DESC`;
+
+    try {
+      const [rows]: any = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('$getStatistics error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getLatestStatistics(): Promise<any> {
+    try {
+      const [rows]: any = await DB.query(`SELECT * FROM lightning_stats ORDER BY added DESC LIMIT 1`);
+      const [rows2]: any = await DB.query(`SELECT * FROM lightning_stats WHERE DATE(added) = DATE(NOW() - INTERVAL 7 DAY)`);
+      return {
+        latest: rows[0],
+        previous: rows2[0],
+      };
+    } catch (e) {
+      logger.err('$getLatestStatistics error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getStatisticsCount(): Promise<number> {
+    try {
+      const [rows]: any = await DB.query(`SELECT count(*) as count FROM lightning_stats`);
+      return rows[0].count;
+    } catch (e) {
+      logger.err('$getLatestStatistics error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+}
+
+export default new StatisticsApi();

backend/src/api/fee-api.ts

@@ -1,23 +1,36 @@
-import config from '../config';
 import { MempoolBlock } from '../mempool.interfaces';
+import config from '../config';
 import mempool from './mempool';
 import projectedBlocks from './mempool-blocks';
 
+const isLiquid = config.MEMPOOL.NETWORK === 'liquid' || config.MEMPOOL.NETWORK === 'liquidtestnet';
+
+interface RecommendedFees {
+  fastestFee: number,
+  halfHourFee: number,
+  hourFee: number,
+  economyFee: number,
+  minimumFee: number,
+}
+
 class FeeApi {
   constructor() { }
 
-  defaultFee = config.MEMPOOL.NETWORK === 'liquid' ? 0.1 : 1;
+  defaultFee = isLiquid ? 0.1 : 1;
+  minimumIncrement = isLiquid ? 0.1 : 1;
 
-  public getRecommendedFee() {
+  public getRecommendedFee(): RecommendedFees {
     const pBlocks = projectedBlocks.getMempoolBlocks();
     const mPool = mempool.getMempoolInfo();
-    const minimumFee = Math.ceil(mPool.mempoolminfee * 100000);
+    const minimumFee = this.roundUpToNearest(mPool.mempoolminfee * 100000, this.minimumIncrement);
+    const defaultMinFee = Math.max(minimumFee, this.defaultFee);
 
     if (!pBlocks.length) {
       return {
-        'fastestFee': this.defaultFee,
-        'halfHourFee': this.defaultFee,
-        'hourFee': this.defaultFee,
+        'fastestFee': defaultMinFee,
+        'halfHourFee': defaultMinFee,
+        'hourFee': defaultMinFee,
+        'economyFee': minimumFee,
         'minimumFee': minimumFee,
       };
     }
@@ -26,10 +39,25 @@ class FeeApi {
     const secondMedianFee = pBlocks[1] ? this.optimizeMedianFee(pBlocks[1], pBlocks[2], firstMedianFee) : this.defaultFee;
     const thirdMedianFee = pBlocks[2] ? this.optimizeMedianFee(pBlocks[2], pBlocks[3], secondMedianFee) : this.defaultFee;
 
+    let fastestFee = Math.max(minimumFee, firstMedianFee);
+    let halfHourFee = Math.max(minimumFee, secondMedianFee);
+    let hourFee = Math.max(minimumFee, thirdMedianFee);
+    const economyFee = Math.max(minimumFee, Math.min(2 * minimumFee, thirdMedianFee));
+
+    // ensure recommendations always increase w/ priority
+    fastestFee = Math.max(fastestFee, halfHourFee, hourFee, economyFee);
+    halfHourFee = Math.max(halfHourFee, hourFee, economyFee);
+    hourFee = Math.max(hourFee, economyFee);
+
+    // explicitly enforce a minimum of ceil(mempoolminfee) on all recommendations.
+    // simply rounding up recommended rates is insufficient, as the purging rate
+    // can exceed the median rate of projected blocks in some extreme scenarios
+    // (see https://bitcoin.stackexchange.com/a/120024)
     return {
-      'fastestFee': firstMedianFee,
-      'halfHourFee': secondMedianFee,
-      'hourFee': thirdMedianFee,
+      'fastestFee': fastestFee,
+      'halfHourFee': halfHourFee,
+      'hourFee': hourFee,
+      'economyFee': economyFee,
       'minimumFee': minimumFee,
     };
   }
@@ -43,7 +71,11 @@ class FeeApi {
       const multiplier = (pBlock.blockVSize - 500000) / 500000;
       return Math.max(Math.round(useFee * multiplier), this.defaultFee);
     }
-    return Math.ceil(useFee);
+    return this.roundUpToNearest(useFee, this.minimumIncrement);
+  }
+
+  private roundUpToNearest(value: number, nearest: number): number {
+    return Math.ceil(value / nearest) * nearest;
   }
 }
 

backend/src/api/fetch-version.ts

@@ -0,0 +1,37 @@
+import fs from 'fs';
+import path from "path";
+const { spawnSync } = require('child_process');
+
+function getVersion(): string {
+  const packageJson = fs.readFileSync('package.json').toString();
+  return JSON.parse(packageJson).version;
+}
+
+function getGitCommit(): string {
+  if (process.env.MEMPOOL_COMMIT_HASH) {
+    return process.env.MEMPOOL_COMMIT_HASH;
+  } else {
+    const gitRevParse = spawnSync('git', ['rev-parse', '--short', 'HEAD']);
+    if (!gitRevParse.error) {
+      const output = gitRevParse.stdout.toString('utf-8').replace(/[\n\r\s]+$/, '');
+      if (output) {
+        return output;
+      } else {
+        console.log('Could not fetch git commit: No repo available');
+      }
+    } else if (gitRevParse.error.code === 'ENOENT') {
+      console.log('Could not fetch git commit: Command `git` is unavailable');
+    }
+  }
+  return '?';
+}
+
+const versionInfo = {
+  version: getVersion(),
+  gitCommit: getGitCommit()
+}
+
+fs.writeFileSync(
+  path.join(__dirname, 'version.json'),
+  JSON.stringify(versionInfo, null, 2) + "\n"
+);

backend/src/api/fiat-conversion.ts

@@ -1,44 +0,0 @@
-import logger from '../logger';
-import axios from 'axios';
-import { IConversionRates } from '../mempool.interfaces';
-import config from '../config';
-
-class FiatConversion {
-  private conversionRates: IConversionRates = {
-    'USD': 0
-  };
-  private ratesChangedCallback: ((rates: IConversionRates) => void) | undefined;
-
-  constructor() { }
-
-  public setProgressChangedCallback(fn: (rates: IConversionRates) => void) {
-    this.ratesChangedCallback = fn;
-  }
-
-  public startService() {
-    logger.info('Starting currency rates service');
-    setInterval(this.updateCurrency.bind(this), 1000 * config.MEMPOOL.PRICE_FEED_UPDATE_INTERVAL);
-    this.updateCurrency();
-  }
-
-  public getConversionRates() {
-    return this.conversionRates;
-  }
-
-  private async updateCurrency(): Promise<void> {
-    try {
-      const response = await axios.get('https://price.bisq.wiz.biz/getAllMarketPrices', { timeout: 10000 });
-      const usd = response.data.data.find((item: any) => item.currencyCode === 'USD');
-      this.conversionRates = {
-        'USD': usd.price,
-      };
-      if (this.ratesChangedCallback) {
-        this.ratesChangedCallback(this.conversionRates);
-      }
-    } catch (e) {
-      logger.err('Error updating fiat conversion rates: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-}
-
-export default new FiatConversion();

backend/src/api/lightning/clightning/clightning-client.ts

@@ -0,0 +1,270 @@
+// Imported from https://github.com/shesek/lightning-client-js
+
+'use strict';
+
+const methods = [
+  'addgossip',
+  'autocleaninvoice',
+  'check',
+  'checkmessage',
+  'close',
+  'connect',
+  'createinvoice',
+  'createinvoicerequest',
+  'createoffer',
+  'createonion',
+  'decode',
+  'decodepay',
+  'delexpiredinvoice',
+  'delinvoice',
+  'delpay',
+  'dev-listaddrs',
+  'dev-rescan-outputs',
+  'disableoffer',
+  'disconnect',
+  'estimatefees',
+  'feerates',
+  'fetchinvoice',
+  'fundchannel',
+  'fundchannel_cancel',
+  'fundchannel_complete',
+  'fundchannel_start',
+  'fundpsbt',
+  'getchaininfo',
+  'getinfo',
+  'getlog',
+  'getrawblockbyheight',
+  'getroute',
+  'getsharedsecret',
+  'getutxout',
+  'help',
+  'invoice',
+  'keysend',
+  'legacypay',
+  'listchannels',
+  'listconfigs',
+  'listforwards',
+  'listfunds',
+  'listinvoices',
+  'listnodes',
+  'listoffers',
+  'listpays',
+  'listpeers',
+  'listsendpays',
+  'listtransactions',
+  'multifundchannel',
+  'multiwithdraw',
+  'newaddr',
+  'notifications',
+  'offer',
+  'offerout',
+  'openchannel_abort',
+  'openchannel_bump',
+  'openchannel_init',
+  'openchannel_signed',
+  'openchannel_update',
+  'pay',
+  'payersign',
+  'paystatus',
+  'ping',
+  'plugin',
+  'reserveinputs',
+  'sendinvoice',
+  'sendonion',
+  'sendonionmessage',
+  'sendpay',
+  'sendpsbt',
+  'sendrawtransaction',
+  'setchannelfee',
+  'signmessage',
+  'signpsbt',
+  'stop',
+  'txdiscard',
+  'txprepare',
+  'txsend',
+  'unreserveinputs',
+  'utxopsbt',
+  'waitanyinvoice',
+  'waitblockheight',
+  'waitinvoice',
+  'waitsendpay',
+  'withdraw'
+];
+
+
+import EventEmitter from 'events';
+import { existsSync, statSync } from 'fs';
+import { createConnection, Socket } from 'net';
+import { homedir } from 'os';
+import path from 'path';
+import { createInterface, Interface } from 'readline';
+import logger from '../../../logger';
+import { AbstractLightningApi } from '../lightning-api-abstract-factory';
+import { ILightningApi } from '../lightning-api.interface';
+import { convertAndmergeBidirectionalChannels, convertNode } from './clightning-convert';
+
+class LightningError extends Error {
+  type: string = 'lightning';
+  message: string = 'lightning-client error';
+
+  constructor(error) {
+    super();
+    this.type = error.type;
+    this.message = error.message;
+  }
+}
+
+const defaultRpcPath = path.join(homedir(), '.lightning')
+  , fStat = (...p) => statSync(path.join(...p))
+  , fExists = (...p) => existsSync(path.join(...p))
+
+export default class CLightningClient extends EventEmitter implements AbstractLightningApi {
+  private rpcPath: string;
+  private reconnectWait: number;
+  private reconnectTimeout;
+  private reqcount: number;
+  private client: Socket;
+  private rl: Interface;
+  private clientConnectionPromise: Promise<unknown>;
+
+  constructor(rpcPath = defaultRpcPath) {
+    if (!path.isAbsolute(rpcPath)) {
+      throw new Error('The rpcPath must be an absolute path');
+    }
+
+    if (!fExists(rpcPath) || !fStat(rpcPath).isSocket()) {
+      // network directory provided, use the lightning-rpc within in
+      if (fExists(rpcPath, 'lightning-rpc')) {
+        rpcPath = path.join(rpcPath, 'lightning-rpc');
+      }
+
+      // main data directory provided, default to using the bitcoin mainnet subdirectory
+      // to be removed in v0.2.0
+      else if (fExists(rpcPath, 'bitcoin', 'lightning-rpc')) {
+        logger.warn(`${rpcPath}/lightning-rpc is missing, using the bitcoin mainnet subdirectory at ${rpcPath}/bitcoin instead.`, logger.tags.ln)
+        logger.warn(`specifying the main lightning data directory is deprecated, please specify the network directory explicitly.\n`, logger.tags.ln)
+        rpcPath = path.join(rpcPath, 'bitcoin', 'lightning-rpc')
+      }
+    }
+
+    logger.debug(`Connecting to ${rpcPath}`, logger.tags.ln);
+
+    super();
+    this.rpcPath = rpcPath;
+    this.reconnectWait = 0.5;
+    this.reconnectTimeout = null;
+    this.reqcount = 0;
+
+    const _self = this;
+
+    this.client = createConnection(rpcPath).on(
+      'error', () => {
+        _self.increaseWaitTime();
+        _self.reconnect();
+      }
+    );
+    this.rl = createInterface({ input: this.client }).on(
+      'error', () => {
+        _self.increaseWaitTime();
+        _self.reconnect();
+      }
+    );
+
+    this.clientConnectionPromise = new Promise<void>(resolve => {
+      _self.client.on('connect', () => {
+        logger.info(`CLightning client connected`, logger.tags.ln);
+        _self.reconnectWait = 1;
+        resolve();
+      });
+
+      _self.client.on('end', () => {
+        logger.err(`CLightning client connection closed, reconnecting`, logger.tags.ln);
+        _self.increaseWaitTime();
+        _self.reconnect();
+      });
+
+      _self.client.on('error', error => {
+        logger.err(`CLightning client connection error: ${error}`, logger.tags.ln);
+        _self.increaseWaitTime();
+        _self.reconnect();
+      });
+    });
+
+    this.rl.on('line', line => {
+      line = line.trim();
+      if (!line) {
+        return;
+      }
+      const data = JSON.parse(line);
+      _self.emit('res:' + data.id, data);
+    });
+  }
+
+  increaseWaitTime(): void {
+    if (this.reconnectWait >= 16) {
+      this.reconnectWait = 16;
+    } else {
+      this.reconnectWait *= 2;
+    }
+  }
+
+  reconnect(): void {
+    const _self = this;
+
+    if (this.reconnectTimeout) {
+      return;
+    }
+
+    this.reconnectTimeout = setTimeout(() => {
+      logger.debug(`Trying to reconnect...`, logger.tags.ln);
+
+      _self.client.connect(_self.rpcPath);
+      _self.reconnectTimeout = null;
+    }, this.reconnectWait * 1000);
+  }
+
+  call(method, args = []): Promise<any> {
+    const _self = this;
+
+    const callInt = ++this.reqcount;
+    const sendObj = {
+      jsonrpc: '2.0',
+      method,
+      params: args,
+      id: '' + callInt
+    };
+
+
+    // Wait for the client to connect
+    return this.clientConnectionPromise
+      .then(() => new Promise((resolve, reject) => {
+        // Wait for a response
+        this.once('res:' + callInt, res => res.error == null
+          ? resolve(res.result)
+          : reject(new LightningError(res.error))
+        );
+
+        // Send the command
+        _self.client.write(JSON.stringify(sendObj));
+      }));
+  }
+
+  async $getNetworkGraph(): Promise<ILightningApi.NetworkGraph> {
+    const listnodes: any[] = await this.call('listnodes');
+    const listchannels: any[] = await this.call('listchannels');
+    const channelsList = await convertAndmergeBidirectionalChannels(listchannels['channels']);
+
+    return {
+      nodes: listnodes['nodes'].map(node => convertNode(node)),
+      edges: channelsList,
+    };
+  }
+}
+
+const protify = s => s.replace(/-([a-z])/g, m => m[1].toUpperCase());
+
+methods.forEach(k => {
+  CLightningClient.prototype[protify(k)] = function (...args: any) {
+    return this.call(k, args);
+  };
+});

backend/src/api/lightning/clightning/clightning-convert.ts

@@ -0,0 +1,275 @@
+import { ILightningApi } from '../lightning-api.interface';
+import FundingTxFetcher from '../../../tasks/lightning/sync-tasks/funding-tx-fetcher';
+import logger from '../../../logger';
+import { Common } from '../../common';
+import { hex2bin } from '../../../utils/format';
+import config from '../../../config';
+
+// https://github.com/lightningnetwork/lnd/blob/master/lnwire/features.go
+export enum FeatureBits {
+	DataLossProtectRequired = 0,
+	DataLossProtectOptional = 1,
+	InitialRoutingSync = 3,
+	UpfrontShutdownScriptRequired = 4,
+	UpfrontShutdownScriptOptional = 5,
+	GossipQueriesRequired = 6,
+	GossipQueriesOptional = 7,
+	TLVOnionPayloadRequired = 8,
+	TLVOnionPayloadOptional = 9,
+	StaticRemoteKeyRequired = 12,
+	StaticRemoteKeyOptional = 13,
+	PaymentAddrRequired = 14,
+	PaymentAddrOptional = 15,
+	MPPRequired = 16,
+	MPPOptional = 17,
+	WumboChannelsRequired = 18,
+	WumboChannelsOptional = 19,
+	AnchorsRequired = 20,
+	AnchorsOptional = 21,
+	AnchorsZeroFeeHtlcTxRequired = 22,
+	AnchorsZeroFeeHtlcTxOptional = 23,
+	ShutdownAnySegwitRequired = 26,
+	ShutdownAnySegwitOptional = 27,
+	AMPRequired = 30,
+	AMPOptional = 31,
+	ExplicitChannelTypeRequired = 44,
+	ExplicitChannelTypeOptional = 45,
+	ScidAliasRequired = 46,
+	ScidAliasOptional = 47,
+	PaymentMetadataRequired = 48,
+	PaymentMetadataOptional = 49,
+	ZeroConfRequired = 50,
+	ZeroConfOptional = 51,
+	KeysendRequired = 54,
+	KeysendOptional = 55,
+	ScriptEnforcedLeaseRequired = 2022,
+	ScriptEnforcedLeaseOptional = 2023,
+	SimpleTaprootChannelsRequiredFinal = 80,
+	SimpleTaprootChannelsOptionalFinal = 81,
+	SimpleTaprootChannelsRequiredStaging = 180,
+	SimpleTaprootChannelsOptionalStaging = 181,
+	MaxBolt11Feature = 5114,
+};
+
+export const FeaturesMap = new Map<FeatureBits, string>([
+	[FeatureBits.DataLossProtectRequired, 'data-loss-protect'],
+	[FeatureBits.DataLossProtectOptional, 'data-loss-protect'],
+	[FeatureBits.InitialRoutingSync, 'initial-routing-sync'],
+	[FeatureBits.UpfrontShutdownScriptRequired, 'upfront-shutdown-script'],
+	[FeatureBits.UpfrontShutdownScriptOptional, 'upfront-shutdown-script'],
+	[FeatureBits.GossipQueriesRequired, 'gossip-queries'],
+	[FeatureBits.GossipQueriesOptional, 'gossip-queries'],
+	[FeatureBits.TLVOnionPayloadRequired, 'tlv-onion'],
+	[FeatureBits.TLVOnionPayloadOptional, 'tlv-onion'],
+	[FeatureBits.StaticRemoteKeyOptional, 'static-remote-key'],
+	[FeatureBits.StaticRemoteKeyRequired, 'static-remote-key'],
+	[FeatureBits.PaymentAddrOptional, 'payment-addr'],
+	[FeatureBits.PaymentAddrRequired, 'payment-addr'],
+	[FeatureBits.MPPOptional, 'multi-path-payments'],
+	[FeatureBits.MPPRequired, 'multi-path-payments'],
+	[FeatureBits.AnchorsRequired, 'anchor-commitments'],
+	[FeatureBits.AnchorsOptional, 'anchor-commitments'],
+	[FeatureBits.AnchorsZeroFeeHtlcTxRequired, 'anchors-zero-fee-htlc-tx'],
+	[FeatureBits.AnchorsZeroFeeHtlcTxOptional, 'anchors-zero-fee-htlc-tx'],
+	[FeatureBits.WumboChannelsRequired, 'wumbo-channels'],
+	[FeatureBits.WumboChannelsOptional, 'wumbo-channels'],
+	[FeatureBits.AMPRequired, 'amp'],
+	[FeatureBits.AMPOptional, 'amp'],
+	[FeatureBits.PaymentMetadataOptional, 'payment-metadata'],
+	[FeatureBits.PaymentMetadataRequired, 'payment-metadata'],
+	[FeatureBits.ExplicitChannelTypeOptional, 'explicit-commitment-type'],
+	[FeatureBits.ExplicitChannelTypeRequired, 'explicit-commitment-type'],
+	[FeatureBits.KeysendOptional, 'keysend'],
+	[FeatureBits.KeysendRequired, 'keysend'],
+	[FeatureBits.ScriptEnforcedLeaseRequired, 'script-enforced-lease'],
+	[FeatureBits.ScriptEnforcedLeaseOptional, 'script-enforced-lease'],
+	[FeatureBits.ScidAliasRequired, 'scid-alias'],
+	[FeatureBits.ScidAliasOptional, 'scid-alias'],
+	[FeatureBits.ZeroConfRequired, 'zero-conf'],
+	[FeatureBits.ZeroConfOptional, 'zero-conf'],
+	[FeatureBits.ShutdownAnySegwitRequired, 'shutdown-any-segwit'],
+	[FeatureBits.ShutdownAnySegwitOptional, 'shutdown-any-segwit'],
+	[FeatureBits.SimpleTaprootChannelsRequiredFinal, 'taproot-channels'],
+	[FeatureBits.SimpleTaprootChannelsOptionalFinal, 'taproot-channels'],
+	[FeatureBits.SimpleTaprootChannelsRequiredStaging, 'taproot-channels-staging'],
+	[FeatureBits.SimpleTaprootChannelsOptionalStaging, 'taproot-channels-staging'],
+]);
+
+/**
+ * Convert a clightning "listnode" entry to a lnd node entry
+ */
+export function convertNode(clNode: any): ILightningApi.Node {
+  let custom_records: { [type: number]: string } | undefined = undefined;
+  if (clNode.option_will_fund) {
+    try {
+      custom_records = { '1': Buffer.from(clNode.option_will_fund.compact_lease || '', 'hex').toString('base64') };
+    } catch (e) {
+      logger.err(`Cannot decode option_will_fund compact_lease for ${clNode.nodeid}). Reason: ` + (e instanceof Error ? e.message : e));
+      custom_records = undefined;
+    }
+  }
+
+  const nodeFeatures: ILightningApi.Feature[] = [];
+  const nodeFeaturesBinary = hex2bin(clNode.features).split('').reverse().join('');
+
+  for (let i = 0; i < nodeFeaturesBinary.length; i++) {
+    if (nodeFeaturesBinary[i] === '0') {
+      continue;
+    }
+    const feature = FeaturesMap.get(i);
+    if (!feature) {
+      nodeFeatures.push({
+        bit: i,
+        name: 'unknown',
+        is_required: i % 2 === 0,
+        is_known: false
+      });
+    } else {
+      nodeFeatures.push({
+        bit: i,
+        name: feature,
+        is_required: i % 2 === 0,
+        is_known: true
+      });
+    }
+  }
+
+  return {
+    alias: clNode.alias ?? '',
+    color: `#${clNode.color ?? ''}`,
+    features: nodeFeatures,
+    pub_key: clNode.nodeid,
+    addresses: clNode.addresses?.map((addr) => {
+      let address = addr.address;
+      if (addr.type === 'ipv6') {
+        address = `[${address}]`;
+      }
+      return {
+        network: addr.type,
+        addr: `${address}:${addr.port}`
+      };
+    }) ?? [],
+    last_update: clNode?.last_timestamp ?? 0,
+    custom_records
+  };
+}
+
+/**
+ * Convert clightning "listchannels" response to lnd "describegraph.edges" format
+ */
+export async function convertAndmergeBidirectionalChannels(clChannels: any[]): Promise<ILightningApi.Channel[]> {
+  logger.debug(`Converting clightning nodes and channels to lnd graph format`, logger.tags.ln);
+
+  let loggerTimer = new Date().getTime() / 1000;
+  let channelProcessed = 0;
+
+  const consolidatedChannelList: ILightningApi.Channel[] = [];
+  const clChannelsDict = {};
+  const clChannelsDictCount = {};
+
+  for (const clChannel of clChannels) {
+    if (!clChannelsDict[clChannel.short_channel_id]) {
+      clChannelsDict[clChannel.short_channel_id] = clChannel;
+      clChannelsDictCount[clChannel.short_channel_id] = 1;
+    } else {
+      const fullChannel = await buildFullChannel(clChannel, clChannelsDict[clChannel.short_channel_id]);
+      if (fullChannel !== null) {
+        consolidatedChannelList.push(fullChannel);
+        delete clChannelsDict[clChannel.short_channel_id];
+        clChannelsDictCount[clChannel.short_channel_id]++;
+      }
+    }
+
+    const elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer);
+    if (elapsedSeconds > config.LIGHTNING.LOGGER_UPDATE_INTERVAL) {
+      logger.info(`Building complete channels from clightning output. Channels processed: ${channelProcessed + 1} of ${clChannels.length}`, logger.tags.ln);
+      loggerTimer = new Date().getTime() / 1000;
+    }
+
+    ++channelProcessed;
+  }
+
+  channelProcessed = 0;
+  const keys = Object.keys(clChannelsDict);
+  for (const short_channel_id of keys) {
+    const incompleteChannel = await buildIncompleteChannel(clChannelsDict[short_channel_id]);
+    if (incompleteChannel !== null) {
+      consolidatedChannelList.push(incompleteChannel);
+    }
+
+    const elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer);
+    if (elapsedSeconds > config.LIGHTNING.LOGGER_UPDATE_INTERVAL) {
+      logger.info(`Building partial channels from clightning output. Channels processed: ${channelProcessed + 1} of ${keys.length}`);
+      loggerTimer = new Date().getTime() / 1000;
+    }
+
+    channelProcessed++;
+  }
+
+  return consolidatedChannelList;
+}
+
+/**
+ * Convert two clightning "getchannels" entries into a full a lnd "describegraph.edges" format
+ * In this case, clightning knows the channel policy for both nodes
+ */
+async function buildFullChannel(clChannelA: any, clChannelB: any): Promise<ILightningApi.Channel | null> {
+  const lastUpdate = Math.max(clChannelA.last_update ?? 0, clChannelB.last_update ?? 0);
+
+  const tx = await FundingTxFetcher.$fetchChannelOpenTx(clChannelA.short_channel_id);
+  if (!tx) {
+    return null;
+  }
+  const parts = clChannelA.short_channel_id.split('x');
+  const outputIdx = parts[2];
+
+  return {
+    channel_id: Common.channelShortIdToIntegerId(clChannelA.short_channel_id),
+    capacity: (clChannelA.amount_msat / 1000).toString(),
+    last_update: lastUpdate,
+    node1_policy: convertPolicy(clChannelA),
+    node2_policy: convertPolicy(clChannelB),
+    chan_point: `${tx.txid}:${outputIdx}`,
+    node1_pub: clChannelA.source,
+    node2_pub: clChannelB.source,
+  };
+}
+
+/**
+ * Convert one clightning "getchannels" entry into a full a lnd "describegraph.edges" format
+ * In this case, clightning knows the channel policy of only one node
+ */
+async function buildIncompleteChannel(clChannel: any): Promise<ILightningApi.Channel | null> {
+  const tx = await FundingTxFetcher.$fetchChannelOpenTx(clChannel.short_channel_id);
+  if (!tx) {
+    return null;
+  }
+  const parts = clChannel.short_channel_id.split('x');
+  const outputIdx = parts[2];
+
+  return {
+    channel_id: Common.channelShortIdToIntegerId(clChannel.short_channel_id),
+    capacity: (clChannel.amount_msat / 1000).toString(),
+    last_update: clChannel.last_update ?? 0,
+    node1_policy: convertPolicy(clChannel),
+    node2_policy: null,
+    chan_point: `${tx.txid}:${outputIdx}`,
+    node1_pub: clChannel.source,
+    node2_pub: clChannel.destination,
+  };
+}
+
+/**
+ * Convert a clightning "listnode" response to a lnd channel policy format
+ */
+function convertPolicy(clChannel: any): ILightningApi.RoutingPolicy {
+  return {
+    time_lock_delta: clChannel.delay,
+    min_htlc: clChannel.htlc_minimum_msat.toString(),
+    max_htlc_msat: clChannel.htlc_maximum_msat.toString(),
+    fee_base_msat: clChannel.base_fee_millisatoshi,
+    fee_rate_milli_msat: clChannel.fee_per_millionth,
+    disabled: !clChannel.active,
+    last_update: clChannel.last_update ?? 0,
+  };
+}

backend/src/api/lightning/lightning-api-abstract-factory.ts

@@ -0,0 +1,5 @@
+import { ILightningApi } from './lightning-api.interface';
+
+export interface AbstractLightningApi {
+  $getNetworkGraph(): Promise<ILightningApi.NetworkGraph>;
+}

backend/src/api/lightning/lightning-api-factory.ts

@@ -0,0 +1,16 @@
+import config from '../../config';
+import CLightningClient from './clightning/clightning-client';
+import { AbstractLightningApi } from './lightning-api-abstract-factory';
+import LndApi from './lnd/lnd-api';
+
+function lightningApiFactory(): AbstractLightningApi {
+  switch (config.LIGHTNING.ENABLED === true && config.LIGHTNING.BACKEND) {
+    case 'cln':
+      return new CLightningClient(config.CLIGHTNING.SOCKET);
+    case 'lnd':
+    default:
+      return new LndApi();
+  }
+}
+
+export default lightningApiFactory();

backend/src/api/lightning/lightning-api.interface.ts

@@ -0,0 +1,93 @@
+export namespace ILightningApi {
+  export interface NetworkInfo {
+    graph_diameter: number;
+    avg_out_degree: number;
+    max_out_degree: number;
+    num_nodes: number;
+    num_channels: number;
+    total_network_capacity: string;
+    avg_channel_size: number;
+    min_channel_size: string;
+    max_channel_size: string;
+    median_channel_size_sat: string;
+    num_zombie_chans: string;
+  }
+
+  export interface NetworkGraph {
+    nodes: Node[];
+    edges: Channel[];
+  }
+
+  export interface Channel {
+    channel_id: string;
+    chan_point: string;
+    last_update: number | null;
+    node1_pub: string;
+    node2_pub: string;
+    capacity: string;
+    node1_policy: RoutingPolicy | null;
+    node2_policy: RoutingPolicy | null;
+  }
+
+  export interface RoutingPolicy {
+    time_lock_delta: number;
+    min_htlc: string;
+    fee_base_msat: string;
+    fee_rate_milli_msat: string;
+    disabled: boolean;
+    max_htlc_msat: string;
+    last_update: number | null;
+  }
+
+  export interface Node {
+    last_update: number | null;
+    pub_key: string;
+    alias: string;
+    addresses: {
+      network: string;
+      addr: string;
+    }[];
+    color: string;
+    features: { [key: number]: Feature };
+    custom_records?: { [type: number]: string };
+  }
+
+  export interface Info {
+    identity_pubkey: string;
+    alias: string;
+    num_pending_channels: number;
+    num_active_channels: number;
+    num_peers: number;
+    block_height: number;
+    block_hash: string;
+    synced_to_chain: boolean;
+    testnet: boolean;
+    uris: string[];
+    best_header_timestamp: string;
+    version: string;
+    num_inactive_channels: number;
+    chains: {
+      chain: string;
+      network: string;
+    }[];
+    color: string;
+    synced_to_graph: boolean;
+    features: { [key: number]: Feature };
+    commit_hash: string;
+    /** Available on LND since v0.15.0-beta */
+    require_htlc_interceptor?: boolean;
+  }
+
+  export interface Feature {
+    bit: number;
+    name: string;
+    is_required: boolean;
+    is_known: boolean;
+  }
+
+  export interface ForensicOutput {
+    node?: 1 | 2;
+    type: number;
+    value: number;
+  }
+}

backend/src/api/lightning/lnd/lnd-api.ts

@@ -0,0 +1,64 @@
+import axios, { AxiosRequestConfig } from 'axios';
+import { Agent } from 'https';
+import * as fs from 'fs';
+import { AbstractLightningApi } from '../lightning-api-abstract-factory';
+import { ILightningApi } from '../lightning-api.interface';
+import config from '../../../config';
+import logger from '../../../logger';
+
+class LndApi implements AbstractLightningApi {
+  axiosConfig: AxiosRequestConfig = {};
+
+  constructor() {
+    if (!config.LIGHTNING.ENABLED) {
+      return;
+    }
+    try {
+      this.axiosConfig = {
+        headers: {
+          'Grpc-Metadata-macaroon': fs.readFileSync(config.LND.MACAROON_PATH).toString('hex'),
+        },
+        httpsAgent: new Agent({
+          ca: fs.readFileSync(config.LND.TLS_CERT_PATH)
+        }),
+        timeout: config.LND.TIMEOUT
+      };
+    } catch (e) {
+      config.LIGHTNING.ENABLED = false;
+      logger.updateNetwork();
+      logger.err(`Could not initialize LND Macaroon/TLS Cert. Disabling LIGHTNING. ` + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  async $getNetworkInfo(): Promise<ILightningApi.NetworkInfo> {
+    return axios.get<ILightningApi.NetworkInfo>(config.LND.REST_API_URL + '/v1/graph/info', this.axiosConfig)
+      .then((response) => response.data);
+  }
+
+  async $getInfo(): Promise<ILightningApi.Info> {
+    return axios.get<ILightningApi.Info>(config.LND.REST_API_URL + '/v1/getinfo', this.axiosConfig)
+      .then((response) => response.data);
+  }
+
+  async $getNetworkGraph(): Promise<ILightningApi.NetworkGraph> {
+    const graph = await axios.get<ILightningApi.NetworkGraph>(config.LND.REST_API_URL + '/v1/graph', this.axiosConfig)
+      .then((response) => response.data);
+
+    for (const node of graph.nodes) {
+      const nodeFeatures: ILightningApi.Feature[] = [];
+      for (const bit in node.features) {        
+        nodeFeatures.push({
+          bit: parseInt(bit, 10),
+          name: node.features[bit].name,  
+          is_required: node.features[bit].is_required,
+          is_known: node.features[bit].is_known,
+        });
+      }
+      node.features = nodeFeatures;
+    }
+
+    return graph;
+  }
+}
+
+export default LndApi;

backend/src/api/liquid/elements-parser.ts

@@ -2,11 +2,15 @@ import { IBitcoinApi } from '../bitcoin/bitcoin-api.interface';
 import bitcoinClient from '../bitcoin/bitcoin-client';
 import bitcoinSecondClient from '../bitcoin/bitcoin-second-client';
 import { Common } from '../common';
-import { DB } from '../../database';
+import DB from '../../database';
 import logger from '../../logger';
 
+const federationChangeAddresses = ['bc1qxvay4an52gcghxq5lavact7r6qe9l4laedsazz8fj2ee2cy47tlqff4aj4', '3EiAcrzq1cELXScc98KeCswGWZaPGceT1d'];
+const auditBlockOffsetWithTip = 1; // Wait for 1 block confirmation before processing the block in the audit process to reduce the risk of reorgs
+
 class ElementsParser {
   private isRunning = false;
+  private isUtxosUpdatingRunning = false;
 
   constructor() { }
 
@@ -18,12 +22,12 @@ class ElementsParser {
       this.isRunning = true;
       const result = await bitcoinClient.getChainTips();
       const tip = result[0].height;
-      const latestBlock = await this.$getLatestBlockFromDatabase();
-      for (let height = latestBlock.block + 1; height <= tip; height++) {
+      const latestBlockHeight = await this.$getLatestBlockHeightFromDatabase();
+      for (let height = latestBlockHeight + 1; height <= tip; height++) {
         const blockHash: IBitcoinApi.ChainTips = await bitcoinClient.getBlockHash(height);
         const block: IBitcoinApi.Block = await bitcoinClient.getBlock(blockHash, 2);
         await this.$parseBlock(block);
-        await this.$saveLatestBlockToDatabase(block.height, block.time, block.hash);
+        await this.$saveLatestBlockToDatabase(block.height);
       }
       this.isRunning = false;
     } catch (e) {
@@ -32,14 +36,6 @@ class ElementsParser {
     }
   }
 
-  public async $getPegDataByMonth(): Promise<any> {
-    const connection = await DB.pool.getConnection();
-    const query = `SELECT SUM(amount) AS amount, DATE_FORMAT(FROM_UNIXTIME(datetime), '%Y-%m-01') AS date FROM elements_pegs GROUP BY DATE_FORMAT(FROM_UNIXTIME(datetime), '%Y%m')`;
-    const [rows] = await connection.query<any>(query);
-    connection.release();
-    return rows;
-  }
-
   protected async $parseBlock(block: IBitcoinApi.Block) {
     for (const tx of block.tx) {
       await this.$parseInputs(tx, block);
@@ -57,54 +53,440 @@ class ElementsParser {
 
   protected async $parsePegIn(input: IBitcoinApi.Vin, vindex: number, txid: string, block: IBitcoinApi.Block) {
     const bitcoinTx: IBitcoinApi.Transaction = await bitcoinSecondClient.getRawTransaction(input.txid, true);
+    const bitcoinBlock: IBitcoinApi.Block = await bitcoinSecondClient.getBlock(bitcoinTx.blockhash);
     const prevout = bitcoinTx.vout[input.vout || 0];
     const outputAddress = prevout.scriptPubKey.address || (prevout.scriptPubKey.addresses && prevout.scriptPubKey.addresses[0]) || '';
     await this.$savePegToDatabase(block.height, block.time, prevout.value * 100000000, txid, vindex,
-      outputAddress, bitcoinTx.txid, prevout.n, 1);
+      outputAddress, bitcoinTx.txid, prevout.n, bitcoinBlock.height, bitcoinBlock.time, 1);
   }
 
   protected async $parseOutputs(tx: IBitcoinApi.Transaction, block: IBitcoinApi.Block) {
     for (const output of tx.vout) {
       if (output.scriptPubKey.pegout_chain) {
         await this.$savePegToDatabase(block.height, block.time, 0 - output.value * 100000000, tx.txid, output.n,
-          (output.scriptPubKey.pegout_addresses && output.scriptPubKey.pegout_addresses[0] || ''), '', 0, 0);
+          (output.scriptPubKey.pegout_address || ''), '', 0, 0, 0, 0);
       }
       if (!output.scriptPubKey.pegout_chain && output.scriptPubKey.type === 'nulldata'
         && output.value && output.value > 0 && output.asset && output.asset === Common.nativeAssetId) {
         await this.$savePegToDatabase(block.height, block.time, 0 - output.value * 100000000, tx.txid, output.n,
-          (output.scriptPubKey.pegout_addresses && output.scriptPubKey.pegout_addresses[0] || ''), '', 0, 1);
+          (output.scriptPubKey.pegout_address || ''), '', 0, 0, 0, 1);
       }
     }
   }
 
   protected async $savePegToDatabase(height: number, blockTime: number, amount: number, txid: string,
-    txindex: number, bitcoinaddress: string, bitcointxid: string, bitcoinindex: number, final_tx: number): Promise<void> {
-    const connection = await DB.pool.getConnection();
-    const query = `INSERT INTO elements_pegs(
+    txindex: number, bitcoinaddress: string, bitcointxid: string, bitcoinindex: number, bitcoinblock: number, bitcoinBlockTime: number, final_tx: number): Promise<void> {
+    const query = `INSERT IGNORE INTO elements_pegs(
         block, datetime, amount, txid, txindex, bitcoinaddress, bitcointxid, bitcoinindex, final_tx
       ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`;
 
     const params: (string | number)[] = [
       height, blockTime, amount, txid, txindex, bitcoinaddress, bitcointxid, bitcoinindex, final_tx
     ];
-    await connection.query(query, params);
-    connection.release();
-    logger.debug(`Saved L-BTC peg from block height #${height} with TXID ${txid}.`);
+    await DB.query(query, params);
+    logger.debug(`Saved L-BTC peg from Liquid block height #${height} with TXID ${txid}.`);
+
+    if (amount > 0) { // Peg-in
+  
+      // Add the address to the federation addresses table
+      await DB.query(`INSERT IGNORE INTO federation_addresses (bitcoinaddress) VALUES (?)`, [bitcoinaddress]);
+
+      // Add the UTXO to the federation txos table
+      const query_utxos = `INSERT IGNORE INTO federation_txos (txid, txindex, bitcoinaddress, amount, blocknumber, blocktime, unspent, lastblockupdate, lasttimeupdate, timelock, expiredAt, emergencyKey, pegtxid, pegindex, pegblocktime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
+      const params_utxos: (string | number)[] = [bitcointxid, bitcoinindex, bitcoinaddress, amount, bitcoinblock, bitcoinBlockTime, 1, bitcoinblock - 1, 0, 4032, 0, 0, txid, txindex, blockTime];
+      await DB.query(query_utxos, params_utxos);
+      const [minBlockUpdate] = await DB.query(`SELECT MIN(lastblockupdate) AS lastblockupdate FROM federation_txos WHERE unspent = 1`)
+      await this.$saveLastBlockAuditToDatabase(minBlockUpdate[0]['lastblockupdate']);
+      logger.debug(`Saved new Federation UTXO ${bitcointxid}:${bitcoinindex} belonging to ${bitcoinaddress} to federation txos`);
+
+    }
   }
 
-  protected async $getLatestBlockFromDatabase(): Promise<any> {
-    const connection = await DB.pool.getConnection();
-    const query = `SELECT block, datetime, block_hash FROM last_elements_block`;
-    const [rows] = await connection.query<any>(query);
-    connection.release();
+  protected async $getLatestBlockHeightFromDatabase(): Promise<number> {
+    const query = `SELECT number FROM state WHERE name = 'last_elements_block'`;
+    const [rows] = await DB.query(query);
+    return rows[0]['number'];
+  }
+
+  protected async $saveLatestBlockToDatabase(blockHeight: number) {
+    const query = `UPDATE state SET number = ? WHERE name = 'last_elements_block'`;
+    await DB.query(query, [blockHeight]);
+  }
+
+  ///////////// FEDERATION AUDIT //////////////
+
+  public async $updateFederationUtxos() {
+    if (this.isUtxosUpdatingRunning) {
+      return;
+    }
+
+    this.isUtxosUpdatingRunning = true;
+
+    try {
+      let auditProgress = await this.$getAuditProgress();
+      // If no peg in transaction was found in the database, return
+      if (!auditProgress.lastBlockAudit) {
+        logger.debug(`No Federation UTXOs found in the database. Waiting for some to be confirmed before starting the Federation UTXOs audit`);
+        this.isUtxosUpdatingRunning = false;
+        return;
+      }
+
+      const bitcoinBlocksToSync = await this.$getBitcoinBlockchainState();
+      // If the bitcoin blockchain is not synced yet, return
+      if (bitcoinBlocksToSync.bitcoinHeaders > bitcoinBlocksToSync.bitcoinBlocks + 1) {
+        logger.debug(`Bitcoin client is not synced yet. ${bitcoinBlocksToSync.bitcoinHeaders - bitcoinBlocksToSync.bitcoinBlocks} blocks remaining to sync before the Federation audit process can start`);
+        this.isUtxosUpdatingRunning = false;
+        return;
+      }
+
+      auditProgress.lastBlockAudit++;
+
+      // Logging
+      let indexedThisRun = 0;
+      let timer = Date.now() / 1000;
+      const startedAt = Date.now() / 1000;
+      const indexingSpeeds: number[] = [];
+
+      while (auditProgress.lastBlockAudit <= auditProgress.confirmedTip) {
+
+        // First, get the current UTXOs that need to be scanned in the block
+        const utxos = await this.$getFederationUtxosToScan(auditProgress.lastBlockAudit);
+
+        // Get the peg-out addresses that need to be scanned
+        const redeemAddresses = await this.$getRedeemAddressesToScan();
+
+        // The fast way: check if these UTXOs are still unspent as of the current block with gettxout
+        let spentAsTip: any[];
+        let unspentAsTip: any[];
+        if (auditProgress.confirmedTip - auditProgress.lastBlockAudit <= 150) { // If the audit status is not too far in the past, we can use gettxout (fast way)
+          const utxosToParse = await this.$getFederationUtxosToParse(utxos);
+          spentAsTip = utxosToParse.spentAsTip;
+          unspentAsTip = utxosToParse.unspentAsTip;
+          logger.debug(`Found ${utxos.length} Federation UTXOs and ${redeemAddresses.length} Peg-Out Addresses to scan in Bitcoin block height #${auditProgress.lastBlockAudit} / #${auditProgress.confirmedTip}`);
+          logger.debug(`${unspentAsTip.length} / ${utxos.length} Federation UTXOs are unspent as of tip`);
+        } else { // If the audit status is too far in the past, it is useless and wasteful to look for still unspent txos since they will all be spent as of the tip
+          spentAsTip = utxos;
+          unspentAsTip = [];
+
+          // Logging
+          const elapsedSeconds = (Date.now() / 1000) - timer;
+          if (elapsedSeconds > 5) {
+            const runningFor = (Date.now() / 1000) - startedAt;
+            const blockPerSeconds = indexedThisRun / elapsedSeconds;
+            indexingSpeeds.push(blockPerSeconds);
+            if (indexingSpeeds.length > 100) indexingSpeeds.shift(); // Keep the length of the up to 100 last indexing speeds
+            const meanIndexingSpeed = indexingSpeeds.reduce((a, b) => a + b, 0) / indexingSpeeds.length;
+            const eta = (auditProgress.confirmedTip - auditProgress.lastBlockAudit) / meanIndexingSpeed;
+            logger.debug(`Scanning ${utxos.length} Federation UTXOs and ${redeemAddresses.length} Peg-Out Addresses at Bitcoin block height #${auditProgress.lastBlockAudit} / #${auditProgress.confirmedTip} | ~${meanIndexingSpeed.toFixed(2)} blocks/sec | elapsed: ${(runningFor / 60).toFixed(0)} minutes | ETA: ${(eta / 60).toFixed(0)} minutes`);
+            timer = Date.now() / 1000;
+            indexedThisRun = 0;
+          }
+        }
+
+        // The slow way: parse the block to look for the spending tx
+        const blockHash: IBitcoinApi.ChainTips = await bitcoinSecondClient.getBlockHash(auditProgress.lastBlockAudit);
+        const block: IBitcoinApi.Block = await bitcoinSecondClient.getBlock(blockHash, 2);
+        await this.$parseBitcoinBlock(block, spentAsTip, unspentAsTip, auditProgress.confirmedTip, redeemAddresses);
+
+        // Finally, update the lastblockupdate of the remaining UTXOs and save to the database
+        const [minBlockUpdate] = await DB.query(`SELECT MIN(lastblockupdate) AS lastblockupdate FROM federation_txos WHERE unspent = 1`)
+        await this.$saveLastBlockAuditToDatabase(minBlockUpdate[0]['lastblockupdate']);
+
+        auditProgress = await this.$getAuditProgress();
+        auditProgress.lastBlockAudit++;
+        indexedThisRun++;
+      }
+
+      this.isUtxosUpdatingRunning = false;
+    } catch (e) {
+      this.isUtxosUpdatingRunning = false;
+      throw new Error(e instanceof Error ? e.message : 'Error');
+    } 
+  }
+
+  // Get the UTXOs that need to be scanned in block height (UTXOs that were last updated in the block height - 1)
+  protected async $getFederationUtxosToScan(height: number) { 
+    const query = `SELECT txid, txindex, bitcoinaddress, amount, blocknumber, timelock, expiredAt FROM federation_txos WHERE lastblockupdate = ? AND unspent = 1`;
+    const [rows] = await DB.query(query, [height - 1]);
+    return rows as any[];
+  }
+
+  // Returns the UTXOs that are spent as of tip and need to be scanned
+  protected async $getFederationUtxosToParse(utxos: any[]): Promise<any> {
+    const spentAsTip: any[] = [];
+    const unspentAsTip: any[] = [];
+
+    for (const utxo of utxos) {
+      const result = await bitcoinSecondClient.getTxOut(utxo.txid, utxo.txindex, false);
+      result ? unspentAsTip.push(utxo) : spentAsTip.push(utxo);
+    }
+    
+    return {spentAsTip, unspentAsTip};
+  }
+
+  protected async $parseBitcoinBlock(block: IBitcoinApi.Block, spentAsTip: any[], unspentAsTip: any[], confirmedTip: number, redeemAddressesData: any[] = []) {
+    const redeemAddresses: string[] = redeemAddressesData.map(redeemAddress => redeemAddress.bitcoinaddress);
+    for (const tx of block.tx) {
+      let mightRedeemInThisTx = false;
+      // Check if the Federation UTXOs that was spent as of tip are spent in this block
+      for (const input of tx.vin) {
+        const txo = spentAsTip.find(txo => txo.txid === input.txid && txo.txindex === input.vout);
+        if (txo) {
+          mightRedeemInThisTx = true; // A Federation UTXO is spent in this block: we might find a peg-out address in the outputs
+          if (txo.expiredAt > 0 ) {
+            if (input.txinwitness?.length !== 13) { // Check if the witness data of the input contains the 11 signatures: if it doesn't, emergency keys are being used
+              await DB.query(`UPDATE federation_txos SET unspent = 0, lastblockupdate = ?, lasttimeupdate = ?, emergencyKey = 1 WHERE txid = ? AND txindex = ?`, [block.height, block.time, txo.txid, txo.txindex]);
+              logger.debug(`Expired Federation UTXO ${txo.txid}:${txo.txindex} (${txo.amount} sats) was spent in block ${block.height} using emergency keys!`);
+             } else {
+              await DB.query(`UPDATE federation_txos SET unspent = 0, lastblockupdate = ?, lasttimeupdate = ? WHERE txid = ? AND txindex = ?`, [block.height, block.time, txo.txid, txo.txindex]);
+              logger.debug(`Expired Federation UTXO ${txo.txid}:${txo.txindex} (${txo.amount} sats) was spent in block ${block.height} using regular 11-of-15 signatures`);
+            }
+          } else {
+            await DB.query(`UPDATE federation_txos SET unspent = 0, lastblockupdate = ?, lasttimeupdate = ? WHERE txid = ? AND txindex = ?`, [block.height, block.time, txo.txid, txo.txindex]);
+            logger.debug(`Federation UTXO ${txo.txid}:${txo.txindex} (${txo.amount} sats) was spent in block ${block.height}`);
+          }
+          // Remove the TXO from the utxo array
+          spentAsTip.splice(spentAsTip.indexOf(txo), 1);
+        }
+      }
+      // Check if an output is sent to a change address of the federation
+      for (const output of tx.vout) {
+        if (output.scriptPubKey.address && federationChangeAddresses.includes(output.scriptPubKey.address)) {
+          // Check that the UTXO was not already added in the DB by previous scans
+          const [rows_check] = await DB.query(`SELECT txid FROM federation_txos WHERE txid = ? AND txindex = ?`, [tx.txid, output.n]) as any[];
+          if (rows_check.length === 0) {
+            const timelock = output.scriptPubKey.address === federationChangeAddresses[0] ? 4032 : 2016; // P2WSH change address has a 4032 timelock, P2SH change address has a 2016 timelock
+            const query_utxos = `INSERT INTO federation_txos (txid, txindex, bitcoinaddress, amount, blocknumber, blocktime, unspent, lastblockupdate, lasttimeupdate, timelock, expiredAt, emergencyKey, pegtxid, pegindex, pegblocktime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
+            const params_utxos: (string | number)[] = [tx.txid, output.n, output.scriptPubKey.address, output.value * 100000000, block.height, block.time, 1, block.height, 0, timelock, 0, 0, '', 0, 0];
+            await DB.query(query_utxos, params_utxos);
+            // Add the UTXO to the utxo array
+            spentAsTip.push({
+              txid: tx.txid,
+              txindex: output.n,
+              bitcoinaddress: output.scriptPubKey.address,
+              amount: output.value * 100000000,
+              blocknumber: block.height,
+              timelock: timelock,
+              expiredAt: 0,
+            });
+            logger.debug(`Added new Federation UTXO ${tx.txid}:${output.n} (${Math.round(output.value * 100000000)} sats), change address: ${output.scriptPubKey.address}`);
+          }
+        }
+        if (mightRedeemInThisTx && output.scriptPubKey.address && redeemAddresses.includes(output.scriptPubKey.address)) {
+          // Find the number of times output.scriptPubKey.address appears in redeemAddresses. There can be address reuse for peg-outs...
+          const matchingAddress: any[] = redeemAddressesData.filter(redeemAddress => redeemAddress.bitcoinaddress === output.scriptPubKey.address && -redeemAddress.amount === Math.round(output.value * 100000000));
+          if (matchingAddress.length > 0) {
+            if (matchingAddress.length > 1) {
+              // If there are more than one peg out address with the same amount, we can't know which one redeemed the UTXO: we take the oldest one
+              matchingAddress.sort((a, b) => a.datetime - b.datetime);
+              logger.debug(`Found redeem txid ${tx.txid}:${output.n} to peg-out address ${matchingAddress[0].bitcoinaddress}, amount ${matchingAddress[0].amount}, datetime ${matchingAddress[0].datetime}`);
+            } else {
+              logger.debug(`Found redeem txid ${tx.txid}:${output.n} to peg-out address ${matchingAddress[0].bitcoinaddress}, amount ${matchingAddress[0].amount}`);
+            }
+            const query_add_redeem = `UPDATE elements_pegs SET bitcointxid = ?, bitcoinindex = ? WHERE bitcoinaddress = ? AND amount = ? AND datetime = ?`;
+            const params_add_redeem: (string | number)[] = [tx.txid, output.n, matchingAddress[0].bitcoinaddress, matchingAddress[0].amount, matchingAddress[0].datetime];
+            await DB.query(query_add_redeem, params_add_redeem);
+            const index = redeemAddressesData.indexOf(matchingAddress[0]);
+            redeemAddressesData.splice(index, 1);
+            redeemAddresses.splice(index, 1);
+          } else { // The output amount does not match the peg-out amount... log it
+            logger.debug(`Found redeem txid ${tx.txid}:${output.n} to peg-out address ${output.scriptPubKey.address} but output amount ${Math.round(output.value * 100000000)} does not match the peg-out amount!`);
+          }
+        }
+      }
+    }
+
+    for (const utxo of spentAsTip) {   
+      if (utxo.expiredAt === 0 && block.height >= utxo.blocknumber + utxo.timelock) { // The UTXO is expiring in this block
+        await DB.query(`UPDATE federation_txos SET lastblockupdate = ?, expiredAt = ? WHERE txid = ? AND txindex = ?`, [block.height, block.time, utxo.txid, utxo.txindex]);
+      } else {
+        await DB.query(`UPDATE federation_txos SET lastblockupdate = ? WHERE txid = ? AND txindex = ?`, [block.height, utxo.txid, utxo.txindex]);
+      }
+    }
+
+    for (const utxo of unspentAsTip) {
+      if (utxo.expiredAt === 0 && block.height >= utxo.blocknumber + utxo.timelock) { // The UTXO is expiring in this block
+        await DB.query(`UPDATE federation_txos SET unspent = 0, lastblockupdate = ?, expiredAt = ? WHERE txid = ? AND txindex = ?`, [confirmedTip, block.time, utxo.txid, utxo.txindex]);
+      } else if (utxo.expiredAt === 0 && confirmedTip >= utxo.blocknumber + utxo.timelock) { // The UTXO is expiring before the tip: we need to keep track of it
+        await DB.query(`UPDATE federation_txos SET lastblockupdate = ? WHERE txid = ? AND txindex = ?`, [utxo.blocknumber + utxo.timelock - 1, utxo.txid, utxo.txindex]); 
+      } else {
+      await DB.query(`UPDATE federation_txos SET lastblockupdate = ? WHERE txid = ? AND txindex = ?`, [confirmedTip, utxo.txid, utxo.txindex]);
+      }
+    }
+  }
+
+  protected async $saveLastBlockAuditToDatabase(blockHeight: number) {
+    const query = `UPDATE state SET number = ? WHERE name = 'last_bitcoin_block_audit'`;
+    await DB.query(query, [blockHeight]);
+  }
+
+  // Get the bitcoin block where the audit process was last updated
+  protected async $getAuditProgress(): Promise<any> {
+    const lastblockaudit = await this.$getLastBlockAudit();
+    const bitcoinBlocksToSync = await this.$getBitcoinBlockchainState();
+    return {
+      lastBlockAudit: lastblockaudit,
+      confirmedTip: bitcoinBlocksToSync.bitcoinBlocks - auditBlockOffsetWithTip,
+    };
+  }
+
+  // Get the bitcoin blocks remaining to be synced
+  protected async $getBitcoinBlockchainState(): Promise<any> {
+    const result = await bitcoinSecondClient.getBlockchainInfo();
+    return {
+      bitcoinBlocks: result.blocks,
+      bitcoinHeaders: result.headers,
+    }
+  }
+
+  protected async $getLastBlockAudit(): Promise<number> {
+    const query = `SELECT number FROM state WHERE name = 'last_bitcoin_block_audit'`;
+    const [rows] = await DB.query(query);
+    return rows[0]['number'];
+  }
+
+  protected async $getRedeemAddressesToScan(): Promise<any[]> {
+    const query = `SELECT datetime, amount, bitcoinaddress FROM elements_pegs where amount < 0 AND bitcoinaddress != '' AND bitcointxid = '';`;
+    const [rows]: any[] = await DB.query(query);
+    return rows;
+  }
+
+  protected isDust(amount: number, feeRate: number): boolean {
+    return amount <= (450 * feeRate); // A P2WSH 11-of-15 multisig input is around 450 bytes
+  }
+
+  ///////////// DATA QUERY //////////////
+
+  public async $getAuditStatus(): Promise<any> {
+    const lastBlockAudit = await this.$getLastBlockAudit();
+    const bitcoinBlocksToSync = await this.$getBitcoinBlockchainState();
+    return {
+      bitcoinBlocks: bitcoinBlocksToSync.bitcoinBlocks,
+      bitcoinHeaders: bitcoinBlocksToSync.bitcoinHeaders,
+      lastBlockAudit: lastBlockAudit,
+      isAuditSynced: bitcoinBlocksToSync.bitcoinHeaders - bitcoinBlocksToSync.bitcoinBlocks <= 2 && bitcoinBlocksToSync.bitcoinBlocks - lastBlockAudit <= 3,
+    };
+  }
+
+  public async $getPegDataByMonth(): Promise<any> {
+    const query = `SELECT SUM(amount) AS amount, DATE_FORMAT(FROM_UNIXTIME(datetime), '%Y-%m-01') AS date FROM elements_pegs GROUP BY DATE_FORMAT(FROM_UNIXTIME(datetime), '%Y%m')`;
+    const [rows] = await DB.query(query);
+    return rows;
+  }
+
+  public async $getFederationReservesByMonth(): Promise<any> {
+    const query = `
+    SELECT SUM(amount) AS amount, DATE_FORMAT(FROM_UNIXTIME(blocktime), '%Y-%m-01') AS date FROM federation_txos 
+    WHERE
+        (blocktime > UNIX_TIMESTAMP(LAST_DAY(FROM_UNIXTIME(blocktime) - INTERVAL 1 MONTH) + INTERVAL 1 DAY))
+      AND 
+        ((unspent = 1) OR (unspent = 0 AND lasttimeupdate > UNIX_TIMESTAMP(LAST_DAY(FROM_UNIXTIME(blocktime)) + INTERVAL 1 DAY)))
+      AND 
+        (expiredAt = 0 OR expiredAt > UNIX_TIMESTAMP(LAST_DAY(FROM_UNIXTIME(blocktime)) + INTERVAL 1 DAY))
+    GROUP BY 
+        date;`;          
+    const [rows] = await DB.query(query);
+    return rows;
+  }
+
+  // Get the current L-BTC pegs and the last Liquid block it was updated
+  public async $getCurrentLbtcSupply(): Promise<any> {
+    const [rows] = await DB.query(`SELECT SUM(amount) AS LBTC_supply FROM elements_pegs;`);
+    const lastblockupdate = await this.$getLatestBlockHeightFromDatabase();
+    const hash = await bitcoinClient.getBlockHash(lastblockupdate);
+    return {
+      amount: rows[0]['LBTC_supply'],
+      lastBlockUpdate: lastblockupdate,
+      hash: hash
+    };
+  }
+
+  // Get the current reserves of the federation and the last Bitcoin block it was updated
+  public async $getCurrentFederationReserves(): Promise<any> {
+    const [rows] = await DB.query(`SELECT SUM(amount) AS total_balance FROM federation_txos WHERE unspent = 1 AND expiredAt = 0;`);
+    const lastblockaudit = await this.$getLastBlockAudit();
+    const hash = await bitcoinSecondClient.getBlockHash(lastblockaudit);
+    return {
+      amount: rows[0]['total_balance'],
+      lastBlockUpdate: lastblockaudit,
+      hash: hash
+    };
+  }
+
+  // Get all of the federation addresses, most balances first
+  public async $getFederationAddresses(): Promise<any> {
+    const query = `SELECT bitcoinaddress, SUM(amount) AS balance FROM federation_txos WHERE unspent = 1 AND expiredAt = 0 GROUP BY bitcoinaddress ORDER BY balance DESC;`;
+    const [rows] = await DB.query(query);
+    return rows;
+  }
+
+  // Get all of the UTXOs held by the federation, most recent first
+  public async $getFederationUtxos(): Promise<any> {
+    const query = `SELECT txid, txindex, bitcoinaddress, amount, blocknumber, blocktime, pegtxid, pegindex, pegblocktime, timelock, expiredAt FROM federation_txos WHERE unspent = 1 AND expiredAt = 0 ORDER BY blocktime DESC;`;
+    const [rows] = await DB.query(query);
+    return rows;
+  }
+
+  // Get expired UTXOs, most recent first
+  public async $getExpiredUtxos(): Promise<any> {
+    const query = `SELECT txid, txindex, bitcoinaddress, amount, blocknumber, blocktime, pegtxid, pegindex, pegblocktime, timelock, expiredAt FROM federation_txos WHERE unspent = 1 AND expiredAt > 0 ORDER BY blocktime DESC;`;
+    const [rows]: any[] = await DB.query(query);
+    const feeRate = Math.round((await bitcoinSecondClient.estimateSmartFee(1)).feerate * 100000000 / 1000);
+    for (const row of rows) {
+      row.isDust = this.isDust(row.amount, feeRate);
+    }
+    return rows;
+  }
+
+    // Get utxos that were spent using emergency keys
+    public async $getEmergencySpentUtxos(): Promise<any> {
+      const query = `SELECT txid, txindex, bitcoinaddress, amount, blocknumber, blocktime, pegtxid, pegindex, pegblocktime, timelock, expiredAt FROM federation_txos WHERE emergencyKey = 1 ORDER BY blocktime DESC;`;
+      const [rows] = await DB.query(query);
+      return rows;
+    }
+  
+  // Get the total number of federation addresses
+  public async $getFederationAddressesNumber(): Promise<any> {
+    const query = `SELECT COUNT(DISTINCT bitcoinaddress) AS address_count FROM federation_txos WHERE unspent = 1 AND expiredAt = 0;`;
+    const [rows] = await DB.query(query);
     return rows[0];
   }
 
-  protected async $saveLatestBlockToDatabase(blockHeight: number, datetime: number, blockHash: string) {
-    const connection = await DB.pool.getConnection();
-    const query = `UPDATE last_elements_block SET block = ?, datetime = ?, block_hash = ?`;
-    await connection.query<any>(query, [blockHeight, datetime, blockHash]);
-    connection.release();
+  // Get the total number of federation utxos
+  public async $getFederationUtxosNumber(): Promise<any> {
+    const query = `SELECT COUNT(*) AS utxo_count FROM federation_txos WHERE unspent = 1 AND expiredAt = 0;`;
+    const [rows] = await DB.query(query);
+    return rows[0];
+  }
+
+  // Get the total number of emergency spent utxos and their total amount
+  public async $getEmergencySpentUtxosStats(): Promise<any> {
+    const query = `SELECT COUNT(*) AS utxo_count, SUM(amount) AS total_amount FROM federation_txos WHERE emergencyKey = 1;`;
+    const [rows] = await DB.query(query);
+    return rows[0];
+  }
+
+  // Get recent pegs in / out
+  public async $getPegsList(count: number = 0): Promise<any> {
+    const query = `SELECT txid, txindex, amount, bitcoinaddress, bitcointxid, bitcoinindex, datetime AS blocktime FROM elements_pegs ORDER BY block DESC LIMIT 15 OFFSET ?;`;
+    const [rows] = await DB.query(query, [count]);
+    return rows;
+  }
+
+  // Get all peg in / out from the last month
+  public async $getPegsVolumeDaily(): Promise<any> {
+    const pegInQuery = await DB.query(`SELECT SUM(amount) AS volume, COUNT(*) AS number FROM elements_pegs WHERE amount > 0 and datetime > UNIX_TIMESTAMP(TIMESTAMPADD(DAY, -1, CURRENT_TIMESTAMP()));`);
+    const pegOutQuery = await DB.query(`SELECT SUM(amount) AS volume, COUNT(*) AS number FROM elements_pegs WHERE amount < 0 and datetime > UNIX_TIMESTAMP(TIMESTAMPADD(DAY, -1, CURRENT_TIMESTAMP()));`);
+    return [
+      pegInQuery[0][0],
+      pegOutQuery[0][0]
+    ];
+  }
+
+  // Get the total pegs number
+  public async $getPegsCount(): Promise<any> {
+    const [rows] = await DB.query(`SELECT COUNT(*) AS pegs_count FROM elements_pegs;`);
+    return rows[0];
   }
 }
 

backend/src/api/liquid/icons.ts

@@ -0,0 +1,38 @@
+import * as fs from 'fs';
+import logger from '../../logger';
+
+class Icons {
+  private static FILE_NAME = '/elements/asset_registry_db/icons.json';
+  private iconIds: string[] = [];
+  private icons: { [assetId: string]: string; } = {};
+
+  constructor() {}
+
+  public loadIcons() {
+    if (!fs.existsSync(Icons.FILE_NAME)) {
+      logger.warn(`${Icons.FILE_NAME} does not exist. No Liquid icons loaded.`);
+      return;
+    }
+    const cacheData = fs.readFileSync(Icons.FILE_NAME, 'utf8');
+    this.icons = JSON.parse(cacheData);
+
+    for (const i in this.icons) {
+      this.iconIds.push(i);
+    }
+    logger.debug(`Liquid icons has been loaded.`);
+  }
+
+  public getIconByAssetId(assetId: string): Buffer | undefined {
+    const icon = this.icons[assetId];
+    if (icon) {
+      return Buffer.from(icon, 'base64');
+    }
+  }
+
+  public getAllIconIds() {
+    return this.iconIds;
+  }
+
+}
+
+export default new Icons();

backend/src/api/liquid/liquid.routes.ts

@@ -0,0 +1,260 @@
+import axios from 'axios';
+import { Application, Request, Response } from 'express';
+import config from '../../config';
+import elementsParser from './elements-parser';
+import icons from './icons';
+import { handleError } from '../../utils/api';
+
+class LiquidRoutes {
+  public initRoutes(app: Application) {
+    app
+      .get(config.MEMPOOL.API_URL_PREFIX + 'assets/icons', this.getAllLiquidIcon)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'assets/featured', this.$getAllFeaturedLiquidAssets)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'asset/:assetId/icon', this.getLiquidIcon)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'assets/group/:id', this.$getAssetGroup)
+      ;
+    
+    if (config.DATABASE.ENABLED) {
+      app
+        .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/pegs', this.$getElementsPegs)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/pegs/month', this.$getElementsPegsByMonth)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/pegs/list/:count', this.$getPegsList)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/pegs/volume', this.$getPegsVolumeDaily)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/pegs/count', this.$getPegsCount)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/reserves', this.$getFederationReserves)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/reserves/month', this.$getFederationReservesByMonth)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/reserves/addresses', this.$getFederationAddresses)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/reserves/addresses/total', this.$getFederationAddressesNumber)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/reserves/utxos', this.$getFederationUtxos)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/reserves/utxos/total', this.$getFederationUtxosNumber)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/reserves/utxos/expired', this.$getExpiredUtxos)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/reserves/utxos/emergency-spent', this.$getEmergencySpentUtxos)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/reserves/utxos/emergency-spent/stats', this.$getEmergencySpentUtxosStats)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/reserves/status', this.$getFederationAuditStatus)
+        ;
+    }
+  }
+
+
+  private getLiquidIcon(req: Request, res: Response) {
+    const result = icons.getIconByAssetId(req.params.assetId);
+    if (result) {
+      res.setHeader('content-type', 'image/png');
+      res.setHeader('content-length', result.length);
+      res.send(result);
+    } else {
+      handleError(req, res, 404, 'Asset icon not found');
+    }
+  }
+
+  private getAllLiquidIcon(req: Request, res: Response) {
+    const result = icons.getAllIconIds();
+    if (result) {
+      res.json(result);
+    } else {
+      handleError(req, res, 404, 'Asset icons not found');
+    }
+  }
+
+  private async $getAllFeaturedLiquidAssets(req: Request, res: Response) {
+    try {
+      const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.LIQUID_API}/assets/featured`, { responseType: 'stream', timeout: 10000 });
+      response.data.pipe(res);
+    } catch (e) {
+      res.status(500).end();
+    }
+  }
+
+  private async $getAssetGroup(req: Request, res: Response) {
+    try {
+      const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.LIQUID_API}/assets/group/${parseInt(req.params.id, 10)}`,
+        { responseType: 'stream', timeout: 10000 });
+      response.data.pipe(res);
+    } catch (e) {
+      res.status(500).end();
+    }
+  }
+
+  private async $getElementsPegsByMonth(req: Request, res: Response) {
+    try {
+      const pegs = await elementsParser.$getPegDataByMonth();
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60 * 60).toUTCString());
+      res.json(pegs);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get pegs by month');
+    }
+  }
+
+  private async $getFederationReservesByMonth(req: Request, res: Response) {
+    try {
+      const reserves = await elementsParser.$getFederationReservesByMonth();
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60 * 60).toUTCString());
+      res.json(reserves);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get reserves by month');
+    }
+  }
+
+  private async $getElementsPegs(req: Request, res: Response) {
+    try {
+      const currentSupply = await elementsParser.$getCurrentLbtcSupply();
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
+      res.json(currentSupply);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get pegs');
+    }
+  }
+
+  private async $getFederationReserves(req: Request, res: Response) {
+    try {
+      const currentReserves = await elementsParser.$getCurrentFederationReserves();
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
+      res.json(currentReserves);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get reserves');
+    }
+  }
+
+  private async $getFederationAuditStatus(req: Request, res: Response) {
+    try {
+      const auditStatus = await elementsParser.$getAuditStatus();
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
+      res.json(auditStatus);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get federation audit status');
+    }
+  }
+
+  private async $getFederationAddresses(req: Request, res: Response) {
+    try {
+      const federationAddresses = await elementsParser.$getFederationAddresses();
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
+      res.json(federationAddresses);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get federation addresses');
+    }
+  }
+
+  private async $getFederationAddressesNumber(req: Request, res: Response) {
+    try {
+      const federationAddresses = await elementsParser.$getFederationAddressesNumber();
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
+      res.json(federationAddresses);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get federation addresses');
+    }
+  }
+
+  private async $getFederationUtxos(req: Request, res: Response) {
+    try {
+      const federationUtxos = await elementsParser.$getFederationUtxos();
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
+      res.json(federationUtxos);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get federation utxos');
+    }
+  }
+
+  private async $getExpiredUtxos(req: Request, res: Response) {
+    try {
+      const expiredUtxos = await elementsParser.$getExpiredUtxos();
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
+      res.json(expiredUtxos);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get expired utxos');
+    }
+  }
+
+  private async $getFederationUtxosNumber(req: Request, res: Response) {
+    try {
+      const federationUtxos = await elementsParser.$getFederationUtxosNumber();
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
+      res.json(federationUtxos);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get federation utxos number');
+    }
+  }
+
+  private async $getEmergencySpentUtxos(req: Request, res: Response) {
+    try {
+      const emergencySpentUtxos = await elementsParser.$getEmergencySpentUtxos();
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
+      res.json(emergencySpentUtxos);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get emergency spent utxos');
+    }
+  }
+
+  private async $getEmergencySpentUtxosStats(req: Request, res: Response) {
+    try {
+      const emergencySpentUtxos = await elementsParser.$getEmergencySpentUtxosStats();
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
+      res.json(emergencySpentUtxos);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get emergency spent utxos stats');
+    }
+  }
+
+  private async $getPegsList(req: Request, res: Response) {
+    try {
+      const recentPegs = await elementsParser.$getPegsList(parseInt(req.params?.count));
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
+      res.json(recentPegs);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get pegs list');
+    }
+  }
+
+  private async $getPegsVolumeDaily(req: Request, res: Response) {
+    try {
+      const pegsVolume = await elementsParser.$getPegsVolumeDaily();
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
+      res.json(pegsVolume);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get pegs volume daily');
+    }
+  }
+
+  private async $getPegsCount(req: Request, res: Response) {
+    try {
+      const pegsCount = await elementsParser.$getPegsCount();
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
+      res.json(pegsCount);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get pegs count');
+    }
+  }
+
+}
+
+export default new LiquidRoutes();

backend/src/api/loading-indicators.ts

@@ -12,8 +12,8 @@ class LoadingIndicators {
     this.progressChangedCallback = fn;
   }
 
-  public setProgress(name: string, progressPercent: number) {
-    const newProgress = Math.round(progressPercent);
+  public setProgress(name: string, progressPercent: number, rounded: boolean = true) {
+    const newProgress = rounded === true ? Math.round(progressPercent) : progressPercent;
     if (newProgress >= 100) {
       delete this.loadingIndicators[name];
     } else {

backend/src/api/memory-cache.ts

@@ -31,7 +31,7 @@ class MemoryCache {
   }
 
   private cleanup() {
-    this.cache = this.cache.filter((cache) => cache.expires < (new Date()));
+    this.cache = this.cache.filter((cache) => cache.expires > (new Date()));
   }
 }
 

backend/src/api/mempool-blocks.ts

@@ -1,12 +1,28 @@
+import { GbtGenerator, GbtResult, ThreadTransaction as RustThreadTransaction, ThreadAcceleration as RustThreadAcceleration } from 'rust-gbt';
 import logger from '../logger';
-import { MempoolBlock, TransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces';
-import { Common } from './common';
+import { MempoolBlock, MempoolTransactionExtended, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats, TransactionClassified, TransactionCompressed, MempoolDeltaChange, GbtCandidates, PoolTag } from '../mempool.interfaces';
+import { Common, OnlineFeeStatsCalculator } from './common';
 import config from '../config';
+import { Worker } from 'worker_threads';
+import path from 'path';
+import mempool from './mempool';
+import { Acceleration } from './services/acceleration';
+import PoolsRepository from '../repositories/PoolsRepository';
+
+const MAX_UINT32 = Math.pow(2, 32) - 1;
 
 class MempoolBlocks {
   private mempoolBlocks: MempoolBlockWithTransactions[] = [];
+  private mempoolBlockDeltas: MempoolBlockDelta[] = [];
+  private txSelectionWorker: Worker | null = null;
+  private rustInitialized: boolean = false;
+  private rustGbtGenerator: GbtGenerator = new GbtGenerator(config.MEMPOOL.BLOCK_WEIGHT_UNITS, config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT);
 
-  constructor() {}
+  private nextUid: number = 1;
+  private uidMap: Map<number, string> = new Map(); // map short numerical uids to full txids
+  private txidMap: Map<string, number> = new Map(); // map full txids back to short numerical uids
+
+  private pools: { [id: number]: PoolTag } = {};
 
   public getMempoolBlocks(): MempoolBlock[] {
     return this.mempoolBlocks.map((block) => {
@@ -25,95 +41,765 @@ class MempoolBlocks {
     return this.mempoolBlocks;
   }
 
-  public updateMempoolBlocks(memPool: { [txid: string]: TransactionExtended }): void {
-    const latestMempool = memPool;
-    const memPoolArray: TransactionExtended[] = [];
-    for (const i in latestMempool) {
-      if (latestMempool.hasOwnProperty(i)) {
-        memPoolArray.push(latestMempool[i]);
-      }
-    }
-    const start = new Date().getTime();
-
-    // Clear bestDescendants & ancestors
-    memPoolArray.forEach((tx) => {
-      tx.bestDescendant = null;
-      tx.ancestors = [];
-      tx.cpfpChecked = false;
-      if (!tx.effectiveFeePerVsize) {
-        tx.effectiveFeePerVsize = tx.feePerVsize;
-      }
-    });
-
-    // First sort
-    memPoolArray.sort((a, b) => b.feePerVsize - a.feePerVsize);
-
-    // Loop through and traverse all ancestors and sum up all the sizes + fees
-    // Pass down size + fee to all unconfirmed children
-    let sizes = 0;
-    memPoolArray.forEach((tx, i) => {
-      sizes += tx.weight;
-      if (sizes > 4000000 * 8) {
-        return;
-      }
-      Common.setRelativesAndGetCpfpInfo(tx, memPool);
-    });
-
-    // Final sort, by effective fee
-    memPoolArray.sort((a, b) => b.effectiveFeePerVsize - a.effectiveFeePerVsize);
-
-    const end = new Date().getTime();
-    const time = end - start;
-    logger.debug('Mempool blocks calculated in ' + time / 1000 + ' seconds');
-
-    this.mempoolBlocks = this.calculateMempoolBlocks(memPoolArray);
+  public getMempoolBlockDeltas(): MempoolBlockDelta[] {
+    return this.mempoolBlockDeltas;
   }
 
-  private calculateMempoolBlocks(transactionsSorted: TransactionExtended[]): MempoolBlockWithTransactions[] {
-    const mempoolBlocks: MempoolBlockWithTransactions[] = [];
-    let blockWeight = 0;
-    let blockSize = 0;
-    let transactions: TransactionExtended[] = [];
-    transactionsSorted.forEach((tx) => {
-      if (blockWeight + tx.weight <= config.MEMPOOL.BLOCK_WEIGHT_UNITS
-        || mempoolBlocks.length === config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT - 1) {
-        blockWeight += tx.weight;
-        blockSize += tx.size;
-        transactions.push(tx);
-      } else {
-        mempoolBlocks.push(this.dataToMempoolBlocks(transactions, blockSize, blockWeight, mempoolBlocks.length));
-        blockWeight = tx.weight;
-        blockSize = tx.size;
-        transactions = [tx];
-      }
-    });
-    if (transactions.length) {
-      mempoolBlocks.push(this.dataToMempoolBlocks(transactions, blockSize, blockWeight, mempoolBlocks.length));
+  public async updatePools$(): Promise<void> {
+    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
+      this.pools = {};
+      return;
     }
+    const allPools = await PoolsRepository.$getPools();
+    this.pools = {};
+    for (const pool of allPools) {
+      this.pools[pool.uniqueId] = pool;
+    }
+  }
+
+  private calculateMempoolDeltas(prevBlocks: MempoolBlockWithTransactions[], mempoolBlocks: MempoolBlockWithTransactions[]): MempoolBlockDelta[] {
+    const mempoolBlockDeltas: MempoolBlockDelta[] = [];
+    for (let i = 0; i < Math.max(mempoolBlocks.length, prevBlocks.length); i++) {
+      let added: TransactionClassified[] = [];
+      let removed: string[] = [];
+      const changed: TransactionClassified[] = [];
+      if (mempoolBlocks[i] && !prevBlocks[i]) {
+        added = mempoolBlocks[i].transactions;
+      } else if (!mempoolBlocks[i] && prevBlocks[i]) {
+        removed = prevBlocks[i].transactions.map(tx => tx.txid);
+      } else if (mempoolBlocks[i] && prevBlocks[i]) {
+        const prevIds = {};
+        const newIds = {};
+        prevBlocks[i].transactions.forEach(tx => {
+          prevIds[tx.txid] = tx;
+        });
+        mempoolBlocks[i].transactions.forEach(tx => {
+          newIds[tx.txid] = true;
+        });
+        prevBlocks[i].transactions.forEach(tx => {
+          if (!newIds[tx.txid]) {
+            removed.push(tx.txid);
+          }
+        });
+        mempoolBlocks[i].transactions.forEach(tx => {
+          if (!prevIds[tx.txid]) {
+            added.push(tx);
+          } else if (tx.rate !== prevIds[tx.txid].rate || tx.acc !== prevIds[tx.txid].acc) {
+            changed.push(tx);
+          }
+        });
+      }
+      mempoolBlockDeltas.push({
+        added: added.map(this.compressTx),
+        removed,
+        changed: changed.map(this.compressDeltaChange),
+      });
+    }
+    return mempoolBlockDeltas;
+  }
+
+  public async $makeBlockTemplates(transactions: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, candidates: GbtCandidates | undefined, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
+    const start = Date.now();
+
+    // reset mempool short ids
+    if (saveResults) {
+      this.resetUids();
+    }
+    // set missing short ids
+    for (const txid of transactions) {
+      const tx = newMempool[txid];
+      this.setUid(tx, !saveResults);
+    }
+
+    const accelerations = useAccelerations ? mempool.getAccelerations() : {};
+
+    // prepare a stripped down version of the mempool with only the minimum necessary data
+    // to reduce the overhead of passing this data to the worker thread
+    const strippedMempool: Map<number, CompactThreadTransaction> = new Map();
+    for (const txid of transactions) {
+      const entry = newMempool[txid];
+      if (entry.uid !== null && entry.uid !== undefined) {
+        const stripped = {
+          uid: entry.uid,
+          fee: entry.fee + (useAccelerations && (!accelerationPool || accelerations[entry.txid]?.pools?.includes(accelerationPool)) ? (accelerations[entry.txid]?.feeDelta || 0) : 0),
+          weight: (entry.adjustedVsize * 4),
+          sigops: entry.sigops,
+          feePerVsize: entry.adjustedFeePerVsize || entry.feePerVsize,
+          effectiveFeePerVsize: entry.effectiveFeePerVsize || entry.adjustedFeePerVsize || entry.feePerVsize,
+          inputs: entry.vin.map(v => this.getUid(newMempool[v.txid])).filter(uid => (uid !== null && uid !== undefined)) as number[],
+        };
+        strippedMempool.set(entry.uid, stripped);
+      }
+    }
+
+    // (re)initialize tx selection worker thread
+    if (!this.txSelectionWorker) {
+      this.txSelectionWorker = new Worker(path.resolve(__dirname, './tx-selection-worker.js'));
+      // if the thread throws an unexpected error, or exits for any other reason,
+      // reset worker state so that it will be re-initialized on the next run
+      this.txSelectionWorker.once('error', () => {
+        this.txSelectionWorker = null;
+      });
+      this.txSelectionWorker.once('exit', () => {
+        this.txSelectionWorker = null;
+      });
+    }
+
+    // run the block construction algorithm in a separate thread, and wait for a result
+    let threadErrorListener;
+    try {
+      const workerResultPromise = new Promise<{ blocks: number[][], rates: Map<number, number>, clusters: Map<number, number[]> }>((resolve, reject) => {
+        threadErrorListener = reject;
+        this.txSelectionWorker?.once('message', (result): void => {
+          resolve(result);
+        });
+        this.txSelectionWorker?.once('error', reject);
+      });
+      this.txSelectionWorker.postMessage({ type: 'set', mempool: strippedMempool });
+      const { blocks, rates, clusters } = this.convertResultTxids(await workerResultPromise);
+
+      // clean up thread error listener
+      this.txSelectionWorker?.removeListener('error', threadErrorListener);
+
+      const processed = this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), candidates, accelerations, accelerationPool, saveResults);
+
+      logger.debug(`makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`);
+
+      return processed;
+    } catch (e) {
+      logger.err('makeBlockTemplates failed. ' + (e instanceof Error ? e.message : e));
+    }
+    return this.mempoolBlocks;
+  }
+
+  public async $updateBlockTemplates(transactions: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], candidates: GbtCandidates | undefined, accelerationDelta: string[] = [], saveResults: boolean = false, useAccelerations: boolean = false): Promise<void> {
+    if (!this.txSelectionWorker) {
+      // need to reset the worker
+      await this.$makeBlockTemplates(transactions, newMempool, candidates, saveResults, useAccelerations);
+      return;
+    }
+
+    const start = Date.now();
+
+    const accelerations = useAccelerations ? mempool.getAccelerations() : {};
+    const addedAndChanged: MempoolTransactionExtended[] = useAccelerations ? accelerationDelta.map(txid => newMempool[txid]).filter(tx => tx != null).concat(added) : added;
+
+    for (const tx of addedAndChanged) {
+      this.setUid(tx, false);
+    }
+    const removedTxs = removed.filter(tx => tx.uid != null) as MempoolTransactionExtended[];
+
+    // prepare a stripped down version of the mempool with only the minimum necessary data
+    // to reduce the overhead of passing this data to the worker thread
+    const addedStripped: CompactThreadTransaction[] = addedAndChanged.filter(entry => entry.uid != null).map(entry => {
+      return {
+        uid: entry.uid || 0,
+        fee: entry.fee + (useAccelerations ? (accelerations[entry.txid]?.feeDelta || 0) : 0),
+        weight: (entry.adjustedVsize * 4),
+        sigops: entry.sigops,
+        feePerVsize: entry.adjustedFeePerVsize || entry.feePerVsize,
+        effectiveFeePerVsize: entry.effectiveFeePerVsize || entry.adjustedFeePerVsize || entry.feePerVsize,
+        inputs: entry.vin.map(v => this.getUid(newMempool[v.txid])).filter(uid => (uid !== null && uid !== undefined)) as number[],
+      };
+    });
+
+    // run the block construction algorithm in a separate thread, and wait for a result
+    let threadErrorListener;
+    try {
+      const workerResultPromise = new Promise<{ blocks: number[][], rates: Map<number, number>, clusters: Map<number, number[]> }>((resolve, reject) => {
+        threadErrorListener = reject;
+        this.txSelectionWorker?.once('message', (result): void => {
+          resolve(result);
+        });
+        this.txSelectionWorker?.once('error', reject);
+      });
+      this.txSelectionWorker.postMessage({ type: 'update', added: addedStripped, removed: removedTxs.map(tx => tx.uid) as number[] });
+      const { blocks, rates, clusters } = this.convertResultTxids(await workerResultPromise);
+
+      this.removeUids(removedTxs);
+
+      // clean up thread error listener
+      this.txSelectionWorker?.removeListener('error', threadErrorListener);
+
+      this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), candidates, accelerations, null, saveResults);
+      logger.debug(`updateBlockTemplates completed in ${(Date.now() - start) / 1000} seconds`);
+    } catch (e) {
+      logger.err('updateBlockTemplates failed. ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  private resetRustGbt(): void {
+    this.rustInitialized = false;
+    this.rustGbtGenerator = new GbtGenerator(config.MEMPOOL.BLOCK_WEIGHT_UNITS, config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT);
+  }
+
+  public async $rustMakeBlockTemplates(txids: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, candidates: GbtCandidates | undefined, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
+    const start = Date.now();
+
+    // reset mempool short ids
+    if (saveResults) {
+      this.resetUids();
+    }
+
+    const transactions = txids.map(txid => newMempool[txid]).filter(tx => tx != null);
+    // set missing short ids
+    for (const tx of transactions) {
+      this.setUid(tx, !saveResults);
+    }
+    // set short ids for transaction inputs
+    for (const tx of transactions) {
+      tx.inputs = tx.vin.map(v => this.getUid(newMempool[v.txid])).filter(uid => (uid !== null && uid !== undefined)) as number[];
+    }
+
+    const accelerations = useAccelerations ? mempool.getAccelerations() : {};
+    const acceleratedList = accelerationPool ? Object.values(accelerations).filter(acc => newMempool[acc.txid] && acc.pools.includes(accelerationPool)) : Object.values(accelerations).filter(acc => newMempool[acc.txid]);
+    const convertedAccelerations = acceleratedList.map(acc => {
+      this.setUid(newMempool[acc.txid], true);
+      return {
+        uid: this.getUid(newMempool[acc.txid]),
+        delta: acc.feeDelta,
+      };
+    });
+
+    // run the block construction algorithm in a separate thread, and wait for a result
+    const rustGbt = saveResults ? this.rustGbtGenerator : new GbtGenerator(config.MEMPOOL.BLOCK_WEIGHT_UNITS, config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT);
+    try {
+      const { blocks, blockWeights, rates, clusters, overflow } = this.convertNapiResultTxids(
+        await rustGbt.make(transactions as RustThreadTransaction[], convertedAccelerations as RustThreadAcceleration[], this.nextUid),
+      );
+      if (saveResults) {
+        this.rustInitialized = true;
+      }
+      const expectedSize = transactions.length;
+      const resultMempoolSize = blocks.reduce((total, block) => total + block.length, 0) + overflow.length;
+      logger.debug(`RUST updateBlockTemplates returned ${resultMempoolSize} txs out of ${expectedSize} in the mempool, ${overflow.length} were unmineable`);
+      const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, candidates, accelerations, accelerationPool, saveResults);
+      logger.debug(`RUST makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`);
+      return processed;
+    } catch (e) {
+      logger.err('RUST makeBlockTemplates failed. ' + (e instanceof Error ? e.message : e));
+      if (saveResults) {
+        this.resetRustGbt();
+      }
+    }
+    return this.mempoolBlocks;
+  }
+
+  public async $oneOffRustBlockTemplates(transactions: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, candidates: GbtCandidates | undefined, useAccelerations: boolean, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
+    return this.$rustMakeBlockTemplates(transactions, newMempool, candidates, false, useAccelerations, accelerationPool);
+  }
+
+  public async $rustUpdateBlockTemplates(transactions: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], candidates: GbtCandidates | undefined, useAccelerations: boolean, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
+    // GBT optimization requires that uids never get too sparse
+    // as a sanity check, we should also explicitly prevent uint32 uid overflow
+    if (this.nextUid + added.length >= Math.min(Math.max(262144, 2 * transactions.length), MAX_UINT32)) {
+      this.resetRustGbt();
+    }
+
+    if (!this.rustInitialized) {
+      // need to reset the worker
+      return this.$rustMakeBlockTemplates(transactions, newMempool, candidates, true, useAccelerations, accelerationPool);
+    }
+
+    const start = Date.now();
+    // set missing short ids
+    for (const tx of added) {
+      this.setUid(tx, false);
+    }
+    // set short ids for transaction inputs
+    for (const tx of added) {
+      tx.inputs = tx.vin.map(v => this.getUid(newMempool[v.txid])).filter(uid => (uid !== null && uid !== undefined)) as number[];
+    }
+    const removedTxs = removed.filter(tx => tx.uid != null) as MempoolTransactionExtended[];
+
+    const accelerations = useAccelerations ? mempool.getAccelerations() : {};
+    const acceleratedList = accelerationPool ? Object.values(accelerations).filter(acc => newMempool[acc.txid] && acc.pools.includes(accelerationPool)) : Object.values(accelerations).filter(acc => newMempool[acc.txid]);
+    const convertedAccelerations = acceleratedList.map(acc => {
+      this.setUid(newMempool[acc.txid], true);
+      return {
+        uid: this.getUid(newMempool[acc.txid]),
+        delta: acc.feeDelta,
+      };
+    });
+
+    // run the block construction algorithm in a separate thread, and wait for a result
+    try {
+      const { blocks, blockWeights, rates, clusters, overflow } = this.convertNapiResultTxids(
+        await this.rustGbtGenerator.update(
+          added as RustThreadTransaction[],
+          removedTxs.map(tx => tx.uid) as number[],
+          convertedAccelerations as RustThreadAcceleration[],
+          this.nextUid,
+        ),
+      );
+      const resultMempoolSize = blocks.reduce((total, block) => total + block.length, 0) + overflow.length;
+      logger.debug(`RUST updateBlockTemplates returned ${resultMempoolSize} txs out of ${transactions.length} candidates, ${overflow.length} were unmineable`);
+      if (transactions.length !== resultMempoolSize) {
+        throw new Error(`GBT returned wrong number of transactions ${transactions.length} vs ${resultMempoolSize}, cache is probably out of sync`);
+      } else {
+        const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, candidates, accelerations, accelerationPool, true);
+        this.removeUids(removedTxs);
+        logger.debug(`RUST updateBlockTemplates completed in ${(Date.now() - start)/1000} seconds`);
+        return processed;
+      }
+    } catch (e) {
+      logger.err('RUST updateBlockTemplates failed. ' + (e instanceof Error ? e.message : e));
+      this.resetRustGbt();
+      return this.mempoolBlocks;
+    }
+  }
+
+  private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], candidates: GbtCandidates | undefined, accelerations: { [txid: string]: Acceleration }, accelerationPool, saveResults): MempoolBlockWithTransactions[] {
+    for (const txid of Object.keys(candidates?.txs ?? mempool)) {
+      if (txid in mempool) {
+        mempool[txid].cpfpDirty = false;
+        mempool[txid].ancestors = [];
+        mempool[txid].descendants = [];
+        mempool[txid].bestDescendant = null;
+      }
+    }
+    for (const [txid, rate] of rates) {
+      if (txid in mempool) {
+        mempool[txid].cpfpDirty = (rate !== mempool[txid].effectiveFeePerVsize);
+        mempool[txid].effectiveFeePerVsize = rate;
+        mempool[txid].cpfpChecked = true;
+      }
+    }
+
+    const lastBlockIndex = blocks.length - 1;
+    let hasBlockStack = blocks.length >= 8;
+    let stackWeight;
+    let feeStatsCalculator: OnlineFeeStatsCalculator | null = null;
+    if (hasBlockStack) {
+      if (blockWeights && blockWeights[7] !== null) {
+        stackWeight = blockWeights[7];
+      } else {
+        stackWeight = blocks[lastBlockIndex].reduce((total, tx) => total + (mempool[tx]?.weight || 0), 0);
+      }
+      hasBlockStack = stackWeight > config.MEMPOOL.BLOCK_WEIGHT_UNITS;
+      feeStatsCalculator = new OnlineFeeStatsCalculator(stackWeight, 0.5, [10, 20, 30, 40, 50, 60, 70, 80, 90]);
+    }
+
+    const ancestors: Ancestor[] = [];
+    const descendants: Ancestor[] = [];
+    let ancestor: MempoolTransactionExtended;
+    for (const cluster of clusters) {
+      for (const memberTxid of cluster) {
+        const mempoolTx = mempool[memberTxid];
+        if (mempoolTx) {
+          // ugly micro-optimization to avoid allocating new arrays
+          ancestors.length = 0;
+          descendants.length = 0;
+          let matched = false;
+          cluster.forEach(txid => {
+            ancestor = mempool[txid];
+            if (txid === memberTxid) {
+              matched = true;
+            } else {
+              if (!ancestor) {
+                console.log('txid missing from mempool! ', txid, candidates?.txs[txid]);
+                return;
+              }
+              const relative = {
+                txid: txid,
+                fee: ancestor.fee,
+                weight: (ancestor.adjustedVsize * 4),
+              };
+              if (matched) {
+                descendants.push(relative);
+                if (!mempoolTx.lastBoosted || (ancestor.firstSeen && ancestor.firstSeen > mempoolTx.lastBoosted)) {
+                  mempoolTx.lastBoosted = ancestor.firstSeen;
+                }
+              } else {
+                ancestors.push(relative);
+              }
+            }
+          });
+          if (mempoolTx.ancestors?.length !== ancestors.length || mempoolTx.descendants?.length !== descendants.length) {
+            mempoolTx.cpfpDirty = true;
+          }
+          // ugly micro-optimization to avoid allocating new arrays or objects
+          if (mempoolTx.ancestors) {
+            mempoolTx.ancestors.length = 0;
+          } else {
+            mempoolTx.ancestors = [];
+          }
+          if (mempoolTx.descendants) {
+            mempoolTx.descendants.length = 0;
+          } else {
+            mempoolTx.descendants = [];
+          }
+          mempoolTx.ancestors.push(...ancestors);
+          mempoolTx.descendants.push(...descendants);
+          mempoolTx.cpfpChecked = true;
+        }
+      }
+    }
+
+    const isAcceleratedBy : { [txid: string]: number[] | false } = {};
+
+    const sizeLimit = (config.MEMPOOL.BLOCK_WEIGHT_UNITS / 4) * 1.2;
+    // update this thread's mempool with the results
+    let mempoolTx: MempoolTransactionExtended;
+    let acceleration: Acceleration;
+    const mempoolBlocks: MempoolBlockWithTransactions[] = [];
+    for (let blockIndex = 0; blockIndex < blocks.length; blockIndex++) {
+      const block = blocks[blockIndex];
+      let totalSize = 0;
+      let totalVsize = 0;
+      let totalWeight = 0;
+      let totalFees = 0;
+      const transactions: MempoolTransactionExtended[] = [];
+
+      // backfill purged transactions
+      if (candidates?.txs && blockIndex === blocks.length - 1) {
+        for (const txid of Object.keys(mempool)) {
+          if (!candidates.txs[txid]) {
+            block.push(txid);
+          }
+        }
+      }
+
+      for (let i = 0; i < block.length; i++) {
+        const txid = block[i];
+        if (txid in mempool) {
+          mempoolTx = mempool[txid];
+          // save position in projected blocks
+          mempoolTx.position = {
+            block: blockIndex,
+            vsize: totalVsize + (mempoolTx.vsize / 2),
+          };
+
+          if (txid in accelerations) {
+            acceleration = accelerations[txid];
+            if (isAcceleratedBy[txid] || (acceleration && (!accelerationPool || acceleration.pools.includes(accelerationPool)))) {
+              if (!mempoolTx.acceleration) {
+                mempoolTx.cpfpDirty = true;
+              }
+              mempoolTx.acceleration = true;
+              mempoolTx.acceleratedBy = isAcceleratedBy[txid] || acceleration?.pools;
+              mempoolTx.acceleratedAt = acceleration?.added;
+              mempoolTx.feeDelta = acceleration?.feeDelta;
+              for (const ancestor of mempoolTx.ancestors || []) {
+                if (!(ancestor.txid in mempool)) {
+                  continue;
+                }
+                if (!mempool[ancestor.txid].acceleration) {
+                  mempool[ancestor.txid].cpfpDirty = true;
+                }
+                mempool[ancestor.txid].acceleration = true;
+                mempool[ancestor.txid].acceleratedBy = mempoolTx.acceleratedBy;
+                mempool[ancestor.txid].acceleratedAt = mempoolTx.acceleratedAt;
+                mempool[ancestor.txid].feeDelta = mempoolTx.feeDelta;
+                isAcceleratedBy[ancestor.txid] = mempoolTx.acceleratedBy;
+              }
+            } else {
+              if (mempoolTx.acceleration) {
+                mempoolTx.cpfpDirty = true;
+                delete mempoolTx.acceleration;
+              }
+            }
+          } else {
+            if (mempoolTx.acceleration) {
+              mempoolTx.cpfpDirty = true;
+              delete mempoolTx.acceleration;
+            }
+          }
+
+          // online calculation of stack-of-blocks fee stats
+          if (hasBlockStack && blockIndex === lastBlockIndex && feeStatsCalculator) {
+            feeStatsCalculator.processNext(mempoolTx);
+          }
+
+          totalSize += mempoolTx.size;
+          totalVsize += mempoolTx.vsize;
+          totalWeight += mempoolTx.weight;
+          totalFees += mempoolTx.fee;
+
+          if (totalVsize <= sizeLimit) {
+            transactions.push(mempoolTx);
+          }
+        }
+      }
+      mempoolBlocks[blockIndex] = this.dataToMempoolBlocks(
+        block,
+        transactions,
+        totalSize,
+        totalWeight,
+        totalFees,
+        (hasBlockStack && blockIndex === lastBlockIndex && feeStatsCalculator) ? feeStatsCalculator.getRawFeeStats() : undefined,
+      );
+    };
+
+    if (saveResults) {
+      const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, mempoolBlocks);
+      this.mempoolBlocks = mempoolBlocks;
+      this.mempoolBlockDeltas = deltas;
+      this.updateAccelerationPositions(mempool, accelerations, mempoolBlocks);
+    }
+
     return mempoolBlocks;
   }
 
-  private dataToMempoolBlocks(transactions: TransactionExtended[],
-    blockSize: number, blockWeight: number, blocksIndex: number): MempoolBlockWithTransactions {
-    let rangeLength = 4;
-    if (blocksIndex === 0) {
-      rangeLength = 8;
-    }
-    if (transactions.length > 4000) {
-      rangeLength = 6;
-    } else if (transactions.length > 10000) {
-      rangeLength = 8;
+  private dataToMempoolBlocks(transactionIds: string[], transactions: MempoolTransactionExtended[], totalSize: number, totalWeight: number, totalFees: number, feeStats?: EffectiveFeeStats ): MempoolBlockWithTransactions {
+    if (!feeStats) {
+      feeStats = Common.calcEffectiveFeeStatistics(transactions);
     }
     return {
-      blockSize: blockSize,
-      blockVSize: blockWeight / 4,
-      nTx: transactions.length,
-      totalFees: transactions.reduce((acc, cur) => acc + cur.fee, 0),
-      medianFee: Common.percentile(transactions.map((tx) => tx.effectiveFeePerVsize), config.MEMPOOL.RECOMMENDED_FEE_PERCENTILE),
-      feeRange: Common.getFeesInRange(transactions, rangeLength),
-      transactionIds: transactions.map((tx) => tx.txid),
+      blockSize: totalSize,
+      blockVSize: (totalWeight / 4), // fractional vsize to avoid rounding errors
+      nTx: transactionIds.length,
+      totalFees: totalFees,
+      medianFee: feeStats.medianFee, // Common.percentile(transactions.map((tx) => tx.effectiveFeePerVsize), config.MEMPOOL.RECOMMENDED_FEE_PERCENTILE),
+      feeRange: feeStats.feeRange, //Common.getFeesInRange(transactions, rangeLength),
+      transactionIds: transactionIds,
+      transactions: transactions.map((tx) => Common.classifyTransaction(tx)),
     };
   }
+
+  private resetUids(): void {
+    this.uidMap.clear();
+    this.txidMap.clear();
+    this.nextUid = 1;
+  }
+
+  private setUid(tx: MempoolTransactionExtended, skipSet = false): number {
+    if (!this.txidMap.has(tx.txid) || !skipSet) {
+      const uid = this.nextUid;
+      this.nextUid++;
+      this.uidMap.set(uid, tx.txid);
+      this.txidMap.set(tx.txid, uid);
+      tx.uid = uid;
+      return uid;
+    } else {
+      tx.uid = this.txidMap.get(tx.txid) as number;
+      return tx.uid;
+    }
+  }
+
+  private getUid(tx: MempoolTransactionExtended): number | void {
+    if (tx) {
+      return this.txidMap.get(tx.txid);
+    }
+  }
+
+  private removeUids(txs: MempoolTransactionExtended[]): void {
+    for (const tx of txs) {
+      const uid = this.txidMap.get(tx.txid);
+      if (uid != null) {
+        this.uidMap.delete(uid);
+        this.txidMap.delete(tx.txid);
+      }
+      tx.uid = undefined;
+    }
+  }
+
+  private convertResultTxids({ blocks, rates, clusters }: { blocks: number[][], rates: Map<number, number>, clusters: Map<number, number[]>})
+    : { blocks: string[][], rates: { [root: string]: number }, clusters: { [root: string]: string[] }} {
+    const convertedBlocks: string[][] = blocks.map(block => block.map(uid => {
+      return this.uidMap.get(uid) || '';
+    }));
+    const convertedRates = {};
+    for (const rateUid of rates.keys()) {
+      const rateTxid = this.uidMap.get(rateUid);
+      if (rateTxid) {
+        convertedRates[rateTxid] = rates.get(rateUid);
+      }
+    }
+    const convertedClusters = {};
+    for (const rootUid of clusters.keys()) {
+      const rootTxid = this.uidMap.get(rootUid);
+      if (rootTxid) {
+        const members = clusters.get(rootUid)?.map(uid => {
+          return this.uidMap.get(uid);
+        });
+        convertedClusters[rootTxid] = members;
+      }
+    }
+    return { blocks: convertedBlocks, rates: convertedRates, clusters: convertedClusters } as { blocks: string[][], rates: { [root: string]: number }, clusters: { [root: string]: string[] }};
+  }
+
+  private convertNapiResultTxids({ blocks, blockWeights, rates, clusters, overflow }: GbtResult)
+    : { blocks: string[][], blockWeights: number[], rates: [string, number][], clusters: string[][], overflow: string[] } {
+    const convertedBlocks: string[][] = blocks.map(block => block.map(uid => {
+      const txid = this.uidMap.get(uid);
+      if (txid !== undefined) {
+        return txid;
+      } else {
+        throw new Error('GBT returned a block containing a transaction with unknown uid');
+      }
+    }));
+    const convertedRates: [string, number][] = [];
+    for (const [rateUid, rate] of rates) {
+      const rateTxid = this.uidMap.get(rateUid) as string;
+      convertedRates.push([rateTxid, rate]);
+    }
+    const convertedClusters: string[][] = [];
+    for (const cluster of clusters) {
+      convertedClusters.push(cluster.map(uid => this.uidMap.get(uid)) as string[]);
+    }
+    const convertedOverflow: string[] = overflow.map(uid => {
+      const txid = this.uidMap.get(uid);
+      if (txid !== undefined) {
+        return txid;
+      } else {
+        throw new Error('GBT returned an unmineable transaction with unknown uid');
+      }
+    });
+    return { blocks: convertedBlocks, blockWeights, rates: convertedRates, clusters: convertedClusters, overflow: convertedOverflow };
+  }
+
+  public compressTx(tx: TransactionClassified): TransactionCompressed {
+    if (tx.acc) {
+      return [
+        tx.txid,
+        tx.fee,
+        tx.vsize,
+        tx.value,
+        Math.round((tx.rate || (tx.fee / tx.vsize)) * 100) / 100,
+        tx.flags,
+        tx.time || 0,
+        1,
+      ];
+    } else {
+      return [
+        tx.txid,
+        tx.fee,
+        tx.vsize,
+        tx.value,
+        Math.round((tx.rate || (tx.fee / tx.vsize)) * 100) / 100,
+        tx.flags,
+        tx.time || 0,
+      ];
+    }
+  }
+
+  public compressDeltaChange(tx: TransactionClassified): MempoolDeltaChange {
+    return [
+      tx.txid,
+      Math.round((tx.rate || (tx.fee / tx.vsize)) * 100) / 100,
+      tx.flags,
+      tx.acc ? 1 : 0,
+    ];
+  }
+
+  // estimates and saves positions of accelerations in mining partner mempools
+  private updateAccelerationPositions(mempoolCache: { [txid: string]: MempoolTransactionExtended }, accelerations: { [txid: string]: Acceleration }, mempoolBlocks: MempoolBlockWithTransactions[]): void {
+    const accelerationPositions: { [txid: string]: { poolId: number, pool: string, block: number, vsize: number }[] } = {};
+    // keep track of simulated mempool blocks for each active pool
+    const pools: {
+      [pool: string]: { name: string, block: number, vsize: number, accelerations: string[], complete: boolean };
+    } = {};
+    // prepare a list of accelerations in ascending order (we'll pop items off the end of the list)
+    const accQueue: { acceleration: Acceleration, rate: number, vsize: number }[] = Object.values(accelerations).filter(acc => acc.txid in mempoolCache).map(acc => {
+      let vsize = mempoolCache[acc.txid].vsize;
+      for (const ancestor of mempoolCache[acc.txid].ancestors || []) {
+        vsize += (ancestor.weight / 4);
+      }
+      return {
+        acceleration: acc,
+        rate: mempoolCache[acc.txid].effectiveFeePerVsize,
+        vsize
+      };
+    }).sort((a, b) => a.rate - b.rate);
+    // initialize the pool tracker
+    for (const { acceleration }  of accQueue) {
+      accelerationPositions[acceleration.txid] = [];
+      for (const pool of acceleration.pools) {
+        if (!pools[pool]) {
+          pools[pool] = {
+            name: this.pools[pool]?.name || 'unknown',
+            block: 0,
+            vsize: 0,
+            accelerations: [],
+            complete: false,
+          };
+        }
+        pools[pool].accelerations.push(acceleration.txid);
+      }
+      for (const ancestor of mempoolCache[acceleration.txid].ancestors || []) {
+        accelerationPositions[ancestor.txid] = [];
+      }
+    }
+
+    for (const pool of Object.keys(pools)) {
+      // if any pools accepted *every* acceleration, we can just use the GBT result positions directly
+      if (pools[pool].accelerations.length === Object.keys(accelerations).length) {
+        pools[pool].complete = true;
+      }
+    }
+
+    let block = 0;
+    let index = 0;
+    let next = accQueue.pop();
+    // build simulated blocks for each pool by taking the best option from
+    // either the mempool or the list of accelerations.
+    while (next && block < mempoolBlocks.length) {
+      while (next && index < mempoolBlocks[block].transactions.length) {
+        const nextTx = mempoolBlocks[block].transactions[index];
+        if (next.rate >= (nextTx.rate || (nextTx.fee / nextTx.vsize))) {
+          for (const pool of next.acceleration.pools) {
+            if (pools[pool].vsize + next.vsize <= 999_000) {
+              pools[pool].vsize += next.vsize;
+            } else {
+              pools[pool].block++;
+              pools[pool].vsize = next.vsize;
+            }
+            // insert the acceleration into matching pool's blocks
+            if (pools[pool].complete && mempoolCache[next.acceleration.txid]?.position !== undefined) {
+              accelerationPositions[next.acceleration.txid].push({
+                ...mempoolCache[next.acceleration.txid].position as { block: number, vsize: number },
+                poolId: pool,
+                pool: pools[pool].name
+              });
+            } else {
+              accelerationPositions[next.acceleration.txid].push({
+                poolId: pool,
+                pool: pools[pool].name,
+                block: pools[pool].block,
+                vsize: pools[pool].vsize - (next.vsize / 2),
+              });
+            }
+            // and any accelerated ancestors
+            for (const ancestor of mempoolCache[next.acceleration.txid].ancestors || []) {
+              if (pools[pool].complete && mempoolCache[ancestor.txid]?.position !== undefined) {
+                accelerationPositions[ancestor.txid].push({
+                  ...mempoolCache[ancestor.txid].position as { block: number, vsize: number },
+                  poolId: pool,
+                  pool: pools[pool].name,
+                });
+              } else {
+                accelerationPositions[ancestor.txid].push({
+                  poolId: pool,
+                  pool: pools[pool].name,
+                  block: pools[pool].block,
+                  vsize: pools[pool].vsize - (next.vsize / 2),
+                });
+              }
+            }
+          }
+          next = accQueue.pop();
+        } else {
+          // skip accelerated transactions and their CPFP ancestors
+          if (accelerationPositions[nextTx.txid] == null) {
+            // insert into all pools' blocks
+            for (const pool of Object.keys(pools)) {
+              if (pools[pool].vsize + nextTx.vsize <= 999_000) {
+                pools[pool].vsize += nextTx.vsize;
+              } else {
+                pools[pool].block++;
+                pools[pool].vsize = nextTx.vsize;
+              }
+            }
+          }
+          index++;
+        }
+      }
+      block++;
+      index = 0;
+    }
+    mempool.setAccelerationPositions(accelerationPositions);
+  }
 }
 
 export default new MempoolBlocks();

backend/src/api/mempool.ts

@@ -1,6 +1,6 @@
 import config from '../config';
 import bitcoinApi from './bitcoin/bitcoin-api-factory';
-import { TransactionExtended, VbytesPerSecond } from '../mempool.interfaces';
+import { MempoolTransactionExtended, TransactionExtended, VbytesPerSecond, GbtCandidates } from '../mempool.interfaces';
 import logger from '../logger';
 import { Common } from './common';
 import transactionUtils from './transaction-utils';
@@ -8,16 +8,28 @@ import { IBitcoinApi } from './bitcoin/bitcoin-api.interface';
 import loadingIndicators from './loading-indicators';
 import bitcoinClient from './bitcoin/bitcoin-client';
 import bitcoinSecondClient from './bitcoin/bitcoin-second-client';
+import rbfCache from './rbf-cache';
+import { Acceleration } from './services/acceleration';
+import accelerationApi from './services/acceleration';
+import redisCache from './redis-cache';
+import blocks from './blocks';
 
 class Mempool {
-  private static WEBSOCKET_REFRESH_RATE_MS = 10000;
-  private static LAZY_DELETE_AFTER_SECONDS = 30;
   private inSync: boolean = false;
-  private mempoolCache: { [txId: string]: TransactionExtended } = {};
-  private mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0,
-                                                    maxmempool: 300000000, mempoolminfee: 0.00001000, minrelaytxfee: 0.00001000 };
-  private mempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[],
-    deletedTransactions: TransactionExtended[]) => void) | undefined;
+  private mempoolCacheDelta: number = -1;
+  private mempoolCache: { [txId: string]: MempoolTransactionExtended } = {};
+  private mempoolCandidates: { [txid: string ]: boolean } = {};
+  private spendMap = new Map<string, MempoolTransactionExtended>();
+  private recentlyDeleted: MempoolTransactionExtended[][] = []; // buffer of transactions deleted in recent mempool updates
+  private mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0, total_fee: 0,
+                                                    maxmempool: 300000000, mempoolminfee: Common.isLiquid() ? 0.00000100 : 0.00001000, minrelaytxfee: Common.isLiquid() ? 0.00000100 : 0.00001000 };
+  private mempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, newTransactions: MempoolTransactionExtended[],
+    deletedTransactions: MempoolTransactionExtended[][], accelerationDelta: string[]) => void) | undefined;
+  private $asyncMempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, mempoolSize: number, newTransactions: MempoolTransactionExtended[],
+    deletedTransactions: MempoolTransactionExtended[][], accelerationDelta: string[], candidates?: GbtCandidates) => Promise<void>) | undefined;
+
+  private accelerations: { [txId: string]: Acceleration } = {};
+  private accelerationPositions: { [txid: string]: { poolId: number, pool: string, block: number, vsize: number }[] } = {};
 
   private txPerSecondArray: number[] = [];
   private txPerSecond: number = 0;
@@ -27,9 +39,27 @@ class Mempool {
   private mempoolProtection = 0;
   private latestTransactions: any[] = [];
 
+  private ESPLORA_MISSING_TX_WARNING_THRESHOLD = 100; 
+  private SAMPLE_TIME = 10000; // In ms
+  private timer = new Date().getTime();
+  private missingTxCount = 0;
+  private mainLoopTimeout: number = 120000;
+
+  public limitGBT = config.MEMPOOL.USE_SECOND_NODE_FOR_MINFEE && config.MEMPOOL.LIMIT_GBT;
+
   constructor() {
     setInterval(this.updateTxPerSecond.bind(this), 1000);
-    setInterval(this.deleteExpiredTransactions.bind(this), 20000);
+  }
+
+  /**
+   * Return true if we should leave resources available for mempool tx caching
+   */
+  public hasPriority(): boolean {
+    if (this.inSync) {
+      return false;
+    } else {
+      return this.mempoolCacheDelta == -1 || this.mempoolCacheDelta > 25;
+    }
   }
 
   public isInSync(): boolean {
@@ -45,20 +75,108 @@ class Mempool {
     return this.latestTransactions;
   }
 
-  public setMempoolChangedCallback(fn: (newMempool: { [txId: string]: TransactionExtended; },
-    newTransactions: TransactionExtended[], deletedTransactions: TransactionExtended[]) => void) {
+  public setMempoolChangedCallback(fn: (newMempool: { [txId: string]: MempoolTransactionExtended; },
+    newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[][], accelerationDelta: string[]) => void): void {
     this.mempoolChangedCallback = fn;
   }
 
-  public getMempool(): { [txid: string]: TransactionExtended } {
+  public setAsyncMempoolChangedCallback(fn: (newMempool: { [txId: string]: MempoolTransactionExtended; }, mempoolSize: number,
+    newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[][], accelerationDelta: string[],
+    candidates?: GbtCandidates) => Promise<void>): void {
+    this.$asyncMempoolChangedCallback = fn;
+  }
+
+  public getMempool(): { [txid: string]: MempoolTransactionExtended } {
     return this.mempoolCache;
   }
 
-  public setMempool(mempoolData: { [txId: string]: TransactionExtended }) {
+  public getSpendMap(): Map<string, MempoolTransactionExtended> {
+    return this.spendMap;
+  }
+
+  public getFromSpendMap(txid, index): MempoolTransactionExtended | void {
+    return this.spendMap.get(`${txid}:${index}`);
+  }
+
+  public async $setMempool(mempoolData: { [txId: string]: MempoolTransactionExtended }) {
     this.mempoolCache = mempoolData;
-    if (this.mempoolChangedCallback) {
-      this.mempoolChangedCallback(this.mempoolCache, [], []);
+    let count = 0;
+    const redisTimer = Date.now();
+    if (config.MEMPOOL.CACHE_ENABLED && config.REDIS.ENABLED) {
+      logger.debug(`Migrating ${Object.keys(this.mempoolCache).length} transactions from disk cache to Redis cache`);
     }
+    for (const txid of Object.keys(this.mempoolCache)) {
+      if (!this.mempoolCache[txid].adjustedVsize || this.mempoolCache[txid].sigops == null || this.mempoolCache[txid].effectiveFeePerVsize == null) {
+        this.mempoolCache[txid] = transactionUtils.extendMempoolTransaction(this.mempoolCache[txid]);
+      }
+      if (this.mempoolCache[txid].order == null) {
+        this.mempoolCache[txid].order = transactionUtils.txidToOrdering(txid);
+      }
+      for (const vin of this.mempoolCache[txid].vin) {
+        transactionUtils.addInnerScriptsToVin(vin);
+      }
+      count++;
+      if (config.MEMPOOL.CACHE_ENABLED && config.REDIS.ENABLED) {
+        await redisCache.$addTransaction(this.mempoolCache[txid]);
+      }
+      this.mempoolCache[txid].flags = Common.getTransactionFlags(this.mempoolCache[txid]);
+      this.mempoolCache[txid].cpfpChecked = false;
+      this.mempoolCache[txid].cpfpDirty = true;
+      this.mempoolCache[txid].cpfpUpdated = undefined;
+    }
+    if (config.MEMPOOL.CACHE_ENABLED && config.REDIS.ENABLED) {
+      await redisCache.$flushTransactions();
+      logger.debug(`Finished migrating cache transactions in ${((Date.now() - redisTimer) / 1000).toFixed(2)} seconds`);
+    }
+    if (this.mempoolChangedCallback) {
+      this.mempoolChangedCallback(this.mempoolCache, [], [], []);
+    }
+    if (this.$asyncMempoolChangedCallback) {
+      await this.$asyncMempoolChangedCallback(this.mempoolCache, count, [], [], [], this.limitGBT ? { txs: {}, added: [], removed: [] } : undefined);
+    }
+    this.addToSpendMap(Object.values(this.mempoolCache));
+  }
+
+  public async $reloadMempool(expectedCount: number): Promise<MempoolTransactionExtended[]> {
+    let count = 0;
+    let done = false;
+    let last_txid;
+    const newTransactions: MempoolTransactionExtended[] = [];
+    loadingIndicators.setProgress('mempool', count / expectedCount * 100);
+    while (!done) {
+      try {
+        const result = await bitcoinApi.$getAllMempoolTransactions(last_txid, config.ESPLORA.BATCH_QUERY_BASE_SIZE);
+        if (result) {
+          for (const tx of result) {
+            const extendedTransaction = transactionUtils.extendMempoolTransaction(tx);
+            if (!this.mempoolCache[extendedTransaction.txid]) {
+              newTransactions.push(extendedTransaction);
+              this.mempoolCache[extendedTransaction.txid] = extendedTransaction;
+            }
+            count++;
+          }
+          logger.info(`Fetched ${count} of ${expectedCount} mempool transactions from esplora`);
+          if (result.length > 0) {
+            last_txid = result[result.length - 1].txid;
+          } else {
+            done = true;
+          }
+          if (Math.floor((count / expectedCount) * 100) < 100) {
+            loadingIndicators.setProgress('mempool', count / expectedCount * 100);
+          }
+        } else {
+          done = true;
+        }
+      } catch(err) {
+        logger.err('failed to fetch bulk mempool transactions from esplora');
+      }
+    }
+    logger.info(`Done inserting loaded mempool transactions into local cache`);
+    return newTransactions;
+  }
+
+  public getMempoolCandidates(): { [txid: string]: boolean } {
+    return this.mempoolCandidates;
   }
 
   public async $updateMemPoolInfo() {
@@ -90,26 +208,65 @@ class Mempool {
     return txTimes;
   }
 
-  public async $updateMempool() {
-    logger.debug('Updating mempool');
+  public async $updateMempool(transactions: string[], accelerations: Record<string, Acceleration> | null, minFeeMempool: string[], minFeeTip: number, pollRate: number): Promise<void> {
+    logger.debug(`Updating mempool...`);
+
+    // warn if this run stalls the main loop for more than 2 minutes
+    const timer = this.startTimer();
+
     const start = new Date().getTime();
     let hasChange: boolean = false;
     const currentMempoolSize = Object.keys(this.mempoolCache).length;
-    let txCount = 0;
-    const transactions = await bitcoinApi.$getRawMempool();
+    this.updateTimerProgress(timer, 'got raw mempool');
     const diff = transactions.length - currentMempoolSize;
-    const newTransactions: TransactionExtended[] = [];
+    let newTransactions: MempoolTransactionExtended[] = [];
+
+    this.mempoolCacheDelta = Math.abs(diff);
 
     if (!this.inSync) {
-      loadingIndicators.setProgress('mempool', Object.keys(this.mempoolCache).length / transactions.length * 100);
+      loadingIndicators.setProgress('mempool', currentMempoolSize / transactions.length * 100);
     }
 
-    for (const txid of transactions) {
-      if (!this.mempoolCache[txid]) {
-        try {
-          const transaction = await transactionUtils.$getTransactionExtended(txid);
-          this.mempoolCache[txid] = transaction;
-          txCount++;
+    // https://github.com/mempool/mempool/issues/3283
+    const logEsplora404 = (missingTxCount, threshold, time) => {
+      const log = `In the past ${time / 1000} seconds, esplora tx API replied ${missingTxCount} times with a 404 error code while updating nodejs backend mempool`;
+      if (missingTxCount >= threshold) {
+        logger.warn(log);
+      } else if (missingTxCount > 0) {
+        logger.debug(log);
+      }
+    };
+
+    let intervalTimer = Date.now();
+
+    let loaded = false;
+    if (config.MEMPOOL.BACKEND === 'esplora' && currentMempoolSize < transactions.length * 0.5 && transactions.length > 20_000) {
+      this.inSync = false;
+      logger.info(`Missing ${transactions.length - currentMempoolSize} mempool transactions, attempting to reload in bulk from esplora`);
+      try {
+        newTransactions = await this.$reloadMempool(transactions.length);
+        if (config.REDIS.ENABLED) {
+          for (const tx of newTransactions) {
+            await redisCache.$addTransaction(tx);
+          }
+        }
+        loaded = true;
+      } catch (e) {
+        logger.err('failed to load mempool in bulk from esplora, falling back to fetching individual transactions');
+      }
+    }
+
+    if (!loaded) {
+      const remainingTxids = transactions.filter(txid => !this.mempoolCache[txid]);
+      const sliceLength = config.ESPLORA.BATCH_QUERY_BASE_SIZE;
+      for (let i = 0; i < Math.ceil(remainingTxids.length / sliceLength); i++) {
+        const slice = remainingTxids.slice(i * sliceLength, (i + 1) * sliceLength);
+        const txs = await transactionUtils.$getMempoolTransactionsExtended(slice, false, false, false);
+        logger.debug(`fetched ${txs.length} transactions`);
+        this.updateTimerProgress(timer, 'fetched new transactions');
+
+        for (const transaction of txs) {
+          this.mempoolCache[transaction.txid] = transaction;
           if (this.inSync) {
             this.txPerSecondArray.push(new Date().getTime());
             this.vBytesPerSecondArray.push({
@@ -118,20 +275,45 @@ class Mempool {
             });
           }
           hasChange = true;
-          if (diff > 0) {
-            logger.debug('Fetched transaction ' + txCount + ' / ' + diff);
-          } else {
-            logger.debug('Fetched transaction ' + txCount);
-          }
           newTransactions.push(transaction);
-        } catch (e) {
-          logger.debug('Error finding transaction in mempool: ' + (e instanceof Error ? e.message : e));
+
+          if (config.REDIS.ENABLED) {
+            await redisCache.$addTransaction(transaction);
+          }
+        }
+
+        if (txs.length < slice.length) {
+          const missing = slice.length - txs.length;
+          if (config.MEMPOOL.BACKEND === 'esplora') {
+            this.missingTxCount += missing;
+          }
+          logger.debug(`Error finding ${missing} transactions in the mempool: `);
+        }
+
+        if (Date.now() - intervalTimer > Math.max(pollRate * 2, 5_000)) {
+          if (this.inSync) {
+            // Break and restart mempool loop if we spend too much time processing
+            // new transactions that may lead to falling behind on block height
+            logger.debug('Breaking mempool loop because the 5s time limit exceeded.');
+            break;
+          } else {
+            const progress = (currentMempoolSize + newTransactions.length) / transactions.length * 100;
+            logger.debug(`Mempool is synchronizing. Processed ${newTransactions.length}/${diff} txs (${Math.round(progress)}%)`);
+            if (Math.floor(progress) < 100) {
+              loadingIndicators.setProgress('mempool', progress);
+            }
+            intervalTimer = Date.now();
+          }
         }
       }
+    }
 
-      if ((new Date().getTime()) - start > Mempool.WEBSOCKET_REFRESH_RATE_MS) {
-        break;
-      }
+    // Reset esplora 404 counter and log a warning if needed
+    const elapsedTime = new Date().getTime() - this.timer;
+    if (elapsedTime > this.SAMPLE_TIME) {
+      logEsplora404(this.missingTxCount, this.ESPLORA_MISSING_TX_WARNING_THRESHOLD, elapsedTime);
+      this.timer = new Date().getTime();
+      this.missingTxCount = 0;
     }
 
     // Prevent mempool from clear on bitcoind restart by delaying the deletion
@@ -144,11 +326,11 @@ class Mempool {
       logger.warn(`Mempool clear protection triggered because transactions.length: ${transactions.length} and currentMempoolSize: ${currentMempoolSize}.`);
       setTimeout(() => {
         this.mempoolProtection = 2;
-        logger.warn('Mempool clear protection resumed.');
+        logger.warn('Mempool clear protection ended, normal operation resumed.');
       }, 1000 * 60 * config.MEMPOOL.CLEAR_PROTECTION_MINUTES);
     }
 
-    const deletedTransactions: TransactionExtended[] = [];
+    const deletedTransactions: MempoolTransactionExtended[] = [];
 
     if (this.mempoolProtection !== 1) {
       this.mempoolProtection = 0;
@@ -156,32 +338,193 @@ class Mempool {
       const transactionsObject = {};
       transactions.forEach((txId) => transactionsObject[txId] = true);
 
-      // Flag transactions for lazy deletion
+      // Delete evicted transactions from mempool
       for (const tx in this.mempoolCache) {
-        if (!transactionsObject[tx] && !this.mempoolCache[tx].deleteAfter) {
+        if (!transactionsObject[tx]) {
           deletedTransactions.push(this.mempoolCache[tx]);
-          this.mempoolCache[tx].deleteAfter = new Date().getTime() + Mempool.LAZY_DELETE_AFTER_SECONDS * 1000;
         }
       }
+      for (const tx of deletedTransactions) {
+        delete this.mempoolCache[tx.txid];
+      }
     }
 
+    const candidates = await this.getNextCandidates(minFeeMempool, minFeeTip, deletedTransactions);
+
+    const newMempoolSize = currentMempoolSize + newTransactions.length - deletedTransactions.length;
     const newTransactionsStripped = newTransactions.map((tx) => Common.stripTransaction(tx));
     this.latestTransactions = newTransactionsStripped.concat(this.latestTransactions).slice(0, 6);
 
-    if (!this.inSync && transactions.length === Object.keys(this.mempoolCache).length) {
+    const accelerationDelta = accelerations != null ? await this.updateAccelerations(accelerations) : [];
+    if (accelerationDelta.length) {
+      hasChange = true;
+    }
+
+    this.mempoolCacheDelta = Math.abs(transactions.length - newMempoolSize);
+
+    const candidatesChanged = candidates?.added?.length || candidates?.removed?.length;
+
+    this.recentlyDeleted.unshift(deletedTransactions);
+    this.recentlyDeleted.length = Math.min(this.recentlyDeleted.length, 10); // truncate to the last 10 mempool updates
+
+    if (this.mempoolChangedCallback && (hasChange || newTransactions.length || deletedTransactions.length)) {
+      this.mempoolChangedCallback(this.mempoolCache, newTransactions, this.recentlyDeleted, accelerationDelta);
+    }
+    if (this.$asyncMempoolChangedCallback && (hasChange || newTransactions.length || deletedTransactions.length || candidatesChanged)) {
+      this.updateTimerProgress(timer, 'running async mempool callback');
+      await this.$asyncMempoolChangedCallback(this.mempoolCache, newMempoolSize, newTransactions, this.recentlyDeleted, accelerationDelta, candidates);
+      this.updateTimerProgress(timer, 'completed async mempool callback');
+    }
+
+    if (!this.inSync && transactions.length === newMempoolSize) {
       this.inSync = true;
       logger.notice('The mempool is now in sync!');
       loadingIndicators.setProgress('mempool', 100);
     }
 
-    if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) {
-      this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions);
+    // Update Redis cache
+    if (config.REDIS.ENABLED) {
+      await redisCache.$flushTransactions();
+      await redisCache.$removeTransactions(deletedTransactions.map(tx => tx.txid));
+      await rbfCache.updateCache();
     }
 
     const end = new Date().getTime();
     const time = end - start;
-    logger.debug(`New mempool size: ${Object.keys(this.mempoolCache).length} Change: ${diff}`);
-    logger.debug('Mempool updated in ' + time / 1000 + ' seconds');
+    logger.debug(`Mempool updated in ${time / 1000} seconds. New size: ${Object.keys(this.mempoolCache).length} (${diff > 0 ? '+' + diff : diff})`);
+
+    this.clearTimer(timer);
+  }
+
+  public getAccelerations(): { [txid: string]: Acceleration } {
+    return this.accelerations;
+  }
+
+  public updateAccelerations(newAccelerationMap: Record<string, Acceleration>): string[] {
+    try {
+      const accelerationDelta = accelerationApi.getAccelerationDelta(this.accelerations, newAccelerationMap);
+      this.accelerations = newAccelerationMap;
+      return accelerationDelta;
+    } catch (e: any) {
+      logger.debug(`Failed to update accelerations: ` + (e instanceof Error ? e.message : e));
+      return [];
+    }
+  }
+
+  public async getNextCandidates(minFeeTransactions: string[], blockHeight: number, deletedTransactions: MempoolTransactionExtended[]): Promise<GbtCandidates | undefined> {
+    if (this.limitGBT) {
+      const deletedTxsMap = {};
+      for (const tx of deletedTransactions) {
+        deletedTxsMap[tx.txid] = tx;
+      }
+      const newCandidateTxMap = {};
+      for (const txid of minFeeTransactions) {
+        if (this.mempoolCache[txid]) {
+          newCandidateTxMap[txid] = true;
+        }
+      }
+      const accelerations = this.getAccelerations();
+      for (const txid of Object.keys(accelerations)) {
+        if (this.mempoolCache[txid]) {
+          newCandidateTxMap[txid] = true;
+        }
+      }
+      const removed: MempoolTransactionExtended[] = [];
+      const added: MempoolTransactionExtended[] = [];
+      // don't prematurely remove txs included in a new block
+      if (blockHeight > blocks.getCurrentBlockHeight()) {
+        for (const txid of Object.keys(this.mempoolCandidates)) {
+          newCandidateTxMap[txid] = true;
+        }
+      } else {
+        for (const txid of Object.keys(this.mempoolCandidates)) {
+          if (!newCandidateTxMap[txid]) {
+            if (this.mempoolCache[txid]) {
+              removed.push(this.mempoolCache[txid]);
+              this.mempoolCache[txid].effectiveFeePerVsize = this.mempoolCache[txid].adjustedFeePerVsize;
+              this.mempoolCache[txid].ancestors = [];
+              this.mempoolCache[txid].descendants = [];
+              this.mempoolCache[txid].bestDescendant = null;
+              this.mempoolCache[txid].cpfpChecked = false;
+              this.mempoolCache[txid].cpfpUpdated = undefined;
+            } else if (deletedTxsMap[txid]) {
+              removed.push(deletedTxsMap[txid]);
+            }
+          }
+        }
+      }
+
+      for (const txid of Object.keys(newCandidateTxMap)) {
+        if (!this.mempoolCandidates[txid]) {
+          added.push(this.mempoolCache[txid]);
+        }
+      }
+
+      this.mempoolCandidates = newCandidateTxMap;
+      return {
+        txs: this.mempoolCandidates,
+        added,
+        removed
+      };
+    }
+  }
+
+  setAccelerationPositions(positions: { [txid: string]: { poolId: number, pool: string, block: number, vsize: number }[] }): void {
+    this.accelerationPositions = positions;
+  }
+
+  getAccelerationPositions(txid: string): { [pool: number]: { poolId: number, pool: string, block: number, vsize: number } } | undefined {
+    return this.accelerationPositions[txid];
+  }
+
+  private startTimer() {
+    const state: any = {
+      start: Date.now(),
+      progress: 'begin $updateMempool',
+      timer: null,
+    };
+    state.timer = setTimeout(() => {
+      logger.err(`$updateMempool stalled at "${state.progress}"`);
+    }, this.mainLoopTimeout);
+    return state;
+  }
+
+  private updateTimerProgress(state, msg) {
+    state.progress = msg;
+  }
+
+  private clearTimer(state) {
+    if (state.timer) {
+      clearTimeout(state.timer);
+    }
+  }
+
+  public handleRbfTransactions(rbfTransactions: { [txid: string]: { replaced: MempoolTransactionExtended[], replacedBy: TransactionExtended }}): void {
+    for (const rbfTransaction in rbfTransactions) {
+      if (rbfTransactions[rbfTransaction].replacedBy && rbfTransactions[rbfTransaction]?.replaced?.length) {
+        // Store replaced transactions
+        rbfCache.add(rbfTransactions[rbfTransaction].replaced, transactionUtils.extendMempoolTransaction(rbfTransactions[rbfTransaction].replacedBy));
+      }
+    }
+  }
+
+  public addToSpendMap(transactions: MempoolTransactionExtended[]): void {
+    for (const tx of transactions) {
+      for (const vin of tx.vin) {
+        this.spendMap.set(`${vin.txid}:${vin.vout}`, tx);
+      }
+    }
+  }
+
+  public removeFromSpendMap(transactions: TransactionExtended[]): void {
+    for (const tx of transactions) {
+      for (const vin of tx.vin) {
+        const key = `${vin.txid}:${vin.vout}`;
+        if (this.spendMap.get(key)?.txid === tx.txid) {
+          this.spendMap.delete(key);
+        }
+      }
+    }
   }
 
   private updateTxPerSecond() {
@@ -197,16 +540,6 @@ class Mempool {
     }
   }
 
-  private deleteExpiredTransactions() {
-    const now = new Date().getTime();
-    for (const tx in this.mempoolCache) {
-      const lazyDeleteAt = this.mempoolCache[tx].deleteAfter;
-      if (lazyDeleteAt && lazyDeleteAt < now) {
-        delete this.mempoolCache[tx];
-      }
-    }
-  }
-
   private $getMempoolInfo() {
     if (config.MEMPOOL.USE_SECOND_NODE_FOR_MINFEE) {
       return Promise.all([

backend/src/api/mini-miner.ts

@@ -0,0 +1,515 @@
+import { Acceleration } from './acceleration/acceleration';
+import { MempoolTransactionExtended } from '../mempool.interfaces';
+import logger from '../logger';
+
+const BLOCK_WEIGHT_UNITS = 4_000_000;
+const BLOCK_SIGOPS = 80_000;
+const MAX_RELATIVE_GRAPH_SIZE = 100;
+
+export interface GraphTx {
+  txid: string;
+  vsize: number;
+  weight: number;
+  depends: string[];
+  spentby: string[];
+
+  ancestorcount: number;
+  ancestorsize: number;
+  fees: { // in sats
+    base: number;
+    ancestor: number;
+  };
+
+  ancestors: Map<string, GraphTx>,
+  ancestorRate: number;
+  individualRate: number;
+  score: number;
+}
+
+interface TemplateTransaction {
+  txid: string;
+  order: number;
+  weight: number;
+  adjustedVsize: number; // sigop-adjusted vsize, rounded up to the nearest integer
+  sigops: number;
+  fee: number;
+  feeDelta: number;
+  ancestors: string[];
+  cluster: string[];
+  effectiveFeePerVsize: number;
+}
+
+interface MinerTransaction extends TemplateTransaction {
+  inputs: string[];
+  feePerVsize: number;
+  relativesSet: boolean;
+  ancestorMap: Map<string, MinerTransaction>;
+  children: Set<MinerTransaction>;
+  ancestorFee: number;
+  ancestorVsize: number;
+  ancestorSigops: number;
+  score: number;
+  used: boolean;
+  modified: boolean;
+  dependencyRate: number;
+}
+
+/**
+ * Takes a raw transaction, and builds a graph of same-block relatives,
+ * and returns as a GraphTx
+ *
+ * @param tx
+ */
+export function getSameBlockRelatives(tx: MempoolTransactionExtended, transactions: MempoolTransactionExtended[]): Map<string, GraphTx> {
+  const blockTxs = new Map<string, MempoolTransactionExtended>(); // map of txs in this block
+  const spendMap = new Map<string, string>(); // map of outpoints to spending txids
+  for (const tx of transactions) {
+    blockTxs.set(tx.txid, tx);
+    for (const vin of tx.vin) {
+      spendMap.set(`${vin.txid}:${vin.vout}`, tx.txid);
+    }
+  }
+
+  const relatives: Map<string, GraphTx> = new Map();
+  const stack: string[] = [tx.txid];
+
+  // build set of same-block ancestors
+  while (stack.length > 0) {
+    const nextTxid = stack.pop();
+    const nextTx = nextTxid ? blockTxs.get(nextTxid) : null;
+    if (!nextTx || relatives.has(nextTx.txid)) {
+      continue;
+    }
+
+    const mempoolTx = convertToGraphTx(nextTx, spendMap);
+
+    for (const txid of [...mempoolTx.depends, ...mempoolTx.spentby]) {
+      if (txid) {
+        stack.push(txid);
+      }
+    }
+
+    relatives.set(mempoolTx.txid, mempoolTx);
+  }
+
+  return relatives;
+}
+
+/**
+ * Takes a raw transaction and converts it to GraphTx format
+ * fee and ancestor data is initialized with dummy/null values
+ *
+ * @param tx
+ */
+export function convertToGraphTx(tx: MempoolTransactionExtended, spendMap?: Map<string, MempoolTransactionExtended | string>): GraphTx {
+  return {
+    txid: tx.txid,
+    vsize: Math.max(tx.sigops * 5, Math.ceil(tx.weight / 4)),
+    weight: tx.weight,
+    fees: {
+      base: tx.fee || 0,
+      ancestor: tx.fee || 0,
+    },
+    depends: (tx.vin.map(vin => vin.txid).filter(depend => depend) as string[]),
+    spentby: spendMap ? (tx.vout.map((vout, index) => { const spend = spendMap.get(`${tx.txid}:${index}`); return (spend?.['txid'] || spend); }).filter(spent => spent) as string[]) : [],
+
+    ancestorcount: 1,
+    ancestorsize: Math.max(tx.sigops * 5, Math.ceil(tx.weight / 4)),
+    ancestors: new Map<string, GraphTx>(),
+    ancestorRate: 0,
+    individualRate: 0,
+    score: 0,
+  };
+}
+
+/**
+ * Takes a map of transaction ancestors, and expands it into a full graph of up to MAX_GRAPH_SIZE in-mempool relatives
+ */
+export function expandRelativesGraph(mempool: { [txid: string]: MempoolTransactionExtended }, ancestors: Map<string, GraphTx>, spendMap: Map<string, MempoolTransactionExtended>): Map<string, GraphTx> {
+  const relatives: Map<string, GraphTx> = new Map();
+  const stack: GraphTx[] = Array.from(ancestors.values());
+  while (stack.length > 0) {
+    if (relatives.size > MAX_RELATIVE_GRAPH_SIZE) {
+      return relatives;
+    }
+
+    const nextTx = stack.pop();
+    if (!nextTx) {
+      continue;
+    }
+    relatives.set(nextTx.txid, nextTx);
+
+    for (const relativeTxid of [...nextTx.depends, ...nextTx.spentby]) {
+      if (relatives.has(relativeTxid)) {
+        // already processed this tx
+        continue;
+      }
+      let ancestorTx = ancestors.get(relativeTxid);
+      if (!ancestorTx && relativeTxid in mempool) {
+        const mempoolTx = mempool[relativeTxid];
+        ancestorTx = convertToGraphTx(mempoolTx, spendMap);
+      }
+      if (ancestorTx) {
+        stack.push(ancestorTx);
+      }
+    }
+  }
+
+  return relatives;
+}
+
+/**
+ * Recursively traverses an in-mempool dependency graph, and sets a Map of in-mempool ancestors
+ * for each transaction.
+ *
+ * @param tx
+ * @param all
+ */
+function setAncestors(tx: GraphTx, all: Map<string, GraphTx>, visited: Map<string, Map<string, GraphTx>>, depth: number = 0): Map<string, GraphTx> {
+  // sanity check for infinite recursion / too many ancestors (should never happen)
+  if (depth > MAX_RELATIVE_GRAPH_SIZE) {
+    logger.warn('cpfp dependency calculation failed: setAncestors reached depth of 100, unable to proceed');
+    return tx.ancestors;
+  }
+
+  // initialize the ancestor map for this tx
+  tx.ancestors = new Map<string, GraphTx>();
+  tx.depends.forEach(parentId => {
+    const parent = all.get(parentId);
+    if (parent) {
+      // add the parent
+      tx.ancestors?.set(parentId, parent);
+      // check for a cached copy of this parent's ancestors
+      let ancestors = visited.get(parent.txid);
+      if (!ancestors) {
+        // recursively fetch the parent's ancestors
+        ancestors = setAncestors(parent, all, visited, depth + 1);
+      }
+      // and add to this tx's map
+      ancestors.forEach((ancestor, ancestorId) => {
+        tx.ancestors?.set(ancestorId, ancestor);
+      });
+    }
+  });
+  visited.set(tx.txid, tx.ancestors);
+
+  return tx.ancestors;
+}
+
+/**
+   * Efficiently sets a Map of in-mempool ancestors for each member of an expanded relative graph
+   * by running setAncestors on each leaf, and caching intermediate results.
+   * then initializes ancestor data for each transaction
+   *
+   * @param all
+   */
+export function initializeRelatives(mempoolTxs: Map<string, GraphTx>): Map<string, GraphTx> {
+  const visited: Map<string, Map<string, GraphTx>> = new Map();
+  const leaves: GraphTx[] = Array.from(mempoolTxs.values()).filter(entry => entry.spentby.length === 0);
+  for (const leaf of leaves) {
+    setAncestors(leaf, mempoolTxs, visited);
+  }
+  mempoolTxs.forEach(entry => {
+    entry.ancestors?.forEach(ancestor => {
+      entry.ancestorcount++;
+      entry.ancestorsize += ancestor.vsize;
+      entry.fees.ancestor += ancestor.fees.base;
+    });
+    setAncestorScores(entry);
+  });
+  return mempoolTxs;
+}
+
+/**
+ * Remove a cluster of transactions from an in-mempool dependency graph
+ * and update the survivors' scores and ancestors
+ *
+ * @param cluster
+ * @param ancestors
+ */
+export function removeAncestors(cluster: Map<string, GraphTx>, all: Map<string, GraphTx>): void {
+  // remove
+  cluster.forEach(tx => {
+    all.delete(tx.txid);
+  });
+
+  // update survivors
+  all.forEach(tx => {
+    cluster.forEach(remove => {
+      if (tx.ancestors?.has(remove.txid)) {
+        // remove as dependency
+        tx.ancestors.delete(remove.txid);
+        tx.depends = tx.depends.filter(parent => parent !== remove.txid);
+        // update ancestor sizes and fees
+        tx.ancestorsize -= remove.vsize;
+        tx.fees.ancestor -= remove.fees.base;
+      }
+    });
+    // recalculate fee rates
+    setAncestorScores(tx);
+  });
+}
+
+/**
+ * Take a mempool transaction, and set the fee rates and ancestor score
+ *
+ * @param tx
+ */
+export function setAncestorScores(tx: GraphTx): void {
+  tx.individualRate = tx.fees.base / tx.vsize;
+  tx.ancestorRate = tx.fees.ancestor / tx.ancestorsize;
+  tx.score = Math.min(tx.individualRate, tx.ancestorRate);
+}
+
+// Sort by descending score
+export function mempoolComparator(a: GraphTx, b: GraphTx): number {
+  return b.score - a.score;
+}
+
+/*
+* Build a block using an approximation of the transaction selection algorithm from Bitcoin Core
+* (see BlockAssembler in https://github.com/bitcoin/bitcoin/blob/master/src/node/miner.cpp)
+*/
+export function makeBlockTemplate(candidates: MempoolTransactionExtended[], accelerations: Acceleration[], maxBlocks: number = 8, weightLimit: number = BLOCK_WEIGHT_UNITS, sigopLimit: number = BLOCK_SIGOPS): TemplateTransaction[] {
+  const auditPool: Map<string, MinerTransaction> = new Map();
+  const mempoolArray: MinerTransaction[] = [];
+
+  candidates.forEach(tx => {
+    // initializing everything up front helps V8 optimize property access later
+    const adjustedVsize = Math.ceil(Math.max(tx.weight / 4, 5 * (tx.sigops || 0)));
+    const feePerVsize = (tx.fee / adjustedVsize);
+    auditPool.set(tx.txid, {
+      txid: tx.txid,
+      order: txidToOrdering(tx.txid),
+      fee: tx.fee,
+      feeDelta: 0,
+      weight: tx.weight,
+      adjustedVsize,
+      feePerVsize: feePerVsize,
+      effectiveFeePerVsize: feePerVsize,
+      dependencyRate: feePerVsize,
+      sigops: tx.sigops || 0,
+      inputs: (tx.vin?.map(vin => vin.txid) || []) as string[],
+      relativesSet: false,
+      ancestors: [],
+      cluster: [],
+      ancestorMap: new Map<string, MinerTransaction>(),
+      children: new Set<MinerTransaction>(),
+      ancestorFee: 0,
+      ancestorVsize: 0,
+      ancestorSigops: 0,
+      score: 0,
+      used: false,
+      modified: false,
+    });
+    mempoolArray.push(auditPool.get(tx.txid) as MinerTransaction);
+  });
+
+  // set accelerated effective fee
+  for (const acceleration of accelerations) {
+    const tx = auditPool.get(acceleration.txid);
+    if (tx) {
+      tx.feeDelta = acceleration.max_bid;
+      tx.feePerVsize = ((tx.fee + tx.feeDelta) / tx.adjustedVsize);
+      tx.effectiveFeePerVsize = tx.feePerVsize;
+      tx.dependencyRate = tx.feePerVsize;
+    }
+  }
+
+  // Build relatives graph & calculate ancestor scores
+  for (const tx of mempoolArray) {
+    if (!tx.relativesSet) {
+      setRelatives(tx, auditPool);
+    }
+  }
+
+  // Sort by descending ancestor score
+  mempoolArray.sort(priorityComparator);
+
+  // Build blocks by greedily choosing the highest feerate package
+  // (i.e. the package rooted in the transaction with the best ancestor score)
+  const blocks: number[][] = [];
+  let blockWeight = 0;
+  let blockSigops = 0;
+  const transactions: MinerTransaction[] = [];
+  let modified: MinerTransaction[] = [];
+  const overflow: MinerTransaction[] = [];
+  let failures = 0;
+  while (mempoolArray.length || modified.length) {
+    // skip invalid transactions
+    while (mempoolArray[0]?.used || mempoolArray[0]?.modified) {
+      mempoolArray.shift();
+    }
+
+    // Select best next package
+    let nextTx;
+    const nextPoolTx = mempoolArray[0];
+    const nextModifiedTx = modified[0];
+    if (nextPoolTx && (!nextModifiedTx || (nextPoolTx.score || 0) > (nextModifiedTx.score || 0))) {
+      nextTx = nextPoolTx;
+      mempoolArray.shift();
+    } else {
+      modified.shift();
+      if (nextModifiedTx) {
+        nextTx = nextModifiedTx;
+      }
+    }
+
+    if (nextTx && !nextTx?.used) {
+      // Check if the package fits into this block
+      if (blocks.length >= (maxBlocks - 1) || ((blockWeight + (4 * nextTx.ancestorVsize) < weightLimit) && (blockSigops + nextTx.ancestorSigops <= sigopLimit))) {
+        const ancestors: MinerTransaction[] = Array.from(nextTx.ancestorMap.values());
+        // sort ancestors by dependency graph (equivalent to sorting by ascending ancestor count)
+        const sortedTxSet = [...ancestors.sort((a, b) => { return (a.ancestorMap.size || 0) - (b.ancestorMap.size || 0); }), nextTx];
+        const clusterTxids = sortedTxSet.map(tx => tx.txid);
+        const effectiveFeeRate = Math.min(nextTx.dependencyRate || Infinity, nextTx.ancestorFee / nextTx.ancestorVsize);
+        const used: MinerTransaction[] = [];
+        while (sortedTxSet.length) {
+          const ancestor = sortedTxSet.pop();
+          if (!ancestor) {
+            continue;
+          }
+          ancestor.used = true;
+          ancestor.usedBy = nextTx.txid;
+          // update this tx with effective fee rate & relatives data
+          if (ancestor.effectiveFeePerVsize !== effectiveFeeRate) {
+            ancestor.effectiveFeePerVsize = effectiveFeeRate;
+          }
+          ancestor.cluster = clusterTxids;
+          transactions.push(ancestor);
+          blockWeight += ancestor.weight;
+          blockSigops += ancestor.sigops;
+          used.push(ancestor);
+        }
+
+        // remove these as valid package ancestors for any descendants remaining in the mempool
+        if (used.length) {
+          used.forEach(tx => {
+            modified = updateDescendants(tx, auditPool, modified, effectiveFeeRate);
+          });
+        }
+
+        failures = 0;
+      } else {
+        // hold this package in an overflow list while we check for smaller options
+        overflow.push(nextTx);
+        failures++;
+      }
+    }
+
+    // this block is full
+    const exceededPackageTries = failures > 1000 && blockWeight > (weightLimit - 4000);
+    const queueEmpty = !mempoolArray.length && !modified.length;
+
+    if (exceededPackageTries || queueEmpty) {
+      break;
+    }
+  }
+
+  for (const tx of transactions) {
+    tx.ancestors = Object.values(tx.ancestorMap);
+  }
+
+  return transactions;
+}
+
+// traverse in-mempool ancestors
+// recursion unavoidable, but should be limited to depth < 25 by mempool policy
+function setRelatives(
+  tx: MinerTransaction,
+  mempool: Map<string, MinerTransaction>,
+): void {
+  for (const parent of tx.inputs) {
+    const parentTx = mempool.get(parent);
+    if (parentTx && !tx.ancestorMap?.has(parent)) {
+      tx.ancestorMap.set(parent, parentTx);
+      parentTx.children.add(tx);
+      // visit each node only once
+      if (!parentTx.relativesSet) {
+        setRelatives(parentTx, mempool);
+      }
+      parentTx.ancestorMap.forEach((ancestor) => {
+        tx.ancestorMap.set(ancestor.txid, ancestor);
+      });
+    }
+  };
+  tx.ancestorFee = (tx.fee + tx.feeDelta);
+  tx.ancestorVsize = tx.adjustedVsize || 0;
+  tx.ancestorSigops = tx.sigops || 0;
+  tx.ancestorMap.forEach((ancestor) => {
+    tx.ancestorFee += (ancestor.fee + ancestor.feeDelta);
+    tx.ancestorVsize += ancestor.adjustedVsize;
+    tx.ancestorSigops += ancestor.sigops;
+  });
+  tx.score = tx.ancestorFee / tx.ancestorVsize;
+  tx.relativesSet = true;
+}
+
+// iterate over remaining descendants, removing the root as a valid ancestor & updating the ancestor score
+// avoids recursion to limit call stack depth
+function updateDescendants(
+  rootTx: MinerTransaction,
+  mempool: Map<string, MinerTransaction>,
+  modified: MinerTransaction[],
+  clusterRate: number,
+): MinerTransaction[] {
+  const descendantSet: Set<MinerTransaction> = new Set();
+  // stack of nodes left to visit
+  const descendants: MinerTransaction[] = [];
+  let descendantTx: MinerTransaction | undefined;
+  rootTx.children.forEach(childTx => {
+    if (!descendantSet.has(childTx)) {
+      descendants.push(childTx);
+      descendantSet.add(childTx);
+    }
+  });
+  while (descendants.length) {
+    descendantTx = descendants.pop();
+    if (descendantTx && descendantTx.ancestorMap && descendantTx.ancestorMap.has(rootTx.txid)) {
+      // remove tx as ancestor
+      descendantTx.ancestorMap.delete(rootTx.txid);
+      descendantTx.ancestorFee -= (rootTx.fee + rootTx.feeDelta);
+      descendantTx.ancestorVsize -= rootTx.adjustedVsize;
+      descendantTx.ancestorSigops -= rootTx.sigops;
+      descendantTx.score = descendantTx.ancestorFee / descendantTx.ancestorVsize;
+      descendantTx.dependencyRate = descendantTx.dependencyRate ? Math.min(descendantTx.dependencyRate, clusterRate) : clusterRate;
+
+      if (!descendantTx.modified) {
+        descendantTx.modified = true;
+        modified.push(descendantTx);
+      }
+
+      // add this node's children to the stack
+      descendantTx.children.forEach(childTx => {
+        // visit each node only once
+        if (!descendantSet.has(childTx)) {
+          descendants.push(childTx);
+          descendantSet.add(childTx);
+        }
+      });
+    }
+  }
+  // return new, resorted modified list
+  return modified.sort(priorityComparator);
+}
+
+// Used to sort an array of MinerTransactions by descending ancestor score
+function priorityComparator(a: MinerTransaction, b: MinerTransaction): number {
+  if (b.score === a.score) {
+    // tie-break by txid for stability
+    return a.order - b.order;
+  } else {
+    return b.score - a.score;
+  }
+}
+
+// returns the most significant 4 bytes of the txid as an integer
+function txidToOrdering(txid: string): number {
+  return parseInt(
+    txid.substring(62, 64) +
+      txid.substring(60, 62) +
+      txid.substring(58, 60) +
+      txid.substring(56, 58),
+    16
+  );
+}