Merge remote-tracking branch 'mempooljs/main' into wiz/subtree-merge-mempool.js-repo

Typo fixed

.github/ISSUE_TEMPLATE/00-bug-issue.md

@@ -1,13 +1,14 @@
 ---
 name: 🐛 Bug Report
-about: Report bugs (no support requests, please)
+about: Report bugs or other issues to us on GitHub
 ---
 
 <!--
    SUPPORT REQUESTS:
    This is for reporting bugs in Mempool, not for support requests. 
-   If you have a support request, please reach out on Matrix:
-   https://matrix.to/#/#mempool.support:bitcoin.kyoto
+   If you have a support request, please join our Keybase or Matrix:
+   https://keybase.io/team/mempool
+   https://matrix.to/#/#mempool:bitcoin.kyoto
 -->
 
 ### Description

.github/ISSUE_TEMPLATE/30-feature-request.md

@@ -1,13 +1,14 @@
 ---
 name: ✨ Feature Request
-about: Request a feature or suggest other enhancements
+about: Request a feature or suggest other enhancements 💡
 ---
 
 <!--
    SUPPORT REQUESTS:
    This is for requesting features in Mempool, not for support requests. 
-   If you have a support request, please reach out on Matrix:
-   https://matrix.to/#/#mempool.support:bitcoin.kyoto
+   If you have a support request, please join our Keybase or Matrix:
+   https://keybase.io/team/mempool
+   https://matrix.to/#/#mempool:bitcoin.kyoto
 -->
 
 ### Description

.github/ISSUE_TEMPLATE/config.yml

@@ -1,8 +1,8 @@
 blank_issues_enabled: false
 contact_links:
-  - name: 🙋 Need help? Chat with us on Matrix
-    url: https://matrix.to/#/#mempool.support:bitcoin.kyoto
+  - name: 💬 Need help? Chat with us on Matrix
+    url: https://matrix.to/#/#mempool:bitcoin.kyoto
+    about: For support requests or general questions
+  - name: 💬 Need help? Chat with us on Keybase
+    url: https://keybase.io/team/mempool
     about: For support requests or general questions
-  - name: 🌐 Want to help with translations? Use Transifex
-    url: https://www.transifex.com/mempool/mempool
-    about: All translations work is done on Transifex

.github/dependabot.yml

@@ -1,20 +0,0 @@
-version: 2
-updates:
-- package-ecosystem: npm
-  directory: "/backend"
-  schedule:
-    interval: daily
-  open-pull-requests-limit: 10
-- package-ecosystem: npm
-  directory: "/frontend"
-  schedule:
-    interval: daily
-  open-pull-requests-limit: 10
-- package-ecosystem: docker
-  directory: "/docker/backend"
-  schedule:
-    interval: weekly
-- package-ecosystem: "github-actions"
-  directory: "/"
-  schedule:
-      interval: "weekly"

.github/pull_request_template.md

@@ -1,6 +0,0 @@
-<!--
-Please do not open pull requests for translations.
-
-All translations work is done on Transifex:
-https://www.transifex.com/mempool/mempool
--->

.github/workflows/cypress.yml

@@ -1,6 +1,7 @@
 name: Cypress Tests
 
 on: [push, pull_request]
+
 jobs:
   cypress:
     runs-on: ${{ matrix.os }}
@@ -17,75 +18,69 @@ jobs:
       - name: Setup node
         uses: actions/setup-node@v2
         with:
-          node-version: 16.15.0
+          node-version: 16.10.0
           cache: 'npm'
           cache-dependency-path: frontend/package-lock.json
       - name: ${{ matrix.browser }} browser tests (Mempool)
-        uses: cypress-io/github-action@v4
+        uses: cypress-io/github-action@v2
         with:
-          tag: ${{ github.event_name }}
           working-directory: frontend
           build: npm run config:defaults:mempool
-          start: npm run start:local-staging
+          start: npm run start:local-prod
           wait-on: 'http://localhost:4200'
           wait-on-timeout: 120
           record: true
           parallel: true
           spec: |
-            cypress/e2e/mainnet/*.spec.ts
-            cypress/e2e/signet/*.spec.ts
-            cypress/e2e/testnet/*.spec.ts
+            cypress/integration/mainnet/*.spec.ts
+            cypress/integration/signet/*.spec.ts
+            cypress/integration/testnet/*.spec.ts
           group: Tests on ${{ matrix.browser }} (Mempool)
           browser: ${{ matrix.browser }}
           ci-build-id: '${{ github.sha }}-${{ github.workflow }}-${{ github.event_name }}'
         env:
-          COMMIT_INFO_MESSAGE: ${{ github.event.pull_request.title }}
           CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}
 
       - name: ${{ matrix.browser }} browser tests (Liquid)
-        uses: cypress-io/github-action@v4
+        uses: cypress-io/github-action@v2
         if: always()
         with:
-          tag: ${{ github.event_name }}
           working-directory: frontend
           build: npm run config:defaults:liquid
-          start: npm run start:local-staging
+          start: npm run start:local-prod
           wait-on: 'http://localhost:4200'
           wait-on-timeout: 120
           record: true
           parallel: true
           spec: |
-            cypress/e2e/liquid/liquid.spec.ts
-            cypress/e2e/liquidtestnet/liquidtestnet.spec.ts
+            cypress/integration/liquid/liquid.spec.ts
+            cypress/integration/liquidtestnet/liquidtestnet.spec.ts
           group: Tests on ${{ matrix.browser }} (Liquid)
           browser: ${{ matrix.browser }}
           ci-build-id: '${{ github.sha }}-${{ github.workflow }}-${{ github.event_name }}'
         env:
-          COMMIT_INFO_MESSAGE: ${{ github.event.pull_request.title }}
           CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}
 
       - name: ${{ matrix.browser }} browser tests (Bisq)
-        uses: cypress-io/github-action@v4
+        uses: cypress-io/github-action@v2
         if: always()
         with:
-          tag: ${{ github.event_name }}
           working-directory: frontend
           build: npm run config:defaults:bisq
-          start: npm run start:local-staging
+          start: npm run start:local-prod
           wait-on: 'http://localhost:4200'
           wait-on-timeout: 120
           record: true
           parallel: true
-          spec: cypress/e2e/bisq/bisq.spec.ts
+          spec: cypress/integration/bisq/bisq.spec.ts
           group: Tests on ${{ matrix.browser }} (Bisq)
           browser: ${{ matrix.browser }}
           ci-build-id: '${{ github.sha }}-${{ github.workflow }}-${{ github.event_name }}'
         env:
-          COMMIT_INFO_MESSAGE: ${{ github.event.pull_request.title }}
           CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}

.github/workflows/on-tag.yml

@@ -11,9 +11,6 @@ on:
       - v[0-9]+.[0-9]+.[0-9]+
       - v[0-9]+.[0-9]+.[0-9]+-*
 
-permissions:
-  contents: read
-
 jobs:
   build:
     strategy:
@@ -38,24 +35,24 @@ jobs:
         run: echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin
 
       - name: Checkout project
-        uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
+        uses: actions/checkout@v2
 
       - name: Init repo for Dockerization
         run: docker/init.sh "$TAG"
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 # v1
+        uses: docker/setup-qemu-action@v1
         id: qemu
 
       - name: Setup Docker buildx action
-        uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 # v1
+        uses: docker/setup-buildx-action@v1
         id: buildx
 
       - name: Available platforms
         run: echo ${{ steps.buildx.outputs.platforms }}
 
       - name: Cache Docker layers
-        uses: actions/cache@661fd3eb7f2f20d8c7c84bc2b0509efd7a826628 # v2
+        uses: actions/cache@v2
         id: cache
         with:
           path: /tmp/.buildx-cache

.nvmrc

@@ -1 +1 @@
-v16.15.0
+v16.10.0

README.md

@@ -1,33 +1,415 @@
-# The Mempool Open Source Project™ [![mempool](https://img.shields.io/endpoint?url=https://dashboard.cypress.io/badge/simple/ry4br7/master&style=flat-square)](https://dashboard.cypress.io/projects/ry4br7/runs)
+# The Mempool Open Source Project™
 
-Mempool is the fully-featured mempool visualizer, explorer, and API service running at [mempool.space](https://mempool.space/). 
+Mempool is the fully featured visualizer, explorer, and API service running on [mempool.space](https://mempool.space/), an open source project developed and operated for the benefit of the Bitcoin community, with a focus on the emerging transaction fee market to help our transition into a multi-layer ecosystem.
 
-It is an open-source project developed and operated for the benefit of the Bitcoin community, with a focus on the emerging transaction fee market that is evolving Bitcoin into a multi-layer ecosystem.
+![mempool](https://mempool.space/resources/screenshots/v2.3.0-dashboard.png)
 
-![mempool](https://mempool.space/resources/screenshots/v2.4.0-dashboard.png)
+## Installation Methods
 
-# Installation Methods
+Mempool can be self-hosted on a wide variety of your own hardware, ranging from a simple one-click installation on a Raspberry Pi distro, all the way to an advanced high availability cluster of powerful servers for a production instance. We support the following installation methods, ranked in order from simple to advanced:
 
-Mempool can be self-hosted on a wide variety of your own hardware, ranging from a simple one-click installation on a Raspberry Pi full-node distro all the way to a robust production instance on a powerful FreeBSD server. 
+1) One-click installation on: [Umbrel](https://github.com/getumbrel/umbrel), [RaspiBlitz](https://github.com/rootzoll/raspiblitz), [RoninDojo](https://code.samourai.io/ronindojo/RoninDojo), or [MyNode](https://github.com/mynodebtc/mynode).
+2) [Docker installation on Linux using docker-compose](https://github.com/mempool/mempool/tree/master/docker)
+3) [Manual installation on Linux or FreeBSD](https://github.com/mempool/mempool#manual-installation)
+4) [Production installation on a powerful FreeBSD server](https://github.com/mempool/mempool/tree/master/production)
+5) [High Availability cluster using powerful FreeBSD servers](https://github.com/mempool/mempool/tree/master/production#high-availability)
 
-**Most people should use a one-click install method.** Other install methods are meant for developers and others with experience managing servers. 
+# Docker Installation
 
-<a id="one-click-installation"></a>
-## One-Click Installation
+The `docker` directory contains the Dockerfiles used to build and release the official images and a `docker-compose.yml` file that is intended for end users to run a Mempool instance with minimal effort.
 
-Mempool can be conveniently installed on the following full-node distros: 
-- [Umbrel](https://github.com/getumbrel/umbrel)
-- [RaspiBlitz](https://github.com/rootzoll/raspiblitz)
-- [RoninDojo](https://code.samourai.io/ronindojo/RoninDojo)
-- [myNode](https://github.com/mynodebtc/mynode)
-- [Start9](https://github.com/Start9Labs/embassy-os)
+## bitcoind only configuration
 
-**We highly recommend you deploy your own Mempool instance this way.** No matter which option you pick, you'll be able to get your own fully-sovereign instance of Mempool up quickly without needing to fiddle with any settings.
+To run an instance with the default settings, use the following command:
 
-## Advanced Installation Methods
+```bash
+$ docker-compose up
+```
 
-Mempool can be installed in other ways too, but we only recommend doing so if you're a developer, have experience managing servers, or otherwise know what you're doing.
+The default configuration will allow you to run Mempool using `bitcoind` as the backend, so address lookups will be disabled. It assumes you have added RPC credentials for the `mempool` user with a `mempool` password in your `bitcoin.conf` file:
 
-- See the [`docker/`](./docker/) directory for instructions on deploying Mempool with Docker.
-- See the [`backend/`](./backend/) and [`frontend/`](./frontend/) directories for manual install instructions oriented for developers and small-scale deployments.
-- See the [`production/`](./production/) directory for guidance on setting up a more serious Mempool instance designed for high performance at scale.
+```
+rpcuser=mempool
+rpcpassword=mempool
+```
+
+If you want to use your current credentials, update them in the `docker-compose.yml` file:
+
+```
+  api:
+    environment:
+      MEMPOOL_BACKEND: "none"
+      RPC_HOST: "172.27.0.1"
+      RPC_PORT: "8332"
+      RPC_USER: "mempool"
+      RPC_PASS: "mempool"
+```
+
+Note: the IP in the example above refers to Docker's default gateway IP address so the container can hit the `bitcoind` instance running on the host machine. If your setup is different, update it accordingly.
+
+You can check if the instance is running by visiting http://localhost - the graphs will be populated as new transactions are detected.
+
+## bitcoind+electrum configuration
+
+In order to run with a `electrum` compatible server as the backend, in addition to the settings required for running with `bitcoind` above, you will need to make the following changes to the `docker-compose.yml` file:
+
+- Under the `api` service, change the value of the `MEMPOOL_BACKEND` key from `none` to `electrum`:
+
+```
+  api:
+    environment:
+      MEMPOOL_BACKEND: "none"
+```
+
+- Under the `api` service, set the `ELECTRUM_HOST` and `ELECTRUM_PORT` keys to your Docker host IP address and set `ELECTRUM_TLS_ENABLED` to `false`:
+
+```
+  api:
+    environment:
+      ELECTRUM_HOST: "172.27.0.1"
+      ELECTRUM_PORT: "50002"
+      ELECTRUM_TLS_ENABLED: "false"
+```
+
+You can update any of the backend settings in the `mempool-config.json` file using the following environment variables to override them under the same `api` `environment` section.
+
+JSON:
+```
+  "MEMPOOL": {
+    "NETWORK": "mainnet",
+    "BACKEND": "electrum",
+    "HTTP_PORT": 8999,
+    "SPAWN_CLUSTER_PROCS": 0,
+    "API_URL_PREFIX": "/api/v1/",
+    "POLL_RATE_MS": 2000,
+    "CACHE_DIR": "./cache",
+    "CLEAR_PROTECTION_MINUTES": 20,
+    "RECOMMENDED_FEE_PERCENTILE": 50,
+    "BLOCK_WEIGHT_UNITS": 4000000,
+    "INITIAL_BLOCKS_AMOUNT": 8,
+    "MEMPOOL_BLOCKS_AMOUNT": 8,
+    "PRICE_FEED_UPDATE_INTERVAL": 3600,
+    "USE_SECOND_NODE_FOR_MINFEE": false,
+    "EXTERNAL_ASSETS": []
+  },
+```
+
+docker-compose overrides::
+```
+      MEMPOOL_NETWORK: ""
+      MEMPOOL_BACKEND: ""
+      MEMPOOL_HTTP_PORT: ""
+      MEMPOOL_SPAWN_CLUSTER_PROCS: ""
+      MEMPOOL_API_URL_PREFIX: ""
+      MEMPOOL_POLL_RATE_MS: ""
+      MEMPOOL_CACHE_DIR: ""
+      MEMPOOL_CLEAR_PROTECTION_MINUTES: ""
+      MEMPOOL_RECOMMENDED_FEE_PERCENTILE: ""
+      MEMPOOL_BLOCK_WEIGHT_UNITS: ""
+      MEMPOOL_INITIAL_BLOCKS_AMOUNT: ""
+      MEMPOOL_MEMPOOL_BLOCKS_AMOUNT: ""
+      MEMPOOL_PRICE_FEED_UPDATE_INTERVAL: ""
+      MEMPOOL_USE_SECOND_NODE_FOR_MINFEE: ""
+      MEMPOOL_EXTERNAL_ASSETS: ""
+```
+
+JSON:
+```
+"CORE_RPC": {
+    "HOST": "127.0.0.1",
+    "PORT": 8332,
+    "USERNAME": "mempool",
+    "PASSWORD": "mempool"
+  },
+```
+docker-compose overrides:
+```
+      CORE_RPC_HOST: ""
+      CORE_RPC_PORT: ""
+      CORE_RPC_USERNAME: ""
+      CORE_RPC_PASSWORD: ""
+```
+
+JSON:
+```
+  "ELECTRUM": {
+    "HOST": "127.0.0.1",
+    "PORT": 50002,
+    "TLS_ENABLED": true
+  },
+```
+
+docker-compose overrides:
+```
+      ELECTRUM_HOST: ""
+      ELECTRUM_PORT: ""
+      ELECTRUM_TLS: ""
+```
+
+JSON:
+```
+  "ESPLORA": {
+    "REST_API_URL": "http://127.0.0.1:3000"
+  },
+```
+docker-compose overrides:
+```
+      ESPLORA_REST_API_URL: ""
+```
+
+JSON:
+```
+  "SECOND_CORE_RPC": {
+    "HOST": "127.0.0.1",
+    "PORT": 8332,
+    "USERNAME": "mempool",
+    "PASSWORD": "mempool"
+  },
+```
+
+docker-compose overrides:
+```
+      SECOND_CORE_RPC_HOST: ""
+      SECOND_CORE_RPC_PORT: ""
+      SECOND_CORE_RPC_USERNAME: ""
+      SECOND_CORE_RPC_PASSWORD: ""
+```
+
+JSON:
+```
+  "DATABASE": {
+    "ENABLED": true,
+    "HOST": "127.0.0.1",
+    "PORT": 3306,
+    "DATABASE": "mempool",
+    "USERNAME": "mempool",
+    "PASSWORD": "mempool"
+  },
+```
+
+docker-compose overrides:
+```
+      DATABASE_ENABLED: ""
+      DATABASE_HOST: ""
+      DATABASE_PORT: ""
+      DATABASE_DATABASE: ""
+      DATABASE_USERAME: ""
+      DATABASE_PASSWORD: ""
+```
+
+JSON:
+```
+  "SYSLOG": {
+    "ENABLED": true,
+    "HOST": "127.0.0.1",
+    "PORT": 514,
+    "MIN_PRIORITY": "info",
+    "FACILITY": "local7"
+  },
+```
+
+docker-compose overrides:
+```
+      SYSLOG_ENABLED: ""
+      SYSLOG_HOST: ""
+      SYSLOG_PORT: ""
+      SYSLOG_MIN_PRIORITY: ""
+      SYSLOG_FACILITY: ""
+```
+
+JSON:
+```
+  "STATISTICS": {
+    "ENABLED": true,
+    "TX_PER_SECOND_SAMPLE_PERIOD": 150
+  },
+```
+
+docker-compose overrides:
+```
+      STATISTICS_ENABLED: ""
+      STATISTICS_TX_PER_SECOND_SAMPLE_PERIOD: ""
+```
+
+JSON:
+```
+  "BISQ": {
+    "ENABLED": false,
+    "DATA_PATH": "/bisq/statsnode-data/btc_mainnet/db"
+  }
+```
+
+docker-compose overrides:
+```
+      BISQ_ENABLED: ""
+      BISQ_DATA_PATH: ""
+```
+
+
+# Manual Installation
+
+The following instructions are for a manual installation on Linux or FreeBSD. The file and directory paths may need to be changed to match your OS.
+
+## Dependencies
+
+* [Bitcoin](https://github.com/bitcoin/bitcoin)
+* [Electrum](https://github.com/romanz/electrs)
+* [NodeJS](https://github.com/nodejs/node)
+* [MariaDB](https://github.com/mariadb/server)
+* [Nginx](https://github.com/nginx/nginx)
+
+## Mempool
+
+Clone the mempool repo, and checkout the latest release tag:
+```bash
+  git clone https://github.com/mempool/mempool
+  cd mempool
+  latestrelease=$(curl -s https://api.github.com/repos/mempool/mempool/releases/latest|grep tag_name|head -1|cut -d '"' -f4)
+  git checkout $latestrelease
+```
+
+## Bitcoin Core (bitcoind)
+
+Enable RPC and txindex in `bitcoin.conf`:
+```bash
+  rpcuser=mempool
+  rpcpassword=mempool
+  txindex=1
+```
+
+## MySQL
+
+Install MariaDB from OS package manager:
+```bash
+  # Linux
+  apt-get install mariadb-server mariadb-client
+
+  # macOS
+  brew install mariadb
+  mysql.server start
+```
+
+Create database and grant privileges:
+```bash
+  MariaDB [(none)]> drop database mempool;
+  Query OK, 0 rows affected (0.00 sec)
+
+  MariaDB [(none)]> create database mempool;
+  Query OK, 1 row affected (0.00 sec)
+
+  MariaDB [(none)]> grant all privileges on mempool.* to 'mempool'@'%' identified by 'mempool';
+  Query OK, 0 rows affected (0.00 sec)
+```
+
+## Mempool Backend
+Install mempool dependencies from npm and build the backend:
+
+```bash
+  # backend
+  cd backend
+  npm install --prod
+  npm run build
+```
+
+In the `backend` folder, make a copy of the sample config and modify it to fit your settings.
+
+```bash
+  cp mempool-config.sample.json mempool-config.json
+```
+
+Edit `mempool-config.json` to add your Bitcoin Core node RPC credentials:
+```bash
+{
+  "MEMPOOL": {
+    "NETWORK": "mainnet",
+    "BACKEND": "electrum",
+    "HTTP_PORT": 8999
+  },
+  "CORE_RPC": {
+    "HOST": "127.0.0.1",
+    "PORT": 8332,
+    "USERNAME": "mempool",
+    "PASSWORD": "mempool"
+  },
+  "ELECTRUM": {
+    "HOST": "127.0.0.1",
+    "PORT": 50002,
+    "TLS_ENABLED": true
+  },
+  "DATABASE": {
+    "ENABLED": true,
+    "HOST": "127.0.0.1",
+    "PORT": 3306,
+    "USERNAME": "mempool",
+    "PASSWORD": "mempool",
+    "DATABASE": "mempool"
+  }
+}
+```
+
+Start the backend:
+
+```bash
+  npm run start
+```
+
+When it's running you should see output like this:
+
+```bash
+  Mempool updated in 0.189 seconds
+  Updating mempool
+  Mempool updated in 0.096 seconds
+  Updating mempool
+  Mempool updated in 0.099 seconds
+  Updating mempool
+  Calculated fee for transaction 1 / 10
+  Calculated fee for transaction 2 / 10
+  Calculated fee for transaction 3 / 10
+  Calculated fee for transaction 4 / 10
+  Calculated fee for transaction 5 / 10
+  Calculated fee for transaction 6 / 10
+  Calculated fee for transaction 7 / 10
+  Calculated fee for transaction 8 / 10
+  Calculated fee for transaction 9 / 10
+  Calculated fee for transaction 10 / 10
+  Mempool updated in 0.243 seconds
+  Updating mempool
+```
+
+## Mempool Frontend
+
+Install mempool dependencies from npm and build the frontend static HTML/CSS/JS:
+
+```bash
+  # frontend
+  cd frontend
+  npm install --prod
+  npm run build
+```
+
+Install the output into nginx webroot folder:
+
+```bash
+  sudo rsync -av --delete dist/ /var/www/
+```
+
+## nginx + certbot
+
+Install the supplied nginx.conf and nginx-mempool.conf in /etc/nginx
+
+```bash
+  # install nginx and certbot
+  apt-get install -y nginx python3-certbot-nginx
+
+  # install the mempool configuration for nginx
+  cp nginx.conf nginx-mempool.conf /etc/nginx/
+
+  # replace example.com with your domain name
+  certbot --nginx -d example.com
+
+```
+
+If everything went okay you should see the beautiful mempool :grin:
+
+If you get stuck on "loading blocks", this means the websocket can't connect.
+Check your nginx proxy setup, firewalls, etc. and open an issue if you need help.

backend/README.md

@@ -1,161 +1,22 @@
-# Mempool Backend
+# Setup backend watchers
 
-These instructions are mostly intended for developers, but can be used as a basis for personal or small-scale production setups. 
+The backend is static. Typescript scripts are compiled into the `dist` folder and served through a node web server.
 
-If you choose to use these instructions for a production setup, be aware that you will still probably need to do additional configuration for your specific OS, environment, use-case, etc. We do our best here to provide a good starting point, but only proceed if you know what you're doing. Mempool does not provide support for custom setups.
+You can avoid the manual shutdown/recompile/restart command line cycle by using a watcher.
 
-See other ways to set up Mempool on [the main README](/../../#installation-methods).
+Make sure you are in the `backend` directory `cd backend`.
 
-Jump to a section in this doc:
-- [Set Up the Backend](#setup)
-- [Development Tips](#development-tips)
-
-## Setup
-
-### 1. Clone Mempool Repository
-
-Get the latest Mempool code:
+1. Install nodemon and ts-node
 
 ```
-git clone https://github.com/mempool/mempool
-cd mempool
+sudo npm install -g ts-node nodemon
 ```
 
-Check out the latest release:
+2. Run the watcher
 
-```
-latestrelease=$(curl -s https://api.github.com/repos/mempool/mempool/releases/latest|grep tag_name|head -1|cut -d '"' -f4)
-git checkout $latestrelease
-```
-
-### 2. Configure Bitcoin Core
-
-Turn on `txindex`, enable RPC, and set RPC credentials in `bitcoin.conf`:
-
-```
-txindex=1
-server=1
-rpcuser=mempool
-rpcpassword=mempool
-```
-
-### 3. Configure Electrum Server
-
-[Pick an Electrum Server implementation](https://mempool.space/docs/faq#address-lookup-issues), configure it, and make sure it's synced.
-
-**This step is optional.** You can run Mempool without configuring an Electrum Server for it, but address lookups will be disabled.
-
-### 4. Configure MariaDB
-
-_Mempool needs MariaDB v10.5 or later. If you already have MySQL installed, make sure to migrate any existing databases **before** installing MariaDB._
-
-Get MariaDB from your operating system's package manager:
-
-```
-# Debian, Ubuntu, etc.
-apt-get install mariadb-server mariadb-client
-
-# macOS
-brew install mariadb
-mysql.server start
-```
-
-Create a database and grant privileges:
-
-```
-MariaDB [(none)]> drop database mempool;
-Query OK, 0 rows affected (0.00 sec)
-
-MariaDB [(none)]> create database mempool;
-Query OK, 1 row affected (0.00 sec)
-
-MariaDB [(none)]> grant all privileges on mempool.* to 'mempool'@'%' identified by 'mempool';
-Query OK, 0 rows affected (0.00 sec)
-```
-
-### 5. Prepare Mempool Backend
-
-#### Build
-
-_Make sure to use Node.js 16.15 and npm 7._
-
-Install dependencies with `npm` and build the backend:
-
-```
-cd backend
-npm install # add --prod for production
-npm run build
-```
-
-#### Configure
-
-In the backend folder, make a copy of the sample config file:
-
-```
-cp mempool-config.sample.json mempool-config.json
-```
-
-Edit `mempool-config.json` as needed. 
-
-In particular, make sure:
-- the correct Bitcoin Core RPC credentials are specified in `CORE_RPC`
-- the correct `BACKEND` is specified in `MEMPOOL`:
-  - "electrum" if you're using [romanz/electrs](https://github.com/romanz/electrs) or [cculianu/Fulcrum](https://github.com/cculianu/Fulcrum)
-  - "esplora" if you're using [Blockstream/electrs](https://github.com/Blockstream/electrs)
-  - "none" if you're not using any Electrum Server
-
-### 6. Run Mempool Backend
-
-Run the Mempool backend:
-
-```
-npm run start
-```
-
-When it's running, you should see output like this:
-
-```
-Mempool updated in 0.189 seconds
-Updating mempool
-Mempool updated in 0.096 seconds
-Updating mempool
-Mempool updated in 0.099 seconds
-Updating mempool
-Calculated fee for transaction 1 / 10
-Calculated fee for transaction 2 / 10
-Calculated fee for transaction 3 / 10
-Calculated fee for transaction 4 / 10
-Calculated fee for transaction 5 / 10
-Calculated fee for transaction 6 / 10
-Calculated fee for transaction 7 / 10
-Calculated fee for transaction 8 / 10
-Calculated fee for transaction 9 / 10
-Calculated fee for transaction 10 / 10
-Mempool updated in 0.243 seconds
-Updating mempool
-```
-
-### 7. Set Up Mempool Frontend
-With the backend configured and running, proceed to set up the [Mempool frontend](../frontend#manual-setup).
-
-## Development Tips
-
-### Set Up Backend Watchers
-
-The Mempool backend is static. TypeScript scripts are compiled into the `dist` folder and served through a Node.js web server. 
-
-As a result, for development purposes, you may find it helpful to set up backend watchers to avoid the manual shutdown/recompile/restart command-line cycle.
-
-First, install `nodemon` and `ts-node`:
-
-```
-npm install -g ts-node nodemon
-```
-
-Then, run the watcher:
+> Note: You can find your npm global binary folder using `npm -g bin`, where nodemon will be installed.
 
 ```
 nodemon src/index.ts --ignore cache/ --ignore pools.json
 ```
 
-`nodemon` should be in npm's global binary folder. If needed, you can determine where that is with `npm -g bin`.

backend/mempool-config.sample.json

@@ -12,14 +12,12 @@
     "BLOCK_WEIGHT_UNITS": 4000000,
     "INITIAL_BLOCKS_AMOUNT": 8,
     "MEMPOOL_BLOCKS_AMOUNT": 8,
-    "INDEXING_BLOCKS_AMOUNT": 11000,
-    "PRICE_FEED_UPDATE_INTERVAL": 600,
+    "INDEXING_BLOCKS_AMOUNT": 1100,
+    "PRICE_FEED_UPDATE_INTERVAL": 3600,
     "USE_SECOND_NODE_FOR_MINFEE": false,
-    "EXTERNAL_ASSETS": [],
-    "EXTERNAL_MAX_RETRY": 1,
-    "EXTERNAL_RETRY_INTERVAL": 0,
-    "USER_AGENT": "mempool",
-    "STDOUT_LOG_MIN_PRIORITY": "debug"
+    "EXTERNAL_ASSETS": [
+      "https://mempool.space/resources/pools.json"
+    ]
   },
   "CORE_RPC": {
     "HOST": "127.0.0.1",
@@ -45,7 +43,6 @@
     "ENABLED": true,
     "HOST": "127.0.0.1",
     "PORT": 3306,
-    "SOCKET": "/var/run/mysql/mysql.sock",
     "DATABASE": "mempool",
     "USERNAME": "mempool",
     "PASSWORD": "mempool"
@@ -64,25 +61,5 @@
   "BISQ": {
     "ENABLED": false,
     "DATA_PATH": "/bisq/statsnode-data/btc_mainnet/db"
-  },
-  "SOCKS5PROXY": {
-    "ENABLED": false,
-    "USE_ONION": true,
-    "HOST": "127.0.0.1",
-    "PORT": 9050,
-    "USERNAME": "",
-    "PASSWORD": ""
-  },
-  "PRICE_DATA_SERVER": {
-    "TOR_URL": "http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices",
-    "CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices"
-  },
-  "EXTERNAL_DATA_SERVER": {
-    "MEMPOOL_API": "https://mempool.space/api/v1",
-    "MEMPOOL_ONION": "http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1",
-    "LIQUID_API": "https://liquid.network/api/v1",
-    "LIQUID_ONION": "http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1",
-    "BISQ_URL": "https://bisq.markets/api",
-    "BISQ_ONION": "http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api"
   }
 }

backend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mempool-backend",
-  "version": "2.4.0",
+  "version": "2.4.0-dev",
   "description": "Bitcoin mempool visualizer and blockchain explorer backend",
   "license": "GNU Affero General Public License v3.0",
   "homepage": "https://mempool.space",
@@ -28,21 +28,23 @@
     "test": "echo \"Error: no test specified\" && exit 1"
   },
   "dependencies": {
+    "@mempool/bitcoin": "^3.0.3",
     "@mempool/electrum-client": "^1.1.7",
-    "axios": "~0.27.2",
+    "@types/ws": "8.2.2",
+    "axios": "0.24.0",
     "bitcoinjs-lib": "6.0.1",
     "crypto-js": "^4.0.0",
-    "express": "^4.18.0",
+    "express": "^4.17.1",
+    "locutus": "^2.0.12",
     "mysql2": "2.3.3",
-    "node-worker-threads-pool": "^1.5.1",
-    "socks-proxy-agent": "^6.2.0",
-    "typescript": "~4.7.2",
-    "ws": "~8.7.0"
+    "node-worker-threads-pool": "^1.4.3",
+    "typescript": "4.4.4",
+    "ws": "8.3.0"
   },
   "devDependencies": {
-    "@types/compression": "^1.7.2",
-    "@types/ws": "~8.5.3",
-    "@types/express": "^4.17.13",
+    "@types/compression": "^1.0.1",
+    "@types/express": "^4.17.2",
+    "@types/locutus": "^0.0.6",
     "tslint": "^6.1.0"
   }
 }

backend/src/api/backend-info.ts

@@ -2,7 +2,6 @@ import * as fs from 'fs';
 import * as os from 'os';
 import logger from '../logger';
 import { IBackendInfo } from '../mempool.interfaces';
-const { spawnSync } = require('child_process');
 
 class BackendInfo {
   private gitCommitHash = '';
@@ -28,23 +27,10 @@ class BackendInfo {
   }
 
   private setLatestCommitHash(): void {
-    //TODO: share this logic with `generate-config.js`
-    if (process.env.DOCKER_COMMIT_HASH) {
-      this.gitCommitHash = process.env.DOCKER_COMMIT_HASH;
-    } else {
-      try {
-        const gitRevParse = spawnSync('git', ['rev-parse', '--short', 'HEAD']);
-        if (!gitRevParse.error) {
-          const output = gitRevParse.stdout.toString('utf-8').replace(/[\n\r\s]+$/, '');
-          this.gitCommitHash = output ? output : '?';
-        } else if (gitRevParse.error.code === 'ENOENT') {
-          console.log('git not found, cannot parse git hash');
-          this.gitCommitHash = '?';
-        }
-      } catch (e: any) {
-        console.log('Could not load git commit info: ' + e.message);
-        this.gitCommitHash = '?';
-      }
+    try {
+      this.gitCommitHash = fs.readFileSync('../.git/refs/heads/master').toString().trim();
+    } catch (e) {
+      logger.err('Could not load git commit info: ' + (e instanceof Error ? e.message : e));
     }
   }
 

backend/src/api/bisq/bisq.ts

@@ -1,14 +1,10 @@
 import config from '../../config';
 import * as fs from 'fs';
-import axios, { AxiosResponse } from 'axios';
-import * as http from 'http';
-import * as https from 'https';
-import { SocksProxyAgent } from 'socks-proxy-agent';
+import axios from 'axios';
 import { BisqBlocks, BisqBlock, BisqTransaction, BisqStats, BisqTrade } from './interfaces';
 import { Common } from '../common';
 import { BlockExtended } from '../../mempool.interfaces';
 import { StaticPool } from 'node-worker-threads-pool';
-import backendInfo from '../backend-info';
 import logger from '../../logger';
 
 class Bisq {
@@ -39,13 +35,7 @@ class Bisq {
   constructor() {}
 
   startBisqService(): void {
-    try {
-      this.checkForBisqDataFolder();
-    } catch (e) {
-      logger.info('Retrying to start bisq service in 3 minutes');
-      setTimeout(this.startBisqService.bind(this), 180000);
-      return;
-    }
+    this.checkForBisqDataFolder();
     this.loadBisqDumpFile();
     setInterval(this.updatePrice.bind(this), 1000 * 60 * 60);
     this.updatePrice();
@@ -100,7 +90,7 @@ class Bisq {
   private checkForBisqDataFolder() {
     if (!fs.existsSync(Bisq.BLOCKS_JSON_FILE_PATH)) {
       logger.warn(Bisq.BLOCKS_JSON_FILE_PATH + ` doesn't exist. Make sure Bisq is running and the config is correct before starting the server.`);
-      throw new Error(`Cannot load BISQ ${Bisq.BLOCKS_JSON_FILE_PATH} file`);
+      return process.exit(1);
     }
   }
 
@@ -147,59 +137,12 @@ class Bisq {
       }, 2000);
     });
   }
-  private async updatePrice() {
-    type axiosOptions = {
-      headers: {
-        'User-Agent': string
-      };
-      timeout: number;
-      httpAgent?: http.Agent;
-      httpsAgent?: https.Agent;
-    }
-    const setDelay = (secs: number = 1): Promise<void> => new Promise(resolve => setTimeout(() => resolve(), secs * 1000));
-    const BISQ_URL = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.EXTERNAL_DATA_SERVER.BISQ_ONION : config.EXTERNAL_DATA_SERVER.BISQ_URL;
-    const isHTTP = (new URL(BISQ_URL).protocol.split(':')[0] === 'http') ? true : false;
-    const axiosOptions: axiosOptions = {
-      headers: {
-        'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
-      },
-      timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000
-    };
-    let retry = 0;
 
-    while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
-      try {
-        if (config.SOCKS5PROXY.ENABLED) {
-          const socksOptions: any = {
-            agentOptions: {
-              keepAlive: true,
-            },
-            hostname: config.SOCKS5PROXY.HOST,
-            port: config.SOCKS5PROXY.PORT
-          };
-
-          if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
-            socksOptions.username = config.SOCKS5PROXY.USERNAME;
-            socksOptions.password = config.SOCKS5PROXY.PASSWORD;
-          } else {
-            // Retry with different tor circuits https://stackoverflow.com/a/64960234
-            socksOptions.username = `circuit${retry}`;
-          }
-
-          // Handle proxy agent for onion addresses
-          if (isHTTP) {
-            axiosOptions.httpAgent = new SocksProxyAgent(socksOptions);
-          } else {
-            axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions);
-          }
-        }
-        
-        const data: AxiosResponse = await axios.get(`${BISQ_URL}/trades/?market=bsq_btc`, axiosOptions);
-        if (data.statusText === 'error' || !data.data) {
-          throw new Error(`Could not fetch data from Bisq market, Error: ${data.status}`);
-        }
+  private updatePrice() {
+    axios.get<BisqTrade[]>('https://bisq.markets/api/trades/?market=bsq_btc', { timeout: 10000 })
+      .then((response) => {
         const prices: number[] = [];
-        data.data.forEach((trade) => {
+        response.data.forEach((trade) => {
           prices.push(parseFloat(trade.price) * 100000000);
         });
         prices.sort((a, b) => a - b);
@@ -207,24 +150,19 @@ class Bisq {
         if (this.priceUpdateCallbackFunction) {
           this.priceUpdateCallbackFunction(this.price);
         }
-        logger.debug('Successfully updated Bisq market price');
-        break;
-      } catch (e) {
-        logger.err('Error updating Bisq market price: '  + (e instanceof Error ? e.message : e));
-        await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL);
-        retry++;
-      }
-    }
+    }).catch((err) => {
+      logger.err('Error updating Bisq market price: ' + err);
+    });
   }
 
   private async loadBisqDumpFile(): Promise<void> {
-    this.allBlocks = [];
     try {
-      await this.loadData();
+      const data = await this.loadData();
+      await this.loadBisqBlocksDump(data);
       this.buildIndex();
       this.calculateStats();
     } catch (e) {
-      logger.info('Cannot load bisq dump file because: ' + (e instanceof Error ? e.message : e));
+      logger.info('loadBisqDumpFile() error.' + (e instanceof Error ? e.message : e));
     }
   }
 
@@ -303,61 +241,36 @@ class Bisq {
     };
   }
 
-  private async loadData(): Promise<any> {
-    if (!fs.existsSync(Bisq.BLOCKS_JSON_FILE_PATH)) {
-      throw new Error(Bisq.BLOCKS_JSON_FILE_PATH + ` doesn't exist`);
-    }
-
-    const readline = require('readline');
-    const events = require('events');
-
-    const rl = readline.createInterface({
-      input: fs.createReadStream(Bisq.BLOCKS_JSON_FILE_PATH),
-      crlfDelay: Infinity
-    });
-
-    let blockBuffer = '';
-    let readingBlock = false;
-    let lineCount = 1;
+  private async loadBisqBlocksDump(cacheData: string): Promise<void> {
     const start = new Date().getTime();
-
-    logger.debug('Processing Bisq data dump...');
-
-    rl.on('line', (line) => {
-      if (lineCount === 2) {
-        line = line.replace('  "chainHeight": ', '');
-        this.latestBlockHeight = parseInt(line, 10);
+    if (cacheData && cacheData.length !== 0) {
+      logger.debug('Processing Bisq data dump...');
+      const data: BisqBlocks = await this.jsonParsePool.exec(cacheData);
+      if (data.blocks && data.blocks.length !== this.allBlocks.length) {
+        this.allBlocks = data.blocks;
+        this.allBlocks.reverse();
+        this.blocks = this.allBlocks.filter((block) => block.txs.length > 0);
+        this.latestBlockHeight = data.chainHeight;
+        const time = new Date().getTime() - start;
+        logger.debug('Bisq dump processed in ' + time + ' ms (worker thread)');
+      } else {
+        throw new Error(`Bisq dump didn't contain any blocks`);
       }
+    }
+  }
 
-      if (line === '    {') {
-        readingBlock = true;
-      } else if (line === '    },') {
-        blockBuffer += '}';
-        try {
-          const block: BisqBlock =  JSON.parse(blockBuffer);
-          this.allBlocks.push(block);
-          readingBlock = false;
-          blockBuffer = '';
-        } catch (e) {
-          logger.debug(blockBuffer);
-          throw Error(`Unable to parse Bisq data dump at line ${lineCount}` + (e instanceof Error ? e.message : e));
+  private loadData(): Promise<string> {
+    return new Promise((resolve, reject) => {
+      if (!fs.existsSync(Bisq.BLOCKS_JSON_FILE_PATH)) {
+        return reject(Bisq.BLOCKS_JSON_FILE_PATH + ` doesn't exist`);
+      }
+      fs.readFile(Bisq.BLOCKS_JSON_FILE_PATH, 'utf8', (err, data) => {
+        if (err) {
+          reject(err);
         }
-      }
-
-      if (readingBlock === true) {
-        blockBuffer += line;
-      }
-
-      ++lineCount;
+        resolve(data);
+      });
     });
-
-    await events.once(rl, 'close');
-
-    this.allBlocks.reverse();
-    this.blocks = this.allBlocks.filter((block) => block.txs.length > 0);
-
-    const time = new Date().getTime() - start;
-    logger.debug('Bisq dump processed in ' + time + ' ms');
   }
 }
 

backend/src/api/bisq/markets-api.ts

@@ -1,7 +1,7 @@
 import { Currencies, OffersData, TradesData, Depth, Currency, Interval, HighLowOpenClose,
   Markets, Offers, Offer, BisqTrade, MarketVolume, Tickers, Ticker, SummarizedIntervals, SummarizedInterval } from './interfaces';
 
-const strtotime = require('./strtotime');
+import * as datetime from 'locutus/php/datetime';
 
 class BisqMarketsApi {
   private cryptoCurrencyData: Currency[] = [];
@@ -312,7 +312,7 @@ class BisqMarketsApi {
 
   getTickerFromMarket(market: string): Ticker | null {
     let ticker: Ticker;
-    const timestamp_from = strtotime('-24 hour');
+    const timestamp_from = datetime.strtotime('-24 hour');
     const timestamp_to = new Date().getTime() / 1000;
     const trades = this.getTradesByCriteria(market, timestamp_to, timestamp_from,
       undefined, undefined, undefined, 'asc', Number.MAX_SAFE_INTEGER);
@@ -638,13 +638,13 @@ class BisqMarketsApi {
         case 'half_day':
             return (ts - (ts % (3600 * 12)));
         case 'day':
-            return strtotime('midnight today', ts);
+            return datetime.strtotime('midnight today', ts);
         case 'week':
-            return strtotime('midnight sunday last week', ts);
+            return datetime.strtotime('midnight sunday last week', ts);
         case 'month':
-            return strtotime('midnight first day of this month', ts);
+            return datetime.strtotime('midnight first day of this month', ts);
         case 'year':
-            return strtotime('midnight first day of january', ts);
+            return datetime.strtotime('midnight first day of january', ts);
         default:
             throw new Error('Unsupported interval: ' + interval);
     }

backend/src/api/bisq/markets.ts

@@ -26,13 +26,7 @@ class Bisq {
   constructor() {}
 
   startBisqService(): void {
-    try {
-      this.checkForBisqDataFolder();
-    } catch (e) {
-      logger.info('Retrying to start bisq service (markets) in 3 minutes');
-      setTimeout(this.startBisqService.bind(this), 180000);
-      return;
-    }
+    this.checkForBisqDataFolder();
     this.loadBisqDumpFile();
     this.startBisqDirectoryWatcher();
   }
@@ -40,7 +34,7 @@ class Bisq {
   private checkForBisqDataFolder() {
     if (!fs.existsSync(Bisq.MARKET_JSON_PATH + Bisq.MARKET_JSON_FILE_PATHS.cryptoCurrency)) {
       logger.err(Bisq.MARKET_JSON_PATH + Bisq.MARKET_JSON_FILE_PATHS.cryptoCurrency + ` doesn't exist. Make sure Bisq is running and the config is correct before starting the server.`);
-      throw new Error(`Cannot load BISQ ${Bisq.MARKET_JSON_FILE_PATHS.cryptoCurrency} file`);
+      return process.exit(1);
     }
   }
 

backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts

@@ -2,7 +2,7 @@ import { IEsploraApi } from './esplora-api.interface';
 
 export interface AbstractBitcoinApi {
   $getRawMempool(): Promise<IEsploraApi.Transaction['txid'][]>;
-  $getRawTransaction(txId: string, skipConversion?: boolean, addPrevout?: boolean, lazyPrevouts?: boolean): Promise<IEsploraApi.Transaction>;
+  $getRawTransaction(txId: string, skipConversion?: boolean, addPrevout?: boolean): Promise<IEsploraApi.Transaction>;
   $getBlockHeightTip(): Promise<number>;
   $getTxIdsForBlock(hash: string): Promise<string[]>;
   $getBlockHash(height: number): Promise<string>;

backend/src/api/bitcoin/bitcoin-api.interface.ts

@@ -4,7 +4,6 @@ export namespace IBitcoinApi {
     size: number;                    //  (numeric) Current tx count
     bytes: number;                   //  (numeric) Sum of all virtual transaction sizes as defined in BIP 141.
     usage: number;                   //  (numeric) Total memory usage for the mempool
-    total_fee: number;               //  (numeric) Total fees of transactions in the mempool
     maxmempool: number;              //  (numeric) Maximum memory usage for the mempool
     mempoolminfee: number;           //  (numeric) Minimum fee rate in BTC/kB for tx to be accepted.
     minrelaytxfee: number;           //  (numeric) Current minimum relay fee for transactions

backend/src/api/bitcoin/bitcoin-api.ts

@@ -14,25 +14,7 @@ class BitcoinApi implements AbstractBitcoinApi {
     this.bitcoindClient = bitcoinClient;
   }
 
-  static convertBlock(block: IBitcoinApi.Block): IEsploraApi.Block {
-    return {
-      id: block.hash,
-      height: block.height,
-      version: block.version,
-      timestamp: block.time,
-      bits: parseInt(block.bits, 16),
-      nonce: block.nonce,
-      difficulty: block.difficulty,
-      merkle_root: block.merkleroot,
-      tx_count: block.nTx,
-      size: block.size,
-      weight: block.weight,
-      previousblockhash: block.previousblockhash,
-    };
-  }
-
-
-  $getRawTransaction(txId: string, skipConversion = false, addPrevout = false, lazyPrevouts = false): Promise<IEsploraApi.Transaction> {
+  $getRawTransaction(txId: string, skipConversion = false, addPrevout = false): Promise<IEsploraApi.Transaction> {
     // If the transaction is in the mempool we already converted and fetched the fee. Only prevouts are missing
     const txInMempool = mempool.getMempool()[txId];
     if (txInMempool && addPrevout) {
@@ -43,11 +25,11 @@ class BitcoinApi implements AbstractBitcoinApi {
       .then((transaction: IBitcoinApi.Transaction) => {
         if (skipConversion) {
           transaction.vout.forEach((vout) => {
-            vout.value = Math.round(vout.value * 100000000);
+            vout.value = vout.value * 100000000;
           });
           return transaction;
         }
-        return this.$convertTransaction(transaction, addPrevout, lazyPrevouts);
+        return this.$convertTransaction(transaction, addPrevout);
       })
       .catch((e: Error) => {
         if (e.message.startsWith('The genesis block coinbase')) {
@@ -59,9 +41,7 @@ class BitcoinApi implements AbstractBitcoinApi {
 
   $getBlockHeightTip(): Promise<number> {
     return this.bitcoindClient.getChainTips()
-      .then((result: IBitcoinApi.ChainTips[]) => {
-        return result.find(tip => tip.status === 'active')!.height;
-      });
+      .then((result: IBitcoinApi.ChainTips[]) => result[0].height);
   }
 
   $getTxIdsForBlock(hash: string): Promise<string[]> {
@@ -88,7 +68,7 @@ class BitcoinApi implements AbstractBitcoinApi {
     }
 
     return this.bitcoindClient.getBlock(hash)
-      .then((block: IBitcoinApi.Block) => BitcoinApi.convertBlock(block));
+      .then((block: IBitcoinApi.Block) => this.convertBlock(block));
   }
 
   $getAddress(address: string): Promise<IEsploraApi.Address> {
@@ -104,19 +84,19 @@ class BitcoinApi implements AbstractBitcoinApi {
   }
 
   $getAddressPrefix(prefix: string): string[] {
-    const found: { [address: string]: string } = {};
+    const found: string[] = [];
     const mp = mempool.getMempool();
     for (const tx in mp) {
       for (const vout of mp[tx].vout) {
         if (vout.scriptpubkey_address.indexOf(prefix) === 0) {
-          found[vout.scriptpubkey_address] = '';
-          if (Object.keys(found).length >= 10) {
-            return Object.keys(found);
+          found.push(vout.scriptpubkey_address);
+          if (found.length >= 10) {
+            return found;
           }
         }
       }
     }
-    return Object.keys(found);
+    return found;
   }
 
   $sendRawTransaction(rawTransaction: string): Promise<string> {
@@ -127,16 +107,10 @@ class BitcoinApi implements AbstractBitcoinApi {
     const outSpends: IEsploraApi.Outspend[] = [];
     const tx = await this.$getRawTransaction(txId, true, false);
     for (let i = 0; i < tx.vout.length; i++) {
-      if (tx.status && tx.status.block_height === 0) {
-        outSpends.push({
-          spent: false
-        });
-      } else {
-        const txOut = await this.bitcoindClient.getTxOut(txId, i);
-        outSpends.push({
-          spent: txOut === null,
-        });
-      }
+      const txOut = await this.bitcoindClient.getTxOut(txId, i);
+      outSpends.push({
+        spent: txOut === null,
+      });
     }
     return outSpends;
   }
@@ -146,7 +120,7 @@ class BitcoinApi implements AbstractBitcoinApi {
     return this.bitcoindClient.getNetworkHashPs(120, blockHeight);
   }
 
-  protected async $convertTransaction(transaction: IBitcoinApi.Transaction, addPrevout: boolean, lazyPrevouts = false): Promise<IEsploraApi.Transaction> {
+  protected async $convertTransaction(transaction: IBitcoinApi.Transaction, addPrevout: boolean): Promise<IEsploraApi.Transaction> {
     let esploraTransaction: IEsploraApi.Transaction = {
       txid: transaction.txid,
       version: transaction.version,
@@ -161,11 +135,11 @@ class BitcoinApi implements AbstractBitcoinApi {
 
     esploraTransaction.vout = transaction.vout.map((vout) => {
       return {
-        value: Math.round(vout.value * 100000000),
+        value: vout.value * 100000000,
         scriptpubkey: vout.scriptPubKey.hex,
         scriptpubkey_address: vout.scriptPubKey && vout.scriptPubKey.address ? vout.scriptPubKey.address
           : vout.scriptPubKey.addresses ? vout.scriptPubKey.addresses[0] : '',
-        scriptpubkey_asm: vout.scriptPubKey.asm ? this.convertScriptSigAsm(vout.scriptPubKey.hex) : '',
+        scriptpubkey_asm: vout.scriptPubKey.asm ? this.convertScriptSigAsm(vout.scriptPubKey.asm) : '',
         scriptpubkey_type: this.translateScriptPubKeyType(vout.scriptPubKey.type),
       };
     });
@@ -175,7 +149,7 @@ class BitcoinApi implements AbstractBitcoinApi {
         is_coinbase: !!vin.coinbase,
         prevout: null,
         scriptsig: vin.scriptSig && vin.scriptSig.hex || vin.coinbase || '',
-        scriptsig_asm: vin.scriptSig && this.convertScriptSigAsm(vin.scriptSig.hex) || '',
+        scriptsig_asm: vin.scriptSig && this.convertScriptSigAsm(vin.scriptSig.asm) || '',
         sequence: vin.sequence,
         txid: vin.txid || '',
         vout: vin.vout || 0,
@@ -192,15 +166,35 @@ class BitcoinApi implements AbstractBitcoinApi {
       };
     }
 
-    if (addPrevout) {
-      esploraTransaction = await this.$calculateFeeFromInputs(esploraTransaction, false, lazyPrevouts);
-    } else if (!transaction.confirmations) {
+    if (transaction.confirmations) {
+      esploraTransaction = await this.$calculateFeeFromInputs(esploraTransaction, addPrevout);
+    } else {
       esploraTransaction = await this.$appendMempoolFeeData(esploraTransaction);
+      if (addPrevout) {
+        esploraTransaction = await this.$calculateFeeFromInputs(esploraTransaction, addPrevout);
+      }
     }
 
     return esploraTransaction;
   }
 
+  private convertBlock(block: IBitcoinApi.Block): IEsploraApi.Block {
+    return {
+      id: block.hash,
+      height: block.height,
+      version: block.version,
+      timestamp: block.time,
+      bits: parseInt(block.bits, 16),
+      nonce: block.nonce,
+      difficulty: block.difficulty,
+      merkle_root: block.merkleroot,
+      tx_count: block.nTx,
+      size: block.size,
+      weight: block.weight,
+      previousblockhash: block.previousblockhash,
+    };
+  }
+
   private translateScriptPubKeyType(outputType: string): string {
     const map = {
       'pubkey': 'p2pk',
@@ -210,14 +204,13 @@ class BitcoinApi implements AbstractBitcoinApi {
       'witness_v0_scripthash': 'v0_p2wsh',
       'witness_v1_taproot': 'v1_p2tr',
       'nonstandard': 'nonstandard',
-      'multisig': 'multisig',
       'nulldata': 'op_return'
     };
 
     if (map[outputType]) {
       return map[outputType];
     } else {
-      return 'unknown';
+      return '';
     }
   }
 
@@ -234,7 +227,7 @@ class BitcoinApi implements AbstractBitcoinApi {
     } else {
       mempoolEntry = await this.$getMempoolEntry(transaction.txid);
     }
-    transaction.fee = Math.round(mempoolEntry.fees.base * 100000000);
+    transaction.fee = mempoolEntry.fees.base * 100000000;
     return transaction;
   }
 
@@ -243,7 +236,7 @@ class BitcoinApi implements AbstractBitcoinApi {
       if (vin.prevout) {
         continue;
       }
-      const innerTx = await this.$getRawTransaction(vin.txid, false, false);
+      const innerTx = await this.$getRawTransaction(vin.txid, false);
       vin.prevout = innerTx.vout[vin.vout];
       this.addInnerScriptsToVin(vin);
     }
@@ -269,95 +262,42 @@ class BitcoinApi implements AbstractBitcoinApi {
     return this.bitcoindClient.getRawMemPool(true);
   }
 
-
-  private async $calculateFeeFromInputs(transaction: IEsploraApi.Transaction, addPrevout: boolean, lazyPrevouts: boolean): Promise<IEsploraApi.Transaction> {
+  private async $calculateFeeFromInputs(transaction: IEsploraApi.Transaction, addPrevout: boolean): Promise<IEsploraApi.Transaction> {
     if (transaction.vin[0].is_coinbase) {
       transaction.fee = 0;
       return transaction;
     }
     let totalIn = 0;
-
-    for (let i = 0; i < transaction.vin.length; i++) {
-      if (lazyPrevouts && i > 12) {
-        transaction.vin[i].lazy = true;
-        continue;
+    for (const vin of transaction.vin) {
+      const innerTx = await this.$getRawTransaction(vin.txid, !addPrevout);
+      if (addPrevout) {
+        vin.prevout = innerTx.vout[vin.vout];
+        this.addInnerScriptsToVin(vin);
       }
-      const innerTx = await this.$getRawTransaction(transaction.vin[i].txid, false, false);
-      transaction.vin[i].prevout = innerTx.vout[transaction.vin[i].vout];
-      this.addInnerScriptsToVin(transaction.vin[i]);
-      totalIn += innerTx.vout[transaction.vin[i].vout].value;
-    }
-    if (lazyPrevouts && transaction.vin.length > 12) {
-      transaction.fee = -1;
-    } else {
-      const totalOut = transaction.vout.reduce((p, output) => p + output.value, 0);
-      transaction.fee = parseFloat((totalIn - totalOut).toFixed(8));
+      totalIn += innerTx.vout[vin.vout].value;
     }
+    const totalOut = transaction.vout.reduce((p, output) => p + output.value, 0);
+    transaction.fee = parseFloat((totalIn - totalOut).toFixed(8));
     return transaction;
   }
 
-  private convertScriptSigAsm(hex: string): string {
-    const buf = Buffer.from(hex, 'hex');
-
+  private convertScriptSigAsm(str: string): string {
+    const a = str.split(' ');
     const b: string[] = [];
-
-    let i = 0;
-    while (i < buf.length) {
-      const op = buf[i];
-      if (op >= 0x01 && op <= 0x4e) {
-        i++;
-        let push: number;
-        if (op === 0x4c) {
-          push = buf.readUInt8(i);
-          b.push('OP_PUSHDATA1');
-          i += 1;
-        } else if (op === 0x4d) {
-          push = buf.readUInt16LE(i);
-          b.push('OP_PUSHDATA2');
-          i += 2;
-        } else if (op === 0x4e) {
-          push = buf.readUInt32LE(i);
-          b.push('OP_PUSHDATA4');
-          i += 4;
-        } else {
-          push = op;
-          b.push('OP_PUSHBYTES_' + push);
-        }
-
-        const data = buf.slice(i, i + push);
-        if (data.length !== push) {
-          break;
-        }
-
-        b.push(data.toString('hex'));
-        i += data.length;
+    a.forEach((chunk) => {
+      if (chunk.substr(0, 3) === 'OP_') {
+        chunk = chunk.replace(/^OP_(\d+)/, 'OP_PUSHNUM_$1');
+        chunk = chunk.replace('OP_CHECKSEQUENCEVERIFY', 'OP_CSV');
+        b.push(chunk);
       } else {
-        if (op === 0x00) {
+        chunk = chunk.replace('[ALL]', '01');
+        if (chunk === '0') {
           b.push('OP_0');
-        } else if (op === 0x4f) {
-          b.push('OP_PUSHNUM_NEG1');
-        } else if (op === 0xb1) {
-          b.push('OP_CLTV');
-        } else if (op === 0xb2) {
-          b.push('OP_CSV');
-        } else if (op === 0xba) {
-          b.push('OP_CHECKSIGADD');
         } else {
-          const opcode = bitcoinjs.script.toASM([ op ]);
-          if (opcode && op < 0xfd) {
-            if (/^OP_(\d+)$/.test(opcode)) {
-              b.push(opcode.replace(/^OP_(\d+)$/, 'OP_PUSHNUM_$1'));
-            } else {
-              b.push(opcode);
-            }
-          } else {
-            b.push('OP_RETURN_' + op);
-          }
+          b.push('OP_PUSHBYTES_' + Math.round(chunk.length / 2) + ' ' + chunk);
         }
-        i += 1;
       }
-    }
-
+    });
     return b.join(' ');
   }
 
@@ -368,21 +308,21 @@ class BitcoinApi implements AbstractBitcoinApi {
 
     if (vin.prevout.scriptpubkey_type === 'p2sh') {
       const redeemScript = vin.scriptsig_asm.split(' ').reverse()[0];
-      vin.inner_redeemscript_asm = this.convertScriptSigAsm(redeemScript);
+      vin.inner_redeemscript_asm = this.convertScriptSigAsm(bitcoinjs.script.toASM(Buffer.from(redeemScript, 'hex')));
       if (vin.witness && vin.witness.length > 2) {
         const witnessScript = vin.witness[vin.witness.length - 1];
-        vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript);
+        vin.inner_witnessscript_asm = this.convertScriptSigAsm(bitcoinjs.script.toASM(Buffer.from(witnessScript, 'hex')));
       }
     }
 
     if (vin.prevout.scriptpubkey_type === 'v0_p2wsh' && vin.witness) {
       const witnessScript = vin.witness[vin.witness.length - 1];
-      vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript);
+      vin.inner_witnessscript_asm = this.convertScriptSigAsm(bitcoinjs.script.toASM(Buffer.from(witnessScript, 'hex')));
     }
 
     if (vin.prevout.scriptpubkey_type === 'v1_p2tr' && vin.witness && vin.witness.length > 1) {
       const witnessScript = vin.witness[vin.witness.length - 2];
-      vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript);
+      vin.inner_witnessscript_asm = this.convertScriptSigAsm(bitcoinjs.script.toASM(Buffer.from(witnessScript, 'hex')));
     }
   }
 

backend/src/api/bitcoin/bitcoin-client.ts

@@ -1,5 +1,5 @@
 import config from '../../config';
-const bitcoin = require('../../rpc-api/index');
+import * as bitcoin from '@mempool/bitcoin';
 import { BitcoinRpcCredentials } from './bitcoin-api-abstract-factory';
 
 const nodeRpcCredentials: BitcoinRpcCredentials = {

backend/src/api/bitcoin/bitcoin-second-client.ts

@@ -1,5 +1,5 @@
 import config from '../../config';
-const bitcoin = require('../../rpc-api/index');
+import * as bitcoin from '@mempool/bitcoin';
 import { BitcoinRpcCredentials } from './bitcoin-api-abstract-factory';
 
 const nodeRpcCredentials: BitcoinRpcCredentials = {

backend/src/api/bitcoin/esplora-api.interface.ts

@@ -33,8 +33,6 @@ export namespace IEsploraApi {
     // Elements
     is_pegin?: boolean;
     issuance?: Issuance;
-    // Custom
-    lazy?: boolean;
   }
 
   interface Issuance {

backend/src/api/blocks.ts

@@ -10,15 +10,6 @@ import bitcoinClient from './bitcoin/bitcoin-client';
 import { IEsploraApi } from './bitcoin/esplora-api.interface';
 import poolsRepository from '../repositories/PoolsRepository';
 import blocksRepository from '../repositories/BlocksRepository';
-import loadingIndicators from './loading-indicators';
-import BitcoinApi from './bitcoin/bitcoin-api';
-import { prepareBlock } from '../utils/blocks-utils';
-import BlocksRepository from '../repositories/BlocksRepository';
-import HashratesRepository from '../repositories/HashratesRepository';
-import indexer from '../indexer';
-import fiatConversion from './fiat-conversion';
-import RatesRepository from '../repositories/RatesRepository';
-import poolsParser from './pools-parser';
 
 class Blocks {
   private blocks: BlockExtended[] = [];
@@ -27,6 +18,7 @@ class Blocks {
   private lastDifficultyAdjustmentTime = 0;
   private previousDifficultyRetarget = 0;
   private newBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => void)[] = [];
+  private blockIndexingStarted = false;
 
   constructor() { }
 
@@ -49,12 +41,7 @@ class Blocks {
    * @param onlyCoinbase - Set to true if you only need the coinbase transaction
    * @returns Promise<TransactionExtended[]>
    */
-  private async $getTransactionsExtended(
-    blockHash: string,
-    blockHeight: number,
-    onlyCoinbase: boolean,
-    quiet: boolean = false,
-  ): Promise<TransactionExtended[]> {
+  private async $getTransactionsExtended(blockHash: string, blockHeight: number, onlyCoinbase: boolean): Promise<TransactionExtended[]> {
     const transactions: TransactionExtended[] = [];
     const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash);
 
@@ -68,9 +55,9 @@ class Blocks {
         // optimize here by directly fetching txs in the "outdated" mempool
         transactions.push(mempool[txIds[i]]);
         transactionsFound++;
-      } else if (config.MEMPOOL.BACKEND === 'esplora' || !memPool.hasPriority() || i === 0) {
+      } else if (config.MEMPOOL.BACKEND === 'esplora' || memPool.isInSync() || i === 0) {
         // Otherwise we fetch the tx data through backend services (esplora, electrum, core rpc...)
-        if (!quiet && (i % (Math.round((txIds.length) / 10)) === 0 || i + 1 === txIds.length)) { // Avoid log spam
+        if (i % (Math.round((txIds.length) / 10)) === 0 || i + 1 === txIds.length) { // Avoid log spam
           logger.debug(`Indexing tx ${i + 1} of ${txIds.length} in block #${blockHeight}`);
         }
         try {
@@ -78,12 +65,9 @@ class Blocks {
           transactions.push(tx);
           transactionsFetched++;
         } catch (e) {
+          logger.debug('Error fetching block tx: ' + (e instanceof Error ? e.message : e));
           if (i === 0) {
-            const msg = `Cannot fetch coinbase tx ${txIds[i]}. Reason: ` + (e instanceof Error ? e.message : e); 
-            logger.err(msg);
-            throw new Error(msg);
-          } else {
-            logger.err(`Cannot fetch tx ${txIds[i]}. Reason: ` + (e instanceof Error ? e.message : e));
+            throw new Error('Failed to fetch Coinbase transaction: ' + txIds[i]);
           }
         }
       }
@@ -99,9 +83,7 @@ class Blocks {
       }
     });
 
-    if (!quiet) {
-      logger.debug(`${transactionsFound} of ${txIds.length} found in mempool. ${transactionsFetched} fetched through backend service.`);
-    }
+    logger.debug(`${transactionsFound} of ${txIds.length} found in mempool. ${transactionsFetched} fetched through backend service.`);
 
     return transactions;
   }
@@ -112,53 +94,16 @@ class Blocks {
    * @param transactions
    * @returns BlockExtended
    */
-  private async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise<BlockExtended> {
-    const blockExtended: BlockExtended = Object.assign({ extras: {} }, block);
-    blockExtended.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
-    blockExtended.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
-    blockExtended.extras.coinbaseRaw = blockExtended.extras.coinbaseTx.vin[0].scriptsig;
+  private getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): BlockExtended {
+    const blockExtended: BlockExtended = Object.assign({}, block);
+    blockExtended.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
+    blockExtended.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
 
-    if (block.height === 0) {
-      blockExtended.extras.medianFee = 0; // 50th percentiles
-      blockExtended.extras.feeRange = [0, 0, 0, 0, 0, 0, 0];
-      blockExtended.extras.totalFees = 0;
-      blockExtended.extras.avgFee = 0;
-      blockExtended.extras.avgFeeRate = 0;
-    } else {
-      const stats = await bitcoinClient.getBlockStats(block.id, [
-        'feerate_percentiles', 'minfeerate', 'maxfeerate', 'totalfee', 'avgfee', 'avgfeerate'
-      ]);
-      blockExtended.extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles
-      blockExtended.extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat();
-      blockExtended.extras.totalFees = stats.totalfee;
-      blockExtended.extras.avgFee = stats.avgfee;
-      blockExtended.extras.avgFeeRate = stats.avgfeerate;
-    }
-
-    if (['mainnet', 'testnet', 'signet', 'regtest'].includes(config.MEMPOOL.NETWORK)) {
-      let pool: PoolTag;
-      if (blockExtended.extras?.coinbaseTx !== undefined) {
-        pool = await this.$findBlockMiner(blockExtended.extras?.coinbaseTx);
-      } else {
-        if (config.DATABASE.ENABLED === true) {
-          pool = await poolsRepository.$getUnknownPool();
-        } else {
-          pool = poolsParser.unknownPool;
-        }
-      }
-
-      if (!pool) { // We should never have this situation in practise
-        logger.warn(`Cannot assign pool to block ${blockExtended.height} and 'unknown' pool does not exist. ` +
-          `Check your "pools" table entries`);
-        return blockExtended;
-      }
-
-      blockExtended.extras.pool = {
-        id: pool.id,
-        name: pool.name,
-        slug: pool.slug,
-      };
-    }
+    const transactionsTmp = [...transactions];
+    transactionsTmp.shift();
+    transactionsTmp.sort((a, b) => b.effectiveFeePerVsize - a.effectiveFeePerVsize);
+    blockExtended.medianFee = transactionsTmp.length > 0 ? Common.median(transactionsTmp.map((tx) => tx.effectiveFeePerVsize)) : 0;
+    blockExtended.feeRange = transactionsTmp.length > 0 ? Common.getFeesInRange(transactionsTmp, 8) : [0, 0];
 
     return blockExtended;
   }
@@ -170,22 +115,13 @@ class Blocks {
    */
   private async $findBlockMiner(txMinerInfo: TransactionMinerInfo | undefined): Promise<PoolTag> {
     if (txMinerInfo === undefined || txMinerInfo.vout.length < 1) {
-      if (config.DATABASE.ENABLED === true) {
-        return await poolsRepository.$getUnknownPool();
-      } else {
-        return poolsParser.unknownPool;
-      }
+      return await poolsRepository.$getUnknownPool();
     }
 
     const asciiScriptSig = transactionUtils.hex2ascii(txMinerInfo.vin[0].scriptsig);
     const address = txMinerInfo.vout[0].scriptpubkey_address;
 
-    let pools: PoolTag[] = [];
-    if (config.DATABASE.ENABLED === true) {
-      pools = await poolsRepository.$getPools();
-    } else {
-      pools = poolsParser.miningPools;
-    }
+    const pools: PoolTag[] = await poolsRepository.$getPools();
     for (let i = 0; i < pools.length; ++i) {
       if (address !== undefined) {
         const addresses: string[] = JSON.parse(pools[i].addresses);
@@ -196,110 +132,93 @@ class Blocks {
 
       const regexes: string[] = JSON.parse(pools[i].regexes);
       for (let y = 0; y < regexes.length; ++y) {
-        const regex = new RegExp(regexes[y], 'i');
-        const match = asciiScriptSig.match(regex);
+        const match = asciiScriptSig.match(regexes[y]);
         if (match !== null) {
           return pools[i];
         }
       }
     }
 
-    if (config.DATABASE.ENABLED === true) {
-      return await poolsRepository.$getUnknownPool();
-    } else {
-      return poolsParser.unknownPool;
-    }
+    return await poolsRepository.$getUnknownPool();
   }
 
   /**
-   * [INDEXING] Index all blocks metadata for the mining dashboard
+   * Index all blocks metadata for the mining dashboard
    */
   public async $generateBlockDatabase() {
-    const blockchainInfo = await bitcoinClient.getBlockchainInfo();
-    if (blockchainInfo.blocks !== blockchainInfo.headers) { // Wait for node to sync
+    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false || // Bitcoin only
+      config.MEMPOOL.INDEXING_BLOCKS_AMOUNT === 0 || // Indexing must be enabled
+      !memPool.isInSync() || // We sync the mempool first
+      this.blockIndexingStarted === true // Indexing must not already be in progress
+    ) {
       return;
     }
 
+    const blockchainInfo = await bitcoinClient.getBlockchainInfo();
+    if (blockchainInfo.blocks !== blockchainInfo.headers) {
+      return;
+    }
+
+    this.blockIndexingStarted = true;
+
     try {
       let currentBlockHeight = blockchainInfo.blocks;
 
-      let indexingBlockAmount = Math.min(config.MEMPOOL.INDEXING_BLOCKS_AMOUNT, blockchainInfo.blocks);
+      let indexingBlockAmount = config.MEMPOOL.INDEXING_BLOCKS_AMOUNT;
       if (indexingBlockAmount <= -1) {
         indexingBlockAmount = currentBlockHeight + 1;
       }
 
       const lastBlockToIndex = Math.max(0, currentBlockHeight - indexingBlockAmount + 1);
 
-      logger.debug(`Indexing blocks from #${currentBlockHeight} to #${lastBlockToIndex}`);
-      loadingIndicators.setProgress('block-indexing', 0);
+      logger.info(`Indexing blocks from #${currentBlockHeight} to #${lastBlockToIndex}`);
 
       const chunkSize = 10000;
-      let totalIndexed = await blocksRepository.$blockCountBetweenHeight(currentBlockHeight, lastBlockToIndex);
-      let indexedThisRun = 0;
-      let newlyIndexed = 0;
-      const startedAt = new Date().getTime() / 1000;
-      let timer = new Date().getTime() / 1000;
-
       while (currentBlockHeight >= lastBlockToIndex) {
         const endBlock = Math.max(0, lastBlockToIndex, currentBlockHeight - chunkSize + 1);
 
         const missingBlockHeights: number[] = await blocksRepository.$getMissingBlocksBetweenHeights(
           currentBlockHeight, endBlock);
         if (missingBlockHeights.length <= 0) {
+          logger.debug(`No missing blocks between #${currentBlockHeight} to #${endBlock}`);
           currentBlockHeight -= chunkSize;
           continue;
         }
 
-        logger.info(`Indexing ${missingBlockHeights.length} blocks from #${currentBlockHeight} to #${endBlock}`);
+        logger.debug(`Indexing ${missingBlockHeights.length} blocks from #${currentBlockHeight} to #${endBlock}`);
 
         for (const blockHeight of missingBlockHeights) {
           if (blockHeight < lastBlockToIndex) {
             break;
           }
-          ++indexedThisRun;
-          ++totalIndexed;
-          const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
-          if (elapsedSeconds > 5 || blockHeight === lastBlockToIndex) {
-            const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
-            const blockPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds));
-            const progress = Math.round(totalIndexed / indexingBlockAmount * 10000) / 100;
-            const timeLeft = Math.round((indexingBlockAmount - totalIndexed) / blockPerSeconds);
-            logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${runningFor} seconds | left: ~${timeLeft} seconds`);
-            timer = new Date().getTime() / 1000;
-            indexedThisRun = 0;
-            loadingIndicators.setProgress('block-indexing', progress, false);
+          try {
+            logger.debug(`Indexing block #${blockHeight}`);
+            const blockHash = await bitcoinApi.$getBlockHash(blockHeight);
+            const block = await bitcoinApi.$getBlock(blockHash);
+            const transactions = await this.$getTransactionsExtended(blockHash, block.height, true);
+            const blockExtended = this.getBlockExtended(block, transactions);
+            const miner = await this.$findBlockMiner(blockExtended.coinbaseTx);
+            const coinbase: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(transactions[0].txid, true);
+            await blocksRepository.$saveBlockInDatabase(blockExtended, blockHash, coinbase.hex, miner);
+          } catch (e) {
+            logger.err(`Something went wrong while indexing blocks.` + e);
           }
-          const blockHash = await bitcoinApi.$getBlockHash(blockHeight);
-          const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash));
-          const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, true);
-          const blockExtended = await this.$getBlockExtended(block, transactions);
-
-          newlyIndexed++;
-          await blocksRepository.$saveBlockInDatabase(blockExtended);
         }
 
         currentBlockHeight -= chunkSize;
       }
-      logger.info(`Indexed ${newlyIndexed} blocks`);
-      loadingIndicators.setProgress('block-indexing', 100);
+      logger.info('Block indexing completed');
     } catch (e) {
-      logger.err('Block indexing failed. Trying again later. Reason: ' + (e instanceof Error ? e.message : e));
-      loadingIndicators.setProgress('block-indexing', 100);
-      return;
-    }
-
-    const chainValid = await BlocksRepository.$validateChain();
-    if (!chainValid) {
-      indexer.reindex();
+      logger.err('An error occured in $generateBlockDatabase(). Skipping block indexing. ' + e);
+      console.log(e);
     }
   }
 
   public async $updateBlocks() {
-    let fastForwarded = false;
     const blockHeightTip = await bitcoinApi.$getBlockHeightTip();
 
     if (this.blocks.length === 0) {
-      this.currentBlockHeight = Math.max(blockHeightTip - config.MEMPOOL.INITIAL_BLOCKS_AMOUNT, -1);
+      this.currentBlockHeight = blockHeightTip - config.MEMPOOL.INITIAL_BLOCKS_AMOUNT;
     } else {
       this.currentBlockHeight = this.blocks[this.blocks.length - 1].height;
     }
@@ -307,9 +226,6 @@ class Blocks {
     if (blockHeightTip - this.currentBlockHeight > config.MEMPOOL.INITIAL_BLOCKS_AMOUNT * 2) {
       logger.info(`${blockHeightTip - this.currentBlockHeight} blocks since tip. Fast forwarding to the ${config.MEMPOOL.INITIAL_BLOCKS_AMOUNT} recent blocks`);
       this.currentBlockHeight = blockHeightTip - config.MEMPOOL.INITIAL_BLOCKS_AMOUNT;
-      fastForwarded = true;
-      logger.info(`Re-indexing skipped blocks and corresponding hashrates data`);
-      indexer.reindex(); // Make sure to index the skipped blocks #1619
     }
 
     if (!this.lastDifficultyAdjustmentTime) {
@@ -317,23 +233,21 @@ class Blocks {
       if (blockchainInfo.blocks === blockchainInfo.headers) {
         const heightDiff = blockHeightTip % 2016;
         const blockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff);
-        const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash));
+        const block = await bitcoinApi.$getBlock(blockHash);
         this.lastDifficultyAdjustmentTime = block.timestamp;
         this.currentDifficulty = block.difficulty;
 
-        if (blockHeightTip >= 2016) {
-          const previousPeriodBlockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff - 2016);
-          const previousPeriodBlock = await bitcoinApi.$getBlock(previousPeriodBlockHash);
-          this.previousDifficultyRetarget = (block.difficulty - previousPeriodBlock.difficulty) / previousPeriodBlock.difficulty * 100;
-          logger.debug(`Initial difficulty adjustment data set.`);
-        }
+        const previousPeriodBlockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff - 2016);
+        const previousPeriodBlock = await bitcoinApi.$getBlock(previousPeriodBlockHash);
+        this.previousDifficultyRetarget = (block.difficulty - previousPeriodBlock.difficulty) / previousPeriodBlock.difficulty * 100;
+        logger.debug(`Initial difficulty adjustment data set.`);
       } else {
         logger.debug(`Blockchain headers (${blockchainInfo.headers}) and blocks (${blockchainInfo.blocks}) not in sync. Waiting...`);
       }
     }
 
     while (this.currentBlockHeight < blockHeightTip) {
-      if (this.currentBlockHeight < blockHeightTip - config.MEMPOOL.INITIAL_BLOCKS_AMOUNT) {
+      if (this.currentBlockHeight === 0) {
         this.currentBlockHeight = blockHeightTip;
       } else {
         this.currentBlockHeight++;
@@ -341,28 +255,15 @@ class Blocks {
       }
 
       const blockHash = await bitcoinApi.$getBlockHash(this.currentBlockHeight);
-      const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash));
+      const block = await bitcoinApi.$getBlock(blockHash);
       const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash);
       const transactions = await this.$getTransactionsExtended(blockHash, block.height, false);
-      const blockExtended: BlockExtended = await this.$getBlockExtended(block, transactions);
+      const blockExtended: BlockExtended = this.getBlockExtended(block, transactions);
+      const coinbase: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(transactions[0].txid, true);
 
-      if (Common.indexingEnabled()) {
-        if (!fastForwarded) {
-          const lastBlock = await blocksRepository.$getBlockByHeight(blockExtended.height - 1);
-          if (lastBlock !== null && blockExtended.previousblockhash !== lastBlock['hash']) {
-            logger.warn(`Chain divergence detected at block ${lastBlock['height']}, re-indexing most recent data`);
-            // We assume there won't be a reorg with more than 10 block depth
-            await BlocksRepository.$deleteBlocksFrom(lastBlock['height'] - 10);
-            await HashratesRepository.$deleteLastEntries();
-            for (let i = 10; i >= 0; --i) {
-              await this.$indexBlock(lastBlock['height'] - i);
-            }
-          }
-          await blocksRepository.$saveBlockInDatabase(blockExtended);
-        }
-      }
-      if (fiatConversion.ratesInitialized === true && config.DATABASE.ENABLED === true) {
-        await RatesRepository.$saveRate(blockExtended.height, fiatConversion.getConversionRates());
+      if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === true) {
+        const miner = await this.$findBlockMiner(blockExtended.coinbaseTx);
+        await blocksRepository.$saveBlockInDatabase(blockExtended, blockHash, coinbase.hex, miner);
       }
 
       if (block.height % 2016 === 0) {
@@ -379,108 +280,11 @@ class Blocks {
       if (this.newBlockCallbacks.length) {
         this.newBlockCallbacks.forEach((cb) => cb(blockExtended, txIds, transactions));
       }
-      if (!memPool.hasPriority()) {
+      if (memPool.isInSync()) {
         diskCache.$saveCacheToDisk();
       }
-    }
-  }
 
-  /**
-   * Index a block if it's missing from the database. Returns the block after indexing
-   */
-  public async $indexBlock(height: number): Promise<BlockExtended> {
-    const dbBlock = await blocksRepository.$getBlockByHeight(height);
-    if (dbBlock != null) {
-      return prepareBlock(dbBlock);
-    }
-
-    const blockHash = await bitcoinApi.$getBlockHash(height);
-    const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash));
-    const transactions = await this.$getTransactionsExtended(blockHash, block.height, true);
-    const blockExtended = await this.$getBlockExtended(block, transactions);
-
-    await blocksRepository.$saveBlockInDatabase(blockExtended);
-
-    return prepareBlock(blockExtended);
-  }
-
-  /**
-   * Index a block by hash if it's missing from the database. Returns the block after indexing
-   */
-  public async $getBlock(hash: string): Promise<BlockExtended | IEsploraApi.Block> {
-    // Check the memory cache
-    const blockByHash = this.getBlocks().find((b) => b.id === hash);
-    if (blockByHash) {
-      return blockByHash;
-    }
-
-    // Block has already been indexed
-    if (Common.indexingEnabled()) {
-      const dbBlock = await blocksRepository.$getBlockByHash(hash);
-      if (dbBlock != null) {
-        return prepareBlock(dbBlock);
-      }
-    }
-
-    const block = await bitcoinApi.$getBlock(hash);
-
-    // Not Bitcoin network, return the block as it
-    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
-      return block;
-    }
-
-    // Bitcoin network, add our custom data on top
-    const transactions = await this.$getTransactionsExtended(hash, block.height, true);
-    const blockExtended = await this.$getBlockExtended(block, transactions);
-    if (Common.indexingEnabled()) {
-      delete(blockExtended['coinbaseTx']);
-      await blocksRepository.$saveBlockInDatabase(blockExtended);
-    }
-
-    return blockExtended;
-  }
-
-  public async $getBlocks(fromHeight?: number, limit: number = 15): Promise<BlockExtended[]> {
-    try {
-      let currentHeight = fromHeight !== undefined ? fromHeight : this.getCurrentBlockHeight();
-      const returnBlocks: BlockExtended[] = [];
-
-      if (currentHeight < 0) {
-        return returnBlocks;
-      }
-
-      if (currentHeight === 0 && Common.indexingEnabled()) {
-        currentHeight = await blocksRepository.$mostRecentBlockHeight();
-      }
-
-      // Check if block height exist in local cache to skip the hash lookup
-      const blockByHeight = this.getBlocks().find((b) => b.height === currentHeight);
-      let startFromHash: string | null = null;
-      if (blockByHeight) {
-        startFromHash = blockByHeight.id;
-      } else if (!Common.indexingEnabled()) {
-        startFromHash = await bitcoinApi.$getBlockHash(currentHeight);
-      }
-
-      let nextHash = startFromHash;
-      for (let i = 0; i < limit && currentHeight >= 0; i++) {
-        let block = this.getBlocks().find((b) => b.height === currentHeight);
-        if (block) {
-          returnBlocks.push(block);
-        } else if (Common.indexingEnabled()) {
-          block = await this.$indexBlock(currentHeight);
-          returnBlocks.push(block);
-        } else if (nextHash != null) {
-          block = prepareBlock(await bitcoinApi.$getBlock(nextHash));
-          nextHash = block.previousblockhash;
-          returnBlocks.push(block);
-        }
-        currentHeight--;
-      }
-
-      return returnBlocks;
-    } catch (e) {
-      throw e;
+      return;
     }
   }
 

backend/src/api/common.ts

@@ -77,7 +77,7 @@ export class Common {
     };
   }
 
-  static sleep$(ms: number): Promise<void> {
+  static sleep(ms: number): Promise<void> {
     return new Promise((resolve) => {
        setTimeout(() => {
          resolve();
@@ -154,27 +154,4 @@ export class Common {
     });
     return parents;
   }
-
-  static getSqlInterval(interval: string | null): string | null {
-    switch (interval) {
-      case '24h': return '1 DAY';
-      case '3d': return '3 DAY';
-      case '1w': return '1 WEEK';
-      case '1m': return '1 MONTH';
-      case '3m': return '3 MONTH';
-      case '6m': return '6 MONTH';
-      case '1y': return '1 YEAR';
-      case '2y': return '2 YEAR';
-      case '3y': return '3 YEAR';
-      default: return null;
-    }
-  }
-
-  static indexingEnabled(): boolean {
-    return (
-      ['mainnet', 'testnet', 'signet', 'regtest'].includes(config.MEMPOOL.NETWORK) &&
-      config.DATABASE.ENABLED === true &&
-      config.MEMPOOL.INDEXING_BLOCKS_AMOUNT !== 0
-    );
-  }
 }

backend/src/api/database-migration.ts

@@ -1,10 +1,12 @@
+import { PoolConnection } from 'mysql2/promise';
 import config from '../config';
-import DB from '../database';
+import { DB } from '../database';
 import logger from '../logger';
-import { Common } from './common';
+
+const sleep = (ms: number) => new Promise(res => setTimeout(res, ms));
 
 class DatabaseMigration {
-  private static currentVersion = 19;
+  private static currentVersion = 4;
   private queryTimeout = 120000;
   private statisticsAddedIndexed = false;
 
@@ -13,21 +15,21 @@ class DatabaseMigration {
    * Entry point
    */
   public async $initializeOrMigrateDatabase(): Promise<void> {
-    logger.debug('MIGRATIONS: Running migrations');
+    logger.info('MIGRATIONS: Running migrations');
 
     await this.$printDatabaseVersion();
 
     // First of all, if the `state` database does not exist, create it so we can track migration version
     if (!await this.$checkIfTableExists('state')) {
-      logger.debug('MIGRATIONS: `state` table does not exist. Creating it.');
+      logger.info('MIGRATIONS: `state` table does not exist. Creating it.');
       try {
         await this.$createMigrationStateTable();
       } catch (e) {
         logger.err('MIGRATIONS: Unable to create `state` table, aborting in 10 seconds. ' + e);
-        await Common.sleep$(10000);
+        await sleep(10000);
         process.exit(-1);
       }
-      logger.debug('MIGRATIONS: `state` table initialized.');
+      logger.info('MIGRATIONS: `state` table initialized.');
     }
 
     let databaseSchemaVersion = 0;
@@ -35,14 +37,14 @@ class DatabaseMigration {
       databaseSchemaVersion = await this.$getSchemaVersionFromDatabase();
     } catch (e) {
       logger.err('MIGRATIONS: Unable to get current database migration version, aborting in 10 seconds. ' + e);
-      await Common.sleep$(10000);
+      await sleep(10000);
       process.exit(-1);
     }
 
-    logger.debug('MIGRATIONS: Current state.schema_version ' + databaseSchemaVersion);
-    logger.debug('MIGRATIONS: Latest DatabaseMigration.version is ' + DatabaseMigration.currentVersion);
+    logger.info('MIGRATIONS: Current state.schema_version ' + databaseSchemaVersion);
+    logger.info('MIGRATIONS: Latest DatabaseMigration.version is ' + DatabaseMigration.currentVersion);
     if (databaseSchemaVersion >= DatabaseMigration.currentVersion) {
-      logger.debug('MIGRATIONS: Nothing to do.');
+      logger.info('MIGRATIONS: Nothing to do.');
       return;
     }
 
@@ -51,15 +53,15 @@ class DatabaseMigration {
       await this.$createMissingTablesAndIndexes(databaseSchemaVersion);
     } catch (e) {
       logger.err('MIGRATIONS: Unable to create required tables, aborting in 10 seconds. ' + e);
-      await Common.sleep$(10000);
+      await sleep(10000);
       process.exit(-1);
     }
 
     if (DatabaseMigration.currentVersion > databaseSchemaVersion) {
-      logger.notice('MIGRATIONS: Upgrading database schema');
+      logger.info('MIGRATIONS: Upgrading datababse schema');
       try {
         await this.$migrateTableSchemaFromVersion(databaseSchemaVersion);
-        logger.notice(`MIGRATIONS: OK. Database schema have been migrated from version ${databaseSchemaVersion} to ${DatabaseMigration.currentVersion} (latest version)`);
+        logger.info(`MIGRATIONS: OK. Database schema have been migrated from version ${databaseSchemaVersion} to ${DatabaseMigration.currentVersion} (latest version)`);
       } catch (e) {
         logger.err('MIGRATIONS: Unable to migrate database, aborting. ' + e);
       }
@@ -74,121 +76,23 @@ class DatabaseMigration {
   private async $createMissingTablesAndIndexes(databaseSchemaVersion: number) {
     await this.$setStatisticsAddedIndexedFlag(databaseSchemaVersion);
 
-    const isBitcoin = ['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK);
+    const connection = await DB.pool.getConnection();
     try {
-      await this.$executeQuery(this.getCreateElementsTableQuery(), await this.$checkIfTableExists('elements_pegs'));
-      await this.$executeQuery(this.getCreateStatisticsQuery(), await this.$checkIfTableExists('statistics'));
+      await this.$executeQuery(connection, this.getCreateElementsTableQuery(), await this.$checkIfTableExists('elements_pegs'));
+      await this.$executeQuery(connection, this.getCreateStatisticsQuery(), await this.$checkIfTableExists('statistics'));
       if (databaseSchemaVersion < 2 && this.statisticsAddedIndexed === false) {
-        await this.$executeQuery(`CREATE INDEX added ON statistics (added);`);
+        await this.$executeQuery(connection, `CREATE INDEX added ON statistics (added);`);
       }
       if (databaseSchemaVersion < 3) {
-        await this.$executeQuery(this.getCreatePoolsTableQuery(), await this.$checkIfTableExists('pools'));
+        await this.$executeQuery(connection, this.getCreatePoolsTableQuery(), await this.$checkIfTableExists('pools'));
       }
       if (databaseSchemaVersion < 4) {
-        await this.$executeQuery('DROP table IF EXISTS blocks;');
-        await this.$executeQuery(this.getCreateBlocksTableQuery(), await this.$checkIfTableExists('blocks'));
-      }
-      if (databaseSchemaVersion < 5 && isBitcoin === true) {
-        logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.`);
-        await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index
-        await this.$executeQuery('ALTER TABLE blocks ADD `reward` double unsigned NOT NULL DEFAULT "0"');
-      }
-
-      if (databaseSchemaVersion < 6 && isBitcoin === true) {
-        logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.`);
-        await this.$executeQuery('TRUNCATE blocks;');  // Need to re-index
-        // Cleanup original blocks fields type
-        await this.$executeQuery('ALTER TABLE blocks MODIFY `height` integer unsigned NOT NULL DEFAULT "0"');
-        await this.$executeQuery('ALTER TABLE blocks MODIFY `tx_count` smallint unsigned NOT NULL DEFAULT "0"');
-        await this.$executeQuery('ALTER TABLE blocks MODIFY `size` integer unsigned NOT NULL DEFAULT "0"');
-        await this.$executeQuery('ALTER TABLE blocks MODIFY `weight` integer unsigned NOT NULL DEFAULT "0"');
-        await this.$executeQuery('ALTER TABLE blocks MODIFY `difficulty` double NOT NULL DEFAULT "0"');
-        // We also fix the pools.id type so we need to drop/re-create the foreign key
-        await this.$executeQuery('ALTER TABLE blocks DROP FOREIGN KEY IF EXISTS `blocks_ibfk_1`');
-        await this.$executeQuery('ALTER TABLE pools MODIFY `id` smallint unsigned AUTO_INCREMENT');
-        await this.$executeQuery('ALTER TABLE blocks MODIFY `pool_id` smallint unsigned NULL');
-        await this.$executeQuery('ALTER TABLE blocks ADD FOREIGN KEY (`pool_id`) REFERENCES `pools` (`id`)');
-        // Add new block indexing fields
-        await this.$executeQuery('ALTER TABLE blocks ADD `version` integer unsigned NOT NULL DEFAULT "0"');
-        await this.$executeQuery('ALTER TABLE blocks ADD `bits` integer unsigned NOT NULL DEFAULT "0"');
-        await this.$executeQuery('ALTER TABLE blocks ADD `nonce` bigint unsigned NOT NULL DEFAULT "0"');
-        await this.$executeQuery('ALTER TABLE blocks ADD `merkle_root` varchar(65) NOT NULL DEFAULT ""');
-        await this.$executeQuery('ALTER TABLE blocks ADD `previous_block_hash` varchar(65) NULL');
-      }
-
-      if (databaseSchemaVersion < 7 && isBitcoin === true) {
-        await this.$executeQuery('DROP table IF EXISTS hashrates;');
-        await this.$executeQuery(this.getCreateDailyStatsTableQuery(), await this.$checkIfTableExists('hashrates'));
-      }
-
-      if (databaseSchemaVersion < 8 && isBitcoin === true) {
-        logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.`);
-        await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index
-        await this.$executeQuery('ALTER TABLE `hashrates` DROP INDEX `PRIMARY`');
-        await this.$executeQuery('ALTER TABLE `hashrates` ADD `id` int NOT NULL AUTO_INCREMENT PRIMARY KEY FIRST');
-        await this.$executeQuery('ALTER TABLE `hashrates` ADD `share` float NOT NULL DEFAULT "0"');
-        await this.$executeQuery('ALTER TABLE `hashrates` ADD `type` enum("daily", "weekly") DEFAULT "daily"');
-      }
-
-      if (databaseSchemaVersion < 9 && isBitcoin === true) {
-        logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.`);
-        await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index
-        await this.$executeQuery('ALTER TABLE `state` CHANGE `name` `name` varchar(100)');
-        await this.$executeQuery('ALTER TABLE `hashrates` ADD UNIQUE `hashrate_timestamp_pool_id` (`hashrate_timestamp`, `pool_id`)');
-      }
-
-      if (databaseSchemaVersion < 10 && isBitcoin === true) {
-        await this.$executeQuery('ALTER TABLE `blocks` ADD INDEX `blockTimestamp` (`blockTimestamp`)');
-      }
-
-      if (databaseSchemaVersion < 11 && isBitcoin === true) {
-        logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.`);
-        await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index
-        await this.$executeQuery(`ALTER TABLE blocks
-          ADD avg_fee INT UNSIGNED NULL,
-          ADD avg_fee_rate INT UNSIGNED NULL
-        `);
-        await this.$executeQuery('ALTER TABLE blocks MODIFY `reward` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
-        await this.$executeQuery('ALTER TABLE blocks MODIFY `median_fee` INT UNSIGNED NOT NULL DEFAULT "0"');
-        await this.$executeQuery('ALTER TABLE blocks MODIFY `fees` INT UNSIGNED NOT NULL DEFAULT "0"');
-      }
-
-      if (databaseSchemaVersion < 12 && isBitcoin === true) {
-        // No need to re-index because the new data type can contain larger values
-        await this.$executeQuery('ALTER TABLE blocks MODIFY `fees` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
-      }
-
-      if (databaseSchemaVersion < 13 && isBitcoin === true) {
-        await this.$executeQuery('ALTER TABLE blocks MODIFY `difficulty` DOUBLE UNSIGNED NOT NULL DEFAULT "0"');
-        await this.$executeQuery('ALTER TABLE blocks MODIFY `median_fee` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
-        await this.$executeQuery('ALTER TABLE blocks MODIFY `avg_fee` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
-        await this.$executeQuery('ALTER TABLE blocks MODIFY `avg_fee_rate` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
-      }
-
-      if (databaseSchemaVersion < 14 && isBitcoin === true) {
-        logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.`);
-        await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index
-        await this.$executeQuery('ALTER TABLE `hashrates` DROP FOREIGN KEY `hashrates_ibfk_1`');
-        await this.$executeQuery('ALTER TABLE `hashrates` MODIFY `pool_id` SMALLINT UNSIGNED NOT NULL DEFAULT "0"');
-      }
-
-      if (databaseSchemaVersion < 16 && isBitcoin === true) {
-        logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.`);
-        await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index because we changed timestamps
-      }
-
-      if (databaseSchemaVersion < 17 && isBitcoin === true) {
-        await this.$executeQuery('ALTER TABLE `pools` ADD `slug` CHAR(50) NULL');
-      }
-
-      if (databaseSchemaVersion < 18 && isBitcoin === true) {
-        await this.$executeQuery('ALTER TABLE `blocks` ADD INDEX `hash` (`hash`);');
-      }
-
-      if (databaseSchemaVersion < 19) {
-        await this.$executeQuery(this.getCreateRatesTableQuery(), await this.$checkIfTableExists('rates'));
+        await this.$executeQuery(connection, 'DROP table IF EXISTS blocks;');
+        await this.$executeQuery(connection, this.getCreateBlocksTableQuery(), await this.$checkIfTableExists('blocks'));
       }
+      connection.release();
     } catch (e) {
+      connection.release();
       throw e;
     }
   }
@@ -205,16 +109,18 @@ class DatabaseMigration {
       return;
     }
 
+    const connection = await DB.pool.getConnection();
+
     try {
       // We don't use "CREATE INDEX IF NOT EXISTS" because it is not supported on old mariadb version 5.X
       const query = `SELECT COUNT(1) hasIndex FROM INFORMATION_SCHEMA.STATISTICS
         WHERE table_schema=DATABASE() AND table_name='statistics' AND index_name='added';`;
-      const [rows] = await this.$executeQuery(query, true);
+      const [rows] = await this.$executeQuery(connection, query, true);
       if (rows[0].hasIndex === 0) {
-        logger.debug('MIGRATIONS: `statistics.added` is not indexed');
+        logger.info('MIGRATIONS: `statistics.added` is not indexed');
         this.statisticsAddedIndexed = false;
       } else if (rows[0].hasIndex === 1) {
-        logger.debug('MIGRATIONS: `statistics.added` is already indexed');
+        logger.info('MIGRATIONS: `statistics.added` is already indexed');
         this.statisticsAddedIndexed = true;
       }
     } catch (e) {
@@ -223,24 +129,28 @@ class DatabaseMigration {
       logger.err('MIGRATIONS: Unable to check if `statistics.added` INDEX exist or not.');
       this.statisticsAddedIndexed = true;
     }
+
+    connection.release();
   }
 
   /**
    * Small query execution wrapper to log all executed queries
    */
-  private async $executeQuery(query: string, silent: boolean = false): Promise<any> {
+  private async $executeQuery(connection: PoolConnection, query: string, silent: boolean = false): Promise<any> {
     if (!silent) {
-      logger.debug('MIGRATIONS: Execute query:\n' + query);
+      logger.info('MIGRATIONS: Execute query:\n' + query);
     }
-    return DB.query({ sql: query, timeout: this.queryTimeout });
+    return connection.query<any>({ sql: query, timeout: this.queryTimeout });
   }
 
   /**
    * Check if 'table' exists in the database
    */
   private async $checkIfTableExists(table: string): Promise<boolean> {
+    const connection = await DB.pool.getConnection();
     const query = `SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = '${config.DATABASE.DATABASE}' AND TABLE_NAME = '${table}'`;
-    const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
+    const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
+    connection.release();
     return rows[0]['COUNT(*)'] === 1;
   }
 
@@ -248,8 +158,10 @@ class DatabaseMigration {
    * Get current database version
    */
   private async $getSchemaVersionFromDatabase(): Promise<number> {
+    const connection = await DB.pool.getConnection();
     const query = `SELECT number FROM state WHERE name = 'schema_version';`;
-    const [rows] = await this.$executeQuery(query, true);
+    const [rows] = await this.$executeQuery(connection, query, true);
+    connection.release();
     return rows[0]['number'];
   }
 
@@ -257,6 +169,8 @@ class DatabaseMigration {
    * Create the `state` table
    */
   private async $createMigrationStateTable(): Promise<void> {
+    const connection = await DB.pool.getConnection();
+
     try {
       const query = `CREATE TABLE IF NOT EXISTS state (
         name varchar(25) NOT NULL,
@@ -264,12 +178,15 @@ class DatabaseMigration {
         string varchar(100) NULL,
         CONSTRAINT name_unique UNIQUE (name)
       ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
-      await this.$executeQuery(query);
+      await this.$executeQuery(connection, query);
 
       // Set initial values
-      await this.$executeQuery(`INSERT INTO state VALUES('schema_version', 0, NULL);`);
-      await this.$executeQuery(`INSERT INTO state VALUES('last_elements_block', 0, NULL);`);
+      await this.$executeQuery(connection, `INSERT INTO state VALUES('schema_version', 0, NULL);`);
+      await this.$executeQuery(connection, `INSERT INTO state VALUES('last_elements_block', 0, NULL);`);
+
+      connection.release();
     } catch (e) {
+      connection.release();
       throw e;
     }
   }
@@ -284,14 +201,18 @@ class DatabaseMigration {
     }
     transactionQueries.push(this.getUpdateToLatestSchemaVersionQuery());
 
+    const connection = await DB.pool.getConnection();
     try {
-      await this.$executeQuery('START TRANSACTION;');
+      await this.$executeQuery(connection, 'START TRANSACTION;');
       for (const query of transactionQueries) {
-        await this.$executeQuery(query);
+        await this.$executeQuery(connection, query);
       }
-      await this.$executeQuery('COMMIT;');
+      await this.$executeQuery(connection, 'COMMIT;');
+
+      connection.release();
     } catch (e) {
-      await this.$executeQuery('ROLLBACK;');
+      await this.$executeQuery(connection, 'ROLLBACK;');
+      connection.release();
       throw e;
     }
   }
@@ -301,7 +222,6 @@ class DatabaseMigration {
    */
   private getMigrationQueriesFromVersion(version: number): string[] {
     const queries: string[] = [];
-    const isBitcoin = ['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK);
 
     if (version < 1) {
       if (config.MEMPOOL.NETWORK !== 'liquid' && config.MEMPOOL.NETWORK !== 'liquidtestnet') {
@@ -309,14 +229,6 @@ class DatabaseMigration {
       }
     }
 
-    if (version < 7 && isBitcoin === true) {
-      queries.push(`INSERT INTO state(name, number, string) VALUES ('last_hashrates_indexing', 0, NULL)`);
-    }
-
-    if (version < 9  && isBitcoin === true) {
-      queries.push(`INSERT INTO state(name, number, string) VALUES ('last_weekly_hashrates_indexing', 0, NULL)`);
-    }
-
     return queries;
   }
 
@@ -331,12 +243,14 @@ class DatabaseMigration {
    * Print current database version
    */
   private async $printDatabaseVersion() {
+    const connection = await DB.pool.getConnection();
     try {
-      const [rows] = await this.$executeQuery('SELECT VERSION() as version;', true);
-      logger.debug(`MIGRATIONS: Database engine version '${rows[0].version}'`);
+      const [rows] = await this.$executeQuery(connection, 'SELECT VERSION() as version;', true);
+      logger.info(`MIGRATIONS: Database engine version '${rows[0].version}'`);
     } catch (e) {
-      logger.debug(`MIGRATIONS: Could not fetch database engine version. ` + e);
+      logger.info(`MIGRATIONS: Could not fetch database engine version. ` + e);
     }
+    connection.release();
   }
 
   // Couple of wrappers to clean the main logic
@@ -458,46 +372,6 @@ class DatabaseMigration {
       FOREIGN KEY (pool_id) REFERENCES pools (id)
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
-
-  private getCreateDailyStatsTableQuery(): string {
-    return `CREATE TABLE IF NOT EXISTS hashrates (
-      hashrate_timestamp timestamp NOT NULL,
-      avg_hashrate double unsigned DEFAULT '0',
-      pool_id smallint unsigned NULL,
-      PRIMARY KEY (hashrate_timestamp),
-      INDEX (pool_id),
-      FOREIGN KEY (pool_id) REFERENCES pools (id)
-    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
-  }
-
-  private getCreateRatesTableQuery(): string {
-    return `CREATE TABLE IF NOT EXISTS rates (
-      height int(10) unsigned NOT NULL,
-      bisq_rates JSON NOT NULL,
-      PRIMARY KEY (height)
-    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
-  }
-
-  public async $truncateIndexedData(tables: string[]) {
-    const allowedTables = ['blocks', 'hashrates'];
-
-    try {
-      for (const table of tables) {
-        if (!allowedTables.includes(table)) {
-          logger.debug(`Table ${table} cannot to be re-indexed (not allowed)`);
-          continue;
-        }
-
-        await this.$executeQuery(`TRUNCATE ${table}`, true);
-        if (table === 'hashrates') {
-          await this.$executeQuery('UPDATE state set number = 0 where name = "last_hashrates_indexing"', true);
-        }
-        logger.notice(`Table ${table} has been truncated`);
-      }
-    } catch (e) {
-      logger.warn(`Unable to erase indexed data`);
-    }
-  }
 }
 
 export default new DatabaseMigration();

backend/src/api/difficulty-adjustment.ts

@@ -1,67 +0,0 @@
-import config from '../config';
-import { IDifficultyAdjustment } from '../mempool.interfaces';
-import blocks from './blocks';
-
-class DifficultyAdjustmentApi {
-  constructor() { }
-
-  public getDifficultyAdjustment(): IDifficultyAdjustment {
-    const DATime = blocks.getLastDifficultyAdjustmentTime();
-    const previousRetarget = blocks.getPreviousDifficultyRetarget();
-    const blockHeight = blocks.getCurrentBlockHeight();
-    const blocksCache = blocks.getBlocks();
-    const latestBlock = blocksCache[blocksCache.length - 1];
-
-    const now = new Date().getTime() / 1000;
-    const diff = now - DATime;
-    const blocksInEpoch = blockHeight % 2016;
-    const progressPercent = (blocksInEpoch >= 0) ? blocksInEpoch / 2016 * 100 : 100;
-    const remainingBlocks = 2016 - blocksInEpoch;
-    const nextRetargetHeight = blockHeight + remainingBlocks;
-
-    let difficultyChange = 0;
-    if (remainingBlocks < 1870) {
-      if (blocksInEpoch > 0) {
-        difficultyChange = (600 / (diff / blocksInEpoch) - 1) * 100;
-      }
-      if (difficultyChange > 300) {
-        difficultyChange = 300;
-      }
-      if (difficultyChange < -75) {
-        difficultyChange = -75;
-      }
-    }
-
-    let timeAvgMins = blocksInEpoch && blocksInEpoch > 146 ? diff / blocksInEpoch / 60 : 10;
-
-    // Testnet difficulty is set to 1 after 20 minutes of no blocks,
-    // therefore the time between blocks will always be below 20 minutes (1200s).
-    let timeOffset = 0;
-    if (config.MEMPOOL.NETWORK === 'testnet') {
-      if (timeAvgMins > 20) {
-        timeAvgMins = 20;
-      }
-      if (now - latestBlock.timestamp + timeAvgMins * 60 > 1200) {
-        timeOffset = -Math.min(now - latestBlock.timestamp, 1200) * 1000;
-      }
-    }
-
-    const timeAvg = timeAvgMins * 60 * 1000 ;
-    const remainingTime = (remainingBlocks * timeAvg) + (now * 1000);
-    const estimatedRetargetDate = remainingTime + now;
-
-    return {
-      progressPercent,
-      difficultyChange,
-      estimatedRetargetDate,
-      remainingBlocks,
-      remainingTime,
-      previousRetarget,
-      nextRetargetHeight,
-      timeAvg,
-      timeOffset,
-    };
-  }
-}
-
-export default new DifficultyAdjustmentApi();

backend/src/api/disk-cache.ts

@@ -9,8 +9,6 @@ import { TransactionExtended } from '../mempool.interfaces';
 import { Common } from './common';
 
 class DiskCache {
-  private cacheSchemaVersion = 1;
-
   private static FILE_NAME = config.MEMPOOL.CACHE_DIR + '/cache.json';
   private static FILE_NAMES = config.MEMPOOL.CACHE_DIR + '/cache{number}.json';
   private static CHUNK_FILES = 25;
@@ -41,7 +39,6 @@ class DiskCache {
       const chunkSize = Math.floor(mempoolArray.length / DiskCache.CHUNK_FILES);
 
       await fsPromises.writeFile(DiskCache.FILE_NAME, JSON.stringify({
-        cacheSchemaVersion: this.cacheSchemaVersion,
         blocks: blocks.getBlocks(),
         mempool: {},
         mempoolArray: mempoolArray.splice(0, chunkSize),
@@ -60,13 +57,6 @@ class DiskCache {
     }
   }
 
-  wipeCache() {
-    fs.unlinkSync(DiskCache.FILE_NAME);
-    for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
-      fs.unlinkSync(DiskCache.FILE_NAMES.replace('{number}', i.toString()));
-    }
-  }
-  
   loadMempoolCache() {
     if (!fs.existsSync(DiskCache.FILE_NAME)) {
       return;
@@ -77,11 +67,6 @@ class DiskCache {
       if (cacheData) {
         logger.info('Restoring mempool and blocks data from disk cache');
         data = JSON.parse(cacheData);
-        if (data.cacheSchemaVersion === undefined || data.cacheSchemaVersion !== this.cacheSchemaVersion) {
-          logger.notice('Disk cache contains an outdated schema version. Clearing it and skipping the cache loading.');
-          return this.wipeCache();
-        }
-
         if (data.mempoolArray) {
           for (const tx of data.mempoolArray) {
             data.mempool[tx.txid] = tx;
@@ -103,14 +88,14 @@ class DiskCache {
             }
           }
         } catch (e) {
-          logger.info('Error parsing ' + fileName + '. Skipping. Reason: ' + (e instanceof Error ? e.message : e));
+          logger.debug('Error parsing ' + fileName + '. Skipping.');
         }
       }
 
       memPool.setMempool(data.mempool);
       blocks.setBlocks(data.blocks);
     } catch (e) {
-      logger.warn('Failed to parse mempoool and blocks cache. Skipping. Reason: ' + (e instanceof Error ? e.message : e));
+      logger.warn('Failed to parse mempoool and blocks cache. Skipping.');
     }
   }
 }

backend/src/api/fee-api.ts

@@ -1,3 +1,4 @@
+import config from '../config';
 import { MempoolBlock } from '../mempool.interfaces';
 import { Common } from './common';
 import mempool from './mempool';
@@ -18,7 +19,6 @@ class FeeApi {
         'fastestFee': this.defaultFee,
         'halfHourFee': this.defaultFee,
         'hourFee': this.defaultFee,
-        'economyFee': minimumFee,
         'minimumFee': minimumFee,
       };
     }
@@ -31,7 +31,6 @@ class FeeApi {
       'fastestFee': firstMedianFee,
       'halfHourFee': secondMedianFee,
       'hourFee': thirdMedianFee,
-      'economyFee': Math.min(2 * minimumFee, thirdMedianFee),
       'minimumFee': minimumFee,
     };
   }

backend/src/api/fiat-conversion.ts

@@ -1,39 +1,22 @@
 import logger from '../logger';
-import * as http from 'http';
-import * as https from 'https';
-import axios, { AxiosResponse } from 'axios';
+import axios from 'axios';
 import { IConversionRates } from '../mempool.interfaces';
 import config from '../config';
-import backendInfo from './backend-info';
-import { SocksProxyAgent } from 'socks-proxy-agent';
 
 class FiatConversion {
-  private debasingFiatCurrencies = ['AED', 'AUD', 'BDT', 'BHD', 'BMD', 'BRL', 'CAD', 'CHF', 'CLP',
-    'CNY', 'CZK', 'DKK', 'EUR', 'GBP', 'HKD', 'HUF', 'IDR', 'ILS', 'INR', 'JPY', 'KRW', 'KWD',
-    'LKR', 'MMK', 'MXN', 'MYR', 'NGN', 'NOK', 'NZD', 'PHP', 'PKR', 'PLN', 'RUB', 'SAR', 'SEK',
-    'SGD', 'THB', 'TRY', 'TWD', 'UAH', 'USD', 'VND', 'ZAR'];
-  private conversionRates: IConversionRates = {};
+  private conversionRates: IConversionRates = {
+    'USD': 0
+  };
   private ratesChangedCallback: ((rates: IConversionRates) => void) | undefined;
-  public ratesInitialized = false; // If true, it means rates are ready for use
 
-  constructor() {
-    for (const fiat of this.debasingFiatCurrencies) {
-      this.conversionRates[fiat] = 0;
-    }
-  }
+  constructor() { }
 
   public setProgressChangedCallback(fn: (rates: IConversionRates) => void) {
     this.ratesChangedCallback = fn;
   }
 
   public startService() {
-    const fiatConversionUrl = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.PRICE_DATA_SERVER.TOR_URL : config.PRICE_DATA_SERVER.CLEARNET_URL;
     logger.info('Starting currency rates service');
-    if (config.SOCKS5PROXY.ENABLED) {
-      logger.info(`Currency rates service will be queried over the Tor network using ${fiatConversionUrl}`);
-    } else {
-      logger.info(`Currency rates service will be queried over clearnet using ${config.PRICE_DATA_SERVER.CLEARNET_URL}`);
-    }
     setInterval(this.updateCurrency.bind(this), 1000 * config.MEMPOOL.PRICE_FEED_UPDATE_INTERVAL);
     this.updateCurrency();
   }
@@ -43,79 +26,17 @@ class FiatConversion {
   }
 
   private async updateCurrency(): Promise<void> {
-    type axiosOptions = {
-      headers: {
-        'User-Agent': string
+    try {
+      const response = await axios.get('https://price.bisq.wiz.biz/getAllMarketPrices', { timeout: 10000 });
+      const usd = response.data.data.find((item: any) => item.currencyCode === 'USD');
+      this.conversionRates = {
+        'USD': usd.price,
       };
-      timeout: number;
-      httpAgent?: http.Agent;
-      httpsAgent?: https.Agent;
-    }
-    const setDelay = (secs: number = 1): Promise<void> => new Promise(resolve => setTimeout(() => resolve(), secs * 1000));
-    const fiatConversionUrl = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.PRICE_DATA_SERVER.TOR_URL : config.PRICE_DATA_SERVER.CLEARNET_URL;
-    const isHTTP = (new URL(fiatConversionUrl).protocol.split(':')[0] === 'http') ? true : false;
-    const axiosOptions: axiosOptions = {
-      headers: {
-        'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
-      },
-      timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000
-    };
-
-    let retry = 0;
-
-    while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
-      try {
-        if (config.SOCKS5PROXY.ENABLED) {
-          let socksOptions: any = {
-            agentOptions: {
-              keepAlive: true,
-            },
-            hostname: config.SOCKS5PROXY.HOST,
-            port: config.SOCKS5PROXY.PORT
-          };
-
-          if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
-            socksOptions.username = config.SOCKS5PROXY.USERNAME;
-            socksOptions.password = config.SOCKS5PROXY.PASSWORD;
-          } else {
-            // Retry with different tor circuits https://stackoverflow.com/a/64960234
-            socksOptions.username = `circuit${retry}`;
-          }
-
-          // Handle proxy agent for onion addresses
-          if (isHTTP) {
-            axiosOptions.httpAgent = new SocksProxyAgent(socksOptions);
-          } else {
-            axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions);
-          }
-        }
-        
-        logger.debug('Querying currency rates service...');
-
-        const response: AxiosResponse = await axios.get(`${fiatConversionUrl}`, axiosOptions);
-
-        if (response.statusText === 'error' || !response.data) {
-          throw new Error(`Could not fetch data from ${fiatConversionUrl}, Error: ${response.status}`);
-        }
-
-        for (const rate of response.data.data) {
-          if (this.debasingFiatCurrencies.includes(rate.currencyCode) && rate.provider === 'Bisq-Aggregate') {
-            this.conversionRates[rate.currencyCode] = Math.round(100 * rate.price) / 100;
-          }
-        }
-
-        this.ratesInitialized = true;
-        logger.debug(`USD Conversion Rate: ${this.conversionRates.USD}`);
-
-        if (this.ratesChangedCallback) {
-          this.ratesChangedCallback(this.conversionRates);
-        }
-        break;
-      } catch (e) {
-        logger.err('Error updating fiat conversion rates: '  + (e instanceof Error ? e.message : e));
-        await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL);
-        retry++;
+      if (this.ratesChangedCallback) {
+        this.ratesChangedCallback(this.conversionRates);
       }
+    } catch (e) {
+      logger.err('Error updating fiat conversion rates: ' + (e instanceof Error ? e.message : e));
     }
   }
 }

backend/src/api/liquid/elements-parser.ts

@@ -2,7 +2,7 @@ import { IBitcoinApi } from '../bitcoin/bitcoin-api.interface';
 import bitcoinClient from '../bitcoin/bitcoin-client';
 import bitcoinSecondClient from '../bitcoin/bitcoin-second-client';
 import { Common } from '../common';
-import DB from '../../database';
+import { DB } from '../../database';
 import logger from '../../logger';
 
 class ElementsParser {
@@ -33,8 +33,10 @@ class ElementsParser {
   }
 
   public async $getPegDataByMonth(): Promise<any> {
+    const connection = await DB.pool.getConnection();
     const query = `SELECT SUM(amount) AS amount, DATE_FORMAT(FROM_UNIXTIME(datetime), '%Y-%m-01') AS date FROM elements_pegs GROUP BY DATE_FORMAT(FROM_UNIXTIME(datetime), '%Y%m')`;
-    const [rows] = await DB.query(query);
+    const [rows] = await connection.query<any>(query);
+    connection.release();
     return rows;
   }
 
@@ -77,6 +79,7 @@ class ElementsParser {
 
   protected async $savePegToDatabase(height: number, blockTime: number, amount: number, txid: string,
     txindex: number, bitcoinaddress: string, bitcointxid: string, bitcoinindex: number, final_tx: number): Promise<void> {
+    const connection = await DB.pool.getConnection();
     const query = `INSERT INTO elements_pegs(
         block, datetime, amount, txid, txindex, bitcoinaddress, bitcointxid, bitcoinindex, final_tx
       ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`;
@@ -84,19 +87,24 @@ class ElementsParser {
     const params: (string | number)[] = [
       height, blockTime, amount, txid, txindex, bitcoinaddress, bitcointxid, bitcoinindex, final_tx
     ];
-    await DB.query(query, params);
+    await connection.query(query, params);
+    connection.release();
     logger.debug(`Saved L-BTC peg from block height #${height} with TXID ${txid}.`);
   }
 
   protected async $getLatestBlockHeightFromDatabase(): Promise<number> {
+    const connection = await DB.pool.getConnection();
     const query = `SELECT number FROM state WHERE name = 'last_elements_block'`;
-    const [rows] = await DB.query(query);
+    const [rows] = await connection.query<any>(query);
+    connection.release();
     return rows[0]['number'];
   }
 
   protected async $saveLatestBlockToDatabase(blockHeight: number) {
+    const connection = await DB.pool.getConnection();
     const query = `UPDATE state SET number = ? WHERE name = 'last_elements_block'`;
-    await DB.query(query, [blockHeight]);
+    await connection.query<any>(query, [blockHeight]);
+    connection.release();
   }
 }
 

backend/src/api/liquid/icons.ts

@@ -1,4 +1,5 @@
 import * as fs from 'fs';
+import config from '../../config';
 import logger from '../../logger';
 
 class Icons {

backend/src/api/loading-indicators.ts

@@ -12,8 +12,8 @@ class LoadingIndicators {
     this.progressChangedCallback = fn;
   }
 
-  public setProgress(name: string, progressPercent: number, rounded: boolean = true) {
-    const newProgress = rounded === true ? Math.round(progressPercent) : progressPercent;
+  public setProgress(name: string, progressPercent: number) {
+    const newProgress = Math.round(progressPercent);
     if (newProgress >= 100) {
       delete this.loadingIndicators[name];
     } else {

backend/src/api/mempool-blocks.ts

@@ -1,11 +1,10 @@
 import logger from '../logger';
-import { MempoolBlock, TransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta } from '../mempool.interfaces';
+import { MempoolBlock, TransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces';
 import { Common } from './common';
 import config from '../config';
 
 class MempoolBlocks {
   private mempoolBlocks: MempoolBlockWithTransactions[] = [];
-  private mempoolBlockDeltas: MempoolBlockDelta[] = [];
 
   constructor() {}
 
@@ -26,10 +25,6 @@ class MempoolBlocks {
     return this.mempoolBlocks;
   }
 
-  public getMempoolBlockDeltas(): MempoolBlockDelta[] {
-    return this.mempoolBlockDeltas;
-  }
-
   public updateMempoolBlocks(memPool: { [txid: string]: TransactionExtended }): void {
     const latestMempool = memPool;
     const memPoolArray: TransactionExtended[] = [];
@@ -71,15 +66,11 @@ class MempoolBlocks {
     const time = end - start;
     logger.debug('Mempool blocks calculated in ' + time / 1000 + ' seconds');
 
-    const { blocks, deltas } = this.calculateMempoolBlocks(memPoolArray, this.mempoolBlocks);
-    this.mempoolBlocks = blocks;
-    this.mempoolBlockDeltas = deltas;
+    this.mempoolBlocks = this.calculateMempoolBlocks(memPoolArray);
   }
 
-  private calculateMempoolBlocks(transactionsSorted: TransactionExtended[], prevBlocks: MempoolBlockWithTransactions[]):
-    { blocks: MempoolBlockWithTransactions[], deltas: MempoolBlockDelta[] } {
+  private calculateMempoolBlocks(transactionsSorted: TransactionExtended[]): MempoolBlockWithTransactions[] {
     const mempoolBlocks: MempoolBlockWithTransactions[] = [];
-    const mempoolBlockDeltas: MempoolBlockDelta[] = [];
     let blockWeight = 0;
     let blockSize = 0;
     let transactions: TransactionExtended[] = [];
@@ -99,43 +90,7 @@ class MempoolBlocks {
     if (transactions.length) {
       mempoolBlocks.push(this.dataToMempoolBlocks(transactions, blockSize, blockWeight, mempoolBlocks.length));
     }
-    // Calculate change from previous block states
-    for (let i = 0; i < Math.max(mempoolBlocks.length, prevBlocks.length); i++) {
-      let added: TransactionStripped[] = [];
-      let removed: string[] = [];
-      if (mempoolBlocks[i] && !prevBlocks[i]) {
-        added = mempoolBlocks[i].transactions;
-      } else if (!mempoolBlocks[i] && prevBlocks[i]) {
-        removed = prevBlocks[i].transactions.map(tx => tx.txid);
-      } else if (mempoolBlocks[i] && prevBlocks[i]) {
-        const prevIds = {};
-        const newIds = {};
-        prevBlocks[i].transactions.forEach(tx => {
-          prevIds[tx.txid] = true;
-        });
-        mempoolBlocks[i].transactions.forEach(tx => {
-          newIds[tx.txid] = true;
-        });
-        prevBlocks[i].transactions.forEach(tx => {
-          if (!newIds[tx.txid]) {
-            removed.push(tx.txid);
-          }
-        });
-        mempoolBlocks[i].transactions.forEach(tx => {
-          if (!prevIds[tx.txid]) {
-            added.push(tx);
-          }
-        });
-      }
-      mempoolBlockDeltas.push({
-        added,
-        removed
-      });
-    }
-    return {
-      blocks: mempoolBlocks,
-      deltas: mempoolBlockDeltas
-    };
+    return mempoolBlocks;
   }
 
   private dataToMempoolBlocks(transactions: TransactionExtended[],
@@ -157,7 +112,6 @@ class MempoolBlocks {
       medianFee: Common.percentile(transactions.map((tx) => tx.effectiveFeePerVsize), config.MEMPOOL.RECOMMENDED_FEE_PERCENTILE),
       feeRange: Common.getFeesInRange(transactions, rangeLength),
       transactionIds: transactions.map((tx) => tx.txid),
-      transactions: transactions.map((tx) => Common.stripTransaction(tx)),
     };
   }
 }

backend/src/api/mempool.ts

@@ -8,15 +8,13 @@ import { IBitcoinApi } from './bitcoin/bitcoin-api.interface';
 import loadingIndicators from './loading-indicators';
 import bitcoinClient from './bitcoin/bitcoin-client';
 import bitcoinSecondClient from './bitcoin/bitcoin-second-client';
-import rbfCache from './rbf-cache';
 
 class Mempool {
   private static WEBSOCKET_REFRESH_RATE_MS = 10000;
   private static LAZY_DELETE_AFTER_SECONDS = 30;
   private inSync: boolean = false;
-  private mempoolCacheDelta: number = -1;
   private mempoolCache: { [txId: string]: TransactionExtended } = {};
-  private mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0, total_fee: 0,
+  private mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0,
                                                     maxmempool: 300000000, mempoolminfee: 0.00001000, minrelaytxfee: 0.00001000 };
   private mempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[],
     deletedTransactions: TransactionExtended[]) => void) | undefined;
@@ -34,17 +32,6 @@ class Mempool {
     setInterval(this.deleteExpiredTransactions.bind(this), 20000);
   }
 
-  /**
-   * Return true if we should leave resources available for mempool tx caching
-   */
-  public hasPriority(): boolean {
-    if (this.inSync) {
-      return false;
-    } else {
-      return this.mempoolCacheDelta == -1 || this.mempoolCacheDelta > 25;
-    }
-  }
-
   public isInSync(): boolean {
     return this.inSync;
   }
@@ -113,8 +100,6 @@ class Mempool {
     const diff = transactions.length - currentMempoolSize;
     const newTransactions: TransactionExtended[] = [];
 
-    this.mempoolCacheDelta = Math.abs(diff);
-
     if (!this.inSync) {
       loadingIndicators.setProgress('mempool', Object.keys(this.mempoolCache).length / transactions.length * 100);
     }
@@ -189,8 +174,6 @@ class Mempool {
       loadingIndicators.setProgress('mempool', 100);
     }
 
-    this.mempoolCacheDelta = Math.abs(transactions.length - Object.keys(this.mempoolCache).length);
-
     if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) {
       this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions);
     }
@@ -201,17 +184,6 @@ class Mempool {
     logger.debug('Mempool updated in ' + time / 1000 + ' seconds');
   }
 
-  public handleRbfTransactions(rbfTransactions: { [txid: string]: TransactionExtended; }) {
-    for (const rbfTransaction in rbfTransactions) {
-      if (this.mempoolCache[rbfTransaction]) {
-        // Store replaced transactions
-        rbfCache.add(rbfTransaction, rbfTransactions[rbfTransaction].txid);
-        // Erase the replaced transactions from the local mempool
-        delete this.mempoolCache[rbfTransaction];
-      }
-    }
-  }
-
   private updateTxPerSecond() {
     const nowMinusTimeSpan = new Date().getTime() - (1000 * config.STATISTICS.TX_PER_SECOND_SAMPLE_PERIOD);
     this.txPerSecondArray = this.txPerSecondArray.filter((unixTime) => unixTime > nowMinusTimeSpan);

backend/src/api/mining.ts

@@ -1,405 +1,69 @@
-import { PoolInfo, PoolStats, RewardStats } from '../mempool.interfaces';
-import BlocksRepository from '../repositories/BlocksRepository';
+import { PoolInfo, PoolStats } from '../mempool.interfaces';
+import BlocksRepository, { EmptyBlocks } from '../repositories/BlocksRepository';
 import PoolsRepository from '../repositories/PoolsRepository';
-import HashratesRepository from '../repositories/HashratesRepository';
 import bitcoinClient from './bitcoin/bitcoin-client';
-import logger from '../logger';
-import { Common } from './common';
-import loadingIndicators from './loading-indicators';
-import { escape } from 'mysql2';
 
 class Mining {
   constructor() {
   }
 
-  /**
-   * Get historical block total fee
-   */
-  public async $getHistoricalBlockFees(interval: string | null = null): Promise<any> {
-    return await BlocksRepository.$getHistoricalBlockFees(
-      this.getTimeRange(interval),
-      Common.getSqlInterval(interval)
-    );
-  }
-
-  /**
-   * Get historical block rewards
-   */
-  public async $getHistoricalBlockRewards(interval: string | null = null): Promise<any> {
-    return await BlocksRepository.$getHistoricalBlockRewards(
-      this.getTimeRange(interval),
-      Common.getSqlInterval(interval)
-    );
-  }
-
-  /**
-   * Get historical block fee rates percentiles
-   */
-   public async $getHistoricalBlockFeeRates(interval: string | null = null): Promise<any> {
-    return await BlocksRepository.$getHistoricalBlockFeeRates(
-      this.getTimeRange(interval),
-      Common.getSqlInterval(interval)
-    );
-  }
-
-  /**
-   * Get historical block sizes
-   */
-   public async $getHistoricalBlockSizes(interval: string | null = null): Promise<any> {
-    return await BlocksRepository.$getHistoricalBlockSizes(
-      this.getTimeRange(interval),
-      Common.getSqlInterval(interval)
-    );
-  }
-
-  /**
-   * Get historical block weights
-   */
-   public async $getHistoricalBlockWeights(interval: string | null = null): Promise<any> {
-    return await BlocksRepository.$getHistoricalBlockWeights(
-      this.getTimeRange(interval),
-      Common.getSqlInterval(interval)
-    );
-  }
-
   /**
    * Generate high level overview of the pool ranks and general stats
    */
-  public async $getPoolsStats(interval: string | null): Promise<object> {
+  public async $getPoolsStats(interval: string | null) : Promise<object> {
+    let sqlInterval: string | null = null;
+    switch (interval) {
+      case '24h': sqlInterval = '1 DAY'; break;
+      case '3d': sqlInterval = '3 DAY'; break;
+      case '1w': sqlInterval = '1 WEEK'; break;
+      case '1m': sqlInterval = '1 MONTH'; break;
+      case '3m': sqlInterval = '3 MONTH'; break;
+      case '6m': sqlInterval = '6 MONTH'; break;
+      case '1y': sqlInterval = '1 YEAR'; break;
+      case '2y': sqlInterval = '2 YEAR'; break;
+      case '3y': sqlInterval = '3 YEAR'; break;
+      default: sqlInterval = null; break;
+    }
+
     const poolsStatistics = {};
 
-    const poolsInfo: PoolInfo[] = await PoolsRepository.$getPoolsInfo(interval);
-    const emptyBlocks: any[] = await BlocksRepository.$countEmptyBlocks(null, interval);
+    const poolsInfo: PoolInfo[] = await PoolsRepository.$getPoolsInfo(sqlInterval);
+    const emptyBlocks: EmptyBlocks[] = await BlocksRepository.$countEmptyBlocks(sqlInterval);
 
     const poolsStats: PoolStats[] = [];
     let rank = 1;
 
     poolsInfo.forEach((poolInfo: PoolInfo) => {
-      const emptyBlocksCount = emptyBlocks.filter((emptyCount) => emptyCount.poolId === poolInfo.poolId);
       const poolStat: PoolStats = {
         poolId: poolInfo.poolId, // mysql row id
         name: poolInfo.name,
         link: poolInfo.link,
         blockCount: poolInfo.blockCount,
         rank: rank++,
-        emptyBlocks: emptyBlocksCount.length > 0 ? emptyBlocksCount[0]['count'] : 0,
-        slug: poolInfo.slug,
-      };
+        emptyBlocks: 0, 
+      }
+      for (let i = 0; i < emptyBlocks.length; ++i) {
+        if (emptyBlocks[i].poolId === poolInfo.poolId) {
+          poolStat.emptyBlocks++;
+        }
+      }
       poolsStats.push(poolStat);
     });
 
     poolsStatistics['pools'] = poolsStats;
 
-    const blockCount: number = await BlocksRepository.$blockCount(null, interval);
+    const oldestBlock = new Date(await BlocksRepository.$oldestBlockTimestamp());
+    poolsStatistics['oldestIndexedBlockTimestamp'] = oldestBlock.getTime();
+
+    const blockCount: number = await BlocksRepository.$blockCount(sqlInterval);
     poolsStatistics['blockCount'] = blockCount;
 
-    const totalBlock24h: number = await BlocksRepository.$blockCount(null, '24h');
-
-    try {
-      poolsStatistics['lastEstimatedHashrate'] = await bitcoinClient.getNetworkHashPs(totalBlock24h);
-    } catch (e) {
-      poolsStatistics['lastEstimatedHashrate'] = 0;
-      logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate');
-    }
+    const blockHeightTip = await bitcoinClient.getBlockCount();
+    const lastBlockHashrate = await bitcoinClient.getNetworkHashPs(120, blockHeightTip);
+    poolsStatistics['lastEstimatedHashrate'] = lastBlockHashrate;
 
     return poolsStatistics;
   }
-
-  /**
-   * Get all mining pool stats for a pool
-   */
-  public async $getPoolStat(slug: string): Promise<object> {
-    const pool = await PoolsRepository.$getPool(slug);
-    if (!pool) {
-      throw new Error('This mining pool does not exist ' + escape(slug));
-    }
-
-    const blockCount: number = await BlocksRepository.$blockCount(pool.id);
-    const totalBlock: number = await BlocksRepository.$blockCount(null, null);
-
-    const blockCount24h: number = await BlocksRepository.$blockCount(pool.id, '24h');
-    const totalBlock24h: number = await BlocksRepository.$blockCount(null, '24h');
-
-    const blockCount1w: number = await BlocksRepository.$blockCount(pool.id, '1w');
-    const totalBlock1w: number = await BlocksRepository.$blockCount(null, '1w');
-
-    let currentEstimatedHashrate = 0;
-    try {
-      currentEstimatedHashrate = await bitcoinClient.getNetworkHashPs(totalBlock24h);
-    } catch (e) {
-      logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate');
-    }
-
-    return {
-      pool: pool,
-      blockCount: {
-        'all': blockCount,
-        '24h': blockCount24h,
-        '1w': blockCount1w,
-      },
-      blockShare: {
-        'all': blockCount / totalBlock,
-        '24h': blockCount24h / totalBlock24h,
-        '1w': blockCount1w / totalBlock1w,
-      },
-      estimatedHashrate: currentEstimatedHashrate * (blockCount24h / totalBlock24h),
-      reportedHashrate: null,
-    };
-  }
-
-  /**
-   * Get miner reward stats
-   */
-  public async $getRewardStats(blockCount: number): Promise<RewardStats> {
-    return await BlocksRepository.$getBlockStats(blockCount);
-  }
-
-  /**
-   * [INDEXING] Generate weekly mining pool hashrate history
-   */
-  public async $generatePoolHashrateHistory(): Promise<void> {
-    const now = new Date();
-
-    try {
-      const lastestRunDate = await HashratesRepository.$getLatestRun('last_weekly_hashrates_indexing');
-
-      // Run only if:
-      // * lastestRunDate is set to 0 (node backend restart, reorg)
-      // * we started a new week (around Monday midnight)
-      const runIndexing = lastestRunDate === 0 || now.getUTCDay() === 1 && lastestRunDate !== now.getUTCDate();
-      if (!runIndexing) {
-        return;
-      }
-    } catch (e) {
-      throw e;
-    }
-
-    try {
-      const indexedTimestamp = await HashratesRepository.$getWeeklyHashrateTimestamps();
-      const hashrates: any[] = [];
-      const genesisTimestamp = 1231006505000; // bitcoin-cli getblock 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f
-
-      const lastMonday = new Date(now.setDate(now.getDate() - (now.getDay() + 6) % 7));
-      const lastMondayMidnight = this.getDateMidnight(lastMonday);
-      let toTimestamp = lastMondayMidnight.getTime();
-
-      const totalWeekIndexed = (await BlocksRepository.$blockCount(null, null)) / 1008;
-      let indexedThisRun = 0;
-      let totalIndexed = 0;
-      let newlyIndexed = 0;
-      const startedAt = new Date().getTime() / 1000;
-      let timer = new Date().getTime() / 1000;
-
-      logger.debug(`Indexing weekly mining pool hashrate`);
-      loadingIndicators.setProgress('weekly-hashrate-indexing', 0);
-
-      while (toTimestamp > genesisTimestamp) {
-        const fromTimestamp = toTimestamp - 604800000;
-
-        // Skip already indexed weeks
-        if (indexedTimestamp.includes(toTimestamp / 1000)) {
-          toTimestamp -= 604800000;
-          ++totalIndexed;
-          continue;
-        }
-
-        // Check if we have blocks for the previous week (which mean that the week
-        // we are currently indexing has complete data)
-        const blockStatsPreviousWeek: any = await BlocksRepository.$blockCountBetweenTimestamp(
-          null, (fromTimestamp - 604800000) / 1000, (toTimestamp - 604800000) / 1000);
-        if (blockStatsPreviousWeek.blockCount === 0) { // We are done indexing
-          break;
-        }
-
-        const blockStats: any = await BlocksRepository.$blockCountBetweenTimestamp(
-          null, fromTimestamp / 1000, toTimestamp / 1000);
-        const lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount,
-          blockStats.lastBlockHeight);
-
-        let pools = await PoolsRepository.$getPoolsInfoBetween(fromTimestamp / 1000, toTimestamp / 1000);
-        const totalBlocks = pools.reduce((acc, pool) => acc + pool.blockCount, 0);
-        pools = pools.map((pool: any) => {
-          pool.hashrate = (pool.blockCount / totalBlocks) * lastBlockHashrate;
-          pool.share = (pool.blockCount / totalBlocks);
-          return pool;
-        });
-
-        for (const pool of pools) {
-          hashrates.push({
-            hashrateTimestamp: toTimestamp / 1000,
-            avgHashrate: pool['hashrate'],
-            poolId: pool.poolId,
-            share: pool['share'],
-            type: 'weekly',
-          });
-        }
-
-        newlyIndexed += hashrates.length;
-        await HashratesRepository.$saveHashrates(hashrates);
-        hashrates.length = 0;
-
-        const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
-        if (elapsedSeconds > 1) {
-          const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
-          const weeksPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds));
-          const progress = Math.round(totalIndexed / totalWeekIndexed * 10000) / 100;
-          const timeLeft = Math.round((totalWeekIndexed - totalIndexed) / weeksPerSeconds);
-          const formattedDate = new Date(fromTimestamp).toUTCString();
-          logger.debug(`Getting weekly pool hashrate for ${formattedDate} | ~${weeksPerSeconds.toFixed(2)} weeks/sec | total: ~${totalIndexed}/${Math.round(totalWeekIndexed)} (${progress}%) | elapsed: ${runningFor} seconds | left: ~${timeLeft} seconds`);
-          timer = new Date().getTime() / 1000;
-          indexedThisRun = 0;
-          loadingIndicators.setProgress('weekly-hashrate-indexing', progress, false);
-        }
-
-        toTimestamp -= 604800000;
-        ++indexedThisRun;
-        ++totalIndexed;
-      }
-      await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', new Date().getUTCDate());
-      if (newlyIndexed > 0) {
-        logger.info(`Indexed ${newlyIndexed} pools weekly hashrate`);
-      }
-      loadingIndicators.setProgress('weekly-hashrate-indexing', 100);
-    } catch (e) {
-      loadingIndicators.setProgress('weekly-hashrate-indexing', 100);
-      throw e;
-    }
-  }
-
-  /**
-   * [INDEXING] Generate daily hashrate data
-   */
-  public async $generateNetworkHashrateHistory(): Promise<void> {
-    try {
-      // We only run this once a day around midnight
-      const latestRunDate = await HashratesRepository.$getLatestRun('last_hashrates_indexing');
-      const now = new Date().getUTCDate();
-      if (now === latestRunDate) {
-        return;
-      }
-    } catch (e) {
-      throw e;
-    }
-
-    try {
-      const indexedTimestamp = (await HashratesRepository.$getNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp);
-      const genesisTimestamp = 1231006505000; // bitcoin-cli getblock 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f
-      const lastMidnight = this.getDateMidnight(new Date());
-      let toTimestamp = Math.round(lastMidnight.getTime());
-      const hashrates: any[] = [];
-
-      const totalDayIndexed = (await BlocksRepository.$blockCount(null, null)) / 144;
-      let indexedThisRun = 0;
-      let totalIndexed = 0;
-      let newlyIndexed = 0;
-      const startedAt = new Date().getTime() / 1000;
-      let timer = new Date().getTime() / 1000;
-
-      logger.debug(`Indexing daily network hashrate`);
-      loadingIndicators.setProgress('daily-hashrate-indexing', 0);
-
-      while (toTimestamp > genesisTimestamp) {
-        const fromTimestamp = toTimestamp - 86400000;
-
-        // Skip already indexed weeks
-        if (indexedTimestamp.includes(toTimestamp / 1000)) {
-          toTimestamp -= 86400000;
-          ++totalIndexed;
-          continue;
-        }
-
-        // Check if we have blocks for the previous day (which mean that the day
-        // we are currently indexing has complete data)
-        const blockStatsPreviousDay: any = await BlocksRepository.$blockCountBetweenTimestamp(
-          null, (fromTimestamp - 86400000) / 1000, (toTimestamp - 86400000) / 1000);
-        if (blockStatsPreviousDay.blockCount === 0) { // We are done indexing
-          break;
-        }
-
-        const blockStats: any = await BlocksRepository.$blockCountBetweenTimestamp(
-          null, fromTimestamp / 1000, toTimestamp / 1000);
-        const lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount,
-          blockStats.lastBlockHeight);
-
-        hashrates.push({
-          hashrateTimestamp: toTimestamp / 1000,
-          avgHashrate: lastBlockHashrate,
-          poolId: 0,
-          share: 1,
-          type: 'daily',
-        });
-
-        if (hashrates.length > 10) {
-          newlyIndexed += hashrates.length;
-          await HashratesRepository.$saveHashrates(hashrates);
-          hashrates.length = 0;
-        }
-
-        const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
-        if (elapsedSeconds > 1) {
-          const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
-          const daysPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds));
-          const progress = Math.round(totalIndexed / totalDayIndexed * 10000) / 100;
-          const timeLeft = Math.round((totalDayIndexed - totalIndexed) / daysPerSeconds);
-          const formattedDate = new Date(fromTimestamp).toUTCString();
-          logger.debug(`Getting network daily hashrate for ${formattedDate} | ~${daysPerSeconds.toFixed(2)} days/sec | total: ~${totalIndexed}/${Math.round(totalDayIndexed)} (${progress}%) | elapsed: ${runningFor} seconds | left: ~${timeLeft} seconds`);
-          timer = new Date().getTime() / 1000;
-          indexedThisRun = 0;
-          loadingIndicators.setProgress('daily-hashrate-indexing', progress);
-        }
-
-        toTimestamp -= 86400000;
-        ++indexedThisRun;
-        ++totalIndexed;
-      }
-
-      // Add genesis block manually
-      if (toTimestamp <= genesisTimestamp && !indexedTimestamp.includes(genesisTimestamp)) {
-        hashrates.push({
-          hashrateTimestamp: genesisTimestamp,
-          avgHashrate: await bitcoinClient.getNetworkHashPs(1, 1),
-          poolId: null,
-          type: 'daily',
-        });
-      }
-
-      newlyIndexed += hashrates.length;
-      await HashratesRepository.$saveHashrates(hashrates);
-
-      await HashratesRepository.$setLatestRun('last_hashrates_indexing', new Date().getUTCDate());
-      if (newlyIndexed > 0) {
-        logger.info(`Indexed ${newlyIndexed} day of network hashrate`);
-      }
-      loadingIndicators.setProgress('daily-hashrate-indexing', 100);
-    } catch (e) {
-      loadingIndicators.setProgress('daily-hashrate-indexing', 100);
-      throw e;
-    }
-  }
-
-  private getDateMidnight(date: Date): Date {
-    date.setUTCHours(0);
-    date.setUTCMinutes(0);
-    date.setUTCSeconds(0);
-    date.setUTCMilliseconds(0);
-
-    return date;
-  }
-
-  private getTimeRange(interval: string | null): number {
-    switch (interval) {
-      case '3y': return 43200; // 12h
-      case '2y': return 28800; // 8h
-      case '1y': return 28800; // 8h
-      case '6m': return 10800; // 3h
-      case '3m': return 7200; // 2h
-      case '1m': return 1800; // 30min
-      case '1w': return 300; // 5min
-      case '3d': return 1;
-      case '24h': return 1;
-      default: return 86400; // 24h
-    }
-  }
 }
 
 export default new Mining();

backend/src/api/pools-parser.ts

@@ -1,4 +1,5 @@
-import DB from '../database';
+import { readFileSync } from 'fs';
+import { DB } from '../database';
 import logger from '../logger';
 import config from '../config';
 
@@ -7,28 +8,29 @@ interface Pool {
   link: string;
   regexes: string[];
   addresses: string[];
-  slug: string;
 }
 
 class PoolsParser {
-  miningPools: any[] = [];
-  unknownPool: any = {
-    'name': "Unknown",
-    'link': "https://learnmeabitcoin.com/technical/coinbase-transaction",
-    'regexes': "[]",
-    'addresses': "[]",
-    'slug': 'unknown'
-  };
-  slugWarnFlag = false;
-
   /**
    * Parse the pools.json file, consolidate the data and dump it into the database
    */
-  public async migratePoolsJson(poolsJson: object) {
+  public async migratePoolsJson() {
     if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
       return;
     }
 
+    logger.debug('Importing pools.json to the database, open ./pools.json');
+
+    let poolsJson: object = {};
+    try {
+      const fileContent: string = readFileSync('./pools.json', 'utf8');
+      poolsJson = JSON.parse(fileContent);
+    } catch (e) {
+      logger.err('Unable to open ./pools.json, does the file exist?');
+      await this.insertUnknownPool();
+      return;
+    }
+
     // First we save every entries without paying attention to pool duplication
     const poolsDuplicated: Pool[] = [];
 
@@ -40,7 +42,6 @@ class PoolsParser {
         'link': (<Pool>coinbaseTags[i][1]).link,
         'regexes': [coinbaseTags[i][0]],
         'addresses': [],
-        'slug': ''
       });
     }
     logger.debug('Parse payout_addresses');
@@ -51,7 +52,6 @@ class PoolsParser {
         'link': (<Pool>addressesTags[i][1]).link,
         'regexes': [],
         'addresses': [addressesTags[i][0]],
-        'slug': ''
       });
     }
 
@@ -66,20 +66,16 @@ class PoolsParser {
     logger.debug(`Found ${poolNames.length} unique mining pools`);
 
     // Get existing pools from the db
+    const connection = await DB.pool.getConnection();
     let existingPools;
     try {
-      if (config.DATABASE.ENABLED === true) {
-        [existingPools] = await DB.query({ sql: 'SELECT * FROM pools;', timeout: 120000 });
-      } else {
-        existingPools = [];
-      }
+      [existingPools] = await connection.query<any>({ sql: 'SELECT * FROM pools;', timeout: 120000 });
     } catch (e) {
-      logger.err('Cannot get existing pools from the database, skipping pools.json import');
+      logger.err('Unable to get existing pools from the database, skipping pools.json import');
+      connection.release();
       return;
     }
 
-    this.miningPools = [];
-
     // Finally, we generate the final consolidated pools data
     const finalPoolDataAdd: Pool[] = [];
     const finalPoolDataUpdate: Pool[] = [];
@@ -95,85 +91,59 @@ class PoolsParser {
 
       const finalPoolName = poolNames[i].replace(`'`, `''`); // To support single quote in names when doing db queries
 
-      let slug: string | undefined;
-      try {
-        slug = poolsJson['slugs'][poolNames[i]];
-      } catch (e) {
-        if (this.slugWarnFlag === false) {
-          logger.warn(`pools.json does not seem to contain the 'slugs' object`);
-          this.slugWarnFlag = true;
-        }
-      }
-
-      if (slug === undefined) {
-        // Only keep alphanumerical
-        slug = poolNames[i].replace(/[^a-z0-9]/gi, '').toLowerCase();
-        logger.warn(`No slug found for '${poolNames[i]}', generating it => '${slug}'`);
-      }
-
-      const poolObj = {
-        'name': finalPoolName,
-        'link': match[0].link,
-        'regexes': allRegexes,
-        'addresses': allAddresses,
-        'slug': slug
-      };
-
       if (existingPools.find((pool) => pool.name === poolNames[i]) !== undefined) {
-        finalPoolDataUpdate.push(poolObj);
+        logger.debug(`Update '${finalPoolName}' mining pool`);
+        finalPoolDataUpdate.push({
+          'name': finalPoolName,
+          'link': match[0].link,
+          'regexes': allRegexes,
+          'addresses': allAddresses,
+        });
       } else {
         logger.debug(`Add '${finalPoolName}' mining pool`);
-        finalPoolDataAdd.push(poolObj);
+        finalPoolDataAdd.push({
+          'name': finalPoolName,
+          'link': match[0].link,
+          'regexes': allRegexes,
+          'addresses': allAddresses,
+        });
       }
-
-      this.miningPools.push({
-        'name': finalPoolName,
-        'link': match[0].link,
-        'regexes': JSON.stringify(allRegexes),
-        'addresses': JSON.stringify(allAddresses),
-        'slug': slug
-      });
-    }
-
-    if (config.DATABASE.ENABLED === false) { // Don't run db operations
-      logger.info('Mining pools.json import completed (no database)');
-      return;
     }
 
     logger.debug(`Update pools table now`);
 
     // Add new mining pools into the database
-    let queryAdd: string = 'INSERT INTO pools(name, link, regexes, addresses, slug) VALUES ';
+    let queryAdd: string = 'INSERT INTO pools(name, link, regexes, addresses) VALUES ';
     for (let i = 0; i < finalPoolDataAdd.length; ++i) {
       queryAdd += `('${finalPoolDataAdd[i].name}', '${finalPoolDataAdd[i].link}',
-      '${JSON.stringify(finalPoolDataAdd[i].regexes)}', '${JSON.stringify(finalPoolDataAdd[i].addresses)}',
-      ${JSON.stringify(finalPoolDataAdd[i].slug)}),`;
+      '${JSON.stringify(finalPoolDataAdd[i].regexes)}', '${JSON.stringify(finalPoolDataAdd[i].addresses)}'),`;
     }
     queryAdd = queryAdd.slice(0, -1) + ';';
 
-    // Updated existing mining pools in the database
+    // Add new mining pools into the database
     const updateQueries: string[] = [];
     for (let i = 0; i < finalPoolDataUpdate.length; ++i) {
       updateQueries.push(`
         UPDATE pools
         SET name='${finalPoolDataUpdate[i].name}', link='${finalPoolDataUpdate[i].link}',
-        regexes='${JSON.stringify(finalPoolDataUpdate[i].regexes)}', addresses='${JSON.stringify(finalPoolDataUpdate[i].addresses)}',
-        slug='${finalPoolDataUpdate[i].slug}'
+        regexes='${JSON.stringify(finalPoolDataUpdate[i].regexes)}', addresses='${JSON.stringify(finalPoolDataUpdate[i].addresses)}'
         WHERE name='${finalPoolDataUpdate[i].name}'
       ;`);
     }
 
     try {
       if (finalPoolDataAdd.length > 0) {
-        await DB.query({ sql: queryAdd, timeout: 120000 });
+        await connection.query<any>({ sql: queryAdd, timeout: 120000 });
       }
       for (const query of updateQueries) {
-        await DB.query({ sql: query, timeout: 120000 });
+        await connection.query<any>({ sql: query, timeout: 120000 });
       }
       await this.insertUnknownPool();
+      connection.release();
       logger.info('Mining pools.json import completed');
     } catch (e) {
-      logger.err(`Cannot import pools in the database`);
+      connection.release();
+      logger.err(`Unable to import pools in the database!`);
       throw e;
     }
   }
@@ -182,24 +152,21 @@ class PoolsParser {
    * Manually add the 'unknown pool'
    */
   private async insertUnknownPool() {
+    const connection = await DB.pool.getConnection();
     try {
-      const [rows]: any[] = await DB.query({ sql: 'SELECT name from pools where name="Unknown"', timeout: 120000 });
+      const [rows]: any[] = await connection.query({ sql: 'SELECT name from pools where name="Unknown"', timeout: 120000 });
       if (rows.length === 0) {
-        await DB.query({
-          sql: `INSERT INTO pools(name, link, regexes, addresses, slug)
-          VALUES("Unknown", "https://learnmeabitcoin.com/technical/coinbase-transaction", "[]", "[]", "unknown");
+        logger.debug('Manually inserting "Unknown" mining pool into the databse');
+        await connection.query({
+          sql: `INSERT INTO pools(name, link, regexes, addresses)
+          VALUES("Unknown", "https://learnmeabitcoin.com/technical/coinbase-transaction", "[]", "[]");
         `});
-      } else {
-        await DB.query(`UPDATE pools
-          SET name='Unknown', link='https://learnmeabitcoin.com/technical/coinbase-transaction',
-          regexes='[]', addresses='[]',
-          slug='unknown'
-          WHERE name='Unknown'
-        `);
       }
     } catch (e) {
       logger.err('Unable to insert "Unknown" mining pool');
     }
+
+    connection.release();
   }
 }
 

backend/src/api/rbf-cache.ts

@@ -1,34 +0,0 @@
-export interface CachedRbf {
-  txid: string;
-  expires: Date;
-}
-
-class RbfCache {
-  private cache: { [txid: string]: CachedRbf; } = {};
-
-  constructor() {
-    setInterval(this.cleanup.bind(this), 1000 * 60 * 60);
-  }
-
-  public add(replacedTxId: string, newTxId: string): void {
-    this.cache[replacedTxId] = {
-      expires: new Date(Date.now() + 1000 * 604800), // 1 week
-      txid: newTxId,
-    };
-  }
-
-  public get(txId: string): CachedRbf | undefined {
-    return this.cache[txId];
-  }
-
-  private cleanup(): void {
-    const currentDate = new Date();
-    for (const c in this.cache) {
-      if (this.cache[c].expires < currentDate) {
-        delete this.cache[c];
-      }
-    }
-  }
-}
-
-export default new RbfCache();

backend/src/api/statistics.ts

@@ -1,5 +1,5 @@
 import memPool from './mempool';
-import DB from '../database';
+import { DB } from '../database';
 import logger from '../logger';
 
 import { Statistic, TransactionExtended, OptimizedStatistic } from '../mempool.interfaces';
@@ -155,6 +155,7 @@ class Statistics {
   }
 
   private async $createZeroedStatistic(): Promise<number | undefined> {
+    const connection = await DB.pool.getConnection();
     try {
       const query = `INSERT INTO statistics(
               added,
@@ -205,14 +206,17 @@ class Statistics {
             )
             VALUES (NOW(), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)`;
-      const [result]: any = await DB.query(query);
+      const [result]: any = await connection.query(query);
+      connection.release();
       return result.insertId;
     } catch (e) {
+      connection.release();
       logger.err('$create() error' + (e instanceof Error ? e.message : e));
     }
   }
 
   private async $create(statistics: Statistic): Promise<number | undefined> {
+    const connection = await DB.pool.getConnection();
     try {
       const query = `INSERT INTO statistics(
               added,
@@ -310,9 +314,11 @@ class Statistics {
         statistics.vsize_1800,
         statistics.vsize_2000,
       ];
-      const [result]: any = await DB.query(query, params);
+      const [result]: any = await connection.query(query, params);
+      connection.release();
       return result.insertId;
     } catch (e) {
+      connection.release();
       logger.err('$create() error' + (e instanceof Error ? e.message : e));
     }
   }
@@ -415,8 +421,10 @@ class Statistics {
 
   private async $get(id: number): Promise<OptimizedStatistic | undefined> {
     try {
+      const connection = await DB.pool.getConnection();
       const query = `SELECT *, UNIX_TIMESTAMP(added) as added FROM statistics WHERE id = ?`;
-      const [rows] = await DB.query(query, [id]);
+      const [rows] = await connection.query<any>(query, [id]);
+      connection.release();
       if (rows[0]) {
         return this.mapStatisticToOptimizedStatistic([rows[0]])[0];
       }
@@ -427,9 +435,11 @@ class Statistics {
 
   public async $list2H(): Promise<OptimizedStatistic[]> {
     try {
+      const connection = await DB.pool.getConnection();
       const query = `SELECT *, UNIX_TIMESTAMP(added) as added FROM statistics ORDER BY statistics.added DESC LIMIT 120`;
-      const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
-      return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
+      const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
+      connection.release();
+      return this.mapStatisticToOptimizedStatistic(rows);
     } catch (e) {
       logger.err('$list2H() error' + (e instanceof Error ? e.message : e));
       return [];
@@ -438,9 +448,11 @@ class Statistics {
 
   public async $list24H(): Promise<OptimizedStatistic[]> {
     try {
+      const connection = await DB.pool.getConnection();
       const query = `SELECT *, UNIX_TIMESTAMP(added) as added FROM statistics ORDER BY statistics.added DESC LIMIT 1440`;
-      const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
-      return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
+      const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
+      connection.release();
+      return this.mapStatisticToOptimizedStatistic(rows);
     } catch (e) {
       logger.err('$list24h() error' + (e instanceof Error ? e.message : e));
       return [];
@@ -449,9 +461,11 @@ class Statistics {
 
   public async $list1W(): Promise<OptimizedStatistic[]> {
     try {
+      const connection = await DB.pool.getConnection();
       const query = this.getQueryForDaysAvg(300, '1 WEEK'); // 5m interval
-      const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
-      return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
+      const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
+      connection.release();
+      return this.mapStatisticToOptimizedStatistic(rows);
     } catch (e) {
       logger.err('$list1W() error' + (e instanceof Error ? e.message : e));
       return [];
@@ -460,9 +474,11 @@ class Statistics {
 
   public async $list1M(): Promise<OptimizedStatistic[]> {
     try {
+      const connection = await DB.pool.getConnection();
       const query = this.getQueryForDaysAvg(1800, '1 MONTH'); // 30m interval
-      const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
-      return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
+      const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
+      connection.release();
+      return this.mapStatisticToOptimizedStatistic(rows);
     } catch (e) {
       logger.err('$list1M() error' + (e instanceof Error ? e.message : e));
       return [];
@@ -471,9 +487,11 @@ class Statistics {
 
   public async $list3M(): Promise<OptimizedStatistic[]> {
     try {
+      const connection = await DB.pool.getConnection();
       const query = this.getQueryForDaysAvg(7200, '3 MONTH'); // 2h interval
-      const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
-      return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
+      const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
+      connection.release();
+      return this.mapStatisticToOptimizedStatistic(rows);
     } catch (e) {
       logger.err('$list3M() error' + (e instanceof Error ? e.message : e));
       return [];
@@ -482,9 +500,11 @@ class Statistics {
 
   public async $list6M(): Promise<OptimizedStatistic[]> {
     try {
-      const query = this.getQueryForDaysAvg(10800, '6 MONTH'); // 3h interval
-      const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
-      return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
+      const connection = await DB.pool.getConnection();
+      const query = this.getQueryForDaysAvg(10800, '6 MONTH'); // 3h interval 
+      const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
+      connection.release();
+      return this.mapStatisticToOptimizedStatistic(rows);
     } catch (e) {
       logger.err('$list6M() error' + (e instanceof Error ? e.message : e));
       return [];
@@ -493,9 +513,11 @@ class Statistics {
 
   public async $list1Y(): Promise<OptimizedStatistic[]> {
     try {
+      const connection = await DB.pool.getConnection();
       const query = this.getQueryForDays(28800, '1 YEAR'); // 8h interval
-      const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
-      return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
+      const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
+      connection.release();
+      return this.mapStatisticToOptimizedStatistic(rows);
     } catch (e) {
       logger.err('$list1Y() error' + (e instanceof Error ? e.message : e));
       return [];
@@ -504,9 +526,11 @@ class Statistics {
 
   public async $list2Y(): Promise<OptimizedStatistic[]> {
     try {
-      const query = this.getQueryForDays(28800, '2 YEAR'); // 8h interval
-      const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
-      return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
+      const connection = await DB.pool.getConnection();
+      const query = this.getQueryForDays(28800, "2 YEAR"); // 8h interval
+      const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
+      connection.release();
+      return this.mapStatisticToOptimizedStatistic(rows);
     } catch (e) {
       logger.err('$list2Y() error' + (e instanceof Error ? e.message : e));
       return [];
@@ -515,9 +539,11 @@ class Statistics {
 
   public async $list3Y(): Promise<OptimizedStatistic[]> {
     try {
-      const query = this.getQueryForDays(43200, '3 YEAR'); // 12h interval
-      const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
-      return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
+      const connection = await DB.pool.getConnection();
+      const query = this.getQueryForDays(43200, "3 YEAR"); // 12h interval
+      const [rows] = await connection.query<any>({ sql: query, timeout: this.queryTimeout });
+      connection.release();
+      return this.mapStatisticToOptimizedStatistic(rows);
     } catch (e) {
       logger.err('$list3Y() error' + (e instanceof Error ? e.message : e));
       return [];

backend/src/api/transaction-utils.ts

@@ -21,8 +21,8 @@ class TransactionUtils {
     };
   }
 
-  public async $getTransactionExtended(txId: string, addPrevouts = false, lazyPrevouts = false): Promise<TransactionExtended> {
-    const transaction: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(txId, false, addPrevouts, lazyPrevouts);
+  public async $getTransactionExtended(txId: string, addPrevouts = false): Promise<TransactionExtended> {
+    const transaction: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(txId, false, addPrevouts);
     return this.extendTransaction(transaction);
   }
 

backend/src/api/websocket-handler.ts

@@ -1,6 +1,6 @@
 import logger from '../logger';
 import * as WebSocket from 'ws';
-import { BlockExtended, TransactionExtended, WebsocketResponse, MempoolBlock, MempoolBlockDelta,
+import { BlockExtended, TransactionExtended, WebsocketResponse, MempoolBlock,
   OptimizedStatistic, ILoadingIndicators, IConversionRates } from '../mempool.interfaces';
 import blocks from './blocks';
 import memPool from './mempool';
@@ -11,9 +11,6 @@ import { Common } from './common';
 import loadingIndicators from './loading-indicators';
 import config from '../config';
 import transactionUtils from './transaction-utils';
-import rbfCache from './rbf-cache';
-import difficultyAdjustment from './difficulty-adjustment';
-import feeApi from './fee-api';
 
 class WebsocketHandler {
   private wss: WebSocket.Server | undefined;
@@ -51,38 +48,29 @@ class WebsocketHandler {
           if (parsedMessage && parsedMessage['track-tx']) {
             if (/^[a-fA-F0-9]{64}$/.test(parsedMessage['track-tx'])) {
               client['track-tx'] = parsedMessage['track-tx'];
-              // Client is telling the transaction wasn't found
+              // Client is telling the transaction wasn't found but it might have appeared before we had the time to start watching for it
               if (parsedMessage['watch-mempool']) {
-                const rbfCacheTx = rbfCache.get(client['track-tx']);
-                if (rbfCacheTx) {
-                  response['txReplaced'] = {
-                    txid: rbfCacheTx.txid,
-                  };
-                  client['track-tx'] = null;
-                } else {
-                  // It might have appeared before we had the time to start watching for it
-                  const tx = memPool.getMempool()[client['track-tx']];
-                  if (tx) {
-                    if (config.MEMPOOL.BACKEND === 'esplora') {
-                      response['tx'] = tx;
-                    } else {
-                      // tx.prevout is missing from transactions when in bitcoind mode
-                      try {
-                        const fullTx = await transactionUtils.$getTransactionExtended(tx.txid, true);
-                        response['tx'] = fullTx;
-                      } catch (e) {
-                        logger.debug('Error finding transaction: ' + (e instanceof Error ? e.message : e));
-                      }
-                    }
+                const tx = memPool.getMempool()[client['track-tx']];
+                if (tx) {
+                  if (config.MEMPOOL.BACKEND === 'esplora') {
+                    response['tx'] = tx;
                   } else {
+                    // tx.prevouts is missing from transactions when in bitcoind mode
                     try {
-                      const fullTx = await transactionUtils.$getTransactionExtended(client['track-tx'], true);
+                      const fullTx = await transactionUtils.$getTransactionExtended(tx.txid, true);
                       response['tx'] = fullTx;
                     } catch (e) {
-                      logger.debug('Error finding transaction. ' + (e instanceof Error ? e.message : e));
-                      client['track-mempool-tx'] = parsedMessage['track-tx'];
+                      logger.debug('Error finding transaction: ' + (e instanceof Error ? e.message : e));
                     }
                   }
+                } else {
+                  try {
+                    const fullTx = await transactionUtils.$getTransactionExtended(client['track-tx'], true);
+                    response['tx'] = fullTx;
+                  } catch (e) {
+                    logger.debug('Error finding transaction. ' + (e instanceof Error ? e.message : e));
+                    client['track-mempool-tx'] = parsedMessage['track-tx'];
+                  }
                 }
               }
             } else {
@@ -111,20 +99,6 @@ class WebsocketHandler {
             }
           }
 
-          if (parsedMessage && parsedMessage['track-mempool-block'] !== undefined) {
-            if (Number.isInteger(parsedMessage['track-mempool-block']) && parsedMessage['track-mempool-block'] >= 0) {
-              const index = parsedMessage['track-mempool-block'];
-              client['track-mempool-block'] = index;
-              const mBlocksWithTransactions = mempoolBlocks.getMempoolBlocksWithTransactions();
-              response['projected-block-transactions'] = {
-                index: index,
-                blockTransactions: mBlocksWithTransactions[index]?.transactions || [],
-              };
-            } else {
-              client['track-mempool-block'] = null;
-            }
-          }
-
           if (parsedMessage.action === 'init') {
             const _blocks = blocks.getBlocks().slice(-config.MEMPOOL.INITIAL_BLOCKS_AMOUNT);
             if (!_blocks) {
@@ -207,14 +181,14 @@ class WebsocketHandler {
     return {
       'mempoolInfo': memPool.getMempoolInfo(),
       'vBytesPerSecond': memPool.getVBytesPerSecond(),
+      'lastDifficultyAdjustment': blocks.getLastDifficultyAdjustmentTime(),
+      'previousRetarget': blocks.getPreviousDifficultyRetarget(),
       'blocks': _blocks,
       'conversions': fiatConversion.getConversionRates(),
       'mempool-blocks': mempoolBlocks.getMempoolBlocks(),
       'transactions': memPool.getLatestTransactions(),
       'backendInfo': backendInfo.getBackendInfo(),
       'loadingIndicators': loadingIndicators.getLoadingIndicators(),
-      'da': difficultyAdjustment.getDifficultyAdjustment(),
-      'fees': feeApi.getRecommendedFee(),
       ...this.extraInitProperties
     };
   }
@@ -247,13 +221,14 @@ class WebsocketHandler {
 
     mempoolBlocks.updateMempoolBlocks(newMempool);
     const mBlocks = mempoolBlocks.getMempoolBlocks();
-    const mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();
+    const mempool = memPool.getMempool();
     const mempoolInfo = memPool.getMempoolInfo();
     const vBytesPerSecond = memPool.getVBytesPerSecond();
     const rbfTransactions = Common.findRbfTransactions(newTransactions, deletedTransactions);
-    const da = difficultyAdjustment.getDifficultyAdjustment();
-    memPool.handleRbfTransactions(rbfTransactions);
-    const recommendedFees = feeApi.getRecommendedFee();
+
+    for (const rbfTransaction in rbfTransactions) {
+      delete mempool[rbfTransaction];
+    }
 
     this.wss.clients.forEach(async (client: WebSocket) => {
       if (client.readyState !== WebSocket.OPEN) {
@@ -266,8 +241,6 @@ class WebsocketHandler {
         response['mempoolInfo'] = mempoolInfo;
         response['vBytesPerSecond'] = vBytesPerSecond;
         response['transactions'] = newTransactions.slice(0, 6).map((tx) => Common.stripTransaction(tx));
-        response['da'] = da;
-        response['fees'] = recommendedFees;
       }
 
       if (client['want-mempool-blocks']) {
@@ -358,43 +331,25 @@ class WebsocketHandler {
         }
       }
 
-      if (client['track-tx']) {
-        const outspends: object = {};
-        newTransactions.forEach((tx) => tx.vin.forEach((vin, i) => {
-          if (vin.txid === client['track-tx']) {
-            outspends[vin.vout] = {
-              vin: i,
-              txid: tx.txid,
-            };
-          }
-        }));
-
-        if (Object.keys(outspends).length) {
-          response['utxoSpent'] = outspends;
-        }
-
-        if (rbfTransactions[client['track-tx']]) {
-          for (const rbfTransaction in rbfTransactions) {
-            if (client['track-tx'] === rbfTransaction) {
-              response['rbfTransaction'] = {
-                txid: rbfTransactions[rbfTransaction].txid,
-              };
-              break;
+      if (client['track-tx'] && rbfTransactions[client['track-tx']]) {
+        for (const rbfTransaction in rbfTransactions) {
+          if (client['track-tx'] === rbfTransaction) {
+            const rbfTx = rbfTransactions[rbfTransaction];
+            if (config.MEMPOOL.BACKEND !== 'esplora') {
+              try {
+                const fullTx = await transactionUtils.$getTransactionExtended(rbfTransaction, true);
+                response['rbfTransaction'] = fullTx;
+              } catch (e) {
+                logger.debug('Error finding transaction in mempool: ' + (e instanceof Error ? e.message : e));
+              }
+            } else {
+              response['rbfTransaction'] = rbfTx;
             }
+            break;
           }
         }
       }
 
-      if (client['track-mempool-block'] >= 0) {
-        const index = client['track-mempool-block'];
-        if (mBlockDeltas[index]) {
-          response['projected-block-transactions'] = {
-            index: index,
-            delta: mBlockDeltas[index],
-          };
-        }
-      }
-
       if (Object.keys(response).length) {
         client.send(JSON.stringify(response));
       }
@@ -407,7 +362,6 @@ class WebsocketHandler {
     }
 
     let mBlocks: undefined | MempoolBlock[];
-    let mBlockDeltas: undefined | MempoolBlockDelta[];
     let matchRate = 0;
     const _memPool = memPool.getMempool();
     const _mempoolBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
@@ -424,15 +378,9 @@ class WebsocketHandler {
       matchRate = Math.round((matches.length / (txIds.length - 1)) * 100);
       mempoolBlocks.updateMempoolBlocks(_memPool);
       mBlocks = mempoolBlocks.getMempoolBlocks();
-      mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();
     }
 
-    if (block.extras) {
-      block.extras.matchRate = matchRate;
-    }
-
-    const da = difficultyAdjustment.getDifficultyAdjustment();
-    const fees = feeApi.getRecommendedFee();
+    block.matchRate = matchRate;
 
     this.wss.clients.forEach((client) => {
       if (client.readyState !== WebSocket.OPEN) {
@@ -446,8 +394,8 @@ class WebsocketHandler {
       const response = {
         'block': block,
         'mempoolInfo': memPool.getMempoolInfo(),
-        'da': da,
-        'fees': fees,
+        'lastDifficultyAdjustment': blocks.getLastDifficultyAdjustmentTime(),
+        'previousRetarget': blocks.getPreviousDifficultyRetarget(),
       };
 
       if (mBlocks && client['want-mempool-blocks']) {
@@ -455,6 +403,7 @@ class WebsocketHandler {
       }
 
       if (client['track-tx'] && txIds.indexOf(client['track-tx']) > -1) {
+        client['track-tx'] = null;
         response['txConfirmed'] = true;
       }
 
@@ -522,16 +471,6 @@ class WebsocketHandler {
         }
       }
 
-      if (client['track-mempool-block'] >= 0) {
-        const index = client['track-mempool-block'];
-        if (mBlockDeltas && mBlockDeltas[index]) {
-          response['projected-block-transactions'] = {
-            index: index,
-            delta: mBlockDeltas[index],
-          };
-        }
-      }
-
       client.send(JSON.stringify(response));
     });
   }

backend/src/config.ts

@@ -18,10 +18,6 @@ interface IConfig {
     PRICE_FEED_UPDATE_INTERVAL: number;
     USE_SECOND_NODE_FOR_MINFEE: boolean;
     EXTERNAL_ASSETS: string[];
-    EXTERNAL_MAX_RETRY: number;
-    EXTERNAL_RETRY_INTERVAL: number;
-    USER_AGENT: string;
-    STDOUT_LOG_MIN_PRIORITY: 'emerg' | 'alert' | 'crit' | 'err' | 'warn' | 'notice' | 'info' | 'debug';
   };
   ESPLORA: {
     REST_API_URL: string;
@@ -46,7 +42,6 @@ interface IConfig {
   DATABASE: {
     ENABLED: boolean;
     HOST: string,
-    SOCKET: string,
     PORT: number;
     DATABASE: string;
     USERNAME: string;
@@ -56,7 +51,7 @@ interface IConfig {
     ENABLED: boolean;
     HOST: string;
     PORT: number;
-    MIN_PRIORITY: 'emerg' | 'alert' | 'crit' | 'err' | 'warn' | 'notice' | 'info' | 'debug';
+    MIN_PRIORITY: 'emerg' | 'alert' | 'crit' | 'err' |'warn' | 'notice' | 'info' | 'debug';
     FACILITY: string;
   };
   STATISTICS: {
@@ -67,26 +62,6 @@ interface IConfig {
     ENABLED: boolean;
     DATA_PATH: string;
   };
-  SOCKS5PROXY: {
-    ENABLED: boolean;
-    USE_ONION: boolean;
-    HOST: string;
-    PORT: number;
-    USERNAME: string;
-    PASSWORD: string;
-  };
-  PRICE_DATA_SERVER: {
-    TOR_URL: string;
-    CLEARNET_URL: string;
-  };
-  EXTERNAL_DATA_SERVER: {
-    MEMPOOL_API: string;
-    MEMPOOL_ONION: string;
-    LIQUID_API: string;
-    LIQUID_ONION: string;
-    BISQ_URL: string;
-    BISQ_ONION: string;
-  };
 }
 
 const defaults: IConfig = {
@@ -103,14 +78,12 @@ const defaults: IConfig = {
     'BLOCK_WEIGHT_UNITS': 4000000,
     'INITIAL_BLOCKS_AMOUNT': 8,
     'MEMPOOL_BLOCKS_AMOUNT': 8,
-    'INDEXING_BLOCKS_AMOUNT': 11000, // 0 = disable indexing, -1 = index all blocks
-    'PRICE_FEED_UPDATE_INTERVAL': 600,
+    'INDEXING_BLOCKS_AMOUNT': 1100, // 0 = disable indexing, -1 = index all blocks
+    'PRICE_FEED_UPDATE_INTERVAL': 3600,
     'USE_SECOND_NODE_FOR_MINFEE': false,
-    'EXTERNAL_ASSETS': [],
-    'EXTERNAL_MAX_RETRY': 1,
-    'EXTERNAL_RETRY_INTERVAL': 0,
-    'USER_AGENT': 'mempool',
-    'STDOUT_LOG_MIN_PRIORITY': 'debug',
+    'EXTERNAL_ASSETS': [
+      'https://mempool.space/resources/pools.json'
+    ]
   },
   'ESPLORA': {
     'REST_API_URL': 'http://127.0.0.1:3000',
@@ -135,7 +108,6 @@ const defaults: IConfig = {
   'DATABASE': {
     'ENABLED': true,
     'HOST': '127.0.0.1',
-    'SOCKET': '',
     'PORT': 3306,
     'DATABASE': 'mempool',
     'USERNAME': 'mempool',
@@ -156,26 +128,6 @@ const defaults: IConfig = {
     'ENABLED': false,
     'DATA_PATH': '/bisq/statsnode-data/btc_mainnet/db'
   },
-  'SOCKS5PROXY': {
-    'ENABLED': false,
-    'USE_ONION': true,
-    'HOST': '127.0.0.1',
-    'PORT': 9050,
-    'USERNAME': '',
-    'PASSWORD': ''
-  },
-  "PRICE_DATA_SERVER": {
-    'TOR_URL': 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices',
-    'CLEARNET_URL': 'https://price.bisq.wiz.biz/getAllMarketPrices'
-  },
-  "EXTERNAL_DATA_SERVER": {
-    'MEMPOOL_API': 'https://mempool.space/api/v1',
-    'MEMPOOL_ONION': 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1',
-    'LIQUID_API': 'https://liquid.network/api/v1',
-    'LIQUID_ONION': 'http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1',
-    'BISQ_URL': 'https://bisq.markets/api',
-    'BISQ_ONION': 'http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api'
-  }
 };
 
 class Config implements IConfig {
@@ -188,9 +140,6 @@ class Config implements IConfig {
   SYSLOG: IConfig['SYSLOG'];
   STATISTICS: IConfig['STATISTICS'];
   BISQ: IConfig['BISQ'];
-  SOCKS5PROXY: IConfig['SOCKS5PROXY'];
-  PRICE_DATA_SERVER: IConfig['PRICE_DATA_SERVER'];
-  EXTERNAL_DATA_SERVER: IConfig['EXTERNAL_DATA_SERVER'];
 
   constructor() {
     const configs = this.merge(configFile, defaults);
@@ -203,9 +152,6 @@ class Config implements IConfig {
     this.SYSLOG = configs.SYSLOG;
     this.STATISTICS = configs.STATISTICS;
     this.BISQ = configs.BISQ;
-    this.SOCKS5PROXY = configs.SOCKS5PROXY;
-    this.PRICE_DATA_SERVER = configs.PRICE_DATA_SERVER;
-    this.EXTERNAL_DATA_SERVER = configs.EXTERNAL_DATA_SERVER;
   }
 
   merge = (...objects: object[]): IConfig => {

backend/src/database.ts

@@ -1,59 +1,26 @@
 import config from './config';
-import { createPool, Pool, PoolConnection } from 'mysql2/promise';
+import { createPool } from 'mysql2/promise';
 import logger from './logger';
-import { PoolOptions } from 'mysql2/typings/mysql';
 
- class DB {
-  constructor() {
-    if (config.DATABASE.SOCKET !== '') {
-      this.poolConfig.socketPath = config.DATABASE.SOCKET;
-    } else {
-      this.poolConfig.host = config.DATABASE.HOST;
-    }
-  }
-  private pool: Pool | null = null;
-  private poolConfig: PoolOptions = {
+export class DB {
+  static pool = createPool({
+    host: config.DATABASE.HOST,
     port: config.DATABASE.PORT,
     database: config.DATABASE.DATABASE,
     user: config.DATABASE.USERNAME,
     password: config.DATABASE.PASSWORD,
     connectionLimit: 10,
     supportBigNumbers: true,
-    timezone: '+00:00',
-  };
-
-  private checkDBFlag() {
-    if (config.DATABASE.ENABLED === false) {
-      logger.err('Trying to use DB feature but config.DATABASE.ENABLED is set to false, please open an issue');
-    }
-  }
-
-  public async query(query, params?) {
-    this.checkDBFlag();
-    const pool = await this.getPool();
-    return pool.query(query, params);
-  }
-
-  public async checkDbConnection() {
-    this.checkDBFlag();
-    try {
-      await this.query('SELECT ?', [1]);
-      logger.info('Database connection established.');
-    } catch (e) {
-      logger.err('Could not connect to database: ' + (e instanceof Error ? e.message : e));
-      process.exit(1);
-    }
-  }
-
-  private async getPool(): Promise<Pool> {
-    if (this.pool === null) {
-      this.pool = createPool(this.poolConfig);
-      this.pool.on('connection', function (newConnection: PoolConnection) {
-        newConnection.query(`SET time_zone='+00:00'`);
-      });
-    }
-    return this.pool;
-  }
+  });
 }
 
-export default new DB();
+export async function checkDbConnection() {
+  try {
+    const connection = await DB.pool.getConnection();
+    logger.info('Database connection established.');
+    connection.release();
+  } catch (e) {
+    logger.err('Could not connect to database: ' + (e instanceof Error ? e.message : e));
+    process.exit(1);
+  }
+}

backend/src/index.ts

@@ -5,7 +5,7 @@ import * as WebSocket from 'ws';
 import * as cluster from 'cluster';
 import axios from 'axios';
 
-import DB from './database';
+import { checkDbConnection } from './database';
 import config from './config';
 import routes from './routes';
 import blocks from './api/blocks';
@@ -22,11 +22,10 @@ import loadingIndicators from './api/loading-indicators';
 import mempool from './api/mempool';
 import elementsParser from './api/liquid/elements-parser';
 import databaseMigration from './api/database-migration';
+import poolsParser from './api/pools-parser';
 import syncAssets from './sync-assets';
 import icons from './api/liquid/icons';
 import { Common } from './api/common';
-import poolsUpdater from './tasks/pools-updater';
-import indexer from './indexer';
 
 class Server {
   private wss: WebSocket.Server | undefined;
@@ -67,7 +66,7 @@ class Server {
   }
 
   async startServer(worker = false) {
-    logger.notice(`Starting Mempool Server${worker ? ' (worker)' : ''}... (${backendInfo.getShortCommitHash()})`);
+    logger.debug(`Starting Mempool Server${worker ? ' (worker)' : ''}... (${backendInfo.getShortCommitHash()})`);
 
     this.app
       .use((req: Request, res: Response, next: NextFunction) => {
@@ -83,22 +82,14 @@ class Server {
 
     this.setUpWebsocketHandling();
 
-    await syncAssets.syncAssets$();
+    await syncAssets.syncAssets();
     diskCache.loadMempoolCache();
 
     if (config.DATABASE.ENABLED) {
-      await DB.checkDbConnection();
+      await checkDbConnection();
       try {
-        if (process.env.npm_config_reindex !== undefined) { // Re-index requests
-          const tables = process.env.npm_config_reindex.split(',');
-          logger.warn(`Indexed data for "${process.env.npm_config_reindex}" tables will be erased in 5 seconds (using '--reindex')`);
-          await Common.sleep$(5000);
-          await databaseMigration.$truncateIndexedData(tables);
-        }
         await databaseMigration.$initializeOrMigrateDatabase();
-        if (Common.indexingEnabled()) {
-          await indexer.$resetHashratesIndexingState();
-        }
+        await poolsParser.migratePoolsJson();
       } catch (e) {
         throw new Error(e instanceof Error ? e.message : 'Error');
       }
@@ -109,11 +100,7 @@ class Server {
     }
 
     if (Common.isLiquid()) {
-      try {
-        icons.loadIcons();
-      } catch (e) {
-        logger.err('Cannot load liquid icons. Ignoring. Reason: ' + (e instanceof Error ? e.message : e));
-      }
+      icons.loadIcons();
     }
 
     fiatConversion.startService();
@@ -149,10 +136,9 @@ class Server {
           logger.debug(msg);
         }
       }
-      await poolsUpdater.updatePoolsJson();
       await blocks.$updateBlocks();
       await memPool.$updateMempool();
-      indexer.$run();
+      blocks.$generateBlockDatabase();
 
       setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS);
       this.currentBackendRetryInterval = 5;
@@ -205,7 +191,7 @@ class Server {
       .post(config.MEMPOOL.API_URL_PREFIX + 'tx/push', routes.$postTransactionForm)
       .get(config.MEMPOOL.API_URL_PREFIX + 'donations', async (req, res) => {
         try {
-          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/donations`, { responseType: 'stream', timeout: 10000 });
+          const response = await axios.get('https://mempool.space/api/v1/donations', { responseType: 'stream', timeout: 10000 });
           response.data.pipe(res);
         } catch (e) {
           res.status(500).end();
@@ -213,7 +199,7 @@ class Server {
       })
       .get(config.MEMPOOL.API_URL_PREFIX + 'donations/images/:id', async (req, res) => {
         try {
-          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/donations/images/${req.params.id}`, {
+          const response = await axios.get('https://mempool.space/api/v1/donations/images/' + req.params.id, {
             responseType: 'stream', timeout: 10000
           });
           response.data.pipe(res);
@@ -223,7 +209,7 @@ class Server {
       })
       .get(config.MEMPOOL.API_URL_PREFIX + 'contributors', async (req, res) => {
         try {
-          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/contributors`, { responseType: 'stream', timeout: 10000 });
+          const response = await axios.get('https://mempool.space/api/v1/contributors', { responseType: 'stream', timeout: 10000 });
           response.data.pipe(res);
         } catch (e) {
           res.status(500).end();
@@ -231,7 +217,7 @@ class Server {
       })
       .get(config.MEMPOOL.API_URL_PREFIX + 'contributors/images/:id', async (req, res) => {
         try {
-          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/contributors/images/${req.params.id}`, {
+          const response = await axios.get('https://mempool.space/api/v1/contributors/images/' + req.params.id, {
             responseType: 'stream', timeout: 10000
           });
           response.data.pipe(res);
@@ -241,7 +227,7 @@ class Server {
       })
       .get(config.MEMPOOL.API_URL_PREFIX + 'translators', async (req, res) => {
         try {
-          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/translators`, { responseType: 'stream', timeout: 10000 });
+          const response = await axios.get('https://mempool.space/api/v1/translators', { responseType: 'stream', timeout: 10000 });
           response.data.pipe(res);
         } catch (e) {
           res.status(500).end();
@@ -249,7 +235,7 @@ class Server {
       })
       .get(config.MEMPOOL.API_URL_PREFIX + 'translators/images/:id', async (req, res) => {
         try {
-          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/translators/images/${req.params.id}`, {
+          const response = await axios.get('https://mempool.space/api/v1/translators/images/' + req.params.id, {
             responseType: 'stream', timeout: 10000
           });
           response.data.pipe(res);
@@ -270,26 +256,10 @@ class Server {
         .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/1y', routes.$getStatisticsByTime.bind(routes, '1y'))
         .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/2y', routes.$getStatisticsByTime.bind(routes, '2y'))
         .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/3y', routes.$getStatisticsByTime.bind(routes, '3y'))
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools', routes.$getPools)
         ;
     }
 
-    if (Common.indexingEnabled()) {
-      this.app
-        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/:interval', routes.$getPools)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug/hashrate', routes.$getPoolHistoricalHashrate)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug/blocks', routes.$getPoolBlocks)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug/blocks/:height', routes.$getPoolBlocks)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug', routes.$getPool)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/hashrate/pools/:interval', routes.$getPoolsHistoricalHashrate)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/hashrate/:interval', routes.$getHistoricalHashrate)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/reward-stats/:blockCount', routes.$getRewardStats)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/fees/:interval', routes.$getHistoricalBlockFees)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/rewards/:interval', routes.$getHistoricalBlockRewards)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/fee-rates/:interval', routes.$getHistoricalBlockFeeRates)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/sizes-weights/:interval', routes.$getHistoricalBlockSizeAndWeight)
-      ;
-    }
-
     if (config.BISQ.ENABLED) {
       this.app
         .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/stats', routes.getBisqStats)
@@ -311,11 +281,6 @@ class Server {
         ;
     }
 
-    this.app
-      .get(config.MEMPOOL.API_URL_PREFIX + 'blocks', routes.getBlocks.bind(routes))
-      .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/:height', routes.getBlocks.bind(routes))
-      .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash', routes.getBlock);
-
     if (config.MEMPOOL.BACKEND !== 'esplora') {
       this.app
         .get(config.MEMPOOL.API_URL_PREFIX + 'mempool', routes.getMempool)
@@ -326,7 +291,10 @@ class Server {
         .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/hex', routes.getRawTransaction)
         .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/status', routes.getTransactionStatus)
         .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/outspends', routes.getTransactionOutspends)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash', routes.getBlock)
         .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/header', routes.getBlockHeader)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'blocks', routes.getBlocks)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/:height', routes.getBlocks)
         .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/height', routes.getBlockTipHeight)
         .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txs', routes.getBlockTransactions)
         .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txs/:index', routes.getBlockTransactions)
@@ -342,9 +310,7 @@ class Server {
     if (Common.isLiquid()) {
       this.app
         .get(config.MEMPOOL.API_URL_PREFIX + 'assets/icons', routes.getAllLiquidIcon)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'assets/featured', routes.$getAllFeaturedLiquidAssets)
         .get(config.MEMPOOL.API_URL_PREFIX + 'asset/:assetId/icon', routes.getLiquidIcon)
-        .get(config.MEMPOOL.API_URL_PREFIX + 'assets/group/:id', routes.$getAssetGroup)
       ;
     }
 

backend/src/indexer.ts

@@ -1,54 +0,0 @@
-import { Common } from './api/common';
-import blocks from './api/blocks';
-import mempool from './api/mempool';
-import mining from './api/mining';
-import logger from './logger';
-import HashratesRepository from './repositories/HashratesRepository';
-
-class Indexer {
-  runIndexer = true;
-  indexerRunning = false;
-
-  constructor() {
-  }
-
-  public reindex() {
-    if (Common.indexingEnabled()) {
-      this.runIndexer = true;
-    }
-  }
-
-  public async $run() {
-    if (!Common.indexingEnabled() || this.runIndexer === false ||
-      this.indexerRunning === true || mempool.hasPriority()
-    ) {
-      return;
-    }
-
-    this.runIndexer = false;
-    this.indexerRunning = true;
-
-    try {
-      await blocks.$generateBlockDatabase();
-      await this.$resetHashratesIndexingState();
-      await mining.$generateNetworkHashrateHistory();
-      await mining.$generatePoolHashrateHistory();
-    } catch (e) {
-      this.reindex();
-      logger.err(`Indexer failed, trying again later. Reason: ` + (e instanceof Error ? e.message : e));
-    }
-
-    this.indexerRunning = false;
-  }
-
-  async $resetHashratesIndexingState() {
-    try {
-      await HashratesRepository.$setLatestRun('last_hashrates_indexing', 0);
-      await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', 0);
-    } catch (e) {
-      logger.err(`Cannot reset hashrate indexing timestamps. Reason: ` + (e instanceof Error ? e.message : e));
-    }
-  }
-}
-
-export default new Indexer();

backend/src/logger.ts

@@ -97,9 +97,6 @@ class Logger {
       syslogmsg = `<${(Logger.facilities[config.SYSLOG.FACILITY] * 8 + prionum)}> ${this.name}[${process.pid}]: ${priority.toUpperCase()}${network} ${msg}`;
       this.syslog(syslogmsg);
     }
-    if (Logger.priorities[priority] > Logger.priorities[config.MEMPOOL.STDOUT_LOG_MIN_PRIORITY]) {
-      return;
-    }
     if (priority === 'warning') {
       priority = 'warn';
     }

backend/src/mempool.interfaces.ts

@@ -1,25 +1,23 @@
 import { IEsploraApi } from './api/bitcoin/esplora-api.interface';
 
 export interface PoolTag {
-  id: number; // mysql row id
-  name: string;
-  link: string;
-  regexes: string; // JSON array
-  addresses: string; // JSON array
-  slug: string;
+  id: number | null, // mysql row id
+  name: string,
+  link: string,
+  regexes: string, // JSON array
+  addresses: string, // JSON array
 }
 
 export interface PoolInfo {
-  poolId: number; // mysql row id
-  name: string;
-  link: string;
-  blockCount: number;
-  slug: string;
+  poolId: number, // mysql row id
+  name: string,
+  link: string,
+  blockCount: number,
 }
 
 export interface PoolStats extends PoolInfo {
-  rank: number;
-  emptyBlocks: number;
+  rank: number,
+  emptyBlocks: number,
 }
 
 export interface MempoolBlock {
@@ -33,12 +31,6 @@ export interface MempoolBlock {
 
 export interface MempoolBlockWithTransactions extends MempoolBlock {
   transactionIds: string[];
-  transactions: TransactionStripped[];
-}
-
-export interface MempoolBlockDelta {
-  added: TransactionStripped[];
-  removed: string[];
 }
 
 interface VinStrippedToScriptsig {
@@ -84,26 +76,12 @@ export interface TransactionStripped {
   vsize: number;
   value: number;
 }
-
-export interface BlockExtension {
-  totalFees?: number;
+export interface BlockExtended extends IEsploraApi.Block {
   medianFee?: number;
   feeRange?: number[];
   reward?: number;
   coinbaseTx?: TransactionMinerInfo;
   matchRate?: number;
-  pool?: {
-    id: number;
-    name: string;
-    slug: string;
-  };
-  avgFee?: number;
-  avgFeeRate?: number;
-  coinbaseRaw?: string;
-}
-
-export interface BlockExtended extends IEsploraApi.Block {
-  extras: BlockExtension;
 }
 
 export interface TransactionMinerInfo {
@@ -206,21 +184,3 @@ export interface IBackendInfo {
   gitCommit: string;
   version: string;
 }
-
-export interface IDifficultyAdjustment {
-  progressPercent: number;
-  difficultyChange: number;
-  estimatedRetargetDate: number;
-  remainingBlocks: number;
-  remainingTime: number;
-  previousRetarget: number;
-  nextRetargetHeight: number;
-  timeAvg: number;
-  timeOffset: number;
-}
-
-export interface RewardStats {
-  totalReward: number;
-  totalFee: number;
-  totalTx: number;
-}

backend/src/repositories/BlocksRepository.ts

@@ -1,64 +1,47 @@
-import { BlockExtended } from '../mempool.interfaces';
-import DB from '../database';
+import { BlockExtended, PoolTag } from '../mempool.interfaces';
+import { DB } from '../database';
 import logger from '../logger';
-import { Common } from '../api/common';
-import { prepareBlock } from '../utils/blocks-utils';
-import PoolsRepository from './PoolsRepository';
-import HashratesRepository from './HashratesRepository';
-import { escape } from 'mysql2';
+
+export interface EmptyBlocks {
+  emptyBlocks: number;
+  poolId: number;
+}
 
 class BlocksRepository {
   /**
    * Save indexed block data in the database
    */
-  public async $saveBlockInDatabase(block: BlockExtended) {
+  public async $saveBlockInDatabase(
+    block: BlockExtended,
+    blockHash: string,
+    coinbaseHex: string | undefined,
+    poolTag: PoolTag
+  ) {
+    const connection = await DB.pool.getConnection();
+
     try {
       const query = `INSERT INTO blocks(
-        height,           hash,                blockTimestamp, size,
-        weight,           tx_count,            coinbase_raw,   difficulty,
-        pool_id,          fees,                fee_span,       median_fee,
-        reward,           version,             bits,           nonce,
-        merkle_root,      previous_block_hash, avg_fee,        avg_fee_rate
+        height,  hash,     blockTimestamp, size,
+        weight,  tx_count, coinbase_raw,   difficulty,
+        pool_id, fees,     fee_span,       median_fee
       ) VALUE (
         ?, ?, FROM_UNIXTIME(?), ?,
         ?, ?, ?, ?,
-        ?, ?, ?, ?,
-        ?, ?, ?, ?,
         ?, ?, ?, ?
       )`;
 
       const params: any[] = [
-        block.height,
-        block.id,
-        block.timestamp,
-        block.size,
-        block.weight,
-        block.tx_count,
-        block.extras.coinbaseRaw,
-        block.difficulty,
-        block.extras.pool?.id, // Should always be set to something
-        block.extras.totalFees,
-        JSON.stringify(block.extras.feeRange),
-        block.extras.medianFee,
-        block.extras.reward,
-        block.version,
-        block.bits,
-        block.nonce,
-        block.merkle_root,
-        block.previousblockhash,
-        block.extras.avgFee,
-        block.extras.avgFeeRate,
+        block.height, blockHash, block.timestamp, block.size,
+        block.weight, block.tx_count, coinbaseHex ? coinbaseHex : '', block.difficulty,
+        poolTag.id, 0, '[]', block.medianFee,
       ];
 
-      await DB.query(query, params);
-    } catch (e: any) {
-      if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
-        logger.debug(`$saveBlockInDatabase() - Block ${block.height} has already been indexed, ignoring`);
-      } else {
-        logger.err('Cannot save indexed block into db. Reason: ' + (e instanceof Error ? e.message : e));
-        throw e;
-      }
+      await connection.query(query, params);
+    } catch (e) {
+      logger.err('$saveBlockInDatabase() error' + (e instanceof Error ? e.message : e));
     }
+
+    connection.release();
   }
 
   /**
@@ -69,589 +52,77 @@ class BlocksRepository {
       return [];
     }
 
-    try {
-      const [rows]: any[] = await DB.query(`
-        SELECT height
-        FROM blocks
-        WHERE height <= ? AND height >= ?
-        ORDER BY height DESC;
-      `, [startHeight, endHeight]);
-
-      const indexedBlockHeights: number[] = [];
-      rows.forEach((row: any) => { indexedBlockHeights.push(row.height); });
-      const seekedBlocks: number[] = Array.from(Array(startHeight - endHeight + 1).keys(), n => n + endHeight).reverse();
-      const missingBlocksHeights = seekedBlocks.filter(x => indexedBlockHeights.indexOf(x) === -1);
-
-      return missingBlocksHeights;
-    } catch (e) {
-      logger.err('Cannot retrieve blocks list to index. Reason: ' + (e instanceof Error ? e.message : e));
-      throw e;
-    }
-  }
-
-  /**
-   * Get empty blocks for one or all pools
-   */
-  public async $countEmptyBlocks(poolId: number | null, interval: string | null = null): Promise<any> {
-    interval = Common.getSqlInterval(interval);
-
-    const params: any[] = [];
-    let query = `SELECT count(height) as count, pools.id as poolId
+    const connection = await DB.pool.getConnection();
+    const [rows] : any[] = await connection.query(`
+      SELECT height
       FROM blocks
-      JOIN pools on pools.id = blocks.pool_id
-      WHERE tx_count = 1`;
+      WHERE height <= ${startHeight} AND height >= ${endHeight}
+      ORDER BY height DESC;
+    `);
+    connection.release();
 
-    if (poolId) {
-      query += ` AND pool_id = ?`;
-      params.push(poolId);
-    }
+    const indexedBlockHeights: number[] = [];
+    rows.forEach((row: any) => { indexedBlockHeights.push(row.height); });
+    const seekedBlocks: number[] = Array.from(Array(startHeight - endHeight + 1).keys(), n => n + endHeight).reverse();
+    const missingBlocksHeights =  seekedBlocks.filter(x => indexedBlockHeights.indexOf(x) === -1);
 
-    if (interval) {
-      query += ` AND blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
-    }
-
-    query += ` GROUP by pools.id`;
-
-    try {
-      const [rows] = await DB.query(query, params);
-      return rows;
-    } catch (e) {
-      logger.err('Cannot count empty blocks. Reason: ' + (e instanceof Error ? e.message : e));
-      throw e;
-    }
+    return missingBlocksHeights;
   }
 
   /**
-   * Return most recent block height
+   * Count empty blocks for all pools
    */
-  public async $mostRecentBlockHeight(): Promise<number> {
-    try {
-      const [row] = await DB.query('SELECT MAX(height) as maxHeight from blocks');
-      return row[0]['maxHeight'];
-    } catch (e) {
-      logger.err(`Cannot count blocks for this pool (using offset). Reason: ` + (e instanceof Error ? e.message : e));
-      throw e;
-    }
+  public async $countEmptyBlocks(interval: string | null): Promise<EmptyBlocks[]> {
+    const query = `
+      SELECT pool_id as poolId
+      FROM blocks
+      WHERE tx_count = 1` +
+      (interval != null ? ` AND blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()` : ``)
+    ;
+
+    const connection = await DB.pool.getConnection();
+    const [rows] = await connection.query(query);
+    connection.release();
+
+    return <EmptyBlocks[]>rows;
   }
 
   /**
    * Get blocks count for a period
    */
-  public async $blockCount(poolId: number | null, interval: string | null = null): Promise<number> {
-    interval = Common.getSqlInterval(interval);
+   public async $blockCount(interval: string | null): Promise<number> {
+    const query = `
+      SELECT count(height) as blockCount
+      FROM blocks` +
+      (interval != null ? ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()` : ``)
+    ;
 
-    const params: any[] = [];
-    let query = `SELECT count(height) as blockCount
-      FROM blocks`;
+    const connection = await DB.pool.getConnection();
+    const [rows] = await connection.query(query);
+    connection.release();
 
-    if (poolId) {
-      query += ` WHERE pool_id = ?`;
-      params.push(poolId);
-    }
-
-    if (interval) {
-      if (poolId) {
-        query += ` AND`;
-      } else {
-        query += ` WHERE`;
-      }
-      query += ` blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
-    }
-
-    try {
-      const [rows] = await DB.query(query, params);
-      return <number>rows[0].blockCount;
-    } catch (e) {
-      logger.err(`Cannot count blocks for this pool (using offset). Reason: ` + (e instanceof Error ? e.message : e));
-      throw e;
-    }
-  }
-
-  /**
-   * Get blocks count between two dates
-   * @param poolId
-   * @param from - The oldest timestamp
-   * @param to - The newest timestamp
-   * @returns
-   */
-  public async $blockCountBetweenTimestamp(poolId: number | null, from: number, to: number): Promise<number> {
-    const params: any[] = [];
-    let query = `SELECT
-      count(height) as blockCount,
-      max(height) as lastBlockHeight
-      FROM blocks`;
-
-    if (poolId) {
-      query += ` WHERE pool_id = ?`;
-      params.push(poolId);
-    }
-
-    if (poolId) {
-      query += ` AND`;
-    } else {
-      query += ` WHERE`;
-    }
-    query += ` blockTimestamp BETWEEN FROM_UNIXTIME('${from}') AND FROM_UNIXTIME('${to}')`;
-
-    try {
-      const [rows] = await DB.query(query, params);
-      return <number>rows[0];
-    } catch (e) {
-      logger.err(`Cannot count blocks for this pool (using timestamps). Reason: ` + (e instanceof Error ? e.message : e));
-      throw e;
-    }
-  }
-
-  /**
-   * Get blocks count for a period
-   */
-   public async $blockCountBetweenHeight(startHeight: number, endHeight: number): Promise<number> {
-    const params: any[] = [];
-    let query = `SELECT count(height) as blockCount
-      FROM blocks
-      WHERE height <= ${startHeight} AND height >= ${endHeight}`;
-
-    try {
-      const [rows] = await DB.query(query, params);
-      return <number>rows[0].blockCount;
-    } catch (e) {
-      logger.err(`Cannot count blocks for this pool (using offset). Reason: ` + (e instanceof Error ? e.message : e));
-      throw e;
-    }
+    return <number>rows[0].blockCount;
   }
 
   /**
    * Get the oldest indexed block
    */
   public async $oldestBlockTimestamp(): Promise<number> {
-    const query = `SELECT UNIX_TIMESTAMP(blockTimestamp) as blockTimestamp
+    const connection = await DB.pool.getConnection();
+    const [rows]: any[] = await connection.query(`
+      SELECT blockTimestamp
       FROM blocks
       ORDER BY height
-      LIMIT 1;`;
+      LIMIT 1;
+    `);
+    connection.release();
 
-    try {
-      const [rows]: any[] = await DB.query(query);
-
-      if (rows.length <= 0) {
-        return -1;
-      }
-
-      return <number>rows[0].blockTimestamp;
-    } catch (e) {
-      logger.err('Cannot get oldest indexed block timestamp. Reason: ' + (e instanceof Error ? e.message : e));
-      throw e;
-    }
-  }
-
-  /**
-   * Get blocks mined by a specific mining pool
-   */
-  public async $getBlocksByPool(slug: string, startHeight?: number): Promise<object[]> {
-    const pool = await PoolsRepository.$getPool(slug);
-    if (!pool) {
-      throw new Error('This mining pool does not exist ' + escape(slug));
+    if (rows.length <= 0) {
+      return -1;
     }
 
-    const params: any[] = [];
-    let query = ` SELECT
-      height,
-      hash as id,
-      UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp,
-      size,
-      weight,
-      tx_count,
-      coinbase_raw,
-      difficulty,
-      fees,
-      fee_span,
-      median_fee,
-      reward,
-      version,
-      bits,
-      nonce,
-      merkle_root,
-      previous_block_hash as previousblockhash,
-      avg_fee,
-      avg_fee_rate
-      FROM blocks
-      WHERE pool_id = ?`;
-    params.push(pool.id);
-
-    if (startHeight !== undefined) {
-      query += ` AND height < ?`;
-      params.push(startHeight);
-    }
-
-    query += ` ORDER BY height DESC
-      LIMIT 10`;
-
-    try {
-      const [rows] = await DB.query(query, params);
-
-      const blocks: BlockExtended[] = [];
-      for (const block of <object[]>rows) {
-        blocks.push(prepareBlock(block));
-      }
-
-      return blocks;
-    } catch (e) {
-      logger.err('Cannot get blocks for this pool. Reason: ' + (e instanceof Error ? e.message : e));
-      throw e;
-    }
-  }
-
-  /**
-   * Get one block by height
-   */
-  public async $getBlockByHeight(height: number): Promise<object | null> {
-    try {
-      const [rows]: any[] = await DB.query(`SELECT
-        height,
-        hash,
-        hash as id,
-        UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp,
-        size,
-        weight,
-        tx_count,
-        coinbase_raw,
-        difficulty,
-        pools.id as pool_id,
-        pools.name as pool_name,
-        pools.link as pool_link,
-        pools.slug as pool_slug,
-        pools.addresses as pool_addresses,
-        pools.regexes as pool_regexes,
-        fees,
-        fee_span,
-        median_fee,
-        reward,
-        version,
-        bits,
-        nonce,
-        merkle_root,
-        previous_block_hash as previousblockhash,
-        avg_fee,
-        avg_fee_rate
-        FROM blocks
-        JOIN pools ON blocks.pool_id = pools.id
-        WHERE height = ${height};
-      `);
-
-      if (rows.length <= 0) {
-        return null;
-      }
-
-      rows[0].fee_span = JSON.parse(rows[0].fee_span);
-      return rows[0];
-    } catch (e) {
-      logger.err(`Cannot get indexed block ${height}. Reason: ` + (e instanceof Error ? e.message : e));
-      throw e;
-    }
-  }
-
-  /**
-   * Get one block by hash
-   */
-  public async $getBlockByHash(hash: string): Promise<object | null> {
-    try {
-      const query = `
-        SELECT *, UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp, hash as id,
-        pools.id as pool_id, pools.name as pool_name, pools.link as pool_link, pools.slug as pool_slug,
-        pools.addresses as pool_addresses, pools.regexes as pool_regexes,
-        previous_block_hash as previousblockhash
-        FROM blocks
-        JOIN pools ON blocks.pool_id = pools.id
-        WHERE hash = '${hash}';
-      `;
-      const [rows]: any[] = await DB.query(query);
-
-      if (rows.length <= 0) {
-        return null;
-      }
-
-      rows[0].fee_span = JSON.parse(rows[0].fee_span);
-      return rows[0];
-    } catch (e) {
-      logger.err(`Cannot get indexed block ${hash}. Reason: ` + (e instanceof Error ? e.message : e));
-      throw e;
-    }
-  }
-
-  /**
-   * Return blocks difficulty
-   */
-  public async $getBlocksDifficulty(interval: string | null): Promise<object[]> {
-    interval = Common.getSqlInterval(interval);
-
-    // :D ... Yeah don't ask me about this one https://stackoverflow.com/a/40303162
-    // Basically, using temporary user defined fields, we are able to extract all
-    // difficulty adjustments from the blocks tables.
-    // This allow use to avoid indexing it in another table.
-    let query = `
-      SELECT
-      *
-      FROM
-      (
-        SELECT
-        UNIX_TIMESTAMP(blockTimestamp) as timestamp, difficulty, height,
-        IF(@prevStatus = YT.difficulty, @rn := @rn + 1,
-          IF(@prevStatus := YT.difficulty, @rn := 1, @rn := 1)
-        ) AS rn
-        FROM blocks YT
-        CROSS JOIN
-        (
-          SELECT @prevStatus := -1, @rn := 1
-        ) AS var
-    `;
-
-    if (interval) {
-      query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
-    }
-
-    query += `
-        ORDER BY YT.height
-      ) AS t
-      WHERE t.rn = 1
-      ORDER BY t.height
-    `;
-
-    try {
-      const [rows]: any[] = await DB.query(query);
-
-      for (const row of rows) {
-        delete row['rn'];
-      }
-
-      return rows;
-    } catch (e) {
-      logger.err('Cannot generate difficulty history. Reason: ' + (e instanceof Error ? e.message : e));
-      throw e;
-    }
-  }
-
-  /**
-   * Get general block stats
-   */
-  public async $getBlockStats(blockCount: number): Promise<any> {
-    try {
-      // We need to use a subquery
-      const query = `
-        SELECT MIN(height) as startBlock, MAX(height) as endBlock, SUM(reward) as totalReward, SUM(fees) as totalFee, SUM(tx_count) as totalTx
-        FROM
-          (SELECT height, reward, fees, tx_count FROM blocks
-          ORDER by height DESC
-          LIMIT ?) as sub`;
-
-      const [rows]: any = await DB.query(query, [blockCount]);
-
-      return rows[0];
-    } catch (e) {
-      logger.err('Cannot generate reward stats. Reason: ' + (e instanceof Error ? e.message : e));
-      throw e;
-    }
-  }
-
-  /*
-   * Check if the last 10 blocks chain is valid
-   */
-  public async $validateRecentBlocks(): Promise<boolean> {
-    try {
-      const [lastBlocks]: any[] = await DB.query(`SELECT height, hash, previous_block_hash FROM blocks ORDER BY height DESC LIMIT 10`);
-
-      for (let i = 0; i < lastBlocks.length - 1; ++i) {
-        if (lastBlocks[i].previous_block_hash !== lastBlocks[i + 1].hash) {
-          logger.warn(`Chain divergence detected at block ${lastBlocks[i].height}, re-indexing most recent data`);
-          return false;
-        }
-      }
-
-      return true;
-    } catch (e) {
-      return true; // Don't do anything if there is a db error
-    }
-  }
-
-  /**
-   * Check if the chain of block hash is valid and delete data from the stale branch if needed
-   */
-  public async $validateChain(): Promise<boolean> {
-    try {
-      const start = new Date().getTime();
-      const [blocks]: any[] = await DB.query(`SELECT height, hash, previous_block_hash,
-        UNIX_TIMESTAMP(blockTimestamp) as timestamp FROM blocks ORDER BY height`);
-
-      let partialMsg = false;
-      let idx = 1;
-      while (idx < blocks.length) {
-        if (blocks[idx].height - 1 !== blocks[idx - 1].height) {
-          if (partialMsg === false) {
-            logger.info('Some blocks are not indexed, skipping missing blocks during chain validation');
-            partialMsg = true;
-          }
-          ++idx;
-          continue;
-        }
-
-        if (blocks[idx].previous_block_hash !== blocks[idx - 1].hash) {
-          logger.warn(`Chain divergence detected at block ${blocks[idx - 1].height}, re-indexing newer blocks and hashrates`);
-          await this.$deleteBlocksFrom(blocks[idx - 1].height);
-          await HashratesRepository.$deleteHashratesFromTimestamp(blocks[idx - 1].timestamp - 604800);
-          return false;
-        }
-        ++idx;
-      }
-
-      logger.info(`${idx} blocks hash validated in ${new Date().getTime() - start} ms`);
-      return true;
-    } catch (e) {
-      logger.err('Cannot validate chain of block hash. Reason: ' + (e instanceof Error ? e.message : e));
-      return true; // Don't do anything if there is a db error
-    }
-  }
-
-  /**
-   * Delete blocks from the database from blockHeight
-   */
-  public async $deleteBlocksFrom(blockHeight: number) {
-    logger.info(`Delete newer blocks from height ${blockHeight} from the database`);
-
-    try {
-      await DB.query(`DELETE FROM blocks where height >= ${blockHeight}`);
-    } catch (e) {
-      logger.err('Cannot delete indexed blocks. Reason: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  /**
-   * Get the historical averaged block fees
-   */
-  public async $getHistoricalBlockFees(div: number, interval: string | null): Promise<any> {
-    try {
-      let query = `SELECT
-        CAST(AVG(height) as INT) as avgHeight,
-        CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
-        CAST(AVG(fees) as INT) as avgFees
-        FROM blocks`;
-
-      if (interval !== null) {
-        query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
-      }
-
-      query += ` GROUP BY UNIX_TIMESTAMP(blockTimestamp) DIV ${div}`;
-
-      const [rows]: any = await DB.query(query);
-      return rows;
-    } catch (e) {
-      logger.err('Cannot generate block fees history. Reason: ' + (e instanceof Error ? e.message : e));
-      throw e;
-    }
-  }
-
-  /**
-   * Get the historical averaged block rewards
-   */
-  public async $getHistoricalBlockRewards(div: number, interval: string | null): Promise<any> {
-    try {
-      let query = `SELECT
-        CAST(AVG(height) as INT) as avgHeight,
-        CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
-        CAST(AVG(reward) as INT) as avgRewards
-        FROM blocks`;
-
-      if (interval !== null) {
-        query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
-      }
-
-      query += ` GROUP BY UNIX_TIMESTAMP(blockTimestamp) DIV ${div}`;
-
-      const [rows]: any = await DB.query(query);
-      return rows;
-    } catch (e) {
-      logger.err('Cannot generate block rewards history. Reason: ' + (e instanceof Error ? e.message : e));
-      throw e;
-    }
-  }
-
-  /**
-   * Get the historical averaged block fee rate percentiles
-   */
-   public async $getHistoricalBlockFeeRates(div: number, interval: string | null): Promise<any> {
-    try {
-      let query = `SELECT
-        CAST(AVG(height) as INT) as avgHeight,
-        CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
-        CAST(AVG(JSON_EXTRACT(fee_span, '$[0]')) as INT) as avgFee_0,
-        CAST(AVG(JSON_EXTRACT(fee_span, '$[1]')) as INT) as avgFee_10,
-        CAST(AVG(JSON_EXTRACT(fee_span, '$[2]')) as INT) as avgFee_25,
-        CAST(AVG(JSON_EXTRACT(fee_span, '$[3]')) as INT) as avgFee_50,
-        CAST(AVG(JSON_EXTRACT(fee_span, '$[4]')) as INT) as avgFee_75,
-        CAST(AVG(JSON_EXTRACT(fee_span, '$[5]')) as INT) as avgFee_90,
-        CAST(AVG(JSON_EXTRACT(fee_span, '$[6]')) as INT) as avgFee_100
-      FROM blocks`;
-
-      if (interval !== null) {
-        query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
-      }
-
-      query += ` GROUP BY UNIX_TIMESTAMP(blockTimestamp) DIV ${div}`;
-
-      const [rows]: any = await DB.query(query);
-      return rows;
-    } catch (e) {
-      logger.err('Cannot generate block fee rates history. Reason: ' + (e instanceof Error ? e.message : e));
-      throw e;
-    }
-  }
-
-  /**
-   * Get the historical averaged block sizes
-   */
-   public async $getHistoricalBlockSizes(div: number, interval: string | null): Promise<any> {
-    try {
-      let query = `SELECT
-        CAST(AVG(height) as INT) as avgHeight,
-        CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
-        CAST(AVG(size) as INT) as avgSize
-      FROM blocks`;
-
-      if (interval !== null) {
-        query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
-      }
-
-      query += ` GROUP BY UNIX_TIMESTAMP(blockTimestamp) DIV ${div}`;
-
-      const [rows]: any = await DB.query(query);
-      return rows;
-    } catch (e) {
-      logger.err('Cannot generate block size and weight history. Reason: ' + (e instanceof Error ? e.message : e));
-      throw e;
-    }
-  }
-
-  /**
-   * Get the historical averaged block weights
-   */
-   public async $getHistoricalBlockWeights(div: number, interval: string | null): Promise<any> {
-    try {
-      let query = `SELECT
-        CAST(AVG(height) as INT) as avgHeight,
-        CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
-        CAST(AVG(weight) as INT) as avgWeight
-      FROM blocks`;
-
-      if (interval !== null) {
-        query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
-      }
-
-      query += ` GROUP BY UNIX_TIMESTAMP(blockTimestamp) DIV ${div}`;
-
-      const [rows]: any = await DB.query(query);
-      return rows;
-    } catch (e) {
-      logger.err('Cannot generate block size and weight history. Reason: ' + (e instanceof Error ? e.message : e));
-      throw e;
-    }
+    return <number>rows[0].blockTimestamp;
   }
 }
 
-export default new BlocksRepository();
+export default new BlocksRepository();

backend/src/repositories/HashratesRepository.ts

@@ -1,217 +0,0 @@
-import { escape } from 'mysql2';
-import { Common } from '../api/common';
-import DB from '../database';
-import logger from '../logger';
-import PoolsRepository from './PoolsRepository';
-
-class HashratesRepository {
-  /**
-   * Save indexed block data in the database
-   */
-  public async $saveHashrates(hashrates: any) {
-    if (hashrates.length === 0) {
-      return;
-    }
-
-    let query = `INSERT INTO
-      hashrates(hashrate_timestamp, avg_hashrate, pool_id, share, type) VALUES`;
-
-    for (const hashrate of hashrates) {
-      query += ` (FROM_UNIXTIME(${hashrate.hashrateTimestamp}), ${hashrate.avgHashrate}, ${hashrate.poolId}, ${hashrate.share}, "${hashrate.type}"),`;
-    }
-    query = query.slice(0, -1);
-
-    try {
-      await DB.query(query);
-    } catch (e: any) {
-      logger.err('Cannot save indexed hashrate into db. Reason: ' + (e instanceof Error ? e.message : e));
-      throw e;
-    }
-  }
-
-  public async $getNetworkDailyHashrate(interval: string | null): Promise<any[]> {
-    interval = Common.getSqlInterval(interval);
-
-    let query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp, avg_hashrate as avgHashrate
-      FROM hashrates`;
-
-    if (interval) {
-      query += ` WHERE hashrate_timestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()
-        AND hashrates.type = 'daily'`;
-    } else {
-      query += ` WHERE hashrates.type = 'daily'`;
-    }
-
-    query += ` ORDER by hashrate_timestamp`;
-
-    try {
-      const [rows]: any[] = await DB.query(query);
-      return rows;
-    } catch (e) {
-      logger.err('Cannot fetch network hashrate history. Reason: ' + (e instanceof Error ? e.message : e));
-      throw e;
-    }
-  }
-
-  public async $getWeeklyHashrateTimestamps(): Promise<number[]> {
-    const query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp
-      FROM hashrates
-      WHERE type = 'weekly'
-      GROUP BY hashrate_timestamp`;
-
-    try {
-      const [rows]: any[] = await DB.query(query);
-      return rows.map(row => row.timestamp);
-    } catch (e) {
-      logger.err('Cannot retreive indexed weekly hashrate timestamps. Reason: ' + (e instanceof Error ? e.message : e));
-      throw e;
-    }
-  }
-
-  /**
-   * Returns the current biggest pool hashrate history
-   */
-  public async $getPoolsWeeklyHashrate(interval: string | null): Promise<any[]> {
-    interval = Common.getSqlInterval(interval);
-
-    const topPoolsId = (await PoolsRepository.$getPoolsInfo('1w')).map((pool) => pool.poolId);
-
-    let query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp, avg_hashrate as avgHashrate, share, pools.name as poolName
-      FROM hashrates
-      JOIN pools on pools.id = pool_id`;
-
-    if (interval) {
-      query += ` WHERE hashrate_timestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()
-        AND hashrates.type = 'weekly'
-        AND pool_id IN (${topPoolsId})`;
-    } else {
-      query += ` WHERE hashrates.type = 'weekly'
-        AND pool_id IN (${topPoolsId})`;
-    }
-
-    query += ` ORDER by hashrate_timestamp, FIELD(pool_id, ${topPoolsId})`;
-
-    try {
-      const [rows]: any[] = await DB.query(query);
-      return rows;
-    } catch (e) {
-      logger.err('Cannot fetch weekly pools hashrate history. Reason: ' + (e instanceof Error ? e.message : e));
-      throw e;
-    }
-  }
-
-  /**
-   * Returns a pool hashrate history
-   */
-  public async $getPoolWeeklyHashrate(slug: string): Promise<any[]> {
-    const pool = await PoolsRepository.$getPool(slug);
-    if (!pool) {
-      throw new Error('This mining pool does not exist ' + escape(slug));
-    }
-
-    // Find hashrate boundaries
-    let query = `SELECT MIN(hashrate_timestamp) as firstTimestamp, MAX(hashrate_timestamp) as lastTimestamp
-      FROM hashrates
-      JOIN pools on pools.id = pool_id
-      WHERE hashrates.type = 'weekly' AND pool_id = ? AND avg_hashrate != 0
-      ORDER by hashrate_timestamp LIMIT 1`;
-
-    let boundaries = {
-      firstTimestamp: '1970-01-01',
-      lastTimestamp: '9999-01-01'
-    };
-
-    try {
-      const [rows]: any[] = await DB.query(query, [pool.id]);
-      boundaries = rows[0];
-    } catch (e) {
-      logger.err('Cannot fetch hashrate start/end timestamps for this pool. Reason: ' + (e instanceof Error ? e.message : e));
-    }
-
-    // Get hashrates entries between boundaries
-    query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp, avg_hashrate as avgHashrate, share, pools.name as poolName
-      FROM hashrates
-      JOIN pools on pools.id = pool_id
-      WHERE hashrates.type = 'weekly' AND hashrate_timestamp BETWEEN ? AND ?
-      AND pool_id = ?
-      ORDER by hashrate_timestamp`;
-
-    try {
-      const [rows]: any[] = await DB.query(query, [boundaries.firstTimestamp, boundaries.lastTimestamp, pool.id]);
-      return rows;
-    } catch (e) {
-      logger.err('Cannot fetch pool hashrate history for this pool. Reason: ' + (e instanceof Error ? e.message : e));
-      throw e;
-    }
-  }
-
-  /**
-   * Set latest run timestamp
-   */
-  public async $setLatestRun(key: string, val: number) {
-    const query = `UPDATE state SET number = ? WHERE name = ?`;
-
-    try {
-      await DB.query(query, [val, key]);
-    } catch (e) {
-      logger.err(`Cannot set last indexing run for ${key}. Reason: ` + (e instanceof Error ? e.message : e));
-      throw e;
-    }
-  }
-
-  /**
-   * Get latest run timestamp
-   */
-  public async $getLatestRun(key: string): Promise<number> {
-    const query = `SELECT number FROM state WHERE name = ?`;
-
-    try {
-      const [rows]: any[] = await DB.query(query, [key]);
-
-      if (rows.length === 0) {
-        return 0;
-      }
-      return rows[0]['number'];
-    } catch (e) {
-      logger.err(`Cannot retrieve last indexing run for ${key}. Reason: ` + (e instanceof Error ? e.message : e));
-      throw e;
-    }
-  }
-
-  /**
-   * Delete most recent data points for re-indexing
-   */
-  public async $deleteLastEntries() {
-    logger.info(`Delete latest hashrates data points from the database`);
-
-    try {
-      const [rows]: any[] = await DB.query(`SELECT MAX(hashrate_timestamp) as timestamp FROM hashrates GROUP BY type`);
-      for (const row of rows) {
-        await DB.query(`DELETE FROM hashrates WHERE hashrate_timestamp = ?`, [row.timestamp]);
-      }
-      // Re-run the hashrate indexing to fill up missing data
-      await this.$setLatestRun('last_hashrates_indexing', 0);
-      await this.$setLatestRun('last_weekly_hashrates_indexing', 0);
-    } catch (e) {
-      logger.err('Cannot delete latest hashrates data points. Reason: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-  
-  /**
-   * Delete hashrates from the database from timestamp
-   */
-  public async $deleteHashratesFromTimestamp(timestamp: number) {
-    logger.info(`Delete newer hashrates from timestamp ${new Date(timestamp * 1000).toUTCString()} from the database`);
-
-    try {
-      await DB.query(`DELETE FROM hashrates WHERE hashrate_timestamp >= FROM_UNIXTIME(?)`, [timestamp]);
-      // Re-run the hashrate indexing to fill up missing data
-      await this.$setLatestRun('last_hashrates_indexing', 0);
-      await this.$setLatestRun('last_weekly_hashrates_indexing', 0);
-    } catch (e) {
-      logger.err('Cannot delete latest hashrates data points. Reason: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-}
-
-export default new HashratesRepository();

backend/src/repositories/PoolsRepository.ts

@@ -1,7 +1,4 @@
-import { Common } from '../api/common';
-import config from '../config';
-import DB from '../database';
-import logger from '../logger';
+import { DB } from '../database';
 import { PoolInfo, PoolTag } from '../mempool.interfaces';
 
 class PoolsRepository {
@@ -9,7 +6,9 @@ class PoolsRepository {
    * Get all pools tagging info
    */
   public async $getPools(): Promise<PoolTag[]> {
-    const [rows] = await DB.query('SELECT id, name, addresses, regexes, slug FROM pools;');
+    const connection = await DB.pool.getConnection();
+    const [rows] = await connection.query('SELECT * FROM pools;');
+    connection.release();
     return <PoolTag[]>rows;
   }
 
@@ -17,82 +16,30 @@ class PoolsRepository {
    * Get unknown pool tagging info
    */
   public async $getUnknownPool(): Promise<PoolTag> {
-    const [rows] = await DB.query('SELECT id, name, slug FROM pools where name = "Unknown"');
+    const connection = await DB.pool.getConnection();
+    const [rows] = await connection.query('SELECT * FROM pools where name = "Unknown"');
+    connection.release();
     return <PoolTag>rows[0];
   }
 
   /**
    * Get basic pool info and block count
    */
-  public async $getPoolsInfo(interval: string | null = null): Promise<PoolInfo[]> {
-    interval = Common.getSqlInterval(interval);
-
-    let query = `SELECT COUNT(height) as blockCount, pool_id as poolId, pools.name as name, pools.link as link, slug
-      FROM blocks
-      JOIN pools on pools.id = pool_id`;
-
-    if (interval) {
-      query += ` WHERE blocks.blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
-    }
-
-    query += ` GROUP BY pool_id
-      ORDER BY COUNT(height) DESC`;
-
-    try {
-      const [rows] = await DB.query(query);
-      return <PoolInfo[]>rows;
-    } catch (e) {
-      logger.err(`Cannot generate pools stats. Reason: ` + (e instanceof Error ? e.message : e));
-      throw e;
-    }
-  }
-
-  /**
-   * Get basic pool info and block count between two timestamp
-   */
-   public async $getPoolsInfoBetween(from: number, to: number): Promise<PoolInfo[]> {
-    const query = `SELECT COUNT(height) as blockCount, pools.id as poolId, pools.name as poolName
-      FROM pools
-      LEFT JOIN blocks on pools.id = blocks.pool_id AND blocks.blockTimestamp BETWEEN FROM_UNIXTIME(?) AND FROM_UNIXTIME(?)
-      GROUP BY pools.id`;
-
-    try {
-      const [rows] = await DB.query(query, [from, to]);
-      return <PoolInfo[]>rows;
-    } catch (e) {
-      logger.err('Cannot generate pools blocks count. Reason: ' + (e instanceof Error ? e.message : e));
-      throw e;
-    }
-  }
-
-  /**
-   * Get mining pool statistics for one pool
-   */
-   public async $getPool(slug: string): Promise<PoolTag | null> {
+  public async $getPoolsInfo(interval: string | null): Promise<PoolInfo[]> {
     const query = `
-      SELECT *
-      FROM pools
-      WHERE pools.slug = ?`;
+      SELECT COUNT(height) as blockCount, pool_id as poolId, pools.name as name, pools.link as link
+      FROM blocks
+      JOIN pools on pools.id = pool_id` +
+      (interval != null ? ` WHERE blocks.blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()` : ``) +
+      ` GROUP BY pool_id
+      ORDER BY COUNT(height) DESC
+    `;
 
-    try {
-      const [rows]: any[] = await DB.query(query, [slug]);
+    const connection = await DB.pool.getConnection();
+    const [rows] = await connection.query(query);
+    connection.release();
 
-      if (rows.length < 1) {
-        return null;
-      }
-
-      rows[0].regexes = JSON.parse(rows[0].regexes);
-      if (['testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
-        rows[0].addresses = []; // pools.json only contains mainnet addresses
-      } else {
-        rows[0].addresses = JSON.parse(rows[0].addresses);
-      }
-
-      return rows[0];
-    } catch (e) {
-      logger.err('Cannot get pool from db. Reason: ' + (e instanceof Error ? e.message : e));
-      throw e;
-    }
+    return <PoolInfo[]>rows;
   }
 }
 

backend/src/repositories/RatesRepository.ts

@@ -1,21 +0,0 @@
-import DB from '../database';
-import logger from '../logger';
-import { IConversionRates } from '../mempool.interfaces';
-
-class RatesRepository {
-  public async $saveRate(height: number, rates: IConversionRates) {
-    try {
-      await DB.query(`INSERT INTO rates(height, bisq_rates) VALUE (?, ?)`, [height, JSON.stringify(rates)]);
-    } catch (e: any) {
-      if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
-        logger.debug(`Rate already exists for block ${height}, ignoring`);
-      } else {
-        logger.err(`Cannot save exchange rate into db for block ${height} Reason: ` + (e instanceof Error ? e.message : e));
-        throw e;
-      }
-    }
-  }
-}
-
-export default new RatesRepository();
-

backend/src/routes.ts

@@ -21,11 +21,6 @@ import bitcoinClient from './api/bitcoin/bitcoin-client';
 import elementsParser from './api/liquid/elements-parser';
 import icons from './api/liquid/icons';
 import miningStats from './api/mining';
-import axios from 'axios';
-import mining from './api/mining';
-import BlocksRepository from './repositories/BlocksRepository';
-import HashratesRepository from './repositories/HashratesRepository';
-import difficultyAdjustment from './api/difficulty-adjustment';
 
 class Routes {
   constructor() {}
@@ -537,48 +532,11 @@ class Routes {
     }
   }
 
-  public async $getPool(req: Request, res: Response) {
-    try {
-      const stats = await mining.$getPoolStat(req.params.slug);
-      res.header('Pragma', 'public');
-      res.header('Cache-control', 'public');
-      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
-      res.json(stats);
-    } catch (e) {
-      if (e instanceof Error && e.message.indexOf('This mining pool does not exist') > -1) {
-        res.status(404).send(e.message);
-      } else {
-        res.status(500).send(e instanceof Error ? e.message : e);
-      }
-    }
-  }
-
-  public async $getPoolBlocks(req: Request, res: Response) {
-    try {
-      const poolBlocks = await BlocksRepository.$getBlocksByPool(
-        req.params.slug,
-        req.params.height === undefined ? undefined : parseInt(req.params.height, 10),
-      );
-      res.header('Pragma', 'public');
-      res.header('Cache-control', 'public');
-      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
-      res.json(poolBlocks);
-    } catch (e) {
-      if (e instanceof Error && e.message.indexOf('This mining pool does not exist') > -1) {
-        res.status(404).send(e.message);
-      } else {
-        res.status(500).send(e instanceof Error ? e.message : e);
-      }
-    }
-  }
-
   public async $getPools(req: Request, res: Response) {
     try {
-      const stats = await miningStats.$getPoolsStats(req.params.interval);
-      const blockCount = await BlocksRepository.$blockCount(null, null);
+      let stats = await miningStats.$getPoolsStats(req.query.interval as string);
       res.header('Pragma', 'public');
       res.header('Cache-control', 'public');
-      res.header('X-total-count', blockCount.toString());
       res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       res.json(stats);
     } catch (e) {
@@ -586,131 +544,10 @@ class Routes {
     }
   }
 
-  public async $getPoolsHistoricalHashrate(req: Request, res: Response) {
-    try {
-      const hashrates = await HashratesRepository.$getPoolsWeeklyHashrate(req.params.interval);
-      const blockCount = await BlocksRepository.$blockCount(null, null);
-      res.header('Pragma', 'public');
-      res.header('Cache-control', 'public');
-      res.header('X-total-count', blockCount.toString());
-      res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
-      res.json(hashrates);
-    } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
-    }
-  }
-
-  public async $getPoolHistoricalHashrate(req: Request, res: Response) {
-    try {
-      const hashrates = await HashratesRepository.$getPoolWeeklyHashrate(req.params.slug);
-      const blockCount = await BlocksRepository.$blockCount(null, null);
-      res.header('Pragma', 'public');
-      res.header('Cache-control', 'public');
-      res.header('X-total-count', blockCount.toString());
-      res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
-      res.json(hashrates);
-    } catch (e) {
-      if (e instanceof Error && e.message.indexOf('This mining pool does not exist') > -1) {
-        res.status(404).send(e.message);
-      } else {
-        res.status(500).send(e instanceof Error ? e.message : e);
-      }
-    }
-  }
-
-  public async $getHistoricalHashrate(req: Request, res: Response) {
-    let currentHashrate = 0, currentDifficulty = 0;
-    try {
-      currentHashrate = await bitcoinClient.getNetworkHashPs();
-      currentDifficulty = await bitcoinClient.getDifficulty();
-    } catch (e) {
-      logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate and difficulty');
-    }
-
-    try {
-      const hashrates = await HashratesRepository.$getNetworkDailyHashrate(req.params.interval);
-      const difficulty = await BlocksRepository.$getBlocksDifficulty(req.params.interval);
-      const blockCount = await BlocksRepository.$blockCount(null, null);
-      res.header('Pragma', 'public');
-      res.header('Cache-control', 'public');
-      res.header('X-total-count', blockCount.toString());
-      res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
-      res.json({
-        hashrates: hashrates,
-        difficulty: difficulty,
-        currentHashrate: currentHashrate,
-        currentDifficulty: currentDifficulty,
-      });
-    } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
-    }
-  }
-
-  public async $getHistoricalBlockFees(req: Request, res: Response) {
-    try {
-      const blockFees = await mining.$getHistoricalBlockFees(req.params.interval);
-      const blockCount = await BlocksRepository.$blockCount(null, null);
-      res.header('Pragma', 'public');
-      res.header('Cache-control', 'public');
-      res.header('X-total-count', blockCount.toString());
-      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
-      res.json(blockFees);
-    } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
-    }
-  }
-
-  public async $getHistoricalBlockRewards(req: Request, res: Response) {
-    try {
-      const blockRewards = await mining.$getHistoricalBlockRewards(req.params.interval);
-      const blockCount = await BlocksRepository.$blockCount(null, null);
-      res.header('Pragma', 'public');
-      res.header('Cache-control', 'public');
-      res.header('X-total-count', blockCount.toString());
-      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
-      res.json(blockRewards);
-    } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
-    }
-  }
-
-  public async $getHistoricalBlockFeeRates(req: Request, res: Response) {
-    try {
-      const blockFeeRates = await mining.$getHistoricalBlockFeeRates(req.params.interval);
-      const blockCount = await BlocksRepository.$blockCount(null, null);
-      res.header('Pragma', 'public');
-      res.header('Cache-control', 'public');
-      res.header('X-total-count', blockCount.toString());
-      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
-      res.json(blockFeeRates);
-    } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
-    }
-  }
-
-  public async $getHistoricalBlockSizeAndWeight(req: Request, res: Response) {
-    try {
-      const blockSizes = await mining.$getHistoricalBlockSizes(req.params.interval);
-      const blockWeights = await mining.$getHistoricalBlockWeights(req.params.interval);
-      const blockCount = await BlocksRepository.$blockCount(null, null);
-      res.header('Pragma', 'public');
-      res.header('Cache-control', 'public');
-      res.header('X-total-count', blockCount.toString());
-      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
-      res.json({
-        sizes: blockSizes,
-        weights: blockWeights
-      });
-    } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
-    }
-  }
-
   public async getBlock(req: Request, res: Response) {
     try {
-      const block = await blocks.$getBlock(req.params.hash);
-      res.setHeader('Expires', new Date(Date.now() + 1000 * 600).toUTCString());
-      res.json(block);
+      const result = await bitcoinApi.$getBlock(req.params.hash);
+      res.json(result);
     } catch (e) {
       res.status(500).send(e instanceof Error ? e.message : e);
     }
@@ -728,20 +565,8 @@ class Routes {
 
   public async getBlocks(req: Request, res: Response) {
     try {
-      if (['mainnet', 'testnet', 'signet', 'regtest'].includes(config.MEMPOOL.NETWORK)) { // Bitcoin
-        const height = req.params.height === undefined ? undefined : parseInt(req.params.height, 10);
-        res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
-        res.json(await blocks.$getBlocks(height, 15));
-      } else { // Liquid, Bisq
-        return await this.getLegacyBlocks(req, res);
-      }
-    } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
-    }
-  }
+      loadingIndicators.setProgress('blocks', 0);
 
-  public async getLegacyBlocks(req: Request, res: Response) {
-    try {
       const returnBlocks: IEsploraApi.Block[] = [];
       const fromHeight = parseInt(req.params.height, 10) || blocks.getCurrentBlockHeight();
 
@@ -755,7 +580,7 @@ class Routes {
       }
 
       let nextHash = startFromHash;
-      for (let i = 0; i < 10 && nextHash; i++) {
+      for (let i = 0; i < 10; i++) {
         const localBlock = blocks.getBlocks().find((b) => b.id === nextHash);
         if (localBlock) {
           returnBlocks.push(localBlock);
@@ -765,15 +590,16 @@ class Routes {
           returnBlocks.push(block);
           nextHash = block.previousblockhash;
         }
+        loadingIndicators.setProgress('blocks', i / 10 * 100);
       }
 
-      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       res.json(returnBlocks);
     } catch (e) {
+      loadingIndicators.setProgress('blocks', 100);
       res.status(500).send(e instanceof Error ? e.message : e);
     }
   }
-  
+
   public async getBlockTransactions(req: Request, res: Response) {
     try {
       loadingIndicators.setProgress('blocktxs-' + req.params.hash, 0);
@@ -785,9 +611,9 @@ class Routes {
       const endIndex = Math.min(startingIndex + 10, txIds.length);
       for (let i = startingIndex; i < endIndex; i++) {
         try {
-          const transaction = await transactionUtils.$getTransactionExtended(txIds[i], true, true);
+          const transaction = await transactionUtils.$getTransactionExtended(txIds[i], true);
           transactions.push(transaction);
-          loadingIndicators.setProgress('blocktxs-' + req.params.hash, (i - startingIndex + 1) / (endIndex - startingIndex) * 100);
+          loadingIndicators.setProgress('blocktxs-' + req.params.hash, (i + 1) / endIndex * 100);
         } catch (e) {
           logger.debug('getBlockTransactions error: ' + (e instanceof Error ? e.message : e));
         }
@@ -864,13 +690,7 @@ class Routes {
   }
 
   public async getMempool(req: Request, res: Response) {
-    const info = mempool.getMempoolInfo();
-    res.json({
-      count: info.size,
-      vsize: info.bytes,
-      total_fee: info.total_fee * 1e8,
-      fee_histogram: []
-    });
+    res.status(501).send('Not implemented');
   }
 
   public async getMempoolTxIds(req: Request, res: Response) {
@@ -920,7 +740,55 @@ class Routes {
 
   public getDifficultyChange(req: Request, res: Response) {
     try {
-      res.json(difficultyAdjustment.getDifficultyAdjustment());
+      const DATime = blocks.getLastDifficultyAdjustmentTime();
+      const previousRetarget = blocks.getPreviousDifficultyRetarget();
+      const blockHeight = blocks.getCurrentBlockHeight();
+
+      const now = new Date().getTime() / 1000;
+      const diff = now - DATime;
+      const blocksInEpoch = blockHeight % 2016;
+      const progressPercent = (blocksInEpoch >= 0) ? blocksInEpoch / 2016 * 100 : 100;
+      const remainingBlocks = 2016 - blocksInEpoch;
+      const nextRetargetHeight = blockHeight + remainingBlocks;
+
+      let difficultyChange = 0;
+      if (remainingBlocks < 1870) {
+        if (blocksInEpoch > 0) {
+          difficultyChange = (600 / (diff / blocksInEpoch ) - 1) * 100;
+        }
+        if (difficultyChange > 300) {
+          difficultyChange = 300;
+        }
+        if (difficultyChange < -75) {
+          difficultyChange = -75;
+        }
+      }
+
+      const timeAvgDiff = difficultyChange * 0.1;
+
+      let timeAvgMins = 10;
+      if (timeAvgDiff > 0) {
+        timeAvgMins -= Math.abs(timeAvgDiff);
+      } else {
+        timeAvgMins += Math.abs(timeAvgDiff);
+      }
+
+      const timeAvg = timeAvgMins * 60;
+      const remainingTime =  remainingBlocks * timeAvg;
+      const estimatedRetargetDate = remainingTime + now;
+
+      const result = {
+        progressPercent,
+        difficultyChange,
+        estimatedRetargetDate,
+        remainingBlocks,
+        remainingTime,
+        previousRetarget,
+        nextRetargetHeight,
+        timeAvg,
+      };
+      res.json(result);
+
     } catch (e) {
       res.status(500).send(e instanceof Error ? e.message : e);
     }
@@ -987,35 +855,6 @@ class Routes {
       res.status(404).send('Asset icons not found');
     }
   }
-
-  public async $getAllFeaturedLiquidAssets(req: Request, res: Response) {
-    try {
-      const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.LIQUID_API}/assets/featured`, { responseType: 'stream', timeout: 10000 });
-      response.data.pipe(res);
-    } catch (e) {
-      res.status(500).end();
-    }
-  }
-
-  public async $getAssetGroup(req: Request, res: Response) {
-    try {
-      const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.LIQUID_API}/assets/group/${parseInt(req.params.id, 10)}`,
-        { responseType: 'stream', timeout: 10000 });
-      response.data.pipe(res);
-    } catch (e) {
-      res.status(500).end();
-    }
-  }
-
-  public async $getRewardStats(req: Request, res: Response) {
-    try {
-      const response = await mining.$getRewardStats(parseInt(req.params.blockCount, 10));
-      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
-      res.json(response);
-    } catch (e) {
-      res.status(500).end();
-    }
-  }
 }
 
 export default new Routes();

backend/src/rpc-api/commands.ts

@@ -1,92 +0,0 @@
-module.exports = {
-  addMultiSigAddress: 'addmultisigaddress',
-  addNode: 'addnode', // bitcoind v0.8.0+
-  backupWallet: 'backupwallet',
-  createMultiSig: 'createmultisig',
-  createRawTransaction: 'createrawtransaction', // bitcoind v0.7.0+
-  decodeRawTransaction: 'decoderawtransaction', // bitcoind v0.7.0+
-  decodeScript: 'decodescript',
-  dumpPrivKey: 'dumpprivkey',
-  dumpWallet: 'dumpwallet', // bitcoind v0.9.0+
-  encryptWallet: 'encryptwallet',
-  estimateFee: 'estimatefee', // bitcoind v0.10.0x
-  estimatePriority: 'estimatepriority', // bitcoind v0.10.0+
-  generate: 'generate', // bitcoind v0.11.0+
-  getAccount: 'getaccount',
-  getAccountAddress: 'getaccountaddress',
-  getAddedNodeInfo: 'getaddednodeinfo', // bitcoind v0.8.0+
-  getAddressesByAccount: 'getaddressesbyaccount',
-  getBalance: 'getbalance',
-  getBestBlockHash: 'getbestblockhash', // bitcoind v0.9.0+
-  getBlock: 'getblock',
-  getBlockStats: 'getblockstats',
-  getBlockFilter: 'getblockfilter',
-  getBlockchainInfo: 'getblockchaininfo', // bitcoind v0.9.2+
-  getBlockCount: 'getblockcount',
-  getBlockHash: 'getblockhash',
-  getBlockHeader: 'getblockheader',
-  getBlockTemplate: 'getblocktemplate', // bitcoind v0.7.0+
-  getChainTips: 'getchaintips', // bitcoind v0.10.0+
-  getChainTxStats: 'getchaintxstats',
-  getConnectionCount: 'getconnectioncount',
-  getDifficulty: 'getdifficulty',
-  getGenerate: 'getgenerate',
-  getInfo: 'getinfo',
-  getMempoolAncestors: 'getmempoolancestors',
-  getMempoolDescendants: 'getmempooldescendants',
-  getMempoolEntry: 'getmempoolentry',
-  getMempoolInfo: 'getmempoolinfo', // bitcoind v0.10+
-  getMiningInfo: 'getmininginfo',
-  getNetTotals: 'getnettotals',
-  getNetworkInfo: 'getnetworkinfo', // bitcoind v0.9.2+
-  getNetworkHashPs: 'getnetworkhashps', // bitcoind v0.9.0+
-  getNewAddress: 'getnewaddress',
-  getPeerInfo: 'getpeerinfo', // bitcoind v0.7.0+
-  getRawChangeAddress: 'getrawchangeaddress', // bitcoin v0.9+
-  getRawMemPool: 'getrawmempool', // bitcoind v0.7.0+
-  getRawTransaction: 'getrawtransaction', // bitcoind v0.7.0+
-  getReceivedByAccount: 'getreceivedbyaccount',
-  getReceivedByAddress: 'getreceivedbyaddress',
-  getTransaction: 'gettransaction',
-  getTxOut: 'gettxout', // bitcoind v0.7.0+
-  getTxOutProof: 'gettxoutproof', // bitcoind v0.11.0+
-  getTxOutSetInfo: 'gettxoutsetinfo', // bitcoind v0.7.0+
-  getUnconfirmedBalance: 'getunconfirmedbalance', // bitcoind v0.9.0+
-  getWalletInfo: 'getwalletinfo', // bitcoind v0.9.2+
-  help: 'help',
-  importAddress: 'importaddress', // bitcoind v0.10.0+
-  importPrivKey: 'importprivkey',
-  importWallet: 'importwallet', // bitcoind v0.9.0+
-  keypoolRefill: 'keypoolrefill',
-  keyPoolRefill: 'keypoolrefill',
-  listAccounts: 'listaccounts',
-  listAddressGroupings: 'listaddressgroupings', // bitcoind v0.7.0+
-  listLockUnspent: 'listlockunspent', // bitcoind v0.8.0+
-  listReceivedByAccount: 'listreceivedbyaccount',
-  listReceivedByAddress: 'listreceivedbyaddress',
-  listSinceBlock: 'listsinceblock',
-  listTransactions: 'listtransactions',
-  listUnspent: 'listunspent', // bitcoind v0.7.0+
-  lockUnspent: 'lockunspent', // bitcoind v0.8.0+
-  move: 'move',
-  ping: 'ping', // bitcoind v0.9.0+
-  prioritiseTransaction: 'prioritisetransaction', // bitcoind v0.10.0+
-  sendFrom: 'sendfrom',
-  sendMany: 'sendmany',
-  sendRawTransaction: 'sendrawtransaction', // bitcoind v0.7.0+
-  sendToAddress: 'sendtoaddress',
-  setAccount: 'setaccount',
-  setGenerate: 'setgenerate',
-  setTxFee: 'settxfee',
-  signMessage: 'signmessage',
-  signRawTransaction: 'signrawtransaction', // bitcoind v0.7.0+
-  stop: 'stop',
-  submitBlock: 'submitblock', // bitcoind v0.7.0+
-  validateAddress: 'validateaddress',
-  verifyChain: 'verifychain', // bitcoind v0.9.0+
-  verifyMessage: 'verifymessage',
-  verifyTxOutProof: 'verifytxoutproof', // bitcoind v0.11.0+
-  walletLock: 'walletlock',
-  walletPassphrase: 'walletpassphrase',
-  walletPassphraseChange: 'walletpassphrasechange'
-}

backend/src/rpc-api/index.ts

@@ -1,61 +0,0 @@
-var commands = require('./commands')
-var rpc = require('./jsonrpc')
-
-// ===----------------------------------------------------------------------===//
-// JsonRPC
-// ===----------------------------------------------------------------------===//
-function Client (opts) {
-  // @ts-ignore
-  this.rpc = new rpc.JsonRPC(opts)
-}
-
-// ===----------------------------------------------------------------------===//
-// cmd
-// ===----------------------------------------------------------------------===//
-Client.prototype.cmd = function () {
-  var args = [].slice.call(arguments)
-  var cmd = args.shift()
-
-  callRpc(cmd, args, this.rpc)
-}
-
-// ===----------------------------------------------------------------------===//
-// callRpc
-// ===----------------------------------------------------------------------===//
-function callRpc (cmd, args, rpc) {
-  var fn = args[args.length - 1]
-
-  // If the last argument is a callback, pop it from the args list
-  if (typeof fn === 'function') {
-    args.pop()
-  } else {
-    fn = function () {}
-  }
-
-  return rpc.call(cmd, args, function () {
-    var args = [].slice.call(arguments)
-      // @ts-ignore
-    args.unshift(null)
-      // @ts-ignore
-    fn.apply(this, args)
-  }, function (err) {
-    fn(err)
-  })
-}
-
-// ===----------------------------------------------------------------------===//
-// Initialize wrappers
-// ===----------------------------------------------------------------------===//
-(function () {
-  for (var protoFn in commands) {
-    (function (protoFn) {
-      Client.prototype[protoFn] = function () {
-        var args = [].slice.call(arguments)
-        return callRpc(commands[protoFn], args, this.rpc)
-      }
-    })(protoFn)
-  }
-})()
-
-// Export!
-module.exports.Client = Client;

backend/src/rpc-api/jsonrpc.ts

@@ -1,162 +0,0 @@
-var http = require('http')
-var https = require('https')
-
-var JsonRPC = function (opts) {
-  // @ts-ignore
-  this.opts = opts || {}
-  // @ts-ignore
-  this.http = this.opts.ssl ? https : http
-}
-
-JsonRPC.prototype.call = function (method, params) {
-  return new Promise((resolve, reject) => {
-    var time = Date.now()
-    var requestJSON
-
-    if (Array.isArray(method)) {
-      // multiple rpc batch call
-      requestJSON = []
-      method.forEach(function (batchCall, i) {
-        requestJSON.push({
-          id: time + '-' + i,
-          method: batchCall.method,
-          params: batchCall.params
-        })
-      })
-    } else {
-      // single rpc call
-      requestJSON = {
-        id: time,
-        method: method,
-        params: params
-      }
-    }
-
-    // First we encode the request into JSON
-    requestJSON = JSON.stringify(requestJSON)
-
-    // prepare request options
-    var requestOptions = {
-      host: this.opts.host || 'localhost',
-      port: this.opts.port || 8332,
-      method: 'POST',
-      path: '/',
-      headers: {
-        'Host': this.opts.host || 'localhost',
-        'Content-Length': requestJSON.length
-      },
-      agent: false,
-      rejectUnauthorized: this.opts.ssl && this.opts.sslStrict !== false
-    }
-
-    if (this.opts.ssl && this.opts.sslCa) {
-    // @ts-ignore 
-      requestOptions.ca = this.opts.sslCa
-    }
-
-    // use HTTP auth if user and password set
-    if (this.opts.user && this.opts.pass) {
-      // @ts-ignore
-      requestOptions.auth = this.opts.user + ':' + this.opts.pass
-    }
-
-    // Now we'll make a request to the server
-    var cbCalled = false
-    var request = this.http.request(requestOptions)
-
-    // start request timeout timer
-    var reqTimeout = setTimeout(function () {
-      if (cbCalled) return
-      cbCalled = true
-      request.abort()
-      var err = new Error('ETIMEDOUT')
-      // @ts-ignore
-      err.code = 'ETIMEDOUT'
-      reject(err)
-    }, this.opts.timeout || 30000)
-
-    // set additional timeout on socket in case of remote freeze after sending headers
-    request.setTimeout(this.opts.timeout || 30000, function () {
-      if (cbCalled) return
-      cbCalled = true
-      request.abort()
-      var err = new Error('ESOCKETTIMEDOUT')
-      // @ts-ignore
-      err.code = 'ESOCKETTIMEDOUT'
-      reject(err)
-    })
-
-    request.on('error', function (err) {
-      if (cbCalled) return
-      cbCalled = true
-      clearTimeout(reqTimeout)
-      reject(err)
-    })
-
-    request.on('response', function (response) {
-      clearTimeout(reqTimeout)
-
-      // We need to buffer the response chunks in a nonblocking way.
-      var buffer = ''
-      response.on('data', function (chunk) {
-        buffer = buffer + chunk
-      })
-      // When all the responses are finished, we decode the JSON and
-      // depending on whether it's got a result or an error, we call
-      // emitSuccess or emitError on the promise.
-      response.on('end', function () {
-        var err
-
-        if (cbCalled) return
-        cbCalled = true
-
-        try {
-          var decoded = JSON.parse(buffer)
-        } catch (e) {
-          if (response.statusCode !== 200) {
-            err = new Error('Invalid params, response status code: ' + response.statusCode)
-            err.code = -32602
-            reject(err)
-          } else {
-            err = new Error('Problem parsing JSON response from server')
-            err.code = -32603
-            reject(err)
-          }
-          return
-        }
-
-        if (!Array.isArray(decoded)) {
-          decoded = [decoded]
-        }
-
-        // iterate over each response, normally there will be just one
-        // unless a batch rpc call response is being processed
-        decoded.forEach(function (decodedResponse, i) {
-          if (decodedResponse.hasOwnProperty('error') && decodedResponse.error != null) {
-            if (reject) {
-              err = new Error(decodedResponse.error.message || '')
-              if (decodedResponse.error.code) {
-                err.code = decodedResponse.error.code
-              }
-              reject(err)
-            }
-          } else if (decodedResponse.hasOwnProperty('result')) {
-            // @ts-ignore
-            resolve(decodedResponse.result, response.headers)
-          } else {
-            if (reject) {
-              err = new Error(decodedResponse.error.message || '')
-              if (decodedResponse.error.code) {
-                err.code = decodedResponse.error.code
-              }
-              reject(err)
-            }
-          }
-        })
-      })
-    })
-    request.end(requestJSON);
-  });
-}
-
-module.exports.JsonRPC = JsonRPC

backend/src/sync-assets.ts

@@ -1,84 +1,31 @@
-import axios, { AxiosResponse } from 'axios';
+import axios from 'axios';
 import * as fs from 'fs';
+const fsPromises = fs.promises;
 import config from './config';
-import backendInfo from './api/backend-info';
 import logger from './logger';
-import { SocksProxyAgent } from 'socks-proxy-agent';
 
 const PATH = './';
 
 class SyncAssets {
   constructor() { }
 
-  public async syncAssets$() {
+  public async syncAssets() {
     for (const url of config.MEMPOOL.EXTERNAL_ASSETS) {
-      try {
-        await this.downloadFile$(url);
-      } catch (e) {
-        throw new Error(`Failed to download external asset. ` + (e instanceof Error ? e.message : e));
-      }
+      await this.downloadFile(url);
     }
   }
 
-  private async downloadFile$(url: string) {
-    return new Promise((resolve, reject) => {
-      const fileName = url.split('/').slice(-1)[0];
-
-      try {
-        if (config.SOCKS5PROXY.ENABLED) {
-          const socksOptions: any = {
-            agentOptions: {
-              keepAlive: true,
-            },
-            hostname: config.SOCKS5PROXY.HOST,
-            port: config.SOCKS5PROXY.PORT
-          };
-
-          if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
-            socksOptions.username = config.SOCKS5PROXY.USERNAME;
-            socksOptions.password = config.SOCKS5PROXY.PASSWORD;
-          }
-
-          const agent = new SocksProxyAgent(socksOptions);
-
-          logger.info(`Downloading external asset ${fileName} over the Tor network...`);
-          return axios.get(url, {
-            headers: {
-              'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
-            },
-            httpAgent: agent,
-            httpsAgent: agent,
-            responseType: 'stream',
-            timeout: 30000
-          }).then(function (response) {
-            const writer = fs.createWriteStream(PATH + fileName);
-            writer.on('finish', () => {
-              logger.info(`External asset ${fileName} saved to ${PATH + fileName}`);
-              resolve(0);
-            });
-            response.data.pipe(writer);
-          });
-        } else {
-          logger.info(`Downloading external asset ${fileName} over clearnet...`);
-          return axios.get(url, {
-            headers: {
-              'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
-            },
-            responseType: 'stream',
-            timeout: 30000
-          }).then(function (response) {
-            const writer = fs.createWriteStream(PATH + fileName);
-            writer.on('finish', () => {
-              logger.info(`External asset ${fileName} saved to ${PATH + fileName}`);
-              resolve(0);
-            });
-            response.data.pipe(writer);
-          });
-        }
-      } catch (e: any) {
-        reject(e);
-      }
-    });
+  private async downloadFile(url: string) {
+    const fileName = url.split('/').slice(-1)[0];
+    logger.info(`Downloading external asset: ${fileName}...`);
+    try {
+      const response = await axios.get(url, {
+        responseType: 'stream', timeout: 30000
+      });
+      await fsPromises.writeFile(PATH + fileName, response.data);
+    } catch (e: any) {
+      throw new Error(`Failed to download external asset. ` + e);
+    }
   }
 }
 

backend/src/tasks/pools-updater.ts

@@ -1,175 +0,0 @@
-import axios, { AxiosResponse } from 'axios';
-import poolsParser from '../api/pools-parser';
-import config from '../config';
-import DB from '../database';
-import backendInfo from '../api/backend-info';
-import logger from '../logger';
-import { SocksProxyAgent } from 'socks-proxy-agent';
-import * as https from 'https';
-
-/**
- * Maintain the most recent version of pools.json
- */
-class PoolsUpdater {
-  lastRun: number = 0;
-  currentSha: any = undefined;
-
-  constructor() {
-  }
-
-  public async updatePoolsJson() {
-    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
-      return;
-    }
-
-    const oneWeek = 604800;
-    const oneDay = 86400;
-
-    const now = new Date().getTime() / 1000;
-    if (now - this.lastRun < oneWeek) { // Execute the PoolsUpdate only once a week, or upon restart
-      return;
-    }
-
-    this.lastRun = now;
-
-    logger.info('Updating latest mining pools from Github');
-    if (config.SOCKS5PROXY.ENABLED) {
-      logger.info('List of public pools will be queried over the Tor network');
-    } else {
-      logger.info('List of public pools will be queried over clearnet');
-    }
-
-    try {
-      const githubSha = await this.fetchPoolsSha(); // Fetch pools.json sha from github
-      if (githubSha === undefined) {
-        return;
-      }
-
-      if (config.DATABASE.ENABLED === true) {
-        this.currentSha = await this.getShaFromDb();      
-      }
-
-      logger.debug(`Pools.json sha | Current: ${this.currentSha} | Github: ${githubSha}`);
-      if (this.currentSha !== undefined && this.currentSha === githubSha) {
-        return;
-      }
-
-      logger.warn('Pools.json is outdated, fetch latest from github');
-      const poolsJson = await this.query('https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json');
-      if (poolsJson === undefined) {
-        return;
-      }
-      await poolsParser.migratePoolsJson(poolsJson);
-      await this.updateDBSha(githubSha);
-      logger.notice('PoolsUpdater completed');
-
-    } catch (e) {
-      this.lastRun = now - (oneWeek - oneDay); // Try again in 24h instead of waiting next week
-      logger.err('PoolsUpdater failed. Will try again in 24h. Reason: '  + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  /**
-   * Fetch our latest pools.json sha from the db
-   */
-  private async updateDBSha(githubSha: string) {
-    this.currentSha = githubSha;
-    if (config.DATABASE.ENABLED === true) {
-      try {
-        await DB.query('DELETE FROM state where name="pools_json_sha"');
-        await DB.query(`INSERT INTO state VALUES('pools_json_sha', NULL, '${githubSha}')`);
-      } catch (e) {
-        logger.err('Cannot save github pools.json sha into the db. Reason: '  + (e instanceof Error ? e.message : e));
-      }
-    }
-  }
-
-  /**
-   * Fetch our latest pools.json sha from the db
-   */
-  private async getShaFromDb(): Promise<string | undefined> {
-    try {
-      const [rows]: any[] = await DB.query('SELECT string FROM state WHERE name="pools_json_sha"');
-      return (rows.length > 0 ? rows[0].string : undefined);
-    } catch (e) {
-      logger.err('Cannot fetch pools.json sha from db. Reason: '  + (e instanceof Error ? e.message : e));
-      return undefined;
-    }
-  }
-
-  /**
-   * Fetch our latest pools.json sha from github
-   */
-  private async fetchPoolsSha(): Promise<string | undefined> {
-    const response = await this.query('https://api.github.com/repos/mempool/mining-pools/git/trees/master');
-
-    if (response !== undefined) {
-      for (const file of response['tree']) {
-        if (file['path'] === 'pools.json') {
-          return file['sha'];
-        }
-      }
-    }
-
-    logger.err('Cannot to find latest pools.json sha from github api response');
-    return undefined;
-  }
-
-  /**
-   * Http request wrapper
-   */
-  private async query(path): Promise<object | undefined> {
-    type axiosOptions = {
-      headers: {
-        'User-Agent': string
-      };
-      timeout: number;
-      httpsAgent?: https.Agent;
-    }
-    const setDelay = (secs: number = 1): Promise<void> => new Promise(resolve => setTimeout(() => resolve(), secs * 1000));
-    const axiosOptions: axiosOptions = {
-      headers: {
-        'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
-      },
-      timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000
-    };
-    let retry = 0;
-
-    while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
-      try {
-        if (config.SOCKS5PROXY.ENABLED) {
-          const socksOptions: any = {
-            agentOptions: {
-              keepAlive: true,
-            },
-            hostname: config.SOCKS5PROXY.HOST,
-            port: config.SOCKS5PROXY.PORT
-          };
-
-          if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
-            socksOptions.username = config.SOCKS5PROXY.USERNAME;
-            socksOptions.password = config.SOCKS5PROXY.PASSWORD;
-          } else {
-            // Retry with different tor circuits https://stackoverflow.com/a/64960234
-            socksOptions.username = `circuit${retry}`;
-          }
-
-          axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions);
-        }
-        
-        const data: AxiosResponse = await axios.get(path, axiosOptions);
-        if (data.statusText === 'error' || !data.data) {
-          throw new Error(`Could not fetch data from Github, Error: ${data.status}`);
-        }
-        return data.data;
-      } catch (e) {
-        logger.err('Could not connect to Github. Reason: '  + (e instanceof Error ? e.message : e));
-        retry++;
-      }
-      await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL);
-    }
-    return undefined;
-  }
-}
-
-export default new PoolsUpdater();

backend/src/utils/blocks-utils.ts

@@ -1,32 +0,0 @@
-import { BlockExtended } from '../mempool.interfaces';
-
-export function prepareBlock(block: any): BlockExtended {
-  return <BlockExtended>{
-    id: block.id ?? block.hash, // hash for indexed block
-    timestamp: block.timestamp ?? block.blockTimestamp, // blockTimestamp for indexed block
-    height: block.height,
-    version: block.version,
-    bits: block.bits,
-    nonce: block.nonce,
-    difficulty: block.difficulty,
-    merkle_root: block.merkle_root,
-    tx_count: block.tx_count,
-    size: block.size,
-    weight: block.weight,
-    previousblockhash: block.previousblockhash,
-    extras: {
-      coinbaseRaw: block.coinbase_raw ?? block.extras?.coinbaseRaw,
-      medianFee: block.medianFee ?? block.median_fee ?? block.extras?.medianFee,
-      feeRange: block.feeRange ?? block.fee_span,
-      reward: block.reward ?? block?.extras?.reward,
-      totalFees: block.totalFees ?? block?.fees ?? block?.extras?.totalFees,
-      avgFee: block?.extras?.avgFee ?? block.avg_fee,
-      avgFeeRate: block?.avgFeeRate ?? block.avg_fee_rate,
-      pool: block?.extras?.pool ?? (block?.pool_id ? {
-        id: block.pool_id,
-        name: block.pool_name,
-        slug: block.pool_slug,
-      } : undefined),
-    }
-  };
-}

backend/tsconfig.json

@@ -10,8 +10,7 @@
     "moduleResolution": "node",
     "typeRoots": [
       "node_modules/@types"
-    ],
-    "allowSyntheticDefaultImports": true
+    ]
   },
   "include": [
     "src/**/*.ts"

contributors/TechMiX.txt

@@ -1,3 +0,0 @@
-I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of January 25, 2022.
-
-Signed: TechMiX

contributors/antonilol.txt

@@ -1,3 +0,0 @@
-I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of January 25, 2022.
-
-Signed: antonilol

contributors/ayanamidev.txt

@@ -1,3 +0,0 @@
-I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of May 15, 2022.
-
-Signed: ayanamidev

contributors/bosch-0.txt

@@ -1,3 +0,0 @@
-I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of January 25, 2022.
-
-Signed: Bosch-0

contributors/dsbaars.txt

@@ -1,3 +0,0 @@
-I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of January 25, 2022.
-
-Signed: dsbaars

contributors/emzy.txt

@@ -1 +0,0 @@
-Mempool Space K.K. has a signed CLA or other agreement on file with @emzy as of January 25, 2022

contributors/hunicus.txt

@@ -1 +0,0 @@
-Mempool Space K.K. has a signed CLA or other agreement on file with @hunicus as of January 25, 2022

contributors/knorrium.txt

@@ -1 +0,0 @@
-Mempool Space K.K. has a signed CLA or other agreement on file with @knorrium as of January 25, 2022

contributors/miguelmedeiros.txt

@@ -1 +0,0 @@
-Mempool Space K.K. has a signed CLA or other agreement on file with @miguelmedeiros as of January 25, 2022

contributors/mononaut.txt

@@ -1,3 +0,0 @@
-I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of May 31, 2022.
-
-Signed: mononaut

contributors/naveensrinivasan.txt

@@ -1,3 +0,0 @@
-I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of March 11, 2022.
-
-Signed: naveensrinivasan

contributors/nymkappa.txt

@@ -1 +0,0 @@
-Mempool Space K.K. has a signed CLA or other agreement on file with @nymkappa as of January 25, 2022

docker/README.md

@@ -1,346 +0,0 @@
-# Docker Installation
-
-This directory contains the Dockerfiles used to build and release the official images, as well as a `docker-compose.yml` to configure environment variables and other settings.
-
-If you are looking to use these Docker images to deploy your own instance of Mempool, note that they only containerize Mempool's frontend and backend. You will still need to deploy and configure Bitcoin Core and an Electrum Server separately, along with any other utilities specific to your use case (e.g., a reverse proxy, etc). Such configuration is mostly beyond the scope of the Mempool project, so please only proceed if you know what you're doing.
-
-Jump to a section in this doc:
-- [Configure with Bitcoin Core Only](#configure-with-bitcoin-core-only)
-- [Configure with Bitcoin Core + Electrum Server](#configure-with-bitcoin-core--electrum-server)
-- [Further Configuration](#further-configuration)
-
-## Configure with Bitcoin Core Only
-
-_Note: address lookups require an Electrum Server and will not work with this configuration. [Add an Electrum Server](#configure-with-bitcoin-core--electrum-server) to your backend for full functionality._
-
-The default Docker configuration assumes you have the following configuration in your `bitcoin.conf` file:
-
-```
-txindex=1
-server=1
-rpcuser=mempool
-rpcpassword=mempool
-```
-
-If you want to use different credentials, specify them in the `docker-compose.yml` file:
-
-```
-  api:
-    environment:
-      MEMPOOL_BACKEND: "none"
-      CORE_RPC_HOST: "172.27.0.1"
-      CORE_RPC_PORT: "8332"
-      CORE_RPC_USERNAME: "customuser"
-      CORE_RPC_PASSWORD: "custompassword"
-```
-
-The IP address in the example above refers to Docker's default gateway IP address so that the container can hit the `bitcoind` instance running on the host machine. If your setup is different, update it accordingly.
-
-Make sure `bitcoind` is running and synced.
-
-Now, run:
-
-```bash
-docker-compose up
-```
-
-Your Mempool instance should be running at http://localhost. The graphs will be populated as new transactions are detected.
-
-## Configure with Bitcoin Core + Electrum Server
-
-First, configure `bitcoind` as specified above, and make sure your Electrum Server is running and synced. See [this FAQ](https://mempool.space/docs/faq#address-lookup-issues) if you need help picking an Electrum Server implementation.
-
-Then, set the following variables in `docker-compose.yml` so Mempool can connect to your Electrum Server:
-
-```
-  api:
-    environment:
-      MEMPOOL_BACKEND: "electrum"
-      ELECTRUM_HOST: "172.27.0.1"
-      ELECTRUM_PORT: "50002"
-      ELECTRUM_TLS_ENABLED: "false"
-```
-
-Eligible values for `MEMPOOL_BACKEND`:
-  - "electrum" if you're using [romanz/electrs](https://github.com/romanz/electrs) or [cculianu/Fulcrum](https://github.com/cculianu/Fulcrum)
-  - "esplora" if you're using [Blockstream/electrs](https://github.com/Blockstream/electrs)
-  - "none" if you're not using any Electrum Server
-
-Of course, if your Docker host IP address is different, update accordingly.
-
-With `bitcoind` and Electrum Server set up, run Mempool with:
-
-```bash
-docker-compose up
-```
-
-## Further Configuration
-
-Optionally, you can override any other backend settings from `mempool-config.json`.
-
-Below we list all settings from `mempool-config.json` and the corresponding overrides you can make in the `api` > `environment` section of `docker-compose.yml`. 
-
-<br/>
-
-`mempool-config.json`:
-```
-  "MEMPOOL": {
-    "NETWORK": "mainnet",
-    "BACKEND": "electrum",
-    "HTTP_PORT": 8999,
-    "SPAWN_CLUSTER_PROCS": 0,
-    "API_URL_PREFIX": "/api/v1/",
-    "POLL_RATE_MS": 2000,
-    "CACHE_DIR": "./cache",
-    "CLEAR_PROTECTION_MINUTES": 20,
-    "RECOMMENDED_FEE_PERCENTILE": 50,
-    "BLOCK_WEIGHT_UNITS": 4000000,
-    "INITIAL_BLOCKS_AMOUNT": 8,
-    "MEMPOOL_BLOCKS_AMOUNT": 8,
-    "PRICE_FEED_UPDATE_INTERVAL": 600,
-    "USE_SECOND_NODE_FOR_MINFEE": false,
-    "EXTERNAL_ASSETS": ["https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json"],
-    "STDOUT_LOG_MIN_PRIORITY": "info"
-  },
-```
-
-Corresponding `docker-compose.yml` overrides:
-```
-  api:
-    environment:
-      MEMPOOL_NETWORK: ""
-      MEMPOOL_BACKEND: ""
-      MEMPOOL_HTTP_PORT: ""
-      MEMPOOL_SPAWN_CLUSTER_PROCS: ""
-      MEMPOOL_API_URL_PREFIX: ""
-      MEMPOOL_POLL_RATE_MS: ""
-      MEMPOOL_CACHE_DIR: ""
-      MEMPOOL_CLEAR_PROTECTION_MINUTES: ""
-      MEMPOOL_RECOMMENDED_FEE_PERCENTILE: ""
-      MEMPOOL_BLOCK_WEIGHT_UNITS: ""
-      MEMPOOL_INITIAL_BLOCKS_AMOUNT: ""
-      MEMPOOL_MEMPOOL_BLOCKS_AMOUNT: ""
-      MEMPOOL_PRICE_FEED_UPDATE_INTERVAL: ""
-      MEMPOOL_USE_SECOND_NODE_FOR_MINFEE: ""
-      MEMPOOL_EXTERNAL_ASSETS: ""
-      MEMPOOL_STDOUT_LOG_MIN_PRIORITY: ""
-      ...
-```
-
-<br/>
-
-`mempool-config.json`:
-```
-"CORE_RPC": {
-    "HOST": "127.0.0.1",
-    "PORT": 8332,
-    "USERNAME": "mempool",
-    "PASSWORD": "mempool"
-  },
-```
-
-Corresponding `docker-compose.yml` overrides:
-```
-  api:
-    environment:
-      CORE_RPC_HOST: ""
-      CORE_RPC_PORT: ""
-      CORE_RPC_USERNAME: ""
-      CORE_RPC_PASSWORD: ""
-      ...
-```
-
-<br/>
-
-`mempool-config.json`:
-```
-  "ELECTRUM": {
-    "HOST": "127.0.0.1",
-    "PORT": 50002,
-    "TLS_ENABLED": true
-  },
-```
-
-Corresponding `docker-compose.yml` overrides:
-```
-  api:
-    environment:
-      ELECTRUM_HOST: ""
-      ELECTRUM_PORT: ""
-      ELECTRUM_TLS_ENABLED: ""
-      ...
-```
-
-<br/>
-
-`mempool-config.json`:
-```
-  "ESPLORA": {
-    "REST_API_URL": "http://127.0.0.1:3000"
-  },
-```
-
-Corresponding `docker-compose.yml` overrides:
-```
-  api:
-    environment:
-      ESPLORA_REST_API_URL: ""
-      ...
-```
-
-<br/>
-
-`mempool-config.json`:
-```
-  "SECOND_CORE_RPC": {
-    "HOST": "127.0.0.1",
-    "PORT": 8332,
-    "USERNAME": "mempool",
-    "PASSWORD": "mempool"
-  },
-```
-
-Corresponding `docker-compose.yml` overrides:
-```
-  api:
-    environment:
-      SECOND_CORE_RPC_HOST: ""
-      SECOND_CORE_RPC_PORT: ""
-      SECOND_CORE_RPC_USERNAME: ""
-      SECOND_CORE_RPC_PASSWORD: ""
-      ...
-```
-
-<br/>
-
-`mempool-config.json`:
-```
-  "DATABASE": {
-    "ENABLED": true,
-    "HOST": "127.0.0.1",
-    "PORT": 3306,
-    "DATABASE": "mempool",
-    "USERNAME": "mempool",
-    "PASSWORD": "mempool"
-  },
-```
-
-Corresponding `docker-compose.yml` overrides:
-```
-  api:
-    environment:
-      DATABASE_ENABLED: ""
-      DATABASE_HOST: ""
-      DATABASE_PORT: ""
-      DATABASE_DATABASE: ""
-      DATABASE_USERAME: ""
-      DATABASE_PASSWORD: ""
-      ...
-```
-
-<br/>
-
-`mempool-config.json`:
-```
-  "SYSLOG": {
-    "ENABLED": true,
-    "HOST": "127.0.0.1",
-    "PORT": 514,
-    "MIN_PRIORITY": "info",
-    "FACILITY": "local7"
-  },
-```
-
-Corresponding `docker-compose.yml` overrides:
-```
-  api:
-    environment:
-      SYSLOG_ENABLED: ""
-      SYSLOG_HOST: ""
-      SYSLOG_PORT: ""
-      SYSLOG_MIN_PRIORITY: ""
-      SYSLOG_FACILITY: ""
-      ...
-```
-
-<br/>
-
-`mempool-config.json`:
-```
-  "STATISTICS": {
-    "ENABLED": true,
-    "TX_PER_SECOND_SAMPLE_PERIOD": 150
-  },
-```
-
-Corresponding `docker-compose.yml` overrides:
-```
-  api:
-    environment:
-      STATISTICS_ENABLED: ""
-      STATISTICS_TX_PER_SECOND_SAMPLE_PERIOD: ""
-      ...
-```
-
-<br/>
-
-`mempool-config.json`:
-```
-  "BISQ": {
-    "ENABLED": false,
-    "DATA_PATH": "/bisq/statsnode-data/btc_mainnet/db"
-  }
-```
-
-Corresponding `docker-compose.yml` overrides:
-```
-  api:
-    environment:
-      BISQ_ENABLED: ""
-      BISQ_DATA_PATH: ""
-      ...
-```
-
-<br/>
-
-`mempool-config.json`:
-```
-  "SOCKS5PROXY": {
-    "ENABLED": false,
-    "HOST": "127.0.0.1",
-    "PORT": "9050",
-    "USERNAME": "",
-    "PASSWORD": ""
-  }
-```
-
-Corresponding `docker-compose.yml` overrides:
-```
-  api:
-    environment:
-      SOCKS5PROXY_ENABLED: ""
-      SOCKS5PROXY_HOST: ""
-      SOCKS5PROXY_PORT: ""
-      SOCKS5PROXY_USERNAME: ""
-      SOCKS5PROXY_PASSWORD: ""
-      ...
-```
-
-<br/>
-
-`mempool-config.json`:
-```
-  "PRICE_DATA_SERVER": {
-    "TOR_URL": "http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices",
-    "CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices"
-  }
-```
-
-Corresponding `docker-compose.yml` overrides:
-```
-  api:
-    environment:
-      PRICE_DATA_SERVER_TOR_URL: ""
-      PRICE_DATA_SERVER_CLEARNET_URL: ""
-      ...
-```

docker/backend/Dockerfile

@@ -1,7 +1,4 @@
-FROM node:16.15.0-buster-slim AS builder
-
-ARG commitHash
-ENV DOCKER_COMMIT_HASH=${commitHash}
+FROM node:16.10.0-buster-slim AS builder
 
 WORKDIR /build
 COPY . .
@@ -11,7 +8,7 @@ RUN apt-get install -y build-essential python3 pkg-config
 RUN npm install
 RUN npm run build
 
-FROM node:16.15.0-buster-slim
+FROM node:16.10.0-buster-slim
 
 WORKDIR /backend
 

docker/backend/mempool-config.json

@@ -14,12 +14,7 @@
     "MEMPOOL_BLOCKS_AMOUNT": __MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__,
     "PRICE_FEED_UPDATE_INTERVAL": __MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__,
     "USE_SECOND_NODE_FOR_MINFEE": __MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__,
-    "EXTERNAL_ASSETS": __MEMPOOL_EXTERNAL_ASSETS__,
-    "EXTERNAL_MAX_RETRY": __MEMPOOL_EXTERNAL_MAX_RETRY__,
-    "EXTERNAL_RETRY_INTERVAL": __MEMPOOL_EXTERNAL_RETRY_INTERVAL__,
-    "USER_AGENT": "__MEMPOOL_USER_AGENT__",
-    "STDOUT_LOG_MIN_PRIORITY": "__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__",
-    "INDEXING_BLOCKS_AMOUNT": __MEMPOOL_INDEXING_BLOCKS_AMOUNT__
+    "EXTERNAL_ASSETS": __MEMPOOL_EXTERNAL_ASSETS__
   },
   "CORE_RPC": {
     "HOST": "__CORE_RPC_HOST__",
@@ -44,7 +39,6 @@
   "DATABASE": {
     "ENABLED": __DATABASE_ENABLED__,
     "HOST": "__DATABASE_HOST__",
-    "SOCKET": "__DATABASE_SOCKET__",
     "PORT": __DATABASE_PORT__,
     "DATABASE": "__DATABASE_DATABASE__",
     "USERNAME": "__DATABASE_USERNAME__",
@@ -64,25 +58,5 @@
   "BISQ": {
     "ENABLED": __BISQ_ENABLED__,
     "DATA_PATH": "__BISQ_DATA_PATH__"
-  },
-  "SOCKS5PROXY": {
-    "ENABLED": __SOCKS5PROXY_ENABLED__,
-    "USE_ONION": __SOCKS5PROXY_USE_ONION__,
-    "HOST": "__SOCKS5PROXY_HOST__",
-    "PORT": "__SOCKS5PROXY_PORT__",
-    "USERNAME": "__SOCKS5PROXY_USERNAME__",
-    "PASSWORD": "__SOCKS5PROXY_PASSWORD__"
-  },
-  "PRICE_DATA_SERVER": {
-    "TOR_URL": "__PRICE_DATA_SERVER_TOR_URL__",
-    "CLEARNET_URL": "__PRICE_DATA_SERVER_CLEARNET_URL__"
-  },
-  "EXTERNAL_DATA_SERVER": {
-    "MEMPOOL_API": "__EXTERNAL_DATA_SERVER_MEMPOOL_API__",
-    "MEMPOOL_ONION": "__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__",
-    "LIQUID_API": "__EXTERNAL_DATA_SERVER_LIQUID_API__",
-    "LIQUID_ONION": "__EXTERNAL_DATA_SERVER_LIQUID_ONION__",
-    "BISQ_URL": "__EXTERNAL_DATA_SERVER_BISQ_URL__",
-    "BISQ_ONION": "__EXTERNAL_DATA_SERVER_BISQ_ONION__"
   }
 }

docker/backend/start.sh

@@ -13,14 +13,10 @@ __MEMPOOL_RECOMMENDED_FEE_PERCENTILE__=${MEMPOOL_RECOMMENDED_FEE_PERCENTILE:=50}
 __MEMPOOL_BLOCK_WEIGHT_UNITS__=${MEMPOOL_BLOCK_WEIGHT_UNITS:=4000000}
 __MEMPOOL_INITIAL_BLOCKS_AMOUNT__=${MEMPOOL_INITIAL_BLOCKS_AMOUNT:=8}
 __MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__=${MEMPOOL_MEMPOOL_BLOCKS_AMOUNT:=8}
-__MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=11000}
-__MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__=${MEMPOOL_PRICE_FEED_UPDATE_INTERVAL:=600}
+__MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=1100}
+__MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__=${MEMPOOL_PRICE_FEED_UPDATE_INTERVAL:=3600}
 __MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__=${MEMPOOL_USE_SECOND_NODE_FOR_MINFEE:=false}
 __MEMPOOL_EXTERNAL_ASSETS__=${MEMPOOL_EXTERNAL_ASSETS:=[]}
-__MEMPOOL_EXTERNAL_MAX_RETRY__=${MEMPOOL_EXTERNAL_MAX_RETRY:=1}
-__MEMPOOL_EXTERNAL_RETRY_INTERVAL__=${MEMPOOL_EXTERNAL_RETRY_INTERVAL:=0}
-__MEMPOOL_USER_AGENT__=${MEMPOOL_USER_AGENT:=mempool}
-__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__=${MEMPOOL_STDOUT_LOG_MIN_PRIORITY:=info}
 
 # CORE_RPC
 __CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1}
@@ -31,7 +27,7 @@ __CORE_RPC_PASSWORD__=${CORE_RPC_PASSWORD:=mempool}
 # ELECTRUM
 __ELECTRUM_HOST__=${ELECTRUM_HOST:=127.0.0.1}
 __ELECTRUM_PORT__=${ELECTRUM_PORT:=50002}
-__ELECTRUM_TLS_ENABLED__=${ELECTRUM_TLS_ENABLED:=false}
+__ELECTRUM_TLS_ENABLED__=${ELECTRUM_TLS:=false}
 
 # ESPLORA
 __ESPLORA_REST_API_URL__=${ESPLORA_REST_API_URL:=http://127.0.0.1:3000}
@@ -45,7 +41,6 @@ __SECOND_CORE_RPC_PASSWORD__=${SECOND_CORE_RPC_PASSWORD:=mempool}
 # DATABASE
 __DATABASE_ENABLED__=${DATABASE_ENABLED:=true}
 __DATABASE_HOST__=${DATABASE_HOST:=127.0.0.1}
-__DATABASE_SOCKET__=${DATABASE_SOCKET:=""}
 __DATABASE_PORT__=${DATABASE_PORT:=3306}
 __DATABASE_DATABASE__=${DATABASE_DATABASE:=mempool}
 __DATABASE_USERNAME__=${DATABASE_USERNAME:=mempool}
@@ -66,26 +61,6 @@ __STATISTICS_TX_PER_SECOND_SAMPLE_PERIOD__=${STATISTICS_TX_PER_SECOND_SAMPLE_PER
 __BISQ_ENABLED__=${BISQ_ENABLED:=false}
 __BISQ_DATA_PATH__=${BISQ_DATA_PATH:=/bisq/statsnode-data/btc_mainnet/db}
 
-# SOCKS5PROXY
-__SOCKS5PROXY_ENABLED__=${SOCKS5PROXY_ENABLED:=false}
-__SOCKS5PROXY_USE_ONION__=${SOCKS5PROXY_USE_ONION:=true}
-__SOCKS5PROXY_HOST__=${SOCKS5PROXY_HOST:=localhost}
-__SOCKS5PROXY_PORT__=${SOCKS5PROXY_PORT:=9050}
-__SOCKS5PROXY_USERNAME__=${SOCKS5PROXY_USERNAME:=""}
-__SOCKS5PROXY_PASSWORD__=${SOCKS5PROXY_PASSWORD:=""}
-
-# PRICE_DATA_SERVER
-__PRICE_DATA_SERVER_TOR_URL__=${PRICE_DATA_SERVER_TOR_URL:=http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices}
-__PRICE_DATA_SERVER_CLEARNET_URL__=${PRICE_DATA_SERVER_CLEARNET_URL:=https://price.bisq.wiz.biz/getAllMarketPrices}
-
-# EXTERNAL_DATA_SERVER
-__EXTERNAL_DATA_SERVER_MEMPOOL_API__=${EXTERNAL_DATA_SERVER_MEMPOOL_API:=https://mempool.space/api/v1}
-__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__=${EXTERNAL_DATA_SERVER_MEMPOOL_ONION:=http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1}
-__EXTERNAL_DATA_SERVER_LIQUID_API__=${EXTERNAL_DATA_SERVER_LIQUID_API:=https://liquid.network/api/v1}
-__EXTERNAL_DATA_SERVER_LIQUID_ONION__=${EXTERNAL_DATA_SERVER_LIQUID_ONION:=http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1}
-__EXTERNAL_DATA_SERVER_BISQ_URL__=${EXTERNAL_DATA_SERVER_BISQ_URL:=https://bisq.markets/api}
-__EXTERNAL_DATA_SERVER_BISQ_ONION__=${EXTERNAL_DATA_SERVER_BISQ_ONION:=http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api}
-
 mkdir -p "${__MEMPOOL_CACHE_DIR__}"
 
 sed -i "s/__MEMPOOL_NETWORK__/${__MEMPOOL_NETWORK__}/g" mempool-config.json
@@ -103,11 +78,7 @@ sed -i "s/__MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__/${__MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__}
 sed -i "s/__MEMPOOL_INDEXING_BLOCKS_AMOUNT__/${__MEMPOOL_INDEXING_BLOCKS_AMOUNT__}/g" mempool-config.json
 sed -i "s/__MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__/${__MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__}/g" mempool-config.json
 sed -i "s/__MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__/${__MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__}/g" mempool-config.json
-sed -i "s!__MEMPOOL_EXTERNAL_ASSETS__!${__MEMPOOL_EXTERNAL_ASSETS__}!g" mempool-config.json
-sed -i "s!__MEMPOOL_EXTERNAL_MAX_RETRY__!${__MEMPOOL_EXTERNAL_MAX_RETRY__}!g" mempool-config.json
-sed -i "s!__MEMPOOL_EXTERNAL_RETRY_INTERVAL__!${__MEMPOOL_EXTERNAL_RETRY_INTERVAL__}!g" mempool-config.json
-sed -i "s!__MEMPOOL_USER_AGENT__!${__MEMPOOL_USER_AGENT__}!g" mempool-config.json
-sed -i "s/__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__/${__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__}/g" mempool-config.json
+sed -i "s/__MEMPOOL_EXTERNAL_ASSETS__/${__MEMPOOL_EXTERNAL_ASSETS__}/g" mempool-config.json
 
 sed -i "s/__CORE_RPC_HOST__/${__CORE_RPC_HOST__}/g" mempool-config.json
 sed -i "s/__CORE_RPC_PORT__/${__CORE_RPC_PORT__}/g" mempool-config.json
@@ -127,8 +98,6 @@ sed -i "s/__SECOND_CORE_RPC_PASSWORD__/${__SECOND_CORE_RPC_PASSWORD__}/g" mempoo
 
 sed -i "s/__DATABASE_ENABLED__/${__DATABASE_ENABLED__}/g" mempool-config.json
 sed -i "s/__DATABASE_HOST__/${__DATABASE_HOST__}/g" mempool-config.json
-sed -i "s!__DATABASE_SOCKET__!${__DATABASE_SOCKET__}!g" mempool-config.json
-
 sed -i "s/__DATABASE_PORT__/${__DATABASE_PORT__}/g" mempool-config.json
 sed -i "s/__DATABASE_DATABASE__/${__DATABASE_DATABASE__}/g" mempool-config.json
 sed -i "s/__DATABASE_USERNAME__/${__DATABASE_USERNAME__}/g" mempool-config.json
@@ -146,21 +115,4 @@ sed -i "s/__STATISTICS_TX_PER_SECOND_SAMPLE_PERIOD__/${__STATISTICS_TX_PER_SECON
 sed -i "s/__BISQ_ENABLED__/${__BISQ_ENABLED__}/g" mempool-config.json
 sed -i "s!__BISQ_DATA_PATH__!${__BISQ_DATA_PATH__}!g" mempool-config.json
 
-sed -i "s/__SOCKS5PROXY_ENABLED__/${__SOCKS5PROXY_ENABLED__}/g" mempool-config.json
-sed -i "s/__SOCKS5PROXY_USE_ONION__/${__SOCKS5PROXY_USE_ONION__}/g" mempool-config.json
-sed -i "s/__SOCKS5PROXY_HOST__/${__SOCKS5PROXY_HOST__}/g" mempool-config.json
-sed -i "s/__SOCKS5PROXY_PORT__/${__SOCKS5PROXY_PORT__}/g" mempool-config.json
-sed -i "s/__SOCKS5PROXY_USERNAME__/${__SOCKS5PROXY_USERNAME__}/g" mempool-config.json
-sed -i "s/__SOCKS5PROXY_PASSWORD__/${__SOCKS5PROXY_PASSWORD__}/g" mempool-config.json
-
-sed -i "s!__PRICE_DATA_SERVER_TOR_URL__!${__PRICE_DATA_SERVER_TOR_URL__}!g" mempool-config.json
-sed -i "s!__PRICE_DATA_SERVER_CLEARNET_URL__!${__PRICE_DATA_SERVER_CLEARNET_URL__}!g" mempool-config.json
-
-sed -i "s!__EXTERNAL_DATA_SERVER_MEMPOOL_API__!${__EXTERNAL_DATA_SERVER_MEMPOOL_API__}!g" mempool-config.json
-sed -i "s!__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__!${__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__}!g" mempool-config.json
-sed -i "s!__EXTERNAL_DATA_SERVER_LIQUID_API__!${__EXTERNAL_DATA_SERVER_LIQUID_API__}!g" mempool-config.json
-sed -i "s!__EXTERNAL_DATA_SERVER_LIQUID_ONION__!${__EXTERNAL_DATA_SERVER_LIQUID_ONION__}!g" mempool-config.json
-sed -i "s!__EXTERNAL_DATA_SERVER_BISQ_URL__!${__EXTERNAL_DATA_SERVER_BISQ_URL__}!g" mempool-config.json
-sed -i "s!__EXTERNAL_DATA_SERVER_BISQ_ONION__!${__EXTERNAL_DATA_SERVER_BISQ_ONION__}!g" mempool-config.json
-
 node /backend/dist/index.js

docker/frontend/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:16.15.0-buster-slim AS builder
+FROM node:16.10.0-buster-slim AS builder
 
 ARG commitHash
 ENV DOCKER_COMMIT_HASH=${commitHash}

frontend/.gitignore

@@ -53,7 +53,6 @@ src/resources/assets.minimal.json
 src/resources/assets-testnet.json
 src/resources/assets-testnet.minimal.json
 src/resources/pools.json
-src/resources/mining-pools/*
 
 # environment config
 mempool-frontend-config.json

frontend/README.md

@@ -1,30 +1,8 @@
-# Mempool Frontend
+# mempool-frontend
 
-You can build and run the Mempool frontend and proxy to the production Mempool backend (for easier frontend development), or you can connect it to your own backend for a full Mempool development instance, custom deployment, etc.
+## Contributing
 
-Jump to a section in this doc:
-- [Quick Setup for Frontend Development](#quick-setup-for-frontend-development)
-- [Manual Frontend Setup](#manual-setup)
-- [Translations](#translations-transifex-project)
-
-## Quick Setup for Frontend Development
-
-If you want to quickly improve the UI, fix typos, or make other updates that don't require any backend changes, you don't need to set up an entire backend—you can simply run the Mempool frontend locally and proxy to the mempool.space backend.
-
-### 1. Clone Mempool Repository
-
-Get the latest Mempool code:
-
-```
-git clone https://github.com/mempool/mempool
-cd mempool
-```
-
-### 2. Specify Website
-
-The same frontend codebase is used for https://mempool.space, https://liquid.network and https://bisq.markets.
-
-Configure the frontend for the site you want by running the corresponding command:
+This package is used for the https://mempool.space, https://liquid.network and https://bisq.markets websites - there are npm scripts to setup all three, which effectively change how BASE_MODULE is  configured:
 
 ```
 $ npm run config:defaults:mempool
@@ -32,22 +10,18 @@ $ npm run config:defaults:liquid
 $ npm run config:defaults:bisq
 ```
 
-### 3. Run the Frontend
+Changes that affect the frontend codebase only can be done using the production backend so you don't need to spin up the entire Mempool infrastructure. This is very convenient in case you want to quickly improve the UI, fix typos or implement new features that don't require any backend changes.
 
-_Make sure to use Node.js 16.15 and npm 7._
-
-Install project dependencies and run the frontend server:
+Make your changes, install the project dependencies and run the frontend server as follows:
 
 ```
 $ npm install
 $ npm run serve:local-prod
 ```
 
-The frontend will be available at http://localhost:4200/ and all API requests will be proxied to the production server at https://mempool.space.
+The frontend will be available at http://localhost:4200/ and all API requests will be proxied to the production server at https://mempool.space
 
-### 4. Test
-
-After making your changes, you can run our end-to-end automation suite and check for possible regressions.
+After making your changes, you can run our end-to-end automation suite and check for possible regressions:
 
 Headless:
 
@@ -63,43 +37,11 @@ $ npm run config:defaults:mempool && npm run cypress:open
 
 This will open the Cypress test runner, where you can select any of the test files to run.
 
-If all tests are green, submit your PR, and it will be reviewed by someone on the team as soon as possible.
-
-## Manual Setup
-
-Set up the [Mempool backend](../backend/) first, if you haven't already.
-
-### 1. Build the Frontend
-
-_Node.js 16 and npm 7 are recommended._
-
-Build the frontend:
-
-```
-cd frontend
-npm install # add --prod for production
-npm run build
-```
-
-### 2. Run the Frontend
-
-#### Development
-
-To run your local Mempool frontend with your local Mempool backend:
-
-```
-npm run serve
-```
-
-#### Production
-
-The `npm run build` command from step 1 above should have generated a `dist` directory. Put the contents of `dist/` onto your web server.
-
-You will probably want to set up a reverse proxy, TLS, etc. There are sample nginx configuration files in the top level of the repository for reference, but note that support for such tasks is outside the scope of this project.
+If all tests are green, submit your PR and it will be reviewed by someone on the team as soon as possible.
 
 ## Translations: Transifex Project
 
-The Mempool frontend strings are localized into 20+ locales:
+The mempool frontend strings are localized into 20+ locales:
 https://www.transifex.com/mempool/mempool/dashboard/
 
 ### Translators

frontend/angular.json

@@ -218,10 +218,6 @@
               "proxyConfig": "proxy.conf.local.js",
               "verbose": true
             },
-            "mixed": {
-              "proxyConfig": "proxy.conf.mixed.js",
-              "verbose": true
-            },
             "staging": {
               "proxyConfig": "proxy.conf.js",
               "disableHostCheck": true,
@@ -233,12 +229,6 @@
               "disableHostCheck": true,
               "host": "0.0.0.0",
               "verbose": false
-            },
-            "local-staging": {
-              "proxyConfig": "proxy.conf.staging.js",
-              "disableHostCheck": true,
-              "host": "0.0.0.0",
-              "verbose": false
             }
           }
         },

frontend/cypress.config.ts

@@ -1,23 +0,0 @@
-import { defineConfig } from 'cypress'
-
-export default defineConfig({
-  projectId: 'ry4br7',
-  videosFolder: 'cypress/videos',
-  screenshotsFolder: 'cypress/screenshots',
-  fixturesFolder: 'cypress/fixtures',
-  video: false,
-  retries: {
-    runMode: 3,
-    openMode: 0,
-  },
-  chromeWebSecurity: false,
-  e2e: {
-    // We've imported your old cypress plugins here.
-    // You may want to clean this up later by importing these.
-    setupNodeEvents(on, config) {
-      return require('./cypress/plugins/index.js')(on, config)
-    },
-    baseUrl: 'http://localhost:4200',
-    specPattern: 'cypress/e2e/**/*.{js,jsx,ts,tsx}',
-  },
-})

frontend/cypress.json

@@ -0,0 +1,16 @@
+{
+  "projectId": "ry4br7",
+  "integrationFolder": "cypress/integration",
+  "supportFile": "cypress/support/index.ts",
+  "videosFolder": "cypress/videos",
+  "screenshotsFolder": "cypress/screenshots",
+  "pluginsFile": "cypress/plugins/index.js",
+  "fixturesFolder": "cypress/fixtures",
+  "baseUrl": "http://localhost:4200",
+  "video": false,
+  "retries": {
+    "runMode": 3,
+    "openMode": 0
+  },
+  "chromeWebSecurity": false
+}

frontend/cypress/integration/bisq/bisq.spec.ts

@@ -35,14 +35,13 @@ describe('Bisq', () => {
         "Proposal", "Reimbursement request", "Transfer BSQ", "Unlock", "Vote reveal"
       ];
       filters.forEach((filter) => {
-        it.only(`filters the transaction screen by ${filter}`, () => {
+        it(`filters the transaction screen by ${filter}`, () => {
           cy.visit(`${basePath}/transactions`);
-          cy.wait('@txs');
           cy.waitForSkeletonGone();
           cy.get('#filter').click();
           cy.contains(filter).find('input').click();
-          cy.wait('@txs');
-          cy.wait(500);
+          //TODO: change this waiter
+          cy.wait(1000);
           cy.get('td:nth-of-type(2)').each(($td) => {
             expect($td.text().trim()).to.eq(filter);
           });
@@ -57,7 +56,7 @@ describe('Bisq', () => {
         filters.forEach((filter) => {
           cy.contains(filter).find('input').click();
           //TODO: change this waiter
-          cy.wait(1500);
+          cy.wait(1000);
         });
         cy.get('td:nth-of-type(2)').each(($td) => {
           const regex = new RegExp(`${filters.join('|')}`, 'g');

frontend/cypress/integration/liquid/liquid.spec.ts

@@ -84,8 +84,8 @@ describe('Liquid', () => {
         cy.visit(`${basePath}/tx/fe764f7bedfc2a37b29d9c8aef67d64a57d253a6b11c5a55555cfd5826483a58`);
         cy.waitForSkeletonGone();
         //TODO: Change to an element id so we don't assert on a string
-        cy.get('.table-tx-vin').should('contain', 'Peg-in');
-        cy.get('.table-tx-vin a').click().then(() => {
+        cy.get('#table-tx-vin').should('contain', 'Peg-in');
+        cy.get('#table-tx-vin a').click().then(() => {
           cy.waitForSkeletonGone();
           if (baseModule === 'liquid') {
             cy.url().should('eq', 'https://mempool.space/tx/f148c0d854db4174ea420655235f910543f0ec3680566dcfdf84fb0a1697b592');
@@ -99,7 +99,7 @@ describe('Liquid', () => {
       it('loads peg out addresses', () => {
         cy.visit(`${basePath}/tx/ecf6eba04ffb3946faa172343c87162df76f1a57b07b0d6dc6ad956b13376dc8`);
         cy.waitForSkeletonGone();
-        cy.get('.table-tx-vout a').first().click().then(() => {
+        cy.get('#table-tx-vout a').first().click().then(() => {
           cy.waitForSkeletonGone();
           if (baseModule === 'liquid') {
             cy.url().should('eq', 'https://mempool.space/address/1BxoGcMg14oaH3CwHD2hF4gU9VcfgX5yoR');
@@ -115,16 +115,17 @@ describe('Liquid', () => {
 
     describe('assets', () => {
       it('shows the assets screen', () => {
-        cy.visit(`${basePath}/assets`);
+        cy.visit(`${basePath}`);
+        cy.get('#btn-assets');
         cy.waitForSkeletonGone();
-        cy.get('.featuredBox .card').should('have.length.at.least', 5);
+        cy.get('table tr').should('have.length.at.least', 5);
       });
 
       it('allows searching assets', () => {
         cy.visit(`${basePath}/assets`);
         cy.waitForSkeletonGone();
         cy.get('.container-xl input').click().type('Liquid Bitcoin').then(() => {
-          cy.get('ngb-typeahead-window', { timeout: 30000 }).should('have.length', 1);
+          cy.get('table tr').should('have.length', 1);
         });
       });
 
@@ -132,7 +133,7 @@ describe('Liquid', () => {
         cy.visit(`${basePath}/assets`);
         cy.waitForSkeletonGone();
         cy.get('.container-xl input').click().type('Liquid AUD').then(() => {
-          cy.get('ngb-typeahead-window:nth-of-type(1) button', { timeout: 30000 }).click();
+          cy.get('table tr td:nth-of-type(1) a').click();
         });
       });
     });
@@ -149,57 +150,57 @@ describe('Liquid', () => {
       it('show unblinded TX', () => {
         cy.visit(`${basePath}/tx/f2f41c0850e8e7e3f1af233161fd596662e67c11ef10ed15943884186fbb7f46#blinded=100000,6f0279e9ed041c3d710a9f57d0c02928416460c4b722ae3457a11eec381c526d,0ab9f70650f16b1db8dfada05237f7d0d65191c3a13183da8a2ddddfbde9a2ad,fd98b2edc5530d76acd553f206a431f4c1fab27e10e290ad719582af878e98fc,2364760,6f0279e9ed041c3d710a9f57d0c02928416460c4b722ae3457a11eec381c526d,90c7a43b15b905bca045ca42a01271cfe71d2efe3133f4197792c24505cb32ed,12eb5959d9293b8842e7dd8bc9aa9639fd3fd031c5de3ba911adeca94eb57a3a`);
         cy.waitForSkeletonGone();
-        cy.get('.table-tx-vin tr:nth-child(1) .amount').should('contain.text', '0.02465000 L-BTC');
-        cy.get('.table-tx-vin tr').should('have.class', 'assetBox');
-        cy.get('.table-tx-vout tr:nth-child(1) .amount').should('contain.text', '0.00100000 L-BTC');
-        cy.get('.table-tx-vout tr:nth-child(2) .amount').should('contain.text', '0.02364760 L-BTC');
-        cy.get('.table-tx-vout tr').should('have.class', 'assetBox');
+        cy.get('#table-tx-vin tr:nth-child(1) .amount').should('contain.text', '0.02465000 L-BTC');
+        cy.get('#table-tx-vin tr').should('have.class', 'assetBox');
+        cy.get('#table-tx-vout tr:nth-child(1) .amount').should('contain.text', '0.00100000 L-BTC');
+        cy.get('#table-tx-vout tr:nth-child(2) .amount').should('contain.text', '0.02364760 L-BTC');
+        cy.get('#table-tx-vout tr').should('have.class', 'assetBox');
       });
 
       it('show empty unblinded TX', () => {
         cy.visit(`${basePath}/tx/f2f41c0850e8e7e3f1af233161fd596662e67c11ef10ed15943884186fbb7f46#blinded=`);
         cy.waitForSkeletonGone();
-        cy.get('.table-tx-vin tr:nth-child(1)').should('have.class', '');
-        cy.get('.table-tx-vin tr:nth-child(1) .amount').should('contain.text', 'Confidential');
-        cy.get('.table-tx-vout tr:nth-child(1)').should('have.class', '');
-        cy.get('.table-tx-vout tr:nth-child(2)').should('have.class', '');
-        cy.get('.table-tx-vout tr:nth-child(1) .amount').should('contain.text', 'Confidential');
-        cy.get('.table-tx-vout tr:nth-child(2) .amount').should('contain.text', 'Confidential');
+        cy.get('#table-tx-vin tr:nth-child(1)').should('have.class', '');
+        cy.get('#table-tx-vin tr:nth-child(1) .amount').should('contain.text', 'Confidential');
+        cy.get('#table-tx-vout tr:nth-child(1)').should('have.class', '');
+        cy.get('#table-tx-vout tr:nth-child(2)').should('have.class', '');
+        cy.get('#table-tx-vout tr:nth-child(1) .amount').should('contain.text', 'Confidential');
+        cy.get('#table-tx-vout tr:nth-child(2) .amount').should('contain.text', 'Confidential');
       });
 
       it('show invalid unblinded TX hex', () => {
         cy.visit(`${basePath}/tx/f2f41c0850e8e7e3f1af233161fd596662e67c11ef10ed15943884186fbb7f46#blinded=123`);
         cy.waitForSkeletonGone();
-        cy.get('.table-tx-vin tr').should('have.class', '');
-        cy.get('.table-tx-vout tr').should('have.class', '');
+        cy.get('#table-tx-vin tr').should('have.class', '');
+        cy.get('#table-tx-vout tr').should('have.class', '');
         cy.get('.error-unblinded').contains('Error: Invalid blinding data (invalid hex)');
       });
 
       it('show first unblinded vout', () => {
         cy.visit(`${basePath}/tx/f2f41c0850e8e7e3f1af233161fd596662e67c11ef10ed15943884186fbb7f46#blinded=100000,6f0279e9ed041c3d710a9f57d0c02928416460c4b722ae3457a11eec381c526d,0ab9f70650f16b1db8dfada05237f7d0d65191c3a13183da8a2ddddfbde9a2ad,fd98b2edc5530d76acd553f206a431f4c1fab27e10e290ad719582af878e98fc`);
         cy.waitForSkeletonGone();
-        cy.get('.table-tx-vout tr:nth-child(1)').should('have.class', 'assetBox');
-        cy.get('.table-tx-vout tr:nth-child(1) .amount').should('contain.text', '0.00100000 L-BTC');
+        cy.get('#table-tx-vout tr:nth-child(1)').should('have.class', 'assetBox');
+        cy.get('#table-tx-vout tr:nth-child(1) .amount').should('contain.text', '0.00100000 L-BTC');
       });
 
       it('show second unblinded vout', () => {
         cy.visit(`${basePath}/tx/f2f41c0850e8e7e3f1af233161fd596662e67c11ef10ed15943884186fbb7f46#blinded=2364760,6f0279e9ed041c3d710a9f57d0c02928416460c4b722ae3457a11eec381c526d,90c7a43b15b905bca045ca42a01271cfe71d2efe3133f4197792c24505cb32ed,12eb5959d9293b8842e7dd8bc9aa9639fd3fd031c5de3ba911adeca94eb57a3a`);
-        cy.get('.table-tx-vout tr:nth-child(2').should('have.class', 'assetBox');
-        cy.get('.table-tx-vout tr:nth-child(2) .amount').should('contain.text', '0.02364760 L-BTC');
+        cy.get('#table-tx-vout tr:nth-child(2').should('have.class', 'assetBox');
+        cy.get('#table-tx-vout tr:nth-child(2) .amount').should('contain.text', '0.02364760 L-BTC');
       });
 
       it('show invalid error unblinded TX', () => {
         cy.visit(`${basePath}/tx/f2f41c0850e8e7e3f1af233161fd596662e67c11ef10ed15943884186fbb7f46#blinded=100000,6f0279e9ed041c3d710a9f57d0c02928416460c4b722ae3457a11eec381c526d,0ab9f70650f16b1db8dfada05237f7d0d65191c3a13183da8a2ddddfbde9a2ad,fd98b2edc5530d76acd553f206a431f4c1fab27e10e290ad719582af878e98fc,2364760,6f0279e9ed041c3d710a9f57d0c02928416460c4b722ae3457a11eec381c526d,90c7a43b15b905bca045ca42a01271cfe71d2efe3133f4197792c24505cb32ed,12eb5959d9293b8842e7dd8bc9aa9639fd3fd031c5de3ba911adeca94eb57a3c`);
         cy.waitForSkeletonGone();
-        cy.get('.table-tx-vout tr').should('have.class', 'assetBox');
+        cy.get('#table-tx-vout tr').should('have.class', 'assetBox');
         cy.get('.error-unblinded').contains('Error: Invalid blinding data.');
       });
 
       it('shows asset peg in/out and burn transactions', () => {
-        cy.visit(`${basePath}/assets/asset/6f0279e9ed041c3d710a9f57d0c02928416460c4b722ae3457a11eec381c526d`);
+        cy.visit(`${basePath}/asset/6f0279e9ed041c3d710a9f57d0c02928416460c4b722ae3457a11eec381c526d`);
         cy.waitForSkeletonGone();
-        cy.get('.table-tx-vout tr').not('.assetBox');
-        cy.get('.table-tx-vin tr').not('.assetBox');
+        cy.get('#table-tx-vout tr').not('.assetBox');
+        cy.get('#table-tx-vin tr').not('.assetBox');
       });
 
       it('prevents regressing issue #644', () => {

frontend/cypress/integration/liquidtestnet/liquidtestnet.spec.ts

@@ -73,11 +73,17 @@ describe('Liquid Testnet', () => {
     });
 
     describe('assets', () => {
+      it('shows the assets screen', () => {
+        cy.visit(`${basePath}/assets`);
+        cy.waitForSkeletonGone();
+        cy.get('table tr').should('have.length.at.least', 5);
+      });
+
       it('allows searching assets', () => {
         cy.visit(`${basePath}/assets`);
         cy.waitForSkeletonGone();
         cy.get('.container-xl input').click().type('Liquid Bitcoin').then(() => {
-          cy.get('ngb-typeahead-window').should('have.length', 1);
+          cy.get('table tr').should('have.length', 1);
         });
       });
 
@@ -85,7 +91,7 @@ describe('Liquid Testnet', () => {
         cy.visit(`${basePath}/assets`);
         cy.waitForSkeletonGone();
         cy.get('.container-xl input').click().type('Liquid CAD').then(() => {
-          cy.get('ngb-typeahead-window:nth-of-type(1) button').click();
+          cy.get('table tr td:nth-of-type(1) a').click();
         });
       });
     });
@@ -100,66 +106,66 @@ describe('Liquid Testnet', () => {
       it('show unblinded TX', () => {
         cy.visit(`${basePath}/tx/c3d908ab77891e4c569b0df71aae90f4720b157019ebb20db176f4f9c4d626b8#blinded=100000,144c654344aa716d6f3abcc1ca90e5641e4e2a7f633bc09fe3baf64585819a49,df290ead654d7d110ebc5aaf0bcf11d5b5d360431a467f1cde0a856fde986893,33cb3a2fd2e76643843691cf44a78c5cd28ec652a414da752160ad63fbd37bc9,49741,144c654344aa716d6f3abcc1ca90e5641e4e2a7f633bc09fe3baf64585819a49,edb0713bcbfcb3daabf601cb50978439667d208e15fed8a5ebbfea5696cda1d5,4de70115501e8c7d6bd763e229bf42781edeacf6e75e1d7bdfa4c63104bc508a`);
         cy.waitForSkeletonGone();
-        cy.get('.table-tx-vin tr:nth-child(1) .amount').should('contain.text', '0.00100000 tL-BTC');
-        cy.get('.table-tx-vin tr').should('have.class', 'assetBox');
-        cy.get('.table-tx-vout tr:nth-child(1) .amount').should('contain.text', '0.00050000 tL-BTC');
-        cy.get('.table-tx-vout tr:nth-child(2) .amount').should('contain.text', '0.00049741 tL-BTC');
-        cy.get('.table-tx-vout tr').should('have.class', 'assetBox');
+        cy.get('#table-tx-vin tr:nth-child(1) .amount').should('contain.text', '0.00100000 tL-BTC');
+        cy.get('#table-tx-vin tr').should('have.class', 'assetBox');
+        cy.get('#table-tx-vout tr:nth-child(1) .amount').should('contain.text', '0.00050000 tL-BTC');
+        cy.get('#table-tx-vout tr:nth-child(2) .amount').should('contain.text', '0.00049741 tL-BTC');
+        cy.get('#table-tx-vout tr').should('have.class', 'assetBox');
       });
 
       it('show empty unblinded TX', () => {
         cy.visit(`${basePath}/tx/c3d908ab77891e4c569b0df71aae90f4720b157019ebb20db176f4f9c4d626b8#blinded=`);
         cy.waitForSkeletonGone();
-        cy.get('.table-tx-vin tr:nth-child(1)').should('have.class', '');
-        cy.get('.table-tx-vin tr:nth-child(1) .amount').should('contain.text', 'Confidential');
-        cy.get('.table-tx-vout tr:nth-child(1)').should('have.class', '');
-        cy.get('.table-tx-vout tr:nth-child(2)').should('have.class', '');
-        cy.get('.table-tx-vout tr:nth-child(1) .amount').should('contain.text', 'Confidential');
-        cy.get('.table-tx-vout tr:nth-child(2) .amount').should('contain.text', 'Confidential');
+        cy.get('#table-tx-vin tr:nth-child(1)').should('have.class', '');
+        cy.get('#table-tx-vin tr:nth-child(1) .amount').should('contain.text', 'Confidential');
+        cy.get('#table-tx-vout tr:nth-child(1)').should('have.class', '');
+        cy.get('#table-tx-vout tr:nth-child(2)').should('have.class', '');
+        cy.get('#table-tx-vout tr:nth-child(1) .amount').should('contain.text', 'Confidential');
+        cy.get('#table-tx-vout tr:nth-child(2) .amount').should('contain.text', 'Confidential');
       });
 
       it('show invalid unblinded TX hex', () => {
         cy.visit(`${basePath}/tx/2477f220eef1d03f8ffa4a2861c275d155c3562adf0d79523aeeb0c59ee611ba#blinded=5000`);
         cy.waitForSkeletonGone();
-        cy.get('.table-tx-vin tr').should('have.class', '');
-        cy.get('.table-tx-vout tr').should('have.class', '');
+        cy.get('#table-tx-vin tr').should('have.class', '');
+        cy.get('#table-tx-vout tr').should('have.class', '');
         cy.get('.error-unblinded').contains('Error: Invalid blinding data (invalid hex)');
       });
 
       it('show first unblinded vout', () => {
         cy.visit(`${basePath}/tx/0877bc0c7aa5c2b8d0e4b15450425879b8783c40e341806037a605ef836fb886#blinded=5000,38fca2d939696061a8f76d4e6b5eecd54e3b4221c846f24a6b279e79952850a5,328de54e90e867a9154b4f1eb7fcab86267e880fa2ee9e53b41a91e61dab86e6,8885831e6b089eaf06889d53a24843f0da533d300a7b1527b136883a6819f3ae,5000,38fca2d939696061a8f76d4e6b5eecd54e3b4221c846f24a6b279e79952850a5,aca78b953615d69ae0ae68c4c5c3c0ee077c10bc20ad3f0c5960706004e6cb56,d2ec175afe5f761e2dbd443faf46abbb7091f341deb3387e5787d812bdb2df9f,100000,144c654344aa716d6f3abcc1ca90e5641e4e2a7f633bc09fe3baf64585819a49,4b54a4ca809b3844f34dd88b68617c4c866d92a02211f02ba355755bac20a1c6,eddd02e92b0cfbad8cab89828570a50f2c643bb2a54d886c86e25ce47e818685,99729,144c654344aa716d6f3abcc1ca90e5641e4e2a7f633bc09fe3baf64585819a49,8b86d565c9549eb0352bb81ee576d01d064435b64fddcc045decebeb1d9913ce,b082ce3448d40d47b5b39f15d72b285f4a1046b636b56c25f32f498ece29d062,10000,38fca2d939696061a8f76d4e6b5eecd54e3b4221c846f24a6b279e79952850a5,62b04ee86198d6b41681cdd0acb450ab366af727a010aaee8ba0b9e69ff43896,3f98429bca9b538dc943c22111f25d9c4448d45a63ff0f4e58b22fd434c0365e`);
         cy.waitForSkeletonGone();
-        cy.get('.table-tx-vout tr:nth-child(1)').should('have.class', 'assetBox');
-        cy.get('.table-tx-vout tr:nth-child(1) .amount').should('contain.text', '0.00099729 tL-BTC');
+        cy.get('#table-tx-vout tr:nth-child(1)').should('have.class', 'assetBox');
+        cy.get('#table-tx-vout tr:nth-child(1) .amount').should('contain.text', '0.00099729 tL-BTC');
       });
 
       it('show second unblinded vout (asset)', () => {
         cy.visit(`${basePath}/tx/0877bc0c7aa5c2b8d0e4b15450425879b8783c40e341806037a605ef836fb886#blinded=5000,38fca2d939696061a8f76d4e6b5eecd54e3b4221c846f24a6b279e79952850a5,328de54e90e867a9154b4f1eb7fcab86267e880fa2ee9e53b41a91e61dab86e6,8885831e6b089eaf06889d53a24843f0da533d300a7b1527b136883a6819f3ae,5000,38fca2d939696061a8f76d4e6b5eecd54e3b4221c846f24a6b279e79952850a5,aca78b953615d69ae0ae68c4c5c3c0ee077c10bc20ad3f0c5960706004e6cb56,d2ec175afe5f761e2dbd443faf46abbb7091f341deb3387e5787d812bdb2df9f,100000,144c654344aa716d6f3abcc1ca90e5641e4e2a7f633bc09fe3baf64585819a49,4b54a4ca809b3844f34dd88b68617c4c866d92a02211f02ba355755bac20a1c6,eddd02e92b0cfbad8cab89828570a50f2c643bb2a54d886c86e25ce47e818685,99729,144c654344aa716d6f3abcc1ca90e5641e4e2a7f633bc09fe3baf64585819a49,8b86d565c9549eb0352bb81ee576d01d064435b64fddcc045decebeb1d9913ce,b082ce3448d40d47b5b39f15d72b285f4a1046b636b56c25f32f498ece29d062,10000,38fca2d939696061a8f76d4e6b5eecd54e3b4221c846f24a6b279e79952850a5,62b04ee86198d6b41681cdd0acb450ab366af727a010aaee8ba0b9e69ff43896,3f98429bca9b538dc943c22111f25d9c4448d45a63ff0f4e58b22fd434c0365e`);
-        cy.get('.table-tx-vout tr:nth-child(2)').should('have.class', 'assetBox');
+        cy.get('#table-tx-vout tr:nth-child(2)').should('have.class', 'assetBox');
         //TODO Update after the precision bug fix is merged
-        cy.get('.table-tx-vout tr:nth-child(2) .amount').should('contain.text', '0 TEST');
+        cy.get('#table-tx-vout tr:nth-child(2) .amount').should('contain.text', '0 TEST');
       });
 
       it('should link to the asset page from the unblinded tx', () => {
         cy.visit(`${basePath}/tx/0877bc0c7aa5c2b8d0e4b15450425879b8783c40e341806037a605ef836fb886#blinded=5000,38fca2d939696061a8f76d4e6b5eecd54e3b4221c846f24a6b279e79952850a5,328de54e90e867a9154b4f1eb7fcab86267e880fa2ee9e53b41a91e61dab86e6,8885831e6b089eaf06889d53a24843f0da533d300a7b1527b136883a6819f3ae,5000,38fca2d939696061a8f76d4e6b5eecd54e3b4221c846f24a6b279e79952850a5,aca78b953615d69ae0ae68c4c5c3c0ee077c10bc20ad3f0c5960706004e6cb56,d2ec175afe5f761e2dbd443faf46abbb7091f341deb3387e5787d812bdb2df9f,100000,144c654344aa716d6f3abcc1ca90e5641e4e2a7f633bc09fe3baf64585819a49,4b54a4ca809b3844f34dd88b68617c4c866d92a02211f02ba355755bac20a1c6,eddd02e92b0cfbad8cab89828570a50f2c643bb2a54d886c86e25ce47e818685,99729,144c654344aa716d6f3abcc1ca90e5641e4e2a7f633bc09fe3baf64585819a49,8b86d565c9549eb0352bb81ee576d01d064435b64fddcc045decebeb1d9913ce,b082ce3448d40d47b5b39f15d72b285f4a1046b636b56c25f32f498ece29d062,10000,38fca2d939696061a8f76d4e6b5eecd54e3b4221c846f24a6b279e79952850a5,62b04ee86198d6b41681cdd0acb450ab366af727a010aaee8ba0b9e69ff43896,3f98429bca9b538dc943c22111f25d9c4448d45a63ff0f4e58b22fd434c0365e`);
-        cy.get('.table-tx-vout tr:nth-child(2) .amount a').click().then(() => {
+        cy.get('#table-tx-vout tr:nth-child(2) .amount a').click().then(() => {
           cy.waitForSkeletonGone();
-          cy.url().should('contain', '/assets/asset/38fca2d939696061a8f76d4e6b5eecd54e3b4221c846f24a6b279e79952850a5');
+          cy.url().should('contain', '/asset/38fca2d939696061a8f76d4e6b5eecd54e3b4221c846f24a6b279e79952850a5');
         });
       });
 
       it('show invalid error unblinded TX', () => {
         cy.visit(`${basePath}/tx/c3d908ab77891e4c569b0df71aae90f4720b157019ebb20db176f4f9c4d626b8#blinded=100000,144c654344aa716d6f3abcc1ca90e5641e4e2a7f633bc09fe3baf64585819a49,df290ead654d7d110ebc5aaf0bcf11d5b5d360431a467f1cde0a856fde986893,33cb3a2fd2e76643843691cf44a78c5cd28ec652a414da752160ad63fbd37bc9,49741,144c654344aa716d6f3abcc1ca90e5641e4e2a7f633bc09fe3baf64585819a49,edb0713bcbfcb3daabf601cb50978439667d208e15fed8a5ebbfea5696cda1d5,4de70115501e8c7d6bd763e229bf42781edeacf6e75e1d7bdfa4c63104bc508c`);
         cy.waitForSkeletonGone();
-        cy.get('.table-tx-vin tr').should('have.class', 'assetBox');
+        cy.get('#table-tx-vin tr').should('have.class', 'assetBox');
         cy.get('.error-unblinded').contains('Error: Invalid blinding data.');
       });
 
       it('shows asset peg in/out and burn transactions', () => {
-        cy.visit(`${basePath}/assets/asset/ac3e0ff248c5051ffd61e00155b7122e5ebc04fd397a0ecbdd4f4e4a56232926`);
+        cy.visit(`${basePath}/asset/ac3e0ff248c5051ffd61e00155b7122e5ebc04fd397a0ecbdd4f4e4a56232926`);
         cy.waitForSkeletonGone();
-        cy.get('.table-tx-vout tr').not('.assetBox');
-        cy.get('.table-tx-vin tr').not('.assetBox');
+        cy.get('#table-tx-vout tr').not('.assetBox');
+        cy.get('#table-tx-vin tr').not('.assetBox');
       });
 
     });

frontend/cypress/integration/mainnet/mainnet.spec.ts

@@ -66,7 +66,7 @@ describe('Mainnet', () => {
       cy.get('[id^="bitcoin-block-"]').should('have.length', 8);
       cy.get('.footer').should('be.visible');
       cy.get('.row > :nth-child(1)').invoke('text').then((text) => {
-        expect(text).to.match(/Incoming transactions.* vB\/s/);
+        expect(text).to.match(/Tx vBytes per second:.* vB\/s/);
       });
       cy.get('.row > :nth-child(2)').invoke('text').then((text) => {
         expect(text).to.match(/Unconfirmed:(.*)/);
@@ -171,50 +171,6 @@ describe('Mainnet', () => {
 
     });
 
-    describe('address highlighting', () => {
-      it('highlights single input addresses', () => {
-        const address = '1wiz32gbHZwMzJCRHMGehJuBgsMTPdaCa';
-        cy.visit(`/address/${address}`);
-        cy.waitForSkeletonGone();
-        cy.get('[data-cy="tx-0"] .table-tx-vin .highlight').should('exist');
-        cy.get('[data-cy="tx-0"] .table-tx-vin .highlight').invoke('text').should('contain', `${address}`);
-      });
-
-      it('highlights multiple input addresses', () => {
-        const address = '1wiz1rtKFBA58qjb582WF5KAFg9mWCuZV';
-        cy.visit(`/address/${address}`);
-        cy.waitForSkeletonGone();
-        cy.get('[data-cy="tx-2"] .table-tx-vin .highlight').should('exist');
-        cy.get('[data-cy="tx-2"] .table-tx-vin .highlight').its('length').should('equal', 2);
-        cy.get('[data-cy="tx-2"] .table-tx-vin .highlight').invoke('text').should('contain', `${address}`);
-      });
-
-      it('highlights both input and output addresses in the same transaction', () => {
-        const address = 'bc1q03u63r6hm7a3v6em58zdqtp446w2pw30nm63mv';
-        cy.visit(`/address/${address}`);
-        cy.waitForSkeletonGone();
-        cy.get('[data-cy="tx-1"] .table-tx-vin .highlight').should('exist');
-        cy.get('[data-cy="tx-1"] .table-tx-vout .highlight').should('exist');
-      });
-
-      it('highlights single output addresses', () => {
-        const address = '1wiz32gbHZwMzJCRHMGehJuBgsMTPdaCa';
-        cy.visit(`/address/${address}`);
-        cy.waitForSkeletonGone();
-        cy.get('[data-cy="tx-1"] .table-tx-vout .highlight').should('exist');
-        cy.get('[data-cy="tx-1"] .table-tx-vout .highlight').invoke('text').should('contain', `${address}`);
-      });
-
-      it('highlights multiple output addresses', () => {
-        const address = '1F3Q3sQmiGsWSqK5K6T9tYnX8yqzYRgQbe';
-        cy.visit(`/address/${address}`);
-        cy.waitForSkeletonGone();
-        cy.get('[data-cy="tx-1"] .table-tx-vout .highlight').should('exist');
-        cy.get('[data-cy="tx-1"] .table-tx-vout .highlight').its('length').should('equal', 2);
-        cy.get('[data-cy="tx-1"] .table-tx-vout .highlight').invoke('text').should('contain', `${address}`);
-      });
-    });
-
     describe('blocks navigation', () => {
 
       describe('keyboard events', () => {
@@ -241,7 +197,7 @@ describe('Mainnet', () => {
             cy.get('[ngbtooltip="Next Block"] > .ng-fa-icon > .svg-inline--fa').should('not.exist');
             cy.get('[ngbtooltip="Previous Block"] > .ng-fa-icon > .svg-inline--fa').should('be.visible');
             cy.document().left();
-            cy.get('.title-block h1').invoke('text').should('equal', 'Next Block');
+            cy.get('.title-block h1').invoke('text').should('equal', 'Next block');
           });
         });
 
@@ -318,19 +274,113 @@ describe('Mainnet', () => {
             });
           });
         });
+      });
+    });
 
-        it('loads genesis block and click on the arrow left', () => {
-          cy.viewport('macbook-16');
-          cy.visit('/block/0');
-          cy.waitForSkeletonGone();
-          cy.waitForPageIdle();
-          cy.get('[ngbtooltip="Next Block"] > .ng-fa-icon > .svg-inline--fa').should('be.visible');
-          cy.get('[ngbtooltip="Previous Block"] > .ng-fa-icon > .svg-inline--fa').should('not.exist');
-          cy.get('[ngbtooltip="Next Block"] > .ng-fa-icon > .svg-inline--fa').click().then(() => {
-            cy.get('[ngbtooltip="Next Block"] > .ng-fa-icon > .svg-inline--fa').should('be.visible');
-            cy.get('[ngbtooltip="Previous Block"] > .ng-fa-icon > .svg-inline--fa').should('be.visible');
-          });
-        });
+
+    it('loads skeleton when changes between networks', () => {
+      cy.visit('/');
+      cy.waitForSkeletonGone();
+
+      cy.changeNetwork("testnet");
+      cy.changeNetwork("signet");
+      cy.changeNetwork("mainnet");
+    });
+
+    it.skip('loads the dashboard with the skeleton blocks', () => {
+      cy.mockMempoolSocket();
+      cy.visit("/");
+      cy.get(':nth-child(1) > #bitcoin-block-0').should('be.visible');
+      cy.get(':nth-child(2) > #bitcoin-block-0').should('be.visible');
+      cy.get(':nth-child(3) > #bitcoin-block-0').should('be.visible');
+      cy.get('#mempool-block-0').should('be.visible');
+      cy.get('#mempool-block-1').should('be.visible');
+      cy.get('#mempool-block-2').should('be.visible');
+
+      emitMempoolInfo({
+        'params': {
+          command: 'init'
+        }
+      });
+
+      cy.get(':nth-child(1) > #bitcoin-block-0').should('not.exist');
+      cy.get(':nth-child(2) > #bitcoin-block-0').should('not.exist');
+      cy.get(':nth-child(3) > #bitcoin-block-0').should('not.exist');
+    });
+
+    it('loads the pools screen', () => {
+      cy.visit('/');
+      cy.waitForSkeletonGone();
+      cy.get('#btn-pools').click().then(() => {
+        cy.waitForPageIdle();
+      });
+    });
+
+    it('loads the graphs screen', () => {
+      cy.visit('/');
+      cy.waitForSkeletonGone();
+      cy.get('#btn-graphs').click().then(() => {
+        cy.wait(1000);
+      });
+    });
+
+    describe('graphs page', () => {
+      it('check buttons - mobile', () => {
+        cy.viewport('iphone-6');
+        cy.visit('/graphs');
+        cy.waitForSkeletonGone();
+        cy.get('.small-buttons > :nth-child(2)').should('be.visible');
+        cy.get('#dropdownFees').should('be.visible');
+        cy.get('.btn-group').should('be.visible');
+      });
+      it('check buttons - tablet', () => {
+        cy.viewport('ipad-2');
+        cy.visit('/graphs');
+        cy.waitForSkeletonGone();
+        cy.get('.small-buttons > :nth-child(2)').should('be.visible');
+        cy.get('#dropdownFees').should('be.visible');
+        cy.get('.btn-group').should('be.visible');
+      });
+      it('check buttons - desktop', () => {
+        cy.viewport('macbook-16');
+        cy.visit('/graphs');
+        cy.waitForSkeletonGone();
+        cy.get('.small-buttons > :nth-child(2)').should('be.visible');
+        cy.get('#dropdownFees').should('be.visible');
+        cy.get('.btn-group').should('be.visible');
+      });
+    });
+
+    it('loads the tv screen - desktop', () => {
+      cy.viewport('macbook-16');
+      cy.visit('/');
+      cy.waitForSkeletonGone();
+      cy.get('#btn-tv').click().then(() => {
+        cy.viewport('macbook-16');
+        cy.get('.chart-holder');
+        cy.get('.blockchain-wrapper').should('be.visible');
+        cy.get('#mempool-block-0').should('be.visible');
+      });
+    });
+
+    it('loads the tv screen - mobile', () => {
+      cy.viewport('iphone-6');
+      cy.visit('/tv');
+      cy.waitForSkeletonGone();
+      cy.get('.chart-holder');
+      cy.get('.blockchain-wrapper').should('not.visible');
+    });
+
+    it('loads genesis block and click on the arrow left', () => {
+      cy.viewport('macbook-16');
+      cy.visit('/block/0');
+      cy.waitForSkeletonGone();
+      cy.waitForPageIdle();
+      cy.get('[ngbtooltip="Next Block"] > .ng-fa-icon > .svg-inline--fa').should('be.visible');
+      cy.get('[ngbtooltip="Previous Block"] > .ng-fa-icon > .svg-inline--fa').should('not.exist');
+      cy.get('[ngbtooltip="Next Block"] > .ng-fa-icon > .svg-inline--fa').click().then(() => {
+        cy.get('[ngbtooltip="Next Block"] > .ng-fa-icon > .svg-inline--fa').should('be.visible');
+        cy.get('[ngbtooltip="Previous Block"] > .ng-fa-icon > .svg-inline--fa').should('be.visible');
       });
     });
 

frontend/cypress/support/commands.ts

@@ -50,98 +50,98 @@ import { mockWebSocket } from './websocket';
 
 /* global Cypress */
 const codes = {
-  ArrowLeft: 37,
-  ArrowUp: 38,
-  ArrowRight: 39,
-  ArrowDown: 40
+    ArrowLeft: 37,
+    ArrowUp: 38,
+    ArrowRight: 39,
+    ArrowDown: 40
 }
 
 Cypress.Commands.add('waitForSkeletonGone', () => {
-  cy.waitUntil(() => {
-    return Cypress.$('.skeleton-loader').length === 0;
-  }, { verbose: true, description: "waitForSkeletonGone", errorMsg: "skeleton loaders never went away", timeout: 15000, interval: 50 });
+    cy.waitUntil(() => {
+        return Cypress.$('.skeleton-loader').length === 0;
+    }, { verbose: true, description: "waitForSkeletonGone", errorMsg: "skeleton loaders never went away", timeout: 15000, interval: 50});
 });
 
 Cypress.Commands.add(
-  "waitForPageIdle",
-  () => {
-    console.warn("Waiting for page idle state");
-    const pageIdleDetector = new PageIdleDetector();
-    pageIdleDetector.WaitForPageToBeIdle();
-  }
+    "waitForPageIdle",
+    () => {
+        console.warn("Waiting for page idle state");
+        const pageIdleDetector = new PageIdleDetector();
+        pageIdleDetector.WaitForPageToBeIdle();
+    }
 );
 
 Cypress.Commands.add('mockMempoolSocket', () => {
   mockWebSocket();
 });
 
-Cypress.Commands.add('changeNetwork', (network: "testnet" | "signet" | "liquid" | "bisq" | "mainnet") => {
-  cy.get('.dropdown-toggle').click().then(() => {
-    cy.get(`a.${network}`).click().then(() => {
-      cy.waitForPageIdle();
-      if (network !== 'bisq') {
-        cy.waitForSkeletonGone();
-      }
+Cypress.Commands.add('changeNetwork', (network: "testnet"|"signet"|"liquid"|"bisq"|"mainnet" ) => {
+    cy.get('.dropdown-toggle').click().then(() => {
+        cy.get(`.${network}`).click().then(() => {
+            cy.waitForPageIdle();
+            if(network !== 'bisq'){
+                cy.waitForSkeletonGone();
+            }
+        });
     });
-  });
 });
 
 // https://github.com/bahmutov/cypress-arrows/blob/8f0303842a343550fbeaf01528d01d1ff213b70c/src/index.js
-function keydownCommand($el, key) {
-  const message = `sending the "${key}" keydown event`
-  const log = Cypress.log({
-    name: `keydown: ${key}`,
-    message: message,
-    consoleProps: function () {
-      return {
-        Subject: $el
+function keydownCommand ($el, key) {
+    const message = `sending the "${key}" keydown event`
+    const log = Cypress.log({
+      name: `keydown: ${key}`,
+      message: message,
+      consoleProps: function () {
+        return {
+          Subject: $el
+        }
       }
-    }
-  })
-
-  const e = $el.createEvent('KeyboardEvent')
-
-  Object.defineProperty(e, 'key', {
-    get: function () {
-      return key
-    }
-  })
-
-  Object.defineProperty(e, 'keyCode', {
-    get: function () {
-      return this.keyCodeVal
-    }
-  })
-  Object.defineProperty(e, 'which', {
-    get: function () {
-      return this.keyCodeVal
-    }
-  })
-  var metaKey = false
-
-  Object.defineProperty(e, 'metaKey', {
-    get: function () {
-      return metaKey
-    }
-  })
-
-  Object.defineProperty(e, 'shiftKey', {
-    get: function () {
-      return false
-    }
-  })
-  e.keyCodeVal = codes[key]
-
-  e.initKeyboardEvent('keydown', true, true,
-    $el.defaultView, false, false, false, false, e.keyCodeVal, e.keyCodeVal)
-
-  $el.dispatchEvent(e)
-  log.snapshot().end()
-  return $el
-}
-
-Cypress.Commands.add('keydown', { prevSubject: "dom" }, keydownCommand)
-Cypress.Commands.add('left', { prevSubject: "dom" }, $el => keydownCommand($el, 'ArrowLeft'))
-Cypress.Commands.add('right', { prevSubject: "dom" }, $el => keydownCommand($el, 'ArrowRight'))
-Cypress.Commands.add('up', { prevSubject: "dom" }, $el => keydownCommand($el, 'ArrowUp'))
-Cypress.Commands.add('down', { prevSubject: "dom" }, $el => keydownCommand($el, 'ArrowDown'))
+    })
+  
+    const e = $el.createEvent('KeyboardEvent')
+  
+    Object.defineProperty(e, 'key', {
+      get: function () {
+        return key
+      }
+    })
+  
+    Object.defineProperty(e, 'keyCode', {
+      get: function () {
+        return this.keyCodeVal
+      }
+    })
+    Object.defineProperty(e, 'which', {
+      get: function () {
+        return this.keyCodeVal
+      }
+    })
+    var metaKey = false
+  
+    Object.defineProperty(e, 'metaKey', {
+      get: function () {
+        return metaKey
+      }
+    })
+  
+    Object.defineProperty(e, 'shiftKey', {
+      get: function () {
+        return false
+      }
+    })
+    e.keyCodeVal = codes[key]
+  
+    e.initKeyboardEvent('keydown', true, true,
+      $el.defaultView, false, false, false, false, e.keyCodeVal, e.keyCodeVal)
+  
+    $el.dispatchEvent(e)
+    log.snapshot().end()
+    return $el
+  }
+  
+  Cypress.Commands.add('keydown', { prevSubject: "dom" }, keydownCommand)
+  Cypress.Commands.add('left', { prevSubject: "dom" }, $el => keydownCommand($el, 'ArrowLeft'))
+  Cypress.Commands.add('right', { prevSubject: "dom" }, $el => keydownCommand($el, 'ArrowRight'))
+  Cypress.Commands.add('up', { prevSubject: "dom" }, $el => keydownCommand($el, 'ArrowUp'))
+  Cypress.Commands.add('down', { prevSubject: "dom" }, $el => keydownCommand($el, 'ArrowDown'))

frontend/generate-config.js

@@ -51,9 +51,9 @@ if (process.env.DOCKER_COMMIT_HASH) {
 } else {
   try {
     const gitRevParse = spawnSync('git', ['rev-parse', '--short', 'HEAD']);
+  
     if (!gitRevParse.error) {
-      const output = gitRevParse.stdout.toString('utf-8').replace(/[\n\r\s]+$/, '');
-      gitCommitHash = output ? output : '?';
+      gitCommitHash = gitRevParse.stdout.toString('utf-8').replace(/[\n\r\s]+$/, '');
       console.log(`mempool revision ${gitCommitHash}`);
     } else if (gitRevParse.error.code === 'ENOENT') {
       console.log('git not found, cannot parse git hash');

frontend/mempool-frontend-config.sample.json

@@ -15,6 +15,5 @@
   "BASE_MODULE": "mempool",
   "MEMPOOL_WEBSITE_URL": "https://mempool.space",
   "LIQUID_WEBSITE_URL": "https://liquid.network",
-  "BISQ_WEBSITE_URL": "https://bisq.markets",
-  "MINING_DASHBOARD": true
+  "BISQ_WEBSITE_URL": "https://bisq.markets"
 }

frontend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mempool-frontend",
-  "version": "2.4.0",
+  "version": "2.4.0-dev",
   "description": "Bitcoin mempool visualizer and blockchain explorer backend",
   "license": "GNU Affero General Public License v3.0",
   "homepage": "https://mempool.space",
@@ -27,12 +27,9 @@
     "serve": "npm run generate-config && ng serve -c local",
     "serve:stg": "npm run generate-config && ng serve -c staging",
     "serve:local-prod": "npm run generate-config && ng serve -c local-prod",
-    "serve:local-staging": "npm run generate-config && ng serve -c local-staging",
     "start": "npm run generate-config && npm run sync-assets-dev && ng serve -c local",
     "start:stg": "npm run generate-config && npm run sync-assets-dev && ng serve -c staging",
     "start:local-prod": "npm run generate-config && npm run sync-assets-dev && ng serve -c local-prod",
-    "start:local-staging": "npm run generate-config && npm run sync-assets-dev && ng serve -c local-staging",
-    "start:mixed": "npm run generate-config && npm run sync-assets-dev && ng serve -c mixed",
     "build": "npm run generate-config && ng build --configuration production --localize && npm run sync-assets && npm run build-mempool.js",
     "sync-assets": "node sync-assets.js && rsync -av ./dist/mempool/browser/en-US/resources ./dist/mempool/browser/resources",
     "sync-assets-dev": "node sync-assets.js dev",
@@ -56,73 +53,71 @@
     "cypress:run": "cypress run",
     "cypress:run:record": "cypress run --record",
     "cypress:open:ci": "node update-config.js TESTNET_ENABLED=true SIGNET_ENABLED=true LIQUID_ENABLED=true BISQ_ENABLED=true ITEMS_PER_PAGE=25 && npm run generate-config && start-server-and-test serve:local-prod 4200 cypress:open",
-    "cypress:run:ci": "node update-config.js TESTNET_ENABLED=true SIGNET_ENABLED=true LIQUID_ENABLED=true BISQ_ENABLED=true ITEMS_PER_PAGE=25 && npm run generate-config && start-server-and-test serve:local-prod 4200 cypress:run:record",
-    "cypress:open:ci:staging": "node update-config.js TESTNET_ENABLED=true SIGNET_ENABLED=true LIQUID_ENABLED=true BISQ_ENABLED=true ITEMS_PER_PAGE=25 && npm run generate-config && start-server-and-test serve:local-staging 4200 cypress:open",
-    "cypress:run:ci:staging": "node update-config.js TESTNET_ENABLED=true SIGNET_ENABLED=true LIQUID_ENABLED=true BISQ_ENABLED=true ITEMS_PER_PAGE=25 && npm run generate-config && start-server-and-test serve:local-staging 4200 cypress:run:record"
+    "cypress:run:ci": "node update-config.js TESTNET_ENABLED=true SIGNET_ENABLED=true LIQUID_ENABLED=true BISQ_ENABLED=true ITEMS_PER_PAGE=25 && npm run generate-config && start-server-and-test serve:local-prod 4200 cypress:run:record"
   },
   "dependencies": {
-    "@angular-devkit/build-angular": "~13.3.7",
-    "@angular/animations": "~13.3.10",
-    "@angular/cli": "~13.3.7",
-    "@angular/common": "~13.3.10",
-    "@angular/compiler": "~13.3.10",
-    "@angular/core": "~13.3.10",
-    "@angular/forms": "~13.3.10",
-    "@angular/localize": "~13.3.10",
-    "@angular/platform-browser": "~13.3.10",
-    "@angular/platform-browser-dynamic": "~13.3.10",
-    "@angular/platform-server": "~13.3.10",
-    "@angular/router": "~13.3.10",
-    "@fortawesome/angular-fontawesome": "~0.10.2",
-    "@fortawesome/fontawesome-common-types": "~6.1.1",
-    "@fortawesome/fontawesome-svg-core": "~6.1.1",
-    "@fortawesome/free-solid-svg-icons": "~6.1.1",
+    "@angular-devkit/build-angular": "^13.1.2",
+    "@angular/animations": "~13.1.1",
+    "@angular/cli": "~13.0.4",
+    "@angular/common": "~13.1.1",
+    "@angular/compiler": "~13.1.1",
+    "@angular/core": "~13.1.1",
+    "@angular/forms": "~13.1.1",
+    "@angular/localize": "^13.1.1",
+    "@angular/platform-browser": "~13.1.1",
+    "@angular/platform-browser-dynamic": "~13.1.1",
+    "@angular/platform-server": "~13.1.1",
+    "@angular/router": "~13.1.1",
+    "@fortawesome/angular-fontawesome": "^0.8.2",
+    "@fortawesome/fontawesome-common-types": "^0.2.35",
+    "@fortawesome/fontawesome-svg-core": "^1.2.35",
+    "@fortawesome/free-solid-svg-icons": "^5.15.3",
     "@juggle/resize-observer": "^3.3.1",
     "@mempool/mempool.js": "2.3.0",
     "@ng-bootstrap/ng-bootstrap": "^11.0.0",
-    "@nguniversal/express-engine": "~13.1.1",
-    "@types/qrcode": "~1.4.2",
-    "bootstrap": "~4.5.0",
+    "@nguniversal/express-engine": "11.2.1",
+    "@types/qrcode": "1.4.1",
+    "bootstrap": "4.5.0",
     "browserify": "^17.0.0",
-    "clipboard": "^2.0.10",
+    "clipboard": "^2.0.4",
     "domino": "^2.1.6",
-    "echarts": "~5.3.2",
+    "echarts": "^5.1.2",
     "express": "^4.17.1",
     "lightweight-charts": "^3.3.0",
     "ngx-bootrap-multiselect": "^2.0.0",
-    "ngx-echarts": "8.0.1",
+    "ngx-echarts": "^7.0.1",
     "ngx-infinite-scroll": "^10.0.1",
     "qrcode": "1.5.0",
-    "rxjs": "~7.5.5",
+    "rxjs": "^6.6.7",
     "tinyify": "^3.0.0",
     "tlite": "^0.1.9",
-    "tslib": "~2.4.0",
-    "zone.js": "~0.11.5"
+    "tslib": "^2.2.0",
+    "zone.js": "~0.11.4"
   },
   "devDependencies": {
-    "@angular/compiler-cli": "~13.3.10",
-    "@angular/language-service": "~13.3.10",
-    "@nguniversal/builders": "~13.1.1",
+    "@angular/compiler-cli": "~13.1.1",
+    "@angular/language-service": "~13.1.1",
+    "@nguniversal/builders": "^11.2.1",
     "@types/express": "^4.17.0",
-    "@types/jasmine": "~4.0.3",
-    "@types/jasminewd2": "~2.0.10",
+    "@types/jasmine": "~3.6.0",
+    "@types/jasminewd2": "~2.0.3",
     "@types/node": "^12.11.1",
-    "codelyzer": "~6.0.2",
+    "codelyzer": "^6.0.1",
     "http-proxy-middleware": "^1.0.5",
-    "jasmine-core": "~4.1.0",
-    "jasmine-spec-reporter": "~7.0.0",
-    "karma": "~6.3.19",
+    "jasmine-core": "~3.6.0",
+    "jasmine-spec-reporter": "~5.0.0",
+    "karma": "~6.3.4",
     "karma-chrome-launcher": "~3.1.0",
     "karma-coverage": "~2.0.3",
-    "karma-jasmine": "~5.0.0",
+    "karma-jasmine": "~4.0.0",
     "karma-jasmine-html-reporter": "^1.5.0",
     "ts-node": "~8.3.0",
     "tslint": "~6.1.0",
-    "typescript": "~4.6.4"
+    "typescript": "~4.4.4"
   },
   "optionalDependencies": {
     "@cypress/schematic": "^1.3.0",
-    "cypress": "^10.0.2",
+    "cypress": "^9.3.1",
     "cypress-fail-on-console-error": "^2.1.3",
     "cypress-wait-until": "^1.7.1",
     "mock-socket": "^9.0.3",

frontend/proxy.conf.js

@@ -20,8 +20,8 @@ try {
 
 PROXY_CONFIG = [
     {
-        context: ['*',
-        '/api/**', '!/api/v1/ws',
+        context: ['*', 
+        '/api/**', '!/api/v1/ws', 
         '!/bisq', '!/bisq/**', '!/bisq/',
         '!/liquid', '!/liquid/**', '!/liquid/',
         '!/liquidtestnet', '!/liquidtestnet/**', '!/liquidtestnet/',
@@ -65,13 +65,7 @@ PROXY_CONFIG = [
         ws: true,
         secure: false,
         changeOrigin: true
-    },
-    {
-      context: ['/resources/mining-pools/**'],
-      target: "https://mempool.space",
-      secure: false,
-      changeOrigin: true
-  }
+    }
 ];
 
 if (configContent && configContent.BASE_MODULE == "liquid") {

frontend/proxy.conf.local.js

@@ -1,13 +1,17 @@
+
 const fs = require('fs');
 
+let PROXY_CONFIG = require('./proxy.conf.js');
+const BACKEND_CONFIG_FILE_NAME = '../backend/mempool-config.json';
 const FRONTEND_CONFIG_FILE_NAME = 'mempool-frontend-config.json';
 
-let configContent;
+let backendConfigContent;
+let frontendConfigContent;
 
 // Read frontend config 
 try {
     const rawConfig = fs.readFileSync(FRONTEND_CONFIG_FILE_NAME);
-    configContent = JSON.parse(rawConfig);
+    frontendConfigContent = JSON.parse(rawConfig);
     console.log(`${FRONTEND_CONFIG_FILE_NAME} file found, using provided config`);
 } catch (e) {
     console.log(e);
@@ -18,109 +22,51 @@ try {
   }
 }
 
-let PROXY_CONFIG = [];
-
-if (configContent && configContent.BASE_MODULE === 'liquid') {
-  PROXY_CONFIG.push(...[
-    {
-      context: ['/liquid/api/v1/**'],
-      target: `http://localhost:8999`,
-      secure: false,
-      ws: true,
-      changeOrigin: true,
-      proxyTimeout: 30000,
-      pathRewrite: {
-          "^/liquid": ""
-      },
-    },
-    {
-      context: ['/liquid/api/**'],
-      target: `http://localhost:8999`,
-      secure: false,
-      changeOrigin: true,
-      proxyTimeout: 30000,
-      pathRewrite: {
-          "^/liquid/api/": "/api/v1/"
-      },
-    },
-    {
-      context: ['/liquidtestnet/api/v1/**'],
-      target: `http://localhost:8999`,
-      secure: false,
-      ws: true,
-      changeOrigin: true,
-      proxyTimeout: 30000,
-      pathRewrite: {
-          "^/liquidtestnet": ""
-      },
-    },
-    {
-      context: ['/liquidtestnet/api/**'],
-      target: `http://localhost:8999`,
-      secure: false,
-      changeOrigin: true,
-      proxyTimeout: 30000,
-      pathRewrite: {
-          "^/liquidtestnet/api/": "/api/v1/"
-      },
-    },
-  ]);
-}
-
-
-if (configContent && configContent.BASE_MODULE === 'bisq') {
-  PROXY_CONFIG.push(...[
-    {
-      context: ['/bisq/api/v1/ws'],
-      target: `http://localhost:8999`,
-      secure: false,
-      ws: true,
-      changeOrigin: true,
-      proxyTimeout: 30000,
-      pathRewrite: {
-          "^/bisq": ""
-      },
-    },
-    {
-      context: ['/bisq/api/v1/**'],
-      target: `http://localhost:8999`,
-      secure: false,
-      changeOrigin: true,
-      proxyTimeout: 30000,
-    },
-    {
-      context: ['/bisq/api/**'],
-      target: `http://localhost:8999`,
-      secure: false,
-      changeOrigin: true,
-      proxyTimeout: 30000,
-      pathRewrite: {
-          "^/bisq/api/": "/api/v1/bisq/"
-      },
-    }
-  ]);
-}
-
-PROXY_CONFIG.push(...[
-  {
-    context: ['/api/v1/**'],
-    target: `http://localhost:8999`,
-    secure: false,
-    ws: true,
-    changeOrigin: true,
-    proxyTimeout: 30000,
-  },
-  {
-    context: ['/api/**'],
-    target: `http://localhost:8999`,
-    secure: false,
-    changeOrigin: true,
-    proxyTimeout: 30000,
-    pathRewrite: {
-        "^/api/": "/api/v1/"
-    },
+// Read backend config
+try {
+    const rawConfig = fs.readFileSync(BACKEND_CONFIG_FILE_NAME);
+    backendConfigContent = JSON.parse(rawConfig);
+    console.log(`${BACKEND_CONFIG_FILE_NAME} file found, using provided config`);
+} catch (e) {
+    console.log(e);
+    if (e.code !== 'ENOENT') {
+      throw new Error(e);
+  } else {
+      console.log(`${BACKEND_CONFIG_FILE_NAME} file not found, using default config`);
   }
-]);
+}
+
+// Remove the "/api/**" entry from the default proxy config
+let localDevContext = PROXY_CONFIG[0].context
+
+localDevContext.splice(PROXY_CONFIG[0].context.indexOf('/api/**'), 1);
+
+PROXY_CONFIG[0].context = localDevContext;
+
+// Change all targets to localhost
+PROXY_CONFIG.map(conf => conf.target = "http://localhost:8999");
+
+// Add rules for local backend
+if (backendConfigContent) {
+    PROXY_CONFIG.push({
+        context: ['/api/address/**', '/api/tx/**', '/api/block/**', '/api/blocks/**'],
+        target: `http://localhost:8999`,
+        secure: false,
+        changeOrigin: true,
+        proxyTimeout: 30000,
+        pathRewrite: {
+            "^/api/": "/api/v1/"
+        },
+    });    
+    PROXY_CONFIG.push({
+        context: ['/api/v1/**'],
+        target: `http://localhost:8999`,
+        secure: false,
+        changeOrigin: true,
+        proxyTimeout: 30000
+    });
+
+}
 
 console.log(PROXY_CONFIG);
 

frontend/proxy.conf.mixed.js

@@ -1,117 +0,0 @@
-const fs = require('fs');
-
-const FRONTEND_CONFIG_FILE_NAME = 'mempool-frontend-config.json';
-
-let configContent;
-
-// Read frontend config 
-try {
-    const rawConfig = fs.readFileSync(FRONTEND_CONFIG_FILE_NAME);
-    configContent = JSON.parse(rawConfig);
-    console.log(`${FRONTEND_CONFIG_FILE_NAME} file found, using provided config`);
-} catch (e) {
-    console.log(e);
-    if (e.code !== 'ENOENT') {
-      throw new Error(e);
-  } else {
-      console.log(`${FRONTEND_CONFIG_FILE_NAME} file not found, using default config`);
-  }
-}
-
-let PROXY_CONFIG = [];
-
-if (configContent && configContent.BASE_MODULE === 'liquid') {
-  PROXY_CONFIG.push(...[
-    {
-      context: ['/liquid/api/v1/**'],
-      target: `http://localhost:8999`,
-      secure: false,
-      ws: true,
-      changeOrigin: true,
-      proxyTimeout: 30000,
-      pathRewrite: {
-          "^/liquid": ""
-      },
-    },
-    {
-      context: ['/liquid/api/**'],
-      target: `https://liquid.network`,
-      secure: false,
-      changeOrigin: true,
-      proxyTimeout: 30000,
-    },
-    {
-      context: ['/liquidtestnet/api/v1/**'],
-      target: `http://localhost:8999`,
-      secure: false,
-      ws: true,
-      changeOrigin: true,
-      proxyTimeout: 30000,
-      pathRewrite: {
-          "^/liquidtestnet": ""
-      },
-    },
-    {
-      context: ['/liquidtestnet/api/**'],
-      target: `https://liquid.network`,
-      secure: false,
-      changeOrigin: true,
-      proxyTimeout: 30000,
-    },
-  ]);
-}
-
-if (configContent && configContent.BASE_MODULE === 'bisq') {
-  PROXY_CONFIG.push(...[
-    {
-      context: ['/bisq/api/v1/ws'],
-      target: `http://localhost:8999`,
-      secure: false,
-      ws: true,
-      changeOrigin: true,
-      proxyTimeout: 30000,
-      pathRewrite: {
-          "^/bisq": ""
-      },
-    },
-    {
-      context: ['/bisq/api/v1/**'],
-      target: `http://localhost:8999`,
-      secure: false,
-      changeOrigin: true,
-      proxyTimeout: 30000,
-    },
-    {
-      context: ['/bisq/api/**'],
-      target: `http://localhost:8999`,
-      secure: false,
-      changeOrigin: true,
-      proxyTimeout: 30000,
-      pathRewrite: {
-          "^/bisq/api/": "/api/v1/bisq/"
-      },
-    },
-  ]);
-}
-
-PROXY_CONFIG.push(...[
-  {
-    context: ['/api/v1/**'],
-    target: `http://localhost:8999`,
-    secure: false,
-    ws: true,
-    changeOrigin: true,
-    proxyTimeout: 30000,
-  },
-  {
-    context: ['/api/**'],
-    target: `https://mempool.space`,
-    secure: false,
-    changeOrigin: true,
-    proxyTimeout: 30000,
-  }
-]);
-
-console.log(PROXY_CONFIG);
-
-module.exports = PROXY_CONFIG;

frontend/proxy.conf.staging.js

@@ -1,11 +0,0 @@
-const fs = require('fs');
-
-let PROXY_CONFIG = require('./proxy.conf');
-
-PROXY_CONFIG.forEach(entry => {
-  entry.target = entry.target.replace("mempool.space", "mempool.ninja");
-  entry.target = entry.target.replace("liquid.network", "liquid.place");
-  entry.target = entry.target.replace("bisq.markets", "bisq.ninja");
-});
-
-module.exports = PROXY_CONFIG;

frontend/server.ts

@@ -66,7 +66,6 @@ export function app(locale: string): express.Express {
   server.get('/address/*', getLocalizedSSR(indexHtml));
   server.get('/blocks', getLocalizedSSR(indexHtml));
   server.get('/mining/pools', getLocalizedSSR(indexHtml));
-  server.get('/mining/pool/*', getLocalizedSSR(indexHtml));
   server.get('/graphs', getLocalizedSSR(indexHtml));
   server.get('/liquid', getLocalizedSSR(indexHtml));
   server.get('/liquid/tx/*', getLocalizedSSR(indexHtml));