Merge pull request #1835 from mempool/simon/transifex-pull-0608

Pulled from Transifex

.gitattributes

@@ -0,0 +1,3 @@
+# ignore all differences in line endings
+package.json eol=crlf -crlf
+*/package.json eol=crlf -crlf

.github/FUNDING.yml

@@ -0,0 +1,12 @@
+# These are supported funding model platforms
+
+github: ['mempool'] # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
+patreon: # Replace with a single Patreon username
+open_collective: # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: ['https://mempool.space/sponsor'] # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.github/ISSUE_TEMPLATE/00-bug-issue.md

@@ -0,0 +1,43 @@
+---
+name: 🐛 Bug Report
+about: Report bugs (no support requests, please)
+---
+
+<!--
+   SUPPORT REQUESTS:
+   This is for reporting bugs in Mempool, not for support requests. 
+   If you have a support request, please reach out on Matrix:
+   https://matrix.to/#/#mempool.support:bitcoin.kyoto
+-->
+
+### Description
+
+<!-- brief description of the bug -->
+
+#### Version
+
+<!-- commit id or version number -->
+
+### Steps to reproduce
+
+<!-- if you can reliably reproduce the bug, list the steps here -->
+
+### Expected behaviour
+
+<!-- description of the expected behavior -->
+
+### Actual behaviour
+
+<!-- explain what happened instead of the expected behaviour -->
+
+### Screenshots
+
+<!-- Screenshots if gui related, drag and drop to add to the issue -->
+
+#### Device or machine
+
+<!-- device/machine used, operating system -->
+
+#### Additional info
+
+<!-- Additional information useful for debugging (e.g. logs) -->

.github/ISSUE_TEMPLATE/30-feature-request.md

@@ -0,0 +1,27 @@
+---
+name: ✨ Feature Request
+about: Request a feature or suggest other enhancements
+---
+
+<!--
+   SUPPORT REQUESTS:
+   This is for requesting features in Mempool, not for support requests. 
+   If you have a support request, please reach out on Matrix:
+   https://matrix.to/#/#mempool.support:bitcoin.kyoto
+-->
+
+### Description
+
+<!-- brief description of the feature request -->
+
+### Problem to be solved
+
+<!-- description of the the problem you're having -->
+
+### Proposed solution
+
+<!-- explain how you think we should solve the problem -->
+
+#### Additional info
+
+<!-- Additional information useful for implementing (e.g. docs, links, etc.) -->

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,8 @@
+blank_issues_enabled: false
+contact_links:
+  - name: 🙋 Need help? Chat with us on Matrix
+    url: https://matrix.to/#/#mempool.support:bitcoin.kyoto
+    about: For support requests or general questions
+  - name: 🌐 Want to help with translations? Use Transifex
+    url: https://www.transifex.com/mempool/mempool
+    about: All translations work is done on Transifex

.github/dependabot.yml

@@ -0,0 +1,20 @@
+version: 2
+updates:
+- package-ecosystem: npm
+  directory: "/backend"
+  schedule:
+    interval: daily
+  open-pull-requests-limit: 10
+- package-ecosystem: npm
+  directory: "/frontend"
+  schedule:
+    interval: daily
+  open-pull-requests-limit: 10
+- package-ecosystem: docker
+  directory: "/docker/backend"
+  schedule:
+    interval: weekly
+- package-ecosystem: "github-actions"
+  directory: "/"
+  schedule:
+      interval: "weekly"

.github/pull_request_template.md

@@ -0,0 +1,6 @@
+<!--
+Please do not open pull requests for translations.
+
+All translations work is done on Transifex:
+https://www.transifex.com/mempool/mempool
+-->

.github/workflows/cypress.yml

@@ -0,0 +1,91 @@
+name: Cypress Tests
+
+on: [push, pull_request]
+jobs:
+  cypress:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        containers: [1, 2, 3, 4, 5]
+        os: ["ubuntu-latest"]
+        browser: [chrome]
+    name: E2E tests on ${{ matrix.browser }} - ${{ matrix.os }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup node
+        uses: actions/setup-node@v2
+        with:
+          node-version: 16.15.0
+          cache: 'npm'
+          cache-dependency-path: frontend/package-lock.json
+      - name: ${{ matrix.browser }} browser tests (Mempool)
+        uses: cypress-io/github-action@v4
+        with:
+          tag: ${{ github.event_name }}
+          working-directory: frontend
+          build: npm run config:defaults:mempool
+          start: npm run start:local-staging
+          wait-on: 'http://localhost:4200'
+          wait-on-timeout: 120
+          record: true
+          parallel: true
+          spec: |
+            cypress/e2e/mainnet/*.spec.ts
+            cypress/e2e/signet/*.spec.ts
+            cypress/e2e/testnet/*.spec.ts
+          group: Tests on ${{ matrix.browser }} (Mempool)
+          browser: ${{ matrix.browser }}
+          ci-build-id: '${{ github.sha }}-${{ github.workflow }}-${{ github.event_name }}'
+        env:
+          COMMIT_INFO_MESSAGE: ${{ github.event.pull_request.title }}
+          CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}
+
+      - name: ${{ matrix.browser }} browser tests (Liquid)
+        uses: cypress-io/github-action@v4
+        if: always()
+        with:
+          tag: ${{ github.event_name }}
+          working-directory: frontend
+          build: npm run config:defaults:liquid
+          start: npm run start:local-staging
+          wait-on: 'http://localhost:4200'
+          wait-on-timeout: 120
+          record: true
+          parallel: true
+          spec: |
+            cypress/e2e/liquid/liquid.spec.ts
+            cypress/e2e/liquidtestnet/liquidtestnet.spec.ts
+          group: Tests on ${{ matrix.browser }} (Liquid)
+          browser: ${{ matrix.browser }}
+          ci-build-id: '${{ github.sha }}-${{ github.workflow }}-${{ github.event_name }}'
+        env:
+          COMMIT_INFO_MESSAGE: ${{ github.event.pull_request.title }}
+          CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}
+
+      - name: ${{ matrix.browser }} browser tests (Bisq)
+        uses: cypress-io/github-action@v4
+        if: always()
+        with:
+          tag: ${{ github.event_name }}
+          working-directory: frontend
+          build: npm run config:defaults:bisq
+          start: npm run start:local-staging
+          wait-on: 'http://localhost:4200'
+          wait-on-timeout: 120
+          record: true
+          parallel: true
+          spec: cypress/e2e/bisq/bisq.spec.ts
+          group: Tests on ${{ matrix.browser }} (Bisq)
+          browser: ${{ matrix.browser }}
+          ci-build-id: '${{ github.sha }}-${{ github.workflow }}-${{ github.event_name }}'
+        env:
+          COMMIT_INFO_MESSAGE: ${{ github.event.pull_request.title }}
+          CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}

.github/workflows/on-tag.yml

@@ -0,0 +1,75 @@
+name: Docker build on tag
+env:
+  DOCKER_CLI_EXPERIMENTAL: enabled
+  TAG_FMT: '^refs/tags/(((.?[0-9]+){3,4}))$'
+  DOCKER_BUILDKIT: 0
+  COMPOSE_DOCKER_CLI_BUILD: 0
+
+on:
+  push:
+    tags:
+      - v[0-9]+.[0-9]+.[0-9]+
+      - v[0-9]+.[0-9]+.[0-9]+-*
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        service:
+          - frontend
+          - backend
+    runs-on: ubuntu-18.04
+    name: Build and push to DockerHub
+    steps:
+      - name: Set env variables
+        run: echo "TAG=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV
+
+      - name: Show set environment variables
+        run: |
+          printf "      TAG: %s\n"  "$TAG"
+      
+      - name: Add SHORT_SHA env property with commit short sha
+        run: echo "SHORT_SHA=`echo ${GITHUB_SHA} | cut -c1-8`" >> $GITHUB_ENV
+
+      - name: Login to Docker for building
+        run: echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin
+
+      - name: Checkout project
+        uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
+
+      - name: Init repo for Dockerization
+        run: docker/init.sh "$TAG"
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 # v1
+        id: qemu
+
+      - name: Setup Docker buildx action
+        uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 # v1
+        id: buildx
+
+      - name: Available platforms
+        run: echo ${{ steps.buildx.outputs.platforms }}
+
+      - name: Cache Docker layers
+        uses: actions/cache@661fd3eb7f2f20d8c7c84bc2b0509efd7a826628 # v2
+        id: cache
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+
+      - name: Run Docker buildx for ${{ matrix.service }} against tag
+        run: |
+          docker buildx build \
+          --cache-from "type=local,src=/tmp/.buildx-cache" \
+          --cache-to "type=local,dest=/tmp/.buildx-cache" \
+          --platform linux/amd64,linux/arm64,linux/arm/v7 \
+          --tag ${{ secrets.DOCKER_HUB_USER }}/${{ matrix.service }}:$TAG \
+          --tag ${{ secrets.DOCKER_HUB_USER }}/${{ matrix.service }}:latest \
+          --output "type=registry" ./${{ matrix.service }}/ \
+          --build-arg commitHash=$SHORT_SHA

.gitignore

@@ -0,0 +1,4 @@
+sitemap
+data
+docker-compose.yml
+backend/mempool-config.json

.nvmrc

@@ -0,0 +1 @@
+v16.15.0

.vscode/settings.json

@@ -0,0 +1,4 @@
+{
+  "editor.tabSize": 2,
+  "typescript.tsdk": "./backend/node_modules/typescript/lib"
+}

CONTRIBUTING.md

@@ -0,0 +1,49 @@
+# Contributing to The Mempool Open Source Project
+
+Thank you for contributing to The Mempool Open Source Project managed by Mempool Space K.K. (“Mempool”).
+
+In order to clarify the intellectual property license granted with Contributions from any person or entity, Mempool must have a statement on file from each Contributor indicating their agreement to the Contributor License Agreement (“Agreement”). This license is for your protection as a Contributor as well as the protection of Mempool and its other contributors and users; it does not change your rights to use your own Contributions for any other purpose.
+
+When submitting a pull request for the first time, please create a file with a name like `/contributors/{github_username}.txt`, and in the content of that file indicate your agreement to the Contributor License Agreement terms below. An example of what that file should contain can be seen in wiz's agreement file. (This method of CLA "signing" is borrowed from Medium's open source project.)
+
+# Contributor License Agreement
+
+Last Updated: January 25, 2022
+
+By accepting this Agreement, You agree to the following terms and conditions for Your present and future Contributions submitted to Mempool. Except for the license granted herein to Mempool and recipients of software distributed by Mempool, You reserve all right, title, and interest in and to Your Contributions.
+
+### 1. Definitions
+
+“You” (or “Your”) shall mean the copyright owner or legal entity authorized by the copyright owner that is making this Agreement with Mempool. For legal entities, the entity making a Contribution and all other entities that control, are controlled by, or are under common control with that entity are considered to be a single Contributor. For the purposes of this definition, “control” means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity.
+
+“Contribution” shall mean any original work of authorship, including any modifications or additions to an existing work, that is intentionally submitted by You to Mempool for inclusion in, or documentation of, any of the products owned or managed by Mempool (“Work”). For the purposes of this definition, “submitted” means any form of electronic, verbal, or written communication sent to Mempool or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, Mempool for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by You as “Not a Contribution.”
+
+### 2. Grant of Copyright License
+
+Subject to the terms and conditions of this Agreement, You hereby grant to Mempool and to recipients of software distributed by Mempool a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare derivative works of, publicly display, publicly perform, sublicense, and distribute Your Contributions and such derivative works.
+
+### 3. Grant of Patent License
+
+Subject to the terms and conditions of this Agreement, You hereby grant to Mempool and to recipients of software distributed by Mempool a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by You that are necessarily infringed by Your Contribution(s) alone or by combination of Your Contribution(s) with the Work to which such Contribution(s) was submitted. If any entity institutes patent litigation against You or any other entity (including a cross-claim or counterclaim in a lawsuit) alleging that your Contribution, or the Work to which you have contributed, constitutes direct or contributory patent infringement, then any patent licenses granted to that entity under this Agreement for that Contribution or Work shall terminate as of the date such litigation is filed.
+
+### 4. Authority
+
+You represent that you are legally entitled to grant the above license. If your employer(s) has rights to intellectual property that you create that includes your Contributions, you represent that you have received permission to make Contributions on behalf of that employer, that your employer has waived such rights for your Contributions to Mempool, or that your employer has executed a separate Corporate Contributor License Agreement with Mempool.
+
+### 5. Originality
+
+You represent that each of Your Contributions is Your original creation (see section 7 for submissions on behalf of others). You represent that Your Contribution submissions include complete details of any third-party license or other restriction (including, but not limited to, related patents and trademarks) of which you are personally aware, and which are associated with any part of Your Contributions.
+
+### 6. Support
+
+You are not expected to provide support for Your Contributions, except to the extent You desire to provide support. You may provide support for free, for a fee, or not at all. Unless required by applicable law or agreed to in writing, You provide Your Contributions on an “AS IS” BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON- INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE.
+
+### 7. Third Party Contributions
+
+Should You wish to submit work that is not Your original creation, You may submit it to Mempool separately from any Contribution, identifying the complete details of its source and of any license or other restriction (including, but not limited to, related patents, trademarks, and license agreements) of which you are personally aware, and conspicuously marking the work as “Submitted on behalf of a third-party: [named here]”.
+
+### 8. Notifications
+
+You agree to notify Mempool of any facts or circumstances of which you become aware that would make these representations inaccurate in any respect.
+
+EOF

Dockerfile

@@ -1,57 +0,0 @@
-FROM alpine:latest
-
-RUN mkdir /mempool.space/
-COPY ./backend /mempool.space/backend/
-COPY ./frontend /mempool.space/frontend/
-COPY ./mariadb-structure.sql /mempool.space/mariadb-structure.sql
-
-RUN apk add mariadb mariadb-client git nginx npm rsync bash
-
-RUN mysql_install_db --user=mysql --datadir=/var/lib/mysql/
-RUN /usr/bin/mysqld_safe --datadir='/var/lib/mysql/'& \
-    sleep 60 && \
-    mysql -e "create database mempool" && \
-    mysql -e "grant all privileges on mempool.* to 'mempool'@'localhost' identified by 'mempool'" && \
-    mysql mempool < /mempool.space/mariadb-structure.sql
-RUN sed -i "/^skip-networking/ c#skip-networking" /etc/my.cnf.d/mariadb-server.cnf
-
-RUN export NG_CLI_ANALYTICS=ci && \
-    npm install -g typescript && \
-    cd /mempool.space/frontend && \
-    npm install && \
-    cd /mempool.space/backend && \
-    npm install && \
-    tsc
-
-COPY ./nginx-nossl-docker.conf /etc/nginx/nginx.conf
-    
-ENV ENV dev
-ENV DB_HOST localhost
-ENV DB_PORT 3306
-ENV DB_USER mempool
-ENV DB_PASSWORD mempool
-ENV DB_DATABASE mempool
-ENV API_ENDPOINT /api/v1/
-ENV CHAT_SSL_ENABLED false
-ENV MEMPOOL_REFRESH_RATE_MS 500
-ENV INITIAL_BLOCK_AMOUNT 8
-ENV DEFAULT_PROJECTED_BLOCKS_AMOUNT 3
-ENV KEEP_BLOCK_AMOUNT 24
-ENV BITCOIN_NODE_HOST bitcoinhost
-ENV BITCOIN_NODE_PORT 8332
-ENV BITCOIN_NODE_USER bitcoinuser
-ENV BITCOIN_NODE_PASS bitcoinpass
-ENV TX_PER_SECOND_SPAN_SECONDS 150
-ENV BACKEND_API bitcoind
-ENV ELECTRS_API_URL https://mempool.space/api
-
-RUN cd /mempool.space/frontend/ && \
-    npm run build && \
-    rsync -av --delete dist/mempool/ /var/www/html/
-
-EXPOSE 80
-
-COPY ./entrypoint.sh /mempool.space/entrypoint.sh
-RUN chmod +x /mempool.space/entrypoint.sh
-WORKDIR /mempool.space
-CMD ["/mempool.space/entrypoint.sh"]

GNUmakefile

@@ -0,0 +1,47 @@
+# If you see pwd_unknown showing up check permissions
+PWD ?= pwd_unknown
+
+# DATABASE DEPLOY FOLDER CONFIG - default ./data
+ifeq ($(data),)
+DATA := data
+export DATA
+else
+DATA := $(data)
+export DATA
+endif
+
+.PHONY: help
+help:
+	@echo ''
+	@echo ''
+	@echo '	Usage: make [COMMAND]'
+	@echo ''
+	@echo '		make all		# build init mempool and electrs'
+	@echo '		make init		# setup some useful configs'
+	@echo '		make mempool		# build q dockerized mempool.space'
+	@echo '		make electrs		# build a docker electrs image'
+	@echo ''
+
+.PHONY: init
+init:
+	@echo ''
+	mkdir -p   $(DATA) $(DATA)/mysql $(DATA)/mysql/data 
+	#REF: https://github.com/mempool/mempool/blob/master/docker/README.md
+	cat docker/docker-compose.yml > docker-compose.yml
+	cat backend/mempool-config.sample.json > backend/mempool-config.json
+.PHONY: mempool
+mempool: init
+	@echo ''
+	docker-compose up --force-recreate --always-recreate-deps
+	@echo ''
+.PHONY: electrs
+electrum:
+	#REF: https://hub.docker.com/r/beli/electrum
+	@echo ''
+	docker build -f docker/electrum/Dockerfile .
+	@echo ''
+.PHONY: all
+all: init
+	make mempool
+#######################
+-include Makefile

LICENSE

@@ -1,21 +1,29 @@
-MIT License
+The Mempool Open Source Project
+Copyright (c) 2019-2022 The Mempool Open Source Project Developers
 
-Copyright (c) 2019 Simon Lindh
+This program is free software; you can redistribute it and/or modify it under
+the terms of (at your option) either:
 
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
+ 1) the GNU Affero General Public License as published by the Free Software
+ Foundation, either version 3 of the License or any later version approved by a
+ proxy statement published on <https://mempool.space/about>; or
 
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
+ 2) the GNU General Public License as published by the Free Software
+ Foundation, either version 3 of the License or any later version approved by a
+ proxy statement published on <https://mempool.space/about>.
 
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+However, this copyright license does not include an implied right or license to
+use our trademarks: The Mempool Open Source Project™, mempool.space™, the
+mempool Logo™, the mempool.space Vertical Logo™, the mempool.space Horizontal
+Logo™, the mempool Square Logo™, and the mempool Blocks logo™ are registered
+trademarks or trademarks of Mempool Space K.K in Japan, the United States,
+and/or other countries. See our full Trademark Policy and Guidelines for more
+details, published on <https://mempool.space/trademark-policy>.
+
+This program is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+PARTICULAR PURPOSE. See the full license terms for more details.
+
+You should have received a copy of both the GNU Affero General Public License
+and the GNU General Public License along with this program.  If not, see
+<http://www.gnu.org/licenses/>.

LICENSE.AGPL-3.md

@@ -0,0 +1,660 @@
+### GNU AFFERO GENERAL PUBLIC LICENSE
+
+Version 3, 19 November 2007
+
+Copyright (C) 2007 Free Software Foundation, Inc.
+<https://fsf.org/>
+
+Everyone is permitted to copy and distribute verbatim copies of this
+license document, but changing it is not allowed.
+
+### Preamble
+
+The GNU Affero General Public License is a free, copyleft license for
+software and other kinds of works, specifically designed to ensure
+cooperation with the community in the case of network server software.
+
+The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works. By contrast,
+our General Public Licenses are intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains
+free software for all its users.
+
+When we speak of free software, we are referring to freedom, not
+price. Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+Developers that use our General Public Licenses protect your rights
+with two steps: (1) assert copyright on the software, and (2) offer
+you this License which gives you legal permission to copy, distribute
+and/or modify the software.
+
+A secondary benefit of defending all users' freedom is that
+improvements made in alternate versions of the program, if they
+receive widespread use, become available for other developers to
+incorporate. Many developers of free software are heartened and
+encouraged by the resulting cooperation. However, in the case of
+software used on network servers, this result may fail to come about.
+The GNU General Public License permits making a modified version and
+letting the public access it on a server without ever releasing its
+source code to the public.
+
+The GNU Affero General Public License is designed specifically to
+ensure that, in such cases, the modified source code becomes available
+to the community. It requires the operator of a network server to
+provide the source code of the modified version running there to the
+users of that server. Therefore, public use of a modified version, on
+a publicly accessible server, gives the public access to the source
+code of the modified version.
+
+An older license, called the Affero General Public License and
+published by Affero, was designed to accomplish similar goals. This is
+a different license, not a version of the Affero GPL, but Affero has
+released a new version of the Affero GPL which permits relicensing
+under this license.
+
+The precise terms and conditions for copying, distribution and
+modification follow.
+
+### TERMS AND CONDITIONS
+
+#### 0. Definitions.
+
+"This License" refers to version 3 of the GNU Affero General Public
+License.
+
+"Copyright" also means copyright-like laws that apply to other kinds
+of works, such as semiconductor masks.
+
+"The Program" refers to any copyrightable work licensed under this
+License. Each licensee is addressed as "you". "Licensees" and
+"recipients" may be individuals or organizations.
+
+To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of
+an exact copy. The resulting work is called a "modified version" of
+the earlier work or a work "based on" the earlier work.
+
+A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy. Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies. Mere interaction with a user
+through a computer network, with no transfer of a copy, is not
+conveying.
+
+An interactive user interface displays "Appropriate Legal Notices" to
+the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License. If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+#### 1. Source Code.
+
+The "source code" for a work means the preferred form of the work for
+making modifications to it. "Object code" means any non-source form of
+a work.
+
+A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form. A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities. However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work. For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+The Corresponding Source need not include anything that users can
+regenerate automatically from other parts of the Corresponding Source.
+
+The Corresponding Source for a work in source code form is that same
+work.
+
+#### 2. Basic Permissions.
+
+All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met. This License explicitly affirms your unlimited
+permission to run the unmodified Program. The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work. This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+You may make, run and propagate covered works that you do not convey,
+without conditions so long as your license otherwise remains in force.
+You may convey covered works to others for the sole purpose of having
+them make modifications exclusively for you, or provide you with
+facilities for running those works, provided that you comply with the
+terms of this License in conveying all material for which you do not
+control copyright. Those thus making or running the covered works for
+you must do so exclusively on your behalf, under your direction and
+control, on terms that prohibit them from making any copies of your
+copyrighted material outside their relationship with you.
+
+Conveying under any other circumstances is permitted solely under the
+conditions stated below. Sublicensing is not allowed; section 10 makes
+it unnecessary.
+
+#### 3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such
+circumvention is effected by exercising rights under this License with
+respect to the covered work, and you disclaim any intention to limit
+operation or modification of the work as a means of enforcing, against
+the work's users, your or third parties' legal rights to forbid
+circumvention of technological measures.
+
+#### 4. Conveying Verbatim Copies.
+
+You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+#### 5. Conveying Modified Source Versions.
+
+You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these
+conditions:
+
+-   a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+-   b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under
+    section 7. This requirement modifies the requirement in section 4
+    to "keep intact all notices".
+-   c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy. This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged. This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+-   d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit. Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+#### 6. Conveying Non-Source Forms.
+
+You may convey a covered work in object code form under the terms of
+sections 4 and 5, provided that you also convey the machine-readable
+Corresponding Source under the terms of this License, in one of these
+ways:
+
+-   a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+-   b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the Corresponding
+    Source from a network server at no charge.
+-   c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source. This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+-   d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge. You need not require recipients to copy the
+    Corresponding Source along with the object code. If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source. Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+-   e) Convey the object code using peer-to-peer transmission,
+    provided you inform other peers where the object code and
+    Corresponding Source of the work are being offered to the general
+    public at no charge under subsection 6d.
+
+A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal,
+family, or household purposes, or (2) anything designed or sold for
+incorporation into a dwelling. In determining whether a product is a
+consumer product, doubtful cases shall be resolved in favor of
+coverage. For a particular product received by a particular user,
+"normally used" refers to a typical or common use of that class of
+product, regardless of the status of the particular user or of the way
+in which the particular user actually uses, or expects or is expected
+to use, the product. A product is a consumer product regardless of
+whether the product has substantial commercial, industrial or
+non-consumer uses, unless such uses represent the only significant
+mode of use of the product.
+
+"Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to
+install and execute modified versions of a covered work in that User
+Product from a modified version of its Corresponding Source. The
+information must suffice to ensure that the continued functioning of
+the modified object code is in no case prevented or interfered with
+solely because modification has been made.
+
+If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information. But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or
+updates for a work that has been modified or installed by the
+recipient, or for the User Product in which it has been modified or
+installed. Access to a network may be denied when the modification
+itself materially and adversely affects the operation of the network
+or violates the rules and protocols for communication across the
+network.
+
+Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+#### 7. Additional Terms.
+
+"Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law. If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it. (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.) You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders
+of that material) supplement the terms of this License with terms:
+
+-   a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+-   b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+-   c) Prohibiting misrepresentation of the origin of that material,
+    or requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+-   d) Limiting the use for publicity purposes of names of licensors
+    or authors of the material; or
+-   e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+-   f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions
+    of it) with contractual assumptions of liability to the recipient,
+    for any liability that these contractual assumptions directly
+    impose on those licensors and authors.
+
+All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10. If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term. If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions; the
+above requirements apply either way.
+
+#### 8. Termination.
+
+You may not propagate or modify a covered work except as expressly
+provided under this License. Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+However, if you cease all violation of this License, then your license
+from a particular copyright holder is reinstated (a) provisionally,
+unless and until the copyright holder explicitly and finally
+terminates your license, and (b) permanently, if the copyright holder
+fails to notify you of the violation by some reasonable means prior to
+60 days after the cessation.
+
+Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License. If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+#### 9. Acceptance Not Required for Having Copies.
+
+You are not required to accept this License in order to receive or run
+a copy of the Program. Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance. However,
+nothing other than this License grants you permission to propagate or
+modify any covered work. These actions infringe copyright if you do
+not accept this License. Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+#### 10. Automatic Licensing of Downstream Recipients.
+
+Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License. You are not responsible
+for enforcing compliance by third parties with this License.
+
+An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations. If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License. For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+#### 11. Patents.
+
+A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based. The
+work thus licensed is called the contributor's "contributor version".
+
+A contributor's "essential patent claims" are all patent claims owned
+or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version. For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement). To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients. "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+A patent license is "discriminatory" if it does not include within the
+scope of its coverage, prohibits the exercise of, or is conditioned on
+the non-exercise of one or more of the rights that are specifically
+granted under this License. You may not convey a covered work if you
+are a party to an arrangement with a third party that is in the
+business of distributing software, under which you make payment to the
+third party based on the extent of your activity of conveying the
+work, and under which the third party grants, to any of the parties
+who would receive the covered work from you, a discriminatory patent
+license (a) in connection with copies of the covered work conveyed by
+you (or copies made from those copies), or (b) primarily for and in
+connection with specific products or compilations that contain the
+covered work, unless you entered into that arrangement, or that patent
+license was granted, prior to 28 March 2007.
+
+Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+#### 12. No Surrender of Others' Freedom.
+
+If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License. If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under
+this License and any other pertinent obligations, then as a
+consequence you may not convey it at all. For example, if you agree to
+terms that obligate you to collect a royalty for further conveying
+from those to whom you convey the Program, the only way you could
+satisfy both those terms and this License would be to refrain entirely
+from conveying the Program.
+
+#### 13. Remote Network Interaction; Use with the GNU General Public License.
+
+Notwithstanding any other provision of this License, if you modify the
+Program, your modified version must prominently offer all users
+interacting with it remotely through a computer network (if your
+version supports such interaction) an opportunity to receive the
+Corresponding Source of your version by providing access to the
+Corresponding Source from a network server at no charge, through some
+standard or customary means of facilitating copying of software. This
+Corresponding Source shall include the Corresponding Source for any
+work covered by version 3 of the GNU General Public License that is
+incorporated pursuant to the following paragraph.
+
+Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU General Public License into a single
+combined work, and to convey the resulting work. The terms of this
+License will continue to apply to the part which is the covered work,
+but the work with which it is combined will remain governed by version
+3 of the GNU General Public License.
+
+#### 14. Revised Versions of this License.
+
+The Free Software Foundation may publish revised and/or new versions
+of the GNU Affero General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+Each version is given a distinguishing version number. If the Program
+specifies that a certain numbered version of the GNU Affero General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation. If the Program does not specify a version number of the
+GNU Affero General Public License, you may choose any version ever
+published by the Free Software Foundation.
+
+If the Program specifies that a proxy can decide which future versions
+of the GNU Affero General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+Later license versions may give you additional or different
+permissions. However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+#### 15. Disclaimer of Warranty.
+
+THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT
+WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND
+PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE
+DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR
+CORRECTION.
+
+#### 16. Limitation of Liability.
+
+IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR
+CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES
+ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT
+NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR
+LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM
+TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER
+PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
+
+#### 17. Interpretation of Sections 15 and 16.
+
+If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+END OF TERMS AND CONDITIONS
+
+### How to Apply These Terms to Your New Programs
+
+If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these
+terms.
+
+To do so, attach the following notices to the program. It is safest to
+attach them to the start of each source file to most effectively state
+the exclusion of warranty; and each file should have at least the
+"copyright" line and a pointer to where the full notice is found.
+
+        <one line to give the program's name and a brief idea of what it does.>
+        Copyright (C) <year>  <name of author>
+
+        This program is free software: you can redistribute it and/or modify
+        it under the terms of the GNU Affero General Public License as
+        published by the Free Software Foundation, either version 3 of the
+        License, or (at your option) any later version.
+
+        This program is distributed in the hope that it will be useful,
+        but WITHOUT ANY WARRANTY; without even the implied warranty of
+        MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+        GNU Affero General Public License for more details.
+
+        You should have received a copy of the GNU Affero General Public License
+        along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper
+mail.
+
+If your software can interact with users remotely through a computer
+network, you should also make sure that it provides a way for users to
+get its source. For example, if your program is a web application, its
+interface could display a "Source" link that leads users to an archive
+of the code. There are many ways you could offer source, and different
+solutions will be better for different programs; see section 13 for
+the specific requirements.
+
+You should also get your employer (if you work as a programmer) or
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary. For more information on this, and how to apply and follow
+the GNU AGPL, see <https://www.gnu.org/licenses/>.

LICENSE.GPL-3.md

@@ -0,0 +1,675 @@
+### GNU GENERAL PUBLIC LICENSE
+
+Version 3, 29 June 2007
+
+Copyright (C) 2007 Free Software Foundation, Inc.
+<https://fsf.org/>
+
+Everyone is permitted to copy and distribute verbatim copies of this
+license document, but changing it is not allowed.
+
+### Preamble
+
+The GNU General Public License is a free, copyleft license for
+software and other kinds of works.
+
+The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works. By contrast,
+the GNU General Public License is intended to guarantee your freedom
+to share and change all versions of a program--to make sure it remains
+free software for all its users. We, the Free Software Foundation, use
+the GNU General Public License for most of our software; it applies
+also to any other work released this way by its authors. You can apply
+it to your programs, too.
+
+When we speak of free software, we are referring to freedom, not
+price. Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+To protect your rights, we need to prevent others from denying you
+these rights or asking you to surrender the rights. Therefore, you
+have certain responsibilities if you distribute copies of the
+software, or if you modify it: responsibilities to respect the freedom
+of others.
+
+For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must pass on to the recipients the same
+freedoms that you received. You must make sure that they, too, receive
+or can get the source code. And you must show them these terms so they
+know their rights.
+
+Developers that use the GNU GPL protect your rights with two steps:
+(1) assert copyright on the software, and (2) offer you this License
+giving you legal permission to copy, distribute and/or modify it.
+
+For the developers' and authors' protection, the GPL clearly explains
+that there is no warranty for this free software. For both users' and
+authors' sake, the GPL requires that modified versions be marked as
+changed, so that their problems will not be attributed erroneously to
+authors of previous versions.
+
+Some devices are designed to deny users access to install or run
+modified versions of the software inside them, although the
+manufacturer can do so. This is fundamentally incompatible with the
+aim of protecting users' freedom to change the software. The
+systematic pattern of such abuse occurs in the area of products for
+individuals to use, which is precisely where it is most unacceptable.
+Therefore, we have designed this version of the GPL to prohibit the
+practice for those products. If such problems arise substantially in
+other domains, we stand ready to extend this provision to those
+domains in future versions of the GPL, as needed to protect the
+freedom of users.
+
+Finally, every program is threatened constantly by software patents.
+States should not allow patents to restrict development and use of
+software on general-purpose computers, but in those that do, we wish
+to avoid the special danger that patents applied to a free program
+could make it effectively proprietary. To prevent this, the GPL
+assures that patents cannot be used to render the program non-free.
+
+The precise terms and conditions for copying, distribution and
+modification follow.
+
+### TERMS AND CONDITIONS
+
+#### 0. Definitions.
+
+"This License" refers to version 3 of the GNU General Public License.
+
+"Copyright" also means copyright-like laws that apply to other kinds
+of works, such as semiconductor masks.
+
+"The Program" refers to any copyrightable work licensed under this
+License. Each licensee is addressed as "you". "Licensees" and
+"recipients" may be individuals or organizations.
+
+To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of
+an exact copy. The resulting work is called a "modified version" of
+the earlier work or a work "based on" the earlier work.
+
+A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy. Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies. Mere interaction with a user
+through a computer network, with no transfer of a copy, is not
+conveying.
+
+An interactive user interface displays "Appropriate Legal Notices" to
+the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License. If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+#### 1. Source Code.
+
+The "source code" for a work means the preferred form of the work for
+making modifications to it. "Object code" means any non-source form of
+a work.
+
+A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form. A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities. However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work. For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+The Corresponding Source need not include anything that users can
+regenerate automatically from other parts of the Corresponding Source.
+
+The Corresponding Source for a work in source code form is that same
+work.
+
+#### 2. Basic Permissions.
+
+All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met. This License explicitly affirms your unlimited
+permission to run the unmodified Program. The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work. This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+You may make, run and propagate covered works that you do not convey,
+without conditions so long as your license otherwise remains in force.
+You may convey covered works to others for the sole purpose of having
+them make modifications exclusively for you, or provide you with
+facilities for running those works, provided that you comply with the
+terms of this License in conveying all material for which you do not
+control copyright. Those thus making or running the covered works for
+you must do so exclusively on your behalf, under your direction and
+control, on terms that prohibit them from making any copies of your
+copyrighted material outside their relationship with you.
+
+Conveying under any other circumstances is permitted solely under the
+conditions stated below. Sublicensing is not allowed; section 10 makes
+it unnecessary.
+
+#### 3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such
+circumvention is effected by exercising rights under this License with
+respect to the covered work, and you disclaim any intention to limit
+operation or modification of the work as a means of enforcing, against
+the work's users, your or third parties' legal rights to forbid
+circumvention of technological measures.
+
+#### 4. Conveying Verbatim Copies.
+
+You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+#### 5. Conveying Modified Source Versions.
+
+You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these
+conditions:
+
+-   a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+-   b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under
+    section 7. This requirement modifies the requirement in section 4
+    to "keep intact all notices".
+-   c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy. This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged. This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+-   d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit. Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+#### 6. Conveying Non-Source Forms.
+
+You may convey a covered work in object code form under the terms of
+sections 4 and 5, provided that you also convey the machine-readable
+Corresponding Source under the terms of this License, in one of these
+ways:
+
+-   a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+-   b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the Corresponding
+    Source from a network server at no charge.
+-   c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source. This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+-   d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge. You need not require recipients to copy the
+    Corresponding Source along with the object code. If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source. Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+-   e) Convey the object code using peer-to-peer transmission,
+    provided you inform other peers where the object code and
+    Corresponding Source of the work are being offered to the general
+    public at no charge under subsection 6d.
+
+A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal,
+family, or household purposes, or (2) anything designed or sold for
+incorporation into a dwelling. In determining whether a product is a
+consumer product, doubtful cases shall be resolved in favor of
+coverage. For a particular product received by a particular user,
+"normally used" refers to a typical or common use of that class of
+product, regardless of the status of the particular user or of the way
+in which the particular user actually uses, or expects or is expected
+to use, the product. A product is a consumer product regardless of
+whether the product has substantial commercial, industrial or
+non-consumer uses, unless such uses represent the only significant
+mode of use of the product.
+
+"Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to
+install and execute modified versions of a covered work in that User
+Product from a modified version of its Corresponding Source. The
+information must suffice to ensure that the continued functioning of
+the modified object code is in no case prevented or interfered with
+solely because modification has been made.
+
+If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information. But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or
+updates for a work that has been modified or installed by the
+recipient, or for the User Product in which it has been modified or
+installed. Access to a network may be denied when the modification
+itself materially and adversely affects the operation of the network
+or violates the rules and protocols for communication across the
+network.
+
+Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+#### 7. Additional Terms.
+
+"Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law. If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it. (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.) You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders
+of that material) supplement the terms of this License with terms:
+
+-   a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+-   b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+-   c) Prohibiting misrepresentation of the origin of that material,
+    or requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+-   d) Limiting the use for publicity purposes of names of licensors
+    or authors of the material; or
+-   e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+-   f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions
+    of it) with contractual assumptions of liability to the recipient,
+    for any liability that these contractual assumptions directly
+    impose on those licensors and authors.
+
+All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10. If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term. If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions; the
+above requirements apply either way.
+
+#### 8. Termination.
+
+You may not propagate or modify a covered work except as expressly
+provided under this License. Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+However, if you cease all violation of this License, then your license
+from a particular copyright holder is reinstated (a) provisionally,
+unless and until the copyright holder explicitly and finally
+terminates your license, and (b) permanently, if the copyright holder
+fails to notify you of the violation by some reasonable means prior to
+60 days after the cessation.
+
+Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License. If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+#### 9. Acceptance Not Required for Having Copies.
+
+You are not required to accept this License in order to receive or run
+a copy of the Program. Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance. However,
+nothing other than this License grants you permission to propagate or
+modify any covered work. These actions infringe copyright if you do
+not accept this License. Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+#### 10. Automatic Licensing of Downstream Recipients.
+
+Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License. You are not responsible
+for enforcing compliance by third parties with this License.
+
+An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations. If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License. For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+#### 11. Patents.
+
+A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based. The
+work thus licensed is called the contributor's "contributor version".
+
+A contributor's "essential patent claims" are all patent claims owned
+or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version. For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement). To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients. "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+A patent license is "discriminatory" if it does not include within the
+scope of its coverage, prohibits the exercise of, or is conditioned on
+the non-exercise of one or more of the rights that are specifically
+granted under this License. You may not convey a covered work if you
+are a party to an arrangement with a third party that is in the
+business of distributing software, under which you make payment to the
+third party based on the extent of your activity of conveying the
+work, and under which the third party grants, to any of the parties
+who would receive the covered work from you, a discriminatory patent
+license (a) in connection with copies of the covered work conveyed by
+you (or copies made from those copies), or (b) primarily for and in
+connection with specific products or compilations that contain the
+covered work, unless you entered into that arrangement, or that patent
+license was granted, prior to 28 March 2007.
+
+Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+#### 12. No Surrender of Others' Freedom.
+
+If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License. If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under
+this License and any other pertinent obligations, then as a
+consequence you may not convey it at all. For example, if you agree to
+terms that obligate you to collect a royalty for further conveying
+from those to whom you convey the Program, the only way you could
+satisfy both those terms and this License would be to refrain entirely
+from conveying the Program.
+
+#### 13. Use with the GNU Affero General Public License.
+
+Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU Affero General Public License into a single
+combined work, and to convey the resulting work. The terms of this
+License will continue to apply to the part which is the covered work,
+but the special requirements of the GNU Affero General Public License,
+section 13, concerning interaction through a network will apply to the
+combination as such.
+
+#### 14. Revised Versions of this License.
+
+The Free Software Foundation may publish revised and/or new versions
+of the GNU General Public License from time to time. Such new versions
+will be similar in spirit to the present version, but may differ in
+detail to address new problems or concerns.
+
+Each version is given a distinguishing version number. If the Program
+specifies that a certain numbered version of the GNU General Public
+License "or any later version" applies to it, you have the option of
+following the terms and conditions either of that numbered version or
+of any later version published by the Free Software Foundation. If the
+Program does not specify a version number of the GNU General Public
+License, you may choose any version ever published by the Free
+Software Foundation.
+
+If the Program specifies that a proxy can decide which future versions
+of the GNU General Public License can be used, that proxy's public
+statement of acceptance of a version permanently authorizes you to
+choose that version for the Program.
+
+Later license versions may give you additional or different
+permissions. However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+#### 15. Disclaimer of Warranty.
+
+THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT
+WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND
+PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE
+DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR
+CORRECTION.
+
+#### 16. Limitation of Liability.
+
+IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR
+CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES
+ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT
+NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR
+LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM
+TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER
+PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
+
+#### 17. Interpretation of Sections 15 and 16.
+
+If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+END OF TERMS AND CONDITIONS
+
+### How to Apply These Terms to Your New Programs
+
+If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these
+terms.
+
+To do so, attach the following notices to the program. It is safest to
+attach them to the start of each source file to most effectively state
+the exclusion of warranty; and each file should have at least the
+"copyright" line and a pointer to where the full notice is found.
+
+        <one line to give the program's name and a brief idea of what it does.>
+        Copyright (C) <year>  <name of author>
+
+        This program is free software: you can redistribute it and/or modify
+        it under the terms of the GNU General Public License as published by
+        the Free Software Foundation, either version 3 of the License, or
+        (at your option) any later version.
+
+        This program is distributed in the hope that it will be useful,
+        but WITHOUT ANY WARRANTY; without even the implied warranty of
+        MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+        GNU General Public License for more details.
+
+        You should have received a copy of the GNU General Public License
+        along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper
+mail.
+
+If the program does terminal interaction, make it output a short
+notice like this when it starts in an interactive mode:
+
+        <program>  Copyright (C) <year>  <name of author>
+        This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+        This is free software, and you are welcome to redistribute it
+        under certain conditions; type `show c' for details.
+
+The hypothetical commands \`show w' and \`show c' should show the
+appropriate parts of the General Public License. Of course, your
+program's commands might be different; for a GUI interface, you would
+use an "about box".
+
+You should also get your employer (if you work as a programmer) or
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary. For more information on this, and how to apply and follow
+the GNU GPL, see <https://www.gnu.org/licenses/>.
+
+The GNU General Public License does not permit incorporating your
+program into proprietary programs. If your program is a subroutine
+library, you may consider it more useful to permit linking proprietary
+applications with the library. If this is what you want to do, use the
+GNU Lesser General Public License instead of this License. But first,
+please read <https://www.gnu.org/licenses/why-not-lgpl.html>.

Makefile

@@ -0,0 +1 @@
+# For additional configs/scripting

README.md

@@ -1,208 +1,33 @@
-# mempool
-## a mempool visualizer and explorer for Bitcoin
+# The Mempool Open Source Project™ [![mempool](https://img.shields.io/endpoint?url=https://dashboard.cypress.io/badge/simple/ry4br7/master&style=flat-square)](https://dashboard.cypress.io/projects/ry4br7/runs)
 
-![mempool](https://pbs.twimg.com/media/EAETXWCU4AAv2v-?format=jpg&name=4096x4096)
-![blockchain](https://pbs.twimg.com/media/EAETXWAU8AAj4IP?format=jpg&name=4096x4096)
+Mempool is the fully-featured mempool visualizer, explorer, and API service running at [mempool.space](https://mempool.space/). 
 
-## Pick the right version for your use case
+It is an open-source project developed and operated for the benefit of the Bitcoin community, with a focus on the emerging transaction fee market that is evolving Bitcoin into a multi-layer ecosystem.
 
-Mempool V1 has basic explorer functionality and can run from a Bitcoin Core full node on a Raspberry Pi (no pruning, txindex=1).
+![mempool](https://mempool.space/resources/screenshots/v2.4.0-dashboard.png)
 
-Mempool V2 is what runs on https://mempool.space and has advanced explorer functionality, but requires a fully synced electrs backend running on powerful server hardware.
+# Installation Methods
 
-# Mempool V1 using Docker (easy)
+Mempool can be self-hosted on a wide variety of your own hardware, ranging from a simple one-click installation on a Raspberry Pi full-node distro all the way to a robust production instance on a powerful FreeBSD server. 
 
-Install from Docker Hub, passing your Bitcoin Core RPC credentials as environment variables:
+**Most people should use a one-click install method.** Other install methods are meant for developers and others with experience managing servers. 
 
-```bash
-docker pull mempool/mempool:v1.0
-docker create -p 80:80 -e BITCOIN_NODE_HOST=192.168.1.102 -e BITCOIN_NODE_USER=foo -e BITCOIN_NODE_PASS=bar --name mempool mempool/mempool:v1.0
-docker start mempool
-docker logs mempool
-```
+<a id="one-click-installation"></a>
+## One-Click Installation
 
-You should see mempool starting up, which takes over an hour (needs 8 blocks). When it's ready, visit http://127.0.0.1/ to see your mempool.
+Mempool can be conveniently installed on the following full-node distros: 
+- [Umbrel](https://github.com/getumbrel/umbrel)
+- [RaspiBlitz](https://github.com/rootzoll/raspiblitz)
+- [RoninDojo](https://code.samourai.io/ronindojo/RoninDojo)
+- [myNode](https://github.com/mynodebtc/mynode)
+- [Start9](https://github.com/Start9Labs/embassy-os)
 
-# Mempool V1 not using Docker (advanced)
+**We highly recommend you deploy your own Mempool instance this way.** No matter which option you pick, you'll be able to get your own fully-sovereign instance of Mempool up quickly without needing to fiddle with any settings.
 
-## Dependencies
+## Advanced Installation Methods
 
-* Bitcoin (full node required, no pruning, txindex=1)
-* NodeJS (official stable LTS)
-* MySQL or MariaDB (default config)
-* Nginx (use supplied nginx.conf)
+Mempool can be installed in other ways too, but we only recommend doing so if you're a developer, have experience managing servers, or otherwise know what you're doing.
 
-## Checking out release tag
-```bash
-  git clone https://github.com/mempool-space/mempool.space
-  cd mempool.space
-  git checkout v1.0.0 # put latest release tag here
-```
-
-## Bitcoin Core (bitcoind)
-
-Enable RPC and txindex in bitcoin.conf
-
-```bash
-  rpcuser=mempool
-  rpcpassword=71b61986da5b03a5694d7c7d5165ece5
-  txindex=1
-```
-
-## NodeJS
-
-Install dependencies and build code:
-
-```bash
-  # Install TypeScript Globally
-  npm install -g typescript
-
-  # Frontend
-  cd frontend
-  npm install
-  npm run build
-
-  # Backend
-  cd ../backend/
-  npm install
-  npm run build
-```
-
-## Mempool Configuration
-In the `backend` folder, make a copy of the sample config and modify it to fit your settings.
-
-```bash
-  cp mempool-config.sample.json mempool-config.json
-```
-
-Edit `mempool-config.json` to add your Bitcoin Core node RPC credentials:
-```bash
-  "BITCOIN_NODE_HOST": "192.168.1.5",
-  "BITCOIN_NODE_PORT": 8332,
-  "BITCOIN_NODE_USER": "mempool",
-  "BITCOIN_NODE_PASS": "71b61986da5b03a5694d7c7d5165ece5",
-```
-
-## MySQL
-
-Install MariaDB:
-
-```bash
-  # Linux
-  apt-get install mariadb-server mariadb-client
-
-  # macOS
-  brew install mariadb
-  brew services start mariadb
-```
-
-Create database and grant privileges:
-```bash
-  MariaDB [(none)]> drop database mempool;
-  Query OK, 0 rows affected (0.00 sec)
-
-  MariaDB [(none)]> create database mempool;
-  Query OK, 1 row affected (0.00 sec)
-
-  MariaDB [(none)]> grant all privileges on mempool.* to 'mempool' identified by 'mempool';
-  Query OK, 0 rows affected (0.00 sec)
-```
-
-From the root folder, initialize database structure:
-
-```bash
-  mysql -u mempool -p mempool < mariadb-structure.sql
-```
-
-## Running (Backend)
-
-Create an initial empty cache and start the app:
-
-```bash
-  touch cache.json
-  npm run start # node dist/index.js
-```
-
-After starting you should see:
-
-```bash
-  Server started on port 8999 :)
-  New block found (#586498)! 0 of 1986 found in mempool. 1985 not found.
-  New block found (#586499)! 0 of 1094 found in mempool. 1093 not found.
-  New block found (#586500)! 0 of 2735 found in mempool. 2734 not found.
-  New block found (#586501)! 0 of 2675 found in mempool. 2674 not found.
-  New block found (#586502)! 0 of 975 found in mempool. 974 not found.
-  New block found (#586503)! 0 of 2130 found in mempool. 2129 not found.
-  New block found (#586504)! 0 of 2770 found in mempool. 2769 not found.
-  New block found (#586505)! 0 of 2759 found in mempool. 2758 not found.
-  Updating mempool
-  Calculated fee for transaction 1 / 3257
-  Calculated fee for transaction 2 / 3257
-  Calculated fee for transaction 3 / 3257
-  Calculated fee for transaction 4 / 3257
-  Calculated fee for transaction 5 / 3257
-  Calculated fee for transaction 6 / 3257
-  Calculated fee for transaction 7 / 3257
-  Calculated fee for transaction 8 / 3257
-  Calculated fee for transaction 9 / 3257
-```
-You need to wait for at least *8 blocks to be mined*, so please wait ~80 minutes.
-The backend also needs to index transactions, calculate fees, etc.
-When it's ready you will see output like this:
-
-```bash
-  Mempool updated in 0.189 seconds
-  Updating mempool
-  Mempool updated in 0.096 seconds
-  Updating mempool
-  Mempool updated in 0.099 seconds
-  Updating mempool
-  Calculated fee for transaction 1 / 10
-  Calculated fee for transaction 2 / 10
-  Calculated fee for transaction 3 / 10
-  Calculated fee for transaction 4 / 10
-  Calculated fee for transaction 5 / 10
-  Calculated fee for transaction 6 / 10
-  Calculated fee for transaction 7 / 10
-  Calculated fee for transaction 8 / 10
-  Calculated fee for transaction 9 / 10
-  Calculated fee for transaction 10 / 10
-  Mempool updated in 0.243 seconds
-  Updating mempool
-```
-
-## nginx + CertBot (LetsEncrypt)
-Setup nginx using the supplied nginx.conf
-
-```bash
-  # install nginx and certbot
-  apt-get install -y nginx python-certbot-nginx
-
-  # replace example.com with your domain name
-  certbot --nginx -d example.com
-
-  # install the mempool configuration for nginx
-  cp nginx.conf /etc/nginx/nginx.conf
-
-  # edit the installed nginx.conf, and replace all
-  # instances of example.com with your domain name
-```
-Make sure you can access https://<your-domain-name>/ in browser before proceeding
-
-
-## Running (Frontend)
-
-Build the frontend static HTML/CSS/JS, rsync the output into nginx folder:
-
-```bash
-  cd frontend/
-  npm run build
-  sudo rsync -av --delete dist/mempool/ /var/www/html/
-```
-
-## Try It Out
-
-If everything went okay you should see the beautiful mempool :grin:
-
-If you get stuck on "loading blocks", this means the websocket can't connect.
-Check your nginx proxy setup, firewalls, etc. and open an issue if you need help.
+- See the [`docker/`](./docker/) directory for instructions on deploying Mempool with Docker.
+- See the [`backend/`](./backend/) and [`frontend/`](./frontend/) directories for manual install instructions oriented for developers and small-scale deployments.
+- See the [`production/`](./production/) directory for guidance on setting up a more serious Mempool instance designed for high performance at scale.

backend/.gitignore

@@ -1,7 +1,10 @@
 # See http://help.github.com/ignore-files/ for more about ignoring files.
 
-# production config
-mempool-config.json
+# production config and external assets
+*.json
+!mempool-config.sample.json
+
+icons.json
 
 # compiled output
 /dist
@@ -41,5 +44,3 @@ testem.log
 #System Files
 .DS_Store
 Thumbs.db
-
-cache.json

backend/.vscode/settings.json

@@ -0,0 +1,4 @@
+{
+  "editor.tabSize": 2,
+  "typescript.tsdk": "../backend/node_modules/typescript/lib"
+}

backend/README.md

@@ -0,0 +1,161 @@
+# Mempool Backend
+
+These instructions are mostly intended for developers, but can be used as a basis for personal or small-scale production setups. 
+
+If you choose to use these instructions for a production setup, be aware that you will still probably need to do additional configuration for your specific OS, environment, use-case, etc. We do our best here to provide a good starting point, but only proceed if you know what you're doing. Mempool does not provide support for custom setups.
+
+See other ways to set up Mempool on [the main README](/../../#installation-methods).
+
+Jump to a section in this doc:
+- [Set Up the Backend](#setup)
+- [Development Tips](#development-tips)
+
+## Setup
+
+### 1. Clone Mempool Repository
+
+Get the latest Mempool code:
+
+```
+git clone https://github.com/mempool/mempool
+cd mempool
+```
+
+Check out the latest release:
+
+```
+latestrelease=$(curl -s https://api.github.com/repos/mempool/mempool/releases/latest|grep tag_name|head -1|cut -d '"' -f4)
+git checkout $latestrelease
+```
+
+### 2. Configure Bitcoin Core
+
+Turn on `txindex`, enable RPC, and set RPC credentials in `bitcoin.conf`:
+
+```
+txindex=1
+server=1
+rpcuser=mempool
+rpcpassword=mempool
+```
+
+### 3. Configure Electrum Server
+
+[Pick an Electrum Server implementation](https://mempool.space/docs/faq#address-lookup-issues), configure it, and make sure it's synced.
+
+**This step is optional.** You can run Mempool without configuring an Electrum Server for it, but address lookups will be disabled.
+
+### 4. Configure MariaDB
+
+_Mempool needs MariaDB v10.5 or later. If you already have MySQL installed, make sure to migrate any existing databases **before** installing MariaDB._
+
+Get MariaDB from your operating system's package manager:
+
+```
+# Debian, Ubuntu, etc.
+apt-get install mariadb-server mariadb-client
+
+# macOS
+brew install mariadb
+mysql.server start
+```
+
+Create a database and grant privileges:
+
+```
+MariaDB [(none)]> drop database mempool;
+Query OK, 0 rows affected (0.00 sec)
+
+MariaDB [(none)]> create database mempool;
+Query OK, 1 row affected (0.00 sec)
+
+MariaDB [(none)]> grant all privileges on mempool.* to 'mempool'@'%' identified by 'mempool';
+Query OK, 0 rows affected (0.00 sec)
+```
+
+### 5. Prepare Mempool Backend
+
+#### Build
+
+_Make sure to use Node.js 16.15 and npm 7._
+
+Install dependencies with `npm` and build the backend:
+
+```
+cd backend
+npm install # add --prod for production
+npm run build
+```
+
+#### Configure
+
+In the backend folder, make a copy of the sample config file:
+
+```
+cp mempool-config.sample.json mempool-config.json
+```
+
+Edit `mempool-config.json` as needed. 
+
+In particular, make sure:
+- the correct Bitcoin Core RPC credentials are specified in `CORE_RPC`
+- the correct `BACKEND` is specified in `MEMPOOL`:
+  - "electrum" if you're using [romanz/electrs](https://github.com/romanz/electrs) or [cculianu/Fulcrum](https://github.com/cculianu/Fulcrum)
+  - "esplora" if you're using [Blockstream/electrs](https://github.com/Blockstream/electrs)
+  - "none" if you're not using any Electrum Server
+
+### 6. Run Mempool Backend
+
+Run the Mempool backend:
+
+```
+npm run start
+```
+
+When it's running, you should see output like this:
+
+```
+Mempool updated in 0.189 seconds
+Updating mempool
+Mempool updated in 0.096 seconds
+Updating mempool
+Mempool updated in 0.099 seconds
+Updating mempool
+Calculated fee for transaction 1 / 10
+Calculated fee for transaction 2 / 10
+Calculated fee for transaction 3 / 10
+Calculated fee for transaction 4 / 10
+Calculated fee for transaction 5 / 10
+Calculated fee for transaction 6 / 10
+Calculated fee for transaction 7 / 10
+Calculated fee for transaction 8 / 10
+Calculated fee for transaction 9 / 10
+Calculated fee for transaction 10 / 10
+Mempool updated in 0.243 seconds
+Updating mempool
+```
+
+### 7. Set Up Mempool Frontend
+With the backend configured and running, proceed to set up the [Mempool frontend](../frontend#manual-setup).
+
+## Development Tips
+
+### Set Up Backend Watchers
+
+The Mempool backend is static. TypeScript scripts are compiled into the `dist` folder and served through a Node.js web server. 
+
+As a result, for development purposes, you may find it helpful to set up backend watchers to avoid the manual shutdown/recompile/restart command-line cycle.
+
+First, install `nodemon` and `ts-node`:
+
+```
+npm install -g ts-node nodemon
+```
+
+Then, run the watcher:
+
+```
+nodemon src/index.ts --ignore cache/ --ignore pools.json
+```
+
+`nodemon` should be in npm's global binary folder. If needed, you can determine where that is with `npm -g bin`.

backend/cache/.gitignore

@@ -0,0 +1 @@
+*.json

backend/mempool-config.sample.json

@@ -1,25 +1,88 @@
 {
-  "ENV": "dev",
-  "DB_HOST": "localhost",
-  "DB_PORT": 3306,
-  "DB_USER": "mempool",
-  "DB_PASSWORD": "mempool",
-  "DB_DATABASE": "mempool",
-  "HTTP_PORT": 3000,
-  "API_ENDPOINT": "/api/v1/",
-  "CHAT_SSL_ENABLED": false,
-  "CHAT_SSL_PRIVKEY": "",
-  "CHAT_SSL_CERT": "",
-  "CHAT_SSL_CHAIN": "",
-  "MEMPOOL_REFRESH_RATE_MS": 500,
-  "INITIAL_BLOCK_AMOUNT": 8,
-  "DEFAULT_PROJECTED_BLOCKS_AMOUNT": 3,
-  "KEEP_BLOCK_AMOUNT": 24,
-  "BITCOIN_NODE_HOST": "localhost",
-  "BITCOIN_NODE_PORT": 8332,
-  "BITCOIN_NODE_USER": "",
-  "BITCOIN_NODE_PASS": "",
-  "BACKEND_API": "bitcoind",
-  "ELECTRS_API_URL": "https://mempool.space/api",
-  "TX_PER_SECOND_SPAN_SECONDS": 150
+  "MEMPOOL": {
+    "NETWORK": "mainnet",
+    "BACKEND": "electrum",
+    "HTTP_PORT": 8999,
+    "SPAWN_CLUSTER_PROCS": 0,
+    "API_URL_PREFIX": "/api/v1/",
+    "POLL_RATE_MS": 2000,
+    "CACHE_DIR": "./cache",
+    "CLEAR_PROTECTION_MINUTES": 20,
+    "RECOMMENDED_FEE_PERCENTILE": 50,
+    "BLOCK_WEIGHT_UNITS": 4000000,
+    "INITIAL_BLOCKS_AMOUNT": 8,
+    "MEMPOOL_BLOCKS_AMOUNT": 8,
+    "INDEXING_BLOCKS_AMOUNT": 11000,
+    "PRICE_FEED_UPDATE_INTERVAL": 600,
+    "USE_SECOND_NODE_FOR_MINFEE": false,
+    "EXTERNAL_ASSETS": [],
+    "EXTERNAL_MAX_RETRY": 1,
+    "EXTERNAL_RETRY_INTERVAL": 0,
+    "USER_AGENT": "mempool",
+    "STDOUT_LOG_MIN_PRIORITY": "debug"
+  },
+  "CORE_RPC": {
+    "HOST": "127.0.0.1",
+    "PORT": 8332,
+    "USERNAME": "mempool",
+    "PASSWORD": "mempool"
+  },
+  "ELECTRUM": {
+    "HOST": "127.0.0.1",
+    "PORT": 50002,
+    "TLS_ENABLED": true
+  },
+  "ESPLORA": {
+    "REST_API_URL": "http://127.0.0.1:3000"
+  },
+  "SECOND_CORE_RPC": {
+    "HOST": "127.0.0.1",
+    "PORT": 8332,
+    "USERNAME": "mempool",
+    "PASSWORD": "mempool"
+  },
+  "DATABASE": {
+    "ENABLED": true,
+    "HOST": "127.0.0.1",
+    "PORT": 3306,
+    "SOCKET": "/var/run/mysql/mysql.sock",
+    "DATABASE": "mempool",
+    "USERNAME": "mempool",
+    "PASSWORD": "mempool"
+  },
+  "SYSLOG": {
+    "ENABLED": true,
+    "HOST": "127.0.0.1",
+    "PORT": 514,
+    "MIN_PRIORITY": "info",
+    "FACILITY": "local7"
+  },
+  "STATISTICS": {
+    "ENABLED": true,
+    "TX_PER_SECOND_SAMPLE_PERIOD": 150
+  },
+  "BISQ": {
+    "ENABLED": false,
+    "DATA_PATH": "/bisq/statsnode-data/btc_mainnet/db"
+  },
+  "SOCKS5PROXY": {
+    "ENABLED": false,
+    "USE_ONION": true,
+    "HOST": "127.0.0.1",
+    "PORT": 9050,
+    "USERNAME": "",
+    "PASSWORD": ""
+  },
+  "PRICE_DATA_SERVER": {
+    "TOR_URL": "http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices",
+    "CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices"
+  },
+  "EXTERNAL_DATA_SERVER": {
+    "MEMPOOL_API": "https://mempool.space/api/v1",
+    "MEMPOOL_ONION": "http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1",
+    "LIQUID_API": "https://liquid.network/api/v1",
+    "LIQUID_ONION": "http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1",
+    "BISQ_URL": "https://bisq.markets/api",
+    "BISQ_ONION": "http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api"
+  }
 }

backend/package.json

@@ -1,31 +1,48 @@
 {
   "name": "mempool-backend",
-  "version": "1.0.0",
-  "description": "Bitcoin Mempool Visualizer",
+  "version": "2.4.0-dev",
+  "description": "Bitcoin mempool visualizer and blockchain explorer backend",
+  "license": "GNU Affero General Public License v3.0",
+  "homepage": "https://mempool.space",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/mempool/mempool"
+  },
+  "bugs": {
+    "url": "https://github.com/mempool/mempool/issues"
+  },
+  "keywords": [
+    "bitcoin",
+    "mempool",
+    "blockchain",
+    "explorer",
+    "liquid"
+  ],
   "main": "index.ts",
   "scripts": {
-    "build": "tsc",
-    "start": "npm run build && node dist/index.js"
+    "ng": "./node_modules/@angular/cli/bin/ng",
+    "tsc": "./node_modules/typescript/bin/tsc",
+    "build": "npm run tsc",
+    "start": "node --max-old-space-size=2048 dist/index.js",
+    "start-production": "node --max-old-space-size=4096 dist/index.js",
+    "test": "echo \"Error: no test specified\" && exit 1"
   },
-  "author": {
-    "name": "Simon Lindh",
-    "url": "https://github.com/mempool-space/mempool.space"
-  },
-  "license": "MIT",
   "dependencies": {
-    "bitcoin": "^3.0.1",
-    "compression": "^1.7.3",
-    "express": "^4.16.3",
-    "mysql2": "^1.6.1",
-    "request": "^2.88.0",
-    "ws": "^6.0.0"
+    "@mempool/electrum-client": "^1.1.7",
+    "axios": "~0.27.2",
+    "bitcoinjs-lib": "6.0.1",
+    "crypto-js": "^4.0.0",
+    "express": "^4.18.0",
+    "mysql2": "2.3.3",
+    "node-worker-threads-pool": "^1.5.1",
+    "socks-proxy-agent": "^6.2.0",
+    "typescript": "~4.7.2",
+    "ws": "~8.7.0"
   },
   "devDependencies": {
-    "@types/express": "^4.16.0",
-    "@types/mysql2": "github:types/mysql2",
-    "@types/request": "^2.48.2",
-    "@types/ws": "^6.0.1",
-    "tslint": "^5.11.0",
-    "typescript": "^3.1.1"
+    "@types/compression": "^1.7.2",
+    "@types/ws": "~8.5.3",
+    "@types/express": "^4.17.13",
+    "tslint": "^6.1.0"
   }
 }

backend/src/api/backend-info.ts

@@ -0,0 +1,61 @@
+import * as fs from 'fs';
+import * as os from 'os';
+import logger from '../logger';
+import { IBackendInfo } from '../mempool.interfaces';
+const { spawnSync } = require('child_process');
+
+class BackendInfo {
+  private gitCommitHash = '';
+  private hostname = '';
+  private version = '';
+
+  constructor() {
+    this.setLatestCommitHash();
+    this.setVersion();
+    this.hostname = os.hostname();
+  }
+
+  public getBackendInfo(): IBackendInfo {
+    return {
+      hostname: this.hostname,
+      gitCommit: this.gitCommitHash,
+      version: this.version,
+    };
+  }
+
+  public getShortCommitHash() {
+    return this.gitCommitHash.slice(0, 7);
+  }
+
+  private setLatestCommitHash(): void {
+    //TODO: share this logic with `generate-config.js`
+    if (process.env.DOCKER_COMMIT_HASH) {
+      this.gitCommitHash = process.env.DOCKER_COMMIT_HASH;
+    } else {
+      try {
+        const gitRevParse = spawnSync('git', ['rev-parse', '--short', 'HEAD']);
+        if (!gitRevParse.error) {
+          const output = gitRevParse.stdout.toString('utf-8').replace(/[\n\r\s]+$/, '');
+          this.gitCommitHash = output ? output : '?';
+        } else if (gitRevParse.error.code === 'ENOENT') {
+          console.log('git not found, cannot parse git hash');
+          this.gitCommitHash = '?';
+        }
+      } catch (e: any) {
+        console.log('Could not load git commit info: ' + e.message);
+        this.gitCommitHash = '?';
+      }
+    }
+  }
+
+  private setVersion(): void {
+    try {
+      const packageJson = fs.readFileSync('package.json').toString();
+      this.version = JSON.parse(packageJson).version;
+    } catch (e) {
+      throw new Error(e instanceof Error ? e.message : 'Error');
+    }
+  }
+}
+
+export default new BackendInfo();

backend/src/api/bisq/bisq.ts

@@ -0,0 +1,364 @@
+import config from '../../config';
+import * as fs from 'fs';
+import axios, { AxiosResponse } from 'axios';
+import * as http from 'http';
+import * as https from 'https';
+import { SocksProxyAgent } from 'socks-proxy-agent';
+import { BisqBlocks, BisqBlock, BisqTransaction, BisqStats, BisqTrade } from './interfaces';
+import { Common } from '../common';
+import { BlockExtended } from '../../mempool.interfaces';
+import { StaticPool } from 'node-worker-threads-pool';
+import backendInfo from '../backend-info';
+import logger from '../../logger';
+
+class Bisq {
+  private static BLOCKS_JSON_FILE_PATH = config.BISQ.DATA_PATH + '/json/all/blocks.json';
+  private latestBlockHeight = 0;
+  private blocks: BisqBlock[] = [];
+  private allBlocks: BisqBlock[] = [];
+  private transactions: BisqTransaction[] = [];
+  private transactionIndex: { [txId: string]: BisqTransaction } = {};
+  private blockIndex: { [hash: string]: BisqBlock } = {};
+  private addressIndex: { [address: string]: BisqTransaction[] } = {};
+  private stats: BisqStats = {
+    minted: 0,
+    burnt: 0,
+    addresses: 0,
+    unspent_txos: 0,
+    spent_txos: 0,
+  };
+  private price: number = 0;
+  private priceUpdateCallbackFunction: ((price: number) => void) | undefined;
+  private topDirectoryWatcher: fs.FSWatcher | undefined;
+  private subdirectoryWatcher: fs.FSWatcher | undefined;
+  private jsonParsePool = new StaticPool({
+    size: 4,
+    task: (blob: string) => JSON.parse(blob),
+  });
+
+  constructor() {}
+
+  startBisqService(): void {
+    try {
+      this.checkForBisqDataFolder();
+    } catch (e) {
+      logger.info('Retrying to start bisq service in 3 minutes');
+      setTimeout(this.startBisqService.bind(this), 180000);
+      return;
+    }
+    this.loadBisqDumpFile();
+    setInterval(this.updatePrice.bind(this), 1000 * 60 * 60);
+    this.updatePrice();
+    this.startTopDirectoryWatcher();
+    this.startSubDirectoryWatcher();
+  }
+
+  handleNewBitcoinBlock(block: BlockExtended): void {
+    if (block.height - 10 > this.latestBlockHeight && this.latestBlockHeight !== 0) {
+      logger.warn(`Bitcoin block height (#${block.height}) has diverged from the latest Bisq block height (#${this.latestBlockHeight}). Restarting watchers...`);
+      this.startTopDirectoryWatcher();
+      this.startSubDirectoryWatcher();
+    }
+  }
+
+  getTransaction(txId: string): BisqTransaction | undefined {
+    return this.transactionIndex[txId];
+  }
+
+  getTransactions(start: number, length: number, types: string[]): [BisqTransaction[], number] {
+    let transactions = this.transactions;
+    if (types.length) {
+      transactions = transactions.filter((tx) => types.indexOf(tx.txType) > -1);
+    }
+    return [transactions.slice(start, length + start), transactions.length];
+  }
+
+  getBlock(hash: string): BisqBlock | undefined {
+    return this.blockIndex[hash];
+  }
+
+  getAddress(hash: string): BisqTransaction[] {
+    return this.addressIndex[hash];
+  }
+
+  getBlocks(start: number, length: number): [BisqBlock[], number] {
+    return [this.blocks.slice(start, length + start), this.blocks.length];
+  }
+
+  getStats(): BisqStats {
+    return this.stats;
+  }
+
+  setPriceCallbackFunction(fn: (price: number) => void) {
+    this.priceUpdateCallbackFunction = fn;
+  }
+
+  getLatestBlockHeight(): number {
+    return this.latestBlockHeight;
+  }
+
+  private checkForBisqDataFolder() {
+    if (!fs.existsSync(Bisq.BLOCKS_JSON_FILE_PATH)) {
+      logger.warn(Bisq.BLOCKS_JSON_FILE_PATH + ` doesn't exist. Make sure Bisq is running and the config is correct before starting the server.`);
+      throw new Error(`Cannot load BISQ ${Bisq.BLOCKS_JSON_FILE_PATH} file`);
+    }
+  }
+
+  private startTopDirectoryWatcher() {
+    if (this.topDirectoryWatcher) {
+      this.topDirectoryWatcher.close();
+    }
+    let fsWait: NodeJS.Timeout | null = null;
+    this.topDirectoryWatcher = fs.watch(config.BISQ.DATA_PATH + '/json', () => {
+      if (fsWait) {
+        clearTimeout(fsWait);
+      }
+      if (this.subdirectoryWatcher) {
+        this.subdirectoryWatcher.close();
+      }
+      fsWait = setTimeout(() => {
+        logger.debug(`Bisq restart detected. Resetting both watchers in 3 minutes.`);
+        setTimeout(() => {
+          this.startTopDirectoryWatcher();
+          this.startSubDirectoryWatcher();
+          this.loadBisqDumpFile();
+        }, 180000);
+      }, 15000);
+    });
+  }
+
+  private startSubDirectoryWatcher() {
+    if (this.subdirectoryWatcher) {
+      this.subdirectoryWatcher.close();
+    }
+    if (!fs.existsSync(Bisq.BLOCKS_JSON_FILE_PATH)) {
+      logger.warn(Bisq.BLOCKS_JSON_FILE_PATH + ` doesn't exist. Trying to restart sub directory watcher again in 3 minutes.`);
+      setTimeout(() => this.startSubDirectoryWatcher(), 180000);
+      return;
+    }
+    let fsWait: NodeJS.Timeout | null = null;
+    this.subdirectoryWatcher = fs.watch(config.BISQ.DATA_PATH + '/json/all', () => {
+      if (fsWait) {
+        clearTimeout(fsWait);
+      }
+      fsWait = setTimeout(() => {
+        logger.debug(`Change detected in the Bisq data folder.`);
+        this.loadBisqDumpFile();
+      }, 2000);
+    });
+  }
+  private async updatePrice() {
+    type axiosOptions = {
+      headers: {
+        'User-Agent': string
+      };
+      timeout: number;
+      httpAgent?: http.Agent;
+      httpsAgent?: https.Agent;
+    }
+    const setDelay = (secs: number = 1): Promise<void> => new Promise(resolve => setTimeout(() => resolve(), secs * 1000));
+    const BISQ_URL = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.EXTERNAL_DATA_SERVER.BISQ_ONION : config.EXTERNAL_DATA_SERVER.BISQ_URL;
+    const isHTTP = (new URL(BISQ_URL).protocol.split(':')[0] === 'http') ? true : false;
+    const axiosOptions: axiosOptions = {
+      headers: {
+        'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
+      },
+      timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000
+    };
+    let retry = 0;
+
+    while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
+      try {
+        if (config.SOCKS5PROXY.ENABLED) {
+          const socksOptions: any = {
+            agentOptions: {
+              keepAlive: true,
+            },
+            hostname: config.SOCKS5PROXY.HOST,
+            port: config.SOCKS5PROXY.PORT
+          };
+
+          if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
+            socksOptions.username = config.SOCKS5PROXY.USERNAME;
+            socksOptions.password = config.SOCKS5PROXY.PASSWORD;
+          } else {
+            // Retry with different tor circuits https://stackoverflow.com/a/64960234
+            socksOptions.username = `circuit${retry}`;
+          }
+
+          // Handle proxy agent for onion addresses
+          if (isHTTP) {
+            axiosOptions.httpAgent = new SocksProxyAgent(socksOptions);
+          } else {
+            axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions);
+          }
+        }
+        
+        const data: AxiosResponse = await axios.get(`${BISQ_URL}/trades/?market=bsq_btc`, axiosOptions);
+        if (data.statusText === 'error' || !data.data) {
+          throw new Error(`Could not fetch data from Bisq market, Error: ${data.status}`);
+        }
+        const prices: number[] = [];
+        data.data.forEach((trade) => {
+          prices.push(parseFloat(trade.price) * 100000000);
+        });
+        prices.sort((a, b) => a - b);
+        this.price = Common.median(prices);
+        if (this.priceUpdateCallbackFunction) {
+          this.priceUpdateCallbackFunction(this.price);
+        }
+        logger.debug('Successfully updated Bisq market price');
+        break;
+      } catch (e) {
+        logger.err('Error updating Bisq market price: '  + (e instanceof Error ? e.message : e));
+        await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL);
+        retry++;
+      }
+    }
+  }
+
+  private async loadBisqDumpFile(): Promise<void> {
+    this.allBlocks = [];
+    try {
+      await this.loadData();
+      this.buildIndex();
+      this.calculateStats();
+    } catch (e) {
+      logger.info('Cannot load bisq dump file because: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  private buildIndex() {
+    const start = new Date().getTime();
+    this.transactions = [];
+    this.transactionIndex = {};
+    this.addressIndex = {};
+
+    this.allBlocks.forEach((block) => {
+      /* Build block index */
+      if (!this.blockIndex[block.hash]) {
+        this.blockIndex[block.hash] = block;
+      }
+
+      /* Build transactions index */
+      block.txs.forEach((tx) => {
+        this.transactions.push(tx);
+        this.transactionIndex[tx.id] = tx;
+      });
+    });
+
+    /* Build address index */
+    this.transactions.forEach((tx) => {
+      tx.inputs.forEach((input) => {
+        if (!this.addressIndex[input.address]) {
+          this.addressIndex[input.address] = [];
+        }
+        if (this.addressIndex[input.address].indexOf(tx) === -1) {
+          this.addressIndex[input.address].push(tx);
+        }
+      });
+      tx.outputs.forEach((output) => {
+        if (!this.addressIndex[output.address]) {
+          this.addressIndex[output.address] = [];
+        }
+        if (this.addressIndex[output.address].indexOf(tx) === -1) {
+          this.addressIndex[output.address].push(tx);
+        }
+      });
+    });
+
+    const time = new Date().getTime() - start;
+    logger.debug('Bisq data index rebuilt in ' + time + ' ms');
+  }
+
+  private calculateStats() {
+    let minted = 0;
+    let burned = 0;
+    let unspent = 0;
+    let spent = 0;
+
+    this.transactions.forEach((tx) => {
+      tx.outputs.forEach((output) => {
+        if (output.opReturn) {
+          return;
+        }
+        if (output.txOutputType === 'GENESIS_OUTPUT' || output.txOutputType === 'ISSUANCE_CANDIDATE_OUTPUT' && output.isVerified) {
+          minted += output.bsqAmount;
+        }
+        if (output.isUnspent) {
+          unspent++;
+        } else {
+          spent++;
+        }
+      });
+      burned += tx['burntFee'];
+    });
+
+    this.stats = {
+      addresses: Object.keys(this.addressIndex).length,
+      minted: minted / 100,
+      burnt: burned / 100,
+      spent_txos: spent,
+      unspent_txos: unspent,
+    };
+  }
+
+  private async loadData(): Promise<any> {
+    if (!fs.existsSync(Bisq.BLOCKS_JSON_FILE_PATH)) {
+      throw new Error(Bisq.BLOCKS_JSON_FILE_PATH + ` doesn't exist`);
+    }
+
+    const readline = require('readline');
+    const events = require('events');
+
+    const rl = readline.createInterface({
+      input: fs.createReadStream(Bisq.BLOCKS_JSON_FILE_PATH),
+      crlfDelay: Infinity
+    });
+
+    let blockBuffer = '';
+    let readingBlock = false;
+    let lineCount = 1;
+    const start = new Date().getTime();
+
+    logger.debug('Processing Bisq data dump...');
+
+    rl.on('line', (line) => {
+      if (lineCount === 2) {
+        line = line.replace('  "chainHeight": ', '');
+        this.latestBlockHeight = parseInt(line, 10);
+      }
+
+      if (line === '    {') {
+        readingBlock = true;
+      } else if (line === '    },') {
+        blockBuffer += '}';
+        try {
+          const block: BisqBlock =  JSON.parse(blockBuffer);
+          this.allBlocks.push(block);
+          readingBlock = false;
+          blockBuffer = '';
+        } catch (e) {
+          logger.debug(blockBuffer);
+          throw Error(`Unable to parse Bisq data dump at line ${lineCount}` + (e instanceof Error ? e.message : e));
+        }
+      }
+
+      if (readingBlock === true) {
+        blockBuffer += line;
+      }
+
+      ++lineCount;
+    });
+
+    await events.once(rl, 'close');
+
+    this.allBlocks.reverse();
+    this.blocks = this.allBlocks.filter((block) => block.txs.length > 0);
+
+    const time = new Date().getTime() - start;
+    logger.debug('Bisq dump processed in ' + time + ' ms');
+  }
+}
+
+export default new Bisq();

backend/src/api/bisq/interfaces.ts

@@ -0,0 +1,258 @@
+
+export interface BisqBlocks {
+  chainHeight: number;
+  blocks: BisqBlock[];
+}
+
+export interface BisqBlock {
+  height: number;
+  time: number;
+  hash: string;
+  previousBlockHash: string;
+  txs: BisqTransaction[];
+}
+
+export interface BisqTransaction {
+  txVersion: string;
+  id: string;
+  blockHeight: number;
+  blockHash: string;
+  time: number;
+  inputs: BisqInput[];
+  outputs: BisqOutput[];
+  txType: string;
+  txTypeDisplayString: string;
+  burntFee: number;
+  invalidatedBsq: number;
+  unlockBlockHeight: number;
+}
+
+export interface BisqStats {
+  minted: number;
+  burnt: number;
+  addresses: number;
+  unspent_txos: number;
+  spent_txos: number;
+}
+
+interface BisqInput {
+  spendingTxOutputIndex: number;
+  spendingTxId: string;
+  bsqAmount: number;
+  isVerified: boolean;
+  address: string;
+  time: number;
+}
+
+interface BisqOutput {
+  txVersion: string;
+  txId: string;
+  index: number;
+  bsqAmount: number;
+  btcAmount: number;
+  height: number;
+  isVerified: boolean;
+  burntFee: number;
+  invalidatedBsq: number;
+  address: string;
+  scriptPubKey: BisqScriptPubKey;
+  time: any;
+  txType: string;
+  txTypeDisplayString: string;
+  txOutputType: string;
+  txOutputTypeDisplayString: string;
+  lockTime: number;
+  isUnspent: boolean;
+  spentInfo: SpentInfo;
+  opReturn?: string;
+}
+
+interface BisqScriptPubKey {
+  addresses: string[];
+  asm: string;
+  hex: string;
+  reqSigs?: number;
+  type: string;
+}
+
+interface SpentInfo {
+  height: number;
+  inputIndex: number;
+  txId: string;
+}
+
+export interface BisqTrade {
+  direction: string;
+  price: string;
+  amount: string;
+  volume: string;
+  payment_method: string;
+  trade_id: string;
+  trade_date: number;
+  market?: string;
+}
+
+export interface Currencies { [txid: string]: Currency; }
+
+export interface Currency {
+  code: string;
+  name: string;
+  precision: number;
+
+  _type: string;
+}
+
+export interface Depth { [market: string]: Market; }
+
+interface Market {
+  'buys': string[];
+  'sells': string[];
+}
+
+export interface HighLowOpenClose {
+  period_start: number | string;
+  open: string;
+  high: string;
+  low: string;
+  close: string;
+  volume_left: string;
+  volume_right: string;
+  avg: string;
+}
+
+export interface Markets { [txid: string]: Pair; }
+
+interface Pair {
+  pair: string;
+  lname: string;
+  rname: string;
+  lsymbol: string;
+  rsymbol: string;
+  lprecision: number;
+  rprecision: number;
+  ltype: string;
+  rtype: string;
+  name: string;
+}
+
+export interface Offers { [market: string]: OffersMarket; }
+
+interface OffersMarket {
+  buys: Offer[] | null;
+  sells: Offer[] | null;
+}
+
+export interface OffersData {
+  direction: string;
+  currencyCode: string;
+  minAmount: number;
+  amount: number;
+  price: number;
+  date: number;
+  useMarketBasedPrice: boolean;
+  marketPriceMargin: number;
+  paymentMethod: string;
+  id: string;
+  currencyPair: string;
+  primaryMarketDirection: string;
+  priceDisplayString: string;
+  primaryMarketAmountDisplayString: string;
+  primaryMarketMinAmountDisplayString: string;
+  primaryMarketVolumeDisplayString: string;
+  primaryMarketMinVolumeDisplayString: string;
+  primaryMarketPrice: number;
+  primaryMarketAmount: number;
+  primaryMarketMinAmount: number;
+  primaryMarketVolume: number;
+  primaryMarketMinVolume: number;
+}
+
+export interface Offer {
+  offer_id: string;
+  offer_date: number;
+  direction: string;
+  min_amount: string;
+  amount: string;
+  price: string;
+  volume: string;
+  payment_method: string;
+  offer_fee_txid: any;
+}
+
+export interface Tickers { [market: string]: Ticker | null; }
+
+export interface Ticker {
+  last: string;
+  high: string;
+  low: string;
+  volume_left: string;
+  volume_right: string;
+  buy: string | null;
+  sell: string | null;
+}
+
+export interface Trade {
+  direction: string;
+  price: string;
+  amount: string;
+  volume: string;
+  payment_method: string;
+  trade_id: string;
+  trade_date: number;
+}
+
+export interface TradesData {
+  currency: string;
+  direction: string;
+  tradePrice: number;
+  tradeAmount: number;
+  tradeDate: number;
+  paymentMethod: string;
+  offerDate: number;
+  useMarketBasedPrice: boolean;
+  marketPriceMargin: number;
+  offerAmount: number;
+  offerMinAmount: number;
+  offerId: string;
+  depositTxId?: string;
+  currencyPair: string;
+  primaryMarketDirection: string;
+  primaryMarketTradePrice: number;
+  primaryMarketTradeAmount: number;
+  primaryMarketTradeVolume: number;
+
+  _market: string;
+  _tradePriceStr: string;
+  _tradeAmountStr: string;
+  _tradeVolumeStr: string;
+  _offerAmountStr: string;
+  _tradePrice: number;
+  _tradeAmount: number;
+  _tradeVolume: number;
+  _offerAmount: number;
+}
+
+export interface MarketVolume {
+  period_start: number;
+  num_trades: number;
+  volume: string;
+}
+
+export interface MarketsApiError {
+  success: number;
+  error: string;
+}
+
+export type Interval = 'minute' | 'half_hour' | 'hour' | 'half_day' | 'day' | 'week' | 'month' | 'year' | 'auto';
+
+export interface SummarizedIntervals { [market: string]: SummarizedInterval; }
+export interface SummarizedInterval {
+  'period_start': number;
+  'open': number;
+  'close': number;
+  'high': number;
+  'low': number;
+  'avg': number;
+  'volume_right': number;
+  'volume_left': number;
+}

backend/src/api/bisq/markets-api.ts

@@ -0,0 +1,679 @@
+import { Currencies, OffersData, TradesData, Depth, Currency, Interval, HighLowOpenClose,
+  Markets, Offers, Offer, BisqTrade, MarketVolume, Tickers, Ticker, SummarizedIntervals, SummarizedInterval } from './interfaces';
+
+const strtotime = require('./strtotime');
+
+class BisqMarketsApi {
+  private cryptoCurrencyData: Currency[] = [];
+  private fiatCurrencyData: Currency[] = [];
+  private activeCryptoCurrencyData: Currency[] = [];
+  private activeFiatCurrencyData: Currency[] = [];
+  private offersData: OffersData[] = [];
+  private tradesData: TradesData[] = [];
+  private fiatCurrenciesIndexed: { [code: string]: true } = {};
+  private allCurrenciesIndexed: { [code: string]: Currency } = {};
+  private tradeDataByMarket: { [market: string]: TradesData[] } = {};
+  private tickersCache: Ticker | Tickers | null = null;
+
+  constructor() { }
+
+  setOffersData(offers: OffersData[]) {
+    this.offersData = offers;
+  }
+
+  setTradesData(trades: TradesData[]) {
+    this.tradesData = trades;
+    this.tradeDataByMarket = {};
+
+    this.tradesData.forEach((trade) => {
+      trade._market = trade.currencyPair.toLowerCase().replace('/', '_');
+      if (!this.tradeDataByMarket[trade._market]) {
+        this.tradeDataByMarket[trade._market] = [];
+      }
+      this.tradeDataByMarket[trade._market].push(trade);
+    });
+  }
+
+  setCurrencyData(cryptoCurrency: Currency[], fiatCurrency: Currency[], activeCryptoCurrency: Currency[], activeFiatCurrency: Currency[]) {
+    this.cryptoCurrencyData = cryptoCurrency,
+    this.fiatCurrencyData = fiatCurrency,
+    this.activeCryptoCurrencyData = activeCryptoCurrency,
+    this.activeFiatCurrencyData = activeFiatCurrency;
+
+    this.fiatCurrenciesIndexed = {};
+    this.allCurrenciesIndexed = {};
+
+    this.fiatCurrencyData.forEach((currency) => {
+      currency._type = 'fiat';
+      this.fiatCurrenciesIndexed[currency.code] = true;
+      this.allCurrenciesIndexed[currency.code] = currency;
+    });
+    this.cryptoCurrencyData.forEach((currency) => {
+       currency._type = 'crypto';
+       this.allCurrenciesIndexed[currency.code] = currency;
+    });
+  }
+
+  updateCache() {
+    this.tickersCache = null;
+    this.tickersCache = this.getTicker();
+  }
+
+  getCurrencies(
+    type: 'crypto' | 'fiat' | 'active' | 'all' = 'all',
+  ): Currencies {
+    let currencies: Currency[];
+
+    switch (type) {
+      case 'fiat':
+        currencies = this.fiatCurrencyData;
+        break;
+      case 'crypto':
+        currencies = this.cryptoCurrencyData;
+        break;
+      case 'active':
+        currencies = this.activeCryptoCurrencyData.concat(this.activeFiatCurrencyData);
+        break;
+      case 'all':
+      default:
+        currencies = this.cryptoCurrencyData.concat(this.fiatCurrencyData);
+    }
+    const result = {};
+    currencies.forEach((currency) => {
+      result[currency.code] = currency;
+    });
+    return result;
+  }
+
+  getDepth(
+    market: string,
+  ): Depth {
+    const currencyPair = market.replace('_', '/').toUpperCase();
+
+    const buys = this.offersData
+      .filter((offer) => offer.currencyPair === currencyPair && offer.primaryMarketDirection === 'BUY')
+      .map((offer) => offer.price)
+      .sort((a, b) => b - a)
+      .map((price) => this.intToBtc(price));
+
+    const sells = this.offersData
+      .filter((offer) => offer.currencyPair === currencyPair && offer.primaryMarketDirection === 'SELL')
+      .map((offer) => offer.price)
+      .sort((a, b) => a - b)
+      .map((price) => this.intToBtc(price));
+
+    const result = {};
+    result[market] = {
+      'buys': buys,
+      'sells': sells,
+    };
+    return result;
+  }
+
+  getOffers(
+    market: string,
+    direction?: 'buy' | 'sell',
+  ): Offers {
+    const currencyPair = market.replace('_', '/').toUpperCase();
+
+    let buys: Offer[] | null = null;
+    let sells: Offer[] | null = null;
+
+    if (!direction || direction === 'buy') {
+      buys = this.offersData
+        .filter((offer) => offer.currencyPair === currencyPair && offer.primaryMarketDirection === 'BUY')
+        .sort((a, b) => b.price - a.price)
+        .map((offer) => this.offerDataToOffer(offer, market));
+    }
+
+    if (!direction || direction === 'sell') {
+      sells = this.offersData
+        .filter((offer) => offer.currencyPair === currencyPair && offer.primaryMarketDirection === 'SELL')
+        .sort((a, b) => a.price - b.price)
+        .map((offer) => this.offerDataToOffer(offer, market));
+    }
+
+    const result: Offers = {};
+    result[market] = {
+      'buys': buys,
+      'sells': sells,
+    };
+    return result;
+  }
+
+  getMarkets(): Markets {
+    const allCurrencies = this.getCurrencies();
+    const activeCurrencies = this.getCurrencies('active');
+    const markets = {};
+
+    for (const currency of Object.keys(activeCurrencies)) {
+      if (allCurrencies[currency].code === 'BTC') {
+        continue;
+      }
+
+      const isFiat = allCurrencies[currency]._type === 'fiat';
+      const pmarketname = allCurrencies['BTC']['name'];
+
+      const lsymbol = isFiat ? 'BTC' : currency;
+      const rsymbol = isFiat ? currency : 'BTC';
+      const lname = isFiat ? pmarketname : allCurrencies[currency].name;
+      const rname = isFiat ? allCurrencies[currency].name : pmarketname;
+      const ltype = isFiat ? 'crypto' : allCurrencies[currency]._type;
+      const rtype = isFiat ? 'fiat' : 'crypto';
+      const lprecision = 8;
+      const rprecision = isFiat ? 2 : 8;
+      const pair = lsymbol.toLowerCase() + '_' + rsymbol.toLowerCase();
+
+      markets[pair] = {
+        'pair': pair,
+        'lname': lname,
+        'rname': rname,
+        'lsymbol': lsymbol,
+        'rsymbol': rsymbol,
+        'lprecision': lprecision,
+        'rprecision': rprecision,
+        'ltype': ltype,
+        'rtype': rtype,
+        'name': lname + '/' + rname,
+      };
+    }
+
+    return markets;
+  }
+
+  getTrades(
+    market: string,
+    timestamp_from?: number,
+    timestamp_to?: number,
+    trade_id_from?: string,
+    trade_id_to?: string,
+    direction?: 'buy' | 'sell',
+    limit: number = 100,
+    sort: 'asc' | 'desc' = 'desc',
+  ): BisqTrade[] {
+      limit = Math.min(limit, 2000);
+      const _market = market === 'all' ? undefined : market;
+
+      if (!timestamp_from) {
+        timestamp_from = new Date('2016-01-01').getTime() / 1000;
+      }
+      if (!timestamp_to) {
+        timestamp_to = new Date().getTime() / 1000;
+      }
+
+      const matches = this.getTradesByCriteria(_market, timestamp_to, timestamp_from,
+        trade_id_to, trade_id_from, direction, sort, limit, false);
+
+      if (sort === 'asc') {
+        matches.sort((a, b) => a.tradeDate - b.tradeDate);
+      } else {
+        matches.sort((a, b) => b.tradeDate - a.tradeDate);
+      }
+
+      return matches.map((trade) => {
+        const bsqTrade: BisqTrade = {
+          direction: trade.primaryMarketDirection,
+          price: trade._tradePriceStr,
+          amount: trade._tradeAmountStr,
+          volume: trade._tradeVolumeStr,
+          payment_method: trade.paymentMethod,
+          trade_id: trade.offerId,
+          trade_date: trade.tradeDate,
+        };
+        if (market === 'all') {
+          bsqTrade.market = trade._market;
+        }
+        return bsqTrade;
+      });
+  }
+
+  getVolumes(
+    market?: string,
+    timestamp_from?: number,
+    timestamp_to?: number,
+    interval: Interval = 'auto',
+    milliseconds?: boolean,
+    timestamp: 'no' | 'yes' = 'yes',
+  ): MarketVolume[] {
+    if (milliseconds) {
+      timestamp_from = timestamp_from ? timestamp_from / 1000 : timestamp_from;
+      timestamp_to = timestamp_to ? timestamp_to / 1000 : timestamp_to;
+    }
+    if (!timestamp_from) {
+      timestamp_from = new Date('2016-01-01').getTime() / 1000;
+    }
+    if (!timestamp_to) {
+      timestamp_to = new Date().getTime() / 1000;
+    }
+
+    const trades = this.getTradesByCriteria(market, timestamp_to, timestamp_from,
+      undefined, undefined, undefined, 'asc', Number.MAX_SAFE_INTEGER);
+
+    if (interval === 'auto') {
+      const range = timestamp_to - timestamp_from;
+      interval = this.getIntervalFromRange(range);
+    }
+
+    const intervals: any = {};
+    const marketVolumes: MarketVolume[] = [];
+
+    for (const trade of trades) {
+      const traded_at = trade['tradeDate'] / 1000;
+      const interval_start = this.intervalStart(traded_at, interval);
+
+      if (!intervals[interval_start]) {
+        intervals[interval_start] = {
+          'volume': 0,
+          'num_trades': 0,
+        };
+      }
+
+      const period = intervals[interval_start];
+      period['period_start'] = interval_start;
+      period['volume'] += this.fiatCurrenciesIndexed[trade.currency] ? trade._tradeAmount : trade._tradeVolume;
+      period['num_trades']++;
+    }
+
+    for (const p in intervals) {
+      if (intervals.hasOwnProperty(p)) {
+        const period = intervals[p];
+        marketVolumes.push({
+          period_start: timestamp === 'no' ? new Date(period['period_start'] * 1000).toISOString() : period['period_start'],
+          num_trades: period['num_trades'],
+          volume: this.intToBtc(period['volume']),
+        });
+      }
+    }
+
+    return marketVolumes;
+  }
+
+  getTicker(
+    market?: string,
+  ): Tickers | Ticker | null {
+    if (market) {
+      return this.getTickerFromMarket(market);
+    }
+
+    if (this.tickersCache) {
+      return this.tickersCache;
+    }
+
+    const allMarkets = this.getMarkets();
+    const tickers = {};
+    for (const m in allMarkets) {
+      if (allMarkets.hasOwnProperty(m)) {
+        tickers[allMarkets[m].pair] = this.getTickerFromMarket(allMarkets[m].pair);
+      }
+    }
+
+    return tickers;
+  }
+
+  getTickerFromMarket(market: string): Ticker | null {
+    let ticker: Ticker;
+    const timestamp_from = strtotime('-24 hour');
+    const timestamp_to = new Date().getTime() / 1000;
+    const trades = this.getTradesByCriteria(market, timestamp_to, timestamp_from,
+      undefined, undefined, undefined, 'asc', Number.MAX_SAFE_INTEGER);
+
+    const periods: SummarizedInterval[] = Object.values(this.getTradesSummarized(trades, timestamp_from));
+
+    const allCurrencies = this.getCurrencies();
+    const currencyRight = allCurrencies[market.split('_')[1].toUpperCase()];
+
+    if (periods[0]) {
+      ticker = {
+        'last': this.intToBtc(periods[0].close),
+        'high': this.intToBtc(periods[0].high),
+        'low': this.intToBtc(periods[0].low),
+        'volume_left': this.intToBtc(periods[0].volume_left),
+        'volume_right': this.intToBtc(periods[0].volume_right),
+        'buy': null,
+        'sell': null,
+      };
+    } else {
+      const lastTrade = this.tradeDataByMarket[market];
+      if (!lastTrade) {
+        return null;
+      }
+      const tradePrice = lastTrade[0].primaryMarketTradePrice * Math.pow(10, 8 - currencyRight.precision);
+
+      const lastTradePrice = this.intToBtc(tradePrice);
+      ticker = {
+        'last': lastTradePrice,
+        'high': lastTradePrice,
+        'low': lastTradePrice,
+        'volume_left': '0',
+        'volume_right': '0',
+        'buy': null,
+        'sell': null,
+      };
+    }
+
+    const timestampFromMilli = timestamp_from * 1000;
+    const timestampToMilli = timestamp_to * 1000;
+
+    const currencyPair = market.replace('_', '/').toUpperCase();
+    const offersData = this.offersData.slice().sort((a, b) => a.price - b.price);
+
+    const buy = offersData.find((offer) => offer.currencyPair === currencyPair
+                                          && offer.primaryMarketDirection === 'BUY'
+                                          && offer.date >= timestampFromMilli
+                                          && offer.date <= timestampToMilli
+                                        );
+    const sell = offersData.find((offer) => offer.currencyPair === currencyPair
+                                            && offer.primaryMarketDirection === 'SELL'
+                                            && offer.date >= timestampFromMilli
+                                            && offer.date <= timestampToMilli
+                                          );
+
+    if (buy) {
+      ticker.buy = this.intToBtc(buy.primaryMarketPrice * Math.pow(10, 8 - currencyRight.precision));
+    }
+    if (sell) {
+      ticker.sell = this.intToBtc(sell.primaryMarketPrice * Math.pow(10, 8 - currencyRight.precision));
+    }
+
+    return ticker;
+  }
+
+  getHloc(
+    market: string,
+    interval: Interval = 'auto',
+    timestamp_from?: number,
+    timestamp_to?: number,
+    milliseconds?: boolean,
+    timestamp: 'no' | 'yes' = 'yes',
+  ): HighLowOpenClose[] {
+    if (milliseconds) {
+      timestamp_from = timestamp_from ? timestamp_from / 1000 : timestamp_from;
+      timestamp_to = timestamp_to ? timestamp_to / 1000 : timestamp_to;
+    }
+    if (!timestamp_from) {
+      timestamp_from = new Date('2016-01-01').getTime() / 1000;
+    }
+    if (!timestamp_to) {
+      timestamp_to = new Date().getTime() / 1000;
+    }
+
+    const trades = this.getTradesByCriteria(market, timestamp_to, timestamp_from,
+      undefined, undefined, undefined, 'asc', Number.MAX_SAFE_INTEGER);
+
+    if (interval === 'auto') {
+      const range = timestamp_to - timestamp_from;
+      interval = this.getIntervalFromRange(range);
+    }
+
+    const intervals = this.getTradesSummarized(trades, timestamp_from, interval);
+
+    const hloc: HighLowOpenClose[] = [];
+
+    for (const p in intervals) {
+      if (intervals.hasOwnProperty(p)) {
+        const period = intervals[p];
+        hloc.push({
+          period_start: timestamp === 'no' ? new Date(period['period_start'] * 1000).toISOString() : period['period_start'],
+          open: this.intToBtc(period['open']),
+          close: this.intToBtc(period['close']),
+          high: this.intToBtc(period['high']),
+          low: this.intToBtc(period['low']),
+          avg: this.intToBtc(period['avg']),
+          volume_right: this.intToBtc(period['volume_right']),
+          volume_left: this.intToBtc(period['volume_left']),
+        });
+      }
+    }
+
+    return hloc;
+  }
+
+  private getIntervalFromRange(range: number): Interval {
+    // two days range loads minute data
+    if (range <= 3600) {
+      // up to one hour range loads minutely data
+      return 'minute';
+    } else if (range <= 1 * 24 * 3600) {
+      // up to one day range loads half-hourly data
+      return 'half_hour';
+    } else if (range <= 3 * 24 * 3600) {
+      // up to 3 day range loads hourly data
+      return 'hour';
+    } else if (range <= 7 * 24 * 3600) {
+      // up to 7 day range loads half-daily data
+      return 'half_day';
+    } else if (range <= 60 * 24 * 3600) {
+      // up to 2 month range loads daily data
+      return 'day';
+    } else if (range <= 12 * 31 * 24 * 3600) {
+      // up to one year range loads weekly data
+      return 'week';
+    } else if (range <= 12 * 31 * 24 * 3600) {
+      // up to 5 year range loads monthly data
+      return 'month';
+    } else {
+      // greater range loads yearly data
+      return 'year';
+    }
+  }
+
+  getVolumesByTime(time: number): MarketVolume[] {
+    const timestamp_from = new Date().getTime() / 1000 - time;
+    const timestamp_to = new Date().getTime() / 1000;
+
+    const trades = this.getTradesByCriteria(undefined, timestamp_to, timestamp_from,
+      undefined, undefined, undefined, 'asc', Number.MAX_SAFE_INTEGER);
+
+    const markets: any = {};
+
+    for (const trade of trades) {
+      if (!markets[trade._market]) {
+        markets[trade._market] = {
+          'volume': 0,
+          'num_trades': 0,
+        };
+      }
+
+      markets[trade._market]['volume'] += this.fiatCurrenciesIndexed[trade.currency] ? trade._tradeAmount : trade._tradeVolume;
+      markets[trade._market]['num_trades']++;
+    }
+
+    return markets;
+  }
+
+  private getTradesSummarized(trades: TradesData[], timestamp_from: number, interval?: string): SummarizedIntervals {
+    const intervals: any = {};
+    const intervals_prices: any = {};
+
+    for (const trade of trades) {
+      const traded_at = trade.tradeDate / 1000;
+      const interval_start = !interval ? timestamp_from : this.intervalStart(traded_at, interval);
+
+      if (!intervals[interval_start]) {
+          intervals[interval_start] = {
+            'open': 0,
+            'close': 0,
+            'high': 0,
+            'low': 0,
+            'avg': 0,
+            'volume_right': 0,
+            'volume_left': 0,
+        };
+        intervals_prices[interval_start] = [];
+      }
+      const period = intervals[interval_start];
+      const price = trade._tradePrice;
+
+      if (!intervals_prices[interval_start]['leftvol']) {
+        intervals_prices[interval_start]['leftvol'] = [];
+      }
+      if (!intervals_prices[interval_start]['rightvol']) {
+        intervals_prices[interval_start]['rightvol'] = [];
+      }
+
+      intervals_prices[interval_start]['leftvol'].push(trade._tradeAmount);
+      intervals_prices[interval_start]['rightvol'].push(trade._tradeVolume);
+
+      if (price) {
+          const plow = period['low'];
+          period['period_start'] = interval_start;
+          period['open'] = period['open'] || price;
+          period['close'] = price;
+          period['high'] = price > period['high'] ? price : period['high'];
+          period['low'] = (plow && price > plow) ? period['low'] : price;
+          period['avg'] = intervals_prices[interval_start]['rightvol'].reduce((p: number, c: number) => c + p, 0)
+            / intervals_prices[interval_start]['leftvol'].reduce((c: number, p: number) => c + p, 0) * 100000000;
+          period['volume_left'] += trade._tradeAmount;
+          period['volume_right'] += trade._tradeVolume;
+      }
+    }
+    return intervals;
+  }
+
+  private getTradesByCriteria(
+    market: string | undefined,
+    timestamp_to: number,
+    timestamp_from: number,
+    trade_id_to: string | undefined,
+    trade_id_from: string | undefined,
+    direction: 'buy' | 'sell' | undefined,
+    sort: string,
+    limit: number,
+    integerAmounts: boolean = true,
+ ): TradesData[] {
+    let trade_id_from_ts: number | null = null;
+    let trade_id_to_ts: number | null = null;
+    const allCurrencies = this.getCurrencies();
+
+    const timestampFromMilli = timestamp_from * 1000;
+    const timestampToMilli = timestamp_to * 1000;
+
+    // note: the offer_id_from/to depends on iterating over trades in
+    // descending chronological order.
+    const tradesDataSorted = this.tradesData.slice();
+    if (sort === 'asc') {
+      tradesDataSorted.reverse();
+    }
+
+    let matches: TradesData[] = [];
+    for (const trade of tradesDataSorted) {
+      if (trade_id_from === trade.offerId) {
+        trade_id_from_ts = trade.tradeDate;
+      }
+      if (trade_id_to === trade.offerId) {
+        trade_id_to_ts = trade.tradeDate;
+      }
+      if (trade_id_to && trade_id_to_ts === null) {
+        continue;
+      }
+      if (trade_id_from && trade_id_from_ts != null && trade_id_from_ts !== trade.tradeDate) {
+        continue;
+      }
+      if (market && market !== trade._market) {
+        continue;
+      }
+      if (timestampFromMilli && timestampFromMilli > trade.tradeDate) {
+        continue;
+      }
+      if (timestampToMilli && timestampToMilli < trade.tradeDate) {
+        continue;
+      }
+      if (direction && direction !== trade.direction.toLowerCase()) {
+        continue;
+      }
+
+      // Filter out bogus trades with BTC/BTC or XXX/XXX market.
+      // See github issue: https://github.com/bitsquare/bitsquare/issues/883
+      const currencyPairs = trade.currencyPair.split('/');
+      if (currencyPairs[0] === currencyPairs[1]) {
+        continue;
+      }
+
+      const currencyLeft = allCurrencies[currencyPairs[0]];
+      const currencyRight = allCurrencies[currencyPairs[1]];
+
+      if (!currencyLeft || !currencyRight) {
+        continue;
+      }
+
+      const tradePrice = trade.primaryMarketTradePrice * Math.pow(10, 8 - currencyRight.precision);
+      const tradeAmount = trade.primaryMarketTradeAmount * Math.pow(10, 8 - currencyLeft.precision);
+      const tradeVolume = trade.primaryMarketTradeVolume * Math.pow(10, 8 - currencyRight.precision);
+
+      if (integerAmounts) {
+        trade._tradePrice = tradePrice;
+        trade._tradeAmount = tradeAmount;
+        trade._tradeVolume = tradeVolume;
+        trade._offerAmount = trade.offerAmount;
+      } else {
+        trade._tradePriceStr = this.intToBtc(tradePrice);
+        trade._tradeAmountStr = this.intToBtc(tradeAmount);
+        trade._tradeVolumeStr = this.intToBtc(tradeVolume);
+        trade._offerAmountStr = this.intToBtc(trade.offerAmount);
+      }
+
+      matches.push(trade);
+
+      if (matches.length >= limit) {
+        break;
+      }
+    }
+
+    if ((trade_id_from && !trade_id_from_ts) || (trade_id_to && !trade_id_to_ts)) {
+      matches = [];
+    }
+    return matches;
+  }
+
+  private intervalStart(ts: number, interval: string): number {
+    switch (interval) {
+        case 'minute':
+            return (ts - (ts % 60));
+        case '10_minute':
+            return (ts - (ts % 600));
+        case 'half_hour':
+            return (ts - (ts % 1800));
+        case 'hour':
+            return (ts - (ts % 3600));
+        case 'half_day':
+            return (ts - (ts % (3600 * 12)));
+        case 'day':
+            return strtotime('midnight today', ts);
+        case 'week':
+            return strtotime('midnight sunday last week', ts);
+        case 'month':
+            return strtotime('midnight first day of this month', ts);
+        case 'year':
+            return strtotime('midnight first day of january', ts);
+        default:
+            throw new Error('Unsupported interval: ' + interval);
+    }
+}
+
+  private offerDataToOffer(offer: OffersData, market: string): Offer {
+    const currencyPairs = market.split('_');
+    const currencyRight = this.allCurrenciesIndexed[currencyPairs[1].toUpperCase()];
+    const currencyLeft = this.allCurrenciesIndexed[currencyPairs[0].toUpperCase()];
+    const price = offer['primaryMarketPrice'] * Math.pow( 10, 8 - currencyRight['precision']);
+    const amount = offer['primaryMarketAmount'] * Math.pow( 10, 8 - currencyLeft['precision']);
+    const volume = offer['primaryMarketVolume'] * Math.pow( 10, 8 - currencyRight['precision']);
+
+    return {
+      offer_id: offer.id,
+      offer_date: offer.date,
+      direction: offer.primaryMarketDirection,
+      min_amount: this.intToBtc(offer.minAmount),
+      amount: this.intToBtc(amount),
+      price: this.intToBtc(price),
+      volume: this.intToBtc(volume),
+      payment_method: offer.paymentMethod,
+      offer_fee_txid: null,
+    };
+  }
+
+  private intToBtc(val: number): string {
+    return (val / 100000000).toFixed(8);
+  }
+}
+
+export default new BisqMarketsApi();

backend/src/api/bisq/markets.ts

@@ -0,0 +1,137 @@
+import config from '../../config';
+import * as fs from 'fs';
+import { OffersData as OffersData, TradesData, Currency } from './interfaces';
+import bisqMarket from './markets-api';
+import logger from '../../logger';
+
+class Bisq {
+  private static FOLDER_WATCH_CHANGE_DETECTION_DEBOUNCE = 4000;
+  private static MARKET_JSON_PATH = config.BISQ.DATA_PATH;
+  private static MARKET_JSON_FILE_PATHS = {
+    activeCryptoCurrency: '/active_crypto_currency_list.json',
+    activeFiatCurrency: '/active_fiat_currency_list.json',
+    cryptoCurrency: '/crypto_currency_list.json',
+    fiatCurrency: '/fiat_currency_list.json',
+    offers: '/offers_statistics.json',
+    trades: '/trade_statistics.json',
+  };
+
+  private cryptoCurrencyLastMtime = new Date('2016-01-01');
+  private fiatCurrencyLastMtime = new Date('2016-01-01');
+  private offersLastMtime = new Date('2016-01-01');
+  private tradesLastMtime = new Date('2016-01-01');
+
+  private subdirectoryWatcher: fs.FSWatcher | undefined;
+
+  constructor() {}
+
+  startBisqService(): void {
+    try {
+      this.checkForBisqDataFolder();
+    } catch (e) {
+      logger.info('Retrying to start bisq service (markets) in 3 minutes');
+      setTimeout(this.startBisqService.bind(this), 180000);
+      return;
+    }
+    this.loadBisqDumpFile();
+    this.startBisqDirectoryWatcher();
+  }
+
+  private checkForBisqDataFolder() {
+    if (!fs.existsSync(Bisq.MARKET_JSON_PATH + Bisq.MARKET_JSON_FILE_PATHS.cryptoCurrency)) {
+      logger.err(Bisq.MARKET_JSON_PATH + Bisq.MARKET_JSON_FILE_PATHS.cryptoCurrency + ` doesn't exist. Make sure Bisq is running and the config is correct before starting the server.`);
+      throw new Error(`Cannot load BISQ ${Bisq.MARKET_JSON_FILE_PATHS.cryptoCurrency} file`);
+    }
+  }
+
+  private startBisqDirectoryWatcher() {
+    if (this.subdirectoryWatcher) {
+      this.subdirectoryWatcher.close();
+    }
+    if (!fs.existsSync(Bisq.MARKET_JSON_PATH + Bisq.MARKET_JSON_FILE_PATHS.cryptoCurrency)) {
+      logger.warn(Bisq.MARKET_JSON_PATH + Bisq.MARKET_JSON_FILE_PATHS.cryptoCurrency + ` doesn't exist. Trying to restart sub directory watcher again in 3 minutes.`);
+      setTimeout(() => this.startBisqDirectoryWatcher(), 180000);
+      return;
+    }
+    let fsWait: NodeJS.Timeout | null = null;
+    this.subdirectoryWatcher = fs.watch(Bisq.MARKET_JSON_PATH, () => {
+      if (fsWait) {
+        clearTimeout(fsWait);
+      }
+      fsWait = setTimeout(() => {
+        logger.debug(`Change detected in the Bisq market data folder.`);
+        this.loadBisqDumpFile();
+      }, Bisq.FOLDER_WATCH_CHANGE_DETECTION_DEBOUNCE);
+    });
+  }
+
+  private async loadBisqDumpFile(): Promise<void> {
+    const start = new Date().getTime();
+    try {
+      let marketsDataUpdated = false;
+      const cryptoMtime = this.getFileMtime(Bisq.MARKET_JSON_FILE_PATHS.cryptoCurrency);
+      const fiatMtime = this.getFileMtime(Bisq.MARKET_JSON_FILE_PATHS.fiatCurrency);
+      if (cryptoMtime > this.cryptoCurrencyLastMtime || fiatMtime > this.fiatCurrencyLastMtime) {
+        const cryptoCurrencyData = await this.loadData<Currency[]>(Bisq.MARKET_JSON_FILE_PATHS.cryptoCurrency);
+        const fiatCurrencyData = await this.loadData<Currency[]>(Bisq.MARKET_JSON_FILE_PATHS.fiatCurrency);
+        const activeCryptoCurrencyData = await this.loadData<Currency[]>(Bisq.MARKET_JSON_FILE_PATHS.activeCryptoCurrency);
+        const activeFiatCurrencyData = await this.loadData<Currency[]>(Bisq.MARKET_JSON_FILE_PATHS.activeFiatCurrency);
+        logger.debug('Updating Bisq Market Currency Data');
+        bisqMarket.setCurrencyData(cryptoCurrencyData, fiatCurrencyData, activeCryptoCurrencyData, activeFiatCurrencyData);
+        if (cryptoMtime > this.cryptoCurrencyLastMtime) {
+          this.cryptoCurrencyLastMtime = cryptoMtime;
+        }
+        if (fiatMtime > this.fiatCurrencyLastMtime) {
+          this.fiatCurrencyLastMtime = fiatMtime;
+        }
+        marketsDataUpdated = true;
+      }
+      const offersMtime = this.getFileMtime(Bisq.MARKET_JSON_FILE_PATHS.offers);
+      if (offersMtime > this.offersLastMtime) {
+        const offersData = await this.loadData<OffersData[]>(Bisq.MARKET_JSON_FILE_PATHS.offers);
+        logger.debug('Updating Bisq Market Offers Data');
+        bisqMarket.setOffersData(offersData);
+        this.offersLastMtime = offersMtime;
+        marketsDataUpdated = true;
+      }
+      const tradesMtime = this.getFileMtime(Bisq.MARKET_JSON_FILE_PATHS.trades);
+      if (tradesMtime > this.tradesLastMtime) {
+        const tradesData = await this.loadData<TradesData[]>(Bisq.MARKET_JSON_FILE_PATHS.trades);
+        logger.debug('Updating Bisq Market Trades Data');
+        bisqMarket.setTradesData(tradesData);
+        this.tradesLastMtime = tradesMtime;
+        marketsDataUpdated = true;
+      }
+      if (marketsDataUpdated) {
+        bisqMarket.updateCache();
+        const time = new Date().getTime() - start;
+        logger.debug('Bisq market data updated in ' + time + ' ms');
+      }
+    } catch (e) {
+      logger.err('loadBisqMarketDataDumpFile() error.' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  private getFileMtime(path: string): Date {
+    const stats = fs.statSync(Bisq.MARKET_JSON_PATH + path);
+    return stats.mtime;
+  }
+
+  private loadData<T>(path: string): Promise<T> {
+    return new Promise((resolve, reject) => {
+      fs.readFile(Bisq.MARKET_JSON_PATH + path, 'utf8', (err, data) => {
+        if (err) {
+          reject(err);
+        }
+        try {
+          const parsedData = JSON.parse(data);
+          resolve(parsedData);
+        } catch (e) {
+          reject('JSON parse error (' + path + ')');
+        }
+      });
+    });
+  }
+}
+
+export default new Bisq();

backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts

@@ -1,19 +1,23 @@
-import { IMempoolInfo, ITransaction, IBlock } from '../../interfaces';
+import { IEsploraApi } from './esplora-api.interface';
 
 export interface AbstractBitcoinApi {
-  getMempoolInfo(): Promise<IMempoolInfo>;
-  getRawMempool(): Promise<ITransaction['txid'][]>;
-  getRawTransaction(txId: string): Promise<ITransaction>;
-  getBlockCount(): Promise<number>;
-  getBlockAndTransactions(hash: string): Promise<IBlock>;
-  getBlockHash(height: number): Promise<string>;
-
-  getBlock(hash: string): Promise<IBlock>;
-  getBlockTransactions(hash: string): Promise<IBlock>;
-  getBlockTransactionsFromIndex(hash: string, index: number): Promise<IBlock>;
-  getBlocks(): Promise<string>;
-  getBlocksFromHeight(height: number): Promise<string>;
-  getAddress(address: string): Promise<IBlock>;
-  getAddressTransactions(address: string): Promise<IBlock>;
-  getAddressTransactionsFromLastSeenTxid(address: string, lastSeenTxid: string): Promise<IBlock>;
+  $getRawMempool(): Promise<IEsploraApi.Transaction['txid'][]>;
+  $getRawTransaction(txId: string, skipConversion?: boolean, addPrevout?: boolean, lazyPrevouts?: boolean): Promise<IEsploraApi.Transaction>;
+  $getBlockHeightTip(): Promise<number>;
+  $getTxIdsForBlock(hash: string): Promise<string[]>;
+  $getBlockHash(height: number): Promise<string>;
+  $getBlockHeader(hash: string): Promise<string>;
+  $getBlock(hash: string): Promise<IEsploraApi.Block>;
+  $getAddress(address: string): Promise<IEsploraApi.Address>;
+  $getAddressTransactions(address: string, lastSeenTxId: string): Promise<IEsploraApi.Transaction[]>;
+  $getAddressPrefix(prefix: string): string[];
+  $sendRawTransaction(rawTransaction: string): Promise<string>;
+  $getOutspends(txId: string): Promise<IEsploraApi.Outspend[]>;
+}
+export interface BitcoinRpcCredentials {
+  host: string;
+  port: number;
+  user: string;
+  pass: string;
+  timeout: number;
 }

backend/src/api/bitcoin/bitcoin-api-factory.ts

@@ -1,16 +1,20 @@
-const config = require('../../../mempool-config.json');
+import config from '../../config';
 import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
-import BitcoindApi from './bitcoind-api';
-import ElectrsApi from './electrs-api';
+import EsploraApi from './esplora-api';
+import BitcoinApi from './bitcoin-api';
+import ElectrumApi from './electrum-api';
+import bitcoinClient from './bitcoin-client';
 
-function factory(): AbstractBitcoinApi {
-  switch (config.BACKEND_API) {
-    case 'electrs':
-      return new ElectrsApi();
-    case 'bitcoind':
+function bitcoinApiFactory(): AbstractBitcoinApi {
+  switch (config.MEMPOOL.BACKEND) {
+    case 'esplora':
+      return new EsploraApi();
+    case 'electrum':
+      return new ElectrumApi(bitcoinClient);
+    case 'none':
     default:
-      return new BitcoindApi();
+      return new BitcoinApi(bitcoinClient);
   }
 }
 
-export default factory();
+export default bitcoinApiFactory();

backend/src/api/bitcoin/bitcoin-api.interface.ts

@@ -0,0 +1,167 @@
+export namespace IBitcoinApi {
+  export interface MempoolInfo {
+    loaded: boolean;                 //  (boolean) True if the mempool is fully loaded
+    size: number;                    //  (numeric) Current tx count
+    bytes: number;                   //  (numeric) Sum of all virtual transaction sizes as defined in BIP 141.
+    usage: number;                   //  (numeric) Total memory usage for the mempool
+    total_fee: number;               //  (numeric) Total fees of transactions in the mempool
+    maxmempool: number;              //  (numeric) Maximum memory usage for the mempool
+    mempoolminfee: number;           //  (numeric) Minimum fee rate in BTC/kB for tx to be accepted.
+    minrelaytxfee: number;           //  (numeric) Current minimum relay fee for transactions
+  }
+
+  export interface RawMempool { [txId: string]: MempoolEntry; }
+
+  export interface MempoolEntry {
+    vsize: number;                   //  (numeric) virtual transaction size as defined in BIP 141.
+    weight: number;                  //  (numeric) transaction weight as defined in BIP 141.
+    time: number;                    //  (numeric) local time transaction entered pool in seconds since 1 Jan 1970 GMT
+    height: number;                  //  (numeric) block height when transaction entered pool
+    descendantcount: number;         //  (numeric) number of in-mempool descendant transactions (including this one)
+    descendantsize: number;          //  (numeric) virtual transaction size of in-mempool descendants (including this one)
+    ancestorcount: number;           //  (numeric) number of in-mempool ancestor transactions (including this one)
+    ancestorsize: number;            //  (numeric) virtual transaction size of in-mempool ancestors (including this one)
+    wtxid: string;                   //  (string) hash of serialized transactionumber; including witness data
+    fees: {
+      base: number;                  //  (numeric) transaction fee in BTC
+      modified: number;              //  (numeric) transaction fee with fee deltas used for mining priority in BTC
+      ancestor: number;              //  (numeric) modified fees (see above) of in-mempool ancestors (including this one) in BTC
+      descendant: number;            //  (numeric) modified fees (see above) of in-mempool descendants (including this one) in BTC
+    };
+    depends: string[];               //  (string) parent transaction id
+    spentby: string[];               //  (array) unconfirmed transactions spending outputs from this transaction
+    'bip125-replaceable': boolean;   //  (boolean) Whether this transaction could be replaced due to BIP125 (replace-by-fee)
+  }
+
+  export interface Block {
+    hash: string;                    //  (string) the block hash (same as provided)
+    confirmations: number;           //  (numeric) The number of confirmations, or -1 if the block is not on the main chain
+    size: number;                    //  (numeric) The block size
+    strippedsize: number;            //  (numeric) The block size excluding witness data
+    weight: number;                  //  (numeric) The block weight as defined in BIP 141
+    height: number;                  //  (numeric) The block height or index
+    version: number;                 //  (numeric) The block version
+    versionHex: string;              //  (string) The block version formatted in hexadecimal
+    merkleroot: string;              //  (string) The merkle root
+    tx: Transaction[];
+    time: number;                    //  (numeric) The block time expressed in UNIX epoch time
+    mediantime: number;              //  (numeric) The median block time expressed in UNIX epoch time
+    nonce: number;                   //  (numeric) The nonce
+    bits: string;                    //  (string) The bits
+    difficulty: number;              //  (numeric) The difficulty
+    chainwork: string;               //  (string) Expected number of hashes required to produce the chain up to this block (in hex)
+    nTx: number;                     //  (numeric) The number of transactions in the block
+    previousblockhash: string;       //  (string) The hash of the previous block
+    nextblockhash: string;           //  (string) The hash of the next block
+  }
+
+  export interface Transaction {
+    in_active_chain: boolean;        //  (boolean) Whether specified block is in the active chain or not
+    hex: string;                     //  (string) The serialized, hex-encoded data for 'txid'
+    txid: string;                    //  (string) The transaction id (same as provided)
+    hash: string;                    //  (string) The transaction hash (differs from txid for witness transactions)
+    size: number;                    //  (numeric) The serialized transaction size
+    vsize: number;                   //  (numeric) The virtual transaction size (differs from size for witness transactions)
+    weight: number;                  //  (numeric) The transaction's weight (between vsize*4-3 and vsize*4)
+    version: number;                 //  (numeric) The version
+    locktime: number;                //  (numeric) The lock time
+    vin: Vin[];
+    vout: Vout[];
+    blockhash: string;               //  (string) the block hash
+    confirmations: number;           //  (numeric) The confirmations
+    blocktime: number;               //  (numeric) The block time expressed in UNIX epoch time
+    time: number;                    //  (numeric) Same as blocktime
+  }
+
+  export interface Vin {
+    txid?: string;                   //  (string) The transaction id
+    vout?: number;                   //  (string)
+    scriptSig?: {                    //  (json object) The script
+      asm: string;                   //  (string) asm
+      hex: string;                   //  (string) hex
+    };
+    sequence: number;                //  (numeric) The script sequence number
+    txinwitness?: string[];          //  (string) hex-encoded witness data
+    coinbase?: string;
+    is_pegin?: boolean;              //  (boolean) Elements peg-in
+  }
+
+  export interface Vout {
+    value: number;                   //  (numeric) The value in BTC
+    n: number;                       //  (numeric) index
+    asset?: string;                  //  (string) Elements asset id
+    scriptPubKey: {                  //  (json object)
+      asm: string;                   //  (string) the asm
+      hex: string;                   //  (string) the hex
+      reqSigs?: number;              //  (numeric) The required sigs
+      type: string;                  //  (string) The type, eg 'pubkeyhash'
+      address?: string;              //  (string) bitcoin address
+      addresses?: string[];           //  (string) bitcoin addresses
+      pegout_chain?: string;         //  (string) Elements peg-out chain
+      pegout_addresses?: string[];   //  (string) Elements peg-out addresses
+    };
+  }
+
+  export interface AddressInformation {
+    isvalid: boolean;                //  (boolean) If the address is valid or not. If not, this is the only property returned.
+    isvalid_parent?: boolean;        //  (boolean) Elements only
+    address: string;                 //  (string) The bitcoin address validated
+    scriptPubKey: string;            //  (string) The hex-encoded scriptPubKey generated by the address
+    isscript: boolean;               //  (boolean) If the key is a script
+    iswitness: boolean;              //  (boolean) If the address is a witness
+    witness_version?: number;        //  (numeric, optional) The version number of the witness program
+    witness_program: string;         //  (string, optional) The hex value of the witness program
+    confidential_key?: string;       //  (string) Elements only
+    unconfidential?: string;         //  (string) Elements only
+  }
+
+  export interface ChainTips {
+    height: number;                  //  (numeric) height of the chain tip
+    hash: string;                    //  (string) block hash of the tip
+    branchlen: number;               //  (numeric) zero for main chain, otherwise length of branch connecting the tip to the main chain
+    status: 'invalid' | 'headers-only' | 'valid-headers' | 'valid-fork' | 'active';
+  }
+
+  export interface BlockchainInfo {
+    chain: number;                   // (string) current network name as defined in BIP70 (main, test, regtest)
+    blocks: number;                  // (numeric) the current number of blocks processed in the server
+    headers: number;                 // (numeric) the current number of headers we have validated
+    bestblockhash: string,           // (string) the hash of the currently best block
+    difficulty: number;              // (numeric) the current difficulty
+    mediantime: number;              // (numeric) median time for the current best block
+    verificationprogress: number;    // (numeric) estimate of verification progress [0..1]
+    initialblockdownload: boolean;   // (bool) (debug information) estimate of whether this node is in Initial Block Download mode.
+    chainwork: string                // (string) total amount of work in active chain, in hexadecimal
+    size_on_disk: number;            // (numeric) the estimated size of the block and undo files on disk
+    pruned: number;                  // (boolean) if the blocks are subject to pruning
+    pruneheight: number;             // (numeric) lowest-height complete block stored (only present if pruning is enabled)
+    automatic_pruning: number;       // (boolean) whether automatic pruning is enabled (only present if pruning is enabled)
+    prune_target_size: number;       // (numeric) the target size used by pruning (only present if automatic pruning is enabled)
+    softforks: SoftFork[];           // (array) status of softforks in progress
+    bip9_softforks: { [name: string]: Bip9SoftForks[] } // (object) status of BIP9 softforks in progress
+    warnings: string;                // (string) any network and blockchain warnings.
+  }
+
+  interface SoftFork {
+    id: string;                      // (string) name of softfork
+    version: number;                 // (numeric) block version
+    reject: {                        // (object) progress toward rejecting pre-softfork blocks
+      status: boolean;               // (boolean) true if threshold reached
+    },
+  }
+  interface Bip9SoftForks {
+    status: number;                  // (string) one of defined, started, locked_in, active, failed
+    bit: number;                     // (numeric) the bit (0-28) in the block version field used to signal this softfork (only for started status)
+    startTime: number;               // (numeric) the minimum median time past of a block at which the bit gains its meaning
+    timeout: number;                 // (numeric) the median time past of a block at which the deployment is considered failed if not yet locked in
+    since: number;                   // (numeric) height of the first block to which the status applies
+    statistics: {                    // (object) numeric statistics about BIP9 signalling for a softfork (only for started status)
+      period: number;                // (numeric) the length in blocks of the BIP9 signalling period 
+      threshold: number;             // (numeric) the number of blocks with the version bit set required to activate the feature 
+      elapsed: number;               // (numeric) the number of blocks elapsed since the beginning of the current period 
+      count: number;                 // (numeric) the number of blocks with the version bit set in the current period 
+      possible: boolean;             // (boolean) returns false if there are not enough blocks left in this period to pass activation threshold 
+    }
+  }
+
+}

backend/src/api/bitcoin/bitcoin-api.ts

@@ -0,0 +1,391 @@
+import * as bitcoinjs from 'bitcoinjs-lib';
+import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
+import { IBitcoinApi } from './bitcoin-api.interface';
+import { IEsploraApi } from './esplora-api.interface';
+import blocks from '../blocks';
+import mempool from '../mempool';
+import { TransactionExtended } from '../../mempool.interfaces';
+
+class BitcoinApi implements AbstractBitcoinApi {
+  private rawMempoolCache: IBitcoinApi.RawMempool | null = null;
+  protected bitcoindClient: any;
+
+  constructor(bitcoinClient: any) {
+    this.bitcoindClient = bitcoinClient;
+  }
+
+  static convertBlock(block: IBitcoinApi.Block): IEsploraApi.Block {
+    return {
+      id: block.hash,
+      height: block.height,
+      version: block.version,
+      timestamp: block.time,
+      bits: parseInt(block.bits, 16),
+      nonce: block.nonce,
+      difficulty: block.difficulty,
+      merkle_root: block.merkleroot,
+      tx_count: block.nTx,
+      size: block.size,
+      weight: block.weight,
+      previousblockhash: block.previousblockhash,
+    };
+  }
+
+
+  $getRawTransaction(txId: string, skipConversion = false, addPrevout = false, lazyPrevouts = false): Promise<IEsploraApi.Transaction> {
+    // If the transaction is in the mempool we already converted and fetched the fee. Only prevouts are missing
+    const txInMempool = mempool.getMempool()[txId];
+    if (txInMempool && addPrevout) {
+      return this.$addPrevouts(txInMempool);
+    }
+
+    return this.bitcoindClient.getRawTransaction(txId, true)
+      .then((transaction: IBitcoinApi.Transaction) => {
+        if (skipConversion) {
+          transaction.vout.forEach((vout) => {
+            vout.value = Math.round(vout.value * 100000000);
+          });
+          return transaction;
+        }
+        return this.$convertTransaction(transaction, addPrevout, lazyPrevouts);
+      })
+      .catch((e: Error) => {
+        if (e.message.startsWith('The genesis block coinbase')) {
+          return this.$returnCoinbaseTransaction();
+        }
+        throw e;
+      });
+  }
+
+  $getBlockHeightTip(): Promise<number> {
+    return this.bitcoindClient.getChainTips()
+      .then((result: IBitcoinApi.ChainTips[]) => {
+        return result.find(tip => tip.status === 'active')!.height;
+      });
+  }
+
+  $getTxIdsForBlock(hash: string): Promise<string[]> {
+    return this.bitcoindClient.getBlock(hash, 1)
+      .then((rpcBlock: IBitcoinApi.Block) => rpcBlock.tx);
+  }
+
+  $getRawBlock(hash: string): Promise<string> {
+    return this.bitcoindClient.getBlock(hash, 0);
+  }
+
+  $getBlockHash(height: number): Promise<string> {
+    return this.bitcoindClient.getBlockHash(height);
+  }
+
+  $getBlockHeader(hash: string): Promise<string> {
+    return this.bitcoindClient.getBlockHeader(hash, false);
+  }
+
+  async $getBlock(hash: string): Promise<IEsploraApi.Block> {
+    const foundBlock = blocks.getBlocks().find((block) => block.id === hash);
+    if (foundBlock) {
+      return foundBlock;
+    }
+
+    return this.bitcoindClient.getBlock(hash)
+      .then((block: IBitcoinApi.Block) => BitcoinApi.convertBlock(block));
+  }
+
+  $getAddress(address: string): Promise<IEsploraApi.Address> {
+    throw new Error('Method getAddress not supported by the Bitcoin RPC API.');
+  }
+
+  $getAddressTransactions(address: string, lastSeenTxId: string): Promise<IEsploraApi.Transaction[]> {
+    throw new Error('Method getAddressTransactions not supported by the Bitcoin RPC API.');
+  }
+
+  $getRawMempool(): Promise<IEsploraApi.Transaction['txid'][]> {
+    return this.bitcoindClient.getRawMemPool();
+  }
+
+  $getAddressPrefix(prefix: string): string[] {
+    const found: { [address: string]: string } = {};
+    const mp = mempool.getMempool();
+    for (const tx in mp) {
+      for (const vout of mp[tx].vout) {
+        if (vout.scriptpubkey_address.indexOf(prefix) === 0) {
+          found[vout.scriptpubkey_address] = '';
+          if (Object.keys(found).length >= 10) {
+            return Object.keys(found);
+          }
+        }
+      }
+    }
+    return Object.keys(found);
+  }
+
+  $sendRawTransaction(rawTransaction: string): Promise<string> {
+    return this.bitcoindClient.sendRawTransaction(rawTransaction);
+  }
+
+  async $getOutspends(txId: string): Promise<IEsploraApi.Outspend[]> {
+    const outSpends: IEsploraApi.Outspend[] = [];
+    const tx = await this.$getRawTransaction(txId, true, false);
+    for (let i = 0; i < tx.vout.length; i++) {
+      if (tx.status && tx.status.block_height === 0) {
+        outSpends.push({
+          spent: false
+        });
+      } else {
+        const txOut = await this.bitcoindClient.getTxOut(txId, i);
+        outSpends.push({
+          spent: txOut === null,
+        });
+      }
+    }
+    return outSpends;
+  }
+
+  $getEstimatedHashrate(blockHeight: number): Promise<number> {
+    // 120 is the default block span in Core
+    return this.bitcoindClient.getNetworkHashPs(120, blockHeight);
+  }
+
+  protected async $convertTransaction(transaction: IBitcoinApi.Transaction, addPrevout: boolean, lazyPrevouts = false): Promise<IEsploraApi.Transaction> {
+    let esploraTransaction: IEsploraApi.Transaction = {
+      txid: transaction.txid,
+      version: transaction.version,
+      locktime: transaction.locktime,
+      size: transaction.size,
+      weight: transaction.weight,
+      fee: 0,
+      vin: [],
+      vout: [],
+      status: { confirmed: false },
+    };
+
+    esploraTransaction.vout = transaction.vout.map((vout) => {
+      return {
+        value: Math.round(vout.value * 100000000),
+        scriptpubkey: vout.scriptPubKey.hex,
+        scriptpubkey_address: vout.scriptPubKey && vout.scriptPubKey.address ? vout.scriptPubKey.address
+          : vout.scriptPubKey.addresses ? vout.scriptPubKey.addresses[0] : '',
+        scriptpubkey_asm: vout.scriptPubKey.asm ? this.convertScriptSigAsm(vout.scriptPubKey.hex) : '',
+        scriptpubkey_type: this.translateScriptPubKeyType(vout.scriptPubKey.type),
+      };
+    });
+
+    esploraTransaction.vin = transaction.vin.map((vin) => {
+      return {
+        is_coinbase: !!vin.coinbase,
+        prevout: null,
+        scriptsig: vin.scriptSig && vin.scriptSig.hex || vin.coinbase || '',
+        scriptsig_asm: vin.scriptSig && this.convertScriptSigAsm(vin.scriptSig.hex) || '',
+        sequence: vin.sequence,
+        txid: vin.txid || '',
+        vout: vin.vout || 0,
+        witness: vin.txinwitness,
+      };
+    });
+
+    if (transaction.confirmations) {
+      esploraTransaction.status = {
+        confirmed: true,
+        block_height: blocks.getCurrentBlockHeight() - transaction.confirmations + 1,
+        block_hash: transaction.blockhash,
+        block_time: transaction.blocktime,
+      };
+    }
+
+    if (addPrevout) {
+      esploraTransaction = await this.$calculateFeeFromInputs(esploraTransaction, false, lazyPrevouts);
+    } else if (!transaction.confirmations) {
+      esploraTransaction = await this.$appendMempoolFeeData(esploraTransaction);
+    }
+
+    return esploraTransaction;
+  }
+
+  private translateScriptPubKeyType(outputType: string): string {
+    const map = {
+      'pubkey': 'p2pk',
+      'pubkeyhash': 'p2pkh',
+      'scripthash': 'p2sh',
+      'witness_v0_keyhash': 'v0_p2wpkh',
+      'witness_v0_scripthash': 'v0_p2wsh',
+      'witness_v1_taproot': 'v1_p2tr',
+      'nonstandard': 'nonstandard',
+      'multisig': 'multisig',
+      'nulldata': 'op_return'
+    };
+
+    if (map[outputType]) {
+      return map[outputType];
+    } else {
+      return 'unknown';
+    }
+  }
+
+  private async $appendMempoolFeeData(transaction: IEsploraApi.Transaction): Promise<IEsploraApi.Transaction> {
+    if (transaction.fee) {
+      return transaction;
+    }
+    let mempoolEntry: IBitcoinApi.MempoolEntry;
+    if (!mempool.isInSync() && !this.rawMempoolCache) {
+      this.rawMempoolCache = await this.$getRawMempoolVerbose();
+    }
+    if (this.rawMempoolCache && this.rawMempoolCache[transaction.txid]) {
+      mempoolEntry = this.rawMempoolCache[transaction.txid];
+    } else {
+      mempoolEntry = await this.$getMempoolEntry(transaction.txid);
+    }
+    transaction.fee = Math.round(mempoolEntry.fees.base * 100000000);
+    return transaction;
+  }
+
+  protected async $addPrevouts(transaction: TransactionExtended): Promise<TransactionExtended> {
+    for (const vin of transaction.vin) {
+      if (vin.prevout) {
+        continue;
+      }
+      const innerTx = await this.$getRawTransaction(vin.txid, false, false);
+      vin.prevout = innerTx.vout[vin.vout];
+      this.addInnerScriptsToVin(vin);
+    }
+    return transaction;
+  }
+
+  protected $returnCoinbaseTransaction(): Promise<IEsploraApi.Transaction> {
+    return this.bitcoindClient.getBlockHash(0).then((hash: string) =>
+      this.bitcoindClient.getBlock(hash, 2)
+        .then((block: IBitcoinApi.Block) => {
+          return this.$convertTransaction(Object.assign(block.tx[0], {
+            confirmations: blocks.getCurrentBlockHeight() + 1,
+            blocktime: block.time }), false);
+        })
+    );
+  }
+
+  private $getMempoolEntry(txid: string): Promise<IBitcoinApi.MempoolEntry> {
+    return this.bitcoindClient.getMempoolEntry(txid);
+  }
+
+  private $getRawMempoolVerbose(): Promise<IBitcoinApi.RawMempool> {
+    return this.bitcoindClient.getRawMemPool(true);
+  }
+
+
+  private async $calculateFeeFromInputs(transaction: IEsploraApi.Transaction, addPrevout: boolean, lazyPrevouts: boolean): Promise<IEsploraApi.Transaction> {
+    if (transaction.vin[0].is_coinbase) {
+      transaction.fee = 0;
+      return transaction;
+    }
+    let totalIn = 0;
+
+    for (let i = 0; i < transaction.vin.length; i++) {
+      if (lazyPrevouts && i > 12) {
+        transaction.vin[i].lazy = true;
+        continue;
+      }
+      const innerTx = await this.$getRawTransaction(transaction.vin[i].txid, false, false);
+      transaction.vin[i].prevout = innerTx.vout[transaction.vin[i].vout];
+      this.addInnerScriptsToVin(transaction.vin[i]);
+      totalIn += innerTx.vout[transaction.vin[i].vout].value;
+    }
+    if (lazyPrevouts && transaction.vin.length > 12) {
+      transaction.fee = -1;
+    } else {
+      const totalOut = transaction.vout.reduce((p, output) => p + output.value, 0);
+      transaction.fee = parseFloat((totalIn - totalOut).toFixed(8));
+    }
+    return transaction;
+  }
+
+  private convertScriptSigAsm(hex: string): string {
+    const buf = Buffer.from(hex, 'hex');
+
+    const b: string[] = [];
+
+    let i = 0;
+    while (i < buf.length) {
+      const op = buf[i];
+      if (op >= 0x01 && op <= 0x4e) {
+        i++;
+        let push: number;
+        if (op === 0x4c) {
+          push = buf.readUInt8(i);
+          b.push('OP_PUSHDATA1');
+          i += 1;
+        } else if (op === 0x4d) {
+          push = buf.readUInt16LE(i);
+          b.push('OP_PUSHDATA2');
+          i += 2;
+        } else if (op === 0x4e) {
+          push = buf.readUInt32LE(i);
+          b.push('OP_PUSHDATA4');
+          i += 4;
+        } else {
+          push = op;
+          b.push('OP_PUSHBYTES_' + push);
+        }
+
+        const data = buf.slice(i, i + push);
+        if (data.length !== push) {
+          break;
+        }
+
+        b.push(data.toString('hex'));
+        i += data.length;
+      } else {
+        if (op === 0x00) {
+          b.push('OP_0');
+        } else if (op === 0x4f) {
+          b.push('OP_PUSHNUM_NEG1');
+        } else if (op === 0xb1) {
+          b.push('OP_CLTV');
+        } else if (op === 0xb2) {
+          b.push('OP_CSV');
+        } else if (op === 0xba) {
+          b.push('OP_CHECKSIGADD');
+        } else {
+          const opcode = bitcoinjs.script.toASM([ op ]);
+          if (opcode && op < 0xfd) {
+            if (/^OP_(\d+)$/.test(opcode)) {
+              b.push(opcode.replace(/^OP_(\d+)$/, 'OP_PUSHNUM_$1'));
+            } else {
+              b.push(opcode);
+            }
+          } else {
+            b.push('OP_RETURN_' + op);
+          }
+        }
+        i += 1;
+      }
+    }
+
+    return b.join(' ');
+  }
+
+  private addInnerScriptsToVin(vin: IEsploraApi.Vin): void {
+    if (!vin.prevout) {
+      return;
+    }
+
+    if (vin.prevout.scriptpubkey_type === 'p2sh') {
+      const redeemScript = vin.scriptsig_asm.split(' ').reverse()[0];
+      vin.inner_redeemscript_asm = this.convertScriptSigAsm(redeemScript);
+      if (vin.witness && vin.witness.length > 2) {
+        const witnessScript = vin.witness[vin.witness.length - 1];
+        vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript);
+      }
+    }
+
+    if (vin.prevout.scriptpubkey_type === 'v0_p2wsh' && vin.witness) {
+      const witnessScript = vin.witness[vin.witness.length - 1];
+      vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript);
+    }
+
+    if (vin.prevout.scriptpubkey_type === 'v1_p2tr' && vin.witness && vin.witness.length > 1) {
+      const witnessScript = vin.witness[vin.witness.length - 2];
+      vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript);
+    }
+  }
+
+}
+
+export default BitcoinApi;

backend/src/api/bitcoin/bitcoin-client.ts

@@ -0,0 +1,13 @@
+import config from '../../config';
+const bitcoin = require('../../rpc-api/index');
+import { BitcoinRpcCredentials } from './bitcoin-api-abstract-factory';
+
+const nodeRpcCredentials: BitcoinRpcCredentials = {
+  host: config.CORE_RPC.HOST,
+  port: config.CORE_RPC.PORT,
+  user: config.CORE_RPC.USERNAME,
+  pass: config.CORE_RPC.PASSWORD,
+  timeout: 60000,
+};
+
+export default new bitcoin.Client(nodeRpcCredentials);

backend/src/api/bitcoin/bitcoin-second-client.ts

@@ -0,0 +1,13 @@
+import config from '../../config';
+const bitcoin = require('../../rpc-api/index');
+import { BitcoinRpcCredentials } from './bitcoin-api-abstract-factory';
+
+const nodeRpcCredentials: BitcoinRpcCredentials = {
+  host: config.SECOND_CORE_RPC.HOST,
+  port: config.SECOND_CORE_RPC.PORT,
+  user: config.SECOND_CORE_RPC.USERNAME,
+  pass: config.SECOND_CORE_RPC.PASSWORD,
+  timeout: 60000,
+};
+
+export default new bitcoin.Client(nodeRpcCredentials);

backend/src/api/bitcoin/bitcoind-api.ts

@@ -1,110 +0,0 @@
-const config = require('../../../mempool-config.json');
-import * as bitcoin from 'bitcoin';
-import { ITransaction, IMempoolInfo, IBlock } from '../../interfaces';
-import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
-
-class BitcoindApi implements AbstractBitcoinApi {
-  client: any;
-
-  constructor() {
-    this.client = new bitcoin.Client({
-      host: config.BITCOIN_NODE_HOST,
-      port: config.BITCOIN_NODE_PORT,
-      user: config.BITCOIN_NODE_USER,
-      pass: config.BITCOIN_NODE_PASS,
-    });
-  }
-
-  getMempoolInfo(): Promise<IMempoolInfo> {
-    return new Promise((resolve, reject) => {
-      this.client.getMempoolInfo((err: Error, mempoolInfo: any) => {
-        if (err) {
-          return reject(err);
-        }
-        resolve(mempoolInfo);
-      });
-    });
-  }
-
-  getRawMempool(): Promise<ITransaction['txid'][]> {
-    return new Promise((resolve, reject) => {
-      this.client.getRawMemPool((err: Error, transactions: ITransaction['txid'][]) => {
-        if (err) {
-          return reject(err);
-        }
-        resolve(transactions);
-      });
-    });
-  }
-
-  getRawTransaction(txId: string): Promise<ITransaction> {
-    return new Promise((resolve, reject) => {
-      this.client.getRawTransaction(txId, true, (err: Error, txData: ITransaction) => {
-        if (err) {
-          return reject(err);
-        }
-        resolve(txData);
-      });
-    });
-  }
-
-  getBlockCount(): Promise<number> {
-    return new Promise((resolve, reject) => {
-      this.client.getBlockCount((err: Error, response: number) => {
-        if (err) {
-          return reject(err);
-        }
-        resolve(response);
-      });
-    });
-  }
-
-  getBlockAndTransactions(hash: string, verbosity:  1 | 2 = 1): Promise<IBlock> {
-    return new Promise((resolve, reject) => {
-      this.client.getBlock(hash, verbosity, (err: Error, block: IBlock) => {
-        if (err) {
-          return reject(err);
-        }
-        resolve(block);
-      });
-    });
-  }
-
-  getBlockHash(height: number): Promise<string> {
-    return new Promise((resolve, reject) => {
-      this.client.getBlockHash(height, (err: Error, response: string) => {
-        if (err) {
-          return reject(err);
-        }
-        resolve(response);
-      });
-    });
-  }
-
-  getBlock(hash: string): Promise<IBlock> {
-    throw new Error('Method not implemented.');
-  }
-  getBlocks(): Promise<string> {
-    throw new Error('Method not implemented.');
-  }
-  getBlocksFromHeight(height: number): Promise<string> {
-    throw new Error('Method not implemented.');
-  }
-  getBlockTransactions(hash: string): Promise<IBlock> {
-    throw new Error('Method not implemented.');
-  }
-  getBlockTransactionsFromIndex(hash: string, index: number): Promise<IBlock> {
-    throw new Error('Method not implemented.');
-  }
-  getAddress(address: string): Promise<IBlock> {
-    throw new Error('Method not implemented.');
-  }
-  getAddressTransactions(address: string): Promise<IBlock> {
-    throw new Error('Method not implemented.');
-  }
-  getAddressTransactionsFromLastSeenTxid(address: string, lastSeenTxid: string): Promise<IBlock> {
-    throw new Error('Method not implemented.');
-  }
-}
-
-export default BitcoindApi;

backend/src/api/bitcoin/electrs-api.ts

@@ -1,229 +0,0 @@
-const config = require('../../../mempool-config.json');
-import { ITransaction, IMempoolInfo, IBlock } from '../../interfaces';
-import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
-import * as request from 'request';
-
-class ElectrsApi implements AbstractBitcoinApi {
-
-  constructor() {
-  }
-
-  getMempoolInfo(): Promise<IMempoolInfo> {
-    return new Promise((resolve, reject) => {
-      request(config.ELECTRS_API_URL + '/mempool', { json: true, timeout: 10000 }, (err, res, response) => {
-        if (err) {
-          reject(err);
-        } else if (res.statusCode !== 200) {
-          reject(response);
-        } else {
-          resolve({
-            size: response.count,
-            bytes: response.vsize,
-          });
-        }
-      });
-    });
-  }
-
-  getRawMempool(): Promise<ITransaction['txid'][]> {
-    return new Promise((resolve, reject) => {
-      request(config.ELECTRS_API_URL + '/mempool/txids', { json: true, timeout: 10000 }, (err, res, response) => {
-        if (err) {
-          reject(err);
-        } else if (res.statusCode !== 200) {
-          reject(response);
-        } else {
-          resolve(response);
-        }
-      });
-    });
-  }
-
-  getRawTransaction(txId: string): Promise<ITransaction> {
-    return new Promise((resolve, reject) => {
-      request(config.ELECTRS_API_URL + '/tx/' + txId, { json: true, timeout: 10000 }, (err, res, response) => {
-        if (err) {
-          reject(err);
-        } else if (res.statusCode !== 200) {
-          reject(response);
-        } else {
-          response.vsize = Math.round(response.weight / 4);
-          response.fee = response.fee / 100000000;
-          response.blockhash = response.status.block_hash;
-          resolve(response);
-        }
-      });
-    });
-  }
-
-  getBlockCount(): Promise<number> {
-    return new Promise((resolve, reject) => {
-      request(config.ELECTRS_API_URL + '/blocks/tip/height', { json: true, timeout: 10000 }, (err, res, response) => {
-        if (err) {
-          reject(err);
-        } else if (res.statusCode !== 200) {
-          reject(response);
-        } else {
-          resolve(response);
-        }
-      });
-    });
-  }
-
-  getBlockAndTransactions(hash: string): Promise<IBlock> {
-    return new Promise((resolve, reject) => {
-      request(config.ELECTRS_API_URL + '/block/' + hash, { json: true, timeout: 10000 }, (err, res, response) => {
-        if (err) {
-          reject(err);
-        } else if (res.statusCode !== 200) {
-          reject(response);
-        } else {
-          request(config.ELECTRS_API_URL + '/block/' + hash + '/txids', { json: true, timeout: 10000 }, (err2, res2, response2) => {
-            if (err2) {
-              reject(err2);
-            } else if (res.statusCode !== 200) {
-              reject(response);
-            } else {
-              const block = response;
-              block.hash = hash;
-              block.nTx = block.tx_count;
-              block.time = block.timestamp;
-              block.tx = response2;
-
-              resolve(block);
-            }
-          });
-        }
-      });
-    });
-  }
-
-  getBlockHash(height: number): Promise<string> {
-    return new Promise((resolve, reject) => {
-      request(config.ELECTRS_API_URL + '/block-height/' + height, { json: true, timeout: 10000 }, (err, res, response) => {
-        if (err) {
-          reject(err);
-        } else if (res.statusCode !== 200) {
-          reject(response);
-        } else  {
-          resolve(response);
-        }
-      });
-    });
-  }
-
-  getBlocks(): Promise<string> {
-    return new Promise((resolve, reject) => {
-      request(config.ELECTRS_API_URL + '/blocks', { json: true, timeout: 10000 }, (err, res, response) => {
-        if (err) {
-          reject(err);
-        } else if (res.statusCode !== 200) {
-          reject(response);
-        } else  {
-          resolve(response);
-        }
-      });
-    });
-  }
-
-  getBlocksFromHeight(height: number): Promise<string> {
-    return new Promise((resolve, reject) => {
-      request(config.ELECTRS_API_URL + '/blocks/' + height, { json: true, timeout: 10000 }, (err, res, response) => {
-        if (err) {
-          reject(err);
-        } else if (res.statusCode !== 200) {
-          reject(response);
-        } else  {
-          resolve(response);
-        }
-      });
-    });
-  }
-
-  getBlock(hash: string): Promise<IBlock> {
-    return new Promise((resolve, reject) => {
-      request(config.ELECTRS_API_URL + '/block/' + hash, { json: true, timeout: 10000 }, (err, res, response) => {
-        if (err) {
-          reject(err);
-        } else if (res.statusCode !== 200) {
-          reject(response);
-        } else  {
-          resolve(response);
-        }
-      });
-    });
-  }
-
-  getBlockTransactions(hash: string): Promise<IBlock> {
-    return new Promise((resolve, reject) => {
-      request(config.ELECTRS_API_URL + '/block/' + hash + '/txs', { json: true, timeout: 10000 }, (err, res, response) => {
-        if (err) {
-          reject(err);
-        } else if (res.statusCode !== 200) {
-          reject(response);
-        } else  {
-          resolve(response);
-        }
-      });
-    });
-  }
-
-  getBlockTransactionsFromIndex(hash: string, index: number): Promise<IBlock> {
-    return new Promise((resolve, reject) => {
-      request(config.ELECTRS_API_URL + '/block/' + hash + '/txs/' + index, { json: true, timeout: 10000 }, (err, res, response) => {
-        if (err) {
-          reject(err);
-        } else if (res.statusCode !== 200) {
-          reject(response);
-        } else  {
-          resolve(response);
-        }
-      });
-    });
-  }
-
-  getAddress(address: string): Promise<IBlock> {
-    return new Promise((resolve, reject) => {
-      request(config.ELECTRS_API_URL + '/address/' + address, { json: true, timeout: 10000 }, (err, res, response) => {
-        if (err) {
-          reject(err);
-        } else if (res.statusCode !== 200) {
-          reject(response);
-        } else  {
-          resolve(response);
-        }
-      });
-    });
-  }
-
-  getAddressTransactions(address: string): Promise<IBlock> {
-    return new Promise((resolve, reject) => {
-      request(config.ELECTRS_API_URL + '/address/' + address + '/txs', { json: true, timeout: 10000 }, (err, res, response) => {
-        if (err) {
-          reject(err);
-        } else if (res.statusCode !== 200) {
-          reject(response);
-        } else  {
-          resolve(response);
-        }
-      });
-    });
-  }
-
-  getAddressTransactionsFromLastSeenTxid(address: string, lastSeenTxid: string): Promise<IBlock> {
-    return new Promise((resolve, reject) => {
-      request(config.ELECTRS_API_URL + '/address/' + address + '/txs/chain/' + lastSeenTxid,
-        { json: true, timeout: 10000 }, (err, res, response) => {
-        if (err) {
-          reject(err);
-        } else if (res.statusCode !== 200) {
-          reject(response);
-        } else  {
-          resolve(response);
-        }
-      });
-    });
-  }
-}
-
-export default ElectrsApi;

backend/src/api/bitcoin/electrum-api.interface.ts

@@ -0,0 +1,12 @@
+export namespace IElectrumApi {
+  export interface ScriptHashBalance {
+    confirmed: number;
+    unconfirmed: number;
+  }
+
+  export interface ScriptHashHistory {
+    height: number;
+    tx_hash: string;
+    fee?: number;
+  }
+}

backend/src/api/bitcoin/electrum-api.ts

@@ -0,0 +1,153 @@
+import config from '../../config';
+import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
+import { IEsploraApi } from './esplora-api.interface';
+import { IElectrumApi } from './electrum-api.interface';
+import BitcoinApi from './bitcoin-api';
+import logger from '../../logger';
+import * as ElectrumClient from '@mempool/electrum-client';
+import * as sha256 from 'crypto-js/sha256';
+import * as hexEnc from 'crypto-js/enc-hex';
+import loadingIndicators from '../loading-indicators';
+import memoryCache from '../memory-cache';
+
+class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
+  private electrumClient: any;
+
+  constructor(bitcoinClient: any) {
+    super(bitcoinClient);
+
+    const electrumConfig = { client: 'mempool-v2', version: '1.4' };
+    const electrumPersistencePolicy = { retryPeriod: 10000, maxRetry: 1000, callback: null };
+
+    const electrumCallbacks = {
+      onConnect: (client, versionInfo) => { logger.info(`Connected to Electrum Server at ${config.ELECTRUM.HOST}:${config.ELECTRUM.PORT} (${JSON.stringify(versionInfo)})`); },
+      onClose: (client) => { logger.info(`Disconnected from Electrum Server at ${config.ELECTRUM.HOST}:${config.ELECTRUM.PORT}`); },
+      onError: (err) => { logger.err(`Electrum error: ${JSON.stringify(err)}`); },
+      onLog: (str) => { logger.debug(str); },
+    };
+
+    this.electrumClient = new ElectrumClient(
+      config.ELECTRUM.PORT,
+      config.ELECTRUM.HOST,
+      config.ELECTRUM.TLS_ENABLED ? 'tls' : 'tcp',
+      null,
+      electrumCallbacks
+    );
+
+    this.electrumClient.initElectrum(electrumConfig, electrumPersistencePolicy)
+      .then(() => {})
+      .catch((err) => {
+        logger.err(`Error connecting to Electrum Server at ${config.ELECTRUM.HOST}:${config.ELECTRUM.PORT}`);
+      });
+  }
+
+  async $getAddress(address: string): Promise<IEsploraApi.Address> {
+    const addressInfo = await this.bitcoindClient.validateAddress(address);
+    if (!addressInfo || !addressInfo.isvalid) {
+      return ({
+        'address': address,
+        'chain_stats': {
+          'funded_txo_count': 0,
+          'funded_txo_sum': 0,
+          'spent_txo_count': 0,
+          'spent_txo_sum': 0,
+          'tx_count': 0
+        },
+        'mempool_stats': {
+          'funded_txo_count': 0,
+          'funded_txo_sum': 0,
+          'spent_txo_count': 0,
+          'spent_txo_sum': 0,
+          'tx_count': 0
+        }
+      });
+    }
+
+    try {
+      const balance = await this.$getScriptHashBalance(addressInfo.scriptPubKey);
+      const history = await this.$getScriptHashHistory(addressInfo.scriptPubKey);
+
+      const unconfirmed = history.filter((h) => h.fee).length;
+
+      return {
+        'address': addressInfo.address,
+        'chain_stats': {
+          'funded_txo_count': 0,
+          'funded_txo_sum': balance.confirmed ? balance.confirmed : 0,
+          'spent_txo_count': 0,
+          'spent_txo_sum': balance.confirmed < 0 ? balance.confirmed : 0,
+          'tx_count': history.length - unconfirmed,
+        },
+        'mempool_stats': {
+          'funded_txo_count': 0,
+          'funded_txo_sum': balance.unconfirmed > 0 ? balance.unconfirmed : 0,
+          'spent_txo_count': 0,
+          'spent_txo_sum': balance.unconfirmed < 0 ? -balance.unconfirmed : 0,
+          'tx_count': unconfirmed,
+        },
+        'electrum': true,
+      };
+    } catch (e: any) {
+      throw new Error(typeof e === 'string' ? e : e && e.message || e);
+    }
+  }
+
+  async $getAddressTransactions(address: string, lastSeenTxId: string): Promise<IEsploraApi.Transaction[]> {
+    const addressInfo = await this.bitcoindClient.validateAddress(address);
+    if (!addressInfo || !addressInfo.isvalid) {
+     return [];
+    }
+
+    try {
+      loadingIndicators.setProgress('address-' + address, 0);
+
+      const transactions: IEsploraApi.Transaction[] = [];
+      const history = await this.$getScriptHashHistory(addressInfo.scriptPubKey);
+      history.sort((a, b) => (b.height || 9999999) - (a.height || 9999999));
+
+      let startingIndex = 0;
+      if (lastSeenTxId) {
+        const pos = history.findIndex((historicalTx) => historicalTx.tx_hash === lastSeenTxId);
+        if (pos) {
+          startingIndex = pos + 1;
+        }
+      }
+      const endIndex = Math.min(startingIndex + 10, history.length);
+
+      for (let i = startingIndex; i < endIndex; i++) {
+        const tx = await this.$getRawTransaction(history[i].tx_hash, false, true);
+        transactions.push(tx);
+        loadingIndicators.setProgress('address-' + address, (i + 1) / endIndex * 100);
+      }
+
+      return transactions;
+    } catch (e: any) {
+      loadingIndicators.setProgress('address-' + address, 100);
+      throw new Error(typeof e === 'string' ? e : e && e.message || e);
+    }
+  }
+
+  private $getScriptHashBalance(scriptHash: string): Promise<IElectrumApi.ScriptHashBalance> {
+    return this.electrumClient.blockchainScripthash_getBalance(this.encodeScriptHash(scriptHash));
+  }
+
+  private $getScriptHashHistory(scriptHash: string): Promise<IElectrumApi.ScriptHashHistory[]> {
+    const fromCache = memoryCache.get<IElectrumApi.ScriptHashHistory[]>('Scripthash_getHistory', scriptHash);
+    if (fromCache) {
+      return Promise.resolve(fromCache);
+    }
+    return this.electrumClient.blockchainScripthash_getHistory(this.encodeScriptHash(scriptHash))
+      .then((history) => {
+        memoryCache.set('Scripthash_getHistory', scriptHash, history, 2);
+        return history;
+      });
+  }
+
+  private encodeScriptHash(scriptPubKey: string): string {
+    const addrScripthash = hexEnc.stringify(sha256(hexEnc.parse(scriptPubKey)));
+    return addrScripthash.match(/.{2}/g).reverse().join('');
+  }
+
+}
+
+export default BitcoindElectrsApi;

backend/src/api/bitcoin/esplora-api.interface.ts

@@ -0,0 +1,172 @@
+export namespace IEsploraApi {
+  export interface Transaction {
+    txid: string;
+    version: number;
+    locktime: number;
+    size: number;
+    weight: number;
+    fee: number;
+    vin: Vin[];
+    vout: Vout[];
+    status: Status;
+    hex?: string;
+  }
+
+  export interface Recent {
+    txid: string;
+    fee: number;
+    vsize: number;
+    value: number;
+  }
+
+  export interface Vin {
+    txid: string;
+    vout: number;
+    is_coinbase: boolean;
+    scriptsig: string;
+    scriptsig_asm: string;
+    inner_redeemscript_asm?: string;
+    inner_witnessscript_asm?: string;
+    sequence: any;
+    witness?: string[];
+    prevout: Vout | null;
+    // Elements
+    is_pegin?: boolean;
+    issuance?: Issuance;
+    // Custom
+    lazy?: boolean;
+  }
+
+  interface Issuance {
+    asset_id: string;
+    is_reissuance: string;
+    asset_blinding_nonce: string;
+    asset_entropy: string;
+    contract_hash: string;
+    assetamount?: number;
+    assetamountcommitment?: string;
+    tokenamount?: number;
+    tokenamountcommitment?: string;
+  }
+
+  export interface Vout {
+    scriptpubkey: string;
+    scriptpubkey_asm: string;
+    scriptpubkey_type: string;
+    scriptpubkey_address: string;
+    value: number;
+    // Elements
+    valuecommitment?: number;
+    asset?: string;
+    pegout?: Pegout;
+  }
+
+  interface Pegout {
+    genesis_hash: string;
+    scriptpubkey: string;
+    scriptpubkey_asm: string;
+    scriptpubkey_address: string;
+  }
+
+  export interface Status {
+    confirmed: boolean;
+    block_height?: number;
+    block_hash?: string;
+    block_time?: number;
+  }
+
+  export interface Block {
+    id: string;
+    height: number;
+    version: number;
+    timestamp: number;
+    bits: number;
+    nonce: number;
+    difficulty: number;
+    merkle_root: string;
+    tx_count: number;
+    size: number;
+    weight: number;
+    previousblockhash: string;
+  }
+
+  export interface Address {
+    address: string;
+    chain_stats: ChainStats;
+    mempool_stats: MempoolStats;
+    electrum?: boolean;
+  }
+
+  export interface ChainStats {
+    funded_txo_count: number;
+    funded_txo_sum: number;
+    spent_txo_count: number;
+    spent_txo_sum: number;
+    tx_count: number;
+  }
+
+  export interface MempoolStats {
+    funded_txo_count: number;
+    funded_txo_sum: number;
+    spent_txo_count: number;
+    spent_txo_sum: number;
+    tx_count: number;
+  }
+
+  export interface Outspend {
+    spent: boolean;
+    txid?: string;
+    vin?: number;
+    status?: Status;
+  }
+
+  export interface Asset {
+    asset_id: string;
+    issuance_txin: IssuanceTxin;
+    issuance_prevout: IssuancePrevout;
+    reissuance_token: string;
+    contract_hash: string;
+    status: Status;
+    chain_stats: AssetStats;
+    mempool_stats: AssetStats;
+  }
+
+  export interface AssetExtended extends Asset {
+    name: string;
+    ticker: string;
+    precision: number;
+    entity: Entity;
+    version: number;
+    issuer_pubkey: string;
+  }
+
+  export interface Entity {
+    domain: string;
+  }
+
+  interface IssuanceTxin {
+    txid: string;
+    vin: number;
+  }
+
+  interface IssuancePrevout {
+    txid: string;
+    vout: number;
+  }
+
+  interface AssetStats {
+    tx_count: number;
+    issuance_count: number;
+    issued_amount: number;
+    burned_amount: number;
+    has_blinded_issuances: boolean;
+    reissuance_tokens: number;
+    burned_reissuance_tokens: number;
+    peg_in_count: number;
+    peg_in_amount: number;
+    peg_out_count: number;
+    peg_out_amount: number;
+    burn_count: number;
+  }
+
+}

backend/src/api/bitcoin/esplora-api.ts

@@ -0,0 +1,69 @@
+import config from '../../config';
+import axios, { AxiosRequestConfig } from 'axios';
+import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
+import { IEsploraApi } from './esplora-api.interface';
+
+class ElectrsApi implements AbstractBitcoinApi {
+  axiosConfig: AxiosRequestConfig = {
+    timeout: 10000,
+  };
+
+  constructor() { }
+
+  $getRawMempool(): Promise<IEsploraApi.Transaction['txid'][]> {
+    return axios.get<IEsploraApi.Transaction['txid'][]>(config.ESPLORA.REST_API_URL + '/mempool/txids', this.axiosConfig)
+      .then((response) => response.data);
+  }
+
+  $getRawTransaction(txId: string): Promise<IEsploraApi.Transaction> {
+    return axios.get<IEsploraApi.Transaction>(config.ESPLORA.REST_API_URL + '/tx/' + txId, this.axiosConfig)
+      .then((response) => response.data);
+  }
+
+  $getBlockHeightTip(): Promise<number> {
+    return axios.get<number>(config.ESPLORA.REST_API_URL + '/blocks/tip/height', this.axiosConfig)
+      .then((response) => response.data);
+  }
+
+  $getTxIdsForBlock(hash: string): Promise<string[]> {
+    return axios.get<string[]>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/txids', this.axiosConfig)
+      .then((response) => response.data);
+  }
+
+  $getBlockHash(height: number): Promise<string> {
+    return axios.get<string>(config.ESPLORA.REST_API_URL + '/block-height/' + height, this.axiosConfig)
+      .then((response) => response.data);
+  }
+
+  $getBlockHeader(hash: string): Promise<string> {
+    return axios.get<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/header', this.axiosConfig)
+      .then((response) => response.data);
+  }
+
+  $getBlock(hash: string): Promise<IEsploraApi.Block> {
+    return axios.get<IEsploraApi.Block>(config.ESPLORA.REST_API_URL + '/block/' + hash, this.axiosConfig)
+      .then((response) => response.data);
+  }
+
+  $getAddress(address: string): Promise<IEsploraApi.Address> {
+    throw new Error('Method getAddress not implemented.');
+  }
+
+  $getAddressTransactions(address: string, txId?: string): Promise<IEsploraApi.Transaction[]> {
+    throw new Error('Method getAddressTransactions not implemented.');
+  }
+
+  $getAddressPrefix(prefix: string): string[] {
+    throw new Error('Method not implemented.');
+  }
+
+  $sendRawTransaction(rawTransaction: string): Promise<string> {
+    throw new Error('Method not implemented.');
+  }
+
+  $getOutspends(): Promise<IEsploraApi.Outspend[]> {
+    throw new Error('Method not implemented.');
+  }
+}
+
+export default ElectrsApi;

backend/src/api/blocks.ts

@@ -1,214 +1,499 @@
-const config = require('../../mempool-config.json');
+import config from '../config';
 import bitcoinApi from './bitcoin/bitcoin-api-factory';
-import { DB } from '../database';
-import { IBlock, ITransaction } from '../interfaces';
+import logger from '../logger';
 import memPool from './mempool';
+import { BlockExtended, PoolTag, TransactionExtended, TransactionMinerInfo } from '../mempool.interfaces';
+import { Common } from './common';
+import diskCache from './disk-cache';
+import transactionUtils from './transaction-utils';
+import bitcoinClient from './bitcoin/bitcoin-client';
+import { IEsploraApi } from './bitcoin/esplora-api.interface';
+import poolsRepository from '../repositories/PoolsRepository';
+import blocksRepository from '../repositories/BlocksRepository';
+import loadingIndicators from './loading-indicators';
+import BitcoinApi from './bitcoin/bitcoin-api';
+import { prepareBlock } from '../utils/blocks-utils';
+import BlocksRepository from '../repositories/BlocksRepository';
+import HashratesRepository from '../repositories/HashratesRepository';
+import indexer from '../indexer';
+import fiatConversion from './fiat-conversion';
+import RatesRepository from '../repositories/RatesRepository';
+import poolsParser from './pools-parser';
 
 class Blocks {
-  private blocks: IBlock[] = [];
-  private newBlockCallback: Function | undefined;
+  private blocks: BlockExtended[] = [];
   private currentBlockHeight = 0;
+  private currentDifficulty = 0;
+  private lastDifficultyAdjustmentTime = 0;
+  private previousDifficultyRetarget = 0;
+  private newBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => void)[] = [];
 
-  constructor() {
-    setInterval(this.$clearOldTransactionsAndBlocksFromDatabase.bind(this), 86400000);
-  }
+  constructor() { }
 
-  public setNewBlockCallback(fn: Function) {
-    this.newBlockCallback = fn;
-  }
-
-  public getBlocks(): IBlock[] {
+  public getBlocks(): BlockExtended[] {
     return this.blocks;
   }
 
-  public formatBlock(block: IBlock) {
-    return {
-      hash: block.hash,
-      height: block.height,
-      nTx: block.nTx - 1,
-      size: block.size,
-      time: block.time,
-      weight: block.weight,
-      fees: block.fees,
-      minFee: block.minFee,
-      maxFee: block.maxFee,
-      medianFee: block.medianFee,
-    };
+  public setBlocks(blocks: BlockExtended[]) {
+    this.blocks = blocks;
   }
 
-  public async updateBlocks() {
-    try {
-      const blockCount = await bitcoinApi.getBlockCount();
+  public setNewBlockCallback(fn: (block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => void) {
+    this.newBlockCallbacks.push(fn);
+  }
 
-      if (this.blocks.length === 0) {
-        this.currentBlockHeight = blockCount - config.INITIAL_BLOCK_AMOUNT;
-      } else {
-        this.currentBlockHeight = this.blocks[this.blocks.length - 1].height;
+  /**
+   * Return the list of transaction for a block
+   * @param blockHash
+   * @param blockHeight
+   * @param onlyCoinbase - Set to true if you only need the coinbase transaction
+   * @returns Promise<TransactionExtended[]>
+   */
+  private async $getTransactionsExtended(
+    blockHash: string,
+    blockHeight: number,
+    onlyCoinbase: boolean,
+    quiet: boolean = false,
+  ): Promise<TransactionExtended[]> {
+    const transactions: TransactionExtended[] = [];
+    const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash);
+
+    const mempool = memPool.getMempool();
+    let transactionsFound = 0;
+    let transactionsFetched = 0;
+
+    for (let i = 0; i < txIds.length; i++) {
+      if (mempool[txIds[i]]) {
+        // We update blocks before the mempool (index.ts), therefore we can
+        // optimize here by directly fetching txs in the "outdated" mempool
+        transactions.push(mempool[txIds[i]]);
+        transactionsFound++;
+      } else if (config.MEMPOOL.BACKEND === 'esplora' || !memPool.hasPriority() || i === 0) {
+        // Otherwise we fetch the tx data through backend services (esplora, electrum, core rpc...)
+        if (!quiet && (i % (Math.round((txIds.length) / 10)) === 0 || i + 1 === txIds.length)) { // Avoid log spam
+          logger.debug(`Indexing tx ${i + 1} of ${txIds.length} in block #${blockHeight}`);
+        }
+        try {
+          const tx = await transactionUtils.$getTransactionExtended(txIds[i]);
+          transactions.push(tx);
+          transactionsFetched++;
+        } catch (e) {
+          if (i === 0) {
+            const msg = `Cannot fetch coinbase tx ${txIds[i]}. Reason: ` + (e instanceof Error ? e.message : e); 
+            logger.err(msg);
+            throw new Error(msg);
+          } else {
+            logger.err(`Cannot fetch tx ${txIds[i]}. Reason: ` + (e instanceof Error ? e.message : e));
+          }
+        }
       }
 
-      while (this.currentBlockHeight < blockCount) {
-        this.currentBlockHeight++;
+      if (onlyCoinbase === true) {
+        break; // Fetch the first transaction and exit
+      }
+    }
 
-        let block: IBlock | undefined;
+    transactions.forEach((tx) => {
+      if (!tx.cpfpChecked) {
+        Common.setRelativesAndGetCpfpInfo(tx, mempool); // Child Pay For Parent
+      }
+    });
 
-        const storedBlock = await this.$getBlockFromDatabase(this.currentBlockHeight);
-        if (storedBlock) {
-          block = storedBlock;
+    if (!quiet) {
+      logger.debug(`${transactionsFound} of ${txIds.length} found in mempool. ${transactionsFetched} fetched through backend service.`);
+    }
+
+    return transactions;
+  }
+
+  /**
+   * Return a block with additional data (reward, coinbase, fees...)
+   * @param block
+   * @param transactions
+   * @returns BlockExtended
+   */
+  private async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise<BlockExtended> {
+    const blockExtended: BlockExtended = Object.assign({ extras: {} }, block);
+    blockExtended.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
+    blockExtended.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
+    blockExtended.extras.coinbaseRaw = blockExtended.extras.coinbaseTx.vin[0].scriptsig;
+
+    if (block.height === 0) {
+      blockExtended.extras.medianFee = 0; // 50th percentiles
+      blockExtended.extras.feeRange = [0, 0, 0, 0, 0, 0, 0];
+      blockExtended.extras.totalFees = 0;
+      blockExtended.extras.avgFee = 0;
+      blockExtended.extras.avgFeeRate = 0;
+    } else {
+      const stats = await bitcoinClient.getBlockStats(block.id, [
+        'feerate_percentiles', 'minfeerate', 'maxfeerate', 'totalfee', 'avgfee', 'avgfeerate'
+      ]);
+      blockExtended.extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles
+      blockExtended.extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat();
+      blockExtended.extras.totalFees = stats.totalfee;
+      blockExtended.extras.avgFee = stats.avgfee;
+      blockExtended.extras.avgFeeRate = stats.avgfeerate;
+    }
+
+    if (['mainnet', 'testnet', 'signet', 'regtest'].includes(config.MEMPOOL.NETWORK)) {
+      let pool: PoolTag;
+      if (blockExtended.extras?.coinbaseTx !== undefined) {
+        pool = await this.$findBlockMiner(blockExtended.extras?.coinbaseTx);
+      } else {
+        if (config.DATABASE.ENABLED === true) {
+          pool = await poolsRepository.$getUnknownPool();
         } else {
-          const blockHash = await bitcoinApi.getBlockHash(this.currentBlockHeight);
-          block = await bitcoinApi.getBlockAndTransactions(blockHash);
+          pool = poolsParser.unknownPool;
+        }
+      }
 
-          const coinbase = await memPool.getRawTransaction(block.tx[0], true);
-          if (coinbase && coinbase.totalOut) {
-            block.fees = coinbase.totalOut;
+      if (!pool) { // We should never have this situation in practise
+        logger.warn(`Cannot assign pool to block ${blockExtended.height} and 'unknown' pool does not exist. ` +
+          `Check your "pools" table entries`);
+        return blockExtended;
+      }
+
+      blockExtended.extras.pool = {
+        id: pool.id,
+        name: pool.name,
+        slug: pool.slug,
+      };
+    }
+
+    return blockExtended;
+  }
+
+  /**
+   * Try to find which miner found the block
+   * @param txMinerInfo
+   * @returns
+   */
+  private async $findBlockMiner(txMinerInfo: TransactionMinerInfo | undefined): Promise<PoolTag> {
+    if (txMinerInfo === undefined || txMinerInfo.vout.length < 1) {
+      if (config.DATABASE.ENABLED === true) {
+        return await poolsRepository.$getUnknownPool();
+      } else {
+        return poolsParser.unknownPool;
+      }
+    }
+
+    const asciiScriptSig = transactionUtils.hex2ascii(txMinerInfo.vin[0].scriptsig);
+    const address = txMinerInfo.vout[0].scriptpubkey_address;
+
+    let pools: PoolTag[] = [];
+    if (config.DATABASE.ENABLED === true) {
+      pools = await poolsRepository.$getPools();
+    } else {
+      pools = poolsParser.miningPools;
+    }
+    for (let i = 0; i < pools.length; ++i) {
+      if (address !== undefined) {
+        const addresses: string[] = JSON.parse(pools[i].addresses);
+        if (addresses.indexOf(address) !== -1) {
+          return pools[i];
+        }
+      }
+
+      const regexes: string[] = JSON.parse(pools[i].regexes);
+      for (let y = 0; y < regexes.length; ++y) {
+        const regex = new RegExp(regexes[y], 'i');
+        const match = asciiScriptSig.match(regex);
+        if (match !== null) {
+          return pools[i];
+        }
+      }
+    }
+
+    if (config.DATABASE.ENABLED === true) {
+      return await poolsRepository.$getUnknownPool();
+    } else {
+      return poolsParser.unknownPool;
+    }
+  }
+
+  /**
+   * [INDEXING] Index all blocks metadata for the mining dashboard
+   */
+  public async $generateBlockDatabase() {
+    const blockchainInfo = await bitcoinClient.getBlockchainInfo();
+    if (blockchainInfo.blocks !== blockchainInfo.headers) { // Wait for node to sync
+      return;
+    }
+
+    try {
+      let currentBlockHeight = blockchainInfo.blocks;
+
+      let indexingBlockAmount = Math.min(config.MEMPOOL.INDEXING_BLOCKS_AMOUNT, blockchainInfo.blocks);
+      if (indexingBlockAmount <= -1) {
+        indexingBlockAmount = currentBlockHeight + 1;
+      }
+
+      const lastBlockToIndex = Math.max(0, currentBlockHeight - indexingBlockAmount + 1);
+
+      logger.debug(`Indexing blocks from #${currentBlockHeight} to #${lastBlockToIndex}`);
+      loadingIndicators.setProgress('block-indexing', 0);
+
+      const chunkSize = 10000;
+      let totalIndexed = await blocksRepository.$blockCountBetweenHeight(currentBlockHeight, lastBlockToIndex);
+      let indexedThisRun = 0;
+      let newlyIndexed = 0;
+      const startedAt = new Date().getTime() / 1000;
+      let timer = new Date().getTime() / 1000;
+
+      while (currentBlockHeight >= lastBlockToIndex) {
+        const endBlock = Math.max(0, lastBlockToIndex, currentBlockHeight - chunkSize + 1);
+
+        const missingBlockHeights: number[] = await blocksRepository.$getMissingBlocksBetweenHeights(
+          currentBlockHeight, endBlock);
+        if (missingBlockHeights.length <= 0) {
+          currentBlockHeight -= chunkSize;
+          continue;
+        }
+
+        logger.info(`Indexing ${missingBlockHeights.length} blocks from #${currentBlockHeight} to #${endBlock}`);
+
+        for (const blockHeight of missingBlockHeights) {
+          if (blockHeight < lastBlockToIndex) {
+            break;
           }
+          ++indexedThisRun;
+          ++totalIndexed;
+          const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
+          if (elapsedSeconds > 5 || blockHeight === lastBlockToIndex) {
+            const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
+            const blockPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds));
+            const progress = Math.round(totalIndexed / indexingBlockAmount * 10000) / 100;
+            const timeLeft = Math.round((indexingBlockAmount - totalIndexed) / blockPerSeconds);
+            logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${runningFor} seconds | left: ~${timeLeft} seconds`);
+            timer = new Date().getTime() / 1000;
+            indexedThisRun = 0;
+            loadingIndicators.setProgress('block-indexing', progress, false);
+          }
+          const blockHash = await bitcoinApi.$getBlockHash(blockHeight);
+          const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash));
+          const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, true);
+          const blockExtended = await this.$getBlockExtended(block, transactions);
 
-          const mempool = memPool.getMempool();
-          let found = 0;
-          let notFound = 0;
+          newlyIndexed++;
+          await blocksRepository.$saveBlockInDatabase(blockExtended);
+        }
 
-          let transactions: ITransaction[] = [];
+        currentBlockHeight -= chunkSize;
+      }
+      logger.info(`Indexed ${newlyIndexed} blocks`);
+      loadingIndicators.setProgress('block-indexing', 100);
+    } catch (e) {
+      logger.err('Block indexing failed. Trying again later. Reason: ' + (e instanceof Error ? e.message : e));
+      loadingIndicators.setProgress('block-indexing', 100);
+      return;
+    }
 
-          for (let i = 1; i < block.tx.length; i++) {
-            if (mempool[block.tx[i]]) {
-              transactions.push(mempool[block.tx[i]]);
-              found++;
-            } else {
-              console.log(`Fetching block tx ${i} of ${block.tx.length}`);
-              const tx = await memPool.getRawTransaction(block.tx[i]);
-              if (tx) {
-                transactions.push(tx);
-              }
-              notFound++;
+    const chainValid = await BlocksRepository.$validateChain();
+    if (!chainValid) {
+      indexer.reindex();
+    }
+  }
+
+  public async $updateBlocks() {
+    let fastForwarded = false;
+    const blockHeightTip = await bitcoinApi.$getBlockHeightTip();
+
+    if (this.blocks.length === 0) {
+      this.currentBlockHeight = Math.max(blockHeightTip - config.MEMPOOL.INITIAL_BLOCKS_AMOUNT, -1);
+    } else {
+      this.currentBlockHeight = this.blocks[this.blocks.length - 1].height;
+    }
+
+    if (blockHeightTip - this.currentBlockHeight > config.MEMPOOL.INITIAL_BLOCKS_AMOUNT * 2) {
+      logger.info(`${blockHeightTip - this.currentBlockHeight} blocks since tip. Fast forwarding to the ${config.MEMPOOL.INITIAL_BLOCKS_AMOUNT} recent blocks`);
+      this.currentBlockHeight = blockHeightTip - config.MEMPOOL.INITIAL_BLOCKS_AMOUNT;
+      fastForwarded = true;
+      logger.info(`Re-indexing skipped blocks and corresponding hashrates data`);
+      indexer.reindex(); // Make sure to index the skipped blocks #1619
+    }
+
+    if (!this.lastDifficultyAdjustmentTime) {
+      const blockchainInfo = await bitcoinClient.getBlockchainInfo();
+      if (blockchainInfo.blocks === blockchainInfo.headers) {
+        const heightDiff = blockHeightTip % 2016;
+        const blockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff);
+        const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash));
+        this.lastDifficultyAdjustmentTime = block.timestamp;
+        this.currentDifficulty = block.difficulty;
+
+        if (blockHeightTip >= 2016) {
+          const previousPeriodBlockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff - 2016);
+          const previousPeriodBlock = await bitcoinApi.$getBlock(previousPeriodBlockHash);
+          this.previousDifficultyRetarget = (block.difficulty - previousPeriodBlock.difficulty) / previousPeriodBlock.difficulty * 100;
+          logger.debug(`Initial difficulty adjustment data set.`);
+        }
+      } else {
+        logger.debug(`Blockchain headers (${blockchainInfo.headers}) and blocks (${blockchainInfo.blocks}) not in sync. Waiting...`);
+      }
+    }
+
+    while (this.currentBlockHeight < blockHeightTip) {
+      if (this.currentBlockHeight < blockHeightTip - config.MEMPOOL.INITIAL_BLOCKS_AMOUNT) {
+        this.currentBlockHeight = blockHeightTip;
+      } else {
+        this.currentBlockHeight++;
+        logger.debug(`New block found (#${this.currentBlockHeight})!`);
+      }
+
+      const blockHash = await bitcoinApi.$getBlockHash(this.currentBlockHeight);
+      const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash));
+      const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash);
+      const transactions = await this.$getTransactionsExtended(blockHash, block.height, false);
+      const blockExtended: BlockExtended = await this.$getBlockExtended(block, transactions);
+
+      if (Common.indexingEnabled()) {
+        if (!fastForwarded) {
+          const lastBlock = await blocksRepository.$getBlockByHeight(blockExtended.height - 1);
+          if (lastBlock !== null && blockExtended.previousblockhash !== lastBlock['hash']) {
+            logger.warn(`Chain divergence detected at block ${lastBlock['height']}, re-indexing most recent data`);
+            // We assume there won't be a reorg with more than 10 block depth
+            await BlocksRepository.$deleteBlocksFrom(lastBlock['height'] - 10);
+            await HashratesRepository.$deleteLastEntries();
+            for (let i = 10; i >= 0; --i) {
+              await this.$indexBlock(lastBlock['height'] - i);
             }
           }
-
-          transactions.sort((a, b) => b.feePerVsize - a.feePerVsize);
-          transactions = transactions.filter((tx: ITransaction) => tx.feePerVsize);
-
-          block.minFee = transactions[transactions.length - 1] ? transactions[transactions.length - 1].feePerVsize : 0;
-          block.maxFee = transactions[0] ? transactions[0].feePerVsize : 0;
-          block.medianFee = this.median(transactions.map((tx) => tx.feePerVsize));
-
-          console.log(`New block found (#${this.currentBlockHeight})! `
-           + `${found} of ${block.tx.length} found in mempool. ${notFound} not found.`);
-
-           if (this.newBlockCallback) {
-            this.newBlockCallback(block);
-          }
-
-          this.$saveBlockToDatabase(block);
-          this.$saveTransactionsToDatabase(block.height, transactions);
+          await blocksRepository.$saveBlockInDatabase(blockExtended);
         }
+      }
+      if (fiatConversion.ratesInitialized === true && config.DATABASE.ENABLED === true) {
+        await RatesRepository.$saveRate(blockExtended.height, fiatConversion.getConversionRates());
+      }
 
-        this.blocks.push(block);
-        if (this.blocks.length > config.KEEP_BLOCK_AMOUNT) {
-          this.blocks.shift();
+      if (block.height % 2016 === 0) {
+        this.previousDifficultyRetarget = (block.difficulty - this.currentDifficulty) / this.currentDifficulty * 100;
+        this.lastDifficultyAdjustmentTime = block.timestamp;
+        this.currentDifficulty = block.difficulty;
+      }
+
+      this.blocks.push(blockExtended);
+      if (this.blocks.length > config.MEMPOOL.INITIAL_BLOCKS_AMOUNT * 4) {
+        this.blocks = this.blocks.slice(-config.MEMPOOL.INITIAL_BLOCKS_AMOUNT * 4);
+      }
+
+      if (this.newBlockCallbacks.length) {
+        this.newBlockCallbacks.forEach((cb) => cb(blockExtended, txIds, transactions));
+      }
+      if (!memPool.hasPriority()) {
+        diskCache.$saveCacheToDisk();
+      }
+    }
+  }
+
+  /**
+   * Index a block if it's missing from the database. Returns the block after indexing
+   */
+  public async $indexBlock(height: number): Promise<BlockExtended> {
+    const dbBlock = await blocksRepository.$getBlockByHeight(height);
+    if (dbBlock != null) {
+      return prepareBlock(dbBlock);
+    }
+
+    const blockHash = await bitcoinApi.$getBlockHash(height);
+    const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash));
+    const transactions = await this.$getTransactionsExtended(blockHash, block.height, true);
+    const blockExtended = await this.$getBlockExtended(block, transactions);
+
+    await blocksRepository.$saveBlockInDatabase(blockExtended);
+
+    return prepareBlock(blockExtended);
+  }
+
+  /**
+   * Index a block by hash if it's missing from the database. Returns the block after indexing
+   */
+  public async $getBlock(hash: string): Promise<BlockExtended | IEsploraApi.Block> {
+    // Check the memory cache
+    const blockByHash = this.getBlocks().find((b) => b.id === hash);
+    if (blockByHash) {
+      return blockByHash;
+    }
+
+    // Block has already been indexed
+    if (Common.indexingEnabled()) {
+      const dbBlock = await blocksRepository.$getBlockByHash(hash);
+      if (dbBlock != null) {
+        return prepareBlock(dbBlock);
+      }
+    }
+
+    const block = await bitcoinApi.$getBlock(hash);
+
+    // Not Bitcoin network, return the block as it
+    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
+      return block;
+    }
+
+    // Bitcoin network, add our custom data on top
+    const transactions = await this.$getTransactionsExtended(hash, block.height, true);
+    const blockExtended = await this.$getBlockExtended(block, transactions);
+    if (Common.indexingEnabled()) {
+      delete(blockExtended['coinbaseTx']);
+      await blocksRepository.$saveBlockInDatabase(blockExtended);
+    }
+
+    return blockExtended;
+  }
+
+  public async $getBlocks(fromHeight?: number, limit: number = 15): Promise<BlockExtended[]> {
+    try {
+      let currentHeight = fromHeight !== undefined ? fromHeight : this.getCurrentBlockHeight();
+      const returnBlocks: BlockExtended[] = [];
+
+      if (currentHeight < 0) {
+        return returnBlocks;
+      }
+
+      if (currentHeight === 0 && Common.indexingEnabled()) {
+        currentHeight = await blocksRepository.$mostRecentBlockHeight();
+      }
+
+      // Check if block height exist in local cache to skip the hash lookup
+      const blockByHeight = this.getBlocks().find((b) => b.height === currentHeight);
+      let startFromHash: string | null = null;
+      if (blockByHeight) {
+        startFromHash = blockByHeight.id;
+      } else if (!Common.indexingEnabled()) {
+        startFromHash = await bitcoinApi.$getBlockHash(currentHeight);
+      }
+
+      let nextHash = startFromHash;
+      for (let i = 0; i < limit && currentHeight >= 0; i++) {
+        let block = this.getBlocks().find((b) => b.height === currentHeight);
+        if (block) {
+          returnBlocks.push(block);
+        } else if (Common.indexingEnabled()) {
+          block = await this.$indexBlock(currentHeight);
+          returnBlocks.push(block);
+        } else if (nextHash != null) {
+          block = prepareBlock(await bitcoinApi.$getBlock(nextHash));
+          nextHash = block.previousblockhash;
+          returnBlocks.push(block);
         }
-
-      }
-    } catch (err) {
-      console.log('Error getBlockCount', err);
-    }
-  }
-
-  private async $getBlockFromDatabase(height: number): Promise<IBlock | undefined> {
-    try {
-      const connection = await DB.pool.getConnection();
-      const query = `
-        SELECT * FROM blocks WHERE height = ?
-      `;
-
-      const [rows] = await connection.query<any>(query, [height]);
-      connection.release();
-
-      if (rows[0]) {
-        return rows[0];
-      }
-    } catch (e) {
-      console.log('$get() block error', e);
-    }
-  }
-
-  private async $saveBlockToDatabase(block: IBlock) {
-    try {
-      const connection = await DB.pool.getConnection();
-      const query = `
-        INSERT IGNORE INTO blocks
-        (height, hash, size, weight, minFee, maxFee, time, fees, nTx, medianFee)
-        VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-      `;
-
-      const params: (any)[] = [
-        block.height,
-        block.hash,
-        block.size,
-        block.weight,
-        block.minFee,
-        block.maxFee,
-        block.time,
-        block.fees,
-        block.nTx - 1,
-        block.medianFee,
-      ];
-
-      await connection.query(query, params);
-      connection.release();
-    } catch (e) {
-      console.log('$create() block error', e);
-    }
-  }
-
-  private async $saveTransactionsToDatabase(blockheight: number, transactions: ITransaction[]) {
-    try {
-      const connection = await DB.pool.getConnection();
-
-      for (let i = 0; i < transactions.length; i++) {
-        const query = `
-          INSERT IGNORE INTO transactions
-          (blockheight, txid, fee, feePerVsize)
-          VALUES(?, ?, ?, ?)
-        `;
-
-        const params: (any)[] = [
-          blockheight,
-          transactions[i].txid,
-          transactions[i].fee,
-          transactions[i].feePerVsize,
-        ];
-
-        await connection.query(query, params);
+        currentHeight--;
       }
 
-      connection.release();
+      return returnBlocks;
     } catch (e) {
-      console.log('$create() transaction error', e);
+      throw e;
     }
   }
 
-  private async $clearOldTransactionsAndBlocksFromDatabase() {
-    try {
-      const connection = await DB.pool.getConnection();
-      let query = `DELETE FROM blocks WHERE height < ?`;
-      await connection.query<any>(query, [this.currentBlockHeight - config.KEEP_BLOCK_AMOUNT]);
-      query = `DELETE FROM transactions WHERE blockheight < ?`;
-      await connection.query<any>(query, [this.currentBlockHeight - config.KEEP_BLOCK_AMOUNT]);
-      connection.release();
-    } catch (e) {
-      console.log('$clearOldTransactionsFromDatabase() error', e);
-    }
+  public getLastDifficultyAdjustmentTime(): number {
+    return this.lastDifficultyAdjustmentTime;
   }
 
-  private median(numbers: number[]) {
-    if (!numbers.length) { return 0; }
-    let medianNr = 0;
-    const numsLen = numbers.length;
-    if (numsLen % 2 === 0) {
-        medianNr = (numbers[numsLen / 2 - 1] + numbers[numsLen / 2]) / 2;
-    } else {
-        medianNr = numbers[(numsLen - 1) / 2];
-    }
-    return medianNr;
+  public getPreviousDifficultyRetarget(): number {
+    return this.previousDifficultyRetarget;
+  }
+
+  public getCurrentBlockHeight(): number {
+    return this.currentBlockHeight;
   }
 }
 

backend/src/api/common.ts

@@ -0,0 +1,180 @@
+import { CpfpInfo, TransactionExtended, TransactionStripped } from '../mempool.interfaces';
+import config from '../config';
+export class Common {
+  static nativeAssetId = config.MEMPOOL.NETWORK === 'liquidtestnet' ?
+    '144c654344aa716d6f3abcc1ca90e5641e4e2a7f633bc09fe3baf64585819a49'
+  : '6f0279e9ed041c3d710a9f57d0c02928416460c4b722ae3457a11eec381c526d';
+  static _isLiquid = config.MEMPOOL.NETWORK === 'liquid' || config.MEMPOOL.NETWORK === 'liquidtestnet';
+
+  static isLiquid(): boolean {
+    return this._isLiquid;
+  }
+
+  static median(numbers: number[]) {
+    let medianNr = 0;
+    const numsLen = numbers.length;
+    if (numsLen % 2 === 0) {
+        medianNr = (numbers[numsLen / 2 - 1] + numbers[numsLen / 2]) / 2;
+    } else {
+        medianNr = numbers[(numsLen - 1) / 2];
+    }
+    return medianNr;
+  }
+
+  static percentile(numbers: number[], percentile: number) {
+    if (percentile === 50) {
+      return this.median(numbers);
+    }
+    const index = Math.ceil(numbers.length * (100 - percentile) * 1e-2);
+    if (index < 0 || index > numbers.length - 1) {
+      return 0;
+    }
+    return numbers[index];
+  }
+
+  static getFeesInRange(transactions: TransactionExtended[], rangeLength: number) {
+    const arr = [transactions[transactions.length - 1].effectiveFeePerVsize];
+    const chunk = 1 / (rangeLength - 1);
+    let itemsToAdd = rangeLength - 2;
+
+    while (itemsToAdd > 0) {
+      arr.push(transactions[Math.floor(transactions.length * chunk * itemsToAdd)].effectiveFeePerVsize);
+      itemsToAdd--;
+    }
+
+    arr.push(transactions[0].effectiveFeePerVsize);
+    return arr;
+  }
+
+  static findRbfTransactions(added: TransactionExtended[], deleted: TransactionExtended[]): { [txid: string]: TransactionExtended } {
+    const matches: { [txid: string]: TransactionExtended } = {};
+    deleted
+      // The replaced tx must have at least one input with nSequence < maxint-1 (That’s the opt-in)
+      .filter((tx) => tx.vin.some((vin) => vin.sequence < 0xfffffffe))
+      .forEach((deletedTx) => {
+        const foundMatches = added.find((addedTx) => {
+          // The new tx must, absolutely speaking, pay at least as much fee as the replaced tx.
+          return addedTx.fee > deletedTx.fee
+            // The new transaction must pay more fee per kB than the replaced tx.
+            && addedTx.feePerVsize > deletedTx.feePerVsize
+            // Spends one or more of the same inputs
+            && deletedTx.vin.some((deletedVin) =>
+              addedTx.vin.some((vin) => vin.txid === deletedVin.txid));
+            });
+        if (foundMatches) {
+          matches[deletedTx.txid] = foundMatches;
+        }
+      });
+    return matches;
+  }
+
+  static stripTransaction(tx: TransactionExtended): TransactionStripped {
+    return {
+      txid: tx.txid,
+      fee: tx.fee,
+      vsize: tx.weight / 4,
+      value: tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0),
+    };
+  }
+
+  static sleep$(ms: number): Promise<void> {
+    return new Promise((resolve) => {
+       setTimeout(() => {
+         resolve();
+       }, ms);
+    });
+  }
+
+  static shuffleArray(array: any[]) {
+    for (let i = array.length - 1; i > 0; i--) {
+        const j = Math.floor(Math.random() * (i + 1));
+        [array[i], array[j]] = [array[j], array[i]];
+    }
+  }
+
+  static setRelativesAndGetCpfpInfo(tx: TransactionExtended, memPool: { [txid: string]: TransactionExtended }): CpfpInfo {
+    const parents = this.findAllParents(tx, memPool);
+    const lowerFeeParents = parents.filter((parent) => parent.feePerVsize < tx.effectiveFeePerVsize);
+
+    let totalWeight = tx.weight + lowerFeeParents.reduce((prev, val) => prev + val.weight, 0);
+    let totalFees = tx.fee + lowerFeeParents.reduce((prev, val) => prev + val.fee, 0);
+
+    tx.ancestors = parents
+      .map((t) => {
+        return {
+          txid: t.txid,
+          weight: t.weight,
+          fee: t.fee,
+        };
+      });
+
+    // Add high (high fee) decendant weight and fees
+    if (tx.bestDescendant) {
+      totalWeight += tx.bestDescendant.weight;
+      totalFees += tx.bestDescendant.fee;
+    }
+
+    tx.effectiveFeePerVsize = Math.max(Common.isLiquid() ? 0.1 : 1, totalFees / (totalWeight / 4));
+    tx.cpfpChecked = true;
+
+    return {
+      ancestors: tx.ancestors,
+      bestDescendant: tx.bestDescendant || null,
+    };
+  }
+
+
+  private static findAllParents(tx: TransactionExtended, memPool: { [txid: string]: TransactionExtended }): TransactionExtended[] {
+    let parents: TransactionExtended[] = [];
+    tx.vin.forEach((parent) => {
+      if (parents.find((p) => p.txid === parent.txid)) {
+        return;
+      }
+
+      const parentTx = memPool[parent.txid];
+      if (parentTx) {
+        if (tx.bestDescendant && tx.bestDescendant.fee / (tx.bestDescendant.weight / 4) > parentTx.feePerVsize) {
+          if (parentTx.bestDescendant && parentTx.bestDescendant.fee < tx.fee + tx.bestDescendant.fee) {
+            parentTx.bestDescendant = {
+              weight: tx.weight + tx.bestDescendant.weight,
+              fee: tx.fee + tx.bestDescendant.fee,
+              txid: tx.txid,
+            };
+          }
+        } else if (tx.feePerVsize > parentTx.feePerVsize) {
+          parentTx.bestDescendant = {
+            weight: tx.weight,
+            fee: tx.fee,
+            txid: tx.txid
+          };
+        }
+        parents.push(parentTx);
+        parents = parents.concat(this.findAllParents(parentTx, memPool));
+      }
+    });
+    return parents;
+  }
+
+  static getSqlInterval(interval: string | null): string | null {
+    switch (interval) {
+      case '24h': return '1 DAY';
+      case '3d': return '3 DAY';
+      case '1w': return '1 WEEK';
+      case '1m': return '1 MONTH';
+      case '3m': return '3 MONTH';
+      case '6m': return '6 MONTH';
+      case '1y': return '1 YEAR';
+      case '2y': return '2 YEAR';
+      case '3y': return '3 YEAR';
+      default: return null;
+    }
+  }
+
+  static indexingEnabled(): boolean {
+    return (
+      ['mainnet', 'testnet', 'signet', 'regtest'].includes(config.MEMPOOL.NETWORK) &&
+      config.DATABASE.ENABLED === true &&
+      config.MEMPOOL.INDEXING_BLOCKS_AMOUNT !== 0
+    );
+  }
+}

backend/src/api/database-migration.ts

@@ -0,0 +1,503 @@
+import config from '../config';
+import DB from '../database';
+import logger from '../logger';
+import { Common } from './common';
+
+class DatabaseMigration {
+  private static currentVersion = 19;
+  private queryTimeout = 120000;
+  private statisticsAddedIndexed = false;
+
+  constructor() { }
+  /**
+   * Entry point
+   */
+  public async $initializeOrMigrateDatabase(): Promise<void> {
+    logger.debug('MIGRATIONS: Running migrations');
+
+    await this.$printDatabaseVersion();
+
+    // First of all, if the `state` database does not exist, create it so we can track migration version
+    if (!await this.$checkIfTableExists('state')) {
+      logger.debug('MIGRATIONS: `state` table does not exist. Creating it.');
+      try {
+        await this.$createMigrationStateTable();
+      } catch (e) {
+        logger.err('MIGRATIONS: Unable to create `state` table, aborting in 10 seconds. ' + e);
+        await Common.sleep$(10000);
+        process.exit(-1);
+      }
+      logger.debug('MIGRATIONS: `state` table initialized.');
+    }
+
+    let databaseSchemaVersion = 0;
+    try {
+      databaseSchemaVersion = await this.$getSchemaVersionFromDatabase();
+    } catch (e) {
+      logger.err('MIGRATIONS: Unable to get current database migration version, aborting in 10 seconds. ' + e);
+      await Common.sleep$(10000);
+      process.exit(-1);
+    }
+
+    logger.debug('MIGRATIONS: Current state.schema_version ' + databaseSchemaVersion);
+    logger.debug('MIGRATIONS: Latest DatabaseMigration.version is ' + DatabaseMigration.currentVersion);
+    if (databaseSchemaVersion >= DatabaseMigration.currentVersion) {
+      logger.debug('MIGRATIONS: Nothing to do.');
+      return;
+    }
+
+    // Now, create missing tables. Those queries cannot be wrapped into a transaction unfortunately
+    try {
+      await this.$createMissingTablesAndIndexes(databaseSchemaVersion);
+    } catch (e) {
+      logger.err('MIGRATIONS: Unable to create required tables, aborting in 10 seconds. ' + e);
+      await Common.sleep$(10000);
+      process.exit(-1);
+    }
+
+    if (DatabaseMigration.currentVersion > databaseSchemaVersion) {
+      logger.notice('MIGRATIONS: Upgrading database schema');
+      try {
+        await this.$migrateTableSchemaFromVersion(databaseSchemaVersion);
+        logger.notice(`MIGRATIONS: OK. Database schema have been migrated from version ${databaseSchemaVersion} to ${DatabaseMigration.currentVersion} (latest version)`);
+      } catch (e) {
+        logger.err('MIGRATIONS: Unable to migrate database, aborting. ' + e);
+      }
+    }
+
+    return;
+  }
+
+  /**
+   * Create all missing tables
+   */
+  private async $createMissingTablesAndIndexes(databaseSchemaVersion: number) {
+    await this.$setStatisticsAddedIndexedFlag(databaseSchemaVersion);
+
+    const isBitcoin = ['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK);
+    try {
+      await this.$executeQuery(this.getCreateElementsTableQuery(), await this.$checkIfTableExists('elements_pegs'));
+      await this.$executeQuery(this.getCreateStatisticsQuery(), await this.$checkIfTableExists('statistics'));
+      if (databaseSchemaVersion < 2 && this.statisticsAddedIndexed === false) {
+        await this.$executeQuery(`CREATE INDEX added ON statistics (added);`);
+      }
+      if (databaseSchemaVersion < 3) {
+        await this.$executeQuery(this.getCreatePoolsTableQuery(), await this.$checkIfTableExists('pools'));
+      }
+      if (databaseSchemaVersion < 4) {
+        await this.$executeQuery('DROP table IF EXISTS blocks;');
+        await this.$executeQuery(this.getCreateBlocksTableQuery(), await this.$checkIfTableExists('blocks'));
+      }
+      if (databaseSchemaVersion < 5 && isBitcoin === true) {
+        logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.`);
+        await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index
+        await this.$executeQuery('ALTER TABLE blocks ADD `reward` double unsigned NOT NULL DEFAULT "0"');
+      }
+
+      if (databaseSchemaVersion < 6 && isBitcoin === true) {
+        logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.`);
+        await this.$executeQuery('TRUNCATE blocks;');  // Need to re-index
+        // Cleanup original blocks fields type
+        await this.$executeQuery('ALTER TABLE blocks MODIFY `height` integer unsigned NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE blocks MODIFY `tx_count` smallint unsigned NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE blocks MODIFY `size` integer unsigned NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE blocks MODIFY `weight` integer unsigned NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE blocks MODIFY `difficulty` double NOT NULL DEFAULT "0"');
+        // We also fix the pools.id type so we need to drop/re-create the foreign key
+        await this.$executeQuery('ALTER TABLE blocks DROP FOREIGN KEY IF EXISTS `blocks_ibfk_1`');
+        await this.$executeQuery('ALTER TABLE pools MODIFY `id` smallint unsigned AUTO_INCREMENT');
+        await this.$executeQuery('ALTER TABLE blocks MODIFY `pool_id` smallint unsigned NULL');
+        await this.$executeQuery('ALTER TABLE blocks ADD FOREIGN KEY (`pool_id`) REFERENCES `pools` (`id`)');
+        // Add new block indexing fields
+        await this.$executeQuery('ALTER TABLE blocks ADD `version` integer unsigned NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE blocks ADD `bits` integer unsigned NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE blocks ADD `nonce` bigint unsigned NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE blocks ADD `merkle_root` varchar(65) NOT NULL DEFAULT ""');
+        await this.$executeQuery('ALTER TABLE blocks ADD `previous_block_hash` varchar(65) NULL');
+      }
+
+      if (databaseSchemaVersion < 7 && isBitcoin === true) {
+        await this.$executeQuery('DROP table IF EXISTS hashrates;');
+        await this.$executeQuery(this.getCreateDailyStatsTableQuery(), await this.$checkIfTableExists('hashrates'));
+      }
+
+      if (databaseSchemaVersion < 8 && isBitcoin === true) {
+        logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.`);
+        await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index
+        await this.$executeQuery('ALTER TABLE `hashrates` DROP INDEX `PRIMARY`');
+        await this.$executeQuery('ALTER TABLE `hashrates` ADD `id` int NOT NULL AUTO_INCREMENT PRIMARY KEY FIRST');
+        await this.$executeQuery('ALTER TABLE `hashrates` ADD `share` float NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE `hashrates` ADD `type` enum("daily", "weekly") DEFAULT "daily"');
+      }
+
+      if (databaseSchemaVersion < 9 && isBitcoin === true) {
+        logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.`);
+        await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index
+        await this.$executeQuery('ALTER TABLE `state` CHANGE `name` `name` varchar(100)');
+        await this.$executeQuery('ALTER TABLE `hashrates` ADD UNIQUE `hashrate_timestamp_pool_id` (`hashrate_timestamp`, `pool_id`)');
+      }
+
+      if (databaseSchemaVersion < 10 && isBitcoin === true) {
+        await this.$executeQuery('ALTER TABLE `blocks` ADD INDEX `blockTimestamp` (`blockTimestamp`)');
+      }
+
+      if (databaseSchemaVersion < 11 && isBitcoin === true) {
+        logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.`);
+        await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index
+        await this.$executeQuery(`ALTER TABLE blocks
+          ADD avg_fee INT UNSIGNED NULL,
+          ADD avg_fee_rate INT UNSIGNED NULL
+        `);
+        await this.$executeQuery('ALTER TABLE blocks MODIFY `reward` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE blocks MODIFY `median_fee` INT UNSIGNED NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE blocks MODIFY `fees` INT UNSIGNED NOT NULL DEFAULT "0"');
+      }
+
+      if (databaseSchemaVersion < 12 && isBitcoin === true) {
+        // No need to re-index because the new data type can contain larger values
+        await this.$executeQuery('ALTER TABLE blocks MODIFY `fees` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
+      }
+
+      if (databaseSchemaVersion < 13 && isBitcoin === true) {
+        await this.$executeQuery('ALTER TABLE blocks MODIFY `difficulty` DOUBLE UNSIGNED NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE blocks MODIFY `median_fee` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE blocks MODIFY `avg_fee` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE blocks MODIFY `avg_fee_rate` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
+      }
+
+      if (databaseSchemaVersion < 14 && isBitcoin === true) {
+        logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.`);
+        await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index
+        await this.$executeQuery('ALTER TABLE `hashrates` DROP FOREIGN KEY `hashrates_ibfk_1`');
+        await this.$executeQuery('ALTER TABLE `hashrates` MODIFY `pool_id` SMALLINT UNSIGNED NOT NULL DEFAULT "0"');
+      }
+
+      if (databaseSchemaVersion < 16 && isBitcoin === true) {
+        logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.`);
+        await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index because we changed timestamps
+      }
+
+      if (databaseSchemaVersion < 17 && isBitcoin === true) {
+        await this.$executeQuery('ALTER TABLE `pools` ADD `slug` CHAR(50) NULL');
+      }
+
+      if (databaseSchemaVersion < 18 && isBitcoin === true) {
+        await this.$executeQuery('ALTER TABLE `blocks` ADD INDEX `hash` (`hash`);');
+      }
+
+      if (databaseSchemaVersion < 19) {
+        await this.$executeQuery(this.getCreateRatesTableQuery(), await this.$checkIfTableExists('rates'));
+      }
+    } catch (e) {
+      throw e;
+    }
+  }
+
+  /**
+   * Special case here for the `statistics` table - It appeared that somehow some dbs already had the `added` field indexed
+   * while it does not appear in previous schemas. The mariadb command "CREATE INDEX IF NOT EXISTS" is not supported on
+   * older mariadb version. Therefore we set a flag here in order to know if the index needs to be created or not before
+   * running the migration process
+   */
+  private async $setStatisticsAddedIndexedFlag(databaseSchemaVersion: number) {
+    if (databaseSchemaVersion >= 2) {
+      this.statisticsAddedIndexed = true;
+      return;
+    }
+
+    try {
+      // We don't use "CREATE INDEX IF NOT EXISTS" because it is not supported on old mariadb version 5.X
+      const query = `SELECT COUNT(1) hasIndex FROM INFORMATION_SCHEMA.STATISTICS
+        WHERE table_schema=DATABASE() AND table_name='statistics' AND index_name='added';`;
+      const [rows] = await this.$executeQuery(query, true);
+      if (rows[0].hasIndex === 0) {
+        logger.debug('MIGRATIONS: `statistics.added` is not indexed');
+        this.statisticsAddedIndexed = false;
+      } else if (rows[0].hasIndex === 1) {
+        logger.debug('MIGRATIONS: `statistics.added` is already indexed');
+        this.statisticsAddedIndexed = true;
+      }
+    } catch (e) {
+      // Should really never happen but just in case it fails, we just don't execute
+      // any query related to this indexing so it won't fail if the index actually already exists
+      logger.err('MIGRATIONS: Unable to check if `statistics.added` INDEX exist or not.');
+      this.statisticsAddedIndexed = true;
+    }
+  }
+
+  /**
+   * Small query execution wrapper to log all executed queries
+   */
+  private async $executeQuery(query: string, silent: boolean = false): Promise<any> {
+    if (!silent) {
+      logger.debug('MIGRATIONS: Execute query:\n' + query);
+    }
+    return DB.query({ sql: query, timeout: this.queryTimeout });
+  }
+
+  /**
+   * Check if 'table' exists in the database
+   */
+  private async $checkIfTableExists(table: string): Promise<boolean> {
+    const query = `SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = '${config.DATABASE.DATABASE}' AND TABLE_NAME = '${table}'`;
+    const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
+    return rows[0]['COUNT(*)'] === 1;
+  }
+
+  /**
+   * Get current database version
+   */
+  private async $getSchemaVersionFromDatabase(): Promise<number> {
+    const query = `SELECT number FROM state WHERE name = 'schema_version';`;
+    const [rows] = await this.$executeQuery(query, true);
+    return rows[0]['number'];
+  }
+
+  /**
+   * Create the `state` table
+   */
+  private async $createMigrationStateTable(): Promise<void> {
+    try {
+      const query = `CREATE TABLE IF NOT EXISTS state (
+        name varchar(25) NOT NULL,
+        number int(11) NULL,
+        string varchar(100) NULL,
+        CONSTRAINT name_unique UNIQUE (name)
+      ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+      await this.$executeQuery(query);
+
+      // Set initial values
+      await this.$executeQuery(`INSERT INTO state VALUES('schema_version', 0, NULL);`);
+      await this.$executeQuery(`INSERT INTO state VALUES('last_elements_block', 0, NULL);`);
+    } catch (e) {
+      throw e;
+    }
+  }
+
+  /**
+   * We actually execute the migrations queries here
+   */
+  private async $migrateTableSchemaFromVersion(version: number): Promise<void> {
+    const transactionQueries: string[] = [];
+    for (const query of this.getMigrationQueriesFromVersion(version)) {
+      transactionQueries.push(query);
+    }
+    transactionQueries.push(this.getUpdateToLatestSchemaVersionQuery());
+
+    try {
+      await this.$executeQuery('START TRANSACTION;');
+      for (const query of transactionQueries) {
+        await this.$executeQuery(query);
+      }
+      await this.$executeQuery('COMMIT;');
+    } catch (e) {
+      await this.$executeQuery('ROLLBACK;');
+      throw e;
+    }
+  }
+
+  /**
+   * Generate migration queries based on schema version
+   */
+  private getMigrationQueriesFromVersion(version: number): string[] {
+    const queries: string[] = [];
+    const isBitcoin = ['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK);
+
+    if (version < 1) {
+      if (config.MEMPOOL.NETWORK !== 'liquid' && config.MEMPOOL.NETWORK !== 'liquidtestnet') {
+        queries.push(this.getShiftStatisticsQuery());
+      }
+    }
+
+    if (version < 7 && isBitcoin === true) {
+      queries.push(`INSERT INTO state(name, number, string) VALUES ('last_hashrates_indexing', 0, NULL)`);
+    }
+
+    if (version < 9  && isBitcoin === true) {
+      queries.push(`INSERT INTO state(name, number, string) VALUES ('last_weekly_hashrates_indexing', 0, NULL)`);
+    }
+
+    return queries;
+  }
+
+  /**
+   * Save the schema version in the database
+   */
+  private getUpdateToLatestSchemaVersionQuery(): string {
+    return `UPDATE state SET number = ${DatabaseMigration.currentVersion} WHERE name = 'schema_version';`;
+  }
+
+  /**
+   * Print current database version
+   */
+  private async $printDatabaseVersion() {
+    try {
+      const [rows] = await this.$executeQuery('SELECT VERSION() as version;', true);
+      logger.debug(`MIGRATIONS: Database engine version '${rows[0].version}'`);
+    } catch (e) {
+      logger.debug(`MIGRATIONS: Could not fetch database engine version. ` + e);
+    }
+  }
+
+  // Couple of wrappers to clean the main logic
+  private getShiftStatisticsQuery(): string {
+    return `UPDATE statistics SET
+      vsize_1 = vsize_1 + vsize_2, vsize_2 = vsize_3,
+      vsize_3 = vsize_4, vsize_4 = vsize_5,
+      vsize_5 = vsize_6, vsize_6 = vsize_8,
+      vsize_8 = vsize_10, vsize_10 = vsize_12,
+      vsize_12 = vsize_15, vsize_15 = vsize_20,
+      vsize_20 = vsize_30, vsize_30 = vsize_40,
+      vsize_40 = vsize_50, vsize_50 = vsize_60,
+      vsize_60 = vsize_70, vsize_70 = vsize_80,
+      vsize_80 = vsize_90, vsize_90 = vsize_100,
+      vsize_100 = vsize_125, vsize_125 = vsize_150,
+      vsize_150 = vsize_175, vsize_175 = vsize_200,
+      vsize_200 = vsize_250, vsize_250 = vsize_300,
+      vsize_300 = vsize_350, vsize_350 = vsize_400,
+      vsize_400 = vsize_500, vsize_500 = vsize_600,
+      vsize_600 = vsize_700, vsize_700 = vsize_800,
+      vsize_800 = vsize_900, vsize_900 = vsize_1000,
+      vsize_1000 = vsize_1200, vsize_1200 = vsize_1400,
+      vsize_1400 = vsize_1800, vsize_1800 = vsize_2000, vsize_2000 = 0;`;
+  }
+
+  private getCreateStatisticsQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS statistics (
+      id int(11) NOT NULL AUTO_INCREMENT,
+      added datetime NOT NULL,
+      unconfirmed_transactions int(11) UNSIGNED NOT NULL,
+      tx_per_second float UNSIGNED NOT NULL,
+      vbytes_per_second int(10) UNSIGNED NOT NULL,
+      mempool_byte_weight int(10) UNSIGNED NOT NULL,
+      fee_data longtext NOT NULL,
+      total_fee double UNSIGNED NOT NULL,
+      vsize_1 int(11) NOT NULL,
+      vsize_2 int(11) NOT NULL,
+      vsize_3 int(11) NOT NULL,
+      vsize_4 int(11) NOT NULL,
+      vsize_5 int(11) NOT NULL,
+      vsize_6 int(11) NOT NULL,
+      vsize_8 int(11) NOT NULL,
+      vsize_10 int(11) NOT NULL,
+      vsize_12 int(11) NOT NULL,
+      vsize_15 int(11) NOT NULL,
+      vsize_20 int(11) NOT NULL,
+      vsize_30 int(11) NOT NULL,
+      vsize_40 int(11) NOT NULL,
+      vsize_50 int(11) NOT NULL,
+      vsize_60 int(11) NOT NULL,
+      vsize_70 int(11) NOT NULL,
+      vsize_80 int(11) NOT NULL,
+      vsize_90 int(11) NOT NULL,
+      vsize_100 int(11) NOT NULL,
+      vsize_125 int(11) NOT NULL,
+      vsize_150 int(11) NOT NULL,
+      vsize_175 int(11) NOT NULL,
+      vsize_200 int(11) NOT NULL,
+      vsize_250 int(11) NOT NULL,
+      vsize_300 int(11) NOT NULL,
+      vsize_350 int(11) NOT NULL,
+      vsize_400 int(11) NOT NULL,
+      vsize_500 int(11) NOT NULL,
+      vsize_600 int(11) NOT NULL,
+      vsize_700 int(11) NOT NULL,
+      vsize_800 int(11) NOT NULL,
+      vsize_900 int(11) NOT NULL,
+      vsize_1000 int(11) NOT NULL,
+      vsize_1200 int(11) NOT NULL,
+      vsize_1400 int(11) NOT NULL,
+      vsize_1600 int(11) NOT NULL,
+      vsize_1800 int(11) NOT NULL,
+      vsize_2000 int(11) NOT NULL,
+      CONSTRAINT PRIMARY KEY (id)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
+  private getCreateElementsTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS elements_pegs (
+      block int(11) NOT NULL,
+      datetime int(11) NOT NULL,
+      amount bigint(20) NOT NULL,
+      txid varchar(65) NOT NULL,
+      txindex int(11) NOT NULL,
+      bitcoinaddress varchar(100) NOT NULL,
+      bitcointxid varchar(65) NOT NULL,
+      bitcoinindex int(11) NOT NULL,
+      final_tx int(11) NOT NULL
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
+  private getCreatePoolsTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS pools (
+      id int(11) NOT NULL AUTO_INCREMENT,
+      name varchar(50) NOT NULL,
+      link varchar(255) NOT NULL,
+      addresses text NOT NULL,
+      regexes text NOT NULL,
+      PRIMARY KEY (id)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;`;
+  }
+
+  private getCreateBlocksTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS blocks (
+      height int(11) unsigned NOT NULL,
+      hash varchar(65) NOT NULL,
+      blockTimestamp timestamp NOT NULL,
+      size int(11) unsigned NOT NULL,
+      weight int(11) unsigned NOT NULL,
+      tx_count int(11) unsigned NOT NULL,
+      coinbase_raw text,
+      difficulty bigint(20) unsigned NOT NULL,
+      pool_id int(11) DEFAULT -1,
+      fees double unsigned NOT NULL,
+      fee_span json NOT NULL,
+      median_fee double unsigned NOT NULL,
+      PRIMARY KEY (height),
+      INDEX (pool_id),
+      FOREIGN KEY (pool_id) REFERENCES pools (id)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
+  private getCreateDailyStatsTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS hashrates (
+      hashrate_timestamp timestamp NOT NULL,
+      avg_hashrate double unsigned DEFAULT '0',
+      pool_id smallint unsigned NULL,
+      PRIMARY KEY (hashrate_timestamp),
+      INDEX (pool_id),
+      FOREIGN KEY (pool_id) REFERENCES pools (id)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
+  private getCreateRatesTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS rates (
+      height int(10) unsigned NOT NULL,
+      bisq_rates JSON NOT NULL,
+      PRIMARY KEY (height)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
+  public async $truncateIndexedData(tables: string[]) {
+    const allowedTables = ['blocks', 'hashrates'];
+
+    try {
+      for (const table of tables) {
+        if (!allowedTables.includes(table)) {
+          logger.debug(`Table ${table} cannot to be re-indexed (not allowed)`);
+          continue;
+        }
+
+        await this.$executeQuery(`TRUNCATE ${table}`, true);
+        if (table === 'hashrates') {
+          await this.$executeQuery('UPDATE state set number = 0 where name = "last_hashrates_indexing"', true);
+        }
+        logger.notice(`Table ${table} has been truncated`);
+      }
+    } catch (e) {
+      logger.warn(`Unable to erase indexed data`);
+    }
+  }
+}
+
+export default new DatabaseMigration();

backend/src/api/difficulty-adjustment.ts

@@ -0,0 +1,67 @@
+import config from '../config';
+import { IDifficultyAdjustment } from '../mempool.interfaces';
+import blocks from './blocks';
+
+class DifficultyAdjustmentApi {
+  constructor() { }
+
+  public getDifficultyAdjustment(): IDifficultyAdjustment {
+    const DATime = blocks.getLastDifficultyAdjustmentTime();
+    const previousRetarget = blocks.getPreviousDifficultyRetarget();
+    const blockHeight = blocks.getCurrentBlockHeight();
+    const blocksCache = blocks.getBlocks();
+    const latestBlock = blocksCache[blocksCache.length - 1];
+
+    const now = new Date().getTime() / 1000;
+    const diff = now - DATime;
+    const blocksInEpoch = blockHeight % 2016;
+    const progressPercent = (blocksInEpoch >= 0) ? blocksInEpoch / 2016 * 100 : 100;
+    const remainingBlocks = 2016 - blocksInEpoch;
+    const nextRetargetHeight = blockHeight + remainingBlocks;
+
+    let difficultyChange = 0;
+    if (remainingBlocks < 1870) {
+      if (blocksInEpoch > 0) {
+        difficultyChange = (600 / (diff / blocksInEpoch) - 1) * 100;
+      }
+      if (difficultyChange > 300) {
+        difficultyChange = 300;
+      }
+      if (difficultyChange < -75) {
+        difficultyChange = -75;
+      }
+    }
+
+    let timeAvgMins = blocksInEpoch && blocksInEpoch > 146 ? diff / blocksInEpoch / 60 : 10;
+
+    // Testnet difficulty is set to 1 after 20 minutes of no blocks,
+    // therefore the time between blocks will always be below 20 minutes (1200s).
+    let timeOffset = 0;
+    if (config.MEMPOOL.NETWORK === 'testnet') {
+      if (timeAvgMins > 20) {
+        timeAvgMins = 20;
+      }
+      if (now - latestBlock.timestamp + timeAvgMins * 60 > 1200) {
+        timeOffset = -Math.min(now - latestBlock.timestamp, 1200) * 1000;
+      }
+    }
+
+    const timeAvg = timeAvgMins * 60 * 1000 ;
+    const remainingTime = (remainingBlocks * timeAvg) + (now * 1000);
+    const estimatedRetargetDate = remainingTime + now;
+
+    return {
+      progressPercent,
+      difficultyChange,
+      estimatedRetargetDate,
+      remainingBlocks,
+      remainingTime,
+      previousRetarget,
+      nextRetargetHeight,
+      timeAvg,
+      timeOffset,
+    };
+  }
+}
+
+export default new DifficultyAdjustmentApi();

backend/src/api/disk-cache.ts

@@ -1,15 +1,117 @@
 import * as fs from 'fs';
+const fsPromises = fs.promises;
+import * as cluster from 'cluster';
+import memPool from './mempool';
+import blocks from './blocks';
+import logger from '../logger';
+import config from '../config';
+import { TransactionExtended } from '../mempool.interfaces';
+import { Common } from './common';
 
 class DiskCache {
-  static FILE_NAME = './cache.json';
+  private cacheSchemaVersion = 1;
+
+  private static FILE_NAME = config.MEMPOOL.CACHE_DIR + '/cache.json';
+  private static FILE_NAMES = config.MEMPOOL.CACHE_DIR + '/cache{number}.json';
+  private static CHUNK_FILES = 25;
+  private isWritingCache = false;
+
   constructor() { }
 
-  saveData(dataBlob: string) {
-    fs.writeFileSync(DiskCache.FILE_NAME, dataBlob, 'utf8');
+  async $saveCacheToDisk(): Promise<void> {
+    if (!cluster.isMaster) {
+      return;
+    }
+    if (this.isWritingCache) {
+      logger.debug('Saving cache already in progress. Skipping.')
+      return;
+    }
+    try {
+      logger.debug('Writing mempool and blocks data to disk cache (async)...');
+      this.isWritingCache = true;
+
+      const mempool = memPool.getMempool();
+      const mempoolArray: TransactionExtended[] = [];
+      for (const tx in mempool) {
+        mempoolArray.push(mempool[tx]);
+      }
+
+      Common.shuffleArray(mempoolArray);
+
+      const chunkSize = Math.floor(mempoolArray.length / DiskCache.CHUNK_FILES);
+
+      await fsPromises.writeFile(DiskCache.FILE_NAME, JSON.stringify({
+        cacheSchemaVersion: this.cacheSchemaVersion,
+        blocks: blocks.getBlocks(),
+        mempool: {},
+        mempoolArray: mempoolArray.splice(0, chunkSize),
+      }), {flag: 'w'});
+      for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
+        await fsPromises.writeFile(DiskCache.FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
+          mempool: {},
+          mempoolArray: mempoolArray.splice(0, chunkSize),
+        }), {flag: 'w'});
+      }
+      logger.debug('Mempool and blocks data saved to disk cache');
+      this.isWritingCache = false;
+    } catch (e) {
+      logger.warn('Error writing to cache file: ' + (e instanceof Error ? e.message : e));
+      this.isWritingCache = false;
+    }
   }
 
-  loadData(): string {
-    return fs.readFileSync(DiskCache.FILE_NAME, 'utf8');
+  wipeCache() {
+    fs.unlinkSync(DiskCache.FILE_NAME);
+    for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
+      fs.unlinkSync(DiskCache.FILE_NAMES.replace('{number}', i.toString()));
+    }
+  }
+  
+  loadMempoolCache() {
+    if (!fs.existsSync(DiskCache.FILE_NAME)) {
+      return;
+    }
+    try {
+      let data: any = {};
+      const cacheData = fs.readFileSync(DiskCache.FILE_NAME, 'utf8');
+      if (cacheData) {
+        logger.info('Restoring mempool and blocks data from disk cache');
+        data = JSON.parse(cacheData);
+        if (data.cacheSchemaVersion === undefined || data.cacheSchemaVersion !== this.cacheSchemaVersion) {
+          logger.notice('Disk cache contains an outdated schema version. Clearing it and skipping the cache loading.');
+          return this.wipeCache();
+        }
+
+        if (data.mempoolArray) {
+          for (const tx of data.mempoolArray) {
+            data.mempool[tx.txid] = tx;
+          }
+        }
+      }
+
+      for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
+        const fileName = DiskCache.FILE_NAMES.replace('{number}', i.toString());
+        try {
+          if (fs.existsSync(fileName)) {
+            const cacheData2 = JSON.parse(fs.readFileSync(fileName, 'utf8'));
+            if (cacheData2.mempoolArray) {
+              for (const tx of cacheData2.mempoolArray) {
+                data.mempool[tx.txid] = tx;
+              }
+            } else {
+              Object.assign(data.mempool, cacheData2.mempool);
+            }
+          }
+        } catch (e) {
+          logger.info('Error parsing ' + fileName + '. Skipping. Reason: ' + (e instanceof Error ? e.message : e));
+        }
+      }
+
+      memPool.setMempool(data.mempool);
+      blocks.setBlocks(data.blocks);
+    } catch (e) {
+      logger.warn('Failed to parse mempoool and blocks cache. Skipping. Reason: ' + (e instanceof Error ? e.message : e));
+    }
   }
 }
 

backend/src/api/fee-api.ts

@@ -1,47 +1,52 @@
-import projectedBlocks from './projected-blocks';
-import { DB } from '../database';
+import { MempoolBlock } from '../mempool.interfaces';
+import { Common } from './common';
+import mempool from './mempool';
+import projectedBlocks from './mempool-blocks';
 
 class FeeApi {
   constructor() { }
 
+  defaultFee = Common.isLiquid() ? 0.1 : 1;
+
   public getRecommendedFee() {
-    const pBlocks = projectedBlocks.getProjectedBlocks();
+    const pBlocks = projectedBlocks.getMempoolBlocks();
+    const mPool = mempool.getMempoolInfo();
+    const minimumFee = Math.ceil(mPool.mempoolminfee * 100000);
+
     if (!pBlocks.length) {
       return {
-        'fastestFee': 0,
-        'halfHourFee': 0,
-        'hourFee': 0,
+        'fastestFee': this.defaultFee,
+        'halfHourFee': this.defaultFee,
+        'hourFee': this.defaultFee,
+        'economyFee': minimumFee,
+        'minimumFee': minimumFee,
       };
     }
-    let firstMedianFee = Math.ceil(pBlocks[0].medianFee);
 
-    if (pBlocks.length === 1 && pBlocks[0].blockWeight <= 2000000) {
-      firstMedianFee = 1;
-    }
-
-    const secondMedianFee = pBlocks[1] ? Math.ceil(pBlocks[1].medianFee) : firstMedianFee;
-    const thirdMedianFee = pBlocks[2] ? Math.ceil(pBlocks[2].medianFee) : secondMedianFee;
+    const firstMedianFee = this.optimizeMedianFee(pBlocks[0], pBlocks[1]);
+    const secondMedianFee = pBlocks[1] ? this.optimizeMedianFee(pBlocks[1], pBlocks[2], firstMedianFee) : this.defaultFee;
+    const thirdMedianFee = pBlocks[2] ? this.optimizeMedianFee(pBlocks[2], pBlocks[3], secondMedianFee) : this.defaultFee;
 
     return {
       'fastestFee': firstMedianFee,
       'halfHourFee': secondMedianFee,
       'hourFee': thirdMedianFee,
+      'economyFee': Math.min(2 * minimumFee, thirdMedianFee),
+      'minimumFee': minimumFee,
     };
   }
 
-  public async $getTransactionsForBlock(blockHeight: number): Promise<any[]> {
-    try {
-      const connection = await DB.pool.getConnection();
-      const query = `SELECT feePerVsize AS fpv FROM transactions WHERE blockheight = ? ORDER BY feePerVsize ASC`;
-      const [rows] = await connection.query<any>(query, [blockHeight]);
-      connection.release();
-      return rows;
-    } catch (e) {
-      console.log('$getTransactionsForBlock() error', e);
-      return [];
+  private optimizeMedianFee(pBlock: MempoolBlock, nextBlock: MempoolBlock | undefined, previousFee?: number): number {
+    const useFee = previousFee ? (pBlock.medianFee + previousFee) / 2 : pBlock.medianFee;
+    if (pBlock.blockVSize <= 500000) {
+      return this.defaultFee;
     }
+    if (pBlock.blockVSize <= 950000 && !nextBlock) {
+      const multiplier = (pBlock.blockVSize - 500000) / 500000;
+      return Math.max(Math.round(useFee * multiplier), this.defaultFee);
+    }
+    return Math.ceil(useFee);
   }
-
 }
 
 export default new FeeApi();

backend/src/api/fiat-conversion.ts

@@ -1,30 +1,122 @@
-import * as request from 'request';
+import logger from '../logger';
+import * as http from 'http';
+import * as https from 'https';
+import axios, { AxiosResponse } from 'axios';
+import { IConversionRates } from '../mempool.interfaces';
+import config from '../config';
+import backendInfo from './backend-info';
+import { SocksProxyAgent } from 'socks-proxy-agent';
 
 class FiatConversion {
-  private tickers = {
-    'BTCUSD': {
-      'USD': 4110.78
-    },
-  };
+  private debasingFiatCurrencies = ['AED', 'AUD', 'BDT', 'BHD', 'BMD', 'BRL', 'CAD', 'CHF', 'CLP',
+    'CNY', 'CZK', 'DKK', 'EUR', 'GBP', 'HKD', 'HUF', 'IDR', 'ILS', 'INR', 'JPY', 'KRW', 'KWD',
+    'LKR', 'MMK', 'MXN', 'MYR', 'NGN', 'NOK', 'NZD', 'PHP', 'PKR', 'PLN', 'RUB', 'SAR', 'SEK',
+    'SGD', 'THB', 'TRY', 'TWD', 'UAH', 'USD', 'VND', 'ZAR'];
+  private conversionRates: IConversionRates = {};
+  private ratesChangedCallback: ((rates: IConversionRates) => void) | undefined;
+  public ratesInitialized = false; // If true, it means rates are ready for use
 
-  constructor() { }
+  constructor() {
+    for (const fiat of this.debasingFiatCurrencies) {
+      this.conversionRates[fiat] = 0;
+    }
+  }
+
+  public setProgressChangedCallback(fn: (rates: IConversionRates) => void) {
+    this.ratesChangedCallback = fn;
+  }
 
   public startService() {
-    setInterval(this.updateCurrency.bind(this), 1000 * 60 * 60);
+    const fiatConversionUrl = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.PRICE_DATA_SERVER.TOR_URL : config.PRICE_DATA_SERVER.CLEARNET_URL;
+    logger.info('Starting currency rates service');
+    if (config.SOCKS5PROXY.ENABLED) {
+      logger.info(`Currency rates service will be queried over the Tor network using ${fiatConversionUrl}`);
+    } else {
+      logger.info(`Currency rates service will be queried over clearnet using ${config.PRICE_DATA_SERVER.CLEARNET_URL}`);
+    }
+    setInterval(this.updateCurrency.bind(this), 1000 * config.MEMPOOL.PRICE_FEED_UPDATE_INTERVAL);
     this.updateCurrency();
   }
 
-  public getTickers() {
-    return this.tickers;
+  public getConversionRates() {
+    return this.conversionRates;
   }
 
-  private updateCurrency() {
-    request('https://api.opennode.co/v1/rates', { json: true }, (err, res, body) => {
-      if (err) { return console.log(err); }
-      if (body && body.data) {
-        this.tickers = body.data;
+  private async updateCurrency(): Promise<void> {
+    type axiosOptions = {
+      headers: {
+        'User-Agent': string
+      };
+      timeout: number;
+      httpAgent?: http.Agent;
+      httpsAgent?: https.Agent;
+    }
+    const setDelay = (secs: number = 1): Promise<void> => new Promise(resolve => setTimeout(() => resolve(), secs * 1000));
+    const fiatConversionUrl = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.PRICE_DATA_SERVER.TOR_URL : config.PRICE_DATA_SERVER.CLEARNET_URL;
+    const isHTTP = (new URL(fiatConversionUrl).protocol.split(':')[0] === 'http') ? true : false;
+    const axiosOptions: axiosOptions = {
+      headers: {
+        'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
+      },
+      timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000
+    };
+
+    let retry = 0;
+
+    while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
+      try {
+        if (config.SOCKS5PROXY.ENABLED) {
+          let socksOptions: any = {
+            agentOptions: {
+              keepAlive: true,
+            },
+            hostname: config.SOCKS5PROXY.HOST,
+            port: config.SOCKS5PROXY.PORT
+          };
+
+          if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
+            socksOptions.username = config.SOCKS5PROXY.USERNAME;
+            socksOptions.password = config.SOCKS5PROXY.PASSWORD;
+          } else {
+            // Retry with different tor circuits https://stackoverflow.com/a/64960234
+            socksOptions.username = `circuit${retry}`;
+          }
+
+          // Handle proxy agent for onion addresses
+          if (isHTTP) {
+            axiosOptions.httpAgent = new SocksProxyAgent(socksOptions);
+          } else {
+            axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions);
+          }
+        }
+        
+        logger.debug('Querying currency rates service...');
+
+        const response: AxiosResponse = await axios.get(`${fiatConversionUrl}`, axiosOptions);
+
+        if (response.statusText === 'error' || !response.data) {
+          throw new Error(`Could not fetch data from ${fiatConversionUrl}, Error: ${response.status}`);
+        }
+
+        for (const rate of response.data.data) {
+          if (this.debasingFiatCurrencies.includes(rate.currencyCode) && rate.provider === 'Bisq-Aggregate') {
+            this.conversionRates[rate.currencyCode] = Math.round(100 * rate.price) / 100;
+          }
+        }
+
+        this.ratesInitialized = true;
+        logger.debug(`USD Conversion Rate: ${this.conversionRates.USD}`);
+
+        if (this.ratesChangedCallback) {
+          this.ratesChangedCallback(this.conversionRates);
+        }
+        break;
+      } catch (e) {
+        logger.err('Error updating fiat conversion rates: '  + (e instanceof Error ? e.message : e));
+        await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL);
+        retry++;
       }
-    });
+    }
   }
 }
 

backend/src/api/liquid/elements-parser.ts

@@ -0,0 +1,103 @@
+import { IBitcoinApi } from '../bitcoin/bitcoin-api.interface';
+import bitcoinClient from '../bitcoin/bitcoin-client';
+import bitcoinSecondClient from '../bitcoin/bitcoin-second-client';
+import { Common } from '../common';
+import DB from '../../database';
+import logger from '../../logger';
+
+class ElementsParser {
+  private isRunning = false;
+
+  constructor() { }
+
+  public async $parse() {
+    if (this.isRunning) {
+      return;
+    }
+    try {
+      this.isRunning = true;
+      const result = await bitcoinClient.getChainTips();
+      const tip = result[0].height;
+      const latestBlockHeight = await this.$getLatestBlockHeightFromDatabase();
+      for (let height = latestBlockHeight + 1; height <= tip; height++) {
+        const blockHash: IBitcoinApi.ChainTips = await bitcoinClient.getBlockHash(height);
+        const block: IBitcoinApi.Block = await bitcoinClient.getBlock(blockHash, 2);
+        await this.$parseBlock(block);
+        await this.$saveLatestBlockToDatabase(block.height);
+      }
+      this.isRunning = false;
+    } catch (e) {
+      this.isRunning = false;
+      throw new Error(e instanceof Error ? e.message : 'Error');
+    }
+  }
+
+  public async $getPegDataByMonth(): Promise<any> {
+    const query = `SELECT SUM(amount) AS amount, DATE_FORMAT(FROM_UNIXTIME(datetime), '%Y-%m-01') AS date FROM elements_pegs GROUP BY DATE_FORMAT(FROM_UNIXTIME(datetime), '%Y%m')`;
+    const [rows] = await DB.query(query);
+    return rows;
+  }
+
+  protected async $parseBlock(block: IBitcoinApi.Block) {
+    for (const tx of block.tx) {
+      await this.$parseInputs(tx, block);
+      await this.$parseOutputs(tx, block);
+    }
+  }
+
+  protected async $parseInputs(tx: IBitcoinApi.Transaction, block: IBitcoinApi.Block) {
+    for (const [index, input] of tx.vin.entries()) {
+      if (input.is_pegin) {
+        await this.$parsePegIn(input, index, tx.txid, block);
+      }
+    }
+  }
+
+  protected async $parsePegIn(input: IBitcoinApi.Vin, vindex: number, txid: string, block: IBitcoinApi.Block) {
+    const bitcoinTx: IBitcoinApi.Transaction = await bitcoinSecondClient.getRawTransaction(input.txid, true);
+    const prevout = bitcoinTx.vout[input.vout || 0];
+    const outputAddress = prevout.scriptPubKey.address || (prevout.scriptPubKey.addresses && prevout.scriptPubKey.addresses[0]) || '';
+    await this.$savePegToDatabase(block.height, block.time, prevout.value * 100000000, txid, vindex,
+      outputAddress, bitcoinTx.txid, prevout.n, 1);
+  }
+
+  protected async $parseOutputs(tx: IBitcoinApi.Transaction, block: IBitcoinApi.Block) {
+    for (const output of tx.vout) {
+      if (output.scriptPubKey.pegout_chain) {
+        await this.$savePegToDatabase(block.height, block.time, 0 - output.value * 100000000, tx.txid, output.n,
+          (output.scriptPubKey.pegout_addresses && output.scriptPubKey.pegout_addresses[0] || ''), '', 0, 0);
+      }
+      if (!output.scriptPubKey.pegout_chain && output.scriptPubKey.type === 'nulldata'
+        && output.value && output.value > 0 && output.asset && output.asset === Common.nativeAssetId) {
+        await this.$savePegToDatabase(block.height, block.time, 0 - output.value * 100000000, tx.txid, output.n,
+          (output.scriptPubKey.pegout_addresses && output.scriptPubKey.pegout_addresses[0] || ''), '', 0, 1);
+      }
+    }
+  }
+
+  protected async $savePegToDatabase(height: number, blockTime: number, amount: number, txid: string,
+    txindex: number, bitcoinaddress: string, bitcointxid: string, bitcoinindex: number, final_tx: number): Promise<void> {
+    const query = `INSERT INTO elements_pegs(
+        block, datetime, amount, txid, txindex, bitcoinaddress, bitcointxid, bitcoinindex, final_tx
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`;
+
+    const params: (string | number)[] = [
+      height, blockTime, amount, txid, txindex, bitcoinaddress, bitcointxid, bitcoinindex, final_tx
+    ];
+    await DB.query(query, params);
+    logger.debug(`Saved L-BTC peg from block height #${height} with TXID ${txid}.`);
+  }
+
+  protected async $getLatestBlockHeightFromDatabase(): Promise<number> {
+    const query = `SELECT number FROM state WHERE name = 'last_elements_block'`;
+    const [rows] = await DB.query(query);
+    return rows[0]['number'];
+  }
+
+  protected async $saveLatestBlockToDatabase(blockHeight: number) {
+    const query = `UPDATE state SET number = ? WHERE name = 'last_elements_block'`;
+    await DB.query(query, [blockHeight]);
+  }
+}
+
+export default new ElementsParser();

backend/src/api/liquid/icons.ts

@@ -0,0 +1,38 @@
+import * as fs from 'fs';
+import logger from '../../logger';
+
+class Icons {
+  private static FILE_NAME = './icons.json';
+  private iconIds: string[] = [];
+  private icons: { [assetId: string]: string; } = {};
+
+  constructor() {}
+
+  public loadIcons() {
+    if (!fs.existsSync(Icons.FILE_NAME)) {
+      logger.warn(`${Icons.FILE_NAME} does not exist. No Liquid icons loaded.`);
+      return;
+    }
+    const cacheData = fs.readFileSync(Icons.FILE_NAME, 'utf8');
+    this.icons = JSON.parse(cacheData);
+
+    for (const i in this.icons) {
+      this.iconIds.push(i);
+    }
+    logger.debug(`Liquid icons has been loaded.`);
+  }
+
+  public getIconByAssetId(assetId: string): Buffer | undefined {
+    const icon = this.icons[assetId];
+    if (icon) {
+      return Buffer.from(icon, 'base64');
+    }
+  }
+
+  public getAllIconIds() {
+    return this.iconIds;
+  }
+
+}
+
+export default new Icons();

backend/src/api/loading-indicators.ts

@@ -0,0 +1,32 @@
+import { ILoadingIndicators } from '../mempool.interfaces';
+
+class LoadingIndicators {
+  private loadingIndicators: ILoadingIndicators = {
+    'mempool': 0,
+  };
+  private progressChangedCallback: ((loadingIndicators: ILoadingIndicators) => void) | undefined;
+
+  constructor() { }
+
+  public setProgressChangedCallback(fn: (loadingIndicators: ILoadingIndicators) => void) {
+    this.progressChangedCallback = fn;
+  }
+
+  public setProgress(name: string, progressPercent: number, rounded: boolean = true) {
+    const newProgress = rounded === true ? Math.round(progressPercent) : progressPercent;
+    if (newProgress >= 100) {
+      delete this.loadingIndicators[name];
+    } else {
+      this.loadingIndicators[name] = newProgress;
+    }
+    if (this.progressChangedCallback) {
+      this.progressChangedCallback(this.loadingIndicators);
+    }
+  }
+
+  public getLoadingIndicators() {
+    return this.loadingIndicators;
+  }
+}
+
+export default new LoadingIndicators();

backend/src/api/memory-cache.ts

@@ -0,0 +1,38 @@
+interface ICache {
+  type: string;
+  id: string;
+  expires: Date;
+  data: any;
+}
+
+class MemoryCache {
+  private cache: ICache[] = [];
+  constructor() {
+    setInterval(this.cleanup.bind(this), 1000);
+  }
+
+  public set(type: string, id: string, data: any, secondsExpiry: number) {
+    const expiry = new Date();
+    expiry.setSeconds(expiry.getSeconds() + secondsExpiry);
+    this.cache.push({
+      type: type,
+      id: id,
+      data: data,
+      expires: expiry,
+    });
+  }
+
+  public get<T>(type: string, id: string): T | null {
+    const found = this.cache.find((cache) => cache.type === type && cache.id === id);
+    if (found) {
+      return found.data;
+    }
+    return null;
+  }
+
+  private cleanup() {
+    this.cache = this.cache.filter((cache) => cache.expires < (new Date()));
+  }
+}
+
+export default new MemoryCache();

backend/src/api/mempool-blocks.ts

@@ -0,0 +1,165 @@
+import logger from '../logger';
+import { MempoolBlock, TransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta } from '../mempool.interfaces';
+import { Common } from './common';
+import config from '../config';
+
+class MempoolBlocks {
+  private mempoolBlocks: MempoolBlockWithTransactions[] = [];
+  private mempoolBlockDeltas: MempoolBlockDelta[] = [];
+
+  constructor() {}
+
+  public getMempoolBlocks(): MempoolBlock[] {
+    return this.mempoolBlocks.map((block) => {
+      return {
+        blockSize: block.blockSize,
+        blockVSize: block.blockVSize,
+        nTx: block.nTx,
+        totalFees: block.totalFees,
+        medianFee: block.medianFee,
+        feeRange: block.feeRange,
+      };
+    });
+  }
+
+  public getMempoolBlocksWithTransactions(): MempoolBlockWithTransactions[] {
+    return this.mempoolBlocks;
+  }
+
+  public getMempoolBlockDeltas(): MempoolBlockDelta[] {
+    return this.mempoolBlockDeltas;
+  }
+
+  public updateMempoolBlocks(memPool: { [txid: string]: TransactionExtended }): void {
+    const latestMempool = memPool;
+    const memPoolArray: TransactionExtended[] = [];
+    for (const i in latestMempool) {
+      if (latestMempool.hasOwnProperty(i)) {
+        memPoolArray.push(latestMempool[i]);
+      }
+    }
+    const start = new Date().getTime();
+
+    // Clear bestDescendants & ancestors
+    memPoolArray.forEach((tx) => {
+      tx.bestDescendant = null;
+      tx.ancestors = [];
+      tx.cpfpChecked = false;
+      if (!tx.effectiveFeePerVsize) {
+        tx.effectiveFeePerVsize = tx.feePerVsize;
+      }
+    });
+
+    // First sort
+    memPoolArray.sort((a, b) => b.feePerVsize - a.feePerVsize);
+
+    // Loop through and traverse all ancestors and sum up all the sizes + fees
+    // Pass down size + fee to all unconfirmed children
+    let sizes = 0;
+    memPoolArray.forEach((tx, i) => {
+      sizes += tx.weight;
+      if (sizes > 4000000 * 8) {
+        return;
+      }
+      Common.setRelativesAndGetCpfpInfo(tx, memPool);
+    });
+
+    // Final sort, by effective fee
+    memPoolArray.sort((a, b) => b.effectiveFeePerVsize - a.effectiveFeePerVsize);
+
+    const end = new Date().getTime();
+    const time = end - start;
+    logger.debug('Mempool blocks calculated in ' + time / 1000 + ' seconds');
+
+    const { blocks, deltas } = this.calculateMempoolBlocks(memPoolArray, this.mempoolBlocks);
+    this.mempoolBlocks = blocks;
+    this.mempoolBlockDeltas = deltas;
+  }
+
+  private calculateMempoolBlocks(transactionsSorted: TransactionExtended[], prevBlocks: MempoolBlockWithTransactions[]):
+    { blocks: MempoolBlockWithTransactions[], deltas: MempoolBlockDelta[] } {
+    const mempoolBlocks: MempoolBlockWithTransactions[] = [];
+    const mempoolBlockDeltas: MempoolBlockDelta[] = [];
+    let blockWeight = 0;
+    let blockSize = 0;
+    let transactions: TransactionExtended[] = [];
+    transactionsSorted.forEach((tx) => {
+      if (blockWeight + tx.weight <= config.MEMPOOL.BLOCK_WEIGHT_UNITS
+        || mempoolBlocks.length === config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT - 1) {
+        blockWeight += tx.weight;
+        blockSize += tx.size;
+        transactions.push(tx);
+      } else {
+        mempoolBlocks.push(this.dataToMempoolBlocks(transactions, blockSize, blockWeight, mempoolBlocks.length));
+        blockWeight = tx.weight;
+        blockSize = tx.size;
+        transactions = [tx];
+      }
+    });
+    if (transactions.length) {
+      mempoolBlocks.push(this.dataToMempoolBlocks(transactions, blockSize, blockWeight, mempoolBlocks.length));
+    }
+    // Calculate change from previous block states
+    for (let i = 0; i < Math.max(mempoolBlocks.length, prevBlocks.length); i++) {
+      let added: TransactionStripped[] = [];
+      let removed: string[] = [];
+      if (mempoolBlocks[i] && !prevBlocks[i]) {
+        added = mempoolBlocks[i].transactions;
+      } else if (!mempoolBlocks[i] && prevBlocks[i]) {
+        removed = prevBlocks[i].transactions.map(tx => tx.txid);
+      } else if (mempoolBlocks[i] && prevBlocks[i]) {
+        const prevIds = {};
+        const newIds = {};
+        prevBlocks[i].transactions.forEach(tx => {
+          prevIds[tx.txid] = true;
+        });
+        mempoolBlocks[i].transactions.forEach(tx => {
+          newIds[tx.txid] = true;
+        });
+        prevBlocks[i].transactions.forEach(tx => {
+          if (!newIds[tx.txid]) {
+            removed.push(tx.txid);
+          }
+        });
+        mempoolBlocks[i].transactions.forEach(tx => {
+          if (!prevIds[tx.txid]) {
+            added.push(tx);
+          }
+        });
+      }
+      mempoolBlockDeltas.push({
+        added,
+        removed
+      });
+    }
+    return {
+      blocks: mempoolBlocks,
+      deltas: mempoolBlockDeltas
+    };
+  }
+
+  private dataToMempoolBlocks(transactions: TransactionExtended[],
+    blockSize: number, blockWeight: number, blocksIndex: number): MempoolBlockWithTransactions {
+    let rangeLength = 4;
+    if (blocksIndex === 0) {
+      rangeLength = 8;
+    }
+    if (transactions.length > 4000) {
+      rangeLength = 6;
+    } else if (transactions.length > 10000) {
+      rangeLength = 8;
+    }
+    return {
+      blockSize: blockSize,
+      blockVSize: blockWeight / 4,
+      nTx: transactions.length,
+      totalFees: transactions.reduce((acc, cur) => acc + cur.fee, 0),
+      medianFee: Common.percentile(transactions.map((tx) => tx.effectiveFeePerVsize), config.MEMPOOL.RECOMMENDED_FEE_PERCENTILE),
+      feeRange: Common.getFeesInRange(transactions, rangeLength),
+      transactionIds: transactions.map((tx) => tx.txid),
+      transactions: transactions.map((tx) => Common.stripTransaction(tx)),
+    };
+  }
+}
+
+export default new MempoolBlocks();

backend/src/api/mempool.ts

@@ -1,35 +1,84 @@
-const config = require('../../mempool-config.json');
+import config from '../config';
 import bitcoinApi from './bitcoin/bitcoin-api-factory';
-import { ITransaction, IMempoolInfo, IMempool } from '../interfaces';
+import { TransactionExtended, VbytesPerSecond } from '../mempool.interfaces';
+import logger from '../logger';
+import { Common } from './common';
+import transactionUtils from './transaction-utils';
+import { IBitcoinApi } from './bitcoin/bitcoin-api.interface';
+import loadingIndicators from './loading-indicators';
+import bitcoinClient from './bitcoin/bitcoin-client';
+import bitcoinSecondClient from './bitcoin/bitcoin-second-client';
+import rbfCache from './rbf-cache';
 
 class Mempool {
-  private mempool: IMempool = {};
-  private mempoolInfo: IMempoolInfo | undefined;
-  private mempoolChangedCallback: Function | undefined;
+  private static WEBSOCKET_REFRESH_RATE_MS = 10000;
+  private static LAZY_DELETE_AFTER_SECONDS = 30;
+  private inSync: boolean = false;
+  private mempoolCacheDelta: number = -1;
+  private mempoolCache: { [txId: string]: TransactionExtended } = {};
+  private mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0, total_fee: 0,
+                                                    maxmempool: 300000000, mempoolminfee: 0.00001000, minrelaytxfee: 0.00001000 };
+  private mempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[],
+    deletedTransactions: TransactionExtended[]) => void) | undefined;
 
   private txPerSecondArray: number[] = [];
   private txPerSecond: number = 0;
 
-  private vBytesPerSecondArray: any[] = [];
+  private vBytesPerSecondArray: VbytesPerSecond[] = [];
   private vBytesPerSecond: number = 0;
+  private mempoolProtection = 0;
+  private latestTransactions: any[] = [];
 
   constructor() {
     setInterval(this.updateTxPerSecond.bind(this), 1000);
+    setInterval(this.deleteExpiredTransactions.bind(this), 20000);
   }
 
-  public setMempoolChangedCallback(fn: Function) {
+  /**
+   * Return true if we should leave resources available for mempool tx caching
+   */
+  public hasPriority(): boolean {
+    if (this.inSync) {
+      return false;
+    } else {
+      return this.mempoolCacheDelta == -1 || this.mempoolCacheDelta > 25;
+    }
+  }
+
+  public isInSync(): boolean {
+    return this.inSync;
+  }
+
+  public setOutOfSync(): void {
+    this.inSync = false;
+    loadingIndicators.setProgress('mempool', 99);
+  }
+
+  public getLatestTransactions() {
+    return this.latestTransactions;
+  }
+
+  public setMempoolChangedCallback(fn: (newMempool: { [txId: string]: TransactionExtended; },
+    newTransactions: TransactionExtended[], deletedTransactions: TransactionExtended[]) => void) {
     this.mempoolChangedCallback = fn;
   }
 
-  public getMempool(): { [txid: string]: ITransaction } {
-    return this.mempool;
+  public getMempool(): { [txid: string]: TransactionExtended } {
+    return this.mempoolCache;
   }
 
-  public setMempool(mempoolData: any) {
-    this.mempool = mempoolData;
+  public setMempool(mempoolData: { [txId: string]: TransactionExtended }) {
+    this.mempoolCache = mempoolData;
+    if (this.mempoolChangedCallback) {
+      this.mempoolChangedCallback(this.mempoolCache, [], []);
+    }
   }
 
-  public getMempoolInfo(): IMempoolInfo | undefined {
+  public async $updateMemPoolInfo() {
+    this.mempoolInfo = await this.$getMempoolInfo();
+  }
+
+  public getMempoolInfo(): IBitcoinApi.MempoolInfo {
     return this.mempoolInfo;
   }
 
@@ -41,128 +90,165 @@ class Mempool {
     return this.vBytesPerSecond;
   }
 
-  public async updateMemPoolInfo() {
-    try {
-      this.mempoolInfo = await bitcoinApi.getMempoolInfo();
-    } catch (err) {
-      console.log('Error getMempoolInfo', err);
-    }
-  }
-
-  public async getRawTransaction(txId: string, isCoinbase = false): Promise<ITransaction | false> {
-    try {
-      const transaction = await bitcoinApi.getRawTransaction(txId);
-
-      let totalOut = 0;
-      transaction.vout.forEach((output) => totalOut += output.value);
-
-      if (config.BACKEND_API === 'electrs') {
-        transaction.feePerWeightUnit = (transaction.fee * 100000000) / transaction.weight || 0;
-        transaction.feePerVsize = (transaction.fee * 100000000) / (transaction.vsize) || 0;
-        transaction.totalOut = totalOut / 100000000;
+  public getFirstSeenForTransactions(txIds: string[]): number[] {
+    const txTimes: number[] = [];
+    txIds.forEach((txId: string) => {
+      const tx = this.mempoolCache[txId];
+      if (tx && tx.firstSeen) {
+        txTimes.push(tx.firstSeen);
       } else {
-        let totalIn = 0;
-        if (!isCoinbase) {
-          for (let i = 0; i < transaction.vin.length; i++) {
-            try {
-              const result = await bitcoinApi.getRawTransaction(transaction.vin[i].txid);
-              transaction.vin[i]['value'] = result.vout[transaction.vin[i].vout].value;
-              totalIn += result.vout[transaction.vin[i].vout].value;
-            } catch (err) {
-              console.log('Locating historical tx error');
-            }
-          }
-        }
-
-        if (totalIn > totalOut) {
-          transaction.fee = parseFloat((totalIn - totalOut).toFixed(8));
-          transaction.feePerWeightUnit = (transaction.fee * 100000000) / (transaction.vsize * 4) || 0;
-          transaction.feePerVsize = (transaction.fee * 100000000) / (transaction.vsize) || 0;
-        } else if (!isCoinbase) {
-          transaction.fee = 0;
-          transaction.feePerVsize = 0;
-          transaction.feePerWeightUnit = 0;
-          console.log('Minus fee error!');
-        }
-        transaction.totalOut = totalOut;
+        txTimes.push(0);
       }
-      return transaction;
-    } catch (e) {
-      console.log(txId + ' not found');
-      return false;
-    }
+    });
+    return txTimes;
   }
 
-  public async updateMempool() {
-    console.log('Updating mempool');
+  public async $updateMempool() {
+    logger.debug('Updating mempool');
     const start = new Date().getTime();
     let hasChange: boolean = false;
+    const currentMempoolSize = Object.keys(this.mempoolCache).length;
     let txCount = 0;
-    try {
-      const transactions = await bitcoinApi.getRawMempool();
-      const diff = transactions.length - Object.keys(this.mempool).length;
-      for (const tx of transactions) {
-        if (!this.mempool[tx]) {
-          const transaction = await this.getRawTransaction(tx);
-          if (transaction) {
-            this.mempool[tx] = transaction;
-            txCount++;
+    const transactions = await bitcoinApi.$getRawMempool();
+    const diff = transactions.length - currentMempoolSize;
+    const newTransactions: TransactionExtended[] = [];
+
+    this.mempoolCacheDelta = Math.abs(diff);
+
+    if (!this.inSync) {
+      loadingIndicators.setProgress('mempool', Object.keys(this.mempoolCache).length / transactions.length * 100);
+    }
+
+    for (const txid of transactions) {
+      if (!this.mempoolCache[txid]) {
+        try {
+          const transaction = await transactionUtils.$getTransactionExtended(txid);
+          this.mempoolCache[txid] = transaction;
+          txCount++;
+          if (this.inSync) {
             this.txPerSecondArray.push(new Date().getTime());
             this.vBytesPerSecondArray.push({
               unixTime: new Date().getTime(),
               vSize: transaction.vsize,
             });
-            hasChange = true;
-            if (diff > 0) {
-              console.log('Calculated fee for transaction ' + txCount + ' / ' + diff);
-            } else {
-              console.log('Calculated fee for transaction ' + txCount);
-            }
-          } else {
-            console.log('Error finding transaction in mempool.');
           }
-        }
-
-        if ((new Date().getTime()) - start > config.MEMPOOL_REFRESH_RATE_MS * 10) {
-          break;
-        }
-      }
-
-      const newMempool: IMempool = {};
-      transactions.forEach((tx) => {
-        if (this.mempool[tx]) {
-          newMempool[tx] = this.mempool[tx];
-        } else {
           hasChange = true;
+          if (diff > 0) {
+            logger.debug('Fetched transaction ' + txCount + ' / ' + diff);
+          } else {
+            logger.debug('Fetched transaction ' + txCount);
+          }
+          newTransactions.push(transaction);
+        } catch (e) {
+          logger.debug('Error finding transaction in mempool: ' + (e instanceof Error ? e.message : e));
         }
-      });
-
-      this.mempool = newMempool;
-
-      if (hasChange && this.mempoolChangedCallback) {
-        this.mempoolChangedCallback(this.mempool);
       }
 
-      const end = new Date().getTime();
-      const time = end - start;
-      console.log('Mempool updated in ' + time / 1000 + ' seconds');
-    } catch (err) {
-      console.log('getRawMempool error.', err);
+      if ((new Date().getTime()) - start > Mempool.WEBSOCKET_REFRESH_RATE_MS) {
+        break;
+      }
+    }
+
+    // Prevent mempool from clear on bitcoind restart by delaying the deletion
+    if (this.mempoolProtection === 0
+      && currentMempoolSize > 20000
+      && transactions.length / currentMempoolSize <= 0.80
+    ) {
+      this.mempoolProtection = 1;
+      this.inSync = false;
+      logger.warn(`Mempool clear protection triggered because transactions.length: ${transactions.length} and currentMempoolSize: ${currentMempoolSize}.`);
+      setTimeout(() => {
+        this.mempoolProtection = 2;
+        logger.warn('Mempool clear protection resumed.');
+      }, 1000 * 60 * config.MEMPOOL.CLEAR_PROTECTION_MINUTES);
+    }
+
+    const deletedTransactions: TransactionExtended[] = [];
+
+    if (this.mempoolProtection !== 1) {
+      this.mempoolProtection = 0;
+      // Index object for faster search
+      const transactionsObject = {};
+      transactions.forEach((txId) => transactionsObject[txId] = true);
+
+      // Flag transactions for lazy deletion
+      for (const tx in this.mempoolCache) {
+        if (!transactionsObject[tx] && !this.mempoolCache[tx].deleteAfter) {
+          deletedTransactions.push(this.mempoolCache[tx]);
+          this.mempoolCache[tx].deleteAfter = new Date().getTime() + Mempool.LAZY_DELETE_AFTER_SECONDS * 1000;
+        }
+      }
+    }
+
+    const newTransactionsStripped = newTransactions.map((tx) => Common.stripTransaction(tx));
+    this.latestTransactions = newTransactionsStripped.concat(this.latestTransactions).slice(0, 6);
+
+    if (!this.inSync && transactions.length === Object.keys(this.mempoolCache).length) {
+      this.inSync = true;
+      logger.notice('The mempool is now in sync!');
+      loadingIndicators.setProgress('mempool', 100);
+    }
+
+    this.mempoolCacheDelta = Math.abs(transactions.length - Object.keys(this.mempoolCache).length);
+
+    if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) {
+      this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions);
+    }
+
+    const end = new Date().getTime();
+    const time = end - start;
+    logger.debug(`New mempool size: ${Object.keys(this.mempoolCache).length} Change: ${diff}`);
+    logger.debug('Mempool updated in ' + time / 1000 + ' seconds');
+  }
+
+  public handleRbfTransactions(rbfTransactions: { [txid: string]: TransactionExtended; }) {
+    for (const rbfTransaction in rbfTransactions) {
+      if (this.mempoolCache[rbfTransaction]) {
+        // Store replaced transactions
+        rbfCache.add(rbfTransaction, rbfTransactions[rbfTransaction].txid);
+        // Erase the replaced transactions from the local mempool
+        delete this.mempoolCache[rbfTransaction];
+      }
     }
   }
 
   private updateTxPerSecond() {
-    const nowMinusTimeSpan = new Date().getTime() - (1000 * config.TX_PER_SECOND_SPAN_SECONDS);
+    const nowMinusTimeSpan = new Date().getTime() - (1000 * config.STATISTICS.TX_PER_SECOND_SAMPLE_PERIOD);
     this.txPerSecondArray = this.txPerSecondArray.filter((unixTime) => unixTime > nowMinusTimeSpan);
-    this.txPerSecond = this.txPerSecondArray.length / config.TX_PER_SECOND_SPAN_SECONDS || 0;
+    this.txPerSecond = this.txPerSecondArray.length / config.STATISTICS.TX_PER_SECOND_SAMPLE_PERIOD || 0;
 
     this.vBytesPerSecondArray = this.vBytesPerSecondArray.filter((data) => data.unixTime > nowMinusTimeSpan);
     if (this.vBytesPerSecondArray.length) {
       this.vBytesPerSecond = Math.round(
-        this.vBytesPerSecondArray.map((data) => data.vSize).reduce((a, b) => a + b) / config.TX_PER_SECOND_SPAN_SECONDS
+        this.vBytesPerSecondArray.map((data) => data.vSize).reduce((a, b) => a + b) / config.STATISTICS.TX_PER_SECOND_SAMPLE_PERIOD
       );
     }
   }
+
+  private deleteExpiredTransactions() {
+    const now = new Date().getTime();
+    for (const tx in this.mempoolCache) {
+      const lazyDeleteAt = this.mempoolCache[tx].deleteAfter;
+      if (lazyDeleteAt && lazyDeleteAt < now) {
+        delete this.mempoolCache[tx];
+      }
+    }
+  }
+
+  private $getMempoolInfo() {
+    if (config.MEMPOOL.USE_SECOND_NODE_FOR_MINFEE) {
+      return Promise.all([
+        bitcoinClient.getMempoolInfo(),
+        bitcoinSecondClient.getMempoolInfo()
+      ]).then(([mempoolInfo, secondMempoolInfo]) => {
+        mempoolInfo.maxmempool = secondMempoolInfo.maxmempool;
+        mempoolInfo.mempoolminfee = secondMempoolInfo.mempoolminfee;
+        mempoolInfo.minrelaytxfee = secondMempoolInfo.minrelaytxfee;
+        return mempoolInfo;
+      });
+    }
+    return bitcoinClient.getMempoolInfo();
+  }
 }
 
 export default new Mempool();

backend/src/api/mining.ts

@@ -0,0 +1,405 @@
+import { PoolInfo, PoolStats, RewardStats } from '../mempool.interfaces';
+import BlocksRepository from '../repositories/BlocksRepository';
+import PoolsRepository from '../repositories/PoolsRepository';
+import HashratesRepository from '../repositories/HashratesRepository';
+import bitcoinClient from './bitcoin/bitcoin-client';
+import logger from '../logger';
+import { Common } from './common';
+import loadingIndicators from './loading-indicators';
+import { escape } from 'mysql2';
+
+class Mining {
+  constructor() {
+  }
+
+  /**
+   * Get historical block total fee
+   */
+  public async $getHistoricalBlockFees(interval: string | null = null): Promise<any> {
+    return await BlocksRepository.$getHistoricalBlockFees(
+      this.getTimeRange(interval),
+      Common.getSqlInterval(interval)
+    );
+  }
+
+  /**
+   * Get historical block rewards
+   */
+  public async $getHistoricalBlockRewards(interval: string | null = null): Promise<any> {
+    return await BlocksRepository.$getHistoricalBlockRewards(
+      this.getTimeRange(interval),
+      Common.getSqlInterval(interval)
+    );
+  }
+
+  /**
+   * Get historical block fee rates percentiles
+   */
+   public async $getHistoricalBlockFeeRates(interval: string | null = null): Promise<any> {
+    return await BlocksRepository.$getHistoricalBlockFeeRates(
+      this.getTimeRange(interval),
+      Common.getSqlInterval(interval)
+    );
+  }
+
+  /**
+   * Get historical block sizes
+   */
+   public async $getHistoricalBlockSizes(interval: string | null = null): Promise<any> {
+    return await BlocksRepository.$getHistoricalBlockSizes(
+      this.getTimeRange(interval),
+      Common.getSqlInterval(interval)
+    );
+  }
+
+  /**
+   * Get historical block weights
+   */
+   public async $getHistoricalBlockWeights(interval: string | null = null): Promise<any> {
+    return await BlocksRepository.$getHistoricalBlockWeights(
+      this.getTimeRange(interval),
+      Common.getSqlInterval(interval)
+    );
+  }
+
+  /**
+   * Generate high level overview of the pool ranks and general stats
+   */
+  public async $getPoolsStats(interval: string | null): Promise<object> {
+    const poolsStatistics = {};
+
+    const poolsInfo: PoolInfo[] = await PoolsRepository.$getPoolsInfo(interval);
+    const emptyBlocks: any[] = await BlocksRepository.$countEmptyBlocks(null, interval);
+
+    const poolsStats: PoolStats[] = [];
+    let rank = 1;
+
+    poolsInfo.forEach((poolInfo: PoolInfo) => {
+      const emptyBlocksCount = emptyBlocks.filter((emptyCount) => emptyCount.poolId === poolInfo.poolId);
+      const poolStat: PoolStats = {
+        poolId: poolInfo.poolId, // mysql row id
+        name: poolInfo.name,
+        link: poolInfo.link,
+        blockCount: poolInfo.blockCount,
+        rank: rank++,
+        emptyBlocks: emptyBlocksCount.length > 0 ? emptyBlocksCount[0]['count'] : 0,
+        slug: poolInfo.slug,
+      };
+      poolsStats.push(poolStat);
+    });
+
+    poolsStatistics['pools'] = poolsStats;
+
+    const blockCount: number = await BlocksRepository.$blockCount(null, interval);
+    poolsStatistics['blockCount'] = blockCount;
+
+    const totalBlock24h: number = await BlocksRepository.$blockCount(null, '24h');
+
+    try {
+      poolsStatistics['lastEstimatedHashrate'] = await bitcoinClient.getNetworkHashPs(totalBlock24h);
+    } catch (e) {
+      poolsStatistics['lastEstimatedHashrate'] = 0;
+      logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate');
+    }
+
+    return poolsStatistics;
+  }
+
+  /**
+   * Get all mining pool stats for a pool
+   */
+  public async $getPoolStat(slug: string): Promise<object> {
+    const pool = await PoolsRepository.$getPool(slug);
+    if (!pool) {
+      throw new Error('This mining pool does not exist ' + escape(slug));
+    }
+
+    const blockCount: number = await BlocksRepository.$blockCount(pool.id);
+    const totalBlock: number = await BlocksRepository.$blockCount(null, null);
+
+    const blockCount24h: number = await BlocksRepository.$blockCount(pool.id, '24h');
+    const totalBlock24h: number = await BlocksRepository.$blockCount(null, '24h');
+
+    const blockCount1w: number = await BlocksRepository.$blockCount(pool.id, '1w');
+    const totalBlock1w: number = await BlocksRepository.$blockCount(null, '1w');
+
+    let currentEstimatedHashrate = 0;
+    try {
+      currentEstimatedHashrate = await bitcoinClient.getNetworkHashPs(totalBlock24h);
+    } catch (e) {
+      logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate');
+    }
+
+    return {
+      pool: pool,
+      blockCount: {
+        'all': blockCount,
+        '24h': blockCount24h,
+        '1w': blockCount1w,
+      },
+      blockShare: {
+        'all': blockCount / totalBlock,
+        '24h': blockCount24h / totalBlock24h,
+        '1w': blockCount1w / totalBlock1w,
+      },
+      estimatedHashrate: currentEstimatedHashrate * (blockCount24h / totalBlock24h),
+      reportedHashrate: null,
+    };
+  }
+
+  /**
+   * Get miner reward stats
+   */
+  public async $getRewardStats(blockCount: number): Promise<RewardStats> {
+    return await BlocksRepository.$getBlockStats(blockCount);
+  }
+
+  /**
+   * [INDEXING] Generate weekly mining pool hashrate history
+   */
+  public async $generatePoolHashrateHistory(): Promise<void> {
+    const now = new Date();
+
+    try {
+      const lastestRunDate = await HashratesRepository.$getLatestRun('last_weekly_hashrates_indexing');
+
+      // Run only if:
+      // * lastestRunDate is set to 0 (node backend restart, reorg)
+      // * we started a new week (around Monday midnight)
+      const runIndexing = lastestRunDate === 0 || now.getUTCDay() === 1 && lastestRunDate !== now.getUTCDate();
+      if (!runIndexing) {
+        return;
+      }
+    } catch (e) {
+      throw e;
+    }
+
+    try {
+      const indexedTimestamp = await HashratesRepository.$getWeeklyHashrateTimestamps();
+      const hashrates: any[] = [];
+      const genesisTimestamp = 1231006505000; // bitcoin-cli getblock 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f
+
+      const lastMonday = new Date(now.setDate(now.getDate() - (now.getDay() + 6) % 7));
+      const lastMondayMidnight = this.getDateMidnight(lastMonday);
+      let toTimestamp = lastMondayMidnight.getTime();
+
+      const totalWeekIndexed = (await BlocksRepository.$blockCount(null, null)) / 1008;
+      let indexedThisRun = 0;
+      let totalIndexed = 0;
+      let newlyIndexed = 0;
+      const startedAt = new Date().getTime() / 1000;
+      let timer = new Date().getTime() / 1000;
+
+      logger.debug(`Indexing weekly mining pool hashrate`);
+      loadingIndicators.setProgress('weekly-hashrate-indexing', 0);
+
+      while (toTimestamp > genesisTimestamp) {
+        const fromTimestamp = toTimestamp - 604800000;
+
+        // Skip already indexed weeks
+        if (indexedTimestamp.includes(toTimestamp / 1000)) {
+          toTimestamp -= 604800000;
+          ++totalIndexed;
+          continue;
+        }
+
+        // Check if we have blocks for the previous week (which mean that the week
+        // we are currently indexing has complete data)
+        const blockStatsPreviousWeek: any = await BlocksRepository.$blockCountBetweenTimestamp(
+          null, (fromTimestamp - 604800000) / 1000, (toTimestamp - 604800000) / 1000);
+        if (blockStatsPreviousWeek.blockCount === 0) { // We are done indexing
+          break;
+        }
+
+        const blockStats: any = await BlocksRepository.$blockCountBetweenTimestamp(
+          null, fromTimestamp / 1000, toTimestamp / 1000);
+        const lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount,
+          blockStats.lastBlockHeight);
+
+        let pools = await PoolsRepository.$getPoolsInfoBetween(fromTimestamp / 1000, toTimestamp / 1000);
+        const totalBlocks = pools.reduce((acc, pool) => acc + pool.blockCount, 0);
+        pools = pools.map((pool: any) => {
+          pool.hashrate = (pool.blockCount / totalBlocks) * lastBlockHashrate;
+          pool.share = (pool.blockCount / totalBlocks);
+          return pool;
+        });
+
+        for (const pool of pools) {
+          hashrates.push({
+            hashrateTimestamp: toTimestamp / 1000,
+            avgHashrate: pool['hashrate'],
+            poolId: pool.poolId,
+            share: pool['share'],
+            type: 'weekly',
+          });
+        }
+
+        newlyIndexed += hashrates.length;
+        await HashratesRepository.$saveHashrates(hashrates);
+        hashrates.length = 0;
+
+        const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
+        if (elapsedSeconds > 1) {
+          const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
+          const weeksPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds));
+          const progress = Math.round(totalIndexed / totalWeekIndexed * 10000) / 100;
+          const timeLeft = Math.round((totalWeekIndexed - totalIndexed) / weeksPerSeconds);
+          const formattedDate = new Date(fromTimestamp).toUTCString();
+          logger.debug(`Getting weekly pool hashrate for ${formattedDate} | ~${weeksPerSeconds.toFixed(2)} weeks/sec | total: ~${totalIndexed}/${Math.round(totalWeekIndexed)} (${progress}%) | elapsed: ${runningFor} seconds | left: ~${timeLeft} seconds`);
+          timer = new Date().getTime() / 1000;
+          indexedThisRun = 0;
+          loadingIndicators.setProgress('weekly-hashrate-indexing', progress, false);
+        }
+
+        toTimestamp -= 604800000;
+        ++indexedThisRun;
+        ++totalIndexed;
+      }
+      await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', new Date().getUTCDate());
+      if (newlyIndexed > 0) {
+        logger.info(`Indexed ${newlyIndexed} pools weekly hashrate`);
+      }
+      loadingIndicators.setProgress('weekly-hashrate-indexing', 100);
+    } catch (e) {
+      loadingIndicators.setProgress('weekly-hashrate-indexing', 100);
+      throw e;
+    }
+  }
+
+  /**
+   * [INDEXING] Generate daily hashrate data
+   */
+  public async $generateNetworkHashrateHistory(): Promise<void> {
+    try {
+      // We only run this once a day around midnight
+      const latestRunDate = await HashratesRepository.$getLatestRun('last_hashrates_indexing');
+      const now = new Date().getUTCDate();
+      if (now === latestRunDate) {
+        return;
+      }
+    } catch (e) {
+      throw e;
+    }
+
+    try {
+      const indexedTimestamp = (await HashratesRepository.$getNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp);
+      const genesisTimestamp = 1231006505000; // bitcoin-cli getblock 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f
+      const lastMidnight = this.getDateMidnight(new Date());
+      let toTimestamp = Math.round(lastMidnight.getTime());
+      const hashrates: any[] = [];
+
+      const totalDayIndexed = (await BlocksRepository.$blockCount(null, null)) / 144;
+      let indexedThisRun = 0;
+      let totalIndexed = 0;
+      let newlyIndexed = 0;
+      const startedAt = new Date().getTime() / 1000;
+      let timer = new Date().getTime() / 1000;
+
+      logger.debug(`Indexing daily network hashrate`);
+      loadingIndicators.setProgress('daily-hashrate-indexing', 0);
+
+      while (toTimestamp > genesisTimestamp) {
+        const fromTimestamp = toTimestamp - 86400000;
+
+        // Skip already indexed weeks
+        if (indexedTimestamp.includes(toTimestamp / 1000)) {
+          toTimestamp -= 86400000;
+          ++totalIndexed;
+          continue;
+        }
+
+        // Check if we have blocks for the previous day (which mean that the day
+        // we are currently indexing has complete data)
+        const blockStatsPreviousDay: any = await BlocksRepository.$blockCountBetweenTimestamp(
+          null, (fromTimestamp - 86400000) / 1000, (toTimestamp - 86400000) / 1000);
+        if (blockStatsPreviousDay.blockCount === 0) { // We are done indexing
+          break;
+        }
+
+        const blockStats: any = await BlocksRepository.$blockCountBetweenTimestamp(
+          null, fromTimestamp / 1000, toTimestamp / 1000);
+        const lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount,
+          blockStats.lastBlockHeight);
+
+        hashrates.push({
+          hashrateTimestamp: toTimestamp / 1000,
+          avgHashrate: lastBlockHashrate,
+          poolId: 0,
+          share: 1,
+          type: 'daily',
+        });
+
+        if (hashrates.length > 10) {
+          newlyIndexed += hashrates.length;
+          await HashratesRepository.$saveHashrates(hashrates);
+          hashrates.length = 0;
+        }
+
+        const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
+        if (elapsedSeconds > 1) {
+          const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
+          const daysPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds));
+          const progress = Math.round(totalIndexed / totalDayIndexed * 10000) / 100;
+          const timeLeft = Math.round((totalDayIndexed - totalIndexed) / daysPerSeconds);
+          const formattedDate = new Date(fromTimestamp).toUTCString();
+          logger.debug(`Getting network daily hashrate for ${formattedDate} | ~${daysPerSeconds.toFixed(2)} days/sec | total: ~${totalIndexed}/${Math.round(totalDayIndexed)} (${progress}%) | elapsed: ${runningFor} seconds | left: ~${timeLeft} seconds`);
+          timer = new Date().getTime() / 1000;
+          indexedThisRun = 0;
+          loadingIndicators.setProgress('daily-hashrate-indexing', progress);
+        }
+
+        toTimestamp -= 86400000;
+        ++indexedThisRun;
+        ++totalIndexed;
+      }
+
+      // Add genesis block manually
+      if (toTimestamp <= genesisTimestamp && !indexedTimestamp.includes(genesisTimestamp)) {
+        hashrates.push({
+          hashrateTimestamp: genesisTimestamp,
+          avgHashrate: await bitcoinClient.getNetworkHashPs(1, 1),
+          poolId: null,
+          type: 'daily',
+        });
+      }
+
+      newlyIndexed += hashrates.length;
+      await HashratesRepository.$saveHashrates(hashrates);
+
+      await HashratesRepository.$setLatestRun('last_hashrates_indexing', new Date().getUTCDate());
+      if (newlyIndexed > 0) {
+        logger.info(`Indexed ${newlyIndexed} day of network hashrate`);
+      }
+      loadingIndicators.setProgress('daily-hashrate-indexing', 100);
+    } catch (e) {
+      loadingIndicators.setProgress('daily-hashrate-indexing', 100);
+      throw e;
+    }
+  }
+
+  private getDateMidnight(date: Date): Date {
+    date.setUTCHours(0);
+    date.setUTCMinutes(0);
+    date.setUTCSeconds(0);
+    date.setUTCMilliseconds(0);
+
+    return date;
+  }
+
+  private getTimeRange(interval: string | null): number {
+    switch (interval) {
+      case '3y': return 43200; // 12h
+      case '2y': return 28800; // 8h
+      case '1y': return 28800; // 8h
+      case '6m': return 10800; // 3h
+      case '3m': return 7200; // 2h
+      case '1m': return 1800; // 30min
+      case '1w': return 300; // 5min
+      case '3d': return 1;
+      case '24h': return 1;
+      default: return 86400; // 24h
+    }
+  }
+}
+
+export default new Mining();

backend/src/api/pools-parser.ts

@@ -0,0 +1,206 @@
+import DB from '../database';
+import logger from '../logger';
+import config from '../config';
+
+interface Pool {
+  name: string;
+  link: string;
+  regexes: string[];
+  addresses: string[];
+  slug: string;
+}
+
+class PoolsParser {
+  miningPools: any[] = [];
+  unknownPool: any = {
+    'name': "Unknown",
+    'link': "https://learnmeabitcoin.com/technical/coinbase-transaction",
+    'regexes': "[]",
+    'addresses': "[]",
+    'slug': 'unknown'
+  };
+  slugWarnFlag = false;
+
+  /**
+   * Parse the pools.json file, consolidate the data and dump it into the database
+   */
+  public async migratePoolsJson(poolsJson: object) {
+    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
+      return;
+    }
+
+    // First we save every entries without paying attention to pool duplication
+    const poolsDuplicated: Pool[] = [];
+
+    logger.debug('Parse coinbase_tags');
+    const coinbaseTags = Object.entries(poolsJson['coinbase_tags']);
+    for (let i = 0; i < coinbaseTags.length; ++i) {
+      poolsDuplicated.push({
+        'name': (<Pool>coinbaseTags[i][1]).name,
+        'link': (<Pool>coinbaseTags[i][1]).link,
+        'regexes': [coinbaseTags[i][0]],
+        'addresses': [],
+        'slug': ''
+      });
+    }
+    logger.debug('Parse payout_addresses');
+    const addressesTags = Object.entries(poolsJson['payout_addresses']);
+    for (let i = 0; i < addressesTags.length; ++i) {
+      poolsDuplicated.push({
+        'name': (<Pool>addressesTags[i][1]).name,
+        'link': (<Pool>addressesTags[i][1]).link,
+        'regexes': [],
+        'addresses': [addressesTags[i][0]],
+        'slug': ''
+      });
+    }
+
+    // Then, we find unique mining pool names
+    logger.debug('Identify unique mining pools');
+    const poolNames: string[] = [];
+    for (let i = 0; i < poolsDuplicated.length; ++i) {
+      if (poolNames.indexOf(poolsDuplicated[i].name) === -1) {
+        poolNames.push(poolsDuplicated[i].name);
+      }
+    }
+    logger.debug(`Found ${poolNames.length} unique mining pools`);
+
+    // Get existing pools from the db
+    let existingPools;
+    try {
+      if (config.DATABASE.ENABLED === true) {
+        [existingPools] = await DB.query({ sql: 'SELECT * FROM pools;', timeout: 120000 });
+      } else {
+        existingPools = [];
+      }
+    } catch (e) {
+      logger.err('Cannot get existing pools from the database, skipping pools.json import');
+      return;
+    }
+
+    this.miningPools = [];
+
+    // Finally, we generate the final consolidated pools data
+    const finalPoolDataAdd: Pool[] = [];
+    const finalPoolDataUpdate: Pool[] = [];
+    for (let i = 0; i < poolNames.length; ++i) {
+      let allAddresses: string[] = [];
+      let allRegexes: string[] = [];
+      const match = poolsDuplicated.filter((pool: Pool) => pool.name === poolNames[i]);
+
+      for (let y = 0; y < match.length; ++y) {
+        allAddresses = allAddresses.concat(match[y].addresses);
+        allRegexes = allRegexes.concat(match[y].regexes);
+      }
+
+      const finalPoolName = poolNames[i].replace(`'`, `''`); // To support single quote in names when doing db queries
+
+      let slug: string | undefined;
+      try {
+        slug = poolsJson['slugs'][poolNames[i]];
+      } catch (e) {
+        if (this.slugWarnFlag === false) {
+          logger.warn(`pools.json does not seem to contain the 'slugs' object`);
+          this.slugWarnFlag = true;
+        }
+      }
+
+      if (slug === undefined) {
+        // Only keep alphanumerical
+        slug = poolNames[i].replace(/[^a-z0-9]/gi, '').toLowerCase();
+        logger.warn(`No slug found for '${poolNames[i]}', generating it => '${slug}'`);
+      }
+
+      const poolObj = {
+        'name': finalPoolName,
+        'link': match[0].link,
+        'regexes': allRegexes,
+        'addresses': allAddresses,
+        'slug': slug
+      };
+
+      if (existingPools.find((pool) => pool.name === poolNames[i]) !== undefined) {
+        finalPoolDataUpdate.push(poolObj);
+      } else {
+        logger.debug(`Add '${finalPoolName}' mining pool`);
+        finalPoolDataAdd.push(poolObj);
+      }
+
+      this.miningPools.push({
+        'name': finalPoolName,
+        'link': match[0].link,
+        'regexes': JSON.stringify(allRegexes),
+        'addresses': JSON.stringify(allAddresses),
+        'slug': slug
+      });
+    }
+
+    if (config.DATABASE.ENABLED === false) { // Don't run db operations
+      logger.info('Mining pools.json import completed (no database)');
+      return;
+    }
+
+    logger.debug(`Update pools table now`);
+
+    // Add new mining pools into the database
+    let queryAdd: string = 'INSERT INTO pools(name, link, regexes, addresses, slug) VALUES ';
+    for (let i = 0; i < finalPoolDataAdd.length; ++i) {
+      queryAdd += `('${finalPoolDataAdd[i].name}', '${finalPoolDataAdd[i].link}',
+      '${JSON.stringify(finalPoolDataAdd[i].regexes)}', '${JSON.stringify(finalPoolDataAdd[i].addresses)}',
+      ${JSON.stringify(finalPoolDataAdd[i].slug)}),`;
+    }
+    queryAdd = queryAdd.slice(0, -1) + ';';
+
+    // Updated existing mining pools in the database
+    const updateQueries: string[] = [];
+    for (let i = 0; i < finalPoolDataUpdate.length; ++i) {
+      updateQueries.push(`
+        UPDATE pools
+        SET name='${finalPoolDataUpdate[i].name}', link='${finalPoolDataUpdate[i].link}',
+        regexes='${JSON.stringify(finalPoolDataUpdate[i].regexes)}', addresses='${JSON.stringify(finalPoolDataUpdate[i].addresses)}',
+        slug='${finalPoolDataUpdate[i].slug}'
+        WHERE name='${finalPoolDataUpdate[i].name}'
+      ;`);
+    }
+
+    try {
+      if (finalPoolDataAdd.length > 0) {
+        await DB.query({ sql: queryAdd, timeout: 120000 });
+      }
+      for (const query of updateQueries) {
+        await DB.query({ sql: query, timeout: 120000 });
+      }
+      await this.insertUnknownPool();
+      logger.info('Mining pools.json import completed');
+    } catch (e) {
+      logger.err(`Cannot import pools in the database`);
+      throw e;
+    }
+  }
+
+  /**
+   * Manually add the 'unknown pool'
+   */
+  private async insertUnknownPool() {
+    try {
+      const [rows]: any[] = await DB.query({ sql: 'SELECT name from pools where name="Unknown"', timeout: 120000 });
+      if (rows.length === 0) {
+        await DB.query({
+          sql: `INSERT INTO pools(name, link, regexes, addresses, slug)
+          VALUES("Unknown", "https://learnmeabitcoin.com/technical/coinbase-transaction", "[]", "[]", "unknown");
+        `});
+      } else {
+        await DB.query(`UPDATE pools
+          SET name='Unknown', link='https://learnmeabitcoin.com/technical/coinbase-transaction',
+          regexes='[]', addresses='[]',
+          slug='unknown'
+          WHERE name='Unknown'
+        `);
+      }
+    } catch (e) {
+      logger.err('Unable to insert "Unknown" mining pool');
+    }
+  }
+}
+
+export default new PoolsParser();

backend/src/api/projected-blocks.ts

@@ -1,101 +0,0 @@
-const config = require('../../mempool-config.json');
-import { ITransaction, IProjectedBlock, IMempool, IProjectedBlockInternal } from '../interfaces';
-
-class ProjectedBlocks {
-  private transactionsSorted: ITransaction[] = [];
-
-  constructor() {}
-
-  public getProjectedBlockFeesForBlock(index: number) {
-    const projectedBlock = this.getProjectedBlocksInternal()[index];
-
-    if (!projectedBlock) {
-      throw new Error('No projected block for that index');
-    }
-
-    return projectedBlock.txFeePerVsizes.map((fpv) => {
-      return {'fpv': fpv};
-    });
-  }
-
-  public updateProjectedBlocks(memPool: IMempool): void {
-    const latestMempool = memPool;
-    const memPoolArray: ITransaction[] = [];
-    for (const i in latestMempool) {
-      if (latestMempool.hasOwnProperty(i)) {
-        memPoolArray.push(latestMempool[i]);
-      }
-    }
-    memPoolArray.sort((a, b) => b.feePerWeightUnit - a.feePerWeightUnit);
-    this.transactionsSorted = memPoolArray.filter((tx) => tx.feePerWeightUnit);
-  }
-
-  public getProjectedBlocks(txId?: string, numberOfBlocks: number = config.DEFAULT_PROJECTED_BLOCKS_AMOUNT): IProjectedBlock[] {
-    return this.getProjectedBlocksInternal(numberOfBlocks).map((projectedBlock) => {
-      return {
-        blockSize: projectedBlock.blockSize,
-        blockWeight: projectedBlock.blockWeight,
-        nTx: projectedBlock.nTx,
-        minFee: projectedBlock.minFee,
-        maxFee: projectedBlock.maxFee,
-        minWeightFee: projectedBlock.minWeightFee,
-        maxWeightFee: projectedBlock.maxWeightFee,
-        medianFee: projectedBlock.medianFee,
-        fees: projectedBlock.fees,
-        hasMytx: txId ? projectedBlock.txIds.some((tx) => tx === txId) : false
-      };
-    });
-  }
-
-  private getProjectedBlocksInternal(numberOfBlocks: number = config.DEFAULT_PROJECTED_BLOCKS_AMOUNT): IProjectedBlockInternal[] {
-    const projectedBlocks: IProjectedBlockInternal[] = [];
-    let blockWeight = 0;
-    let blockSize = 0;
-    let transactions: ITransaction[] = [];
-    this.transactionsSorted.forEach((tx) => {
-      if (blockWeight + tx.vsize * 4 < 4000000 || projectedBlocks.length === numberOfBlocks) {
-        blockWeight += tx.weight || tx.vsize * 4;
-        blockSize += tx.size;
-        transactions.push(tx);
-      } else {
-        projectedBlocks.push(this.dataToProjectedBlock(transactions, blockSize, blockWeight));
-        blockWeight = 0;
-        blockSize = 0;
-        transactions = [];
-      }
-    });
-    if (transactions.length) {
-      projectedBlocks.push(this.dataToProjectedBlock(transactions, blockSize, blockWeight));
-    }
-    return projectedBlocks;
-  }
-
-  private dataToProjectedBlock(transactions: ITransaction[], blockSize: number, blockWeight: number): IProjectedBlockInternal {
-    return {
-      blockSize: blockSize,
-      blockWeight: blockWeight,
-      nTx: transactions.length,
-      minFee: transactions[transactions.length - 1].feePerVsize,
-      maxFee: transactions[0].feePerVsize,
-      minWeightFee: transactions[transactions.length - 1].feePerWeightUnit,
-      maxWeightFee: transactions[0].feePerWeightUnit,
-      medianFee: this.median(transactions.map((tx) => tx.feePerVsize)),
-      txIds: transactions.map((tx) => tx.txid),
-      txFeePerVsizes: transactions.map((tx) => tx.feePerVsize).reverse(),
-      fees: transactions.map((tx) => tx.fee).reduce((acc, currValue) => acc + currValue),
-    };
-  }
-
-  private median(numbers: number[]) {
-    let medianNr = 0;
-    const numsLen = numbers.length;
-    if (numsLen % 2 === 0) {
-        medianNr = (numbers[numsLen / 2 - 1] + numbers[numsLen / 2]) / 2;
-    } else {
-        medianNr = numbers[(numsLen - 1) / 2];
-    }
-    return medianNr;
-  }
-}
-
-export default new ProjectedBlocks();

backend/src/api/rbf-cache.ts

@@ -0,0 +1,34 @@
+export interface CachedRbf {
+  txid: string;
+  expires: Date;
+}
+
+class RbfCache {
+  private cache: { [txid: string]: CachedRbf; } = {};
+
+  constructor() {
+    setInterval(this.cleanup.bind(this), 1000 * 60 * 60);
+  }
+
+  public add(replacedTxId: string, newTxId: string): void {
+    this.cache[replacedTxId] = {
+      expires: new Date(Date.now() + 1000 * 604800), // 1 week
+      txid: newTxId,
+    };
+  }
+
+  public get(txId: string): CachedRbf | undefined {
+    return this.cache[txId];
+  }
+
+  private cleanup(): void {
+    const currentDate = new Date();
+    for (const c in this.cache) {
+      if (this.cache[c].expires < currentDate) {
+        delete this.cache[c];
+      }
+    }
+  }
+}
+
+export default new RbfCache();

backend/src/api/statistics.ts

@@ -1,20 +1,25 @@
 import memPool from './mempool';
-import { DB } from '../database';
+import DB from '../database';
+import logger from '../logger';
 
-import { ITransaction, IMempoolStats } from '../interfaces';
+import { Statistic, TransactionExtended, OptimizedStatistic } from '../mempool.interfaces';
+import config from '../config';
+import { Common } from './common';
 
 class Statistics {
   protected intervalTimer: NodeJS.Timer | undefined;
-  protected newStatisticsEntryCallback: Function | undefined;
+  protected newStatisticsEntryCallback: ((stats: OptimizedStatistic) => void) | undefined;
+  protected queryTimeout = 120000;
 
-  public setNewStatisticsEntryCallback(fn: Function) {
+  public setNewStatisticsEntryCallback(fn: (stats: OptimizedStatistic) => void) {
     this.newStatisticsEntryCallback = fn;
   }
 
-  constructor() {
-  }
+  constructor() { }
 
   public startStatistics(): void {
+    logger.info('Starting statistics service');
+
     const now = new Date();
     const nextInterval = new Date(now.getFullYear(), now.getMonth(), now.getDate(), now.getHours(),
       Math.floor(now.getMinutes() / 1) * 1 + 1, 0, 0);
@@ -22,60 +27,63 @@ class Statistics {
 
     setTimeout(() => {
       this.runStatistics();
-      this.intervalTimer = setInterval(() => { this.runStatistics(); }, 1 * 60 * 1000);
+      this.intervalTimer = setInterval(() => {
+        this.runStatistics();
+      }, 1 * 60 * 1000);
     }, difference);
   }
 
   private async runStatistics(): Promise<void> {
+    if (!memPool.isInSync()) {
+      return;
+    }
     const currentMempool = memPool.getMempool();
     const txPerSecond = memPool.getTxPerSecond();
     const vBytesPerSecond = memPool.getVBytesPerSecond();
 
-    if (txPerSecond === 0) {
-      return;
-    }
+    logger.debug('Running statistics');
 
-    console.log('Running statistics');
-
-    let memPoolArray: ITransaction[] = [];
+    let memPoolArray: TransactionExtended[] = [];
     for (const i in currentMempool) {
       if (currentMempool.hasOwnProperty(i)) {
         memPoolArray.push(currentMempool[i]);
       }
     }
     // Remove 0 and undefined
-    memPoolArray = memPoolArray.filter((tx) => tx.feePerWeightUnit);
+    memPoolArray = memPoolArray.filter((tx) => tx.effectiveFeePerVsize);
 
     if (!memPoolArray.length) {
+      try {
+        const insertIdZeroed = await this.$createZeroedStatistic();
+        if (this.newStatisticsEntryCallback && insertIdZeroed) {
+          const newStats = await this.$get(insertIdZeroed);
+          if (newStats) {
+            this.newStatisticsEntryCallback(newStats);
+          }
+        }
+      } catch (e) {
+        logger.err('Unable to insert zeroed statistics. ' + e);
+      }
       return;
     }
 
-    memPoolArray.sort((a, b) => a.feePerWeightUnit - b.feePerWeightUnit);
+    memPoolArray.sort((a, b) => a.effectiveFeePerVsize - b.effectiveFeePerVsize);
     const totalWeight = memPoolArray.map((tx) => tx.vsize).reduce((acc, curr) => acc + curr) * 4;
     const totalFee = memPoolArray.map((tx) => tx.fee).reduce((acc, curr) => acc + curr);
 
     const logFees = [1, 2, 3, 4, 5, 6, 8, 10, 12, 15, 20, 30, 40, 50, 60, 70, 80, 90, 100, 125, 150, 175, 200,
       250, 300, 350, 400, 500, 600, 700, 800, 900, 1000, 1200, 1400, 1600, 1800, 2000];
 
-    const weightUnitFees: { [feePerWU: number]: number } = {};
     const weightVsizeFees: { [feePerWU: number]: number } = {};
+    const lastItem = logFees.length - 1;
 
     memPoolArray.forEach((transaction) => {
       for (let i = 0; i < logFees.length; i++) {
-        if ((logFees[i] === 2000 && transaction.feePerWeightUnit >= 2000) || transaction.feePerWeightUnit <= logFees[i]) {
-          if (weightUnitFees[logFees[i]]) {
-            weightUnitFees[logFees[i]] += transaction.vsize * 4;
-          } else {
-            weightUnitFees[logFees[i]] = transaction.vsize * 4;
-          }
-          break;
-        }
-      }
-    });
-
-    memPoolArray.forEach((transaction) => {
-      for (let i = 0; i < logFees.length; i++) {
-        if ((logFees[i] === 2000 && transaction.feePerVsize >= 2000) || transaction.feePerVsize <= logFees[i]) {
+        if (
+          (Common.isLiquid() && (i === lastItem || transaction.effectiveFeePerVsize * 10 < logFees[i + 1]))
+          ||
+          (!Common.isLiquid() && (i === lastItem || transaction.effectiveFeePerVsize < logFees[i + 1]))
+        ) {
           if (weightVsizeFees[logFees[i]]) {
             weightVsizeFees[logFees[i]] += transaction.vsize;
           } else {
@@ -86,66 +94,126 @@ class Statistics {
       }
     });
 
-    const insertId = await this.$create({
-      added: 'NOW()',
-      unconfirmed_transactions: memPoolArray.length,
-      tx_per_second: txPerSecond,
-      vbytes_per_second: Math.round(vBytesPerSecond),
-      mempool_byte_weight: totalWeight,
-      total_fee: totalFee,
-      fee_data: JSON.stringify({
-        'wu': weightUnitFees,
-        'vsize': weightVsizeFees
-      }),
-      vsize_1: weightVsizeFees['1'] || 0,
-      vsize_2: weightVsizeFees['2'] || 0,
-      vsize_3: weightVsizeFees['3'] || 0,
-      vsize_4: weightVsizeFees['4'] || 0,
-      vsize_5: weightVsizeFees['5'] || 0,
-      vsize_6: weightVsizeFees['6'] || 0,
-      vsize_8: weightVsizeFees['8'] || 0,
-      vsize_10: weightVsizeFees['10'] || 0,
-      vsize_12: weightVsizeFees['12'] || 0,
-      vsize_15: weightVsizeFees['15'] || 0,
-      vsize_20: weightVsizeFees['20'] || 0,
-      vsize_30: weightVsizeFees['30'] || 0,
-      vsize_40: weightVsizeFees['40'] || 0,
-      vsize_50: weightVsizeFees['50'] || 0,
-      vsize_60: weightVsizeFees['60'] || 0,
-      vsize_70: weightVsizeFees['70'] || 0,
-      vsize_80: weightVsizeFees['80'] || 0,
-      vsize_90: weightVsizeFees['90'] || 0,
-      vsize_100: weightVsizeFees['100'] || 0,
-      vsize_125: weightVsizeFees['125'] || 0,
-      vsize_150: weightVsizeFees['150'] || 0,
-      vsize_175: weightVsizeFees['175'] || 0,
-      vsize_200: weightVsizeFees['200'] || 0,
-      vsize_250: weightVsizeFees['250'] || 0,
-      vsize_300: weightVsizeFees['300'] || 0,
-      vsize_350: weightVsizeFees['350'] || 0,
-      vsize_400: weightVsizeFees['400'] || 0,
-      vsize_500: weightVsizeFees['500'] || 0,
-      vsize_600: weightVsizeFees['600'] || 0,
-      vsize_700: weightVsizeFees['700'] || 0,
-      vsize_800: weightVsizeFees['800'] || 0,
-      vsize_900: weightVsizeFees['900'] || 0,
-      vsize_1000: weightVsizeFees['1000'] || 0,
-      vsize_1200: weightVsizeFees['1200'] || 0,
-      vsize_1400: weightVsizeFees['1400'] || 0,
-      vsize_1600: weightVsizeFees['1600'] || 0,
-      vsize_1800: weightVsizeFees['1800'] || 0,
-      vsize_2000: weightVsizeFees['2000'] || 0,
-    });
+    try {
+      const insertId = await this.$create({
+        added: 'NOW()',
+        unconfirmed_transactions: memPoolArray.length,
+        tx_per_second: txPerSecond,
+        vbytes_per_second: Math.round(vBytesPerSecond),
+        mempool_byte_weight: totalWeight,
+        total_fee: totalFee,
+        fee_data: '',
+        vsize_1: weightVsizeFees['1'] || 0,
+        vsize_2: weightVsizeFees['2'] || 0,
+        vsize_3: weightVsizeFees['3'] || 0,
+        vsize_4: weightVsizeFees['4'] || 0,
+        vsize_5: weightVsizeFees['5'] || 0,
+        vsize_6: weightVsizeFees['6'] || 0,
+        vsize_8: weightVsizeFees['8'] || 0,
+        vsize_10: weightVsizeFees['10'] || 0,
+        vsize_12: weightVsizeFees['12'] || 0,
+        vsize_15: weightVsizeFees['15'] || 0,
+        vsize_20: weightVsizeFees['20'] || 0,
+        vsize_30: weightVsizeFees['30'] || 0,
+        vsize_40: weightVsizeFees['40'] || 0,
+        vsize_50: weightVsizeFees['50'] || 0,
+        vsize_60: weightVsizeFees['60'] || 0,
+        vsize_70: weightVsizeFees['70'] || 0,
+        vsize_80: weightVsizeFees['80'] || 0,
+        vsize_90: weightVsizeFees['90'] || 0,
+        vsize_100: weightVsizeFees['100'] || 0,
+        vsize_125: weightVsizeFees['125'] || 0,
+        vsize_150: weightVsizeFees['150'] || 0,
+        vsize_175: weightVsizeFees['175'] || 0,
+        vsize_200: weightVsizeFees['200'] || 0,
+        vsize_250: weightVsizeFees['250'] || 0,
+        vsize_300: weightVsizeFees['300'] || 0,
+        vsize_350: weightVsizeFees['350'] || 0,
+        vsize_400: weightVsizeFees['400'] || 0,
+        vsize_500: weightVsizeFees['500'] || 0,
+        vsize_600: weightVsizeFees['600'] || 0,
+        vsize_700: weightVsizeFees['700'] || 0,
+        vsize_800: weightVsizeFees['800'] || 0,
+        vsize_900: weightVsizeFees['900'] || 0,
+        vsize_1000: weightVsizeFees['1000'] || 0,
+        vsize_1200: weightVsizeFees['1200'] || 0,
+        vsize_1400: weightVsizeFees['1400'] || 0,
+        vsize_1600: weightVsizeFees['1600'] || 0,
+        vsize_1800: weightVsizeFees['1800'] || 0,
+        vsize_2000: weightVsizeFees['2000'] || 0,
+      });
 
-    if (this.newStatisticsEntryCallback && insertId) {
-      const newStats = await this.$get(insertId);
-      this.newStatisticsEntryCallback(newStats);
+      if (this.newStatisticsEntryCallback && insertId) {
+        const newStats = await this.$get(insertId);
+        if (newStats) {
+          this.newStatisticsEntryCallback(newStats);
+        }
+      }
+    } catch (e) {
+      logger.err('Unable to insert statistics. ' + e);
     }
   }
 
-  private async $create(statistics: IMempoolStats): Promise<number | undefined> {
+  private async $createZeroedStatistic(): Promise<number | undefined> {
+    try {
+      const query = `INSERT INTO statistics(
+              added,
+              unconfirmed_transactions,
+              tx_per_second,
+              vbytes_per_second,
+              mempool_byte_weight,
+              fee_data,
+              total_fee,
+              vsize_1,
+              vsize_2,
+              vsize_3,
+              vsize_4,
+              vsize_5,
+              vsize_6,
+              vsize_8,
+              vsize_10,
+              vsize_12,
+              vsize_15,
+              vsize_20,
+              vsize_30,
+              vsize_40,
+              vsize_50,
+              vsize_60,
+              vsize_70,
+              vsize_80,
+              vsize_90,
+              vsize_100,
+              vsize_125,
+              vsize_150,
+              vsize_175,
+              vsize_200,
+              vsize_250,
+              vsize_300,
+              vsize_350,
+              vsize_400,
+              vsize_500,
+              vsize_600,
+              vsize_700,
+              vsize_800,
+              vsize_900,
+              vsize_1000,
+              vsize_1200,
+              vsize_1400,
+              vsize_1600,
+              vsize_1800,
+              vsize_2000
+            )
+            VALUES (NOW(), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+               0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)`;
+      const [result]: any = await DB.query(query);
+      return result.insertId;
+    } catch (e) {
+      logger.err('$create() error' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  private async $create(statistics: Statistic): Promise<number | undefined> {
     try {
-      const connection = await DB.pool.getConnection();
       const query = `INSERT INTO statistics(
               added,
               unconfirmed_transactions,
@@ -242,148 +310,271 @@ class Statistics {
         statistics.vsize_1800,
         statistics.vsize_2000,
       ];
-      const [result]: any = await connection.query(query, params);
-      connection.release();
+      const [result]: any = await DB.query(query, params);
       return result.insertId;
     } catch (e) {
-      console.log('$create() error', e);
+      logger.err('$create() error' + (e instanceof Error ? e.message : e));
     }
   }
 
-  private getQueryForDays(days: number, groupBy: number) {
-
-    return `SELECT id, added, unconfirmed_transactions,
-            AVG(tx_per_second) AS tx_per_second,
-            AVG(vbytes_per_second) AS vbytes_per_second,
-            AVG(vsize_1) AS vsize_1,
-            AVG(vsize_2) AS vsize_2,
-            AVG(vsize_3) AS vsize_3,
-            AVG(vsize_4) AS vsize_4,
-            AVG(vsize_5) AS vsize_5,
-            AVG(vsize_6) AS vsize_6,
-            AVG(vsize_8) AS vsize_8,
-            AVG(vsize_10) AS vsize_10,
-            AVG(vsize_12) AS vsize_12,
-            AVG(vsize_15) AS vsize_15,
-            AVG(vsize_20) AS vsize_20,
-            AVG(vsize_30) AS vsize_30,
-            AVG(vsize_40) AS vsize_40,
-            AVG(vsize_50) AS vsize_50,
-            AVG(vsize_60) AS vsize_60,
-            AVG(vsize_70) AS vsize_70,
-            AVG(vsize_80) AS vsize_80,
-            AVG(vsize_90) AS vsize_90,
-            AVG(vsize_100) AS vsize_100,
-            AVG(vsize_125) AS vsize_125,
-            AVG(vsize_150) AS vsize_150,
-            AVG(vsize_175) AS vsize_175,
-            AVG(vsize_200) AS vsize_200,
-            AVG(vsize_250) AS vsize_250,
-            AVG(vsize_300) AS vsize_300,
-            AVG(vsize_350) AS vsize_350,
-            AVG(vsize_400) AS vsize_400,
-            AVG(vsize_500) AS vsize_500,
-            AVG(vsize_600) AS vsize_600,
-            AVG(vsize_700) AS vsize_700,
-            AVG(vsize_800) AS vsize_800,
-            AVG(vsize_900) AS vsize_900,
-            AVG(vsize_1000) AS vsize_1000,
-            AVG(vsize_1200) AS vsize_1200,
-            AVG(vsize_1400) AS vsize_1400,
-            AVG(vsize_1600) AS vsize_1600,
-            AVG(vsize_1800) AS vsize_1800,
-            AVG(vsize_2000) AS vsize_2000 FROM statistics GROUP BY UNIX_TIMESTAMP(added) DIV ${groupBy} ORDER BY id DESC LIMIT ${days}`;
+  private getQueryForDaysAvg(div: number, interval: string) {
+    return `SELECT
+      UNIX_TIMESTAMP(added) as added,
+      CAST(avg(vbytes_per_second) as DOUBLE) as vbytes_per_second,
+      CAST(avg(vsize_1) as DOUBLE) as vsize_1,
+      CAST(avg(vsize_2) as DOUBLE) as vsize_2,
+      CAST(avg(vsize_3) as DOUBLE) as vsize_3,
+      CAST(avg(vsize_4) as DOUBLE) as vsize_4,
+      CAST(avg(vsize_5) as DOUBLE) as vsize_5,
+      CAST(avg(vsize_6) as DOUBLE) as vsize_6,
+      CAST(avg(vsize_8) as DOUBLE) as vsize_8,
+      CAST(avg(vsize_10) as DOUBLE) as vsize_10,
+      CAST(avg(vsize_12) as DOUBLE) as vsize_12,
+      CAST(avg(vsize_15) as DOUBLE) as vsize_15,
+      CAST(avg(vsize_20) as DOUBLE) as vsize_20,
+      CAST(avg(vsize_30) as DOUBLE) as vsize_30,
+      CAST(avg(vsize_40) as DOUBLE) as vsize_40,
+      CAST(avg(vsize_50) as DOUBLE) as vsize_50,
+      CAST(avg(vsize_60) as DOUBLE) as vsize_60,
+      CAST(avg(vsize_70) as DOUBLE) as vsize_70,
+      CAST(avg(vsize_80) as DOUBLE) as vsize_80,
+      CAST(avg(vsize_90) as DOUBLE) as vsize_90,
+      CAST(avg(vsize_100) as DOUBLE) as vsize_100,
+      CAST(avg(vsize_125) as DOUBLE) as vsize_125,
+      CAST(avg(vsize_150) as DOUBLE) as vsize_150,
+      CAST(avg(vsize_175) as DOUBLE) as vsize_175,
+      CAST(avg(vsize_200) as DOUBLE) as vsize_200,
+      CAST(avg(vsize_250) as DOUBLE) as vsize_250,
+      CAST(avg(vsize_300) as DOUBLE) as vsize_300,
+      CAST(avg(vsize_350) as DOUBLE) as vsize_350,
+      CAST(avg(vsize_400) as DOUBLE) as vsize_400,
+      CAST(avg(vsize_500) as DOUBLE) as vsize_500,
+      CAST(avg(vsize_600) as DOUBLE) as vsize_600,
+      CAST(avg(vsize_700) as DOUBLE) as vsize_700,
+      CAST(avg(vsize_800) as DOUBLE) as vsize_800,
+      CAST(avg(vsize_900) as DOUBLE) as vsize_900,
+      CAST(avg(vsize_1000) as DOUBLE) as vsize_1000,
+      CAST(avg(vsize_1200) as DOUBLE) as vsize_1200,
+      CAST(avg(vsize_1400) as DOUBLE) as vsize_1400,
+      CAST(avg(vsize_1600) as DOUBLE) as vsize_1600,
+      CAST(avg(vsize_1800) as DOUBLE) as vsize_1800,
+      CAST(avg(vsize_2000) as DOUBLE) as vsize_2000 \
+      FROM statistics \
+      WHERE added BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW() \
+      GROUP BY UNIX_TIMESTAMP(added) DIV ${div} \
+      ORDER BY statistics.added DESC;`;
   }
 
-  public async $get(id: number): Promise<IMempoolStats | undefined> {
+  private getQueryForDays(div: number, interval: string) {
+    return `SELECT
+      UNIX_TIMESTAMP(added) as added,
+      CAST(avg(vbytes_per_second) as DOUBLE) as vbytes_per_second,
+      vsize_1,
+      vsize_2,
+      vsize_3,
+      vsize_4,
+      vsize_5,
+      vsize_6,
+      vsize_8,
+      vsize_10,
+      vsize_12,
+      vsize_15,
+      vsize_20,
+      vsize_30,
+      vsize_40,
+      vsize_50,
+      vsize_60,
+      vsize_70,
+      vsize_80,
+      vsize_90,
+      vsize_100,
+      vsize_125,
+      vsize_150,
+      vsize_175,
+      vsize_200,
+      vsize_250,
+      vsize_300,
+      vsize_350,
+      vsize_400,
+      vsize_500,
+      vsize_600,
+      vsize_700,
+      vsize_800,
+      vsize_900,
+      vsize_1000,
+      vsize_1200,
+      vsize_1400,
+      vsize_1600,
+      vsize_1800,
+      vsize_2000 \
+      FROM statistics \
+      WHERE added BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW() \
+      GROUP BY UNIX_TIMESTAMP(added) DIV ${div} \
+      ORDER BY statistics.added DESC;`;
+  }
+
+  private async $get(id: number): Promise<OptimizedStatistic | undefined> {
     try {
-      const connection = await DB.pool.getConnection();
-      const query = `SELECT * FROM statistics WHERE id = ?`;
-      const [rows] = await connection.query<any>(query, [id]);
-      connection.release();
-      return rows[0];
+      const query = `SELECT *, UNIX_TIMESTAMP(added) as added FROM statistics WHERE id = ?`;
+      const [rows] = await DB.query(query, [id]);
+      if (rows[0]) {
+        return this.mapStatisticToOptimizedStatistic([rows[0]])[0];
+      }
     } catch (e) {
-      console.log('$list2H() error', e);
+      logger.err('$list2H() error' + (e instanceof Error ? e.message : e));
     }
   }
 
-  public async $list2H(): Promise<IMempoolStats[]> {
+  public async $list2H(): Promise<OptimizedStatistic[]> {
     try {
-      const connection = await DB.pool.getConnection();
-      const query = `SELECT * FROM statistics ORDER BY id DESC LIMIT 120`;
-      const [rows] = await connection.query<any>(query);
-      connection.release();
-      return rows;
+      const query = `SELECT *, UNIX_TIMESTAMP(added) as added FROM statistics ORDER BY statistics.added DESC LIMIT 120`;
+      const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
+      return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
     } catch (e) {
-      console.log('$list2H() error', e);
+      logger.err('$list2H() error' + (e instanceof Error ? e.message : e));
       return [];
     }
   }
 
-  public async $list24H(): Promise<IMempoolStats[]> {
+  public async $list24H(): Promise<OptimizedStatistic[]> {
     try {
-      const connection = await DB.pool.getConnection();
-      const query = this.getQueryForDays(120, 720);
-      const [rows] = await connection.query<any>(query);
-      connection.release();
-      return rows;
+      const query = `SELECT *, UNIX_TIMESTAMP(added) as added FROM statistics ORDER BY statistics.added DESC LIMIT 1440`;
+      const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
+      return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
     } catch (e) {
+      logger.err('$list24h() error' + (e instanceof Error ? e.message : e));
       return [];
     }
   }
 
-  public async $list1W(): Promise<IMempoolStats[]> {
+  public async $list1W(): Promise<OptimizedStatistic[]> {
     try {
-      const connection = await DB.pool.getConnection();
-      const query = this.getQueryForDays(120, 5040);
-      const [rows] = await connection.query<any>(query);
-      connection.release();
-      return rows;
+      const query = this.getQueryForDaysAvg(300, '1 WEEK'); // 5m interval
+      const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
+      return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
     } catch (e) {
-      console.log('$list1W() error', e);
+      logger.err('$list1W() error' + (e instanceof Error ? e.message : e));
       return [];
     }
   }
 
-  public async $list1M(): Promise<IMempoolStats[]> {
+  public async $list1M(): Promise<OptimizedStatistic[]> {
     try {
-      const connection = await DB.pool.getConnection();
-      const query = this.getQueryForDays(120, 20160);
-      const [rows] = await connection.query<any>(query);
-      connection.release();
-      return rows;
+      const query = this.getQueryForDaysAvg(1800, '1 MONTH'); // 30m interval
+      const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
+      return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
     } catch (e) {
-      console.log('$list1M() error', e);
+      logger.err('$list1M() error' + (e instanceof Error ? e.message : e));
       return [];
     }
   }
 
-  public async $list3M(): Promise<IMempoolStats[]> {
+  public async $list3M(): Promise<OptimizedStatistic[]> {
     try {
-      const connection = await DB.pool.getConnection();
-      const query = this.getQueryForDays(120, 60480);
-      const [rows] = await connection.query<any>(query);
-      connection.release();
-      return rows;
+      const query = this.getQueryForDaysAvg(7200, '3 MONTH'); // 2h interval
+      const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
+      return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
     } catch (e) {
-      console.log('$list3M() error', e);
+      logger.err('$list3M() error' + (e instanceof Error ? e.message : e));
       return [];
     }
   }
 
-  public async $list6M(): Promise<IMempoolStats[]> {
+  public async $list6M(): Promise<OptimizedStatistic[]> {
     try {
-      const connection = await DB.pool.getConnection();
-      const query = this.getQueryForDays(120, 120960);
-      const [rows] = await connection.query<any>(query);
-      connection.release();
-      return rows;
+      const query = this.getQueryForDaysAvg(10800, '6 MONTH'); // 3h interval
+      const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
+      return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
     } catch (e) {
-      console.log('$list6M() error', e);
+      logger.err('$list6M() error' + (e instanceof Error ? e.message : e));
       return [];
     }
   }
 
+  public async $list1Y(): Promise<OptimizedStatistic[]> {
+    try {
+      const query = this.getQueryForDays(28800, '1 YEAR'); // 8h interval
+      const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
+      return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
+    } catch (e) {
+      logger.err('$list1Y() error' + (e instanceof Error ? e.message : e));
+      return [];
+    }
+  }
+
+  public async $list2Y(): Promise<OptimizedStatistic[]> {
+    try {
+      const query = this.getQueryForDays(28800, '2 YEAR'); // 8h interval
+      const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
+      return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
+    } catch (e) {
+      logger.err('$list2Y() error' + (e instanceof Error ? e.message : e));
+      return [];
+    }
+  }
+
+  public async $list3Y(): Promise<OptimizedStatistic[]> {
+    try {
+      const query = this.getQueryForDays(43200, '3 YEAR'); // 12h interval
+      const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
+      return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
+    } catch (e) {
+      logger.err('$list3Y() error' + (e instanceof Error ? e.message : e));
+      return [];
+    }
+  }
+
+  private mapStatisticToOptimizedStatistic(statistic: Statistic[]): OptimizedStatistic[] {
+    return statistic.map((s) => {
+      return {
+        added: s.added,
+        vbytes_per_second: s.vbytes_per_second,
+        mempool_byte_weight: s.mempool_byte_weight,
+        total_fee: s.total_fee,
+        vsizes: [
+          s.vsize_1,
+          s.vsize_2,
+          s.vsize_3,
+          s.vsize_4,
+          s.vsize_5,
+          s.vsize_6,
+          s.vsize_8,
+          s.vsize_10,
+          s.vsize_12,
+          s.vsize_15,
+          s.vsize_20,
+          s.vsize_30,
+          s.vsize_40,
+          s.vsize_50,
+          s.vsize_60,
+          s.vsize_70,
+          s.vsize_80,
+          s.vsize_90,
+          s.vsize_100,
+          s.vsize_125,
+          s.vsize_150,
+          s.vsize_175,
+          s.vsize_200,
+          s.vsize_250,
+          s.vsize_300,
+          s.vsize_350,
+          s.vsize_400,
+          s.vsize_500,
+          s.vsize_600,
+          s.vsize_700,
+          s.vsize_800,
+          s.vsize_900,
+          s.vsize_1000,
+          s.vsize_1200,
+          s.vsize_1400,
+          s.vsize_1600,
+          s.vsize_1800,
+          s.vsize_2000,
+        ]
+      };
+    });
+  }
+
 }
 
 export default new Statistics();

backend/src/api/transaction-utils.ts

@@ -0,0 +1,57 @@
+import bitcoinApi from './bitcoin/bitcoin-api-factory';
+import { TransactionExtended, TransactionMinerInfo } from '../mempool.interfaces';
+import { IEsploraApi } from './bitcoin/esplora-api.interface';
+import config from '../config';
+import { Common } from './common';
+
+class TransactionUtils {
+  constructor() { }
+
+  public stripCoinbaseTransaction(tx: TransactionExtended): TransactionMinerInfo {
+    return {
+      vin: [{
+        scriptsig: tx.vin[0].scriptsig || tx.vin[0]['coinbase']
+      }],
+      vout: tx.vout
+        .map((vout) => ({
+          scriptpubkey_address: vout.scriptpubkey_address,
+          value: vout.value
+        }))
+        .filter((vout) => vout.value)
+    };
+  }
+
+  public async $getTransactionExtended(txId: string, addPrevouts = false, lazyPrevouts = false): Promise<TransactionExtended> {
+    const transaction: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(txId, false, addPrevouts, lazyPrevouts);
+    return this.extendTransaction(transaction);
+  }
+
+  private extendTransaction(transaction: IEsploraApi.Transaction): TransactionExtended {
+    // @ts-ignore
+    if (transaction.vsize) {
+      // @ts-ignore
+      return transaction;
+    }
+    const feePerVbytes = Math.max(Common.isLiquid() ? 0.1 : 1,
+      (transaction.fee || 0) / (transaction.weight / 4));
+    const transactionExtended: TransactionExtended = Object.assign({
+      vsize: Math.round(transaction.weight / 4),
+      feePerVsize: feePerVbytes,
+      effectiveFeePerVsize: feePerVbytes,
+    }, transaction);
+    if (!transaction.status.confirmed) {
+      transactionExtended.firstSeen = Math.round((new Date().getTime() / 1000));
+    }
+    return transactionExtended;
+  }
+
+  public hex2ascii(hex: string) {
+    let str = '';
+    for (let i = 0; i < hex.length; i += 2) {
+      str += String.fromCharCode(parseInt(hex.substr(i, 2), 16));
+    }
+    return str;
+  }
+}
+
+export default new TransactionUtils();

backend/src/api/websocket-handler.ts

@@ -0,0 +1,540 @@
+import logger from '../logger';
+import * as WebSocket from 'ws';
+import { BlockExtended, TransactionExtended, WebsocketResponse, MempoolBlock, MempoolBlockDelta,
+  OptimizedStatistic, ILoadingIndicators, IConversionRates } from '../mempool.interfaces';
+import blocks from './blocks';
+import memPool from './mempool';
+import backendInfo from './backend-info';
+import mempoolBlocks from './mempool-blocks';
+import fiatConversion from './fiat-conversion';
+import { Common } from './common';
+import loadingIndicators from './loading-indicators';
+import config from '../config';
+import transactionUtils from './transaction-utils';
+import rbfCache from './rbf-cache';
+import difficultyAdjustment from './difficulty-adjustment';
+import feeApi from './fee-api';
+
+class WebsocketHandler {
+  private wss: WebSocket.Server | undefined;
+  private extraInitProperties = {};
+
+  constructor() { }
+
+  setWebsocketServer(wss: WebSocket.Server) {
+    this.wss = wss;
+  }
+
+  setExtraInitProperties(property: string, value: any) {
+    this.extraInitProperties[property] = value;
+  }
+
+  setupConnectionHandling() {
+    if (!this.wss) {
+      throw new Error('WebSocket.Server is not set');
+    }
+
+    this.wss.on('connection', (client: WebSocket) => {
+      client.on('error', logger.info);
+      client.on('message', async (message: string) => {
+        try {
+          const parsedMessage: WebsocketResponse = JSON.parse(message);
+          const response = {};
+
+          if (parsedMessage.action === 'want') {
+            client['want-blocks'] = parsedMessage.data.indexOf('blocks') > -1;
+            client['want-mempool-blocks'] = parsedMessage.data.indexOf('mempool-blocks') > -1;
+            client['want-live-2h-chart'] = parsedMessage.data.indexOf('live-2h-chart') > -1;
+            client['want-stats'] = parsedMessage.data.indexOf('stats') > -1;
+          }
+
+          if (parsedMessage && parsedMessage['track-tx']) {
+            if (/^[a-fA-F0-9]{64}$/.test(parsedMessage['track-tx'])) {
+              client['track-tx'] = parsedMessage['track-tx'];
+              // Client is telling the transaction wasn't found
+              if (parsedMessage['watch-mempool']) {
+                const rbfCacheTx = rbfCache.get(client['track-tx']);
+                if (rbfCacheTx) {
+                  response['txReplaced'] = {
+                    txid: rbfCacheTx.txid,
+                  };
+                  client['track-tx'] = null;
+                } else {
+                  // It might have appeared before we had the time to start watching for it
+                  const tx = memPool.getMempool()[client['track-tx']];
+                  if (tx) {
+                    if (config.MEMPOOL.BACKEND === 'esplora') {
+                      response['tx'] = tx;
+                    } else {
+                      // tx.prevout is missing from transactions when in bitcoind mode
+                      try {
+                        const fullTx = await transactionUtils.$getTransactionExtended(tx.txid, true);
+                        response['tx'] = fullTx;
+                      } catch (e) {
+                        logger.debug('Error finding transaction: ' + (e instanceof Error ? e.message : e));
+                      }
+                    }
+                  } else {
+                    try {
+                      const fullTx = await transactionUtils.$getTransactionExtended(client['track-tx'], true);
+                      response['tx'] = fullTx;
+                    } catch (e) {
+                      logger.debug('Error finding transaction. ' + (e instanceof Error ? e.message : e));
+                      client['track-mempool-tx'] = parsedMessage['track-tx'];
+                    }
+                  }
+                }
+              }
+            } else {
+              client['track-tx'] = null;
+            }
+          }
+
+          if (parsedMessage && parsedMessage['track-address']) {
+            if (/^([a-km-zA-HJ-NP-Z1-9]{26,35}|[a-km-zA-HJ-NP-Z1-9]{80}|[a-z]{2,5}1[ac-hj-np-z02-9]{8,100}|[A-Z]{2,5}1[AC-HJ-NP-Z02-9]{8,100})$/
+              .test(parsedMessage['track-address'])) {
+              let matchedAddress = parsedMessage['track-address'];
+              if (/^[A-Z]{2,5}1[AC-HJ-NP-Z02-9]{8,100}$/.test(parsedMessage['track-address'])) {
+                matchedAddress = matchedAddress.toLowerCase();
+              }
+              client['track-address'] = matchedAddress;
+            } else {
+              client['track-address'] = null;
+            }
+          }
+
+          if (parsedMessage && parsedMessage['track-asset']) {
+            if (/^[a-fA-F0-9]{64}$/.test(parsedMessage['track-asset'])) {
+              client['track-asset'] = parsedMessage['track-asset'];
+            } else {
+              client['track-asset'] = null;
+            }
+          }
+
+          if (parsedMessage && parsedMessage['track-mempool-block'] !== undefined) {
+            if (Number.isInteger(parsedMessage['track-mempool-block']) && parsedMessage['track-mempool-block'] >= 0) {
+              const index = parsedMessage['track-mempool-block'];
+              client['track-mempool-block'] = index;
+              const mBlocksWithTransactions = mempoolBlocks.getMempoolBlocksWithTransactions();
+              response['projected-block-transactions'] = {
+                index: index,
+                blockTransactions: mBlocksWithTransactions[index]?.transactions || [],
+              };
+            } else {
+              client['track-mempool-block'] = null;
+            }
+          }
+
+          if (parsedMessage.action === 'init') {
+            const _blocks = blocks.getBlocks().slice(-config.MEMPOOL.INITIAL_BLOCKS_AMOUNT);
+            if (!_blocks) {
+              return;
+            }
+            client.send(JSON.stringify(this.getInitData(_blocks)));
+          }
+
+          if (parsedMessage.action === 'ping') {
+            response['pong'] = true;
+          }
+
+          if (parsedMessage['track-donation'] && parsedMessage['track-donation'].length === 22) {
+            client['track-donation'] = parsedMessage['track-donation'];
+          }
+
+          if (parsedMessage['track-bisq-market']) {
+            if (/^[a-z]{3}_[a-z]{3}$/.test(parsedMessage['track-bisq-market'])) {
+              client['track-bisq-market'] = parsedMessage['track-bisq-market'];
+            } else {
+              client['track-bisq-market'] = null;
+            }
+          }
+
+          if (Object.keys(response).length) {
+            client.send(JSON.stringify(response));
+          }
+        } catch (e) {
+          logger.debug('Error parsing websocket message: ' + (e instanceof Error ? e.message : e));
+        }
+      });
+    });
+  }
+
+  handleNewDonation(id: string) {
+    if (!this.wss) {
+      throw new Error('WebSocket.Server is not set');
+    }
+
+    this.wss.clients.forEach((client: WebSocket) => {
+      if (client.readyState !== WebSocket.OPEN) {
+        return;
+      }
+      if (client['track-donation'] === id) {
+        client.send(JSON.stringify({ donationConfirmed: true }));
+      }
+    });
+  }
+
+  handleLoadingChanged(indicators: ILoadingIndicators) {
+    if (!this.wss) {
+      throw new Error('WebSocket.Server is not set');
+    }
+
+    this.wss.clients.forEach((client: WebSocket) => {
+      if (client.readyState !== WebSocket.OPEN) {
+        return;
+      }
+      client.send(JSON.stringify({ loadingIndicators: indicators }));
+    });
+  }
+
+  handleNewConversionRates(conversionRates: IConversionRates) {
+    if (!this.wss) {
+      throw new Error('WebSocket.Server is not set');
+    }
+
+    this.wss.clients.forEach((client: WebSocket) => {
+      if (client.readyState !== WebSocket.OPEN) {
+        return;
+      }
+      client.send(JSON.stringify({ conversions: conversionRates }));
+    });
+  }
+
+  getInitData(_blocks?: BlockExtended[]) {
+    if (!_blocks) {
+      _blocks = blocks.getBlocks().slice(-config.MEMPOOL.INITIAL_BLOCKS_AMOUNT);
+    }
+    return {
+      'mempoolInfo': memPool.getMempoolInfo(),
+      'vBytesPerSecond': memPool.getVBytesPerSecond(),
+      'blocks': _blocks,
+      'conversions': fiatConversion.getConversionRates(),
+      'mempool-blocks': mempoolBlocks.getMempoolBlocks(),
+      'transactions': memPool.getLatestTransactions(),
+      'backendInfo': backendInfo.getBackendInfo(),
+      'loadingIndicators': loadingIndicators.getLoadingIndicators(),
+      'da': difficultyAdjustment.getDifficultyAdjustment(),
+      'fees': feeApi.getRecommendedFee(),
+      ...this.extraInitProperties
+    };
+  }
+
+  handleNewStatistic(stats: OptimizedStatistic) {
+    if (!this.wss) {
+      throw new Error('WebSocket.Server is not set');
+    }
+
+    this.wss.clients.forEach((client: WebSocket) => {
+      if (client.readyState !== WebSocket.OPEN) {
+        return;
+      }
+
+      if (!client['want-live-2h-chart']) {
+        return;
+      }
+
+      client.send(JSON.stringify({
+        'live-2h-chart': stats
+      }));
+    });
+  }
+
+  handleMempoolChange(newMempool: { [txid: string]: TransactionExtended },
+    newTransactions: TransactionExtended[], deletedTransactions: TransactionExtended[]) {
+    if (!this.wss) {
+      throw new Error('WebSocket.Server is not set');
+    }
+
+    mempoolBlocks.updateMempoolBlocks(newMempool);
+    const mBlocks = mempoolBlocks.getMempoolBlocks();
+    const mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();
+    const mempoolInfo = memPool.getMempoolInfo();
+    const vBytesPerSecond = memPool.getVBytesPerSecond();
+    const rbfTransactions = Common.findRbfTransactions(newTransactions, deletedTransactions);
+    const da = difficultyAdjustment.getDifficultyAdjustment();
+    memPool.handleRbfTransactions(rbfTransactions);
+    const recommendedFees = feeApi.getRecommendedFee();
+
+    this.wss.clients.forEach(async (client: WebSocket) => {
+      if (client.readyState !== WebSocket.OPEN) {
+        return;
+      }
+
+      const response = {};
+
+      if (client['want-stats']) {
+        response['mempoolInfo'] = mempoolInfo;
+        response['vBytesPerSecond'] = vBytesPerSecond;
+        response['transactions'] = newTransactions.slice(0, 6).map((tx) => Common.stripTransaction(tx));
+        response['da'] = da;
+        response['fees'] = recommendedFees;
+      }
+
+      if (client['want-mempool-blocks']) {
+        response['mempool-blocks'] = mBlocks;
+      }
+
+      if (client['track-mempool-tx']) {
+        const tx = newTransactions.find((t) => t.txid === client['track-mempool-tx']);
+        if (tx) {
+          if (config.MEMPOOL.BACKEND !== 'esplora') {
+            try {
+              const fullTx = await transactionUtils.$getTransactionExtended(tx.txid, true);
+              response['tx'] = fullTx;
+            } catch (e) {
+              logger.debug('Error finding transaction in mempool: ' + (e instanceof Error ? e.message : e));
+            }
+          } else {
+            response['tx'] = tx;
+          }
+          client['track-mempool-tx'] = null;
+        }
+      }
+
+      if (client['track-address']) {
+        const foundTransactions: TransactionExtended[] = [];
+
+        for (const tx of newTransactions) {
+          const someVin = tx.vin.some((vin) => !!vin.prevout && vin.prevout.scriptpubkey_address === client['track-address']);
+          if (someVin) {
+            if (config.MEMPOOL.BACKEND !== 'esplora') {
+              try {
+                const fullTx = await transactionUtils.$getTransactionExtended(tx.txid, true);
+                foundTransactions.push(fullTx);
+              } catch (e) {
+                logger.debug('Error finding transaction in mempool: ' + (e instanceof Error ? e.message : e));
+              }
+            } else {
+              foundTransactions.push(tx);
+            }
+            return;
+          }
+          const someVout = tx.vout.some((vout) => vout.scriptpubkey_address === client['track-address']);
+          if (someVout) {
+            if (config.MEMPOOL.BACKEND !== 'esplora') {
+              try {
+                const fullTx = await transactionUtils.$getTransactionExtended(tx.txid, true);
+                foundTransactions.push(fullTx);
+              } catch (e) {
+                logger.debug('Error finding transaction in mempool: ' + (e instanceof Error ? e.message : e));
+              }
+            } else {
+              foundTransactions.push(tx);
+            }
+          }
+        }
+
+        if (foundTransactions.length) {
+          response['address-transactions'] = foundTransactions;
+        }
+      }
+
+      if (client['track-asset']) {
+        const foundTransactions: TransactionExtended[] = [];
+
+        newTransactions.forEach((tx) => {
+
+          if (client['track-asset'] === Common.nativeAssetId) {
+            if (tx.vin.some((vin) => !!vin.is_pegin)) {
+              foundTransactions.push(tx);
+              return;
+            }
+            if (tx.vout.some((vout) => !!vout.pegout)) {
+              foundTransactions.push(tx);
+            }
+          } else {
+            if (tx.vin.some((vin) => !!vin.issuance && vin.issuance.asset_id === client['track-asset'])) {
+              foundTransactions.push(tx);
+              return;
+            }
+            if (tx.vout.some((vout) => !!vout.asset && vout.asset === client['track-asset'])) {
+              foundTransactions.push(tx);
+            }
+          }
+        });
+
+        if (foundTransactions.length) {
+          response['address-transactions'] = foundTransactions;
+        }
+      }
+
+      if (client['track-tx']) {
+        const outspends: object = {};
+        newTransactions.forEach((tx) => tx.vin.forEach((vin, i) => {
+          if (vin.txid === client['track-tx']) {
+            outspends[vin.vout] = {
+              vin: i,
+              txid: tx.txid,
+            };
+          }
+        }));
+
+        if (Object.keys(outspends).length) {
+          response['utxoSpent'] = outspends;
+        }
+
+        if (rbfTransactions[client['track-tx']]) {
+          for (const rbfTransaction in rbfTransactions) {
+            if (client['track-tx'] === rbfTransaction) {
+              response['rbfTransaction'] = {
+                txid: rbfTransactions[rbfTransaction].txid,
+              };
+              break;
+            }
+          }
+        }
+      }
+
+      if (client['track-mempool-block'] >= 0) {
+        const index = client['track-mempool-block'];
+        if (mBlockDeltas[index]) {
+          response['projected-block-transactions'] = {
+            index: index,
+            delta: mBlockDeltas[index],
+          };
+        }
+      }
+
+      if (Object.keys(response).length) {
+        client.send(JSON.stringify(response));
+      }
+    });
+  }
+
+  handleNewBlock(block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) {
+    if (!this.wss) {
+      throw new Error('WebSocket.Server is not set');
+    }
+
+    let mBlocks: undefined | MempoolBlock[];
+    let mBlockDeltas: undefined | MempoolBlockDelta[];
+    let matchRate = 0;
+    const _memPool = memPool.getMempool();
+    const _mempoolBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
+
+    if (_mempoolBlocks[0]) {
+      const matches: string[] = [];
+      for (const txId of txIds) {
+        if (_mempoolBlocks[0].transactionIds.indexOf(txId) > -1) {
+          matches.push(txId);
+        }
+        delete _memPool[txId];
+      }
+
+      matchRate = Math.round((matches.length / (txIds.length - 1)) * 100);
+      mempoolBlocks.updateMempoolBlocks(_memPool);
+      mBlocks = mempoolBlocks.getMempoolBlocks();
+      mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();
+    }
+
+    if (block.extras) {
+      block.extras.matchRate = matchRate;
+    }
+
+    const da = difficultyAdjustment.getDifficultyAdjustment();
+    const fees = feeApi.getRecommendedFee();
+
+    this.wss.clients.forEach((client) => {
+      if (client.readyState !== WebSocket.OPEN) {
+        return;
+      }
+
+      if (!client['want-blocks']) {
+        return;
+      }
+
+      const response = {
+        'block': block,
+        'mempoolInfo': memPool.getMempoolInfo(),
+        'da': da,
+        'fees': fees,
+      };
+
+      if (mBlocks && client['want-mempool-blocks']) {
+        response['mempool-blocks'] = mBlocks;
+      }
+
+      if (client['track-tx'] && txIds.indexOf(client['track-tx']) > -1) {
+        response['txConfirmed'] = true;
+      }
+
+      if (client['track-address']) {
+        const foundTransactions: TransactionExtended[] = [];
+
+        transactions.forEach((tx) => {
+          if (tx.vin && tx.vin.some((vin) => !!vin.prevout && vin.prevout.scriptpubkey_address === client['track-address'])) {
+            foundTransactions.push(tx);
+            return;
+          }
+          if (tx.vout && tx.vout.some((vout) => vout.scriptpubkey_address === client['track-address'])) {
+            foundTransactions.push(tx);
+          }
+        });
+
+        if (foundTransactions.length) {
+          foundTransactions.forEach((tx) => {
+            tx.status = {
+              confirmed: true,
+              block_height: block.height,
+              block_hash: block.id,
+              block_time: block.timestamp,
+            };
+          });
+
+          response['block-transactions'] = foundTransactions;
+        }
+      }
+
+      if (client['track-asset']) {
+        const foundTransactions: TransactionExtended[] = [];
+
+        transactions.forEach((tx) => {
+          if (client['track-asset'] === Common.nativeAssetId) {
+            if (tx.vin && tx.vin.some((vin) => !!vin.is_pegin)) {
+              foundTransactions.push(tx);
+              return;
+            }
+            if (tx.vout && tx.vout.some((vout) => !!vout.pegout)) {
+              foundTransactions.push(tx);
+            }
+          } else {
+            if (tx.vin && tx.vin.some((vin) => !!vin.issuance && vin.issuance.asset_id === client['track-asset'])) {
+              foundTransactions.push(tx);
+              return;
+            }
+            if (tx.vout && tx.vout.some((vout) => !!vout.asset && vout.asset === client['track-asset'])) {
+              foundTransactions.push(tx);
+            }
+          }
+        });
+
+        if (foundTransactions.length) {
+          foundTransactions.forEach((tx) => {
+            tx.status = {
+              confirmed: true,
+              block_height: block.height,
+              block_hash: block.id,
+              block_time: block.timestamp,
+            };
+          });
+
+          response['block-transactions'] = foundTransactions;
+        }
+      }
+
+      if (client['track-mempool-block'] >= 0) {
+        const index = client['track-mempool-block'];
+        if (mBlockDeltas && mBlockDeltas[index]) {
+          response['projected-block-transactions'] = {
+            index: index,
+            delta: mBlockDeltas[index],
+          };
+        }
+      }
+
+      client.send(JSON.stringify(response));
+    });
+  }
+}
+
+export default new WebsocketHandler();

backend/src/config.ts

@@ -0,0 +1,222 @@
+const configFile = require('../mempool-config.json');
+
+interface IConfig {
+  MEMPOOL: {
+    NETWORK: 'mainnet' | 'testnet' | 'signet' | 'liquid' | 'liquidtestnet';
+    BACKEND: 'esplora' | 'electrum' | 'none';
+    HTTP_PORT: number;
+    SPAWN_CLUSTER_PROCS: number;
+    API_URL_PREFIX: string;
+    POLL_RATE_MS: number;
+    CACHE_DIR: string;
+    CLEAR_PROTECTION_MINUTES: number;
+    RECOMMENDED_FEE_PERCENTILE: number;
+    BLOCK_WEIGHT_UNITS: number;
+    INITIAL_BLOCKS_AMOUNT: number;
+    MEMPOOL_BLOCKS_AMOUNT: number;
+    INDEXING_BLOCKS_AMOUNT: number;
+    PRICE_FEED_UPDATE_INTERVAL: number;
+    USE_SECOND_NODE_FOR_MINFEE: boolean;
+    EXTERNAL_ASSETS: string[];
+    EXTERNAL_MAX_RETRY: number;
+    EXTERNAL_RETRY_INTERVAL: number;
+    USER_AGENT: string;
+    STDOUT_LOG_MIN_PRIORITY: 'emerg' | 'alert' | 'crit' | 'err' | 'warn' | 'notice' | 'info' | 'debug';
+  };
+  ESPLORA: {
+    REST_API_URL: string;
+  };
+  ELECTRUM: {
+    HOST: string;
+    PORT: number;
+    TLS_ENABLED: boolean;
+  };
+  CORE_RPC: {
+    HOST: string;
+    PORT: number;
+    USERNAME: string;
+    PASSWORD: string;
+  };
+  SECOND_CORE_RPC: {
+    HOST: string;
+    PORT: number;
+    USERNAME: string;
+    PASSWORD: string;
+  };
+  DATABASE: {
+    ENABLED: boolean;
+    HOST: string,
+    SOCKET: string,
+    PORT: number;
+    DATABASE: string;
+    USERNAME: string;
+    PASSWORD: string;
+  };
+  SYSLOG: {
+    ENABLED: boolean;
+    HOST: string;
+    PORT: number;
+    MIN_PRIORITY: 'emerg' | 'alert' | 'crit' | 'err' | 'warn' | 'notice' | 'info' | 'debug';
+    FACILITY: string;
+  };
+  STATISTICS: {
+    ENABLED: boolean;
+    TX_PER_SECOND_SAMPLE_PERIOD: number;
+  };
+  BISQ: {
+    ENABLED: boolean;
+    DATA_PATH: string;
+  };
+  SOCKS5PROXY: {
+    ENABLED: boolean;
+    USE_ONION: boolean;
+    HOST: string;
+    PORT: number;
+    USERNAME: string;
+    PASSWORD: string;
+  };
+  PRICE_DATA_SERVER: {
+    TOR_URL: string;
+    CLEARNET_URL: string;
+  };
+  EXTERNAL_DATA_SERVER: {
+    MEMPOOL_API: string;
+    MEMPOOL_ONION: string;
+    LIQUID_API: string;
+    LIQUID_ONION: string;
+    BISQ_URL: string;
+    BISQ_ONION: string;
+  };
+}
+
+const defaults: IConfig = {
+  'MEMPOOL': {
+    'NETWORK': 'mainnet',
+    'BACKEND': 'none',
+    'HTTP_PORT': 8999,
+    'SPAWN_CLUSTER_PROCS': 0,
+    'API_URL_PREFIX': '/api/v1/',
+    'POLL_RATE_MS': 2000,
+    'CACHE_DIR': './cache',
+    'CLEAR_PROTECTION_MINUTES': 20,
+    'RECOMMENDED_FEE_PERCENTILE': 50,
+    'BLOCK_WEIGHT_UNITS': 4000000,
+    'INITIAL_BLOCKS_AMOUNT': 8,
+    'MEMPOOL_BLOCKS_AMOUNT': 8,
+    'INDEXING_BLOCKS_AMOUNT': 11000, // 0 = disable indexing, -1 = index all blocks
+    'PRICE_FEED_UPDATE_INTERVAL': 600,
+    'USE_SECOND_NODE_FOR_MINFEE': false,
+    'EXTERNAL_ASSETS': [],
+    'EXTERNAL_MAX_RETRY': 1,
+    'EXTERNAL_RETRY_INTERVAL': 0,
+    'USER_AGENT': 'mempool',
+    'STDOUT_LOG_MIN_PRIORITY': 'debug',
+  },
+  'ESPLORA': {
+    'REST_API_URL': 'http://127.0.0.1:3000',
+  },
+  'ELECTRUM': {
+    'HOST': '127.0.0.1',
+    'PORT': 3306,
+    'TLS_ENABLED': true,
+  },
+  'CORE_RPC': {
+    'HOST': '127.0.0.1',
+    'PORT': 8332,
+    'USERNAME': 'mempool',
+    'PASSWORD': 'mempool'
+  },
+  'SECOND_CORE_RPC': {
+    'HOST': '127.0.0.1',
+    'PORT': 8332,
+    'USERNAME': 'mempool',
+    'PASSWORD': 'mempool'
+  },
+  'DATABASE': {
+    'ENABLED': true,
+    'HOST': '127.0.0.1',
+    'SOCKET': '',
+    'PORT': 3306,
+    'DATABASE': 'mempool',
+    'USERNAME': 'mempool',
+    'PASSWORD': 'mempool'
+  },
+  'SYSLOG': {
+    'ENABLED': true,
+    'HOST': '127.0.0.1',
+    'PORT': 514,
+    'MIN_PRIORITY': 'info',
+    'FACILITY': 'local7'
+  },
+  'STATISTICS': {
+    'ENABLED': true,
+    'TX_PER_SECOND_SAMPLE_PERIOD': 150
+  },
+  'BISQ': {
+    'ENABLED': false,
+    'DATA_PATH': '/bisq/statsnode-data/btc_mainnet/db'
+  },
+  'SOCKS5PROXY': {
+    'ENABLED': false,
+    'USE_ONION': true,
+    'HOST': '127.0.0.1',
+    'PORT': 9050,
+    'USERNAME': '',
+    'PASSWORD': ''
+  },
+  "PRICE_DATA_SERVER": {
+    'TOR_URL': 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices',
+    'CLEARNET_URL': 'https://price.bisq.wiz.biz/getAllMarketPrices'
+  },
+  "EXTERNAL_DATA_SERVER": {
+    'MEMPOOL_API': 'https://mempool.space/api/v1',
+    'MEMPOOL_ONION': 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1',
+    'LIQUID_API': 'https://liquid.network/api/v1',
+    'LIQUID_ONION': 'http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1',
+    'BISQ_URL': 'https://bisq.markets/api',
+    'BISQ_ONION': 'http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api'
+  }
+};
+
+class Config implements IConfig {
+  MEMPOOL: IConfig['MEMPOOL'];
+  ESPLORA: IConfig['ESPLORA'];
+  ELECTRUM: IConfig['ELECTRUM'];
+  CORE_RPC: IConfig['CORE_RPC'];
+  SECOND_CORE_RPC: IConfig['SECOND_CORE_RPC'];
+  DATABASE: IConfig['DATABASE'];
+  SYSLOG: IConfig['SYSLOG'];
+  STATISTICS: IConfig['STATISTICS'];
+  BISQ: IConfig['BISQ'];
+  SOCKS5PROXY: IConfig['SOCKS5PROXY'];
+  PRICE_DATA_SERVER: IConfig['PRICE_DATA_SERVER'];
+  EXTERNAL_DATA_SERVER: IConfig['EXTERNAL_DATA_SERVER'];
+
+  constructor() {
+    const configs = this.merge(configFile, defaults);
+    this.MEMPOOL = configs.MEMPOOL;
+    this.ESPLORA = configs.ESPLORA;
+    this.ELECTRUM = configs.ELECTRUM;
+    this.CORE_RPC = configs.CORE_RPC;
+    this.SECOND_CORE_RPC = configs.SECOND_CORE_RPC;
+    this.DATABASE = configs.DATABASE;
+    this.SYSLOG = configs.SYSLOG;
+    this.STATISTICS = configs.STATISTICS;
+    this.BISQ = configs.BISQ;
+    this.SOCKS5PROXY = configs.SOCKS5PROXY;
+    this.PRICE_DATA_SERVER = configs.PRICE_DATA_SERVER;
+    this.EXTERNAL_DATA_SERVER = configs.EXTERNAL_DATA_SERVER;
+  }
+
+  merge = (...objects: object[]): IConfig => {
+    // @ts-ignore
+    return objects.reduce((prev, next) => {
+      Object.keys(prev).forEach(key => {
+        next[key] = { ...next[key], ...prev[key] };
+      });
+      return next;
+    });
+  }
+}
+
+export default new Config();

backend/src/database.ts

@@ -1,26 +1,59 @@
-const config = require('../mempool-config.json');
-import { createPool } from 'mysql2/promise';
+import config from './config';
+import { createPool, Pool, PoolConnection } from 'mysql2/promise';
+import logger from './logger';
+import { PoolOptions } from 'mysql2/typings/mysql';
 
-export class DB {
-  static pool = createPool({
-    host: config.DB_HOST,
-    port: config.DB_PORT,
-    database: config.DB_DATABASE,
-    user: config.DB_USER,
-    password: config.DB_PASSWORD,
+ class DB {
+  constructor() {
+    if (config.DATABASE.SOCKET !== '') {
+      this.poolConfig.socketPath = config.DATABASE.SOCKET;
+    } else {
+      this.poolConfig.host = config.DATABASE.HOST;
+    }
+  }
+  private pool: Pool | null = null;
+  private poolConfig: PoolOptions = {
+    port: config.DATABASE.PORT,
+    database: config.DATABASE.DATABASE,
+    user: config.DATABASE.USERNAME,
+    password: config.DATABASE.PASSWORD,
     connectionLimit: 10,
     supportBigNumbers: true,
-  });
-}
+    timezone: '+00:00',
+  };
 
-export async function checkDbConnection() {
-  try {
-    const connection = await DB.pool.getConnection();
-    console.log('MySQL connection established.');
-    connection.release();
-  } catch (e) {
-    console.log('Could not connect to MySQL.');
-    console.log(e);
-    process.exit(1);
+  private checkDBFlag() {
+    if (config.DATABASE.ENABLED === false) {
+      logger.err('Trying to use DB feature but config.DATABASE.ENABLED is set to false, please open an issue');
+    }
+  }
+
+  public async query(query, params?) {
+    this.checkDBFlag();
+    const pool = await this.getPool();
+    return pool.query(query, params);
+  }
+
+  public async checkDbConnection() {
+    this.checkDBFlag();
+    try {
+      await this.query('SELECT ?', [1]);
+      logger.info('Database connection established.');
+    } catch (e) {
+      logger.err('Could not connect to database: ' + (e instanceof Error ? e.message : e));
+      process.exit(1);
+    }
+  }
+
+  private async getPool(): Promise<Pool> {
+    if (this.pool === null) {
+      this.pool = createPool(this.poolConfig);
+      this.pool.on('connection', function (newConnection: PoolConnection) {
+        newConnection.query(`SET time_zone='+00:00'`);
+      });
+    }
+    return this.pool;
   }
 }
+
+export default new DB();

backend/src/index.ts

@@ -1,284 +1,359 @@
-const config = require('../mempool-config.json');
-import * as fs from 'fs';
+import { Express, Request, Response, NextFunction } from 'express';
 import * as express from 'express';
-import * as compression from 'compression';
 import * as http from 'http';
-import * as https from 'https';
 import * as WebSocket from 'ws';
+import * as cluster from 'cluster';
+import axios from 'axios';
 
-import bitcoinApi from './api/bitcoin/bitcoin-api-factory';
-import diskCache from './api/disk-cache';
-import memPool from './api/mempool';
-import blocks from './api/blocks';
-import projectedBlocks from './api/projected-blocks';
-import statistics from './api/statistics';
-import { IBlock, IMempool, ITransaction, IMempoolStats } from './interfaces';
-
+import DB from './database';
+import config from './config';
 import routes from './routes';
+import blocks from './api/blocks';
+import memPool from './api/mempool';
+import diskCache from './api/disk-cache';
+import statistics from './api/statistics';
+import websocketHandler from './api/websocket-handler';
 import fiatConversion from './api/fiat-conversion';
+import bisq from './api/bisq/bisq';
+import bisqMarkets from './api/bisq/markets';
+import logger from './logger';
+import backendInfo from './api/backend-info';
+import loadingIndicators from './api/loading-indicators';
+import mempool from './api/mempool';
+import elementsParser from './api/liquid/elements-parser';
+import databaseMigration from './api/database-migration';
+import syncAssets from './sync-assets';
+import icons from './api/liquid/icons';
+import { Common } from './api/common';
+import poolsUpdater from './tasks/pools-updater';
+import indexer from './indexer';
 
-class MempoolSpace {
-  private wss: WebSocket.Server;
-  private server: https.Server | http.Server;
-  private app: any;
+class Server {
+  private wss: WebSocket.Server | undefined;
+  private server: http.Server | undefined;
+  private app: Express;
+  private currentBackendRetryInterval = 5;
 
   constructor() {
     this.app = express();
+
+    if (!config.MEMPOOL.SPAWN_CLUSTER_PROCS) {
+      this.startServer();
+      return;
+    }
+
+    if (cluster.isMaster) {
+      logger.notice(`Mempool Server (Master) is running on port ${config.MEMPOOL.HTTP_PORT} (${backendInfo.getShortCommitHash()})`);
+
+      const numCPUs = config.MEMPOOL.SPAWN_CLUSTER_PROCS;
+      for (let i = 0; i < numCPUs; i++) {
+        const env = { workerId: i };
+        const worker = cluster.fork(env);
+        worker.process['env'] = env;
+      }
+
+      cluster.on('exit', (worker, code, signal) => {
+        const workerId = worker.process['env'].workerId;
+        logger.warn(`Mempool Worker PID #${worker.process.pid} workerId: ${workerId} died. Restarting in 10 seconds... ${signal || code}`);
+        setTimeout(() => {
+          const env = { workerId: workerId };
+          const newWorker = cluster.fork(env);
+          newWorker.process['env'] = env;
+        }, 10000);
+      });
+    } else {
+      this.startServer(true);
+    }
+  }
+
+  async startServer(worker = false) {
+    logger.notice(`Starting Mempool Server${worker ? ' (worker)' : ''}... (${backendInfo.getShortCommitHash()})`);
+
     this.app
-      .use((req, res, next)  => {
+      .use((req: Request, res: Response, next: NextFunction) => {
         res.setHeader('Access-Control-Allow-Origin', '*');
         next();
       })
-      .use(compression());
-    if (config.ENV === 'dev') {
-      this.server = http.createServer(this.app);
-      this.wss = new WebSocket.Server({ server: this.server });
-    } else {
-      const credentials = {
-        cert: fs.readFileSync('/etc/letsencrypt/live/mempool.space/fullchain.pem'),
-        key: fs.readFileSync('/etc/letsencrypt/live/mempool.space/privkey.pem'),
-      };
-      this.server = https.createServer(credentials, this.app);
-      this.wss = new WebSocket.Server({ server: this.server });
+      .use(express.urlencoded({ extended: true }))
+      .use(express.text())
+    ;
+
+    this.server = http.createServer(this.app);
+    this.wss = new WebSocket.Server({ server: this.server });
+
+    this.setUpWebsocketHandling();
+
+    await syncAssets.syncAssets$();
+    diskCache.loadMempoolCache();
+
+    if (config.DATABASE.ENABLED) {
+      await DB.checkDbConnection();
+      try {
+        if (process.env.npm_config_reindex !== undefined) { // Re-index requests
+          const tables = process.env.npm_config_reindex.split(',');
+          logger.warn(`Indexed data for "${process.env.npm_config_reindex}" tables will be erased in 5 seconds (using '--reindex')`);
+          await Common.sleep$(5000);
+          await databaseMigration.$truncateIndexedData(tables);
+        }
+        await databaseMigration.$initializeOrMigrateDatabase();
+        if (Common.indexingEnabled()) {
+          await indexer.$resetHashratesIndexingState();
+        }
+      } catch (e) {
+        throw new Error(e instanceof Error ? e.message : 'Error');
+      }
     }
 
-    this.setUpRoutes();
-    this.setUpWebsocketHandling();
-    this.setUpMempoolCache();
-    this.runMempoolIntervalFunctions();
+    if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED && cluster.isMaster) {
+      statistics.startStatistics();
+    }
+
+    if (Common.isLiquid()) {
+      try {
+        icons.loadIcons();
+      } catch (e) {
+        logger.err('Cannot load liquid icons. Ignoring. Reason: ' + (e instanceof Error ? e.message : e));
+      }
+    }
 
-    statistics.startStatistics();
     fiatConversion.startService();
 
-    const opts = {
-        host: '127.0.0.1',
-        port: 8999
-    };
-    this.server.listen(opts, () => {
-      console.log(`Server started on ${opts.host}:${opts.port}`);
-    });
-  }
+    this.setUpHttpApiRoutes();
+    this.runMainUpdateLoop();
 
-  private async runMempoolIntervalFunctions() {
-    await blocks.updateBlocks();
-    await memPool.updateMemPoolInfo();
-    await memPool.updateMempool();
-    setTimeout(this.runMempoolIntervalFunctions.bind(this), config.MEMPOOL_REFRESH_RATE_MS);
-  }
-
-  private setUpMempoolCache() {
-    const cacheData = diskCache.loadData();
-    if (cacheData) {
-      memPool.setMempool(JSON.parse(cacheData));
+    if (config.BISQ.ENABLED) {
+      bisq.startBisqService();
+      bisq.setPriceCallbackFunction((price) => websocketHandler.setExtraInitProperties('bsq-price', price));
+      blocks.setNewBlockCallback(bisq.handleNewBitcoinBlock.bind(bisq));
+      bisqMarkets.startBisqService();
     }
 
-    process.on('SIGINT', (options) => {
-      console.log('SIGINT');
-      diskCache.saveData(JSON.stringify(memPool.getMempool()));
-      process.exit(2);
+    this.server.listen(config.MEMPOOL.HTTP_PORT, () => {
+      if (worker) {
+        logger.info(`Mempool Server worker #${process.pid} started`);
+      } else {
+        logger.notice(`Mempool Server is running on port ${config.MEMPOOL.HTTP_PORT}`);
+      }
     });
   }
 
-  private setUpWebsocketHandling() {
-    this.wss.on('connection', (client: WebSocket) => {
-      let theBlocks = blocks.getBlocks();
-      theBlocks = theBlocks.concat([]).splice(theBlocks.length - config.INITIAL_BLOCK_AMOUNT);
-      const formatedBlocks = theBlocks.map((b) => blocks.formatBlock(b));
+  async runMainUpdateLoop() {
+    try {
+      try {
+        await memPool.$updateMemPoolInfo();
+      } catch (e) {
+        const msg = `updateMempoolInfo: ${(e instanceof Error ? e.message : e)}`;
+        if (config.MEMPOOL.USE_SECOND_NODE_FOR_MINFEE) {
+          logger.warn(msg);
+        } else {
+          logger.debug(msg);
+        }
+      }
+      await poolsUpdater.updatePoolsJson();
+      await blocks.$updateBlocks();
+      await memPool.$updateMempool();
+      indexer.$run();
 
-      client.send(JSON.stringify({
-        'mempoolInfo': memPool.getMempoolInfo(),
-        'blocks': formatedBlocks,
-        'projectedBlocks': projectedBlocks.getProjectedBlocks(),
-        'txPerSecond': memPool.getTxPerSecond(),
-        'vBytesPerSecond': memPool.getVBytesPerSecond(),
-        'conversions': fiatConversion.getTickers()['BTCUSD'],
-      }));
+      setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS);
+      this.currentBackendRetryInterval = 5;
+    } catch (e) {
+      const loggerMsg = `runMainLoop error: ${(e instanceof Error ? e.message : e)}. Retrying in ${this.currentBackendRetryInterval} sec.`;
+      if (this.currentBackendRetryInterval > 5) {
+        logger.warn(loggerMsg);
+        mempool.setOutOfSync();
+      } else {
+        logger.debug(loggerMsg);
+      }
+      logger.debug(JSON.stringify(e));
+      setTimeout(this.runMainUpdateLoop.bind(this), 1000 * this.currentBackendRetryInterval);
+      this.currentBackendRetryInterval *= 2;
+      this.currentBackendRetryInterval = Math.min(this.currentBackendRetryInterval, 60);
+    }
+  }
 
-      client.on('message', async (message: any) => {
+  setUpWebsocketHandling() {
+    if (this.wss) {
+      websocketHandler.setWebsocketServer(this.wss);
+    }
+    if (Common.isLiquid() && config.DATABASE.ENABLED) {
+      blocks.setNewBlockCallback(async () => {
         try {
-          const parsedMessage = JSON.parse(message);
-
-          if (parsedMessage.action === 'want') {
-            client['want-stats'] = parsedMessage.data.indexOf('stats') > -1;
-            client['want-blocks'] = parsedMessage.data.indexOf('blocks') > -1;
-            client['want-projected-blocks'] = parsedMessage.data.indexOf('projected-blocks') > -1;
-            client['want-live-2h-chart'] = parsedMessage.data.indexOf('live-2h-chart') > -1;
-          }
-
-          if (parsedMessage.action === 'track-tx' && parsedMessage.txId && /^[a-fA-F0-9]{64}$/.test(parsedMessage.txId)) {
-            const tx = await memPool.getRawTransaction(parsedMessage.txId);
-            if (tx) {
-              console.log('Now tracking: ' + parsedMessage.txId);
-              client['trackingTx'] = true;
-              client['txId'] = parsedMessage.txId;
-              client['tx'] = tx;
-
-              if (tx.blockhash) {
-                const currentBlocks = blocks.getBlocks();
-                const foundBlock = currentBlocks.find((block) => block.tx && block.tx.some((i: string) => i === parsedMessage.txId));
-                if (foundBlock) {
-                  console.log('Found block by looking in local cache');
-                  client['blockHeight'] = foundBlock.height;
-                } else {
-                  const theBlock = await bitcoinApi.getBlockAndTransactions(tx.blockhash);
-                  if (theBlock) {
-                    client['blockHeight'] = theBlock.height;
-                  }
-                }
-              } else {
-                client['blockHeight'] = 0;
-              }
-              client.send(JSON.stringify({
-                'projectedBlocks': projectedBlocks.getProjectedBlocks(client['txId']),
-                'track-tx': {
-                  tracking: true,
-                  blockHeight: client['blockHeight'],
-                  tx: client['tx'],
-                }
-              }));
-            } else {
-              console.log('TX NOT FOUND, NOT TRACKING');
-              client['trackingTx'] = false;
-              client['blockHeight'] = 0;
-              client['tx'] = null;
-              client.send(JSON.stringify({
-                'track-tx': {
-                  tracking: false,
-                  blockHeight: 0,
-                  message: 'not-found',
-                }
-              }));
-            }
-          }
-          if (parsedMessage.action === 'stop-tracking-tx') {
-            console.log('STOP TRACKING');
-            client['trackingTx'] = false;
-            client.send(JSON.stringify({
-              'track-tx': {
-                tracking: false,
-                blockHeight: 0,
-                message: 'not-found',
-              }
-            }));
-          }
+          await elementsParser.$parse();
         } catch (e) {
-          console.log(e);
+          logger.warn('Elements parsing error: ' + (e instanceof Error ? e.message : e));
         }
       });
-
-      client.on('close', () => {
-        client['trackingTx'] = false;
-      });
-    });
-
-    blocks.setNewBlockCallback((block: IBlock) => {
-      const formattedBlocks = blocks.formatBlock(block);
-
-      this.wss.clients.forEach((client) => {
-        if (client.readyState !== WebSocket.OPEN) {
-          return;
-        }
-
-        const response = {};
-
-        if (client['trackingTx'] === true && client['blockHeight'] === 0) {
-          if (block.tx.some((tx: ITransaction) => tx === client['txId'])) {
-            client['blockHeight'] = block.height;
-          }
-        }
-
-        response['track-tx'] = {
-          tracking: client['trackingTx'] || false,
-          blockHeight: client['blockHeight'],
-        };
-
-        response['block'] = formattedBlocks;
-
-        client.send(JSON.stringify(response));
-      });
-    });
-
-    memPool.setMempoolChangedCallback((newMempool: IMempool) => {
-      projectedBlocks.updateProjectedBlocks(newMempool);
-
-      const pBlocks = projectedBlocks.getProjectedBlocks();
-      const mempoolInfo = memPool.getMempoolInfo();
-      const txPerSecond = memPool.getTxPerSecond();
-      const vBytesPerSecond = memPool.getVBytesPerSecond();
-
-      this.wss.clients.forEach((client: WebSocket) => {
-        if (client.readyState !== WebSocket.OPEN) {
-          return;
-        }
-
-        const response = {};
-
-        if (client['want-stats']) {
-          response['mempoolInfo'] = mempoolInfo;
-          response['txPerSecond'] = txPerSecond;
-          response['vBytesPerSecond'] = vBytesPerSecond;
-          response['track-tx'] = {
-            tracking: client['trackingTx'] || false,
-            blockHeight: client['blockHeight'],
-          };
-        }
-
-        if (client['want-projected-blocks'] && client['trackingTx'] && client['blockHeight'] === 0) {
-          response['projectedBlocks'] = projectedBlocks.getProjectedBlocks(client['txId']);
-        } else if (client['want-projected-blocks']) {
-          response['projectedBlocks'] = pBlocks;
-        }
-
-        if (Object.keys(response).length) {
-          client.send(JSON.stringify(response));
-        }
-      });
-    });
-
-    statistics.setNewStatisticsEntryCallback((stats: IMempoolStats) => {
-      this.wss.clients.forEach((client: WebSocket) => {
-        if (client.readyState !== WebSocket.OPEN) {
-          return;
-        }
-
-        if (client['want-live-2h-chart']) {
-          client.send(JSON.stringify({
-            'live-2h-chart': stats
-          }));
-        }
-      });
-    });
+    }
+    websocketHandler.setupConnectionHandling();
+    statistics.setNewStatisticsEntryCallback(websocketHandler.handleNewStatistic.bind(websocketHandler));
+    blocks.setNewBlockCallback(websocketHandler.handleNewBlock.bind(websocketHandler));
+    memPool.setMempoolChangedCallback(websocketHandler.handleMempoolChange.bind(websocketHandler));
+    fiatConversion.setProgressChangedCallback(websocketHandler.handleNewConversionRates.bind(websocketHandler));
+    loadingIndicators.setProgressChangedCallback(websocketHandler.handleLoadingChanged.bind(websocketHandler));
   }
 
-  private setUpRoutes() {
+  setUpHttpApiRoutes() {
     this.app
-      .get(config.API_ENDPOINT + 'transactions/height/:id', routes.$getgetTransactionsForBlock)
-      .get(config.API_ENDPOINT + 'transactions/projected/:id', routes.getgetTransactionsForProjectedBlock)
-      .get(config.API_ENDPOINT + 'fees/recommended', routes.getRecommendedFees)
-      .get(config.API_ENDPOINT + 'fees/projected-blocks', routes.getProjectedBlocks)
-      .get(config.API_ENDPOINT + 'statistics/2h', routes.get2HStatistics)
-      .get(config.API_ENDPOINT + 'statistics/24h', routes.get24HStatistics.bind(routes))
-      .get(config.API_ENDPOINT + 'statistics/1w', routes.get1WHStatistics.bind(routes))
-      .get(config.API_ENDPOINT + 'statistics/1m', routes.get1MStatistics.bind(routes))
-      .get(config.API_ENDPOINT + 'statistics/3m', routes.get3MStatistics.bind(routes))
-      .get(config.API_ENDPOINT + 'statistics/6m', routes.get6MStatistics.bind(routes))
-      ;
+      .get(config.MEMPOOL.API_URL_PREFIX + 'transaction-times', routes.getTransactionTimes)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'cpfp/:txId', routes.getCpfpInfo)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'difficulty-adjustment', routes.getDifficultyChange)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'fees/recommended', routes.getRecommendedFees)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'fees/mempool-blocks', routes.getMempoolBlocks)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'backend-info', routes.getBackendInfo)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'init-data', routes.getInitData)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'validate-address/:address', routes.validateAddress)
+      .post(config.MEMPOOL.API_URL_PREFIX + 'tx/push', routes.$postTransactionForm)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'donations', async (req, res) => {
+        try {
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/donations`, { responseType: 'stream', timeout: 10000 });
+          response.data.pipe(res);
+        } catch (e) {
+          res.status(500).end();
+        }
+      })
+      .get(config.MEMPOOL.API_URL_PREFIX + 'donations/images/:id', async (req, res) => {
+        try {
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/donations/images/${req.params.id}`, {
+            responseType: 'stream', timeout: 10000
+          });
+          response.data.pipe(res);
+        } catch (e) {
+          res.status(500).end();
+        }
+      })
+      .get(config.MEMPOOL.API_URL_PREFIX + 'contributors', async (req, res) => {
+        try {
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/contributors`, { responseType: 'stream', timeout: 10000 });
+          response.data.pipe(res);
+        } catch (e) {
+          res.status(500).end();
+        }
+      })
+      .get(config.MEMPOOL.API_URL_PREFIX + 'contributors/images/:id', async (req, res) => {
+        try {
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/contributors/images/${req.params.id}`, {
+            responseType: 'stream', timeout: 10000
+          });
+          response.data.pipe(res);
+        } catch (e) {
+          res.status(500).end();
+        }
+      })
+      .get(config.MEMPOOL.API_URL_PREFIX + 'translators', async (req, res) => {
+        try {
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/translators`, { responseType: 'stream', timeout: 10000 });
+          response.data.pipe(res);
+        } catch (e) {
+          res.status(500).end();
+        }
+      })
+      .get(config.MEMPOOL.API_URL_PREFIX + 'translators/images/:id', async (req, res) => {
+        try {
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/translators/images/${req.params.id}`, {
+            responseType: 'stream', timeout: 10000
+          });
+          response.data.pipe(res);
+        } catch (e) {
+          res.status(500).end();
+        }
+      })
+    ;
 
-    if (config.BACKEND_API === 'electrs') {
+    if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED) {
       this.app
-        .get(config.API_ENDPOINT + 'explorer/blocks', routes.getBlocks)
-        .get(config.API_ENDPOINT + 'explorer/blocks/:height', routes.getBlocks)
-        .get(config.API_ENDPOINT + 'explorer/tx/:id', routes.getRawTransaction)
-        .get(config.API_ENDPOINT + 'explorer/block/:hash', routes.getBlock)
-        .get(config.API_ENDPOINT + 'explorer/block/:hash/tx', routes.getBlockTransactions)
-        .get(config.API_ENDPOINT + 'explorer/block/:hash/tx/:index', routes.getBlockTransactionsFromIndex)
-        .get(config.API_ENDPOINT + 'explorer/address/:address', routes.getAddress)
-        .get(config.API_ENDPOINT + 'explorer/address/:address/tx', routes.getAddressTransactions)
-        .get(config.API_ENDPOINT + 'explorer/address/:address/tx/chain/:txid', routes.getAddressTransactionsFromTxid)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/2h', routes.$getStatisticsByTime.bind(routes, '2h'))
+        .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/24h', routes.$getStatisticsByTime.bind(routes, '24h'))
+        .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/1w', routes.$getStatisticsByTime.bind(routes, '1w'))
+        .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/1m', routes.$getStatisticsByTime.bind(routes, '1m'))
+        .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/3m', routes.$getStatisticsByTime.bind(routes, '3m'))
+        .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/6m', routes.$getStatisticsByTime.bind(routes, '6m'))
+        .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/1y', routes.$getStatisticsByTime.bind(routes, '1y'))
+        .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/2y', routes.$getStatisticsByTime.bind(routes, '2y'))
+        .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/3y', routes.$getStatisticsByTime.bind(routes, '3y'))
         ;
     }
 
+    if (Common.indexingEnabled()) {
+      this.app
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/:interval', routes.$getPools)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug/hashrate', routes.$getPoolHistoricalHashrate)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug/blocks', routes.$getPoolBlocks)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug/blocks/:height', routes.$getPoolBlocks)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug', routes.$getPool)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/hashrate/pools/:interval', routes.$getPoolsHistoricalHashrate)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/hashrate/:interval', routes.$getHistoricalHashrate)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/reward-stats/:blockCount', routes.$getRewardStats)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/fees/:interval', routes.$getHistoricalBlockFees)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/rewards/:interval', routes.$getHistoricalBlockRewards)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/fee-rates/:interval', routes.$getHistoricalBlockFeeRates)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/sizes-weights/:interval', routes.$getHistoricalBlockSizeAndWeight)
+      ;
+    }
+
+    if (config.BISQ.ENABLED) {
+      this.app
+        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/stats', routes.getBisqStats)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/tx/:txId', routes.getBisqTransaction)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/block/:hash', routes.getBisqBlock)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/blocks/tip/height', routes.getBisqTip)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/blocks/:index/:length', routes.getBisqBlocks)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/address/:address', routes.getBisqAddress)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/txs/:index/:length', routes.getBisqTransactions)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/currencies', routes.getBisqMarketCurrencies.bind(routes))
+        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/depth', routes.getBisqMarketDepth.bind(routes))
+        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/hloc', routes.getBisqMarketHloc.bind(routes))
+        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/markets', routes.getBisqMarketMarkets.bind(routes))
+        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/offers', routes.getBisqMarketOffers.bind(routes))
+        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/ticker', routes.getBisqMarketTicker.bind(routes))
+        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/trades', routes.getBisqMarketTrades.bind(routes))
+        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/volumes', routes.getBisqMarketVolumes.bind(routes))
+        .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/volumes/7d', routes.getBisqMarketVolumes7d.bind(routes))
+        ;
+    }
+
+    this.app
+      .get(config.MEMPOOL.API_URL_PREFIX + 'blocks', routes.getBlocks.bind(routes))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/:height', routes.getBlocks.bind(routes))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash', routes.getBlock);
+
+    if (config.MEMPOOL.BACKEND !== 'esplora') {
+      this.app
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mempool', routes.getMempool)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mempool/txids', routes.getMempoolTxIds)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mempool/recent', routes.getRecentMempoolTransactions)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId', routes.getTransaction)
+        .post(config.MEMPOOL.API_URL_PREFIX + 'tx', routes.$postTransaction)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/hex', routes.getRawTransaction)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/status', routes.getTransactionStatus)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/outspends', routes.getTransactionOutspends)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/header', routes.getBlockHeader)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/height', routes.getBlockTipHeight)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txs', routes.getBlockTransactions)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txs/:index', routes.getBlockTransactions)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txids', routes.getTxIdsForBlock)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'block-height/:height', routes.getBlockHeight)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address', routes.getAddress)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs', routes.getAddressTransactions)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs/chain/:txId', routes.getAddressTransactions)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'address-prefix/:prefix', routes.getAddressPrefix)
+      ;
+    }
+
+    if (Common.isLiquid()) {
+      this.app
+        .get(config.MEMPOOL.API_URL_PREFIX + 'assets/icons', routes.getAllLiquidIcon)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'assets/featured', routes.$getAllFeaturedLiquidAssets)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'asset/:assetId/icon', routes.getLiquidIcon)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'assets/group/:id', routes.$getAssetGroup)
+      ;
+    }
+
+    if (Common.isLiquid() && config.DATABASE.ENABLED) {
+      this.app
+        .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/pegs/month', routes.$getElementsPegsByMonth)
+      ;
     }
   }
+}
 
-const mempoolSpace = new MempoolSpace();
+const server = new Server();

backend/src/indexer.ts

@@ -0,0 +1,54 @@
+import { Common } from './api/common';
+import blocks from './api/blocks';
+import mempool from './api/mempool';
+import mining from './api/mining';
+import logger from './logger';
+import HashratesRepository from './repositories/HashratesRepository';
+
+class Indexer {
+  runIndexer = true;
+  indexerRunning = false;
+
+  constructor() {
+  }
+
+  public reindex() {
+    if (Common.indexingEnabled()) {
+      this.runIndexer = true;
+    }
+  }
+
+  public async $run() {
+    if (!Common.indexingEnabled() || this.runIndexer === false ||
+      this.indexerRunning === true || mempool.hasPriority()
+    ) {
+      return;
+    }
+
+    this.runIndexer = false;
+    this.indexerRunning = true;
+
+    try {
+      await blocks.$generateBlockDatabase();
+      await this.$resetHashratesIndexingState();
+      await mining.$generateNetworkHashrateHistory();
+      await mining.$generatePoolHashrateHistory();
+    } catch (e) {
+      this.reindex();
+      logger.err(`Indexer failed, trying again later. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+
+    this.indexerRunning = false;
+  }
+
+  async $resetHashratesIndexingState() {
+    try {
+      await HashratesRepository.$setLatestRun('last_hashrates_indexing', 0);
+      await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', 0);
+    } catch (e) {
+      logger.err(`Cannot reset hashrate indexing timestamps. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+  }
+}
+
+export default new Indexer();

backend/src/interfaces.ts

@@ -1,152 +0,0 @@
-export interface IMempoolInfo {
-  size: number;
-  bytes: number;
-  usage?: number;
-  maxmempool?: number;
-  mempoolminfee?: number;
-  minrelaytxfee?: number;
-}
-
-export interface ITransaction {
-  txid: string;
-  hash: string;
-  version: number;
-  size: number;
-  vsize: number;
-  weight: number;
-  locktime: number;
-  vin: Vin[];
-  vout: Vout[];
-  hex: string;
-  fee: number;
-  feePerWeightUnit: number;
-  feePerVsize: number;
-  blockhash?: string;
-  confirmations?: number;
-  time?: number;
-  blocktime?: number;
-  totalOut?: number;
-}
-
-export interface IBlock {
-  hash: string;
-  confirmations: number;
-  strippedsize: number;
-  size: number;
-  weight: number;
-  height: number;
-  version: number;
-  versionHex: string;
-  merkleroot: string;
-  tx: any;
-  time: number;
-  mediantime: number;
-  nonce: number;
-  bits: string;
-  difficulty: number;
-  chainwork: string;
-  nTx: number;
-  previousblockhash: string;
-  fees: number;
-
-  minFee?: number;
-  maxFee?: number;
-  medianFee?: number;
-}
-
-interface ScriptSig {
-  asm: string;
-  hex: string;
-}
-
-interface Vin {
-  txid: string;
-  vout: number;
-  scriptSig: ScriptSig;
-  sequence: number;
-}
-
-interface ScriptPubKey {
-  asm: string;
-  hex: string;
-  reqSigs: number;
-  type: string;
-  addresses: string[];
-}
-
-interface Vout {
-  value: number;
-  n: number;
-  scriptPubKey: ScriptPubKey;
-}
-
-export interface IMempoolStats {
-  id?: number;
-  added: string;
-  unconfirmed_transactions: number;
-  tx_per_second: number;
-  vbytes_per_second: number;
-  total_fee: number;
-  mempool_byte_weight: number;
-  fee_data: string;
-
-  vsize_1: number;
-  vsize_2: number;
-  vsize_3: number;
-  vsize_4: number;
-  vsize_5: number;
-  vsize_6: number;
-  vsize_8: number;
-  vsize_10: number;
-  vsize_12: number;
-  vsize_15: number;
-  vsize_20: number;
-  vsize_30: number;
-  vsize_40: number;
-  vsize_50: number;
-  vsize_60: number;
-  vsize_70: number;
-  vsize_80: number;
-  vsize_90: number;
-  vsize_100: number;
-  vsize_125: number;
-  vsize_150: number;
-  vsize_175: number;
-  vsize_200: number;
-  vsize_250: number;
-  vsize_300: number;
-  vsize_350: number;
-  vsize_400: number;
-  vsize_500: number;
-  vsize_600: number;
-  vsize_700: number;
-  vsize_800: number;
-  vsize_900: number;
-  vsize_1000: number;
-  vsize_1200: number;
-  vsize_1400: number;
-  vsize_1600: number;
-  vsize_1800: number;
-  vsize_2000: number;
-}
-
-export interface IProjectedBlockInternal extends IProjectedBlock {
-  txIds: string[];
-  txFeePerVsizes: number[];
-}
-
-export interface IProjectedBlock {
-  blockSize: number;
-  blockWeight: number;
-  maxFee: number;
-  maxWeightFee: number;
-  medianFee: number;
-  minFee: number;
-  minWeightFee: number;
-  nTx: number;
-  fees: number;
-  hasMyTxId?: boolean;
-}
-
-export interface IMempool { [txid: string]: ITransaction; }
-

backend/src/logger.ts

@@ -0,0 +1,148 @@
+import config from './config';
+import * as dgram from 'dgram';
+
+class Logger {
+  static priorities = {
+    emerg: 0,
+    alert: 1,
+    crit: 2,
+    err: 3,
+    warn: 4,
+    notice: 5,
+    info: 6,
+    debug: 7
+  };
+  static facilities = {
+    kern: 0,
+    user: 1,
+    mail: 2,
+    daemon: 3,
+    auth: 4,
+    syslog: 5,
+    lpr: 6,
+    news: 7,
+    uucp: 8,
+    local0: 16,
+    local1: 17,
+    local2: 18,
+    local3: 19,
+    local4: 20,
+    local5: 21,
+    local6: 22,
+    local7: 23
+  };
+
+  // @ts-ignore
+  public emerg: ((msg: string) => void);
+  // @ts-ignore
+  public alert: ((msg: string) => void);
+  // @ts-ignore
+  public crit: ((msg: string) => void);
+  // @ts-ignore
+  public err: ((msg: string) => void);
+  // @ts-ignore
+  public warn: ((msg: string) => void);
+  // @ts-ignore
+  public notice: ((msg: string) => void);
+  // @ts-ignore
+  public info: ((msg: string) => void);
+  // @ts-ignore
+  public debug: ((msg: string) => void);
+
+  private name = 'mempool';
+  private client: dgram.Socket;
+  private network: string;
+
+  constructor() {
+    let prio;
+    for (prio in Logger.priorities) {
+      if (true) {
+        this.addprio(prio);
+      }
+    }
+    this.client = dgram.createSocket('udp4');
+    this.network = this.getNetwork();
+  }
+
+  private addprio(prio): void {
+    this[prio] = (function(_this) {
+      return function(msg) {
+        return _this.msg(prio, msg);
+      };
+    })(this);
+  }
+
+  private getNetwork(): string {
+    if (config.BISQ.ENABLED) {
+      return 'bisq';
+    }
+    if (config.MEMPOOL.NETWORK && config.MEMPOOL.NETWORK !== 'mainnet') {
+      return config.MEMPOOL.NETWORK;
+    }
+    return '';
+  }
+
+  private msg(priority, msg) {
+    let consolemsg, prionum, syslogmsg;
+    if (typeof msg === 'string' && msg.length > 0) {
+      while (msg[msg.length - 1].charCodeAt(0) === 10) {
+        msg = msg.slice(0, msg.length - 1);
+      }
+    }
+    const network = this.network ? ' <' + this.network + '>' : '';
+    prionum = Logger.priorities[priority] || Logger.priorities.info;
+    consolemsg = `${this.ts()} [${process.pid}] ${priority.toUpperCase()}:${network} ${msg}`;
+
+    if (config.SYSLOG.ENABLED && Logger.priorities[priority] <= Logger.priorities[config.SYSLOG.MIN_PRIORITY]) {
+      syslogmsg = `<${(Logger.facilities[config.SYSLOG.FACILITY] * 8 + prionum)}> ${this.name}[${process.pid}]: ${priority.toUpperCase()}${network} ${msg}`;
+      this.syslog(syslogmsg);
+    }
+    if (Logger.priorities[priority] > Logger.priorities[config.MEMPOOL.STDOUT_LOG_MIN_PRIORITY]) {
+      return;
+    }
+    if (priority === 'warning') {
+      priority = 'warn';
+    }
+    if (priority === 'debug') {
+      priority = 'info';
+    }
+    if (priority === 'err') {
+      priority = 'error';
+    }
+    return (console[priority] || console.error)(consolemsg);
+  }
+
+  private syslog(msg) {
+    let msgbuf;
+    msgbuf = Buffer.from(msg);
+    this.client.send(msgbuf, 0, msgbuf.length, config.SYSLOG.PORT, config.SYSLOG.HOST, function(err, bytes) {
+      if (err) {
+        console.log(err);
+      }
+    });
+  }
+
+  private leadZero(n: number): number | string {
+    if (n < 10) {
+      return '0' + n;
+    }
+    return n;
+  }
+
+  private ts() {
+    let day, dt, hours, minutes, month, months, seconds;
+    dt = new Date();
+    hours = this.leadZero(dt.getHours());
+    minutes = this.leadZero(dt.getMinutes());
+    seconds = this.leadZero(dt.getSeconds());
+    month = dt.getMonth();
+    day = dt.getDate();
+    if (day < 10) {
+      day = ' ' + day;
+    }
+    months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
+    return months[month] + ' ' + day + ' ' + hours + ':' + minutes + ':' + seconds;
+  }
+}
+
+export default new Logger();

backend/src/mempool.interfaces.ts

@@ -0,0 +1,226 @@
+import { IEsploraApi } from './api/bitcoin/esplora-api.interface';
+
+export interface PoolTag {
+  id: number; // mysql row id
+  name: string;
+  link: string;
+  regexes: string; // JSON array
+  addresses: string; // JSON array
+  slug: string;
+}
+
+export interface PoolInfo {
+  poolId: number; // mysql row id
+  name: string;
+  link: string;
+  blockCount: number;
+  slug: string;
+}
+
+export interface PoolStats extends PoolInfo {
+  rank: number;
+  emptyBlocks: number;
+}
+
+export interface MempoolBlock {
+  blockSize: number;
+  blockVSize: number;
+  nTx: number;
+  medianFee: number;
+  totalFees: number;
+  feeRange: number[];
+}
+
+export interface MempoolBlockWithTransactions extends MempoolBlock {
+  transactionIds: string[];
+  transactions: TransactionStripped[];
+}
+
+export interface MempoolBlockDelta {
+  added: TransactionStripped[];
+  removed: string[];
+}
+
+interface VinStrippedToScriptsig {
+  scriptsig: string;
+}
+
+interface VoutStrippedToScriptPubkey {
+  scriptpubkey_address: string | undefined;
+  value: number;
+}
+
+export interface TransactionExtended extends IEsploraApi.Transaction {
+  vsize: number;
+  feePerVsize: number;
+  firstSeen?: number;
+  effectiveFeePerVsize: number;
+  ancestors?: Ancestor[];
+  bestDescendant?: BestDescendant | null;
+  cpfpChecked?: boolean;
+  deleteAfter?: number;
+}
+
+interface Ancestor {
+  txid: string;
+  weight: number;
+  fee: number;
+}
+
+interface BestDescendant {
+  txid: string;
+  weight: number;
+  fee: number;
+}
+
+export interface CpfpInfo {
+  ancestors: Ancestor[];
+  bestDescendant: BestDescendant | null;
+}
+
+export interface TransactionStripped {
+  txid: string;
+  fee: number;
+  vsize: number;
+  value: number;
+}
+
+export interface BlockExtension {
+  totalFees?: number;
+  medianFee?: number;
+  feeRange?: number[];
+  reward?: number;
+  coinbaseTx?: TransactionMinerInfo;
+  matchRate?: number;
+  pool?: {
+    id: number;
+    name: string;
+    slug: string;
+  };
+  avgFee?: number;
+  avgFeeRate?: number;
+  coinbaseRaw?: string;
+}
+
+export interface BlockExtended extends IEsploraApi.Block {
+  extras: BlockExtension;
+}
+
+export interface TransactionMinerInfo {
+  vin: VinStrippedToScriptsig[];
+  vout: VoutStrippedToScriptPubkey[];
+}
+
+export interface MempoolStats {
+  funded_txo_count: number;
+  funded_txo_sum: number;
+  spent_txo_count: number;
+  spent_txo_sum: number;
+  tx_count: number;
+}
+
+export interface Statistic {
+  id?: number;
+  added: string;
+  unconfirmed_transactions: number;
+  tx_per_second: number;
+  vbytes_per_second: number;
+  total_fee: number;
+  mempool_byte_weight: number;
+  fee_data: string;
+
+  vsize_1: number;
+  vsize_2: number;
+  vsize_3: number;
+  vsize_4: number;
+  vsize_5: number;
+  vsize_6: number;
+  vsize_8: number;
+  vsize_10: number;
+  vsize_12: number;
+  vsize_15: number;
+  vsize_20: number;
+  vsize_30: number;
+  vsize_40: number;
+  vsize_50: number;
+  vsize_60: number;
+  vsize_70: number;
+  vsize_80: number;
+  vsize_90: number;
+  vsize_100: number;
+  vsize_125: number;
+  vsize_150: number;
+  vsize_175: number;
+  vsize_200: number;
+  vsize_250: number;
+  vsize_300: number;
+  vsize_350: number;
+  vsize_400: number;
+  vsize_500: number;
+  vsize_600: number;
+  vsize_700: number;
+  vsize_800: number;
+  vsize_900: number;
+  vsize_1000: number;
+  vsize_1200: number;
+  vsize_1400: number;
+  vsize_1600: number;
+  vsize_1800: number;
+  vsize_2000: number;
+}
+
+export interface OptimizedStatistic {
+  added: string;
+  vbytes_per_second: number;
+  total_fee: number;
+  mempool_byte_weight: number;
+  vsizes: number[];
+}
+
+export interface WebsocketResponse {
+  action: string;
+  data: string[];
+  'track-tx': string;
+  'track-address': string;
+  'watch-mempool': boolean;
+  'track-bisq-market': string;
+}
+
+export interface VbytesPerSecond {
+  unixTime: number;
+  vSize: number;
+}
+
+export interface RequiredSpec { [name: string]: RequiredParams; }
+
+interface RequiredParams {
+  required: boolean;
+  types: ('@string' | '@number' | '@boolean' | string)[];
+}
+
+export interface ILoadingIndicators { [name: string]: number; }
+export interface IConversionRates { [currency: string]: number; }
+
+export interface IBackendInfo {
+  hostname: string;
+  gitCommit: string;
+  version: string;
+}
+
+export interface IDifficultyAdjustment {
+  progressPercent: number;
+  difficultyChange: number;
+  estimatedRetargetDate: number;
+  remainingBlocks: number;
+  remainingTime: number;
+  previousRetarget: number;
+  nextRetargetHeight: number;
+  timeAvg: number;
+  timeOffset: number;
+}
+
+export interface RewardStats {
+  totalReward: number;
+  totalFee: number;
+  totalTx: number;
+}

backend/src/repositories/BlocksRepository.ts

@@ -0,0 +1,657 @@
+import { BlockExtended } from '../mempool.interfaces';
+import DB from '../database';
+import logger from '../logger';
+import { Common } from '../api/common';
+import { prepareBlock } from '../utils/blocks-utils';
+import PoolsRepository from './PoolsRepository';
+import HashratesRepository from './HashratesRepository';
+import { escape } from 'mysql2';
+
+class BlocksRepository {
+  /**
+   * Save indexed block data in the database
+   */
+  public async $saveBlockInDatabase(block: BlockExtended) {
+    try {
+      const query = `INSERT INTO blocks(
+        height,           hash,                blockTimestamp, size,
+        weight,           tx_count,            coinbase_raw,   difficulty,
+        pool_id,          fees,                fee_span,       median_fee,
+        reward,           version,             bits,           nonce,
+        merkle_root,      previous_block_hash, avg_fee,        avg_fee_rate
+      ) VALUE (
+        ?, ?, FROM_UNIXTIME(?), ?,
+        ?, ?, ?, ?,
+        ?, ?, ?, ?,
+        ?, ?, ?, ?,
+        ?, ?, ?, ?
+      )`;
+
+      const params: any[] = [
+        block.height,
+        block.id,
+        block.timestamp,
+        block.size,
+        block.weight,
+        block.tx_count,
+        block.extras.coinbaseRaw,
+        block.difficulty,
+        block.extras.pool?.id, // Should always be set to something
+        block.extras.totalFees,
+        JSON.stringify(block.extras.feeRange),
+        block.extras.medianFee,
+        block.extras.reward,
+        block.version,
+        block.bits,
+        block.nonce,
+        block.merkle_root,
+        block.previousblockhash,
+        block.extras.avgFee,
+        block.extras.avgFeeRate,
+      ];
+
+      await DB.query(query, params);
+    } catch (e: any) {
+      if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
+        logger.debug(`$saveBlockInDatabase() - Block ${block.height} has already been indexed, ignoring`);
+      } else {
+        logger.err('Cannot save indexed block into db. Reason: ' + (e instanceof Error ? e.message : e));
+        throw e;
+      }
+    }
+  }
+
+  /**
+   * Get all block height that have not been indexed between [startHeight, endHeight]
+   */
+  public async $getMissingBlocksBetweenHeights(startHeight: number, endHeight: number): Promise<number[]> {
+    if (startHeight < endHeight) {
+      return [];
+    }
+
+    try {
+      const [rows]: any[] = await DB.query(`
+        SELECT height
+        FROM blocks
+        WHERE height <= ? AND height >= ?
+        ORDER BY height DESC;
+      `, [startHeight, endHeight]);
+
+      const indexedBlockHeights: number[] = [];
+      rows.forEach((row: any) => { indexedBlockHeights.push(row.height); });
+      const seekedBlocks: number[] = Array.from(Array(startHeight - endHeight + 1).keys(), n => n + endHeight).reverse();
+      const missingBlocksHeights = seekedBlocks.filter(x => indexedBlockHeights.indexOf(x) === -1);
+
+      return missingBlocksHeights;
+    } catch (e) {
+      logger.err('Cannot retrieve blocks list to index. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Get empty blocks for one or all pools
+   */
+  public async $countEmptyBlocks(poolId: number | null, interval: string | null = null): Promise<any> {
+    interval = Common.getSqlInterval(interval);
+
+    const params: any[] = [];
+    let query = `SELECT count(height) as count, pools.id as poolId
+      FROM blocks
+      JOIN pools on pools.id = blocks.pool_id
+      WHERE tx_count = 1`;
+
+    if (poolId) {
+      query += ` AND pool_id = ?`;
+      params.push(poolId);
+    }
+
+    if (interval) {
+      query += ` AND blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
+    }
+
+    query += ` GROUP by pools.id`;
+
+    try {
+      const [rows] = await DB.query(query, params);
+      return rows;
+    } catch (e) {
+      logger.err('Cannot count empty blocks. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Return most recent block height
+   */
+  public async $mostRecentBlockHeight(): Promise<number> {
+    try {
+      const [row] = await DB.query('SELECT MAX(height) as maxHeight from blocks');
+      return row[0]['maxHeight'];
+    } catch (e) {
+      logger.err(`Cannot count blocks for this pool (using offset). Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Get blocks count for a period
+   */
+  public async $blockCount(poolId: number | null, interval: string | null = null): Promise<number> {
+    interval = Common.getSqlInterval(interval);
+
+    const params: any[] = [];
+    let query = `SELECT count(height) as blockCount
+      FROM blocks`;
+
+    if (poolId) {
+      query += ` WHERE pool_id = ?`;
+      params.push(poolId);
+    }
+
+    if (interval) {
+      if (poolId) {
+        query += ` AND`;
+      } else {
+        query += ` WHERE`;
+      }
+      query += ` blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
+    }
+
+    try {
+      const [rows] = await DB.query(query, params);
+      return <number>rows[0].blockCount;
+    } catch (e) {
+      logger.err(`Cannot count blocks for this pool (using offset). Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Get blocks count between two dates
+   * @param poolId
+   * @param from - The oldest timestamp
+   * @param to - The newest timestamp
+   * @returns
+   */
+  public async $blockCountBetweenTimestamp(poolId: number | null, from: number, to: number): Promise<number> {
+    const params: any[] = [];
+    let query = `SELECT
+      count(height) as blockCount,
+      max(height) as lastBlockHeight
+      FROM blocks`;
+
+    if (poolId) {
+      query += ` WHERE pool_id = ?`;
+      params.push(poolId);
+    }
+
+    if (poolId) {
+      query += ` AND`;
+    } else {
+      query += ` WHERE`;
+    }
+    query += ` blockTimestamp BETWEEN FROM_UNIXTIME('${from}') AND FROM_UNIXTIME('${to}')`;
+
+    try {
+      const [rows] = await DB.query(query, params);
+      return <number>rows[0];
+    } catch (e) {
+      logger.err(`Cannot count blocks for this pool (using timestamps). Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Get blocks count for a period
+   */
+   public async $blockCountBetweenHeight(startHeight: number, endHeight: number): Promise<number> {
+    const params: any[] = [];
+    let query = `SELECT count(height) as blockCount
+      FROM blocks
+      WHERE height <= ${startHeight} AND height >= ${endHeight}`;
+
+    try {
+      const [rows] = await DB.query(query, params);
+      return <number>rows[0].blockCount;
+    } catch (e) {
+      logger.err(`Cannot count blocks for this pool (using offset). Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Get the oldest indexed block
+   */
+  public async $oldestBlockTimestamp(): Promise<number> {
+    const query = `SELECT UNIX_TIMESTAMP(blockTimestamp) as blockTimestamp
+      FROM blocks
+      ORDER BY height
+      LIMIT 1;`;
+
+    try {
+      const [rows]: any[] = await DB.query(query);
+
+      if (rows.length <= 0) {
+        return -1;
+      }
+
+      return <number>rows[0].blockTimestamp;
+    } catch (e) {
+      logger.err('Cannot get oldest indexed block timestamp. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Get blocks mined by a specific mining pool
+   */
+  public async $getBlocksByPool(slug: string, startHeight?: number): Promise<object[]> {
+    const pool = await PoolsRepository.$getPool(slug);
+    if (!pool) {
+      throw new Error('This mining pool does not exist ' + escape(slug));
+    }
+
+    const params: any[] = [];
+    let query = ` SELECT
+      height,
+      hash as id,
+      UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp,
+      size,
+      weight,
+      tx_count,
+      coinbase_raw,
+      difficulty,
+      fees,
+      fee_span,
+      median_fee,
+      reward,
+      version,
+      bits,
+      nonce,
+      merkle_root,
+      previous_block_hash as previousblockhash,
+      avg_fee,
+      avg_fee_rate
+      FROM blocks
+      WHERE pool_id = ?`;
+    params.push(pool.id);
+
+    if (startHeight !== undefined) {
+      query += ` AND height < ?`;
+      params.push(startHeight);
+    }
+
+    query += ` ORDER BY height DESC
+      LIMIT 10`;
+
+    try {
+      const [rows] = await DB.query(query, params);
+
+      const blocks: BlockExtended[] = [];
+      for (const block of <object[]>rows) {
+        blocks.push(prepareBlock(block));
+      }
+
+      return blocks;
+    } catch (e) {
+      logger.err('Cannot get blocks for this pool. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Get one block by height
+   */
+  public async $getBlockByHeight(height: number): Promise<object | null> {
+    try {
+      const [rows]: any[] = await DB.query(`SELECT
+        height,
+        hash,
+        hash as id,
+        UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp,
+        size,
+        weight,
+        tx_count,
+        coinbase_raw,
+        difficulty,
+        pools.id as pool_id,
+        pools.name as pool_name,
+        pools.link as pool_link,
+        pools.slug as pool_slug,
+        pools.addresses as pool_addresses,
+        pools.regexes as pool_regexes,
+        fees,
+        fee_span,
+        median_fee,
+        reward,
+        version,
+        bits,
+        nonce,
+        merkle_root,
+        previous_block_hash as previousblockhash,
+        avg_fee,
+        avg_fee_rate
+        FROM blocks
+        JOIN pools ON blocks.pool_id = pools.id
+        WHERE height = ${height};
+      `);
+
+      if (rows.length <= 0) {
+        return null;
+      }
+
+      rows[0].fee_span = JSON.parse(rows[0].fee_span);
+      return rows[0];
+    } catch (e) {
+      logger.err(`Cannot get indexed block ${height}. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Get one block by hash
+   */
+  public async $getBlockByHash(hash: string): Promise<object | null> {
+    try {
+      const query = `
+        SELECT *, UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp, hash as id,
+        pools.id as pool_id, pools.name as pool_name, pools.link as pool_link, pools.slug as pool_slug,
+        pools.addresses as pool_addresses, pools.regexes as pool_regexes,
+        previous_block_hash as previousblockhash
+        FROM blocks
+        JOIN pools ON blocks.pool_id = pools.id
+        WHERE hash = '${hash}';
+      `;
+      const [rows]: any[] = await DB.query(query);
+
+      if (rows.length <= 0) {
+        return null;
+      }
+
+      rows[0].fee_span = JSON.parse(rows[0].fee_span);
+      return rows[0];
+    } catch (e) {
+      logger.err(`Cannot get indexed block ${hash}. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Return blocks difficulty
+   */
+  public async $getBlocksDifficulty(interval: string | null): Promise<object[]> {
+    interval = Common.getSqlInterval(interval);
+
+    // :D ... Yeah don't ask me about this one https://stackoverflow.com/a/40303162
+    // Basically, using temporary user defined fields, we are able to extract all
+    // difficulty adjustments from the blocks tables.
+    // This allow use to avoid indexing it in another table.
+    let query = `
+      SELECT
+      *
+      FROM
+      (
+        SELECT
+        UNIX_TIMESTAMP(blockTimestamp) as timestamp, difficulty, height,
+        IF(@prevStatus = YT.difficulty, @rn := @rn + 1,
+          IF(@prevStatus := YT.difficulty, @rn := 1, @rn := 1)
+        ) AS rn
+        FROM blocks YT
+        CROSS JOIN
+        (
+          SELECT @prevStatus := -1, @rn := 1
+        ) AS var
+    `;
+
+    if (interval) {
+      query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
+    }
+
+    query += `
+        ORDER BY YT.height
+      ) AS t
+      WHERE t.rn = 1
+      ORDER BY t.height
+    `;
+
+    try {
+      const [rows]: any[] = await DB.query(query);
+
+      for (const row of rows) {
+        delete row['rn'];
+      }
+
+      return rows;
+    } catch (e) {
+      logger.err('Cannot generate difficulty history. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Get general block stats
+   */
+  public async $getBlockStats(blockCount: number): Promise<any> {
+    try {
+      // We need to use a subquery
+      const query = `
+        SELECT MIN(height) as startBlock, MAX(height) as endBlock, SUM(reward) as totalReward, SUM(fees) as totalFee, SUM(tx_count) as totalTx
+        FROM
+          (SELECT height, reward, fees, tx_count FROM blocks
+          ORDER by height DESC
+          LIMIT ?) as sub`;
+
+      const [rows]: any = await DB.query(query, [blockCount]);
+
+      return rows[0];
+    } catch (e) {
+      logger.err('Cannot generate reward stats. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /*
+   * Check if the last 10 blocks chain is valid
+   */
+  public async $validateRecentBlocks(): Promise<boolean> {
+    try {
+      const [lastBlocks]: any[] = await DB.query(`SELECT height, hash, previous_block_hash FROM blocks ORDER BY height DESC LIMIT 10`);
+
+      for (let i = 0; i < lastBlocks.length - 1; ++i) {
+        if (lastBlocks[i].previous_block_hash !== lastBlocks[i + 1].hash) {
+          logger.warn(`Chain divergence detected at block ${lastBlocks[i].height}, re-indexing most recent data`);
+          return false;
+        }
+      }
+
+      return true;
+    } catch (e) {
+      return true; // Don't do anything if there is a db error
+    }
+  }
+
+  /**
+   * Check if the chain of block hash is valid and delete data from the stale branch if needed
+   */
+  public async $validateChain(): Promise<boolean> {
+    try {
+      const start = new Date().getTime();
+      const [blocks]: any[] = await DB.query(`SELECT height, hash, previous_block_hash,
+        UNIX_TIMESTAMP(blockTimestamp) as timestamp FROM blocks ORDER BY height`);
+
+      let partialMsg = false;
+      let idx = 1;
+      while (idx < blocks.length) {
+        if (blocks[idx].height - 1 !== blocks[idx - 1].height) {
+          if (partialMsg === false) {
+            logger.info('Some blocks are not indexed, skipping missing blocks during chain validation');
+            partialMsg = true;
+          }
+          ++idx;
+          continue;
+        }
+
+        if (blocks[idx].previous_block_hash !== blocks[idx - 1].hash) {
+          logger.warn(`Chain divergence detected at block ${blocks[idx - 1].height}, re-indexing newer blocks and hashrates`);
+          await this.$deleteBlocksFrom(blocks[idx - 1].height);
+          await HashratesRepository.$deleteHashratesFromTimestamp(blocks[idx - 1].timestamp - 604800);
+          return false;
+        }
+        ++idx;
+      }
+
+      logger.info(`${idx} blocks hash validated in ${new Date().getTime() - start} ms`);
+      return true;
+    } catch (e) {
+      logger.err('Cannot validate chain of block hash. Reason: ' + (e instanceof Error ? e.message : e));
+      return true; // Don't do anything if there is a db error
+    }
+  }
+
+  /**
+   * Delete blocks from the database from blockHeight
+   */
+  public async $deleteBlocksFrom(blockHeight: number) {
+    logger.info(`Delete newer blocks from height ${blockHeight} from the database`);
+
+    try {
+      await DB.query(`DELETE FROM blocks where height >= ${blockHeight}`);
+    } catch (e) {
+      logger.err('Cannot delete indexed blocks. Reason: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  /**
+   * Get the historical averaged block fees
+   */
+  public async $getHistoricalBlockFees(div: number, interval: string | null): Promise<any> {
+    try {
+      let query = `SELECT
+        CAST(AVG(height) as INT) as avgHeight,
+        CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
+        CAST(AVG(fees) as INT) as avgFees
+        FROM blocks`;
+
+      if (interval !== null) {
+        query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
+      }
+
+      query += ` GROUP BY UNIX_TIMESTAMP(blockTimestamp) DIV ${div}`;
+
+      const [rows]: any = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('Cannot generate block fees history. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Get the historical averaged block rewards
+   */
+  public async $getHistoricalBlockRewards(div: number, interval: string | null): Promise<any> {
+    try {
+      let query = `SELECT
+        CAST(AVG(height) as INT) as avgHeight,
+        CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
+        CAST(AVG(reward) as INT) as avgRewards
+        FROM blocks`;
+
+      if (interval !== null) {
+        query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
+      }
+
+      query += ` GROUP BY UNIX_TIMESTAMP(blockTimestamp) DIV ${div}`;
+
+      const [rows]: any = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('Cannot generate block rewards history. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Get the historical averaged block fee rate percentiles
+   */
+   public async $getHistoricalBlockFeeRates(div: number, interval: string | null): Promise<any> {
+    try {
+      let query = `SELECT
+        CAST(AVG(height) as INT) as avgHeight,
+        CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
+        CAST(AVG(JSON_EXTRACT(fee_span, '$[0]')) as INT) as avgFee_0,
+        CAST(AVG(JSON_EXTRACT(fee_span, '$[1]')) as INT) as avgFee_10,
+        CAST(AVG(JSON_EXTRACT(fee_span, '$[2]')) as INT) as avgFee_25,
+        CAST(AVG(JSON_EXTRACT(fee_span, '$[3]')) as INT) as avgFee_50,
+        CAST(AVG(JSON_EXTRACT(fee_span, '$[4]')) as INT) as avgFee_75,
+        CAST(AVG(JSON_EXTRACT(fee_span, '$[5]')) as INT) as avgFee_90,
+        CAST(AVG(JSON_EXTRACT(fee_span, '$[6]')) as INT) as avgFee_100
+      FROM blocks`;
+
+      if (interval !== null) {
+        query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
+      }
+
+      query += ` GROUP BY UNIX_TIMESTAMP(blockTimestamp) DIV ${div}`;
+
+      const [rows]: any = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('Cannot generate block fee rates history. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Get the historical averaged block sizes
+   */
+   public async $getHistoricalBlockSizes(div: number, interval: string | null): Promise<any> {
+    try {
+      let query = `SELECT
+        CAST(AVG(height) as INT) as avgHeight,
+        CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
+        CAST(AVG(size) as INT) as avgSize
+      FROM blocks`;
+
+      if (interval !== null) {
+        query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
+      }
+
+      query += ` GROUP BY UNIX_TIMESTAMP(blockTimestamp) DIV ${div}`;
+
+      const [rows]: any = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('Cannot generate block size and weight history. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Get the historical averaged block weights
+   */
+   public async $getHistoricalBlockWeights(div: number, interval: string | null): Promise<any> {
+    try {
+      let query = `SELECT
+        CAST(AVG(height) as INT) as avgHeight,
+        CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
+        CAST(AVG(weight) as INT) as avgWeight
+      FROM blocks`;
+
+      if (interval !== null) {
+        query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
+      }
+
+      query += ` GROUP BY UNIX_TIMESTAMP(blockTimestamp) DIV ${div}`;
+
+      const [rows]: any = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('Cannot generate block size and weight history. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+}
+
+export default new BlocksRepository();

backend/src/repositories/HashratesRepository.ts

@@ -0,0 +1,217 @@
+import { escape } from 'mysql2';
+import { Common } from '../api/common';
+import DB from '../database';
+import logger from '../logger';
+import PoolsRepository from './PoolsRepository';
+
+class HashratesRepository {
+  /**
+   * Save indexed block data in the database
+   */
+  public async $saveHashrates(hashrates: any) {
+    if (hashrates.length === 0) {
+      return;
+    }
+
+    let query = `INSERT INTO
+      hashrates(hashrate_timestamp, avg_hashrate, pool_id, share, type) VALUES`;
+
+    for (const hashrate of hashrates) {
+      query += ` (FROM_UNIXTIME(${hashrate.hashrateTimestamp}), ${hashrate.avgHashrate}, ${hashrate.poolId}, ${hashrate.share}, "${hashrate.type}"),`;
+    }
+    query = query.slice(0, -1);
+
+    try {
+      await DB.query(query);
+    } catch (e: any) {
+      logger.err('Cannot save indexed hashrate into db. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getNetworkDailyHashrate(interval: string | null): Promise<any[]> {
+    interval = Common.getSqlInterval(interval);
+
+    let query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp, avg_hashrate as avgHashrate
+      FROM hashrates`;
+
+    if (interval) {
+      query += ` WHERE hashrate_timestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()
+        AND hashrates.type = 'daily'`;
+    } else {
+      query += ` WHERE hashrates.type = 'daily'`;
+    }
+
+    query += ` ORDER by hashrate_timestamp`;
+
+    try {
+      const [rows]: any[] = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('Cannot fetch network hashrate history. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getWeeklyHashrateTimestamps(): Promise<number[]> {
+    const query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp
+      FROM hashrates
+      WHERE type = 'weekly'
+      GROUP BY hashrate_timestamp`;
+
+    try {
+      const [rows]: any[] = await DB.query(query);
+      return rows.map(row => row.timestamp);
+    } catch (e) {
+      logger.err('Cannot retreive indexed weekly hashrate timestamps. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Returns the current biggest pool hashrate history
+   */
+  public async $getPoolsWeeklyHashrate(interval: string | null): Promise<any[]> {
+    interval = Common.getSqlInterval(interval);
+
+    const topPoolsId = (await PoolsRepository.$getPoolsInfo('1w')).map((pool) => pool.poolId);
+
+    let query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp, avg_hashrate as avgHashrate, share, pools.name as poolName
+      FROM hashrates
+      JOIN pools on pools.id = pool_id`;
+
+    if (interval) {
+      query += ` WHERE hashrate_timestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()
+        AND hashrates.type = 'weekly'
+        AND pool_id IN (${topPoolsId})`;
+    } else {
+      query += ` WHERE hashrates.type = 'weekly'
+        AND pool_id IN (${topPoolsId})`;
+    }
+
+    query += ` ORDER by hashrate_timestamp, FIELD(pool_id, ${topPoolsId})`;
+
+    try {
+      const [rows]: any[] = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('Cannot fetch weekly pools hashrate history. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Returns a pool hashrate history
+   */
+  public async $getPoolWeeklyHashrate(slug: string): Promise<any[]> {
+    const pool = await PoolsRepository.$getPool(slug);
+    if (!pool) {
+      throw new Error('This mining pool does not exist ' + escape(slug));
+    }
+
+    // Find hashrate boundaries
+    let query = `SELECT MIN(hashrate_timestamp) as firstTimestamp, MAX(hashrate_timestamp) as lastTimestamp
+      FROM hashrates
+      JOIN pools on pools.id = pool_id
+      WHERE hashrates.type = 'weekly' AND pool_id = ? AND avg_hashrate != 0
+      ORDER by hashrate_timestamp LIMIT 1`;
+
+    let boundaries = {
+      firstTimestamp: '1970-01-01',
+      lastTimestamp: '9999-01-01'
+    };
+
+    try {
+      const [rows]: any[] = await DB.query(query, [pool.id]);
+      boundaries = rows[0];
+    } catch (e) {
+      logger.err('Cannot fetch hashrate start/end timestamps for this pool. Reason: ' + (e instanceof Error ? e.message : e));
+    }
+
+    // Get hashrates entries between boundaries
+    query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp, avg_hashrate as avgHashrate, share, pools.name as poolName
+      FROM hashrates
+      JOIN pools on pools.id = pool_id
+      WHERE hashrates.type = 'weekly' AND hashrate_timestamp BETWEEN ? AND ?
+      AND pool_id = ?
+      ORDER by hashrate_timestamp`;
+
+    try {
+      const [rows]: any[] = await DB.query(query, [boundaries.firstTimestamp, boundaries.lastTimestamp, pool.id]);
+      return rows;
+    } catch (e) {
+      logger.err('Cannot fetch pool hashrate history for this pool. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Set latest run timestamp
+   */
+  public async $setLatestRun(key: string, val: number) {
+    const query = `UPDATE state SET number = ? WHERE name = ?`;
+
+    try {
+      await DB.query(query, [val, key]);
+    } catch (e) {
+      logger.err(`Cannot set last indexing run for ${key}. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Get latest run timestamp
+   */
+  public async $getLatestRun(key: string): Promise<number> {
+    const query = `SELECT number FROM state WHERE name = ?`;
+
+    try {
+      const [rows]: any[] = await DB.query(query, [key]);
+
+      if (rows.length === 0) {
+        return 0;
+      }
+      return rows[0]['number'];
+    } catch (e) {
+      logger.err(`Cannot retrieve last indexing run for ${key}. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Delete most recent data points for re-indexing
+   */
+  public async $deleteLastEntries() {
+    logger.info(`Delete latest hashrates data points from the database`);
+
+    try {
+      const [rows]: any[] = await DB.query(`SELECT MAX(hashrate_timestamp) as timestamp FROM hashrates GROUP BY type`);
+      for (const row of rows) {
+        await DB.query(`DELETE FROM hashrates WHERE hashrate_timestamp = ?`, [row.timestamp]);
+      }
+      // Re-run the hashrate indexing to fill up missing data
+      await this.$setLatestRun('last_hashrates_indexing', 0);
+      await this.$setLatestRun('last_weekly_hashrates_indexing', 0);
+    } catch (e) {
+      logger.err('Cannot delete latest hashrates data points. Reason: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+  
+  /**
+   * Delete hashrates from the database from timestamp
+   */
+  public async $deleteHashratesFromTimestamp(timestamp: number) {
+    logger.info(`Delete newer hashrates from timestamp ${new Date(timestamp * 1000).toUTCString()} from the database`);
+
+    try {
+      await DB.query(`DELETE FROM hashrates WHERE hashrate_timestamp >= FROM_UNIXTIME(?)`, [timestamp]);
+      // Re-run the hashrate indexing to fill up missing data
+      await this.$setLatestRun('last_hashrates_indexing', 0);
+      await this.$setLatestRun('last_weekly_hashrates_indexing', 0);
+    } catch (e) {
+      logger.err('Cannot delete latest hashrates data points. Reason: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+}
+
+export default new HashratesRepository();

backend/src/repositories/PoolsRepository.ts

@@ -0,0 +1,99 @@
+import { Common } from '../api/common';
+import config from '../config';
+import DB from '../database';
+import logger from '../logger';
+import { PoolInfo, PoolTag } from '../mempool.interfaces';
+
+class PoolsRepository {
+  /**
+   * Get all pools tagging info
+   */
+  public async $getPools(): Promise<PoolTag[]> {
+    const [rows] = await DB.query('SELECT id, name, addresses, regexes, slug FROM pools;');
+    return <PoolTag[]>rows;
+  }
+
+  /**
+   * Get unknown pool tagging info
+   */
+  public async $getUnknownPool(): Promise<PoolTag> {
+    const [rows] = await DB.query('SELECT id, name, slug FROM pools where name = "Unknown"');
+    return <PoolTag>rows[0];
+  }
+
+  /**
+   * Get basic pool info and block count
+   */
+  public async $getPoolsInfo(interval: string | null = null): Promise<PoolInfo[]> {
+    interval = Common.getSqlInterval(interval);
+
+    let query = `SELECT COUNT(height) as blockCount, pool_id as poolId, pools.name as name, pools.link as link, slug
+      FROM blocks
+      JOIN pools on pools.id = pool_id`;
+
+    if (interval) {
+      query += ` WHERE blocks.blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
+    }
+
+    query += ` GROUP BY pool_id
+      ORDER BY COUNT(height) DESC`;
+
+    try {
+      const [rows] = await DB.query(query);
+      return <PoolInfo[]>rows;
+    } catch (e) {
+      logger.err(`Cannot generate pools stats. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Get basic pool info and block count between two timestamp
+   */
+   public async $getPoolsInfoBetween(from: number, to: number): Promise<PoolInfo[]> {
+    const query = `SELECT COUNT(height) as blockCount, pools.id as poolId, pools.name as poolName
+      FROM pools
+      LEFT JOIN blocks on pools.id = blocks.pool_id AND blocks.blockTimestamp BETWEEN FROM_UNIXTIME(?) AND FROM_UNIXTIME(?)
+      GROUP BY pools.id`;
+
+    try {
+      const [rows] = await DB.query(query, [from, to]);
+      return <PoolInfo[]>rows;
+    } catch (e) {
+      logger.err('Cannot generate pools blocks count. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Get mining pool statistics for one pool
+   */
+   public async $getPool(slug: string): Promise<PoolTag | null> {
+    const query = `
+      SELECT *
+      FROM pools
+      WHERE pools.slug = ?`;
+
+    try {
+      const [rows]: any[] = await DB.query(query, [slug]);
+
+      if (rows.length < 1) {
+        return null;
+      }
+
+      rows[0].regexes = JSON.parse(rows[0].regexes);
+      if (['testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
+        rows[0].addresses = []; // pools.json only contains mainnet addresses
+      } else {
+        rows[0].addresses = JSON.parse(rows[0].addresses);
+      }
+
+      return rows[0];
+    } catch (e) {
+      logger.err('Cannot get pool from db. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+}
+
+export default new PoolsRepository();

backend/src/repositories/RatesRepository.ts

@@ -0,0 +1,21 @@
+import DB from '../database';
+import logger from '../logger';
+import { IConversionRates } from '../mempool.interfaces';
+
+class RatesRepository {
+  public async $saveRate(height: number, rates: IConversionRates) {
+    try {
+      await DB.query(`INSERT INTO rates(height, bisq_rates) VALUE (?, ?)`, [height, JSON.stringify(rates)]);
+    } catch (e: any) {
+      if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
+        logger.debug(`Rate already exists for block ${height}, ignoring`);
+      } else {
+        logger.err(`Cannot save exchange rate into db for block ${height} Reason: ` + (e instanceof Error ? e.message : e));
+        throw e;
+      }
+    }
+  }
+}
+
+export default new RatesRepository();
+

backend/src/rpc-api/commands.ts

@@ -0,0 +1,92 @@
+module.exports = {
+  addMultiSigAddress: 'addmultisigaddress',
+  addNode: 'addnode', // bitcoind v0.8.0+
+  backupWallet: 'backupwallet',
+  createMultiSig: 'createmultisig',
+  createRawTransaction: 'createrawtransaction', // bitcoind v0.7.0+
+  decodeRawTransaction: 'decoderawtransaction', // bitcoind v0.7.0+
+  decodeScript: 'decodescript',
+  dumpPrivKey: 'dumpprivkey',
+  dumpWallet: 'dumpwallet', // bitcoind v0.9.0+
+  encryptWallet: 'encryptwallet',
+  estimateFee: 'estimatefee', // bitcoind v0.10.0x
+  estimatePriority: 'estimatepriority', // bitcoind v0.10.0+
+  generate: 'generate', // bitcoind v0.11.0+
+  getAccount: 'getaccount',
+  getAccountAddress: 'getaccountaddress',
+  getAddedNodeInfo: 'getaddednodeinfo', // bitcoind v0.8.0+
+  getAddressesByAccount: 'getaddressesbyaccount',
+  getBalance: 'getbalance',
+  getBestBlockHash: 'getbestblockhash', // bitcoind v0.9.0+
+  getBlock: 'getblock',
+  getBlockStats: 'getblockstats',
+  getBlockFilter: 'getblockfilter',
+  getBlockchainInfo: 'getblockchaininfo', // bitcoind v0.9.2+
+  getBlockCount: 'getblockcount',
+  getBlockHash: 'getblockhash',
+  getBlockHeader: 'getblockheader',
+  getBlockTemplate: 'getblocktemplate', // bitcoind v0.7.0+
+  getChainTips: 'getchaintips', // bitcoind v0.10.0+
+  getChainTxStats: 'getchaintxstats',
+  getConnectionCount: 'getconnectioncount',
+  getDifficulty: 'getdifficulty',
+  getGenerate: 'getgenerate',
+  getInfo: 'getinfo',
+  getMempoolAncestors: 'getmempoolancestors',
+  getMempoolDescendants: 'getmempooldescendants',
+  getMempoolEntry: 'getmempoolentry',
+  getMempoolInfo: 'getmempoolinfo', // bitcoind v0.10+
+  getMiningInfo: 'getmininginfo',
+  getNetTotals: 'getnettotals',
+  getNetworkInfo: 'getnetworkinfo', // bitcoind v0.9.2+
+  getNetworkHashPs: 'getnetworkhashps', // bitcoind v0.9.0+
+  getNewAddress: 'getnewaddress',
+  getPeerInfo: 'getpeerinfo', // bitcoind v0.7.0+
+  getRawChangeAddress: 'getrawchangeaddress', // bitcoin v0.9+
+  getRawMemPool: 'getrawmempool', // bitcoind v0.7.0+
+  getRawTransaction: 'getrawtransaction', // bitcoind v0.7.0+
+  getReceivedByAccount: 'getreceivedbyaccount',
+  getReceivedByAddress: 'getreceivedbyaddress',
+  getTransaction: 'gettransaction',
+  getTxOut: 'gettxout', // bitcoind v0.7.0+
+  getTxOutProof: 'gettxoutproof', // bitcoind v0.11.0+
+  getTxOutSetInfo: 'gettxoutsetinfo', // bitcoind v0.7.0+
+  getUnconfirmedBalance: 'getunconfirmedbalance', // bitcoind v0.9.0+
+  getWalletInfo: 'getwalletinfo', // bitcoind v0.9.2+
+  help: 'help',
+  importAddress: 'importaddress', // bitcoind v0.10.0+
+  importPrivKey: 'importprivkey',
+  importWallet: 'importwallet', // bitcoind v0.9.0+
+  keypoolRefill: 'keypoolrefill',
+  keyPoolRefill: 'keypoolrefill',
+  listAccounts: 'listaccounts',
+  listAddressGroupings: 'listaddressgroupings', // bitcoind v0.7.0+
+  listLockUnspent: 'listlockunspent', // bitcoind v0.8.0+
+  listReceivedByAccount: 'listreceivedbyaccount',
+  listReceivedByAddress: 'listreceivedbyaddress',
+  listSinceBlock: 'listsinceblock',
+  listTransactions: 'listtransactions',
+  listUnspent: 'listunspent', // bitcoind v0.7.0+
+  lockUnspent: 'lockunspent', // bitcoind v0.8.0+
+  move: 'move',
+  ping: 'ping', // bitcoind v0.9.0+
+  prioritiseTransaction: 'prioritisetransaction', // bitcoind v0.10.0+
+  sendFrom: 'sendfrom',
+  sendMany: 'sendmany',
+  sendRawTransaction: 'sendrawtransaction', // bitcoind v0.7.0+
+  sendToAddress: 'sendtoaddress',
+  setAccount: 'setaccount',
+  setGenerate: 'setgenerate',
+  setTxFee: 'settxfee',
+  signMessage: 'signmessage',
+  signRawTransaction: 'signrawtransaction', // bitcoind v0.7.0+
+  stop: 'stop',
+  submitBlock: 'submitblock', // bitcoind v0.7.0+
+  validateAddress: 'validateaddress',
+  verifyChain: 'verifychain', // bitcoind v0.9.0+
+  verifyMessage: 'verifymessage',
+  verifyTxOutProof: 'verifytxoutproof', // bitcoind v0.11.0+
+  walletLock: 'walletlock',
+  walletPassphrase: 'walletpassphrase',
+  walletPassphraseChange: 'walletpassphrasechange'
+}

backend/src/rpc-api/index.ts

@@ -0,0 +1,61 @@
+var commands = require('./commands')
+var rpc = require('./jsonrpc')
+
+// ===----------------------------------------------------------------------===//
+// JsonRPC
+// ===----------------------------------------------------------------------===//
+function Client (opts) {
+  // @ts-ignore
+  this.rpc = new rpc.JsonRPC(opts)
+}
+
+// ===----------------------------------------------------------------------===//
+// cmd
+// ===----------------------------------------------------------------------===//
+Client.prototype.cmd = function () {
+  var args = [].slice.call(arguments)
+  var cmd = args.shift()
+
+  callRpc(cmd, args, this.rpc)
+}
+
+// ===----------------------------------------------------------------------===//
+// callRpc
+// ===----------------------------------------------------------------------===//
+function callRpc (cmd, args, rpc) {
+  var fn = args[args.length - 1]
+
+  // If the last argument is a callback, pop it from the args list
+  if (typeof fn === 'function') {
+    args.pop()
+  } else {
+    fn = function () {}
+  }
+
+  return rpc.call(cmd, args, function () {
+    var args = [].slice.call(arguments)
+      // @ts-ignore
+    args.unshift(null)
+      // @ts-ignore
+    fn.apply(this, args)
+  }, function (err) {
+    fn(err)
+  })
+}
+
+// ===----------------------------------------------------------------------===//
+// Initialize wrappers
+// ===----------------------------------------------------------------------===//
+(function () {
+  for (var protoFn in commands) {
+    (function (protoFn) {
+      Client.prototype[protoFn] = function () {
+        var args = [].slice.call(arguments)
+        return callRpc(commands[protoFn], args, this.rpc)
+      }
+    })(protoFn)
+  }
+})()
+
+// Export!
+module.exports.Client = Client;

backend/src/rpc-api/jsonrpc.ts

@@ -0,0 +1,162 @@
+var http = require('http')
+var https = require('https')
+
+var JsonRPC = function (opts) {
+  // @ts-ignore
+  this.opts = opts || {}
+  // @ts-ignore
+  this.http = this.opts.ssl ? https : http
+}
+
+JsonRPC.prototype.call = function (method, params) {
+  return new Promise((resolve, reject) => {
+    var time = Date.now()
+    var requestJSON
+
+    if (Array.isArray(method)) {
+      // multiple rpc batch call
+      requestJSON = []
+      method.forEach(function (batchCall, i) {
+        requestJSON.push({
+          id: time + '-' + i,
+          method: batchCall.method,
+          params: batchCall.params
+        })
+      })
+    } else {
+      // single rpc call
+      requestJSON = {
+        id: time,
+        method: method,
+        params: params
+      }
+    }
+
+    // First we encode the request into JSON
+    requestJSON = JSON.stringify(requestJSON)
+
+    // prepare request options
+    var requestOptions = {
+      host: this.opts.host || 'localhost',
+      port: this.opts.port || 8332,
+      method: 'POST',
+      path: '/',
+      headers: {
+        'Host': this.opts.host || 'localhost',
+        'Content-Length': requestJSON.length
+      },
+      agent: false,
+      rejectUnauthorized: this.opts.ssl && this.opts.sslStrict !== false
+    }
+
+    if (this.opts.ssl && this.opts.sslCa) {
+    // @ts-ignore 
+      requestOptions.ca = this.opts.sslCa
+    }
+
+    // use HTTP auth if user and password set
+    if (this.opts.user && this.opts.pass) {
+      // @ts-ignore
+      requestOptions.auth = this.opts.user + ':' + this.opts.pass
+    }
+
+    // Now we'll make a request to the server
+    var cbCalled = false
+    var request = this.http.request(requestOptions)
+
+    // start request timeout timer
+    var reqTimeout = setTimeout(function () {
+      if (cbCalled) return
+      cbCalled = true
+      request.abort()
+      var err = new Error('ETIMEDOUT')
+      // @ts-ignore
+      err.code = 'ETIMEDOUT'
+      reject(err)
+    }, this.opts.timeout || 30000)
+
+    // set additional timeout on socket in case of remote freeze after sending headers
+    request.setTimeout(this.opts.timeout || 30000, function () {
+      if (cbCalled) return
+      cbCalled = true
+      request.abort()
+      var err = new Error('ESOCKETTIMEDOUT')
+      // @ts-ignore
+      err.code = 'ESOCKETTIMEDOUT'
+      reject(err)
+    })
+
+    request.on('error', function (err) {
+      if (cbCalled) return
+      cbCalled = true
+      clearTimeout(reqTimeout)
+      reject(err)
+    })
+
+    request.on('response', function (response) {
+      clearTimeout(reqTimeout)
+
+      // We need to buffer the response chunks in a nonblocking way.
+      var buffer = ''
+      response.on('data', function (chunk) {
+        buffer = buffer + chunk
+      })
+      // When all the responses are finished, we decode the JSON and
+      // depending on whether it's got a result or an error, we call
+      // emitSuccess or emitError on the promise.
+      response.on('end', function () {
+        var err
+
+        if (cbCalled) return
+        cbCalled = true
+
+        try {
+          var decoded = JSON.parse(buffer)
+        } catch (e) {
+          if (response.statusCode !== 200) {
+            err = new Error('Invalid params, response status code: ' + response.statusCode)
+            err.code = -32602
+            reject(err)
+          } else {
+            err = new Error('Problem parsing JSON response from server')
+            err.code = -32603
+            reject(err)
+          }
+          return
+        }
+
+        if (!Array.isArray(decoded)) {
+          decoded = [decoded]
+        }
+
+        // iterate over each response, normally there will be just one
+        // unless a batch rpc call response is being processed
+        decoded.forEach(function (decodedResponse, i) {
+          if (decodedResponse.hasOwnProperty('error') && decodedResponse.error != null) {
+            if (reject) {
+              err = new Error(decodedResponse.error.message || '')
+              if (decodedResponse.error.code) {
+                err.code = decodedResponse.error.code
+              }
+              reject(err)
+            }
+          } else if (decodedResponse.hasOwnProperty('result')) {
+            // @ts-ignore
+            resolve(decodedResponse.result, response.headers)
+          } else {
+            if (reject) {
+              err = new Error(decodedResponse.error.message || '')
+              if (decodedResponse.error.code) {
+                err.code = decodedResponse.error.code
+              }
+              reject(err)
+            }
+          }
+        })
+      })
+    })
+    request.end(requestJSON);
+  });
+}
+
+module.exports.JsonRPC = JsonRPC

backend/src/sync-assets.ts

@@ -0,0 +1,85 @@
+import axios, { AxiosResponse } from 'axios';
+import * as fs from 'fs';
+import config from './config';
+import backendInfo from './api/backend-info';
+import logger from './logger';
+import { SocksProxyAgent } from 'socks-proxy-agent';
+
+const PATH = './';
+
+class SyncAssets {
+  constructor() { }
+
+  public async syncAssets$() {
+    for (const url of config.MEMPOOL.EXTERNAL_ASSETS) {
+      try {
+        await this.downloadFile$(url);
+      } catch (e) {
+        throw new Error(`Failed to download external asset. ` + (e instanceof Error ? e.message : e));
+      }
+    }
+  }
+
+  private async downloadFile$(url: string) {
+    return new Promise((resolve, reject) => {
+      const fileName = url.split('/').slice(-1)[0];
+
+      try {
+        if (config.SOCKS5PROXY.ENABLED) {
+          const socksOptions: any = {
+            agentOptions: {
+              keepAlive: true,
+            },
+            hostname: config.SOCKS5PROXY.HOST,
+            port: config.SOCKS5PROXY.PORT
+          };
+
+          if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
+            socksOptions.username = config.SOCKS5PROXY.USERNAME;
+            socksOptions.password = config.SOCKS5PROXY.PASSWORD;
+          }
+
+          const agent = new SocksProxyAgent(socksOptions);
+
+          logger.info(`Downloading external asset ${fileName} over the Tor network...`);
+          return axios.get(url, {
+            headers: {
+              'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
+            },
+            httpAgent: agent,
+            httpsAgent: agent,
+            responseType: 'stream',
+            timeout: 30000
+          }).then(function (response) {
+            const writer = fs.createWriteStream(PATH + fileName);
+            writer.on('finish', () => {
+              logger.info(`External asset ${fileName} saved to ${PATH + fileName}`);
+              resolve(0);
+            });
+            response.data.pipe(writer);
+          });
+        } else {
+          logger.info(`Downloading external asset ${fileName} over clearnet...`);
+          return axios.get(url, {
+            headers: {
+              'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
+            },
+            responseType: 'stream',
+            timeout: 30000
+          }).then(function (response) {
+            const writer = fs.createWriteStream(PATH + fileName);
+            writer.on('finish', () => {
+              logger.info(`External asset ${fileName} saved to ${PATH + fileName}`);
+              resolve(0);
+            });
+            response.data.pipe(writer);
+          });
+        }
+      } catch (e: any) {
+        reject(e);
+      }
+    });
+  }
+}
+
+export default new SyncAssets();

backend/src/tasks/pools-updater.ts

@@ -0,0 +1,175 @@
+import axios, { AxiosResponse } from 'axios';
+import poolsParser from '../api/pools-parser';
+import config from '../config';
+import DB from '../database';
+import backendInfo from '../api/backend-info';
+import logger from '../logger';
+import { SocksProxyAgent } from 'socks-proxy-agent';
+import * as https from 'https';
+
+/**
+ * Maintain the most recent version of pools.json
+ */
+class PoolsUpdater {
+  lastRun: number = 0;
+  currentSha: any = undefined;
+
+  constructor() {
+  }
+
+  public async updatePoolsJson() {
+    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
+      return;
+    }
+
+    const oneWeek = 604800;
+    const oneDay = 86400;
+
+    const now = new Date().getTime() / 1000;
+    if (now - this.lastRun < oneWeek) { // Execute the PoolsUpdate only once a week, or upon restart
+      return;
+    }
+
+    this.lastRun = now;
+
+    logger.info('Updating latest mining pools from Github');
+    if (config.SOCKS5PROXY.ENABLED) {
+      logger.info('List of public pools will be queried over the Tor network');
+    } else {
+      logger.info('List of public pools will be queried over clearnet');
+    }
+
+    try {
+      const githubSha = await this.fetchPoolsSha(); // Fetch pools.json sha from github
+      if (githubSha === undefined) {
+        return;
+      }
+
+      if (config.DATABASE.ENABLED === true) {
+        this.currentSha = await this.getShaFromDb();      
+      }
+
+      logger.debug(`Pools.json sha | Current: ${this.currentSha} | Github: ${githubSha}`);
+      if (this.currentSha !== undefined && this.currentSha === githubSha) {
+        return;
+      }
+
+      logger.warn('Pools.json is outdated, fetch latest from github');
+      const poolsJson = await this.query('https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json');
+      if (poolsJson === undefined) {
+        return;
+      }
+      await poolsParser.migratePoolsJson(poolsJson);
+      await this.updateDBSha(githubSha);
+      logger.notice('PoolsUpdater completed');
+
+    } catch (e) {
+      this.lastRun = now - (oneWeek - oneDay); // Try again in 24h instead of waiting next week
+      logger.err('PoolsUpdater failed. Will try again in 24h. Reason: '  + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  /**
+   * Fetch our latest pools.json sha from the db
+   */
+  private async updateDBSha(githubSha: string) {
+    this.currentSha = githubSha;
+    if (config.DATABASE.ENABLED === true) {
+      try {
+        await DB.query('DELETE FROM state where name="pools_json_sha"');
+        await DB.query(`INSERT INTO state VALUES('pools_json_sha', NULL, '${githubSha}')`);
+      } catch (e) {
+        logger.err('Cannot save github pools.json sha into the db. Reason: '  + (e instanceof Error ? e.message : e));
+      }
+    }
+  }
+
+  /**
+   * Fetch our latest pools.json sha from the db
+   */
+  private async getShaFromDb(): Promise<string | undefined> {
+    try {
+      const [rows]: any[] = await DB.query('SELECT string FROM state WHERE name="pools_json_sha"');
+      return (rows.length > 0 ? rows[0].string : undefined);
+    } catch (e) {
+      logger.err('Cannot fetch pools.json sha from db. Reason: '  + (e instanceof Error ? e.message : e));
+      return undefined;
+    }
+  }
+
+  /**
+   * Fetch our latest pools.json sha from github
+   */
+  private async fetchPoolsSha(): Promise<string | undefined> {
+    const response = await this.query('https://api.github.com/repos/mempool/mining-pools/git/trees/master');
+
+    if (response !== undefined) {
+      for (const file of response['tree']) {
+        if (file['path'] === 'pools.json') {
+          return file['sha'];
+        }
+      }
+    }
+
+    logger.err('Cannot to find latest pools.json sha from github api response');
+    return undefined;
+  }
+
+  /**
+   * Http request wrapper
+   */
+  private async query(path): Promise<object | undefined> {
+    type axiosOptions = {
+      headers: {
+        'User-Agent': string
+      };
+      timeout: number;
+      httpsAgent?: https.Agent;
+    }
+    const setDelay = (secs: number = 1): Promise<void> => new Promise(resolve => setTimeout(() => resolve(), secs * 1000));
+    const axiosOptions: axiosOptions = {
+      headers: {
+        'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
+      },
+      timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000
+    };
+    let retry = 0;
+
+    while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
+      try {
+        if (config.SOCKS5PROXY.ENABLED) {
+          const socksOptions: any = {
+            agentOptions: {
+              keepAlive: true,
+            },
+            hostname: config.SOCKS5PROXY.HOST,
+            port: config.SOCKS5PROXY.PORT
+          };
+
+          if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
+            socksOptions.username = config.SOCKS5PROXY.USERNAME;
+            socksOptions.password = config.SOCKS5PROXY.PASSWORD;
+          } else {
+            // Retry with different tor circuits https://stackoverflow.com/a/64960234
+            socksOptions.username = `circuit${retry}`;
+          }
+
+          axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions);
+        }
+        
+        const data: AxiosResponse = await axios.get(path, axiosOptions);
+        if (data.statusText === 'error' || !data.data) {
+          throw new Error(`Could not fetch data from Github, Error: ${data.status}`);
+        }
+        return data.data;
+      } catch (e) {
+        logger.err('Could not connect to Github. Reason: '  + (e instanceof Error ? e.message : e));
+        retry++;
+      }
+      await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL);
+    }
+    return undefined;
+  }
+}
+
+export default new PoolsUpdater();

backend/src/utils/blocks-utils.ts

@@ -0,0 +1,32 @@
+import { BlockExtended } from '../mempool.interfaces';
+
+export function prepareBlock(block: any): BlockExtended {
+  return <BlockExtended>{
+    id: block.id ?? block.hash, // hash for indexed block
+    timestamp: block.timestamp ?? block.blockTimestamp, // blockTimestamp for indexed block
+    height: block.height,
+    version: block.version,
+    bits: block.bits,
+    nonce: block.nonce,
+    difficulty: block.difficulty,
+    merkle_root: block.merkle_root,
+    tx_count: block.tx_count,
+    size: block.size,
+    weight: block.weight,
+    previousblockhash: block.previousblockhash,
+    extras: {
+      coinbaseRaw: block.coinbase_raw ?? block.extras?.coinbaseRaw,
+      medianFee: block.medianFee ?? block.median_fee ?? block.extras?.medianFee,
+      feeRange: block.feeRange ?? block.fee_span,
+      reward: block.reward ?? block?.extras?.reward,
+      totalFees: block.totalFees ?? block?.fees ?? block?.extras?.totalFees,
+      avgFee: block?.extras?.avgFee ?? block.avg_fee,
+      avgFeeRate: block?.avgFeeRate ?? block.avg_fee_rate,
+      pool: block?.extras?.pool ?? (block?.pool_id ? {
+        id: block.pool_id,
+        name: block.pool_name,
+        slug: block.pool_slug,
+      } : undefined),
+    }
+  };
+}

backend/tsconfig.json

@@ -1,7 +1,8 @@
 {
   "compilerOptions": {
     "module": "commonjs",
-    "target": "es2015",
+    "target": "esnext",
+    "lib": ["es2019", "dom"],
     "strict": true,
     "noImplicitAny": false,
     "sourceMap": false,
@@ -9,7 +10,8 @@
     "moduleResolution": "node",
     "typeRoots": [
       "node_modules/@types"
-    ]
+    ],
+    "allowSyntheticDefaultImports": true
   },
   "include": [
     "src/**/*.ts"

backend/tslint.json

@@ -12,7 +12,7 @@
       "severity": "warn"
     },
     "eofline": true,
-    "forin": true,
+    "forin": false,
     "import-blacklist": [
       true,
       "rxjs",

contributors/TechMiX.txt

@@ -0,0 +1,3 @@
+I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of January 25, 2022.
+
+Signed: TechMiX

contributors/antonilol.txt

@@ -0,0 +1,3 @@
+I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of January 25, 2022.
+
+Signed: antonilol

contributors/ayanamidev.txt

@@ -0,0 +1,3 @@
+I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of May 15, 2022.
+
+Signed: ayanamidev

contributors/bosch-0.txt

@@ -0,0 +1,3 @@
+I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of January 25, 2022.
+
+Signed: Bosch-0

contributors/dsbaars.txt

@@ -0,0 +1,3 @@
+I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of January 25, 2022.
+
+Signed: dsbaars

contributors/emzy.txt

@@ -0,0 +1 @@
+Mempool Space K.K. has a signed CLA or other agreement on file with @emzy as of January 25, 2022

contributors/hunicus.txt

@@ -0,0 +1 @@
+Mempool Space K.K. has a signed CLA or other agreement on file with @hunicus as of January 25, 2022

contributors/knorrium.txt

@@ -0,0 +1 @@
+Mempool Space K.K. has a signed CLA or other agreement on file with @knorrium as of January 25, 2022

contributors/miguelmedeiros.txt

@@ -0,0 +1 @@
+Mempool Space K.K. has a signed CLA or other agreement on file with @miguelmedeiros as of January 25, 2022

contributors/mononaut.txt

@@ -0,0 +1,3 @@
+I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of May 31, 2022.
+
+Signed: mononaut

contributors/naveensrinivasan.txt

@@ -0,0 +1,3 @@
+I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of March 11, 2022.
+
+Signed: naveensrinivasan

contributors/nymkappa.txt

@@ -0,0 +1 @@
+Mempool Space K.K. has a signed CLA or other agreement on file with @nymkappa as of January 25, 2022

contributors/softsimon.txt

@@ -0,0 +1,3 @@
+I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of January 25, 2022.
+
+Signed: softsimon

contributors/wiz.txt

@@ -0,0 +1,3 @@
+I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of January 25, 2022.
+
+Signed: wiz