Merge branch 'master' into simon/block-error-code

.github/workflows/ci.yml

@@ -0,0 +1,48 @@
+name: CI Pipeline for the Backend and Frontend
+on:
+  push:
+env:
+  NODE_VERSION: 16.15.0
+jobs:
+  build_backend:
+    name: Build backend
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Setup Node
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+          registry-url: 'https://registry.npmjs.org'
+      - name: Install 
+        run: npm install --prod
+        working-directory: backend
+    #  - name: Lint 
+    #    run: npm run lint
+    #  - name: Test 
+    #    run: npm run test
+      - name: Build
+        run: npm run build
+        working-directory: backend
+  build_frontend:
+    name: Build frontend
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Setup Node
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+          registry-url: 'https://registry.npmjs.org'
+      - name: Install 
+        run: npm install --prod
+        working-directory: frontend
+      # - name: Lint 
+      #   run: npm run lint
+      # - name: Test 
+      #   run: npm run test
+      - name: Build
+        run: npm run build
+        working-directory: frontend

.github/workflows/cypress.yml

@@ -1,8 +1,11 @@
 name: Cypress Tests
 
-on: [push, pull_request]
+on:
+  pull_request:
+    types: [ opened, review_requested, synchronize ]
 jobs:
   cypress:
+    if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')"
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false

backend/package-lock.json

@@ -10,6 +10,7 @@
       "license": "GNU Affero General Public License v3.0",
       "dependencies": {
         "@mempool/electrum-client": "^1.1.7",
+        "@types/node": "^16.11.41",
         "axios": "~0.27.2",
         "bitcoinjs-lib": "6.0.1",
         "crypto-js": "^4.0.0",
@@ -119,10 +120,9 @@
       "dev": true
     },
     "node_modules/@types/node": {
-      "version": "14.14.20",
+      "version": "16.11.41",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.20.tgz",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.41.tgz",
-      "integrity": "sha512-Y93R97Ouif9JEOWPIUyU+eyIdyRqQR0I8Ez1dzku4hDx34NWh4HbtIc3WNzwB1Y9ULvNGeu5B8h8bVL5cAk4/A==",
+      "integrity": "sha512-mqoYK2TnVjdkGk8qXAVGc/x9nSaTpSrFaGFm43BUH3IdoBV0nta6hYaGmdOvIMlbHJbUEVen3gvwpwovAZKNdQ=="
-      "dev": true
     },
     "node_modules/@types/qs": {
       "version": "6.9.7",
@@ -1647,10 +1647,9 @@
       "dev": true
     },
     "@types/node": {
-      "version": "14.14.20",
+      "version": "16.11.41",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.20.tgz",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.41.tgz",
-      "integrity": "sha512-Y93R97Ouif9JEOWPIUyU+eyIdyRqQR0I8Ez1dzku4hDx34NWh4HbtIc3WNzwB1Y9ULvNGeu5B8h8bVL5cAk4/A==",
+      "integrity": "sha512-mqoYK2TnVjdkGk8qXAVGc/x9nSaTpSrFaGFm43BUH3IdoBV0nta6hYaGmdOvIMlbHJbUEVen3gvwpwovAZKNdQ=="
-      "dev": true
     },
     "@types/qs": {
       "version": "6.9.7",

backend/package.json

@@ -20,7 +20,6 @@
   ],
   "main": "index.ts",
   "scripts": {
-    "ng": "./node_modules/@angular/cli/bin/ng",
     "tsc": "./node_modules/typescript/bin/tsc",
     "build": "npm run tsc",
     "start": "node --max-old-space-size=2048 dist/index.js",
@@ -29,6 +28,7 @@
   },
   "dependencies": {
     "@mempool/electrum-client": "^1.1.7",
+    "@types/node": "^16.11.41",
     "axios": "~0.27.2",
     "bitcoinjs-lib": "6.0.1",
     "crypto-js": "^4.0.0",
@@ -41,8 +41,8 @@
   },
   "devDependencies": {
     "@types/compression": "^1.7.2",
-    "@types/ws": "~8.5.3",
     "@types/express": "^4.17.13",
+    "@types/ws": "~8.5.3",
     "tslint": "^6.1.0"
   }
 }

backend/src/api/bitcoin/electrum-api.ts

@@ -1,10 +1,10 @@
 import config from '../../config';
+import Client from '@mempool/electrum-client';
 import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
 import { IEsploraApi } from './esplora-api.interface';
 import { IElectrumApi } from './electrum-api.interface';
 import BitcoinApi from './bitcoin-api';
 import logger from '../../logger';
-import * as ElectrumClient from '@mempool/electrum-client';
 import * as sha256 from 'crypto-js/sha256';
 import * as hexEnc from 'crypto-js/enc-hex';
 import loadingIndicators from '../loading-indicators';
@@ -26,7 +26,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
       onLog: (str) => { logger.debug(str); },
     };
 
-    this.electrumClient = new ElectrumClient(
+    this.electrumClient = new Client(
       config.ELECTRUM.PORT,
       config.ELECTRUM.HOST,
       config.ELECTRUM.TLS_ENABLED ? 'tls' : 'tcp',
@@ -35,7 +35,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
     );
 
     this.electrumClient.initElectrum(electrumConfig, electrumPersistencePolicy)
-      .then(() => {})
+      .then(() => { })
       .catch((err) => {
         logger.err(`Error connecting to Electrum Server at ${config.ELECTRUM.HOST}:${config.ELECTRUM.PORT}`);
       });

backend/src/api/disk-cache.ts

@@ -1,6 +1,6 @@
 import * as fs from 'fs';
 const fsPromises = fs.promises;
-import * as cluster from 'cluster';
+import cluster from 'cluster';
 import memPool from './mempool';
 import blocks from './blocks';
 import logger from '../logger';
@@ -19,7 +19,7 @@ class DiskCache {
   constructor() { }
 
   async $saveCacheToDisk(): Promise<void> {
-    if (!cluster.isMaster) {
+    if (!cluster.isPrimary) {
       return;
     }
     if (this.isWritingCache) {
@@ -46,12 +46,12 @@ class DiskCache {
         blockSummaries: blocks.getBlockSummaries(),
         mempool: {},
         mempoolArray: mempoolArray.splice(0, chunkSize),
-      }), {flag: 'w'});
+      }), { flag: 'w' });
       for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
         await fsPromises.writeFile(DiskCache.FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
           mempool: {},
           mempoolArray: mempoolArray.splice(0, chunkSize),
-        }), {flag: 'w'});
+        }), { flag: 'w' });
       }
       logger.debug('Mempool and blocks data saved to disk cache');
       this.isWritingCache = false;

backend/src/api/websocket-handler.ts

@@ -1,7 +1,9 @@
 import logger from '../logger';
 import * as WebSocket from 'ws';
-import { BlockExtended, TransactionExtended, WebsocketResponse, MempoolBlock, MempoolBlockDelta,
+import {
-  OptimizedStatistic, ILoadingIndicators, IConversionRates } from '../mempool.interfaces';
+  BlockExtended, TransactionExtended, WebsocketResponse, MempoolBlock, MempoolBlockDelta,
+  OptimizedStatistic, ILoadingIndicators, IConversionRates
+} from '../mempool.interfaces';
 import blocks from './blocks';
 import memPool from './mempool';
 import backendInfo from './backend-info';
@@ -164,7 +166,7 @@ class WebsocketHandler {
       throw new Error('WebSocket.Server is not set');
     }
 
-    this.wss.clients.forEach((client: WebSocket) => {
+    this.wss.clients.forEach((client) => {
       if (client.readyState !== WebSocket.OPEN) {
         return;
       }
@@ -179,7 +181,7 @@ class WebsocketHandler {
       throw new Error('WebSocket.Server is not set');
     }
 
-    this.wss.clients.forEach((client: WebSocket) => {
+    this.wss.clients.forEach((client) => {
       if (client.readyState !== WebSocket.OPEN) {
         return;
       }
@@ -192,7 +194,7 @@ class WebsocketHandler {
       throw new Error('WebSocket.Server is not set');
     }
 
-    this.wss.clients.forEach((client: WebSocket) => {
+    this.wss.clients.forEach((client) => {
       if (client.readyState !== WebSocket.OPEN) {
         return;
       }
@@ -224,7 +226,7 @@ class WebsocketHandler {
       throw new Error('WebSocket.Server is not set');
     }
 
-    this.wss.clients.forEach((client: WebSocket) => {
+    this.wss.clients.forEach((client) => {
       if (client.readyState !== WebSocket.OPEN) {
         return;
       }
@@ -255,7 +257,7 @@ class WebsocketHandler {
     memPool.handleRbfTransactions(rbfTransactions);
     const recommendedFees = feeApi.getRecommendedFee();
 
-    this.wss.clients.forEach(async (client: WebSocket) => {
+    this.wss.clients.forEach(async (client) => {
       if (client.readyState !== WebSocket.OPEN) {
         return;
       }

backend/src/index.ts

@@ -1,8 +1,8 @@
-import { Express, Request, Response, NextFunction } from 'express';
+import express from "express";
-import * as express from 'express';
+import { Application, Request, Response, NextFunction, Express } from 'express';
 import * as http from 'http';
 import * as WebSocket from 'ws';
-import * as cluster from 'cluster';
+import cluster from 'cluster';
 import axios from 'axios';
 
 import DB from './database';
@@ -32,7 +32,7 @@ import priceUpdater from './tasks/price-updater';
 class Server {
   private wss: WebSocket.Server | undefined;
   private server: http.Server | undefined;
-  private app: Express;
+  private app: Application;
   private currentBackendRetryInterval = 5;
 
   constructor() {
@@ -43,7 +43,7 @@ class Server {
       return;
     }
 
-    if (cluster.isMaster) {
+    if (cluster.isPrimary) {
       logger.notice(`Mempool Server (Master) is running on port ${config.MEMPOOL.HTTP_PORT} (${backendInfo.getShortCommitHash()})`);
 
       const numCPUs = config.MEMPOOL.SPAWN_CLUSTER_PROCS;
@@ -105,7 +105,7 @@ class Server {
       }
     }
 
-    if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED && cluster.isMaster) {
+    if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED && cluster.isPrimary) {
       statistics.startStatistics();
     }
 

backend/tsconfig.json

@@ -1,5 +1,6 @@
 {
   "compilerOptions": {
+    "types": ["node"],
     "module": "commonjs",
     "target": "esnext",
     "lib": ["es2019", "dom"],
@@ -11,7 +12,8 @@
     "typeRoots": [
       "node_modules/@types"
     ],
-    "allowSyntheticDefaultImports": true
+    "allowSyntheticDefaultImports": true,
+    "esModuleInterop": true
   },
   "include": [
     "src/**/*.ts"

frontend/package.json

@@ -20,20 +20,20 @@
   ],
   "main": "index.ts",
   "scripts": {
-    "ng": "./node_modules/@angular/cli/bin/ng",
+    "ng": "./node_modules/@angular/cli/bin/ng.js",
     "tsc": "./node_modules/typescript/bin/tsc",
     "i18n-extract-from-source": "./node_modules/@angular/cli/bin/ng extract-i18n --out-file ./src/locale/messages.xlf",
     "i18n-pull-from-transifex": "tx pull -a --parallel --minimum-perc 1 --force",
-    "serve": "npm run generate-config && ng serve -c local",
+    "serve": "npm run generate-config && npm run ng -- serve -c local",
-    "serve:stg": "npm run generate-config && ng serve -c staging",
+    "serve:stg": "npm run generate-config && npm run ng -- serve -c staging",
-    "serve:local-prod": "npm run generate-config && ng serve -c local-prod",
+    "serve:local-prod": "npm run generate-config && npm run ng -- serve -c local-prod",
-    "serve:local-staging": "npm run generate-config && ng serve -c local-staging",
+    "serve:local-staging": "npm run generate-config && npm run ng -- serve -c local-staging",
-    "start": "npm run generate-config && npm run sync-assets-dev && ng serve -c local",
+    "start": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c local",
-    "start:stg": "npm run generate-config && npm run sync-assets-dev && ng serve -c staging",
+    "start:stg": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c staging",
-    "start:local-prod": "npm run generate-config && npm run sync-assets-dev && ng serve -c local-prod",
+    "start:local-prod": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c local-prod",
-    "start:local-staging": "npm run generate-config && npm run sync-assets-dev && ng serve -c local-staging",
+    "start:local-staging": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c local-staging",
-    "start:mixed": "npm run generate-config && npm run sync-assets-dev && ng serve -c mixed",
+    "start:mixed": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c mixed",
-    "build": "npm run generate-config && ng build --configuration production --localize && npm run sync-assets && npm run build-mempool.js",
+    "build": "npm run generate-config && npm run ng -- build --configuration production --localize && npm run sync-assets && npm run build-mempool.js",
     "sync-assets": "node sync-assets.js && rsync -av ./dist/mempool/browser/en-US/resources ./dist/mempool/browser/resources",
     "sync-assets-dev": "node sync-assets.js dev",
     "generate-config": "node generate-config.js",
@@ -41,17 +41,17 @@
     "build-mempool-js": "browserify -p tinyify ./node_modules/@mempool/mempool.js/lib/index.js --standalone mempoolJS > ./dist/mempool/browser/en-US/mempool.js",
     "build-mempool-bisq-js": "browserify -p tinyify ./node_modules/@mempool/mempool.js/lib/index-bisq.js --standalone bisqJS > ./dist/mempool/browser/en-US/bisq.js",
     "build-mempool-liquid-js": "browserify -p tinyify ./node_modules/@mempool/mempool.js/lib/index-liquid.js --standalone liquidJS > ./dist/mempool/browser/en-US/liquid.js",
-    "test": "ng test",
+    "test": "npm run ng -- test",
-    "lint": "ng lint",
+    "lint": "npm run ng -- lint",
-    "e2e": "npm run generate-config && ng e2e",
+    "e2e": "npm run generate-config && npm run ng -- e2e",
     "e2e:ci": "npm run cypress:run:ci",
     "config:defaults:mempool": "node update-config.js TESTNET_ENABLED=true SIGNET_ENABLED=true LIQUID_ENABLED=true LIQUID_TESTNET_ENABLED=true BISQ_ENABLED=true ITEMS_PER_PAGE=25 BASE_MODULE=mempool BLOCK_WEIGHT_UNITS=4000000 && npm run generate-config",
     "config:defaults:liquid": "node update-config.js TESTNET_ENABLED=true SIGNET_ENABLED=true LIQUID_ENABLED=true LIQUID_TESTNET_ENABLED=true BISQ_ENABLED=true ITEMS_PER_PAGE=25 BASE_MODULE=liquid BLOCK_WEIGHT_UNITS=300000 && npm run generate-config",
     "config:defaults:bisq": "node update-config.js TESTNET_ENABLED=true SIGNET_ENABLED=true LIQUID_ENABLED=true BISQ_ENABLED=true ITEMS_PER_PAGE=25 BASE_MODULE=bisq BLOCK_WEIGHT_UNITS=4000000 && npm run generate-config",
-    "dev:ssr": "npm run generate-config && ng run mempool:serve-ssr",
+    "dev:ssr": "npm run generate-config && npm run ng -- run mempool:serve-ssr",
     "serve:ssr": "node server.run.js",
-    "build:ssr": "npm run build && ng run mempool:server:production && ./node_modules/typescript/bin/tsc server.run.ts",
+    "build:ssr": "npm run build && npm run ng -- run mempool:server:production && npm run tsc -- server.run.ts",
-    "prerender": "ng run mempool:prerender",
+    "prerender": "npm run ng -- run mempool:prerender",
     "cypress:open": "cypress open",
     "cypress:run": "cypress run",
     "cypress:run:record": "cypress run --record",

production/bitcoin.conf

@@ -22,6 +22,8 @@ bind=0.0.0.0:8333
 bind=[::]:8333
 rpcbind=127.0.0.1:8332
 rpcbind=[::1]:8332
+zmqpubrawblock=tcp://127.0.0.1:18332
+zmqpubrawtx=tcp://127.0.0.1:18333
 #addnode=[2401:b140:2::92:201]:8333
 #addnode=[2401:b140:2::92:202]:8333
 #addnode=[2401:b140:2::92:203]:8333

production/install

@@ -282,7 +282,7 @@ BITCOIN_REPO_URL=https://github.com/bitcoin/bitcoin
 BITCOIN_REPO_NAME=bitcoin
 BITCOIN_REPO_BRANCH=master
 #BITCOIN_LATEST_RELEASE=$(curl -s https://api.github.com/repos/bitcoin/bitcoin/releases/latest|grep tag_name|head -1|cut -d '"' -f4)
-BITCOIN_LATEST_RELEASE=v22.0
+BITCOIN_LATEST_RELEASE=v23.0
 echo -n '.'
 
 BISQ_REPO_URL=https://github.com/bisq-network/bisq
@@ -330,7 +330,7 @@ DEBIAN_PKG+=(nodejs npm mariadb-server nginx-core python-certbot-nginx rsync ufw
 # packages needed for mempool ecosystem
 FREEBSD_PKG=()
 FREEBSD_PKG+=(zsh sudo git screen curl wget calc neovim)
-FREEBSD_PKG+=(openssh-portable py38-pip rust llvm90 jq base64)
+FREEBSD_PKG+=(openssh-portable py38-pip rust llvm90 jq base64 libzmq4)
 FREEBSD_PKG+=(boost-libs autoconf automake gmake gcc libevent libtool pkgconf)
 FREEBSD_PKG+=(nginx rsync py38-certbot-nginx mariadb105-server keybase)