machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into simon/block-error-code

Browse Source
This commit is contained in:
Felipe Knorr Kuhn 2022-07-04 09:21:03 -07:00 committed by GitHub
commit 32cd93b689
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 115 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
.github/workflows/ci.yml vendored Normal file
View File

@ -0,0 +1,48 @@
name: CI Pipeline for the Backend and Frontend
on:
push:
env:
NODE_VERSION: 16.15.0
jobs:
build_backend:
name: Build backend
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Setup Node
uses: actions/setup-node@v3
with:
node-version: ${{ env.NODE_VERSION }}
registry-url: 'https://registry.npmjs.org'
- name: Install
run: npm install --prod
working-directory: backend
# - name: Lint
# run: npm run lint
# - name: Test
# run: npm run test
- name: Build
run: npm run build
working-directory: backend
build_frontend:
name: Build frontend
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Setup Node
uses: actions/setup-node@v3
with:
node-version: ${{ env.NODE_VERSION }}
registry-url: 'https://registry.npmjs.org'
- name: Install
run: npm install --prod
working-directory: frontend
# - name: Lint
# run: npm run lint
# - name: Test
# run: npm run test
- name: Build
run: npm run build
working-directory: frontend

5
.github/workflows/cypress.yml vendored
View File

@ -1,8 +1,11 @@
name: Cypress Tests
on: [push, pull_request]
on:
pull_request:
types: [ opened, review_requested, synchronize ]
jobs:
cypress:
if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')"
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false

15
backend/package-lock.json generated
View File

@ -10,6 +10,7 @@
"license": "GNU Affero General Public License v3.0",
"dependencies": {
"@mempool/electrum-client": "^1.1.7",
"@types/node": "^16.11.41",
"axios": "~0.27.2",
"bitcoinjs-lib": "6.0.1",
"crypto-js": "^4.0.0",
@ -119,10 +120,9 @@
"dev": true
},
"node_modules/@types/node": {
"version": "14.14.20",
"resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.20.tgz",
"integrity": "sha512-Y93R97Ouif9JEOWPIUyU+eyIdyRqQR0I8Ez1dzku4hDx34NWh4HbtIc3WNzwB1Y9ULvNGeu5B8h8bVL5cAk4/A==",
"dev": true
"version": "16.11.41",
"resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.41.tgz",
"integrity": "sha512-mqoYK2TnVjdkGk8qXAVGc/x9nSaTpSrFaGFm43BUH3IdoBV0nta6hYaGmdOvIMlbHJbUEVen3gvwpwovAZKNdQ=="
},
"node_modules/@types/qs": {
"version": "6.9.7",
@ -1647,10 +1647,9 @@
"dev": true
},
"@types/node": {
"version": "14.14.20",
"resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.20.tgz",
"integrity": "sha512-Y93R97Ouif9JEOWPIUyU+eyIdyRqQR0I8Ez1dzku4hDx34NWh4HbtIc3WNzwB1Y9ULvNGeu5B8h8bVL5cAk4/A==",
"dev": true
"version": "16.11.41",
"resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.41.tgz",
"integrity": "sha512-mqoYK2TnVjdkGk8qXAVGc/x9nSaTpSrFaGFm43BUH3IdoBV0nta6hYaGmdOvIMlbHJbUEVen3gvwpwovAZKNdQ=="
},
"@types/qs": {
"version": "6.9.7",

4
backend/package.json
View File

@ -20,7 +20,6 @@
],
"main": "index.ts",
"scripts": {
"ng": "./node_modules/@angular/cli/bin/ng",
"tsc": "./node_modules/typescript/bin/tsc",
"build": "npm run tsc",
"start": "node --max-old-space-size=2048 dist/index.js",
@ -29,6 +28,7 @@
},
"dependencies": {
"@mempool/electrum-client": "^1.1.7",
"@types/node": "^16.11.41",
"axios": "~0.27.2",
"bitcoinjs-lib": "6.0.1",
"crypto-js": "^4.0.0",
@ -41,8 +41,8 @@
},
"devDependencies": {
"@types/compression": "^1.7.2",
"@types/ws": "~8.5.3",
"@types/express": "^4.17.13",
"@types/ws": "~8.5.3",
"tslint": "^6.1.0"
}
}

8
backend/src/api/bitcoin/electrum-api.ts
View File

@ -1,10 +1,10 @@
import config from '../../config';
import Client from '@mempool/electrum-client';
import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
import { IEsploraApi } from './esplora-api.interface';
import { IElectrumApi } from './electrum-api.interface';
import BitcoinApi from './bitcoin-api';
import logger from '../../logger';
import * as ElectrumClient from '@mempool/electrum-client';
import * as sha256 from 'crypto-js/sha256';
import * as hexEnc from 'crypto-js/enc-hex';
import loadingIndicators from '../loading-indicators';
@ -26,7 +26,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
onLog: (str) => { logger.debug(str); },
};
this.electrumClient = new ElectrumClient(
this.electrumClient = new Client(
config.ELECTRUM.PORT,
config.ELECTRUM.HOST,
config.ELECTRUM.TLS_ENABLED ? 'tls' : 'tcp',
@ -35,7 +35,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
);
this.electrumClient.initElectrum(electrumConfig, electrumPersistencePolicy)
.then(() => {})
.then(() => { })
.catch((err) => {
logger.err(`Error connecting to Electrum Server at ${config.ELECTRUM.HOST}:${config.ELECTRUM.PORT}`);
});
@ -95,7 +95,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
async $getAddressTransactions(address: string, lastSeenTxId: string): Promise<IEsploraApi.Transaction[]> {
const addressInfo = await this.bitcoindClient.validateAddress(address);
if (!addressInfo || !addressInfo.isvalid) {
return [];
return [];
}
try {

10
backend/src/api/disk-cache.ts
View File

@ -1,6 +1,6 @@
import * as fs from 'fs';
const fsPromises = fs.promises;
import * as cluster from 'cluster';
import cluster from 'cluster';
import memPool from './mempool';
import blocks from './blocks';
import logger from '../logger';
@ -19,7 +19,7 @@ class DiskCache {
constructor() { }
async $saveCacheToDisk(): Promise<void> {
if (!cluster.isMaster) {
if (!cluster.isPrimary) {
return;
}
if (this.isWritingCache) {
@ -46,12 +46,12 @@ class DiskCache {
blockSummaries: blocks.getBlockSummaries(),
mempool: {},
mempoolArray: mempoolArray.splice(0, chunkSize),
}), {flag: 'w'});
}), { flag: 'w' });
for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
await fsPromises.writeFile(DiskCache.FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
mempool: {},
mempoolArray: mempoolArray.splice(0, chunkSize),
}), {flag: 'w'});
}), { flag: 'w' });
}
logger.debug('Mempool and blocks data saved to disk cache');
this.isWritingCache = false;
@ -67,7 +67,7 @@ class DiskCache {
fs.unlinkSync(DiskCache.FILE_NAMES.replace('{number}', i.toString()));
}
}
loadMempoolCache() {
if (!fs.existsSync(DiskCache.FILE_NAME)) {
return;

16
backend/src/api/websocket-handler.ts
View File

@ -1,7 +1,9 @@
import logger from '../logger';
import * as WebSocket from 'ws';
import { BlockExtended, TransactionExtended, WebsocketResponse, MempoolBlock, MempoolBlockDelta,
OptimizedStatistic, ILoadingIndicators, IConversionRates } from '../mempool.interfaces';
import {
BlockExtended, TransactionExtended, WebsocketResponse, MempoolBlock, MempoolBlockDelta,
OptimizedStatistic, ILoadingIndicators, IConversionRates
} from '../mempool.interfaces';
import blocks from './blocks';
import memPool from './mempool';
import backendInfo from './backend-info';
@ -164,7 +166,7 @@ class WebsocketHandler {
throw new Error('WebSocket.Server is not set');
}
this.wss.clients.forEach((client: WebSocket) => {
this.wss.clients.forEach((client) => {
if (client.readyState !== WebSocket.OPEN) {
return;
}
@ -179,7 +181,7 @@ class WebsocketHandler {
throw new Error('WebSocket.Server is not set');
}
this.wss.clients.forEach((client: WebSocket) => {
this.wss.clients.forEach((client) => {
if (client.readyState !== WebSocket.OPEN) {
return;
}
@ -192,7 +194,7 @@ class WebsocketHandler {
throw new Error('WebSocket.Server is not set');
}
this.wss.clients.forEach((client: WebSocket) => {
this.wss.clients.forEach((client) => {
if (client.readyState !== WebSocket.OPEN) {
return;
}
@ -224,7 +226,7 @@ class WebsocketHandler {
throw new Error('WebSocket.Server is not set');
}
this.wss.clients.forEach((client: WebSocket) => {
this.wss.clients.forEach((client) => {
if (client.readyState !== WebSocket.OPEN) {
return;
}
@ -255,7 +257,7 @@ class WebsocketHandler {
memPool.handleRbfTransactions(rbfTransactions);
const recommendedFees = feeApi.getRecommendedFee();
this.wss.clients.forEach(async (client: WebSocket) => {
this.wss.clients.forEach(async (client) => {
if (client.readyState !== WebSocket.OPEN) {
return;
}

24
backend/src/index.ts
View File

@ -1,8 +1,8 @@
import { Express, Request, Response, NextFunction } from 'express';
import * as express from 'express';
import express from "express";
import { Application, Request, Response, NextFunction, Express } from 'express';
import * as http from 'http';
import * as WebSocket from 'ws';
import * as cluster from 'cluster';
import cluster from 'cluster';
import axios from 'axios';
import DB from './database';
@ -32,7 +32,7 @@ import priceUpdater from './tasks/price-updater';
class Server {
private wss: WebSocket.Server | undefined;
private server: http.Server | undefined;
private app: Express;
private app: Application;
private currentBackendRetryInterval = 5;
constructor() {
@ -43,7 +43,7 @@ class Server {
return;
}
if (cluster.isMaster) {
if (cluster.isPrimary) {
logger.notice(`Mempool Server (Master) is running on port ${config.MEMPOOL.HTTP_PORT} (${backendInfo.getShortCommitHash()})`);
const numCPUs = config.MEMPOOL.SPAWN_CLUSTER_PROCS;
@ -77,7 +77,7 @@ class Server {
})
.use(express.urlencoded({ extended: true }))
.use(express.text())
;
;
this.server = http.createServer(this.app);
this.wss = new WebSocket.Server({ server: this.server });
@ -105,7 +105,7 @@ class Server {
}
}
if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED && cluster.isMaster) {
if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED && cluster.isPrimary) {
statistics.startStatistics();
}
@ -260,7 +260,7 @@ class Server {
res.status(500).end();
}
})
;
;
if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED) {
this.app
@ -290,7 +290,7 @@ class Server {
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/rewards/:interval', routes.$getHistoricalBlockRewards)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/fee-rates/:interval', routes.$getHistoricalBlockFeeRates)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/sizes-weights/:interval', routes.$getHistoricalBlockSizeAndWeight)
;
;
}
if (config.BISQ.ENABLED) {
@ -340,7 +340,7 @@ class Server {
.get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs', routes.getAddressTransactions)
.get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs/chain/:txId', routes.getAddressTransactions)
.get(config.MEMPOOL.API_URL_PREFIX + 'address-prefix/:prefix', routes.getAddressPrefix)
;
;
}
if (Common.isLiquid()) {
@ -349,13 +349,13 @@ class Server {
.get(config.MEMPOOL.API_URL_PREFIX + 'assets/featured', routes.$getAllFeaturedLiquidAssets)
.get(config.MEMPOOL.API_URL_PREFIX + 'asset/:assetId/icon', routes.getLiquidIcon)
.get(config.MEMPOOL.API_URL_PREFIX + 'assets/group/:id', routes.$getAssetGroup)
;
;
}
if (Common.isLiquid() && config.DATABASE.ENABLED) {
this.app
.get(config.MEMPOOL.API_URL_PREFIX + 'liquid/pegs/month', routes.$getElementsPegsByMonth)
;
;
}
}
}

4
backend/tsconfig.json
View File

@ -1,5 +1,6 @@
{
"compilerOptions": {
"types": ["node"],
"module": "commonjs",
"target": "esnext",
"lib": ["es2019", "dom"],
@ -11,7 +12,8 @@
"typeRoots": [
"node_modules/@types"
],
"allowSyntheticDefaultImports": true
"allowSyntheticDefaultImports": true,
"esModuleInterop": true
},
"include": [
"src/**/*.ts"

34
frontend/package.json
View File

@ -20,20 +20,20 @@
],
"main": "index.ts",
"scripts": {
"ng": "./node_modules/@angular/cli/bin/ng",
"ng": "./node_modules/@angular/cli/bin/ng.js",
"tsc": "./node_modules/typescript/bin/tsc",
"i18n-extract-from-source": "./node_modules/@angular/cli/bin/ng extract-i18n --out-file ./src/locale/messages.xlf",
"i18n-pull-from-transifex": "tx pull -a --parallel --minimum-perc 1 --force",
"serve": "npm run generate-config && ng serve -c local",
"serve:stg": "npm run generate-config && ng serve -c staging",
"serve:local-prod": "npm run generate-config && ng serve -c local-prod",
"serve:local-staging": "npm run generate-config && ng serve -c local-staging",
"start": "npm run generate-config && npm run sync-assets-dev && ng serve -c local",
"start:stg": "npm run generate-config && npm run sync-assets-dev && ng serve -c staging",
"start:local-prod": "npm run generate-config && npm run sync-assets-dev && ng serve -c local-prod",
"start:local-staging": "npm run generate-config && npm run sync-assets-dev && ng serve -c local-staging",
"start:mixed": "npm run generate-config && npm run sync-assets-dev && ng serve -c mixed",
"build": "npm run generate-config && ng build --configuration production --localize && npm run sync-assets && npm run build-mempool.js",
"serve": "npm run generate-config && npm run ng -- serve -c local",
"serve:stg": "npm run generate-config && npm run ng -- serve -c staging",
"serve:local-prod": "npm run generate-config && npm run ng -- serve -c local-prod",
"serve:local-staging": "npm run generate-config && npm run ng -- serve -c local-staging",
"start": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c local",
"start:stg": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c staging",
"start:local-prod": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c local-prod",
"start:local-staging": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c local-staging",
"start:mixed": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c mixed",
"build": "npm run generate-config && npm run ng -- build --configuration production --localize && npm run sync-assets && npm run build-mempool.js",
"sync-assets": "node sync-assets.js && rsync -av ./dist/mempool/browser/en-US/resources ./dist/mempool/browser/resources",
"sync-assets-dev": "node sync-assets.js dev",
"generate-config": "node generate-config.js",
@ -41,17 +41,17 @@
"build-mempool-js": "browserify -p tinyify ./node_modules/@mempool/mempool.js/lib/index.js --standalone mempoolJS > ./dist/mempool/browser/en-US/mempool.js",
"build-mempool-bisq-js": "browserify -p tinyify ./node_modules/@mempool/mempool.js/lib/index-bisq.js --standalone bisqJS > ./dist/mempool/browser/en-US/bisq.js",
"build-mempool-liquid-js": "browserify -p tinyify ./node_modules/@mempool/mempool.js/lib/index-liquid.js --standalone liquidJS > ./dist/mempool/browser/en-US/liquid.js",
"test": "ng test",
"lint": "ng lint",
"e2e": "npm run generate-config && ng e2e",
"test": "npm run ng -- test",
"lint": "npm run ng -- lint",
"e2e": "npm run generate-config && npm run ng -- e2e",
"e2e:ci": "npm run cypress:run:ci",
"config:defaults:mempool": "node update-config.js TESTNET_ENABLED=true SIGNET_ENABLED=true LIQUID_ENABLED=true LIQUID_TESTNET_ENABLED=true BISQ_ENABLED=true ITEMS_PER_PAGE=25 BASE_MODULE=mempool BLOCK_WEIGHT_UNITS=4000000 && npm run generate-config",
"config:defaults:liquid": "node update-config.js TESTNET_ENABLED=true SIGNET_ENABLED=true LIQUID_ENABLED=true LIQUID_TESTNET_ENABLED=true BISQ_ENABLED=true ITEMS_PER_PAGE=25 BASE_MODULE=liquid BLOCK_WEIGHT_UNITS=300000 && npm run generate-config",
"config:defaults:bisq": "node update-config.js TESTNET_ENABLED=true SIGNET_ENABLED=true LIQUID_ENABLED=true BISQ_ENABLED=true ITEMS_PER_PAGE=25 BASE_MODULE=bisq BLOCK_WEIGHT_UNITS=4000000 && npm run generate-config",
"dev:ssr": "npm run generate-config && ng run mempool:serve-ssr",
"dev:ssr": "npm run generate-config && npm run ng -- run mempool:serve-ssr",
"serve:ssr": "node server.run.js",
"build:ssr": "npm run build && ng run mempool:server:production && ./node_modules/typescript/bin/tsc server.run.ts",
"prerender": "ng run mempool:prerender",
"build:ssr": "npm run build && npm run ng -- run mempool:server:production && npm run tsc -- server.run.ts",
"prerender": "npm run ng -- run mempool:prerender",
"cypress:open": "cypress open",
"cypress:run": "cypress run",
"cypress:run:record": "cypress run --record",

2
production/bitcoin.conf
View File

@ -22,6 +22,8 @@ bind=0.0.0.0:8333
bind=[::]:8333
rpcbind=127.0.0.1:8332
rpcbind=[::1]:8332
zmqpubrawblock=tcp://127.0.0.1:18332
zmqpubrawtx=tcp://127.0.0.1:18333
#addnode=[2401:b140:2::92:201]:8333
#addnode=[2401:b140:2::92:202]:8333
#addnode=[2401:b140:2::92:203]:8333

4
production/install
View File

@ -282,7 +282,7 @@ BITCOIN_REPO_URL=https://github.com/bitcoin/bitcoin
BITCOIN_REPO_NAME=bitcoin
BITCOIN_REPO_BRANCH=master
#BITCOIN_LATEST_RELEASE=$(curl -s https://api.github.com/repos/bitcoin/bitcoin/releases/latest|grep tag_name|head -1|cut -d '"' -f4)
BITCOIN_LATEST_RELEASE=v22.0
BITCOIN_LATEST_RELEASE=v23.0
echo -n '.'
BISQ_REPO_URL=https://github.com/bisq-network/bisq
@ -330,7 +330,7 @@ DEBIAN_PKG+=(nodejs npm mariadb-server nginx-core python-certbot-nginx rsync ufw
# packages needed for mempool ecosystem
FREEBSD_PKG=()
FREEBSD_PKG+=(zsh sudo git screen curl wget calc neovim)
FREEBSD_PKG+=(openssh-portable py38-pip rust llvm90 jq base64)
FREEBSD_PKG+=(openssh-portable py38-pip rust llvm90 jq base64 libzmq4)
FREEBSD_PKG+=(boost-libs autoconf automake gmake gcc libevent libtool pkgconf)
FREEBSD_PKG+=(nginx rsync py38-certbot-nginx mariadb105-server keybase)