machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Revamping configuration file.

Browse Source 
fixes #141
This commit is contained in:
softsimon 2020-10-19 11:57:02 +07:00
parent 0abe62128e
commit 45efb604c1
No known key found for this signature in database
GPG Key ID: 488D7DCFB5A430D7
17 changed files with 218 additions and 146 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
backend/mempool-config.sample.json
View File

@ -1,30 +1,41 @@
{ {
"HTTP_PORT": 8999, "MEMPOOL": {
"DB_HOST": "localhost", "NETWORK": "mainnet",
"DB_PORT": 3306, "HTTP_PORT": 8999,
"DB_USER": "mempool", "MINED_BLOCKS_CACHE": 144,
"DB_PASSWORD": "mempool", "SPAWN_CLUSTER_PROCS": 0,
"DB_DATABASE": "mempool", "API_URL_PREFIX": "/api/v1/",
"DB_DISABLED": false, "WEBSOCKET_REFRESH_RATE_MS": 2000
"API_ENDPOINT": "/api/v1/", },
"ELECTRS_POLL_RATE_MS": 2000, "ELECTRS": {
"NETWORK": "mainnet", "REST_API_URL": "http://localhost:50001",
"MEMPOOL_REFRESH_RATE_MS": 2000, "POLL_RATE_MS": 2000
"DEFAULT_PROJECTED_BLOCKS_AMOUNT": 8, },
"KEEP_BLOCK_AMOUNT": 24, "DATABASE": {
"INITIAL_BLOCK_AMOUNT": 8, "ENABLED": true,
"TX_PER_SECOND_SPAN_SECONDS": 150, "HOST": "localhost",
"ELECTRS_API_URL": "http://localhost:50001", "PORT": 3306,
"CLUSTER_NUM_CORES": 1, "DATABASE": "mempool",
"BISQ_ENABLED": false, "USERNAME": "mempool",
"BISQ_BLOCKS_DATA_PATH": "/bisq/seednode-data/btc_mainnet/db/json", "PASSWORD": "mempool"
"BISQ_MARKET_ENABLED": false, },
"BISQ_MARKETS_DATA_PATH": "/bisq/seednode-data/btc_mainnet/db", "STATISTICS": {
"SSL": false, "ENABLED": true,
"SSL_CERT_FILE_PATH": "/etc/letsencrypt/live/mysite/fullchain.pem", "TX_PER_SECOND_SAMPLE_PERIOD": 150
"SSL_KEY_FILE_PATH": "/etc/letsencrypt/live/mysite/privkey.pem", },
"BTCPAY_URL": "", "BISQ_BLOCKS": {
"BTCPAY_WEBHOOK_URL": "", "ENABLED": true,
"BTCPAY_AUTH": "", "DATA_PATH": "/bisq/statsnode-data/btc_mainnet/db/json"
"TWITTER_BEARER_AUTH": "" },
"BISQ_MARKETS": {
"ENABLED": true,
"DATA_PATH": "/bisq/statsnode-data/btc_mainnet/db"
},
"SPONSORS": {
"ENABLED": true,
"BTCPAY_URL": "",
"BTCPAY_AUTH": "",
"BTCPAY_WEBHOOK_URL": "",
"TWITTER_BEARER_AUTH": ""
}
} }

22
backend/src/api/bisq/bisq.ts
View File

@ -1,4 +1,4 @@
const config = require('../../../mempool-config.json'); import config from '../../config';
import * as fs from 'fs'; import * as fs from 'fs';
import * as request from 'request'; import * as request from 'request';
import { BisqBlocks, BisqBlock, BisqTransaction, BisqStats, BisqTrade } from './interfaces'; import { BisqBlocks, BisqBlock, BisqTransaction, BisqStats, BisqTrade } from './interfaces';
@ -8,7 +8,7 @@ import { StaticPool } from 'node-worker-threads-pool';
import logger from '../../logger'; import logger from '../../logger';
class Bisq { class Bisq {
private static BLOCKS_JSON_FILE_PATH = '/all/blocks.json'; private static BLOCKS_JSON_FILE_PATH = config.BISQ_BLOCKS.DATA_PATH + '/all/blocks.json';
private latestBlockHeight = 0; private latestBlockHeight = 0;
private blocks: BisqBlock[] = []; private blocks: BisqBlock[] = [];
private transactions: BisqTransaction[] = []; private transactions: BisqTransaction[] = [];
@ -87,8 +87,8 @@ class Bisq {
} }
private checkForBisqDataFolder() { private checkForBisqDataFolder() {
if (!fs.existsSync(config.BISQ_BLOCKS_DATA_PATH + Bisq.BLOCKS_JSON_FILE_PATH)) { if (!fs.existsSync(Bisq.BLOCKS_JSON_FILE_PATH)) {
logger.warn(config.BISQ_BLOCKS_DATA_PATH + Bisq.BLOCKS_JSON_FILE_PATH + ` doesn't exist. Make sure Bisq is running and the config is correct before starting the server.`); logger.warn(Bisq.BLOCKS_JSON_FILE_PATH + ` doesn't exist. Make sure Bisq is running and the config is correct before starting the server.`);
return process.exit(1); return process.exit(1);
} }
} }
@ -98,7 +98,7 @@ class Bisq {
this.topDirectoryWatcher.close(); this.topDirectoryWatcher.close();
} }
let fsWait: NodeJS.Timeout | null = null; let fsWait: NodeJS.Timeout | null = null;
this.topDirectoryWatcher = fs.watch(config.BISQ_BLOCKS_DATA_PATH, () => { this.topDirectoryWatcher = fs.watch(config.BISQ_BLOCKS.DATA_PATH, () => {
if (fsWait) { if (fsWait) {
clearTimeout(fsWait); clearTimeout(fsWait);
} }
@ -120,13 +120,13 @@ class Bisq {
if (this.subdirectoryWatcher) { if (this.subdirectoryWatcher) {
this.subdirectoryWatcher.close(); this.subdirectoryWatcher.close();
} }
if (!fs.existsSync(config.BISQ_BLOCKS_DATA_PATH + Bisq.BLOCKS_JSON_FILE_PATH)) { if (!fs.existsSync(Bisq.BLOCKS_JSON_FILE_PATH)) {
logger.warn(config.BISQ_BLOCKS_DATA_PATH + Bisq.BLOCKS_JSON_FILE_PATH + ` doesn't exist. Trying to restart sub directory watcher again in 3 minutes.`); logger.warn(Bisq.BLOCKS_JSON_FILE_PATH + ` doesn't exist. Trying to restart sub directory watcher again in 3 minutes.`);
setTimeout(() => this.startSubDirectoryWatcher(), 180000); setTimeout(() => this.startSubDirectoryWatcher(), 180000);
return; return;
} }
let fsWait: NodeJS.Timeout | null = null; let fsWait: NodeJS.Timeout | null = null;
this.subdirectoryWatcher = fs.watch(config.BISQ_BLOCKS_DATA_PATH + '/all', () => { this.subdirectoryWatcher = fs.watch(config.BISQ_BLOCKS.DATA_PATH + '/all', () => {
if (fsWait) { if (fsWait) {
clearTimeout(fsWait); clearTimeout(fsWait);
} }
@ -249,7 +249,7 @@ class Bisq {
this.blocks.reverse(); this.blocks.reverse();
this.latestBlockHeight = data.chainHeight; this.latestBlockHeight = data.chainHeight;
const time = new Date().getTime() - start; const time = new Date().getTime() - start;
logger.debug('Bisq dump processed in ' + time + ' ms'); logger.debug('Bisq dump processed in ' + time + ' ms (worker thread)');
} else { } else {
throw new Error(`Bisq dump didn't contain any blocks`); throw new Error(`Bisq dump didn't contain any blocks`);
} }
@ -258,10 +258,10 @@ class Bisq {
private loadData(): Promise<string> { private loadData(): Promise<string> {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
if (!fs.existsSync(config.BISQ_BLOCKS_DATA_PATH + Bisq.BLOCKS_JSON_FILE_PATH)) { if (!fs.existsSync(Bisq.BLOCKS_JSON_FILE_PATH)) {
return reject(Bisq.BLOCKS_JSON_FILE_PATH + ` doesn't exist`); return reject(Bisq.BLOCKS_JSON_FILE_PATH + ` doesn't exist`);
} }
fs.readFile(config.BISQ_BLOCKS_DATA_PATH + Bisq.BLOCKS_JSON_FILE_PATH, 'utf8', (err, data) => { fs.readFile(Bisq.BLOCKS_JSON_FILE_PATH, 'utf8', (err, data) => {
if (err) { if (err) {
reject(err); reject(err);
} }

4
backend/src/api/bisq/markets.ts
View File

@ -1,4 +1,4 @@
const config = require('../../../mempool-config.json'); import config from '../../config';
import * as fs from 'fs'; import * as fs from 'fs';
import { OffersData as OffersData, TradesData, Currency } from './interfaces'; import { OffersData as OffersData, TradesData, Currency } from './interfaces';
import bisqMarket from './markets-api'; import bisqMarket from './markets-api';
@ -6,7 +6,7 @@ import logger from '../../logger';
class Bisq { class Bisq {
private static FOLDER_WATCH_CHANGE_DETECTION_DEBOUNCE = 4000; private static FOLDER_WATCH_CHANGE_DETECTION_DEBOUNCE = 4000;
private static MARKET_JSON_PATH = config.BISQ_MARKETS_DATA_PATH + '/btc_mainnet/db'; private static MARKET_JSON_PATH = config.BISQ_MARKETS.DATA_PATH + '/btc_mainnet/db';
private static MARKET_JSON_FILE_PATHS = { private static MARKET_JSON_FILE_PATHS = {
cryptoCurrency: '/crypto_currency_list.json', cryptoCurrency: '/crypto_currency_list.json',
fiatCurrency: '/fiat_currency_list.json', fiatCurrency: '/fiat_currency_list.json',

18
backend/src/api/bitcoin/electrs-api.ts
View File

@ -1,4 +1,4 @@
const config = require('../../../mempool-config.json'); import config from '../../config';
import { Transaction, Block, MempoolInfo } from '../../interfaces'; import { Transaction, Block, MempoolInfo } from '../../interfaces';
import * as request from 'request'; import * as request from 'request';
@ -9,7 +9,7 @@ class ElectrsApi {
getMempoolInfo(): Promise<MempoolInfo> { getMempoolInfo(): Promise<MempoolInfo> {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
request(config.ELECTRS_API_URL + '/mempool', { json: true, timeout: 10000 }, (err, res, response) => { request(config.ELECTRS.REST_API_URL + '/mempool', { json: true, timeout: 10000 }, (err, res, response) => {
if (err) { if (err) {
reject('getMempoolInfo error: ' + err.message || err); reject('getMempoolInfo error: ' + err.message || err);
} else if (res.statusCode !== 200) { } else if (res.statusCode !== 200) {
@ -30,7 +30,7 @@ class ElectrsApi {
getRawMempool(): Promise<Transaction['txid'][]> { getRawMempool(): Promise<Transaction['txid'][]> {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
request(config.ELECTRS_API_URL + '/mempool/txids', { json: true, timeout: 10000, forever: true }, (err, res, response) => { request(config.ELECTRS.REST_API_URL + '/mempool/txids', { json: true, timeout: 10000, forever: true }, (err, res, response) => {
if (err) { if (err) {
reject('getRawMempool error: ' + err.message || err); reject('getRawMempool error: ' + err.message || err);
} else if (res.statusCode !== 200) { } else if (res.statusCode !== 200) {
@ -48,7 +48,7 @@ class ElectrsApi {
getRawTransaction(txId: string): Promise<Transaction> { getRawTransaction(txId: string): Promise<Transaction> {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
request(config.ELECTRS_API_URL + '/tx/' + txId, { json: true, timeout: 10000, forever: true }, (err, res, response) => { request(config.ELECTRS.REST_API_URL + '/tx/' + txId, { json: true, timeout: 10000, forever: true }, (err, res, response) => {
if (err) { if (err) {
reject('getRawTransaction error: ' + err.message || err); reject('getRawTransaction error: ' + err.message || err);
} else if (res.statusCode !== 200) { } else if (res.statusCode !== 200) {
@ -66,7 +66,7 @@ class ElectrsApi {
getBlockHeightTip(): Promise<number> { getBlockHeightTip(): Promise<number> {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
request(config.ELECTRS_API_URL + '/blocks/tip/height', { json: true, timeout: 10000 }, (err, res, response) => { request(config.ELECTRS.REST_API_URL + '/blocks/tip/height', { json: true, timeout: 10000 }, (err, res, response) => {
if (err) { if (err) {
reject('getBlockHeightTip error: ' + err.message || err); reject('getBlockHeightTip error: ' + err.message || err);
} else if (res.statusCode !== 200) { } else if (res.statusCode !== 200) {
@ -80,7 +80,7 @@ class ElectrsApi {
getTxIdsForBlock(hash: string): Promise<string[]> { getTxIdsForBlock(hash: string): Promise<string[]> {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
request(config.ELECTRS_API_URL + '/block/' + hash + '/txids', { json: true, timeout: 10000 }, (err, res, response) => { request(config.ELECTRS.REST_API_URL + '/block/' + hash + '/txids', { json: true, timeout: 10000 }, (err, res, response) => {
if (err) { if (err) {
reject('getTxIdsForBlock error: ' + err.message || err); reject('getTxIdsForBlock error: ' + err.message || err);
} else if (res.statusCode !== 200) { } else if (res.statusCode !== 200) {
@ -98,7 +98,7 @@ class ElectrsApi {
getBlockHash(height: number): Promise<string> { getBlockHash(height: number): Promise<string> {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
request(config.ELECTRS_API_URL + '/block-height/' + height, { json: true, timeout: 10000 }, (err, res, response) => { request(config.ELECTRS.REST_API_URL + '/block-height/' + height, { json: true, timeout: 10000 }, (err, res, response) => {
if (err) { if (err) {
reject('getBlockHash error: ' + err.message || err); reject('getBlockHash error: ' + err.message || err);
} else if (res.statusCode !== 200) { } else if (res.statusCode !== 200) {
@ -112,7 +112,7 @@ class ElectrsApi {
getBlocksFromHeight(height: number): Promise<string> { getBlocksFromHeight(height: number): Promise<string> {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
request(config.ELECTRS_API_URL + '/blocks/' + height, { json: true, timeout: 10000 }, (err, res, response) => { request(config.ELECTRS.REST_API_URL + '/blocks/' + height, { json: true, timeout: 10000 }, (err, res, response) => {
if (err) { if (err) {
reject('getBlocksFromHeight error: ' + err.message || err); reject('getBlocksFromHeight error: ' + err.message || err);
} else if (res.statusCode !== 200) { } else if (res.statusCode !== 200) {
@ -126,7 +126,7 @@ class ElectrsApi {
getBlock(hash: string): Promise<Block> { getBlock(hash: string): Promise<Block> {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
request(config.ELECTRS_API_URL + '/block/' + hash, { json: true, timeout: 10000 }, (err, res, response) => { request(config.ELECTRS.REST_API_URL + '/block/' + hash, { json: true, timeout: 10000 }, (err, res, response) => {
if (err) { if (err) {
reject('getBlock error: ' + err.message || err); reject('getBlock error: ' + err.message || err);
} else if (res.statusCode !== 200) { } else if (res.statusCode !== 200) {

13
backend/src/api/blocks.ts
View File

@ -1,4 +1,3 @@
const config = require('../../mempool-config.json');
import bitcoinApi from './bitcoin/electrs-api'; import bitcoinApi from './bitcoin/electrs-api';
import logger from '../logger'; import logger from '../logger';
import memPool from './mempool'; import memPool from './mempool';
@ -6,6 +5,8 @@ import { Block, TransactionExtended, TransactionMinerInfo } from '../interfaces'
import { Common } from './common'; import { Common } from './common';
class Blocks { class Blocks {
private static INITIAL_BLOCK_AMOUNT = 8;
private static KEEP_BLOCK_AMOUNT = 24;
private blocks: Block[] = []; private blocks: Block[] = [];
private currentBlockHeight = 0; private currentBlockHeight = 0;
private lastDifficultyAdjustmentTime = 0; private lastDifficultyAdjustmentTime = 0;
@ -29,14 +30,14 @@ class Blocks {
const blockHeightTip = await bitcoinApi.getBlockHeightTip(); const blockHeightTip = await bitcoinApi.getBlockHeightTip();
if (this.blocks.length === 0) { if (this.blocks.length === 0) {
this.currentBlockHeight = blockHeightTip - config.INITIAL_BLOCK_AMOUNT; this.currentBlockHeight = blockHeightTip - Blocks.INITIAL_BLOCK_AMOUNT;
} else { } else {
this.currentBlockHeight = this.blocks[this.blocks.length - 1].height; this.currentBlockHeight = this.blocks[this.blocks.length - 1].height;
} }
if (blockHeightTip - this.currentBlockHeight > config.INITIAL_BLOCK_AMOUNT * 2) { if (blockHeightTip - this.currentBlockHeight > Blocks.INITIAL_BLOCK_AMOUNT * 2) {
logger.info(`${blockHeightTip - this.currentBlockHeight} blocks since tip. Fast forwarding to the ${config.INITIAL_BLOCK_AMOUNT} recent blocks`); logger.info(`${blockHeightTip - this.currentBlockHeight} blocks since tip. Fast forwarding to the ${Blocks.INITIAL_BLOCK_AMOUNT} recent blocks`);
this.currentBlockHeight = blockHeightTip - config.INITIAL_BLOCK_AMOUNT; this.currentBlockHeight = blockHeightTip - Blocks.INITIAL_BLOCK_AMOUNT;
} }
if (!this.lastDifficultyAdjustmentTime) { if (!this.lastDifficultyAdjustmentTime) {
@ -91,7 +92,7 @@ class Blocks {
} }
this.blocks.push(block); this.blocks.push(block);
if (this.blocks.length > config.KEEP_BLOCK_AMOUNT) { if (this.blocks.length > Blocks.KEEP_BLOCK_AMOUNT) {
this.blocks.shift(); this.blocks.shift();
} }

4
backend/src/api/disk-cache.ts
View File

@ -1,4 +1,4 @@
const config = require('../../mempool-config.json'); import config from '../config';
import * as fs from 'fs'; import * as fs from 'fs';
import * as process from 'process'; import * as process from 'process';
import memPool from './mempool'; import memPool from './mempool';
@ -9,7 +9,7 @@ class DiskCache {
static FILE_NAME = './cache.json'; static FILE_NAME = './cache.json';
constructor() { constructor() {
if (process.env.workerId === '0' || !config.CLUSTER_NUM_CORES || config.CLUSTER_NUM_CORES === 1) { if (process.env.workerId === '0' || !config.MEMPOOL.SPAWN_CLUSTER_PROCS) {
process.on('SIGINT', () => { process.on('SIGINT', () => {
this.saveCacheToDisk(); this.saveCacheToDisk();
process.exit(2); process.exit(2);

13
backend/src/api/donations.ts
View File

@ -1,4 +1,4 @@
const config = require('../../mempool-config.json'); import config from '../config';
import * as request from 'request'; import * as request from 'request';
import { DB } from '../database'; import { DB } from '../database';
import logger from '../logger'; import logger from '../logger';
@ -6,16 +6,19 @@ import logger from '../logger';
class Donations { class Donations {
private notifyDonationStatusCallback: ((invoiceId: string) => void) | undefined; private notifyDonationStatusCallback: ((invoiceId: string) => void) | undefined;
private options = { private options = {
baseUrl: config.BTCPAY_URL, baseUrl: config.SPONSORS.BTCPAY_URL,
headers: { headers: {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
'Authorization': config.BTCPAY_AUTH, 'Authorization': config.SPONSORS.BTCPAY_AUTH,
}, },
}; };
sponsorsCache: any[] = []; sponsorsCache: any[] = [];
constructor() { constructor() {
if (!config.SPONSORS.ENABLED) {
return;
}
this.$updateCache(); this.$updateCache();
} }
@ -39,7 +42,7 @@ class Donations {
'orderId': orderId, 'orderId': orderId,
'currency': 'BTC', 'currency': 'BTC',
'itemDesc': 'Sponsor mempool.space', 'itemDesc': 'Sponsor mempool.space',
'notificationUrl': config.BTCPAY_WEBHOOK_URL, 'notificationUrl': config.SPONSORS.BTCPAY_WEBHOOK_URL,
'redirectURL': 'https://mempool.space/about' 'redirectURL': 'https://mempool.space/about'
}; };
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
@ -185,7 +188,7 @@ class Donations {
uri: `https://api.twitter.com/1.1/users/show.json?screen_name=${handle}`, uri: `https://api.twitter.com/1.1/users/show.json?screen_name=${handle}`,
json: true, json: true,
headers: { headers: {
Authorization: 'Bearer ' + config.TWITTER_BEARER_AUTH Authorization: 'Bearer ' + config.SPONSORS.TWITTER_BEARER_AUTH
}, },
}, (err, res, body) => { }, (err, res, body) => {
if (err) { return reject(err); } if (err) { return reject(err); }

4
backend/src/api/fee-api.ts
View File

@ -1,11 +1,11 @@
const config = require('../../mempool-config.json'); import config from '../config';
import { MempoolBlock } from '../interfaces'; import { MempoolBlock } from '../interfaces';
import projectedBlocks from './mempool-blocks'; import projectedBlocks from './mempool-blocks';
class FeeApi { class FeeApi {
constructor() { } constructor() { }
defaultFee = config.NETWORK === 'liquid' ? 0.1 : 1; defaultFee = config.MEMPOOL.NETWORK === 'liquid' ? 0.1 : 1;
public getRecommendedFee() { public getRecommendedFee() {
const pBlocks = projectedBlocks.getMempoolBlocks(); const pBlocks = projectedBlocks.getMempoolBlocks();

4
backend/src/api/mempool-blocks.ts
View File

@ -1,8 +1,8 @@
const config = require('../../mempool-config.json');
import { MempoolBlock, TransactionExtended, MempoolBlockWithTransactions } from '../interfaces'; import { MempoolBlock, TransactionExtended, MempoolBlockWithTransactions } from '../interfaces';
import { Common } from './common'; import { Common } from './common';
class MempoolBlocks { class MempoolBlocks {
private static DEFAULT_PROJECTED_BLOCKS_AMOUNT = 8;
private mempoolBlocks: MempoolBlockWithTransactions[] = []; private mempoolBlocks: MempoolBlockWithTransactions[] = [];
constructor() {} constructor() {}
@ -43,7 +43,7 @@ class MempoolBlocks {
let blockSize = 0; let blockSize = 0;
let transactions: TransactionExtended[] = []; let transactions: TransactionExtended[] = [];
transactionsSorted.forEach((tx) => { transactionsSorted.forEach((tx) => {
if (blockVSize + tx.vsize <= 1000000 || mempoolBlocks.length === config.DEFAULT_PROJECTED_BLOCKS_AMOUNT - 1) { if (blockVSize + tx.vsize <= 1000000 || mempoolBlocks.length === MempoolBlocks.DEFAULT_PROJECTED_BLOCKS_AMOUNT - 1) {
blockVSize += tx.vsize; blockVSize += tx.vsize;
blockSize += tx.size; blockSize += tx.size;
transactions.push(tx); transactions.push(tx);

13
backend/src/api/mempool.ts
View File

@ -1,4 +1,4 @@
const config = require('../../mempool-config.json'); import config from '../config';
import bitcoinApi from './bitcoin/electrs-api'; import bitcoinApi from './bitcoin/electrs-api';
import { MempoolInfo, TransactionExtended, Transaction, VbytesPerSecond } from '../interfaces'; import { MempoolInfo, TransactionExtended, Transaction, VbytesPerSecond } from '../interfaces';
import logger from '../logger'; import logger from '../logger';
@ -124,14 +124,13 @@ class Mempool {
} }
} }
if ((new Date().getTime()) - start > config.MEMPOOL_REFRESH_RATE_MS * 10) { if ((new Date().getTime()) - start > config.MEMPOOL.WEBSOCKET_REFRESH_RATE_MS * 10) {
break; break;
} }
} }
// Prevent mempool from clear on bitcoind restart by delaying the deletion // Prevent mempool from clear on bitcoind restart by delaying the deletion
if ((config.NETWORK === 'mainnet' || !config.NETWORK) if (config.MEMPOOL.NETWORK === 'mainnet' && this.mempoolProtection === 0 && transactions.length / currentMempoolSize <= 0.80) {
&& this.mempoolProtection === 0 && transactions.length / currentMempoolSize <= 0.80) {
this.mempoolProtection = 1; this.mempoolProtection = 1;
this.inSync = false; this.inSync = false;
logger.warn(`Mempool clear protection triggered because transactions.length: ${transactions.length} and currentMempoolSize: ${currentMempoolSize}.`); logger.warn(`Mempool clear protection triggered because transactions.length: ${transactions.length} and currentMempoolSize: ${currentMempoolSize}.`);
@ -182,14 +181,14 @@ class Mempool {
} }
private updateTxPerSecond() { private updateTxPerSecond() {
const nowMinusTimeSpan = new Date().getTime() - (1000 * config.TX_PER_SECOND_SPAN_SECONDS); const nowMinusTimeSpan = new Date().getTime() - (1000 * config.STATISTICS.TX_PER_SECOND_SAMPLE_PERIOD);
this.txPerSecondArray = this.txPerSecondArray.filter((unixTime) => unixTime > nowMinusTimeSpan); this.txPerSecondArray = this.txPerSecondArray.filter((unixTime) => unixTime > nowMinusTimeSpan);
this.txPerSecond = this.txPerSecondArray.length / config.TX_PER_SECOND_SPAN_SECONDS || 0; this.txPerSecond = this.txPerSecondArray.length / config.STATISTICS.TX_PER_SECOND_SAMPLE_PERIOD || 0;
this.vBytesPerSecondArray = this.vBytesPerSecondArray.filter((data) => data.unixTime > nowMinusTimeSpan); this.vBytesPerSecondArray = this.vBytesPerSecondArray.filter((data) => data.unixTime > nowMinusTimeSpan);
if (this.vBytesPerSecondArray.length) { if (this.vBytesPerSecondArray.length) {
this.vBytesPerSecond = Math.round( this.vBytesPerSecond = Math.round(
this.vBytesPerSecondArray.map((data) => data.vSize).reduce((a, b) => a + b) / config.TX_PER_SECOND_SPAN_SECONDS this.vBytesPerSecondArray.map((data) => data.vSize).reduce((a, b) => a + b) / config.STATISTICS.TX_PER_SECOND_SAMPLE_PERIOD
); );
} }
} }

4
backend/src/api/websocket-handler.ts
View File

@ -1,4 +1,3 @@
const config = require('../../mempool-config.json');
import logger from '../logger'; import logger from '../logger';
import * as WebSocket from 'ws'; import * as WebSocket from 'ws';
import { Block, TransactionExtended, WebsocketResponse, MempoolBlock, OptimizedStatistic } from '../interfaces'; import { Block, TransactionExtended, WebsocketResponse, MempoolBlock, OptimizedStatistic } from '../interfaces';
@ -10,6 +9,7 @@ import fiatConversion from './fiat-conversion';
import { Common } from './common'; import { Common } from './common';
class WebsocketHandler { class WebsocketHandler {
private static INITIAL_BLOCK_AMOUNT = 8;
private wss: WebSocket.Server | undefined; private wss: WebSocket.Server | undefined;
private nativeAssetId = '6f0279e9ed041c3d710a9f57d0c02928416460c4b722ae3457a11eec381c526d'; private nativeAssetId = '6f0279e9ed041c3d710a9f57d0c02928416460c4b722ae3457a11eec381c526d';
private extraInitProperties = {}; private extraInitProperties = {};
@ -85,7 +85,7 @@ class WebsocketHandler {
'mempoolInfo': memPool.getMempoolInfo(), 'mempoolInfo': memPool.getMempoolInfo(),
'vBytesPerSecond': memPool.getVBytesPerSecond(), 'vBytesPerSecond': memPool.getVBytesPerSecond(),
'lastDifficultyAdjustment': blocks.getLastDifficultyAdjustmentTime(), 'lastDifficultyAdjustment': blocks.getLastDifficultyAdjustmentTime(),
'blocks': _blocks.slice(Math.max(_blocks.length - config.INITIAL_BLOCK_AMOUNT, 0)), 'blocks': _blocks.slice(Math.max(_blocks.length - WebsocketHandler.INITIAL_BLOCK_AMOUNT, 0)),
'conversions': fiatConversion.getTickers()['BTCUSD'], 'conversions': fiatConversion.getTickers()['BTCUSD'],
'mempool-blocks': mempoolBlocks.getMempoolBlocks(), 'mempool-blocks': mempoolBlocks.getMempoolBlocks(),
'transactions': memPool.getLatestTransactions(), 'transactions': memPool.getLatestTransactions(),

65
backend/src/config.ts Normal file
View File

@ -0,0 +1,65 @@
const configFile = require('../mempool-config.json');
export interface IConfig {
MEMPOOL: {
NETWORK: 'mainnet' | 'testnet' | 'liquid';
HTTP_PORT: number;
MINED_BLOCKS_CACHE: number;
SPAWN_CLUSTER_PROCS: number;
API_URL_PREFIX: string;
WEBSOCKET_REFRESH_RATE_MS: number;
};
ELECTRS: {
REST_API_URL: string;
POLL_RATE_MS: number;
};
DATABASE: {
ENABLED: boolean;
HOST: string,
PORT: number;
DATABASE: string;
USERNAME: string;
PASSWORD: string;
};
STATISTICS: {
ENABLED: boolean;
TX_PER_SECOND_SAMPLE_PERIOD: number;
};
BISQ_BLOCKS: {
ENABLED: boolean;
DATA_PATH: string;
};
BISQ_MARKETS: {
ENABLED: boolean;
DATA_PATH: string;
};
SPONSORS: {
ENABLED: boolean;
BTCPAY_URL: string;
BTCPAY_AUTH: string;
BTCPAY_WEBHOOK_URL: string;
TWITTER_BEARER_AUTH: string;
};
}
class Config implements IConfig {
MEMPOOL: IConfig['MEMPOOL'];
ELECTRS: IConfig['ELECTRS'];
DATABASE: IConfig['DATABASE'];
STATISTICS: IConfig['STATISTICS'];
BISQ_BLOCKS: IConfig['BISQ_BLOCKS'];
BISQ_MARKETS: IConfig['BISQ_MARKETS'];
SPONSORS: IConfig['SPONSORS'];
constructor() {
this.MEMPOOL = configFile.MEMPOOL;
this.ELECTRS = configFile.ELECTRS;
this.DATABASE = configFile.DATABASE;
this.STATISTICS = configFile.STATISTICS;
this.BISQ_BLOCKS = configFile.BISQ_BLOCKS;
this.BISQ_MARKETS = configFile.BISQ_MARKETS;
this.SPONSORS = configFile.SPONSORS;
}
}
export default new Config();

12
backend/src/database.ts
View File

@ -1,14 +1,14 @@
const config = require('../mempool-config.json'); import config from './config';
import { createPool } from 'mysql2/promise'; import { createPool } from 'mysql2/promise';
import logger from './logger'; import logger from './logger';
export class DB { export class DB {
static pool = createPool({ static pool = createPool({
host: config.DB_HOST, host: config.DATABASE.HOST,
port: config.DB_PORT, port: config.DATABASE.PORT,
database: config.DB_DATABASE, database: config.DATABASE.DATABASE,
user: config.DB_USER, user: config.DATABASE.USERNAME,
password: config.DB_PASSWORD, password: config.DATABASE.PASSWORD,
connectionLimit: 10, connectionLimit: 10,
supportBigNumbers: true, supportBigNumbers: true,
}); });

107
backend/src/index.ts
View File

@ -1,6 +1,4 @@
const config = require('../mempool-config.json');
import { Express, Request, Response, NextFunction } from 'express'; import { Express, Request, Response, NextFunction } from 'express';
import * as fs from 'fs';
import * as express from 'express'; import * as express from 'express';
import * as compression from 'compression'; import * as compression from 'compression';
import * as http from 'http'; import * as http from 'http';
@ -10,6 +8,7 @@ import * as cluster from 'cluster';
import * as request from 'request'; import * as request from 'request';
import { checkDbConnection } from './database'; import { checkDbConnection } from './database';
import config from './config';
import routes from './routes'; import routes from './routes';
import blocks from './api/blocks'; import blocks from './api/blocks';
import memPool from './api/mempool'; import memPool from './api/mempool';
@ -32,15 +31,15 @@ class Server {
constructor() { constructor() {
this.app = express(); this.app = express();
if (!config.CLUSTER_NUM_CORES || config.CLUSTER_NUM_CORES === 1) { if (!config.MEMPOOL.SPAWN_CLUSTER_PROCS) {
this.startServer(); this.startServer();
return; return;
} }
if (cluster.isMaster) { if (cluster.isMaster) {
logger.notice(`Mempool Server is running on port ${config.HTTP_PORT} (${backendInfo.getShortCommitHash()})`); logger.notice(`Mempool Server (Master) is running on port ${config.MEMPOOL.HTTP_PORT} (${backendInfo.getShortCommitHash()})`);
const numCPUs = config.CLUSTER_NUM_CORES; const numCPUs = config.MEMPOOL.SPAWN_CLUSTER_PROCS;
for (let i = 0; i < numCPUs; i++) { for (let i = 0; i < numCPUs; i++) {
const env = { workerId: i }; const env = { workerId: i };
const worker = cluster.fork(env); const worker = cluster.fork(env);
@ -71,20 +70,14 @@ class Server {
.use(express.urlencoded({ extended: true })) .use(express.urlencoded({ extended: true }))
.use(express.json()); .use(express.json());
if (config.SSL === true) { this.server = http.createServer(this.app);
const credentials = { this.wss = new WebSocket.Server({ server: this.server });
cert: fs.readFileSync(config.SSL_CERT_FILE_PATH),
key: fs.readFileSync(config.SSL_KEY_FILE_PATH), if (config.DATABASE.ENABLED) {
}; checkDbConnection();
this.server = https.createServer(credentials, this.app);
this.wss = new WebSocket.Server({ server: this.server });
} else {
this.server = http.createServer(this.app);
this.wss = new WebSocket.Server({ server: this.server });
} }
if (!config.DB_DISABLED) { if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED) {
checkDbConnection();
statistics.startStatistics(); statistics.startStatistics();
} }
@ -95,21 +88,21 @@ class Server {
fiatConversion.startService(); fiatConversion.startService();
diskCache.loadMempoolCache(); diskCache.loadMempoolCache();
if (config.BISQ_ENABLED) { if (config.BISQ_BLOCKS.ENABLED) {
bisq.startBisqService(); bisq.startBisqService();
bisq.setPriceCallbackFunction((price) => websocketHandler.setExtraInitProperties('bsq-price', price)); bisq.setPriceCallbackFunction((price) => websocketHandler.setExtraInitProperties('bsq-price', price));
blocks.setNewBlockCallback(bisq.handleNewBitcoinBlock.bind(bisq)); blocks.setNewBlockCallback(bisq.handleNewBitcoinBlock.bind(bisq));
} }
if (config.BISQ_MARKET_ENABLED) { if (config.BISQ_MARKETS.ENABLED) {
bisqMarkets.startBisqService(); bisqMarkets.startBisqService();
} }
this.server.listen(config.HTTP_PORT, () => { this.server.listen(config.MEMPOOL.HTTP_PORT, () => {
if (worker) { if (worker) {
logger.info(`Mempool Server worker #${process.pid} started`); logger.info(`Mempool Server worker #${process.pid} started`);
} else { } else {
logger.notice(`Mempool Server is running on port ${config.HTTP_PORT} (${backendInfo.getShortCommitHash()})`); logger.notice(`Mempool Server is running on port ${config.MEMPOOL.HTTP_PORT} (${backendInfo.getShortCommitHash()})`);
} }
}); });
} }
@ -119,7 +112,7 @@ class Server {
await memPool.$updateMemPoolInfo(); await memPool.$updateMemPoolInfo();
await blocks.$updateBlocks(); await blocks.$updateBlocks();
await memPool.$updateMempool(); await memPool.$updateMempool();
setTimeout(this.runMempoolIntervalFunctions.bind(this), config.ELECTRS_POLL_RATE_MS); setTimeout(this.runMempoolIntervalFunctions.bind(this), config.ELECTRS.POLL_RATE_MS);
this.retryOnElectrsErrorAfterSeconds = 5; this.retryOnElectrsErrorAfterSeconds = 5;
} catch (e) { } catch (e) {
logger.warn(`runMempoolIntervalFunctions error: ${(e.message || e)}. Retrying in ${this.retryOnElectrsErrorAfterSeconds} sec.`); logger.warn(`runMempoolIntervalFunctions error: ${(e.message || e)}. Retrying in ${this.retryOnElectrsErrorAfterSeconds} sec.`);
@ -142,57 +135,57 @@ class Server {
setUpHttpApiRoutes() { setUpHttpApiRoutes() {
this.app this.app
.get(config.API_ENDPOINT + 'transaction-times', routes.getTransactionTimes) .get(config.MEMPOOL.API_URL_PREFIX + 'transaction-times', routes.getTransactionTimes)
.get(config.API_ENDPOINT + 'fees/recommended', routes.getRecommendedFees) .get(config.MEMPOOL.API_URL_PREFIX + 'fees/recommended', routes.getRecommendedFees)
.get(config.API_ENDPOINT + 'fees/mempool-blocks', routes.getMempoolBlocks) .get(config.MEMPOOL.API_URL_PREFIX + 'fees/mempool-blocks', routes.getMempoolBlocks)
.get(config.API_ENDPOINT + 'statistics/2h', routes.get2HStatistics) .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/2h', routes.get2HStatistics)
.get(config.API_ENDPOINT + 'statistics/24h', routes.get24HStatistics.bind(routes)) .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/24h', routes.get24HStatistics.bind(routes))
.get(config.API_ENDPOINT + 'statistics/1w', routes.get1WHStatistics.bind(routes)) .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/1w', routes.get1WHStatistics.bind(routes))
.get(config.API_ENDPOINT + 'statistics/1m', routes.get1MStatistics.bind(routes)) .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/1m', routes.get1MStatistics.bind(routes))
.get(config.API_ENDPOINT + 'statistics/3m', routes.get3MStatistics.bind(routes)) .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/3m', routes.get3MStatistics.bind(routes))
.get(config.API_ENDPOINT + 'statistics/6m', routes.get6MStatistics.bind(routes)) .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/6m', routes.get6MStatistics.bind(routes))
.get(config.API_ENDPOINT + 'statistics/1y', routes.get1YStatistics.bind(routes)) .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/1y', routes.get1YStatistics.bind(routes))
.get(config.API_ENDPOINT + 'backend-info', routes.getBackendInfo) .get(config.MEMPOOL.API_URL_PREFIX + 'backend-info', routes.getBackendInfo)
; ;
if (config.BISQ_ENABLED) { if (config.BISQ_BLOCKS.ENABLED) {
this.app this.app
.get(config.API_ENDPOINT + 'bisq/stats', routes.getBisqStats) .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/stats', routes.getBisqStats)
.get(config.API_ENDPOINT + 'bisq/tx/:txId', routes.getBisqTransaction) .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/tx/:txId', routes.getBisqTransaction)
.get(config.API_ENDPOINT + 'bisq/block/:hash', routes.getBisqBlock) .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/block/:hash', routes.getBisqBlock)
.get(config.API_ENDPOINT + 'bisq/blocks/tip/height', routes.getBisqTip) .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/blocks/tip/height', routes.getBisqTip)
.get(config.API_ENDPOINT + 'bisq/blocks/:index/:length', routes.getBisqBlocks) .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/blocks/:index/:length', routes.getBisqBlocks)
.get(config.API_ENDPOINT + 'bisq/address/:address', routes.getBisqAddress) .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/address/:address', routes.getBisqAddress)
.get(config.API_ENDPOINT + 'bisq/txs/:index/:length', routes.getBisqTransactions) .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/txs/:index/:length', routes.getBisqTransactions)
; ;
} }
if (config.BISQ_MARKET_ENABLED) { if (config.BISQ_MARKETS.ENABLED) {
this.app this.app
.get(config.API_ENDPOINT + 'bisq/markets/currencies', routes.getBisqMarketCurrencies.bind(routes)) .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/currencies', routes.getBisqMarketCurrencies.bind(routes))
.get(config.API_ENDPOINT + 'bisq/markets/depth', routes.getBisqMarketDepth.bind(routes)) .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/depth', routes.getBisqMarketDepth.bind(routes))
.get(config.API_ENDPOINT + 'bisq/markets/hloc', routes.getBisqMarketHloc.bind(routes)) .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/hloc', routes.getBisqMarketHloc.bind(routes))
.get(config.API_ENDPOINT + 'bisq/markets/markets', routes.getBisqMarketMarkets.bind(routes)) .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/markets', routes.getBisqMarketMarkets.bind(routes))
.get(config.API_ENDPOINT + 'bisq/markets/offers', routes.getBisqMarketOffers.bind(routes)) .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/offers', routes.getBisqMarketOffers.bind(routes))
.get(config.API_ENDPOINT + 'bisq/markets/ticker', routes.getBisqMarketTicker.bind(routes)) .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/ticker', routes.getBisqMarketTicker.bind(routes))
.get(config.API_ENDPOINT + 'bisq/markets/trades', routes.getBisqMarketTrades.bind(routes)) .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/trades', routes.getBisqMarketTrades.bind(routes))
.get(config.API_ENDPOINT + 'bisq/markets/volumes', routes.getBisqMarketVolumes.bind(routes)) .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/volumes', routes.getBisqMarketVolumes.bind(routes))
; ;
} }
if (config.BTCPAY_URL) { if (config.SPONSORS.ENABLED) {
this.app this.app
.get(config.API_ENDPOINT + 'donations', routes.getDonations.bind(routes)) .get(config.MEMPOOL.API_URL_PREFIX + 'donations', routes.getDonations.bind(routes))
.get(config.API_ENDPOINT + 'donations/images/:id', routes.getSponsorImage.bind(routes)) .get(config.MEMPOOL.API_URL_PREFIX + 'donations/images/:id', routes.getSponsorImage.bind(routes))
.post(config.API_ENDPOINT + 'donations', routes.createDonationRequest.bind(routes)) .post(config.MEMPOOL.API_URL_PREFIX + 'donations', routes.createDonationRequest.bind(routes))
.post(config.API_ENDPOINT + 'donations-webhook', routes.donationWebhook.bind(routes)) .post(config.MEMPOOL.API_URL_PREFIX + 'donations-webhook', routes.donationWebhook.bind(routes))
; ;
} else { } else {
this.app this.app
.get(config.API_ENDPOINT + 'donations', (req, res) => { .get(config.MEMPOOL.API_URL_PREFIX + 'donations', (req, res) => {
req.pipe(request('https://mempool.space/api/v1/donations')).pipe(res); req.pipe(request('https://mempool.space/api/v1/donations')).pipe(res);
}) })
.get(config.API_ENDPOINT + 'donations/images/:id', (req, res) => { .get(config.MEMPOOL.API_URL_PREFIX + 'donations/images/:id', (req, res) => {
req.pipe(request('https://mempool.space/api/v1/donations/images/' + req.params.id)).pipe(res); req.pipe(request('https://mempool.space/api/v1/donations/images/' + req.params.id)).pipe(res);
}); });
} }

8
backend/src/logger.ts
View File

@ -1,4 +1,4 @@
const config = require('../mempool-config.json'); import config from './config';
import * as dgram from 'dgram'; import * as dgram from 'dgram';
class Logger { class Logger {
@ -79,11 +79,11 @@ class Logger {
} }
private getNetwork(): string { private getNetwork(): string {
if (config.BISQ_ENABLED) { if (config.BISQ_BLOCKS.ENABLED) {
return 'bisq'; return 'bisq';
} }
if (config.NETWORK && config.NETWORK !== 'mainnet') { if (config.MEMPOOL.NETWORK && config.MEMPOOL.NETWORK !== 'mainnet') {
return config.NETWORK; return config.MEMPOOL.NETWORK;
} }
return ''; return '';
} }

4
backend/src/routes.ts
View File

@ -1,4 +1,4 @@
const config = require('../mempool-config.json'); import config from './config';
import { Request, Response } from 'express'; import { Request, Response } from 'express';
import statistics from './api/statistics'; import statistics from './api/statistics';
import feeApi from './api/fee-api'; import feeApi from './api/fee-api';
@ -16,7 +16,7 @@ class Routes {
private cache = {}; private cache = {};
constructor() { constructor() {
if (!config.DB_DISABLED) { if (!config.DATABASE.ENABLED) {
this.createCache(); this.createCache();
setInterval(this.createCache.bind(this), 600000); setInterval(this.createCache.bind(this), 600000);
} }

2
frontend/proxy.conf.json
View File

@ -56,7 +56,7 @@
"target": "http://localhost:8999/", "target": "http://localhost:8999/",
"secure": false, "secure": false,
"pathRewrite": { "pathRewrite": {
"^/bisq/api/": "/api/v1/bisq" "^/bisq/api/": "/api/v1/bisq/"
} }
}, },
"/bisq/api/v1/ws": { "/bisq/api/v1/ws": {