machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1691 from ayanamitech/cache-static

Browse Source 
Handle Error with basic retry while syncing external assets ( Price Data )
This commit is contained in:
wiz 2022-06-08 00:46:10 +09:00 committed by GitHub
commit da6c72e9b7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 249 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
backend/mempool-config.sample.json
View File

@ -15,9 +15,10 @@
"INDEXING_BLOCKS_AMOUNT": 11000, "INDEXING_BLOCKS_AMOUNT": 11000,
"PRICE_FEED_UPDATE_INTERVAL": 600, "PRICE_FEED_UPDATE_INTERVAL": 600,
"USE_SECOND_NODE_FOR_MINFEE": false, "USE_SECOND_NODE_FOR_MINFEE": false,
"EXTERNAL_ASSETS": [ "EXTERNAL_ASSETS": [],
"https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json" "EXTERNAL_MAX_RETRY": 1,
], "EXTERNAL_RETRY_INTERVAL": 0,
"USER_AGENT": "mempool",
"STDOUT_LOG_MIN_PRIORITY": "debug" "STDOUT_LOG_MIN_PRIORITY": "debug"
}, },
"CORE_RPC": { "CORE_RPC": {
@ -66,6 +67,7 @@
}, },
"SOCKS5PROXY": { "SOCKS5PROXY": {
"ENABLED": false, "ENABLED": false,
"USE_ONION": true,
"HOST": "127.0.0.1", "HOST": "127.0.0.1",
"PORT": 9050, "PORT": 9050,
"USERNAME": "", "USERNAME": "",
@ -74,5 +76,13 @@
"PRICE_DATA_SERVER": { "PRICE_DATA_SERVER": {
"TOR_URL": "http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices", "TOR_URL": "http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices",
"CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices" "CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices"
},
"EXTERNAL_DATA_SERVER": {
"MEMPOOL_API": "https://mempool.space/api/v1",
"MEMPOOL_ONION": "http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1",
"LIQUID_API": "https://liquid.network/api/v1",
"LIQUID_ONION": "http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1",
"BISQ_URL": "https://bisq.markets/api",
"BISQ_ONION": "http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api"
} }
} }

72
backend/src/api/bisq/bisq.ts
View File

@ -1,10 +1,14 @@
import config from '../../config'; import config from '../../config';
import * as fs from 'fs'; import * as fs from 'fs';
import axios from 'axios'; import axios, { AxiosResponse } from 'axios';
import * as http from 'http';
import * as https from 'https';
import { SocksProxyAgent } from 'socks-proxy-agent';
import { BisqBlocks, BisqBlock, BisqTransaction, BisqStats, BisqTrade } from './interfaces'; import { BisqBlocks, BisqBlock, BisqTransaction, BisqStats, BisqTrade } from './interfaces';
import { Common } from '../common'; import { Common } from '../common';
import { BlockExtended } from '../../mempool.interfaces'; import { BlockExtended } from '../../mempool.interfaces';
import { StaticPool } from 'node-worker-threads-pool'; import { StaticPool } from 'node-worker-threads-pool';
import backendInfo from '../backend-info';
import logger from '../../logger'; import logger from '../../logger';
class Bisq { class Bisq {
@ -143,12 +147,59 @@ class Bisq {
}, 2000); }, 2000);
}); });
} }
private async updatePrice() {
type axiosOptions = {
headers: {
'User-Agent': string
};
timeout: number;
httpAgent?: http.Agent;
httpsAgent?: https.Agent;
}
const setDelay = (secs: number = 1): Promise<void> => new Promise(resolve => setTimeout(() => resolve(), secs * 1000));
const BISQ_URL = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.EXTERNAL_DATA_SERVER.BISQ_ONION : config.EXTERNAL_DATA_SERVER.BISQ_URL;
const isHTTP = (new URL(BISQ_URL).protocol.split(':')[0] === 'http') ? true : false;
const axiosOptions: axiosOptions = {
headers: {
'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
},
timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000
};
let retry = 0;
private updatePrice() { while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
axios.get<BisqTrade[]>('https://bisq.markets/api/trades/?market=bsq_btc', { timeout: 10000 }) try {
.then((response) => { if (config.SOCKS5PROXY.ENABLED) {
const socksOptions: any = {
agentOptions: {
keepAlive: true,
},
hostname: config.SOCKS5PROXY.HOST,
port: config.SOCKS5PROXY.PORT
};
if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
socksOptions.username = config.SOCKS5PROXY.USERNAME;
socksOptions.password = config.SOCKS5PROXY.PASSWORD;
} else {
// Retry with different tor circuits https://stackoverflow.com/a/64960234
socksOptions.username = `circuit${retry}`;
}
// Handle proxy agent for onion addresses
if (isHTTP) {
axiosOptions.httpAgent = new SocksProxyAgent(socksOptions);
} else {
axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions);
}
}
const data: AxiosResponse = await axios.get(`${BISQ_URL}/trades/?market=bsq_btc`, axiosOptions);
if (data.statusText === 'error' || !data.data) {
throw new Error(`Could not fetch data from Bisq market, Error: ${data.status}`);
}
const prices: number[] = []; const prices: number[] = [];
response.data.forEach((trade) => { data.data.forEach((trade) => {
prices.push(parseFloat(trade.price) * 100000000); prices.push(parseFloat(trade.price) * 100000000);
}); });
prices.sort((a, b) => a - b); prices.sort((a, b) => a - b);
@ -156,9 +207,14 @@ class Bisq {
if (this.priceUpdateCallbackFunction) { if (this.priceUpdateCallbackFunction) {
this.priceUpdateCallbackFunction(this.price); this.priceUpdateCallbackFunction(this.price);
} }
}).catch((err) => { logger.debug('Successfully updated Bisq market price');
logger.err('Error updating Bisq market price: ' + err); break;
}); } catch (e) {
logger.err('Error updating Bisq market price: ' + (e instanceof Error ? e.message : e));
await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL);
retry++;
}
}
} }
private async loadBisqDumpFile(): Promise<void> { private async loadBisqDumpFile(): Promise<void> {

53
backend/src/api/fiat-conversion.ts
View File

@ -1,4 +1,6 @@
import logger from '../logger'; import logger from '../logger';
import * as http from 'http';
import * as https from 'https';
import axios, { AxiosResponse } from 'axios'; import axios, { AxiosResponse } from 'axios';
import { IConversionRates } from '../mempool.interfaces'; import { IConversionRates } from '../mempool.interfaces';
import config from '../config'; import config from '../config';
@ -25,9 +27,10 @@ class FiatConversion {
} }
public startService() { public startService() {
const fiatConversionUrl = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.PRICE_DATA_SERVER.TOR_URL : config.PRICE_DATA_SERVER.CLEARNET_URL;
logger.info('Starting currency rates service'); logger.info('Starting currency rates service');
if (config.SOCKS5PROXY.ENABLED) { if (config.SOCKS5PROXY.ENABLED) {
logger.info(`Currency rates service will be queried over the Tor network using ${config.PRICE_DATA_SERVER.TOR_URL}`); logger.info(`Currency rates service will be queried over the Tor network using ${fiatConversionUrl}`);
} else { } else {
logger.info(`Currency rates service will be queried over clearnet using ${config.PRICE_DATA_SERVER.CLEARNET_URL}`); logger.info(`Currency rates service will be queried over clearnet using ${config.PRICE_DATA_SERVER.CLEARNET_URL}`);
} }
@ -40,10 +43,27 @@ class FiatConversion {
} }
private async updateCurrency(): Promise<void> { private async updateCurrency(): Promise<void> {
const headers = { 'User-Agent': `mempool/v${backendInfo.getBackendInfo().version}` }; type axiosOptions = {
let fiatConversionUrl: string; headers: {
let response: AxiosResponse; 'User-Agent': string
};
timeout: number;
httpAgent?: http.Agent;
httpsAgent?: https.Agent;
}
const setDelay = (secs: number = 1): Promise<void> => new Promise(resolve => setTimeout(() => resolve(), secs * 1000));
const fiatConversionUrl = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.PRICE_DATA_SERVER.TOR_URL : config.PRICE_DATA_SERVER.CLEARNET_URL;
const isHTTP = (new URL(fiatConversionUrl).protocol.split(':')[0] === 'http') ? true : false;
const axiosOptions: axiosOptions = {
headers: {
'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
},
timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000
};
let retry = 0;
while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
try { try {
if (config.SOCKS5PROXY.ENABLED) { if (config.SOCKS5PROXY.ENABLED) {
let socksOptions: any = { let socksOptions: any = {
@ -57,16 +77,25 @@ class FiatConversion {
if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) { if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
socksOptions.username = config.SOCKS5PROXY.USERNAME; socksOptions.username = config.SOCKS5PROXY.USERNAME;
socksOptions.password = config.SOCKS5PROXY.PASSWORD; socksOptions.password = config.SOCKS5PROXY.PASSWORD;
} else {
// Retry with different tor circuits https://stackoverflow.com/a/64960234
socksOptions.username = `circuit${retry}`;
} }
const agent = new SocksProxyAgent(socksOptions); // Handle proxy agent for onion addresses
fiatConversionUrl = config.PRICE_DATA_SERVER.TOR_URL; if (isHTTP) {
logger.debug('Querying currency rates service...'); axiosOptions.httpAgent = new SocksProxyAgent(socksOptions);
response = await axios.get(fiatConversionUrl, { httpAgent: agent, headers: headers, timeout: 30000 });
} else { } else {
fiatConversionUrl = config.PRICE_DATA_SERVER.CLEARNET_URL; axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions);
}
}
logger.debug('Querying currency rates service...'); logger.debug('Querying currency rates service...');
response = await axios.get(fiatConversionUrl, { headers: headers, timeout: 10000 });
const response: AxiosResponse = await axios.get(`${fiatConversionUrl}`, axiosOptions);
if (response.statusText === 'error' || !response.data) {
throw new Error(`Could not fetch data from ${fiatConversionUrl}, Error: ${response.status}`);
} }
for (const rate of response.data.data) { for (const rate of response.data.data) {
@ -81,8 +110,12 @@ class FiatConversion {
if (this.ratesChangedCallback) { if (this.ratesChangedCallback) {
this.ratesChangedCallback(this.conversionRates); this.ratesChangedCallback(this.conversionRates);
} }
break;
} catch (e) { } catch (e) {
logger.err('Error updating fiat conversion rates: ' + (e instanceof Error ? e.message : e)); logger.err('Error updating fiat conversion rates: ' + (e instanceof Error ? e.message : e));
await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL);
retry++;
}
} }
} }
} }

30
backend/src/config.ts
View File

@ -18,6 +18,9 @@ interface IConfig {
PRICE_FEED_UPDATE_INTERVAL: number; PRICE_FEED_UPDATE_INTERVAL: number;
USE_SECOND_NODE_FOR_MINFEE: boolean; USE_SECOND_NODE_FOR_MINFEE: boolean;
EXTERNAL_ASSETS: string[]; EXTERNAL_ASSETS: string[];
EXTERNAL_MAX_RETRY: number;
EXTERNAL_RETRY_INTERVAL: number;
USER_AGENT: string;
STDOUT_LOG_MIN_PRIORITY: 'emerg' | 'alert' | 'crit' | 'err' | 'warn' | 'notice' | 'info' | 'debug'; STDOUT_LOG_MIN_PRIORITY: 'emerg' | 'alert' | 'crit' | 'err' | 'warn' | 'notice' | 'info' | 'debug';
}; };
ESPLORA: { ESPLORA: {
@ -66,6 +69,7 @@ interface IConfig {
}; };
SOCKS5PROXY: { SOCKS5PROXY: {
ENABLED: boolean; ENABLED: boolean;
USE_ONION: boolean;
HOST: string; HOST: string;
PORT: number; PORT: number;
USERNAME: string; USERNAME: string;
@ -75,6 +79,14 @@ interface IConfig {
TOR_URL: string; TOR_URL: string;
CLEARNET_URL: string; CLEARNET_URL: string;
}; };
EXTERNAL_DATA_SERVER: {
MEMPOOL_API: string;
MEMPOOL_ONION: string;
LIQUID_API: string;
LIQUID_ONION: string;
BISQ_URL: string;
BISQ_ONION: string;
};
} }
const defaults: IConfig = { const defaults: IConfig = {
@ -94,9 +106,10 @@ const defaults: IConfig = {
'INDEXING_BLOCKS_AMOUNT': 11000, // 0 = disable indexing, -1 = index all blocks 'INDEXING_BLOCKS_AMOUNT': 11000, // 0 = disable indexing, -1 = index all blocks
'PRICE_FEED_UPDATE_INTERVAL': 600, 'PRICE_FEED_UPDATE_INTERVAL': 600,
'USE_SECOND_NODE_FOR_MINFEE': false, 'USE_SECOND_NODE_FOR_MINFEE': false,
'EXTERNAL_ASSETS': [ 'EXTERNAL_ASSETS': [],
'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json' 'EXTERNAL_MAX_RETRY': 1,
], 'EXTERNAL_RETRY_INTERVAL': 0,
'USER_AGENT': 'mempool',
'STDOUT_LOG_MIN_PRIORITY': 'debug', 'STDOUT_LOG_MIN_PRIORITY': 'debug',
}, },
'ESPLORA': { 'ESPLORA': {
@ -145,6 +158,7 @@ const defaults: IConfig = {
}, },
'SOCKS5PROXY': { 'SOCKS5PROXY': {
'ENABLED': false, 'ENABLED': false,
'USE_ONION': true,
'HOST': '127.0.0.1', 'HOST': '127.0.0.1',
'PORT': 9050, 'PORT': 9050,
'USERNAME': '', 'USERNAME': '',
@ -153,6 +167,14 @@ const defaults: IConfig = {
"PRICE_DATA_SERVER": { "PRICE_DATA_SERVER": {
'TOR_URL': 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices', 'TOR_URL': 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices',
'CLEARNET_URL': 'https://price.bisq.wiz.biz/getAllMarketPrices' 'CLEARNET_URL': 'https://price.bisq.wiz.biz/getAllMarketPrices'
},
"EXTERNAL_DATA_SERVER": {
'MEMPOOL_API': 'https://mempool.space/api/v1',
'MEMPOOL_ONION': 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1',
'LIQUID_API': 'https://liquid.network/api/v1',
'LIQUID_ONION': 'http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1',
'BISQ_URL': 'https://bisq.markets/api',
'BISQ_ONION': 'http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api'
} }
}; };
@ -168,6 +190,7 @@ class Config implements IConfig {
BISQ: IConfig['BISQ']; BISQ: IConfig['BISQ'];
SOCKS5PROXY: IConfig['SOCKS5PROXY']; SOCKS5PROXY: IConfig['SOCKS5PROXY'];
PRICE_DATA_SERVER: IConfig['PRICE_DATA_SERVER']; PRICE_DATA_SERVER: IConfig['PRICE_DATA_SERVER'];
EXTERNAL_DATA_SERVER: IConfig['EXTERNAL_DATA_SERVER'];
constructor() { constructor() {
const configs = this.merge(configFile, defaults); const configs = this.merge(configFile, defaults);
@ -182,6 +205,7 @@ class Config implements IConfig {
this.BISQ = configs.BISQ; this.BISQ = configs.BISQ;
this.SOCKS5PROXY = configs.SOCKS5PROXY; this.SOCKS5PROXY = configs.SOCKS5PROXY;
this.PRICE_DATA_SERVER = configs.PRICE_DATA_SERVER; this.PRICE_DATA_SERVER = configs.PRICE_DATA_SERVER;
this.EXTERNAL_DATA_SERVER = configs.EXTERNAL_DATA_SERVER;
} }
merge = (...objects: object[]): IConfig => { merge = (...objects: object[]): IConfig => {

12
backend/src/index.ts
View File

@ -205,7 +205,7 @@ class Server {
.post(config.MEMPOOL.API_URL_PREFIX + 'tx/push', routes.$postTransactionForm) .post(config.MEMPOOL.API_URL_PREFIX + 'tx/push', routes.$postTransactionForm)
.get(config.MEMPOOL.API_URL_PREFIX + 'donations', async (req, res) => { .get(config.MEMPOOL.API_URL_PREFIX + 'donations', async (req, res) => {
try { try {
const response = await axios.get('https://mempool.space/api/v1/donations', { responseType: 'stream', timeout: 10000 }); const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/donations`, { responseType: 'stream', timeout: 10000 });
response.data.pipe(res); response.data.pipe(res);
} catch (e) { } catch (e) {
res.status(500).end(); res.status(500).end();
@ -213,7 +213,7 @@ class Server {
}) })
.get(config.MEMPOOL.API_URL_PREFIX + 'donations/images/:id', async (req, res) => { .get(config.MEMPOOL.API_URL_PREFIX + 'donations/images/:id', async (req, res) => {
try { try {
const response = await axios.get('https://mempool.space/api/v1/donations/images/' + req.params.id, { const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/donations/images/${req.params.id}`, {
responseType: 'stream', timeout: 10000 responseType: 'stream', timeout: 10000
}); });
response.data.pipe(res); response.data.pipe(res);
@ -223,7 +223,7 @@ class Server {
}) })
.get(config.MEMPOOL.API_URL_PREFIX + 'contributors', async (req, res) => { .get(config.MEMPOOL.API_URL_PREFIX + 'contributors', async (req, res) => {
try { try {
const response = await axios.get('https://mempool.space/api/v1/contributors', { responseType: 'stream', timeout: 10000 }); const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/contributors`, { responseType: 'stream', timeout: 10000 });
response.data.pipe(res); response.data.pipe(res);
} catch (e) { } catch (e) {
res.status(500).end(); res.status(500).end();
@ -231,7 +231,7 @@ class Server {
}) })
.get(config.MEMPOOL.API_URL_PREFIX + 'contributors/images/:id', async (req, res) => { .get(config.MEMPOOL.API_URL_PREFIX + 'contributors/images/:id', async (req, res) => {
try { try {
const response = await axios.get('https://mempool.space/api/v1/contributors/images/' + req.params.id, { const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/contributors/images/${req.params.id}`, {
responseType: 'stream', timeout: 10000 responseType: 'stream', timeout: 10000
}); });
response.data.pipe(res); response.data.pipe(res);
@ -241,7 +241,7 @@ class Server {
}) })
.get(config.MEMPOOL.API_URL_PREFIX + 'translators', async (req, res) => { .get(config.MEMPOOL.API_URL_PREFIX + 'translators', async (req, res) => {
try { try {
const response = await axios.get('https://mempool.space/api/v1/translators', { responseType: 'stream', timeout: 10000 }); const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/translators`, { responseType: 'stream', timeout: 10000 });
response.data.pipe(res); response.data.pipe(res);
} catch (e) { } catch (e) {
res.status(500).end(); res.status(500).end();
@ -249,7 +249,7 @@ class Server {
}) })
.get(config.MEMPOOL.API_URL_PREFIX + 'translators/images/:id', async (req, res) => { .get(config.MEMPOOL.API_URL_PREFIX + 'translators/images/:id', async (req, res) => {
try { try {
const response = await axios.get('https://mempool.space/api/v1/translators/images/' + req.params.id, { const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/translators/images/${req.params.id}`, {
responseType: 'stream', timeout: 10000 responseType: 'stream', timeout: 10000
}); });
response.data.pipe(res); response.data.pipe(res);

4
backend/src/routes.ts
View File

@ -990,7 +990,7 @@ class Routes {
public async $getAllFeaturedLiquidAssets(req: Request, res: Response) { public async $getAllFeaturedLiquidAssets(req: Request, res: Response) {
try { try {
const response = await axios.get('https://liquid.network/api/v1/assets/featured', { responseType: 'stream', timeout: 10000 }); const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.LIQUID_API}/assets/featured`, { responseType: 'stream', timeout: 10000 });
response.data.pipe(res); response.data.pipe(res);
} catch (e) { } catch (e) {
res.status(500).end(); res.status(500).end();
@ -999,7 +999,7 @@ class Routes {
public async $getAssetGroup(req: Request, res: Response) { public async $getAssetGroup(req: Request, res: Response) {
try { try {
const response = await axios.get('https://liquid.network/api/v1/assets/group/' + parseInt(req.params.id, 10), const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.LIQUID_API}/assets/group/${parseInt(req.params.id, 10)}`,
{ responseType: 'stream', timeout: 10000 }); { responseType: 'stream', timeout: 10000 });
response.data.pipe(res); response.data.pipe(res);
} catch (e) { } catch (e) {

7
backend/src/sync-assets.ts
View File

@ -1,6 +1,7 @@
import axios, { AxiosResponse } from 'axios'; import axios, { AxiosResponse } from 'axios';
import * as fs from 'fs'; import * as fs from 'fs';
import config from './config'; import config from './config';
import backendInfo from './api/backend-info';
import logger from './logger'; import logger from './logger';
import { SocksProxyAgent } from 'socks-proxy-agent'; import { SocksProxyAgent } from 'socks-proxy-agent';
@ -42,6 +43,9 @@ class SyncAssets {
logger.info(`Downloading external asset ${fileName} over the Tor network...`); logger.info(`Downloading external asset ${fileName} over the Tor network...`);
return axios.get(url, { return axios.get(url, {
headers: {
'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
},
httpAgent: agent, httpAgent: agent,
httpsAgent: agent, httpsAgent: agent,
responseType: 'stream', responseType: 'stream',
@ -57,6 +61,9 @@ class SyncAssets {
} else { } else {
logger.info(`Downloading external asset ${fileName} over clearnet...`); logger.info(`Downloading external asset ${fileName} over clearnet...`);
return axios.get(url, { return axios.get(url, {
headers: {
'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
},
responseType: 'stream', responseType: 'stream',
timeout: 30000 timeout: 30000
}).then(function (response) { }).then(function (response) {

27
backend/src/tasks/pools-updater.ts
View File

@ -1,7 +1,8 @@
import axios from 'axios'; import axios, { AxiosResponse } from 'axios';
import poolsParser from '../api/pools-parser'; import poolsParser from '../api/pools-parser';
import config from '../config'; import config from '../config';
import DB from '../database'; import DB from '../database';
import backendInfo from '../api/backend-info';
import logger from '../logger'; import logger from '../logger';
import { SocksProxyAgent } from 'socks-proxy-agent'; import { SocksProxyAgent } from 'socks-proxy-agent';
import * as https from 'https'; import * as https from 'https';
@ -113,12 +114,23 @@ class PoolsUpdater {
*/ */
private async query(path): Promise<object | undefined> { private async query(path): Promise<object | undefined> {
type axiosOptions = { type axiosOptions = {
headers: {
'User-Agent': string
};
timeout: number;
httpsAgent?: https.Agent; httpsAgent?: https.Agent;
} }
const setDelay = (secs: number = 1): Promise<void> => new Promise(resolve => setTimeout(() => resolve(), secs * 1000)); const setDelay = (secs: number = 1): Promise<void> => new Promise(resolve => setTimeout(() => resolve(), secs * 1000));
const axiosOptions: axiosOptions = {}; const axiosOptions: axiosOptions = {
headers: {
'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
},
timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000
};
let retry = 0; let retry = 0;
while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
try {
if (config.SOCKS5PROXY.ENABLED) { if (config.SOCKS5PROXY.ENABLED) {
const socksOptions: any = { const socksOptions: any = {
agentOptions: { agentOptions: {
@ -131,15 +143,16 @@ class PoolsUpdater {
if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) { if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
socksOptions.username = config.SOCKS5PROXY.USERNAME; socksOptions.username = config.SOCKS5PROXY.USERNAME;
socksOptions.password = config.SOCKS5PROXY.PASSWORD; socksOptions.password = config.SOCKS5PROXY.PASSWORD;
} else {
// Retry with different tor circuits https://stackoverflow.com/a/64960234
socksOptions.username = `circuit${retry}`;
} }
axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions); axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions);
} }
while(retry < 5) { const data: AxiosResponse = await axios.get(path, axiosOptions);
try { if (data.statusText === 'error' || !data.data) {
const data = await axios.get(path, axiosOptions);
if (data.statusText !== 'OK' || !data.data) {
throw new Error(`Could not fetch data from Github, Error: ${data.status}`); throw new Error(`Could not fetch data from Github, Error: ${data.status}`);
} }
return data.data; return data.data;
@ -147,7 +160,7 @@ class PoolsUpdater {
logger.err('Could not connect to Github. Reason: ' + (e instanceof Error ? e.message : e)); logger.err('Could not connect to Github. Reason: ' + (e instanceof Error ? e.message : e));
retry++; retry++;
} }
await setDelay(); await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL);
} }
return undefined; return undefined;
} }

12
docker/backend/mempool-config.json
View File

@ -15,6 +15,9 @@
"PRICE_FEED_UPDATE_INTERVAL": __MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__, "PRICE_FEED_UPDATE_INTERVAL": __MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__,
"USE_SECOND_NODE_FOR_MINFEE": __MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__, "USE_SECOND_NODE_FOR_MINFEE": __MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__,
"EXTERNAL_ASSETS": __MEMPOOL_EXTERNAL_ASSETS__, "EXTERNAL_ASSETS": __MEMPOOL_EXTERNAL_ASSETS__,
"EXTERNAL_MAX_RETRY": __MEMPOOL_EXTERNAL_MAX_RETRY__,
"EXTERNAL_RETRY_INTERVAL": __MEMPOOL_EXTERNAL_RETRY_INTERVAL__,
"USER_AGENT": "__MEMPOOL_USER_AGENT__",
"STDOUT_LOG_MIN_PRIORITY": "__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__", "STDOUT_LOG_MIN_PRIORITY": "__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__",
"INDEXING_BLOCKS_AMOUNT": __MEMPOOL_INDEXING_BLOCKS_AMOUNT__ "INDEXING_BLOCKS_AMOUNT": __MEMPOOL_INDEXING_BLOCKS_AMOUNT__
}, },
@ -64,6 +67,7 @@
}, },
"SOCKS5PROXY": { "SOCKS5PROXY": {
"ENABLED": __SOCKS5PROXY_ENABLED__, "ENABLED": __SOCKS5PROXY_ENABLED__,
"USE_ONION": __SOCKS5PROXY_USE_ONION__,
"HOST": "__SOCKS5PROXY_HOST__", "HOST": "__SOCKS5PROXY_HOST__",
"PORT": "__SOCKS5PROXY_PORT__", "PORT": "__SOCKS5PROXY_PORT__",
"USERNAME": "__SOCKS5PROXY_USERNAME__", "USERNAME": "__SOCKS5PROXY_USERNAME__",
@ -72,5 +76,13 @@
"PRICE_DATA_SERVER": { "PRICE_DATA_SERVER": {
"TOR_URL": "__PRICE_DATA_SERVER_TOR_URL__", "TOR_URL": "__PRICE_DATA_SERVER_TOR_URL__",
"CLEARNET_URL": "__PRICE_DATA_SERVER_CLEARNET_URL__" "CLEARNET_URL": "__PRICE_DATA_SERVER_CLEARNET_URL__"
},
"EXTERNAL_DATA_SERVER": {
"MEMPOOL_API": "__EXTERNAL_DATA_SERVER_MEMPOOL_API__",
"MEMPOOL_ONION": "__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__",
"LIQUID_API": "__EXTERNAL_DATA_SERVER_LIQUID_API__",
"LIQUID_ONION": "__EXTERNAL_DATA_SERVER_LIQUID_ONION__",
"BISQ_URL": "__EXTERNAL_DATA_SERVER_BISQ_URL__",
"BISQ_ONION": "__EXTERNAL_DATA_SERVER_BISQ_ONION__"
} }
} }

14
docker/backend/start.sh
View File

@ -16,7 +16,10 @@ __MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__=${MEMPOOL_MEMPOOL_BLOCKS_AMOUNT:=8}
__MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=11000} __MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=11000}
__MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__=${MEMPOOL_PRICE_FEED_UPDATE_INTERVAL:=600} __MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__=${MEMPOOL_PRICE_FEED_UPDATE_INTERVAL:=600}
__MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__=${MEMPOOL_USE_SECOND_NODE_FOR_MINFEE:=false} __MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__=${MEMPOOL_USE_SECOND_NODE_FOR_MINFEE:=false}
__MEMPOOL_EXTERNAL_ASSETS__=${MEMPOOL_EXTERNAL_ASSETS:=[\"https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json\"]} __MEMPOOL_EXTERNAL_ASSETS__=${MEMPOOL_EXTERNAL_ASSETS:=[]}
__MEMPOOL_EXTERNAL_MAX_RETRY__=${MEMPOOL_EXTERNAL_MAX_RETRY:=1}
__MEMPOOL_EXTERNAL_RETRY_INTERVAL__=${MEMPOOL_EXTERNAL_RETRY_INTERVAL:=0}
__MEMPOOL_USER_AGENT__=${MEMPOOL_USER_AGENT:=mempool}
__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__=${MEMPOOL_STDOUT_LOG_MIN_PRIORITY:=info} __MEMPOOL_STDOUT_LOG_MIN_PRIORITY__=${MEMPOOL_STDOUT_LOG_MIN_PRIORITY:=info}
# CORE_RPC # CORE_RPC
@ -65,6 +68,7 @@ __BISQ_DATA_PATH__=${BISQ_DATA_PATH:=/bisq/statsnode-data/btc_mainnet/db}
# SOCKS5PROXY # SOCKS5PROXY
__SOCKS5PROXY_ENABLED__=${SOCKS5PROXY_ENABLED:=false} __SOCKS5PROXY_ENABLED__=${SOCKS5PROXY_ENABLED:=false}
__SOCKS5PROXY_USE_ONION__=${SOCKS5PROXY_USE_ONION:=true}
__SOCKS5PROXY_HOST__=${SOCKS5PROXY_HOST:=localhost} __SOCKS5PROXY_HOST__=${SOCKS5PROXY_HOST:=localhost}
__SOCKS5PROXY_PORT__=${SOCKS5PROXY_PORT:=9050} __SOCKS5PROXY_PORT__=${SOCKS5PROXY_PORT:=9050}
__SOCKS5PROXY_USERNAME__=${SOCKS5PROXY_USERNAME:=""} __SOCKS5PROXY_USERNAME__=${SOCKS5PROXY_USERNAME:=""}
@ -74,6 +78,14 @@ __SOCKS5PROXY_PASSWORD__=${SOCKS5PROXY_PASSWORD:=""}
__PRICE_DATA_SERVER_TOR_URL__=${PRICE_DATA_SERVER_TOR_URL:=http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices} __PRICE_DATA_SERVER_TOR_URL__=${PRICE_DATA_SERVER_TOR_URL:=http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices}
__PRICE_DATA_SERVER_CLEARNET_URL__=${PRICE_DATA_SERVER_CLEARNET_URL:=https://price.bisq.wiz.biz/getAllMarketPrices} __PRICE_DATA_SERVER_CLEARNET_URL__=${PRICE_DATA_SERVER_CLEARNET_URL:=https://price.bisq.wiz.biz/getAllMarketPrices}
# EXTERNAL_DATA_SERVER
__EXTERNAL_DATA_SERVER_MEMPOOL_API__=${EXTERNAL_DATA_SERVER_MEMPOOL_API:=https://mempool.space/api/v1}
__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__=${EXTERNAL_DATA_SERVER_MEMPOOL_ONION:=http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1}
__EXTERNAL_DATA_SERVER_LIQUID_API__=${EXTERNAL_DATA_SERVER_LIQUID_API:=https://liquid.network/api/v1}
__EXTERNAL_DATA_SERVER_LIQUID_ONION__=${EXTERNAL_DATA_SERVER_LIQUID_ONION:=http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1}
__EXTERNAL_DATA_SERVER_BISQ_URL__=${EXTERNAL_DATA_SERVER_BISQ_URL:=https://bisq.markets/api}
__EXTERNAL_DATA_SERVER_BISQ_ONION__=${EXTERNAL_DATA_SERVER_BISQ_ONION:=http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api}
mkdir -p "${__MEMPOOL_CACHE_DIR__}" mkdir -p "${__MEMPOOL_CACHE_DIR__}"
sed -i "s/__MEMPOOL_NETWORK__/${__MEMPOOL_NETWORK__}/g" mempool-config.json sed -i "s/__MEMPOOL_NETWORK__/${__MEMPOOL_NETWORK__}/g" mempool-config.json