From 9d5bbf1f44137efb0112522486be531a860d1379 Mon Sep 17 00:00:00 2001 From: Ayanami Date: Fri, 20 May 2022 12:55:28 +0900 Subject: [PATCH 1/9] Handle Error with basic retry while syncing external assets ( Price Data ) + Removed unused External Assets value + Make static URL dynamic + Added config options for syncing pool data + Added retry interval & max retry --- backend/mempool-config.sample.json | 17 ++++-- backend/src/api/bisq/bisq.ts | 59 ++++++++++++++++--- backend/src/api/fiat-conversion.ts | 95 +++++++++++++++++++----------- backend/src/config.ts | 30 ++++++++-- backend/src/index.ts | 12 ++-- backend/src/routes.ts | 4 +- backend/src/tasks/pools-updater.ts | 10 ++-- 7 files changed, 165 insertions(+), 62 deletions(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 6be8b1674..19758c0ed 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -15,9 +15,9 @@ "INDEXING_BLOCKS_AMOUNT": 11000, "PRICE_FEED_UPDATE_INTERVAL": 600, "USE_SECOND_NODE_FOR_MINFEE": false, - "EXTERNAL_ASSETS": [ - "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json" - ], + "EXTERNAL_ASSETS": [], + "EXTERNAL_MAX_RETRY": 10, + "EXTERNAL_RETRY_INTERVAL": 60, "STDOUT_LOG_MIN_PRIORITY": "debug" }, "CORE_RPC": { @@ -66,6 +66,7 @@ }, "SOCKS5PROXY": { "ENABLED": false, + "USE_ONION": true, "HOST": "127.0.0.1", "PORT": 9050, "USERNAME": "", @@ -73,6 +74,14 @@ }, "PRICE_DATA_SERVER": { "TOR_URL": "http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices", - "CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices" + "CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices", + "BISQ_URL": "https://bisq.markets/api", + "BISQ_ONION": "http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api" + }, + "EXTERNAL_DATA_SERVER": { + "MEMPOOL_API": "https://mempool.space/api/v1", + "MEMPOOL_ONION": "http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1", + "LIQUID_API": "https://liquid.network/api/v1", + "LIQUID_ONION": "http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1" } } diff --git a/backend/src/api/bisq/bisq.ts b/backend/src/api/bisq/bisq.ts index f16356d1c..332dd5e81 100644 --- a/backend/src/api/bisq/bisq.ts +++ b/backend/src/api/bisq/bisq.ts @@ -1,6 +1,9 @@ import config from '../../config'; import * as fs from 'fs'; -import axios from 'axios'; +import axios, { AxiosResponse } from 'axios'; +import * as http from 'http'; +import * as https from 'https'; +import { SocksProxyAgent } from 'socks-proxy-agent'; import { BisqBlocks, BisqBlock, BisqTransaction, BisqStats, BisqTrade } from './interfaces'; import { Common } from '../common'; import { BlockExtended } from '../../mempool.interfaces'; @@ -143,12 +146,47 @@ class Bisq { }, 2000); }); } + private async updatePrice() { + type axiosOptions = { + httpAgent?: http.Agent; + httpsAgent?: https.Agent; + } + const setDelay = (secs: number = 1): Promise => new Promise(resolve => setTimeout(() => resolve(), secs * 1000)); + const BISQ_URL = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.PRICE_DATA_SERVER.BISQ_ONION : config.PRICE_DATA_SERVER.BISQ_URL; + const isHTTP = (new URL(BISQ_URL).protocol.split(':')[0] === 'http') ? true : false; + const axiosOptions: axiosOptions = {}; + let retry = 0; - private updatePrice() { - axios.get('https://bisq.markets/api/trades/?market=bsq_btc', { timeout: 10000 }) - .then((response) => { + if (config.SOCKS5PROXY.ENABLED) { + const socksOptions: any = { + agentOptions: { + keepAlive: true, + }, + hostname: config.SOCKS5PROXY.HOST, + port: config.SOCKS5PROXY.PORT + }; + + if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) { + socksOptions.username = config.SOCKS5PROXY.USERNAME; + socksOptions.password = config.SOCKS5PROXY.PASSWORD; + } + + // Handle proxy agent for onion addresses + if (isHTTP) { + axiosOptions.httpAgent = new SocksProxyAgent(socksOptions); + } else { + axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions); + } + } + + while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) { + try { + const data: AxiosResponse = await axios.get(`${BISQ_URL}/trades/?market=bsq_btc`, axiosOptions); + if (data.statusText === 'error' || !data.data) { + throw new Error(`Could not fetch data from Bisq market, Error: ${data.status}`); + } const prices: number[] = []; - response.data.forEach((trade) => { + data.data.forEach((trade) => { prices.push(parseFloat(trade.price) * 100000000); }); prices.sort((a, b) => a - b); @@ -156,9 +194,14 @@ class Bisq { if (this.priceUpdateCallbackFunction) { this.priceUpdateCallbackFunction(this.price); } - }).catch((err) => { - logger.err('Error updating Bisq market price: ' + err); - }); + logger.debug('Successfully updated Bisq market price'); + break; + } catch (e) { + logger.err('Error updating Bisq market price: ' + (e instanceof Error ? e.message : e)); + await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL); + retry++; + } + } } private async loadBisqDumpFile(): Promise { diff --git a/backend/src/api/fiat-conversion.ts b/backend/src/api/fiat-conversion.ts index 3562760f1..5bc664099 100644 --- a/backend/src/api/fiat-conversion.ts +++ b/backend/src/api/fiat-conversion.ts @@ -1,4 +1,6 @@ import logger from '../logger'; +import * as http from 'http'; +import * as https from 'https'; import axios, { AxiosResponse } from 'axios'; import { IConversionRates } from '../mempool.interfaces'; import config from '../config'; @@ -40,49 +42,76 @@ class FiatConversion { } private async updateCurrency(): Promise { - const headers = { 'User-Agent': `mempool/v${backendInfo.getBackendInfo().version}` }; - let fiatConversionUrl: string; - let response: AxiosResponse; + type axiosOptions = { + headers: { + 'User-Agent': string + }; + timeout: number; + httpAgent?: http.Agent; + httpsAgent?: https.Agent; + } + const setDelay = (secs: number = 1): Promise => new Promise(resolve => setTimeout(() => resolve(), secs * 1000)); + const fiatConversionUrl = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.PRICE_DATA_SERVER.TOR_URL : config.PRICE_DATA_SERVER.CLEARNET_URL; + const isHTTP = (new URL(fiatConversionUrl).protocol.split(':')[0] === 'http') ? true : false; + const axiosOptions: axiosOptions = { + headers: { + 'User-Agent': `mempool/v${backendInfo.getBackendInfo().version}` + }, + timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000 + }; - try { - if (config.SOCKS5PROXY.ENABLED) { - let socksOptions: any = { - agentOptions: { - keepAlive: true, - }, - hostname: config.SOCKS5PROXY.HOST, - port: config.SOCKS5PROXY.PORT - }; + let retry = 0; - if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) { - socksOptions.username = config.SOCKS5PROXY.USERNAME; - socksOptions.password = config.SOCKS5PROXY.PASSWORD; - } + if (config.SOCKS5PROXY.ENABLED) { + let socksOptions: any = { + agentOptions: { + keepAlive: true, + }, + hostname: config.SOCKS5PROXY.HOST, + port: config.SOCKS5PROXY.PORT + }; - const agent = new SocksProxyAgent(socksOptions); - fiatConversionUrl = config.PRICE_DATA_SERVER.TOR_URL; - logger.debug('Querying currency rates service...'); - response = await axios.get(fiatConversionUrl, { httpAgent: agent, headers: headers, timeout: 30000 }); + if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) { + socksOptions.username = config.SOCKS5PROXY.USERNAME; + socksOptions.password = config.SOCKS5PROXY.PASSWORD; + } + + // Handle proxy agent for onion addresses + if (isHTTP) { + axiosOptions.httpAgent = new SocksProxyAgent(socksOptions); } else { - fiatConversionUrl = config.PRICE_DATA_SERVER.CLEARNET_URL; + axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions); + } + } + + while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) { + try { logger.debug('Querying currency rates service...'); - response = await axios.get(fiatConversionUrl, { headers: headers, timeout: 10000 }); - } - for (const rate of response.data.data) { - if (this.debasingFiatCurrencies.includes(rate.currencyCode) && rate.provider === 'Bisq-Aggregate') { - this.conversionRates[rate.currencyCode] = Math.round(100 * rate.price) / 100; + const response: AxiosResponse = await axios.get(`${fiatConversionUrl}`, axiosOptions); + + if (response.statusText === 'error' || !response.data) { + throw new Error(`Could not fetch data from ${fiatConversionUrl}, Error: ${response.status}`); } - } - this.ratesInitialized = true; - logger.debug(`USD Conversion Rate: ${this.conversionRates.USD}`); + for (const rate of response.data.data) { + if (this.debasingFiatCurrencies.includes(rate.currencyCode) && rate.provider === 'Bisq-Aggregate') { + this.conversionRates[rate.currencyCode] = Math.round(100 * rate.price) / 100; + } + } - if (this.ratesChangedCallback) { - this.ratesChangedCallback(this.conversionRates); + this.ratesInitialized = true; + logger.debug(`USD Conversion Rate: ${this.conversionRates.USD}`); + + if (this.ratesChangedCallback) { + this.ratesChangedCallback(this.conversionRates); + } + break; + } catch (e) { + logger.err('Error updating fiat conversion rates: ' + (e instanceof Error ? e.message : e)); + await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL); + retry++; } - } catch (e) { - logger.err('Error updating fiat conversion rates: ' + (e instanceof Error ? e.message : e)); } } } diff --git a/backend/src/config.ts b/backend/src/config.ts index 1aa37b8be..8295faeaa 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -18,6 +18,8 @@ interface IConfig { PRICE_FEED_UPDATE_INTERVAL: number; USE_SECOND_NODE_FOR_MINFEE: boolean; EXTERNAL_ASSETS: string[]; + EXTERNAL_MAX_RETRY: number; + EXTERNAL_RETRY_INTERVAL: number; STDOUT_LOG_MIN_PRIORITY: 'emerg' | 'alert' | 'crit' | 'err' | 'warn' | 'notice' | 'info' | 'debug'; }; ESPLORA: { @@ -66,6 +68,7 @@ interface IConfig { }; SOCKS5PROXY: { ENABLED: boolean; + USE_ONION: boolean; HOST: string; PORT: number; USERNAME: string; @@ -74,6 +77,14 @@ interface IConfig { PRICE_DATA_SERVER: { TOR_URL: string; CLEARNET_URL: string; + BISQ_URL: string; + BISQ_ONION: string; + }; + EXTERNAL_DATA_SERVER: { + MEMPOOL_API: string; + MEMPOOL_ONION: string; + LIQUID_API: string; + LIQUID_ONION: string; }; } @@ -94,9 +105,9 @@ const defaults: IConfig = { 'INDEXING_BLOCKS_AMOUNT': 11000, // 0 = disable indexing, -1 = index all blocks 'PRICE_FEED_UPDATE_INTERVAL': 600, 'USE_SECOND_NODE_FOR_MINFEE': false, - 'EXTERNAL_ASSETS': [ - 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json' - ], + 'EXTERNAL_ASSETS': [], + 'EXTERNAL_MAX_RETRY': 10, + 'EXTERNAL_RETRY_INTERVAL': 60, 'STDOUT_LOG_MIN_PRIORITY': 'debug', }, 'ESPLORA': { @@ -145,6 +156,7 @@ const defaults: IConfig = { }, 'SOCKS5PROXY': { 'ENABLED': false, + 'USE_ONION': true, 'HOST': '127.0.0.1', 'PORT': 9050, 'USERNAME': '', @@ -152,7 +164,15 @@ const defaults: IConfig = { }, "PRICE_DATA_SERVER": { 'TOR_URL': 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices', - 'CLEARNET_URL': 'https://price.bisq.wiz.biz/getAllMarketPrices' + 'CLEARNET_URL': 'https://price.bisq.wiz.biz/getAllMarketPrices', + 'BISQ_URL': 'https://bisq.markets/api', + 'BISQ_ONION': 'http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api' + }, + "EXTERNAL_DATA_SERVER": { + 'MEMPOOL_API': 'https://mempool.space/api/v1', + 'MEMPOOL_ONION': 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1', + 'LIQUID_API': 'https://liquid.network/api/v1', + 'LIQUID_ONION': 'http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1' } }; @@ -168,6 +188,7 @@ class Config implements IConfig { BISQ: IConfig['BISQ']; SOCKS5PROXY: IConfig['SOCKS5PROXY']; PRICE_DATA_SERVER: IConfig['PRICE_DATA_SERVER']; + EXTERNAL_DATA_SERVER: IConfig['EXTERNAL_DATA_SERVER']; constructor() { const configs = this.merge(configFile, defaults); @@ -182,6 +203,7 @@ class Config implements IConfig { this.BISQ = configs.BISQ; this.SOCKS5PROXY = configs.SOCKS5PROXY; this.PRICE_DATA_SERVER = configs.PRICE_DATA_SERVER; + this.EXTERNAL_DATA_SERVER = configs.EXTERNAL_DATA_SERVER; } merge = (...objects: object[]): IConfig => { diff --git a/backend/src/index.ts b/backend/src/index.ts index 42cfe9aea..d421a6fba 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -205,7 +205,7 @@ class Server { .post(config.MEMPOOL.API_URL_PREFIX + 'tx/push', routes.$postTransactionForm) .get(config.MEMPOOL.API_URL_PREFIX + 'donations', async (req, res) => { try { - const response = await axios.get('https://mempool.space/api/v1/donations', { responseType: 'stream', timeout: 10000 }); + const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/donations`, { responseType: 'stream', timeout: 10000 }); response.data.pipe(res); } catch (e) { res.status(500).end(); @@ -213,7 +213,7 @@ class Server { }) .get(config.MEMPOOL.API_URL_PREFIX + 'donations/images/:id', async (req, res) => { try { - const response = await axios.get('https://mempool.space/api/v1/donations/images/' + req.params.id, { + const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/donations/images/${req.params.id}`, { responseType: 'stream', timeout: 10000 }); response.data.pipe(res); @@ -223,7 +223,7 @@ class Server { }) .get(config.MEMPOOL.API_URL_PREFIX + 'contributors', async (req, res) => { try { - const response = await axios.get('https://mempool.space/api/v1/contributors', { responseType: 'stream', timeout: 10000 }); + const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/contributors`, { responseType: 'stream', timeout: 10000 }); response.data.pipe(res); } catch (e) { res.status(500).end(); @@ -231,7 +231,7 @@ class Server { }) .get(config.MEMPOOL.API_URL_PREFIX + 'contributors/images/:id', async (req, res) => { try { - const response = await axios.get('https://mempool.space/api/v1/contributors/images/' + req.params.id, { + const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/contributors/images/${req.params.id}`, { responseType: 'stream', timeout: 10000 }); response.data.pipe(res); @@ -241,7 +241,7 @@ class Server { }) .get(config.MEMPOOL.API_URL_PREFIX + 'translators', async (req, res) => { try { - const response = await axios.get('https://mempool.space/api/v1/translators', { responseType: 'stream', timeout: 10000 }); + const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/translators`, { responseType: 'stream', timeout: 10000 }); response.data.pipe(res); } catch (e) { res.status(500).end(); @@ -249,7 +249,7 @@ class Server { }) .get(config.MEMPOOL.API_URL_PREFIX + 'translators/images/:id', async (req, res) => { try { - const response = await axios.get('https://mempool.space/api/v1/translators/images/' + req.params.id, { + const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/translators/images/${req.params.id}`, { responseType: 'stream', timeout: 10000 }); response.data.pipe(res); diff --git a/backend/src/routes.ts b/backend/src/routes.ts index 91c41faa6..12368b1ee 100644 --- a/backend/src/routes.ts +++ b/backend/src/routes.ts @@ -990,7 +990,7 @@ class Routes { public async $getAllFeaturedLiquidAssets(req: Request, res: Response) { try { - const response = await axios.get('https://liquid.network/api/v1/assets/featured', { responseType: 'stream', timeout: 10000 }); + const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.LIQUID_API}/assets/featured`, { responseType: 'stream', timeout: 10000 }); response.data.pipe(res); } catch (e) { res.status(500).end(); @@ -999,7 +999,7 @@ class Routes { public async $getAssetGroup(req: Request, res: Response) { try { - const response = await axios.get('https://liquid.network/api/v1/assets/group/' + parseInt(req.params.id, 10), + const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.LIQUID_API}/assets/group/${parseInt(req.params.id, 10)}`, { responseType: 'stream', timeout: 10000 }); response.data.pipe(res); } catch (e) { diff --git a/backend/src/tasks/pools-updater.ts b/backend/src/tasks/pools-updater.ts index 05a1da5dc..6ff56548d 100644 --- a/backend/src/tasks/pools-updater.ts +++ b/backend/src/tasks/pools-updater.ts @@ -1,4 +1,4 @@ -import axios from 'axios'; +import axios, { AxiosResponse } from 'axios'; import poolsParser from '../api/pools-parser'; import config from '../config'; import DB from '../database'; @@ -136,10 +136,10 @@ class PoolsUpdater { axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions); } - while(retry < 5) { + while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) { try { - const data = await axios.get(path, axiosOptions); - if (data.statusText !== 'OK' || !data.data) { + const data: AxiosResponse = await axios.get(path, axiosOptions); + if (data.statusText === 'error' || !data.data) { throw new Error(`Could not fetch data from Github, Error: ${data.status}`); } return data.data; @@ -147,7 +147,7 @@ class PoolsUpdater { logger.err('Could not connect to Github. Reason: ' + (e instanceof Error ? e.message : e)); retry++; } - await setDelay(); + await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL); } return undefined; } From e41a08789abd663c3457ab11a5be10dfe27a5b79 Mon Sep 17 00:00:00 2001 From: Ayanami Date: Fri, 20 May 2022 23:38:16 +0900 Subject: [PATCH 2/9] Added configurable user-agent for axios Will use `mempool/v${backendInfo.getBackendInfo().version}` for default --- backend/mempool-config.sample.json | 1 + backend/src/api/bisq/bisq.ts | 12 +++++++++++- backend/src/api/fiat-conversion.ts | 2 +- backend/src/config.ts | 2 ++ backend/src/sync-assets.ts | 7 +++++++ backend/src/tasks/pools-updater.ts | 12 +++++++++++- 6 files changed, 33 insertions(+), 3 deletions(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 19758c0ed..625886f13 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -18,6 +18,7 @@ "EXTERNAL_ASSETS": [], "EXTERNAL_MAX_RETRY": 10, "EXTERNAL_RETRY_INTERVAL": 60, + "USER_AGENT": "mempool", "STDOUT_LOG_MIN_PRIORITY": "debug" }, "CORE_RPC": { diff --git a/backend/src/api/bisq/bisq.ts b/backend/src/api/bisq/bisq.ts index 332dd5e81..dcde77b06 100644 --- a/backend/src/api/bisq/bisq.ts +++ b/backend/src/api/bisq/bisq.ts @@ -8,6 +8,7 @@ import { BisqBlocks, BisqBlock, BisqTransaction, BisqStats, BisqTrade } from './ import { Common } from '../common'; import { BlockExtended } from '../../mempool.interfaces'; import { StaticPool } from 'node-worker-threads-pool'; +import backendInfo from '../backend-info'; import logger from '../../logger'; class Bisq { @@ -148,13 +149,22 @@ class Bisq { } private async updatePrice() { type axiosOptions = { + headers: { + 'User-Agent': string + }; + timeout: number; httpAgent?: http.Agent; httpsAgent?: https.Agent; } const setDelay = (secs: number = 1): Promise => new Promise(resolve => setTimeout(() => resolve(), secs * 1000)); const BISQ_URL = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.PRICE_DATA_SERVER.BISQ_ONION : config.PRICE_DATA_SERVER.BISQ_URL; const isHTTP = (new URL(BISQ_URL).protocol.split(':')[0] === 'http') ? true : false; - const axiosOptions: axiosOptions = {}; + const axiosOptions: axiosOptions = { + headers: { + 'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}` + }, + timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000 + }; let retry = 0; if (config.SOCKS5PROXY.ENABLED) { diff --git a/backend/src/api/fiat-conversion.ts b/backend/src/api/fiat-conversion.ts index 5bc664099..fb8717d78 100644 --- a/backend/src/api/fiat-conversion.ts +++ b/backend/src/api/fiat-conversion.ts @@ -55,7 +55,7 @@ class FiatConversion { const isHTTP = (new URL(fiatConversionUrl).protocol.split(':')[0] === 'http') ? true : false; const axiosOptions: axiosOptions = { headers: { - 'User-Agent': `mempool/v${backendInfo.getBackendInfo().version}` + 'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}` }, timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000 }; diff --git a/backend/src/config.ts b/backend/src/config.ts index 8295faeaa..c3d3a45a7 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -20,6 +20,7 @@ interface IConfig { EXTERNAL_ASSETS: string[]; EXTERNAL_MAX_RETRY: number; EXTERNAL_RETRY_INTERVAL: number; + USER_AGENT: string; STDOUT_LOG_MIN_PRIORITY: 'emerg' | 'alert' | 'crit' | 'err' | 'warn' | 'notice' | 'info' | 'debug'; }; ESPLORA: { @@ -108,6 +109,7 @@ const defaults: IConfig = { 'EXTERNAL_ASSETS': [], 'EXTERNAL_MAX_RETRY': 10, 'EXTERNAL_RETRY_INTERVAL': 60, + 'USER_AGENT': 'mempool', 'STDOUT_LOG_MIN_PRIORITY': 'debug', }, 'ESPLORA': { diff --git a/backend/src/sync-assets.ts b/backend/src/sync-assets.ts index 74d15e901..f4cc1f266 100644 --- a/backend/src/sync-assets.ts +++ b/backend/src/sync-assets.ts @@ -1,6 +1,7 @@ import axios, { AxiosResponse } from 'axios'; import * as fs from 'fs'; import config from './config'; +import backendInfo from './api/backend-info'; import logger from './logger'; import { SocksProxyAgent } from 'socks-proxy-agent'; @@ -42,6 +43,9 @@ class SyncAssets { logger.info(`Downloading external asset ${fileName} over the Tor network...`); return axios.get(url, { + headers: { + 'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}` + }, httpAgent: agent, httpsAgent: agent, responseType: 'stream', @@ -57,6 +61,9 @@ class SyncAssets { } else { logger.info(`Downloading external asset ${fileName} over clearnet...`); return axios.get(url, { + headers: { + 'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}` + }, responseType: 'stream', timeout: 30000 }).then(function (response) { diff --git a/backend/src/tasks/pools-updater.ts b/backend/src/tasks/pools-updater.ts index 6ff56548d..6140c3ed3 100644 --- a/backend/src/tasks/pools-updater.ts +++ b/backend/src/tasks/pools-updater.ts @@ -2,6 +2,7 @@ import axios, { AxiosResponse } from 'axios'; import poolsParser from '../api/pools-parser'; import config from '../config'; import DB from '../database'; +import backendInfo from '../api/backend-info'; import logger from '../logger'; import { SocksProxyAgent } from 'socks-proxy-agent'; import * as https from 'https'; @@ -113,10 +114,19 @@ class PoolsUpdater { */ private async query(path): Promise { type axiosOptions = { + headers: { + 'User-Agent': string + }; + timeout: number; httpsAgent?: https.Agent; } const setDelay = (secs: number = 1): Promise => new Promise(resolve => setTimeout(() => resolve(), secs * 1000)); - const axiosOptions: axiosOptions = {}; + const axiosOptions: axiosOptions = { + headers: { + 'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}` + }, + timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000 + }; let retry = 0; if (config.SOCKS5PROXY.ENABLED) { From c610cacee41dda5ff5f8b685d42397f095ab050c Mon Sep 17 00:00:00 2001 From: Ayanami Date: Sat, 21 May 2022 15:38:22 +0900 Subject: [PATCH 3/9] Added missing config value addressing comments from @knorrium --- backend/mempool-config.sample.json | 8 ++++---- backend/src/api/bisq/bisq.ts | 2 +- backend/src/config.ts | 12 ++++++------ docker/backend/mempool-config.json | 12 ++++++++++++ docker/backend/start.sh | 14 +++++++++++++- 5 files changed, 36 insertions(+), 12 deletions(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 625886f13..3f949491d 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -75,14 +75,14 @@ }, "PRICE_DATA_SERVER": { "TOR_URL": "http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices", - "CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices", - "BISQ_URL": "https://bisq.markets/api", - "BISQ_ONION": "http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api" + "CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices" }, "EXTERNAL_DATA_SERVER": { "MEMPOOL_API": "https://mempool.space/api/v1", "MEMPOOL_ONION": "http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1", "LIQUID_API": "https://liquid.network/api/v1", - "LIQUID_ONION": "http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1" + "LIQUID_ONION": "http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1", + "BISQ_URL": "https://bisq.markets/api", + "BISQ_ONION": "http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api" } } diff --git a/backend/src/api/bisq/bisq.ts b/backend/src/api/bisq/bisq.ts index dcde77b06..10b011b37 100644 --- a/backend/src/api/bisq/bisq.ts +++ b/backend/src/api/bisq/bisq.ts @@ -157,7 +157,7 @@ class Bisq { httpsAgent?: https.Agent; } const setDelay = (secs: number = 1): Promise => new Promise(resolve => setTimeout(() => resolve(), secs * 1000)); - const BISQ_URL = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.PRICE_DATA_SERVER.BISQ_ONION : config.PRICE_DATA_SERVER.BISQ_URL; + const BISQ_URL = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.EXTERNAL_DATA_SERVER.BISQ_ONION : config.EXTERNAL_DATA_SERVER.BISQ_URL; const isHTTP = (new URL(BISQ_URL).protocol.split(':')[0] === 'http') ? true : false; const axiosOptions: axiosOptions = { headers: { diff --git a/backend/src/config.ts b/backend/src/config.ts index c3d3a45a7..ee28a7da8 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -78,14 +78,14 @@ interface IConfig { PRICE_DATA_SERVER: { TOR_URL: string; CLEARNET_URL: string; - BISQ_URL: string; - BISQ_ONION: string; }; EXTERNAL_DATA_SERVER: { MEMPOOL_API: string; MEMPOOL_ONION: string; LIQUID_API: string; LIQUID_ONION: string; + BISQ_URL: string; + BISQ_ONION: string; }; } @@ -166,15 +166,15 @@ const defaults: IConfig = { }, "PRICE_DATA_SERVER": { 'TOR_URL': 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices', - 'CLEARNET_URL': 'https://price.bisq.wiz.biz/getAllMarketPrices', - 'BISQ_URL': 'https://bisq.markets/api', - 'BISQ_ONION': 'http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api' + 'CLEARNET_URL': 'https://price.bisq.wiz.biz/getAllMarketPrices' }, "EXTERNAL_DATA_SERVER": { 'MEMPOOL_API': 'https://mempool.space/api/v1', 'MEMPOOL_ONION': 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1', 'LIQUID_API': 'https://liquid.network/api/v1', - 'LIQUID_ONION': 'http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1' + 'LIQUID_ONION': 'http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1', + 'BISQ_URL': 'https://bisq.markets/api', + 'BISQ_ONION': 'http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api' } }; diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index 39d0e5fad..9a929a4f0 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -15,6 +15,9 @@ "PRICE_FEED_UPDATE_INTERVAL": __MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__, "USE_SECOND_NODE_FOR_MINFEE": __MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__, "EXTERNAL_ASSETS": __MEMPOOL_EXTERNAL_ASSETS__, + "EXTERNAL_MAX_RETRY": __MEMPOOL_EXTERNAL_MAX_RETRY__, + "EXTERNAL_RETRY_INTERVAL": __MEMPOOL_EXTERNAL_RETRY_INTERVAL__, + "USER_AGENT": "__MEMPOOL_USER_AGENT__", "STDOUT_LOG_MIN_PRIORITY": "__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__", "INDEXING_BLOCKS_AMOUNT": __MEMPOOL_INDEXING_BLOCKS_AMOUNT__ }, @@ -64,6 +67,7 @@ }, "SOCKS5PROXY": { "ENABLED": __SOCKS5PROXY_ENABLED__, + "USE_ONION": __SOCKS5PROXY_USE_ONION__, "HOST": "__SOCKS5PROXY_HOST__", "PORT": "__SOCKS5PROXY_PORT__", "USERNAME": "__SOCKS5PROXY_USERNAME__", @@ -72,5 +76,13 @@ "PRICE_DATA_SERVER": { "TOR_URL": "__PRICE_DATA_SERVER_TOR_URL__", "CLEARNET_URL": "__PRICE_DATA_SERVER_CLEARNET_URL__" + }, + "EXTERNAL_DATA_SERVER": { + "MEMPOOL_API": "__EXTERNAL_DATA_SERVER_MEMPOOL_API__", + "MEMPOOL_ONION": "__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__", + "LIQUID_API": "__EXTERNAL_DATA_SERVER_LIQUID_API__", + "LIQUID_ONION": "__EXTERNAL_DATA_SERVER_LIQUID_ONION__", + "BISQ_URL": "__EXTERNAL_DATA_SERVER_BISQ_URL__", + "BISQ_ONION": "__EXTERNAL_DATA_SERVER_BISQ_ONION__" } } diff --git a/docker/backend/start.sh b/docker/backend/start.sh index 960171e43..a8beeae5f 100644 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -16,7 +16,10 @@ __MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__=${MEMPOOL_MEMPOOL_BLOCKS_AMOUNT:=8} __MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=11000} __MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__=${MEMPOOL_PRICE_FEED_UPDATE_INTERVAL:=600} __MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__=${MEMPOOL_USE_SECOND_NODE_FOR_MINFEE:=false} -__MEMPOOL_EXTERNAL_ASSETS__=${MEMPOOL_EXTERNAL_ASSETS:=[\"https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json\"]} +__MEMPOOL_EXTERNAL_ASSETS__=${MEMPOOL_EXTERNAL_ASSETS:=[]} +__MEMPOOL_EXTERNAL_MAX_RETRY__=${MEMPOOL_EXTERNAL_MAX_RETRY:=10} +__MEMPOOL_EXTERNAL_RETRY_INTERVAL__=${MEMPOOL_EXTERNAL_RETRY_INTERVAL:=60} +__MEMPOOL_USER_AGENT__=${MEMPOOL_USER_AGENT:=mempool} __MEMPOOL_STDOUT_LOG_MIN_PRIORITY__=${MEMPOOL_STDOUT_LOG_MIN_PRIORITY:=info} # CORE_RPC @@ -65,6 +68,7 @@ __BISQ_DATA_PATH__=${BISQ_DATA_PATH:=/bisq/statsnode-data/btc_mainnet/db} # SOCKS5PROXY __SOCKS5PROXY_ENABLED__=${SOCKS5PROXY_ENABLED:=false} +__SOCKS5PROXY_USE_ONION__=${SOCKS5PROXY_USE_ONION:=true} __SOCKS5PROXY_HOST__=${SOCKS5PROXY_HOST:=localhost} __SOCKS5PROXY_PORT__=${SOCKS5PROXY_PORT:=9050} __SOCKS5PROXY_USERNAME__=${SOCKS5PROXY_USERNAME:=""} @@ -74,6 +78,14 @@ __SOCKS5PROXY_PASSWORD__=${SOCKS5PROXY_PASSWORD:=""} __PRICE_DATA_SERVER_TOR_URL__=${PRICE_DATA_SERVER_TOR_URL:=http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices} __PRICE_DATA_SERVER_CLEARNET_URL__=${PRICE_DATA_SERVER_CLEARNET_URL:=https://price.bisq.wiz.biz/getAllMarketPrices} +# EXTERNAL_DATA_SERVER +__EXTERNAL_DATA_SERVER_MEMPOOL_API__=${EXTERNAL_DATA_SERVER_MEMPOOL_API:=https://mempool.space/api/v1} +__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__=${EXTERNAL_DATA_SERVER_MEMPOOL_ONION:=http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1} +__EXTERNAL_DATA_SERVER_LIQUID_API__=${EXTERNAL_DATA_SERVER_LIQUID_API:=https://liquid.network/api/v1} +__EXTERNAL_DATA_SERVER_LIQUID_ONION__=${EXTERNAL_DATA_SERVER_LIQUID_ONION:=http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1} +__EXTERNAL_DATA_SERVER_BISQ_URL__=${EXTERNAL_DATA_SERVER_BISQ_URL:=https://bisq.markets/api} +__EXTERNAL_DATA_SERVER_BISQ_ONION__=${EXTERNAL_DATA_SERVER_BISQ_ONION:=http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api} + mkdir -p "${__MEMPOOL_CACHE_DIR__}" sed -i "s/__MEMPOOL_NETWORK__/${__MEMPOOL_NETWORK__}/g" mempool-config.json From 40bfc6bff3c8fef0dc9a1f13f02b9a137f82c7ec Mon Sep 17 00:00:00 2001 From: Ayanami Date: Mon, 23 May 2022 20:05:32 +0900 Subject: [PATCH 4/9] Include SocksProxyAgent inside while loop To address error Socks5 proxy rejected connection - Failure --- backend/src/api/bisq/bisq.ts | 44 +++++++++++++++--------------- backend/src/api/fiat-conversion.ts | 44 +++++++++++++++--------------- backend/src/tasks/pools-updater.ts | 34 +++++++++++------------ 3 files changed, 61 insertions(+), 61 deletions(-) diff --git a/backend/src/api/bisq/bisq.ts b/backend/src/api/bisq/bisq.ts index 10b011b37..4e2eb5fdd 100644 --- a/backend/src/api/bisq/bisq.ts +++ b/backend/src/api/bisq/bisq.ts @@ -167,30 +167,30 @@ class Bisq { }; let retry = 0; - if (config.SOCKS5PROXY.ENABLED) { - const socksOptions: any = { - agentOptions: { - keepAlive: true, - }, - hostname: config.SOCKS5PROXY.HOST, - port: config.SOCKS5PROXY.PORT - }; - - if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) { - socksOptions.username = config.SOCKS5PROXY.USERNAME; - socksOptions.password = config.SOCKS5PROXY.PASSWORD; - } - - // Handle proxy agent for onion addresses - if (isHTTP) { - axiosOptions.httpAgent = new SocksProxyAgent(socksOptions); - } else { - axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions); - } - } - while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) { try { + if (config.SOCKS5PROXY.ENABLED) { + const socksOptions: any = { + agentOptions: { + keepAlive: true, + }, + hostname: config.SOCKS5PROXY.HOST, + port: config.SOCKS5PROXY.PORT + }; + + if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) { + socksOptions.username = config.SOCKS5PROXY.USERNAME; + socksOptions.password = config.SOCKS5PROXY.PASSWORD; + } + + // Handle proxy agent for onion addresses + if (isHTTP) { + axiosOptions.httpAgent = new SocksProxyAgent(socksOptions); + } else { + axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions); + } + } + const data: AxiosResponse = await axios.get(`${BISQ_URL}/trades/?market=bsq_btc`, axiosOptions); if (data.statusText === 'error' || !data.data) { throw new Error(`Could not fetch data from Bisq market, Error: ${data.status}`); diff --git a/backend/src/api/fiat-conversion.ts b/backend/src/api/fiat-conversion.ts index fb8717d78..e8a504eb7 100644 --- a/backend/src/api/fiat-conversion.ts +++ b/backend/src/api/fiat-conversion.ts @@ -62,30 +62,30 @@ class FiatConversion { let retry = 0; - if (config.SOCKS5PROXY.ENABLED) { - let socksOptions: any = { - agentOptions: { - keepAlive: true, - }, - hostname: config.SOCKS5PROXY.HOST, - port: config.SOCKS5PROXY.PORT - }; - - if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) { - socksOptions.username = config.SOCKS5PROXY.USERNAME; - socksOptions.password = config.SOCKS5PROXY.PASSWORD; - } - - // Handle proxy agent for onion addresses - if (isHTTP) { - axiosOptions.httpAgent = new SocksProxyAgent(socksOptions); - } else { - axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions); - } - } - while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) { try { + if (config.SOCKS5PROXY.ENABLED) { + let socksOptions: any = { + agentOptions: { + keepAlive: true, + }, + hostname: config.SOCKS5PROXY.HOST, + port: config.SOCKS5PROXY.PORT + }; + + if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) { + socksOptions.username = config.SOCKS5PROXY.USERNAME; + socksOptions.password = config.SOCKS5PROXY.PASSWORD; + } + + // Handle proxy agent for onion addresses + if (isHTTP) { + axiosOptions.httpAgent = new SocksProxyAgent(socksOptions); + } else { + axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions); + } + } + logger.debug('Querying currency rates service...'); const response: AxiosResponse = await axios.get(`${fiatConversionUrl}`, axiosOptions); diff --git a/backend/src/tasks/pools-updater.ts b/backend/src/tasks/pools-updater.ts index 6140c3ed3..0211948d6 100644 --- a/backend/src/tasks/pools-updater.ts +++ b/backend/src/tasks/pools-updater.ts @@ -129,25 +129,25 @@ class PoolsUpdater { }; let retry = 0; - if (config.SOCKS5PROXY.ENABLED) { - const socksOptions: any = { - agentOptions: { - keepAlive: true, - }, - hostname: config.SOCKS5PROXY.HOST, - port: config.SOCKS5PROXY.PORT - }; - - if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) { - socksOptions.username = config.SOCKS5PROXY.USERNAME; - socksOptions.password = config.SOCKS5PROXY.PASSWORD; - } - - axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions); - } - while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) { try { + if (config.SOCKS5PROXY.ENABLED) { + const socksOptions: any = { + agentOptions: { + keepAlive: true, + }, + hostname: config.SOCKS5PROXY.HOST, + port: config.SOCKS5PROXY.PORT + }; + + if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) { + socksOptions.username = config.SOCKS5PROXY.USERNAME; + socksOptions.password = config.SOCKS5PROXY.PASSWORD; + } + + axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions); + } + const data: AxiosResponse = await axios.get(path, axiosOptions); if (data.statusText === 'error' || !data.data) { throw new Error(`Could not fetch data from Github, Error: ${data.status}`); From 6ff473ab5d951c099a7c9621ee61034cd5b3d5f4 Mon Sep 17 00:00:00 2001 From: Ayanami Date: Tue, 24 May 2022 15:46:15 +0900 Subject: [PATCH 5/9] Add an ability to change circuits --- backend/src/api/bisq/bisq.ts | 3 +++ backend/src/api/fiat-conversion.ts | 3 +++ backend/src/tasks/pools-updater.ts | 3 +++ 3 files changed, 9 insertions(+) diff --git a/backend/src/api/bisq/bisq.ts b/backend/src/api/bisq/bisq.ts index 4e2eb5fdd..ed9c4fc3e 100644 --- a/backend/src/api/bisq/bisq.ts +++ b/backend/src/api/bisq/bisq.ts @@ -181,6 +181,9 @@ class Bisq { if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) { socksOptions.username = config.SOCKS5PROXY.USERNAME; socksOptions.password = config.SOCKS5PROXY.PASSWORD; + } else { + // Retry with different tor circuits https://stackoverflow.com/a/64960234 + socksOptions.username = `circuit${retry}`; } // Handle proxy agent for onion addresses diff --git a/backend/src/api/fiat-conversion.ts b/backend/src/api/fiat-conversion.ts index e8a504eb7..bed86e92f 100644 --- a/backend/src/api/fiat-conversion.ts +++ b/backend/src/api/fiat-conversion.ts @@ -76,6 +76,9 @@ class FiatConversion { if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) { socksOptions.username = config.SOCKS5PROXY.USERNAME; socksOptions.password = config.SOCKS5PROXY.PASSWORD; + } else { + // Retry with different tor circuits https://stackoverflow.com/a/64960234 + socksOptions.username = `circuit${retry}`; } // Handle proxy agent for onion addresses diff --git a/backend/src/tasks/pools-updater.ts b/backend/src/tasks/pools-updater.ts index 0211948d6..471c066ad 100644 --- a/backend/src/tasks/pools-updater.ts +++ b/backend/src/tasks/pools-updater.ts @@ -143,6 +143,9 @@ class PoolsUpdater { if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) { socksOptions.username = config.SOCKS5PROXY.USERNAME; socksOptions.password = config.SOCKS5PROXY.PASSWORD; + } else { + // Retry with different tor circuits https://stackoverflow.com/a/64960234 + socksOptions.username = `circuit${retry}`; } axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions); From a04bafdb4cacab1e7196ddcac3c18b302f963518 Mon Sep 17 00:00:00 2001 From: Ayanami Date: Sat, 28 May 2022 00:26:27 +0900 Subject: [PATCH 6/9] Correct the log if the onion address is enabled or not --- backend/src/api/fiat-conversion.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/src/api/fiat-conversion.ts b/backend/src/api/fiat-conversion.ts index bed86e92f..ffbe6a758 100644 --- a/backend/src/api/fiat-conversion.ts +++ b/backend/src/api/fiat-conversion.ts @@ -27,9 +27,10 @@ class FiatConversion { } public startService() { + const fiatConversionUrl = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.PRICE_DATA_SERVER.TOR_URL : config.PRICE_DATA_SERVER.CLEARNET_URL; logger.info('Starting currency rates service'); if (config.SOCKS5PROXY.ENABLED) { - logger.info(`Currency rates service will be queried over the Tor network using ${config.PRICE_DATA_SERVER.TOR_URL}`); + logger.info(`Currency rates service will be queried over the Tor network using ${fiatConversionUrl}`); } else { logger.info(`Currency rates service will be queried over clearnet using ${config.PRICE_DATA_SERVER.CLEARNET_URL}`); } From 56dc3376722f01f018dc4ca4530b76bbf7c1e6cf Mon Sep 17 00:00:00 2001 From: Ayanami Date: Wed, 1 Jun 2022 23:38:28 +0900 Subject: [PATCH 7/9] Temporary disable retries Until we find out how to sync async --- backend/mempool-config.sample.json | 4 ++-- backend/src/config.ts | 4 ++-- docker/backend/start.sh | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 3f949491d..77b571136 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -16,8 +16,8 @@ "PRICE_FEED_UPDATE_INTERVAL": 600, "USE_SECOND_NODE_FOR_MINFEE": false, "EXTERNAL_ASSETS": [], - "EXTERNAL_MAX_RETRY": 10, - "EXTERNAL_RETRY_INTERVAL": 60, + "EXTERNAL_MAX_RETRY": 1, + "EXTERNAL_RETRY_INTERVAL": 0, "USER_AGENT": "mempool", "STDOUT_LOG_MIN_PRIORITY": "debug" }, diff --git a/backend/src/config.ts b/backend/src/config.ts index ee28a7da8..e49da3dc9 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -107,8 +107,8 @@ const defaults: IConfig = { 'PRICE_FEED_UPDATE_INTERVAL': 600, 'USE_SECOND_NODE_FOR_MINFEE': false, 'EXTERNAL_ASSETS': [], - 'EXTERNAL_MAX_RETRY': 10, - 'EXTERNAL_RETRY_INTERVAL': 60, + 'EXTERNAL_MAX_RETRY': 1, + 'EXTERNAL_RETRY_INTERVAL': 0, 'USER_AGENT': 'mempool', 'STDOUT_LOG_MIN_PRIORITY': 'debug', }, diff --git a/docker/backend/start.sh b/docker/backend/start.sh index a8beeae5f..bdfed68ae 100644 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -17,8 +17,8 @@ __MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=11000} __MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__=${MEMPOOL_PRICE_FEED_UPDATE_INTERVAL:=600} __MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__=${MEMPOOL_USE_SECOND_NODE_FOR_MINFEE:=false} __MEMPOOL_EXTERNAL_ASSETS__=${MEMPOOL_EXTERNAL_ASSETS:=[]} -__MEMPOOL_EXTERNAL_MAX_RETRY__=${MEMPOOL_EXTERNAL_MAX_RETRY:=10} -__MEMPOOL_EXTERNAL_RETRY_INTERVAL__=${MEMPOOL_EXTERNAL_RETRY_INTERVAL:=60} +__MEMPOOL_EXTERNAL_MAX_RETRY__=${MEMPOOL_EXTERNAL_MAX_RETRY:=1} +__MEMPOOL_EXTERNAL_RETRY_INTERVAL__=${MEMPOOL_EXTERNAL_RETRY_INTERVAL:=0} __MEMPOOL_USER_AGENT__=${MEMPOOL_USER_AGENT:=mempool} __MEMPOOL_STDOUT_LOG_MIN_PRIORITY__=${MEMPOOL_STDOUT_LOG_MIN_PRIORITY:=info} From e133467ea1081318ad69bed498e94805dcf709cc Mon Sep 17 00:00:00 2001 From: hunicus <93150691+hunicus@users.noreply.github.com> Date: Tue, 7 Jun 2022 08:41:07 -0400 Subject: [PATCH 8/9] Add pull request template to discourage tx prs --- .github/pull_request_template.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .github/pull_request_template.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..0b3668cf1 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,6 @@ + From c318993a79784ceeccb7cf8f3158b3a18ea427e2 Mon Sep 17 00:00:00 2001 From: hunicus <93150691+hunicus@users.noreply.github.com> Date: Tue, 7 Jun 2022 08:25:12 -0400 Subject: [PATCH 9/9] Add transifex note to issue template --- .github/ISSUE_TEMPLATE/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index d26923e74..1d01fd18d 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -3,3 +3,6 @@ contact_links: - name: 🙋 Need help? Chat with us on Matrix url: https://matrix.to/#/#mempool.support:bitcoin.kyoto about: For support requests or general questions + - name: 🌐 Want to help with translations? Use Transifex + url: https://www.transifex.com/mempool/mempool + about: All translations work is done on Transifex