From 1aad89ac97c08504aafbebd994574ec9747c94ed Mon Sep 17 00:00:00 2001 From: hunicus <93150691+hunicus@users.noreply.github.com> Date: Mon, 6 Jun 2022 12:29:02 -0400 Subject: [PATCH 01/11] Update fee faq --- frontend/src/app/docs/api-docs/api-docs-data.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/app/docs/api-docs/api-docs-data.ts b/frontend/src/app/docs/api-docs/api-docs-data.ts index eb98c6596..1927c62d9 100644 --- a/frontend/src/app/docs/api-docs/api-docs-data.ts +++ b/frontend/src/app/docs/api-docs/api-docs-data.ts @@ -6107,7 +6107,7 @@ export const faqData = [ showConditions: bitcoinNetworks, fragment: "looking-up-fee-estimates", title: "How can I look up fee estimates?", - answer: "

See real-time fee estimates on the main dashboard.

Low priority is suggested for confirmation within 6 blocks (~1 hour), Medium priority is suggested for confirmation within 3 blocks (~30 minutes), and High priority is suggested for confirmation in the next block (~10 minutes).

" + answer: "

See real-time fee estimates on the main dashboard.

Here is an overview of Mempool's feerate suggestions:

In all cases, the suggested feerate is adjusted lower if any of the projected blocks involved in the calculation are not full (e.g. if there is only 1 projected block with a size of 1.2 MWU, Mempool will suggest a feerate of 1 sat/vB—not the median feerate of transactions in the block).

Projected blocks use feerates, transaction sizes, and other metrics to forecast which transactions will be in future blocks. Actual blocks will turn out to be different: miners have their own views of the mempool, their own algorithms for determining which transactions to include in a block, etc.

Ultimately, the Bitcoin network is not perfectly predictable, so fee estimation cannot be perfectly precise.

Use Mempool's feerate suggestions as a guide, and understand that they do not guarantee transaction confirmation in any period of time.

" }, { type: "endpoint", From 9d5bbf1f44137efb0112522486be531a860d1379 Mon Sep 17 00:00:00 2001 From: Ayanami Date: Fri, 20 May 2022 12:55:28 +0900 Subject: [PATCH 02/11] Handle Error with basic retry while syncing external assets ( Price Data ) + Removed unused External Assets value + Make static URL dynamic + Added config options for syncing pool data + Added retry interval & max retry --- backend/mempool-config.sample.json | 17 ++++-- backend/src/api/bisq/bisq.ts | 59 ++++++++++++++++--- backend/src/api/fiat-conversion.ts | 95 +++++++++++++++++++----------- backend/src/config.ts | 30 ++++++++-- backend/src/index.ts | 12 ++-- backend/src/routes.ts | 4 +- backend/src/tasks/pools-updater.ts | 10 ++-- 7 files changed, 165 insertions(+), 62 deletions(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 6be8b1674..19758c0ed 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -15,9 +15,9 @@ "INDEXING_BLOCKS_AMOUNT": 11000, "PRICE_FEED_UPDATE_INTERVAL": 600, "USE_SECOND_NODE_FOR_MINFEE": false, - "EXTERNAL_ASSETS": [ - "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json" - ], + "EXTERNAL_ASSETS": [], + "EXTERNAL_MAX_RETRY": 10, + "EXTERNAL_RETRY_INTERVAL": 60, "STDOUT_LOG_MIN_PRIORITY": "debug" }, "CORE_RPC": { @@ -66,6 +66,7 @@ }, "SOCKS5PROXY": { "ENABLED": false, + "USE_ONION": true, "HOST": "127.0.0.1", "PORT": 9050, "USERNAME": "", @@ -73,6 +74,14 @@ }, "PRICE_DATA_SERVER": { "TOR_URL": "http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices", - "CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices" + "CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices", + "BISQ_URL": "https://bisq.markets/api", + "BISQ_ONION": "http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api" + }, + "EXTERNAL_DATA_SERVER": { + "MEMPOOL_API": "https://mempool.space/api/v1", + "MEMPOOL_ONION": "http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1", + "LIQUID_API": "https://liquid.network/api/v1", + "LIQUID_ONION": "http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1" } } diff --git a/backend/src/api/bisq/bisq.ts b/backend/src/api/bisq/bisq.ts index f16356d1c..332dd5e81 100644 --- a/backend/src/api/bisq/bisq.ts +++ b/backend/src/api/bisq/bisq.ts @@ -1,6 +1,9 @@ import config from '../../config'; import * as fs from 'fs'; -import axios from 'axios'; +import axios, { AxiosResponse } from 'axios'; +import * as http from 'http'; +import * as https from 'https'; +import { SocksProxyAgent } from 'socks-proxy-agent'; import { BisqBlocks, BisqBlock, BisqTransaction, BisqStats, BisqTrade } from './interfaces'; import { Common } from '../common'; import { BlockExtended } from '../../mempool.interfaces'; @@ -143,12 +146,47 @@ class Bisq { }, 2000); }); } + private async updatePrice() { + type axiosOptions = { + httpAgent?: http.Agent; + httpsAgent?: https.Agent; + } + const setDelay = (secs: number = 1): Promise => new Promise(resolve => setTimeout(() => resolve(), secs * 1000)); + const BISQ_URL = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.PRICE_DATA_SERVER.BISQ_ONION : config.PRICE_DATA_SERVER.BISQ_URL; + const isHTTP = (new URL(BISQ_URL).protocol.split(':')[0] === 'http') ? true : false; + const axiosOptions: axiosOptions = {}; + let retry = 0; - private updatePrice() { - axios.get('https://bisq.markets/api/trades/?market=bsq_btc', { timeout: 10000 }) - .then((response) => { + if (config.SOCKS5PROXY.ENABLED) { + const socksOptions: any = { + agentOptions: { + keepAlive: true, + }, + hostname: config.SOCKS5PROXY.HOST, + port: config.SOCKS5PROXY.PORT + }; + + if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) { + socksOptions.username = config.SOCKS5PROXY.USERNAME; + socksOptions.password = config.SOCKS5PROXY.PASSWORD; + } + + // Handle proxy agent for onion addresses + if (isHTTP) { + axiosOptions.httpAgent = new SocksProxyAgent(socksOptions); + } else { + axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions); + } + } + + while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) { + try { + const data: AxiosResponse = await axios.get(`${BISQ_URL}/trades/?market=bsq_btc`, axiosOptions); + if (data.statusText === 'error' || !data.data) { + throw new Error(`Could not fetch data from Bisq market, Error: ${data.status}`); + } const prices: number[] = []; - response.data.forEach((trade) => { + data.data.forEach((trade) => { prices.push(parseFloat(trade.price) * 100000000); }); prices.sort((a, b) => a - b); @@ -156,9 +194,14 @@ class Bisq { if (this.priceUpdateCallbackFunction) { this.priceUpdateCallbackFunction(this.price); } - }).catch((err) => { - logger.err('Error updating Bisq market price: ' + err); - }); + logger.debug('Successfully updated Bisq market price'); + break; + } catch (e) { + logger.err('Error updating Bisq market price: ' + (e instanceof Error ? e.message : e)); + await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL); + retry++; + } + } } private async loadBisqDumpFile(): Promise { diff --git a/backend/src/api/fiat-conversion.ts b/backend/src/api/fiat-conversion.ts index 3562760f1..5bc664099 100644 --- a/backend/src/api/fiat-conversion.ts +++ b/backend/src/api/fiat-conversion.ts @@ -1,4 +1,6 @@ import logger from '../logger'; +import * as http from 'http'; +import * as https from 'https'; import axios, { AxiosResponse } from 'axios'; import { IConversionRates } from '../mempool.interfaces'; import config from '../config'; @@ -40,49 +42,76 @@ class FiatConversion { } private async updateCurrency(): Promise { - const headers = { 'User-Agent': `mempool/v${backendInfo.getBackendInfo().version}` }; - let fiatConversionUrl: string; - let response: AxiosResponse; + type axiosOptions = { + headers: { + 'User-Agent': string + }; + timeout: number; + httpAgent?: http.Agent; + httpsAgent?: https.Agent; + } + const setDelay = (secs: number = 1): Promise => new Promise(resolve => setTimeout(() => resolve(), secs * 1000)); + const fiatConversionUrl = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.PRICE_DATA_SERVER.TOR_URL : config.PRICE_DATA_SERVER.CLEARNET_URL; + const isHTTP = (new URL(fiatConversionUrl).protocol.split(':')[0] === 'http') ? true : false; + const axiosOptions: axiosOptions = { + headers: { + 'User-Agent': `mempool/v${backendInfo.getBackendInfo().version}` + }, + timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000 + }; - try { - if (config.SOCKS5PROXY.ENABLED) { - let socksOptions: any = { - agentOptions: { - keepAlive: true, - }, - hostname: config.SOCKS5PROXY.HOST, - port: config.SOCKS5PROXY.PORT - }; + let retry = 0; - if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) { - socksOptions.username = config.SOCKS5PROXY.USERNAME; - socksOptions.password = config.SOCKS5PROXY.PASSWORD; - } + if (config.SOCKS5PROXY.ENABLED) { + let socksOptions: any = { + agentOptions: { + keepAlive: true, + }, + hostname: config.SOCKS5PROXY.HOST, + port: config.SOCKS5PROXY.PORT + }; - const agent = new SocksProxyAgent(socksOptions); - fiatConversionUrl = config.PRICE_DATA_SERVER.TOR_URL; - logger.debug('Querying currency rates service...'); - response = await axios.get(fiatConversionUrl, { httpAgent: agent, headers: headers, timeout: 30000 }); + if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) { + socksOptions.username = config.SOCKS5PROXY.USERNAME; + socksOptions.password = config.SOCKS5PROXY.PASSWORD; + } + + // Handle proxy agent for onion addresses + if (isHTTP) { + axiosOptions.httpAgent = new SocksProxyAgent(socksOptions); } else { - fiatConversionUrl = config.PRICE_DATA_SERVER.CLEARNET_URL; + axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions); + } + } + + while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) { + try { logger.debug('Querying currency rates service...'); - response = await axios.get(fiatConversionUrl, { headers: headers, timeout: 10000 }); - } - for (const rate of response.data.data) { - if (this.debasingFiatCurrencies.includes(rate.currencyCode) && rate.provider === 'Bisq-Aggregate') { - this.conversionRates[rate.currencyCode] = Math.round(100 * rate.price) / 100; + const response: AxiosResponse = await axios.get(`${fiatConversionUrl}`, axiosOptions); + + if (response.statusText === 'error' || !response.data) { + throw new Error(`Could not fetch data from ${fiatConversionUrl}, Error: ${response.status}`); } - } - this.ratesInitialized = true; - logger.debug(`USD Conversion Rate: ${this.conversionRates.USD}`); + for (const rate of response.data.data) { + if (this.debasingFiatCurrencies.includes(rate.currencyCode) && rate.provider === 'Bisq-Aggregate') { + this.conversionRates[rate.currencyCode] = Math.round(100 * rate.price) / 100; + } + } - if (this.ratesChangedCallback) { - this.ratesChangedCallback(this.conversionRates); + this.ratesInitialized = true; + logger.debug(`USD Conversion Rate: ${this.conversionRates.USD}`); + + if (this.ratesChangedCallback) { + this.ratesChangedCallback(this.conversionRates); + } + break; + } catch (e) { + logger.err('Error updating fiat conversion rates: ' + (e instanceof Error ? e.message : e)); + await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL); + retry++; } - } catch (e) { - logger.err('Error updating fiat conversion rates: ' + (e instanceof Error ? e.message : e)); } } } diff --git a/backend/src/config.ts b/backend/src/config.ts index 1aa37b8be..8295faeaa 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -18,6 +18,8 @@ interface IConfig { PRICE_FEED_UPDATE_INTERVAL: number; USE_SECOND_NODE_FOR_MINFEE: boolean; EXTERNAL_ASSETS: string[]; + EXTERNAL_MAX_RETRY: number; + EXTERNAL_RETRY_INTERVAL: number; STDOUT_LOG_MIN_PRIORITY: 'emerg' | 'alert' | 'crit' | 'err' | 'warn' | 'notice' | 'info' | 'debug'; }; ESPLORA: { @@ -66,6 +68,7 @@ interface IConfig { }; SOCKS5PROXY: { ENABLED: boolean; + USE_ONION: boolean; HOST: string; PORT: number; USERNAME: string; @@ -74,6 +77,14 @@ interface IConfig { PRICE_DATA_SERVER: { TOR_URL: string; CLEARNET_URL: string; + BISQ_URL: string; + BISQ_ONION: string; + }; + EXTERNAL_DATA_SERVER: { + MEMPOOL_API: string; + MEMPOOL_ONION: string; + LIQUID_API: string; + LIQUID_ONION: string; }; } @@ -94,9 +105,9 @@ const defaults: IConfig = { 'INDEXING_BLOCKS_AMOUNT': 11000, // 0 = disable indexing, -1 = index all blocks 'PRICE_FEED_UPDATE_INTERVAL': 600, 'USE_SECOND_NODE_FOR_MINFEE': false, - 'EXTERNAL_ASSETS': [ - 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json' - ], + 'EXTERNAL_ASSETS': [], + 'EXTERNAL_MAX_RETRY': 10, + 'EXTERNAL_RETRY_INTERVAL': 60, 'STDOUT_LOG_MIN_PRIORITY': 'debug', }, 'ESPLORA': { @@ -145,6 +156,7 @@ const defaults: IConfig = { }, 'SOCKS5PROXY': { 'ENABLED': false, + 'USE_ONION': true, 'HOST': '127.0.0.1', 'PORT': 9050, 'USERNAME': '', @@ -152,7 +164,15 @@ const defaults: IConfig = { }, "PRICE_DATA_SERVER": { 'TOR_URL': 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices', - 'CLEARNET_URL': 'https://price.bisq.wiz.biz/getAllMarketPrices' + 'CLEARNET_URL': 'https://price.bisq.wiz.biz/getAllMarketPrices', + 'BISQ_URL': 'https://bisq.markets/api', + 'BISQ_ONION': 'http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api' + }, + "EXTERNAL_DATA_SERVER": { + 'MEMPOOL_API': 'https://mempool.space/api/v1', + 'MEMPOOL_ONION': 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1', + 'LIQUID_API': 'https://liquid.network/api/v1', + 'LIQUID_ONION': 'http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1' } }; @@ -168,6 +188,7 @@ class Config implements IConfig { BISQ: IConfig['BISQ']; SOCKS5PROXY: IConfig['SOCKS5PROXY']; PRICE_DATA_SERVER: IConfig['PRICE_DATA_SERVER']; + EXTERNAL_DATA_SERVER: IConfig['EXTERNAL_DATA_SERVER']; constructor() { const configs = this.merge(configFile, defaults); @@ -182,6 +203,7 @@ class Config implements IConfig { this.BISQ = configs.BISQ; this.SOCKS5PROXY = configs.SOCKS5PROXY; this.PRICE_DATA_SERVER = configs.PRICE_DATA_SERVER; + this.EXTERNAL_DATA_SERVER = configs.EXTERNAL_DATA_SERVER; } merge = (...objects: object[]): IConfig => { diff --git a/backend/src/index.ts b/backend/src/index.ts index 42cfe9aea..d421a6fba 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -205,7 +205,7 @@ class Server { .post(config.MEMPOOL.API_URL_PREFIX + 'tx/push', routes.$postTransactionForm) .get(config.MEMPOOL.API_URL_PREFIX + 'donations', async (req, res) => { try { - const response = await axios.get('https://mempool.space/api/v1/donations', { responseType: 'stream', timeout: 10000 }); + const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/donations`, { responseType: 'stream', timeout: 10000 }); response.data.pipe(res); } catch (e) { res.status(500).end(); @@ -213,7 +213,7 @@ class Server { }) .get(config.MEMPOOL.API_URL_PREFIX + 'donations/images/:id', async (req, res) => { try { - const response = await axios.get('https://mempool.space/api/v1/donations/images/' + req.params.id, { + const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/donations/images/${req.params.id}`, { responseType: 'stream', timeout: 10000 }); response.data.pipe(res); @@ -223,7 +223,7 @@ class Server { }) .get(config.MEMPOOL.API_URL_PREFIX + 'contributors', async (req, res) => { try { - const response = await axios.get('https://mempool.space/api/v1/contributors', { responseType: 'stream', timeout: 10000 }); + const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/contributors`, { responseType: 'stream', timeout: 10000 }); response.data.pipe(res); } catch (e) { res.status(500).end(); @@ -231,7 +231,7 @@ class Server { }) .get(config.MEMPOOL.API_URL_PREFIX + 'contributors/images/:id', async (req, res) => { try { - const response = await axios.get('https://mempool.space/api/v1/contributors/images/' + req.params.id, { + const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/contributors/images/${req.params.id}`, { responseType: 'stream', timeout: 10000 }); response.data.pipe(res); @@ -241,7 +241,7 @@ class Server { }) .get(config.MEMPOOL.API_URL_PREFIX + 'translators', async (req, res) => { try { - const response = await axios.get('https://mempool.space/api/v1/translators', { responseType: 'stream', timeout: 10000 }); + const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/translators`, { responseType: 'stream', timeout: 10000 }); response.data.pipe(res); } catch (e) { res.status(500).end(); @@ -249,7 +249,7 @@ class Server { }) .get(config.MEMPOOL.API_URL_PREFIX + 'translators/images/:id', async (req, res) => { try { - const response = await axios.get('https://mempool.space/api/v1/translators/images/' + req.params.id, { + const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/translators/images/${req.params.id}`, { responseType: 'stream', timeout: 10000 }); response.data.pipe(res); diff --git a/backend/src/routes.ts b/backend/src/routes.ts index 91c41faa6..12368b1ee 100644 --- a/backend/src/routes.ts +++ b/backend/src/routes.ts @@ -990,7 +990,7 @@ class Routes { public async $getAllFeaturedLiquidAssets(req: Request, res: Response) { try { - const response = await axios.get('https://liquid.network/api/v1/assets/featured', { responseType: 'stream', timeout: 10000 }); + const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.LIQUID_API}/assets/featured`, { responseType: 'stream', timeout: 10000 }); response.data.pipe(res); } catch (e) { res.status(500).end(); @@ -999,7 +999,7 @@ class Routes { public async $getAssetGroup(req: Request, res: Response) { try { - const response = await axios.get('https://liquid.network/api/v1/assets/group/' + parseInt(req.params.id, 10), + const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.LIQUID_API}/assets/group/${parseInt(req.params.id, 10)}`, { responseType: 'stream', timeout: 10000 }); response.data.pipe(res); } catch (e) { diff --git a/backend/src/tasks/pools-updater.ts b/backend/src/tasks/pools-updater.ts index 05a1da5dc..6ff56548d 100644 --- a/backend/src/tasks/pools-updater.ts +++ b/backend/src/tasks/pools-updater.ts @@ -1,4 +1,4 @@ -import axios from 'axios'; +import axios, { AxiosResponse } from 'axios'; import poolsParser from '../api/pools-parser'; import config from '../config'; import DB from '../database'; @@ -136,10 +136,10 @@ class PoolsUpdater { axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions); } - while(retry < 5) { + while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) { try { - const data = await axios.get(path, axiosOptions); - if (data.statusText !== 'OK' || !data.data) { + const data: AxiosResponse = await axios.get(path, axiosOptions); + if (data.statusText === 'error' || !data.data) { throw new Error(`Could not fetch data from Github, Error: ${data.status}`); } return data.data; @@ -147,7 +147,7 @@ class PoolsUpdater { logger.err('Could not connect to Github. Reason: ' + (e instanceof Error ? e.message : e)); retry++; } - await setDelay(); + await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL); } return undefined; } From e41a08789abd663c3457ab11a5be10dfe27a5b79 Mon Sep 17 00:00:00 2001 From: Ayanami Date: Fri, 20 May 2022 23:38:16 +0900 Subject: [PATCH 03/11] Added configurable user-agent for axios Will use `mempool/v${backendInfo.getBackendInfo().version}` for default --- backend/mempool-config.sample.json | 1 + backend/src/api/bisq/bisq.ts | 12 +++++++++++- backend/src/api/fiat-conversion.ts | 2 +- backend/src/config.ts | 2 ++ backend/src/sync-assets.ts | 7 +++++++ backend/src/tasks/pools-updater.ts | 12 +++++++++++- 6 files changed, 33 insertions(+), 3 deletions(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 19758c0ed..625886f13 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -18,6 +18,7 @@ "EXTERNAL_ASSETS": [], "EXTERNAL_MAX_RETRY": 10, "EXTERNAL_RETRY_INTERVAL": 60, + "USER_AGENT": "mempool", "STDOUT_LOG_MIN_PRIORITY": "debug" }, "CORE_RPC": { diff --git a/backend/src/api/bisq/bisq.ts b/backend/src/api/bisq/bisq.ts index 332dd5e81..dcde77b06 100644 --- a/backend/src/api/bisq/bisq.ts +++ b/backend/src/api/bisq/bisq.ts @@ -8,6 +8,7 @@ import { BisqBlocks, BisqBlock, BisqTransaction, BisqStats, BisqTrade } from './ import { Common } from '../common'; import { BlockExtended } from '../../mempool.interfaces'; import { StaticPool } from 'node-worker-threads-pool'; +import backendInfo from '../backend-info'; import logger from '../../logger'; class Bisq { @@ -148,13 +149,22 @@ class Bisq { } private async updatePrice() { type axiosOptions = { + headers: { + 'User-Agent': string + }; + timeout: number; httpAgent?: http.Agent; httpsAgent?: https.Agent; } const setDelay = (secs: number = 1): Promise => new Promise(resolve => setTimeout(() => resolve(), secs * 1000)); const BISQ_URL = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.PRICE_DATA_SERVER.BISQ_ONION : config.PRICE_DATA_SERVER.BISQ_URL; const isHTTP = (new URL(BISQ_URL).protocol.split(':')[0] === 'http') ? true : false; - const axiosOptions: axiosOptions = {}; + const axiosOptions: axiosOptions = { + headers: { + 'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}` + }, + timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000 + }; let retry = 0; if (config.SOCKS5PROXY.ENABLED) { diff --git a/backend/src/api/fiat-conversion.ts b/backend/src/api/fiat-conversion.ts index 5bc664099..fb8717d78 100644 --- a/backend/src/api/fiat-conversion.ts +++ b/backend/src/api/fiat-conversion.ts @@ -55,7 +55,7 @@ class FiatConversion { const isHTTP = (new URL(fiatConversionUrl).protocol.split(':')[0] === 'http') ? true : false; const axiosOptions: axiosOptions = { headers: { - 'User-Agent': `mempool/v${backendInfo.getBackendInfo().version}` + 'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}` }, timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000 }; diff --git a/backend/src/config.ts b/backend/src/config.ts index 8295faeaa..c3d3a45a7 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -20,6 +20,7 @@ interface IConfig { EXTERNAL_ASSETS: string[]; EXTERNAL_MAX_RETRY: number; EXTERNAL_RETRY_INTERVAL: number; + USER_AGENT: string; STDOUT_LOG_MIN_PRIORITY: 'emerg' | 'alert' | 'crit' | 'err' | 'warn' | 'notice' | 'info' | 'debug'; }; ESPLORA: { @@ -108,6 +109,7 @@ const defaults: IConfig = { 'EXTERNAL_ASSETS': [], 'EXTERNAL_MAX_RETRY': 10, 'EXTERNAL_RETRY_INTERVAL': 60, + 'USER_AGENT': 'mempool', 'STDOUT_LOG_MIN_PRIORITY': 'debug', }, 'ESPLORA': { diff --git a/backend/src/sync-assets.ts b/backend/src/sync-assets.ts index 74d15e901..f4cc1f266 100644 --- a/backend/src/sync-assets.ts +++ b/backend/src/sync-assets.ts @@ -1,6 +1,7 @@ import axios, { AxiosResponse } from 'axios'; import * as fs from 'fs'; import config from './config'; +import backendInfo from './api/backend-info'; import logger from './logger'; import { SocksProxyAgent } from 'socks-proxy-agent'; @@ -42,6 +43,9 @@ class SyncAssets { logger.info(`Downloading external asset ${fileName} over the Tor network...`); return axios.get(url, { + headers: { + 'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}` + }, httpAgent: agent, httpsAgent: agent, responseType: 'stream', @@ -57,6 +61,9 @@ class SyncAssets { } else { logger.info(`Downloading external asset ${fileName} over clearnet...`); return axios.get(url, { + headers: { + 'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}` + }, responseType: 'stream', timeout: 30000 }).then(function (response) { diff --git a/backend/src/tasks/pools-updater.ts b/backend/src/tasks/pools-updater.ts index 6ff56548d..6140c3ed3 100644 --- a/backend/src/tasks/pools-updater.ts +++ b/backend/src/tasks/pools-updater.ts @@ -2,6 +2,7 @@ import axios, { AxiosResponse } from 'axios'; import poolsParser from '../api/pools-parser'; import config from '../config'; import DB from '../database'; +import backendInfo from '../api/backend-info'; import logger from '../logger'; import { SocksProxyAgent } from 'socks-proxy-agent'; import * as https from 'https'; @@ -113,10 +114,19 @@ class PoolsUpdater { */ private async query(path): Promise { type axiosOptions = { + headers: { + 'User-Agent': string + }; + timeout: number; httpsAgent?: https.Agent; } const setDelay = (secs: number = 1): Promise => new Promise(resolve => setTimeout(() => resolve(), secs * 1000)); - const axiosOptions: axiosOptions = {}; + const axiosOptions: axiosOptions = { + headers: { + 'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}` + }, + timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000 + }; let retry = 0; if (config.SOCKS5PROXY.ENABLED) { From c610cacee41dda5ff5f8b685d42397f095ab050c Mon Sep 17 00:00:00 2001 From: Ayanami Date: Sat, 21 May 2022 15:38:22 +0900 Subject: [PATCH 04/11] Added missing config value addressing comments from @knorrium --- backend/mempool-config.sample.json | 8 ++++---- backend/src/api/bisq/bisq.ts | 2 +- backend/src/config.ts | 12 ++++++------ docker/backend/mempool-config.json | 12 ++++++++++++ docker/backend/start.sh | 14 +++++++++++++- 5 files changed, 36 insertions(+), 12 deletions(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 625886f13..3f949491d 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -75,14 +75,14 @@ }, "PRICE_DATA_SERVER": { "TOR_URL": "http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices", - "CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices", - "BISQ_URL": "https://bisq.markets/api", - "BISQ_ONION": "http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api" + "CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices" }, "EXTERNAL_DATA_SERVER": { "MEMPOOL_API": "https://mempool.space/api/v1", "MEMPOOL_ONION": "http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1", "LIQUID_API": "https://liquid.network/api/v1", - "LIQUID_ONION": "http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1" + "LIQUID_ONION": "http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1", + "BISQ_URL": "https://bisq.markets/api", + "BISQ_ONION": "http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api" } } diff --git a/backend/src/api/bisq/bisq.ts b/backend/src/api/bisq/bisq.ts index dcde77b06..10b011b37 100644 --- a/backend/src/api/bisq/bisq.ts +++ b/backend/src/api/bisq/bisq.ts @@ -157,7 +157,7 @@ class Bisq { httpsAgent?: https.Agent; } const setDelay = (secs: number = 1): Promise => new Promise(resolve => setTimeout(() => resolve(), secs * 1000)); - const BISQ_URL = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.PRICE_DATA_SERVER.BISQ_ONION : config.PRICE_DATA_SERVER.BISQ_URL; + const BISQ_URL = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.EXTERNAL_DATA_SERVER.BISQ_ONION : config.EXTERNAL_DATA_SERVER.BISQ_URL; const isHTTP = (new URL(BISQ_URL).protocol.split(':')[0] === 'http') ? true : false; const axiosOptions: axiosOptions = { headers: { diff --git a/backend/src/config.ts b/backend/src/config.ts index c3d3a45a7..ee28a7da8 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -78,14 +78,14 @@ interface IConfig { PRICE_DATA_SERVER: { TOR_URL: string; CLEARNET_URL: string; - BISQ_URL: string; - BISQ_ONION: string; }; EXTERNAL_DATA_SERVER: { MEMPOOL_API: string; MEMPOOL_ONION: string; LIQUID_API: string; LIQUID_ONION: string; + BISQ_URL: string; + BISQ_ONION: string; }; } @@ -166,15 +166,15 @@ const defaults: IConfig = { }, "PRICE_DATA_SERVER": { 'TOR_URL': 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices', - 'CLEARNET_URL': 'https://price.bisq.wiz.biz/getAllMarketPrices', - 'BISQ_URL': 'https://bisq.markets/api', - 'BISQ_ONION': 'http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api' + 'CLEARNET_URL': 'https://price.bisq.wiz.biz/getAllMarketPrices' }, "EXTERNAL_DATA_SERVER": { 'MEMPOOL_API': 'https://mempool.space/api/v1', 'MEMPOOL_ONION': 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1', 'LIQUID_API': 'https://liquid.network/api/v1', - 'LIQUID_ONION': 'http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1' + 'LIQUID_ONION': 'http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1', + 'BISQ_URL': 'https://bisq.markets/api', + 'BISQ_ONION': 'http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api' } }; diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index 39d0e5fad..9a929a4f0 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -15,6 +15,9 @@ "PRICE_FEED_UPDATE_INTERVAL": __MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__, "USE_SECOND_NODE_FOR_MINFEE": __MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__, "EXTERNAL_ASSETS": __MEMPOOL_EXTERNAL_ASSETS__, + "EXTERNAL_MAX_RETRY": __MEMPOOL_EXTERNAL_MAX_RETRY__, + "EXTERNAL_RETRY_INTERVAL": __MEMPOOL_EXTERNAL_RETRY_INTERVAL__, + "USER_AGENT": "__MEMPOOL_USER_AGENT__", "STDOUT_LOG_MIN_PRIORITY": "__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__", "INDEXING_BLOCKS_AMOUNT": __MEMPOOL_INDEXING_BLOCKS_AMOUNT__ }, @@ -64,6 +67,7 @@ }, "SOCKS5PROXY": { "ENABLED": __SOCKS5PROXY_ENABLED__, + "USE_ONION": __SOCKS5PROXY_USE_ONION__, "HOST": "__SOCKS5PROXY_HOST__", "PORT": "__SOCKS5PROXY_PORT__", "USERNAME": "__SOCKS5PROXY_USERNAME__", @@ -72,5 +76,13 @@ "PRICE_DATA_SERVER": { "TOR_URL": "__PRICE_DATA_SERVER_TOR_URL__", "CLEARNET_URL": "__PRICE_DATA_SERVER_CLEARNET_URL__" + }, + "EXTERNAL_DATA_SERVER": { + "MEMPOOL_API": "__EXTERNAL_DATA_SERVER_MEMPOOL_API__", + "MEMPOOL_ONION": "__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__", + "LIQUID_API": "__EXTERNAL_DATA_SERVER_LIQUID_API__", + "LIQUID_ONION": "__EXTERNAL_DATA_SERVER_LIQUID_ONION__", + "BISQ_URL": "__EXTERNAL_DATA_SERVER_BISQ_URL__", + "BISQ_ONION": "__EXTERNAL_DATA_SERVER_BISQ_ONION__" } } diff --git a/docker/backend/start.sh b/docker/backend/start.sh index 960171e43..a8beeae5f 100644 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -16,7 +16,10 @@ __MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__=${MEMPOOL_MEMPOOL_BLOCKS_AMOUNT:=8} __MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=11000} __MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__=${MEMPOOL_PRICE_FEED_UPDATE_INTERVAL:=600} __MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__=${MEMPOOL_USE_SECOND_NODE_FOR_MINFEE:=false} -__MEMPOOL_EXTERNAL_ASSETS__=${MEMPOOL_EXTERNAL_ASSETS:=[\"https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json\"]} +__MEMPOOL_EXTERNAL_ASSETS__=${MEMPOOL_EXTERNAL_ASSETS:=[]} +__MEMPOOL_EXTERNAL_MAX_RETRY__=${MEMPOOL_EXTERNAL_MAX_RETRY:=10} +__MEMPOOL_EXTERNAL_RETRY_INTERVAL__=${MEMPOOL_EXTERNAL_RETRY_INTERVAL:=60} +__MEMPOOL_USER_AGENT__=${MEMPOOL_USER_AGENT:=mempool} __MEMPOOL_STDOUT_LOG_MIN_PRIORITY__=${MEMPOOL_STDOUT_LOG_MIN_PRIORITY:=info} # CORE_RPC @@ -65,6 +68,7 @@ __BISQ_DATA_PATH__=${BISQ_DATA_PATH:=/bisq/statsnode-data/btc_mainnet/db} # SOCKS5PROXY __SOCKS5PROXY_ENABLED__=${SOCKS5PROXY_ENABLED:=false} +__SOCKS5PROXY_USE_ONION__=${SOCKS5PROXY_USE_ONION:=true} __SOCKS5PROXY_HOST__=${SOCKS5PROXY_HOST:=localhost} __SOCKS5PROXY_PORT__=${SOCKS5PROXY_PORT:=9050} __SOCKS5PROXY_USERNAME__=${SOCKS5PROXY_USERNAME:=""} @@ -74,6 +78,14 @@ __SOCKS5PROXY_PASSWORD__=${SOCKS5PROXY_PASSWORD:=""} __PRICE_DATA_SERVER_TOR_URL__=${PRICE_DATA_SERVER_TOR_URL:=http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices} __PRICE_DATA_SERVER_CLEARNET_URL__=${PRICE_DATA_SERVER_CLEARNET_URL:=https://price.bisq.wiz.biz/getAllMarketPrices} +# EXTERNAL_DATA_SERVER +__EXTERNAL_DATA_SERVER_MEMPOOL_API__=${EXTERNAL_DATA_SERVER_MEMPOOL_API:=https://mempool.space/api/v1} +__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__=${EXTERNAL_DATA_SERVER_MEMPOOL_ONION:=http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1} +__EXTERNAL_DATA_SERVER_LIQUID_API__=${EXTERNAL_DATA_SERVER_LIQUID_API:=https://liquid.network/api/v1} +__EXTERNAL_DATA_SERVER_LIQUID_ONION__=${EXTERNAL_DATA_SERVER_LIQUID_ONION:=http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1} +__EXTERNAL_DATA_SERVER_BISQ_URL__=${EXTERNAL_DATA_SERVER_BISQ_URL:=https://bisq.markets/api} +__EXTERNAL_DATA_SERVER_BISQ_ONION__=${EXTERNAL_DATA_SERVER_BISQ_ONION:=http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api} + mkdir -p "${__MEMPOOL_CACHE_DIR__}" sed -i "s/__MEMPOOL_NETWORK__/${__MEMPOOL_NETWORK__}/g" mempool-config.json From 40bfc6bff3c8fef0dc9a1f13f02b9a137f82c7ec Mon Sep 17 00:00:00 2001 From: Ayanami Date: Mon, 23 May 2022 20:05:32 +0900 Subject: [PATCH 05/11] Include SocksProxyAgent inside while loop To address error Socks5 proxy rejected connection - Failure --- backend/src/api/bisq/bisq.ts | 44 +++++++++++++++--------------- backend/src/api/fiat-conversion.ts | 44 +++++++++++++++--------------- backend/src/tasks/pools-updater.ts | 34 +++++++++++------------ 3 files changed, 61 insertions(+), 61 deletions(-) diff --git a/backend/src/api/bisq/bisq.ts b/backend/src/api/bisq/bisq.ts index 10b011b37..4e2eb5fdd 100644 --- a/backend/src/api/bisq/bisq.ts +++ b/backend/src/api/bisq/bisq.ts @@ -167,30 +167,30 @@ class Bisq { }; let retry = 0; - if (config.SOCKS5PROXY.ENABLED) { - const socksOptions: any = { - agentOptions: { - keepAlive: true, - }, - hostname: config.SOCKS5PROXY.HOST, - port: config.SOCKS5PROXY.PORT - }; - - if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) { - socksOptions.username = config.SOCKS5PROXY.USERNAME; - socksOptions.password = config.SOCKS5PROXY.PASSWORD; - } - - // Handle proxy agent for onion addresses - if (isHTTP) { - axiosOptions.httpAgent = new SocksProxyAgent(socksOptions); - } else { - axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions); - } - } - while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) { try { + if (config.SOCKS5PROXY.ENABLED) { + const socksOptions: any = { + agentOptions: { + keepAlive: true, + }, + hostname: config.SOCKS5PROXY.HOST, + port: config.SOCKS5PROXY.PORT + }; + + if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) { + socksOptions.username = config.SOCKS5PROXY.USERNAME; + socksOptions.password = config.SOCKS5PROXY.PASSWORD; + } + + // Handle proxy agent for onion addresses + if (isHTTP) { + axiosOptions.httpAgent = new SocksProxyAgent(socksOptions); + } else { + axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions); + } + } + const data: AxiosResponse = await axios.get(`${BISQ_URL}/trades/?market=bsq_btc`, axiosOptions); if (data.statusText === 'error' || !data.data) { throw new Error(`Could not fetch data from Bisq market, Error: ${data.status}`); diff --git a/backend/src/api/fiat-conversion.ts b/backend/src/api/fiat-conversion.ts index fb8717d78..e8a504eb7 100644 --- a/backend/src/api/fiat-conversion.ts +++ b/backend/src/api/fiat-conversion.ts @@ -62,30 +62,30 @@ class FiatConversion { let retry = 0; - if (config.SOCKS5PROXY.ENABLED) { - let socksOptions: any = { - agentOptions: { - keepAlive: true, - }, - hostname: config.SOCKS5PROXY.HOST, - port: config.SOCKS5PROXY.PORT - }; - - if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) { - socksOptions.username = config.SOCKS5PROXY.USERNAME; - socksOptions.password = config.SOCKS5PROXY.PASSWORD; - } - - // Handle proxy agent for onion addresses - if (isHTTP) { - axiosOptions.httpAgent = new SocksProxyAgent(socksOptions); - } else { - axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions); - } - } - while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) { try { + if (config.SOCKS5PROXY.ENABLED) { + let socksOptions: any = { + agentOptions: { + keepAlive: true, + }, + hostname: config.SOCKS5PROXY.HOST, + port: config.SOCKS5PROXY.PORT + }; + + if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) { + socksOptions.username = config.SOCKS5PROXY.USERNAME; + socksOptions.password = config.SOCKS5PROXY.PASSWORD; + } + + // Handle proxy agent for onion addresses + if (isHTTP) { + axiosOptions.httpAgent = new SocksProxyAgent(socksOptions); + } else { + axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions); + } + } + logger.debug('Querying currency rates service...'); const response: AxiosResponse = await axios.get(`${fiatConversionUrl}`, axiosOptions); diff --git a/backend/src/tasks/pools-updater.ts b/backend/src/tasks/pools-updater.ts index 6140c3ed3..0211948d6 100644 --- a/backend/src/tasks/pools-updater.ts +++ b/backend/src/tasks/pools-updater.ts @@ -129,25 +129,25 @@ class PoolsUpdater { }; let retry = 0; - if (config.SOCKS5PROXY.ENABLED) { - const socksOptions: any = { - agentOptions: { - keepAlive: true, - }, - hostname: config.SOCKS5PROXY.HOST, - port: config.SOCKS5PROXY.PORT - }; - - if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) { - socksOptions.username = config.SOCKS5PROXY.USERNAME; - socksOptions.password = config.SOCKS5PROXY.PASSWORD; - } - - axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions); - } - while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) { try { + if (config.SOCKS5PROXY.ENABLED) { + const socksOptions: any = { + agentOptions: { + keepAlive: true, + }, + hostname: config.SOCKS5PROXY.HOST, + port: config.SOCKS5PROXY.PORT + }; + + if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) { + socksOptions.username = config.SOCKS5PROXY.USERNAME; + socksOptions.password = config.SOCKS5PROXY.PASSWORD; + } + + axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions); + } + const data: AxiosResponse = await axios.get(path, axiosOptions); if (data.statusText === 'error' || !data.data) { throw new Error(`Could not fetch data from Github, Error: ${data.status}`); From 6ff473ab5d951c099a7c9621ee61034cd5b3d5f4 Mon Sep 17 00:00:00 2001 From: Ayanami Date: Tue, 24 May 2022 15:46:15 +0900 Subject: [PATCH 06/11] Add an ability to change circuits --- backend/src/api/bisq/bisq.ts | 3 +++ backend/src/api/fiat-conversion.ts | 3 +++ backend/src/tasks/pools-updater.ts | 3 +++ 3 files changed, 9 insertions(+) diff --git a/backend/src/api/bisq/bisq.ts b/backend/src/api/bisq/bisq.ts index 4e2eb5fdd..ed9c4fc3e 100644 --- a/backend/src/api/bisq/bisq.ts +++ b/backend/src/api/bisq/bisq.ts @@ -181,6 +181,9 @@ class Bisq { if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) { socksOptions.username = config.SOCKS5PROXY.USERNAME; socksOptions.password = config.SOCKS5PROXY.PASSWORD; + } else { + // Retry with different tor circuits https://stackoverflow.com/a/64960234 + socksOptions.username = `circuit${retry}`; } // Handle proxy agent for onion addresses diff --git a/backend/src/api/fiat-conversion.ts b/backend/src/api/fiat-conversion.ts index e8a504eb7..bed86e92f 100644 --- a/backend/src/api/fiat-conversion.ts +++ b/backend/src/api/fiat-conversion.ts @@ -76,6 +76,9 @@ class FiatConversion { if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) { socksOptions.username = config.SOCKS5PROXY.USERNAME; socksOptions.password = config.SOCKS5PROXY.PASSWORD; + } else { + // Retry with different tor circuits https://stackoverflow.com/a/64960234 + socksOptions.username = `circuit${retry}`; } // Handle proxy agent for onion addresses diff --git a/backend/src/tasks/pools-updater.ts b/backend/src/tasks/pools-updater.ts index 0211948d6..471c066ad 100644 --- a/backend/src/tasks/pools-updater.ts +++ b/backend/src/tasks/pools-updater.ts @@ -143,6 +143,9 @@ class PoolsUpdater { if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) { socksOptions.username = config.SOCKS5PROXY.USERNAME; socksOptions.password = config.SOCKS5PROXY.PASSWORD; + } else { + // Retry with different tor circuits https://stackoverflow.com/a/64960234 + socksOptions.username = `circuit${retry}`; } axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions); From a04bafdb4cacab1e7196ddcac3c18b302f963518 Mon Sep 17 00:00:00 2001 From: Ayanami Date: Sat, 28 May 2022 00:26:27 +0900 Subject: [PATCH 07/11] Correct the log if the onion address is enabled or not --- backend/src/api/fiat-conversion.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/src/api/fiat-conversion.ts b/backend/src/api/fiat-conversion.ts index bed86e92f..ffbe6a758 100644 --- a/backend/src/api/fiat-conversion.ts +++ b/backend/src/api/fiat-conversion.ts @@ -27,9 +27,10 @@ class FiatConversion { } public startService() { + const fiatConversionUrl = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.PRICE_DATA_SERVER.TOR_URL : config.PRICE_DATA_SERVER.CLEARNET_URL; logger.info('Starting currency rates service'); if (config.SOCKS5PROXY.ENABLED) { - logger.info(`Currency rates service will be queried over the Tor network using ${config.PRICE_DATA_SERVER.TOR_URL}`); + logger.info(`Currency rates service will be queried over the Tor network using ${fiatConversionUrl}`); } else { logger.info(`Currency rates service will be queried over clearnet using ${config.PRICE_DATA_SERVER.CLEARNET_URL}`); } From 56dc3376722f01f018dc4ca4530b76bbf7c1e6cf Mon Sep 17 00:00:00 2001 From: Ayanami Date: Wed, 1 Jun 2022 23:38:28 +0900 Subject: [PATCH 08/11] Temporary disable retries Until we find out how to sync async --- backend/mempool-config.sample.json | 4 ++-- backend/src/config.ts | 4 ++-- docker/backend/start.sh | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 3f949491d..77b571136 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -16,8 +16,8 @@ "PRICE_FEED_UPDATE_INTERVAL": 600, "USE_SECOND_NODE_FOR_MINFEE": false, "EXTERNAL_ASSETS": [], - "EXTERNAL_MAX_RETRY": 10, - "EXTERNAL_RETRY_INTERVAL": 60, + "EXTERNAL_MAX_RETRY": 1, + "EXTERNAL_RETRY_INTERVAL": 0, "USER_AGENT": "mempool", "STDOUT_LOG_MIN_PRIORITY": "debug" }, diff --git a/backend/src/config.ts b/backend/src/config.ts index ee28a7da8..e49da3dc9 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -107,8 +107,8 @@ const defaults: IConfig = { 'PRICE_FEED_UPDATE_INTERVAL': 600, 'USE_SECOND_NODE_FOR_MINFEE': false, 'EXTERNAL_ASSETS': [], - 'EXTERNAL_MAX_RETRY': 10, - 'EXTERNAL_RETRY_INTERVAL': 60, + 'EXTERNAL_MAX_RETRY': 1, + 'EXTERNAL_RETRY_INTERVAL': 0, 'USER_AGENT': 'mempool', 'STDOUT_LOG_MIN_PRIORITY': 'debug', }, diff --git a/docker/backend/start.sh b/docker/backend/start.sh index a8beeae5f..bdfed68ae 100644 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -17,8 +17,8 @@ __MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=11000} __MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__=${MEMPOOL_PRICE_FEED_UPDATE_INTERVAL:=600} __MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__=${MEMPOOL_USE_SECOND_NODE_FOR_MINFEE:=false} __MEMPOOL_EXTERNAL_ASSETS__=${MEMPOOL_EXTERNAL_ASSETS:=[]} -__MEMPOOL_EXTERNAL_MAX_RETRY__=${MEMPOOL_EXTERNAL_MAX_RETRY:=10} -__MEMPOOL_EXTERNAL_RETRY_INTERVAL__=${MEMPOOL_EXTERNAL_RETRY_INTERVAL:=60} +__MEMPOOL_EXTERNAL_MAX_RETRY__=${MEMPOOL_EXTERNAL_MAX_RETRY:=1} +__MEMPOOL_EXTERNAL_RETRY_INTERVAL__=${MEMPOOL_EXTERNAL_RETRY_INTERVAL:=0} __MEMPOOL_USER_AGENT__=${MEMPOOL_USER_AGENT:=mempool} __MEMPOOL_STDOUT_LOG_MIN_PRIORITY__=${MEMPOOL_STDOUT_LOG_MIN_PRIORITY:=info} From e133467ea1081318ad69bed498e94805dcf709cc Mon Sep 17 00:00:00 2001 From: hunicus <93150691+hunicus@users.noreply.github.com> Date: Tue, 7 Jun 2022 08:41:07 -0400 Subject: [PATCH 09/11] Add pull request template to discourage tx prs --- .github/pull_request_template.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .github/pull_request_template.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..0b3668cf1 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,6 @@ + From 87c6e957f0c08f3325b3ad9d1e13b813d80911b6 Mon Sep 17 00:00:00 2001 From: hunicus <93150691+hunicus@users.noreply.github.com> Date: Tue, 7 Jun 2022 08:54:11 -0400 Subject: [PATCH 10/11] Remove unnecessary detail in fee faq --- frontend/src/app/docs/api-docs/api-docs-data.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/app/docs/api-docs/api-docs-data.ts b/frontend/src/app/docs/api-docs/api-docs-data.ts index 1927c62d9..68415188e 100644 --- a/frontend/src/app/docs/api-docs/api-docs-data.ts +++ b/frontend/src/app/docs/api-docs/api-docs-data.ts @@ -6107,7 +6107,7 @@ export const faqData = [ showConditions: bitcoinNetworks, fragment: "looking-up-fee-estimates", title: "How can I look up fee estimates?", - answer: "

See real-time fee estimates on the main dashboard.

Here is an overview of Mempool's feerate suggestions:

  • High Priority. This figure is the median feerate of transactions in the first projected block. Consider using this feerate if you want confirmation as soon as possible.
  • Medium Priority. This figure is the average of the median feerate of the first projected block and the median feerate of the second projected block.
  • Low Priority. This figure is the average of the Medium Priority feerate and the median feerate of the third projected block. Consider using this feerate if you want confirmation soon but don't need it particularly quickly.
  • No Priority. This figure is either 2x the minimum feerate, or the Low Priority feerate (whichever is lower). Consider using this feerate if you are in no rush and don't mind if confirmation takes a while.

In all cases, the suggested feerate is adjusted lower if any of the projected blocks involved in the calculation are not full (e.g. if there is only 1 projected block with a size of 1.2 MWU, Mempool will suggest a feerate of 1 sat/vB—not the median feerate of transactions in the block).

Projected blocks use feerates, transaction sizes, and other metrics to forecast which transactions will be in future blocks. Actual blocks will turn out to be different: miners have their own views of the mempool, their own algorithms for determining which transactions to include in a block, etc.

Ultimately, the Bitcoin network is not perfectly predictable, so fee estimation cannot be perfectly precise.

Use Mempool's feerate suggestions as a guide, and understand that they do not guarantee transaction confirmation in any period of time.

" + answer: "

See real-time fee estimates on the main dashboard.

Here is an overview of Mempool's feerate suggestions:

  • High Priority. This figure is the median feerate of transactions in the first projected block. Consider using this feerate if you want confirmation as soon as possible.
  • Medium Priority. This figure is the average of the median feerate of the first projected block and the median feerate of the second projected block.
  • Low Priority. This figure is the average of the Medium Priority feerate and the median feerate of the third projected block. Consider using this feerate if you want confirmation soon but don't need it particularly quickly.
  • No Priority. This figure is either 2x the minimum feerate, or the Low Priority feerate (whichever is lower). Consider using this feerate if you are in no rush and don't mind if confirmation takes a while.

In all cases, the suggested feerate is adjusted lower if any of the projected blocks involved in the calculation are not full (example: if there is only 1 projected block that's less than half-full, Mempool will suggest a feerate of 1 sat/vB—not the median feerate of transactions in the block).

Projected blocks use feerates, transaction sizes, and other metrics to forecast which transactions will be in future blocks. Actual blocks will turn out to be different: miners have their own views of the mempool, their own algorithms for determining which transactions to include in a block, etc.

Ultimately, the Bitcoin network is not perfectly predictable, so fee estimation cannot be perfectly precise.

Use Mempool's feerate suggestions as a guide, and understand that they do not guarantee transaction confirmation in any period of time.

" }, { type: "endpoint", From c318993a79784ceeccb7cf8f3158b3a18ea427e2 Mon Sep 17 00:00:00 2001 From: hunicus <93150691+hunicus@users.noreply.github.com> Date: Tue, 7 Jun 2022 08:25:12 -0400 Subject: [PATCH 11/11] Add transifex note to issue template --- .github/ISSUE_TEMPLATE/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index d26923e74..1d01fd18d 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -3,3 +3,6 @@ contact_links: - name: 🙋 Need help? Chat with us on Matrix url: https://matrix.to/#/#mempool.support:bitcoin.kyoto about: For support requests or general questions + - name: 🌐 Want to help with translations? Use Transifex + url: https://www.transifex.com/mempool/mempool + about: All translations work is done on Transifex