diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index d26923e74..1d01fd18d 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -3,3 +3,6 @@ contact_links: - name: 🙋 Need help? Chat with us on Matrix url: https://matrix.to/#/#mempool.support:bitcoin.kyoto about: For support requests or general questions + - name: 🌐 Want to help with translations? Use Transifex + url: https://www.transifex.com/mempool/mempool + about: All translations work is done on Transifex diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..0b3668cf1 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,6 @@ + diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 6be8b1674..77b571136 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -15,9 +15,10 @@ "INDEXING_BLOCKS_AMOUNT": 11000, "PRICE_FEED_UPDATE_INTERVAL": 600, "USE_SECOND_NODE_FOR_MINFEE": false, - "EXTERNAL_ASSETS": [ - "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json" - ], + "EXTERNAL_ASSETS": [], + "EXTERNAL_MAX_RETRY": 1, + "EXTERNAL_RETRY_INTERVAL": 0, + "USER_AGENT": "mempool", "STDOUT_LOG_MIN_PRIORITY": "debug" }, "CORE_RPC": { @@ -66,6 +67,7 @@ }, "SOCKS5PROXY": { "ENABLED": false, + "USE_ONION": true, "HOST": "127.0.0.1", "PORT": 9050, "USERNAME": "", @@ -74,5 +76,13 @@ "PRICE_DATA_SERVER": { "TOR_URL": "http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices", "CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices" + }, + "EXTERNAL_DATA_SERVER": { + "MEMPOOL_API": "https://mempool.space/api/v1", + "MEMPOOL_ONION": "http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1", + "LIQUID_API": "https://liquid.network/api/v1", + "LIQUID_ONION": "http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1", + "BISQ_URL": "https://bisq.markets/api", + "BISQ_ONION": "http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api" } } diff --git a/backend/src/api/bisq/bisq.ts b/backend/src/api/bisq/bisq.ts index f16356d1c..ed9c4fc3e 100644 --- a/backend/src/api/bisq/bisq.ts +++ b/backend/src/api/bisq/bisq.ts @@ -1,10 +1,14 @@ import config from '../../config'; import * as fs from 'fs'; -import axios from 'axios'; +import axios, { AxiosResponse } from 'axios'; +import * as http from 'http'; +import * as https from 'https'; +import { SocksProxyAgent } from 'socks-proxy-agent'; import { BisqBlocks, BisqBlock, BisqTransaction, BisqStats, BisqTrade } from './interfaces'; import { Common } from '../common'; import { BlockExtended } from '../../mempool.interfaces'; import { StaticPool } from 'node-worker-threads-pool'; +import backendInfo from '../backend-info'; import logger from '../../logger'; class Bisq { @@ -143,12 +147,59 @@ class Bisq { }, 2000); }); } + private async updatePrice() { + type axiosOptions = { + headers: { + 'User-Agent': string + }; + timeout: number; + httpAgent?: http.Agent; + httpsAgent?: https.Agent; + } + const setDelay = (secs: number = 1): Promise => new Promise(resolve => setTimeout(() => resolve(), secs * 1000)); + const BISQ_URL = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.EXTERNAL_DATA_SERVER.BISQ_ONION : config.EXTERNAL_DATA_SERVER.BISQ_URL; + const isHTTP = (new URL(BISQ_URL).protocol.split(':')[0] === 'http') ? true : false; + const axiosOptions: axiosOptions = { + headers: { + 'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}` + }, + timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000 + }; + let retry = 0; - private updatePrice() { - axios.get('https://bisq.markets/api/trades/?market=bsq_btc', { timeout: 10000 }) - .then((response) => { + while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) { + try { + if (config.SOCKS5PROXY.ENABLED) { + const socksOptions: any = { + agentOptions: { + keepAlive: true, + }, + hostname: config.SOCKS5PROXY.HOST, + port: config.SOCKS5PROXY.PORT + }; + + if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) { + socksOptions.username = config.SOCKS5PROXY.USERNAME; + socksOptions.password = config.SOCKS5PROXY.PASSWORD; + } else { + // Retry with different tor circuits https://stackoverflow.com/a/64960234 + socksOptions.username = `circuit${retry}`; + } + + // Handle proxy agent for onion addresses + if (isHTTP) { + axiosOptions.httpAgent = new SocksProxyAgent(socksOptions); + } else { + axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions); + } + } + + const data: AxiosResponse = await axios.get(`${BISQ_URL}/trades/?market=bsq_btc`, axiosOptions); + if (data.statusText === 'error' || !data.data) { + throw new Error(`Could not fetch data from Bisq market, Error: ${data.status}`); + } const prices: number[] = []; - response.data.forEach((trade) => { + data.data.forEach((trade) => { prices.push(parseFloat(trade.price) * 100000000); }); prices.sort((a, b) => a - b); @@ -156,9 +207,14 @@ class Bisq { if (this.priceUpdateCallbackFunction) { this.priceUpdateCallbackFunction(this.price); } - }).catch((err) => { - logger.err('Error updating Bisq market price: ' + err); - }); + logger.debug('Successfully updated Bisq market price'); + break; + } catch (e) { + logger.err('Error updating Bisq market price: ' + (e instanceof Error ? e.message : e)); + await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL); + retry++; + } + } } private async loadBisqDumpFile(): Promise { diff --git a/backend/src/api/fiat-conversion.ts b/backend/src/api/fiat-conversion.ts index 3562760f1..ffbe6a758 100644 --- a/backend/src/api/fiat-conversion.ts +++ b/backend/src/api/fiat-conversion.ts @@ -1,4 +1,6 @@ import logger from '../logger'; +import * as http from 'http'; +import * as https from 'https'; import axios, { AxiosResponse } from 'axios'; import { IConversionRates } from '../mempool.interfaces'; import config from '../config'; @@ -25,9 +27,10 @@ class FiatConversion { } public startService() { + const fiatConversionUrl = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.PRICE_DATA_SERVER.TOR_URL : config.PRICE_DATA_SERVER.CLEARNET_URL; logger.info('Starting currency rates service'); if (config.SOCKS5PROXY.ENABLED) { - logger.info(`Currency rates service will be queried over the Tor network using ${config.PRICE_DATA_SERVER.TOR_URL}`); + logger.info(`Currency rates service will be queried over the Tor network using ${fiatConversionUrl}`); } else { logger.info(`Currency rates service will be queried over clearnet using ${config.PRICE_DATA_SERVER.CLEARNET_URL}`); } @@ -40,49 +43,79 @@ class FiatConversion { } private async updateCurrency(): Promise { - const headers = { 'User-Agent': `mempool/v${backendInfo.getBackendInfo().version}` }; - let fiatConversionUrl: string; - let response: AxiosResponse; + type axiosOptions = { + headers: { + 'User-Agent': string + }; + timeout: number; + httpAgent?: http.Agent; + httpsAgent?: https.Agent; + } + const setDelay = (secs: number = 1): Promise => new Promise(resolve => setTimeout(() => resolve(), secs * 1000)); + const fiatConversionUrl = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.PRICE_DATA_SERVER.TOR_URL : config.PRICE_DATA_SERVER.CLEARNET_URL; + const isHTTP = (new URL(fiatConversionUrl).protocol.split(':')[0] === 'http') ? true : false; + const axiosOptions: axiosOptions = { + headers: { + 'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}` + }, + timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000 + }; - try { - if (config.SOCKS5PROXY.ENABLED) { - let socksOptions: any = { - agentOptions: { - keepAlive: true, - }, - hostname: config.SOCKS5PROXY.HOST, - port: config.SOCKS5PROXY.PORT - }; + let retry = 0; - if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) { - socksOptions.username = config.SOCKS5PROXY.USERNAME; - socksOptions.password = config.SOCKS5PROXY.PASSWORD; + while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) { + try { + if (config.SOCKS5PROXY.ENABLED) { + let socksOptions: any = { + agentOptions: { + keepAlive: true, + }, + hostname: config.SOCKS5PROXY.HOST, + port: config.SOCKS5PROXY.PORT + }; + + if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) { + socksOptions.username = config.SOCKS5PROXY.USERNAME; + socksOptions.password = config.SOCKS5PROXY.PASSWORD; + } else { + // Retry with different tor circuits https://stackoverflow.com/a/64960234 + socksOptions.username = `circuit${retry}`; + } + + // Handle proxy agent for onion addresses + if (isHTTP) { + axiosOptions.httpAgent = new SocksProxyAgent(socksOptions); + } else { + axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions); + } + } + + logger.debug('Querying currency rates service...'); + + const response: AxiosResponse = await axios.get(`${fiatConversionUrl}`, axiosOptions); + + if (response.statusText === 'error' || !response.data) { + throw new Error(`Could not fetch data from ${fiatConversionUrl}, Error: ${response.status}`); } - const agent = new SocksProxyAgent(socksOptions); - fiatConversionUrl = config.PRICE_DATA_SERVER.TOR_URL; - logger.debug('Querying currency rates service...'); - response = await axios.get(fiatConversionUrl, { httpAgent: agent, headers: headers, timeout: 30000 }); - } else { - fiatConversionUrl = config.PRICE_DATA_SERVER.CLEARNET_URL; - logger.debug('Querying currency rates service...'); - response = await axios.get(fiatConversionUrl, { headers: headers, timeout: 10000 }); - } - - for (const rate of response.data.data) { - if (this.debasingFiatCurrencies.includes(rate.currencyCode) && rate.provider === 'Bisq-Aggregate') { - this.conversionRates[rate.currencyCode] = Math.round(100 * rate.price) / 100; + for (const rate of response.data.data) { + if (this.debasingFiatCurrencies.includes(rate.currencyCode) && rate.provider === 'Bisq-Aggregate') { + this.conversionRates[rate.currencyCode] = Math.round(100 * rate.price) / 100; + } } - } - this.ratesInitialized = true; - logger.debug(`USD Conversion Rate: ${this.conversionRates.USD}`); + this.ratesInitialized = true; + logger.debug(`USD Conversion Rate: ${this.conversionRates.USD}`); - if (this.ratesChangedCallback) { - this.ratesChangedCallback(this.conversionRates); + if (this.ratesChangedCallback) { + this.ratesChangedCallback(this.conversionRates); + } + break; + } catch (e) { + logger.err('Error updating fiat conversion rates: ' + (e instanceof Error ? e.message : e)); + await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL); + retry++; } - } catch (e) { - logger.err('Error updating fiat conversion rates: ' + (e instanceof Error ? e.message : e)); } } } diff --git a/backend/src/config.ts b/backend/src/config.ts index 1aa37b8be..e49da3dc9 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -18,6 +18,9 @@ interface IConfig { PRICE_FEED_UPDATE_INTERVAL: number; USE_SECOND_NODE_FOR_MINFEE: boolean; EXTERNAL_ASSETS: string[]; + EXTERNAL_MAX_RETRY: number; + EXTERNAL_RETRY_INTERVAL: number; + USER_AGENT: string; STDOUT_LOG_MIN_PRIORITY: 'emerg' | 'alert' | 'crit' | 'err' | 'warn' | 'notice' | 'info' | 'debug'; }; ESPLORA: { @@ -66,6 +69,7 @@ interface IConfig { }; SOCKS5PROXY: { ENABLED: boolean; + USE_ONION: boolean; HOST: string; PORT: number; USERNAME: string; @@ -75,6 +79,14 @@ interface IConfig { TOR_URL: string; CLEARNET_URL: string; }; + EXTERNAL_DATA_SERVER: { + MEMPOOL_API: string; + MEMPOOL_ONION: string; + LIQUID_API: string; + LIQUID_ONION: string; + BISQ_URL: string; + BISQ_ONION: string; + }; } const defaults: IConfig = { @@ -94,9 +106,10 @@ const defaults: IConfig = { 'INDEXING_BLOCKS_AMOUNT': 11000, // 0 = disable indexing, -1 = index all blocks 'PRICE_FEED_UPDATE_INTERVAL': 600, 'USE_SECOND_NODE_FOR_MINFEE': false, - 'EXTERNAL_ASSETS': [ - 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json' - ], + 'EXTERNAL_ASSETS': [], + 'EXTERNAL_MAX_RETRY': 1, + 'EXTERNAL_RETRY_INTERVAL': 0, + 'USER_AGENT': 'mempool', 'STDOUT_LOG_MIN_PRIORITY': 'debug', }, 'ESPLORA': { @@ -145,6 +158,7 @@ const defaults: IConfig = { }, 'SOCKS5PROXY': { 'ENABLED': false, + 'USE_ONION': true, 'HOST': '127.0.0.1', 'PORT': 9050, 'USERNAME': '', @@ -153,6 +167,14 @@ const defaults: IConfig = { "PRICE_DATA_SERVER": { 'TOR_URL': 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices', 'CLEARNET_URL': 'https://price.bisq.wiz.biz/getAllMarketPrices' + }, + "EXTERNAL_DATA_SERVER": { + 'MEMPOOL_API': 'https://mempool.space/api/v1', + 'MEMPOOL_ONION': 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1', + 'LIQUID_API': 'https://liquid.network/api/v1', + 'LIQUID_ONION': 'http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1', + 'BISQ_URL': 'https://bisq.markets/api', + 'BISQ_ONION': 'http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api' } }; @@ -168,6 +190,7 @@ class Config implements IConfig { BISQ: IConfig['BISQ']; SOCKS5PROXY: IConfig['SOCKS5PROXY']; PRICE_DATA_SERVER: IConfig['PRICE_DATA_SERVER']; + EXTERNAL_DATA_SERVER: IConfig['EXTERNAL_DATA_SERVER']; constructor() { const configs = this.merge(configFile, defaults); @@ -182,6 +205,7 @@ class Config implements IConfig { this.BISQ = configs.BISQ; this.SOCKS5PROXY = configs.SOCKS5PROXY; this.PRICE_DATA_SERVER = configs.PRICE_DATA_SERVER; + this.EXTERNAL_DATA_SERVER = configs.EXTERNAL_DATA_SERVER; } merge = (...objects: object[]): IConfig => { diff --git a/backend/src/index.ts b/backend/src/index.ts index 42cfe9aea..d421a6fba 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -205,7 +205,7 @@ class Server { .post(config.MEMPOOL.API_URL_PREFIX + 'tx/push', routes.$postTransactionForm) .get(config.MEMPOOL.API_URL_PREFIX + 'donations', async (req, res) => { try { - const response = await axios.get('https://mempool.space/api/v1/donations', { responseType: 'stream', timeout: 10000 }); + const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/donations`, { responseType: 'stream', timeout: 10000 }); response.data.pipe(res); } catch (e) { res.status(500).end(); @@ -213,7 +213,7 @@ class Server { }) .get(config.MEMPOOL.API_URL_PREFIX + 'donations/images/:id', async (req, res) => { try { - const response = await axios.get('https://mempool.space/api/v1/donations/images/' + req.params.id, { + const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/donations/images/${req.params.id}`, { responseType: 'stream', timeout: 10000 }); response.data.pipe(res); @@ -223,7 +223,7 @@ class Server { }) .get(config.MEMPOOL.API_URL_PREFIX + 'contributors', async (req, res) => { try { - const response = await axios.get('https://mempool.space/api/v1/contributors', { responseType: 'stream', timeout: 10000 }); + const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/contributors`, { responseType: 'stream', timeout: 10000 }); response.data.pipe(res); } catch (e) { res.status(500).end(); @@ -231,7 +231,7 @@ class Server { }) .get(config.MEMPOOL.API_URL_PREFIX + 'contributors/images/:id', async (req, res) => { try { - const response = await axios.get('https://mempool.space/api/v1/contributors/images/' + req.params.id, { + const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/contributors/images/${req.params.id}`, { responseType: 'stream', timeout: 10000 }); response.data.pipe(res); @@ -241,7 +241,7 @@ class Server { }) .get(config.MEMPOOL.API_URL_PREFIX + 'translators', async (req, res) => { try { - const response = await axios.get('https://mempool.space/api/v1/translators', { responseType: 'stream', timeout: 10000 }); + const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/translators`, { responseType: 'stream', timeout: 10000 }); response.data.pipe(res); } catch (e) { res.status(500).end(); @@ -249,7 +249,7 @@ class Server { }) .get(config.MEMPOOL.API_URL_PREFIX + 'translators/images/:id', async (req, res) => { try { - const response = await axios.get('https://mempool.space/api/v1/translators/images/' + req.params.id, { + const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/translators/images/${req.params.id}`, { responseType: 'stream', timeout: 10000 }); response.data.pipe(res); diff --git a/backend/src/routes.ts b/backend/src/routes.ts index 91c41faa6..12368b1ee 100644 --- a/backend/src/routes.ts +++ b/backend/src/routes.ts @@ -990,7 +990,7 @@ class Routes { public async $getAllFeaturedLiquidAssets(req: Request, res: Response) { try { - const response = await axios.get('https://liquid.network/api/v1/assets/featured', { responseType: 'stream', timeout: 10000 }); + const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.LIQUID_API}/assets/featured`, { responseType: 'stream', timeout: 10000 }); response.data.pipe(res); } catch (e) { res.status(500).end(); @@ -999,7 +999,7 @@ class Routes { public async $getAssetGroup(req: Request, res: Response) { try { - const response = await axios.get('https://liquid.network/api/v1/assets/group/' + parseInt(req.params.id, 10), + const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.LIQUID_API}/assets/group/${parseInt(req.params.id, 10)}`, { responseType: 'stream', timeout: 10000 }); response.data.pipe(res); } catch (e) { diff --git a/backend/src/sync-assets.ts b/backend/src/sync-assets.ts index 74d15e901..f4cc1f266 100644 --- a/backend/src/sync-assets.ts +++ b/backend/src/sync-assets.ts @@ -1,6 +1,7 @@ import axios, { AxiosResponse } from 'axios'; import * as fs from 'fs'; import config from './config'; +import backendInfo from './api/backend-info'; import logger from './logger'; import { SocksProxyAgent } from 'socks-proxy-agent'; @@ -42,6 +43,9 @@ class SyncAssets { logger.info(`Downloading external asset ${fileName} over the Tor network...`); return axios.get(url, { + headers: { + 'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}` + }, httpAgent: agent, httpsAgent: agent, responseType: 'stream', @@ -57,6 +61,9 @@ class SyncAssets { } else { logger.info(`Downloading external asset ${fileName} over clearnet...`); return axios.get(url, { + headers: { + 'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}` + }, responseType: 'stream', timeout: 30000 }).then(function (response) { diff --git a/backend/src/tasks/pools-updater.ts b/backend/src/tasks/pools-updater.ts index 87487b17e..f17239358 100644 --- a/backend/src/tasks/pools-updater.ts +++ b/backend/src/tasks/pools-updater.ts @@ -1,7 +1,8 @@ -import axios from 'axios'; +import axios, { AxiosResponse } from 'axios'; import poolsParser from '../api/pools-parser'; import config from '../config'; import DB from '../database'; +import backendInfo from '../api/backend-info'; import logger from '../logger'; import { SocksProxyAgent } from 'socks-proxy-agent'; import * as https from 'https'; @@ -119,33 +120,45 @@ class PoolsUpdater { */ private async query(path): Promise { type axiosOptions = { + headers: { + 'User-Agent': string + }; + timeout: number; httpsAgent?: https.Agent; } const setDelay = (secs: number = 1): Promise => new Promise(resolve => setTimeout(() => resolve(), secs * 1000)); - const axiosOptions: axiosOptions = {}; + const axiosOptions: axiosOptions = { + headers: { + 'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}` + }, + timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000 + }; let retry = 0; - if (config.SOCKS5PROXY.ENABLED) { - const socksOptions: any = { - agentOptions: { - keepAlive: true, - }, - hostname: config.SOCKS5PROXY.HOST, - port: config.SOCKS5PROXY.PORT - }; - - if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) { - socksOptions.username = config.SOCKS5PROXY.USERNAME; - socksOptions.password = config.SOCKS5PROXY.PASSWORD; - } - - axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions); - } - - while(retry < 5) { + while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) { try { - const data = await axios.get(path, axiosOptions); - if (data.statusText !== 'OK' || !data.data) { + if (config.SOCKS5PROXY.ENABLED) { + const socksOptions: any = { + agentOptions: { + keepAlive: true, + }, + hostname: config.SOCKS5PROXY.HOST, + port: config.SOCKS5PROXY.PORT + }; + + if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) { + socksOptions.username = config.SOCKS5PROXY.USERNAME; + socksOptions.password = config.SOCKS5PROXY.PASSWORD; + } else { + // Retry with different tor circuits https://stackoverflow.com/a/64960234 + socksOptions.username = `circuit${retry}`; + } + + axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions); + } + + const data: AxiosResponse = await axios.get(path, axiosOptions); + if (data.statusText === 'error' || !data.data) { throw new Error(`Could not fetch data from Github, Error: ${data.status}`); } return data.data; @@ -153,7 +166,7 @@ class PoolsUpdater { logger.err('Could not connect to Github. Reason: ' + (e instanceof Error ? e.message : e)); retry++; } - await setDelay(); + await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL); } return undefined; } diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index 39d0e5fad..9a929a4f0 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -15,6 +15,9 @@ "PRICE_FEED_UPDATE_INTERVAL": __MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__, "USE_SECOND_NODE_FOR_MINFEE": __MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__, "EXTERNAL_ASSETS": __MEMPOOL_EXTERNAL_ASSETS__, + "EXTERNAL_MAX_RETRY": __MEMPOOL_EXTERNAL_MAX_RETRY__, + "EXTERNAL_RETRY_INTERVAL": __MEMPOOL_EXTERNAL_RETRY_INTERVAL__, + "USER_AGENT": "__MEMPOOL_USER_AGENT__", "STDOUT_LOG_MIN_PRIORITY": "__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__", "INDEXING_BLOCKS_AMOUNT": __MEMPOOL_INDEXING_BLOCKS_AMOUNT__ }, @@ -64,6 +67,7 @@ }, "SOCKS5PROXY": { "ENABLED": __SOCKS5PROXY_ENABLED__, + "USE_ONION": __SOCKS5PROXY_USE_ONION__, "HOST": "__SOCKS5PROXY_HOST__", "PORT": "__SOCKS5PROXY_PORT__", "USERNAME": "__SOCKS5PROXY_USERNAME__", @@ -72,5 +76,13 @@ "PRICE_DATA_SERVER": { "TOR_URL": "__PRICE_DATA_SERVER_TOR_URL__", "CLEARNET_URL": "__PRICE_DATA_SERVER_CLEARNET_URL__" + }, + "EXTERNAL_DATA_SERVER": { + "MEMPOOL_API": "__EXTERNAL_DATA_SERVER_MEMPOOL_API__", + "MEMPOOL_ONION": "__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__", + "LIQUID_API": "__EXTERNAL_DATA_SERVER_LIQUID_API__", + "LIQUID_ONION": "__EXTERNAL_DATA_SERVER_LIQUID_ONION__", + "BISQ_URL": "__EXTERNAL_DATA_SERVER_BISQ_URL__", + "BISQ_ONION": "__EXTERNAL_DATA_SERVER_BISQ_ONION__" } } diff --git a/docker/backend/start.sh b/docker/backend/start.sh index 960171e43..bdfed68ae 100644 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -16,7 +16,10 @@ __MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__=${MEMPOOL_MEMPOOL_BLOCKS_AMOUNT:=8} __MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=11000} __MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__=${MEMPOOL_PRICE_FEED_UPDATE_INTERVAL:=600} __MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__=${MEMPOOL_USE_SECOND_NODE_FOR_MINFEE:=false} -__MEMPOOL_EXTERNAL_ASSETS__=${MEMPOOL_EXTERNAL_ASSETS:=[\"https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json\"]} +__MEMPOOL_EXTERNAL_ASSETS__=${MEMPOOL_EXTERNAL_ASSETS:=[]} +__MEMPOOL_EXTERNAL_MAX_RETRY__=${MEMPOOL_EXTERNAL_MAX_RETRY:=1} +__MEMPOOL_EXTERNAL_RETRY_INTERVAL__=${MEMPOOL_EXTERNAL_RETRY_INTERVAL:=0} +__MEMPOOL_USER_AGENT__=${MEMPOOL_USER_AGENT:=mempool} __MEMPOOL_STDOUT_LOG_MIN_PRIORITY__=${MEMPOOL_STDOUT_LOG_MIN_PRIORITY:=info} # CORE_RPC @@ -65,6 +68,7 @@ __BISQ_DATA_PATH__=${BISQ_DATA_PATH:=/bisq/statsnode-data/btc_mainnet/db} # SOCKS5PROXY __SOCKS5PROXY_ENABLED__=${SOCKS5PROXY_ENABLED:=false} +__SOCKS5PROXY_USE_ONION__=${SOCKS5PROXY_USE_ONION:=true} __SOCKS5PROXY_HOST__=${SOCKS5PROXY_HOST:=localhost} __SOCKS5PROXY_PORT__=${SOCKS5PROXY_PORT:=9050} __SOCKS5PROXY_USERNAME__=${SOCKS5PROXY_USERNAME:=""} @@ -74,6 +78,14 @@ __SOCKS5PROXY_PASSWORD__=${SOCKS5PROXY_PASSWORD:=""} __PRICE_DATA_SERVER_TOR_URL__=${PRICE_DATA_SERVER_TOR_URL:=http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices} __PRICE_DATA_SERVER_CLEARNET_URL__=${PRICE_DATA_SERVER_CLEARNET_URL:=https://price.bisq.wiz.biz/getAllMarketPrices} +# EXTERNAL_DATA_SERVER +__EXTERNAL_DATA_SERVER_MEMPOOL_API__=${EXTERNAL_DATA_SERVER_MEMPOOL_API:=https://mempool.space/api/v1} +__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__=${EXTERNAL_DATA_SERVER_MEMPOOL_ONION:=http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1} +__EXTERNAL_DATA_SERVER_LIQUID_API__=${EXTERNAL_DATA_SERVER_LIQUID_API:=https://liquid.network/api/v1} +__EXTERNAL_DATA_SERVER_LIQUID_ONION__=${EXTERNAL_DATA_SERVER_LIQUID_ONION:=http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1} +__EXTERNAL_DATA_SERVER_BISQ_URL__=${EXTERNAL_DATA_SERVER_BISQ_URL:=https://bisq.markets/api} +__EXTERNAL_DATA_SERVER_BISQ_ONION__=${EXTERNAL_DATA_SERVER_BISQ_ONION:=http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api} + mkdir -p "${__MEMPOOL_CACHE_DIR__}" sed -i "s/__MEMPOOL_NETWORK__/${__MEMPOOL_NETWORK__}/g" mempool-config.json diff --git a/frontend/src/app/docs/api-docs/api-docs-data.ts b/frontend/src/app/docs/api-docs/api-docs-data.ts index eb98c6596..68415188e 100644 --- a/frontend/src/app/docs/api-docs/api-docs-data.ts +++ b/frontend/src/app/docs/api-docs/api-docs-data.ts @@ -6107,7 +6107,7 @@ export const faqData = [ showConditions: bitcoinNetworks, fragment: "looking-up-fee-estimates", title: "How can I look up fee estimates?", - answer: "

See real-time fee estimates on the main dashboard.

Low priority is suggested for confirmation within 6 blocks (~1 hour), Medium priority is suggested for confirmation within 3 blocks (~30 minutes), and High priority is suggested for confirmation in the next block (~10 minutes).

" + answer: "

See real-time fee estimates on the main dashboard.

Here is an overview of Mempool's feerate suggestions:

  • High Priority. This figure is the median feerate of transactions in the first projected block. Consider using this feerate if you want confirmation as soon as possible.
  • Medium Priority. This figure is the average of the median feerate of the first projected block and the median feerate of the second projected block.
  • Low Priority. This figure is the average of the Medium Priority feerate and the median feerate of the third projected block. Consider using this feerate if you want confirmation soon but don't need it particularly quickly.
  • No Priority. This figure is either 2x the minimum feerate, or the Low Priority feerate (whichever is lower). Consider using this feerate if you are in no rush and don't mind if confirmation takes a while.

In all cases, the suggested feerate is adjusted lower if any of the projected blocks involved in the calculation are not full (example: if there is only 1 projected block that's less than half-full, Mempool will suggest a feerate of 1 sat/vB—not the median feerate of transactions in the block).

Projected blocks use feerates, transaction sizes, and other metrics to forecast which transactions will be in future blocks. Actual blocks will turn out to be different: miners have their own views of the mempool, their own algorithms for determining which transactions to include in a block, etc.

Ultimately, the Bitcoin network is not perfectly predictable, so fee estimation cannot be perfectly precise.

Use Mempool's feerate suggestions as a guide, and understand that they do not guarantee transaction confirmation in any period of time.

" }, { type: "endpoint",