Merge branch 'master' into update-fee-faq-2

.github/ISSUE_TEMPLATE/config.yml

@@ -3,3 +3,6 @@ contact_links:
   - name: 🙋 Need help? Chat with us on Matrix
     url: https://matrix.to/#/#mempool.support:bitcoin.kyoto
     about: For support requests or general questions
+  - name: 🌐 Want to help with translations? Use Transifex
+    url: https://www.transifex.com/mempool/mempool
+    about: All translations work is done on Transifex

.github/pull_request_template.md

@@ -0,0 +1,6 @@
+<!--
+Please do not open pull requests for translations.
+
+All translations work is done on Transifex:
+https://www.transifex.com/mempool/mempool
+-->

backend/mempool-config.sample.json

@@ -15,9 +15,10 @@
     "INDEXING_BLOCKS_AMOUNT": 11000,
     "PRICE_FEED_UPDATE_INTERVAL": 600,
     "USE_SECOND_NODE_FOR_MINFEE": false,
-    "EXTERNAL_ASSETS": [
+    "EXTERNAL_ASSETS": [],
-      "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json"
+    "EXTERNAL_MAX_RETRY": 1,
-    ],
+    "EXTERNAL_RETRY_INTERVAL": 0,
+    "USER_AGENT": "mempool",
     "STDOUT_LOG_MIN_PRIORITY": "debug"
   },
   "CORE_RPC": {
@@ -66,6 +67,7 @@
   },
   "SOCKS5PROXY": {
     "ENABLED": false,
+    "USE_ONION": true,
     "HOST": "127.0.0.1",
     "PORT": 9050,
     "USERNAME": "",
@@ -74,5 +76,13 @@
   "PRICE_DATA_SERVER": {
     "TOR_URL": "http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices",
     "CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices"
+  },
+  "EXTERNAL_DATA_SERVER": {
+    "MEMPOOL_API": "https://mempool.space/api/v1",
+    "MEMPOOL_ONION": "http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1",
+    "LIQUID_API": "https://liquid.network/api/v1",
+    "LIQUID_ONION": "http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1",
+    "BISQ_URL": "https://bisq.markets/api",
+    "BISQ_ONION": "http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api"
   }
 }

backend/src/api/bisq/bisq.ts

@@ -1,10 +1,14 @@
 import config from '../../config';
 import * as fs from 'fs';
-import axios from 'axios';
+import axios, { AxiosResponse } from 'axios';
+import * as http from 'http';
+import * as https from 'https';
+import { SocksProxyAgent } from 'socks-proxy-agent';
 import { BisqBlocks, BisqBlock, BisqTransaction, BisqStats, BisqTrade } from './interfaces';
 import { Common } from '../common';
 import { BlockExtended } from '../../mempool.interfaces';
 import { StaticPool } from 'node-worker-threads-pool';
+import backendInfo from '../backend-info';
 import logger from '../../logger';
 
 class Bisq {
@@ -143,12 +147,59 @@ class Bisq {
       }, 2000);
     });
   }
+  private async updatePrice() {
+    type axiosOptions = {
+      headers: {
+        'User-Agent': string
+      };
+      timeout: number;
+      httpAgent?: http.Agent;
+      httpsAgent?: https.Agent;
+    }
+    const setDelay = (secs: number = 1): Promise<void> => new Promise(resolve => setTimeout(() => resolve(), secs * 1000));
+    const BISQ_URL = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.EXTERNAL_DATA_SERVER.BISQ_ONION : config.EXTERNAL_DATA_SERVER.BISQ_URL;
+    const isHTTP = (new URL(BISQ_URL).protocol.split(':')[0] === 'http') ? true : false;
+    const axiosOptions: axiosOptions = {
+      headers: {
+        'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
+      },
+      timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000
+    };
+    let retry = 0;
 
-  private updatePrice() {
+    while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
-    axios.get<BisqTrade[]>('https://bisq.markets/api/trades/?market=bsq_btc', { timeout: 10000 })
+      try {
-      .then((response) => {
+        if (config.SOCKS5PROXY.ENABLED) {
+          const socksOptions: any = {
+            agentOptions: {
+              keepAlive: true,
+            },
+            hostname: config.SOCKS5PROXY.HOST,
+            port: config.SOCKS5PROXY.PORT
+          };
+
+          if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
+            socksOptions.username = config.SOCKS5PROXY.USERNAME;
+            socksOptions.password = config.SOCKS5PROXY.PASSWORD;
+          } else {
+            // Retry with different tor circuits https://stackoverflow.com/a/64960234
+            socksOptions.username = `circuit${retry}`;
+          }
+
+          // Handle proxy agent for onion addresses
+          if (isHTTP) {
+            axiosOptions.httpAgent = new SocksProxyAgent(socksOptions);
+          } else {
+            axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions);
+          }
+        }
+        
+        const data: AxiosResponse = await axios.get(`${BISQ_URL}/trades/?market=bsq_btc`, axiosOptions);
+        if (data.statusText === 'error' || !data.data) {
+          throw new Error(`Could not fetch data from Bisq market, Error: ${data.status}`);
+        }
         const prices: number[] = [];
-        response.data.forEach((trade) => {
+        data.data.forEach((trade) => {
           prices.push(parseFloat(trade.price) * 100000000);
         });
         prices.sort((a, b) => a - b);
@@ -156,9 +207,14 @@ class Bisq {
         if (this.priceUpdateCallbackFunction) {
           this.priceUpdateCallbackFunction(this.price);
         }
-    }).catch((err) => {
+        logger.debug('Successfully updated Bisq market price');
-      logger.err('Error updating Bisq market price: ' + err);
+        break;
-    });
+      } catch (e) {
+        logger.err('Error updating Bisq market price: '  + (e instanceof Error ? e.message : e));
+        await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL);
+        retry++;
+      }
+    }
   }
 
   private async loadBisqDumpFile(): Promise<void> {

backend/src/api/fiat-conversion.ts

@@ -1,4 +1,6 @@
 import logger from '../logger';
+import * as http from 'http';
+import * as https from 'https';
 import axios, { AxiosResponse } from 'axios';
 import { IConversionRates } from '../mempool.interfaces';
 import config from '../config';
@@ -25,9 +27,10 @@ class FiatConversion {
   }
 
   public startService() {
+    const fiatConversionUrl = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.PRICE_DATA_SERVER.TOR_URL : config.PRICE_DATA_SERVER.CLEARNET_URL;
     logger.info('Starting currency rates service');
     if (config.SOCKS5PROXY.ENABLED) {
-      logger.info(`Currency rates service will be queried over the Tor network using ${config.PRICE_DATA_SERVER.TOR_URL}`);
+      logger.info(`Currency rates service will be queried over the Tor network using ${fiatConversionUrl}`);
     } else {
       logger.info(`Currency rates service will be queried over clearnet using ${config.PRICE_DATA_SERVER.CLEARNET_URL}`);
     }
@@ -40,49 +43,79 @@ class FiatConversion {
   }
 
   private async updateCurrency(): Promise<void> {
-    const headers = { 'User-Agent': `mempool/v${backendInfo.getBackendInfo().version}` };
+    type axiosOptions = {
-    let fiatConversionUrl: string;
+      headers: {
-    let response: AxiosResponse;
+        'User-Agent': string
+      };
+      timeout: number;
+      httpAgent?: http.Agent;
+      httpsAgent?: https.Agent;
+    }
+    const setDelay = (secs: number = 1): Promise<void> => new Promise(resolve => setTimeout(() => resolve(), secs * 1000));
+    const fiatConversionUrl = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.PRICE_DATA_SERVER.TOR_URL : config.PRICE_DATA_SERVER.CLEARNET_URL;
+    const isHTTP = (new URL(fiatConversionUrl).protocol.split(':')[0] === 'http') ? true : false;
+    const axiosOptions: axiosOptions = {
+      headers: {
+        'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
+      },
+      timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000
+    };
 
-    try {
+    let retry = 0;
-      if (config.SOCKS5PROXY.ENABLED) {
-        let socksOptions: any = {
-          agentOptions: {
-            keepAlive: true,
-          },
-          hostname: config.SOCKS5PROXY.HOST,
-          port: config.SOCKS5PROXY.PORT
-        };
 
-        if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
+    while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
-          socksOptions.username = config.SOCKS5PROXY.USERNAME;
+      try {
-          socksOptions.password = config.SOCKS5PROXY.PASSWORD;
+        if (config.SOCKS5PROXY.ENABLED) {
+          let socksOptions: any = {
+            agentOptions: {
+              keepAlive: true,
+            },
+            hostname: config.SOCKS5PROXY.HOST,
+            port: config.SOCKS5PROXY.PORT
+          };
+
+          if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
+            socksOptions.username = config.SOCKS5PROXY.USERNAME;
+            socksOptions.password = config.SOCKS5PROXY.PASSWORD;
+          } else {
+            // Retry with different tor circuits https://stackoverflow.com/a/64960234
+            socksOptions.username = `circuit${retry}`;
+          }
+
+          // Handle proxy agent for onion addresses
+          if (isHTTP) {
+            axiosOptions.httpAgent = new SocksProxyAgent(socksOptions);
+          } else {
+            axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions);
+          }
+        }
+        
+        logger.debug('Querying currency rates service...');
+
+        const response: AxiosResponse = await axios.get(`${fiatConversionUrl}`, axiosOptions);
+
+        if (response.statusText === 'error' || !response.data) {
+          throw new Error(`Could not fetch data from ${fiatConversionUrl}, Error: ${response.status}`);
         }
 
-        const agent = new SocksProxyAgent(socksOptions);
+        for (const rate of response.data.data) {
-        fiatConversionUrl = config.PRICE_DATA_SERVER.TOR_URL;
+          if (this.debasingFiatCurrencies.includes(rate.currencyCode) && rate.provider === 'Bisq-Aggregate') {
-        logger.debug('Querying currency rates service...');
+            this.conversionRates[rate.currencyCode] = Math.round(100 * rate.price) / 100;
-        response = await axios.get(fiatConversionUrl, { httpAgent: agent, headers: headers, timeout: 30000 });
+          }
-      } else {
-        fiatConversionUrl = config.PRICE_DATA_SERVER.CLEARNET_URL;
-        logger.debug('Querying currency rates service...');
-        response = await axios.get(fiatConversionUrl, { headers: headers, timeout: 10000 });
-      }
-
-      for (const rate of response.data.data) {
-        if (this.debasingFiatCurrencies.includes(rate.currencyCode) && rate.provider === 'Bisq-Aggregate') {
-          this.conversionRates[rate.currencyCode] = Math.round(100 * rate.price) / 100;
         }
-      }
 
-      this.ratesInitialized = true;
+        this.ratesInitialized = true;
-      logger.debug(`USD Conversion Rate: ${this.conversionRates.USD}`);
+        logger.debug(`USD Conversion Rate: ${this.conversionRates.USD}`);
 
-      if (this.ratesChangedCallback) {
+        if (this.ratesChangedCallback) {
-        this.ratesChangedCallback(this.conversionRates);
+          this.ratesChangedCallback(this.conversionRates);
+        }
+        break;
+      } catch (e) {
+        logger.err('Error updating fiat conversion rates: '  + (e instanceof Error ? e.message : e));
+        await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL);
+        retry++;
       }
-    } catch (e) {
-      logger.err('Error updating fiat conversion rates: ' + (e instanceof Error ? e.message : e));
     }
   }
 }

backend/src/config.ts

@@ -18,6 +18,9 @@ interface IConfig {
     PRICE_FEED_UPDATE_INTERVAL: number;
     USE_SECOND_NODE_FOR_MINFEE: boolean;
     EXTERNAL_ASSETS: string[];
+    EXTERNAL_MAX_RETRY: number;
+    EXTERNAL_RETRY_INTERVAL: number;
+    USER_AGENT: string;
     STDOUT_LOG_MIN_PRIORITY: 'emerg' | 'alert' | 'crit' | 'err' | 'warn' | 'notice' | 'info' | 'debug';
   };
   ESPLORA: {
@@ -66,6 +69,7 @@ interface IConfig {
   };
   SOCKS5PROXY: {
     ENABLED: boolean;
+    USE_ONION: boolean;
     HOST: string;
     PORT: number;
     USERNAME: string;
@@ -75,6 +79,14 @@ interface IConfig {
     TOR_URL: string;
     CLEARNET_URL: string;
   };
+  EXTERNAL_DATA_SERVER: {
+    MEMPOOL_API: string;
+    MEMPOOL_ONION: string;
+    LIQUID_API: string;
+    LIQUID_ONION: string;
+    BISQ_URL: string;
+    BISQ_ONION: string;
+  };
 }
 
 const defaults: IConfig = {
@@ -94,9 +106,10 @@ const defaults: IConfig = {
     'INDEXING_BLOCKS_AMOUNT': 11000, // 0 = disable indexing, -1 = index all blocks
     'PRICE_FEED_UPDATE_INTERVAL': 600,
     'USE_SECOND_NODE_FOR_MINFEE': false,
-    'EXTERNAL_ASSETS': [
+    'EXTERNAL_ASSETS': [],
-      'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json'
+    'EXTERNAL_MAX_RETRY': 1,
-    ],
+    'EXTERNAL_RETRY_INTERVAL': 0,
+    'USER_AGENT': 'mempool',
     'STDOUT_LOG_MIN_PRIORITY': 'debug',
   },
   'ESPLORA': {
@@ -145,6 +158,7 @@ const defaults: IConfig = {
   },
   'SOCKS5PROXY': {
     'ENABLED': false,
+    'USE_ONION': true,
     'HOST': '127.0.0.1',
     'PORT': 9050,
     'USERNAME': '',
@@ -153,6 +167,14 @@ const defaults: IConfig = {
   "PRICE_DATA_SERVER": {
     'TOR_URL': 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices',
     'CLEARNET_URL': 'https://price.bisq.wiz.biz/getAllMarketPrices'
+  },
+  "EXTERNAL_DATA_SERVER": {
+    'MEMPOOL_API': 'https://mempool.space/api/v1',
+    'MEMPOOL_ONION': 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1',
+    'LIQUID_API': 'https://liquid.network/api/v1',
+    'LIQUID_ONION': 'http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1',
+    'BISQ_URL': 'https://bisq.markets/api',
+    'BISQ_ONION': 'http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api'
   }
 };
 
@@ -168,6 +190,7 @@ class Config implements IConfig {
   BISQ: IConfig['BISQ'];
   SOCKS5PROXY: IConfig['SOCKS5PROXY'];
   PRICE_DATA_SERVER: IConfig['PRICE_DATA_SERVER'];
+  EXTERNAL_DATA_SERVER: IConfig['EXTERNAL_DATA_SERVER'];
 
   constructor() {
     const configs = this.merge(configFile, defaults);
@@ -182,6 +205,7 @@ class Config implements IConfig {
     this.BISQ = configs.BISQ;
     this.SOCKS5PROXY = configs.SOCKS5PROXY;
     this.PRICE_DATA_SERVER = configs.PRICE_DATA_SERVER;
+    this.EXTERNAL_DATA_SERVER = configs.EXTERNAL_DATA_SERVER;
   }
 
   merge = (...objects: object[]): IConfig => {

backend/src/index.ts

@@ -205,7 +205,7 @@ class Server {
       .post(config.MEMPOOL.API_URL_PREFIX + 'tx/push', routes.$postTransactionForm)
       .get(config.MEMPOOL.API_URL_PREFIX + 'donations', async (req, res) => {
         try {
-          const response = await axios.get('https://mempool.space/api/v1/donations', { responseType: 'stream', timeout: 10000 });
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/donations`, { responseType: 'stream', timeout: 10000 });
           response.data.pipe(res);
         } catch (e) {
           res.status(500).end();
@@ -213,7 +213,7 @@ class Server {
       })
       .get(config.MEMPOOL.API_URL_PREFIX + 'donations/images/:id', async (req, res) => {
         try {
-          const response = await axios.get('https://mempool.space/api/v1/donations/images/' + req.params.id, {
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/donations/images/${req.params.id}`, {
             responseType: 'stream', timeout: 10000
           });
           response.data.pipe(res);
@@ -223,7 +223,7 @@ class Server {
       })
       .get(config.MEMPOOL.API_URL_PREFIX + 'contributors', async (req, res) => {
         try {
-          const response = await axios.get('https://mempool.space/api/v1/contributors', { responseType: 'stream', timeout: 10000 });
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/contributors`, { responseType: 'stream', timeout: 10000 });
           response.data.pipe(res);
         } catch (e) {
           res.status(500).end();
@@ -231,7 +231,7 @@ class Server {
       })
       .get(config.MEMPOOL.API_URL_PREFIX + 'contributors/images/:id', async (req, res) => {
         try {
-          const response = await axios.get('https://mempool.space/api/v1/contributors/images/' + req.params.id, {
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/contributors/images/${req.params.id}`, {
             responseType: 'stream', timeout: 10000
           });
           response.data.pipe(res);
@@ -241,7 +241,7 @@ class Server {
       })
       .get(config.MEMPOOL.API_URL_PREFIX + 'translators', async (req, res) => {
         try {
-          const response = await axios.get('https://mempool.space/api/v1/translators', { responseType: 'stream', timeout: 10000 });
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/translators`, { responseType: 'stream', timeout: 10000 });
           response.data.pipe(res);
         } catch (e) {
           res.status(500).end();
@@ -249,7 +249,7 @@ class Server {
       })
       .get(config.MEMPOOL.API_URL_PREFIX + 'translators/images/:id', async (req, res) => {
         try {
-          const response = await axios.get('https://mempool.space/api/v1/translators/images/' + req.params.id, {
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/translators/images/${req.params.id}`, {
             responseType: 'stream', timeout: 10000
           });
           response.data.pipe(res);

backend/src/routes.ts

@@ -990,7 +990,7 @@ class Routes {
 
   public async $getAllFeaturedLiquidAssets(req: Request, res: Response) {
     try {
-      const response = await axios.get('https://liquid.network/api/v1/assets/featured', { responseType: 'stream', timeout: 10000 });
+      const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.LIQUID_API}/assets/featured`, { responseType: 'stream', timeout: 10000 });
       response.data.pipe(res);
     } catch (e) {
       res.status(500).end();
@@ -999,7 +999,7 @@ class Routes {
 
   public async $getAssetGroup(req: Request, res: Response) {
     try {
-      const response = await axios.get('https://liquid.network/api/v1/assets/group/' + parseInt(req.params.id, 10),
+      const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.LIQUID_API}/assets/group/${parseInt(req.params.id, 10)}`,
         { responseType: 'stream', timeout: 10000 });
       response.data.pipe(res);
     } catch (e) {

backend/src/sync-assets.ts

@@ -1,6 +1,7 @@
 import axios, { AxiosResponse } from 'axios';
 import * as fs from 'fs';
 import config from './config';
+import backendInfo from './api/backend-info';
 import logger from './logger';
 import { SocksProxyAgent } from 'socks-proxy-agent';
 
@@ -42,6 +43,9 @@ class SyncAssets {
 
           logger.info(`Downloading external asset ${fileName} over the Tor network...`);
           return axios.get(url, {
+            headers: {
+              'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
+            },
             httpAgent: agent,
             httpsAgent: agent,
             responseType: 'stream',
@@ -57,6 +61,9 @@ class SyncAssets {
         } else {
           logger.info(`Downloading external asset ${fileName} over clearnet...`);
           return axios.get(url, {
+            headers: {
+              'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
+            },
             responseType: 'stream',
             timeout: 30000
           }).then(function (response) {

backend/src/tasks/pools-updater.ts

@@ -1,7 +1,8 @@
-import axios from 'axios';
+import axios, { AxiosResponse } from 'axios';
 import poolsParser from '../api/pools-parser';
 import config from '../config';
 import DB from '../database';
+import backendInfo from '../api/backend-info';
 import logger from '../logger';
 import { SocksProxyAgent } from 'socks-proxy-agent';
 import * as https from 'https';
@@ -113,33 +114,45 @@ class PoolsUpdater {
    */
   private async query(path): Promise<object | undefined> {
     type axiosOptions = {
+      headers: {
+        'User-Agent': string
+      };
+      timeout: number;
       httpsAgent?: https.Agent;
     }
     const setDelay = (secs: number = 1): Promise<void> => new Promise(resolve => setTimeout(() => resolve(), secs * 1000));
-    const axiosOptions: axiosOptions = {};
+    const axiosOptions: axiosOptions = {
+      headers: {
+        'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
+      },
+      timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000
+    };
     let retry = 0;
 
-    if (config.SOCKS5PROXY.ENABLED) {
+    while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
-      const socksOptions: any = {
-        agentOptions: {
-          keepAlive: true,
-        },
-        hostname: config.SOCKS5PROXY.HOST,
-        port: config.SOCKS5PROXY.PORT
-      };
-
-      if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
-        socksOptions.username = config.SOCKS5PROXY.USERNAME;
-        socksOptions.password = config.SOCKS5PROXY.PASSWORD;
-      }
-
-      axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions);
-    }
-
-    while(retry < 5) {
       try {
-        const data = await axios.get(path, axiosOptions);
+        if (config.SOCKS5PROXY.ENABLED) {
-        if (data.statusText !== 'OK' || !data.data) {
+          const socksOptions: any = {
+            agentOptions: {
+              keepAlive: true,
+            },
+            hostname: config.SOCKS5PROXY.HOST,
+            port: config.SOCKS5PROXY.PORT
+          };
+
+          if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
+            socksOptions.username = config.SOCKS5PROXY.USERNAME;
+            socksOptions.password = config.SOCKS5PROXY.PASSWORD;
+          } else {
+            // Retry with different tor circuits https://stackoverflow.com/a/64960234
+            socksOptions.username = `circuit${retry}`;
+          }
+
+          axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions);
+        }
+        
+        const data: AxiosResponse = await axios.get(path, axiosOptions);
+        if (data.statusText === 'error' || !data.data) {
           throw new Error(`Could not fetch data from Github, Error: ${data.status}`);
         }
         return data.data;
@@ -147,7 +160,7 @@ class PoolsUpdater {
         logger.err('Could not connect to Github. Reason: '  + (e instanceof Error ? e.message : e));
         retry++;
       }
-      await setDelay();
+      await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL);
     }
     return undefined;
   }

docker/backend/mempool-config.json

@@ -15,6 +15,9 @@
     "PRICE_FEED_UPDATE_INTERVAL": __MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__,
     "USE_SECOND_NODE_FOR_MINFEE": __MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__,
     "EXTERNAL_ASSETS": __MEMPOOL_EXTERNAL_ASSETS__,
+    "EXTERNAL_MAX_RETRY": __MEMPOOL_EXTERNAL_MAX_RETRY__,
+    "EXTERNAL_RETRY_INTERVAL": __MEMPOOL_EXTERNAL_RETRY_INTERVAL__,
+    "USER_AGENT": "__MEMPOOL_USER_AGENT__",
     "STDOUT_LOG_MIN_PRIORITY": "__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__",
     "INDEXING_BLOCKS_AMOUNT": __MEMPOOL_INDEXING_BLOCKS_AMOUNT__
   },
@@ -64,6 +67,7 @@
   },
   "SOCKS5PROXY": {
     "ENABLED": __SOCKS5PROXY_ENABLED__,
+    "USE_ONION": __SOCKS5PROXY_USE_ONION__,
     "HOST": "__SOCKS5PROXY_HOST__",
     "PORT": "__SOCKS5PROXY_PORT__",
     "USERNAME": "__SOCKS5PROXY_USERNAME__",
@@ -72,5 +76,13 @@
   "PRICE_DATA_SERVER": {
     "TOR_URL": "__PRICE_DATA_SERVER_TOR_URL__",
     "CLEARNET_URL": "__PRICE_DATA_SERVER_CLEARNET_URL__"
+  },
+  "EXTERNAL_DATA_SERVER": {
+    "MEMPOOL_API": "__EXTERNAL_DATA_SERVER_MEMPOOL_API__",
+    "MEMPOOL_ONION": "__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__",
+    "LIQUID_API": "__EXTERNAL_DATA_SERVER_LIQUID_API__",
+    "LIQUID_ONION": "__EXTERNAL_DATA_SERVER_LIQUID_ONION__",
+    "BISQ_URL": "__EXTERNAL_DATA_SERVER_BISQ_URL__",
+    "BISQ_ONION": "__EXTERNAL_DATA_SERVER_BISQ_ONION__"
   }
 }

docker/backend/start.sh

@@ -16,7 +16,10 @@ __MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__=${MEMPOOL_MEMPOOL_BLOCKS_AMOUNT:=8}
 __MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=11000}
 __MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__=${MEMPOOL_PRICE_FEED_UPDATE_INTERVAL:=600}
 __MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__=${MEMPOOL_USE_SECOND_NODE_FOR_MINFEE:=false}
-__MEMPOOL_EXTERNAL_ASSETS__=${MEMPOOL_EXTERNAL_ASSETS:=[\"https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json\"]}
+__MEMPOOL_EXTERNAL_ASSETS__=${MEMPOOL_EXTERNAL_ASSETS:=[]}
+__MEMPOOL_EXTERNAL_MAX_RETRY__=${MEMPOOL_EXTERNAL_MAX_RETRY:=1}
+__MEMPOOL_EXTERNAL_RETRY_INTERVAL__=${MEMPOOL_EXTERNAL_RETRY_INTERVAL:=0}
+__MEMPOOL_USER_AGENT__=${MEMPOOL_USER_AGENT:=mempool}
 __MEMPOOL_STDOUT_LOG_MIN_PRIORITY__=${MEMPOOL_STDOUT_LOG_MIN_PRIORITY:=info}
 
 # CORE_RPC
@@ -65,6 +68,7 @@ __BISQ_DATA_PATH__=${BISQ_DATA_PATH:=/bisq/statsnode-data/btc_mainnet/db}
 
 # SOCKS5PROXY
 __SOCKS5PROXY_ENABLED__=${SOCKS5PROXY_ENABLED:=false}
+__SOCKS5PROXY_USE_ONION__=${SOCKS5PROXY_USE_ONION:=true}
 __SOCKS5PROXY_HOST__=${SOCKS5PROXY_HOST:=localhost}
 __SOCKS5PROXY_PORT__=${SOCKS5PROXY_PORT:=9050}
 __SOCKS5PROXY_USERNAME__=${SOCKS5PROXY_USERNAME:=""}
@@ -74,6 +78,14 @@ __SOCKS5PROXY_PASSWORD__=${SOCKS5PROXY_PASSWORD:=""}
 __PRICE_DATA_SERVER_TOR_URL__=${PRICE_DATA_SERVER_TOR_URL:=http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices}
 __PRICE_DATA_SERVER_CLEARNET_URL__=${PRICE_DATA_SERVER_CLEARNET_URL:=https://price.bisq.wiz.biz/getAllMarketPrices}
 
+# EXTERNAL_DATA_SERVER
+__EXTERNAL_DATA_SERVER_MEMPOOL_API__=${EXTERNAL_DATA_SERVER_MEMPOOL_API:=https://mempool.space/api/v1}
+__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__=${EXTERNAL_DATA_SERVER_MEMPOOL_ONION:=http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1}
+__EXTERNAL_DATA_SERVER_LIQUID_API__=${EXTERNAL_DATA_SERVER_LIQUID_API:=https://liquid.network/api/v1}
+__EXTERNAL_DATA_SERVER_LIQUID_ONION__=${EXTERNAL_DATA_SERVER_LIQUID_ONION:=http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1}
+__EXTERNAL_DATA_SERVER_BISQ_URL__=${EXTERNAL_DATA_SERVER_BISQ_URL:=https://bisq.markets/api}
+__EXTERNAL_DATA_SERVER_BISQ_ONION__=${EXTERNAL_DATA_SERVER_BISQ_ONION:=http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api}
+
 mkdir -p "${__MEMPOOL_CACHE_DIR__}"
 
 sed -i "s/__MEMPOOL_NETWORK__/${__MEMPOOL_NETWORK__}/g" mempool-config.json