Merge branch 'master' into bugfix/fix-duplicate-indexing

backend/src/api/fiat-conversion.ts

@@ -1,7 +1,9 @@
 import logger from '../logger';
-import axios from 'axios';
+import axios, { AxiosResponse } from 'axios';
 import { IConversionRates } from '../mempool.interfaces';
 import config from '../config';
+import backendInfo from './backend-info';
+import { SocksProxyAgent } from 'socks-proxy-agent';
 
 class FiatConversion {
   private conversionRates: IConversionRates = {
@@ -17,6 +19,11 @@ class FiatConversion {
 
   public startService() {
     logger.info('Starting currency rates service');
+    if (config.SOCKS5PROXY.ENABLED) {
+      logger.info(`Currency rates service will be queried over the Tor network using ${config.PRICE_DATA_SERVER.TOR_URL}`);
+    } else {
+      logger.info(`Currency rates service will be queried over clearnet using ${config.PRICE_DATA_SERVER.CLEARNET_URL}`);
+    }
     setInterval(this.updateCurrency.bind(this), 1000 * config.MEMPOOL.PRICE_FEED_UPDATE_INTERVAL);
     this.updateCurrency();
   }
@@ -26,12 +33,43 @@ class FiatConversion {
   }
 
   private async updateCurrency(): Promise<void> {
+    const headers = { 'User-Agent': `mempool/v${backendInfo.getBackendInfo().version}` };
+    let fiatConversionUrl: string;
+    let response: AxiosResponse;
+
     try {
-      const response = await axios.get('https://price.bisq.wiz.biz/getAllMarketPrices', { timeout: 10000 });
+      if (config.SOCKS5PROXY.ENABLED) {
+        let socksOptions: any = {
+          agentOptions: {
+            keepAlive: true,
+          },
+          host: config.SOCKS5PROXY.HOST,
+          port: config.SOCKS5PROXY.PORT
+        };
+
+        if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
+          socksOptions.username = config.SOCKS5PROXY.USERNAME;
+          socksOptions.password = config.SOCKS5PROXY.PASSWORD;
+        }
+
+        const agent = new SocksProxyAgent(socksOptions);
+        fiatConversionUrl = config.PRICE_DATA_SERVER.TOR_URL;
+        logger.debug('Querying currency rates service...');
+        response = await axios.get(fiatConversionUrl, { httpAgent: agent, headers: headers, timeout: 30000 });
+      } else {
+        fiatConversionUrl = config.PRICE_DATA_SERVER.CLEARNET_URL;
+        logger.debug('Querying currency rates service...');
+        response = await axios.get(fiatConversionUrl, { headers: headers, timeout: 10000 });
+      }
+
       const usd = response.data.data.find((item: any) => item.currencyCode === 'USD');
+
       this.conversionRates = {
         'USD': usd.price,
       };
+
+      logger.debug(`USD Conversion Rate: ${usd.price}`);
+
       if (this.ratesChangedCallback) {
         this.ratesChangedCallback(this.conversionRates);
       }

backend/src/config.ts

@@ -18,6 +18,7 @@ interface IConfig {
     PRICE_FEED_UPDATE_INTERVAL: number;
     USE_SECOND_NODE_FOR_MINFEE: boolean;
     EXTERNAL_ASSETS: string[];
+    STDOUT_LOG_MIN_PRIORITY: 'emerg' | 'alert' | 'crit' | 'err' | 'warn' | 'notice' | 'info' | 'debug';
   };
   ESPLORA: {
     REST_API_URL: string;
@@ -51,7 +52,7 @@ interface IConfig {
     ENABLED: boolean;
     HOST: string;
     PORT: number;
-    MIN_PRIORITY: 'emerg' | 'alert' | 'crit' | 'err' |'warn' | 'notice' | 'info' | 'debug';
+    MIN_PRIORITY: 'emerg' | 'alert' | 'crit' | 'err' | 'warn' | 'notice' | 'info' | 'debug';
     FACILITY: string;
   };
   STATISTICS: {
@@ -62,6 +63,17 @@ interface IConfig {
     ENABLED: boolean;
     DATA_PATH: string;
   };
+  SOCKS5PROXY: {
+    ENABLED: boolean;
+    HOST: string;
+    PORT: number;
+    USERNAME: string;
+    PASSWORD: string;
+  };
+  PRICE_DATA_SERVER: {
+    TOR_URL: string;
+    CLEARNET_URL: string;
+  };
 }
 
 const defaults: IConfig = {
@@ -79,11 +91,12 @@ const defaults: IConfig = {
     'INITIAL_BLOCKS_AMOUNT': 8,
     'MEMPOOL_BLOCKS_AMOUNT': 8,
     'INDEXING_BLOCKS_AMOUNT': 1100, // 0 = disable indexing, -1 = index all blocks
-    'PRICE_FEED_UPDATE_INTERVAL': 3600,
+    'PRICE_FEED_UPDATE_INTERVAL': 600,
     'USE_SECOND_NODE_FOR_MINFEE': false,
     'EXTERNAL_ASSETS': [
       'https://mempool.space/resources/pools.json'
-    ]
+    ],
+    'STDOUT_LOG_MIN_PRIORITY': 'debug',
   },
   'ESPLORA': {
     'REST_API_URL': 'http://127.0.0.1:3000',
@@ -128,6 +141,17 @@ const defaults: IConfig = {
     'ENABLED': false,
     'DATA_PATH': '/bisq/statsnode-data/btc_mainnet/db'
   },
+  'SOCKS5PROXY': {
+    'ENABLED': false,
+    'HOST': '127.0.0.1',
+    'PORT': 9050,
+    'USERNAME': '',
+    'PASSWORD': ''
+  },
+  "PRICE_DATA_SERVER": {
+    'TOR_URL': 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices',
+    'CLEARNET_URL': 'https://price.bisq.wiz.biz/getAllMarketPrices'
+  }
 };
 
 class Config implements IConfig {
@@ -140,6 +164,8 @@ class Config implements IConfig {
   SYSLOG: IConfig['SYSLOG'];
   STATISTICS: IConfig['STATISTICS'];
   BISQ: IConfig['BISQ'];
+  SOCKS5PROXY: IConfig['SOCKS5PROXY'];
+  PRICE_DATA_SERVER: IConfig['PRICE_DATA_SERVER'];
 
   constructor() {
     const configs = this.merge(configFile, defaults);
@@ -152,6 +178,8 @@ class Config implements IConfig {
     this.SYSLOG = configs.SYSLOG;
     this.STATISTICS = configs.STATISTICS;
     this.BISQ = configs.BISQ;
+    this.SOCKS5PROXY = configs.SOCKS5PROXY;
+    this.PRICE_DATA_SERVER = configs.PRICE_DATA_SERVER;
   }
 
   merge = (...objects: object[]): IConfig => {

backend/src/index.ts

@@ -319,7 +319,9 @@ class Server {
     if (Common.isLiquid()) {
       this.app
         .get(config.MEMPOOL.API_URL_PREFIX + 'assets/icons', routes.getAllLiquidIcon)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'assets/featured', routes.$getAllFeaturedLiquidAssets)
         .get(config.MEMPOOL.API_URL_PREFIX + 'asset/:assetId/icon', routes.getLiquidIcon)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'assets/group/:id', routes.$getAssetGroup)
       ;
     }
 

backend/src/logger.ts

@@ -97,6 +97,9 @@ class Logger {
       syslogmsg = `<${(Logger.facilities[config.SYSLOG.FACILITY] * 8 + prionum)}> ${this.name}[${process.pid}]: ${priority.toUpperCase()}${network} ${msg}`;
       this.syslog(syslogmsg);
     }
+    if (Logger.priorities[priority] > Logger.priorities[config.MEMPOOL.STDOUT_LOG_MIN_PRIORITY]) {
+      return;
+    }
     if (priority === 'warning') {
       priority = 'warn';
     }

backend/src/routes.ts

@@ -21,6 +21,7 @@ import bitcoinClient from './api/bitcoin/bitcoin-client';
 import elementsParser from './api/liquid/elements-parser';
 import icons from './api/liquid/icons';
 import miningStats from './api/mining';
+import axios from 'axios';
 
 class Routes {
   constructor() {}
@@ -855,6 +856,25 @@ class Routes {
       res.status(404).send('Asset icons not found');
     }
   }
+
+  public async $getAllFeaturedLiquidAssets(req: Request, res: Response) {
+    try {
+      const response = await axios.get('https://liquid.network/api/v1/assets/featured', { responseType: 'stream', timeout: 10000 });
+      response.data.pipe(res);
+    } catch (e) {
+      res.status(500).end();
+    }
+  }
+
+  public async $getAssetGroup(req: Request, res: Response) {
+    try {
+      const response = await axios.get('https://liquid.network/api/v1/assets/group/' + parseInt(req.params.id, 10),
+        { responseType: 'stream', timeout: 10000 });
+      response.data.pipe(res);
+    } catch (e) {
+      res.status(500).end();
+    }
+  }
 }
 
 export default new Routes();