From 0400deacf29e2141f856cd5806955010025195e1 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn Date: Thu, 3 Feb 2022 23:14:56 -0800 Subject: [PATCH 1/8] Add socks-proxy-agent dependency --- backend/package-lock.json | 161 ++++++++++++++++++++++++++++++++++++++ backend/package.json | 1 + 2 files changed, 162 insertions(+) diff --git a/backend/package-lock.json b/backend/package-lock.json index a93e977a3..f2ef1abb3 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -19,6 +19,7 @@ "locutus": "^2.0.12", "mysql2": "2.3.3", "node-worker-threads-pool": "^1.4.3", + "socks-proxy-agent": "^6.1.1", "typescript": "4.4.4", "ws": "8.3.0" }, @@ -181,6 +182,38 @@ "node": ">= 0.6" } }, + "node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/agent-base/node_modules/debug": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.3.tgz", + "integrity": "sha512-/zxw5+vh1Tfv+4Qn7a5nsbcJKPaSvCDhojn6FEl9vupwK2VCSDtEiEtqr8DFtzYFOdz63LBkxec7DYuc2jon6Q==", + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/agent-base/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, "node_modules/ansi-styles": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", @@ -731,6 +764,11 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, + "node_modules/ip": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/ip/-/ip-1.1.5.tgz", + "integrity": "sha1-vd7XARQpCCjAoDnnLvJfWq7ENUo=" + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -1189,6 +1227,62 @@ "sha.js": "bin.js" } }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.6.1.tgz", + "integrity": "sha512-kLQ9N5ucj8uIcxrDwjm0Jsqk06xdpBjGNQtpXy4Q8/QY2k+fY7nZH8CARy+hkbG+SGAovmzzuauCpBlb8FrnBA==", + "dependencies": { + "ip": "^1.1.5", + "smart-buffer": "^4.1.0" + }, + "engines": { + "node": ">= 10.13.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks-proxy-agent": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-6.1.1.tgz", + "integrity": "sha512-t8J0kG3csjA4g6FTbsMOWws+7R7vuRC8aQ/wy3/1OWmsgwA68zs/+cExQ0koSitUDXqhufF/YJr9wtNMZHw5Ew==", + "dependencies": { + "agent-base": "^6.0.2", + "debug": "^4.3.1", + "socks": "^2.6.1" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/socks-proxy-agent/node_modules/debug": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.3.tgz", + "integrity": "sha512-/zxw5+vh1Tfv+4Qn7a5nsbcJKPaSvCDhojn6FEl9vupwK2VCSDtEiEtqr8DFtzYFOdz63LBkxec7DYuc2jon6Q==", + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/socks-proxy-agent/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, "node_modules/sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", @@ -1538,6 +1632,29 @@ "negotiator": "0.6.2" } }, + "agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "requires": { + "debug": "4" + }, + "dependencies": { + "debug": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.3.tgz", + "integrity": "sha512-/zxw5+vh1Tfv+4Qn7a5nsbcJKPaSvCDhojn6FEl9vupwK2VCSDtEiEtqr8DFtzYFOdz63LBkxec7DYuc2jon6Q==", + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } + } + }, "ansi-styles": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", @@ -1992,6 +2109,11 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, + "ip": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/ip/-/ip-1.1.5.tgz", + "integrity": "sha1-vd7XARQpCCjAoDnnLvJfWq7ENUo=" + }, "ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -2352,6 +2474,45 @@ "safe-buffer": "^5.0.1" } }, + "smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==" + }, + "socks": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.6.1.tgz", + "integrity": "sha512-kLQ9N5ucj8uIcxrDwjm0Jsqk06xdpBjGNQtpXy4Q8/QY2k+fY7nZH8CARy+hkbG+SGAovmzzuauCpBlb8FrnBA==", + "requires": { + "ip": "^1.1.5", + "smart-buffer": "^4.1.0" + } + }, + "socks-proxy-agent": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-6.1.1.tgz", + "integrity": "sha512-t8J0kG3csjA4g6FTbsMOWws+7R7vuRC8aQ/wy3/1OWmsgwA68zs/+cExQ0koSitUDXqhufF/YJr9wtNMZHw5Ew==", + "requires": { + "agent-base": "^6.0.2", + "debug": "^4.3.1", + "socks": "^2.6.1" + }, + "dependencies": { + "debug": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.3.tgz", + "integrity": "sha512-/zxw5+vh1Tfv+4Qn7a5nsbcJKPaSvCDhojn6FEl9vupwK2VCSDtEiEtqr8DFtzYFOdz63LBkxec7DYuc2jon6Q==", + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } + } + }, "sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", diff --git a/backend/package.json b/backend/package.json index a5c9a32ff..e72d163b2 100644 --- a/backend/package.json +++ b/backend/package.json @@ -38,6 +38,7 @@ "locutus": "^2.0.12", "mysql2": "2.3.3", "node-worker-threads-pool": "^1.4.3", + "socks-proxy-agent": "^6.1.1", "typescript": "4.4.4", "ws": "8.3.0" }, From 0a954e8bcf90f91e17766241160af5094d491431 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn Date: Thu, 3 Feb 2022 23:16:30 -0800 Subject: [PATCH 2/8] Allow synthetic default imports due to the socks library --- backend/tsconfig.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/tsconfig.json b/backend/tsconfig.json index 85a3cb358..8b4cbea2e 100644 --- a/backend/tsconfig.json +++ b/backend/tsconfig.json @@ -10,7 +10,8 @@ "moduleResolution": "node", "typeRoots": [ "node_modules/@types" - ] + ], + "allowSyntheticDefaultImports": true }, "include": [ "src/**/*.ts" From 8a996cedb4e2df50bdc6fe63d98c975a5f5fc678 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn Date: Thu, 3 Feb 2022 23:21:19 -0800 Subject: [PATCH 3/8] Update README, reference config files and Docker with the new SOCKS config --- README.md | 19 +++++++++++++++++++ backend/mempool-config.sample.json | 7 +++++++ docker/backend/mempool-config.json | 7 +++++++ docker/backend/start.sh | 13 +++++++++++++ 4 files changed, 46 insertions(+) diff --git a/README.md b/README.md index 34dca4862..151edfab7 100644 --- a/README.md +++ b/README.md @@ -245,6 +245,25 @@ docker-compose overrides: BISQ_DATA_PATH: "" ``` +JSON: +``` + "SOCKS5PROXY": { + "ENABLED": false, + "HOST": "127.0.0.1", + "PORT": "9050", + "USERNAME": "", + "PASSWORD": "" + } +``` + +docker-compose overrides: +``` + SOCKS5PROXY_ENABLED: "" + SOCKS5PROXY_HOST: "" + SOCKS5PROXY_PORT: "" + SOCKS5PROXY_USERNAME: "" + SOCKS5PROXY_PASSWORD: "" +``` # Manual Installation diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 8a9295b3a..e977d8046 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -61,5 +61,12 @@ "BISQ": { "ENABLED": false, "DATA_PATH": "/bisq/statsnode-data/btc_mainnet/db" + }, + "SOCKS5PROXY": { + "ENABLED": false, + "HOST": "127.0.0.1", + "PORT": 9050, + "USERNAME": "", + "PASSWORD": "" } } diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index 00dc31aad..788ab6505 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -58,5 +58,12 @@ "BISQ": { "ENABLED": __BISQ_ENABLED__, "DATA_PATH": "__BISQ_DATA_PATH__" + }, + "SOCKS5PROXY": { + "ENABLED": __SOCKS5PROXY_ENABLED__, + "HOST": "__SOCKS5PROXY_HOST__", + "PORT": "__SOCKS5PROXY_PORT__", + "USERNAME": "__SOCKS5PROXY_USERNAME__", + "PASSWORD": "__SOCKS5PROXY_PASSWORD__" } } diff --git a/docker/backend/start.sh b/docker/backend/start.sh index 372be346e..1d1155a66 100644 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -61,6 +61,13 @@ __STATISTICS_TX_PER_SECOND_SAMPLE_PERIOD__=${STATISTICS_TX_PER_SECOND_SAMPLE_PER __BISQ_ENABLED__=${BISQ_ENABLED:=false} __BISQ_DATA_PATH__=${BISQ_DATA_PATH:=/bisq/statsnode-data/btc_mainnet/db} +# SOCKS5PROXY +__SOCKS5PROXY_ENABLED__=${SOCKS5PROXY_ENABLED:=false} +__SOCKS5PROXY_HOST__=${SOCKS5PROXY_HOST:=localhost} +__SOCKS5PROXY_PORT__=${SOCKS5PROXY_PORT:=9050} +__SOCKS5PROXY_USERNAME__=${SOCKS5PROXY_USERNAME:=""} +__SOCKS5PROXY_PASSWORD__=${SOCKS5PROXY_PASSWORD:=""} + mkdir -p "${__MEMPOOL_CACHE_DIR__}" sed -i "s/__MEMPOOL_NETWORK__/${__MEMPOOL_NETWORK__}/g" mempool-config.json @@ -115,4 +122,10 @@ sed -i "s/__STATISTICS_TX_PER_SECOND_SAMPLE_PERIOD__/${__STATISTICS_TX_PER_SECON sed -i "s/__BISQ_ENABLED__/${__BISQ_ENABLED__}/g" mempool-config.json sed -i "s!__BISQ_DATA_PATH__!${__BISQ_DATA_PATH__}!g" mempool-config.json +sed -i "s/__SOCKS5PROXY_ENABLED__/${__SOCKS5PROXY_ENABLED__}/g" mempool-config.json +sed -i "s/__SOCKS5PROXY_HOST__/${__SOCKS5PROXY_HOST__}/g" mempool-config.json +sed -i "s/__SOCKS5PROXY_PORT__/${__SOCKS5PROXY_PORT__}/g" mempool-config.json +sed -i "s/__SOCKS5PROXY_USERNAME__/${__SOCKS5PROXY_USERNAME__}/g" mempool-config.json +sed -i "s/__SOCKS5PROXY_PASSWORD__/${__SOCKS5PROXY_PASSWORD__}/g" mempool-config.json + node /backend/dist/index.js From b1dde4d8b1c5382c43349a12dc840885f24fbd2f Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn Date: Thu, 3 Feb 2022 23:23:57 -0800 Subject: [PATCH 4/8] Query conversion rates service over clearnet or Tor with mempool User-Agent --- backend/src/api/fiat-conversion.ts | 27 ++++++++++++++++++++++++++- backend/src/config.ts | 18 +++++++++++++++++- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/backend/src/api/fiat-conversion.ts b/backend/src/api/fiat-conversion.ts index d779e31fd..e6b44300a 100644 --- a/backend/src/api/fiat-conversion.ts +++ b/backend/src/api/fiat-conversion.ts @@ -2,6 +2,8 @@ import logger from '../logger'; import axios from 'axios'; import { IConversionRates } from '../mempool.interfaces'; import config from '../config'; +import backendInfo from './backend-info'; +import { SocksProxyAgent } from 'socks-proxy-agent'; class FiatConversion { private conversionRates: IConversionRates = { @@ -17,6 +19,9 @@ class FiatConversion { public startService() { logger.info('Starting currency rates service'); + if (config.SOCKS5PROXY.ENABLED) { + logger.info('Currency rates service will be queried over the Tor network'); + } setInterval(this.updateCurrency.bind(this), 1000 * config.MEMPOOL.PRICE_FEED_UPDATE_INTERVAL); this.updateCurrency(); } @@ -26,8 +31,28 @@ class FiatConversion { } private async updateCurrency(): Promise { + const headers = { 'User-Agent': `mempool/v${backendInfo.getBackendInfo().version}` }; + let response; try { - const response = await axios.get('https://price.bisq.wiz.biz/getAllMarketPrices', { timeout: 10000 }); + let fiatConversionUrl = 'https://price.bisq.wiz.biz/getAllMarketPrices'; + if (config.SOCKS5PROXY.ENABLED) { + let socksOptions: any = { + agentOptions: { + keepAlive: true, + }, + host: config.SOCKS5PROXY.HOST, + port: config.SOCKS5PROXY.PORT + }; + if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) { + socksOptions.username = config.SOCKS5PROXY.USERNAME; + socksOptions.password = config.SOCKS5PROXY.PASSWORD; + } + const agent = new SocksProxyAgent(socksOptions); + fiatConversionUrl = 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices'; + response = await axios.get(fiatConversionUrl, { httpAgent: agent, headers: headers, timeout: 30000 }); + } else { + response = await axios.get(fiatConversionUrl, { headers: headers, timeout: 10000 }); + } const usd = response.data.data.find((item: any) => item.currencyCode === 'USD'); this.conversionRates = { 'USD': usd.price, diff --git a/backend/src/config.ts b/backend/src/config.ts index 085d538c4..e610e9f47 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -51,7 +51,7 @@ interface IConfig { ENABLED: boolean; HOST: string; PORT: number; - MIN_PRIORITY: 'emerg' | 'alert' | 'crit' | 'err' |'warn' | 'notice' | 'info' | 'debug'; + MIN_PRIORITY: 'emerg' | 'alert' | 'crit' | 'err' | 'warn' | 'notice' | 'info' | 'debug'; FACILITY: string; }; STATISTICS: { @@ -62,6 +62,13 @@ interface IConfig { ENABLED: boolean; DATA_PATH: string; }; + SOCKS5PROXY: { + ENABLED: boolean; + HOST: string; + PORT: number; + USERNAME: string; + PASSWORD: string; + }; } const defaults: IConfig = { @@ -128,6 +135,13 @@ const defaults: IConfig = { 'ENABLED': false, 'DATA_PATH': '/bisq/statsnode-data/btc_mainnet/db' }, + 'SOCKS5PROXY': { + 'ENABLED': false, + 'HOST': '127.0.0.1', + 'PORT': 9050, + 'USERNAME': '', + 'PASSWORD': '' + } }; class Config implements IConfig { @@ -140,6 +154,7 @@ class Config implements IConfig { SYSLOG: IConfig['SYSLOG']; STATISTICS: IConfig['STATISTICS']; BISQ: IConfig['BISQ']; + SOCKS5PROXY: IConfig['SOCKS5PROXY']; constructor() { const configs = this.merge(configFile, defaults); @@ -152,6 +167,7 @@ class Config implements IConfig { this.SYSLOG = configs.SYSLOG; this.STATISTICS = configs.STATISTICS; this.BISQ = configs.BISQ; + this.SOCKS5PROXY = configs.SOCKS5PROXY; } merge = (...objects: object[]): IConfig => { From 452375aaf7aadd3fc0f6350d4a306fa6d4c62adb Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn Date: Fri, 4 Feb 2022 22:48:16 -0800 Subject: [PATCH 5/8] Make the Currency Conversion Service URLs configurable and log when queried --- README.md | 14 ++++++++++++++ backend/mempool-config.sample.json | 4 ++++ backend/src/api/fiat-conversion.ts | 23 ++++++++++++++++++----- backend/src/config.ts | 10 ++++++++++ docker/backend/mempool-config.json | 4 ++++ docker/backend/start.sh | 7 +++++++ 6 files changed, 57 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 151edfab7..c50282311 100644 --- a/README.md +++ b/README.md @@ -265,6 +265,20 @@ docker-compose overrides: SOCKS5PROXY_PASSWORD: "" ``` +JSON: +``` + "PRICENODE": { + "TOR_URL": "http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices", + "CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices" + } +``` + +docker-compose overrides: +``` + PRICENODE_TOR_URL: "" + PRICENODE_CLEARNET_URL: "" +``` + # Manual Installation The following instructions are for a manual installation on Linux or FreeBSD. The file and directory paths may need to be changed to match your OS. diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index e977d8046..1b874c665 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -68,5 +68,9 @@ "PORT": 9050, "USERNAME": "", "PASSWORD": "" + }, + "PRICENODE": { + "TOR_URL": "http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices", + "CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices" } } diff --git a/backend/src/api/fiat-conversion.ts b/backend/src/api/fiat-conversion.ts index e6b44300a..df10d7698 100644 --- a/backend/src/api/fiat-conversion.ts +++ b/backend/src/api/fiat-conversion.ts @@ -1,5 +1,5 @@ import logger from '../logger'; -import axios from 'axios'; +import axios, { AxiosResponse } from 'axios'; import { IConversionRates } from '../mempool.interfaces'; import config from '../config'; import backendInfo from './backend-info'; @@ -20,7 +20,9 @@ class FiatConversion { public startService() { logger.info('Starting currency rates service'); if (config.SOCKS5PROXY.ENABLED) { - logger.info('Currency rates service will be queried over the Tor network'); + logger.info(`Currency rates service will be queried over the Tor network using ${config.PRICENODE.TOR_URL}`); + } else { + logger.info(`Currency rates service will be queried over clearnet using ${config.PRICENODE.CLEARNET_URL}`); } setInterval(this.updateCurrency.bind(this), 1000 * config.MEMPOOL.PRICE_FEED_UPDATE_INTERVAL); this.updateCurrency(); @@ -32,9 +34,10 @@ class FiatConversion { private async updateCurrency(): Promise { const headers = { 'User-Agent': `mempool/v${backendInfo.getBackendInfo().version}` }; - let response; + let fiatConversionUrl: string; + let response: AxiosResponse; + try { - let fiatConversionUrl = 'https://price.bisq.wiz.biz/getAllMarketPrices'; if (config.SOCKS5PROXY.ENABLED) { let socksOptions: any = { agentOptions: { @@ -43,20 +46,30 @@ class FiatConversion { host: config.SOCKS5PROXY.HOST, port: config.SOCKS5PROXY.PORT }; + if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) { socksOptions.username = config.SOCKS5PROXY.USERNAME; socksOptions.password = config.SOCKS5PROXY.PASSWORD; } + const agent = new SocksProxyAgent(socksOptions); - fiatConversionUrl = 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices'; + fiatConversionUrl = config.PRICENODE.TOR_URL; + logger.info('Querying currency rates service...'); response = await axios.get(fiatConversionUrl, { httpAgent: agent, headers: headers, timeout: 30000 }); } else { + fiatConversionUrl = config.PRICENODE.CLEARNET_URL; + logger.info('Querying currency rates service...'); response = await axios.get(fiatConversionUrl, { headers: headers, timeout: 10000 }); } + const usd = response.data.data.find((item: any) => item.currencyCode === 'USD'); + this.conversionRates = { 'USD': usd.price, }; + + logger.info(`USD Conversion Rate: ${usd.price}`); + if (this.ratesChangedCallback) { this.ratesChangedCallback(this.conversionRates); } diff --git a/backend/src/config.ts b/backend/src/config.ts index e610e9f47..058895efb 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -69,6 +69,10 @@ interface IConfig { USERNAME: string; PASSWORD: string; }; + PRICENODE: { + TOR_URL: string; + CLEARNET_URL: string; + }; } const defaults: IConfig = { @@ -141,6 +145,10 @@ const defaults: IConfig = { 'PORT': 9050, 'USERNAME': '', 'PASSWORD': '' + }, + "PRICENODE": { + 'TOR_URL': 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices', + 'CLEARNET_URL': 'https://price.bisq.wiz.biz/getAllMarketPrices' } }; @@ -155,6 +163,7 @@ class Config implements IConfig { STATISTICS: IConfig['STATISTICS']; BISQ: IConfig['BISQ']; SOCKS5PROXY: IConfig['SOCKS5PROXY']; + PRICENODE: IConfig['PRICENODE']; constructor() { const configs = this.merge(configFile, defaults); @@ -168,6 +177,7 @@ class Config implements IConfig { this.STATISTICS = configs.STATISTICS; this.BISQ = configs.BISQ; this.SOCKS5PROXY = configs.SOCKS5PROXY; + this.PRICENODE = configs.PRICENODE; } merge = (...objects: object[]): IConfig => { diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index 788ab6505..5a069455d 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -65,5 +65,9 @@ "PORT": "__SOCKS5PROXY_PORT__", "USERNAME": "__SOCKS5PROXY_USERNAME__", "PASSWORD": "__SOCKS5PROXY_PASSWORD__" + }, + "PRICENODE": { + "TOR_URL": "__PRICENODE_TOR_URL__", + "CLEARNET_URL": "__PRICENODE_CLEARNET_URL__" } } diff --git a/docker/backend/start.sh b/docker/backend/start.sh index 1d1155a66..0ae1e32a2 100644 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -68,6 +68,10 @@ __SOCKS5PROXY_PORT__=${SOCKS5PROXY_PORT:=9050} __SOCKS5PROXY_USERNAME__=${SOCKS5PROXY_USERNAME:=""} __SOCKS5PROXY_PASSWORD__=${SOCKS5PROXY_PASSWORD:=""} +# PRICENODE +__PRICENODE_TOR_URL__=${PRICENODE_TOR_URL:=http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices} +__PRICENODE_CLEARNET_URL__=${PRICENODE_CLEARNET_URL:=https://price.bisq.wiz.biz/getAllMarketPrices} + mkdir -p "${__MEMPOOL_CACHE_DIR__}" sed -i "s/__MEMPOOL_NETWORK__/${__MEMPOOL_NETWORK__}/g" mempool-config.json @@ -128,4 +132,7 @@ sed -i "s/__SOCKS5PROXY_PORT__/${__SOCKS5PROXY_PORT__}/g" mempool-config.json sed -i "s/__SOCKS5PROXY_USERNAME__/${__SOCKS5PROXY_USERNAME__}/g" mempool-config.json sed -i "s/__SOCKS5PROXY_PASSWORD__/${__SOCKS5PROXY_PASSWORD__}/g" mempool-config.json +sed -i "s!__PRICENODE_TOR_URL__!${__PRICENODE_TOR_URL__}!g" mempool-config.json +sed -i "s!__PRICENODE_CLEARNET_URL__!${__PRICENODE_CLEARNET_URL__}!g" mempool-config.json + node /backend/dist/index.js From ee7f8d8d1845239aa691710aff1c65f21e0fb5f1 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn Date: Sat, 5 Feb 2022 00:33:16 -0800 Subject: [PATCH 6/8] Reduce currency rates polling interval from 1 hour to 10 minutes --- README.md | 2 +- backend/mempool-config.sample.json | 2 +- backend/src/config.ts | 2 +- docker/backend/start.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c50282311..3dc04b81b 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ JSON: "BLOCK_WEIGHT_UNITS": 4000000, "INITIAL_BLOCKS_AMOUNT": 8, "MEMPOOL_BLOCKS_AMOUNT": 8, - "PRICE_FEED_UPDATE_INTERVAL": 3600, + "PRICE_FEED_UPDATE_INTERVAL": 600, "USE_SECOND_NODE_FOR_MINFEE": false, "EXTERNAL_ASSETS": [] }, diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 1b874c665..b9d7fb982 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -13,7 +13,7 @@ "INITIAL_BLOCKS_AMOUNT": 8, "MEMPOOL_BLOCKS_AMOUNT": 8, "INDEXING_BLOCKS_AMOUNT": 1100, - "PRICE_FEED_UPDATE_INTERVAL": 3600, + "PRICE_FEED_UPDATE_INTERVAL": 600, "USE_SECOND_NODE_FOR_MINFEE": false, "EXTERNAL_ASSETS": [ "https://mempool.space/resources/pools.json" diff --git a/backend/src/config.ts b/backend/src/config.ts index 058895efb..f2787977b 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -90,7 +90,7 @@ const defaults: IConfig = { 'INITIAL_BLOCKS_AMOUNT': 8, 'MEMPOOL_BLOCKS_AMOUNT': 8, 'INDEXING_BLOCKS_AMOUNT': 1100, // 0 = disable indexing, -1 = index all blocks - 'PRICE_FEED_UPDATE_INTERVAL': 3600, + 'PRICE_FEED_UPDATE_INTERVAL': 600, 'USE_SECOND_NODE_FOR_MINFEE': false, 'EXTERNAL_ASSETS': [ 'https://mempool.space/resources/pools.json' diff --git a/docker/backend/start.sh b/docker/backend/start.sh index 0ae1e32a2..d1422bf45 100644 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -14,7 +14,7 @@ __MEMPOOL_BLOCK_WEIGHT_UNITS__=${MEMPOOL_BLOCK_WEIGHT_UNITS:=4000000} __MEMPOOL_INITIAL_BLOCKS_AMOUNT__=${MEMPOOL_INITIAL_BLOCKS_AMOUNT:=8} __MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__=${MEMPOOL_MEMPOOL_BLOCKS_AMOUNT:=8} __MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=1100} -__MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__=${MEMPOOL_PRICE_FEED_UPDATE_INTERVAL:=3600} +__MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__=${MEMPOOL_PRICE_FEED_UPDATE_INTERVAL:=600} __MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__=${MEMPOOL_USE_SECOND_NODE_FOR_MINFEE:=false} __MEMPOOL_EXTERNAL_ASSETS__=${MEMPOOL_EXTERNAL_ASSETS:=[]} From 6b2900345a55ed6676bed9e6a3087a4257539915 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn <100320+knorrium@users.noreply.github.com> Date: Sat, 5 Feb 2022 07:58:35 -0800 Subject: [PATCH 7/8] Rename PRICENODE to PRICE_DATA_SERVER --- README.md | 6 +++--- backend/mempool-config.sample.json | 2 +- backend/src/api/fiat-conversion.ts | 8 ++++---- backend/src/config.ts | 8 ++++---- docker/backend/mempool-config.json | 6 +++--- docker/backend/start.sh | 10 +++++----- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 3dc04b81b..75146d4b1 100644 --- a/README.md +++ b/README.md @@ -267,7 +267,7 @@ docker-compose overrides: JSON: ``` - "PRICENODE": { + "PRICE_DATA_SERVER": { "TOR_URL": "http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices", "CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices" } @@ -275,8 +275,8 @@ JSON: docker-compose overrides: ``` - PRICENODE_TOR_URL: "" - PRICENODE_CLEARNET_URL: "" + PRICE_DATA_SERVER_TOR_URL: "" + PRICE_DATA_SERVER_CLEARNET_URL: "" ``` # Manual Installation diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index b9d7fb982..dc0f1a69e 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -69,7 +69,7 @@ "USERNAME": "", "PASSWORD": "" }, - "PRICENODE": { + "PRICE_DATA_SERVER": { "TOR_URL": "http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices", "CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices" } diff --git a/backend/src/api/fiat-conversion.ts b/backend/src/api/fiat-conversion.ts index df10d7698..be5331d7a 100644 --- a/backend/src/api/fiat-conversion.ts +++ b/backend/src/api/fiat-conversion.ts @@ -20,9 +20,9 @@ class FiatConversion { public startService() { logger.info('Starting currency rates service'); if (config.SOCKS5PROXY.ENABLED) { - logger.info(`Currency rates service will be queried over the Tor network using ${config.PRICENODE.TOR_URL}`); + logger.info(`Currency rates service will be queried over the Tor network using ${config.PRICE_DATA_SERVER.TOR_URL}`); } else { - logger.info(`Currency rates service will be queried over clearnet using ${config.PRICENODE.CLEARNET_URL}`); + logger.info(`Currency rates service will be queried over clearnet using ${config.PRICE_DATA_SERVER.CLEARNET_URL}`); } setInterval(this.updateCurrency.bind(this), 1000 * config.MEMPOOL.PRICE_FEED_UPDATE_INTERVAL); this.updateCurrency(); @@ -53,11 +53,11 @@ class FiatConversion { } const agent = new SocksProxyAgent(socksOptions); - fiatConversionUrl = config.PRICENODE.TOR_URL; + fiatConversionUrl = config.PRICE_DATA_SERVER.TOR_URL; logger.info('Querying currency rates service...'); response = await axios.get(fiatConversionUrl, { httpAgent: agent, headers: headers, timeout: 30000 }); } else { - fiatConversionUrl = config.PRICENODE.CLEARNET_URL; + fiatConversionUrl = config.PRICE_DATA_SERVER.CLEARNET_URL; logger.info('Querying currency rates service...'); response = await axios.get(fiatConversionUrl, { headers: headers, timeout: 10000 }); } diff --git a/backend/src/config.ts b/backend/src/config.ts index f2787977b..040d94bf1 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -69,7 +69,7 @@ interface IConfig { USERNAME: string; PASSWORD: string; }; - PRICENODE: { + PRICE_DATA_SERVER: { TOR_URL: string; CLEARNET_URL: string; }; @@ -146,7 +146,7 @@ const defaults: IConfig = { 'USERNAME': '', 'PASSWORD': '' }, - "PRICENODE": { + "PRICE_DATA_SERVER": { 'TOR_URL': 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices', 'CLEARNET_URL': 'https://price.bisq.wiz.biz/getAllMarketPrices' } @@ -163,7 +163,7 @@ class Config implements IConfig { STATISTICS: IConfig['STATISTICS']; BISQ: IConfig['BISQ']; SOCKS5PROXY: IConfig['SOCKS5PROXY']; - PRICENODE: IConfig['PRICENODE']; + PRICE_DATA_SERVER: IConfig['PRICE_DATA_SERVER']; constructor() { const configs = this.merge(configFile, defaults); @@ -177,7 +177,7 @@ class Config implements IConfig { this.STATISTICS = configs.STATISTICS; this.BISQ = configs.BISQ; this.SOCKS5PROXY = configs.SOCKS5PROXY; - this.PRICENODE = configs.PRICENODE; + this.PRICE_DATA_SERVER = configs.PRICE_DATA_SERVER; } merge = (...objects: object[]): IConfig => { diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index 5a069455d..b97c9e571 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -66,8 +66,8 @@ "USERNAME": "__SOCKS5PROXY_USERNAME__", "PASSWORD": "__SOCKS5PROXY_PASSWORD__" }, - "PRICENODE": { - "TOR_URL": "__PRICENODE_TOR_URL__", - "CLEARNET_URL": "__PRICENODE_CLEARNET_URL__" + "PRICE_DATA_SERVER": { + "TOR_URL": "__PRICE_DATA_SERVER_TOR_URL__", + "CLEARNET_URL": "__PRICE_DATA_SERVER_CLEARNET_URL__" } } diff --git a/docker/backend/start.sh b/docker/backend/start.sh index d1422bf45..d12978970 100644 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -68,9 +68,9 @@ __SOCKS5PROXY_PORT__=${SOCKS5PROXY_PORT:=9050} __SOCKS5PROXY_USERNAME__=${SOCKS5PROXY_USERNAME:=""} __SOCKS5PROXY_PASSWORD__=${SOCKS5PROXY_PASSWORD:=""} -# PRICENODE -__PRICENODE_TOR_URL__=${PRICENODE_TOR_URL:=http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices} -__PRICENODE_CLEARNET_URL__=${PRICENODE_CLEARNET_URL:=https://price.bisq.wiz.biz/getAllMarketPrices} +# PRICE_DATA_SERVER +__PRICE_DATA_SERVER_TOR_URL__=${PRICE_DATA_SERVER_TOR_URL:=http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices} +__PRICE_DATA_SERVER_CLEARNET_URL__=${PRICE_DATA_SERVER_CLEARNET_URL:=https://price.bisq.wiz.biz/getAllMarketPrices} mkdir -p "${__MEMPOOL_CACHE_DIR__}" @@ -132,7 +132,7 @@ sed -i "s/__SOCKS5PROXY_PORT__/${__SOCKS5PROXY_PORT__}/g" mempool-config.json sed -i "s/__SOCKS5PROXY_USERNAME__/${__SOCKS5PROXY_USERNAME__}/g" mempool-config.json sed -i "s/__SOCKS5PROXY_PASSWORD__/${__SOCKS5PROXY_PASSWORD__}/g" mempool-config.json -sed -i "s!__PRICENODE_TOR_URL__!${__PRICENODE_TOR_URL__}!g" mempool-config.json -sed -i "s!__PRICENODE_CLEARNET_URL__!${__PRICENODE_CLEARNET_URL__}!g" mempool-config.json +sed -i "s!__PRICE_DATA_SERVER_TOR_URL__!${__PRICE_DATA_SERVER_TOR_URL__}!g" mempool-config.json +sed -i "s!__PRICE_DATA_SERVER_CLEARNET_URL__!${__PRICE_DATA_SERVER_CLEARNET_URL__}!g" mempool-config.json node /backend/dist/index.js From 4afeb3998a247dbd49bdfdbbd1631268ca5eb8d9 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn <100320+knorrium@users.noreply.github.com> Date: Sat, 5 Feb 2022 15:38:03 -0800 Subject: [PATCH 8/8] Change fiat conversion logs to DEBUG --- backend/src/api/fiat-conversion.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/src/api/fiat-conversion.ts b/backend/src/api/fiat-conversion.ts index be5331d7a..f28f34a0b 100644 --- a/backend/src/api/fiat-conversion.ts +++ b/backend/src/api/fiat-conversion.ts @@ -54,11 +54,11 @@ class FiatConversion { const agent = new SocksProxyAgent(socksOptions); fiatConversionUrl = config.PRICE_DATA_SERVER.TOR_URL; - logger.info('Querying currency rates service...'); + logger.debug('Querying currency rates service...'); response = await axios.get(fiatConversionUrl, { httpAgent: agent, headers: headers, timeout: 30000 }); } else { fiatConversionUrl = config.PRICE_DATA_SERVER.CLEARNET_URL; - logger.info('Querying currency rates service...'); + logger.debug('Querying currency rates service...'); response = await axios.get(fiatConversionUrl, { headers: headers, timeout: 10000 }); } @@ -68,7 +68,7 @@ class FiatConversion { 'USD': usd.price, }; - logger.info(`USD Conversion Rate: ${usd.price}`); + logger.debug(`USD Conversion Rate: ${usd.price}`); if (this.ratesChangedCallback) { this.ratesChangedCallback(this.conversionRates);