diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 83de897ca..902f1f13b 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -17,7 +17,6 @@ import { prepareBlock } from '../utils/blocks-utils'; import BlocksRepository from '../repositories/BlocksRepository'; import HashratesRepository from '../repositories/HashratesRepository'; import indexer from '../indexer'; -import fiatConversion from './fiat-conversion'; import poolsParser from './pools-parser'; import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository'; import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository'; @@ -170,7 +169,7 @@ class Blocks { blockExtended.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0); blockExtended.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]); blockExtended.extras.coinbaseRaw = blockExtended.extras.coinbaseTx.vin[0].scriptsig; - blockExtended.extras.usd = fiatConversion.getConversionRates().USD; + blockExtended.extras.usd = priceUpdater.latestPrices.USD; if (block.height === 0) { blockExtended.extras.medianFee = 0; // 50th percentiles diff --git a/backend/src/api/fiat-conversion.ts b/backend/src/api/fiat-conversion.ts deleted file mode 100644 index ffbe6a758..000000000 --- a/backend/src/api/fiat-conversion.ts +++ /dev/null @@ -1,123 +0,0 @@ -import logger from '../logger'; -import * as http from 'http'; -import * as https from 'https'; -import axios, { AxiosResponse } from 'axios'; -import { IConversionRates } from '../mempool.interfaces'; -import config from '../config'; -import backendInfo from './backend-info'; -import { SocksProxyAgent } from 'socks-proxy-agent'; - -class FiatConversion { - private debasingFiatCurrencies = ['AED', 'AUD', 'BDT', 'BHD', 'BMD', 'BRL', 'CAD', 'CHF', 'CLP', - 'CNY', 'CZK', 'DKK', 'EUR', 'GBP', 'HKD', 'HUF', 'IDR', 'ILS', 'INR', 'JPY', 'KRW', 'KWD', - 'LKR', 'MMK', 'MXN', 'MYR', 'NGN', 'NOK', 'NZD', 'PHP', 'PKR', 'PLN', 'RUB', 'SAR', 'SEK', - 'SGD', 'THB', 'TRY', 'TWD', 'UAH', 'USD', 'VND', 'ZAR']; - private conversionRates: IConversionRates = {}; - private ratesChangedCallback: ((rates: IConversionRates) => void) | undefined; - public ratesInitialized = false; // If true, it means rates are ready for use - - constructor() { - for (const fiat of this.debasingFiatCurrencies) { - this.conversionRates[fiat] = 0; - } - } - - public setProgressChangedCallback(fn: (rates: IConversionRates) => void) { - this.ratesChangedCallback = fn; - } - - public startService() { - const fiatConversionUrl = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.PRICE_DATA_SERVER.TOR_URL : config.PRICE_DATA_SERVER.CLEARNET_URL; - logger.info('Starting currency rates service'); - if (config.SOCKS5PROXY.ENABLED) { - logger.info(`Currency rates service will be queried over the Tor network using ${fiatConversionUrl}`); - } else { - logger.info(`Currency rates service will be queried over clearnet using ${config.PRICE_DATA_SERVER.CLEARNET_URL}`); - } - setInterval(this.updateCurrency.bind(this), 1000 * config.MEMPOOL.PRICE_FEED_UPDATE_INTERVAL); - this.updateCurrency(); - } - - public getConversionRates() { - return this.conversionRates; - } - - private async updateCurrency(): Promise { - type axiosOptions = { - headers: { - 'User-Agent': string - }; - timeout: number; - httpAgent?: http.Agent; - httpsAgent?: https.Agent; - } - const setDelay = (secs: number = 1): Promise => new Promise(resolve => setTimeout(() => resolve(), secs * 1000)); - const fiatConversionUrl = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.PRICE_DATA_SERVER.TOR_URL : config.PRICE_DATA_SERVER.CLEARNET_URL; - const isHTTP = (new URL(fiatConversionUrl).protocol.split(':')[0] === 'http') ? true : false; - const axiosOptions: axiosOptions = { - headers: { - 'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}` - }, - timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000 - }; - - let retry = 0; - - while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) { - try { - if (config.SOCKS5PROXY.ENABLED) { - let socksOptions: any = { - agentOptions: { - keepAlive: true, - }, - hostname: config.SOCKS5PROXY.HOST, - port: config.SOCKS5PROXY.PORT - }; - - if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) { - socksOptions.username = config.SOCKS5PROXY.USERNAME; - socksOptions.password = config.SOCKS5PROXY.PASSWORD; - } else { - // Retry with different tor circuits https://stackoverflow.com/a/64960234 - socksOptions.username = `circuit${retry}`; - } - - // Handle proxy agent for onion addresses - if (isHTTP) { - axiosOptions.httpAgent = new SocksProxyAgent(socksOptions); - } else { - axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions); - } - } - - logger.debug('Querying currency rates service...'); - - const response: AxiosResponse = await axios.get(`${fiatConversionUrl}`, axiosOptions); - - if (response.statusText === 'error' || !response.data) { - throw new Error(`Could not fetch data from ${fiatConversionUrl}, Error: ${response.status}`); - } - - for (const rate of response.data.data) { - if (this.debasingFiatCurrencies.includes(rate.currencyCode) && rate.provider === 'Bisq-Aggregate') { - this.conversionRates[rate.currencyCode] = Math.round(100 * rate.price) / 100; - } - } - - this.ratesInitialized = true; - logger.debug(`USD Conversion Rate: ${this.conversionRates.USD}`); - - if (this.ratesChangedCallback) { - this.ratesChangedCallback(this.conversionRates); - } - break; - } catch (e) { - logger.err('Error updating fiat conversion rates: ' + (e instanceof Error ? e.message : e)); - await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL); - retry++; - } - } - } -} - -export default new FiatConversion(); diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 599c068a6..3b6c62513 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -8,7 +8,6 @@ import blocks from './blocks'; import memPool from './mempool'; import backendInfo from './backend-info'; import mempoolBlocks from './mempool-blocks'; -import fiatConversion from './fiat-conversion'; import { Common } from './common'; import loadingIndicators from './loading-indicators'; import config from '../config'; @@ -20,6 +19,7 @@ import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository'; import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository'; import Audit from './audit'; import { deepClone } from '../utils/clone'; +import priceUpdater from '../tasks/price-updater'; class WebsocketHandler { private wss: WebSocket.Server | undefined; @@ -214,7 +214,7 @@ class WebsocketHandler { 'mempoolInfo': memPool.getMempoolInfo(), 'vBytesPerSecond': memPool.getVBytesPerSecond(), 'blocks': _blocks, - 'conversions': fiatConversion.getConversionRates(), + 'conversions': priceUpdater.latestPrices, 'mempool-blocks': mempoolBlocks.getMempoolBlocks(), 'transactions': memPool.getLatestTransactions(), 'backendInfo': backendInfo.getBackendInfo(), diff --git a/backend/src/index.ts b/backend/src/index.ts index 8371e927f..7c0d7ee48 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -10,7 +10,6 @@ import memPool from './api/mempool'; import diskCache from './api/disk-cache'; import statistics from './api/statistics/statistics'; import websocketHandler from './api/websocket-handler'; -import fiatConversion from './api/fiat-conversion'; import bisq from './api/bisq/bisq'; import bisqMarkets from './api/bisq/markets'; import logger from './logger'; @@ -36,6 +35,7 @@ import liquidRoutes from './api/liquid/liquid.routes'; import bitcoinRoutes from './api/bitcoin/bitcoin.routes'; import fundingTxFetcher from './tasks/lightning/sync-tasks/funding-tx-fetcher'; import forensicsService from './tasks/lightning/forensics.service'; +import priceUpdater from './tasks/price-updater'; class Server { private wss: WebSocket.Server | undefined; @@ -87,6 +87,8 @@ class Server { .use(express.text({ type: ['text/plain', 'application/base64'] })) ; + await priceUpdater.$initializeLatestPriceWithDb(); + this.server = http.createServer(this.app); this.wss = new WebSocket.Server({ server: this.server }); @@ -127,7 +129,7 @@ class Server { } } - fiatConversion.startService(); + priceUpdater.$run(); this.setUpHttpApiRoutes(); @@ -221,7 +223,7 @@ class Server { memPool.setAsyncMempoolChangedCallback(websocketHandler.handleMempoolChange.bind(websocketHandler)); blocks.setNewAsyncBlockCallback(websocketHandler.handleNewBlock.bind(websocketHandler)); } - fiatConversion.setProgressChangedCallback(websocketHandler.handleNewConversionRates.bind(websocketHandler)); + priceUpdater.setRatesChangedCallback(websocketHandler.handleNewConversionRates.bind(websocketHandler)); loadingIndicators.setProgressChangedCallback(websocketHandler.handleLoadingChanged.bind(websocketHandler)); } diff --git a/backend/src/repositories/PricesRepository.ts b/backend/src/repositories/PricesRepository.ts index cc79ff2a6..bc606e68b 100644 --- a/backend/src/repositories/PricesRepository.ts +++ b/backend/src/repositories/PricesRepository.ts @@ -1,12 +1,13 @@ import DB from '../database'; import logger from '../logger'; -import { Prices } from '../tasks/price-updater'; +import { IConversionRates } from '../mempool.interfaces'; +import priceUpdater from '../tasks/price-updater'; class PricesRepository { - public async $savePrices(time: number, prices: Prices): Promise { - if (prices.USD === -1) { - // Some historical price entries have not USD prices, so we just ignore them to avoid future UX issues - // As of today there are only 4 (on 2013-09-05, 2013-09-19, 2013-09-12 and 2013-09-26) so that's fine + public async $savePrices(time: number, prices: IConversionRates): Promise { + if (prices.USD === 0) { + // Some historical price entries have no USD prices, so we just ignore them to avoid future UX issues + // As of today there are only 4 (on 2013-09-05, 2013-0909, 2013-09-12 and 2013-09-26) so that's fine return; } @@ -23,22 +24,22 @@ class PricesRepository { } public async $getOldestPriceTime(): Promise { - const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1 ORDER BY time LIMIT 1`); + const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != 0 ORDER BY time LIMIT 1`); return oldestRow[0] ? oldestRow[0].time : 0; } public async $getLatestPriceId(): Promise { - const [oldestRow] = await DB.query(`SELECT id from prices WHERE USD != -1 ORDER BY time DESC LIMIT 1`); + const [oldestRow] = await DB.query(`SELECT id from prices WHERE USD != 0 ORDER BY time DESC LIMIT 1`); return oldestRow[0] ? oldestRow[0].id : null; } public async $getLatestPriceTime(): Promise { - const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1 ORDER BY time DESC LIMIT 1`); + const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != 0 ORDER BY time DESC LIMIT 1`); return oldestRow[0] ? oldestRow[0].time : 0; } public async $getPricesTimes(): Promise { - const [times]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1 ORDER BY time`); + const [times]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != 0 ORDER BY time`); return times.map(time => time.time); } @@ -46,6 +47,19 @@ class PricesRepository { const [times]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time, id, USD from prices ORDER BY time`); return times; } + + public async $getLatestConversionRates(): Promise { + const [rates]: any[] = await DB.query(` + SELECT USD, EUR, GBP, CAD, CHF, AUD, JPY + FROM prices + ORDER BY time DESC + LIMIT 1` + ); + if (!rates || rates.length === 0) { + return priceUpdater.getEmptyPricesObj(); + } + return rates[0]; + } } export default new PricesRepository(); diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index 9e7e5910a..e42c5887a 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -1,7 +1,8 @@ import * as fs from 'fs'; -import path from "path"; +import path from 'path'; import config from '../config'; import logger from '../logger'; +import { IConversionRates } from '../mempool.interfaces'; import PricesRepository from '../repositories/PricesRepository'; import BitfinexApi from './price-feeds/bitfinex-api'; import BitflyerApi from './price-feeds/bitflyer-api'; @@ -20,17 +21,7 @@ export interface PriceFeed { } export interface PriceHistory { - [timestamp: number]: Prices; -} - -export interface Prices { - USD: number; - EUR: number; - GBP: number; - CAD: number; - CHF: number; - AUD: number; - JPY: number; + [timestamp: number]: IConversionRates; } class PriceUpdater { @@ -40,7 +31,8 @@ class PriceUpdater { running = false; feeds: PriceFeed[] = []; currencies: string[] = ['USD', 'EUR', 'GBP', 'CAD', 'CHF', 'AUD', 'JPY']; - latestPrices: Prices; + latestPrices: IConversionRates; + private ratesChangedCallback: ((rates: IConversionRates) => void) | undefined; constructor() { this.latestPrices = this.getEmptyPricesObj(); @@ -52,18 +44,30 @@ class PriceUpdater { this.feeds.push(new GeminiApi()); } - public getEmptyPricesObj(): Prices { + public getEmptyPricesObj(): IConversionRates { return { - USD: -1, - EUR: -1, - GBP: -1, - CAD: -1, - CHF: -1, - AUD: -1, - JPY: -1, + USD: 0, + EUR: 0, + GBP: 0, + CAD: 0, + CHF: 0, + AUD: 0, + JPY: 0, }; } + public setRatesChangedCallback(fn: (rates: IConversionRates) => void) { + this.ratesChangedCallback = fn; + } + + /** + * We execute this function before the websocket initialization since + * the websocket init is not done asyncronously + */ + public async $initializeLatestPriceWithDb(): Promise { + this.latestPrices = await PricesRepository.$getLatestConversionRates(); + } + public async $run(): Promise { if (this.running === true) { return; @@ -76,10 +80,9 @@ class PriceUpdater { } try { + await this.$updatePrice(); if (this.historyInserted === false && config.DATABASE.ENABLED === true) { await this.$insertHistoricalPrices(); - } else { - await this.$updatePrice(); } } catch (e) { logger.err(`Cannot save BTC prices in db. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.mining); @@ -144,6 +147,10 @@ class PriceUpdater { } } + if (this.ratesChangedCallback) { + this.ratesChangedCallback(this.latestPrices); + } + this.lastRun = new Date().getTime() / 1000; } @@ -213,7 +220,7 @@ class PriceUpdater { // Group them by timestamp and currency, for example // grouped[123456789]['USD'] = [1, 2, 3, 4]; - const grouped: Object = {}; + const grouped: any = {}; for (const historicalEntry of historicalPrices) { for (const time in historicalEntry) { if (existingPriceTimes.includes(parseInt(time, 10))) { @@ -229,7 +236,7 @@ class PriceUpdater { for (const currency of this.currencies) { const price = historicalEntry[time][currency]; if (price > 0) { - grouped[time][currency].push(parseInt(price, 10)); + grouped[time][currency].push(typeof price === 'string' ? parseInt(price, 10) : price); } } } @@ -238,7 +245,7 @@ class PriceUpdater { // Average prices and insert everything into the db let totalInserted = 0; for (const time in grouped) { - const prices: Prices = this.getEmptyPricesObj(); + const prices: IConversionRates = this.getEmptyPricesObj(); for (const currency in grouped[time]) { if (grouped[time][currency].length === 0) { continue;