diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 91006c3f6..0e43bda94 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,6 +6,8 @@ In order to clarify the intellectual property license granted with Contributions When submitting a pull request for the first time, please create a file with a name like `/contributors/{github_username}.txt`, and in the content of that file indicate your agreement to the Contributor License Agreement terms below. An example of what that file should contain can be seen in wiz's agreement file. (This method of CLA "signing" is borrowed from Medium's open source project.) +Also, please GPG-sign all your commits (`git config commit.gpgsign true`). + # Contributor License Agreement Last Updated: January 25, 2022 diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index b71f3586d..214121ed4 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -63,6 +63,11 @@ "ENABLED": true, "TX_PER_SECOND_SAMPLE_PERIOD": 150 }, + "MAXMIND": { + "ENABLED": false, + "GEOLITE2_CITY": "/usr/local/share/GeoIP/GeoLite2-City.mmdb", + "GEOLITE2_ASN": "/usr/local/share/GeoIP/GeoLite2-ASN.mmdb" + }, "BISQ": { "ENABLED": false, "DATA_PATH": "/bisq/statsnode-data/btc_mainnet/db" diff --git a/backend/package-lock.json b/backend/package-lock.json index 7ea9cf43b..e724ac35b 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -17,6 +17,7 @@ "crypto-js": "^4.0.0", "express": "^4.18.0", "lightning": "^5.16.3", + "maxmind": "^4.3.6", "mysql2": "2.3.3", "node-worker-threads-pool": "^1.5.1", "socks-proxy-agent": "~7.0.0", @@ -2222,6 +2223,19 @@ "node": ">=10" } }, + "node_modules/maxmind": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/maxmind/-/maxmind-4.3.6.tgz", + "integrity": "sha512-CwnEZqJX0T6b2rWrc0/V3n9hL/hWAMEn7fY09077YJUHiHx7cn/esA2ZIz8BpYLSJUf7cGVel0oUJa9jMwyQpg==", + "dependencies": { + "mmdb-lib": "2.0.2", + "tiny-lru": "8.0.2" + }, + "engines": { + "node": ">=10", + "npm": ">=6" + } + }, "node_modules/md5.js": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/md5.js/-/md5.js-1.3.5.tgz", @@ -2317,6 +2331,15 @@ "node": "*" } }, + "node_modules/mmdb-lib": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/mmdb-lib/-/mmdb-lib-2.0.2.tgz", + "integrity": "sha512-shi1I+fCPQonhTi7qyb6hr7hi87R7YS69FlfJiMFuJ12+grx0JyL56gLNzGTYXPU7EhAPkMLliGeyHer0K+AVA==", + "engines": { + "node": ">=10", + "npm": ">=6" + } + }, "node_modules/ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", @@ -3027,6 +3050,14 @@ "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==", "dev": true }, + "node_modules/tiny-lru": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-8.0.2.tgz", + "integrity": "sha512-ApGvZ6vVvTNdsmt676grvCkUCGwzG9IqXma5Z07xJgiC5L7akUMof5U8G2JTI9Rz/ovtVhJBlY6mNhEvtjzOIg==", + "engines": { + "node": ">=6" + } + }, "node_modules/tiny-secp256k1": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/tiny-secp256k1/-/tiny-secp256k1-2.2.1.tgz", @@ -4971,6 +5002,15 @@ "yallist": "^4.0.0" } }, + "maxmind": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/maxmind/-/maxmind-4.3.6.tgz", + "integrity": "sha512-CwnEZqJX0T6b2rWrc0/V3n9hL/hWAMEn7fY09077YJUHiHx7cn/esA2ZIz8BpYLSJUf7cGVel0oUJa9jMwyQpg==", + "requires": { + "mmdb-lib": "2.0.2", + "tiny-lru": "8.0.2" + } + }, "md5.js": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/md5.js/-/md5.js-1.3.5.tgz", @@ -5039,6 +5079,11 @@ "brace-expansion": "^1.1.7" } }, + "mmdb-lib": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/mmdb-lib/-/mmdb-lib-2.0.2.tgz", + "integrity": "sha512-shi1I+fCPQonhTi7qyb6hr7hi87R7YS69FlfJiMFuJ12+grx0JyL56gLNzGTYXPU7EhAPkMLliGeyHer0K+AVA==" + }, "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", @@ -5549,6 +5594,11 @@ "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==", "dev": true }, + "tiny-lru": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-8.0.2.tgz", + "integrity": "sha512-ApGvZ6vVvTNdsmt676grvCkUCGwzG9IqXma5Z07xJgiC5L7akUMof5U8G2JTI9Rz/ovtVhJBlY6mNhEvtjzOIg==" + }, "tiny-secp256k1": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/tiny-secp256k1/-/tiny-secp256k1-2.2.1.tgz", diff --git a/backend/package.json b/backend/package.json index 5023d6029..b8930d6e5 100644 --- a/backend/package.json +++ b/backend/package.json @@ -38,6 +38,7 @@ "crypto-js": "^4.0.0", "express": "^4.18.0", "lightning": "^5.16.3", + "maxmind": "^4.3.6", "mysql2": "2.3.3", "node-worker-threads-pool": "^1.5.1", "socks-proxy-agent": "~7.0.0", diff --git a/backend/src/api/bisq/bisq.routes.ts b/backend/src/api/bisq/bisq.routes.ts new file mode 100644 index 000000000..8f002836f --- /dev/null +++ b/backend/src/api/bisq/bisq.routes.ts @@ -0,0 +1,381 @@ +import { Application, Request, Response } from 'express'; +import config from '../../config'; +import { RequiredSpec } from '../../mempool.interfaces'; +import bisq from './bisq'; +import { MarketsApiError } from './interfaces'; +import marketsApi from './markets-api'; + +class BisqRoutes { + public initRoutes(app: Application) { + app + .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/stats', this.getBisqStats) + .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/tx/:txId', this.getBisqTransaction) + .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/block/:hash', this.getBisqBlock) + .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/blocks/tip/height', this.getBisqTip) + .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/blocks/:index/:length', this.getBisqBlocks) + .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/address/:address', this.getBisqAddress) + .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/txs/:index/:length', this.getBisqTransactions) + .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/currencies', this.getBisqMarketCurrencies.bind(this)) + .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/depth', this.getBisqMarketDepth.bind(this)) + .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/hloc', this.getBisqMarketHloc.bind(this)) + .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/markets', this.getBisqMarketMarkets.bind(this)) + .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/offers', this.getBisqMarketOffers.bind(this)) + .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/ticker', this.getBisqMarketTicker.bind(this)) + .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/trades', this.getBisqMarketTrades.bind(this)) + .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/volumes', this.getBisqMarketVolumes.bind(this)) + .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/volumes/7d', this.getBisqMarketVolumes7d.bind(this)) + ; + } + + + private getBisqStats(req: Request, res: Response) { + const result = bisq.getStats(); + res.json(result); + } + + private getBisqTip(req: Request, res: Response) { + const result = bisq.getLatestBlockHeight(); + res.type('text/plain'); + res.send(result.toString()); + } + + private getBisqTransaction(req: Request, res: Response) { + const result = bisq.getTransaction(req.params.txId); + if (result) { + res.json(result); + } else { + res.status(404).send('Bisq transaction not found'); + } + } + + private getBisqTransactions(req: Request, res: Response) { + const types: string[] = []; + req.query.types = req.query.types || []; + if (!Array.isArray(req.query.types)) { + res.status(500).send('Types is not an array'); + return; + } + + for (const _type in req.query.types) { + if (typeof req.query.types[_type] === 'string') { + types.push(req.query.types[_type].toString()); + } + } + + const index = parseInt(req.params.index, 10) || 0; + const length = parseInt(req.params.length, 10) > 100 ? 100 : parseInt(req.params.length, 10) || 25; + const [transactions, count] = bisq.getTransactions(index, length, types); + res.header('X-Total-Count', count.toString()); + res.json(transactions); + } + + private getBisqBlock(req: Request, res: Response) { + const result = bisq.getBlock(req.params.hash); + if (result) { + res.json(result); + } else { + res.status(404).send('Bisq block not found'); + } + } + + private getBisqBlocks(req: Request, res: Response) { + const index = parseInt(req.params.index, 10) || 0; + const length = parseInt(req.params.length, 10) > 100 ? 100 : parseInt(req.params.length, 10) || 25; + const [transactions, count] = bisq.getBlocks(index, length); + res.header('X-Total-Count', count.toString()); + res.json(transactions); + } + + private getBisqAddress(req: Request, res: Response) { + const result = bisq.getAddress(req.params.address.substr(1)); + if (result) { + res.json(result); + } else { + res.status(404).send('Bisq address not found'); + } + } + + private getBisqMarketCurrencies(req: Request, res: Response) { + const constraints: RequiredSpec = { + 'type': { + required: false, + types: ['crypto', 'fiat', 'all'] + }, + }; + + const p = this.parseRequestParameters(req.query, constraints); + if (p.error) { + res.status(400).json(this.getBisqMarketErrorResponse(p.error)); + return; + } + + const result = marketsApi.getCurrencies(p.type); + if (result) { + res.json(result); + } else { + res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketCurrencies error')); + } + } + + private getBisqMarketDepth(req: Request, res: Response) { + const constraints: RequiredSpec = { + 'market': { + required: true, + types: ['@string'] + }, + }; + + const p = this.parseRequestParameters(req.query, constraints); + if (p.error) { + res.status(400).json(this.getBisqMarketErrorResponse(p.error)); + return; + } + + const result = marketsApi.getDepth(p.market); + if (result) { + res.json(result); + } else { + res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketDepth error')); + } + } + + private getBisqMarketMarkets(req: Request, res: Response) { + const result = marketsApi.getMarkets(); + if (result) { + res.json(result); + } else { + res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketMarkets error')); + } + } + + private getBisqMarketTrades(req: Request, res: Response) { + const constraints: RequiredSpec = { + 'market': { + required: true, + types: ['@string'] + }, + 'timestamp_from': { + required: false, + types: ['@number'] + }, + 'timestamp_to': { + required: false, + types: ['@number'] + }, + 'trade_id_to': { + required: false, + types: ['@string'] + }, + 'trade_id_from': { + required: false, + types: ['@string'] + }, + 'direction': { + required: false, + types: ['buy', 'sell'] + }, + 'limit': { + required: false, + types: ['@number'] + }, + 'sort': { + required: false, + types: ['asc', 'desc'] + } + }; + + const p = this.parseRequestParameters(req.query, constraints); + if (p.error) { + res.status(400).json(this.getBisqMarketErrorResponse(p.error)); + return; + } + + const result = marketsApi.getTrades(p.market, p.timestamp_from, + p.timestamp_to, p.trade_id_from, p.trade_id_to, p.direction, p.limit, p.sort); + if (result) { + res.json(result); + } else { + res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketTrades error')); + } + } + + private getBisqMarketOffers(req: Request, res: Response) { + const constraints: RequiredSpec = { + 'market': { + required: true, + types: ['@string'] + }, + 'direction': { + required: false, + types: ['buy', 'sell'] + }, + }; + + const p = this.parseRequestParameters(req.query, constraints); + if (p.error) { + res.status(400).json(this.getBisqMarketErrorResponse(p.error)); + return; + } + + const result = marketsApi.getOffers(p.market, p.direction); + if (result) { + res.json(result); + } else { + res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketOffers error')); + } + } + + private getBisqMarketVolumes(req: Request, res: Response) { + const constraints: RequiredSpec = { + 'market': { + required: false, + types: ['@string'] + }, + 'interval': { + required: false, + types: ['minute', 'half_hour', 'hour', 'half_day', 'day', 'week', 'month', 'year', 'auto'] + }, + 'timestamp_from': { + required: false, + types: ['@number'] + }, + 'timestamp_to': { + required: false, + types: ['@number'] + }, + 'milliseconds': { + required: false, + types: ['@boolean'] + }, + 'timestamp': { + required: false, + types: ['no', 'yes'] + }, + }; + + const p = this.parseRequestParameters(req.query, constraints); + if (p.error) { + res.status(400).json(this.getBisqMarketErrorResponse(p.error)); + return; + } + + const result = marketsApi.getVolumes(p.market, p.timestamp_from, p.timestamp_to, p.interval, p.milliseconds, p.timestamp); + if (result) { + res.json(result); + } else { + res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketVolumes error')); + } + } + + private getBisqMarketHloc(req: Request, res: Response) { + const constraints: RequiredSpec = { + 'market': { + required: true, + types: ['@string'] + }, + 'interval': { + required: false, + types: ['minute', 'half_hour', 'hour', 'half_day', 'day', 'week', 'month', 'year', 'auto'] + }, + 'timestamp_from': { + required: false, + types: ['@number'] + }, + 'timestamp_to': { + required: false, + types: ['@number'] + }, + 'milliseconds': { + required: false, + types: ['@boolean'] + }, + 'timestamp': { + required: false, + types: ['no', 'yes'] + }, + }; + + const p = this.parseRequestParameters(req.query, constraints); + if (p.error) { + res.status(400).json(this.getBisqMarketErrorResponse(p.error)); + return; + } + + const result = marketsApi.getHloc(p.market, p.interval, p.timestamp_from, p.timestamp_to, p.milliseconds, p.timestamp); + if (result) { + res.json(result); + } else { + res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketHloc error')); + } + } + + private getBisqMarketTicker(req: Request, res: Response) { + const constraints: RequiredSpec = { + 'market': { + required: false, + types: ['@string'] + }, + }; + + const p = this.parseRequestParameters(req.query, constraints); + if (p.error) { + res.status(400).json(this.getBisqMarketErrorResponse(p.error)); + return; + } + + const result = marketsApi.getTicker(p.market); + if (result) { + res.json(result); + } else { + res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketTicker error')); + } + } + + private getBisqMarketVolumes7d(req: Request, res: Response) { + const result = marketsApi.getVolumesByTime(604800); + if (result) { + res.json(result); + } else { + res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketVolumes7d error')); + } + } + + private parseRequestParameters(requestParams: object, params: RequiredSpec): { [name: string]: any; } { + const final = {}; + for (const i in params) { + if (params.hasOwnProperty(i)) { + if (params[i].required && requestParams[i] === undefined) { + return { error: i + ' parameter missing'}; + } + if (typeof requestParams[i] === 'string') { + const str = (requestParams[i] || '').toString().toLowerCase(); + if (params[i].types.indexOf('@number') > -1) { + const number = parseInt((str).toString(), 10); + final[i] = number; + } else if (params[i].types.indexOf('@string') > -1) { + final[i] = str; + } else if (params[i].types.indexOf('@boolean') > -1) { + final[i] = str === 'true' || str === 'yes'; + } else if (params[i].types.indexOf(str) > -1) { + final[i] = str; + } else { + return { error: i + ' parameter invalid'}; + } + } else if (typeof requestParams[i] === 'number') { + final[i] = requestParams[i]; + } + } + } + return final; + } + + private getBisqMarketErrorResponse(message: string): MarketsApiError { + return { + 'success': 0, + 'error': message + }; + } + +} + +export default new BisqRoutes; diff --git a/backend/src/api/bitcoin/bitcoin.routes.ts b/backend/src/api/bitcoin/bitcoin.routes.ts new file mode 100644 index 000000000..a04a78117 --- /dev/null +++ b/backend/src/api/bitcoin/bitcoin.routes.ts @@ -0,0 +1,543 @@ +import { Application, Request, Response } from 'express'; +import axios from 'axios'; +import config from '../../config'; +import websocketHandler from '../websocket-handler'; +import mempool from '../mempool'; +import feeApi from '../fee-api'; +import mempoolBlocks from '../mempool-blocks'; +import bitcoinApi from './bitcoin-api-factory'; +import { Common } from '../common'; +import backendInfo from '../backend-info'; +import transactionUtils from '../transaction-utils'; +import { IEsploraApi } from './esplora-api.interface'; +import loadingIndicators from '../loading-indicators'; +import { TransactionExtended } from '../../mempool.interfaces'; +import logger from '../../logger'; +import blocks from '../blocks'; +import bitcoinClient from './bitcoin-client'; +import difficultyAdjustment from '../difficulty-adjustment'; + +class BitcoinRoutes { + public initRoutes(app: Application) { + app + .get(config.MEMPOOL.API_URL_PREFIX + 'transaction-times', this.getTransactionTimes) + .get(config.MEMPOOL.API_URL_PREFIX + 'outspends', this.$getBatchedOutspends) + .get(config.MEMPOOL.API_URL_PREFIX + 'cpfp/:txId', this.getCpfpInfo) + .get(config.MEMPOOL.API_URL_PREFIX + 'difficulty-adjustment', this.getDifficultyChange) + .get(config.MEMPOOL.API_URL_PREFIX + 'fees/recommended', this.getRecommendedFees) + .get(config.MEMPOOL.API_URL_PREFIX + 'fees/mempool-blocks', this.getMempoolBlocks) + .get(config.MEMPOOL.API_URL_PREFIX + 'backend-info', this.getBackendInfo) + .get(config.MEMPOOL.API_URL_PREFIX + 'init-data', this.getInitData) + .get(config.MEMPOOL.API_URL_PREFIX + 'validate-address/:address', this.validateAddress) + .post(config.MEMPOOL.API_URL_PREFIX + 'tx/push', this.$postTransactionForm) + .get(config.MEMPOOL.API_URL_PREFIX + 'donations', async (req, res) => { + try { + const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/donations`, { responseType: 'stream', timeout: 10000 }); + response.data.pipe(res); + } catch (e) { + res.status(500).end(); + } + }) + .get(config.MEMPOOL.API_URL_PREFIX + 'donations/images/:id', async (req, res) => { + try { + const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/donations/images/${req.params.id}`, { + responseType: 'stream', timeout: 10000 + }); + response.data.pipe(res); + } catch (e) { + res.status(500).end(); + } + }) + .get(config.MEMPOOL.API_URL_PREFIX + 'contributors', async (req, res) => { + try { + const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/contributors`, { responseType: 'stream', timeout: 10000 }); + response.data.pipe(res); + } catch (e) { + res.status(500).end(); + } + }) + .get(config.MEMPOOL.API_URL_PREFIX + 'contributors/images/:id', async (req, res) => { + try { + const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/contributors/images/${req.params.id}`, { + responseType: 'stream', timeout: 10000 + }); + response.data.pipe(res); + } catch (e) { + res.status(500).end(); + } + }) + .get(config.MEMPOOL.API_URL_PREFIX + 'translators', async (req, res) => { + try { + const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/translators`, { responseType: 'stream', timeout: 10000 }); + response.data.pipe(res); + } catch (e) { + res.status(500).end(); + } + }) + .get(config.MEMPOOL.API_URL_PREFIX + 'translators/images/:id', async (req, res) => { + try { + const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/translators/images/${req.params.id}`, { + responseType: 'stream', timeout: 10000 + }); + response.data.pipe(res); + } catch (e) { + res.status(500).end(); + } + }) + .get(config.MEMPOOL.API_URL_PREFIX + 'blocks', this.getBlocks.bind(this)) + .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/:height', this.getBlocks.bind(this)) + .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash', this.getBlock) + .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/summary', this.getStrippedBlockTransactions); + ; + + if (config.MEMPOOL.BACKEND !== 'esplora') { + app + .get(config.MEMPOOL.API_URL_PREFIX + 'mempool', this.getMempool) + .get(config.MEMPOOL.API_URL_PREFIX + 'mempool/txids', this.getMempoolTxIds) + .get(config.MEMPOOL.API_URL_PREFIX + 'mempool/recent', this.getRecentMempoolTransactions) + .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId', this.getTransaction) + .post(config.MEMPOOL.API_URL_PREFIX + 'tx', this.$postTransaction) + .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/hex', this.getRawTransaction) + .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/status', this.getTransactionStatus) + .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/outspends', this.getTransactionOutspends) + .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/header', this.getBlockHeader) + .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/height', this.getBlockTipHeight) + .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/hash', this.getBlockTipHash) + .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txs', this.getBlockTransactions) + .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txs/:index', this.getBlockTransactions) + .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txids', this.getTxIdsForBlock) + .get(config.MEMPOOL.API_URL_PREFIX + 'block-height/:height', this.getBlockHeight) + .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address', this.getAddress) + .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs', this.getAddressTransactions) + .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs/chain/:txId', this.getAddressTransactions) + .get(config.MEMPOOL.API_URL_PREFIX + 'address-prefix/:prefix', this.getAddressPrefix) + ; + } + } + + + private getInitData(req: Request, res: Response) { + try { + const result = websocketHandler.getInitData(); + res.json(result); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private getRecommendedFees(req: Request, res: Response) { + if (!mempool.isInSync()) { + res.statusCode = 503; + res.send('Service Unavailable'); + return; + } + const result = feeApi.getRecommendedFee(); + res.json(result); + } + + private getMempoolBlocks(req: Request, res: Response) { + try { + const result = mempoolBlocks.getMempoolBlocks(); + res.json(result); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private getTransactionTimes(req: Request, res: Response) { + if (!Array.isArray(req.query.txId)) { + res.status(500).send('Not an array'); + return; + } + const txIds: string[] = []; + for (const _txId in req.query.txId) { + if (typeof req.query.txId[_txId] === 'string') { + txIds.push(req.query.txId[_txId].toString()); + } + } + + const times = mempool.getFirstSeenForTransactions(txIds); + res.json(times); + } + + private async $getBatchedOutspends(req: Request, res: Response) { + if (!Array.isArray(req.query.txId)) { + res.status(500).send('Not an array'); + return; + } + if (req.query.txId.length > 50) { + res.status(400).send('Too many txids requested'); + return; + } + const txIds: string[] = []; + for (const _txId in req.query.txId) { + if (typeof req.query.txId[_txId] === 'string') { + txIds.push(req.query.txId[_txId].toString()); + } + } + + try { + const batchedOutspends = await bitcoinApi.$getBatchedOutspends(txIds); + res.json(batchedOutspends); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private getCpfpInfo(req: Request, res: Response) { + if (!/^[a-fA-F0-9]{64}$/.test(req.params.txId)) { + res.status(501).send(`Invalid transaction ID.`); + return; + } + + const tx = mempool.getMempool()[req.params.txId]; + if (!tx) { + res.status(404).send(`Transaction doesn't exist in the mempool.`); + return; + } + + if (tx.cpfpChecked) { + res.json({ + ancestors: tx.ancestors, + bestDescendant: tx.bestDescendant || null, + }); + return; + } + + const cpfpInfo = Common.setRelativesAndGetCpfpInfo(tx, mempool.getMempool()); + + res.json(cpfpInfo); + } + + private getBackendInfo(req: Request, res: Response) { + res.json(backendInfo.getBackendInfo()); + } + + private async getTransaction(req: Request, res: Response) { + try { + const transaction = await transactionUtils.$getTransactionExtended(req.params.txId, true); + res.json(transaction); + } catch (e) { + let statusCode = 500; + if (e instanceof Error && e instanceof Error && e.message && e.message.indexOf('No such mempool or blockchain transaction') > -1) { + statusCode = 404; + } + res.status(statusCode).send(e instanceof Error ? e.message : e); + } + } + + private async getRawTransaction(req: Request, res: Response) { + try { + const transaction: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(req.params.txId, true); + res.setHeader('content-type', 'text/plain'); + res.send(transaction.hex); + } catch (e) { + let statusCode = 500; + if (e instanceof Error && e.message && e.message.indexOf('No such mempool or blockchain transaction') > -1) { + statusCode = 404; + } + res.status(statusCode).send(e instanceof Error ? e.message : e); + } + } + + private async getTransactionStatus(req: Request, res: Response) { + try { + const transaction = await transactionUtils.$getTransactionExtended(req.params.txId, true); + res.json(transaction.status); + } catch (e) { + let statusCode = 500; + if (e instanceof Error && e.message && e.message.indexOf('No such mempool or blockchain transaction') > -1) { + statusCode = 404; + } + res.status(statusCode).send(e instanceof Error ? e.message : e); + } + } + + private async getBlock(req: Request, res: Response) { + try { + const block = await blocks.$getBlock(req.params.hash); + + const blockAge = new Date().getTime() / 1000 - block.timestamp; + const day = 24 * 3600; + let cacheDuration; + if (blockAge > 365 * day) { + cacheDuration = 30 * day; + } else if (blockAge > 30 * day) { + cacheDuration = 10 * day; + } else { + cacheDuration = 600 + } + + res.setHeader('Expires', new Date(Date.now() + 1000 * cacheDuration).toUTCString()); + res.json(block); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async getBlockHeader(req: Request, res: Response) { + try { + const blockHeader = await bitcoinApi.$getBlockHeader(req.params.hash); + res.setHeader('content-type', 'text/plain'); + res.send(blockHeader); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async getStrippedBlockTransactions(req: Request, res: Response) { + try { + const transactions = await blocks.$getStrippedBlockTransactions(req.params.hash); + res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24 * 30).toUTCString()); + res.json(transactions); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async getBlocks(req: Request, res: Response) { + try { + if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) { // Bitcoin + const height = req.params.height === undefined ? undefined : parseInt(req.params.height, 10); + res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); + res.json(await blocks.$getBlocks(height, 15)); + } else { // Liquid, Bisq + return await this.getLegacyBlocks(req, res); + } + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async getLegacyBlocks(req: Request, res: Response) { + try { + const returnBlocks: IEsploraApi.Block[] = []; + const fromHeight = parseInt(req.params.height, 10) || blocks.getCurrentBlockHeight(); + + // Check if block height exist in local cache to skip the hash lookup + const blockByHeight = blocks.getBlocks().find((b) => b.height === fromHeight); + let startFromHash: string | null = null; + if (blockByHeight) { + startFromHash = blockByHeight.id; + } else { + startFromHash = await bitcoinApi.$getBlockHash(fromHeight); + } + + let nextHash = startFromHash; + for (let i = 0; i < 10 && nextHash; i++) { + const localBlock = blocks.getBlocks().find((b) => b.id === nextHash); + if (localBlock) { + returnBlocks.push(localBlock); + nextHash = localBlock.previousblockhash; + } else { + const block = await bitcoinApi.$getBlock(nextHash); + returnBlocks.push(block); + nextHash = block.previousblockhash; + } + } + + res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); + res.json(returnBlocks); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async getBlockTransactions(req: Request, res: Response) { + try { + loadingIndicators.setProgress('blocktxs-' + req.params.hash, 0); + + const txIds = await bitcoinApi.$getTxIdsForBlock(req.params.hash); + const transactions: TransactionExtended[] = []; + const startingIndex = Math.max(0, parseInt(req.params.index || '0', 10)); + + const endIndex = Math.min(startingIndex + 10, txIds.length); + for (let i = startingIndex; i < endIndex; i++) { + try { + const transaction = await transactionUtils.$getTransactionExtended(txIds[i], true, true); + transactions.push(transaction); + loadingIndicators.setProgress('blocktxs-' + req.params.hash, (i - startingIndex + 1) / (endIndex - startingIndex) * 100); + } catch (e) { + logger.debug('getBlockTransactions error: ' + (e instanceof Error ? e.message : e)); + } + } + res.json(transactions); + } catch (e) { + loadingIndicators.setProgress('blocktxs-' + req.params.hash, 100); + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async getBlockHeight(req: Request, res: Response) { + try { + const blockHash = await bitcoinApi.$getBlockHash(parseInt(req.params.height, 10)); + res.send(blockHash); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async getAddress(req: Request, res: Response) { + if (config.MEMPOOL.BACKEND === 'none') { + res.status(405).send('Address lookups cannot be used with bitcoind as backend.'); + return; + } + + try { + const addressData = await bitcoinApi.$getAddress(req.params.address); + res.json(addressData); + } catch (e) { + if (e instanceof Error && e.message && (e.message.indexOf('too long') > 0 || e.message.indexOf('confirmed status') > 0)) { + return res.status(413).send(e instanceof Error ? e.message : e); + } + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async getAddressTransactions(req: Request, res: Response) { + if (config.MEMPOOL.BACKEND === 'none') { + res.status(405).send('Address lookups cannot be used with bitcoind as backend.'); + return; + } + + try { + const transactions = await bitcoinApi.$getAddressTransactions(req.params.address, req.params.txId); + res.json(transactions); + } catch (e) { + if (e instanceof Error && e.message && (e.message.indexOf('too long') > 0 || e.message.indexOf('confirmed status') > 0)) { + return res.status(413).send(e instanceof Error ? e.message : e); + } + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async getAdressTxChain(req: Request, res: Response) { + res.status(501).send('Not implemented'); + } + + private async getAddressPrefix(req: Request, res: Response) { + try { + const blockHash = await bitcoinApi.$getAddressPrefix(req.params.prefix); + res.send(blockHash); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async getRecentMempoolTransactions(req: Request, res: Response) { + const latestTransactions = Object.entries(mempool.getMempool()) + .sort((a, b) => (b[1].firstSeen || 0) - (a[1].firstSeen || 0)) + .slice(0, 10).map((tx) => Common.stripTransaction(tx[1])); + + res.json(latestTransactions); + } + + private async getMempool(req: Request, res: Response) { + const info = mempool.getMempoolInfo(); + res.json({ + count: info.size, + vsize: info.bytes, + total_fee: info.total_fee * 1e8, + fee_histogram: [] + }); + } + + private async getMempoolTxIds(req: Request, res: Response) { + try { + const rawMempool = await bitcoinApi.$getRawMempool(); + res.send(rawMempool); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async getBlockTipHeight(req: Request, res: Response) { + try { + const result = await bitcoinApi.$getBlockHeightTip(); + res.json(result); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async getBlockTipHash(req: Request, res: Response) { + try { + const result = await bitcoinApi.$getBlockHashTip(); + res.setHeader('content-type', 'text/plain'); + res.send(result); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async getTxIdsForBlock(req: Request, res: Response) { + try { + const result = await bitcoinApi.$getTxIdsForBlock(req.params.hash); + res.json(result); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async validateAddress(req: Request, res: Response) { + try { + const result = await bitcoinClient.validateAddress(req.params.address); + res.json(result); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async getTransactionOutspends(req: Request, res: Response) { + try { + const result = await bitcoinApi.$getOutspends(req.params.txId); + res.json(result); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private getDifficultyChange(req: Request, res: Response) { + try { + res.json(difficultyAdjustment.getDifficultyAdjustment()); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async $postTransaction(req: Request, res: Response) { + res.setHeader('content-type', 'text/plain'); + try { + let rawTx; + if (typeof req.body === 'object') { + rawTx = Object.keys(req.body)[0]; + } else { + rawTx = req.body; + } + const txIdResult = await bitcoinApi.$sendRawTransaction(rawTx); + res.send(txIdResult); + } catch (e: any) { + res.status(400).send(e.message && e.code ? 'sendrawtransaction RPC error: ' + JSON.stringify({ code: e.code, message: e.message }) + : (e.message || 'Error')); + } + } + + private async $postTransactionForm(req: Request, res: Response) { + res.setHeader('content-type', 'text/plain'); + const matches = /tx=([a-z0-9]+)/.exec(req.body); + let txHex = ''; + if (matches && matches[1]) { + txHex = matches[1]; + } + try { + const txIdResult = await bitcoinClient.sendRawTransaction(txHex); + res.send(txIdResult); + } catch (e: any) { + res.status(400).send(e.message && e.code ? 'sendrawtransaction RPC error: ' + JSON.stringify({ code: e.code, message: e.message }) + : (e.message || 'Error')); + } + } + +} + +export default new BitcoinRoutes(); diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 4b756d6ed..30f9fbf78 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -17,9 +17,10 @@ import { prepareBlock } from '../utils/blocks-utils'; import BlocksRepository from '../repositories/BlocksRepository'; import HashratesRepository from '../repositories/HashratesRepository'; import indexer from '../indexer'; +import fiatConversion from './fiat-conversion'; import poolsParser from './pools-parser'; import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository'; -import mining from './mining'; +import mining from './mining/mining'; import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository'; class Blocks { @@ -149,6 +150,7 @@ class Blocks { blockExtended.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0); blockExtended.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]); blockExtended.extras.coinbaseRaw = blockExtended.extras.coinbaseTx.vin[0].scriptsig; + blockExtended.extras.usd = fiatConversion.getConversionRates().USD; if (block.height === 0) { blockExtended.extras.medianFee = 0; // 50th percentiles @@ -279,8 +281,7 @@ class Blocks { const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); const blockPerSeconds = Math.max(1, indexedThisRun / elapsedSeconds); const progress = Math.round(totalIndexed / indexedBlocks.length * 10000) / 100; - const timeLeft = Math.round((indexedBlocks.length - totalIndexed) / blockPerSeconds); - logger.debug(`Indexing block summary for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexedBlocks.length} (${progress}%) | elapsed: ${runningFor} seconds | left: ~${timeLeft} seconds`); + logger.debug(`Indexing block summary for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexedBlocks.length} (${progress}%) | elapsed: ${runningFor} seconds`); timer = new Date().getTime() / 1000; indexedThisRun = 0; } @@ -292,7 +293,11 @@ class Blocks { totalIndexed++; newlyIndexed++; } - logger.notice(`Blocks summaries indexing completed: indexed ${newlyIndexed} blocks`); + if (newlyIndexed > 0) { + logger.notice(`Blocks summaries indexing completed: indexed ${newlyIndexed} blocks`); + } else { + logger.debug(`Blocks summaries indexing completed: indexed ${newlyIndexed} blocks`); + } } catch (e) { logger.err(`Blocks summaries indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`); throw e; @@ -347,8 +352,7 @@ class Blocks { const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); const blockPerSeconds = Math.max(1, indexedThisRun / elapsedSeconds); const progress = Math.round(totalIndexed / indexingBlockAmount * 10000) / 100; - const timeLeft = Math.round((indexingBlockAmount - totalIndexed) / blockPerSeconds); - logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${runningFor} seconds | left: ~${timeLeft} seconds`); + logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${runningFor} seconds`); timer = new Date().getTime() / 1000; indexedThisRun = 0; loadingIndicators.setProgress('block-indexing', progress, false); @@ -364,7 +368,11 @@ class Blocks { currentBlockHeight -= chunkSize; } - logger.notice(`Block indexing completed: indexed ${newlyIndexed} blocks`); + if (newlyIndexed > 0) { + logger.notice(`Block indexing completed: indexed ${newlyIndexed} blocks`); + } else { + logger.debug(`Block indexing completed: indexed ${newlyIndexed} blocks`); + } loadingIndicators.setProgress('block-indexing', 100); } catch (e) { logger.err('Block indexing failed. Trying again in 10 seconds. Reason: ' + (e instanceof Error ? e.message : e)); @@ -526,13 +534,12 @@ class Blocks { } } - let block = await bitcoinClient.getBlock(hash); - // Not Bitcoin network, return the block as it if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { - return block; + return await bitcoinApi.$getBlock(hash); } + let block = await bitcoinClient.getBlock(hash); block = prepareBlock(block); // Bitcoin network, add our custom data on top @@ -546,8 +553,8 @@ class Blocks { return blockExtended; } - public async $getStrippedBlockTransactions(hash: string, skipMemoryCache: boolean = false, - skipDBLookup: boolean = false): Promise + public async $getStrippedBlockTransactions(hash: string, skipMemoryCache = false, + skipDBLookup = false): Promise { if (skipMemoryCache === false) { // Check the memory cache diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index 1dc9f66ea..fe6b858e0 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -172,7 +172,7 @@ export class Common { static indexingEnabled(): boolean { return ( - ['mainnet', 'testnet', 'signet', 'regtest'].includes(config.MEMPOOL.NETWORK) && + ['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) && config.DATABASE.ENABLED === true && config.MEMPOOL.INDEXING_BLOCKS_AMOUNT !== 0 ); diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 32c726f32..0b43095cb 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -4,7 +4,7 @@ import logger from '../logger'; import { Common } from './common'; class DatabaseMigration { - private static currentVersion = 28; + private static currentVersion = 32; private queryTimeout = 120000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -12,8 +12,6 @@ class DatabaseMigration { private blocksTruncatedMessage = `'blocks' table has been truncated. Re-indexing from scratch.`; private hashratesTruncatedMessage = `'hashrates' table has been truncated. Re-indexing from scratch.`; - constructor() { } - /** * Avoid printing multiple time the same message */ @@ -104,181 +102,205 @@ class DatabaseMigration { await this.$setStatisticsAddedIndexedFlag(databaseSchemaVersion); const isBitcoin = ['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK); - try { - await this.$executeQuery(this.getCreateElementsTableQuery(), await this.$checkIfTableExists('elements_pegs')); - await this.$executeQuery(this.getCreateStatisticsQuery(), await this.$checkIfTableExists('statistics')); - if (databaseSchemaVersion < 2 && this.statisticsAddedIndexed === false) { - await this.$executeQuery(`CREATE INDEX added ON statistics (added);`); - } - if (databaseSchemaVersion < 3) { - await this.$executeQuery(this.getCreatePoolsTableQuery(), await this.$checkIfTableExists('pools')); - } - if (databaseSchemaVersion < 4) { - await this.$executeQuery('DROP table IF EXISTS blocks;'); - await this.$executeQuery(this.getCreateBlocksTableQuery(), await this.$checkIfTableExists('blocks')); - } - if (databaseSchemaVersion < 5 && isBitcoin === true) { - this.uniqueLog(logger.notice, this.blocksTruncatedMessage); - await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index - await this.$executeQuery('ALTER TABLE blocks ADD `reward` double unsigned NOT NULL DEFAULT "0"'); - } - if (databaseSchemaVersion < 6 && isBitcoin === true) { - this.uniqueLog(logger.notice, this.blocksTruncatedMessage); - await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index - // Cleanup original blocks fields type - await this.$executeQuery('ALTER TABLE blocks MODIFY `height` integer unsigned NOT NULL DEFAULT "0"'); - await this.$executeQuery('ALTER TABLE blocks MODIFY `tx_count` smallint unsigned NOT NULL DEFAULT "0"'); - await this.$executeQuery('ALTER TABLE blocks MODIFY `size` integer unsigned NOT NULL DEFAULT "0"'); - await this.$executeQuery('ALTER TABLE blocks MODIFY `weight` integer unsigned NOT NULL DEFAULT "0"'); - await this.$executeQuery('ALTER TABLE blocks MODIFY `difficulty` double NOT NULL DEFAULT "0"'); - // We also fix the pools.id type so we need to drop/re-create the foreign key - await this.$executeQuery('ALTER TABLE blocks DROP FOREIGN KEY IF EXISTS `blocks_ibfk_1`'); - await this.$executeQuery('ALTER TABLE pools MODIFY `id` smallint unsigned AUTO_INCREMENT'); - await this.$executeQuery('ALTER TABLE blocks MODIFY `pool_id` smallint unsigned NULL'); - await this.$executeQuery('ALTER TABLE blocks ADD FOREIGN KEY (`pool_id`) REFERENCES `pools` (`id`)'); - // Add new block indexing fields - await this.$executeQuery('ALTER TABLE blocks ADD `version` integer unsigned NOT NULL DEFAULT "0"'); - await this.$executeQuery('ALTER TABLE blocks ADD `bits` integer unsigned NOT NULL DEFAULT "0"'); - await this.$executeQuery('ALTER TABLE blocks ADD `nonce` bigint unsigned NOT NULL DEFAULT "0"'); - await this.$executeQuery('ALTER TABLE blocks ADD `merkle_root` varchar(65) NOT NULL DEFAULT ""'); - await this.$executeQuery('ALTER TABLE blocks ADD `previous_block_hash` varchar(65) NULL'); - } + await this.$executeQuery(this.getCreateElementsTableQuery(), await this.$checkIfTableExists('elements_pegs')); + await this.$executeQuery(this.getCreateStatisticsQuery(), await this.$checkIfTableExists('statistics')); + if (databaseSchemaVersion < 2 && this.statisticsAddedIndexed === false) { + await this.$executeQuery(`CREATE INDEX added ON statistics (added);`); + } + if (databaseSchemaVersion < 3) { + await this.$executeQuery(this.getCreatePoolsTableQuery(), await this.$checkIfTableExists('pools')); + } + if (databaseSchemaVersion < 4) { + await this.$executeQuery('DROP table IF EXISTS blocks;'); + await this.$executeQuery(this.getCreateBlocksTableQuery(), await this.$checkIfTableExists('blocks')); + } + if (databaseSchemaVersion < 5 && isBitcoin === true) { + this.uniqueLog(logger.notice, this.blocksTruncatedMessage); + await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index + await this.$executeQuery('ALTER TABLE blocks ADD `reward` double unsigned NOT NULL DEFAULT "0"'); + } - if (databaseSchemaVersion < 7 && isBitcoin === true) { - await this.$executeQuery('DROP table IF EXISTS hashrates;'); - await this.$executeQuery(this.getCreateDailyStatsTableQuery(), await this.$checkIfTableExists('hashrates')); - } + if (databaseSchemaVersion < 6 && isBitcoin === true) { + this.uniqueLog(logger.notice, this.blocksTruncatedMessage); + await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index + // Cleanup original blocks fields type + await this.$executeQuery('ALTER TABLE blocks MODIFY `height` integer unsigned NOT NULL DEFAULT "0"'); + await this.$executeQuery('ALTER TABLE blocks MODIFY `tx_count` smallint unsigned NOT NULL DEFAULT "0"'); + await this.$executeQuery('ALTER TABLE blocks MODIFY `size` integer unsigned NOT NULL DEFAULT "0"'); + await this.$executeQuery('ALTER TABLE blocks MODIFY `weight` integer unsigned NOT NULL DEFAULT "0"'); + await this.$executeQuery('ALTER TABLE blocks MODIFY `difficulty` double NOT NULL DEFAULT "0"'); + // We also fix the pools.id type so we need to drop/re-create the foreign key + await this.$executeQuery('ALTER TABLE blocks DROP FOREIGN KEY IF EXISTS `blocks_ibfk_1`'); + await this.$executeQuery('ALTER TABLE pools MODIFY `id` smallint unsigned AUTO_INCREMENT'); + await this.$executeQuery('ALTER TABLE blocks MODIFY `pool_id` smallint unsigned NULL'); + await this.$executeQuery('ALTER TABLE blocks ADD FOREIGN KEY (`pool_id`) REFERENCES `pools` (`id`)'); + // Add new block indexing fields + await this.$executeQuery('ALTER TABLE blocks ADD `version` integer unsigned NOT NULL DEFAULT "0"'); + await this.$executeQuery('ALTER TABLE blocks ADD `bits` integer unsigned NOT NULL DEFAULT "0"'); + await this.$executeQuery('ALTER TABLE blocks ADD `nonce` bigint unsigned NOT NULL DEFAULT "0"'); + await this.$executeQuery('ALTER TABLE blocks ADD `merkle_root` varchar(65) NOT NULL DEFAULT ""'); + await this.$executeQuery('ALTER TABLE blocks ADD `previous_block_hash` varchar(65) NULL'); + } - if (databaseSchemaVersion < 8 && isBitcoin === true) { - this.uniqueLog(logger.notice, this.blocksTruncatedMessage); - await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index - await this.$executeQuery('ALTER TABLE `hashrates` DROP INDEX `PRIMARY`'); - await this.$executeQuery('ALTER TABLE `hashrates` ADD `id` int NOT NULL AUTO_INCREMENT PRIMARY KEY FIRST'); - await this.$executeQuery('ALTER TABLE `hashrates` ADD `share` float NOT NULL DEFAULT "0"'); - await this.$executeQuery('ALTER TABLE `hashrates` ADD `type` enum("daily", "weekly") DEFAULT "daily"'); - } + if (databaseSchemaVersion < 7 && isBitcoin === true) { + await this.$executeQuery('DROP table IF EXISTS hashrates;'); + await this.$executeQuery(this.getCreateDailyStatsTableQuery(), await this.$checkIfTableExists('hashrates')); + } - if (databaseSchemaVersion < 9 && isBitcoin === true) { - this.uniqueLog(logger.notice, this.hashratesTruncatedMessage); - await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index - await this.$executeQuery('ALTER TABLE `state` CHANGE `name` `name` varchar(100)'); - await this.$executeQuery('ALTER TABLE `hashrates` ADD UNIQUE `hashrate_timestamp_pool_id` (`hashrate_timestamp`, `pool_id`)'); - } + if (databaseSchemaVersion < 8 && isBitcoin === true) { + this.uniqueLog(logger.notice, this.blocksTruncatedMessage); + await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index + await this.$executeQuery('ALTER TABLE `hashrates` DROP INDEX `PRIMARY`'); + await this.$executeQuery('ALTER TABLE `hashrates` ADD `id` int NOT NULL AUTO_INCREMENT PRIMARY KEY FIRST'); + await this.$executeQuery('ALTER TABLE `hashrates` ADD `share` float NOT NULL DEFAULT "0"'); + await this.$executeQuery('ALTER TABLE `hashrates` ADD `type` enum("daily", "weekly") DEFAULT "daily"'); + } - if (databaseSchemaVersion < 10 && isBitcoin === true) { - await this.$executeQuery('ALTER TABLE `blocks` ADD INDEX `blockTimestamp` (`blockTimestamp`)'); - } + if (databaseSchemaVersion < 9 && isBitcoin === true) { + this.uniqueLog(logger.notice, this.hashratesTruncatedMessage); + await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index + await this.$executeQuery('ALTER TABLE `state` CHANGE `name` `name` varchar(100)'); + await this.$executeQuery('ALTER TABLE `hashrates` ADD UNIQUE `hashrate_timestamp_pool_id` (`hashrate_timestamp`, `pool_id`)'); + } - if (databaseSchemaVersion < 11 && isBitcoin === true) { - this.uniqueLog(logger.notice, this.blocksTruncatedMessage); - await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index - await this.$executeQuery(`ALTER TABLE blocks - ADD avg_fee INT UNSIGNED NULL, - ADD avg_fee_rate INT UNSIGNED NULL - `); - await this.$executeQuery('ALTER TABLE blocks MODIFY `reward` BIGINT UNSIGNED NOT NULL DEFAULT "0"'); - await this.$executeQuery('ALTER TABLE blocks MODIFY `median_fee` INT UNSIGNED NOT NULL DEFAULT "0"'); - await this.$executeQuery('ALTER TABLE blocks MODIFY `fees` INT UNSIGNED NOT NULL DEFAULT "0"'); - } + if (databaseSchemaVersion < 10 && isBitcoin === true) { + await this.$executeQuery('ALTER TABLE `blocks` ADD INDEX `blockTimestamp` (`blockTimestamp`)'); + } - if (databaseSchemaVersion < 12 && isBitcoin === true) { - // No need to re-index because the new data type can contain larger values - await this.$executeQuery('ALTER TABLE blocks MODIFY `fees` BIGINT UNSIGNED NOT NULL DEFAULT "0"'); - } + if (databaseSchemaVersion < 11 && isBitcoin === true) { + this.uniqueLog(logger.notice, this.blocksTruncatedMessage); + await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index + await this.$executeQuery(`ALTER TABLE blocks + ADD avg_fee INT UNSIGNED NULL, + ADD avg_fee_rate INT UNSIGNED NULL + `); + await this.$executeQuery('ALTER TABLE blocks MODIFY `reward` BIGINT UNSIGNED NOT NULL DEFAULT "0"'); + await this.$executeQuery('ALTER TABLE blocks MODIFY `median_fee` INT UNSIGNED NOT NULL DEFAULT "0"'); + await this.$executeQuery('ALTER TABLE blocks MODIFY `fees` INT UNSIGNED NOT NULL DEFAULT "0"'); + } - if (databaseSchemaVersion < 13 && isBitcoin === true) { - await this.$executeQuery('ALTER TABLE blocks MODIFY `difficulty` DOUBLE UNSIGNED NOT NULL DEFAULT "0"'); - await this.$executeQuery('ALTER TABLE blocks MODIFY `median_fee` BIGINT UNSIGNED NOT NULL DEFAULT "0"'); - await this.$executeQuery('ALTER TABLE blocks MODIFY `avg_fee` BIGINT UNSIGNED NOT NULL DEFAULT "0"'); - await this.$executeQuery('ALTER TABLE blocks MODIFY `avg_fee_rate` BIGINT UNSIGNED NOT NULL DEFAULT "0"'); - } + if (databaseSchemaVersion < 12 && isBitcoin === true) { + // No need to re-index because the new data type can contain larger values + await this.$executeQuery('ALTER TABLE blocks MODIFY `fees` BIGINT UNSIGNED NOT NULL DEFAULT "0"'); + } - if (databaseSchemaVersion < 14 && isBitcoin === true) { - this.uniqueLog(logger.notice, this.hashratesTruncatedMessage); - await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index - await this.$executeQuery('ALTER TABLE `hashrates` DROP FOREIGN KEY `hashrates_ibfk_1`'); - await this.$executeQuery('ALTER TABLE `hashrates` MODIFY `pool_id` SMALLINT UNSIGNED NOT NULL DEFAULT "0"'); - } + if (databaseSchemaVersion < 13 && isBitcoin === true) { + await this.$executeQuery('ALTER TABLE blocks MODIFY `difficulty` DOUBLE UNSIGNED NOT NULL DEFAULT "0"'); + await this.$executeQuery('ALTER TABLE blocks MODIFY `median_fee` BIGINT UNSIGNED NOT NULL DEFAULT "0"'); + await this.$executeQuery('ALTER TABLE blocks MODIFY `avg_fee` BIGINT UNSIGNED NOT NULL DEFAULT "0"'); + await this.$executeQuery('ALTER TABLE blocks MODIFY `avg_fee_rate` BIGINT UNSIGNED NOT NULL DEFAULT "0"'); + } - if (databaseSchemaVersion < 16 && isBitcoin === true) { - this.uniqueLog(logger.notice, this.hashratesTruncatedMessage); - await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index because we changed timestamps - } + if (databaseSchemaVersion < 14 && isBitcoin === true) { + this.uniqueLog(logger.notice, this.hashratesTruncatedMessage); + await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index + await this.$executeQuery('ALTER TABLE `hashrates` DROP FOREIGN KEY `hashrates_ibfk_1`'); + await this.$executeQuery('ALTER TABLE `hashrates` MODIFY `pool_id` SMALLINT UNSIGNED NOT NULL DEFAULT "0"'); + } - if (databaseSchemaVersion < 17 && isBitcoin === true) { - await this.$executeQuery('ALTER TABLE `pools` ADD `slug` CHAR(50) NULL'); - } + if (databaseSchemaVersion < 16 && isBitcoin === true) { + this.uniqueLog(logger.notice, this.hashratesTruncatedMessage); + await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index because we changed timestamps + } - if (databaseSchemaVersion < 18 && isBitcoin === true) { - await this.$executeQuery('ALTER TABLE `blocks` ADD INDEX `hash` (`hash`);'); - } + if (databaseSchemaVersion < 17 && isBitcoin === true) { + await this.$executeQuery('ALTER TABLE `pools` ADD `slug` CHAR(50) NULL'); + } - if (databaseSchemaVersion < 19) { - await this.$executeQuery(this.getCreateRatesTableQuery(), await this.$checkIfTableExists('rates')); - } + if (databaseSchemaVersion < 18 && isBitcoin === true) { + await this.$executeQuery('ALTER TABLE `blocks` ADD INDEX `hash` (`hash`);'); + } - if (databaseSchemaVersion < 20 && isBitcoin === true) { - await this.$executeQuery(this.getCreateBlocksSummariesTableQuery(), await this.$checkIfTableExists('blocks_summaries')); - } + if (databaseSchemaVersion < 19) { + await this.$executeQuery(this.getCreateRatesTableQuery(), await this.$checkIfTableExists('rates')); + } - if (databaseSchemaVersion < 21) { - await this.$executeQuery('DROP TABLE IF EXISTS `rates`'); - await this.$executeQuery(this.getCreatePricesTableQuery(), await this.$checkIfTableExists('prices')); - } + if (databaseSchemaVersion < 20 && isBitcoin === true) { + await this.$executeQuery(this.getCreateBlocksSummariesTableQuery(), await this.$checkIfTableExists('blocks_summaries')); + } - if (databaseSchemaVersion < 22 && isBitcoin === true) { - await this.$executeQuery('DROP TABLE IF EXISTS `difficulty_adjustments`'); - await this.$executeQuery(this.getCreateDifficultyAdjustmentsTableQuery(), await this.$checkIfTableExists('difficulty_adjustments')); - } + if (databaseSchemaVersion < 21) { + await this.$executeQuery('DROP TABLE IF EXISTS `rates`'); + await this.$executeQuery(this.getCreatePricesTableQuery(), await this.$checkIfTableExists('prices')); + } - if (databaseSchemaVersion < 23) { - await this.$executeQuery('TRUNCATE `prices`'); - await this.$executeQuery('ALTER TABLE `prices` DROP `avg_prices`'); - await this.$executeQuery('ALTER TABLE `prices` ADD `USD` float DEFAULT "0"'); - await this.$executeQuery('ALTER TABLE `prices` ADD `EUR` float DEFAULT "0"'); - await this.$executeQuery('ALTER TABLE `prices` ADD `GBP` float DEFAULT "0"'); - await this.$executeQuery('ALTER TABLE `prices` ADD `CAD` float DEFAULT "0"'); - await this.$executeQuery('ALTER TABLE `prices` ADD `CHF` float DEFAULT "0"'); - await this.$executeQuery('ALTER TABLE `prices` ADD `AUD` float DEFAULT "0"'); - await this.$executeQuery('ALTER TABLE `prices` ADD `JPY` float DEFAULT "0"'); - } + if (databaseSchemaVersion < 22 && isBitcoin === true) { + await this.$executeQuery('DROP TABLE IF EXISTS `difficulty_adjustments`'); + await this.$executeQuery(this.getCreateDifficultyAdjustmentsTableQuery(), await this.$checkIfTableExists('difficulty_adjustments')); + } - if (databaseSchemaVersion < 24 && isBitcoin == true) { - await this.$executeQuery('DROP TABLE IF EXISTS `blocks_audits`'); - await this.$executeQuery(this.getCreateBlocksAuditsTableQuery(), await this.$checkIfTableExists('blocks_audits')); - } + if (databaseSchemaVersion < 23) { + await this.$executeQuery('TRUNCATE `prices`'); + await this.$executeQuery('ALTER TABLE `prices` DROP `avg_prices`'); + await this.$executeQuery('ALTER TABLE `prices` ADD `USD` float DEFAULT "0"'); + await this.$executeQuery('ALTER TABLE `prices` ADD `EUR` float DEFAULT "0"'); + await this.$executeQuery('ALTER TABLE `prices` ADD `GBP` float DEFAULT "0"'); + await this.$executeQuery('ALTER TABLE `prices` ADD `CAD` float DEFAULT "0"'); + await this.$executeQuery('ALTER TABLE `prices` ADD `CHF` float DEFAULT "0"'); + await this.$executeQuery('ALTER TABLE `prices` ADD `AUD` float DEFAULT "0"'); + await this.$executeQuery('ALTER TABLE `prices` ADD `JPY` float DEFAULT "0"'); + } - if (databaseSchemaVersion < 25 && isBitcoin === true) { - await this.$executeQuery(`INSERT INTO state VALUES('last_node_stats', 0, '1970-01-01');`); - await this.$executeQuery(this.getCreateLightningStatisticsQuery(), await this.$checkIfTableExists('lightning_stats')); - await this.$executeQuery(this.getCreateNodesQuery(), await this.$checkIfTableExists('nodes')); - await this.$executeQuery(this.getCreateChannelsQuery(), await this.$checkIfTableExists('channels')); - await this.$executeQuery(this.getCreateNodesStatsQuery(), await this.$checkIfTableExists('node_stats')); - } + if (databaseSchemaVersion < 24 && isBitcoin == true) { + await this.$executeQuery('DROP TABLE IF EXISTS `blocks_audits`'); + await this.$executeQuery(this.getCreateBlocksAuditsTableQuery(), await this.$checkIfTableExists('blocks_audits')); + } - if (databaseSchemaVersion < 26 && isBitcoin === true) { - this.uniqueLog(logger.notice, `'lightning_stats' table has been truncated. Will re-generate historical data from scratch.`); - await this.$executeQuery(`TRUNCATE lightning_stats`); - await this.$executeQuery('ALTER TABLE `lightning_stats` ADD tor_nodes int(11) NOT NULL DEFAULT "0"'); - await this.$executeQuery('ALTER TABLE `lightning_stats` ADD clearnet_nodes int(11) NOT NULL DEFAULT "0"'); - await this.$executeQuery('ALTER TABLE `lightning_stats` ADD unannounced_nodes int(11) NOT NULL DEFAULT "0"'); - } + if (databaseSchemaVersion < 25 && isBitcoin === true) { + await this.$executeQuery(`INSERT INTO state VALUES('last_node_stats', 0, '1970-01-01');`); + await this.$executeQuery(this.getCreateLightningStatisticsQuery(), await this.$checkIfTableExists('lightning_stats')); + await this.$executeQuery(this.getCreateNodesQuery(), await this.$checkIfTableExists('nodes')); + await this.$executeQuery(this.getCreateChannelsQuery(), await this.$checkIfTableExists('channels')); + await this.$executeQuery(this.getCreateNodesStatsQuery(), await this.$checkIfTableExists('node_stats')); + } - if (databaseSchemaVersion < 27 && isBitcoin === true) { - await this.$executeQuery('ALTER TABLE `lightning_stats` ADD avg_capacity bigint(20) unsigned NOT NULL DEFAULT "0"'); - await this.$executeQuery('ALTER TABLE `lightning_stats` ADD avg_fee_rate int(11) unsigned NOT NULL DEFAULT "0"'); - await this.$executeQuery('ALTER TABLE `lightning_stats` ADD avg_base_fee_mtokens bigint(20) unsigned NOT NULL DEFAULT "0"'); - await this.$executeQuery('ALTER TABLE `lightning_stats` ADD med_capacity bigint(20) unsigned NOT NULL DEFAULT "0"'); - await this.$executeQuery('ALTER TABLE `lightning_stats` ADD med_fee_rate int(11) unsigned NOT NULL DEFAULT "0"'); - await this.$executeQuery('ALTER TABLE `lightning_stats` ADD med_base_fee_mtokens bigint(20) unsigned NOT NULL DEFAULT "0"'); - } + if (databaseSchemaVersion < 26 && isBitcoin === true) { + this.uniqueLog(logger.notice, `'lightning_stats' table has been truncated. Will re-generate historical data from scratch.`); + await this.$executeQuery(`TRUNCATE lightning_stats`); + await this.$executeQuery('ALTER TABLE `lightning_stats` ADD tor_nodes int(11) NOT NULL DEFAULT "0"'); + await this.$executeQuery('ALTER TABLE `lightning_stats` ADD clearnet_nodes int(11) NOT NULL DEFAULT "0"'); + await this.$executeQuery('ALTER TABLE `lightning_stats` ADD unannounced_nodes int(11) NOT NULL DEFAULT "0"'); + } - if (databaseSchemaVersion < 28 && isBitcoin == true) { - await this.$executeQuery('ALTER TABLE `blocks_summaries` ADD `template` JSON DEFAULT "[]"'); - } - } catch (e) { - throw e; + if (databaseSchemaVersion < 27 && isBitcoin === true) { + await this.$executeQuery('ALTER TABLE `lightning_stats` ADD avg_capacity bigint(20) unsigned NOT NULL DEFAULT "0"'); + await this.$executeQuery('ALTER TABLE `lightning_stats` ADD avg_fee_rate int(11) unsigned NOT NULL DEFAULT "0"'); + await this.$executeQuery('ALTER TABLE `lightning_stats` ADD avg_base_fee_mtokens bigint(20) unsigned NOT NULL DEFAULT "0"'); + await this.$executeQuery('ALTER TABLE `lightning_stats` ADD med_capacity bigint(20) unsigned NOT NULL DEFAULT "0"'); + await this.$executeQuery('ALTER TABLE `lightning_stats` ADD med_fee_rate int(11) unsigned NOT NULL DEFAULT "0"'); + await this.$executeQuery('ALTER TABLE `lightning_stats` ADD med_base_fee_mtokens bigint(20) unsigned NOT NULL DEFAULT "0"'); + } + + if (databaseSchemaVersion < 28 && isBitcoin === true) { + await this.$executeQuery(`TRUNCATE lightning_stats`); + await this.$executeQuery(`TRUNCATE node_stats`); + await this.$executeQuery(`ALTER TABLE lightning_stats MODIFY added DATE`); + } + + if (databaseSchemaVersion < 29 && isBitcoin === true) { + await this.$executeQuery(this.getCreateGeoNamesTableQuery(), await this.$checkIfTableExists('geo_names')); + await this.$executeQuery('ALTER TABLE `nodes` ADD as_number int(11) unsigned NULL DEFAULT NULL'); + await this.$executeQuery('ALTER TABLE `nodes` ADD city_id int(11) unsigned NULL DEFAULT NULL'); + await this.$executeQuery('ALTER TABLE `nodes` ADD country_id int(11) unsigned NULL DEFAULT NULL'); + await this.$executeQuery('ALTER TABLE `nodes` ADD accuracy_radius int(11) unsigned NULL DEFAULT NULL'); + await this.$executeQuery('ALTER TABLE `nodes` ADD subdivision_id int(11) unsigned NULL DEFAULT NULL'); + await this.$executeQuery('ALTER TABLE `nodes` ADD longitude double NULL DEFAULT NULL'); + await this.$executeQuery('ALTER TABLE `nodes` ADD latitude double NULL DEFAULT NULL'); + } + + if (databaseSchemaVersion < 30 && isBitcoin === true) { + await this.$executeQuery('ALTER TABLE `geo_names` CHANGE `type` `type` enum("city","country","division","continent","as_organization") NOT NULL'); + } + + if (databaseSchemaVersion < 31 && isBitcoin == true) { // Link blocks to prices + await this.$executeQuery('ALTER TABLE `prices` ADD `id` int NULL AUTO_INCREMENT UNIQUE'); + await this.$executeQuery('DROP TABLE IF EXISTS `blocks_prices`'); + await this.$executeQuery(this.getCreateBlocksPricesTableQuery(), await this.$checkIfTableExists('blocks_prices')); + } + + if (databaseSchemaVersion < 32 && isBitcoin == true) { + await this.$executeQuery('ALTER TABLE `blocks_summaries` ADD `template` JSON DEFAULT "[]"'); } } @@ -317,7 +339,7 @@ class DatabaseMigration { /** * Small query execution wrapper to log all executed queries */ - private async $executeQuery(query: string, silent: boolean = false): Promise { + private async $executeQuery(query: string, silent = false): Promise { if (!silent) { logger.debug('MIGRATIONS: Execute query:\n' + query); } @@ -346,21 +368,17 @@ class DatabaseMigration { * Create the `state` table */ private async $createMigrationStateTable(): Promise { - try { - const query = `CREATE TABLE IF NOT EXISTS state ( - name varchar(25) NOT NULL, - number int(11) NULL, - string varchar(100) NULL, - CONSTRAINT name_unique UNIQUE (name) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; - await this.$executeQuery(query); + const query = `CREATE TABLE IF NOT EXISTS state ( + name varchar(25) NOT NULL, + number int(11) NULL, + string varchar(100) NULL, + CONSTRAINT name_unique UNIQUE (name) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; + await this.$executeQuery(query); - // Set initial values - await this.$executeQuery(`INSERT INTO state VALUES('schema_version', 0, NULL);`); - await this.$executeQuery(`INSERT INTO state VALUES('last_elements_block', 0, NULL);`); - } catch (e) { - throw e; - } + // Set initial values + await this.$executeQuery(`INSERT INTO state VALUES('schema_version', 0, NULL);`); + await this.$executeQuery(`INSERT INTO state VALUES('last_elements_block', 0, NULL);`); } /** @@ -690,6 +708,25 @@ class DatabaseMigration { ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; } + private getCreateGeoNamesTableQuery(): string { + return `CREATE TABLE geo_names ( + id int(11) unsigned NOT NULL, + type enum('city','country','division','continent') NOT NULL, + names text DEFAULT NULL, + UNIQUE KEY id (id,type), + KEY id_2 (id) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8;` + } + + private getCreateBlocksPricesTableQuery(): string { + return `CREATE TABLE IF NOT EXISTS blocks_prices ( + height int(10) unsigned NOT NULL, + price_id int(10) unsigned NOT NULL, + PRIMARY KEY (height), + INDEX (price_id) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; + } + public async $truncateIndexedData(tables: string[]) { const allowedTables = ['blocks', 'hashrates', 'prices']; diff --git a/backend/src/api/explorer/nodes.api.ts b/backend/src/api/explorer/nodes.api.ts index 1bf9ce12d..590ed1f20 100644 --- a/backend/src/api/explorer/nodes.api.ts +++ b/backend/src/api/explorer/nodes.api.ts @@ -4,15 +4,51 @@ import DB from '../../database'; class NodesApi { public async $getNode(public_key: string): Promise { try { - const query = `SELECT nodes.*, (SELECT COUNT(*) FROM channels WHERE channels.status < 2 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)) AS channel_count, (SELECT SUM(capacity) FROM channels WHERE channels.status < 2 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)) AS capacity, (SELECT AVG(capacity) FROM channels WHERE status < 2 AND (node1_public_key = ? OR node2_public_key = ?)) AS channels_capacity_avg FROM nodes WHERE public_key = ?`; + const query = ` + SELECT nodes.*, geo_names_as.names as as_organization, geo_names_city.names as city, + geo_names_country.names as country, geo_names_subdivision.names as subdivision, + (SELECT Count(*) + FROM channels + WHERE channels.status < 2 AND ( channels.node1_public_key = ? OR channels.node2_public_key = ? )) AS channel_count, + (SELECT Sum(capacity) + FROM channels + WHERE channels.status < 2 AND ( channels.node1_public_key = ? OR channels.node2_public_key = ? )) AS capacity, + (SELECT Avg(capacity) + FROM channels + WHERE status < 2 AND ( node1_public_key = ? OR node2_public_key = ? )) AS channels_capacity_avg + FROM nodes + LEFT JOIN geo_names geo_names_as on geo_names_as.id = as_number + LEFT JOIN geo_names geo_names_city on geo_names_city.id = city_id + LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = subdivision_id + LEFT JOIN geo_names geo_names_country on geo_names_country.id = country_id + WHERE public_key = ? + `; const [rows]: any = await DB.query(query, [public_key, public_key, public_key, public_key, public_key, public_key, public_key]); - return rows[0]; + if (rows.length > 0) { + rows[0].as_organization = JSON.parse(rows[0].as_organization); + rows[0].subdivision = JSON.parse(rows[0].subdivision); + rows[0].city = JSON.parse(rows[0].city); + rows[0].country = JSON.parse(rows[0].country); + return rows[0]; + } + return null; } catch (e) { logger.err('$getNode error: ' + (e instanceof Error ? e.message : e)); throw e; } } + public async $getAllNodes(): Promise { + try { + const query = `SELECT * FROM nodes`; + const [rows]: any = await DB.query(query); + return rows; + } catch (e) { + logger.err('$getAllNodes error: ' + (e instanceof Error ? e.message : e)); + throw e; + } + } + public async $getNodeStats(public_key: string): Promise { try { const query = `SELECT UNIX_TIMESTAMP(added) AS added, capacity, channels FROM node_stats WHERE public_key = ? ORDER BY added DESC`; diff --git a/backend/src/api/liquid/liquid.routes.ts b/backend/src/api/liquid/liquid.routes.ts new file mode 100644 index 000000000..b130373e1 --- /dev/null +++ b/backend/src/api/liquid/liquid.routes.ts @@ -0,0 +1,73 @@ +import axios from 'axios'; +import { Application, Request, Response } from 'express'; +import config from '../../config'; +import elementsParser from './elements-parser'; +import icons from './icons'; + +class LiquidRoutes { + public initRoutes(app: Application) { + app + .get(config.MEMPOOL.API_URL_PREFIX + 'assets/icons', this.getAllLiquidIcon) + .get(config.MEMPOOL.API_URL_PREFIX + 'assets/featured', this.$getAllFeaturedLiquidAssets) + .get(config.MEMPOOL.API_URL_PREFIX + 'asset/:assetId/icon', this.getLiquidIcon) + .get(config.MEMPOOL.API_URL_PREFIX + 'assets/group/:id', this.$getAssetGroup) + ; + + if (config.DATABASE.ENABLED) { + app + .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/pegs/month', this.$getElementsPegsByMonth) + ; + } + } + + + private getLiquidIcon(req: Request, res: Response) { + const result = icons.getIconByAssetId(req.params.assetId); + if (result) { + res.setHeader('content-type', 'image/png'); + res.setHeader('content-length', result.length); + res.send(result); + } else { + res.status(404).send('Asset icon not found'); + } + } + + private getAllLiquidIcon(req: Request, res: Response) { + const result = icons.getAllIconIds(); + if (result) { + res.json(result); + } else { + res.status(404).send('Asset icons not found'); + } + } + + private async $getAllFeaturedLiquidAssets(req: Request, res: Response) { + try { + const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.LIQUID_API}/assets/featured`, { responseType: 'stream', timeout: 10000 }); + response.data.pipe(res); + } catch (e) { + res.status(500).end(); + } + } + + private async $getAssetGroup(req: Request, res: Response) { + try { + const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.LIQUID_API}/assets/group/${parseInt(req.params.id, 10)}`, + { responseType: 'stream', timeout: 10000 }); + response.data.pipe(res); + } catch (e) { + res.status(500).end(); + } + } + + private async $getElementsPegsByMonth(req: Request, res: Response) { + try { + const pegs = await elementsParser.$getPegDataByMonth(); + res.json(pegs); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } +} + +export default new LiquidRoutes(); diff --git a/backend/src/api/mining/mining-routes.ts b/backend/src/api/mining/mining-routes.ts new file mode 100644 index 000000000..02262353f --- /dev/null +++ b/backend/src/api/mining/mining-routes.ts @@ -0,0 +1,238 @@ +import { Application, Request, Response } from 'express'; +import config from "../../config"; +import logger from '../../logger'; +import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository'; +import BlocksRepository from '../../repositories/BlocksRepository'; +import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjustmentsRepository'; +import HashratesRepository from '../../repositories/HashratesRepository'; +import bitcoinClient from '../bitcoin/bitcoin-client'; +import mining from "./mining"; + +class MiningRoutes { + public initRoutes(app: Application) { + app + .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/:interval', this.$getPools) + .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug/hashrate', this.$getPoolHistoricalHashrate) + .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug/blocks', this.$getPoolBlocks) + .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug/blocks/:height', this.$getPoolBlocks) + .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug', this.$getPool) + .get(config.MEMPOOL.API_URL_PREFIX + 'mining/hashrate/pools/:interval', this.$getPoolsHistoricalHashrate) + .get(config.MEMPOOL.API_URL_PREFIX + 'mining/hashrate/:interval', this.$getHistoricalHashrate) + .get(config.MEMPOOL.API_URL_PREFIX + 'mining/difficulty-adjustments', this.$getDifficultyAdjustments) + .get(config.MEMPOOL.API_URL_PREFIX + 'mining/reward-stats/:blockCount', this.$getRewardStats) + .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/fees/:interval', this.$getHistoricalBlockFees) + .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/rewards/:interval', this.$getHistoricalBlockRewards) + .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/fee-rates/:interval', this.$getHistoricalBlockFeeRates) + .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/sizes-weights/:interval', this.$getHistoricalBlockSizeAndWeight) + .get(config.MEMPOOL.API_URL_PREFIX + 'mining/difficulty-adjustments/:interval', this.$getDifficultyAdjustments) + .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/predictions/:interval', this.$getHistoricalBlockPrediction) + ; + } + + private async $getPool(req: Request, res: Response): Promise { + try { + const stats = await mining.$getPoolStat(req.params.slug); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); + res.json(stats); + } catch (e) { + if (e instanceof Error && e.message.indexOf('This mining pool does not exist') > -1) { + res.status(404).send(e.message); + } else { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + } + + private async $getPoolBlocks(req: Request, res: Response) { + try { + const poolBlocks = await BlocksRepository.$getBlocksByPool( + req.params.slug, + req.params.height === undefined ? undefined : parseInt(req.params.height, 10), + ); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); + res.json(poolBlocks); + } catch (e) { + if (e instanceof Error && e.message.indexOf('This mining pool does not exist') > -1) { + res.status(404).send(e.message); + } else { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + } + + private async $getPools(req: Request, res: Response) { + try { + const stats = await mining.$getPoolsStats(req.params.interval); + const blockCount = await BlocksRepository.$blockCount(null, null); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.header('X-total-count', blockCount.toString()); + res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); + res.json(stats); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async $getPoolsHistoricalHashrate(req: Request, res: Response) { + try { + const hashrates = await HashratesRepository.$getPoolsWeeklyHashrate(req.params.interval); + const blockCount = await BlocksRepository.$blockCount(null, null); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.header('X-total-count', blockCount.toString()); + res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString()); + res.json(hashrates); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async $getPoolHistoricalHashrate(req: Request, res: Response) { + try { + const hashrates = await HashratesRepository.$getPoolWeeklyHashrate(req.params.slug); + const blockCount = await BlocksRepository.$blockCount(null, null); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.header('X-total-count', blockCount.toString()); + res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString()); + res.json(hashrates); + } catch (e) { + if (e instanceof Error && e.message.indexOf('This mining pool does not exist') > -1) { + res.status(404).send(e.message); + } else { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + } + + private async $getHistoricalHashrate(req: Request, res: Response) { + let currentHashrate = 0, currentDifficulty = 0; + try { + currentHashrate = await bitcoinClient.getNetworkHashPs(); + currentDifficulty = await bitcoinClient.getDifficulty(); + } catch (e) { + logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate and difficulty'); + } + + try { + const hashrates = await HashratesRepository.$getNetworkDailyHashrate(req.params.interval); + const difficulty = await DifficultyAdjustmentsRepository.$getAdjustments(req.params.interval, false); + const blockCount = await BlocksRepository.$blockCount(null, null); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.header('X-total-count', blockCount.toString()); + res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString()); + res.json({ + hashrates: hashrates, + difficulty: difficulty, + currentHashrate: currentHashrate, + currentDifficulty: currentDifficulty, + }); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async $getHistoricalBlockFees(req: Request, res: Response) { + try { + const blockFees = await mining.$getHistoricalBlockFees(req.params.interval); + const blockCount = await BlocksRepository.$blockCount(null, null); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.header('X-total-count', blockCount.toString()); + res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); + res.json(blockFees); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async $getHistoricalBlockRewards(req: Request, res: Response) { + try { + const blockRewards = await mining.$getHistoricalBlockRewards(req.params.interval); + const blockCount = await BlocksRepository.$blockCount(null, null); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.header('X-total-count', blockCount.toString()); + res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); + res.json(blockRewards); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async $getHistoricalBlockFeeRates(req: Request, res: Response) { + try { + const blockFeeRates = await mining.$getHistoricalBlockFeeRates(req.params.interval); + const blockCount = await BlocksRepository.$blockCount(null, null); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.header('X-total-count', blockCount.toString()); + res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); + res.json(blockFeeRates); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async $getHistoricalBlockSizeAndWeight(req: Request, res: Response) { + try { + const blockSizes = await mining.$getHistoricalBlockSizes(req.params.interval); + const blockWeights = await mining.$getHistoricalBlockWeights(req.params.interval); + const blockCount = await BlocksRepository.$blockCount(null, null); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.header('X-total-count', blockCount.toString()); + res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); + res.json({ + sizes: blockSizes, + weights: blockWeights + }); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async $getDifficultyAdjustments(req: Request, res: Response) { + try { + const difficulty = await DifficultyAdjustmentsRepository.$getRawAdjustments(req.params.interval, true); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString()); + res.json(difficulty.map(adj => [adj.time, adj.height, adj.difficulty, adj.adjustment])); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async $getRewardStats(req: Request, res: Response) { + try { + const response = await mining.$getRewardStats(parseInt(req.params.blockCount, 10)); + res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); + res.json(response); + } catch (e) { + res.status(500).end(); + } + } + + private async $getHistoricalBlockPrediction(req: Request, res: Response) { + try { + const blockPredictions = await mining.$getBlockPredictionsHistory(req.params.interval); + const blockCount = await BlocksAuditsRepository.$getPredictionsCount(); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.header('X-total-count', blockCount.toString()); + res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); + res.json(blockPredictions.map(prediction => [prediction.time, prediction.height, prediction.match_rate])); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } +} + +export default new MiningRoutes(); diff --git a/backend/src/api/mining.ts b/backend/src/api/mining/mining.ts similarity index 80% rename from backend/src/api/mining.ts rename to backend/src/api/mining/mining.ts index 50a86e73e..55e749596 100644 --- a/backend/src/api/mining.ts +++ b/backend/src/api/mining/mining.ts @@ -1,18 +1,20 @@ -import { IndexedDifficultyAdjustment, PoolInfo, PoolStats, RewardStats } from '../mempool.interfaces'; -import BlocksRepository from '../repositories/BlocksRepository'; -import PoolsRepository from '../repositories/PoolsRepository'; -import HashratesRepository from '../repositories/HashratesRepository'; -import bitcoinClient from './bitcoin/bitcoin-client'; -import logger from '../logger'; -import { Common } from './common'; -import loadingIndicators from './loading-indicators'; +import { BlockPrice, PoolInfo, PoolStats, RewardStats } from '../../mempool.interfaces'; +import BlocksRepository from '../../repositories/BlocksRepository'; +import PoolsRepository from '../../repositories/PoolsRepository'; +import HashratesRepository from '../../repositories/HashratesRepository'; +import bitcoinClient from '../bitcoin/bitcoin-client'; +import logger from '../../logger'; +import { Common } from '../common'; +import loadingIndicators from '../loading-indicators'; import { escape } from 'mysql2'; -import indexer from '../indexer'; -import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository'; -import config from '../config'; -import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository'; +import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjustmentsRepository'; +import config from '../../config'; +import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository'; +import PricesRepository from '../../repositories/PricesRepository'; class Mining { + blocksPriceIndexingRunning = false; + constructor() { } @@ -31,7 +33,7 @@ class Mining { */ public async $getHistoricalBlockFees(interval: string | null = null): Promise { return await BlocksRepository.$getHistoricalBlockFees( - this.getTimeRange(interval), + this.getTimeRange(interval, 5), Common.getSqlInterval(interval) ); } @@ -250,9 +252,8 @@ class Mining { const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); const weeksPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds)); const progress = Math.round(totalIndexed / totalWeekIndexed * 10000) / 100; - const timeLeft = Math.round((totalWeekIndexed - totalIndexed) / weeksPerSeconds); const formattedDate = new Date(fromTimestamp).toUTCString(); - logger.debug(`Getting weekly pool hashrate for ${formattedDate} | ~${weeksPerSeconds.toFixed(2)} weeks/sec | total: ~${totalIndexed}/${Math.round(totalWeekIndexed)} (${progress}%) | elapsed: ${runningFor} seconds | left: ~${timeLeft} seconds`); + logger.debug(`Getting weekly pool hashrate for ${formattedDate} | ~${weeksPerSeconds.toFixed(2)} weeks/sec | total: ~${totalIndexed}/${Math.round(totalWeekIndexed)} (${progress}%) | elapsed: ${runningFor} seconds`); timer = new Date().getTime() / 1000; indexedThisRun = 0; loadingIndicators.setProgress('weekly-hashrate-indexing', progress, false); @@ -265,6 +266,8 @@ class Mining { await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', new Date().getUTCDate()); if (newlyIndexed > 0) { logger.notice(`Weekly mining pools hashrates indexing completed: indexed ${newlyIndexed}`); + } else { + logger.debug(`Weekly mining pools hashrates indexing completed: indexed ${newlyIndexed}`); } loadingIndicators.setProgress('weekly-hashrate-indexing', 100); } catch (e) { @@ -339,9 +342,8 @@ class Mining { const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); const daysPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds)); const progress = Math.round(totalIndexed / totalDayIndexed * 10000) / 100; - const timeLeft = Math.round((totalDayIndexed - totalIndexed) / daysPerSeconds); const formattedDate = new Date(fromTimestamp).toUTCString(); - logger.debug(`Getting network daily hashrate for ${formattedDate} | ~${daysPerSeconds.toFixed(2)} days/sec | total: ~${totalIndexed}/${Math.round(totalDayIndexed)} (${progress}%) | elapsed: ${runningFor} seconds | left: ~${timeLeft} seconds`); + logger.debug(`Getting network daily hashrate for ${formattedDate} | ~${daysPerSeconds.toFixed(2)} days/sec | total: ~${totalIndexed}/${Math.round(totalDayIndexed)} (${progress}%) | elapsed: ${runningFor} seconds`); timer = new Date().getTime() / 1000; indexedThisRun = 0; loadingIndicators.setProgress('daily-hashrate-indexing', progress); @@ -369,6 +371,8 @@ class Mining { await HashratesRepository.$setLatestRun('last_hashrates_indexing', new Date().getUTCDate()); if (newlyIndexed > 0) { logger.notice(`Daily network hashrate indexing completed: indexed ${newlyIndexed} days`); + } else { + logger.debug(`Daily network hashrate indexing completed: indexed ${newlyIndexed} days`); } loadingIndicators.setProgress('daily-hashrate-indexing', 100); } catch (e) { @@ -446,9 +450,75 @@ class Mining { if (totalIndexed > 0) { logger.notice(`Indexed ${totalIndexed} difficulty adjustments`); + } else { + logger.debug(`Indexed ${totalIndexed} difficulty adjustments`); } } + /** + * Create a link between blocks and the latest price at when they were mined + */ + public async $indexBlockPrices() { + if (this.blocksPriceIndexingRunning === true) { + return; + } + this.blocksPriceIndexingRunning = true; + + try { + const prices: any[] = await PricesRepository.$getPricesTimesAndId(); + const blocksWithoutPrices: any[] = await BlocksRepository.$getBlocksWithoutPrice(); + + let totalInserted = 0; + const blocksPrices: BlockPrice[] = []; + + for (const block of blocksWithoutPrices) { + // Quick optimisation, out mtgox feed only goes back to 2010-07-19 02:00:00, so skip the first 68951 blocks + if (block.height < 68951) { + blocksPrices.push({ + height: block.height, + priceId: prices[0].id, + }); + continue; + } + for (const price of prices) { + if (block.timestamp < price.time) { + blocksPrices.push({ + height: block.height, + priceId: price.id, + }); + break; + }; + } + + if (blocksPrices.length >= 100000) { + totalInserted += blocksPrices.length; + if (blocksWithoutPrices.length > 200000) { + logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`); + } else { + logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price`); + } + await BlocksRepository.$saveBlockPrices(blocksPrices); + blocksPrices.length = 0; + } + } + + if (blocksPrices.length > 0) { + totalInserted += blocksPrices.length; + if (blocksWithoutPrices.length > 200000) { + logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`); + } else { + logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price`); + } + await BlocksRepository.$saveBlockPrices(blocksPrices); + } + } catch (e) { + this.blocksPriceIndexingRunning = false; + throw e; + } + + this.blocksPriceIndexingRunning = false; + } + private getDateMidnight(date: Date): Date { date.setUTCHours(0); date.setUTCMinutes(0); @@ -458,18 +528,18 @@ class Mining { return date; } - private getTimeRange(interval: string | null): number { + private getTimeRange(interval: string | null, scale = 1): number { switch (interval) { - case '3y': return 43200; // 12h - case '2y': return 28800; // 8h - case '1y': return 28800; // 8h - case '6m': return 10800; // 3h - case '3m': return 7200; // 2h - case '1m': return 1800; // 30min - case '1w': return 300; // 5min - case '3d': return 1; - case '24h': return 1; - default: return 86400; // 24h + case '3y': return 43200 * scale; // 12h + case '2y': return 28800 * scale; // 8h + case '1y': return 28800 * scale; // 8h + case '6m': return 10800 * scale; // 3h + case '3m': return 7200 * scale; // 2h + case '1m': return 1800 * scale; // 30min + case '1w': return 300 * scale; // 5min + case '3d': return 1 * scale; + case '24h': return 1 * scale; + default: return 86400 * scale; } } } diff --git a/backend/src/api/statistics.ts b/backend/src/api/statistics/statistics-api.ts similarity index 69% rename from backend/src/api/statistics.ts rename to backend/src/api/statistics/statistics-api.ts index bd93b4c6e..56a868f8f 100644 --- a/backend/src/api/statistics.ts +++ b/backend/src/api/statistics/statistics-api.ts @@ -1,160 +1,11 @@ -import memPool from './mempool'; -import DB from '../database'; -import logger from '../logger'; +import DB from '../../database'; +import logger from '../../logger'; +import { Statistic, OptimizedStatistic } from '../../mempool.interfaces'; -import { Statistic, TransactionExtended, OptimizedStatistic } from '../mempool.interfaces'; -import config from '../config'; -import { Common } from './common'; - -class Statistics { - protected intervalTimer: NodeJS.Timer | undefined; - protected newStatisticsEntryCallback: ((stats: OptimizedStatistic) => void) | undefined; +class StatisticsApi { protected queryTimeout = 120000; - public setNewStatisticsEntryCallback(fn: (stats: OptimizedStatistic) => void) { - this.newStatisticsEntryCallback = fn; - } - - constructor() { } - - public startStatistics(): void { - logger.info('Starting statistics service'); - - const now = new Date(); - const nextInterval = new Date(now.getFullYear(), now.getMonth(), now.getDate(), now.getHours(), - Math.floor(now.getMinutes() / 1) * 1 + 1, 0, 0); - const difference = nextInterval.getTime() - now.getTime(); - - setTimeout(() => { - this.runStatistics(); - this.intervalTimer = setInterval(() => { - this.runStatistics(); - }, 1 * 60 * 1000); - }, difference); - } - - private async runStatistics(): Promise { - if (!memPool.isInSync()) { - return; - } - const currentMempool = memPool.getMempool(); - const txPerSecond = memPool.getTxPerSecond(); - const vBytesPerSecond = memPool.getVBytesPerSecond(); - - logger.debug('Running statistics'); - - let memPoolArray: TransactionExtended[] = []; - for (const i in currentMempool) { - if (currentMempool.hasOwnProperty(i)) { - memPoolArray.push(currentMempool[i]); - } - } - // Remove 0 and undefined - memPoolArray = memPoolArray.filter((tx) => tx.effectiveFeePerVsize); - - if (!memPoolArray.length) { - try { - const insertIdZeroed = await this.$createZeroedStatistic(); - if (this.newStatisticsEntryCallback && insertIdZeroed) { - const newStats = await this.$get(insertIdZeroed); - if (newStats) { - this.newStatisticsEntryCallback(newStats); - } - } - } catch (e) { - logger.err('Unable to insert zeroed statistics. ' + e); - } - return; - } - - memPoolArray.sort((a, b) => a.effectiveFeePerVsize - b.effectiveFeePerVsize); - const totalWeight = memPoolArray.map((tx) => tx.vsize).reduce((acc, curr) => acc + curr) * 4; - const totalFee = memPoolArray.map((tx) => tx.fee).reduce((acc, curr) => acc + curr); - - const logFees = [1, 2, 3, 4, 5, 6, 8, 10, 12, 15, 20, 30, 40, 50, 60, 70, 80, 90, 100, 125, 150, 175, 200, - 250, 300, 350, 400, 500, 600, 700, 800, 900, 1000, 1200, 1400, 1600, 1800, 2000]; - - const weightVsizeFees: { [feePerWU: number]: number } = {}; - const lastItem = logFees.length - 1; - - memPoolArray.forEach((transaction) => { - for (let i = 0; i < logFees.length; i++) { - if ( - (Common.isLiquid() && (i === lastItem || transaction.effectiveFeePerVsize * 10 < logFees[i + 1])) - || - (!Common.isLiquid() && (i === lastItem || transaction.effectiveFeePerVsize < logFees[i + 1])) - ) { - if (weightVsizeFees[logFees[i]]) { - weightVsizeFees[logFees[i]] += transaction.vsize; - } else { - weightVsizeFees[logFees[i]] = transaction.vsize; - } - break; - } - } - }); - - try { - const insertId = await this.$create({ - added: 'NOW()', - unconfirmed_transactions: memPoolArray.length, - tx_per_second: txPerSecond, - vbytes_per_second: Math.round(vBytesPerSecond), - mempool_byte_weight: totalWeight, - total_fee: totalFee, - fee_data: '', - vsize_1: weightVsizeFees['1'] || 0, - vsize_2: weightVsizeFees['2'] || 0, - vsize_3: weightVsizeFees['3'] || 0, - vsize_4: weightVsizeFees['4'] || 0, - vsize_5: weightVsizeFees['5'] || 0, - vsize_6: weightVsizeFees['6'] || 0, - vsize_8: weightVsizeFees['8'] || 0, - vsize_10: weightVsizeFees['10'] || 0, - vsize_12: weightVsizeFees['12'] || 0, - vsize_15: weightVsizeFees['15'] || 0, - vsize_20: weightVsizeFees['20'] || 0, - vsize_30: weightVsizeFees['30'] || 0, - vsize_40: weightVsizeFees['40'] || 0, - vsize_50: weightVsizeFees['50'] || 0, - vsize_60: weightVsizeFees['60'] || 0, - vsize_70: weightVsizeFees['70'] || 0, - vsize_80: weightVsizeFees['80'] || 0, - vsize_90: weightVsizeFees['90'] || 0, - vsize_100: weightVsizeFees['100'] || 0, - vsize_125: weightVsizeFees['125'] || 0, - vsize_150: weightVsizeFees['150'] || 0, - vsize_175: weightVsizeFees['175'] || 0, - vsize_200: weightVsizeFees['200'] || 0, - vsize_250: weightVsizeFees['250'] || 0, - vsize_300: weightVsizeFees['300'] || 0, - vsize_350: weightVsizeFees['350'] || 0, - vsize_400: weightVsizeFees['400'] || 0, - vsize_500: weightVsizeFees['500'] || 0, - vsize_600: weightVsizeFees['600'] || 0, - vsize_700: weightVsizeFees['700'] || 0, - vsize_800: weightVsizeFees['800'] || 0, - vsize_900: weightVsizeFees['900'] || 0, - vsize_1000: weightVsizeFees['1000'] || 0, - vsize_1200: weightVsizeFees['1200'] || 0, - vsize_1400: weightVsizeFees['1400'] || 0, - vsize_1600: weightVsizeFees['1600'] || 0, - vsize_1800: weightVsizeFees['1800'] || 0, - vsize_2000: weightVsizeFees['2000'] || 0, - }); - - if (this.newStatisticsEntryCallback && insertId) { - const newStats = await this.$get(insertId); - if (newStats) { - this.newStatisticsEntryCallback(newStats); - } - } - } catch (e) { - logger.err('Unable to insert statistics. ' + e); - } - } - - private async $createZeroedStatistic(): Promise { + public async $createZeroedStatistic(): Promise { try { const query = `INSERT INTO statistics( added, @@ -212,7 +63,7 @@ class Statistics { } } - private async $create(statistics: Statistic): Promise { + public async $create(statistics: Statistic): Promise { try { const query = `INSERT INTO statistics( added, @@ -413,7 +264,7 @@ class Statistics { ORDER BY statistics.added DESC;`; } - private async $get(id: number): Promise { + public async $get(id: number): Promise { try { const query = `SELECT *, UNIX_TIMESTAMP(added) as added FROM statistics WHERE id = ?`; const [rows] = await DB.query(query, [id]); @@ -574,7 +425,6 @@ class Statistics { }; }); } - } -export default new Statistics(); +export default new StatisticsApi(); diff --git a/backend/src/api/statistics/statistics.routes.ts b/backend/src/api/statistics/statistics.routes.ts new file mode 100644 index 000000000..4b1b91ce9 --- /dev/null +++ b/backend/src/api/statistics/statistics.routes.ts @@ -0,0 +1,67 @@ +import { Application, Request, Response } from 'express'; +import config from '../../config'; +import statisticsApi from './statistics-api'; + +class StatisticsRoutes { + public initRoutes(app: Application) { + app + .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/2h', this.$getStatisticsByTime.bind(this, '2h')) + .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/24h', this.$getStatisticsByTime.bind(this, '24h')) + .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/1w', this.$getStatisticsByTime.bind(this, '1w')) + .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/1m', this.$getStatisticsByTime.bind(this, '1m')) + .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/3m', this.$getStatisticsByTime.bind(this, '3m')) + .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/6m', this.$getStatisticsByTime.bind(this, '6m')) + .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/1y', this.$getStatisticsByTime.bind(this, '1y')) + .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/2y', this.$getStatisticsByTime.bind(this, '2y')) + .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/3y', this.$getStatisticsByTime.bind(this, '3y')) + ; + } + + private async $getStatisticsByTime(time: '2h' | '24h' | '1w' | '1m' | '3m' | '6m' | '1y' | '2y' | '3y', req: Request, res: Response) { + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString()); + + try { + let result; + switch (time as string) { + case '2h': + result = await statisticsApi.$list2H(); + res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString()); + break; + case '24h': + result = await statisticsApi.$list24H(); + res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); + break; + case '1w': + result = await statisticsApi.$list1W(); + break; + case '1m': + result = await statisticsApi.$list1M(); + break; + case '3m': + result = await statisticsApi.$list3M(); + break; + case '6m': + result = await statisticsApi.$list6M(); + break; + case '1y': + result = await statisticsApi.$list1Y(); + break; + case '2y': + result = await statisticsApi.$list2Y(); + break; + case '3y': + result = await statisticsApi.$list3Y(); + break; + default: + result = await statisticsApi.$list2H(); + } + res.json(result); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } +} + +export default new StatisticsRoutes(); diff --git a/backend/src/api/statistics/statistics.ts b/backend/src/api/statistics/statistics.ts new file mode 100644 index 000000000..27554f36d --- /dev/null +++ b/backend/src/api/statistics/statistics.ts @@ -0,0 +1,153 @@ +import memPool from '../mempool'; +import logger from '../../logger'; +import { TransactionExtended, OptimizedStatistic } from '../../mempool.interfaces'; +import { Common } from '../common'; +import statisticsApi from './statistics-api'; + +class Statistics { + protected intervalTimer: NodeJS.Timer | undefined; + protected newStatisticsEntryCallback: ((stats: OptimizedStatistic) => void) | undefined; + + public setNewStatisticsEntryCallback(fn: (stats: OptimizedStatistic) => void) { + this.newStatisticsEntryCallback = fn; + } + + public startStatistics(): void { + logger.info('Starting statistics service'); + + const now = new Date(); + const nextInterval = new Date(now.getFullYear(), now.getMonth(), now.getDate(), now.getHours(), + Math.floor(now.getMinutes() / 1) * 1 + 1, 0, 0); + const difference = nextInterval.getTime() - now.getTime(); + + setTimeout(() => { + this.runStatistics(); + this.intervalTimer = setInterval(() => { + this.runStatistics(); + }, 1 * 60 * 1000); + }, difference); + } + + private async runStatistics(): Promise { + if (!memPool.isInSync()) { + return; + } + const currentMempool = memPool.getMempool(); + const txPerSecond = memPool.getTxPerSecond(); + const vBytesPerSecond = memPool.getVBytesPerSecond(); + + logger.debug('Running statistics'); + + let memPoolArray: TransactionExtended[] = []; + for (const i in currentMempool) { + if (currentMempool.hasOwnProperty(i)) { + memPoolArray.push(currentMempool[i]); + } + } + // Remove 0 and undefined + memPoolArray = memPoolArray.filter((tx) => tx.effectiveFeePerVsize); + + if (!memPoolArray.length) { + try { + const insertIdZeroed = await statisticsApi.$createZeroedStatistic(); + if (this.newStatisticsEntryCallback && insertIdZeroed) { + const newStats = await statisticsApi.$get(insertIdZeroed); + if (newStats) { + this.newStatisticsEntryCallback(newStats); + } + } + } catch (e) { + logger.err('Unable to insert zeroed statistics. ' + e); + } + return; + } + + memPoolArray.sort((a, b) => a.effectiveFeePerVsize - b.effectiveFeePerVsize); + const totalWeight = memPoolArray.map((tx) => tx.vsize).reduce((acc, curr) => acc + curr) * 4; + const totalFee = memPoolArray.map((tx) => tx.fee).reduce((acc, curr) => acc + curr); + + const logFees = [1, 2, 3, 4, 5, 6, 8, 10, 12, 15, 20, 30, 40, 50, 60, 70, 80, 90, 100, 125, 150, 175, 200, + 250, 300, 350, 400, 500, 600, 700, 800, 900, 1000, 1200, 1400, 1600, 1800, 2000]; + + const weightVsizeFees: { [feePerWU: number]: number } = {}; + const lastItem = logFees.length - 1; + + memPoolArray.forEach((transaction) => { + for (let i = 0; i < logFees.length; i++) { + if ( + (Common.isLiquid() && (i === lastItem || transaction.effectiveFeePerVsize * 10 < logFees[i + 1])) + || + (!Common.isLiquid() && (i === lastItem || transaction.effectiveFeePerVsize < logFees[i + 1])) + ) { + if (weightVsizeFees[logFees[i]]) { + weightVsizeFees[logFees[i]] += transaction.vsize; + } else { + weightVsizeFees[logFees[i]] = transaction.vsize; + } + break; + } + } + }); + + try { + const insertId = await statisticsApi.$create({ + added: 'NOW()', + unconfirmed_transactions: memPoolArray.length, + tx_per_second: txPerSecond, + vbytes_per_second: Math.round(vBytesPerSecond), + mempool_byte_weight: totalWeight, + total_fee: totalFee, + fee_data: '', + vsize_1: weightVsizeFees['1'] || 0, + vsize_2: weightVsizeFees['2'] || 0, + vsize_3: weightVsizeFees['3'] || 0, + vsize_4: weightVsizeFees['4'] || 0, + vsize_5: weightVsizeFees['5'] || 0, + vsize_6: weightVsizeFees['6'] || 0, + vsize_8: weightVsizeFees['8'] || 0, + vsize_10: weightVsizeFees['10'] || 0, + vsize_12: weightVsizeFees['12'] || 0, + vsize_15: weightVsizeFees['15'] || 0, + vsize_20: weightVsizeFees['20'] || 0, + vsize_30: weightVsizeFees['30'] || 0, + vsize_40: weightVsizeFees['40'] || 0, + vsize_50: weightVsizeFees['50'] || 0, + vsize_60: weightVsizeFees['60'] || 0, + vsize_70: weightVsizeFees['70'] || 0, + vsize_80: weightVsizeFees['80'] || 0, + vsize_90: weightVsizeFees['90'] || 0, + vsize_100: weightVsizeFees['100'] || 0, + vsize_125: weightVsizeFees['125'] || 0, + vsize_150: weightVsizeFees['150'] || 0, + vsize_175: weightVsizeFees['175'] || 0, + vsize_200: weightVsizeFees['200'] || 0, + vsize_250: weightVsizeFees['250'] || 0, + vsize_300: weightVsizeFees['300'] || 0, + vsize_350: weightVsizeFees['350'] || 0, + vsize_400: weightVsizeFees['400'] || 0, + vsize_500: weightVsizeFees['500'] || 0, + vsize_600: weightVsizeFees['600'] || 0, + vsize_700: weightVsizeFees['700'] || 0, + vsize_800: weightVsizeFees['800'] || 0, + vsize_900: weightVsizeFees['900'] || 0, + vsize_1000: weightVsizeFees['1000'] || 0, + vsize_1200: weightVsizeFees['1200'] || 0, + vsize_1400: weightVsizeFees['1400'] || 0, + vsize_1600: weightVsizeFees['1600'] || 0, + vsize_1800: weightVsizeFees['1800'] || 0, + vsize_2000: weightVsizeFees['2000'] || 0, + }); + + if (this.newStatisticsEntryCallback && insertId) { + const newStats = await statisticsApi.$get(insertId); + if (newStats) { + this.newStatisticsEntryCallback(newStats); + } + } + } catch (e) { + logger.err('Unable to insert statistics. ' + e); + } + } +} + +export default new Statistics(); diff --git a/backend/src/config.ts b/backend/src/config.ts index 49892f064..bfd89d6a7 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -98,6 +98,11 @@ interface IConfig { BISQ_URL: string; BISQ_ONION: string; }; + MAXMIND: { + ENABLED: boolean; + GEOLITE2_CITY: string; + GEOLITE2_ASN: string; + }, } const defaults: IConfig = { @@ -197,7 +202,12 @@ const defaults: IConfig = { 'LIQUID_ONION': 'http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1', 'BISQ_URL': 'https://bisq.markets/api', 'BISQ_ONION': 'http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api' - } + }, + "MAXMIND": { + 'ENABLED': false, + "GEOLITE2_CITY": "/usr/local/share/GeoIP/GeoLite2-City.mmdb", + "GEOLITE2_ASN": "/usr/local/share/GeoIP/GeoLite2-ASN.mmdb" + }, }; class Config implements IConfig { @@ -215,6 +225,7 @@ class Config implements IConfig { SOCKS5PROXY: IConfig['SOCKS5PROXY']; PRICE_DATA_SERVER: IConfig['PRICE_DATA_SERVER']; EXTERNAL_DATA_SERVER: IConfig['EXTERNAL_DATA_SERVER']; + MAXMIND: IConfig['MAXMIND']; constructor() { const configs = this.merge(configFile, defaults); @@ -232,6 +243,7 @@ class Config implements IConfig { this.SOCKS5PROXY = configs.SOCKS5PROXY; this.PRICE_DATA_SERVER = configs.PRICE_DATA_SERVER; this.EXTERNAL_DATA_SERVER = configs.EXTERNAL_DATA_SERVER; + this.MAXMIND = configs.MAXMIND; } merge = (...objects: object[]): IConfig => { diff --git a/backend/src/index.ts b/backend/src/index.ts index af80c72cf..b7159afaf 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -3,15 +3,12 @@ import { Application, Request, Response, NextFunction } from 'express'; import * as http from 'http'; import * as WebSocket from 'ws'; import cluster from 'cluster'; -import axios from 'axios'; - import DB from './database'; import config from './config'; -import routes from './routes'; import blocks from './api/blocks'; import memPool from './api/mempool'; import diskCache from './api/disk-cache'; -import statistics from './api/statistics'; +import statistics from './api/statistics/statistics'; import websocketHandler from './api/websocket-handler'; import fiatConversion from './api/fiat-conversion'; import bisq from './api/bisq/bisq'; @@ -27,13 +24,16 @@ import icons from './api/liquid/icons'; import { Common } from './api/common'; import poolsUpdater from './tasks/pools-updater'; import indexer from './indexer'; -import priceUpdater from './tasks/price-updater'; import nodesRoutes from './api/explorer/nodes.routes'; import channelsRoutes from './api/explorer/channels.routes'; import generalLightningRoutes from './api/explorer/general.routes'; import lightningStatsUpdater from './tasks/lightning/stats-updater.service'; import nodeSyncService from './tasks/lightning/node-sync.service'; -import BlocksAuditsRepository from './repositories/BlocksAuditsRepository'; +import statisticsRoutes from "./api/statistics/statistics.routes"; +import miningRoutes from "./api/mining/mining-routes"; +import bisqRoutes from "./api/bisq/bisq.routes"; +import liquidRoutes from "./api/liquid/liquid.routes"; +import bitcoinRoutes from "./api/bitcoin/bitcoin.routes"; class Server { private wss: WebSocket.Server | undefined; @@ -165,7 +165,6 @@ class Server { await blocks.$updateBlocks(); await memPool.$updateMempool(); indexer.$run(); - priceUpdater.$run(); setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS); this.currentBackendRetryInterval = 5; @@ -206,174 +205,19 @@ class Server { } setUpHttpApiRoutes() { - this.app - .get(config.MEMPOOL.API_URL_PREFIX + 'transaction-times', routes.getTransactionTimes) - .get(config.MEMPOOL.API_URL_PREFIX + 'outspends', routes.$getBatchedOutspends) - .get(config.MEMPOOL.API_URL_PREFIX + 'cpfp/:txId', routes.getCpfpInfo) - .get(config.MEMPOOL.API_URL_PREFIX + 'difficulty-adjustment', routes.getDifficultyChange) - .get(config.MEMPOOL.API_URL_PREFIX + 'fees/recommended', routes.getRecommendedFees) - .get(config.MEMPOOL.API_URL_PREFIX + 'fees/mempool-blocks', routes.getMempoolBlocks) - .get(config.MEMPOOL.API_URL_PREFIX + 'backend-info', routes.getBackendInfo) - .get(config.MEMPOOL.API_URL_PREFIX + 'init-data', routes.getInitData) - .get(config.MEMPOOL.API_URL_PREFIX + 'validate-address/:address', routes.validateAddress) - .post(config.MEMPOOL.API_URL_PREFIX + 'tx/push', routes.$postTransactionForm) - .get(config.MEMPOOL.API_URL_PREFIX + 'donations', async (req, res) => { - try { - const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/donations`, { responseType: 'stream', timeout: 10000 }); - response.data.pipe(res); - } catch (e) { - res.status(500).end(); - } - }) - .get(config.MEMPOOL.API_URL_PREFIX + 'donations/images/:id', async (req, res) => { - try { - const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/donations/images/${req.params.id}`, { - responseType: 'stream', timeout: 10000 - }); - response.data.pipe(res); - } catch (e) { - res.status(500).end(); - } - }) - .get(config.MEMPOOL.API_URL_PREFIX + 'contributors', async (req, res) => { - try { - const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/contributors`, { responseType: 'stream', timeout: 10000 }); - response.data.pipe(res); - } catch (e) { - res.status(500).end(); - } - }) - .get(config.MEMPOOL.API_URL_PREFIX + 'contributors/images/:id', async (req, res) => { - try { - const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/contributors/images/${req.params.id}`, { - responseType: 'stream', timeout: 10000 - }); - response.data.pipe(res); - } catch (e) { - res.status(500).end(); - } - }) - .get(config.MEMPOOL.API_URL_PREFIX + 'translators', async (req, res) => { - try { - const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/translators`, { responseType: 'stream', timeout: 10000 }); - response.data.pipe(res); - } catch (e) { - res.status(500).end(); - } - }) - .get(config.MEMPOOL.API_URL_PREFIX + 'translators/images/:id', async (req, res) => { - try { - const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/translators/images/${req.params.id}`, { - responseType: 'stream', timeout: 10000 - }); - response.data.pipe(res); - } catch (e) { - res.status(500).end(); - } - }) - ; - + bitcoinRoutes.initRoutes(this.app); if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED) { - this.app - .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/2h', routes.$getStatisticsByTime.bind(routes, '2h')) - .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/24h', routes.$getStatisticsByTime.bind(routes, '24h')) - .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/1w', routes.$getStatisticsByTime.bind(routes, '1w')) - .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/1m', routes.$getStatisticsByTime.bind(routes, '1m')) - .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/3m', routes.$getStatisticsByTime.bind(routes, '3m')) - .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/6m', routes.$getStatisticsByTime.bind(routes, '6m')) - .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/1y', routes.$getStatisticsByTime.bind(routes, '1y')) - .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/2y', routes.$getStatisticsByTime.bind(routes, '2y')) - .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/3y', routes.$getStatisticsByTime.bind(routes, '3y')) - ; + statisticsRoutes.initRoutes(this.app); } - if (Common.indexingEnabled()) { - this.app - .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/:interval', routes.$getPools) - .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug/hashrate', routes.$getPoolHistoricalHashrate) - .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug/blocks', routes.$getPoolBlocks) - .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug/blocks/:height', routes.$getPoolBlocks) - .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug', routes.$getPool) - .get(config.MEMPOOL.API_URL_PREFIX + 'mining/hashrate/pools/:interval', routes.$getPoolsHistoricalHashrate) - .get(config.MEMPOOL.API_URL_PREFIX + 'mining/hashrate/:interval', routes.$getHistoricalHashrate) - .get(config.MEMPOOL.API_URL_PREFIX + 'mining/difficulty-adjustments', routes.$getDifficultyAdjustments) - .get(config.MEMPOOL.API_URL_PREFIX + 'mining/reward-stats/:blockCount', routes.$getRewardStats) - .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/fees/:interval', routes.$getHistoricalBlockFees) - .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/rewards/:interval', routes.$getHistoricalBlockRewards) - .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/fee-rates/:interval', routes.$getHistoricalBlockFeeRates) - .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/sizes-weights/:interval', routes.$getHistoricalBlockSizeAndWeight) - .get(config.MEMPOOL.API_URL_PREFIX + 'mining/difficulty-adjustments/:interval', routes.$getDifficultyAdjustments) - .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/predictions/:interval', routes.$getHistoricalBlockPrediction) - .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/:hash', routes.$getBlockAudit) - ; + miningRoutes.initRoutes(this.app); } - if (config.BISQ.ENABLED) { - this.app - .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/stats', routes.getBisqStats) - .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/tx/:txId', routes.getBisqTransaction) - .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/block/:hash', routes.getBisqBlock) - .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/blocks/tip/height', routes.getBisqTip) - .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/blocks/:index/:length', routes.getBisqBlocks) - .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/address/:address', routes.getBisqAddress) - .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/txs/:index/:length', routes.getBisqTransactions) - .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/currencies', routes.getBisqMarketCurrencies.bind(routes)) - .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/depth', routes.getBisqMarketDepth.bind(routes)) - .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/hloc', routes.getBisqMarketHloc.bind(routes)) - .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/markets', routes.getBisqMarketMarkets.bind(routes)) - .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/offers', routes.getBisqMarketOffers.bind(routes)) - .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/ticker', routes.getBisqMarketTicker.bind(routes)) - .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/trades', routes.getBisqMarketTrades.bind(routes)) - .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/volumes', routes.getBisqMarketVolumes.bind(routes)) - .get(config.MEMPOOL.API_URL_PREFIX + 'bisq/markets/volumes/7d', routes.getBisqMarketVolumes7d.bind(routes)) - ; + bisqRoutes.initRoutes(this.app); } - - this.app - .get(config.MEMPOOL.API_URL_PREFIX + 'blocks', routes.getBlocks.bind(routes)) - .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/:height', routes.getBlocks.bind(routes)) - .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash', routes.getBlock) - .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/summary', routes.getStrippedBlockTransactions); - - if (config.MEMPOOL.BACKEND !== 'esplora') { - this.app - .get(config.MEMPOOL.API_URL_PREFIX + 'mempool', routes.getMempool) - .get(config.MEMPOOL.API_URL_PREFIX + 'mempool/txids', routes.getMempoolTxIds) - .get(config.MEMPOOL.API_URL_PREFIX + 'mempool/recent', routes.getRecentMempoolTransactions) - .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId', routes.getTransaction) - .post(config.MEMPOOL.API_URL_PREFIX + 'tx', routes.$postTransaction) - .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/hex', routes.getRawTransaction) - .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/status', routes.getTransactionStatus) - .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/outspends', routes.getTransactionOutspends) - .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/header', routes.getBlockHeader) - .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/height', routes.getBlockTipHeight) - .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/hash', routes.getBlockTipHash) - .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txs', routes.getBlockTransactions) - .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txs/:index', routes.getBlockTransactions) - .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txids', routes.getTxIdsForBlock) - .get(config.MEMPOOL.API_URL_PREFIX + 'block-height/:height', routes.getBlockHeight) - .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address', routes.getAddress) - .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs', routes.getAddressTransactions) - .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs/chain/:txId', routes.getAddressTransactions) - .get(config.MEMPOOL.API_URL_PREFIX + 'address-prefix/:prefix', routes.getAddressPrefix) - ; - } - if (Common.isLiquid()) { - this.app - .get(config.MEMPOOL.API_URL_PREFIX + 'assets/icons', routes.getAllLiquidIcon) - .get(config.MEMPOOL.API_URL_PREFIX + 'assets/featured', routes.$getAllFeaturedLiquidAssets) - .get(config.MEMPOOL.API_URL_PREFIX + 'asset/:assetId/icon', routes.getLiquidIcon) - .get(config.MEMPOOL.API_URL_PREFIX + 'assets/group/:id', routes.$getAssetGroup) - ; + liquidRoutes.initRoutes(this.app); } - - if (Common.isLiquid() && config.DATABASE.ENABLED) { - this.app - .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/pegs/month', routes.$getElementsPegsByMonth) - ; - } - if (config.LIGHTNING.ENABLED) { generalLightningRoutes.initRoutes(this.app); nodesRoutes.initRoutes(this.app); diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index e34c43e8d..e452a42f4 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -1,10 +1,11 @@ import { Common } from './api/common'; import blocks from './api/blocks'; import mempool from './api/mempool'; -import mining from './api/mining'; +import mining from './api/mining/mining'; import logger from './logger'; import HashratesRepository from './repositories/HashratesRepository'; import bitcoinClient from './api/bitcoin/bitcoin-client'; +import priceUpdater from './tasks/price-updater'; class Indexer { runIndexer = true; @@ -38,6 +39,8 @@ class Indexer { logger.debug(`Running mining indexer`); try { + await priceUpdater.$run(); + const chainValid = await blocks.$generateBlockDatabase(); if (chainValid === false) { // Chain of block hash was invalid, so we need to reindex. Stop here and continue at the next iteration @@ -47,8 +50,9 @@ class Indexer { return; } + await mining.$indexBlockPrices(); await mining.$indexDifficultyAdjustments(); - await this.$resetHashratesIndexingState(); + await this.$resetHashratesIndexingState(); // TODO - Remove this as it's not efficient await mining.$generateNetworkHashrateHistory(); await mining.$generatePoolHashrateHistory(); await blocks.$generateBlocksSummariesDatabase(); diff --git a/backend/src/logger.ts b/backend/src/logger.ts index 43373e043..ea7e8cd3d 100644 --- a/backend/src/logger.ts +++ b/backend/src/logger.ts @@ -73,6 +73,9 @@ class Logger { } private getNetwork(): string { + if (config.LIGHTNING.ENABLED) { + return 'lightning'; + } if (config.BISQ.ENABLED) { return 'bisq'; } diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index 477c6a920..c2d2ee747 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -109,6 +109,7 @@ export interface BlockExtension { avgFee?: number; avgFeeRate?: number; coinbaseRaw?: string; + usd?: number | null; } export interface BlockExtended extends IEsploraApi.Block { @@ -120,6 +121,11 @@ export interface BlockSummary { transactions: TransactionStripped[]; } +export interface BlockPrice { + height: number; + priceId: number; +} + export interface TransactionMinerInfo { vin: VinStrippedToScriptsig[]; vout: VoutStrippedToScriptPubkey[]; diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 4388c46f6..40f670833 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -1,4 +1,4 @@ -import { BlockExtended } from '../mempool.interfaces'; +import { BlockExtended, BlockPrice } from '../mempool.interfaces'; import DB from '../database'; import logger from '../logger'; import { Common } from '../api/common'; @@ -256,7 +256,7 @@ class BlocksRepository { const params: any[] = []; let query = ` SELECT - height, + blocks.height, hash as id, UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp, size, @@ -308,7 +308,7 @@ class BlocksRepository { public async $getBlockByHeight(height: number): Promise { try { const [rows]: any[] = await DB.query(`SELECT - height, + blocks.height, hash, hash as id, UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp, @@ -336,7 +336,7 @@ class BlocksRepository { avg_fee_rate FROM blocks JOIN pools ON blocks.pool_id = pools.id - WHERE height = ${height}; + WHERE blocks.height = ${height} `); if (rows.length <= 0) { @@ -357,15 +357,15 @@ class BlocksRepository { public async $getBlockByHash(hash: string): Promise { try { const query = ` - SELECT *, UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp, hash as id, + SELECT *, blocks.height, UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp, hash as id, pools.id as pool_id, pools.name as pool_name, pools.link as pool_link, pools.slug as pool_slug, pools.addresses as pool_addresses, pools.regexes as pool_regexes, previous_block_hash as previousblockhash FROM blocks JOIN pools ON blocks.pool_id = pools.id - WHERE hash = '${hash}'; + WHERE hash = ?; `; - const [rows]: any[] = await DB.query(query); + const [rows]: any[] = await DB.query(query, [hash]); if (rows.length <= 0) { return null; @@ -387,7 +387,20 @@ class BlocksRepository { const [rows]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(blockTimestamp) as time, height, difficulty FROM blocks`); return rows; } catch (e) { - logger.err('Cannot generate difficulty history. Reason: ' + (e instanceof Error ? e.message : e)); + logger.err('Cannot get blocks difficulty list from the db. Reason: ' + (e instanceof Error ? e.message : e)); + throw e; + } + } + + /** + * Return blocks height + */ + public async $getBlocksHeightsAndTimestamp(): Promise { + try { + const [rows]: any[] = await DB.query(`SELECT height, blockTimestamp as timestamp FROM blocks`); + return rows; + } catch (e) { + logger.err('Cannot get blocks height and timestamp from the db. Reason: ' + (e instanceof Error ? e.message : e)); throw e; } } @@ -446,7 +459,7 @@ class BlocksRepository { ++idx; } - logger.info(`${idx} blocks hash validated in ${new Date().getTime() - start} ms`); + logger.debug(`${idx} blocks hash validated in ${new Date().getTime() - start} ms`); return true; } catch (e) { logger.err('Cannot validate chain of block hash. Reason: ' + (e instanceof Error ? e.message : e)); @@ -473,10 +486,14 @@ class BlocksRepository { public async $getHistoricalBlockFees(div: number, interval: string | null): Promise { try { let query = `SELECT - CAST(AVG(height) as INT) as avgHeight, + CAST(AVG(blocks.height) as INT) as avgHeight, CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp, - CAST(AVG(fees) as INT) as avgFees - FROM blocks`; + CAST(AVG(fees) as INT) as avgFees, + prices.USD + FROM blocks + JOIN blocks_prices on blocks_prices.height = blocks.height + JOIN prices on prices.id = blocks_prices.price_id + `; if (interval !== null) { query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`; @@ -498,10 +515,14 @@ class BlocksRepository { public async $getHistoricalBlockRewards(div: number, interval: string | null): Promise { try { let query = `SELECT - CAST(AVG(height) as INT) as avgHeight, + CAST(AVG(blocks.height) as INT) as avgHeight, CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp, - CAST(AVG(reward) as INT) as avgRewards - FROM blocks`; + CAST(AVG(reward) as INT) as avgRewards, + prices.USD + FROM blocks + JOIN blocks_prices on blocks_prices.height = blocks.height + JOIN prices on prices.id = blocks_prices.price_id + `; if (interval !== null) { query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`; @@ -628,6 +649,46 @@ class BlocksRepository { throw e; } } + + /** + * Get all blocks which have not be linked to a price yet + */ + public async $getBlocksWithoutPrice(): Promise { + try { + const [rows]: any[] = await DB.query(` + SELECT UNIX_TIMESTAMP(blocks.blockTimestamp) as timestamp, blocks.height + FROM blocks + LEFT JOIN blocks_prices ON blocks.height = blocks_prices.height + WHERE blocks_prices.height IS NULL + ORDER BY blocks.height + `); + return rows; + } catch (e) { + logger.err('Cannot get blocks height and timestamp from the db. Reason: ' + (e instanceof Error ? e.message : e)); + throw e; + } + } + + /** + * Save block price by batch + */ + public async $saveBlockPrices(blockPrices: BlockPrice[]): Promise { + try { + let query = `INSERT INTO blocks_prices(height, price_id) VALUES`; + for (const price of blockPrices) { + query += ` (${price.height}, ${price.priceId}),` + } + query = query.slice(0, -1); + await DB.query(query); + } catch (e: any) { + if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart + logger.debug(`Cannot save blocks prices for blocks [${blockPrices[0].height} to ${blockPrices[blockPrices.length - 1].height}] because it has already been indexed, ignoring`); + } else { + logger.err(`Cannot save blocks prices for blocks [${blockPrices[0].height} to ${blockPrices[blockPrices.length - 1].height}] into db. Reason: ` + (e instanceof Error ? e.message : e)); + throw e; + } + } + } } export default new BlocksRepository(); diff --git a/backend/src/repositories/PricesRepository.ts b/backend/src/repositories/PricesRepository.ts index a3b4f2fb9..92fb4860f 100644 --- a/backend/src/repositories/PricesRepository.ts +++ b/backend/src/repositories/PricesRepository.ts @@ -33,9 +33,14 @@ class PricesRepository { } public async $getPricesTimes(): Promise { - const [times]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1`); + const [times]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1 ORDER BY time`); return times.map(time => time.time); } + + public async $getPricesTimesAndId(): Promise { + const [times]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time, id, USD from prices ORDER BY time`); + return times; + } } export default new PricesRepository(); diff --git a/backend/src/routes.ts b/backend/src/routes.ts deleted file mode 100644 index 2257b4449..000000000 --- a/backend/src/routes.ts +++ /dev/null @@ -1,1105 +0,0 @@ -import config from './config'; -import { Request, Response } from 'express'; -import statistics from './api/statistics'; -import feeApi from './api/fee-api'; -import backendInfo from './api/backend-info'; -import mempoolBlocks from './api/mempool-blocks'; -import mempool from './api/mempool'; -import bisq from './api/bisq/bisq'; -import websocketHandler from './api/websocket-handler'; -import bisqMarket from './api/bisq/markets-api'; -import { RequiredSpec, TransactionExtended } from './mempool.interfaces'; -import { MarketsApiError } from './api/bisq/interfaces'; -import { IEsploraApi } from './api/bitcoin/esplora-api.interface'; -import logger from './logger'; -import bitcoinApi from './api/bitcoin/bitcoin-api-factory'; -import transactionUtils from './api/transaction-utils'; -import blocks from './api/blocks'; -import loadingIndicators from './api/loading-indicators'; -import { Common } from './api/common'; -import bitcoinClient from './api/bitcoin/bitcoin-client'; -import elementsParser from './api/liquid/elements-parser'; -import icons from './api/liquid/icons'; -import miningStats from './api/mining'; -import axios from 'axios'; -import mining from './api/mining'; -import BlocksRepository from './repositories/BlocksRepository'; -import HashratesRepository from './repositories/HashratesRepository'; -import difficultyAdjustment from './api/difficulty-adjustment'; -import DifficultyAdjustmentsRepository from './repositories/DifficultyAdjustmentsRepository'; -import BlocksAuditsRepository from './repositories/BlocksAuditsRepository'; - -class Routes { - constructor() {} - - public async $getStatisticsByTime(time: '2h' | '24h' | '1w' | '1m' | '3m' | '6m' | '1y' | '2y' | '3y', req: Request, res: Response) { - res.header('Pragma', 'public'); - res.header('Cache-control', 'public'); - res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString()); - - try { - let result; - switch (time as string) { - case '2h': - result = await statistics.$list2H(); - res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString()); - break; - case '24h': - result = await statistics.$list24H(); - res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); - break; - case '1w': - result = await statistics.$list1W(); - break; - case '1m': - result = await statistics.$list1M(); - break; - case '3m': - result = await statistics.$list3M(); - break; - case '6m': - result = await statistics.$list6M(); - break; - case '1y': - result = await statistics.$list1Y(); - break; - case '2y': - result = await statistics.$list2Y(); - break; - case '3y': - result = await statistics.$list3Y(); - break; - default: - result = await statistics.$list2H(); - } - res.json(result); - } catch (e) { - res.status(500).send(e instanceof Error ? e.message : e); - } - } - - public getInitData(req: Request, res: Response) { - try { - const result = websocketHandler.getInitData(); - res.json(result); - } catch (e) { - res.status(500).send(e instanceof Error ? e.message : e); - } - } - - public getRecommendedFees(req: Request, res: Response) { - if (!mempool.isInSync()) { - res.statusCode = 503; - res.send('Service Unavailable'); - return; - } - const result = feeApi.getRecommendedFee(); - res.json(result); - } - - public getMempoolBlocks(req: Request, res: Response) { - try { - const result = mempoolBlocks.getMempoolBlocks(); - res.json(result); - } catch (e) { - res.status(500).send(e instanceof Error ? e.message : e); - } - } - - public getTransactionTimes(req: Request, res: Response) { - if (!Array.isArray(req.query.txId)) { - res.status(500).send('Not an array'); - return; - } - const txIds: string[] = []; - for (const _txId in req.query.txId) { - if (typeof req.query.txId[_txId] === 'string') { - txIds.push(req.query.txId[_txId].toString()); - } - } - - const times = mempool.getFirstSeenForTransactions(txIds); - res.json(times); - } - - public async $getBatchedOutspends(req: Request, res: Response) { - if (!Array.isArray(req.query.txId)) { - res.status(500).send('Not an array'); - return; - } - if (req.query.txId.length > 50) { - res.status(400).send('Too many txids requested'); - return; - } - const txIds: string[] = []; - for (const _txId in req.query.txId) { - if (typeof req.query.txId[_txId] === 'string') { - txIds.push(req.query.txId[_txId].toString()); - } - } - - try { - const batchedOutspends = await bitcoinApi.$getBatchedOutspends(txIds); - res.json(batchedOutspends); - } catch (e) { - res.status(500).send(e instanceof Error ? e.message : e); - } - } - - public getCpfpInfo(req: Request, res: Response) { - if (!/^[a-fA-F0-9]{64}$/.test(req.params.txId)) { - res.status(501).send(`Invalid transaction ID.`); - return; - } - - const tx = mempool.getMempool()[req.params.txId]; - if (!tx) { - res.status(404).send(`Transaction doesn't exist in the mempool.`); - return; - } - - if (tx.cpfpChecked) { - res.json({ - ancestors: tx.ancestors, - bestDescendant: tx.bestDescendant || null, - }); - return; - } - - const cpfpInfo = Common.setRelativesAndGetCpfpInfo(tx, mempool.getMempool()); - - res.json(cpfpInfo); - } - - public getBackendInfo(req: Request, res: Response) { - res.json(backendInfo.getBackendInfo()); - } - - public getBisqStats(req: Request, res: Response) { - const result = bisq.getStats(); - res.json(result); - } - - public getBisqTip(req: Request, res: Response) { - const result = bisq.getLatestBlockHeight(); - res.type('text/plain'); - res.send(result.toString()); - } - - public getBisqTransaction(req: Request, res: Response) { - const result = bisq.getTransaction(req.params.txId); - if (result) { - res.json(result); - } else { - res.status(404).send('Bisq transaction not found'); - } - } - - public getBisqTransactions(req: Request, res: Response) { - const types: string[] = []; - req.query.types = req.query.types || []; - if (!Array.isArray(req.query.types)) { - res.status(500).send('Types is not an array'); - return; - } - - for (const _type in req.query.types) { - if (typeof req.query.types[_type] === 'string') { - types.push(req.query.types[_type].toString()); - } - } - - const index = parseInt(req.params.index, 10) || 0; - const length = parseInt(req.params.length, 10) > 100 ? 100 : parseInt(req.params.length, 10) || 25; - const [transactions, count] = bisq.getTransactions(index, length, types); - res.header('X-Total-Count', count.toString()); - res.json(transactions); - } - - public getBisqBlock(req: Request, res: Response) { - const result = bisq.getBlock(req.params.hash); - if (result) { - res.json(result); - } else { - res.status(404).send('Bisq block not found'); - } - } - - public getBisqBlocks(req: Request, res: Response) { - const index = parseInt(req.params.index, 10) || 0; - const length = parseInt(req.params.length, 10) > 100 ? 100 : parseInt(req.params.length, 10) || 25; - const [transactions, count] = bisq.getBlocks(index, length); - res.header('X-Total-Count', count.toString()); - res.json(transactions); - } - - public getBisqAddress(req: Request, res: Response) { - const result = bisq.getAddress(req.params.address.substr(1)); - if (result) { - res.json(result); - } else { - res.status(404).send('Bisq address not found'); - } - } - - public getBisqMarketCurrencies(req: Request, res: Response) { - const constraints: RequiredSpec = { - 'type': { - required: false, - types: ['crypto', 'fiat', 'all'] - }, - }; - - const p = this.parseRequestParameters(req.query, constraints); - if (p.error) { - res.status(400).json(this.getBisqMarketErrorResponse(p.error)); - return; - } - - const result = bisqMarket.getCurrencies(p.type); - if (result) { - res.json(result); - } else { - res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketCurrencies error')); - } - } - - public getBisqMarketDepth(req: Request, res: Response) { - const constraints: RequiredSpec = { - 'market': { - required: true, - types: ['@string'] - }, - }; - - const p = this.parseRequestParameters(req.query, constraints); - if (p.error) { - res.status(400).json(this.getBisqMarketErrorResponse(p.error)); - return; - } - - const result = bisqMarket.getDepth(p.market); - if (result) { - res.json(result); - } else { - res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketDepth error')); - } - } - - public getBisqMarketMarkets(req: Request, res: Response) { - const result = bisqMarket.getMarkets(); - if (result) { - res.json(result); - } else { - res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketMarkets error')); - } - } - - public getBisqMarketTrades(req: Request, res: Response) { - const constraints: RequiredSpec = { - 'market': { - required: true, - types: ['@string'] - }, - 'timestamp_from': { - required: false, - types: ['@number'] - }, - 'timestamp_to': { - required: false, - types: ['@number'] - }, - 'trade_id_to': { - required: false, - types: ['@string'] - }, - 'trade_id_from': { - required: false, - types: ['@string'] - }, - 'direction': { - required: false, - types: ['buy', 'sell'] - }, - 'limit': { - required: false, - types: ['@number'] - }, - 'sort': { - required: false, - types: ['asc', 'desc'] - } - }; - - const p = this.parseRequestParameters(req.query, constraints); - if (p.error) { - res.status(400).json(this.getBisqMarketErrorResponse(p.error)); - return; - } - - const result = bisqMarket.getTrades(p.market, p.timestamp_from, - p.timestamp_to, p.trade_id_from, p.trade_id_to, p.direction, p.limit, p.sort); - if (result) { - res.json(result); - } else { - res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketTrades error')); - } - } - - public getBisqMarketOffers(req: Request, res: Response) { - const constraints: RequiredSpec = { - 'market': { - required: true, - types: ['@string'] - }, - 'direction': { - required: false, - types: ['buy', 'sell'] - }, - }; - - const p = this.parseRequestParameters(req.query, constraints); - if (p.error) { - res.status(400).json(this.getBisqMarketErrorResponse(p.error)); - return; - } - - const result = bisqMarket.getOffers(p.market, p.direction); - if (result) { - res.json(result); - } else { - res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketOffers error')); - } - } - - public getBisqMarketVolumes(req: Request, res: Response) { - const constraints: RequiredSpec = { - 'market': { - required: false, - types: ['@string'] - }, - 'interval': { - required: false, - types: ['minute', 'half_hour', 'hour', 'half_day', 'day', 'week', 'month', 'year', 'auto'] - }, - 'timestamp_from': { - required: false, - types: ['@number'] - }, - 'timestamp_to': { - required: false, - types: ['@number'] - }, - 'milliseconds': { - required: false, - types: ['@boolean'] - }, - 'timestamp': { - required: false, - types: ['no', 'yes'] - }, - }; - - const p = this.parseRequestParameters(req.query, constraints); - if (p.error) { - res.status(400).json(this.getBisqMarketErrorResponse(p.error)); - return; - } - - const result = bisqMarket.getVolumes(p.market, p.timestamp_from, p.timestamp_to, p.interval, p.milliseconds, p.timestamp); - if (result) { - res.json(result); - } else { - res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketVolumes error')); - } - } - - public getBisqMarketHloc(req: Request, res: Response) { - const constraints: RequiredSpec = { - 'market': { - required: true, - types: ['@string'] - }, - 'interval': { - required: false, - types: ['minute', 'half_hour', 'hour', 'half_day', 'day', 'week', 'month', 'year', 'auto'] - }, - 'timestamp_from': { - required: false, - types: ['@number'] - }, - 'timestamp_to': { - required: false, - types: ['@number'] - }, - 'milliseconds': { - required: false, - types: ['@boolean'] - }, - 'timestamp': { - required: false, - types: ['no', 'yes'] - }, - }; - - const p = this.parseRequestParameters(req.query, constraints); - if (p.error) { - res.status(400).json(this.getBisqMarketErrorResponse(p.error)); - return; - } - - const result = bisqMarket.getHloc(p.market, p.interval, p.timestamp_from, p.timestamp_to, p.milliseconds, p.timestamp); - if (result) { - res.json(result); - } else { - res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketHloc error')); - } - } - - public getBisqMarketTicker(req: Request, res: Response) { - const constraints: RequiredSpec = { - 'market': { - required: false, - types: ['@string'] - }, - }; - - const p = this.parseRequestParameters(req.query, constraints); - if (p.error) { - res.status(400).json(this.getBisqMarketErrorResponse(p.error)); - return; - } - - const result = bisqMarket.getTicker(p.market); - if (result) { - res.json(result); - } else { - res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketTicker error')); - } - } - - public getBisqMarketVolumes7d(req: Request, res: Response) { - const result = bisqMarket.getVolumesByTime(604800); - if (result) { - res.json(result); - } else { - res.status(500).json(this.getBisqMarketErrorResponse('getBisqMarketVolumes7d error')); - } - } - - private parseRequestParameters(requestParams: object, params: RequiredSpec): { [name: string]: any; } { - const final = {}; - for (const i in params) { - if (params.hasOwnProperty(i)) { - if (params[i].required && requestParams[i] === undefined) { - return { error: i + ' parameter missing'}; - } - if (typeof requestParams[i] === 'string') { - const str = (requestParams[i] || '').toString().toLowerCase(); - if (params[i].types.indexOf('@number') > -1) { - const number = parseInt((str).toString(), 10); - final[i] = number; - } else if (params[i].types.indexOf('@string') > -1) { - final[i] = str; - } else if (params[i].types.indexOf('@boolean') > -1) { - final[i] = str === 'true' || str === 'yes'; - } else if (params[i].types.indexOf(str) > -1) { - final[i] = str; - } else { - return { error: i + ' parameter invalid'}; - } - } else if (typeof requestParams[i] === 'number') { - final[i] = requestParams[i]; - } - } - } - return final; - } - - private getBisqMarketErrorResponse(message: string): MarketsApiError { - return { - 'success': 0, - 'error': message - }; - } - - public async getTransaction(req: Request, res: Response) { - try { - const transaction = await transactionUtils.$getTransactionExtended(req.params.txId, true); - res.json(transaction); - } catch (e) { - let statusCode = 500; - if (e instanceof Error && e instanceof Error && e.message && e.message.indexOf('No such mempool or blockchain transaction') > -1) { - statusCode = 404; - } - res.status(statusCode).send(e instanceof Error ? e.message : e); - } - } - - public async getRawTransaction(req: Request, res: Response) { - try { - const transaction: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(req.params.txId, true); - res.setHeader('content-type', 'text/plain'); - res.send(transaction.hex); - } catch (e) { - let statusCode = 500; - if (e instanceof Error && e.message && e.message.indexOf('No such mempool or blockchain transaction') > -1) { - statusCode = 404; - } - res.status(statusCode).send(e instanceof Error ? e.message : e); - } - } - - public async getTransactionStatus(req: Request, res: Response) { - try { - const transaction = await transactionUtils.$getTransactionExtended(req.params.txId, true); - res.json(transaction.status); - } catch (e) { - let statusCode = 500; - if (e instanceof Error && e.message && e.message.indexOf('No such mempool or blockchain transaction') > -1) { - statusCode = 404; - } - res.status(statusCode).send(e instanceof Error ? e.message : e); - } - } - - public async $getPool(req: Request, res: Response) { - try { - const stats = await mining.$getPoolStat(req.params.slug); - res.header('Pragma', 'public'); - res.header('Cache-control', 'public'); - res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); - res.json(stats); - } catch (e) { - if (e instanceof Error && e.message.indexOf('This mining pool does not exist') > -1) { - res.status(404).send(e.message); - } else { - res.status(500).send(e instanceof Error ? e.message : e); - } - } - } - - public async $getPoolBlocks(req: Request, res: Response) { - try { - const poolBlocks = await BlocksRepository.$getBlocksByPool( - req.params.slug, - req.params.height === undefined ? undefined : parseInt(req.params.height, 10), - ); - res.header('Pragma', 'public'); - res.header('Cache-control', 'public'); - res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); - res.json(poolBlocks); - } catch (e) { - if (e instanceof Error && e.message.indexOf('This mining pool does not exist') > -1) { - res.status(404).send(e.message); - } else { - res.status(500).send(e instanceof Error ? e.message : e); - } - } - } - - public async $getPools(req: Request, res: Response) { - try { - const stats = await miningStats.$getPoolsStats(req.params.interval); - const blockCount = await BlocksRepository.$blockCount(null, null); - res.header('Pragma', 'public'); - res.header('Cache-control', 'public'); - res.header('X-total-count', blockCount.toString()); - res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); - res.json(stats); - } catch (e) { - res.status(500).send(e instanceof Error ? e.message : e); - } - } - - public async $getPoolsHistoricalHashrate(req: Request, res: Response) { - try { - const hashrates = await HashratesRepository.$getPoolsWeeklyHashrate(req.params.interval); - const blockCount = await BlocksRepository.$blockCount(null, null); - res.header('Pragma', 'public'); - res.header('Cache-control', 'public'); - res.header('X-total-count', blockCount.toString()); - res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString()); - res.json(hashrates); - } catch (e) { - res.status(500).send(e instanceof Error ? e.message : e); - } - } - - public async $getPoolHistoricalHashrate(req: Request, res: Response) { - try { - const hashrates = await HashratesRepository.$getPoolWeeklyHashrate(req.params.slug); - const blockCount = await BlocksRepository.$blockCount(null, null); - res.header('Pragma', 'public'); - res.header('Cache-control', 'public'); - res.header('X-total-count', blockCount.toString()); - res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString()); - res.json(hashrates); - } catch (e) { - if (e instanceof Error && e.message.indexOf('This mining pool does not exist') > -1) { - res.status(404).send(e.message); - } else { - res.status(500).send(e instanceof Error ? e.message : e); - } - } - } - - public async $getHistoricalHashrate(req: Request, res: Response) { - let currentHashrate = 0, currentDifficulty = 0; - try { - currentHashrate = await bitcoinClient.getNetworkHashPs(); - currentDifficulty = await bitcoinClient.getDifficulty(); - } catch (e) { - logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate and difficulty'); - } - - try { - const hashrates = await HashratesRepository.$getNetworkDailyHashrate(req.params.interval); - const difficulty = await DifficultyAdjustmentsRepository.$getAdjustments(req.params.interval, false); - const blockCount = await BlocksRepository.$blockCount(null, null); - res.header('Pragma', 'public'); - res.header('Cache-control', 'public'); - res.header('X-total-count', blockCount.toString()); - res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString()); - res.json({ - hashrates: hashrates, - difficulty: difficulty, - currentHashrate: currentHashrate, - currentDifficulty: currentDifficulty, - }); - } catch (e) { - res.status(500).send(e instanceof Error ? e.message : e); - } - } - - public async $getHistoricalBlockFees(req: Request, res: Response) { - try { - const blockFees = await mining.$getHistoricalBlockFees(req.params.interval); - const blockCount = await BlocksRepository.$blockCount(null, null); - res.header('Pragma', 'public'); - res.header('Cache-control', 'public'); - res.header('X-total-count', blockCount.toString()); - res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); - res.json(blockFees); - } catch (e) { - res.status(500).send(e instanceof Error ? e.message : e); - } - } - - public async $getHistoricalBlockRewards(req: Request, res: Response) { - try { - const blockRewards = await mining.$getHistoricalBlockRewards(req.params.interval); - const blockCount = await BlocksRepository.$blockCount(null, null); - res.header('Pragma', 'public'); - res.header('Cache-control', 'public'); - res.header('X-total-count', blockCount.toString()); - res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); - res.json(blockRewards); - } catch (e) { - res.status(500).send(e instanceof Error ? e.message : e); - } - } - - public async $getHistoricalBlockFeeRates(req: Request, res: Response) { - try { - const blockFeeRates = await mining.$getHistoricalBlockFeeRates(req.params.interval); - const blockCount = await BlocksRepository.$blockCount(null, null); - res.header('Pragma', 'public'); - res.header('Cache-control', 'public'); - res.header('X-total-count', blockCount.toString()); - res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); - res.json(blockFeeRates); - } catch (e) { - res.status(500).send(e instanceof Error ? e.message : e); - } - } - - public async $getHistoricalBlockSizeAndWeight(req: Request, res: Response) { - try { - const blockSizes = await mining.$getHistoricalBlockSizes(req.params.interval); - const blockWeights = await mining.$getHistoricalBlockWeights(req.params.interval); - const blockCount = await BlocksRepository.$blockCount(null, null); - res.header('Pragma', 'public'); - res.header('Cache-control', 'public'); - res.header('X-total-count', blockCount.toString()); - res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); - res.json({ - sizes: blockSizes, - weights: blockWeights - }); - } catch (e) { - res.status(500).send(e instanceof Error ? e.message : e); - } - } - - public async $getDifficultyAdjustments(req: Request, res: Response) { - try { - const difficulty = await DifficultyAdjustmentsRepository.$getRawAdjustments(req.params.interval, true); - res.header('Pragma', 'public'); - res.header('Cache-control', 'public'); - res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString()); - res.json(difficulty.map(adj => [adj.time, adj.height, adj.difficulty, adj.adjustment])); - } catch (e) { - res.status(500).send(e instanceof Error ? e.message : e); - } - } - - public async $getHistoricalBlockPrediction(req: Request, res: Response) { - try { - const blockPredictions = await mining.$getBlockPredictionsHistory(req.params.interval); - const blockCount = await BlocksAuditsRepository.$getPredictionsCount(); - res.header('Pragma', 'public'); - res.header('Cache-control', 'public'); - res.header('X-total-count', blockCount.toString()); - res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); - res.json(blockPredictions.map(prediction => [prediction.time, prediction.height, prediction.match_rate])); - } catch (e) { - res.status(500).send(e instanceof Error ? e.message : e); - } - } - - public async $getBlockAudit(req: Request, res: Response) { - try { - const audit = await BlocksAuditsRepository.$getBlockAudit(req.params.hash); - res.header('Pragma', 'public'); - res.header('Cache-control', 'public'); - res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24).toUTCString()); - res.json(audit); - } catch (e) { - res.status(500).send(e instanceof Error ? e.message : e); - } - } - - public async getBlock(req: Request, res: Response) { - try { - const block = await blocks.$getBlock(req.params.hash); - res.setHeader('Expires', new Date(Date.now() + 1000 * 600).toUTCString()); - res.json(block); - } catch (e) { - res.status(500).send(e instanceof Error ? e.message : e); - } - } - - public async getBlockHeader(req: Request, res: Response) { - try { - const blockHeader = await bitcoinApi.$getBlockHeader(req.params.hash); - res.setHeader('content-type', 'text/plain'); - res.send(blockHeader); - } catch (e) { - res.status(500).send(e instanceof Error ? e.message : e); - } - } - - public async getStrippedBlockTransactions(req: Request, res: Response) { - try { - const transactions = await blocks.$getStrippedBlockTransactions(req.params.hash); - res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24 * 30).toUTCString()); - res.json(transactions); - } catch (e) { - res.status(500).send(e instanceof Error ? e.message : e); - } - } - - public async getBlocks(req: Request, res: Response) { - try { - if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) { // Bitcoin - const height = req.params.height === undefined ? undefined : parseInt(req.params.height, 10); - res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); - res.json(await blocks.$getBlocks(height, 15)); - } else { // Liquid, Bisq - return await this.getLegacyBlocks(req, res); - } - } catch (e) { - res.status(500).send(e instanceof Error ? e.message : e); - } - } - - public async getLegacyBlocks(req: Request, res: Response) { - try { - const returnBlocks: IEsploraApi.Block[] = []; - const fromHeight = parseInt(req.params.height, 10) || blocks.getCurrentBlockHeight(); - - // Check if block height exist in local cache to skip the hash lookup - const blockByHeight = blocks.getBlocks().find((b) => b.height === fromHeight); - let startFromHash: string | null = null; - if (blockByHeight) { - startFromHash = blockByHeight.id; - } else { - startFromHash = await bitcoinApi.$getBlockHash(fromHeight); - } - - let nextHash = startFromHash; - for (let i = 0; i < 10 && nextHash; i++) { - const localBlock = blocks.getBlocks().find((b) => b.id === nextHash); - if (localBlock) { - returnBlocks.push(localBlock); - nextHash = localBlock.previousblockhash; - } else { - const block = await bitcoinApi.$getBlock(nextHash); - returnBlocks.push(block); - nextHash = block.previousblockhash; - } - } - - res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); - res.json(returnBlocks); - } catch (e) { - res.status(500).send(e instanceof Error ? e.message : e); - } - } - - public async getBlockTransactions(req: Request, res: Response) { - try { - loadingIndicators.setProgress('blocktxs-' + req.params.hash, 0); - - const txIds = await bitcoinApi.$getTxIdsForBlock(req.params.hash); - const transactions: TransactionExtended[] = []; - const startingIndex = Math.max(0, parseInt(req.params.index || '0', 10)); - - const endIndex = Math.min(startingIndex + 10, txIds.length); - for (let i = startingIndex; i < endIndex; i++) { - try { - const transaction = await transactionUtils.$getTransactionExtended(txIds[i], true, true); - transactions.push(transaction); - loadingIndicators.setProgress('blocktxs-' + req.params.hash, (i - startingIndex + 1) / (endIndex - startingIndex) * 100); - } catch (e) { - logger.debug('getBlockTransactions error: ' + (e instanceof Error ? e.message : e)); - } - } - res.json(transactions); - } catch (e) { - loadingIndicators.setProgress('blocktxs-' + req.params.hash, 100); - res.status(500).send(e instanceof Error ? e.message : e); - } - } - - public async getBlockHeight(req: Request, res: Response) { - try { - const blockHash = await bitcoinApi.$getBlockHash(parseInt(req.params.height, 10)); - res.send(blockHash); - } catch (e) { - res.status(500).send(e instanceof Error ? e.message : e); - } - } - - public async getAddress(req: Request, res: Response) { - if (config.MEMPOOL.BACKEND === 'none') { - res.status(405).send('Address lookups cannot be used with bitcoind as backend.'); - return; - } - - try { - const addressData = await bitcoinApi.$getAddress(req.params.address); - res.json(addressData); - } catch (e) { - if (e instanceof Error && e.message && (e.message.indexOf('too long') > 0 || e.message.indexOf('confirmed status') > 0)) { - return res.status(413).send(e instanceof Error ? e.message : e); - } - res.status(500).send(e instanceof Error ? e.message : e); - } - } - - public async getAddressTransactions(req: Request, res: Response) { - if (config.MEMPOOL.BACKEND === 'none') { - res.status(405).send('Address lookups cannot be used with bitcoind as backend.'); - return; - } - - try { - const transactions = await bitcoinApi.$getAddressTransactions(req.params.address, req.params.txId); - res.json(transactions); - } catch (e) { - if (e instanceof Error && e.message && (e.message.indexOf('too long') > 0 || e.message.indexOf('confirmed status') > 0)) { - return res.status(413).send(e instanceof Error ? e.message : e); - } - res.status(500).send(e instanceof Error ? e.message : e); - } - } - - public async getAdressTxChain(req: Request, res: Response) { - res.status(501).send('Not implemented'); - } - - public async getAddressPrefix(req: Request, res: Response) { - try { - const blockHash = await bitcoinApi.$getAddressPrefix(req.params.prefix); - res.send(blockHash); - } catch (e) { - res.status(500).send(e instanceof Error ? e.message : e); - } - } - - public async getRecentMempoolTransactions(req: Request, res: Response) { - const latestTransactions = Object.entries(mempool.getMempool()) - .sort((a, b) => (b[1].firstSeen || 0) - (a[1].firstSeen || 0)) - .slice(0, 10).map((tx) => Common.stripTransaction(tx[1])); - - res.json(latestTransactions); - } - - public async getMempool(req: Request, res: Response) { - const info = mempool.getMempoolInfo(); - res.json({ - count: info.size, - vsize: info.bytes, - total_fee: info.total_fee * 1e8, - fee_histogram: [] - }); - } - - public async getMempoolTxIds(req: Request, res: Response) { - try { - const rawMempool = await bitcoinApi.$getRawMempool(); - res.send(rawMempool); - } catch (e) { - res.status(500).send(e instanceof Error ? e.message : e); - } - } - - public async getBlockTipHeight(req: Request, res: Response) { - try { - const result = await bitcoinApi.$getBlockHeightTip(); - res.json(result); - } catch (e) { - res.status(500).send(e instanceof Error ? e.message : e); - } - } - - public async getBlockTipHash(req: Request, res: Response) { - try { - const result = await bitcoinApi.$getBlockHashTip(); - res.setHeader('content-type', 'text/plain'); - res.send(result); - } catch (e) { - res.status(500).send(e instanceof Error ? e.message : e); - } - } - - public async getTxIdsForBlock(req: Request, res: Response) { - try { - const result = await bitcoinApi.$getTxIdsForBlock(req.params.hash); - res.json(result); - } catch (e) { - res.status(500).send(e instanceof Error ? e.message : e); - } - } - - public async validateAddress(req: Request, res: Response) { - try { - const result = await bitcoinClient.validateAddress(req.params.address); - res.json(result); - } catch (e) { - res.status(500).send(e instanceof Error ? e.message : e); - } - } - - public async getTransactionOutspends(req: Request, res: Response) { - try { - const result = await bitcoinApi.$getOutspends(req.params.txId); - res.json(result); - } catch (e) { - res.status(500).send(e instanceof Error ? e.message : e); - } - } - - public getDifficultyChange(req: Request, res: Response) { - try { - res.json(difficultyAdjustment.getDifficultyAdjustment()); - } catch (e) { - res.status(500).send(e instanceof Error ? e.message : e); - } - } - - public async $getElementsPegsByMonth(req: Request, res: Response) { - try { - const pegs = await elementsParser.$getPegDataByMonth(); - res.json(pegs); - } catch (e) { - res.status(500).send(e instanceof Error ? e.message : e); - } - } - - public async $postTransaction(req: Request, res: Response) { - res.setHeader('content-type', 'text/plain'); - try { - let rawTx; - if (typeof req.body === 'object') { - rawTx = Object.keys(req.body)[0]; - } else { - rawTx = req.body; - } - const txIdResult = await bitcoinApi.$sendRawTransaction(rawTx); - res.send(txIdResult); - } catch (e: any) { - res.status(400).send(e.message && e.code ? 'sendrawtransaction RPC error: ' + JSON.stringify({ code: e.code, message: e.message }) - : (e.message || 'Error')); - } - } - - public async $postTransactionForm(req: Request, res: Response) { - res.setHeader('content-type', 'text/plain'); - const matches = /tx=([a-z0-9]+)/.exec(req.body); - let txHex = ''; - if (matches && matches[1]) { - txHex = matches[1]; - } - try { - const txIdResult = await bitcoinClient.sendRawTransaction(txHex); - res.send(txIdResult); - } catch (e: any) { - res.status(400).send(e.message && e.code ? 'sendrawtransaction RPC error: ' + JSON.stringify({ code: e.code, message: e.message }) - : (e.message || 'Error')); - } - } - - public getLiquidIcon(req: Request, res: Response) { - const result = icons.getIconByAssetId(req.params.assetId); - if (result) { - res.setHeader('content-type', 'image/png'); - res.setHeader('content-length', result.length); - res.send(result); - } else { - res.status(404).send('Asset icon not found'); - } - } - - public getAllLiquidIcon(req: Request, res: Response) { - const result = icons.getAllIconIds(); - if (result) { - res.json(result); - } else { - res.status(404).send('Asset icons not found'); - } - } - - public async $getAllFeaturedLiquidAssets(req: Request, res: Response) { - try { - const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.LIQUID_API}/assets/featured`, { responseType: 'stream', timeout: 10000 }); - response.data.pipe(res); - } catch (e) { - res.status(500).end(); - } - } - - public async $getAssetGroup(req: Request, res: Response) { - try { - const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.LIQUID_API}/assets/group/${parseInt(req.params.id, 10)}`, - { responseType: 'stream', timeout: 10000 }); - response.data.pipe(res); - } catch (e) { - res.status(500).end(); - } - } - - public async $getRewardStats(req: Request, res: Response) { - try { - const response = await mining.$getRewardStats(parseInt(req.params.blockCount, 10)); - res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); - res.json(response); - } catch (e) { - res.status(500).end(); - } - } -} - -export default new Routes(); diff --git a/backend/src/tasks/lightning/node-sync.service.ts b/backend/src/tasks/lightning/node-sync.service.ts index c5a6c8a9d..3b2eb18e2 100644 --- a/backend/src/tasks/lightning/node-sync.service.ts +++ b/backend/src/tasks/lightning/node-sync.service.ts @@ -8,6 +8,7 @@ import config from '../../config'; import { IEsploraApi } from '../../api/bitcoin/esplora-api.interface'; import lightningApi from '../../api/lightning/lightning-api-factory'; import { ILightningApi } from '../../api/lightning/lightning-api.interface'; +import { $lookupNodeLocation } from './sync-tasks/node-locations'; class NodeSyncService { constructor() {} @@ -33,6 +34,10 @@ class NodeSyncService { } logger.info(`Nodes updated.`); + if (config.MAXMIND.ENABLED) { + await $lookupNodeLocation(); + } + await this.$setChannelsInactive(); for (const channel of networkGraph.channels) { @@ -44,7 +49,9 @@ class NodeSyncService { await this.$lookUpCreationDateFromChain(); await this.$updateNodeFirstSeen(); await this.$scanForClosedChannels(); - await this.$runClosedChannelsForensics(); + if (config.MEMPOOL.BACKEND === 'esplora') { + await this.$runClosedChannelsForensics(); + } } catch (e) { logger.err('$updateNodes() error: ' + (e instanceof Error ? e.message : e)); diff --git a/backend/src/tasks/lightning/stats-updater.service.ts b/backend/src/tasks/lightning/stats-updater.service.ts index 01de7ede1..b44f4820c 100644 --- a/backend/src/tasks/lightning/stats-updater.service.ts +++ b/backend/src/tasks/lightning/stats-updater.service.ts @@ -6,7 +6,7 @@ import channelsApi from '../../api/explorer/channels.api'; import * as net from 'net'; class LightningStatsUpdater { - constructor() {} + hardCodedStartTime = '2018-01-12'; public async $startService() { logger.info('Starting Lightning Stats service'); @@ -28,17 +28,26 @@ class LightningStatsUpdater { return; } - const now = new Date(); - const nextHourInterval = new Date(now.getFullYear(), now.getMonth(), now.getDate(), Math.floor(now.getHours() / 1) + 1, 0, 0, 0); - const difference = nextHourInterval.getTime() - now.getTime(); + await this.$populateHistoricalStatistics(); + await this.$populateHistoricalNodeStatistics(); setTimeout(() => { - setInterval(async () => { - await this.$runTasks(); - }, 1000 * 60 * 60); - }, difference); + this.$runTasks(); + }, this.timeUntilMidnight()); + } - await this.$runTasks(); + private timeUntilMidnight(): number { + const date = new Date(); + this.setDateMidnight(date); + date.setUTCHours(24); + return date.getTime() - new Date().getTime(); + } + + private setDateMidnight(date: Date): void { + date.setUTCHours(0); + date.setUTCMinutes(0); + date.setUTCSeconds(0); + date.setUTCMilliseconds(0); } private async $lightningIsSynced(): Promise { @@ -46,161 +55,17 @@ class LightningStatsUpdater { return nodeInfo.is_synced_to_chain && nodeInfo.is_synced_to_graph; } - private async $runTasks() { - await this.$populateHistoricalData(); + private async $runTasks(): Promise { await this.$logLightningStatsDaily(); await this.$logNodeStatsDaily(); - } - private async $logNodeStatsDaily() { - const currentDate = new Date().toISOString().split('T')[0]; - try { - const [state]: any = await DB.query(`SELECT string FROM state WHERE name = 'last_node_stats'`); - // Only store once per day - if (state[0].string === currentDate) { - return; - } - - logger.info(`Running daily node stats update...`); - - const query = `SELECT nodes.public_key, c1.channels_count_left, c2.channels_count_right, c1.channels_capacity_left, c2.channels_capacity_right FROM nodes LEFT JOIN (SELECT node1_public_key, COUNT(id) AS channels_count_left, SUM(capacity) AS channels_capacity_left FROM channels WHERE channels.status < 2 GROUP BY node1_public_key) c1 ON c1.node1_public_key = nodes.public_key LEFT JOIN (SELECT node2_public_key, COUNT(id) AS channels_count_right, SUM(capacity) AS channels_capacity_right FROM channels WHERE channels.status < 2 GROUP BY node2_public_key) c2 ON c2.node2_public_key = nodes.public_key`; - const [nodes]: any = await DB.query(query); - - // First run we won't have any nodes yet - if (nodes.length < 10) { - return; - } - - for (const node of nodes) { - await DB.query( - `INSERT INTO node_stats(public_key, added, capacity, channels) VALUES (?, NOW(), ?, ?)`, - [node.public_key, (parseInt(node.channels_capacity_left || 0, 10)) + (parseInt(node.channels_capacity_right || 0, 10)), - node.channels_count_left + node.channels_count_right]); - } - await DB.query(`UPDATE state SET string = ? WHERE name = 'last_node_stats'`, [currentDate]); - logger.info('Daily node stats has updated.'); - } catch (e) { - logger.err('$logNodeStatsDaily() error: ' + (e instanceof Error ? e.message : e)); - } - } - - // We only run this on first launch - private async $populateHistoricalData() { - const startTime = '2018-01-13'; - try { - const [rows]: any = await DB.query(`SELECT COUNT(*) FROM lightning_stats`); - // Only store once per day - if (rows[0]['COUNT(*)'] > 0) { - return; - } - logger.info(`Running historical stats population...`); - - const [channels]: any = await DB.query(`SELECT capacity, created, closing_date FROM channels ORDER BY created ASC`); - - let date: Date = new Date(startTime); - const currentDate = new Date(); - - while (date < currentDate) { - let totalCapacity = 0; - let channelsCount = 0; - for (const channel of channels) { - if (new Date(channel.created) > date) { - break; - } - if (channel.closing_date !== null && new Date(channel.closing_date) < date) { - continue; - } - totalCapacity += channel.capacity; - channelsCount++; - } - - const query = `INSERT INTO lightning_stats( - added, - channel_count, - node_count, - total_capacity, - tor_nodes, - clearnet_nodes, - unannounced_nodes - ) - VALUES (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?)`; - - await DB.query(query, [ - date.getTime() / 1000, - channelsCount, - 0, - totalCapacity, - 0, - 0, - 0 - ]); - - // Add one day and continue - date.setDate(date.getDate() + 1); - } - - const [nodes]: any = await DB.query(`SELECT first_seen, sockets FROM nodes ORDER BY first_seen ASC`); - date = new Date(startTime); - - while (date < currentDate) { - let nodeCount = 0; - let clearnetNodes = 0; - let torNodes = 0; - let unannouncedNodes = 0; - for (const node of nodes) { - if (new Date(node.first_seen) > date) { - break; - } - nodeCount++; - - const sockets = node.sockets.split(','); - let isUnnanounced = true; - for (const socket of sockets) { - const hasOnion = socket.indexOf('.onion') !== -1; - if (hasOnion) { - torNodes++; - isUnnanounced = false; - } - const hasClearnet = [4, 6].includes(net.isIP(socket.split(':')[0])); - if (hasClearnet) { - clearnetNodes++; - isUnnanounced = false; - } - } - if (isUnnanounced) { - unannouncedNodes++; - } - } - - const query = `UPDATE lightning_stats SET node_count = ?, tor_nodes = ?, clearnet_nodes = ?, unannounced_nodes = ? WHERE added = FROM_UNIXTIME(?)`; - - await DB.query(query, [ - nodeCount, - torNodes, - clearnetNodes, - unannouncedNodes, - date.getTime() / 1000, - ]); - - // Add one day and continue - date.setDate(date.getDate() + 1); - } - - logger.info('Historical stats populated.'); - } catch (e) { - logger.err('$populateHistoricalData() error: ' + (e instanceof Error ? e.message : e)); - } + setTimeout(() => { + this.$runTasks(); + }, this.timeUntilMidnight()); } private async $logLightningStatsDaily() { - const currentDate = new Date().toISOString().split('T')[0]; try { - const [state]: any = await DB.query(`SELECT string FROM state WHERE name = 'last_node_stats'`); - // Only store once per day - if (state[0].string === currentDate) { - return; - } - logger.info(`Running lightning daily stats log...`); const networkGraph = await lightningApi.$getNetworkGraph(); @@ -250,7 +115,7 @@ class LightningStatsUpdater { med_fee_rate, med_base_fee_mtokens ) - VALUES (NOW(), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`; + VALUES (NOW() - INTERVAL 1 DAY, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`; await DB.query(query, [ networkGraph.channels.length, @@ -271,6 +136,184 @@ class LightningStatsUpdater { logger.err('$logLightningStatsDaily() error: ' + (e instanceof Error ? e.message : e)); } } + + private async $logNodeStatsDaily() { + try { + logger.info(`Running daily node stats update...`); + + const query = `SELECT nodes.public_key, c1.channels_count_left, c2.channels_count_right, c1.channels_capacity_left, c2.channels_capacity_right FROM nodes LEFT JOIN (SELECT node1_public_key, COUNT(id) AS channels_count_left, SUM(capacity) AS channels_capacity_left FROM channels WHERE channels.status < 2 GROUP BY node1_public_key) c1 ON c1.node1_public_key = nodes.public_key LEFT JOIN (SELECT node2_public_key, COUNT(id) AS channels_count_right, SUM(capacity) AS channels_capacity_right FROM channels WHERE channels.status < 2 GROUP BY node2_public_key) c2 ON c2.node2_public_key = nodes.public_key`; + const [nodes]: any = await DB.query(query); + + for (const node of nodes) { + await DB.query( + `INSERT INTO node_stats(public_key, added, capacity, channels) VALUES (?, NOW() - INTERVAL 1 DAY, ?, ?)`, + [node.public_key, (parseInt(node.channels_capacity_left || 0, 10)) + (parseInt(node.channels_capacity_right || 0, 10)), + node.channels_count_left + node.channels_count_right]); + } + logger.info('Daily node stats has updated.'); + } catch (e) { + logger.err('$logNodeStatsDaily() error: ' + (e instanceof Error ? e.message : e)); + } + } + + // We only run this on first launch + private async $populateHistoricalStatistics() { + try { + const [rows]: any = await DB.query(`SELECT COUNT(*) FROM lightning_stats`); + // Only run if table is empty + if (rows[0]['COUNT(*)'] > 0) { + return; + } + logger.info(`Running historical stats population...`); + + const [channels]: any = await DB.query(`SELECT capacity, created, closing_date FROM channels ORDER BY created ASC`); + const [nodes]: any = await DB.query(`SELECT first_seen, sockets FROM nodes ORDER BY first_seen ASC`); + + const date: Date = new Date(this.hardCodedStartTime); + const currentDate = new Date(); + this.setDateMidnight(currentDate); + + while (date < currentDate) { + let totalCapacity = 0; + let channelsCount = 0; + + for (const channel of channels) { + if (new Date(channel.created) > date) { + break; + } + if (channel.closing_date === null || new Date(channel.closing_date) > date) { + totalCapacity += channel.capacity; + channelsCount++; + } + } + + let nodeCount = 0; + let clearnetNodes = 0; + let torNodes = 0; + let unannouncedNodes = 0; + + for (const node of nodes) { + if (new Date(node.first_seen) > date) { + break; + } + nodeCount++; + + const sockets = node.sockets.split(','); + let isUnnanounced = true; + for (const socket of sockets) { + const hasOnion = socket.indexOf('.onion') !== -1; + if (hasOnion) { + torNodes++; + isUnnanounced = false; + } + const hasClearnet = [4, 6].includes(net.isIP(socket.substring(0, socket.lastIndexOf(':')))); + if (hasClearnet) { + clearnetNodes++; + isUnnanounced = false; + } + } + if (isUnnanounced) { + unannouncedNodes++; + } + } + + const query = `INSERT INTO lightning_stats( + added, + channel_count, + node_count, + total_capacity, + tor_nodes, + clearnet_nodes, + unannounced_nodes + ) + VALUES (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?)`; + + await DB.query(query, [ + date.getTime() / 1000, + channelsCount, + nodeCount, + totalCapacity, + torNodes, + clearnetNodes, + unannouncedNodes, + ]); + + date.setUTCDate(date.getUTCDate() + 1); + } + + logger.info('Historical stats populated.'); + } catch (e) { + logger.err('$populateHistoricalData() error: ' + (e instanceof Error ? e.message : e)); + } + } + + private async $populateHistoricalNodeStatistics() { + try { + const [rows]: any = await DB.query(`SELECT COUNT(*) FROM node_stats`); + // Only run if table is empty + if (rows[0]['COUNT(*)'] > 0) { + return; + } + logger.info(`Running historical node stats population...`); + + const [nodes]: any = await DB.query(`SELECT public_key, first_seen, alias FROM nodes ORDER BY first_seen ASC`); + + for (const node of nodes) { + const [channels]: any = await DB.query(`SELECT capacity, created, closing_date FROM channels WHERE node1_public_key = ? OR node2_public_key = ? ORDER BY created ASC`, [node.public_key, node.public_key]); + + const date: Date = new Date(this.hardCodedStartTime); + const currentDate = new Date(); + this.setDateMidnight(currentDate); + + let lastTotalCapacity = 0; + let lastChannelsCount = 0; + + while (date < currentDate) { + let totalCapacity = 0; + let channelsCount = 0; + for (const channel of channels) { + if (new Date(channel.created) > date) { + break; + } + if (channel.closing_date !== null && new Date(channel.closing_date) < date) { + date.setUTCDate(date.getUTCDate() + 1); + continue; + } + totalCapacity += channel.capacity; + channelsCount++; + } + + if (lastTotalCapacity === totalCapacity && lastChannelsCount === channelsCount) { + date.setUTCDate(date.getUTCDate() + 1); + continue; + } + + lastTotalCapacity = totalCapacity; + lastChannelsCount = channelsCount; + + const query = `INSERT INTO node_stats( + public_key, + added, + capacity, + channels + ) + VALUES (?, FROM_UNIXTIME(?), ?, ?)`; + + await DB.query(query, [ + node.public_key, + date.getTime() / 1000, + totalCapacity, + channelsCount, + ]); + date.setUTCDate(date.getUTCDate() + 1); + } + logger.debug('Updated node_stats for: ' + node.alias); + } + logger.info('Historical stats populated.'); + } catch (e) { + logger.err('$populateHistoricalNodeData() error: ' + (e instanceof Error ? e.message : e)); + } + } } export default new LightningStatsUpdater(); diff --git a/backend/src/tasks/lightning/sync-tasks/node-locations.ts b/backend/src/tasks/lightning/sync-tasks/node-locations.ts new file mode 100644 index 000000000..444bd6557 --- /dev/null +++ b/backend/src/tasks/lightning/sync-tasks/node-locations.ts @@ -0,0 +1,70 @@ +import * as net from 'net'; +import maxmind, { CityResponse, AsnResponse } from 'maxmind'; +import nodesApi from '../../../api/explorer/nodes.api'; +import config from '../../../config'; +import DB from '../../../database'; +import logger from '../../../logger'; + +export async function $lookupNodeLocation(): Promise { + logger.info(`Running node location updater using Maxmind...`); + try { + const nodes = await nodesApi.$getAllNodes(); + const lookupCity = await maxmind.open(config.MAXMIND.GEOLITE2_CITY); + const lookupAsn = await maxmind.open(config.MAXMIND.GEOLITE2_ASN); + + for (const node of nodes) { + const sockets: string[] = node.sockets.split(','); + for (const socket of sockets) { + const ip = socket.substring(0, socket.lastIndexOf(':')).replace('[', '').replace(']', ''); + const hasClearnet = [4, 6].includes(net.isIP(ip)); + if (hasClearnet && ip !== '127.0.1.1' && ip !== '127.0.0.1') { + const city = lookupCity.get(ip); + const asn = lookupAsn.get(ip); + if (city && asn) { + const query = `UPDATE nodes SET as_number = ?, city_id = ?, country_id = ?, subdivision_id = ?, longitude = ?, latitude = ?, accuracy_radius = ? WHERE public_key = ?`; + const params = [asn.autonomous_system_number, city.city?.geoname_id, city.country?.geoname_id, city.subdivisions ? city.subdivisions[0].geoname_id : null, city.location?.longitude, city.location?.latitude, city.location?.accuracy_radius, node.public_key]; + await DB.query(query, params); + + // Store Continent + if (city.continent?.geoname_id) { + await DB.query( + `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'continent', ?)`, + [city.continent?.geoname_id, JSON.stringify(city.continent?.names)]); + } + + // Store Country + if (city.country?.geoname_id) { + await DB.query( + `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'country', ?)`, + [city.country?.geoname_id, JSON.stringify(city.country?.names)]); + } + + // Store Division + if (city.subdivisions && city.subdivisions[0]) { + await DB.query( + `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'division', ?)`, + [city.subdivisions[0].geoname_id, JSON.stringify(city.subdivisions[0]?.names)]); + } + + // Store City + if (city.city?.geoname_id) { + await DB.query( + `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'city', ?)`, + [city.city?.geoname_id, JSON.stringify(city.city?.names)]); + } + + // Store AS name + if (asn.autonomous_system_organization) { + await DB.query( + `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'as_organization', ?)`, + [asn.autonomous_system_number, JSON.stringify(asn.autonomous_system_organization)]); + } + } + } + } + } + logger.info(`Node location data updated.`); + } catch (e) { + logger.err('$lookupNodeLocation() error: ' + (e instanceof Error ? e.message : e)); + } +} \ No newline at end of file diff --git a/backend/src/tasks/price-feeds/bitfinex-api.ts b/backend/src/tasks/price-feeds/bitfinex-api.ts index be3f5617b..04bd47732 100644 --- a/backend/src/tasks/price-feeds/bitfinex-api.ts +++ b/backend/src/tasks/price-feeds/bitfinex-api.ts @@ -16,7 +16,7 @@ class BitfinexApi implements PriceFeed { return response ? parseInt(response['last_price'], 10) : -1; } - public async $fetchRecentHourlyPrice(currencies: string[]): Promise { + public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise { const priceHistory: PriceHistory = {}; for (const currency of currencies) { @@ -24,7 +24,7 @@ class BitfinexApi implements PriceFeed { continue; } - const response = await query(this.urlHist.replace('{GRANULARITY}', '1h').replace('{CURRENCY}', currency)); + const response = await query(this.urlHist.replace('{GRANULARITY}', type === 'hour' ? '1h' : '1D').replace('{CURRENCY}', currency)); const pricesRaw = response ? response : []; for (const price of pricesRaw as any[]) { diff --git a/backend/src/tasks/price-feeds/bitflyer-api.ts b/backend/src/tasks/price-feeds/bitflyer-api.ts index d87661abb..143fbe8d9 100644 --- a/backend/src/tasks/price-feeds/bitflyer-api.ts +++ b/backend/src/tasks/price-feeds/bitflyer-api.ts @@ -16,7 +16,7 @@ class BitflyerApi implements PriceFeed { return response ? parseInt(response['ltp'], 10) : -1; } - public async $fetchRecentHourlyPrice(currencies: string[]): Promise { + public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise { return []; } } diff --git a/backend/src/tasks/price-feeds/coinbase-api.ts b/backend/src/tasks/price-feeds/coinbase-api.ts index b9abf860e..ef28b0d80 100644 --- a/backend/src/tasks/price-feeds/coinbase-api.ts +++ b/backend/src/tasks/price-feeds/coinbase-api.ts @@ -16,7 +16,7 @@ class CoinbaseApi implements PriceFeed { return response ? parseInt(response['data']['amount'], 10) : -1; } - public async $fetchRecentHourlyPrice(currencies: string[]): Promise { + public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise { const priceHistory: PriceHistory = {}; for (const currency of currencies) { @@ -24,7 +24,7 @@ class CoinbaseApi implements PriceFeed { continue; } - const response = await query(this.urlHist.replace('{GRANULARITY}', '3600').replace('{CURRENCY}', currency)); + const response = await query(this.urlHist.replace('{GRANULARITY}', type === 'hour' ? '3600' : '86400').replace('{CURRENCY}', currency)); const pricesRaw = response ? response : []; for (const price of pricesRaw as any[]) { diff --git a/backend/src/tasks/price-feeds/ftx-api.ts b/backend/src/tasks/price-feeds/ftx-api.ts index db58c8800..193d3e881 100644 --- a/backend/src/tasks/price-feeds/ftx-api.ts +++ b/backend/src/tasks/price-feeds/ftx-api.ts @@ -16,7 +16,7 @@ class FtxApi implements PriceFeed { return response ? parseInt(response['result']['last'], 10) : -1; } - public async $fetchRecentHourlyPrice(currencies: string[]): Promise { + public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise { const priceHistory: PriceHistory = {}; for (const currency of currencies) { @@ -24,7 +24,7 @@ class FtxApi implements PriceFeed { continue; } - const response = await query(this.urlHist.replace('{GRANULARITY}', '3600').replace('{CURRENCY}', currency)); + const response = await query(this.urlHist.replace('{GRANULARITY}', type === 'hour' ? '3600' : '86400').replace('{CURRENCY}', currency)); const pricesRaw = response ? response['result'] : []; for (const price of pricesRaw as any[]) { diff --git a/backend/src/tasks/price-feeds/gemini-api.ts b/backend/src/tasks/price-feeds/gemini-api.ts index 6b5742a7a..abd8e0939 100644 --- a/backend/src/tasks/price-feeds/gemini-api.ts +++ b/backend/src/tasks/price-feeds/gemini-api.ts @@ -16,7 +16,7 @@ class GeminiApi implements PriceFeed { return response ? parseInt(response['last'], 10) : -1; } - public async $fetchRecentHourlyPrice(currencies: string[]): Promise { + public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise { const priceHistory: PriceHistory = {}; for (const currency of currencies) { @@ -24,7 +24,7 @@ class GeminiApi implements PriceFeed { continue; } - const response = await query(this.urlHist.replace('{GRANULARITY}', '1hr').replace('{CURRENCY}', currency)); + const response = await query(this.urlHist.replace('{GRANULARITY}', type === 'hour' ? '1hr' : '1day').replace('{CURRENCY}', currency)); const pricesRaw = response ? response : []; for (const price of pricesRaw as any[]) { diff --git a/backend/src/tasks/price-feeds/kraken-api.ts b/backend/src/tasks/price-feeds/kraken-api.ts index 6c3cf93da..ce76d62c2 100644 --- a/backend/src/tasks/price-feeds/kraken-api.ts +++ b/backend/src/tasks/price-feeds/kraken-api.ts @@ -26,7 +26,7 @@ class KrakenApi implements PriceFeed { return response ? parseInt(response['result'][this.getTicker(currency)]['c'][0], 10) : -1; } - public async $fetchRecentHourlyPrice(currencies: string[]): Promise { + public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise { const priceHistory: PriceHistory = {}; for (const currency of currencies) { diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index caad6c54b..a5901d7f7 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -16,7 +16,7 @@ export interface PriceFeed { currencies: string[]; $fetchPrice(currency): Promise; - $fetchRecentHourlyPrice(currencies: string[]): Promise; + $fetchRecentPrice(currencies: string[], type: string): Promise; } export interface PriceHistory { @@ -177,13 +177,16 @@ class PriceUpdater { } if (insertedCount > 0) { logger.notice(`Inserted ${insertedCount} MtGox USD weekly price history into db`); + } else { + logger.debug(`Inserted ${insertedCount} MtGox USD weekly price history into db`); } // Insert Kraken weekly prices await new KrakenApi().$insertHistoricalPrice(); // Insert missing recent hourly prices - await this.$insertMissingRecentPrices(); + await this.$insertMissingRecentPrices('day'); + await this.$insertMissingRecentPrices('hour'); this.historyInserted = true; this.lastHistoricalRun = new Date().getTime(); @@ -193,17 +196,17 @@ class PriceUpdater { * Find missing hourly prices and insert them in the database * It has a limited backward range and it depends on which API are available */ - private async $insertMissingRecentPrices(): Promise { + private async $insertMissingRecentPrices(type: 'hour' | 'day'): Promise { const existingPriceTimes = await PricesRepository.$getPricesTimes(); - logger.info(`Fetching hourly price history from exchanges and saving missing ones into the database, this may take a while`); + logger.info(`Fetching ${type === 'day' ? 'dai' : 'hour'}ly price history from exchanges and saving missing ones into the database, this may take a while`); const historicalPrices: PriceHistory[] = []; // Fetch all historical hourly prices for (const feed of this.feeds) { try { - historicalPrices.push(await feed.$fetchRecentHourlyPrice(this.currencies)); + historicalPrices.push(await feed.$fetchRecentPrice(this.currencies, type)); } catch (e) { logger.err(`Cannot fetch hourly historical price from ${feed.name}. Ignoring this feed. Reason: ${e instanceof Error ? e.message : e}`); } @@ -250,7 +253,9 @@ class PriceUpdater { } if (totalInserted > 0) { - logger.notice(`Inserted ${totalInserted} hourly historical prices into the db`); + logger.notice(`Inserted ${totalInserted} ${type === 'day' ? 'dai' : 'hour'}ly historical prices into the db`); + } else { + logger.debug(`Inserted ${totalInserted} ${type === 'day' ? 'dai' : 'hour'}ly historical prices into the db`); } } } diff --git a/backend/src/utils/blocks-utils.ts b/backend/src/utils/blocks-utils.ts index 0f282bdeb..b933d6ae7 100644 --- a/backend/src/utils/blocks-utils.ts +++ b/backend/src/utils/blocks-utils.ts @@ -27,6 +27,7 @@ export function prepareBlock(block: any): BlockExtended { name: block.pool_name, slug: block.pool_slug, } : undefined), + usd: block?.extras?.usd ?? block.usd ?? null, } }; } diff --git a/frontend/src/app/app.module.ts b/frontend/src/app/app.module.ts index 8845f4255..8f95920f3 100644 --- a/frontend/src/app/app.module.ts +++ b/frontend/src/app/app.module.ts @@ -13,6 +13,7 @@ import { SharedModule } from './shared/shared.module'; import { StorageService } from './services/storage.service'; import { HttpCacheInterceptor } from './services/http-cache.interceptor'; import { LanguageService } from './services/language.service'; +import { FiatShortenerPipe } from './shared/pipes/fiat-shortener.pipe'; import { ShortenStringPipe } from './shared/pipes/shorten-string-pipe/shorten-string.pipe'; import { CapAddressPipe } from './shared/pipes/cap-address-pipe/cap-address-pipe'; @@ -37,6 +38,7 @@ import { CapAddressPipe } from './shared/pipes/cap-address-pipe/cap-address-pipe StorageService, LanguageService, ShortenStringPipe, + FiatShortenerPipe, CapAddressPipe, { provide: HTTP_INTERCEPTORS, useClass: HttpCacheInterceptor, multi: true } ], diff --git a/frontend/src/app/components/about/about.component.html b/frontend/src/app/components/about/about.component.html index 7fb796939..e09eddc78 100644 --- a/frontend/src/app/components/about/about.component.html +++ b/frontend/src/app/components/about/about.component.html @@ -2,7 +2,7 @@
™ - +
v{{ packetJsonVersion }} [{{ frontendGitCommitHash }}]
diff --git a/frontend/src/app/components/bisq-master-page/bisq-master-page.component.html b/frontend/src/app/components/bisq-master-page/bisq-master-page.component.html index b1735adee..dfd0f0695 100644 --- a/frontend/src/app/components/bisq-master-page/bisq-master-page.component.html +++ b/frontend/src/app/components/bisq-master-page/bisq-master-page.component.html @@ -2,7 +2,7 @@