From 641d2ad028cd1bc3bb0ba54a359e0480d48121e6 Mon Sep 17 00:00:00 2001 From: softsimon Date: Wed, 15 Sep 2021 01:47:24 +0400 Subject: [PATCH 1/4] Refactoring Bitcoin RPC client implementation --- backend/mempool-config.sample.json | 2 +- .../bitcoin/bitcoin-api-abstract-factory.ts | 7 +++ .../src/api/bitcoin/bitcoin-api-factory.ts | 5 +- backend/src/api/bitcoin/bitcoin-api.ts | 14 ++--- backend/src/api/bitcoin/bitcoin-base.api.ts | 53 ------------------- backend/src/api/bitcoin/bitcoin-client.ts | 13 +++++ .../src/api/bitcoin/bitcoin-second-client.ts | 13 +++++ backend/src/api/bitcoin/electrum-api.ts | 11 ++-- backend/src/api/blocks.ts | 4 +- backend/src/api/mempool.ts | 20 ++++++- backend/src/config.ts | 8 +-- backend/src/index.ts | 2 +- backend/src/routes.ts | 4 +- 13 files changed, 71 insertions(+), 85 deletions(-) delete mode 100644 backend/src/api/bitcoin/bitcoin-base.api.ts create mode 100644 backend/src/api/bitcoin/bitcoin-client.ts create mode 100644 backend/src/api/bitcoin/bitcoin-second-client.ts diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 00644c14c..72e5d28ae 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -28,7 +28,7 @@ "ESPLORA": { "REST_API_URL": "http://127.0.0.1:3000" }, - "CORE_RPC_MINFEE": { + "SECOND_CORE_RPC": { "ENABLED": false, "HOST": "127.0.0.1", "PORT": 8332, diff --git a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts index 906cc0f95..72c7150cc 100644 --- a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts +++ b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts @@ -12,3 +12,10 @@ export interface AbstractBitcoinApi { $getAddressTransactions(address: string, lastSeenTxId: string): Promise; $getAddressPrefix(prefix: string): string[]; } +export interface BitcoinRpcCredentials { + host: string; + port: number; + user: string; + pass: string; + timeout: number; +} diff --git a/backend/src/api/bitcoin/bitcoin-api-factory.ts b/backend/src/api/bitcoin/bitcoin-api-factory.ts index a16521a0e..f89d07b50 100644 --- a/backend/src/api/bitcoin/bitcoin-api-factory.ts +++ b/backend/src/api/bitcoin/bitcoin-api-factory.ts @@ -3,16 +3,17 @@ import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory'; import EsploraApi from './esplora-api'; import BitcoinApi from './bitcoin-api'; import ElectrumApi from './electrum-api'; +import bitcoinClient from './bitcoin-client'; function bitcoinApiFactory(): AbstractBitcoinApi { switch (config.MEMPOOL.BACKEND) { case 'esplora': return new EsploraApi(); case 'electrum': - return new ElectrumApi(); + return new ElectrumApi(bitcoinClient); case 'none': default: - return new BitcoinApi(); + return new BitcoinApi(bitcoinClient); } } diff --git a/backend/src/api/bitcoin/bitcoin-api.ts b/backend/src/api/bitcoin/bitcoin-api.ts index 2f0caa7f9..4ad5d5cd5 100644 --- a/backend/src/api/bitcoin/bitcoin-api.ts +++ b/backend/src/api/bitcoin/bitcoin-api.ts @@ -1,5 +1,3 @@ -import config from '../../config'; -import * as bitcoin from '@mempool/bitcoin'; import * as bitcoinjs from 'bitcoinjs-lib'; import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory'; import { IBitcoinApi } from './bitcoin-api.interface'; @@ -10,16 +8,10 @@ import { TransactionExtended } from '../../mempool.interfaces'; class BitcoinApi implements AbstractBitcoinApi { private rawMempoolCache: IBitcoinApi.RawMempool | null = null; - private bitcoindClient: any; + protected bitcoindClient: any; - constructor() { - this.bitcoindClient = new bitcoin.Client({ - host: config.CORE_RPC.HOST, - port: config.CORE_RPC.PORT, - user: config.CORE_RPC.USERNAME, - pass: config.CORE_RPC.PASSWORD, - timeout: 60000, - }); + constructor(bitcoinClient: any) { + this.bitcoindClient = bitcoinClient; } $getRawTransaction(txId: string, skipConversion = false, addPrevout = false): Promise { diff --git a/backend/src/api/bitcoin/bitcoin-base.api.ts b/backend/src/api/bitcoin/bitcoin-base.api.ts deleted file mode 100644 index c544e5225..000000000 --- a/backend/src/api/bitcoin/bitcoin-base.api.ts +++ /dev/null @@ -1,53 +0,0 @@ -import config from '../../config'; -import * as bitcoin from '@mempool/bitcoin'; -import { IBitcoinApi } from './bitcoin-api.interface'; - -class BitcoinBaseApi { - bitcoindClient: any; - bitcoindClientMempoolInfo: any; - - constructor() { - this.bitcoindClient = new bitcoin.Client({ - host: config.CORE_RPC.HOST, - port: config.CORE_RPC.PORT, - user: config.CORE_RPC.USERNAME, - pass: config.CORE_RPC.PASSWORD, - timeout: 60000, - }); - - if (config.CORE_RPC_MINFEE.ENABLED) { - this.bitcoindClientMempoolInfo = new bitcoin.Client({ - host: config.CORE_RPC_MINFEE.HOST, - port: config.CORE_RPC_MINFEE.PORT, - user: config.CORE_RPC_MINFEE.USERNAME, - pass: config.CORE_RPC_MINFEE.PASSWORD, - timeout: 60000, - }); - } - } - - $getMempoolInfo(): Promise { - if (config.CORE_RPC_MINFEE.ENABLED) { - return Promise.all([ - this.bitcoindClient.getMempoolInfo(), - this.bitcoindClientMempoolInfo.getMempoolInfo() - ]).then(([mempoolInfo, secondMempoolInfo]) => { - mempoolInfo.maxmempool = secondMempoolInfo.maxmempool; - mempoolInfo.mempoolminfee = secondMempoolInfo.mempoolminfee; - mempoolInfo.minrelaytxfee = secondMempoolInfo.minrelaytxfee; - return mempoolInfo; - }); - } - return this.bitcoindClient.getMempoolInfo(); - } - - $getBlockchainInfo(): Promise { - return this.bitcoindClient.getBlockchainInfo(); - } - - $validateAddress(address: string): Promise { - return this.bitcoindClient.validateAddress(address); - } -} - -export default new BitcoinBaseApi(); diff --git a/backend/src/api/bitcoin/bitcoin-client.ts b/backend/src/api/bitcoin/bitcoin-client.ts new file mode 100644 index 000000000..f4aa93bf2 --- /dev/null +++ b/backend/src/api/bitcoin/bitcoin-client.ts @@ -0,0 +1,13 @@ +import config from '../../config'; +import * as bitcoin from '@mempool/bitcoin'; +import { BitcoinRpcCredentials } from './bitcoin-api-abstract-factory'; + +const nodeRpcCredentials: BitcoinRpcCredentials = { + host: config.CORE_RPC.HOST, + port: config.CORE_RPC.PORT, + user: config.CORE_RPC.USERNAME, + pass: config.CORE_RPC.PASSWORD, + timeout: 60000, +}; + +export default new bitcoin.Client(nodeRpcCredentials); diff --git a/backend/src/api/bitcoin/bitcoin-second-client.ts b/backend/src/api/bitcoin/bitcoin-second-client.ts new file mode 100644 index 000000000..6fc00c7c6 --- /dev/null +++ b/backend/src/api/bitcoin/bitcoin-second-client.ts @@ -0,0 +1,13 @@ +import config from '../../config'; +import * as bitcoin from '@mempool/bitcoin'; +import { BitcoinRpcCredentials } from './bitcoin-api-abstract-factory'; + +const nodeRpcCredentials: BitcoinRpcCredentials = { + host: config.SECOND_CORE_RPC.HOST, + port: config.SECOND_CORE_RPC.PORT, + user: config.SECOND_CORE_RPC.USERNAME, + pass: config.SECOND_CORE_RPC.PASSWORD, + timeout: 60000, +}; + +export default new bitcoin.Client(nodeRpcCredentials); diff --git a/backend/src/api/bitcoin/electrum-api.ts b/backend/src/api/bitcoin/electrum-api.ts index 2f3d4c7eb..0ebadd3ef 100644 --- a/backend/src/api/bitcoin/electrum-api.ts +++ b/backend/src/api/bitcoin/electrum-api.ts @@ -1,23 +1,20 @@ import config from '../../config'; import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory'; -import { IBitcoinApi } from './bitcoin-api.interface'; import { IEsploraApi } from './esplora-api.interface'; import { IElectrumApi } from './electrum-api.interface'; import BitcoinApi from './bitcoin-api'; -import mempool from '../mempool'; import logger from '../../logger'; import * as ElectrumClient from '@mempool/electrum-client'; import * as sha256 from 'crypto-js/sha256'; import * as hexEnc from 'crypto-js/enc-hex'; import loadingIndicators from '../loading-indicators'; import memoryCache from '../memory-cache'; -import bitcoinBaseApi from './bitcoin-base.api'; class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi { private electrumClient: any; - constructor() { - super(); + constructor(bitcoinClient: any) { + super(bitcoinClient); const electrumConfig = { client: 'mempool-v2', version: '1.4' }; const electrumPersistencePolicy = { retryPeriod: 10000, maxRetry: 1000, callback: null }; @@ -45,7 +42,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi { } async $getAddress(address: string): Promise { - const addressInfo = await bitcoinBaseApi.$validateAddress(address); + const addressInfo = await this.bitcoindClient.validateAddress(address); if (!addressInfo || !addressInfo.isvalid) { return ({ 'address': address, @@ -99,7 +96,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi { } async $getAddressTransactions(address: string, lastSeenTxId: string): Promise { - const addressInfo = await bitcoinBaseApi.$validateAddress(address); + const addressInfo = await this.bitcoindClient.validateAddress(address); if (!addressInfo || !addressInfo.isvalid) { return []; } diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 785bc12eb..1043c344f 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -6,7 +6,7 @@ import { BlockExtended, TransactionExtended } from '../mempool.interfaces'; import { Common } from './common'; import diskCache from './disk-cache'; import transactionUtils from './transaction-utils'; -import bitcoinBaseApi from './bitcoin/bitcoin-base.api'; +import bitcoinClient from './bitcoin/bitcoin-client'; class Blocks { private blocks: BlockExtended[] = []; @@ -45,7 +45,7 @@ class Blocks { } if (!this.lastDifficultyAdjustmentTime) { - const blockchainInfo = await bitcoinBaseApi.$getBlockchainInfo(); + const blockchainInfo = await bitcoinClient.getBlockchainInfo(); if (blockchainInfo.blocks === blockchainInfo.headers) { const heightDiff = blockHeightTip % 2016; const blockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff); diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index 30a3ad356..e6bd13abb 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -5,8 +5,9 @@ import logger from '../logger'; import { Common } from './common'; import transactionUtils from './transaction-utils'; import { IBitcoinApi } from './bitcoin/bitcoin-api.interface'; -import bitcoinBaseApi from './bitcoin/bitcoin-base.api'; import loadingIndicators from './loading-indicators'; +import bitcoinClient from './bitcoin/bitcoin-client'; +import bitcoinSecondClient from './bitcoin/bitcoin-second-client'; class Mempool { private static WEBSOCKET_REFRESH_RATE_MS = 10000; @@ -61,7 +62,7 @@ class Mempool { } public async $updateMemPoolInfo() { - this.mempoolInfo = await bitcoinBaseApi.$getMempoolInfo(); + this.mempoolInfo = await this.$getMempoolInfo(); } public getMempoolInfo(): IBitcoinApi.MempoolInfo { @@ -205,6 +206,21 @@ class Mempool { } } } + + private $getMempoolInfo() { + if (config.SECOND_CORE_RPC.ENABLED) { + return Promise.all([ + bitcoinClient.getMempoolInfo(), + bitcoinSecondClient.getMempoolInfo() + ]).then(([mempoolInfo, secondMempoolInfo]) => { + mempoolInfo.maxmempool = secondMempoolInfo.maxmempool; + mempoolInfo.mempoolminfee = secondMempoolInfo.mempoolminfee; + mempoolInfo.minrelaytxfee = secondMempoolInfo.minrelaytxfee; + return mempoolInfo; + }); + } + return bitcoinClient.getMempoolInfo(); + } } export default new Mempool(); diff --git a/backend/src/config.ts b/backend/src/config.ts index f690dba58..47d38492d 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -30,7 +30,7 @@ interface IConfig { USERNAME: string; PASSWORD: string; }; - CORE_RPC_MINFEE: { + SECOND_CORE_RPC: { ENABLED: boolean; HOST: string; PORT: number; @@ -92,7 +92,7 @@ const defaults: IConfig = { 'USERNAME': 'mempool', 'PASSWORD': 'mempool' }, - 'CORE_RPC_MINFEE': { + 'SECOND_CORE_RPC': { 'ENABLED': false, 'HOST': '127.0.0.1', 'PORT': 8332, @@ -129,7 +129,7 @@ class Config implements IConfig { ESPLORA: IConfig['ESPLORA']; ELECTRUM: IConfig['ELECTRUM']; CORE_RPC: IConfig['CORE_RPC']; - CORE_RPC_MINFEE: IConfig['CORE_RPC_MINFEE']; + SECOND_CORE_RPC: IConfig['SECOND_CORE_RPC']; DATABASE: IConfig['DATABASE']; SYSLOG: IConfig['SYSLOG']; STATISTICS: IConfig['STATISTICS']; @@ -141,7 +141,7 @@ class Config implements IConfig { this.ESPLORA = configs.ESPLORA; this.ELECTRUM = configs.ELECTRUM; this.CORE_RPC = configs.CORE_RPC; - this.CORE_RPC_MINFEE = configs.CORE_RPC_MINFEE; + this.SECOND_CORE_RPC = configs.SECOND_CORE_RPC; this.DATABASE = configs.DATABASE; this.SYSLOG = configs.SYSLOG; this.STATISTICS = configs.STATISTICS; diff --git a/backend/src/index.ts b/backend/src/index.ts index bc2e47e0f..9756f343a 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -112,7 +112,7 @@ class Server { await memPool.$updateMemPoolInfo(); } catch (e) { const msg = `updateMempoolInfo: ${(e instanceof Error ? e.message : e)}`; - if (config.CORE_RPC_MINFEE.ENABLED) { + if (config.SECOND_CORE_RPC.ENABLED) { logger.warn(msg); } else { logger.debug(msg); diff --git a/backend/src/routes.ts b/backend/src/routes.ts index 849fe76b0..26d88eade 100644 --- a/backend/src/routes.ts +++ b/backend/src/routes.ts @@ -17,7 +17,7 @@ import transactionUtils from './api/transaction-utils'; import blocks from './api/blocks'; import loadingIndicators from './api/loading-indicators'; import { Common } from './api/common'; -import bitcoinBaseApi from './api/bitcoin/bitcoin-base.api'; +import bitcoinClient from './api/bitcoin/bitcoin-client'; class Routes { constructor() {} @@ -690,7 +690,7 @@ class Routes { public async validateAddress(req: Request, res: Response) { try { - const result = await bitcoinBaseApi.$validateAddress(req.params.address); + const result = await bitcoinClient.validateAddress(req.params.address); res.json(result); } catch (e) { res.status(500).send(e instanceof Error ? e.message : e); From aa39bbd0913fa9ed0645a606c5c87c355c77c3fc Mon Sep 17 00:00:00 2001 From: softsimon Date: Sat, 18 Sep 2021 13:37:25 +0400 Subject: [PATCH 2/4] Elements blockchain parser. Save all peg in/out i the database. --- .../src/api/bitcoin/bitcoin-api.interface.ts | 10 +- backend/src/api/common.ts | 2 + backend/src/api/liquid/elements-parser.ts | 105 ++++++++++++++++++ backend/src/api/websocket-handler.ts | 5 +- backend/src/index.ts | 16 +++ backend/src/routes.ts | 10 ++ mariadb-structure.sql | 20 ++++ 7 files changed, 162 insertions(+), 6 deletions(-) create mode 100644 backend/src/api/liquid/elements-parser.ts diff --git a/backend/src/api/bitcoin/bitcoin-api.interface.ts b/backend/src/api/bitcoin/bitcoin-api.interface.ts index 04c158211..ad603835a 100644 --- a/backend/src/api/bitcoin/bitcoin-api.interface.ts +++ b/backend/src/api/bitcoin/bitcoin-api.interface.ts @@ -72,7 +72,7 @@ export namespace IBitcoinApi { time: number; // (numeric) Same as blocktime } - interface Vin { + export interface Vin { txid?: string; // (string) The transaction id vout?: number; // (string) scriptSig?: { // (json object) The script @@ -82,18 +82,22 @@ export namespace IBitcoinApi { sequence: number; // (numeric) The script sequence number txinwitness?: string[]; // (string) hex-encoded witness data coinbase?: string; + is_pegin?: boolean; // (boolean) Elements peg-in } - interface Vout { + export interface Vout { value: number; // (numeric) The value in BTC n: number; // (numeric) index + asset?: string; // (string) Elements asset id scriptPubKey: { // (json object) asm: string; // (string) the asm hex: string; // (string) the hex reqSigs?: number; // (numeric) The required sigs type: string; // (string) The type, eg 'pubkeyhash' - addresses?: string[]; // (string) bitcoin addresses address?: string; // (string) bitcoin address + addresses?: string[]; // (string) bitcoin addresses + pegout_chain?: string; // (string) Elements peg-out chain + pegout_addresses?: string[]; // (string) Elements peg-out addresses }; } diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index 5c59582a1..0a4a210d1 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -1,6 +1,8 @@ import { CpfpInfo, TransactionExtended, TransactionStripped } from '../mempool.interfaces'; import config from '../config'; export class Common { + static nativeAssetId = '6f0279e9ed041c3d710a9f57d0c02928416460c4b722ae3457a11eec381c526d'; + static median(numbers: number[]) { let medianNr = 0; const numsLen = numbers.length; diff --git a/backend/src/api/liquid/elements-parser.ts b/backend/src/api/liquid/elements-parser.ts new file mode 100644 index 000000000..b2618a66e --- /dev/null +++ b/backend/src/api/liquid/elements-parser.ts @@ -0,0 +1,105 @@ +import { IBitcoinApi } from '../bitcoin/bitcoin-api.interface'; +import bitcoinClient from '../bitcoin/bitcoin-client'; +import bitcoinSecondClient from '../bitcoin/bitcoin-second-client'; +import { Common } from '../common'; +import { DB } from '../../database'; +import logger from '../../logger'; + +class ElementsParser { + isRunning = false; + constructor() { } + + public async $parse() { + if (this.isRunning) { + return; + } + this.isRunning = true; + const result = await bitcoinClient.getChainTips(); + const tip = result[0].height; + const latestBlock = await this.$getLatestBlockFromDatabase(); + for (let height = latestBlock.block + 1; height <= tip; height++) { + const blockHash: IBitcoinApi.ChainTips = await bitcoinClient.getBlockHash(height); + const block: IBitcoinApi.Block = await bitcoinClient.getBlock(blockHash, 2); + await this.$parseBlock(block); + await this.$saveLatestBlockToDatabase(block.height, block.time, block.hash); + } + this.isRunning = false; + } + + public async $getPegDataByMonth(): Promise { + const connection = await DB.pool.getConnection(); + const query = `SELECT SUM(amount) AS amount, DATE_FORMAT(FROM_UNIXTIME(datetime), '%Y-%m-01') AS date FROM elements_pegs GROUP BY DATE_FORMAT(FROM_UNIXTIME(datetime), '%Y%m')`; + const [rows] = await connection.query(query); + connection.release(); + return rows; + } + + protected async $parseBlock(block: IBitcoinApi.Block) { + for (const tx of block.tx) { + await this.$parseInputs(tx, block); + await this.$parseOutputs(tx, block); + } + } + + protected async $parseInputs(tx: IBitcoinApi.Transaction, block: IBitcoinApi.Block) { + for (const [index, input] of tx.vin.entries()) { + if (input.is_pegin) { + await this.$parsePegIn(input, index, tx.txid, block); + } + } + } + + protected async $parsePegIn(input: IBitcoinApi.Vin, vindex: number, txid: string, block: IBitcoinApi.Block) { + const bitcoinTx: IBitcoinApi.Transaction = await bitcoinSecondClient.getRawTransaction(input.txid, true); + const prevout = bitcoinTx.vout[input.vout || 0]; + const outputAddress = prevout.scriptPubKey.address || (prevout.scriptPubKey.addresses && prevout.scriptPubKey.addresses[0]) || ''; + await this.$savePegToDatabase(block.height, block.time, prevout.value * 100000000, txid, vindex, + outputAddress, bitcoinTx.txid, prevout.n, 1); + } + + protected async $parseOutputs(tx: IBitcoinApi.Transaction, block: IBitcoinApi.Block) { + for (const output of tx.vout) { + if (output.scriptPubKey.pegout_chain) { + await this.$savePegToDatabase(block.height, block.time, 0 - output.value * 100000000, tx.txid, output.n, + (output.scriptPubKey.pegout_addresses && output.scriptPubKey.pegout_addresses[0] || ''), '', 0, 0); + } + if (!output.scriptPubKey.pegout_chain && output.scriptPubKey.type === 'nulldata' + && output.value && output.value > 0 && output.asset && output.asset === Common.nativeAssetId) { + await this.$savePegToDatabase(block.height, block.time, 0 - output.value * 100000000, tx.txid, output.n, + (output.scriptPubKey.pegout_addresses && output.scriptPubKey.pegout_addresses[0] || ''), '', 0, 1); + } + } + } + + protected async $savePegToDatabase(height: number, blockTime: number, amount: number, txid: string, + txindex: number, bitcoinaddress: string, bitcointxid: string, bitcoinindex: number, final_tx: number): Promise { + const connection = await DB.pool.getConnection(); + const query = `INSERT INTO elements_pegs( + block, datetime, amount, txid, txindex, bitcoinaddress, bitcointxid, bitcoinindex, final_tx + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`; + + const params: (string | number)[] = [ + height, blockTime, amount, txid, txindex, bitcoinaddress, bitcointxid, bitcoinindex, final_tx + ]; + await connection.query(query, params); + connection.release(); + logger.debug(`Saved L-BTC peg from block height #${height} with TXID ${txid}.`); + } + + protected async $getLatestBlockFromDatabase(): Promise { + const connection = await DB.pool.getConnection(); + const query = `SELECT block, datetime, block_hash FROM last_elements_block`; + const [rows] = await connection.query(query); + connection.release(); + return rows[0]; + } + + protected async $saveLatestBlockToDatabase(blockHeight: number, datetime: number, blockHash: string) { + const connection = await DB.pool.getConnection(); + const query = `UPDATE last_elements_block SET block = ?, datetime = ?, block_hash = ?`; + await connection.query(query, [blockHeight, datetime, blockHash]); + connection.release(); + } +} + +export default new ElementsParser(); diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 29b3b7d8f..7891a606a 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -14,7 +14,6 @@ import transactionUtils from './transaction-utils'; class WebsocketHandler { private wss: WebSocket.Server | undefined; - private nativeAssetId = '6f0279e9ed041c3d710a9f57d0c02928416460c4b722ae3457a11eec381c526d'; private extraInitProperties = {}; constructor() { } @@ -308,7 +307,7 @@ class WebsocketHandler { newTransactions.forEach((tx) => { - if (client['track-asset'] === this.nativeAssetId) { + if (client['track-asset'] === Common.nativeAssetId) { if (tx.vin.some((vin) => !!vin.is_pegin)) { foundTransactions.push(tx); return; @@ -439,7 +438,7 @@ class WebsocketHandler { const foundTransactions: TransactionExtended[] = []; transactions.forEach((tx) => { - if (client['track-asset'] === this.nativeAssetId) { + if (client['track-asset'] === Common.nativeAssetId) { if (tx.vin && tx.vin.some((vin) => !!vin.is_pegin)) { foundTransactions.push(tx); return; diff --git a/backend/src/index.ts b/backend/src/index.ts index 9756f343a..a07903a7b 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -20,6 +20,7 @@ import logger from './logger'; import backendInfo from './api/backend-info'; import loadingIndicators from './api/loading-indicators'; import mempool from './api/mempool'; +import elementsParser from './api/liquid/elements-parser'; class Server { private wss: WebSocket.Server | undefined; @@ -141,6 +142,15 @@ class Server { if (this.wss) { websocketHandler.setWebsocketServer(this.wss); } + if (config.MEMPOOL.NETWORK === 'liquid') { + blocks.setNewBlockCallback(async () => { + try { + await elementsParser.$parse(); + } catch (e) { + logger.warn('Elements parsing error: ' + (e instanceof Error ? e.message : e)); + } + }); + } websocketHandler.setupConnectionHandling(); statistics.setNewStatisticsEntryCallback(websocketHandler.handleNewStatistic.bind(websocketHandler)); blocks.setNewBlockCallback(websocketHandler.handleNewBlock.bind(websocketHandler)); @@ -254,6 +264,12 @@ class Server { .get(config.MEMPOOL.API_URL_PREFIX + 'address-prefix/:prefix', routes.getAddressPrefix) ; } + + if (config.MEMPOOL.NETWORK === 'liquid') { + this.app + .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/pegs/month', routes.$getElementsPegsByMonth) + ; + } } } diff --git a/backend/src/routes.ts b/backend/src/routes.ts index 26d88eade..8db3e4b89 100644 --- a/backend/src/routes.ts +++ b/backend/src/routes.ts @@ -18,6 +18,7 @@ import blocks from './api/blocks'; import loadingIndicators from './api/loading-indicators'; import { Common } from './api/common'; import bitcoinClient from './api/bitcoin/bitcoin-client'; +import elementsParser from './api/liquid/elements-parser'; class Routes { constructor() {} @@ -754,6 +755,15 @@ class Routes { res.status(500).send(e instanceof Error ? e.message : e); } } + + public async $getElementsPegsByMonth(req: Request, res: Response) { + try { + const pegs = await elementsParser.$getPegDataByMonth(); + res.json(pegs); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } } export default new Routes(); diff --git a/mariadb-structure.sql b/mariadb-structure.sql index 4d567ed91..f652a0a54 100644 --- a/mariadb-structure.sql +++ b/mariadb-structure.sql @@ -84,3 +84,23 @@ ALTER TABLE `transactions` ALTER TABLE `statistics` MODIFY `id` int(11) NOT NULL AUTO_INCREMENT; + +CREATE TABLE `last_elements_block` ( + `block` int(11) NOT NULL, + `datetime` int(11) NOT NULL, + `block_hash` varchar(65) NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +INSERT INTO `last_elements_block` VALUES(0, 0, ''); + +CREATE TABLE `elements_pegs` ( + `block` int(11) NOT NULL, + `datetime` int(11) NOT NULL, + `amount` bigint(20) NOT NULL, + `txid` varchar(65) NOT NULL, + `txindex` int(11) NOT NULL, + `bitcoinaddress` varchar(100) NOT NULL, + `bitcointxid` varchar(65) NOT NULL, + `bitcoinindex` int(11) NOT NULL, + `final_tx` int(11) NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8; From 0d67bc36eeadb7e1bbbaf5c7d8fb892971afbd68 Mon Sep 17 00:00:00 2001 From: softsimon Date: Sun, 19 Sep 2021 02:40:16 +0400 Subject: [PATCH 3/4] Refactoring the MINFEE node configuration into new configs. --- backend/mempool-config.sample.json | 4 ++-- backend/src/api/mempool.ts | 2 +- backend/src/config.ts | 4 ++-- backend/src/index.ts | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 72e5d28ae..6ff103bf1 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -12,7 +12,8 @@ "BLOCK_WEIGHT_UNITS": 4000000, "INITIAL_BLOCKS_AMOUNT": 8, "MEMPOOL_BLOCKS_AMOUNT": 8, - "PRICE_FEED_UPDATE_INTERVAL": 3600 + "PRICE_FEED_UPDATE_INTERVAL": 3600, + "USE_SECOND_NODE_FOR_MINFEE": false }, "CORE_RPC": { "HOST": "127.0.0.1", @@ -29,7 +30,6 @@ "REST_API_URL": "http://127.0.0.1:3000" }, "SECOND_CORE_RPC": { - "ENABLED": false, "HOST": "127.0.0.1", "PORT": 8332, "USERNAME": "mempool", diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index e6bd13abb..4d6c35860 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -208,7 +208,7 @@ class Mempool { } private $getMempoolInfo() { - if (config.SECOND_CORE_RPC.ENABLED) { + if (config.MEMPOOL.USE_SECOND_NODE_FOR_MINFEE) { return Promise.all([ bitcoinClient.getMempoolInfo(), bitcoinSecondClient.getMempoolInfo() diff --git a/backend/src/config.ts b/backend/src/config.ts index 47d38492d..2addc176a 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -15,6 +15,7 @@ interface IConfig { INITIAL_BLOCKS_AMOUNT: number; MEMPOOL_BLOCKS_AMOUNT: number; PRICE_FEED_UPDATE_INTERVAL: number; + USE_SECOND_NODE_FOR_MINFEE: boolean; }; ESPLORA: { REST_API_URL: string; @@ -31,7 +32,6 @@ interface IConfig { PASSWORD: string; }; SECOND_CORE_RPC: { - ENABLED: boolean; HOST: string; PORT: number; USERNAME: string; @@ -77,6 +77,7 @@ const defaults: IConfig = { 'INITIAL_BLOCKS_AMOUNT': 8, 'MEMPOOL_BLOCKS_AMOUNT': 8, 'PRICE_FEED_UPDATE_INTERVAL': 3600, + 'USE_SECOND_NODE_FOR_MINFEE': false, }, 'ESPLORA': { 'REST_API_URL': 'http://127.0.0.1:3000', @@ -93,7 +94,6 @@ const defaults: IConfig = { 'PASSWORD': 'mempool' }, 'SECOND_CORE_RPC': { - 'ENABLED': false, 'HOST': '127.0.0.1', 'PORT': 8332, 'USERNAME': 'mempool', diff --git a/backend/src/index.ts b/backend/src/index.ts index a07903a7b..cfca7cc65 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -113,7 +113,7 @@ class Server { await memPool.$updateMemPoolInfo(); } catch (e) { const msg = `updateMempoolInfo: ${(e instanceof Error ? e.message : e)}`; - if (config.SECOND_CORE_RPC.ENABLED) { + if (config.MEMPOOL.USE_SECOND_NODE_FOR_MINFEE) { logger.warn(msg); } else { logger.debug(msg); From f3c8e2134b6a841e20f2f9de0ad93cfc0f339578 Mon Sep 17 00:00:00 2001 From: softsimon Date: Mon, 20 Sep 2021 01:02:07 +0400 Subject: [PATCH 4/4] Handle errors gracefully. --- backend/src/api/liquid/elements-parser.ts | 28 ++++++++++++++--------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/backend/src/api/liquid/elements-parser.ts b/backend/src/api/liquid/elements-parser.ts index b2618a66e..37e2362e1 100644 --- a/backend/src/api/liquid/elements-parser.ts +++ b/backend/src/api/liquid/elements-parser.ts @@ -6,24 +6,30 @@ import { DB } from '../../database'; import logger from '../../logger'; class ElementsParser { - isRunning = false; + private isRunning = false; + constructor() { } public async $parse() { if (this.isRunning) { return; } - this.isRunning = true; - const result = await bitcoinClient.getChainTips(); - const tip = result[0].height; - const latestBlock = await this.$getLatestBlockFromDatabase(); - for (let height = latestBlock.block + 1; height <= tip; height++) { - const blockHash: IBitcoinApi.ChainTips = await bitcoinClient.getBlockHash(height); - const block: IBitcoinApi.Block = await bitcoinClient.getBlock(blockHash, 2); - await this.$parseBlock(block); - await this.$saveLatestBlockToDatabase(block.height, block.time, block.hash); + try { + this.isRunning = true; + const result = await bitcoinClient.getChainTips(); + const tip = result[0].height; + const latestBlock = await this.$getLatestBlockFromDatabase(); + for (let height = latestBlock.block + 1; height <= tip; height++) { + const blockHash: IBitcoinApi.ChainTips = await bitcoinClient.getBlockHash(height); + const block: IBitcoinApi.Block = await bitcoinClient.getBlock(blockHash, 2); + await this.$parseBlock(block); + await this.$saveLatestBlockToDatabase(block.height, block.time, block.hash); + } + this.isRunning = false; + } catch (e) { + this.isRunning = false; + throw new Error(e instanceof Error ? e.message : 'Error'); } - this.isRunning = false; } public async $getPegDataByMonth(): Promise {