From ca0c6b5e6e431ebfd27f1c31de61ffcaa0923c42 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Wed, 2 Aug 2023 13:24:56 +0900 Subject: [PATCH] use bulk mempool/txs post api to batch mempool update requests --- .../bitcoin/bitcoin-api-abstract-factory.ts | 3 +- backend/src/api/bitcoin/bitcoin-api.ts | 9 +++- backend/src/api/bitcoin/esplora-api.ts | 25 ++++++++- backend/src/api/mempool.ts | 54 ++++++++++--------- backend/src/api/transaction-utils.ts | 19 +++++++ 5 files changed, 82 insertions(+), 28 deletions(-) diff --git a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts index 7f4a5e53a..f14c5525d 100644 --- a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts +++ b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts @@ -3,7 +3,8 @@ import { IEsploraApi } from './esplora-api.interface'; export interface AbstractBitcoinApi { $getRawMempool(): Promise; $getRawTransaction(txId: string, skipConversion?: boolean, addPrevout?: boolean, lazyPrevouts?: boolean): Promise; - $getMempoolTransactions(lastTxid: string); + $getMempoolTransactions(txids: string[]): Promise; + $getAllMempoolTransactions(lastTxid: string); $getTransactionHex(txId: string): Promise; $getBlockHeightTip(): Promise; $getBlockHashTip(): Promise; diff --git a/backend/src/api/bitcoin/bitcoin-api.ts b/backend/src/api/bitcoin/bitcoin-api.ts index 132cda91a..b315ed0f7 100644 --- a/backend/src/api/bitcoin/bitcoin-api.ts +++ b/backend/src/api/bitcoin/bitcoin-api.ts @@ -60,8 +60,13 @@ class BitcoinApi implements AbstractBitcoinApi { }); } - $getMempoolTransactions(lastTxid: string): Promise { - return Promise.resolve([]); + $getMempoolTransactions(txids: string[]): Promise { + throw new Error('Method getMempoolTransactions not supported by the Bitcoin RPC API.'); + } + + $getAllMempoolTransactions(lastTxid: string): Promise { + throw new Error('Method getAllMempoolTransactions not supported by the Bitcoin RPC API.'); + } async $getTransactionHex(txId: string): Promise { diff --git a/backend/src/api/bitcoin/esplora-api.ts b/backend/src/api/bitcoin/esplora-api.ts index ff10751e0..77c6d80fc 100644 --- a/backend/src/api/bitcoin/esplora-api.ts +++ b/backend/src/api/bitcoin/esplora-api.ts @@ -61,6 +61,25 @@ class ElectrsApi implements AbstractBitcoinApi { }); } + $postWrapper(url, body, responseType = 'json', params: any = undefined): Promise { + return axiosConnection.post(url, body, { ...this.activeAxiosConfig, responseType: responseType, params }) + .then((response) => response.data) + .catch((e) => { + if (e?.code === 'ECONNREFUSED') { + this.fallbackToTcpSocket(); + // Retry immediately + return axiosConnection.post(url, body, this.activeAxiosConfig) + .then((response) => response.data) + .catch((e) => { + logger.warn(`Cannot query esplora through the unix socket nor the tcp socket. Exception ${e}`); + throw e; + }); + } else { + throw e; + } + }); + } + $getRawMempool(): Promise { return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/mempool/txids'); } @@ -69,7 +88,11 @@ class ElectrsApi implements AbstractBitcoinApi { return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/tx/' + txId); } - async $getMempoolTransactions(lastSeenTxid?: string): Promise { + async $getMempoolTransactions(txids: string[]): Promise { + return this.$postWrapper(config.ESPLORA.REST_API_URL + '/mempool/txs', txids, 'json'); + } + + async $getAllMempoolTransactions(lastSeenTxid?: string): Promise { return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/mempool/txs' + (lastSeenTxid ? '/' + lastSeenTxid : '')); } diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index 85b6e6101..ccdd307f8 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -123,7 +123,7 @@ class Mempool { loadingIndicators.setProgress('mempool', count / expectedCount * 100); while (!done) { try { - const result = await bitcoinApi.$getMempoolTransactions(last_txid); + const result = await bitcoinApi.$getAllMempoolTransactions(last_txid); if (result) { for (const tx of result) { const extendedTransaction = transactionUtils.extendMempoolTransaction(tx); @@ -231,31 +231,37 @@ class Mempool { } if (!loaded) { - for (const txid of transactions) { - if (!this.mempoolCache[txid]) { - try { - const transaction = await transactionUtils.$getMempoolTransactionExtended(txid, false, false, false); - this.updateTimerProgress(timer, 'fetched new transaction'); - this.mempoolCache[txid] = transaction; - if (this.inSync) { - this.txPerSecondArray.push(new Date().getTime()); - this.vBytesPerSecondArray.push({ - unixTime: new Date().getTime(), - vSize: transaction.vsize, - }); - } - hasChange = true; - newTransactions.push(transaction); + const remainingTxids = transactions.filter(txid => !this.mempoolCache[txid]); + const sliceLength = 10000; + for (let i = 0; i < Math.ceil(remainingTxids.length / sliceLength); i++) { + const slice = remainingTxids.slice(i * sliceLength, (i + 1) * sliceLength); + const txs = await transactionUtils.$getMempoolTransactionsExtended(slice, false, false, false); + logger.debug(`fetched ${txs.length} transactions`); + this.updateTimerProgress(timer, 'fetched new transactions'); - if (config.REDIS.ENABLED) { - await redisCache.$addTransaction(transaction); - } - } catch (e: any) { - if (config.MEMPOOL.BACKEND === 'esplora' && e.response?.status === 404) { - this.missingTxCount++; - } - logger.debug(`Error finding transaction '${txid}' in the mempool: ` + (e instanceof Error ? e.message : e)); + for (const transaction of txs) { + this.mempoolCache[transaction.txid] = transaction; + if (this.inSync) { + this.txPerSecondArray.push(new Date().getTime()); + this.vBytesPerSecondArray.push({ + unixTime: new Date().getTime(), + vSize: transaction.vsize, + }); } + hasChange = true; + newTransactions.push(transaction); + + if (config.REDIS.ENABLED) { + await redisCache.$addTransaction(transaction); + } + } + + if (txs.length < slice.length) { + const missing = slice.length - txs.length; + if (config.MEMPOOL.BACKEND === 'esplora') { + this.missingTxCount += missing; + } + logger.debug(`Error finding ${missing} transactions in the mempool: `); } if (Date.now() - intervalTimer > Math.max(pollRate * 2, 5_000)) { diff --git a/backend/src/api/transaction-utils.ts b/backend/src/api/transaction-utils.ts index e141a6076..02ee7c055 100644 --- a/backend/src/api/transaction-utils.ts +++ b/backend/src/api/transaction-utils.ts @@ -4,6 +4,7 @@ import { Common } from './common'; import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory'; import * as bitcoinjs from 'bitcoinjs-lib'; import logger from '../logger'; +import config from '../config'; class TransactionUtils { constructor() { } @@ -71,6 +72,24 @@ class TransactionUtils { return (await this.$getTransactionExtended(txId, addPrevouts, lazyPrevouts, forceCore, true)) as MempoolTransactionExtended; } + public async $getMempoolTransactionsExtended(txids: string[], addPrevouts = false, lazyPrevouts = false, forceCore = false): Promise { + if (forceCore || config.MEMPOOL.BACKEND !== 'esplora') { + const results = await Promise.allSettled(txids.map(txid => this.$getTransactionExtended(txid, addPrevouts, lazyPrevouts, forceCore, true))); + return (results.filter(r => r.status === 'fulfilled') as PromiseFulfilledResult[]).map(r => r.value); + } else { + const transactions = await bitcoinApi.$getMempoolTransactions(txids); + return transactions.map(transaction => { + if (Common.isLiquid()) { + if (!isFinite(Number(transaction.fee))) { + transaction.fee = Object.values(transaction.fee || {}).reduce((total, output) => total + output, 0); + } + } + + return this.extendMempoolTransaction(transaction); + }); + } + } + public extendTransaction(transaction: IEsploraApi.Transaction): TransactionExtended { // @ts-ignore if (transaction.vsize) {