From 202d4122b4d811dab03ab9eaf0b79c4df6b64be0 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sat, 22 Jul 2023 14:09:11 +0900 Subject: [PATCH 1/7] load mempool txs in bulk from esplora --- .../bitcoin/bitcoin-api-abstract-factory.ts | 1 + backend/src/api/bitcoin/bitcoin-api.ts | 4 + backend/src/api/bitcoin/esplora-api.ts | 23 +++++ backend/src/api/mempool.ts | 93 ++++++++++++------- 4 files changed, 86 insertions(+), 35 deletions(-) diff --git a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts index c233ed5d7..d195b0eeb 100644 --- a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts +++ b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts @@ -3,6 +3,7 @@ import { IEsploraApi } from './esplora-api.interface'; export interface AbstractBitcoinApi { $getRawMempool(): Promise; $getRawTransaction(txId: string, skipConversion?: boolean, addPrevout?: boolean, lazyPrevouts?: boolean): Promise; + $getMempoolTransactions(expectedCount: number); $getTransactionHex(txId: string): Promise; $getBlockHeightTip(): Promise; $getBlockHashTip(): Promise; diff --git a/backend/src/api/bitcoin/bitcoin-api.ts b/backend/src/api/bitcoin/bitcoin-api.ts index c045d8664..237c69834 100644 --- a/backend/src/api/bitcoin/bitcoin-api.ts +++ b/backend/src/api/bitcoin/bitcoin-api.ts @@ -59,6 +59,10 @@ class BitcoinApi implements AbstractBitcoinApi { }); } + $getMempoolTransactions(expectedCount: number): Promise { + return Promise.resolve([]); + } + $getTransactionHex(txId: string): Promise { return this.$getRawTransaction(txId, true) .then((tx) => tx.hex || ''); diff --git a/backend/src/api/bitcoin/esplora-api.ts b/backend/src/api/bitcoin/esplora-api.ts index 5bfff5730..34de4f94f 100644 --- a/backend/src/api/bitcoin/esplora-api.ts +++ b/backend/src/api/bitcoin/esplora-api.ts @@ -5,6 +5,8 @@ import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory'; import { IEsploraApi } from './esplora-api.interface'; import logger from '../../logger'; +import JsonStream from 'JSONStream'; + const axiosConnection = axios.create({ httpAgent: new http.Agent({ keepAlive: true, }) }); @@ -69,6 +71,27 @@ class ElectrsApi implements AbstractBitcoinApi { return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/tx/' + txId); } + async $getMempoolTransactions(expectedCount: number): Promise { + const transactions: IEsploraApi.Transaction[] = []; + let count = 0; + return new Promise((resolve, reject) => { + axiosConnection.get(config.ESPLORA.REST_API_URL + '/mempool/txs', { ...this.activeAxiosConfig, timeout: 60000, responseType: 'stream' }).then(response => { + response.data.pipe(JsonStream.parse('*')).on('data', transaction => { + count++; + if (count % 10000 === 0) { + logger.info(`Fetched ${count} of ${expectedCount} mempool transactions from esplora`); + } + transactions.push(transaction); + }).on('end', () => { + logger.info(`Fetched all ${count} of ${expectedCount} mempool transactions from esplora`); + resolve(transactions); + }).on('error', (err) => { + reject(err); + }); + }); + }); + } + $getTransactionHex(txId: string): Promise { return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/hex'); } diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index d988ea47a..d0e63ae78 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -1,5 +1,5 @@ import config from '../config'; -import bitcoinApi from './bitcoin/bitcoin-api-factory'; +import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory'; import { MempoolTransactionExtended, TransactionExtended, VbytesPerSecond } from '../mempool.interfaces'; import logger from '../logger'; import { Common } from './common'; @@ -103,6 +103,16 @@ class Mempool { this.addToSpendMap(Object.values(this.mempoolCache)); } + public async $reloadMempool(expectedCount: number): Promise { + const rawTransactions = await bitcoinApi.$getMempoolTransactions(expectedCount); + logger.info(`Inserting loaded mempool transactions into local cache`); + for (const transaction of rawTransactions) { + const extendedTransaction = transactionUtils.extendMempoolTransaction(transaction); + this.mempoolCache[extendedTransaction.txid] = extendedTransaction; + } + logger.info(`Done inserting loaded mempool transactions into local cache`); + } + public async $updateMemPoolInfo() { this.mempoolInfo = await this.$getMempoolInfo(); } @@ -162,41 +172,54 @@ class Mempool { }; let intervalTimer = Date.now(); - for (const txid of transactions) { - if (!this.mempoolCache[txid]) { - try { - const transaction = await transactionUtils.$getMempoolTransactionExtended(txid, false, false, false); - this.updateTimerProgress(timer, 'fetched new transaction'); - this.mempoolCache[txid] = transaction; - if (this.inSync) { - this.txPerSecondArray.push(new Date().getTime()); - this.vBytesPerSecondArray.push({ - unixTime: new Date().getTime(), - vSize: transaction.vsize, - }); - } - hasChange = true; - newTransactions.push(transaction); - } catch (e: any) { - if (config.MEMPOOL.BACKEND === 'esplora' && e.response?.status === 404) { - this.missingTxCount++; - } - logger.debug(`Error finding transaction '${txid}' in the mempool: ` + (e instanceof Error ? e.message : e)); - } - } - if (Date.now() - intervalTimer > 5_000) { - - if (this.inSync) { - // Break and restart mempool loop if we spend too much time processing - // new transactions that may lead to falling behind on block height - logger.debug('Breaking mempool loop because the 5s time limit exceeded.'); - break; - } else { - const progress = (currentMempoolSize + newTransactions.length) / transactions.length * 100; - logger.debug(`Mempool is synchronizing. Processed ${newTransactions.length}/${diff} txs (${Math.round(progress)}%)`); - loadingIndicators.setProgress('mempool', progress); - intervalTimer = Date.now() + let loaded = false; + if (config.MEMPOOL.BACKEND === 'esplora' && currentMempoolSize < transactions.length * 0.5 && transactions.length > 20_000) { + logger.info(`Missing ${transactions.length - currentMempoolSize} mempool transactions, attempting to reload in bulk from esplora`); + try { + await this.$reloadMempool(transactions.length); + loaded = true; + } catch (e) { + logger.err('failed to load mempool in bulk from esplora, falling back to fetching individual transactions'); + } + } + + if (!loaded) { + for (const txid of transactions) { + if (!this.mempoolCache[txid]) { + try { + const transaction = await transactionUtils.$getMempoolTransactionExtended(txid, false, false, false); + this.updateTimerProgress(timer, 'fetched new transaction'); + this.mempoolCache[txid] = transaction; + if (this.inSync) { + this.txPerSecondArray.push(new Date().getTime()); + this.vBytesPerSecondArray.push({ + unixTime: new Date().getTime(), + vSize: transaction.vsize, + }); + } + hasChange = true; + newTransactions.push(transaction); + } catch (e: any) { + if (config.MEMPOOL.BACKEND === 'esplora' && e.response?.status === 404) { + this.missingTxCount++; + } + logger.debug(`Error finding transaction '${txid}' in the mempool: ` + (e instanceof Error ? e.message : e)); + } + } + + if (Date.now() - intervalTimer > 5_000) { + if (this.inSync) { + // Break and restart mempool loop if we spend too much time processing + // new transactions that may lead to falling behind on block height + logger.debug('Breaking mempool loop because the 5s time limit exceeded.'); + break; + } else { + const progress = (currentMempoolSize + newTransactions.length) / transactions.length * 100; + logger.debug(`Mempool is synchronizing. Processed ${newTransactions.length}/${diff} txs (${Math.round(progress)}%)`); + loadingIndicators.setProgress('mempool', progress); + intervalTimer = Date.now() + } } } } From db715a1dbacd3498150d054e9a5426a328eddc89 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sun, 23 Jul 2023 15:15:16 +0900 Subject: [PATCH 2/7] Switch to batch mempool/txs/:txid endpoint --- backend/src/api/bitcoin/esplora-api.ts | 40 +++++++++++++++----------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/backend/src/api/bitcoin/esplora-api.ts b/backend/src/api/bitcoin/esplora-api.ts index 34de4f94f..46b17a4d2 100644 --- a/backend/src/api/bitcoin/esplora-api.ts +++ b/backend/src/api/bitcoin/esplora-api.ts @@ -5,8 +5,6 @@ import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory'; import { IEsploraApi } from './esplora-api.interface'; import logger from '../../logger'; -import JsonStream from 'JSONStream'; - const axiosConnection = axios.create({ httpAgent: new http.Agent({ keepAlive: true, }) }); @@ -74,22 +72,30 @@ class ElectrsApi implements AbstractBitcoinApi { async $getMempoolTransactions(expectedCount: number): Promise { const transactions: IEsploraApi.Transaction[] = []; let count = 0; - return new Promise((resolve, reject) => { - axiosConnection.get(config.ESPLORA.REST_API_URL + '/mempool/txs', { ...this.activeAxiosConfig, timeout: 60000, responseType: 'stream' }).then(response => { - response.data.pipe(JsonStream.parse('*')).on('data', transaction => { - count++; - if (count % 10000 === 0) { - logger.info(`Fetched ${count} of ${expectedCount} mempool transactions from esplora`); + let done = false; + let last_txid = ''; + while (!done) { + try { + const result = await this.$queryWrapper(config.ESPLORA.REST_API_URL + '/mempool/txs' + (last_txid ? '/' + last_txid : '')); + if (result) { + for (const tx of result) { + transactions.push(tx); + count++; } - transactions.push(transaction); - }).on('end', () => { - logger.info(`Fetched all ${count} of ${expectedCount} mempool transactions from esplora`); - resolve(transactions); - }).on('error', (err) => { - reject(err); - }); - }); - }); + logger.info(`Fetched ${count} of ${expectedCount} mempool transactions from esplora`); + if (result.length > 0) { + last_txid = result[result.length - 1].txid; + } else { + done = true; + } + } else { + done = true; + } + } catch(err) { + logger.err('failed to fetch bulk mempool transactions from esplora'); + } + } + return transactions; } $getTransactionHex(txId: string): Promise { From e59c961f25317d4af2bf394a3116d0ddd16b6d17 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 24 Jul 2023 14:59:51 +0900 Subject: [PATCH 3/7] Add electrs sync progress updates --- .../bitcoin/bitcoin-api-abstract-factory.ts | 2 +- backend/src/api/bitcoin/bitcoin-api.ts | 2 +- backend/src/api/bitcoin/esplora-api.ts | 29 ++-------------- backend/src/api/mempool.ts | 34 ++++++++++++++++--- 4 files changed, 33 insertions(+), 34 deletions(-) diff --git a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts index d195b0eeb..f610ed883 100644 --- a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts +++ b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts @@ -3,7 +3,7 @@ import { IEsploraApi } from './esplora-api.interface'; export interface AbstractBitcoinApi { $getRawMempool(): Promise; $getRawTransaction(txId: string, skipConversion?: boolean, addPrevout?: boolean, lazyPrevouts?: boolean): Promise; - $getMempoolTransactions(expectedCount: number); + $getMempoolTransactions(lastTxid: string); $getTransactionHex(txId: string): Promise; $getBlockHeightTip(): Promise; $getBlockHashTip(): Promise; diff --git a/backend/src/api/bitcoin/bitcoin-api.ts b/backend/src/api/bitcoin/bitcoin-api.ts index 237c69834..3ccea01ef 100644 --- a/backend/src/api/bitcoin/bitcoin-api.ts +++ b/backend/src/api/bitcoin/bitcoin-api.ts @@ -59,7 +59,7 @@ class BitcoinApi implements AbstractBitcoinApi { }); } - $getMempoolTransactions(expectedCount: number): Promise { + $getMempoolTransactions(lastTxid: string): Promise { return Promise.resolve([]); } diff --git a/backend/src/api/bitcoin/esplora-api.ts b/backend/src/api/bitcoin/esplora-api.ts index 46b17a4d2..73a44a845 100644 --- a/backend/src/api/bitcoin/esplora-api.ts +++ b/backend/src/api/bitcoin/esplora-api.ts @@ -69,33 +69,8 @@ class ElectrsApi implements AbstractBitcoinApi { return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/tx/' + txId); } - async $getMempoolTransactions(expectedCount: number): Promise { - const transactions: IEsploraApi.Transaction[] = []; - let count = 0; - let done = false; - let last_txid = ''; - while (!done) { - try { - const result = await this.$queryWrapper(config.ESPLORA.REST_API_URL + '/mempool/txs' + (last_txid ? '/' + last_txid : '')); - if (result) { - for (const tx of result) { - transactions.push(tx); - count++; - } - logger.info(`Fetched ${count} of ${expectedCount} mempool transactions from esplora`); - if (result.length > 0) { - last_txid = result[result.length - 1].txid; - } else { - done = true; - } - } else { - done = true; - } - } catch(err) { - logger.err('failed to fetch bulk mempool transactions from esplora'); - } - } - return transactions; + async $getMempoolTransactions(lastSeenTxid?: string): Promise { + return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/mempool/txs' + (lastSeenTxid ? '/' + lastSeenTxid : '')); } $getTransactionHex(txId: string): Promise { diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index d0e63ae78..88533365e 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -9,6 +9,7 @@ import loadingIndicators from './loading-indicators'; import bitcoinClient from './bitcoin/bitcoin-client'; import bitcoinSecondClient from './bitcoin/bitcoin-second-client'; import rbfCache from './rbf-cache'; +import { IEsploraApi } from './bitcoin/esplora-api.interface'; class Mempool { private inSync: boolean = false; @@ -104,11 +105,34 @@ class Mempool { } public async $reloadMempool(expectedCount: number): Promise { - const rawTransactions = await bitcoinApi.$getMempoolTransactions(expectedCount); - logger.info(`Inserting loaded mempool transactions into local cache`); - for (const transaction of rawTransactions) { - const extendedTransaction = transactionUtils.extendMempoolTransaction(transaction); - this.mempoolCache[extendedTransaction.txid] = extendedTransaction; + let count = 0; + let done = false; + let last_txid; + loadingIndicators.setProgress('mempool', count / expectedCount * 100); + while (!done) { + try { + const result = await bitcoinApi.$getMempoolTransactions(last_txid); + if (result) { + for (const tx of result) { + const extendedTransaction = transactionUtils.extendMempoolTransaction(tx); + this.mempoolCache[extendedTransaction.txid] = extendedTransaction; + count++; + } + logger.info(`Fetched ${count} of ${expectedCount} mempool transactions from esplora`); + if (result.length > 0) { + last_txid = result[result.length - 1].txid; + } else { + done = true; + } + if (count < expectedCount) { + loadingIndicators.setProgress('mempool', count / expectedCount * 100); + } + } else { + done = true; + } + } catch(err) { + logger.err('failed to fetch bulk mempool transactions from esplora'); + } } logger.info(`Done inserting loaded mempool transactions into local cache`); } From de4265a6d1b616555d271540ff899fb4db9919e6 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 24 Jul 2023 16:19:19 +0900 Subject: [PATCH 4/7] More conservative mempool inSync status --- backend/src/api/mempool.ts | 15 ++++++++------- backend/src/api/websocket-handler.ts | 4 ++-- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index 88533365e..9239d6b3b 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -124,7 +124,7 @@ class Mempool { } else { done = true; } - if (count < expectedCount) { + if (Math.floor(count / expectedCount) < 1) { loadingIndicators.setProgress('mempool', count / expectedCount * 100); } } else { @@ -199,6 +199,7 @@ class Mempool { let loaded = false; if (config.MEMPOOL.BACKEND === 'esplora' && currentMempoolSize < transactions.length * 0.5 && transactions.length > 20_000) { + this.inSync = false; logger.info(`Missing ${transactions.length - currentMempoolSize} mempool transactions, attempting to reload in bulk from esplora`); try { await this.$reloadMempool(transactions.length); @@ -293,12 +294,6 @@ class Mempool { const newTransactionsStripped = newTransactions.map((tx) => Common.stripTransaction(tx)); this.latestTransactions = newTransactionsStripped.concat(this.latestTransactions).slice(0, 6); - if (!this.inSync && transactions.length === newMempoolSize) { - this.inSync = true; - logger.notice('The mempool is now in sync!'); - loadingIndicators.setProgress('mempool', 100); - } - this.mempoolCacheDelta = Math.abs(transactions.length - newMempoolSize); if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) { @@ -310,6 +305,12 @@ class Mempool { this.updateTimerProgress(timer, 'completed async mempool callback'); } + if (!this.inSync && transactions.length === newMempoolSize) { + this.inSync = true; + logger.notice('The mempool is now in sync!'); + loadingIndicators.setProgress('mempool', 100); + } + const end = new Date().getTime(); const time = end - start; logger.debug(`Mempool updated in ${time / 1000} seconds. New size: ${Object.keys(this.mempoolCache).length} (${diff > 0 ? '+' + diff : diff})`); diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index a0c031175..ccaeb4a8b 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -644,7 +644,7 @@ class WebsocketHandler { memPool.handleMinedRbfTransactions(rbfTransactions); memPool.removeFromSpendMap(transactions); - if (config.MEMPOOL.AUDIT) { + if (config.MEMPOOL.AUDIT && memPool.isInSync()) { let projectedBlocks; let auditMempool = _memPool; // template calculation functions have mempool side effects, so calculate audits using @@ -665,7 +665,7 @@ class WebsocketHandler { projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions(); } - if (Common.indexingEnabled() && memPool.isInSync()) { + if (Common.indexingEnabled()) { const { censored, added, fresh, sigop, fullrbf, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool); const matchRate = Math.round(score * 100 * 100) / 100; From a6edfcc2723b40172d4006123e3170b1feb19d30 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 24 Jul 2023 16:22:35 +0900 Subject: [PATCH 5/7] show mempool skeleton while not inSync --- .../mempool-blocks/mempool-blocks.component.ts | 9 ++++++++- frontend/src/app/services/state.service.ts | 1 + frontend/src/app/services/websocket.service.ts | 5 +++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts b/frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts index e6d5a4bf6..71075b261 100644 --- a/frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts +++ b/frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts @@ -117,7 +117,14 @@ export class MempoolBlocksComponent implements OnInit, OnChanges, OnDestroy { }); this.reduceMempoolBlocksToFitScreen(this.mempoolBlocks); this.stateService.isTabHidden$.subscribe((tabHidden) => this.tabHidden = tabHidden); - this.loadingBlocks$ = this.stateService.isLoadingWebSocket$; + this.loadingBlocks$ = combineLatest([ + this.stateService.isLoadingWebSocket$, + this.stateService.isLoadingMempool$ + ]).pipe( + switchMap(([loadingBlocks, loadingMempool]) => { + return of(loadingBlocks || loadingMempool); + }) + ); this.mempoolBlocks$ = merge( of(true), diff --git a/frontend/src/app/services/state.service.ts b/frontend/src/app/services/state.service.ts index 5ebca9ba1..ea00f12ab 100644 --- a/frontend/src/app/services/state.service.ts +++ b/frontend/src/app/services/state.service.ts @@ -113,6 +113,7 @@ export class StateService { mempoolTxPosition$ = new Subject<{ txid: string, position: MempoolPosition}>(); blockTransactions$ = new Subject(); isLoadingWebSocket$ = new ReplaySubject(1); + isLoadingMempool$ = new BehaviorSubject(true); vbytesPerSecond$ = new ReplaySubject(1); previousRetarget$ = new ReplaySubject(1); backendInfo$ = new ReplaySubject(1); diff --git a/frontend/src/app/services/websocket.service.ts b/frontend/src/app/services/websocket.service.ts index f32f772ac..e70424cdc 100644 --- a/frontend/src/app/services/websocket.service.ts +++ b/frontend/src/app/services/websocket.service.ts @@ -368,6 +368,11 @@ export class WebsocketService { if (response.loadingIndicators) { this.stateService.loadingIndicators$.next(response.loadingIndicators); + if (response.loadingIndicators.mempool != null && response.loadingIndicators.mempool < 100) { + this.stateService.isLoadingMempool$.next(true); + } else { + this.stateService.isLoadingMempool$.next(false); + } } if (response.mempoolInfo) { From 2d463326e05d28db251dedacb3418605fe27c5d3 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 24 Jul 2023 17:22:38 +0900 Subject: [PATCH 6/7] fix gbt mempool size mismatch bug --- backend/src/api/mempool.ts | 13 +++++++++---- backend/src/api/websocket-handler.ts | 4 ++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index 9239d6b3b..b6ac9e074 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -104,10 +104,11 @@ class Mempool { this.addToSpendMap(Object.values(this.mempoolCache)); } - public async $reloadMempool(expectedCount: number): Promise { + public async $reloadMempool(expectedCount: number): Promise { let count = 0; let done = false; let last_txid; + const newTransactions: MempoolTransactionExtended[] = []; loadingIndicators.setProgress('mempool', count / expectedCount * 100); while (!done) { try { @@ -115,7 +116,10 @@ class Mempool { if (result) { for (const tx of result) { const extendedTransaction = transactionUtils.extendMempoolTransaction(tx); - this.mempoolCache[extendedTransaction.txid] = extendedTransaction; + if (!this.mempoolCache[extendedTransaction.txid]) { + newTransactions.push(extendedTransaction); + this.mempoolCache[extendedTransaction.txid] = extendedTransaction; + } count++; } logger.info(`Fetched ${count} of ${expectedCount} mempool transactions from esplora`); @@ -134,6 +138,7 @@ class Mempool { logger.err('failed to fetch bulk mempool transactions from esplora'); } } + return newTransactions; logger.info(`Done inserting loaded mempool transactions into local cache`); } @@ -177,7 +182,7 @@ class Mempool { const currentMempoolSize = Object.keys(this.mempoolCache).length; this.updateTimerProgress(timer, 'got raw mempool'); const diff = transactions.length - currentMempoolSize; - const newTransactions: MempoolTransactionExtended[] = []; + let newTransactions: MempoolTransactionExtended[] = []; this.mempoolCacheDelta = Math.abs(diff); @@ -202,7 +207,7 @@ class Mempool { this.inSync = false; logger.info(`Missing ${transactions.length - currentMempoolSize} mempool transactions, attempting to reload in bulk from esplora`); try { - await this.$reloadMempool(transactions.length); + newTransactions = await this.$reloadMempool(transactions.length); loaded = true; } catch (e) { logger.err('failed to load mempool in bulk from esplora, falling back to fetching individual transactions'); diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index ccaeb4a8b..56c8513cd 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -604,7 +604,7 @@ class WebsocketHandler { } } - if (client['track-mempool-block'] >= 0) { + if (client['track-mempool-block'] >= 0 && memPool.isInSync()) { const index = client['track-mempool-block']; if (mBlockDeltas[index]) { response['projected-block-transactions'] = getCachedResponse(`projected-block-transactions-${index}`, { @@ -858,7 +858,7 @@ class WebsocketHandler { } } - if (client['track-mempool-block'] >= 0) { + if (client['track-mempool-block'] >= 0 && memPool.isInSync()) { const index = client['track-mempool-block']; if (mBlockDeltas && mBlockDeltas[index]) { response['projected-block-transactions'] = getCachedResponse(`projected-block-transactions-${index}`, { From 36fe5627c70a8ba9b350244f49c5228a000ac62c Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 24 Jul 2023 17:49:34 +0900 Subject: [PATCH 7/7] fix mempool sync skeleton loaders on Core backend --- backend/src/api/mempool.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index b6ac9e074..945b78738 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -128,7 +128,7 @@ class Mempool { } else { done = true; } - if (Math.floor(count / expectedCount) < 1) { + if (Math.floor((count / expectedCount) * 100) < 100) { loadingIndicators.setProgress('mempool', count / expectedCount * 100); } } else { @@ -247,7 +247,9 @@ class Mempool { } else { const progress = (currentMempoolSize + newTransactions.length) / transactions.length * 100; logger.debug(`Mempool is synchronizing. Processed ${newTransactions.length}/${diff} txs (${Math.round(progress)}%)`); - loadingIndicators.setProgress('mempool', progress); + if (Math.floor(progress) < 100) { + loadingIndicators.setProgress('mempool', progress); + } intervalTimer = Date.now() } }