From 70badaf4615e3d258a6cd15d4b12a09f085cbd6c Mon Sep 17 00:00:00 2001 From: Mononaut Date: Fri, 18 Aug 2023 02:47:32 +0900 Subject: [PATCH] Speed up $scanForClosedChannels, use internal outspends apis --- .../bitcoin/bitcoin-api-abstract-factory.ts | 1 + backend/src/api/bitcoin/bitcoin-api.ts | 9 ++++++ backend/src/api/bitcoin/esplora-api.ts | 15 +++------ .../src/tasks/lightning/forensics.service.ts | 2 +- .../tasks/lightning/network-sync.service.ts | 32 ++++++++++++------- 5 files changed, 37 insertions(+), 22 deletions(-) diff --git a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts index 6f20dad92..f008e5ed8 100644 --- a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts +++ b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts @@ -25,6 +25,7 @@ export interface AbstractBitcoinApi { $getOutspends(txId: string): Promise; $getBatchedOutspends(txId: string[]): Promise; $getBatchedOutspendsInternal(txId: string[]): Promise; + $getOutSpendsByOutpoint(outpoints: { txid: string, vout: number }[]): Promise; startHealthChecks(): void; } diff --git a/backend/src/api/bitcoin/bitcoin-api.ts b/backend/src/api/bitcoin/bitcoin-api.ts index 9e4cbdd8b..1722334df 100644 --- a/backend/src/api/bitcoin/bitcoin-api.ts +++ b/backend/src/api/bitcoin/bitcoin-api.ts @@ -215,6 +215,15 @@ class BitcoinApi implements AbstractBitcoinApi { return this.$getBatchedOutspends(txId); } + async $getOutSpendsByOutpoint(outpoints: { txid: string, vout: number }[]): Promise { + const outspends: IEsploraApi.Outspend[] = []; + for (const outpoint of outpoints) { + const outspend = await this.$getOutspend(outpoint.txid, outpoint.vout); + outspends.push(outspend); + } + return outspends; + } + $getEstimatedHashrate(blockHeight: number): Promise { // 120 is the default block span in Core return this.bitcoindClient.getNetworkHashPs(120, blockHeight); diff --git a/backend/src/api/bitcoin/esplora-api.ts b/backend/src/api/bitcoin/esplora-api.ts index 574113ae6..2beebe270 100644 --- a/backend/src/api/bitcoin/esplora-api.ts +++ b/backend/src/api/bitcoin/esplora-api.ts @@ -302,16 +302,11 @@ class ElectrsApi implements AbstractBitcoinApi { } async $getBatchedOutspendsInternal(txids: string[]): Promise { - const allOutspends: IEsploraApi.Outspend[][] = []; - const sliceLength = 50; - for (let i = 0; i < Math.ceil(txids.length / sliceLength); i++) { - const slice = txids.slice(i * sliceLength, (i + 1) * sliceLength); - const sliceOutspends = await this.failoverRouter.$get('/txs/outspends', 'json', { txids: slice.join(',') }); - for (const outspends of sliceOutspends) { - allOutspends.push(outspends); - } - } - return allOutspends; + return this.failoverRouter.$post('/internal-api/txs/outspends/by-txid', txids, 'json'); + } + + async $getOutSpendsByOutpoint(outpoints: { txid: string, vout: number }[]): Promise { + return this.failoverRouter.$post('/internal-api/txs/outspends/by-outpoint', outpoints.map(out => `${out.txid}:${out.vout}`), 'json'); } public startHealthChecks(): void { diff --git a/backend/src/tasks/lightning/forensics.service.ts b/backend/src/tasks/lightning/forensics.service.ts index c83f1720c..1cbf7d647 100644 --- a/backend/src/tasks/lightning/forensics.service.ts +++ b/backend/src/tasks/lightning/forensics.service.ts @@ -94,7 +94,7 @@ class ForensicsService { logger.info(`Fetched outspends for ${allOutspends.length} txs from esplora for LN forensics`); await Common.sleep$(config.LIGHTNING.FORENSICS_RATE_LIMIT); } catch (e) { - logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/txs/outspends'}. Reason ${e instanceof Error ? e.message : e}`); + logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/internal-api/txs/outspends/by-txid'}. Reason ${e instanceof Error ? e.message : e}`); } // fetch spending transactions in bulk and load into txCache const newSpendingTxids: { [txid: string]: boolean } = {}; diff --git a/backend/src/tasks/lightning/network-sync.service.ts b/backend/src/tasks/lightning/network-sync.service.ts index 963b9e8c2..dc0d609fa 100644 --- a/backend/src/tasks/lightning/network-sync.service.ts +++ b/backend/src/tasks/lightning/network-sync.service.ts @@ -288,22 +288,32 @@ class NetworkSyncService { } logger.debug(`${log}`, logger.tags.ln); - const channels = await channelsApi.$getChannelsByStatus([0, 1]); - for (const channel of channels) { - const spendingTx = await bitcoinApi.$getOutspend(channel.transaction_id, channel.transaction_vout); - if (spendingTx.spent === true && spendingTx.status?.confirmed === true) { - logger.debug(`Marking channel: ${channel.id} as closed.`, logger.tags.ln); - await DB.query(`UPDATE channels SET status = 2, closing_date = FROM_UNIXTIME(?) WHERE id = ?`, - [spendingTx.status.block_time, channel.id]); - if (spendingTx.txid && !channel.closing_transaction_id) { - await DB.query(`UPDATE channels SET closing_transaction_id = ? WHERE id = ?`, [spendingTx.txid, channel.id]); + const allChannels = await channelsApi.$getChannelsByStatus([0, 1]); + + const sliceLength = 5000; + // process batches of 5000 channels + for (let i = 0; i < Math.ceil(allChannels.length / sliceLength); i++) { + const channels = allChannels.slice(i * sliceLength, (i + 1) * sliceLength); + const outspends = await bitcoinApi.$getOutSpendsByOutpoint(channels.map(channel => { + return { txid: channel.transaction_id, vout: channel.transaction_vout }; + })); + + for (const [index, channel] of channels.entries()) { + const spendingTx = outspends[index]; + if (spendingTx.spent === true && spendingTx.status?.confirmed === true) { + // logger.debug(`Marking channel: ${channel.id} as closed.`, logger.tags.ln); + await DB.query(`UPDATE channels SET status = 2, closing_date = FROM_UNIXTIME(?) WHERE id = ?`, + [spendingTx.status.block_time, channel.id]); + if (spendingTx.txid && !channel.closing_transaction_id) { + await DB.query(`UPDATE channels SET closing_transaction_id = ? WHERE id = ?`, [spendingTx.txid, channel.id]); + } } } - ++progress; + progress += channels.length; const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer); if (elapsedSeconds > config.LIGHTNING.LOGGER_UPDATE_INTERVAL) { - logger.debug(`Checking if channel has been closed ${progress}/${channels.length}`, logger.tags.ln); + logger.debug(`Checking if channel has been closed ${progress}/${allChannels.length}`, logger.tags.ln); this.loggerTimer = new Date().getTime() / 1000; } }