Speed up $scanForClosedChannels, use internal outspends apis

backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts

@@ -25,6 +25,7 @@ export interface AbstractBitcoinApi {
   $getOutspends(txId: string): Promise<IEsploraApi.Outspend[]>;
   $getBatchedOutspends(txId: string[]): Promise<IEsploraApi.Outspend[][]>;
   $getBatchedOutspendsInternal(txId: string[]): Promise<IEsploraApi.Outspend[][]>;
+  $getOutSpendsByOutpoint(outpoints: { txid: string, vout: number }[]): Promise<IEsploraApi.Outspend[]>;
 
   startHealthChecks(): void;
 }

backend/src/api/bitcoin/bitcoin-api.ts

@@ -215,6 +215,15 @@ class BitcoinApi implements AbstractBitcoinApi {
     return this.$getBatchedOutspends(txId);
   }
 
+  async $getOutSpendsByOutpoint(outpoints: { txid: string, vout: number }[]): Promise<IEsploraApi.Outspend[]> {
+    const outspends: IEsploraApi.Outspend[] = [];
+    for (const outpoint of outpoints) {
+      const outspend = await this.$getOutspend(outpoint.txid, outpoint.vout);
+      outspends.push(outspend);
+    }
+    return outspends;
+  }
+
   $getEstimatedHashrate(blockHeight: number): Promise<number> {
     // 120 is the default block span in Core
     return this.bitcoindClient.getNetworkHashPs(120, blockHeight);

backend/src/api/bitcoin/esplora-api.ts

@@ -302,16 +302,11 @@ class ElectrsApi implements AbstractBitcoinApi {
   }
 
   async $getBatchedOutspendsInternal(txids: string[]): Promise<IEsploraApi.Outspend[][]> {
-    const allOutspends: IEsploraApi.Outspend[][] = [];
+    return this.failoverRouter.$post<IEsploraApi.Outspend[][]>('/internal-api/txs/outspends/by-txid', txids, 'json');
-    const sliceLength = 50;
+  }
-    for (let i = 0; i < Math.ceil(txids.length / sliceLength); i++) {
+
-      const slice = txids.slice(i * sliceLength, (i + 1) * sliceLength);
+  async $getOutSpendsByOutpoint(outpoints: { txid: string, vout: number }[]): Promise<IEsploraApi.Outspend[]> {
-      const sliceOutspends = await this.failoverRouter.$get<IEsploraApi.Outspend[][]>('/txs/outspends', 'json', { txids: slice.join(',') });
+    return this.failoverRouter.$post<IEsploraApi.Outspend[]>('/internal-api/txs/outspends/by-outpoint', outpoints.map(out => `${out.txid}:${out.vout}`), 'json');
-      for (const outspends of sliceOutspends) {
-        allOutspends.push(outspends);
-      }
-    }
-    return allOutspends;
   }
 
   public startHealthChecks(): void {

backend/src/tasks/lightning/forensics.service.ts

@@ -94,7 +94,7 @@ class ForensicsService {
           logger.info(`Fetched outspends for ${allOutspends.length} txs from esplora for LN forensics`);
           await Common.sleep$(config.LIGHTNING.FORENSICS_RATE_LIMIT);
         } catch (e) {
-          logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/txs/outspends'}. Reason ${e instanceof Error ? e.message : e}`);
+          logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/internal-api/txs/outspends/by-txid'}. Reason ${e instanceof Error ? e.message : e}`);
         }
         // fetch spending transactions in bulk and load into txCache
         const newSpendingTxids: { [txid: string]: boolean } = {};

backend/src/tasks/lightning/network-sync.service.ts

@@ -288,22 +288,32 @@ class NetworkSyncService {
       }
       logger.debug(`${log}`, logger.tags.ln);
 
-      const channels = await channelsApi.$getChannelsByStatus([0, 1]);
+      const allChannels = await channelsApi.$getChannelsByStatus([0, 1]);
-      for (const channel of channels) {
+
-        const spendingTx = await bitcoinApi.$getOutspend(channel.transaction_id, channel.transaction_vout);
+      const sliceLength = 5000;
-        if (spendingTx.spent === true && spendingTx.status?.confirmed === true) {
+      // process batches of 5000 channels
-          logger.debug(`Marking channel: ${channel.id} as closed.`, logger.tags.ln);
+      for (let i = 0; i < Math.ceil(allChannels.length / sliceLength); i++) {
-          await DB.query(`UPDATE channels SET status = 2, closing_date = FROM_UNIXTIME(?) WHERE id = ?`,
+        const channels = allChannels.slice(i * sliceLength, (i + 1) * sliceLength);
-            [spendingTx.status.block_time, channel.id]);
+        const outspends = await bitcoinApi.$getOutSpendsByOutpoint(channels.map(channel => {
-          if (spendingTx.txid && !channel.closing_transaction_id) {
+          return { txid: channel.transaction_id, vout: channel.transaction_vout };
-            await DB.query(`UPDATE channels SET closing_transaction_id = ? WHERE id = ?`, [spendingTx.txid, channel.id]);
+        }));
+
+        for (const [index, channel] of channels.entries()) {
+          const spendingTx = outspends[index];
+          if (spendingTx.spent === true && spendingTx.status?.confirmed === true) {
+            // logger.debug(`Marking channel: ${channel.id} as closed.`, logger.tags.ln);
+            await DB.query(`UPDATE channels SET status = 2, closing_date = FROM_UNIXTIME(?) WHERE id = ?`,
+              [spendingTx.status.block_time, channel.id]);
+            if (spendingTx.txid && !channel.closing_transaction_id) {
+              await DB.query(`UPDATE channels SET closing_transaction_id = ? WHERE id = ?`, [spendingTx.txid, channel.id]);
+            }
           }
         }
 
-        ++progress;
+        progress += channels.length;
         const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
         if (elapsedSeconds > config.LIGHTNING.LOGGER_UPDATE_INTERVAL) {
-          logger.debug(`Checking if channel has been closed ${progress}/${channels.length}`, logger.tags.ln);
+          logger.debug(`Checking if channel has been closed ${progress}/${allChannels.length}`, logger.tags.ln);
           this.loggerTimer = new Date().getTime() / 1000;
         }
       }