Merge branch 'master' into taproot-channels

2023-11-14 13:41:44 +09:00
parent 2bc3352785 cbe1ec4e72
commit 562cd5683a
111 changed files with 2487 additions and 1412 deletions
--- a/backend/src/fixtures/mempool-config.template.json
+++ b/backend/src/fixtures/mempool-config.template.json
@@ -41,7 +41,9 @@
    "PORT": 15,
    "USERNAME": "__CORE_RPC_USERNAME__",
    "PASSWORD": "__CORE_RPC_PASSWORD__",
-    "TIMEOUT": 1000
+    "TIMEOUT": 1000,
+    "COOKIE": false,
+    "COOKIE_PATH": "__CORE_RPC_COOKIE_PATH__"
  },
  "ELECTRUM": {
    "HOST": "__ELECTRUM_HOST__",
@@ -52,6 +54,8 @@
    "REST_API_URL": "__ESPLORA_REST_API_URL__",
    "UNIX_SOCKET_PATH": "__ESPLORA_UNIX_SOCKET_PATH__",
    "RETRY_UNIX_SOCKET_AFTER": 888,
+    "REQUEST_TIMEOUT": 10000,
+    "FALLBACK_TIMEOUT": 5000,
    "FALLBACK": []
  },
  "SECOND_CORE_RPC": {
@@ -59,7 +63,9 @@
    "PORT": 17,
    "USERNAME": "__SECOND_CORE_RPC_USERNAME__",
    "PASSWORD": "__SECOND_CORE_RPC_PASSWORD__",
-    "TIMEOUT": 2000
+    "TIMEOUT": 2000,
+    "COOKIE": false,
+    "COOKIE_PATH": "__SECOND_CORE_RPC_COOKIE_PATH__"
  },
  "DATABASE": {
    "ENABLED": false,
--- a/backend/src/tests/config.test.ts
+++ b/backend/src/tests/config.test.ts
@@ -56,6 +56,8 @@ describe('Mempool Backend Config', () => {
        REST_API_URL: 'http://127.0.0.1:3000',
        UNIX_SOCKET_PATH: null,
        RETRY_UNIX_SOCKET_AFTER: 30000,
+        REQUEST_TIMEOUT: 10000,
+        FALLBACK_TIMEOUT: 5000,
        FALLBACK: [],
       });

@@ -64,7 +66,9 @@ describe('Mempool Backend Config', () => {
        PORT: 8332,
        USERNAME: 'mempool',
        PASSWORD: 'mempool',
-        TIMEOUT: 60000
+        TIMEOUT: 60000,
+        COOKIE: false,
+        COOKIE_PATH: '/bitcoin/.cookie'
      });

      expect(config.SECOND_CORE_RPC).toStrictEqual({
@@ -72,7 +76,9 @@ describe('Mempool Backend Config', () => {
        PORT: 8332,
        USERNAME: 'mempool',
        PASSWORD: 'mempool',
-        TIMEOUT: 60000
+        TIMEOUT: 60000,
+        COOKIE: false,
+        COOKIE_PATH: '/bitcoin/.cookie'
      });

      expect(config.DATABASE).toStrictEqual({
--- a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts
+++ b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts
@@ -3,6 +3,7 @@ import { IEsploraApi } from './esplora-api.interface';
 export interface AbstractBitcoinApi {
  $getRawMempool(): Promise<IEsploraApi.Transaction['txid'][]>;
  $getRawTransaction(txId: string, skipConversion?: boolean, addPrevout?: boolean, lazyPrevouts?: boolean): Promise<IEsploraApi.Transaction>;
+  $getRawTransactions(txids: string[]): Promise<IEsploraApi.Transaction[]>;
  $getMempoolTransactions(txids: string[]): Promise<IEsploraApi.Transaction[]>;
  $getAllMempoolTransactions(lastTxid: string);
  $getTransactionHex(txId: string): Promise<string>;
@@ -23,6 +24,8 @@ export interface AbstractBitcoinApi {
  $getOutspend(txId: string, vout: number): Promise<IEsploraApi.Outspend>;
  $getOutspends(txId: string): Promise<IEsploraApi.Outspend[]>;
  $getBatchedOutspends(txId: string[]): Promise<IEsploraApi.Outspend[][]>;
+  $getBatchedOutspendsInternal(txId: string[]): Promise<IEsploraApi.Outspend[][]>;
+  $getOutSpendsByOutpoint(outpoints: { txid: string, vout: number }[]): Promise<IEsploraApi.Outspend[]>;

  startHealthChecks(): void;
 }
@@ -32,4 +35,5 @@ export interface BitcoinRpcCredentials {
  user: string;
  pass: string;
  timeout: number;
+  cookie?: string;
 }
--- a/backend/src/api/bitcoin/bitcoin-api.ts
+++ b/backend/src/api/bitcoin/bitcoin-api.ts
@@ -60,6 +60,19 @@ class BitcoinApi implements AbstractBitcoinApi {
      });
  }

+  async $getRawTransactions(txids: string[]): Promise<IEsploraApi.Transaction[]> {
+    const txs: IEsploraApi.Transaction[] = [];
+    for (const txid of txids) {
+      try {
+        const tx = await this.$getRawTransaction(txid, false, true);
+        txs.push(tx);
+      } catch (err) {
+        // skip failures
+      }
+    }
+    return txs;
+  }
+
  $getMempoolTransactions(txids: string[]): Promise<IEsploraApi.Transaction[]> {
    throw new Error('Method getMempoolTransactions not supported by the Bitcoin RPC API.');
  }
@@ -198,6 +211,19 @@ class BitcoinApi implements AbstractBitcoinApi {
    return outspends;
  }

+  async $getBatchedOutspendsInternal(txId: string[]): Promise<IEsploraApi.Outspend[][]> {
+    return this.$getBatchedOutspends(txId);
+  }
+
+  async $getOutSpendsByOutpoint(outpoints: { txid: string, vout: number }[]): Promise<IEsploraApi.Outspend[]> {
+    const outspends: IEsploraApi.Outspend[] = [];
+    for (const outpoint of outpoints) {
+      const outspend = await this.$getOutspend(outpoint.txid, outpoint.vout);
+      outspends.push(outspend);
+    }
+    return outspends;
+  }
+
  $getEstimatedHashrate(blockHeight: number): Promise<number> {
    // 120 is the default block span in Core
    return this.bitcoindClient.getNetworkHashPs(120, blockHeight);
--- a/backend/src/api/bitcoin/bitcoin-client.ts
+++ b/backend/src/api/bitcoin/bitcoin-client.ts
@@ -8,6 +8,7 @@ const nodeRpcCredentials: BitcoinRpcCredentials = {
  user: config.CORE_RPC.USERNAME,
  pass: config.CORE_RPC.PASSWORD,
  timeout: config.CORE_RPC.TIMEOUT,
+  cookie: config.CORE_RPC.COOKIE ? config.CORE_RPC.COOKIE_PATH : undefined,
 };

 export default new bitcoin.Client(nodeRpcCredentials);
--- a/backend/src/api/bitcoin/bitcoin-second-client.ts
+++ b/backend/src/api/bitcoin/bitcoin-second-client.ts
@@ -8,6 +8,7 @@ const nodeRpcCredentials: BitcoinRpcCredentials = {
  user: config.SECOND_CORE_RPC.USERNAME,
  pass: config.SECOND_CORE_RPC.PASSWORD,
  timeout: config.SECOND_CORE_RPC.TIMEOUT,
+  cookie: config.SECOND_CORE_RPC.COOKIE ? config.SECOND_CORE_RPC.COOKIE_PATH : undefined,
 };

 export default new bitcoin.Client(nodeRpcCredentials);
--- a/backend/src/api/bitcoin/bitcoin.routes.ts
+++ b/backend/src/api/bitcoin/bitcoin.routes.ts
@@ -24,7 +24,6 @@ class BitcoinRoutes {
  public initRoutes(app: Application) {
    app
      .get(config.MEMPOOL.API_URL_PREFIX + 'transaction-times', this.getTransactionTimes)
-      .get(config.MEMPOOL.API_URL_PREFIX + 'outspends', this.$getBatchedOutspends)
      .get(config.MEMPOOL.API_URL_PREFIX + 'cpfp/:txId', this.$getCpfpInfo)
      .get(config.MEMPOOL.API_URL_PREFIX + 'difficulty-adjustment', this.getDifficultyChange)
      .get(config.MEMPOOL.API_URL_PREFIX + 'fees/recommended', this.getRecommendedFees)
@@ -112,6 +111,7 @@ class BitcoinRoutes {
          .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/hex', this.getRawTransaction)
          .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/status', this.getTransactionStatus)
          .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/outspends', this.getTransactionOutspends)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'txs/outspends', this.$getBatchedOutspends)
          .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/header', this.getBlockHeader)
          .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/hash', this.getBlockTipHash)
          .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/raw', this.getRawBlock)
@@ -174,24 +174,20 @@ class BitcoinRoutes {
    res.json(times);
  }

-  private async $getBatchedOutspends(req: Request, res: Response) {
-    if (!Array.isArray(req.query.txId)) {
-      res.status(500).send('Not an array');
+  private async $getBatchedOutspends(req: Request, res: Response): Promise<IEsploraApi.Outspend[][] | void> {
+    const txids_csv = req.query.txids;
+    if (!txids_csv || typeof txids_csv !== 'string') {
+      res.status(500).send('Invalid txids format');
      return;
    }
-    if (req.query.txId.length > 50) {
+    const txids = txids_csv.split(',');
+    if (txids.length > 50) {
      res.status(400).send('Too many txids requested');
      return;
    }
-    const txIds: string[] = [];
-    for (const _txId in req.query.txId) {
-      if (typeof req.query.txId[_txId] === 'string') {
-        txIds.push(req.query.txId[_txId].toString());
-      }
-    }

    try {
-      const batchedOutspends = await bitcoinApi.$getBatchedOutspends(txIds);
+      const batchedOutspends = await bitcoinApi.$getBatchedOutspends(txids);
      res.json(batchedOutspends);
    } catch (e) {
      res.status(500).send(e instanceof Error ? e.message : e);
@@ -251,7 +247,7 @@ class BitcoinRoutes {

  private async getTransaction(req: Request, res: Response) {
    try {
-      const transaction = await transactionUtils.$getTransactionExtended(req.params.txId, true);
+      const transaction = await transactionUtils.$getTransactionExtended(req.params.txId, true, false, false, true);
      res.json(transaction);
    } catch (e) {
      let statusCode = 500;
@@ -478,7 +474,7 @@ class BitcoinRoutes {
      }

      let nextHash = startFromHash;
-      for (let i = 0; i < 10 && nextHash; i++) {
+      for (let i = 0; i < 15 && nextHash; i++) {
        const localBlock = blocks.getBlocks().find((b) => b.id === nextHash);
        if (localBlock) {
          returnBlocks.push(localBlock);
--- a/backend/src/api/bitcoin/esplora-api.interface.ts
+++ b/backend/src/api/bitcoin/esplora-api.interface.ts
@@ -6,6 +6,7 @@ export namespace IEsploraApi {
    size: number;
    weight: number;
    fee: number;
+    sigops?: number;
    vin: Vin[];
    vout: Vout[];
    status: Status;
--- a/backend/src/api/bitcoin/esplora-api.ts
+++ b/backend/src/api/bitcoin/esplora-api.ts
@@ -75,9 +75,9 @@ class FailoverRouter {

    const results = await Promise.allSettled(this.hosts.map(async (host) => {
      if (host.socket) {
-        return this.pollConnection.get<number>('/blocks/tip/height', { socketPath: host.host, timeout: 5000 });
+        return this.pollConnection.get<number>('/blocks/tip/height', { socketPath: host.host, timeout: config.ESPLORA.FALLBACK_TIMEOUT });
      } else {
-        return this.pollConnection.get<number>(host.host + '/blocks/tip/height', { timeout: 5000 });
+        return this.pollConnection.get<number>(host.host + '/blocks/tip/height', { timeout: config.ESPLORA.FALLBACK_TIMEOUT });
      }
    }));
    const maxHeight = results.reduce((max, result) => Math.max(max, result.status === 'fulfilled' ? result.value?.data || 0 : 0), 0);
@@ -168,12 +168,15 @@ class FailoverRouter {
    let axiosConfig;
    let url;
    if (host.socket) {
-      axiosConfig = { socketPath: host.host, timeout: 10000, responseType };
+      axiosConfig = { socketPath: host.host, timeout: config.ESPLORA.REQUEST_TIMEOUT, responseType };
      url = path;
    } else {
-      axiosConfig = { timeout: 10000, responseType };
+      axiosConfig = { timeout: config.ESPLORA.REQUEST_TIMEOUT, responseType };
      url = host.host + path;
    }
+    if (data?.params) {
+      axiosConfig.params = data.params;
+    }
    return (method === 'post'
        ? this.requestConnection.post<T>(url, data, axiosConfig)
        : this.requestConnection.get<T>(url, axiosConfig)
@@ -181,7 +184,8 @@ class FailoverRouter {
      .catch((e) => {
        let fallbackHost = this.fallbackHost;
        if (e?.response?.status !== 404) {
-          logger.warn(`esplora request failed ${e?.response?.status || 500} ${host.host}${path}`);
+          logger.warn(`esplora request failed ${e?.response?.status} ${host.host}${path}`);
+          logger.warn(e instanceof Error ? e.message : e);
          fallbackHost = this.addFailure(host);
        }
        if (retry && e?.code === 'ECONNREFUSED' && this.multihost) {
@@ -193,8 +197,8 @@ class FailoverRouter {
      });
  }

-  public async $get<T>(path, responseType = 'json'): Promise<T> {
-    return this.$query<T>('get', path, null, responseType);
+  public async $get<T>(path, responseType = 'json', params: any = null): Promise<T> {
+    return this.$query<T>('get', path, params ? { params } : null, responseType);
  }

  public async $post<T>(path, data: any, responseType = 'json'): Promise<T> {
@@ -213,12 +217,16 @@ class ElectrsApi implements AbstractBitcoinApi {
    return this.failoverRouter.$get<IEsploraApi.Transaction>('/tx/' + txId);
  }

+  async $getRawTransactions(txids: string[]): Promise<IEsploraApi.Transaction[]> {
+    return this.failoverRouter.$post<IEsploraApi.Transaction[]>('/internal/txs', txids, 'json');
+  }
+
  async $getMempoolTransactions(txids: string[]): Promise<IEsploraApi.Transaction[]> {
-    return this.failoverRouter.$post<IEsploraApi.Transaction[]>('/mempool/txs', txids, 'json');
+    return this.failoverRouter.$post<IEsploraApi.Transaction[]>('/internal/mempool/txs', txids, 'json');
  }

  async $getAllMempoolTransactions(lastSeenTxid?: string): Promise<IEsploraApi.Transaction[]> {
-    return this.failoverRouter.$get<IEsploraApi.Transaction[]>('/mempool/txs' + (lastSeenTxid ? '/' + lastSeenTxid : ''));
+    return this.failoverRouter.$get<IEsploraApi.Transaction[]>('/internal/mempool/txs' + (lastSeenTxid ? '/' + lastSeenTxid : ''));
  }

  $getTransactionHex(txId: string): Promise<string> {
@@ -238,7 +246,7 @@ class ElectrsApi implements AbstractBitcoinApi {
  }

  $getTxsForBlock(hash: string): Promise<IEsploraApi.Transaction[]> {
-    return this.failoverRouter.$get<IEsploraApi.Transaction[]>('/block/' + hash + '/txs');
+    return this.failoverRouter.$get<IEsploraApi.Transaction[]>('/internal/block/' + hash + '/txs');
  }

  $getBlockHash(height: number): Promise<string> {
@@ -290,13 +298,16 @@ class ElectrsApi implements AbstractBitcoinApi {
    return this.failoverRouter.$get<IEsploraApi.Outspend[]>('/tx/' + txId + '/outspends');
  }

-  async $getBatchedOutspends(txId: string[]): Promise<IEsploraApi.Outspend[][]> {
-    const outspends: IEsploraApi.Outspend[][] = [];
-    for (const tx of txId) {
-      const outspend = await this.$getOutspends(tx);
-      outspends.push(outspend);
-    }
-    return outspends;
+  async $getBatchedOutspends(txids: string[]): Promise<IEsploraApi.Outspend[][]> {
+    throw new Error('Method not implemented.');
+  }
+
+  async $getBatchedOutspendsInternal(txids: string[]): Promise<IEsploraApi.Outspend[][]> {
+    return this.failoverRouter.$post<IEsploraApi.Outspend[][]>('/internal/txs/outspends/by-txid', txids, 'json');
+  }
+
+  async $getOutSpendsByOutpoint(outpoints: { txid: string, vout: number }[]): Promise<IEsploraApi.Outspend[]> {
+    return this.failoverRouter.$post<IEsploraApi.Outspend[]>('/internal/txs/outspends/by-outpoint', outpoints.map(out => `${out.txid}:${out.vout}`), 'json');
  }

  public startHealthChecks(): void {
--- a/backend/src/api/blocks.ts
+++ b/backend/src/api/blocks.ts
@@ -81,6 +81,7 @@ class Blocks {
  private async $getTransactionsExtended(
    blockHash: string,
    blockHeight: number,
+    blockTime: number,
    onlyCoinbase: boolean,
    txIds: string[] | null = null,
    quiet: boolean = false,
@@ -101,6 +102,12 @@ class Blocks {
    if (!onlyCoinbase) {
      for (const txid of txIds) {
        if (mempool[txid]) {
+          mempool[txid].status = {
+            confirmed: true,
+            block_height: blockHeight,
+            block_hash: blockHash,
+            block_time: blockTime,
+          };
          transactionMap[txid] = mempool[txid];
          foundInMempool++;
          totalFound++;
@@ -608,7 +615,7 @@ class Blocks {
          }
          const blockHash = await bitcoinApi.$getBlockHash(blockHeight);
          const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash);
-          const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, null, true);
+          const transactions = await this.$getTransactionsExtended(blockHash, block.height, block.timestamp, true, null, true);
          const blockExtended = await this.$getBlockExtended(block, transactions);

          newlyIndexed++;
@@ -701,7 +708,7 @@ class Blocks {
      const verboseBlock = await bitcoinClient.getBlock(blockHash, 2);
      const block = BitcoinApi.convertBlock(verboseBlock);
      const txIds: string[] = verboseBlock.tx.map(tx => tx.txid);
-      const transactions = await this.$getTransactionsExtended(blockHash, block.height, false, txIds, false, true) as MempoolTransactionExtended[];
+      const transactions = await this.$getTransactionsExtended(blockHash, block.height, block.timestamp, false, txIds, false, true) as MempoolTransactionExtended[];

      // fill in missing transaction fee data from verboseBlock
      for (let i = 0; i < transactions.length; i++) {
@@ -890,7 +897,7 @@ class Blocks {

    const blockHash = await bitcoinApi.$getBlockHash(height);
    const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash);
-    const transactions = await this.$getTransactionsExtended(blockHash, block.height, true);
+    const transactions = await this.$getTransactionsExtended(blockHash, block.height, block.timestamp, true);
    const blockExtended = await this.$getBlockExtended(block, transactions);

    if (Common.indexingEnabled()) {
@@ -902,7 +909,7 @@ class Blocks {

  public async $indexStaleBlock(hash: string): Promise<BlockExtended> {
    const block: IEsploraApi.Block = await bitcoinApi.$getBlock(hash);
-    const transactions = await this.$getTransactionsExtended(hash, block.height, true);
+    const transactions = await this.$getTransactionsExtended(hash, block.height, block.timestamp, true);
    const blockExtended = await this.$getBlockExtended(block, transactions);

    blockExtended.canonical = await bitcoinApi.$getBlockHash(block.height);
--- a/backend/src/api/disk-cache.ts
+++ b/backend/src/api/disk-cache.ts
@@ -252,7 +252,11 @@ class DiskCache {
      }

      if (rbfData?.rbf) {
-        rbfCache.load(rbfData.rbf);
+        rbfCache.load({
+          txs: rbfData.rbf.txs.map(([txid, entry]) => ({ value: entry })),
+          trees: rbfData.rbf.trees,
+          expiring: rbfData.rbf.expiring.map(([txid, value]) => ({ key: txid, value })),
+        });
      }
    } catch (e) {
      logger.warn('Failed to parse rbf cache. Skipping. Reason: ' + (e instanceof Error ? e.message : e));
--- a/backend/src/api/fee-api.ts
+++ b/backend/src/api/fee-api.ts
@@ -3,21 +3,30 @@ import { Common } from './common';
 import mempool from './mempool';
 import projectedBlocks from './mempool-blocks';

+interface RecommendedFees {
+  fastestFee: number,
+  halfHourFee: number,
+  hourFee: number,
+  economyFee: number,
+  minimumFee: number,
+}
+
 class FeeApi {
  constructor() { }

  defaultFee = Common.isLiquid() ? 0.1 : 1;

-  public getRecommendedFee() {
+  public getRecommendedFee(): RecommendedFees {
    const pBlocks = projectedBlocks.getMempoolBlocks();
    const mPool = mempool.getMempoolInfo();
    const minimumFee = Math.ceil(mPool.mempoolminfee * 100000);
+    const defaultMinFee = Math.max(minimumFee, this.defaultFee);

    if (!pBlocks.length) {
      return {
-        'fastestFee': this.defaultFee,
-        'halfHourFee': this.defaultFee,
-        'hourFee': this.defaultFee,
+        'fastestFee': defaultMinFee,
+        'halfHourFee': defaultMinFee,
+        'hourFee': defaultMinFee,
        'economyFee': minimumFee,
        'minimumFee': minimumFee,
      };
@@ -27,11 +36,15 @@ class FeeApi {
    const secondMedianFee = pBlocks[1] ? this.optimizeMedianFee(pBlocks[1], pBlocks[2], firstMedianFee) : this.defaultFee;
    const thirdMedianFee = pBlocks[2] ? this.optimizeMedianFee(pBlocks[2], pBlocks[3], secondMedianFee) : this.defaultFee;

+    // explicitly enforce a minimum of ceil(mempoolminfee) on all recommendations.
+    // simply rounding up recommended rates is insufficient, as the purging rate
+    // can exceed the median rate of projected blocks in some extreme scenarios
+    // (see https://bitcoin.stackexchange.com/a/120024)
    return {
-      'fastestFee': firstMedianFee,
-      'halfHourFee': secondMedianFee,
-      'hourFee': thirdMedianFee,
-      'economyFee': Math.min(2 * minimumFee, thirdMedianFee),
+      'fastestFee': Math.max(minimumFee, firstMedianFee),
+      'halfHourFee': Math.max(minimumFee, secondMedianFee),
+      'hourFee': Math.max(minimumFee, thirdMedianFee),
+      'economyFee': Math.max(minimumFee, Math.min(2 * minimumFee, thirdMedianFee)),
      'minimumFee': minimumFee,
    };
  }
--- a/backend/src/api/memory-cache.ts
+++ b/backend/src/api/memory-cache.ts
@@ -31,7 +31,7 @@ class MemoryCache {
  }

  private cleanup() {
-    this.cache = this.cache.filter((cache) => cache.expires < (new Date()));
+    this.cache = this.cache.filter((cache) => cache.expires > (new Date()));
  }
 }

--- a/backend/src/api/mempool.ts
+++ b/backend/src/api/mempool.ts
@@ -94,7 +94,7 @@ class Mempool {
      logger.debug(`Migrating ${Object.keys(this.mempoolCache).length} transactions from disk cache to Redis cache`);
    }
    for (const txid of Object.keys(this.mempoolCache)) {
-      if (!this.mempoolCache[txid].sigops || this.mempoolCache[txid].effectiveFeePerVsize == null) {
+      if (!this.mempoolCache[txid].adjustedVsize || this.mempoolCache[txid].sigops == null || this.mempoolCache[txid].effectiveFeePerVsize == null) {
        this.mempoolCache[txid] = transactionUtils.extendMempoolTransaction(this.mempoolCache[txid]);
      }
      if (this.mempoolCache[txid].order == null) {
--- a/backend/src/api/rbf-cache.ts
+++ b/backend/src/api/rbf-cache.ts
@@ -2,6 +2,7 @@ import config from "../config";
 import logger from "../logger";
 import { MempoolTransactionExtended, TransactionStripped } from "../mempool.interfaces";
 import bitcoinApi from './bitcoin/bitcoin-api-factory';
+import { IEsploraApi } from "./bitcoin/esplora-api.interface";
 import { Common } from "./common";
 import redisCache from "./redis-cache";

@@ -53,6 +54,9 @@ class RbfCache {
  private expiring: Map<string, number> = new Map();
  private cacheQueue: CacheEvent[] = [];

+  private evictionCount = 0;
+  private staleCount = 0;
+
  constructor() {
    setInterval(this.cleanup.bind(this), 1000 * 60 * 10);
  }
@@ -245,6 +249,7 @@ class RbfCache {

  // flag a transaction as removed from the mempool
  public evict(txid: string, fast: boolean = false): void {
+    this.evictionCount++;
    if (this.txs.has(txid) && (fast || !this.expiring.has(txid))) {
      const expiryTime = fast ? Date.now() + (1000 * 60 * 10) : Date.now() + (1000 * 86400); // 24 hours
      this.addExpiration(txid, expiryTime);
@@ -272,18 +277,23 @@ class RbfCache {
        this.remove(txid);
      }
    }
-    logger.debug(`rbf cache contains ${this.txs.size} txs, ${this.rbfTrees.size} trees, ${this.expiring.size} due to expire`);
+    logger.debug(`rbf cache contains ${this.txs.size} txs, ${this.rbfTrees.size} trees, ${this.expiring.size} due to expire (${this.evictionCount} newly expired)`);
+    this.evictionCount = 0;
  }

  // remove a transaction & all previous versions from the cache
  private remove(txid): void {
    // don't remove a transaction if a newer version remains in the mempool
    if (!this.replacedBy.has(txid)) {
+      const root = this.treeMap.get(txid);
      const replaces = this.replaces.get(txid);
      this.replaces.delete(txid);
      this.treeMap.delete(txid);
      this.removeTx(txid);
      this.removeExpiration(txid);
+      if (root === txid) {
+        this.removeTree(txid);
+      }
      for (const tx of (replaces || [])) {
        // recursively remove prior versions from the cache
        this.replacedBy.delete(tx);
@@ -359,18 +369,27 @@ class RbfCache {
  }

  public async load({ txs, trees, expiring }): Promise<void> {
-    txs.forEach(txEntry => {
-      this.txs.set(txEntry.key, txEntry.value);
-    });
-    for (const deflatedTree of trees) {
-      await this.importTree(deflatedTree.root, deflatedTree.root, deflatedTree, this.txs);
-    }
-    expiring.forEach(expiringEntry => {
-      if (this.txs.has(expiringEntry.key)) {
-        this.expiring.set(expiringEntry.key, new Date(expiringEntry.value).getTime());
+    try {
+      txs.forEach(txEntry => {
+        this.txs.set(txEntry.value.txid, txEntry.value);
+      });
+      this.staleCount = 0;
+      for (const deflatedTree of trees) {
+        await this.importTree(deflatedTree.root, deflatedTree.root, deflatedTree, this.txs);
      }
-    });
-    this.cleanup();
+      expiring.forEach(expiringEntry => {
+        if (this.txs.has(expiringEntry.key)) {
+          this.expiring.set(expiringEntry.key, new Date(expiringEntry.value).getTime());
+        }
+      });
+      this.staleCount = 0;
+      await this.checkTrees();
+      logger.debug(`loaded ${txs.length} txs, ${trees.length} trees into rbf cache, ${expiring.length} due to expire, ${this.staleCount} were stale`);
+      this.cleanup();
+
+    } catch (e) {
+      logger.err('failed to restore RBF cache: ' + (e instanceof Error ? e.message : e));
+    }
  }

  exportTree(tree: RbfTree, deflated: any = null) {
@@ -398,29 +417,11 @@ class RbfCache {
    const treeInfo = deflated[txid];
    const replaces: RbfTree[] = [];

-    // check if any transactions in this tree have already been confirmed
-    mined = mined || treeInfo.mined;
-    let exists = mined;
-    if (!mined) {
-      try {
-        const apiTx = await bitcoinApi.$getRawTransaction(txid);
-        if (apiTx) {
-          exists = true;
-        }
-        if (apiTx?.status?.confirmed) {
-          mined = true;
-          treeInfo.txMined = true;
-          this.evict(txid, true);
-        }
-      } catch (e) {
-        // most transactions do not exist
-      }
-    }
-
-    // if the root tx is not in the mempool or the blockchain
-    // evict this tree as soon as possible
-    if (root === txid && !exists) {
-      this.evict(txid, true);
+    // if the root tx is unknown, remove this tree and return early
+    if (root === txid && !txs.has(txid)) {
+      this.staleCount++;
+      this.removeTree(deflated.key);
+      return;
    }

    // recursively reconstruct child trees
@@ -458,6 +459,59 @@ class RbfCache {
    return tree;
  }

+  private async checkTrees(): Promise<void> {
+    const found: { [txid: string]: boolean } = {};
+    const txids = Array.from(this.txs.values()).map(tx => tx.txid).filter(txid => {
+      return !this.expiring.has(txid) && !this.getRbfTree(txid)?.mined;
+    });
+
+    const processTxs = (txs: IEsploraApi.Transaction[]): void => {
+      for (const tx of txs) {
+        found[tx.txid] = true;
+        if (tx.status?.confirmed) {
+          const tree = this.getRbfTree(tx.txid);
+          if (tree) {
+            this.setTreeMined(tree, tx.txid);
+            tree.mined = true;
+            this.evict(tx.txid, false);
+          }
+        }
+      }
+    };
+
+    if (config.MEMPOOL.BACKEND === 'esplora') {
+      const sliceLength = 250;
+      for (let i = 0; i < Math.ceil(txids.length / sliceLength); i++) {
+        const slice = txids.slice(i * sliceLength, (i + 1) * sliceLength);
+        try {
+          const txs = await bitcoinApi.$getRawTransactions(slice);
+          logger.debug(`fetched ${slice.length} cached rbf transactions`);
+          processTxs(txs);
+          logger.debug(`processed ${slice.length} cached rbf transactions`);
+        } catch (err) {
+          logger.err(`failed to fetch or process ${slice.length} cached rbf transactions`);
+        }
+      }
+    } else {
+      const txs: IEsploraApi.Transaction[] = [];
+      for (const txid of txids) {
+        try {
+          const tx = await bitcoinApi.$getRawTransaction(txid, true, false);
+          txs.push(tx);
+        } catch (err) {
+          // some 404s are expected, so continue quietly
+        }
+      }
+      processTxs(txs);
+    }
+
+    for (const txid of txids) {
+      if (!found[txid]) {
+        this.evict(txid, false);
+      }
+    }
+  }
+
  public getLatestRbfSummary(): ReplacementInfo[] {
    const rbfList = this.getRbfTrees(false);
    return rbfList.slice(0, 6).map(rbfTree => {
--- a/backend/src/api/redis-cache.ts
+++ b/backend/src/api/redis-cache.ts
@@ -219,7 +219,7 @@ class RedisCache {
    await memPool.$setMempool(loadedMempool);
    await rbfCache.load({
      txs: rbfTxs,
-      trees: rbfTrees.map(loadedTree => loadedTree.value),
+      trees: rbfTrees.map(loadedTree => { loadedTree.value.key = loadedTree.key; return loadedTree.value; }),
      expiring: rbfExpirations,
    });
  }
--- a/backend/src/api/transaction-utils.ts
+++ b/backend/src/api/transaction-utils.ts
@@ -116,7 +116,7 @@ class TransactionUtils {
  public extendMempoolTransaction(transaction: IEsploraApi.Transaction): MempoolTransactionExtended {
    const vsize = Math.ceil(transaction.weight / 4);
    const fractionalVsize = (transaction.weight / 4);
-    const sigops = !Common.isLiquid() ? this.countSigops(transaction) : 0;
+    let sigops = Common.isLiquid() ? 0 : (transaction.sigops != null ? transaction.sigops : this.countSigops(transaction));
    // https://github.com/bitcoin/bitcoin/blob/e9262ea32a6e1d364fb7974844fadc36f931f8c6/src/policy/policy.cpp#L295-L298
    const adjustedVsize = Math.max(fractionalVsize, sigops *  5); // adjusted vsize = Max(weight, sigops * bytes_per_sigop) / witness_scale_factor
    const feePerVbytes = (transaction.fee || 0) / fractionalVsize;
@@ -155,7 +155,7 @@ class TransactionUtils {
      sigops += 20 * (script.match(/OP_CHECKMULTISIG/g)?.length || 0);
    } else {
      // in redeem scripts and witnesses, worth N if preceded by OP_N, 20 otherwise
-      const matches = script.matchAll(/(?:OP_(\d+))? OP_CHECKMULTISIG/g);
+      const matches = script.matchAll(/(?:OP_(?:PUSHNUM_)?(\d+))? OP_CHECKMULTISIG/g);
      for (const match of matches) {
        const n = parseInt(match[1]);
        if (Number.isInteger(n)) {
@@ -189,6 +189,12 @@ class TransactionUtils {
              sigops += this.countScriptSigops(bitcoinjs.script.toASM(Buffer.from(input.witness[input.witness.length - 1], 'hex')), false, true);
            }
            break;
+
+          case input.prevout.scriptpubkey_type === 'p2sh':
+            if (input.inner_redeemscript_asm) {
+              sigops += this.countScriptSigops(input.inner_redeemscript_asm);
+            }
+            break;
        }
      }
    }
--- a/backend/src/api/websocket-handler.ts
+++ b/backend/src/api/websocket-handler.ts
@@ -577,7 +577,7 @@ class WebsocketHandler {
          response['utxoSpent'] = JSON.stringify(outspends);
        }

-        const rbfReplacedBy = rbfCache.getReplacedBy(client['track-tx']);
+        const rbfReplacedBy = rbfChanges.map[client['track-tx']] ? rbfCache.getReplacedBy(client['track-tx']) : false;
        if (rbfReplacedBy) {
          response['rbfTransaction'] = JSON.stringify({
            txid: rbfReplacedBy,
--- a/backend/src/config.ts
+++ b/backend/src/config.ts
@@ -44,6 +44,8 @@ interface IConfig {
    REST_API_URL: string;
    UNIX_SOCKET_PATH: string | void | null;
    RETRY_UNIX_SOCKET_AFTER: number;
+    REQUEST_TIMEOUT: number;
+    FALLBACK_TIMEOUT: number;
    FALLBACK: string[];
  };
  LIGHTNING: {
@@ -76,6 +78,8 @@ interface IConfig {
    USERNAME: string;
    PASSWORD: string;
    TIMEOUT: number;
+    COOKIE: boolean;
+    COOKIE_PATH: string;
  };
  SECOND_CORE_RPC: {
    HOST: string;
@@ -83,6 +87,8 @@ interface IConfig {
    USERNAME: string;
    PASSWORD: string;
    TIMEOUT: number;
+    COOKIE: boolean;
+    COOKIE_PATH: string;
  };
  DATABASE: {
    ENABLED: boolean;
@@ -190,6 +196,8 @@ const defaults: IConfig = {
    'REST_API_URL': 'http://127.0.0.1:3000',
    'UNIX_SOCKET_PATH': null,
    'RETRY_UNIX_SOCKET_AFTER': 30000,
+    'REQUEST_TIMEOUT': 10000,
+    'FALLBACK_TIMEOUT': 5000,
    'FALLBACK': [],
  },
  'ELECTRUM': {
@@ -203,6 +211,8 @@ const defaults: IConfig = {
    'USERNAME': 'mempool',
    'PASSWORD': 'mempool',
    'TIMEOUT': 60000,
+    'COOKIE': false,
+    'COOKIE_PATH': '/bitcoin/.cookie'
  },
  'SECOND_CORE_RPC': {
    'HOST': '127.0.0.1',
@@ -210,6 +220,8 @@ const defaults: IConfig = {
    'USERNAME': 'mempool',
    'PASSWORD': 'mempool',
    'TIMEOUT': 60000,
+    'COOKIE': false,
+    'COOKIE_PATH': '/bitcoin/.cookie'
  },
  'DATABASE': {
    'ENABLED': true,
--- a/backend/src/database.ts
+++ b/backend/src/database.ts
@@ -4,6 +4,7 @@ import config from './config';
 import { createPool, Pool, PoolConnection } from 'mysql2/promise';
 import logger from './logger';
 import { FieldPacket, OkPacket, PoolOptions, ResultSetHeader, RowDataPacket } from 'mysql2/typings/mysql';
+import { execSync } from 'child_process';

 class DB {
  constructor() {
@@ -105,26 +106,43 @@ import { FieldPacket, OkPacket, PoolOptions, ResultSetHeader, RowDataPacket } fr

  public getPidLock(): boolean {
    const filePath = path.join(config.DATABASE.PID_DIR || __dirname, `/mempool-${config.DATABASE.DATABASE}.pid`);
+    this.enforcePidLock(filePath);
+    fs.writeFileSync(filePath, `${process.pid}`);
+    return true;
+  }
+
+  private enforcePidLock(filePath: string): void {
    if (fs.existsSync(filePath)) {
-      const pid = fs.readFileSync(filePath).toString();
-      if (pid !== `${process.pid}`) {
-        const msg = `Already running on PID ${pid} (or pid file '${filePath}' is stale)`;
+      const pid = parseInt(fs.readFileSync(filePath, 'utf-8'));
+      if (pid === process.pid) {
+        logger.warn('PID file already exists for this process');
+        return;
+      }
+
+      let cmd;
+      try {
+        cmd = execSync(`ps -p ${pid} -o args=`);
+      } catch (e) {
+        logger.warn(`Stale PID file at ${filePath}, but no process running on that PID ${pid}`);
+        return;
+      }
+
+      if (cmd && cmd.toString()?.includes('node')) {
+        const msg = `Another mempool nodejs process is already running on PID ${pid}`;
        logger.err(msg);
        throw new Error(msg);
      } else {
-        return true;
+        logger.warn(`Stale PID file at ${filePath}, but the PID ${pid} does not belong to a running mempool instance`);
      }
-    } else {
-      fs.writeFileSync(filePath, `${process.pid}`);
-      return true;
    }
  }

  public releasePidLock(): void {
    const filePath = path.join(config.DATABASE.PID_DIR || __dirname, `/mempool-${config.DATABASE.DATABASE}.pid`);
    if (fs.existsSync(filePath)) {
-      const pid = fs.readFileSync(filePath).toString();
-      if (pid === `${process.pid}`) {
+      const pid = parseInt(fs.readFileSync(filePath, 'utf-8'));
+      // only release our own pid file
+      if (pid === process.pid) {
        fs.unlinkSync(filePath);
      }
    }
--- a/backend/src/index.ts
+++ b/backend/src/index.ts
@@ -92,7 +92,7 @@ class Server {
    logger.notice(`Starting Mempool Server${worker ? ' (worker)' : ''}... (${backendInfo.getShortCommitHash()})`);

    // Register cleanup listeners for exit events
-    ['exit', 'SIGINT', 'SIGTERM', 'SIGUSR1', 'SIGUSR2', 'uncaughtException', 'unhandledRejection'].forEach(event => {
+    ['exit', 'SIGHUP', 'SIGINT', 'SIGTERM', 'SIGUSR1', 'SIGUSR2', 'uncaughtException', 'unhandledRejection'].forEach(event => {
      process.on(event, () => { this.onExit(event); });
    });

--- a/backend/src/rpc-api/jsonrpc.ts
+++ b/backend/src/rpc-api/jsonrpc.ts
@@ -1,5 +1,6 @@
 var http = require('http')
 var https = require('https')
+import { readFileSync } from 'fs';

 var JsonRPC = function (opts) {
  // @ts-ignore
@@ -55,7 +56,13 @@ JsonRPC.prototype.call = function (method, params) {
    }

    // use HTTP auth if user and password set
-    if (this.opts.user && this.opts.pass) {
+    if (this.opts.cookie) {
+      if (!this.cachedCookie) {
+        this.cachedCookie = readFileSync(this.opts.cookie).toString();
+      }
+      // @ts-ignore
+      requestOptions.auth = this.cachedCookie;
+    } else if (this.opts.user && this.opts.pass) {
      // @ts-ignore
      requestOptions.auth = this.opts.user + ':' + this.opts.pass
    }
@@ -93,7 +100,7 @@ JsonRPC.prototype.call = function (method, params) {
      reject(err)
    })

-    request.on('response', function (response) {
+    request.on('response', (response) => {
      clearTimeout(reqTimeout)

      // We need to buffer the response chunks in a nonblocking way.
@@ -104,7 +111,7 @@ JsonRPC.prototype.call = function (method, params) {
      // When all the responses are finished, we decode the JSON and
      // depending on whether it's got a result or an error, we call
      // emitSuccess or emitError on the promise.
-      response.on('end', function () {
+      response.on('end', () => {
        var err

        if (cbCalled) return
@@ -113,6 +120,14 @@ JsonRPC.prototype.call = function (method, params) {
        try {
          var decoded = JSON.parse(buffer)
        } catch (e) {
+          // if we authenticated using a cookie and it failed, read the cookie file again
+          if (
+            response.statusCode === 401 /* Unauthorized */ &&
+            this.opts.cookie
+          ) {
+            this.cachedCookie = undefined;
+          }
+
          if (response.statusCode !== 200) {
            err = new Error('Invalid params, response status code: ' + response.statusCode)
            err.code = -32602
--- a/backend/src/tasks/lightning/forensics.service.ts
+++ b/backend/src/tasks/lightning/forensics.service.ts
@@ -15,8 +15,6 @@ class ForensicsService {
  txCache: { [txid: string]: IEsploraApi.Transaction } = {};
  tempCached: string[] = [];

-  constructor() {}
-
  public async $startService(): Promise<void> {
    logger.info('Starting lightning network forensics service');

@@ -66,93 +64,138 @@ class ForensicsService {
  */

  public async $runClosedChannelsForensics(onlyNewChannels: boolean = false): Promise<void> {
+    // Only Esplora backend can retrieve spent transaction outputs
    if (config.MEMPOOL.BACKEND !== 'esplora') {
      return;
    }

-    let progress = 0;
-
    try {
      logger.debug(`Started running closed channel forensics...`);
-      let channels;
+      let allChannels;
      if (onlyNewChannels) {
-        channels = await channelsApi.$getClosedChannelsWithoutReason();
+        allChannels = await channelsApi.$getClosedChannelsWithoutReason();
      } else {
-        channels = await channelsApi.$getUnresolvedClosedChannels();
+        allChannels = await channelsApi.$getUnresolvedClosedChannels();
      }

-      for (const channel of channels) {
-        let reason = 0;
-        let resolvedForceClose = false;
-        // Only Esplora backend can retrieve spent transaction outputs
-        const cached: string[] = [];
+      let progress = 0;
+      const sliceLength = 1000;
+      // process batches of 1000 channels
+      for (let i = 0; i < Math.ceil(allChannels.length / sliceLength); i++) {
+        const channels = allChannels.slice(i * sliceLength, (i + 1) * sliceLength);
+
+        let allOutspends: IEsploraApi.Outspend[][] = [];
+        const forceClosedChannels: { channel: any, cachedSpends: string[] }[] = [];
+
+        // fetch outspends in bulk
        try {
-          let outspends: IEsploraApi.Outspend[] | undefined;
-          try {
-            outspends = await bitcoinApi.$getOutspends(channel.closing_transaction_id);
-            await Common.sleep$(config.LIGHTNING.FORENSICS_RATE_LIMIT);
-          } catch (e) {
-            logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + channel.closing_transaction_id + '/outspends'}. Reason ${e instanceof Error ? e.message : e}`);
-            continue;
-          }
-          const lightningScriptReasons: number[] = [];
+          const outspendTxids = channels.map(channel => channel.closing_transaction_id);
+          allOutspends = await bitcoinApi.$getBatchedOutspendsInternal(outspendTxids);
+          logger.info(`Fetched outspends for ${allOutspends.length} txs from esplora for LN forensics`);
+          await Common.sleep$(config.LIGHTNING.FORENSICS_RATE_LIMIT);
+        } catch (e) {
+          logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/internal/txs/outspends/by-txid'}. Reason ${e instanceof Error ? e.message : e}`);
+        }
+        // fetch spending transactions in bulk and load into txCache
+        const newSpendingTxids: { [txid: string]: boolean } = {};
+        for (const outspends of allOutspends) {
          for (const outspend of outspends) {
            if (outspend.spent && outspend.txid) {
-              let spendingTx = await this.fetchTransaction(outspend.txid);
-              if (!spendingTx) {
-                continue;
-              }
-              cached.push(spendingTx.txid);
-              const lightningScript = this.findLightningScript(spendingTx.vin[outspend.vin || 0]);
-              lightningScriptReasons.push(lightningScript);
+              newSpendingTxids[outspend.txid] = true;
            }
          }
-          const filteredReasons = lightningScriptReasons.filter((r) => r !== 1);
-          if (filteredReasons.length) {
-            if (filteredReasons.some((r) => r === 2 || r === 4)) {
-              reason = 3;
-            } else {
-              reason = 2;
-              resolvedForceClose = true;
-            }
-          } else {
-            /*
-              We can detect a commitment transaction (force close) by reading Sequence and Locktime
-              https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction
-            */
-            let closingTx = await this.fetchTransaction(channel.closing_transaction_id, true);
-            if (!closingTx) {
+        }
+        const allOutspendTxs = await this.fetchTransactions(
+          allOutspends.flatMap(outspends =>
+            outspends
+              .filter(outspend => outspend.spent && outspend.txid)
+              .map(outspend => outspend.txid)
+          )
+        );
+        logger.info(`Fetched ${allOutspendTxs.length} out-spending txs from esplora for LN forensics`);
+
+        // process each outspend
+        for (const [index, channel] of channels.entries()) {
+          let reason = 0;
+          const cached: string[] = [];
+          try {
+            const outspends = allOutspends[index];
+            if (!outspends || !outspends.length) {
+              // outspends are missing
              continue;
            }
-            cached.push(closingTx.txid);
-            const sequenceHex: string = closingTx.vin[0].sequence.toString(16);
-            const locktimeHex: string = closingTx.locktime.toString(16);
-            if (sequenceHex.substring(0, 2) === '80' && locktimeHex.substring(0, 2) === '20') {
-              reason = 2; // Here we can't be sure if it's a penalty or not
-            } else {
-              reason = 1;
+            const lightningScriptReasons: number[] = [];
+            for (const outspend of outspends) {
+              if (outspend.spent && outspend.txid) {
+                const spendingTx = this.txCache[outspend.txid];
+                if (!spendingTx) {
+                  continue;
+                }
+                cached.push(spendingTx.txid);
+                const lightningScript = this.findLightningScript(spendingTx.vin[outspend.vin || 0]);
+                lightningScriptReasons.push(lightningScript);
+              }
            }
-          }
-          if (reason) {
-            logger.debug('Setting closing reason ' + reason + ' for channel: ' + channel.id + '.');
-            await DB.query(`UPDATE channels SET closing_reason = ? WHERE id = ?`, [reason, channel.id]);
-            if (reason === 2 && resolvedForceClose) {
-              await DB.query(`UPDATE channels SET closing_resolved = ? WHERE id = ?`, [true, channel.id]);
-            }
-            if (reason !== 2 || resolvedForceClose) {
+            const filteredReasons = lightningScriptReasons.filter((r) => r !== 1);
+            if (filteredReasons.length) {
+              if (filteredReasons.some((r) => r === 2 || r === 4)) {
+                // Force closed with penalty
+                reason = 3;
+              } else {
+                // Force closed without penalty
+                reason = 2;
+                await DB.query(`UPDATE channels SET closing_resolved = ? WHERE id = ?`, [true, channel.id]);
+              }
+              await DB.query(`UPDATE channels SET closing_reason = ? WHERE id = ?`, [reason, channel.id]);
+              // clean up cached transactions
              cached.forEach(txid => {
                delete this.txCache[txid];
              });
+            } else {
+              forceClosedChannels.push({ channel, cachedSpends: cached });
            }
+          } catch (e) {
+            logger.err(`$runClosedChannelsForensics() failed for channel ${channel.short_id}. Reason: ${e instanceof Error ? e.message : e}`);
          }
-        } catch (e) {
-          logger.err(`$runClosedChannelsForensics() failed for channel ${channel.short_id}. Reason: ${e instanceof Error ? e.message : e}`);
        }

-        ++progress;
+        // fetch force-closing transactions in bulk
+        const closingTxs = await this.fetchTransactions(forceClosedChannels.map(x => x.channel.closing_transaction_id));
+        logger.info(`Fetched ${closingTxs.length} closing txs from esplora for LN forensics`);
+
+        // process channels with no lightning script reasons
+        for (const { channel, cachedSpends } of forceClosedChannels) {
+          const closingTx = this.txCache[channel.closing_transaction_id];
+          if (!closingTx) {
+            // no channel close transaction found yet
+            continue;
+          }
+          /*
+            We can detect a commitment transaction (force close) by reading Sequence and Locktime
+            https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction
+          */
+          const sequenceHex: string = closingTx.vin[0].sequence.toString(16);
+          const locktimeHex: string = closingTx.locktime.toString(16);
+          let reason;
+          if (sequenceHex.substring(0, 2) === '80' && locktimeHex.substring(0, 2) === '20') {
+            // Force closed, but we can't be sure if it's a penalty or not
+            reason = 2;
+          } else {
+            // Mutually closed
+            reason = 1;
+            // clean up cached transactions
+            delete this.txCache[closingTx.txid];
+            for (const txid of cachedSpends) {
+              delete this.txCache[txid];
+            }
+          }
+          await DB.query(`UPDATE channels SET closing_reason = ? WHERE id = ?`, [reason, channel.id]);
+        }
+
+        progress += channels.length;
        const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
        if (elapsedSeconds > 10) {
-          logger.debug(`Updating channel closed channel forensics ${progress}/${channels.length}`);
+          logger.debug(`Updating channel closed channel forensics ${progress}/${allChannels.length}`);
          this.loggerTimer = new Date().getTime() / 1000;
        }
      }
@@ -220,8 +263,11 @@ class ForensicsService {
      logger.debug(`Started running open channel forensics...`);
      const channels = await channelsApi.$getChannelsWithoutSourceChecked();

+      // preload open channel transactions
+      await this.fetchTransactions(channels.map(channel => channel.transaction_id), true);
+
      for (const openChannel of channels) {
-        let openTx = await this.fetchTransaction(openChannel.transaction_id, true);
+        const openTx = this.txCache[openChannel.transaction_id];
        if (!openTx) {
          continue;
        }
@@ -276,7 +322,7 @@ class ForensicsService {

  // Check if a channel open tx input spends the result of a swept channel close output
  private async $attributeSweptChannelCloses(openChannel: ILightningApi.Channel, input: IEsploraApi.Vin): Promise<void> {
-    let sweepTx = await this.fetchTransaction(input.txid, true);
+    const sweepTx = await this.fetchTransaction(input.txid, true);
    if (!sweepTx) {
      logger.err(`couldn't find input transaction for channel forensics ${openChannel.channel_id} ${input.txid}`);
      return;
@@ -335,7 +381,7 @@ class ForensicsService {

    if (matched && !ambiguous) {
      // fetch closing channel transaction and perform forensics on the outputs
-      let prevChannelTx = await this.fetchTransaction(input.txid, true);
+      const prevChannelTx = await this.fetchTransaction(input.txid, true);
      let outspends: IEsploraApi.Outspend[] | undefined;
      try {
        outspends = await bitcoinApi.$getOutspends(input.txid);
@@ -355,17 +401,17 @@ class ForensicsService {
            };
          });
        }
+
+        // preload outspend transactions
+        await this.fetchTransactions(outspends.filter(o => o.spent && o.txid).map(o => o.txid), true);
+
        for (let i = 0; i < outspends?.length; i++) {
          const outspend = outspends[i];
          const output = prevChannel.outputs[i];
          if (outspend.spent && outspend.txid) {
-            try {
-              const spendingTx = await this.fetchTransaction(outspend.txid, true);
-              if (spendingTx) {
-                output.type = this.findLightningScript(spendingTx.vin[outspend.vin || 0]);
-              }
-            } catch (e) {
-              logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + outspend.txid}. Reason ${e instanceof Error ? e.message : e}`);
+            const spendingTx = this.txCache[outspend.txid];
+            if (spendingTx) {
+              output.type = this.findLightningScript(spendingTx.vin[outspend.vin || 0]);
            }
          } else {
            output.type = 0;
@@ -430,13 +476,36 @@ class ForensicsService {
        }
        await Common.sleep$(config.LIGHTNING.FORENSICS_RATE_LIMIT);
      } catch (e) {
-        logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + txid + '/outspends'}. Reason ${e instanceof Error ? e.message : e}`);
+        logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + txid}. Reason ${e instanceof Error ? e.message : e}`);
        return null;
      }
    }
    return tx;
  }

+  // fetches a batch of transactions and adds them to the txCache
+  // the returned list of txs does *not* preserve ordering or number
+  async fetchTransactions(txids, temp: boolean = false): Promise<(IEsploraApi.Transaction | null)[]> {
+    // deduplicate txids
+    const uniqueTxids = [...new Set<string>(txids)];
+    // filter out any transactions we already have in the cache
+    const needToFetch: string[] = uniqueTxids.filter(txid => !this.txCache[txid]);
+    try {
+      const txs = await bitcoinApi.$getRawTransactions(needToFetch);
+      for (const tx of txs) {
+        this.txCache[tx.txid] = tx;
+        if (temp) {
+          this.tempCached.push(tx.txid);
+        }
+      }
+      await Common.sleep$(config.LIGHTNING.FORENSICS_RATE_LIMIT);
+    } catch (e) {
+      logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/txs'}. Reason ${e instanceof Error ? e.message : e}`);
+      return [];
+    }
+    return txids.map(txid => this.txCache[txid]);
+  }
+
  clearTempCache(): void {
    for (const txid of this.tempCached) {
      delete this.txCache[txid];
--- a/backend/src/tasks/lightning/network-sync.service.ts
+++ b/backend/src/tasks/lightning/network-sync.service.ts
@@ -288,22 +288,32 @@ class NetworkSyncService {
      }
      logger.debug(`${log}`, logger.tags.ln);

-      const channels = await channelsApi.$getChannelsByStatus([0, 1]);
-      for (const channel of channels) {
-        const spendingTx = await bitcoinApi.$getOutspend(channel.transaction_id, channel.transaction_vout);
-        if (spendingTx.spent === true && spendingTx.status?.confirmed === true) {
-          logger.debug(`Marking channel: ${channel.id} as closed.`, logger.tags.ln);
-          await DB.query(`UPDATE channels SET status = 2, closing_date = FROM_UNIXTIME(?) WHERE id = ?`,
-            [spendingTx.status.block_time, channel.id]);
-          if (spendingTx.txid && !channel.closing_transaction_id) {
-            await DB.query(`UPDATE channels SET closing_transaction_id = ? WHERE id = ?`, [spendingTx.txid, channel.id]);
+      const allChannels = await channelsApi.$getChannelsByStatus([0, 1]);
+
+      const sliceLength = 5000;
+      // process batches of 5000 channels
+      for (let i = 0; i < Math.ceil(allChannels.length / sliceLength); i++) {
+        const channels = allChannels.slice(i * sliceLength, (i + 1) * sliceLength);
+        const outspends = await bitcoinApi.$getOutSpendsByOutpoint(channels.map(channel => {
+          return { txid: channel.transaction_id, vout: channel.transaction_vout };
+        }));
+
+        for (const [index, channel] of channels.entries()) {
+          const spendingTx = outspends[index];
+          if (spendingTx.spent === true && spendingTx.status?.confirmed === true) {
+            // logger.debug(`Marking channel: ${channel.id} as closed.`, logger.tags.ln);
+            await DB.query(`UPDATE channels SET status = 2, closing_date = FROM_UNIXTIME(?) WHERE id = ?`,
+              [spendingTx.status.block_time, channel.id]);
+            if (spendingTx.txid && !channel.closing_transaction_id) {
+              await DB.query(`UPDATE channels SET closing_transaction_id = ? WHERE id = ?`, [spendingTx.txid, channel.id]);
+            }
          }
        }

-        ++progress;
+        progress += channels.length;
        const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
        if (elapsedSeconds > config.LIGHTNING.LOGGER_UPDATE_INTERVAL) {
-          logger.debug(`Checking if channel has been closed ${progress}/${channels.length}`, logger.tags.ln);
+          logger.debug(`Checking if channel has been closed ${progress}/${allChannels.length}`, logger.tags.ln);
          this.loggerTimer = new Date().getTime() / 1000;
        }
      }