Merge branch 'master' into taproot-channels

2023-11-14 13:41:44 +09:00
parent 2bc3352785 cbe1ec4e72
commit 562cd5683a
111 changed files with 2487 additions and 1412 deletions
--- a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts
+++ b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts
@@ -3,6 +3,7 @@ import { IEsploraApi } from './esplora-api.interface';
 export interface AbstractBitcoinApi {
  $getRawMempool(): Promise<IEsploraApi.Transaction['txid'][]>;
  $getRawTransaction(txId: string, skipConversion?: boolean, addPrevout?: boolean, lazyPrevouts?: boolean): Promise<IEsploraApi.Transaction>;
+  $getRawTransactions(txids: string[]): Promise<IEsploraApi.Transaction[]>;
  $getMempoolTransactions(txids: string[]): Promise<IEsploraApi.Transaction[]>;
  $getAllMempoolTransactions(lastTxid: string);
  $getTransactionHex(txId: string): Promise<string>;
@@ -23,6 +24,8 @@ export interface AbstractBitcoinApi {
  $getOutspend(txId: string, vout: number): Promise<IEsploraApi.Outspend>;
  $getOutspends(txId: string): Promise<IEsploraApi.Outspend[]>;
  $getBatchedOutspends(txId: string[]): Promise<IEsploraApi.Outspend[][]>;
+  $getBatchedOutspendsInternal(txId: string[]): Promise<IEsploraApi.Outspend[][]>;
+  $getOutSpendsByOutpoint(outpoints: { txid: string, vout: number }[]): Promise<IEsploraApi.Outspend[]>;

  startHealthChecks(): void;
 }
@@ -32,4 +35,5 @@ export interface BitcoinRpcCredentials {
  user: string;
  pass: string;
  timeout: number;
+  cookie?: string;
 }
--- a/backend/src/api/bitcoin/bitcoin-api.ts
+++ b/backend/src/api/bitcoin/bitcoin-api.ts
@@ -60,6 +60,19 @@ class BitcoinApi implements AbstractBitcoinApi {
      });
  }

+  async $getRawTransactions(txids: string[]): Promise<IEsploraApi.Transaction[]> {
+    const txs: IEsploraApi.Transaction[] = [];
+    for (const txid of txids) {
+      try {
+        const tx = await this.$getRawTransaction(txid, false, true);
+        txs.push(tx);
+      } catch (err) {
+        // skip failures
+      }
+    }
+    return txs;
+  }
+
  $getMempoolTransactions(txids: string[]): Promise<IEsploraApi.Transaction[]> {
    throw new Error('Method getMempoolTransactions not supported by the Bitcoin RPC API.');
  }
@@ -198,6 +211,19 @@ class BitcoinApi implements AbstractBitcoinApi {
    return outspends;
  }

+  async $getBatchedOutspendsInternal(txId: string[]): Promise<IEsploraApi.Outspend[][]> {
+    return this.$getBatchedOutspends(txId);
+  }
+
+  async $getOutSpendsByOutpoint(outpoints: { txid: string, vout: number }[]): Promise<IEsploraApi.Outspend[]> {
+    const outspends: IEsploraApi.Outspend[] = [];
+    for (const outpoint of outpoints) {
+      const outspend = await this.$getOutspend(outpoint.txid, outpoint.vout);
+      outspends.push(outspend);
+    }
+    return outspends;
+  }
+
  $getEstimatedHashrate(blockHeight: number): Promise<number> {
    // 120 is the default block span in Core
    return this.bitcoindClient.getNetworkHashPs(120, blockHeight);
--- a/backend/src/api/bitcoin/bitcoin-client.ts
+++ b/backend/src/api/bitcoin/bitcoin-client.ts
@@ -8,6 +8,7 @@ const nodeRpcCredentials: BitcoinRpcCredentials = {
  user: config.CORE_RPC.USERNAME,
  pass: config.CORE_RPC.PASSWORD,
  timeout: config.CORE_RPC.TIMEOUT,
+  cookie: config.CORE_RPC.COOKIE ? config.CORE_RPC.COOKIE_PATH : undefined,
 };

 export default new bitcoin.Client(nodeRpcCredentials);
--- a/backend/src/api/bitcoin/bitcoin-second-client.ts
+++ b/backend/src/api/bitcoin/bitcoin-second-client.ts
@@ -8,6 +8,7 @@ const nodeRpcCredentials: BitcoinRpcCredentials = {
  user: config.SECOND_CORE_RPC.USERNAME,
  pass: config.SECOND_CORE_RPC.PASSWORD,
  timeout: config.SECOND_CORE_RPC.TIMEOUT,
+  cookie: config.SECOND_CORE_RPC.COOKIE ? config.SECOND_CORE_RPC.COOKIE_PATH : undefined,
 };

 export default new bitcoin.Client(nodeRpcCredentials);
--- a/backend/src/api/bitcoin/bitcoin.routes.ts
+++ b/backend/src/api/bitcoin/bitcoin.routes.ts
@@ -24,7 +24,6 @@ class BitcoinRoutes {
  public initRoutes(app: Application) {
    app
      .get(config.MEMPOOL.API_URL_PREFIX + 'transaction-times', this.getTransactionTimes)
-      .get(config.MEMPOOL.API_URL_PREFIX + 'outspends', this.$getBatchedOutspends)
      .get(config.MEMPOOL.API_URL_PREFIX + 'cpfp/:txId', this.$getCpfpInfo)
      .get(config.MEMPOOL.API_URL_PREFIX + 'difficulty-adjustment', this.getDifficultyChange)
      .get(config.MEMPOOL.API_URL_PREFIX + 'fees/recommended', this.getRecommendedFees)
@@ -112,6 +111,7 @@ class BitcoinRoutes {
          .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/hex', this.getRawTransaction)
          .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/status', this.getTransactionStatus)
          .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/outspends', this.getTransactionOutspends)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'txs/outspends', this.$getBatchedOutspends)
          .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/header', this.getBlockHeader)
          .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/hash', this.getBlockTipHash)
          .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/raw', this.getRawBlock)
@@ -174,24 +174,20 @@ class BitcoinRoutes {
    res.json(times);
  }

-  private async $getBatchedOutspends(req: Request, res: Response) {
-    if (!Array.isArray(req.query.txId)) {
-      res.status(500).send('Not an array');
+  private async $getBatchedOutspends(req: Request, res: Response): Promise<IEsploraApi.Outspend[][] | void> {
+    const txids_csv = req.query.txids;
+    if (!txids_csv || typeof txids_csv !== 'string') {
+      res.status(500).send('Invalid txids format');
      return;
    }
-    if (req.query.txId.length > 50) {
+    const txids = txids_csv.split(',');
+    if (txids.length > 50) {
      res.status(400).send('Too many txids requested');
      return;
    }
-    const txIds: string[] = [];
-    for (const _txId in req.query.txId) {
-      if (typeof req.query.txId[_txId] === 'string') {
-        txIds.push(req.query.txId[_txId].toString());
-      }
-    }

    try {
-      const batchedOutspends = await bitcoinApi.$getBatchedOutspends(txIds);
+      const batchedOutspends = await bitcoinApi.$getBatchedOutspends(txids);
      res.json(batchedOutspends);
    } catch (e) {
      res.status(500).send(e instanceof Error ? e.message : e);
@@ -251,7 +247,7 @@ class BitcoinRoutes {

  private async getTransaction(req: Request, res: Response) {
    try {
-      const transaction = await transactionUtils.$getTransactionExtended(req.params.txId, true);
+      const transaction = await transactionUtils.$getTransactionExtended(req.params.txId, true, false, false, true);
      res.json(transaction);
    } catch (e) {
      let statusCode = 500;
@@ -478,7 +474,7 @@ class BitcoinRoutes {
      }

      let nextHash = startFromHash;
-      for (let i = 0; i < 10 && nextHash; i++) {
+      for (let i = 0; i < 15 && nextHash; i++) {
        const localBlock = blocks.getBlocks().find((b) => b.id === nextHash);
        if (localBlock) {
          returnBlocks.push(localBlock);
--- a/backend/src/api/bitcoin/esplora-api.interface.ts
+++ b/backend/src/api/bitcoin/esplora-api.interface.ts
@@ -6,6 +6,7 @@ export namespace IEsploraApi {
    size: number;
    weight: number;
    fee: number;
+    sigops?: number;
    vin: Vin[];
    vout: Vout[];
    status: Status;
--- a/backend/src/api/bitcoin/esplora-api.ts
+++ b/backend/src/api/bitcoin/esplora-api.ts
@@ -75,9 +75,9 @@ class FailoverRouter {

    const results = await Promise.allSettled(this.hosts.map(async (host) => {
      if (host.socket) {
-        return this.pollConnection.get<number>('/blocks/tip/height', { socketPath: host.host, timeout: 5000 });
+        return this.pollConnection.get<number>('/blocks/tip/height', { socketPath: host.host, timeout: config.ESPLORA.FALLBACK_TIMEOUT });
      } else {
-        return this.pollConnection.get<number>(host.host + '/blocks/tip/height', { timeout: 5000 });
+        return this.pollConnection.get<number>(host.host + '/blocks/tip/height', { timeout: config.ESPLORA.FALLBACK_TIMEOUT });
      }
    }));
    const maxHeight = results.reduce((max, result) => Math.max(max, result.status === 'fulfilled' ? result.value?.data || 0 : 0), 0);
@@ -168,12 +168,15 @@ class FailoverRouter {
    let axiosConfig;
    let url;
    if (host.socket) {
-      axiosConfig = { socketPath: host.host, timeout: 10000, responseType };
+      axiosConfig = { socketPath: host.host, timeout: config.ESPLORA.REQUEST_TIMEOUT, responseType };
      url = path;
    } else {
-      axiosConfig = { timeout: 10000, responseType };
+      axiosConfig = { timeout: config.ESPLORA.REQUEST_TIMEOUT, responseType };
      url = host.host + path;
    }
+    if (data?.params) {
+      axiosConfig.params = data.params;
+    }
    return (method === 'post'
        ? this.requestConnection.post<T>(url, data, axiosConfig)
        : this.requestConnection.get<T>(url, axiosConfig)
@@ -181,7 +184,8 @@ class FailoverRouter {
      .catch((e) => {
        let fallbackHost = this.fallbackHost;
        if (e?.response?.status !== 404) {
-          logger.warn(`esplora request failed ${e?.response?.status || 500} ${host.host}${path}`);
+          logger.warn(`esplora request failed ${e?.response?.status} ${host.host}${path}`);
+          logger.warn(e instanceof Error ? e.message : e);
          fallbackHost = this.addFailure(host);
        }
        if (retry && e?.code === 'ECONNREFUSED' && this.multihost) {
@@ -193,8 +197,8 @@ class FailoverRouter {
      });
  }

-  public async $get<T>(path, responseType = 'json'): Promise<T> {
-    return this.$query<T>('get', path, null, responseType);
+  public async $get<T>(path, responseType = 'json', params: any = null): Promise<T> {
+    return this.$query<T>('get', path, params ? { params } : null, responseType);
  }

  public async $post<T>(path, data: any, responseType = 'json'): Promise<T> {
@@ -213,12 +217,16 @@ class ElectrsApi implements AbstractBitcoinApi {
    return this.failoverRouter.$get<IEsploraApi.Transaction>('/tx/' + txId);
  }

+  async $getRawTransactions(txids: string[]): Promise<IEsploraApi.Transaction[]> {
+    return this.failoverRouter.$post<IEsploraApi.Transaction[]>('/internal/txs', txids, 'json');
+  }
+
  async $getMempoolTransactions(txids: string[]): Promise<IEsploraApi.Transaction[]> {
-    return this.failoverRouter.$post<IEsploraApi.Transaction[]>('/mempool/txs', txids, 'json');
+    return this.failoverRouter.$post<IEsploraApi.Transaction[]>('/internal/mempool/txs', txids, 'json');
  }

  async $getAllMempoolTransactions(lastSeenTxid?: string): Promise<IEsploraApi.Transaction[]> {
-    return this.failoverRouter.$get<IEsploraApi.Transaction[]>('/mempool/txs' + (lastSeenTxid ? '/' + lastSeenTxid : ''));
+    return this.failoverRouter.$get<IEsploraApi.Transaction[]>('/internal/mempool/txs' + (lastSeenTxid ? '/' + lastSeenTxid : ''));
  }

  $getTransactionHex(txId: string): Promise<string> {
@@ -238,7 +246,7 @@ class ElectrsApi implements AbstractBitcoinApi {
  }

  $getTxsForBlock(hash: string): Promise<IEsploraApi.Transaction[]> {
-    return this.failoverRouter.$get<IEsploraApi.Transaction[]>('/block/' + hash + '/txs');
+    return this.failoverRouter.$get<IEsploraApi.Transaction[]>('/internal/block/' + hash + '/txs');
  }

  $getBlockHash(height: number): Promise<string> {
@@ -290,13 +298,16 @@ class ElectrsApi implements AbstractBitcoinApi {
    return this.failoverRouter.$get<IEsploraApi.Outspend[]>('/tx/' + txId + '/outspends');
  }

-  async $getBatchedOutspends(txId: string[]): Promise<IEsploraApi.Outspend[][]> {
-    const outspends: IEsploraApi.Outspend[][] = [];
-    for (const tx of txId) {
-      const outspend = await this.$getOutspends(tx);
-      outspends.push(outspend);
-    }
-    return outspends;
+  async $getBatchedOutspends(txids: string[]): Promise<IEsploraApi.Outspend[][]> {
+    throw new Error('Method not implemented.');
+  }
+
+  async $getBatchedOutspendsInternal(txids: string[]): Promise<IEsploraApi.Outspend[][]> {
+    return this.failoverRouter.$post<IEsploraApi.Outspend[][]>('/internal/txs/outspends/by-txid', txids, 'json');
+  }
+
+  async $getOutSpendsByOutpoint(outpoints: { txid: string, vout: number }[]): Promise<IEsploraApi.Outspend[]> {
+    return this.failoverRouter.$post<IEsploraApi.Outspend[]>('/internal/txs/outspends/by-outpoint', outpoints.map(out => `${out.txid}:${out.vout}`), 'json');
  }

  public startHealthChecks(): void {
--- a/backend/src/api/blocks.ts
+++ b/backend/src/api/blocks.ts
@@ -81,6 +81,7 @@ class Blocks {
  private async $getTransactionsExtended(
    blockHash: string,
    blockHeight: number,
+    blockTime: number,
    onlyCoinbase: boolean,
    txIds: string[] | null = null,
    quiet: boolean = false,
@@ -101,6 +102,12 @@ class Blocks {
    if (!onlyCoinbase) {
      for (const txid of txIds) {
        if (mempool[txid]) {
+          mempool[txid].status = {
+            confirmed: true,
+            block_height: blockHeight,
+            block_hash: blockHash,
+            block_time: blockTime,
+          };
          transactionMap[txid] = mempool[txid];
          foundInMempool++;
          totalFound++;
@@ -608,7 +615,7 @@ class Blocks {
          }
          const blockHash = await bitcoinApi.$getBlockHash(blockHeight);
          const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash);
-          const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, null, true);
+          const transactions = await this.$getTransactionsExtended(blockHash, block.height, block.timestamp, true, null, true);
          const blockExtended = await this.$getBlockExtended(block, transactions);

          newlyIndexed++;
@@ -701,7 +708,7 @@ class Blocks {
      const verboseBlock = await bitcoinClient.getBlock(blockHash, 2);
      const block = BitcoinApi.convertBlock(verboseBlock);
      const txIds: string[] = verboseBlock.tx.map(tx => tx.txid);
-      const transactions = await this.$getTransactionsExtended(blockHash, block.height, false, txIds, false, true) as MempoolTransactionExtended[];
+      const transactions = await this.$getTransactionsExtended(blockHash, block.height, block.timestamp, false, txIds, false, true) as MempoolTransactionExtended[];

      // fill in missing transaction fee data from verboseBlock
      for (let i = 0; i < transactions.length; i++) {
@@ -890,7 +897,7 @@ class Blocks {

    const blockHash = await bitcoinApi.$getBlockHash(height);
    const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash);
-    const transactions = await this.$getTransactionsExtended(blockHash, block.height, true);
+    const transactions = await this.$getTransactionsExtended(blockHash, block.height, block.timestamp, true);
    const blockExtended = await this.$getBlockExtended(block, transactions);

    if (Common.indexingEnabled()) {
@@ -902,7 +909,7 @@ class Blocks {

  public async $indexStaleBlock(hash: string): Promise<BlockExtended> {
    const block: IEsploraApi.Block = await bitcoinApi.$getBlock(hash);
-    const transactions = await this.$getTransactionsExtended(hash, block.height, true);
+    const transactions = await this.$getTransactionsExtended(hash, block.height, block.timestamp, true);
    const blockExtended = await this.$getBlockExtended(block, transactions);

    blockExtended.canonical = await bitcoinApi.$getBlockHash(block.height);
--- a/backend/src/api/disk-cache.ts
+++ b/backend/src/api/disk-cache.ts
@@ -252,7 +252,11 @@ class DiskCache {
      }

      if (rbfData?.rbf) {
-        rbfCache.load(rbfData.rbf);
+        rbfCache.load({
+          txs: rbfData.rbf.txs.map(([txid, entry]) => ({ value: entry })),
+          trees: rbfData.rbf.trees,
+          expiring: rbfData.rbf.expiring.map(([txid, value]) => ({ key: txid, value })),
+        });
      }
    } catch (e) {
      logger.warn('Failed to parse rbf cache. Skipping. Reason: ' + (e instanceof Error ? e.message : e));
--- a/backend/src/api/fee-api.ts
+++ b/backend/src/api/fee-api.ts
@@ -3,21 +3,30 @@ import { Common } from './common';
 import mempool from './mempool';
 import projectedBlocks from './mempool-blocks';

+interface RecommendedFees {
+  fastestFee: number,
+  halfHourFee: number,
+  hourFee: number,
+  economyFee: number,
+  minimumFee: number,
+}
+
 class FeeApi {
  constructor() { }

  defaultFee = Common.isLiquid() ? 0.1 : 1;

-  public getRecommendedFee() {
+  public getRecommendedFee(): RecommendedFees {
    const pBlocks = projectedBlocks.getMempoolBlocks();
    const mPool = mempool.getMempoolInfo();
    const minimumFee = Math.ceil(mPool.mempoolminfee * 100000);
+    const defaultMinFee = Math.max(minimumFee, this.defaultFee);

    if (!pBlocks.length) {
      return {
-        'fastestFee': this.defaultFee,
-        'halfHourFee': this.defaultFee,
-        'hourFee': this.defaultFee,
+        'fastestFee': defaultMinFee,
+        'halfHourFee': defaultMinFee,
+        'hourFee': defaultMinFee,
        'economyFee': minimumFee,
        'minimumFee': minimumFee,
      };
@@ -27,11 +36,15 @@ class FeeApi {
    const secondMedianFee = pBlocks[1] ? this.optimizeMedianFee(pBlocks[1], pBlocks[2], firstMedianFee) : this.defaultFee;
    const thirdMedianFee = pBlocks[2] ? this.optimizeMedianFee(pBlocks[2], pBlocks[3], secondMedianFee) : this.defaultFee;

+    // explicitly enforce a minimum of ceil(mempoolminfee) on all recommendations.
+    // simply rounding up recommended rates is insufficient, as the purging rate
+    // can exceed the median rate of projected blocks in some extreme scenarios
+    // (see https://bitcoin.stackexchange.com/a/120024)
    return {
-      'fastestFee': firstMedianFee,
-      'halfHourFee': secondMedianFee,
-      'hourFee': thirdMedianFee,
-      'economyFee': Math.min(2 * minimumFee, thirdMedianFee),
+      'fastestFee': Math.max(minimumFee, firstMedianFee),
+      'halfHourFee': Math.max(minimumFee, secondMedianFee),
+      'hourFee': Math.max(minimumFee, thirdMedianFee),
+      'economyFee': Math.max(minimumFee, Math.min(2 * minimumFee, thirdMedianFee)),
      'minimumFee': minimumFee,
    };
  }
--- a/backend/src/api/memory-cache.ts
+++ b/backend/src/api/memory-cache.ts
@@ -31,7 +31,7 @@ class MemoryCache {
  }

  private cleanup() {
-    this.cache = this.cache.filter((cache) => cache.expires < (new Date()));
+    this.cache = this.cache.filter((cache) => cache.expires > (new Date()));
  }
 }

--- a/backend/src/api/mempool.ts
+++ b/backend/src/api/mempool.ts
@@ -94,7 +94,7 @@ class Mempool {
      logger.debug(`Migrating ${Object.keys(this.mempoolCache).length} transactions from disk cache to Redis cache`);
    }
    for (const txid of Object.keys(this.mempoolCache)) {
-      if (!this.mempoolCache[txid].sigops || this.mempoolCache[txid].effectiveFeePerVsize == null) {
+      if (!this.mempoolCache[txid].adjustedVsize || this.mempoolCache[txid].sigops == null || this.mempoolCache[txid].effectiveFeePerVsize == null) {
        this.mempoolCache[txid] = transactionUtils.extendMempoolTransaction(this.mempoolCache[txid]);
      }
      if (this.mempoolCache[txid].order == null) {
--- a/backend/src/api/rbf-cache.ts
+++ b/backend/src/api/rbf-cache.ts
@@ -2,6 +2,7 @@ import config from "../config";
 import logger from "../logger";
 import { MempoolTransactionExtended, TransactionStripped } from "../mempool.interfaces";
 import bitcoinApi from './bitcoin/bitcoin-api-factory';
+import { IEsploraApi } from "./bitcoin/esplora-api.interface";
 import { Common } from "./common";
 import redisCache from "./redis-cache";

@@ -53,6 +54,9 @@ class RbfCache {
  private expiring: Map<string, number> = new Map();
  private cacheQueue: CacheEvent[] = [];

+  private evictionCount = 0;
+  private staleCount = 0;
+
  constructor() {
    setInterval(this.cleanup.bind(this), 1000 * 60 * 10);
  }
@@ -245,6 +249,7 @@ class RbfCache {

  // flag a transaction as removed from the mempool
  public evict(txid: string, fast: boolean = false): void {
+    this.evictionCount++;
    if (this.txs.has(txid) && (fast || !this.expiring.has(txid))) {
      const expiryTime = fast ? Date.now() + (1000 * 60 * 10) : Date.now() + (1000 * 86400); // 24 hours
      this.addExpiration(txid, expiryTime);
@@ -272,18 +277,23 @@ class RbfCache {
        this.remove(txid);
      }
    }
-    logger.debug(`rbf cache contains ${this.txs.size} txs, ${this.rbfTrees.size} trees, ${this.expiring.size} due to expire`);
+    logger.debug(`rbf cache contains ${this.txs.size} txs, ${this.rbfTrees.size} trees, ${this.expiring.size} due to expire (${this.evictionCount} newly expired)`);
+    this.evictionCount = 0;
  }

  // remove a transaction & all previous versions from the cache
  private remove(txid): void {
    // don't remove a transaction if a newer version remains in the mempool
    if (!this.replacedBy.has(txid)) {
+      const root = this.treeMap.get(txid);
      const replaces = this.replaces.get(txid);
      this.replaces.delete(txid);
      this.treeMap.delete(txid);
      this.removeTx(txid);
      this.removeExpiration(txid);
+      if (root === txid) {
+        this.removeTree(txid);
+      }
      for (const tx of (replaces || [])) {
        // recursively remove prior versions from the cache
        this.replacedBy.delete(tx);
@@ -359,18 +369,27 @@ class RbfCache {
  }

  public async load({ txs, trees, expiring }): Promise<void> {
-    txs.forEach(txEntry => {
-      this.txs.set(txEntry.key, txEntry.value);
-    });
-    for (const deflatedTree of trees) {
-      await this.importTree(deflatedTree.root, deflatedTree.root, deflatedTree, this.txs);
-    }
-    expiring.forEach(expiringEntry => {
-      if (this.txs.has(expiringEntry.key)) {
-        this.expiring.set(expiringEntry.key, new Date(expiringEntry.value).getTime());
+    try {
+      txs.forEach(txEntry => {
+        this.txs.set(txEntry.value.txid, txEntry.value);
+      });
+      this.staleCount = 0;
+      for (const deflatedTree of trees) {
+        await this.importTree(deflatedTree.root, deflatedTree.root, deflatedTree, this.txs);
      }
-    });
-    this.cleanup();
+      expiring.forEach(expiringEntry => {
+        if (this.txs.has(expiringEntry.key)) {
+          this.expiring.set(expiringEntry.key, new Date(expiringEntry.value).getTime());
+        }
+      });
+      this.staleCount = 0;
+      await this.checkTrees();
+      logger.debug(`loaded ${txs.length} txs, ${trees.length} trees into rbf cache, ${expiring.length} due to expire, ${this.staleCount} were stale`);
+      this.cleanup();
+
+    } catch (e) {
+      logger.err('failed to restore RBF cache: ' + (e instanceof Error ? e.message : e));
+    }
  }

  exportTree(tree: RbfTree, deflated: any = null) {
@@ -398,29 +417,11 @@ class RbfCache {
    const treeInfo = deflated[txid];
    const replaces: RbfTree[] = [];

-    // check if any transactions in this tree have already been confirmed
-    mined = mined || treeInfo.mined;
-    let exists = mined;
-    if (!mined) {
-      try {
-        const apiTx = await bitcoinApi.$getRawTransaction(txid);
-        if (apiTx) {
-          exists = true;
-        }
-        if (apiTx?.status?.confirmed) {
-          mined = true;
-          treeInfo.txMined = true;
-          this.evict(txid, true);
-        }
-      } catch (e) {
-        // most transactions do not exist
-      }
-    }
-
-    // if the root tx is not in the mempool or the blockchain
-    // evict this tree as soon as possible
-    if (root === txid && !exists) {
-      this.evict(txid, true);
+    // if the root tx is unknown, remove this tree and return early
+    if (root === txid && !txs.has(txid)) {
+      this.staleCount++;
+      this.removeTree(deflated.key);
+      return;
    }

    // recursively reconstruct child trees
@@ -458,6 +459,59 @@ class RbfCache {
    return tree;
  }

+  private async checkTrees(): Promise<void> {
+    const found: { [txid: string]: boolean } = {};
+    const txids = Array.from(this.txs.values()).map(tx => tx.txid).filter(txid => {
+      return !this.expiring.has(txid) && !this.getRbfTree(txid)?.mined;
+    });
+
+    const processTxs = (txs: IEsploraApi.Transaction[]): void => {
+      for (const tx of txs) {
+        found[tx.txid] = true;
+        if (tx.status?.confirmed) {
+          const tree = this.getRbfTree(tx.txid);
+          if (tree) {
+            this.setTreeMined(tree, tx.txid);
+            tree.mined = true;
+            this.evict(tx.txid, false);
+          }
+        }
+      }
+    };
+
+    if (config.MEMPOOL.BACKEND === 'esplora') {
+      const sliceLength = 250;
+      for (let i = 0; i < Math.ceil(txids.length / sliceLength); i++) {
+        const slice = txids.slice(i * sliceLength, (i + 1) * sliceLength);
+        try {
+          const txs = await bitcoinApi.$getRawTransactions(slice);
+          logger.debug(`fetched ${slice.length} cached rbf transactions`);
+          processTxs(txs);
+          logger.debug(`processed ${slice.length} cached rbf transactions`);
+        } catch (err) {
+          logger.err(`failed to fetch or process ${slice.length} cached rbf transactions`);
+        }
+      }
+    } else {
+      const txs: IEsploraApi.Transaction[] = [];
+      for (const txid of txids) {
+        try {
+          const tx = await bitcoinApi.$getRawTransaction(txid, true, false);
+          txs.push(tx);
+        } catch (err) {
+          // some 404s are expected, so continue quietly
+        }
+      }
+      processTxs(txs);
+    }
+
+    for (const txid of txids) {
+      if (!found[txid]) {
+        this.evict(txid, false);
+      }
+    }
+  }
+
  public getLatestRbfSummary(): ReplacementInfo[] {
    const rbfList = this.getRbfTrees(false);
    return rbfList.slice(0, 6).map(rbfTree => {
--- a/backend/src/api/redis-cache.ts
+++ b/backend/src/api/redis-cache.ts
@@ -219,7 +219,7 @@ class RedisCache {
    await memPool.$setMempool(loadedMempool);
    await rbfCache.load({
      txs: rbfTxs,
-      trees: rbfTrees.map(loadedTree => loadedTree.value),
+      trees: rbfTrees.map(loadedTree => { loadedTree.value.key = loadedTree.key; return loadedTree.value; }),
      expiring: rbfExpirations,
    });
  }
--- a/backend/src/api/transaction-utils.ts
+++ b/backend/src/api/transaction-utils.ts
@@ -116,7 +116,7 @@ class TransactionUtils {
  public extendMempoolTransaction(transaction: IEsploraApi.Transaction): MempoolTransactionExtended {
    const vsize = Math.ceil(transaction.weight / 4);
    const fractionalVsize = (transaction.weight / 4);
-    const sigops = !Common.isLiquid() ? this.countSigops(transaction) : 0;
+    let sigops = Common.isLiquid() ? 0 : (transaction.sigops != null ? transaction.sigops : this.countSigops(transaction));
    // https://github.com/bitcoin/bitcoin/blob/e9262ea32a6e1d364fb7974844fadc36f931f8c6/src/policy/policy.cpp#L295-L298
    const adjustedVsize = Math.max(fractionalVsize, sigops *  5); // adjusted vsize = Max(weight, sigops * bytes_per_sigop) / witness_scale_factor
    const feePerVbytes = (transaction.fee || 0) / fractionalVsize;
@@ -155,7 +155,7 @@ class TransactionUtils {
      sigops += 20 * (script.match(/OP_CHECKMULTISIG/g)?.length || 0);
    } else {
      // in redeem scripts and witnesses, worth N if preceded by OP_N, 20 otherwise
-      const matches = script.matchAll(/(?:OP_(\d+))? OP_CHECKMULTISIG/g);
+      const matches = script.matchAll(/(?:OP_(?:PUSHNUM_)?(\d+))? OP_CHECKMULTISIG/g);
      for (const match of matches) {
        const n = parseInt(match[1]);
        if (Number.isInteger(n)) {
@@ -189,6 +189,12 @@ class TransactionUtils {
              sigops += this.countScriptSigops(bitcoinjs.script.toASM(Buffer.from(input.witness[input.witness.length - 1], 'hex')), false, true);
            }
            break;
+
+          case input.prevout.scriptpubkey_type === 'p2sh':
+            if (input.inner_redeemscript_asm) {
+              sigops += this.countScriptSigops(input.inner_redeemscript_asm);
+            }
+            break;
        }
      }
    }
--- a/backend/src/api/websocket-handler.ts
+++ b/backend/src/api/websocket-handler.ts
@@ -577,7 +577,7 @@ class WebsocketHandler {
          response['utxoSpent'] = JSON.stringify(outspends);
        }

-        const rbfReplacedBy = rbfCache.getReplacedBy(client['track-tx']);
+        const rbfReplacedBy = rbfChanges.map[client['track-tx']] ? rbfCache.getReplacedBy(client['track-tx']) : false;
        if (rbfReplacedBy) {
          response['rbfTransaction'] = JSON.stringify({
            txid: rbfReplacedBy,