Merge pull request #4555 from mempool/mononaut/limit-gbt-input-redux

Limited GBT

backend/mempool-config.sample.json

@@ -28,9 +28,8 @@
     "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json",
     "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
     "AUDIT": false,
-    "ADVANCED_GBT_AUDIT": false,
-    "ADVANCED_GBT_MEMPOOL": false,
     "RUST_GBT": false,
+    "LIMIT_GBT": false,
     "CPFP_INDEXING": false,
     "DISK_CACHE_BLOCK_INTERVAL": 6,
     "MAX_PUSH_TX_SIZE_WEIGHT": 4000000,

backend/src/__fixtures__/mempool-config.template.json

@@ -28,9 +28,8 @@
     "POOLS_JSON_TREE_URL": "__MEMPOOL_POOLS_JSON_TREE_URL__",
     "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__",
     "AUDIT": true,
-    "ADVANCED_GBT_AUDIT": true,
-    "ADVANCED_GBT_MEMPOOL": true,
     "RUST_GBT": false,
+    "LIMIT_GBT": false,
     "CPFP_INDEXING": true,
     "MAX_BLOCKS_BULK_QUERY": 999,
     "DISK_CACHE_BLOCK_INTERVAL": 999,

backend/src/__tests__/config.test.ts

@@ -41,9 +41,8 @@ describe('Mempool Backend Config', () => {
         POOLS_JSON_TREE_URL: 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
         POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json',
         AUDIT: false,
-        ADVANCED_GBT_AUDIT: false,
-        ADVANCED_GBT_MEMPOOL: false,
         RUST_GBT: false,
+        LIMIT_GBT: false,
         CPFP_INDEXING: false,
         MAX_BLOCKS_BULK_QUERY: 0,
         DISK_CACHE_BLOCK_INTERVAL: 6,

backend/src/api/bitcoin/bitcoin.routes.ts

@@ -19,6 +19,7 @@ import bitcoinClient from './bitcoin-client';
 import difficultyAdjustment from '../difficulty-adjustment';
 import transactionRepository from '../../repositories/TransactionRepository';
 import rbfCache from '../rbf-cache';
+import { calculateCpfp } from '../cpfp';
 
 class BitcoinRoutes {
   public initRoutes(app: Application) {
@@ -217,7 +218,7 @@ class BitcoinRoutes {
         return;
       }
 
-      const cpfpInfo = Common.setRelativesAndGetCpfpInfo(tx, mempool.getMempool());
+      const cpfpInfo = calculateCpfp(tx, mempool.getMempool());
 
       res.json(cpfpInfo);
       return;

backend/src/api/common.ts

@@ -574,69 +574,6 @@ export class Common {
     }
   }
 
-  static setRelativesAndGetCpfpInfo(tx: MempoolTransactionExtended, memPool: { [txid: string]: MempoolTransactionExtended }): CpfpInfo {
-    const parents = this.findAllParents(tx, memPool);
-    const lowerFeeParents = parents.filter((parent) => parent.adjustedFeePerVsize < tx.effectiveFeePerVsize);
-
-    let totalWeight = (tx.adjustedVsize * 4) + lowerFeeParents.reduce((prev, val) => prev + (val.adjustedVsize * 4), 0);
-    let totalFees = tx.fee + lowerFeeParents.reduce((prev, val) => prev + val.fee, 0);
-
-    tx.ancestors = parents
-      .map((t) => {
-        return {
-          txid: t.txid,
-          weight: (t.adjustedVsize * 4),
-          fee: t.fee,
-        };
-      });
-
-    // Add high (high fee) decendant weight and fees
-    if (tx.bestDescendant) {
-      totalWeight += tx.bestDescendant.weight;
-      totalFees += tx.bestDescendant.fee;
-    }
-
-    tx.effectiveFeePerVsize = Math.max(0, totalFees / (totalWeight / 4));
-    tx.cpfpChecked = true;
-
-    return {
-      ancestors: tx.ancestors,
-      bestDescendant: tx.bestDescendant || null,
-    };
-  }
-
-
-  private static findAllParents(tx: MempoolTransactionExtended, memPool: { [txid: string]: MempoolTransactionExtended }): MempoolTransactionExtended[] {
-    let parents: MempoolTransactionExtended[] = [];
-    tx.vin.forEach((parent) => {
-      if (parents.find((p) => p.txid === parent.txid)) {
-        return;
-      }
-
-      const parentTx = memPool[parent.txid];
-      if (parentTx) {
-        if (tx.bestDescendant && tx.bestDescendant.fee / (tx.bestDescendant.weight / 4) > parentTx.adjustedFeePerVsize) {
-          if (parentTx.bestDescendant && parentTx.bestDescendant.fee < tx.fee + tx.bestDescendant.fee) {
-            parentTx.bestDescendant = {
-              weight: (tx.adjustedVsize * 4) + tx.bestDescendant.weight,
-              fee: tx.fee + tx.bestDescendant.fee,
-              txid: tx.txid,
-            };
-          }
-        } else if (tx.adjustedFeePerVsize > parentTx.adjustedFeePerVsize) {
-          parentTx.bestDescendant = {
-            weight: (tx.adjustedVsize * 4),
-            fee: tx.fee,
-            txid: tx.txid
-          };
-        }
-        parents.push(parentTx);
-        parents = parents.concat(this.findAllParents(parentTx, memPool));
-      }
-    });
-    return parents;
-  }
-
   // calculates the ratio of matched transactions to projected transactions by weight
   static getSimilarity(projectedBlock: MempoolBlockWithTransactions, transactions: TransactionExtended[]): number {
     let matchedWeight = 0;

backend/src/api/cpfp.ts

@@ -0,0 +1,286 @@
+import { CpfpInfo, MempoolTransactionExtended } from '../mempool.interfaces';
+import memPool from './mempool';
+
+const CPFP_UPDATE_INTERVAL = 60_000; // update CPFP info at most once per 60s per transaction
+const MAX_GRAPH_SIZE = 50; // the maximum number of in-mempool relatives to consider
+
+interface GraphTx extends MempoolTransactionExtended {
+  depends: string[];
+  spentby: string[];
+  ancestorMap: Map<string, GraphTx>;
+  fees: {
+    base: number;
+    ancestor: number;
+  };
+  ancestorcount: number;
+  ancestorsize: number;
+  ancestorRate: number;
+  individualRate: number;
+  score: number;
+}
+
+/**
+ * Takes a mempool transaction and a copy of the current mempool, and calculates the CPFP data for
+ * that transaction (and all others in the same cluster)
+ */
+export function calculateCpfp(tx: MempoolTransactionExtended, mempool: { [txid: string]: MempoolTransactionExtended }): CpfpInfo {
+  if (tx.cpfpUpdated && Date.now() < (tx.cpfpUpdated + CPFP_UPDATE_INTERVAL)) {
+    tx.cpfpDirty = false;
+    return {
+      ancestors: tx.ancestors || [],
+      bestDescendant: tx.bestDescendant || null,
+      descendants: tx.descendants || [],
+      effectiveFeePerVsize: tx.effectiveFeePerVsize || tx.adjustedFeePerVsize || tx.feePerVsize,
+      sigops: tx.sigops,
+      adjustedVsize: tx.adjustedVsize,
+      acceleration: tx.acceleration
+    };
+  }
+
+  const ancestorMap = new Map<string, GraphTx>();
+  const graphTx = mempoolToGraphTx(tx);
+  ancestorMap.set(tx.txid, graphTx);
+
+  const allRelatives = expandRelativesGraph(mempool, ancestorMap);
+  const relativesMap = initializeRelatives(allRelatives);
+  const cluster = calculateCpfpCluster(tx.txid, relativesMap);
+
+  let totalVsize = 0;
+  let totalFee = 0;
+  for (const tx of cluster.values()) {
+    totalVsize += tx.adjustedVsize;
+    totalFee += tx.fee;
+  }
+  const effectiveFeePerVsize = totalFee / totalVsize;
+  for (const tx of cluster.values()) {
+    mempool[tx.txid].effectiveFeePerVsize = effectiveFeePerVsize;
+    mempool[tx.txid].ancestors = Array.from(tx.ancestorMap.values()).map(tx => ({ txid: tx.txid, weight: tx.weight, fee: tx.fee }));
+    mempool[tx.txid].descendants = Array.from(cluster.values()).filter(entry => entry.txid !== tx.txid && !tx.ancestorMap.has(entry.txid)).map(tx => ({ txid: tx.txid, weight: tx.weight, fee: tx.fee }));
+    mempool[tx.txid].bestDescendant = null;
+    mempool[tx.txid].cpfpChecked = true;
+    mempool[tx.txid].cpfpDirty = true;
+    mempool[tx.txid].cpfpUpdated = Date.now();
+  }
+
+  tx = mempool[tx.txid];
+
+  return {
+    ancestors: tx.ancestors || [],
+    bestDescendant: tx.bestDescendant || null,
+    descendants: tx.descendants || [],
+    effectiveFeePerVsize: tx.effectiveFeePerVsize || tx.adjustedFeePerVsize || tx.feePerVsize,
+    sigops: tx.sigops,
+    adjustedVsize: tx.adjustedVsize,
+    acceleration: tx.acceleration
+  };
+}
+
+function mempoolToGraphTx(tx: MempoolTransactionExtended): GraphTx {
+  return {
+    ...tx,
+    depends: tx.vin.map(v => v.txid),
+    spentby: tx.vout.map((v, i) => memPool.getFromSpendMap(tx.txid, i)).map(tx => tx?.txid).filter(txid => txid != null) as string[],
+    ancestorMap: new Map(),
+    fees: {
+      base: tx.fee,
+      ancestor: tx.fee,
+    },
+    ancestorcount: 1,
+    ancestorsize: tx.adjustedVsize,
+    ancestorRate: 0,
+    individualRate: 0,
+    score: 0,
+  };
+}
+
+/**
+ * Takes a map of transaction ancestors, and expands it into a full graph of up to MAX_GRAPH_SIZE in-mempool relatives
+ */
+function expandRelativesGraph(mempool: { [txid: string]: MempoolTransactionExtended }, ancestors: Map<string, GraphTx>): Map<string, GraphTx> {
+  const relatives: Map<string, GraphTx> = new Map();
+  const stack: GraphTx[] = Array.from(ancestors.values());
+  while (stack.length > 0) {
+    if (relatives.size > MAX_GRAPH_SIZE) {
+      return relatives;
+    }
+
+    const nextTx = stack.pop();
+    if (!nextTx) {
+      continue;
+    }
+    relatives.set(nextTx.txid, nextTx);
+
+    for (const relativeTxid of [...nextTx.depends, ...nextTx.spentby]) {
+      if (relatives.has(relativeTxid)) {
+        // already processed this tx
+        continue;
+      }
+      let mempoolTx = ancestors.get(relativeTxid);
+      if (!mempoolTx && mempool[relativeTxid]) {
+        mempoolTx = mempoolToGraphTx(mempool[relativeTxid]);
+      }
+      if (mempoolTx) {
+        stack.push(mempoolTx);
+      }
+    }
+  }
+
+  return relatives;
+}
+
+ /**
+   * Efficiently sets a Map of in-mempool ancestors for each member of an expanded relative graph
+   * by running setAncestors on each leaf, and caching intermediate results.
+   * then initializes ancestor data for each transaction
+   * 
+   * @param all 
+   */
+ function initializeRelatives(mempoolTxs: Map<string, GraphTx>): Map<string, GraphTx> {
+  const visited: Map<string, Map<string, GraphTx>> = new Map();
+  const leaves: GraphTx[] = Array.from(mempoolTxs.values()).filter(entry => entry.spentby.length === 0);
+  for (const leaf of leaves) {
+    setAncestors(leaf, mempoolTxs, visited);
+  }
+  mempoolTxs.forEach(entry => {
+    entry.ancestorMap?.forEach(ancestor => {
+      entry.ancestorcount++;
+      entry.ancestorsize += ancestor.adjustedVsize;
+      entry.fees.ancestor += ancestor.fees.base;
+    });
+    setAncestorScores(entry);
+  });
+  return mempoolTxs;
+}
+
+/**
+   * Given a root transaction and a list of in-mempool ancestors,
+   * Calculate the CPFP cluster
+   * 
+   * @param tx
+   * @param ancestors
+   */
+function calculateCpfpCluster(txid: string, graph: Map<string, GraphTx>): Map<string, GraphTx> {
+  const tx = graph.get(txid);
+  if (!tx) {
+    return new Map<string, GraphTx>([]);
+  }
+
+  // Initialize individual & ancestor fee rates
+  graph.forEach(entry => setAncestorScores(entry));
+
+  // Sort by descending ancestor score
+  let sortedRelatives = Array.from(graph.values()).sort(mempoolComparator);
+
+  // Iterate until we reach a cluster that includes our target tx
+  let maxIterations = MAX_GRAPH_SIZE;
+  let best = sortedRelatives.shift();
+  let bestCluster = new Map<string, GraphTx>(best?.ancestorMap?.entries() || []);
+  while (sortedRelatives.length && best && (best.txid !== tx.txid && !best.ancestorMap.has(tx.txid)) && maxIterations > 0) {
+    maxIterations--;
+    if ((best && best.txid === tx.txid) || (bestCluster && bestCluster.has(tx.txid))) {
+      break;
+    } else {
+      // Remove this cluster (it doesn't include our target tx)
+      // and update scores, ancestor totals and dependencies for the survivors
+      removeAncestors(bestCluster, graph);
+
+      // re-sort
+      sortedRelatives = Array.from(graph.values()).sort(mempoolComparator);
+
+      // Grab the next highest scoring entry
+      best = sortedRelatives.shift();
+      if (best) {
+        bestCluster = new Map<string, GraphTx>(best?.ancestorMap?.entries() || []);
+        bestCluster.set(best?.txid, best);
+      }
+    }
+  }
+
+  bestCluster.set(tx.txid, tx);
+
+  return bestCluster;
+}
+
+ /**
+   * Remove a cluster of transactions from an in-mempool dependency graph
+   * and update the survivors' scores and ancestors
+   * 
+   * @param cluster 
+   * @param ancestors 
+   */
+ function removeAncestors(cluster: Map<string, GraphTx>, all: Map<string, GraphTx>): void {
+  // remove
+  cluster.forEach(tx => {
+    all.delete(tx.txid);
+  });
+
+  // update survivors
+  all.forEach(tx => {
+    cluster.forEach(remove => {
+      if (tx.ancestorMap?.has(remove.txid)) {
+        // remove as dependency
+        tx.ancestorMap.delete(remove.txid);
+        tx.depends = tx.depends.filter(parent => parent !== remove.txid);
+        // update ancestor sizes and fees
+        tx.ancestorsize -= remove.adjustedVsize;
+        tx.fees.ancestor -= remove.fees.base;
+      }
+    });
+    // recalculate fee rates
+    setAncestorScores(tx);
+  });
+}
+
+/**
+   * Recursively traverses an in-mempool dependency graph, and sets a Map of in-mempool ancestors
+   * for each transaction.
+   * 
+   * @param tx 
+   * @param all 
+   */
+function setAncestors(tx: GraphTx, all: Map<string, GraphTx>, visited: Map<string, Map<string, GraphTx>>, depth: number = 0): Map<string, GraphTx> {
+  // sanity check for infinite recursion / too many ancestors (should never happen)
+  if (depth > MAX_GRAPH_SIZE) {
+    return tx.ancestorMap;
+  }
+  
+  // initialize the ancestor map for this tx
+  tx.ancestorMap = new Map<string, GraphTx>();
+  tx.depends.forEach(parentId => {
+    const parent = all.get(parentId);
+    if (parent) {
+      // add the parent
+      tx.ancestorMap?.set(parentId, parent);
+      // check for a cached copy of this parent's ancestors
+      let ancestors = visited.get(parent.txid);
+      if (!ancestors) {
+        // recursively fetch the parent's ancestors
+        ancestors = setAncestors(parent, all, visited, depth + 1);
+      }
+      // and add to this tx's map
+      ancestors.forEach((ancestor, ancestorId) => {
+        tx.ancestorMap?.set(ancestorId, ancestor);
+      });
+    }
+  });
+  visited.set(tx.txid, tx.ancestorMap);
+
+  return tx.ancestorMap;
+}
+
+/**
+   * Take a mempool transaction, and set the fee rates and ancestor score
+   * 
+   * @param tx 
+   */
+function setAncestorScores(tx: GraphTx): GraphTx {
+  tx.individualRate = (tx.fees.base * 100_000_000) / tx.adjustedVsize;
+  tx.ancestorRate = (tx.fees.ancestor * 100_000_000) / tx.ancestorsize;
+  tx.score = Math.min(tx.individualRate, tx.ancestorRate);
+  return tx;
+}
+
+// Sort by descending score
+function mempoolComparator(a: GraphTx, b: GraphTx): number {
+  return b.score - a.score;
+}

backend/src/api/mempool-blocks.ts

@@ -1,6 +1,6 @@
 import { GbtGenerator, GbtResult, ThreadTransaction as RustThreadTransaction, ThreadAcceleration as RustThreadAcceleration } from 'rust-gbt';
 import logger from '../logger';
-import { MempoolBlock, MempoolTransactionExtended, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats, PoolTag, TransactionClassified, TransactionCompressed, MempoolDeltaChange } from '../mempool.interfaces';
+import { MempoolBlock, MempoolTransactionExtended, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats, TransactionClassified, TransactionCompressed, MempoolDeltaChange, GbtCandidates } from '../mempool.interfaces';
 import { Common, OnlineFeeStatsCalculator } from './common';
 import config from '../config';
 import { Worker } from 'worker_threads';
@@ -18,6 +18,7 @@ class MempoolBlocks {
 
   private nextUid: number = 1;
   private uidMap: Map<number, string> = new Map(); // map short numerical uids to full txids
+  private txidMap: Map<string, number> = new Map(); // map full txids back to short numerical uids
 
   public getMempoolBlocks(): MempoolBlock[] {
     return this.mempoolBlocks.map((block) => {
@@ -40,132 +41,6 @@ class MempoolBlocks {
     return this.mempoolBlockDeltas;
   }
 
-  public updateMempoolBlocks(memPool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false): MempoolBlockWithTransactions[] {
-    const latestMempool = memPool;
-    const memPoolArray: MempoolTransactionExtended[] = [];
-    for (const i in latestMempool) {
-      memPoolArray.push(latestMempool[i]);
-    }
-    const start = new Date().getTime();
-
-    // Clear bestDescendants & ancestors
-    memPoolArray.forEach((tx) => {
-      tx.bestDescendant = null;
-      tx.ancestors = [];
-      tx.cpfpChecked = false;
-      if (!tx.effectiveFeePerVsize) {
-        tx.effectiveFeePerVsize = tx.adjustedFeePerVsize;
-      }
-    });
-
-    // First sort
-    memPoolArray.sort((a, b) => {
-      if (a.adjustedFeePerVsize === b.adjustedFeePerVsize) {
-        // tie-break by lexicographic txid order for stability
-        return a.txid < b.txid ? -1 : 1;
-      } else {
-        return b.adjustedFeePerVsize - a.adjustedFeePerVsize;
-      }
-    });
-
-    // Loop through and traverse all ancestors and sum up all the sizes + fees
-    // Pass down size + fee to all unconfirmed children
-    let sizes = 0;
-    memPoolArray.forEach((tx) => {
-      sizes += tx.weight;
-      if (sizes > 4000000 * 8) {
-        return;
-      }
-      Common.setRelativesAndGetCpfpInfo(tx, memPool);
-    });
-
-    // Final sort, by effective fee
-    memPoolArray.sort((a, b) => {
-      if (a.effectiveFeePerVsize === b.effectiveFeePerVsize) {
-        // tie-break by lexicographic txid order for stability
-        return a.txid < b.txid ? -1 : 1;
-      } else {
-        return b.effectiveFeePerVsize - a.effectiveFeePerVsize;
-      }
-    });
-
-    const end = new Date().getTime();
-    const time = end - start;
-    logger.debug('Mempool blocks calculated in ' + time / 1000 + ' seconds');
-
-    const blocks = this.calculateMempoolBlocks(memPoolArray);
-
-    if (saveResults) {
-      const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, blocks);
-      this.mempoolBlocks = blocks;
-      this.mempoolBlockDeltas = deltas;
-    }
-
-    return blocks;
-  }
-
-  private calculateMempoolBlocks(transactionsSorted: MempoolTransactionExtended[]): MempoolBlockWithTransactions[] {
-    const mempoolBlocks: MempoolBlockWithTransactions[] = [];
-    let feeStatsCalculator: OnlineFeeStatsCalculator = new OnlineFeeStatsCalculator(config.MEMPOOL.BLOCK_WEIGHT_UNITS);
-    let onlineStats = false;
-    let blockSize = 0;
-    let blockWeight = 0;
-    let blockVsize = 0;
-    let blockFees = 0;
-    const sizeLimit = (config.MEMPOOL.BLOCK_WEIGHT_UNITS / 4) * 1.2;
-    let transactionIds: string[] = [];
-    let transactions: MempoolTransactionExtended[] = [];
-    transactionsSorted.forEach((tx, index) => {
-      if (blockWeight + tx.weight <= config.MEMPOOL.BLOCK_WEIGHT_UNITS
-        || mempoolBlocks.length === config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT - 1) {
-        tx.position = {
-          block: mempoolBlocks.length,
-          vsize: blockVsize + (tx.vsize / 2),
-        };
-        blockWeight += tx.weight;
-        blockVsize += tx.vsize;
-        blockSize += tx.size;
-        blockFees += tx.fee;
-        if (blockVsize <= sizeLimit) {
-          transactions.push(tx);
-        }
-        transactionIds.push(tx.txid);
-        if (onlineStats) {
-          feeStatsCalculator.processNext(tx);
-        }
-      } else {
-        mempoolBlocks.push(this.dataToMempoolBlocks(transactionIds, transactions, blockSize, blockWeight, blockFees));
-        blockVsize = 0;
-        tx.position = {
-          block: mempoolBlocks.length,
-          vsize: blockVsize + (tx.vsize / 2),
-        };
-
-        if (mempoolBlocks.length === config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT - 1) {
-          const stackWeight = transactionsSorted.slice(index).reduce((total, tx) => total + (tx.weight || 0), 0);
-          if (stackWeight > config.MEMPOOL.BLOCK_WEIGHT_UNITS) {
-            onlineStats = true;
-            feeStatsCalculator = new OnlineFeeStatsCalculator(stackWeight, 0.5, [10, 20, 30, 40, 50, 60, 70, 80, 90]);
-            feeStatsCalculator.processNext(tx);
-          }
-        }
-
-        blockVsize += tx.vsize;
-        blockWeight = tx.weight;
-        blockSize = tx.size;
-        blockFees = tx.fee;
-        transactionIds = [tx.txid];
-        transactions = [tx];
-      }
-    });
-    if (transactions.length) {
-      const feeStats = onlineStats ? feeStatsCalculator.getRawFeeStats() : undefined;
-      mempoolBlocks.push(this.dataToMempoolBlocks(transactionIds, transactions, blockSize, blockWeight, blockFees, feeStats));
-    }
-
-    return mempoolBlocks;
-  }
-
   private calculateMempoolDeltas(prevBlocks: MempoolBlockWithTransactions[], mempoolBlocks: MempoolBlockWithTransactions[]): MempoolBlockDelta[] {
     const mempoolBlockDeltas: MempoolBlockDelta[] = [];
     for (let i = 0; i < Math.max(mempoolBlocks.length, prevBlocks.length); i++) {
@@ -207,7 +82,7 @@ class MempoolBlocks {
     return mempoolBlockDeltas;
   }
 
-  public async $makeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
+  public async $makeBlockTemplates(transactions: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, candidates: GbtCandidates | undefined, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
     const start = Date.now();
 
     // reset mempool short ids
@@ -215,7 +90,8 @@ class MempoolBlocks {
       this.resetUids();
     }
     // set missing short ids
-    for (const tx of Object.values(newMempool)) {
+    for (const txid of transactions) {
+      const tx = newMempool[txid];
       this.setUid(tx, !saveResults);
     }
 
@@ -224,7 +100,8 @@ class MempoolBlocks {
     // prepare a stripped down version of the mempool with only the minimum necessary data
     // to reduce the overhead of passing this data to the worker thread
     const strippedMempool: Map<number, CompactThreadTransaction> = new Map();
-    Object.values(newMempool).forEach(entry => {
+    for (const txid of transactions) {
+      const entry = newMempool[txid];
       if (entry.uid !== null && entry.uid !== undefined) {
         const stripped = {
           uid: entry.uid,
@@ -237,7 +114,7 @@ class MempoolBlocks {
         };
         strippedMempool.set(entry.uid, stripped);
       }
-    });
+    }
 
     // (re)initialize tx selection worker thread
     if (!this.txSelectionWorker) {
@@ -268,7 +145,7 @@ class MempoolBlocks {
       // clean up thread error listener
       this.txSelectionWorker?.removeListener('error', threadErrorListener);
 
-      const processed = this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), accelerations, accelerationPool, saveResults);
+      const processed = this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), candidates, accelerations, accelerationPool, saveResults);
 
       logger.debug(`makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`);
 
@@ -279,10 +156,10 @@ class MempoolBlocks {
     return this.mempoolBlocks;
   }
 
-  public async $updateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], accelerationDelta: string[] = [], saveResults: boolean = false, useAccelerations: boolean = false): Promise<void> {
+  public async $updateBlockTemplates(transactions: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], candidates: GbtCandidates | undefined, accelerationDelta: string[] = [], saveResults: boolean = false, useAccelerations: boolean = false): Promise<void> {
     if (!this.txSelectionWorker) {
       // need to reset the worker
-      await this.$makeBlockTemplates(newMempool, saveResults, useAccelerations);
+      await this.$makeBlockTemplates(transactions, newMempool, candidates, saveResults, useAccelerations);
       return;
     }
 
@@ -292,9 +169,9 @@ class MempoolBlocks {
     const addedAndChanged: MempoolTransactionExtended[] = useAccelerations ? accelerationDelta.map(txid => newMempool[txid]).filter(tx => tx != null).concat(added) : added;
 
     for (const tx of addedAndChanged) {
-      this.setUid(tx, true);
+      this.setUid(tx, false);
     }
-    const removedUids = removed.map(tx => this.getUid(tx)).filter(uid => uid != null) as number[];
+    const removedTxs = removed.filter(tx => tx.uid != null) as MempoolTransactionExtended[];
 
     // prepare a stripped down version of the mempool with only the minimum necessary data
     // to reduce the overhead of passing this data to the worker thread
@@ -320,15 +197,15 @@ class MempoolBlocks {
         });
         this.txSelectionWorker?.once('error', reject);
       });
-      this.txSelectionWorker.postMessage({ type: 'update', added: addedStripped, removed: removedUids });
+      this.txSelectionWorker.postMessage({ type: 'update', added: addedStripped, removed: removedTxs.map(tx => tx.uid) as number[] });
       const { blocks, rates, clusters } = this.convertResultTxids(await workerResultPromise);
 
-      this.removeUids(removedUids);
+      this.removeUids(removedTxs);
 
       // clean up thread error listener
       this.txSelectionWorker?.removeListener('error', threadErrorListener);
 
-      this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), accelerations, null, saveResults);
+      this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), candidates, accelerations, null, saveResults);
       logger.debug(`updateBlockTemplates completed in ${(Date.now() - start) / 1000} seconds`);
     } catch (e) {
       logger.err('updateBlockTemplates failed. ' + (e instanceof Error ? e.message : e));
@@ -340,25 +217,28 @@ class MempoolBlocks {
     this.rustGbtGenerator = new GbtGenerator();
   }
 
-  public async $rustMakeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
+  public async $rustMakeBlockTemplates(txids: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, candidates: GbtCandidates | undefined, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
     const start = Date.now();
 
     // reset mempool short ids
     if (saveResults) {
       this.resetUids();
     }
+
+    const transactions = txids.map(txid => newMempool[txid]).filter(tx => tx != null);
     // set missing short ids
-    for (const tx of Object.values(newMempool)) {
+    for (const tx of transactions) {
       this.setUid(tx, !saveResults);
     }
     // set short ids for transaction inputs
-    for (const tx of Object.values(newMempool)) {
+    for (const tx of transactions) {
       tx.inputs = tx.vin.map(v => this.getUid(newMempool[v.txid])).filter(uid => (uid !== null && uid !== undefined)) as number[];
     }
 
     const accelerations = useAccelerations ? mempool.getAccelerations() : {};
     const acceleratedList = accelerationPool ? Object.values(accelerations).filter(acc => newMempool[acc.txid] && acc.pools.includes(accelerationPool)) : Object.values(accelerations).filter(acc => newMempool[acc.txid]);
     const convertedAccelerations = acceleratedList.map(acc => {
+      this.setUid(newMempool[acc.txid], true);
       return {
         uid: this.getUid(newMempool[acc.txid]),
         delta: acc.feeDelta,
@@ -369,15 +249,15 @@ class MempoolBlocks {
     const rustGbt = saveResults ? this.rustGbtGenerator : new GbtGenerator();
     try {
       const { blocks, blockWeights, rates, clusters, overflow } = this.convertNapiResultTxids(
-        await rustGbt.make(Object.values(newMempool) as RustThreadTransaction[], convertedAccelerations as RustThreadAcceleration[], this.nextUid),
+        await rustGbt.make(transactions as RustThreadTransaction[], convertedAccelerations as RustThreadAcceleration[], this.nextUid),
       );
       if (saveResults) {
         this.rustInitialized = true;
       }
-      const mempoolSize = Object.keys(newMempool).length;
+      const expectedSize = transactions.length;
       const resultMempoolSize = blocks.reduce((total, block) => total + block.length, 0) + overflow.length;
-      logger.debug(`RUST updateBlockTemplates returned ${resultMempoolSize} txs out of ${mempoolSize} in the mempool, ${overflow.length} were unmineable`);
+      logger.debug(`RUST updateBlockTemplates returned ${resultMempoolSize} txs out of ${expectedSize} in the mempool, ${overflow.length} were unmineable`);
-      const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, accelerations, accelerationPool, saveResults);
+      const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, candidates, accelerations, accelerationPool, saveResults);
       logger.debug(`RUST makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`);
       return processed;
     } catch (e) {
@@ -389,36 +269,37 @@ class MempoolBlocks {
     return this.mempoolBlocks;
   }
 
-  public async $oneOffRustBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, useAccelerations: boolean, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
+  public async $oneOffRustBlockTemplates(transactions: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, candidates: GbtCandidates | undefined, useAccelerations: boolean, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
-    return this.$rustMakeBlockTemplates(newMempool, false, useAccelerations, accelerationPool);
+    return this.$rustMakeBlockTemplates(transactions, newMempool, candidates, false, useAccelerations, accelerationPool);
   }
 
-  public async $rustUpdateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, mempoolSize: number, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], useAccelerations: boolean, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
+  public async $rustUpdateBlockTemplates(transactions: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], candidates: GbtCandidates | undefined, useAccelerations: boolean, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
     // GBT optimization requires that uids never get too sparse
     // as a sanity check, we should also explicitly prevent uint32 uid overflow
-    if (this.nextUid + added.length >= Math.min(Math.max(262144, 2 * mempoolSize), MAX_UINT32)) {
+    if (this.nextUid + added.length >= Math.min(Math.max(262144, 2 * transactions.length), MAX_UINT32)) {
       this.resetRustGbt();
     }
 
     if (!this.rustInitialized) {
       // need to reset the worker
-      return this.$rustMakeBlockTemplates(newMempool, true, useAccelerations, accelerationPool);
+      return this.$rustMakeBlockTemplates(transactions, newMempool, candidates, true, useAccelerations, accelerationPool);
     }
 
     const start = Date.now();
     // set missing short ids
     for (const tx of added) {
-      this.setUid(tx, true);
+      this.setUid(tx, false);
     }
     // set short ids for transaction inputs
     for (const tx of added) {
       tx.inputs = tx.vin.map(v => this.getUid(newMempool[v.txid])).filter(uid => (uid !== null && uid !== undefined)) as number[];
     }
-    const removedUids = removed.map(tx => this.getUid(tx)).filter(uid => (uid !== null && uid !== undefined)) as number[];
+    const removedTxs = removed.filter(tx => tx.uid != null) as MempoolTransactionExtended[];
 
     const accelerations = useAccelerations ? mempool.getAccelerations() : {};
     const acceleratedList = accelerationPool ? Object.values(accelerations).filter(acc => newMempool[acc.txid] && acc.pools.includes(accelerationPool)) : Object.values(accelerations).filter(acc => newMempool[acc.txid]);
     const convertedAccelerations = acceleratedList.map(acc => {
+      this.setUid(newMempool[acc.txid], true);
       return {
         uid: this.getUid(newMempool[acc.txid]),
         delta: acc.feeDelta,
@@ -430,18 +311,18 @@ class MempoolBlocks {
       const { blocks, blockWeights, rates, clusters, overflow } = this.convertNapiResultTxids(
         await this.rustGbtGenerator.update(
           added as RustThreadTransaction[],
-          removedUids,
+          removedTxs.map(tx => tx.uid) as number[],
           convertedAccelerations as RustThreadAcceleration[],
           this.nextUid,
         ),
       );
       const resultMempoolSize = blocks.reduce((total, block) => total + block.length, 0) + overflow.length;
-      logger.debug(`RUST updateBlockTemplates returned ${resultMempoolSize} txs out of ${mempoolSize} in the mempool, ${overflow.length} were unmineable`);
+      logger.debug(`RUST updateBlockTemplates returned ${resultMempoolSize} txs out of ${transactions.length} candidates, ${overflow.length} were unmineable`);
-      if (mempoolSize !== resultMempoolSize) {
+      if (transactions.length !== resultMempoolSize) {
-        throw new Error('GBT returned wrong number of transactions , cache is probably out of sync');
+        throw new Error(`GBT returned wrong number of transactions ${transactions.length} vs ${resultMempoolSize}, cache is probably out of sync`);
       } else {
-        const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, accelerations, accelerationPool, true);
+        const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, candidates, accelerations, accelerationPool, true);
-        this.removeUids(removedUids);
+        this.removeUids(removedTxs);
         logger.debug(`RUST updateBlockTemplates completed in ${(Date.now() - start)/1000} seconds`);
         return processed;
       }
@@ -452,7 +333,7 @@ class MempoolBlocks {
     }
   }
 
-  private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], accelerations, accelerationPool, saveResults): MempoolBlockWithTransactions[] {
+  private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], candidates: GbtCandidates | undefined, accelerations, accelerationPool, saveResults): MempoolBlockWithTransactions[] {
     for (const [txid, rate] of rates) {
       if (txid in mempool) {
         mempool[txid].cpfpDirty = (rate !== mempool[txid].effectiveFeePerVsize);
@@ -486,6 +367,9 @@ class MempoolBlocks {
             if (txid === memberTxid) {
               matched = true;
             } else {
+              if (!mempool[txid]) {
+                console.log('txid missing from mempool! ', txid, candidates?.txs[txid]);
+              }
               const relative = {
                 txid: txid,
                 fee: mempool[txid].fee,
@@ -518,6 +402,16 @@ class MempoolBlocks {
       let totalWeight = 0;
       let totalFees = 0;
       const transactions: MempoolTransactionExtended[] = [];
+
+      // backfill purged transactions
+      if (candidates?.txs && blockIndex === blocks.length - 1) {
+        for (const txid of Object.keys(mempool)) {
+          if (!candidates.txs[txid]) {
+            block.push(txid);
+          }
+        }
+      }
+
       for (const txid of block) {
         if (txid) {
           mempoolTx = mempool[txid];
@@ -526,16 +420,6 @@ class MempoolBlocks {
             block: blockIndex,
             vsize: totalVsize + (mempoolTx.vsize / 2),
           };
-          if (!mempoolTx.cpfpChecked) {
-            if (mempoolTx.ancestors?.length) {
-              mempoolTx.ancestors = [];
-            }
-            if (mempoolTx.descendants?.length) {
-              mempoolTx.descendants = [];
-            }
-            mempoolTx.bestDescendant = null;
-            mempoolTx.cpfpChecked = true;
-          }
 
           const acceleration = accelerations[txid];
           if (isAccelerated[txid] || (acceleration && (!accelerationPool || acceleration.pools.includes(accelerationPool)))) {
@@ -610,30 +494,38 @@ class MempoolBlocks {
 
   private resetUids(): void {
     this.uidMap.clear();
+    this.txidMap.clear();
     this.nextUid = 1;
   }
 
   private setUid(tx: MempoolTransactionExtended, skipSet = false): number {
-    if (tx.uid === null || tx.uid === undefined || !skipSet) {
+    if (!this.txidMap.has(tx.txid) || !skipSet) {
       const uid = this.nextUid;
       this.nextUid++;
       this.uidMap.set(uid, tx.txid);
+      this.txidMap.set(tx.txid, uid);
       tx.uid = uid;
       return uid;
     } else {
+      tx.uid = this.txidMap.get(tx.txid) as number;
       return tx.uid;
     }
   }
 
   private getUid(tx: MempoolTransactionExtended): number | void {
-    if (tx?.uid !== null && tx?.uid !== undefined && this.uidMap.has(tx.uid)) {
+    if (tx) {
-      return tx.uid;
+      return this.txidMap.get(tx.txid);
     }
   }
 
-  private removeUids(uids: number[]): void {
+  private removeUids(txs: MempoolTransactionExtended[]): void {
-    for (const uid of uids) {
+    for (const tx of txs) {
-      this.uidMap.delete(uid);
+      const uid = this.txidMap.get(tx.txid);
+      if (uid != null) {
+        this.uidMap.delete(uid);
+        this.txidMap.delete(tx.txid);
+      }
+      tx.uid = undefined;
     }
   }
 

backend/src/api/mempool.ts

@@ -1,6 +1,6 @@
 import config from '../config';
 import bitcoinApi from './bitcoin/bitcoin-api-factory';
-import { MempoolTransactionExtended, TransactionExtended, VbytesPerSecond } from '../mempool.interfaces';
+import { MempoolTransactionExtended, TransactionExtended, VbytesPerSecond, GbtCandidates } from '../mempool.interfaces';
 import logger from '../logger';
 import { Common } from './common';
 import transactionUtils from './transaction-utils';
@@ -11,18 +11,20 @@ import bitcoinSecondClient from './bitcoin/bitcoin-second-client';
 import rbfCache from './rbf-cache';
 import { Acceleration } from './services/acceleration';
 import redisCache from './redis-cache';
+import blocks from './blocks';
 
 class Mempool {
   private inSync: boolean = false;
   private mempoolCacheDelta: number = -1;
   private mempoolCache: { [txId: string]: MempoolTransactionExtended } = {};
+  private mempoolCandidates: { [txid: string ]: boolean } = {};
   private spendMap = new Map<string, MempoolTransactionExtended>();
   private mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0, total_fee: 0,
                                                     maxmempool: 300000000, mempoolminfee: Common.isLiquid() ? 0.00000100 : 0.00001000, minrelaytxfee: Common.isLiquid() ? 0.00000100 : 0.00001000 };
   private mempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, newTransactions: MempoolTransactionExtended[],
     deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => void) | undefined;
   private $asyncMempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, mempoolSize: number, newTransactions: MempoolTransactionExtended[],
-    deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => Promise<void>) | undefined;
+    deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[], candidates?: GbtCandidates) => Promise<void>) | undefined;
 
   private accelerations: { [txId: string]: Acceleration } = {};
 
@@ -40,6 +42,8 @@ class Mempool {
   private missingTxCount = 0;
   private mainLoopTimeout: number = 120000;
 
+  public limitGBT = config.MEMPOOL.USE_SECOND_NODE_FOR_MINFEE && config.MEMPOOL.LIMIT_GBT;
+
   constructor() {
     setInterval(this.updateTxPerSecond.bind(this), 1000);
   }
@@ -74,7 +78,8 @@ class Mempool {
   }
 
   public setAsyncMempoolChangedCallback(fn: (newMempool: { [txId: string]: MempoolTransactionExtended; }, mempoolSize: number,
-    newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => Promise<void>): void {
+    newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[],
+    candidates?: GbtCandidates) => Promise<void>): void {
     this.$asyncMempoolChangedCallback = fn;
   }
 
@@ -86,6 +91,10 @@ class Mempool {
     return this.spendMap;
   }
 
+  public getFromSpendMap(txid, index): MempoolTransactionExtended | void {
+    return this.spendMap.get(`${txid}:${index}`);
+  }
+
   public async $setMempool(mempoolData: { [txId: string]: MempoolTransactionExtended }) {
     this.mempoolCache = mempoolData;
     let count = 0;
@@ -108,6 +117,9 @@ class Mempool {
         await redisCache.$addTransaction(this.mempoolCache[txid]);
       }
       this.mempoolCache[txid].flags = Common.getTransactionFlags(this.mempoolCache[txid]);
+      this.mempoolCache[txid].cpfpChecked = false;
+      this.mempoolCache[txid].cpfpDirty = true;
+      this.mempoolCache[txid].cpfpUpdated = undefined;
     }
     if (config.MEMPOOL.CACHE_ENABLED && config.REDIS.ENABLED) {
       await redisCache.$flushTransactions();
@@ -117,7 +129,7 @@ class Mempool {
       this.mempoolChangedCallback(this.mempoolCache, [], [], []);
     }
     if (this.$asyncMempoolChangedCallback) {
-      await this.$asyncMempoolChangedCallback(this.mempoolCache, count, [], [], []);
+      await this.$asyncMempoolChangedCallback(this.mempoolCache, count, [], [], [], this.limitGBT ? { txs: {}, added: [], removed: [] } : undefined);
     }
     this.addToSpendMap(Object.values(this.mempoolCache));
   }
@@ -160,6 +172,10 @@ class Mempool {
     return newTransactions;
   }
 
+  public getMempoolCandidates(): { [txid: string]: boolean } {
+    return this.mempoolCandidates;
+  }
+
   public async $updateMemPoolInfo() {
     this.mempoolInfo = await this.$getMempoolInfo();
   }
@@ -189,7 +205,7 @@ class Mempool {
     return txTimes;
   }
 
-  public async $updateMempool(transactions: string[], accelerations: Acceleration[] | null, pollRate: number): Promise<void> {
+  public async $updateMempool(transactions: string[], accelerations: Acceleration[] | null, minFeeMempool: string[], minFeeTip: number, pollRate: number): Promise<void> {
     logger.debug(`Updating mempool...`);
 
     // warn if this run stalls the main loop for more than 2 minutes
@@ -330,6 +346,8 @@ class Mempool {
       }
     }
 
+    const candidates = await this.getNextCandidates(minFeeMempool, minFeeTip, deletedTransactions);
+
     const newMempoolSize = currentMempoolSize + newTransactions.length - deletedTransactions.length;
     const newTransactionsStripped = newTransactions.map((tx) => Common.stripTransaction(tx));
     this.latestTransactions = newTransactionsStripped.concat(this.latestTransactions).slice(0, 6);
@@ -341,12 +359,14 @@ class Mempool {
 
     this.mempoolCacheDelta = Math.abs(transactions.length - newMempoolSize);
 
+    const candidatesChanged = candidates?.added?.length || candidates?.removed?.length;
+
     if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) {
       this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions, accelerationDelta);
     }
-    if (this.$asyncMempoolChangedCallback && (hasChange || deletedTransactions.length)) {
+    if (this.$asyncMempoolChangedCallback && (hasChange || deletedTransactions.length || candidatesChanged)) {
       this.updateTimerProgress(timer, 'running async mempool callback');
-      await this.$asyncMempoolChangedCallback(this.mempoolCache, newMempoolSize, newTransactions, deletedTransactions, accelerationDelta);
+      await this.$asyncMempoolChangedCallback(this.mempoolCache, newMempoolSize, newTransactions, deletedTransactions, accelerationDelta, candidates);
       this.updateTimerProgress(timer, 'completed async mempool callback');
     }
 
@@ -432,6 +452,64 @@ class Mempool {
     }
   }
 
+  public async getNextCandidates(minFeeTransactions: string[], blockHeight: number, deletedTransactions: MempoolTransactionExtended[]): Promise<GbtCandidates | undefined> {
+    if (this.limitGBT) {
+      const deletedTxsMap = {};
+      for (const tx of deletedTransactions) {
+        deletedTxsMap[tx.txid] = tx;
+      }
+      const newCandidateTxMap = {};
+      for (const txid of minFeeTransactions) {
+        if (this.mempoolCache[txid]) {
+          newCandidateTxMap[txid] = true;
+        }
+      }
+      const accelerations = this.getAccelerations();
+      for (const txid of Object.keys(accelerations)) {
+        if (this.mempoolCache[txid]) {
+          newCandidateTxMap[txid] = true;
+        }
+      }
+      const removed: MempoolTransactionExtended[] = [];
+      const added: MempoolTransactionExtended[] = [];
+      // don't prematurely remove txs included in a new block
+      if (blockHeight > blocks.getCurrentBlockHeight()) {
+        for (const txid of Object.keys(this.mempoolCandidates)) {
+          newCandidateTxMap[txid] = true;
+        }
+      } else {
+        for (const txid of Object.keys(this.mempoolCandidates)) {
+          if (!newCandidateTxMap[txid]) {
+            if (this.mempoolCache[txid]) {
+              removed.push(this.mempoolCache[txid]);
+              this.mempoolCache[txid].effectiveFeePerVsize = this.mempoolCache[txid].adjustedFeePerVsize;
+              this.mempoolCache[txid].ancestors = [];
+              this.mempoolCache[txid].descendants = [];
+              this.mempoolCache[txid].bestDescendant = null;
+              this.mempoolCache[txid].cpfpChecked = false;
+              this.mempoolCache[txid].cpfpUpdated = undefined;
+            } else if (deletedTxsMap[txid]) {
+              removed.push(deletedTxsMap[txid]);
+            }
+          }
+        }
+      }
+
+      for (const txid of Object.keys(newCandidateTxMap)) {
+        if (!this.mempoolCandidates[txid]) {
+          added.push(this.mempoolCache[txid]);
+        }
+      }
+
+      this.mempoolCandidates = newCandidateTxMap;
+      return {
+        txs: this.mempoolCandidates,
+        added,
+        removed
+      };
+    }
+  }
+
   private startTimer() {
     const state: any = {
       start: Date.now(),

backend/src/api/websocket-handler.ts

@@ -2,7 +2,7 @@ import logger from '../logger';
 import * as WebSocket from 'ws';
 import {
   BlockExtended, TransactionExtended, MempoolTransactionExtended, WebsocketResponse,
-  OptimizedStatistic, ILoadingIndicators
+  OptimizedStatistic, ILoadingIndicators, GbtCandidates,
 } from '../mempool.interfaces';
 import blocks from './blocks';
 import memPool from './mempool';
@@ -18,7 +18,6 @@ import feeApi from './fee-api';
 import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository';
 import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
 import Audit from './audit';
-import { deepClone } from '../utils/clone';
 import priceUpdater from '../tasks/price-updater';
 import { ApiPrice } from '../repositories/PricesRepository';
 import accelerationApi from './services/acceleration';
@@ -32,6 +31,8 @@ interface AddressTransactions {
   confirmed: MempoolTransactionExtended[],
   removed: MempoolTransactionExtended[],
 }
+import bitcoinSecondClient from './bitcoin/bitcoin-second-client';
+import { calculateCpfp } from './cpfp';
 
 // valid 'want' subscriptions
 const wantable = [
@@ -436,21 +437,26 @@ class WebsocketHandler {
   }
 
   async $handleMempoolChange(newMempool: { [txid: string]: MempoolTransactionExtended }, mempoolSize: number,
-    newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]): Promise<void> {
+    newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[],
+    candidates?: GbtCandidates): Promise<void> {
     if (!this.wss) {
       throw new Error('WebSocket.Server is not set');
     }
 
     this.printLogs();
 
-    if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) {
+    const transactionIds = (memPool.limitGBT && candidates) ? Object.keys(candidates?.txs || {}) : Object.keys(newMempool);
-      if (config.MEMPOOL.RUST_GBT) {
+    let added = newTransactions;
-        await mempoolBlocks.$rustUpdateBlockTemplates(newMempool, mempoolSize, newTransactions, deletedTransactions, config.MEMPOOL_SERVICES.ACCELERATIONS);
+    let removed = deletedTransactions;
-      } else {
+    if (memPool.limitGBT) {
-        await mempoolBlocks.$updateBlockTemplates(newMempool, newTransactions, deletedTransactions, accelerationDelta, true, config.MEMPOOL_SERVICES.ACCELERATIONS);
+      added = candidates?.added || [];
-      }
+      removed = candidates?.removed || [];
+    }
+
+    if (config.MEMPOOL.RUST_GBT) {
+      await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, newMempool, added, removed, candidates, config.MEMPOOL_SERVICES.ACCELERATIONS);
     } else {
-      mempoolBlocks.updateMempoolBlocks(newMempool, true);
+      await mempoolBlocks.$updateBlockTemplates(transactionIds, newMempool, added, removed, candidates, accelerationDelta, true, config.MEMPOOL_SERVICES.ACCELERATIONS);
     }
 
     const mBlocks = mempoolBlocks.getMempoolBlocks();
@@ -689,6 +695,9 @@ class WebsocketHandler {
               accelerated: mempoolTx.acceleration || undefined,
             }
           };
+          if (!mempoolTx.cpfpChecked) {
+            calculateCpfp(mempoolTx, newMempool);
+          }
           if (mempoolTx.cpfpDirty) {
             positionData['cpfp'] = {
               ancestors: mempoolTx.ancestors,
@@ -739,8 +748,9 @@ class WebsocketHandler {
     await statistics.runStatistics();
 
     const _memPool = memPool.getMempool();
-
+    const candidateTxs = await memPool.getMempoolCandidates();
-    const isAccelerated = config.MEMPOOL_SERVICES.ACCELERATIONS && accelerationApi.isAcceleratedBlock(block, Object.values(mempool.getAccelerations()));
+    let candidates: GbtCandidates | undefined = (memPool.limitGBT && candidateTxs) ? { txs: candidateTxs, added: [], removed: [] } : undefined;
+    let transactionIds: string[] = (memPool.limitGBT) ? Object.keys(candidates?.txs || {}) : Object.keys(_memPool);
 
     const accelerations = Object.values(mempool.getAccelerations());
     await accelerationRepository.$indexAccelerationsForBlock(block, accelerations, transactions);
@@ -751,31 +761,19 @@ class WebsocketHandler {
 
     if (config.MEMPOOL.AUDIT && memPool.isInSync()) {
       let projectedBlocks;
-      let auditMempool = _memPool;
+      const auditMempool = _memPool;
-      // template calculation functions have mempool side effects, so calculate audits using
+      const isAccelerated = config.MEMPOOL_SERVICES.ACCELERATIONS && accelerationApi.isAcceleratedBlock(block, Object.values(mempool.getAccelerations()));
-      // a cloned copy of the mempool if we're running a different algorithm for mempool updates
+
-      const separateAudit = config.MEMPOOL.ADVANCED_GBT_AUDIT !== config.MEMPOOL.ADVANCED_GBT_MEMPOOL;
+      if ((config.MEMPOOL_SERVICES.ACCELERATIONS)) {
-      if (separateAudit) {
+        if (config.MEMPOOL.RUST_GBT) {
-        auditMempool = deepClone(_memPool);
+          const added = memPool.limitGBT ? (candidates?.added || []) : [];
-        if (config.MEMPOOL.ADVANCED_GBT_AUDIT) {
+          const removed = memPool.limitGBT ? (candidates?.removed || []) : [];
-          if (config.MEMPOOL.RUST_GBT) {
+          projectedBlocks = await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, auditMempool, added, removed, candidates, isAccelerated, block.extras.pool.id);
-            projectedBlocks = await mempoolBlocks.$oneOffRustBlockTemplates(auditMempool, isAccelerated, block.extras.pool.id);
-          } else {
-            projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, isAccelerated, block.extras.pool.id);
-          }
         } else {
-          projectedBlocks = mempoolBlocks.updateMempoolBlocks(auditMempool, false);
+          projectedBlocks = await mempoolBlocks.$makeBlockTemplates(transactionIds, auditMempool, candidates, false, isAccelerated, block.extras.pool.id);
         }
       } else {
-        if ((config.MEMPOOL_SERVICES.ACCELERATIONS)) {
+        projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
-          if (config.MEMPOOL.RUST_GBT) {
-            projectedBlocks = await mempoolBlocks.$rustUpdateBlockTemplates(auditMempool, Object.keys(auditMempool).length, [], [], isAccelerated, block.extras.pool.id);
-          } else {
-            projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, isAccelerated, block.extras.pool.id);
-          }
-        } else {
-          projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
-        }
       }
 
       if (Common.indexingEnabled()) {
@@ -838,14 +836,23 @@ class WebsocketHandler {
       confirmedTxids[txId] = true;
     }
 
-    if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) {
+    if (memPool.limitGBT) {
-      if (config.MEMPOOL.RUST_GBT) {
+      const minFeeMempool = memPool.limitGBT ? await bitcoinSecondClient.getRawMemPool() : null;
-        await mempoolBlocks.$rustUpdateBlockTemplates(_memPool, Object.keys(_memPool).length, [], transactions, true);
+      const minFeeTip = memPool.limitGBT ? await bitcoinSecondClient.getBlockCount() : -1;
-      } else {
+      candidates = await memPool.getNextCandidates(minFeeMempool, minFeeTip, transactions);
-        await mempoolBlocks.$makeBlockTemplates(_memPool, true, config.MEMPOOL_SERVICES.ACCELERATIONS);
+      transactionIds = Object.keys(candidates?.txs || {});
-      }
     } else {
-      mempoolBlocks.updateMempoolBlocks(_memPool, true);
+      candidates = undefined;
+      transactionIds = Object.keys(memPool.getMempool());
+    }
+
+
+    if (config.MEMPOOL.RUST_GBT) {
+      const added = memPool.limitGBT ? (candidates?.added || []) : [];
+      const removed = memPool.limitGBT ? (candidates?.removed || []) : transactions;
+      await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, _memPool, added, removed, candidates, true);
+    } else {
+      await mempoolBlocks.$makeBlockTemplates(transactionIds, _memPool, candidates, true, config.MEMPOOL_SERVICES.ACCELERATIONS);
     }
     const mBlocks = mempoolBlocks.getMempoolBlocks();
     const mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();

backend/src/config.ts

@@ -32,9 +32,8 @@ interface IConfig {
     POOLS_JSON_URL: string,
     POOLS_JSON_TREE_URL: string,
     AUDIT: boolean;
-    ADVANCED_GBT_AUDIT: boolean;
-    ADVANCED_GBT_MEMPOOL: boolean;
     RUST_GBT: boolean;
+    LIMIT_GBT: boolean;
     CPFP_INDEXING: boolean;
     MAX_BLOCKS_BULK_QUERY: number;
     DISK_CACHE_BLOCK_INTERVAL: number;
@@ -194,9 +193,8 @@ const defaults: IConfig = {
     'POOLS_JSON_URL': 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json',
     'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
     'AUDIT': false,
-    'ADVANCED_GBT_AUDIT': false,
-    'ADVANCED_GBT_MEMPOOL': false,
     'RUST_GBT': false,
+    'LIMIT_GBT': false,
     'CPFP_INDEXING': false,
     'MAX_BLOCKS_BULK_QUERY': 0,
     'DISK_CACHE_BLOCK_INTERVAL': 6,

backend/src/index.ts

@@ -45,6 +45,7 @@ import { formatBytes, getBytesUnit } from './utils/format';
 import redisCache from './api/redis-cache';
 import accelerationApi from './api/services/acceleration';
 import bitcoinCoreRoutes from './api/bitcoin/bitcoin-core.routes';
+import bitcoinSecondClient from './api/bitcoin/bitcoin-second-client';
 
 class Server {
   private wss: WebSocket.Server | undefined;
@@ -215,11 +216,13 @@ class Server {
         }
       }
       const newMempool = await bitcoinApi.$getRawMempool();
+      const minFeeMempool = memPool.limitGBT ? await bitcoinSecondClient.getRawMemPool() : null;
+      const minFeeTip = memPool.limitGBT ? await bitcoinSecondClient.getBlockCount() : -1;
       const newAccelerations = await accelerationApi.$fetchAccelerations();
       const numHandledBlocks = await blocks.$updateBlocks();
       const pollRate = config.MEMPOOL.POLL_RATE_MS * (indexer.indexerIsRunning() ? 10 : 1);
       if (numHandledBlocks === 0) {
-        await memPool.$updateMempool(newMempool, newAccelerations, pollRate);
+        await memPool.$updateMempool(newMempool, newAccelerations, minFeeMempool, minFeeTip, pollRate);
       }
       indexer.$run();
       if (config.FIAT_PRICE.ENABLED) {

backend/src/mempool.interfaces.ts

@@ -107,6 +107,7 @@ export interface MempoolTransactionExtended extends TransactionExtended {
   inputs?: number[];
   lastBoosted?: number;
   cpfpDirty?: boolean;
+  cpfpUpdated?: number;
 }
 
 export interface AuditTransaction {
@@ -143,6 +144,12 @@ export interface CompactThreadTransaction {
   dirty?: boolean;
 }
 
+export interface GbtCandidates {
+  txs: { [txid: string ]: boolean },
+  added: MempoolTransactionExtended[];
+  removed: MempoolTransactionExtended[];
+}
+
 export interface ThreadTransaction {
   txid: string;
   fee: number;
@@ -181,6 +188,9 @@ export interface CpfpInfo {
   bestDescendant?: BestDescendant | null;
   descendants?: Ancestor[];
   effectiveFeePerVsize?: number;
+  sigops?: number;
+  adjustedVsize?: number,
+  acceleration?: boolean,
 }
 
 export interface TransactionStripped {

docker/README.md

@@ -109,8 +109,6 @@ Below we list all settings from `mempool-config.json` and the corresponding over
     "AUTOMATIC_BLOCK_REINDEXING": false,
     "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json",
     "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
-    "ADVANCED_GBT_AUDIT": false,
-    "ADVANCED_GBT_MEMPOOL": false,
     "CPFP_INDEXING": false,
     "MAX_BLOCKS_BULK_QUERY": 0,
     "DISK_CACHE_BLOCK_INTERVAL": 6,
@@ -142,8 +140,6 @@ Corresponding `docker-compose.yml` overrides:
       MEMPOOL_AUTOMATIC_BLOCK_REINDEXING: ""
       MEMPOOL_POOLS_JSON_URL: ""
       MEMPOOL_POOLS_JSON_TREE_URL: ""
-      MEMPOOL_ADVANCED_GBT_AUDIT: ""
-      MEMPOOL_ADVANCED_GBT_MEMPOOL: ""
       MEMPOOL_CPFP_INDEXING: ""
       MEMPOOL_MAX_BLOCKS_BULK_QUERY: ""
       MEMPOOL_DISK_CACHE_BLOCK_INTERVAL: ""
@@ -151,8 +147,6 @@ Corresponding `docker-compose.yml` overrides:
       ...
 ```
 
-`ADVANCED_GBT_AUDIT` AND `ADVANCED_GBT_MEMPOOL` enable a more accurate (but slower) block prediction algorithm for the block audit feature and the projected mempool-blocks respectively.
-
 `CPFP_INDEXING` enables indexing CPFP (Child Pays For Parent) information for the last `INDEXING_BLOCKS_AMOUNT` blocks.
 
 <br/>

docker/backend/mempool-config.json

@@ -26,9 +26,8 @@
     "GOGGLES_INDEXING": __MEMPOOL_GOGGLES_INDEXING__,
     "AUTOMATIC_BLOCK_REINDEXING": __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__,
     "AUDIT": __MEMPOOL_AUDIT__,
-    "ADVANCED_GBT_AUDIT": __MEMPOOL_ADVANCED_GBT_AUDIT__,
-    "ADVANCED_GBT_MEMPOOL": __MEMPOOL_ADVANCED_GBT_MEMPOOL__,
     "RUST_GBT": __MEMPOOL_RUST_GBT__,
+    "LIMIT_GBT": __MEMPOOL_LIMIT_GBT__,
     "CPFP_INDEXING": __MEMPOOL_CPFP_INDEXING__,
     "MAX_BLOCKS_BULK_QUERY": __MEMPOOL_MAX_BLOCKS_BULK_QUERY__,
     "DISK_CACHE_BLOCK_INTERVAL": __MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__,

docker/backend/start.sh

@@ -29,9 +29,8 @@ __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__=${MEMPOOL_AUTOMATIC_BLOCK_REINDEXING:=fal
 __MEMPOOL_POOLS_JSON_URL__=${MEMPOOL_POOLS_JSON_URL:=https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json}
 __MEMPOOL_POOLS_JSON_TREE_URL__=${MEMPOOL_POOLS_JSON_TREE_URL:=https://api.github.com/repos/mempool/mining-pools/git/trees/master}
 __MEMPOOL_AUDIT__=${MEMPOOL_AUDIT:=false}
-__MEMPOOL_ADVANCED_GBT_AUDIT__=${MEMPOOL_ADVANCED_GBT_AUDIT:=false}
-__MEMPOOL_ADVANCED_GBT_MEMPOOL__=${MEMPOOL_ADVANCED_GBT_MEMPOOL:=false}
 __MEMPOOL_RUST_GBT__=${MEMPOOL_RUST_GBT:=false}
+__MEMPOOL_LIMIT_GBT__=${MEMPOOL_LIMIT_GBT:=false}
 __MEMPOOL_CPFP_INDEXING__=${MEMPOOL_CPFP_INDEXING:=false}
 __MEMPOOL_MAX_BLOCKS_BULK_QUERY__=${MEMPOOL_MAX_BLOCKS_BULK_QUERY:=0}
 __MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__=${MEMPOOL_DISK_CACHE_BLOCK_INTERVAL:=6}
@@ -189,9 +188,8 @@ sed -i "s!__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__!${__MEMPOOL_AUTOMATIC_BLOCK_REI
 sed -i "s!__MEMPOOL_POOLS_JSON_URL__!${__MEMPOOL_POOLS_JSON_URL__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_POOLS_JSON_TREE_URL__!${__MEMPOOL_POOLS_JSON_TREE_URL__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_AUDIT__!${__MEMPOOL_AUDIT__}!g" mempool-config.json
-sed -i "s!__MEMPOOL_ADVANCED_GBT_MEMPOOL__!${__MEMPOOL_ADVANCED_GBT_MEMPOOL__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_RUST_GBT__!${__MEMPOOL_RUST_GBT__}!g" mempool-config.json
-sed -i "s!__MEMPOOL_ADVANCED_GBT_AUDIT__!${__MEMPOOL_ADVANCED_GBT_AUDIT__}!g" mempool-config.json
+sed -i "s!__MEMPOOL_LIMIT_GBT__!${__MEMPOOL_LIMIT_GBT__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_CPFP_INDEXING__!${__MEMPOOL_CPFP_INDEXING__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_MAX_BLOCKS_BULK_QUERY__!${__MEMPOOL_MAX_BLOCKS_BULK_QUERY__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__!${__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__}!g" mempool-config.json

frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts

@@ -206,6 +206,10 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On
 
   update(add: TransactionStripped[], remove: string[], change: { txid: string, rate: number | undefined, acc: boolean | undefined }[], direction: string = 'left', resetLayout: boolean = false): void {
     if (this.scene) {
+      add = add.filter(tx => !this.scene.txs[tx.txid]);
+      remove = remove.filter(txid => this.scene.txs[txid]);
+      change = change.filter(tx => this.scene.txs[tx.txid]);
+
       this.scene.update(add, remove, change, direction, resetLayout);
       this.start();
       this.updateSearchHighlight();

production/mempool-config.mainnet.json

@@ -15,8 +15,6 @@
     "GOGGLES_INDEXING": true,
     "AUDIT": true,
     "CPFP_INDEXING": true,
-    "ADVANCED_GBT_AUDIT": true,
-    "ADVANCED_GBT_MEMPOOL": true,
     "RUST_GBT": true,
     "USE_SECOND_NODE_FOR_MINFEE": true,
     "DISK_CACHE_BLOCK_INTERVAL": 1,

production/mempool-config.signet.json

@@ -9,8 +9,6 @@
     "API_URL_PREFIX": "/api/v1/",
     "INDEXING_BLOCKS_AMOUNT": -1,
     "AUDIT": true,
-    "ADVANCED_GBT_AUDIT": true,
-    "ADVANCED_GBT_MEMPOOL": true,
     "RUST_GBT": true,
     "POLL_RATE_MS": 1000,
     "DISK_CACHE_BLOCK_INTERVAL": 1,

production/mempool-config.testnet.json

@@ -9,8 +9,6 @@
     "API_URL_PREFIX": "/api/v1/",
     "INDEXING_BLOCKS_AMOUNT": -1,
     "AUDIT": true,
-    "ADVANCED_GBT_AUDIT": true,
-    "ADVANCED_GBT_MEMPOOL": true,
     "RUST_GBT": true,
     "POLL_RATE_MS": 1000,
     "DISK_CACHE_BLOCK_INTERVAL": 1,