From e2d3bb4cc571ccff4e20512620aa84477096b777 Mon Sep 17 00:00:00 2001
From: Mononaut <github@monospace.live>
Date: Fri, 17 Feb 2023 17:54:29 -0600
Subject: [PATCH 1/9] Use minfee node to limit gbt input size

---
 backend/src/api/mempool-blocks.ts    | 89 +++++++++++++++++-----------
 backend/src/api/mempool.ts           | 70 +++++++++++++++++++---
 backend/src/api/websocket-handler.ts | 60 +++++++++++++++----
 backend/src/config.ts                |  2 +
 backend/src/index.ts                 |  5 +-
 backend/src/mempool.interfaces.ts    |  6 ++
 6 files changed, 176 insertions(+), 56 deletions(-)

diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts
index b9da7d4e8..09c5d8e79 100644
--- a/backend/src/api/mempool-blocks.ts
+++ b/backend/src/api/mempool-blocks.ts
@@ -1,6 +1,6 @@
 import { GbtGenerator, GbtResult, ThreadTransaction as RustThreadTransaction, ThreadAcceleration as RustThreadAcceleration } from 'rust-gbt';
 import logger from '../logger';
-import { MempoolBlock, MempoolTransactionExtended, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats, PoolTag, TransactionClassified, TransactionCompressed, MempoolDeltaChange } from '../mempool.interfaces';
+import { MempoolBlock, MempoolTransactionExtended, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats, TransactionClassified, TransactionCompressed, MempoolDeltaChange, GbtCandidates } from '../mempool.interfaces';
 import { Common, OnlineFeeStatsCalculator } from './common';
 import config from '../config';
 import { Worker } from 'worker_threads';
@@ -40,12 +40,9 @@ class MempoolBlocks {
     return this.mempoolBlockDeltas;
   }
 
-  public updateMempoolBlocks(memPool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false): MempoolBlockWithTransactions[] {
+  public updateMempoolBlocks(transactions: string[], memPool: { [txid: string]: MempoolTransactionExtended }, candidates: GbtCandidates | undefined, saveResults: boolean = false): MempoolBlockWithTransactions[] {
     const latestMempool = memPool;
-    const memPoolArray: MempoolTransactionExtended[] = [];
-    for (const i in latestMempool) {
-      memPoolArray.push(latestMempool[i]);
-    }
+    const memPoolArray: MempoolTransactionExtended[] = transactions.map(txid => memPool[txid]);
     const start = new Date().getTime();
 
     // Clear bestDescendants & ancestors
@@ -207,7 +204,7 @@ class MempoolBlocks {
     return mempoolBlockDeltas;
   }
 
-  public async $makeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
+  public async $makeBlockTemplates(transactions: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, candidates: GbtCandidates | undefined, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
     const start = Date.now();
 
     // reset mempool short ids
@@ -215,8 +212,9 @@ class MempoolBlocks {
       this.resetUids();
     }
     // set missing short ids
-    for (const tx of Object.values(newMempool)) {
-      this.setUid(tx, !saveResults);
+    for (const txid of transactions) {
+      const tx = newMempool[txid];
+      this.setUid(tx, false);
     }
 
     const accelerations = useAccelerations ? mempool.getAccelerations() : {};
@@ -224,7 +222,8 @@ class MempoolBlocks {
     // prepare a stripped down version of the mempool with only the minimum necessary data
     // to reduce the overhead of passing this data to the worker thread
     const strippedMempool: Map<number, CompactThreadTransaction> = new Map();
-    Object.values(newMempool).forEach(entry => {
+    for (const txid of transactions) {
+      const entry = newMempool[txid];
       if (entry.uid !== null && entry.uid !== undefined) {
         const stripped = {
           uid: entry.uid,
@@ -237,7 +236,7 @@ class MempoolBlocks {
         };
         strippedMempool.set(entry.uid, stripped);
       }
-    });
+    }
 
     // (re)initialize tx selection worker thread
     if (!this.txSelectionWorker) {
@@ -268,7 +267,7 @@ class MempoolBlocks {
       // clean up thread error listener
       this.txSelectionWorker?.removeListener('error', threadErrorListener);
 
-      const processed = this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), accelerations, accelerationPool, saveResults);
+      const processed = this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), candidates, accelerations, accelerationPool, saveResults);
 
       logger.debug(`makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`);
 
@@ -279,10 +278,10 @@ class MempoolBlocks {
     return this.mempoolBlocks;
   }
 
-  public async $updateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], accelerationDelta: string[] = [], saveResults: boolean = false, useAccelerations: boolean = false): Promise<void> {
+  public async $updateBlockTemplates(transactions: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], candidates: GbtCandidates | undefined, accelerationDelta: string[] = [], saveResults: boolean = false, useAccelerations: boolean = false): Promise<void> {
     if (!this.txSelectionWorker) {
       // need to reset the worker
-      await this.$makeBlockTemplates(newMempool, saveResults, useAccelerations);
+      await this.$makeBlockTemplates(transactions, newMempool, candidates, saveResults, useAccelerations);
       return;
     }
 
@@ -292,7 +291,7 @@ class MempoolBlocks {
     const addedAndChanged: MempoolTransactionExtended[] = useAccelerations ? accelerationDelta.map(txid => newMempool[txid]).filter(tx => tx != null).concat(added) : added;
 
     for (const tx of addedAndChanged) {
-      this.setUid(tx, true);
+      this.setUid(tx, false);
     }
     const removedUids = removed.map(tx => this.getUid(tx)).filter(uid => uid != null) as number[];
 
@@ -328,7 +327,7 @@ class MempoolBlocks {
       // clean up thread error listener
       this.txSelectionWorker?.removeListener('error', threadErrorListener);
 
-      this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), accelerations, null, saveResults);
+      this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), candidates, accelerations, null, saveResults);
       logger.debug(`updateBlockTemplates completed in ${(Date.now() - start) / 1000} seconds`);
     } catch (e) {
       logger.err('updateBlockTemplates failed. ' + (e instanceof Error ? e.message : e));
@@ -336,23 +335,26 @@ class MempoolBlocks {
   }
 
   private resetRustGbt(): void {
+    console.log('reset rust gbt');
     this.rustInitialized = false;
     this.rustGbtGenerator = new GbtGenerator();
   }
 
-  public async $rustMakeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
+  public async $rustMakeBlockTemplates(txids: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, candidates: GbtCandidates | undefined, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
     const start = Date.now();
 
     // reset mempool short ids
     if (saveResults) {
       this.resetUids();
     }
+
+    const transactions = txids.map(txid => newMempool[txid]).filter(tx => tx != null);
     // set missing short ids
-    for (const tx of Object.values(newMempool)) {
-      this.setUid(tx, !saveResults);
+    for (const tx of transactions) {
+      this.setUid(tx, false);
     }
     // set short ids for transaction inputs
-    for (const tx of Object.values(newMempool)) {
+    for (const tx of transactions) {
       tx.inputs = tx.vin.map(v => this.getUid(newMempool[v.txid])).filter(uid => (uid !== null && uid !== undefined)) as number[];
     }
 
@@ -369,15 +371,15 @@ class MempoolBlocks {
     const rustGbt = saveResults ? this.rustGbtGenerator : new GbtGenerator();
     try {
       const { blocks, blockWeights, rates, clusters, overflow } = this.convertNapiResultTxids(
-        await rustGbt.make(Object.values(newMempool) as RustThreadTransaction[], convertedAccelerations as RustThreadAcceleration[], this.nextUid),
+        await rustGbt.make(transactions as RustThreadTransaction[], convertedAccelerations as RustThreadAcceleration[], this.nextUid),
       );
       if (saveResults) {
         this.rustInitialized = true;
       }
-      const mempoolSize = Object.keys(newMempool).length;
+      const expectedSize = transactions.length;
       const resultMempoolSize = blocks.reduce((total, block) => total + block.length, 0) + overflow.length;
-      logger.debug(`RUST updateBlockTemplates returned ${resultMempoolSize} txs out of ${mempoolSize} in the mempool, ${overflow.length} were unmineable`);
-      const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, accelerations, accelerationPool, saveResults);
+      logger.debug(`RUST updateBlockTemplates returned ${resultMempoolSize} txs out of ${expectedSize} in the mempool, ${overflow.length} were unmineable`);
+      const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, candidates, accelerations, accelerationPool, saveResults);
       logger.debug(`RUST makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`);
       return processed;
     } catch (e) {
@@ -389,26 +391,26 @@ class MempoolBlocks {
     return this.mempoolBlocks;
   }
 
-  public async $oneOffRustBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, useAccelerations: boolean, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
-    return this.$rustMakeBlockTemplates(newMempool, false, useAccelerations, accelerationPool);
+  public async $oneOffRustBlockTemplates(transactions: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, candidates: GbtCandidates | undefined, useAccelerations: boolean, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
+    return this.$rustMakeBlockTemplates(transactions, newMempool, candidates, false, useAccelerations, accelerationPool);
   }
 
-  public async $rustUpdateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, mempoolSize: number, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], useAccelerations: boolean, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
+  public async $rustUpdateBlockTemplates(transactions: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], candidates: GbtCandidates | undefined, useAccelerations: boolean, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
     // GBT optimization requires that uids never get too sparse
     // as a sanity check, we should also explicitly prevent uint32 uid overflow
-    if (this.nextUid + added.length >= Math.min(Math.max(262144, 2 * mempoolSize), MAX_UINT32)) {
+    if (this.nextUid + added.length >= Math.min(Math.max(262144, 2 * transactions.length), MAX_UINT32)) {
       this.resetRustGbt();
     }
 
     if (!this.rustInitialized) {
       // need to reset the worker
-      return this.$rustMakeBlockTemplates(newMempool, true, useAccelerations, accelerationPool);
+      return this.$rustMakeBlockTemplates(transactions, newMempool, candidates, true, useAccelerations, accelerationPool);
     }
 
     const start = Date.now();
     // set missing short ids
     for (const tx of added) {
-      this.setUid(tx, true);
+      this.setUid(tx, false);
     }
     // set short ids for transaction inputs
     for (const tx of added) {
@@ -416,6 +418,8 @@ class MempoolBlocks {
     }
     const removedUids = removed.map(tx => this.getUid(tx)).filter(uid => (uid !== null && uid !== undefined)) as number[];
 
+    console.log(`removing ${removed.length} (${removedUids.length} with uids)`);
+
     const accelerations = useAccelerations ? mempool.getAccelerations() : {};
     const acceleratedList = accelerationPool ? Object.values(accelerations).filter(acc => newMempool[acc.txid] && acc.pools.includes(accelerationPool)) : Object.values(accelerations).filter(acc => newMempool[acc.txid]);
     const convertedAccelerations = acceleratedList.map(acc => {
@@ -425,6 +429,8 @@ class MempoolBlocks {
       };
     });
 
+    console.log(`${acceleratedList.length} accelerations`);
+
     // run the block construction algorithm in a separate thread, and wait for a result
     try {
       const { blocks, blockWeights, rates, clusters, overflow } = this.convertNapiResultTxids(
@@ -436,11 +442,11 @@ class MempoolBlocks {
         ),
       );
       const resultMempoolSize = blocks.reduce((total, block) => total + block.length, 0) + overflow.length;
-      logger.debug(`RUST updateBlockTemplates returned ${resultMempoolSize} txs out of ${mempoolSize} in the mempool, ${overflow.length} were unmineable`);
-      if (mempoolSize !== resultMempoolSize) {
-        throw new Error('GBT returned wrong number of transactions , cache is probably out of sync');
+      logger.debug(`RUST updateBlockTemplates returned ${resultMempoolSize} txs out of ${transactions.length} candidates, ${overflow.length} were unmineable`);
+      if (transactions.length !== resultMempoolSize) {
+        throw new Error(`GBT returned wrong number of transactions ${transactions.length} vs ${resultMempoolSize}, cache is probably out of sync`);
       } else {
-        const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, accelerations, accelerationPool, true);
+        const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, candidates, accelerations, accelerationPool, true);
         this.removeUids(removedUids);
         logger.debug(`RUST updateBlockTemplates completed in ${(Date.now() - start)/1000} seconds`);
         return processed;
@@ -452,7 +458,7 @@ class MempoolBlocks {
     }
   }
 
-  private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], accelerations, accelerationPool, saveResults): MempoolBlockWithTransactions[] {
+  private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], candidates: GbtCandidates | undefined, accelerations, accelerationPool, saveResults): MempoolBlockWithTransactions[] {    
     for (const [txid, rate] of rates) {
       if (txid in mempool) {
         mempool[txid].cpfpDirty = (rate !== mempool[txid].effectiveFeePerVsize);
@@ -486,6 +492,9 @@ class MempoolBlocks {
             if (txid === memberTxid) {
               matched = true;
             } else {
+              if (!mempool[txid]) {
+                console.log('txid missing from mempool! ', txid, candidates?.txs[txid]);
+              }
               const relative = {
                 txid: txid,
                 fee: mempool[txid].fee,
@@ -518,6 +527,16 @@ class MempoolBlocks {
       let totalWeight = 0;
       let totalFees = 0;
       const transactions: MempoolTransactionExtended[] = [];
+
+      // backfill purged transactions
+      if (candidates?.txs && blockIndex === blocks.length - 1) {
+        for (const txid of Object.keys(mempool)) {
+          if (!candidates.txs[txid]) {
+            block.push(txid);
+          }
+        }
+      }
+
       for (const txid of block) {
         if (txid) {
           mempoolTx = mempool[txid];
diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts
index 4c2f248ac..540e0828b 100644
--- a/backend/src/api/mempool.ts
+++ b/backend/src/api/mempool.ts
@@ -1,6 +1,6 @@
 import config from '../config';
 import bitcoinApi from './bitcoin/bitcoin-api-factory';
-import { MempoolTransactionExtended, TransactionExtended, VbytesPerSecond } from '../mempool.interfaces';
+import { MempoolTransactionExtended, TransactionExtended, VbytesPerSecond, GbtCandidates } from '../mempool.interfaces';
 import logger from '../logger';
 import { Common } from './common';
 import transactionUtils from './transaction-utils';
@@ -11,18 +11,21 @@ import bitcoinSecondClient from './bitcoin/bitcoin-second-client';
 import rbfCache from './rbf-cache';
 import { Acceleration } from './services/acceleration';
 import redisCache from './redis-cache';
+import blocks from './blocks';
 
 class Mempool {
   private inSync: boolean = false;
   private mempoolCacheDelta: number = -1;
   private mempoolCache: { [txId: string]: MempoolTransactionExtended } = {};
+  private mempoolCandidates: { [txid: string ]: boolean } = {};
+  private minFeeMempool: { [txId: string]: boolean } = {};
   private spendMap = new Map<string, MempoolTransactionExtended>();
   private mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0, total_fee: 0,
                                                     maxmempool: 300000000, mempoolminfee: Common.isLiquid() ? 0.00000100 : 0.00001000, minrelaytxfee: Common.isLiquid() ? 0.00000100 : 0.00001000 };
   private mempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, newTransactions: MempoolTransactionExtended[],
     deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => void) | undefined;
   private $asyncMempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, mempoolSize: number, newTransactions: MempoolTransactionExtended[],
-    deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => Promise<void>) | undefined;
+    deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[], candidates?: GbtCandidates) => Promise<void>) | undefined;
 
   private accelerations: { [txId: string]: Acceleration } = {};
 
@@ -40,6 +43,8 @@ class Mempool {
   private missingTxCount = 0;
   private mainLoopTimeout: number = 120000;
 
+  public limitGBT = config.MEMPOOL.USE_SECOND_NODE_FOR_MINFEE && config.MEMPOOL.LIMIT_GBT;
+
   constructor() {
     setInterval(this.updateTxPerSecond.bind(this), 1000);
   }
@@ -74,7 +79,8 @@ class Mempool {
   }
 
   public setAsyncMempoolChangedCallback(fn: (newMempool: { [txId: string]: MempoolTransactionExtended; }, mempoolSize: number,
-    newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => Promise<void>): void {
+    newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[],
+    candidates?: GbtCandidates) => Promise<void>): void {
     this.$asyncMempoolChangedCallback = fn;
   }
 
@@ -117,7 +123,7 @@ class Mempool {
       this.mempoolChangedCallback(this.mempoolCache, [], [], []);
     }
     if (this.$asyncMempoolChangedCallback) {
-      await this.$asyncMempoolChangedCallback(this.mempoolCache, count, [], [], []);
+      await this.$asyncMempoolChangedCallback(this.mempoolCache, count, [], [], [], this.limitGBT ? { txs: {}, added: [], removed: [] } : undefined);
     }
     this.addToSpendMap(Object.values(this.mempoolCache));
   }
@@ -160,6 +166,10 @@ class Mempool {
     return newTransactions;
   }
 
+  public getMempoolCandidates(): { [txid: string]: boolean } {
+    return this.mempoolCandidates;
+  }
+
   public async $updateMemPoolInfo() {
     this.mempoolInfo = await this.$getMempoolInfo();
   }
@@ -189,7 +199,7 @@ class Mempool {
     return txTimes;
   }
 
-  public async $updateMempool(transactions: string[], accelerations: Acceleration[] | null, pollRate: number): Promise<void> {
+  public async $updateMempool(transactions: string[], accelerations: Acceleration[] | null, minFeeMempool: string[], minFeeTip: number, pollRate: number): Promise<void> {
     logger.debug(`Updating mempool...`);
 
     // warn if this run stalls the main loop for more than 2 minutes
@@ -311,6 +321,8 @@ class Mempool {
       }, 1000 * 60 * config.MEMPOOL.CLEAR_PROTECTION_MINUTES);
     }
 
+    const candidates = await this.getNextCandidates(minFeeMempool, minFeeTip);
+
     const deletedTransactions: MempoolTransactionExtended[] = [];
 
     if (this.mempoolProtection !== 1) {
@@ -341,12 +353,14 @@ class Mempool {
 
     this.mempoolCacheDelta = Math.abs(transactions.length - newMempoolSize);
 
+    const candidatesChanged = candidates?.added?.length || candidates?.removed?.length;
+
     if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) {
       this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions, accelerationDelta);
     }
-    if (this.$asyncMempoolChangedCallback && (hasChange || deletedTransactions.length)) {
+    if (this.$asyncMempoolChangedCallback && (hasChange || deletedTransactions.length || candidatesChanged)) {
       this.updateTimerProgress(timer, 'running async mempool callback');
-      await this.$asyncMempoolChangedCallback(this.mempoolCache, newMempoolSize, newTransactions, deletedTransactions, accelerationDelta);
+      await this.$asyncMempoolChangedCallback(this.mempoolCache, newMempoolSize, newTransactions, deletedTransactions, accelerationDelta, candidates);
       this.updateTimerProgress(timer, 'completed async mempool callback');
     }
 
@@ -432,6 +446,48 @@ class Mempool {
     }
   }
 
+  public async getNextCandidates(minFeeTransactions: string[], blockHeight: number): Promise<GbtCandidates | undefined> {
+    if (this.limitGBT) {
+      const newCandidateTxMap = {};
+      this.minFeeMempool = {};
+      for (const txid of minFeeTransactions) {
+        if (this.mempoolCache[txid]) {
+          newCandidateTxMap[txid] = true;
+        }
+        this.minFeeMempool[txid] = true;
+      }
+      const removed: MempoolTransactionExtended[] = [];
+      const added: MempoolTransactionExtended[] = [];
+      // don't prematurely remove txs included in a new block
+      if (blockHeight > blocks.getCurrentBlockHeight()) {
+        for (const txid of Object.keys(this.mempoolCandidates)) {
+          newCandidateTxMap[txid] = true;
+        }
+      } else {
+        for (const txid of Object.keys(this.mempoolCandidates)) {
+          if (!newCandidateTxMap[txid]) {
+            const tx = this.mempoolCache[txid];
+            removed.push(tx);
+          }
+        }
+      }
+
+      for (const txid of Object.keys(newCandidateTxMap)) {
+        if (!this.mempoolCandidates[txid]) {
+          const tx = this.mempoolCache[txid];
+          added.push(tx);
+        }
+      }
+
+      this.mempoolCandidates = newCandidateTxMap;
+      return {
+        txs: this.mempoolCandidates,
+        added,
+        removed
+      };
+    }
+  }
+
   private startTimer() {
     const state: any = {
       start: Date.now(),
diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts
index 2884f72eb..ac79d828a 100644
--- a/backend/src/api/websocket-handler.ts
+++ b/backend/src/api/websocket-handler.ts
@@ -2,7 +2,7 @@ import logger from '../logger';
 import * as WebSocket from 'ws';
 import {
   BlockExtended, TransactionExtended, MempoolTransactionExtended, WebsocketResponse,
-  OptimizedStatistic, ILoadingIndicators
+  OptimizedStatistic, ILoadingIndicators, GbtCandidates,
 } from '../mempool.interfaces';
 import blocks from './blocks';
 import memPool from './mempool';
@@ -32,6 +32,7 @@ interface AddressTransactions {
   confirmed: MempoolTransactionExtended[],
   removed: MempoolTransactionExtended[],
 }
+import bitcoinSecondClient from './bitcoin/bitcoin-second-client';
 
 // valid 'want' subscriptions
 const wantable = [
@@ -436,21 +437,33 @@ class WebsocketHandler {
   }
 
   async $handleMempoolChange(newMempool: { [txid: string]: MempoolTransactionExtended }, mempoolSize: number,
-    newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]): Promise<void> {
+    newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[],
+    candidates?: GbtCandidates): Promise<void> {
     if (!this.wss) {
       throw new Error('WebSocket.Server is not set');
     }
 
     this.printLogs();
 
+    const transactionIds = (memPool.limitGBT && candidates) ? Object.keys(candidates?.txs || {}) : Object.keys(newMempool);
+    let added = newTransactions;
+    let removed = deletedTransactions;
+    console.log(`handleMempoolChange: ${newTransactions.length} new txs, ${deletedTransactions.length} removed, ${candidates?.added.length} candidate added, ${candidates?.removed.length} candidate removed`);
+    console.log(`mempool size ${Object.keys(newMempool).length}, candidates: ${transactionIds.length}`);
+    if (memPool.limitGBT) {
+      console.log('GBT on limited mempool...');
+      added = candidates?.added || [];
+      removed = candidates?.removed || [];
+    }
+
     if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) {
       if (config.MEMPOOL.RUST_GBT) {
-        await mempoolBlocks.$rustUpdateBlockTemplates(newMempool, mempoolSize, newTransactions, deletedTransactions, config.MEMPOOL_SERVICES.ACCELERATIONS);
+        await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, newMempool, added, removed, candidates, config.MEMPOOL_SERVICES.ACCELERATIONS);
       } else {
-        await mempoolBlocks.$updateBlockTemplates(newMempool, newTransactions, deletedTransactions, accelerationDelta, true, config.MEMPOOL_SERVICES.ACCELERATIONS);
+        await mempoolBlocks.$updateBlockTemplates(transactionIds, newMempool, added, removed, candidates, accelerationDelta, true, config.MEMPOOL_SERVICES.ACCELERATIONS);
       }
     } else {
-      mempoolBlocks.updateMempoolBlocks(newMempool, true);
+      mempoolBlocks.updateMempoolBlocks(transactionIds, newMempool, candidates, true);
     }
 
     const mBlocks = mempoolBlocks.getMempoolBlocks();
@@ -739,6 +752,9 @@ class WebsocketHandler {
     await statistics.runStatistics();
 
     const _memPool = memPool.getMempool();
+    const candidateTxs = await memPool.getMempoolCandidates();
+    let candidates: GbtCandidates | undefined = (memPool.limitGBT && candidateTxs) ? { txs: candidateTxs, added: [], removed: [] } : undefined;
+    let transactionIds: string[] = (memPool.limitGBT && candidates) ? Object.keys(_memPool) : Object.keys(candidates?.txs || {});
 
     const isAccelerated = config.MEMPOOL_SERVICES.ACCELERATIONS && accelerationApi.isAcceleratedBlock(block, Object.values(mempool.getAccelerations()));
 
@@ -759,19 +775,23 @@ class WebsocketHandler {
         auditMempool = deepClone(_memPool);
         if (config.MEMPOOL.ADVANCED_GBT_AUDIT) {
           if (config.MEMPOOL.RUST_GBT) {
-            projectedBlocks = await mempoolBlocks.$oneOffRustBlockTemplates(auditMempool, isAccelerated, block.extras.pool.id);
+            projectedBlocks = await mempoolBlocks.$oneOffRustBlockTemplates(transactionIds, auditMempool, candidates, isAccelerated, block.extras.pool.id);
           } else {
-            projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, isAccelerated, block.extras.pool.id);
+            projectedBlocks = await mempoolBlocks.$makeBlockTemplates(transactionIds, auditMempool, candidates, false, isAccelerated, block.extras.pool.id);
           }
         } else {
-          projectedBlocks = mempoolBlocks.updateMempoolBlocks(auditMempool, false);
+          projectedBlocks = mempoolBlocks.updateMempoolBlocks(transactionIds, auditMempool, candidates, false);
         }
       } else {
         if ((config.MEMPOOL_SERVICES.ACCELERATIONS)) {
           if (config.MEMPOOL.RUST_GBT) {
-            projectedBlocks = await mempoolBlocks.$rustUpdateBlockTemplates(auditMempool, Object.keys(auditMempool).length, [], [], isAccelerated, block.extras.pool.id);
+            console.log(`handleNewBlock: ${transactions.length} mined txs, ${candidates?.added.length} candidate added, ${candidates?.removed.length} candidate removed`);
+            console.log(`mempool size ${Object.keys(auditMempool).length}, candidates: ${transactionIds.length}`);
+            let added = memPool.limitGBT ? (candidates?.added || []) : [];
+            let removed = memPool.limitGBT ? (candidates?.removed || []) : [];
+            projectedBlocks = await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, auditMempool, added, removed, candidates, isAccelerated, block.extras.pool.id);
           } else {
-            projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, isAccelerated, block.extras.pool.id);
+            projectedBlocks = await mempoolBlocks.$makeBlockTemplates(transactionIds, auditMempool, candidates, false, isAccelerated, block.extras.pool.id);
           }
         } else {
           projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
@@ -838,14 +858,28 @@ class WebsocketHandler {
       confirmedTxids[txId] = true;
     }
 
+    if (memPool.limitGBT) {
+      const minFeeMempool = memPool.limitGBT ? await bitcoinSecondClient.getRawMemPool() : null;
+      const minFeeTip = memPool.limitGBT ? await bitcoinSecondClient.getBlockCount() : -1;
+      candidates = await memPool.getNextCandidates(minFeeMempool, minFeeTip);
+      transactionIds = Object.keys(candidates?.txs || {});
+    } else {
+      candidates = undefined;
+      transactionIds = Object.keys(memPool.getMempool());
+    }
+
     if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) {
       if (config.MEMPOOL.RUST_GBT) {
-        await mempoolBlocks.$rustUpdateBlockTemplates(_memPool, Object.keys(_memPool).length, [], transactions, true);
+        console.log(`handleNewBlock (after): ${transactions.length} mined txs, ${candidates?.added.length} candidate added, ${candidates?.removed.length} candidate removed`);
+        console.log(`mempool size ${Object.keys(_memPool).length}, candidates: ${transactionIds.length}`);
+        let added = memPool.limitGBT ? (candidates?.added || []) : [];
+        let removed = memPool.limitGBT ? (candidates?.removed || []) : transactions;
+        await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, _memPool, added, removed, candidates, true);
       } else {
-        await mempoolBlocks.$makeBlockTemplates(_memPool, true, config.MEMPOOL_SERVICES.ACCELERATIONS);
+        await mempoolBlocks.$makeBlockTemplates(transactionIds, _memPool, candidates, true, config.MEMPOOL_SERVICES.ACCELERATIONS);
       }
     } else {
-      mempoolBlocks.updateMempoolBlocks(_memPool, true);
+      mempoolBlocks.updateMempoolBlocks(transactionIds, _memPool, candidates, true);
     }
     const mBlocks = mempoolBlocks.getMempoolBlocks();
     const mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();
diff --git a/backend/src/config.ts b/backend/src/config.ts
index ee3645e58..6a202a392 100644
--- a/backend/src/config.ts
+++ b/backend/src/config.ts
@@ -35,6 +35,7 @@ interface IConfig {
     ADVANCED_GBT_AUDIT: boolean;
     ADVANCED_GBT_MEMPOOL: boolean;
     RUST_GBT: boolean;
+    LIMIT_GBT: boolean;
     CPFP_INDEXING: boolean;
     MAX_BLOCKS_BULK_QUERY: number;
     DISK_CACHE_BLOCK_INTERVAL: number;
@@ -197,6 +198,7 @@ const defaults: IConfig = {
     'ADVANCED_GBT_AUDIT': false,
     'ADVANCED_GBT_MEMPOOL': false,
     'RUST_GBT': false,
+    'LIMIT_GBT': false,
     'CPFP_INDEXING': false,
     'MAX_BLOCKS_BULK_QUERY': 0,
     'DISK_CACHE_BLOCK_INTERVAL': 6,
diff --git a/backend/src/index.ts b/backend/src/index.ts
index 1988c7c56..ae5113029 100644
--- a/backend/src/index.ts
+++ b/backend/src/index.ts
@@ -45,6 +45,7 @@ import { formatBytes, getBytesUnit } from './utils/format';
 import redisCache from './api/redis-cache';
 import accelerationApi from './api/services/acceleration';
 import bitcoinCoreRoutes from './api/bitcoin/bitcoin-core.routes';
+import bitcoinSecondClient from './api/bitcoin/bitcoin-second-client';
 
 class Server {
   private wss: WebSocket.Server | undefined;
@@ -215,11 +216,13 @@ class Server {
         }
       }
       const newMempool = await bitcoinApi.$getRawMempool();
+      const minFeeMempool = memPool.limitGBT ? await bitcoinSecondClient.getRawMemPool() : null;
+      const minFeeTip = memPool.limitGBT ? await bitcoinSecondClient.getBlockCount() : -1;
       const newAccelerations = await accelerationApi.$fetchAccelerations();
       const numHandledBlocks = await blocks.$updateBlocks();
       const pollRate = config.MEMPOOL.POLL_RATE_MS * (indexer.indexerIsRunning() ? 10 : 1);
       if (numHandledBlocks === 0) {
-        await memPool.$updateMempool(newMempool, newAccelerations, pollRate);
+        await memPool.$updateMempool(newMempool, newAccelerations, minFeeMempool, minFeeTip, pollRate);
       }
       indexer.$run();
       if (config.FIAT_PRICE.ENABLED) {
diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts
index d04858607..64c10d322 100644
--- a/backend/src/mempool.interfaces.ts
+++ b/backend/src/mempool.interfaces.ts
@@ -143,6 +143,12 @@ export interface CompactThreadTransaction {
   dirty?: boolean;
 }
 
+export interface GbtCandidates {
+  txs: { [txid: string ]: boolean },
+  added: MempoolTransactionExtended[];
+  removed: MempoolTransactionExtended[];
+}
+
 export interface ThreadTransaction {
   txid: string;
   fee: number;

From 62653086e92424ddc62ac0795ab1fc58fad3ebe0 Mon Sep 17 00:00:00 2001
From: Mononaut <github@monospace.live>
Date: Fri, 5 Jan 2024 22:25:07 +0000
Subject: [PATCH 2/9] Limit GBT - calculate purged tx cpfp on demand

---
 backend/src/api/bitcoin/bitcoin.routes.ts |   3 +-
 backend/src/api/cpfp.ts                   | 282 ++++++++++++++++++++++
 backend/src/api/mempool-blocks.ts         |   7 +-
 backend/src/api/mempool.ts                |  17 +-
 backend/src/api/websocket-handler.ts      |   4 +
 backend/src/mempool.interfaces.ts         |   4 +
 6 files changed, 303 insertions(+), 14 deletions(-)
 create mode 100644 backend/src/api/cpfp.ts

diff --git a/backend/src/api/bitcoin/bitcoin.routes.ts b/backend/src/api/bitcoin/bitcoin.routes.ts
index 93b614006..c65363315 100644
--- a/backend/src/api/bitcoin/bitcoin.routes.ts
+++ b/backend/src/api/bitcoin/bitcoin.routes.ts
@@ -19,6 +19,7 @@ import bitcoinClient from './bitcoin-client';
 import difficultyAdjustment from '../difficulty-adjustment';
 import transactionRepository from '../../repositories/TransactionRepository';
 import rbfCache from '../rbf-cache';
+import { calculateCpfp } from '../cpfp';
 
 class BitcoinRoutes {
   public initRoutes(app: Application) {
@@ -217,7 +218,7 @@ class BitcoinRoutes {
         return;
       }
 
-      const cpfpInfo = Common.setRelativesAndGetCpfpInfo(tx, mempool.getMempool());
+      const cpfpInfo = calculateCpfp(tx, mempool.getMempool());
 
       res.json(cpfpInfo);
       return;
diff --git a/backend/src/api/cpfp.ts b/backend/src/api/cpfp.ts
new file mode 100644
index 000000000..cf54771bb
--- /dev/null
+++ b/backend/src/api/cpfp.ts
@@ -0,0 +1,282 @@
+import { CpfpInfo, MempoolTransactionExtended } from '../mempool.interfaces';
+
+const CPFP_UPDATE_INTERVAL = 60_000; // update CPFP info at most once per 60s per transaction
+
+interface GraphTx extends MempoolTransactionExtended {
+  depends: string[];
+  spentby: string[];
+  ancestorMap: Map<string, GraphTx>;
+  fees: {
+    base: number;
+    ancestor: number;
+  };
+  ancestorcount: number;
+  ancestorsize: number;
+  ancestorRate: number;
+  individualRate: number;
+  score: number;
+}
+
+/**
+ * Takes a mempool transaction and a copy of the current mempool, and calculates the CPFP data for
+ * that transaction (and all others in the same cluster)
+ */
+export function calculateCpfp(tx: MempoolTransactionExtended, mempool: { [txid: string]: MempoolTransactionExtended }): CpfpInfo {
+  if (tx.cpfpUpdated && Date.now() < (tx.cpfpUpdated + CPFP_UPDATE_INTERVAL)) {
+    tx.cpfpDirty = false;
+    return {
+      ancestors: tx.ancestors || [],
+      bestDescendant: tx.bestDescendant || null,
+      descendants: tx.descendants || [],
+      effectiveFeePerVsize: tx.effectiveFeePerVsize || tx.adjustedFeePerVsize || tx.feePerVsize,
+      sigops: tx.sigops,
+      adjustedVsize: tx.adjustedVsize,
+      acceleration: tx.acceleration
+    };
+  }
+
+  const ancestorMap = new Map<string, GraphTx>();
+  const graphTx = mempoolToGraphTx(tx);
+  ancestorMap.set(tx.txid, graphTx);
+
+  const allRelatives = expandRelativesGraph(mempool, ancestorMap);
+  const relativesMap = initializeRelatives(allRelatives);
+  const cluster = calculateCpfpCluster(tx.txid, relativesMap);
+
+  let totalVsize = 0;
+  let totalFee = 0;
+  for (const tx of cluster.values()) {
+    totalVsize += tx.adjustedVsize;
+    totalFee += tx.fee;
+  }
+  const effectiveFeePerVsize = totalFee / totalVsize;
+  for (const tx of cluster.values()) {
+    mempool[tx.txid].effectiveFeePerVsize = effectiveFeePerVsize;
+    mempool[tx.txid].ancestors = Array.from(tx.ancestorMap.values()).map(tx => ({ txid: tx.txid, weight: tx.weight, fee: tx.fee }));
+    mempool[tx.txid].descendants = Array.from(cluster.values()).filter(entry => entry.txid !== tx.txid && !tx.ancestorMap.has(entry.txid)).map(tx => ({ txid: tx.txid, weight: tx.weight, fee: tx.fee }));
+    mempool[tx.txid].bestDescendant = null;
+    mempool[tx.txid].cpfpChecked = true;
+    mempool[tx.txid].cpfpDirty = true;
+    mempool[tx.txid].cpfpUpdated = Date.now();
+  }
+
+  tx = mempool[tx.txid];
+
+  return {
+    ancestors: tx.ancestors || [],
+    bestDescendant: tx.bestDescendant || null,
+    descendants: tx.descendants || [],
+    effectiveFeePerVsize: tx.effectiveFeePerVsize || tx.adjustedFeePerVsize || tx.feePerVsize,
+    sigops: tx.sigops,
+    adjustedVsize: tx.adjustedVsize,
+    acceleration: tx.acceleration
+  };
+}
+
+function mempoolToGraphTx(tx: MempoolTransactionExtended): GraphTx {
+  return {
+    ...tx,
+    depends: [],
+    spentby: [],
+    ancestorMap: new Map(),
+    fees: {
+      base: tx.fee,
+      ancestor: tx.fee,
+    },
+    ancestorcount: 1,
+    ancestorsize: tx.adjustedVsize,
+    ancestorRate: 0,
+    individualRate: 0,
+    score: 0,
+  };
+}
+
+/**
+ * Takes a map of transaction ancestors, and expands it into a full graph of up to 50 in-mempool relatives
+ */
+function expandRelativesGraph(mempool: { [txid: string]: MempoolTransactionExtended }, ancestors: Map<string, GraphTx>): Map<string, GraphTx> {
+  const relatives: Map<string, GraphTx> = new Map();
+  const stack: GraphTx[] = Array.from(ancestors.values());
+  while (stack.length > 0) {
+    if (relatives.size > 50) {
+      return relatives;
+    }
+
+    const nextTx = stack.pop();
+    if (!nextTx) {
+      continue;
+    }
+    relatives.set(nextTx.txid, nextTx);
+
+    for (const relativeTxid of [...nextTx.depends, ...nextTx.spentby]) {
+      if (relatives.has(relativeTxid)) {
+        // already processed this tx
+        continue;
+      }
+      let mempoolTx = ancestors.get(relativeTxid);
+      if (!mempoolTx && mempool[relativeTxid]) {
+        mempoolTx = mempoolToGraphTx(mempool[relativeTxid]);
+      }
+      if (mempoolTx) {
+        stack.push(mempoolTx);
+      }
+    }
+  }
+
+  return relatives;
+}
+
+ /**
+   * Efficiently sets a Map of in-mempool ancestors for each member of an expanded relative graph
+   * by running setAncestors on each leaf, and caching intermediate results.
+   * then initializes ancestor data for each transaction
+   * 
+   * @param all 
+   */
+ function initializeRelatives(mempoolTxs: Map<string, GraphTx>): Map<string, GraphTx> {
+  const visited: Map<string, Map<string, GraphTx>> = new Map();
+  const leaves: GraphTx[] = Array.from(mempoolTxs.values()).filter(entry => entry.spentby.length === 0);
+  for (const leaf of leaves) {
+    setAncestors(leaf, mempoolTxs, visited);
+  }
+  mempoolTxs.forEach(entry => {
+    entry.ancestorMap?.forEach(ancestor => {
+      entry.ancestorcount++;
+      entry.ancestorsize += ancestor.adjustedVsize;
+      entry.fees.ancestor += ancestor.fees.base;
+    });
+    setAncestorScores(entry);
+  });
+  return mempoolTxs;
+}
+
+/**
+   * Given a root transaction and a list of in-mempool ancestors,
+   * Calculate the CPFP cluster
+   * 
+   * @param tx
+   * @param ancestors
+   */
+function calculateCpfpCluster(txid: string, graph: Map<string, GraphTx>): Map<string, GraphTx> {
+  const tx = graph.get(txid);
+  if (!tx) {
+    return new Map<string, GraphTx>([]);
+  }
+
+  // Initialize individual & ancestor fee rates
+  graph.forEach(entry => setAncestorScores(entry));
+
+  // Sort by descending ancestor score
+  let sortedRelatives = Array.from(graph.values()).sort(mempoolComparator);
+
+  // Iterate until we reach a cluster that includes our target tx
+  let maxIterations = 50;
+  let best = sortedRelatives.shift();
+  let bestCluster = new Map<string, GraphTx>(best?.ancestorMap?.entries() || []);
+  while (sortedRelatives.length && best && (best.txid !== tx.txid && !best.ancestorMap.has(tx.txid)) && maxIterations > 0) {
+    maxIterations--;
+    if (bestCluster && bestCluster.has(tx.txid)) {
+      break;
+    } else {
+      // Remove this cluster (it doesn't include our target tx)
+      // and update scores, ancestor totals and dependencies for the survivors
+      removeAncestors(bestCluster, graph);
+
+      // re-sort
+      sortedRelatives = Array.from(graph.values()).sort(mempoolComparator);
+
+      // Grab the next highest scoring entry
+      best = sortedRelatives.shift();
+      if (best) {
+        bestCluster = new Map<string, GraphTx>(best?.ancestorMap?.entries() || []);
+        bestCluster.set(best?.txid, best);
+      }
+    }
+  }
+
+  return bestCluster;
+}
+
+ /**
+   * Remove a cluster of transactions from an in-mempool dependency graph
+   * and update the survivors' scores and ancestors
+   * 
+   * @param cluster 
+   * @param ancestors 
+   */
+ function removeAncestors(cluster: Map<string, GraphTx>, all: Map<string, GraphTx>): void {
+  // remove
+  cluster.forEach(tx => {
+    all.delete(tx.txid);
+  });
+
+  // update survivors
+  all.forEach(tx => {
+    cluster.forEach(remove => {
+      if (tx.ancestorMap?.has(remove.txid)) {
+        // remove as dependency
+        tx.ancestorMap.delete(remove.txid);
+        tx.depends = tx.depends.filter(parent => parent !== remove.txid);
+        // update ancestor sizes and fees
+        tx.ancestorsize -= remove.adjustedVsize;
+        tx.fees.ancestor -= remove.fees.base;
+      }
+    });
+    // recalculate fee rates
+    setAncestorScores(tx);
+  });
+}
+
+/**
+   * Recursively traverses an in-mempool dependency graph, and sets a Map of in-mempool ancestors
+   * for each transaction.
+   * 
+   * @param tx 
+   * @param all 
+   */
+function setAncestors(tx: GraphTx, all: Map<string, GraphTx>, visited: Map<string, Map<string, GraphTx>>, depth: number = 0): Map<string, GraphTx> {
+  // sanity check for infinite recursion / too many ancestors (should never happen)
+  if (depth > 50) {
+    return tx.ancestorMap;
+  }
+  
+  // initialize the ancestor map for this tx
+  tx.ancestorMap = new Map<string, GraphTx>();
+  tx.depends.forEach(parentId => {
+    const parent = all.get(parentId);
+    if (parent) {
+      // add the parent
+      tx.ancestorMap?.set(parentId, parent);
+      // check for a cached copy of this parent's ancestors
+      let ancestors = visited.get(parent.txid);
+      if (!ancestors) {
+        // recursively fetch the parent's ancestors
+        ancestors = setAncestors(parent, all, visited, depth + 1);
+      }
+      // and add to this tx's map
+      ancestors.forEach((ancestor, ancestorId) => {
+        tx.ancestorMap?.set(ancestorId, ancestor);
+      });
+    }
+  });
+  visited.set(tx.txid, tx.ancestorMap);
+
+  return tx.ancestorMap;
+}
+
+/**
+   * Take a mempool transaction, and set the fee rates and ancestor score
+   * 
+   * @param tx 
+   */
+function setAncestorScores(tx: GraphTx): GraphTx {
+  tx.individualRate = (tx.fees.base * 100_000_000) / tx.adjustedVsize;
+  tx.ancestorRate = (tx.fees.ancestor * 100_000_000) / tx.ancestorsize;
+  tx.score = Math.min(tx.individualRate, tx.ancestorRate);
+  return tx;
+}
+
+// Sort by descending score
+function mempoolComparator(a: GraphTx, b: GraphTx): number {
+  return b.score - a.score;
+}
\ No newline at end of file
diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts
index 09c5d8e79..af071321e 100644
--- a/backend/src/api/mempool-blocks.ts
+++ b/backend/src/api/mempool-blocks.ts
@@ -335,7 +335,6 @@ class MempoolBlocks {
   }
 
   private resetRustGbt(): void {
-    console.log('reset rust gbt');
     this.rustInitialized = false;
     this.rustGbtGenerator = new GbtGenerator();
   }
@@ -418,8 +417,6 @@ class MempoolBlocks {
     }
     const removedUids = removed.map(tx => this.getUid(tx)).filter(uid => (uid !== null && uid !== undefined)) as number[];
 
-    console.log(`removing ${removed.length} (${removedUids.length} with uids)`);
-
     const accelerations = useAccelerations ? mempool.getAccelerations() : {};
     const acceleratedList = accelerationPool ? Object.values(accelerations).filter(acc => newMempool[acc.txid] && acc.pools.includes(accelerationPool)) : Object.values(accelerations).filter(acc => newMempool[acc.txid]);
     const convertedAccelerations = acceleratedList.map(acc => {
@@ -429,8 +426,6 @@ class MempoolBlocks {
       };
     });
 
-    console.log(`${acceleratedList.length} accelerations`);
-
     // run the block construction algorithm in a separate thread, and wait for a result
     try {
       const { blocks, blockWeights, rates, clusters, overflow } = this.convertNapiResultTxids(
@@ -458,7 +453,7 @@ class MempoolBlocks {
     }
   }
 
-  private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], candidates: GbtCandidates | undefined, accelerations, accelerationPool, saveResults): MempoolBlockWithTransactions[] {    
+  private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], candidates: GbtCandidates | undefined, accelerations, accelerationPool, saveResults): MempoolBlockWithTransactions[] {
     for (const [txid, rate] of rates) {
       if (txid in mempool) {
         mempool[txid].cpfpDirty = (rate !== mempool[txid].effectiveFeePerVsize);
diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts
index 540e0828b..b08848b26 100644
--- a/backend/src/api/mempool.ts
+++ b/backend/src/api/mempool.ts
@@ -18,7 +18,6 @@ class Mempool {
   private mempoolCacheDelta: number = -1;
   private mempoolCache: { [txId: string]: MempoolTransactionExtended } = {};
   private mempoolCandidates: { [txid: string ]: boolean } = {};
-  private minFeeMempool: { [txId: string]: boolean } = {};
   private spendMap = new Map<string, MempoolTransactionExtended>();
   private mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0, total_fee: 0,
                                                     maxmempool: 300000000, mempoolminfee: Common.isLiquid() ? 0.00000100 : 0.00001000, minrelaytxfee: Common.isLiquid() ? 0.00000100 : 0.00001000 };
@@ -449,12 +448,10 @@ class Mempool {
   public async getNextCandidates(minFeeTransactions: string[], blockHeight: number): Promise<GbtCandidates | undefined> {
     if (this.limitGBT) {
       const newCandidateTxMap = {};
-      this.minFeeMempool = {};
       for (const txid of minFeeTransactions) {
         if (this.mempoolCache[txid]) {
           newCandidateTxMap[txid] = true;
         }
-        this.minFeeMempool[txid] = true;
       }
       const removed: MempoolTransactionExtended[] = [];
       const added: MempoolTransactionExtended[] = [];
@@ -466,16 +463,22 @@ class Mempool {
       } else {
         for (const txid of Object.keys(this.mempoolCandidates)) {
           if (!newCandidateTxMap[txid]) {
-            const tx = this.mempoolCache[txid];
-            removed.push(tx);
+            removed.push(this.mempoolCache[txid]);
+            if (this.mempoolCache[txid]) {
+              this.mempoolCache[txid].effectiveFeePerVsize = this.mempoolCache[txid].adjustedFeePerVsize;
+              this.mempoolCache[txid].ancestors = [];
+              this.mempoolCache[txid].descendants = [];
+              this.mempoolCache[txid].bestDescendant = null;
+              this.mempoolCache[txid].cpfpChecked = false;
+              this.mempoolCache[txid].cpfpUpdated = undefined;
+            }
           }
         }
       }
 
       for (const txid of Object.keys(newCandidateTxMap)) {
         if (!this.mempoolCandidates[txid]) {
-          const tx = this.mempoolCache[txid];
-          added.push(tx);
+          added.push(this.mempoolCache[txid]);
         }
       }
 
diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts
index ac79d828a..f34f7e764 100644
--- a/backend/src/api/websocket-handler.ts
+++ b/backend/src/api/websocket-handler.ts
@@ -33,6 +33,7 @@ interface AddressTransactions {
   removed: MempoolTransactionExtended[],
 }
 import bitcoinSecondClient from './bitcoin/bitcoin-second-client';
+import { calculateCpfp } from './cpfp';
 
 // valid 'want' subscriptions
 const wantable = [
@@ -702,6 +703,9 @@ class WebsocketHandler {
               accelerated: mempoolTx.acceleration || undefined,
             }
           };
+          if (!mempoolTx.cpfpChecked) {
+            calculateCpfp(mempoolTx, newMempool);
+          }
           if (mempoolTx.cpfpDirty) {
             positionData['cpfp'] = {
               ancestors: mempoolTx.ancestors,
diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts
index 64c10d322..a12351fe3 100644
--- a/backend/src/mempool.interfaces.ts
+++ b/backend/src/mempool.interfaces.ts
@@ -107,6 +107,7 @@ export interface MempoolTransactionExtended extends TransactionExtended {
   inputs?: number[];
   lastBoosted?: number;
   cpfpDirty?: boolean;
+  cpfpUpdated?: number;
 }
 
 export interface AuditTransaction {
@@ -187,6 +188,9 @@ export interface CpfpInfo {
   bestDescendant?: BestDescendant | null;
   descendants?: Ancestor[];
   effectiveFeePerVsize?: number;
+  sigops?: number;
+  adjustedVsize?: number,
+  acceleration?: boolean,
 }
 
 export interface TransactionStripped {

From b3e07e0c22cbcdf642f20e153a017282a1a051e0 Mon Sep 17 00:00:00 2001
From: Mononaut <github@monospace.live>
Date: Sat, 6 Jan 2024 18:26:03 +0000
Subject: [PATCH 3/9] Retire "getSimonTemplate" GBT option

---
 backend/mempool-config.sample.json            |   2 -
 .../__fixtures__/mempool-config.template.json |   2 -
 backend/src/__tests__/config.test.ts          |   2 -
 backend/src/api/common.ts                     |  63 ---------
 backend/src/api/mempool-blocks.ts             | 123 ------------------
 backend/src/api/websocket-handler.ts          |  69 +++-------
 backend/src/config.ts                         |   4 -
 docker/README.md                              |   6 -
 docker/backend/mempool-config.json            |   2 -
 docker/backend/start.sh                       |   4 -
 production/mempool-config.mainnet.json        |   2 -
 production/mempool-config.signet.json         |   2 -
 production/mempool-config.testnet.json        |   2 -
 13 files changed, 19 insertions(+), 264 deletions(-)

diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json
index 781973055..c98a070f1 100644
--- a/backend/mempool-config.sample.json
+++ b/backend/mempool-config.sample.json
@@ -28,8 +28,6 @@
     "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json",
     "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
     "AUDIT": false,
-    "ADVANCED_GBT_AUDIT": false,
-    "ADVANCED_GBT_MEMPOOL": false,
     "RUST_GBT": false,
     "CPFP_INDEXING": false,
     "DISK_CACHE_BLOCK_INTERVAL": 6,
diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json
index 5cbff22a3..7d571c169 100644
--- a/backend/src/__fixtures__/mempool-config.template.json
+++ b/backend/src/__fixtures__/mempool-config.template.json
@@ -28,8 +28,6 @@
     "POOLS_JSON_TREE_URL": "__MEMPOOL_POOLS_JSON_TREE_URL__",
     "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__",
     "AUDIT": true,
-    "ADVANCED_GBT_AUDIT": true,
-    "ADVANCED_GBT_MEMPOOL": true,
     "RUST_GBT": false,
     "CPFP_INDEXING": true,
     "MAX_BLOCKS_BULK_QUERY": 999,
diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts
index e261e2adc..84c2e763b 100644
--- a/backend/src/__tests__/config.test.ts
+++ b/backend/src/__tests__/config.test.ts
@@ -41,8 +41,6 @@ describe('Mempool Backend Config', () => {
         POOLS_JSON_TREE_URL: 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
         POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json',
         AUDIT: false,
-        ADVANCED_GBT_AUDIT: false,
-        ADVANCED_GBT_MEMPOOL: false,
         RUST_GBT: false,
         CPFP_INDEXING: false,
         MAX_BLOCKS_BULK_QUERY: 0,
diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts
index 3021d947a..d8cf0d73f 100644
--- a/backend/src/api/common.ts
+++ b/backend/src/api/common.ts
@@ -574,69 +574,6 @@ export class Common {
     }
   }
 
-  static setRelativesAndGetCpfpInfo(tx: MempoolTransactionExtended, memPool: { [txid: string]: MempoolTransactionExtended }): CpfpInfo {
-    const parents = this.findAllParents(tx, memPool);
-    const lowerFeeParents = parents.filter((parent) => parent.adjustedFeePerVsize < tx.effectiveFeePerVsize);
-
-    let totalWeight = (tx.adjustedVsize * 4) + lowerFeeParents.reduce((prev, val) => prev + (val.adjustedVsize * 4), 0);
-    let totalFees = tx.fee + lowerFeeParents.reduce((prev, val) => prev + val.fee, 0);
-
-    tx.ancestors = parents
-      .map((t) => {
-        return {
-          txid: t.txid,
-          weight: (t.adjustedVsize * 4),
-          fee: t.fee,
-        };
-      });
-
-    // Add high (high fee) decendant weight and fees
-    if (tx.bestDescendant) {
-      totalWeight += tx.bestDescendant.weight;
-      totalFees += tx.bestDescendant.fee;
-    }
-
-    tx.effectiveFeePerVsize = Math.max(0, totalFees / (totalWeight / 4));
-    tx.cpfpChecked = true;
-
-    return {
-      ancestors: tx.ancestors,
-      bestDescendant: tx.bestDescendant || null,
-    };
-  }
-
-
-  private static findAllParents(tx: MempoolTransactionExtended, memPool: { [txid: string]: MempoolTransactionExtended }): MempoolTransactionExtended[] {
-    let parents: MempoolTransactionExtended[] = [];
-    tx.vin.forEach((parent) => {
-      if (parents.find((p) => p.txid === parent.txid)) {
-        return;
-      }
-
-      const parentTx = memPool[parent.txid];
-      if (parentTx) {
-        if (tx.bestDescendant && tx.bestDescendant.fee / (tx.bestDescendant.weight / 4) > parentTx.adjustedFeePerVsize) {
-          if (parentTx.bestDescendant && parentTx.bestDescendant.fee < tx.fee + tx.bestDescendant.fee) {
-            parentTx.bestDescendant = {
-              weight: (tx.adjustedVsize * 4) + tx.bestDescendant.weight,
-              fee: tx.fee + tx.bestDescendant.fee,
-              txid: tx.txid,
-            };
-          }
-        } else if (tx.adjustedFeePerVsize > parentTx.adjustedFeePerVsize) {
-          parentTx.bestDescendant = {
-            weight: (tx.adjustedVsize * 4),
-            fee: tx.fee,
-            txid: tx.txid
-          };
-        }
-        parents.push(parentTx);
-        parents = parents.concat(this.findAllParents(parentTx, memPool));
-      }
-    });
-    return parents;
-  }
-
   // calculates the ratio of matched transactions to projected transactions by weight
   static getSimilarity(projectedBlock: MempoolBlockWithTransactions, transactions: TransactionExtended[]): number {
     let matchedWeight = 0;
diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts
index af071321e..fd9918f71 100644
--- a/backend/src/api/mempool-blocks.ts
+++ b/backend/src/api/mempool-blocks.ts
@@ -40,129 +40,6 @@ class MempoolBlocks {
     return this.mempoolBlockDeltas;
   }
 
-  public updateMempoolBlocks(transactions: string[], memPool: { [txid: string]: MempoolTransactionExtended }, candidates: GbtCandidates | undefined, saveResults: boolean = false): MempoolBlockWithTransactions[] {
-    const latestMempool = memPool;
-    const memPoolArray: MempoolTransactionExtended[] = transactions.map(txid => memPool[txid]);
-    const start = new Date().getTime();
-
-    // Clear bestDescendants & ancestors
-    memPoolArray.forEach((tx) => {
-      tx.bestDescendant = null;
-      tx.ancestors = [];
-      tx.cpfpChecked = false;
-      if (!tx.effectiveFeePerVsize) {
-        tx.effectiveFeePerVsize = tx.adjustedFeePerVsize;
-      }
-    });
-
-    // First sort
-    memPoolArray.sort((a, b) => {
-      if (a.adjustedFeePerVsize === b.adjustedFeePerVsize) {
-        // tie-break by lexicographic txid order for stability
-        return a.txid < b.txid ? -1 : 1;
-      } else {
-        return b.adjustedFeePerVsize - a.adjustedFeePerVsize;
-      }
-    });
-
-    // Loop through and traverse all ancestors and sum up all the sizes + fees
-    // Pass down size + fee to all unconfirmed children
-    let sizes = 0;
-    memPoolArray.forEach((tx) => {
-      sizes += tx.weight;
-      if (sizes > 4000000 * 8) {
-        return;
-      }
-      Common.setRelativesAndGetCpfpInfo(tx, memPool);
-    });
-
-    // Final sort, by effective fee
-    memPoolArray.sort((a, b) => {
-      if (a.effectiveFeePerVsize === b.effectiveFeePerVsize) {
-        // tie-break by lexicographic txid order for stability
-        return a.txid < b.txid ? -1 : 1;
-      } else {
-        return b.effectiveFeePerVsize - a.effectiveFeePerVsize;
-      }
-    });
-
-    const end = new Date().getTime();
-    const time = end - start;
-    logger.debug('Mempool blocks calculated in ' + time / 1000 + ' seconds');
-
-    const blocks = this.calculateMempoolBlocks(memPoolArray);
-
-    if (saveResults) {
-      const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, blocks);
-      this.mempoolBlocks = blocks;
-      this.mempoolBlockDeltas = deltas;
-    }
-
-    return blocks;
-  }
-
-  private calculateMempoolBlocks(transactionsSorted: MempoolTransactionExtended[]): MempoolBlockWithTransactions[] {
-    const mempoolBlocks: MempoolBlockWithTransactions[] = [];
-    let feeStatsCalculator: OnlineFeeStatsCalculator = new OnlineFeeStatsCalculator(config.MEMPOOL.BLOCK_WEIGHT_UNITS);
-    let onlineStats = false;
-    let blockSize = 0;
-    let blockWeight = 0;
-    let blockVsize = 0;
-    let blockFees = 0;
-    const sizeLimit = (config.MEMPOOL.BLOCK_WEIGHT_UNITS / 4) * 1.2;
-    let transactionIds: string[] = [];
-    let transactions: MempoolTransactionExtended[] = [];
-    transactionsSorted.forEach((tx, index) => {
-      if (blockWeight + tx.weight <= config.MEMPOOL.BLOCK_WEIGHT_UNITS
-        || mempoolBlocks.length === config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT - 1) {
-        tx.position = {
-          block: mempoolBlocks.length,
-          vsize: blockVsize + (tx.vsize / 2),
-        };
-        blockWeight += tx.weight;
-        blockVsize += tx.vsize;
-        blockSize += tx.size;
-        blockFees += tx.fee;
-        if (blockVsize <= sizeLimit) {
-          transactions.push(tx);
-        }
-        transactionIds.push(tx.txid);
-        if (onlineStats) {
-          feeStatsCalculator.processNext(tx);
-        }
-      } else {
-        mempoolBlocks.push(this.dataToMempoolBlocks(transactionIds, transactions, blockSize, blockWeight, blockFees));
-        blockVsize = 0;
-        tx.position = {
-          block: mempoolBlocks.length,
-          vsize: blockVsize + (tx.vsize / 2),
-        };
-
-        if (mempoolBlocks.length === config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT - 1) {
-          const stackWeight = transactionsSorted.slice(index).reduce((total, tx) => total + (tx.weight || 0), 0);
-          if (stackWeight > config.MEMPOOL.BLOCK_WEIGHT_UNITS) {
-            onlineStats = true;
-            feeStatsCalculator = new OnlineFeeStatsCalculator(stackWeight, 0.5, [10, 20, 30, 40, 50, 60, 70, 80, 90]);
-            feeStatsCalculator.processNext(tx);
-          }
-        }
-
-        blockVsize += tx.vsize;
-        blockWeight = tx.weight;
-        blockSize = tx.size;
-        blockFees = tx.fee;
-        transactionIds = [tx.txid];
-        transactions = [tx];
-      }
-    });
-    if (transactions.length) {
-      const feeStats = onlineStats ? feeStatsCalculator.getRawFeeStats() : undefined;
-      mempoolBlocks.push(this.dataToMempoolBlocks(transactionIds, transactions, blockSize, blockWeight, blockFees, feeStats));
-    }
-
-    return mempoolBlocks;
-  }
-
   private calculateMempoolDeltas(prevBlocks: MempoolBlockWithTransactions[], mempoolBlocks: MempoolBlockWithTransactions[]): MempoolBlockDelta[] {
     const mempoolBlockDeltas: MempoolBlockDelta[] = [];
     for (let i = 0; i < Math.max(mempoolBlocks.length, prevBlocks.length); i++) {
diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts
index f34f7e764..d42fc11f0 100644
--- a/backend/src/api/websocket-handler.ts
+++ b/backend/src/api/websocket-handler.ts
@@ -18,7 +18,6 @@ import feeApi from './fee-api';
 import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository';
 import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
 import Audit from './audit';
-import { deepClone } from '../utils/clone';
 import priceUpdater from '../tasks/price-updater';
 import { ApiPrice } from '../repositories/PricesRepository';
 import accelerationApi from './services/acceleration';
@@ -449,22 +448,15 @@ class WebsocketHandler {
     const transactionIds = (memPool.limitGBT && candidates) ? Object.keys(candidates?.txs || {}) : Object.keys(newMempool);
     let added = newTransactions;
     let removed = deletedTransactions;
-    console.log(`handleMempoolChange: ${newTransactions.length} new txs, ${deletedTransactions.length} removed, ${candidates?.added.length} candidate added, ${candidates?.removed.length} candidate removed`);
-    console.log(`mempool size ${Object.keys(newMempool).length}, candidates: ${transactionIds.length}`);
     if (memPool.limitGBT) {
-      console.log('GBT on limited mempool...');
       added = candidates?.added || [];
       removed = candidates?.removed || [];
     }
 
-    if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) {
-      if (config.MEMPOOL.RUST_GBT) {
-        await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, newMempool, added, removed, candidates, config.MEMPOOL_SERVICES.ACCELERATIONS);
-      } else {
-        await mempoolBlocks.$updateBlockTemplates(transactionIds, newMempool, added, removed, candidates, accelerationDelta, true, config.MEMPOOL_SERVICES.ACCELERATIONS);
-      }
+    if (config.MEMPOOL.RUST_GBT) {
+      await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, newMempool, added, removed, candidates, config.MEMPOOL_SERVICES.ACCELERATIONS);
     } else {
-      mempoolBlocks.updateMempoolBlocks(transactionIds, newMempool, candidates, true);
+      await mempoolBlocks.$updateBlockTemplates(transactionIds, newMempool, added, removed, candidates, accelerationDelta, true, config.MEMPOOL_SERVICES.ACCELERATIONS);
     }
 
     const mBlocks = mempoolBlocks.getMempoolBlocks();
@@ -760,8 +752,6 @@ class WebsocketHandler {
     let candidates: GbtCandidates | undefined = (memPool.limitGBT && candidateTxs) ? { txs: candidateTxs, added: [], removed: [] } : undefined;
     let transactionIds: string[] = (memPool.limitGBT && candidates) ? Object.keys(_memPool) : Object.keys(candidates?.txs || {});
 
-    const isAccelerated = config.MEMPOOL_SERVICES.ACCELERATIONS && accelerationApi.isAcceleratedBlock(block, Object.values(mempool.getAccelerations()));
-
     const accelerations = Object.values(mempool.getAccelerations());
     await accelerationRepository.$indexAccelerationsForBlock(block, accelerations, transactions);
 
@@ -771,35 +761,19 @@ class WebsocketHandler {
 
     if (config.MEMPOOL.AUDIT && memPool.isInSync()) {
       let projectedBlocks;
-      let auditMempool = _memPool;
-      // template calculation functions have mempool side effects, so calculate audits using
-      // a cloned copy of the mempool if we're running a different algorithm for mempool updates
-      const separateAudit = config.MEMPOOL.ADVANCED_GBT_AUDIT !== config.MEMPOOL.ADVANCED_GBT_MEMPOOL;
-      if (separateAudit) {
-        auditMempool = deepClone(_memPool);
-        if (config.MEMPOOL.ADVANCED_GBT_AUDIT) {
-          if (config.MEMPOOL.RUST_GBT) {
-            projectedBlocks = await mempoolBlocks.$oneOffRustBlockTemplates(transactionIds, auditMempool, candidates, isAccelerated, block.extras.pool.id);
-          } else {
-            projectedBlocks = await mempoolBlocks.$makeBlockTemplates(transactionIds, auditMempool, candidates, false, isAccelerated, block.extras.pool.id);
-          }
+      const auditMempool = _memPool;
+      const isAccelerated = config.MEMPOOL_SERVICES.ACCELERATIONS && accelerationApi.isAcceleratedBlock(block, Object.values(mempool.getAccelerations()));
+
+      if ((config.MEMPOOL_SERVICES.ACCELERATIONS)) {
+        if (config.MEMPOOL.RUST_GBT) {
+          const added = memPool.limitGBT ? (candidates?.added || []) : [];
+          const removed = memPool.limitGBT ? (candidates?.removed || []) : [];
+          projectedBlocks = await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, auditMempool, added, removed, candidates, isAccelerated, block.extras.pool.id);
         } else {
-          projectedBlocks = mempoolBlocks.updateMempoolBlocks(transactionIds, auditMempool, candidates, false);
+          projectedBlocks = await mempoolBlocks.$makeBlockTemplates(transactionIds, auditMempool, candidates, false, isAccelerated, block.extras.pool.id);
         }
       } else {
-        if ((config.MEMPOOL_SERVICES.ACCELERATIONS)) {
-          if (config.MEMPOOL.RUST_GBT) {
-            console.log(`handleNewBlock: ${transactions.length} mined txs, ${candidates?.added.length} candidate added, ${candidates?.removed.length} candidate removed`);
-            console.log(`mempool size ${Object.keys(auditMempool).length}, candidates: ${transactionIds.length}`);
-            let added = memPool.limitGBT ? (candidates?.added || []) : [];
-            let removed = memPool.limitGBT ? (candidates?.removed || []) : [];
-            projectedBlocks = await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, auditMempool, added, removed, candidates, isAccelerated, block.extras.pool.id);
-          } else {
-            projectedBlocks = await mempoolBlocks.$makeBlockTemplates(transactionIds, auditMempool, candidates, false, isAccelerated, block.extras.pool.id);
-          }
-        } else {
-          projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
-        }
+        projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
       }
 
       if (Common.indexingEnabled()) {
@@ -872,18 +846,13 @@ class WebsocketHandler {
       transactionIds = Object.keys(memPool.getMempool());
     }
 
-    if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) {
-      if (config.MEMPOOL.RUST_GBT) {
-        console.log(`handleNewBlock (after): ${transactions.length} mined txs, ${candidates?.added.length} candidate added, ${candidates?.removed.length} candidate removed`);
-        console.log(`mempool size ${Object.keys(_memPool).length}, candidates: ${transactionIds.length}`);
-        let added = memPool.limitGBT ? (candidates?.added || []) : [];
-        let removed = memPool.limitGBT ? (candidates?.removed || []) : transactions;
-        await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, _memPool, added, removed, candidates, true);
-      } else {
-        await mempoolBlocks.$makeBlockTemplates(transactionIds, _memPool, candidates, true, config.MEMPOOL_SERVICES.ACCELERATIONS);
-      }
+
+    if (config.MEMPOOL.RUST_GBT) {
+      const added = memPool.limitGBT ? (candidates?.added || []) : [];
+      const removed = memPool.limitGBT ? (candidates?.removed || []) : transactions;
+      await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, _memPool, added, removed, candidates, true);
     } else {
-      mempoolBlocks.updateMempoolBlocks(transactionIds, _memPool, candidates, true);
+      await mempoolBlocks.$makeBlockTemplates(transactionIds, _memPool, candidates, true, config.MEMPOOL_SERVICES.ACCELERATIONS);
     }
     const mBlocks = mempoolBlocks.getMempoolBlocks();
     const mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();
diff --git a/backend/src/config.ts b/backend/src/config.ts
index 6a202a392..41dd0aadb 100644
--- a/backend/src/config.ts
+++ b/backend/src/config.ts
@@ -32,8 +32,6 @@ interface IConfig {
     POOLS_JSON_URL: string,
     POOLS_JSON_TREE_URL: string,
     AUDIT: boolean;
-    ADVANCED_GBT_AUDIT: boolean;
-    ADVANCED_GBT_MEMPOOL: boolean;
     RUST_GBT: boolean;
     LIMIT_GBT: boolean;
     CPFP_INDEXING: boolean;
@@ -195,8 +193,6 @@ const defaults: IConfig = {
     'POOLS_JSON_URL': 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json',
     'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
     'AUDIT': false,
-    'ADVANCED_GBT_AUDIT': false,
-    'ADVANCED_GBT_MEMPOOL': false,
     'RUST_GBT': false,
     'LIMIT_GBT': false,
     'CPFP_INDEXING': false,
diff --git a/docker/README.md b/docker/README.md
index 444324af8..c8ac91d38 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -109,8 +109,6 @@ Below we list all settings from `mempool-config.json` and the corresponding over
     "AUTOMATIC_BLOCK_REINDEXING": false,
     "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json",
     "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
-    "ADVANCED_GBT_AUDIT": false,
-    "ADVANCED_GBT_MEMPOOL": false,
     "CPFP_INDEXING": false,
     "MAX_BLOCKS_BULK_QUERY": 0,
     "DISK_CACHE_BLOCK_INTERVAL": 6,
@@ -142,8 +140,6 @@ Corresponding `docker-compose.yml` overrides:
       MEMPOOL_AUTOMATIC_BLOCK_REINDEXING: ""
       MEMPOOL_POOLS_JSON_URL: ""
       MEMPOOL_POOLS_JSON_TREE_URL: ""
-      MEMPOOL_ADVANCED_GBT_AUDIT: ""
-      MEMPOOL_ADVANCED_GBT_MEMPOOL: ""
       MEMPOOL_CPFP_INDEXING: ""
       MEMPOOL_MAX_BLOCKS_BULK_QUERY: ""
       MEMPOOL_DISK_CACHE_BLOCK_INTERVAL: ""
@@ -151,8 +147,6 @@ Corresponding `docker-compose.yml` overrides:
       ...
 ```
 
-`ADVANCED_GBT_AUDIT` AND `ADVANCED_GBT_MEMPOOL` enable a more accurate (but slower) block prediction algorithm for the block audit feature and the projected mempool-blocks respectively.
-
 `CPFP_INDEXING` enables indexing CPFP (Child Pays For Parent) information for the last `INDEXING_BLOCKS_AMOUNT` blocks.
 
 <br/>
diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json
index f8935706c..0403ba42e 100644
--- a/docker/backend/mempool-config.json
+++ b/docker/backend/mempool-config.json
@@ -26,8 +26,6 @@
     "GOGGLES_INDEXING": __MEMPOOL_GOGGLES_INDEXING__,
     "AUTOMATIC_BLOCK_REINDEXING": __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__,
     "AUDIT": __MEMPOOL_AUDIT__,
-    "ADVANCED_GBT_AUDIT": __MEMPOOL_ADVANCED_GBT_AUDIT__,
-    "ADVANCED_GBT_MEMPOOL": __MEMPOOL_ADVANCED_GBT_MEMPOOL__,
     "RUST_GBT": __MEMPOOL_RUST_GBT__,
     "CPFP_INDEXING": __MEMPOOL_CPFP_INDEXING__,
     "MAX_BLOCKS_BULK_QUERY": __MEMPOOL_MAX_BLOCKS_BULK_QUERY__,
diff --git a/docker/backend/start.sh b/docker/backend/start.sh
index 50ecc17ab..39870ab0c 100755
--- a/docker/backend/start.sh
+++ b/docker/backend/start.sh
@@ -29,8 +29,6 @@ __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__=${MEMPOOL_AUTOMATIC_BLOCK_REINDEXING:=fal
 __MEMPOOL_POOLS_JSON_URL__=${MEMPOOL_POOLS_JSON_URL:=https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json}
 __MEMPOOL_POOLS_JSON_TREE_URL__=${MEMPOOL_POOLS_JSON_TREE_URL:=https://api.github.com/repos/mempool/mining-pools/git/trees/master}
 __MEMPOOL_AUDIT__=${MEMPOOL_AUDIT:=false}
-__MEMPOOL_ADVANCED_GBT_AUDIT__=${MEMPOOL_ADVANCED_GBT_AUDIT:=false}
-__MEMPOOL_ADVANCED_GBT_MEMPOOL__=${MEMPOOL_ADVANCED_GBT_MEMPOOL:=false}
 __MEMPOOL_RUST_GBT__=${MEMPOOL_RUST_GBT:=false}
 __MEMPOOL_CPFP_INDEXING__=${MEMPOOL_CPFP_INDEXING:=false}
 __MEMPOOL_MAX_BLOCKS_BULK_QUERY__=${MEMPOOL_MAX_BLOCKS_BULK_QUERY:=0}
@@ -189,9 +187,7 @@ sed -i "s!__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__!${__MEMPOOL_AUTOMATIC_BLOCK_REI
 sed -i "s!__MEMPOOL_POOLS_JSON_URL__!${__MEMPOOL_POOLS_JSON_URL__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_POOLS_JSON_TREE_URL__!${__MEMPOOL_POOLS_JSON_TREE_URL__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_AUDIT__!${__MEMPOOL_AUDIT__}!g" mempool-config.json
-sed -i "s!__MEMPOOL_ADVANCED_GBT_MEMPOOL__!${__MEMPOOL_ADVANCED_GBT_MEMPOOL__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_RUST_GBT__!${__MEMPOOL_RUST_GBT__}!g" mempool-config.json
-sed -i "s!__MEMPOOL_ADVANCED_GBT_AUDIT__!${__MEMPOOL_ADVANCED_GBT_AUDIT__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_CPFP_INDEXING__!${__MEMPOOL_CPFP_INDEXING__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_MAX_BLOCKS_BULK_QUERY__!${__MEMPOOL_MAX_BLOCKS_BULK_QUERY__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__!${__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__}!g" mempool-config.json
diff --git a/production/mempool-config.mainnet.json b/production/mempool-config.mainnet.json
index 5038d9bfb..385f8cbdc 100644
--- a/production/mempool-config.mainnet.json
+++ b/production/mempool-config.mainnet.json
@@ -15,8 +15,6 @@
     "GOGGLES_INDEXING": true,
     "AUDIT": true,
     "CPFP_INDEXING": true,
-    "ADVANCED_GBT_AUDIT": true,
-    "ADVANCED_GBT_MEMPOOL": true,
     "RUST_GBT": true,
     "USE_SECOND_NODE_FOR_MINFEE": true,
     "DISK_CACHE_BLOCK_INTERVAL": 1,
diff --git a/production/mempool-config.signet.json b/production/mempool-config.signet.json
index 0a711d16f..6ebd9e8b3 100644
--- a/production/mempool-config.signet.json
+++ b/production/mempool-config.signet.json
@@ -9,8 +9,6 @@
     "API_URL_PREFIX": "/api/v1/",
     "INDEXING_BLOCKS_AMOUNT": -1,
     "AUDIT": true,
-    "ADVANCED_GBT_AUDIT": true,
-    "ADVANCED_GBT_MEMPOOL": true,
     "RUST_GBT": true,
     "POLL_RATE_MS": 1000,
     "DISK_CACHE_BLOCK_INTERVAL": 1,
diff --git a/production/mempool-config.testnet.json b/production/mempool-config.testnet.json
index adc93c0e9..2394ac467 100644
--- a/production/mempool-config.testnet.json
+++ b/production/mempool-config.testnet.json
@@ -9,8 +9,6 @@
     "API_URL_PREFIX": "/api/v1/",
     "INDEXING_BLOCKS_AMOUNT": -1,
     "AUDIT": true,
-    "ADVANCED_GBT_AUDIT": true,
-    "ADVANCED_GBT_MEMPOOL": true,
     "RUST_GBT": true,
     "POLL_RATE_MS": 1000,
     "DISK_CACHE_BLOCK_INTERVAL": 1,

From 0533953f54326ae3a2e353f84a9f990070d44b8e Mon Sep 17 00:00:00 2001
From: Mononaut <github@monospace.live>
Date: Sat, 6 Jan 2024 22:32:32 +0000
Subject: [PATCH 4/9] Add LIMIT_GBT config

---
 backend/mempool-config.sample.json                    | 1 +
 backend/src/__fixtures__/mempool-config.template.json | 1 +
 backend/src/__tests__/config.test.ts                  | 1 +
 docker/backend/mempool-config.json                    | 1 +
 docker/backend/start.sh                               | 2 ++
 5 files changed, 6 insertions(+)

diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json
index c98a070f1..79afbae13 100644
--- a/backend/mempool-config.sample.json
+++ b/backend/mempool-config.sample.json
@@ -29,6 +29,7 @@
     "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
     "AUDIT": false,
     "RUST_GBT": false,
+    "LIMIT_GBT": false,
     "CPFP_INDEXING": false,
     "DISK_CACHE_BLOCK_INTERVAL": 6,
     "MAX_PUSH_TX_SIZE_WEIGHT": 4000000,
diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json
index 7d571c169..41e9cce4e 100644
--- a/backend/src/__fixtures__/mempool-config.template.json
+++ b/backend/src/__fixtures__/mempool-config.template.json
@@ -29,6 +29,7 @@
     "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__",
     "AUDIT": true,
     "RUST_GBT": false,
+    "LIMIT_GBT": false,
     "CPFP_INDEXING": true,
     "MAX_BLOCKS_BULK_QUERY": 999,
     "DISK_CACHE_BLOCK_INTERVAL": 999,
diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts
index 84c2e763b..75661951e 100644
--- a/backend/src/__tests__/config.test.ts
+++ b/backend/src/__tests__/config.test.ts
@@ -42,6 +42,7 @@ describe('Mempool Backend Config', () => {
         POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json',
         AUDIT: false,
         RUST_GBT: false,
+        LIMIT_GBT: false,
         CPFP_INDEXING: false,
         MAX_BLOCKS_BULK_QUERY: 0,
         DISK_CACHE_BLOCK_INTERVAL: 6,
diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json
index 0403ba42e..0d6017788 100644
--- a/docker/backend/mempool-config.json
+++ b/docker/backend/mempool-config.json
@@ -27,6 +27,7 @@
     "AUTOMATIC_BLOCK_REINDEXING": __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__,
     "AUDIT": __MEMPOOL_AUDIT__,
     "RUST_GBT": __MEMPOOL_RUST_GBT__,
+    "LIMIT_GBT": __MEMPOOL_LIMIT_GBT__,
     "CPFP_INDEXING": __MEMPOOL_CPFP_INDEXING__,
     "MAX_BLOCKS_BULK_QUERY": __MEMPOOL_MAX_BLOCKS_BULK_QUERY__,
     "DISK_CACHE_BLOCK_INTERVAL": __MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__,
diff --git a/docker/backend/start.sh b/docker/backend/start.sh
index 39870ab0c..aecdac8f1 100755
--- a/docker/backend/start.sh
+++ b/docker/backend/start.sh
@@ -30,6 +30,7 @@ __MEMPOOL_POOLS_JSON_URL__=${MEMPOOL_POOLS_JSON_URL:=https://raw.githubuserconte
 __MEMPOOL_POOLS_JSON_TREE_URL__=${MEMPOOL_POOLS_JSON_TREE_URL:=https://api.github.com/repos/mempool/mining-pools/git/trees/master}
 __MEMPOOL_AUDIT__=${MEMPOOL_AUDIT:=false}
 __MEMPOOL_RUST_GBT__=${MEMPOOL_RUST_GBT:=false}
+__MEMPOOL_LIMIT_GBT__=${MEMPOOL_LIMIT_GBT:=false}
 __MEMPOOL_CPFP_INDEXING__=${MEMPOOL_CPFP_INDEXING:=false}
 __MEMPOOL_MAX_BLOCKS_BULK_QUERY__=${MEMPOOL_MAX_BLOCKS_BULK_QUERY:=0}
 __MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__=${MEMPOOL_DISK_CACHE_BLOCK_INTERVAL:=6}
@@ -188,6 +189,7 @@ sed -i "s!__MEMPOOL_POOLS_JSON_URL__!${__MEMPOOL_POOLS_JSON_URL__}!g" mempool-co
 sed -i "s!__MEMPOOL_POOLS_JSON_TREE_URL__!${__MEMPOOL_POOLS_JSON_TREE_URL__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_AUDIT__!${__MEMPOOL_AUDIT__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_RUST_GBT__!${__MEMPOOL_RUST_GBT__}!g" mempool-config.json
+sed -i "s!__MEMPOOL_LIMIT_GBT__!${__MEMPOOL_LIMIT_GBT__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_CPFP_INDEXING__!${__MEMPOOL_CPFP_INDEXING__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_MAX_BLOCKS_BULK_QUERY__!${__MEMPOOL_MAX_BLOCKS_BULK_QUERY__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__!${__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__}!g" mempool-config.json

From b10bd05207c82001621edb07f9d068a9f5d5c11a Mon Sep 17 00:00:00 2001
From: Mononaut <github@monospace.live>
Date: Sat, 6 Jan 2024 22:41:03 +0000
Subject: [PATCH 5/9] Move max cpfp graph size to named const

---
 backend/src/api/cpfp.ts | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/backend/src/api/cpfp.ts b/backend/src/api/cpfp.ts
index cf54771bb..aefdad5a0 100644
--- a/backend/src/api/cpfp.ts
+++ b/backend/src/api/cpfp.ts
@@ -1,6 +1,7 @@
 import { CpfpInfo, MempoolTransactionExtended } from '../mempool.interfaces';
 
 const CPFP_UPDATE_INTERVAL = 60_000; // update CPFP info at most once per 60s per transaction
+const MAX_GRAPH_SIZE = 50; // the maximum number of in-mempool relatives to consider
 
 interface GraphTx extends MempoolTransactionExtended {
   depends: string[];
@@ -92,13 +93,13 @@ function mempoolToGraphTx(tx: MempoolTransactionExtended): GraphTx {
 }
 
 /**
- * Takes a map of transaction ancestors, and expands it into a full graph of up to 50 in-mempool relatives
+ * Takes a map of transaction ancestors, and expands it into a full graph of up to MAX_GRAPH_SIZE in-mempool relatives
  */
 function expandRelativesGraph(mempool: { [txid: string]: MempoolTransactionExtended }, ancestors: Map<string, GraphTx>): Map<string, GraphTx> {
   const relatives: Map<string, GraphTx> = new Map();
   const stack: GraphTx[] = Array.from(ancestors.values());
   while (stack.length > 0) {
-    if (relatives.size > 50) {
+    if (relatives.size > MAX_GRAPH_SIZE) {
       return relatives;
     }
 
@@ -170,7 +171,7 @@ function calculateCpfpCluster(txid: string, graph: Map<string, GraphTx>): Map<st
   let sortedRelatives = Array.from(graph.values()).sort(mempoolComparator);
 
   // Iterate until we reach a cluster that includes our target tx
-  let maxIterations = 50;
+  let maxIterations = MAX_GRAPH_SIZE;
   let best = sortedRelatives.shift();
   let bestCluster = new Map<string, GraphTx>(best?.ancestorMap?.entries() || []);
   while (sortedRelatives.length && best && (best.txid !== tx.txid && !best.ancestorMap.has(tx.txid)) && maxIterations > 0) {
@@ -236,7 +237,7 @@ function calculateCpfpCluster(txid: string, graph: Map<string, GraphTx>): Map<st
    */
 function setAncestors(tx: GraphTx, all: Map<string, GraphTx>, visited: Map<string, Map<string, GraphTx>>, depth: number = 0): Map<string, GraphTx> {
   // sanity check for infinite recursion / too many ancestors (should never happen)
-  if (depth > 50) {
+  if (depth > MAX_GRAPH_SIZE) {
     return tx.ancestorMap;
   }
   

From 5411eb491f9326425aa7d6a72f5186b0d2d9691e Mon Sep 17 00:00:00 2001
From: Mononaut <github@monospace.live>
Date: Sun, 7 Jan 2024 17:57:36 +0000
Subject: [PATCH 6/9] Limit GBT: fix candidate set inconsistency

---
 backend/src/api/mempool-blocks.ts    | 27 ++++++++++++++++-----------
 backend/src/api/mempool.ts           | 14 ++++++++++----
 backend/src/api/websocket-handler.ts |  4 ++--
 3 files changed, 28 insertions(+), 17 deletions(-)

diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts
index fd9918f71..2308f6671 100644
--- a/backend/src/api/mempool-blocks.ts
+++ b/backend/src/api/mempool-blocks.ts
@@ -91,7 +91,7 @@ class MempoolBlocks {
     // set missing short ids
     for (const txid of transactions) {
       const tx = newMempool[txid];
-      this.setUid(tx, false);
+      this.setUid(tx, !saveResults);
     }
 
     const accelerations = useAccelerations ? mempool.getAccelerations() : {};
@@ -170,7 +170,7 @@ class MempoolBlocks {
     for (const tx of addedAndChanged) {
       this.setUid(tx, false);
     }
-    const removedUids = removed.map(tx => this.getUid(tx)).filter(uid => uid != null) as number[];
+    const removedTxs = removed.filter(tx => tx.uid != null) as MempoolTransactionExtended[];
 
     // prepare a stripped down version of the mempool with only the minimum necessary data
     // to reduce the overhead of passing this data to the worker thread
@@ -196,10 +196,10 @@ class MempoolBlocks {
         });
         this.txSelectionWorker?.once('error', reject);
       });
-      this.txSelectionWorker.postMessage({ type: 'update', added: addedStripped, removed: removedUids });
+      this.txSelectionWorker.postMessage({ type: 'update', added: addedStripped, removed: removedTxs.map(tx => tx.uid) as number[] });
       const { blocks, rates, clusters } = this.convertResultTxids(await workerResultPromise);
 
-      this.removeUids(removedUids);
+      this.removeUids(removedTxs);
 
       // clean up thread error listener
       this.txSelectionWorker?.removeListener('error', threadErrorListener);
@@ -227,7 +227,7 @@ class MempoolBlocks {
     const transactions = txids.map(txid => newMempool[txid]).filter(tx => tx != null);
     // set missing short ids
     for (const tx of transactions) {
-      this.setUid(tx, false);
+      this.setUid(tx, !saveResults);
     }
     // set short ids for transaction inputs
     for (const tx of transactions) {
@@ -237,6 +237,7 @@ class MempoolBlocks {
     const accelerations = useAccelerations ? mempool.getAccelerations() : {};
     const acceleratedList = accelerationPool ? Object.values(accelerations).filter(acc => newMempool[acc.txid] && acc.pools.includes(accelerationPool)) : Object.values(accelerations).filter(acc => newMempool[acc.txid]);
     const convertedAccelerations = acceleratedList.map(acc => {
+      this.setUid(newMempool[acc.txid], true);
       return {
         uid: this.getUid(newMempool[acc.txid]),
         delta: acc.feeDelta,
@@ -292,11 +293,12 @@ class MempoolBlocks {
     for (const tx of added) {
       tx.inputs = tx.vin.map(v => this.getUid(newMempool[v.txid])).filter(uid => (uid !== null && uid !== undefined)) as number[];
     }
-    const removedUids = removed.map(tx => this.getUid(tx)).filter(uid => (uid !== null && uid !== undefined)) as number[];
+    const removedTxs = removed.filter(tx => tx.uid != null) as MempoolTransactionExtended[];
 
     const accelerations = useAccelerations ? mempool.getAccelerations() : {};
     const acceleratedList = accelerationPool ? Object.values(accelerations).filter(acc => newMempool[acc.txid] && acc.pools.includes(accelerationPool)) : Object.values(accelerations).filter(acc => newMempool[acc.txid]);
     const convertedAccelerations = acceleratedList.map(acc => {
+      this.setUid(newMempool[acc.txid], true);
       return {
         uid: this.getUid(newMempool[acc.txid]),
         delta: acc.feeDelta,
@@ -308,7 +310,7 @@ class MempoolBlocks {
       const { blocks, blockWeights, rates, clusters, overflow } = this.convertNapiResultTxids(
         await this.rustGbtGenerator.update(
           added as RustThreadTransaction[],
-          removedUids,
+          removedTxs.map(tx => tx.uid) as number[],
           convertedAccelerations as RustThreadAcceleration[],
           this.nextUid,
         ),
@@ -319,7 +321,7 @@ class MempoolBlocks {
         throw new Error(`GBT returned wrong number of transactions ${transactions.length} vs ${resultMempoolSize}, cache is probably out of sync`);
       } else {
         const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, candidates, accelerations, accelerationPool, true);
-        this.removeUids(removedUids);
+        this.removeUids(removedTxs);
         logger.debug(`RUST updateBlockTemplates completed in ${(Date.now() - start)/1000} seconds`);
         return processed;
       }
@@ -522,9 +524,12 @@ class MempoolBlocks {
     }
   }
 
-  private removeUids(uids: number[]): void {
-    for (const uid of uids) {
-      this.uidMap.delete(uid);
+  private removeUids(txs: MempoolTransactionExtended[]): void {
+    for (const tx of txs) {
+      if (tx.uid != null) {
+        this.uidMap.delete(tx.uid);
+        tx.uid = undefined;
+      }
     }
   }
 
diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts
index b08848b26..68f91b5b0 100644
--- a/backend/src/api/mempool.ts
+++ b/backend/src/api/mempool.ts
@@ -320,8 +320,6 @@ class Mempool {
       }, 1000 * 60 * config.MEMPOOL.CLEAR_PROTECTION_MINUTES);
     }
 
-    const candidates = await this.getNextCandidates(minFeeMempool, minFeeTip);
-
     const deletedTransactions: MempoolTransactionExtended[] = [];
 
     if (this.mempoolProtection !== 1) {
@@ -341,6 +339,8 @@ class Mempool {
       }
     }
 
+    const candidates = await this.getNextCandidates(minFeeMempool, minFeeTip, deletedTransactions);
+
     const newMempoolSize = currentMempoolSize + newTransactions.length - deletedTransactions.length;
     const newTransactionsStripped = newTransactions.map((tx) => Common.stripTransaction(tx));
     this.latestTransactions = newTransactionsStripped.concat(this.latestTransactions).slice(0, 6);
@@ -445,8 +445,12 @@ class Mempool {
     }
   }
 
-  public async getNextCandidates(minFeeTransactions: string[], blockHeight: number): Promise<GbtCandidates | undefined> {
+  public async getNextCandidates(minFeeTransactions: string[], blockHeight: number, deletedTransactions: MempoolTransactionExtended[]): Promise<GbtCandidates | undefined> {
     if (this.limitGBT) {
+      const deletedTxsMap = {};
+      for (const tx of deletedTransactions) {
+        deletedTxsMap[tx.txid] = tx;
+      }
       const newCandidateTxMap = {};
       for (const txid of minFeeTransactions) {
         if (this.mempoolCache[txid]) {
@@ -463,14 +467,16 @@ class Mempool {
       } else {
         for (const txid of Object.keys(this.mempoolCandidates)) {
           if (!newCandidateTxMap[txid]) {
-            removed.push(this.mempoolCache[txid]);
             if (this.mempoolCache[txid]) {
+              removed.push(this.mempoolCache[txid]);
               this.mempoolCache[txid].effectiveFeePerVsize = this.mempoolCache[txid].adjustedFeePerVsize;
               this.mempoolCache[txid].ancestors = [];
               this.mempoolCache[txid].descendants = [];
               this.mempoolCache[txid].bestDescendant = null;
               this.mempoolCache[txid].cpfpChecked = false;
               this.mempoolCache[txid].cpfpUpdated = undefined;
+            } else if (deletedTxsMap[txid]) {
+              removed.push(deletedTxsMap[txid]);
             }
           }
         }
diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts
index d42fc11f0..63e7f383c 100644
--- a/backend/src/api/websocket-handler.ts
+++ b/backend/src/api/websocket-handler.ts
@@ -750,7 +750,7 @@ class WebsocketHandler {
     const _memPool = memPool.getMempool();
     const candidateTxs = await memPool.getMempoolCandidates();
     let candidates: GbtCandidates | undefined = (memPool.limitGBT && candidateTxs) ? { txs: candidateTxs, added: [], removed: [] } : undefined;
-    let transactionIds: string[] = (memPool.limitGBT && candidates) ? Object.keys(_memPool) : Object.keys(candidates?.txs || {});
+    let transactionIds: string[] = (memPool.limitGBT) ? Object.keys(candidates?.txs || {}) : Object.keys(_memPool);
 
     const accelerations = Object.values(mempool.getAccelerations());
     await accelerationRepository.$indexAccelerationsForBlock(block, accelerations, transactions);
@@ -839,7 +839,7 @@ class WebsocketHandler {
     if (memPool.limitGBT) {
       const minFeeMempool = memPool.limitGBT ? await bitcoinSecondClient.getRawMemPool() : null;
       const minFeeTip = memPool.limitGBT ? await bitcoinSecondClient.getBlockCount() : -1;
-      candidates = await memPool.getNextCandidates(minFeeMempool, minFeeTip);
+      candidates = await memPool.getNextCandidates(minFeeMempool, minFeeTip, transactions);
       transactionIds = Object.keys(candidates?.txs || {});
     } else {
       candidates = undefined;

From 07c76d084e8f1b7fa12e79488bb1ac0e120ed28d Mon Sep 17 00:00:00 2001
From: Mononaut <github@monospace.live>
Date: Mon, 8 Jan 2024 00:03:05 +0000
Subject: [PATCH 7/9] Limit GBT: handle accelerations beneath the purge rate

---
 backend/src/api/mempool-blocks.ts | 18 ++++++++++++------
 backend/src/api/mempool.ts        |  6 ++++++
 2 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts
index 2308f6671..d88328161 100644
--- a/backend/src/api/mempool-blocks.ts
+++ b/backend/src/api/mempool-blocks.ts
@@ -18,6 +18,7 @@ class MempoolBlocks {
 
   private nextUid: number = 1;
   private uidMap: Map<number, string> = new Map(); // map short numerical uids to full txids
+  private txidMap: Map<string, number> = new Map(); // map full txids back to short numerical uids
 
   public getMempoolBlocks(): MempoolBlock[] {
     return this.mempoolBlocks.map((block) => {
@@ -503,33 +504,38 @@ class MempoolBlocks {
 
   private resetUids(): void {
     this.uidMap.clear();
+    this.txidMap.clear();
     this.nextUid = 1;
   }
 
   private setUid(tx: MempoolTransactionExtended, skipSet = false): number {
-    if (tx.uid === null || tx.uid === undefined || !skipSet) {
+    if (!this.txidMap.has(tx.txid) || !skipSet) {
       const uid = this.nextUid;
       this.nextUid++;
       this.uidMap.set(uid, tx.txid);
+      this.txidMap.set(tx.txid, uid);
       tx.uid = uid;
       return uid;
     } else {
+      tx.uid = this.txidMap.get(tx.txid) as number;
       return tx.uid;
     }
   }
 
   private getUid(tx: MempoolTransactionExtended): number | void {
-    if (tx?.uid !== null && tx?.uid !== undefined && this.uidMap.has(tx.uid)) {
-      return tx.uid;
+    if (tx) {
+      return this.txidMap.get(tx.txid);
     }
   }
 
   private removeUids(txs: MempoolTransactionExtended[]): void {
     for (const tx of txs) {
-      if (tx.uid != null) {
-        this.uidMap.delete(tx.uid);
-        tx.uid = undefined;
+      const uid = this.txidMap.get(tx.txid);
+      if (uid != null) {
+        this.uidMap.delete(uid);
+        this.txidMap.delete(tx.txid);
       }
+      tx.uid = undefined;
     }
   }
 
diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts
index 68f91b5b0..9fab83080 100644
--- a/backend/src/api/mempool.ts
+++ b/backend/src/api/mempool.ts
@@ -457,6 +457,12 @@ class Mempool {
           newCandidateTxMap[txid] = true;
         }
       }
+      const accelerations = this.getAccelerations();
+      for (const txid of Object.keys(accelerations)) {
+        if (this.mempoolCache[txid]) {
+          newCandidateTxMap[txid] = true;
+        }
+      }
       const removed: MempoolTransactionExtended[] = [];
       const added: MempoolTransactionExtended[] = [];
       // don't prematurely remove txs included in a new block

From 8f2e1de578b25ec9e28bdbd0528e30a77401db52 Mon Sep 17 00:00:00 2001
From: Mononaut <github@monospace.live>
Date: Mon, 8 Jan 2024 00:56:48 +0000
Subject: [PATCH 8/9] Limit GBT: fix on-demand CPFP calculation

---
 backend/src/api/cpfp.ts           |  9 ++++++---
 backend/src/api/mempool-blocks.ts | 10 ----------
 backend/src/api/mempool.ts        |  7 +++++++
 3 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/backend/src/api/cpfp.ts b/backend/src/api/cpfp.ts
index aefdad5a0..604c1b3c9 100644
--- a/backend/src/api/cpfp.ts
+++ b/backend/src/api/cpfp.ts
@@ -1,4 +1,5 @@
 import { CpfpInfo, MempoolTransactionExtended } from '../mempool.interfaces';
+import memPool from './mempool';
 
 const CPFP_UPDATE_INTERVAL = 60_000; // update CPFP info at most once per 60s per transaction
 const MAX_GRAPH_SIZE = 50; // the maximum number of in-mempool relatives to consider
@@ -77,8 +78,8 @@ export function calculateCpfp(tx: MempoolTransactionExtended, mempool: { [txid:
 function mempoolToGraphTx(tx: MempoolTransactionExtended): GraphTx {
   return {
     ...tx,
-    depends: [],
-    spentby: [],
+    depends: tx.vin.map(v => v.txid),
+    spentby: tx.vout.map((v, i) => memPool.getFromSpendMap(tx.txid, i)).map(tx => tx?.txid).filter(txid => txid != null) as string[],
     ancestorMap: new Map(),
     fees: {
       base: tx.fee,
@@ -176,7 +177,7 @@ function calculateCpfpCluster(txid: string, graph: Map<string, GraphTx>): Map<st
   let bestCluster = new Map<string, GraphTx>(best?.ancestorMap?.entries() || []);
   while (sortedRelatives.length && best && (best.txid !== tx.txid && !best.ancestorMap.has(tx.txid)) && maxIterations > 0) {
     maxIterations--;
-    if (bestCluster && bestCluster.has(tx.txid)) {
+    if ((best && best.txid === tx.txid) || (bestCluster && bestCluster.has(tx.txid))) {
       break;
     } else {
       // Remove this cluster (it doesn't include our target tx)
@@ -195,6 +196,8 @@ function calculateCpfpCluster(txid: string, graph: Map<string, GraphTx>): Map<st
     }
   }
 
+  bestCluster.set(tx.txid, tx);
+
   return bestCluster;
 }
 
diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts
index d88328161..1132663f7 100644
--- a/backend/src/api/mempool-blocks.ts
+++ b/backend/src/api/mempool-blocks.ts
@@ -420,16 +420,6 @@ class MempoolBlocks {
             block: blockIndex,
             vsize: totalVsize + (mempoolTx.vsize / 2),
           };
-          if (!mempoolTx.cpfpChecked) {
-            if (mempoolTx.ancestors?.length) {
-              mempoolTx.ancestors = [];
-            }
-            if (mempoolTx.descendants?.length) {
-              mempoolTx.descendants = [];
-            }
-            mempoolTx.bestDescendant = null;
-            mempoolTx.cpfpChecked = true;
-          }
 
           const acceleration = accelerations[txid];
           if (isAccelerated[txid] || (acceleration && (!accelerationPool || acceleration.pools.includes(accelerationPool)))) {
diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts
index 9fab83080..176bedddb 100644
--- a/backend/src/api/mempool.ts
+++ b/backend/src/api/mempool.ts
@@ -91,6 +91,10 @@ class Mempool {
     return this.spendMap;
   }
 
+  public getFromSpendMap(txid, index): MempoolTransactionExtended | void {
+    return this.spendMap.get(`${txid}:${index}`);
+  }
+
   public async $setMempool(mempoolData: { [txId: string]: MempoolTransactionExtended }) {
     this.mempoolCache = mempoolData;
     let count = 0;
@@ -113,6 +117,9 @@ class Mempool {
         await redisCache.$addTransaction(this.mempoolCache[txid]);
       }
       this.mempoolCache[txid].flags = Common.getTransactionFlags(this.mempoolCache[txid]);
+      this.mempoolCache[txid].cpfpChecked = false;
+      this.mempoolCache[txid].cpfpDirty = true;
+      this.mempoolCache[txid].cpfpUpdated = undefined;
     }
     if (config.MEMPOOL.CACHE_ENABLED && config.REDIS.ENABLED) {
       await redisCache.$flushTransactions();

From 3732406ce8744c03b26239a898e3fcc80c91c4f7 Mon Sep 17 00:00:00 2001
From: Mononaut <github@monospace.live>
Date: Tue, 9 Jan 2024 00:27:04 +0000
Subject: [PATCH 9/9] Sanity checks for mempool visualization deltas

---
 .../block-overview-graph/block-overview-graph.component.ts    | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts b/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts
index ab6985a25..a5f6403ad 100644
--- a/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts
+++ b/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts
@@ -206,6 +206,10 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On
 
   update(add: TransactionStripped[], remove: string[], change: { txid: string, rate: number | undefined, acc: boolean | undefined }[], direction: string = 'left', resetLayout: boolean = false): void {
     if (this.scene) {
+      add = add.filter(tx => !this.scene.txs[tx.txid]);
+      remove = remove.filter(txid => this.scene.txs[txid]);
+      change = change.filter(tx => this.scene.txs[tx.txid]);
+
       this.scene.update(add, remove, change, direction, resetLayout);
       this.start();
       this.updateSearchHighlight();