Limit GBT: fix candidate set inconsistency

backend/src/api/mempool-blocks.ts

@@ -91,7 +91,7 @@ class MempoolBlocks {
     // set missing short ids
     for (const txid of transactions) {
       const tx = newMempool[txid];
-      this.setUid(tx, false);
+      this.setUid(tx, !saveResults);
     }
 
     const accelerations = useAccelerations ? mempool.getAccelerations() : {};
@@ -170,7 +170,7 @@ class MempoolBlocks {
     for (const tx of addedAndChanged) {
       this.setUid(tx, false);
     }
-    const removedUids = removed.map(tx => this.getUid(tx)).filter(uid => uid != null) as number[];
+    const removedTxs = removed.filter(tx => tx.uid != null) as MempoolTransactionExtended[];
 
     // prepare a stripped down version of the mempool with only the minimum necessary data
     // to reduce the overhead of passing this data to the worker thread
@@ -196,10 +196,10 @@ class MempoolBlocks {
         });
         this.txSelectionWorker?.once('error', reject);
       });
-      this.txSelectionWorker.postMessage({ type: 'update', added: addedStripped, removed: removedUids });
+      this.txSelectionWorker.postMessage({ type: 'update', added: addedStripped, removed: removedTxs.map(tx => tx.uid) as number[] });
       const { blocks, rates, clusters } = this.convertResultTxids(await workerResultPromise);
 
-      this.removeUids(removedUids);
+      this.removeUids(removedTxs);
 
       // clean up thread error listener
       this.txSelectionWorker?.removeListener('error', threadErrorListener);
@@ -227,7 +227,7 @@ class MempoolBlocks {
     const transactions = txids.map(txid => newMempool[txid]).filter(tx => tx != null);
     // set missing short ids
     for (const tx of transactions) {
-      this.setUid(tx, false);
+      this.setUid(tx, !saveResults);
     }
     // set short ids for transaction inputs
     for (const tx of transactions) {
@@ -237,6 +237,7 @@ class MempoolBlocks {
     const accelerations = useAccelerations ? mempool.getAccelerations() : {};
     const acceleratedList = accelerationPool ? Object.values(accelerations).filter(acc => newMempool[acc.txid] && acc.pools.includes(accelerationPool)) : Object.values(accelerations).filter(acc => newMempool[acc.txid]);
     const convertedAccelerations = acceleratedList.map(acc => {
+      this.setUid(newMempool[acc.txid], true);
       return {
         uid: this.getUid(newMempool[acc.txid]),
         delta: acc.feeDelta,
@@ -292,11 +293,12 @@ class MempoolBlocks {
     for (const tx of added) {
       tx.inputs = tx.vin.map(v => this.getUid(newMempool[v.txid])).filter(uid => (uid !== null && uid !== undefined)) as number[];
     }
-    const removedUids = removed.map(tx => this.getUid(tx)).filter(uid => (uid !== null && uid !== undefined)) as number[];
+    const removedTxs = removed.filter(tx => tx.uid != null) as MempoolTransactionExtended[];
 
     const accelerations = useAccelerations ? mempool.getAccelerations() : {};
     const acceleratedList = accelerationPool ? Object.values(accelerations).filter(acc => newMempool[acc.txid] && acc.pools.includes(accelerationPool)) : Object.values(accelerations).filter(acc => newMempool[acc.txid]);
     const convertedAccelerations = acceleratedList.map(acc => {
+      this.setUid(newMempool[acc.txid], true);
       return {
         uid: this.getUid(newMempool[acc.txid]),
         delta: acc.feeDelta,
@@ -308,7 +310,7 @@ class MempoolBlocks {
       const { blocks, blockWeights, rates, clusters, overflow } = this.convertNapiResultTxids(
         await this.rustGbtGenerator.update(
           added as RustThreadTransaction[],
-          removedUids,
+          removedTxs.map(tx => tx.uid) as number[],
           convertedAccelerations as RustThreadAcceleration[],
           this.nextUid,
         ),
@@ -319,7 +321,7 @@ class MempoolBlocks {
         throw new Error(`GBT returned wrong number of transactions ${transactions.length} vs ${resultMempoolSize}, cache is probably out of sync`);
       } else {
         const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, candidates, accelerations, accelerationPool, true);
-        this.removeUids(removedUids);
+        this.removeUids(removedTxs);
         logger.debug(`RUST updateBlockTemplates completed in ${(Date.now() - start)/1000} seconds`);
         return processed;
       }
@@ -522,9 +524,12 @@ class MempoolBlocks {
     }
   }
 
-  private removeUids(uids: number[]): void {
+  private removeUids(txs: MempoolTransactionExtended[]): void {
-    for (const uid of uids) {
+    for (const tx of txs) {
-      this.uidMap.delete(uid);
+      if (tx.uid != null) {
+        this.uidMap.delete(tx.uid);
+        tx.uid = undefined;
+      }
     }
   }
 

backend/src/api/mempool.ts

@@ -320,8 +320,6 @@ class Mempool {
       }, 1000 * 60 * config.MEMPOOL.CLEAR_PROTECTION_MINUTES);
     }
 
-    const candidates = await this.getNextCandidates(minFeeMempool, minFeeTip);
-
     const deletedTransactions: MempoolTransactionExtended[] = [];
 
     if (this.mempoolProtection !== 1) {
@@ -341,6 +339,8 @@ class Mempool {
       }
     }
 
+    const candidates = await this.getNextCandidates(minFeeMempool, minFeeTip, deletedTransactions);
+
     const newMempoolSize = currentMempoolSize + newTransactions.length - deletedTransactions.length;
     const newTransactionsStripped = newTransactions.map((tx) => Common.stripTransaction(tx));
     this.latestTransactions = newTransactionsStripped.concat(this.latestTransactions).slice(0, 6);
@@ -445,8 +445,12 @@ class Mempool {
     }
   }
 
-  public async getNextCandidates(minFeeTransactions: string[], blockHeight: number): Promise<GbtCandidates | undefined> {
+  public async getNextCandidates(minFeeTransactions: string[], blockHeight: number, deletedTransactions: MempoolTransactionExtended[]): Promise<GbtCandidates | undefined> {
     if (this.limitGBT) {
+      const deletedTxsMap = {};
+      for (const tx of deletedTransactions) {
+        deletedTxsMap[tx.txid] = tx;
+      }
       const newCandidateTxMap = {};
       for (const txid of minFeeTransactions) {
         if (this.mempoolCache[txid]) {
@@ -463,14 +467,16 @@ class Mempool {
       } else {
         for (const txid of Object.keys(this.mempoolCandidates)) {
           if (!newCandidateTxMap[txid]) {
-            removed.push(this.mempoolCache[txid]);
             if (this.mempoolCache[txid]) {
+              removed.push(this.mempoolCache[txid]);
               this.mempoolCache[txid].effectiveFeePerVsize = this.mempoolCache[txid].adjustedFeePerVsize;
               this.mempoolCache[txid].ancestors = [];
               this.mempoolCache[txid].descendants = [];
               this.mempoolCache[txid].bestDescendant = null;
               this.mempoolCache[txid].cpfpChecked = false;
               this.mempoolCache[txid].cpfpUpdated = undefined;
+            } else if (deletedTxsMap[txid]) {
+              removed.push(deletedTxsMap[txid]);
             }
           }
         }

backend/src/api/websocket-handler.ts

@@ -750,7 +750,7 @@ class WebsocketHandler {
     const _memPool = memPool.getMempool();
     const candidateTxs = await memPool.getMempoolCandidates();
     let candidates: GbtCandidates | undefined = (memPool.limitGBT && candidateTxs) ? { txs: candidateTxs, added: [], removed: [] } : undefined;
-    let transactionIds: string[] = (memPool.limitGBT && candidates) ? Object.keys(_memPool) : Object.keys(candidates?.txs || {});
+    let transactionIds: string[] = (memPool.limitGBT) ? Object.keys(candidates?.txs || {}) : Object.keys(_memPool);
 
     const accelerations = Object.values(mempool.getAccelerations());
     await accelerationRepository.$indexAccelerationsForBlock(block, accelerations, transactions);
@@ -839,7 +839,7 @@ class WebsocketHandler {
     if (memPool.limitGBT) {
       const minFeeMempool = memPool.limitGBT ? await bitcoinSecondClient.getRawMemPool() : null;
       const minFeeTip = memPool.limitGBT ? await bitcoinSecondClient.getBlockCount() : -1;
-      candidates = await memPool.getNextCandidates(minFeeMempool, minFeeTip);
+      candidates = await memPool.getNextCandidates(minFeeMempool, minFeeTip, transactions);
       transactionIds = Object.keys(candidates?.txs || {});
     } else {
       candidates = undefined;