strip non-essential data from redis cache txs

Faster blocks

backend/package-lock.json

@@ -19,6 +19,7 @@
         "maxmind": "~4.3.11",
         "mysql2": "~3.5.2",
         "rust-gbt": "file:./rust-gbt",
+        "redis": "^4.6.6",
         "socks-proxy-agent": "~7.0.0",
         "typescript": "~4.9.3",
         "ws": "~8.13.0"
@@ -1555,6 +1556,64 @@
         "node": ">= 8"
       }
     },
+    "node_modules/@redis/bloom": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz",
+      "integrity": "sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==",
+      "peerDependencies": {
+        "@redis/client": "^1.0.0"
+      }
+    },
+    "node_modules/@redis/client": {
+      "version": "1.5.7",
+      "resolved": "https://registry.npmjs.org/@redis/client/-/client-1.5.7.tgz",
+      "integrity": "sha512-gaOBOuJPjK5fGtxSseaKgSvjiZXQCdLlGg9WYQst+/GRUjmXaiB5kVkeQMRtPc7Q2t93XZcJfBMSwzs/XS9UZw==",
+      "dependencies": {
+        "cluster-key-slot": "1.1.2",
+        "generic-pool": "3.9.0",
+        "yallist": "4.0.0"
+      },
+      "engines": {
+        "node": ">=14"
+      }
+    },
+    "node_modules/@redis/client/node_modules/yallist": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
+    },
+    "node_modules/@redis/graph": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.0.tgz",
+      "integrity": "sha512-16yZWngxyXPd+MJxeSr0dqh2AIOi8j9yXKcKCwVaKDbH3HTuETpDVPcLujhFYVPtYrngSco31BUcSa9TH31Gqg==",
+      "peerDependencies": {
+        "@redis/client": "^1.0.0"
+      }
+    },
+    "node_modules/@redis/json": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.4.tgz",
+      "integrity": "sha512-LUZE2Gdrhg0Rx7AN+cZkb1e6HjoSKaeeW8rYnt89Tly13GBI5eP4CwDVr+MY8BAYfCg4/N15OUrtLoona9uSgw==",
+      "peerDependencies": {
+        "@redis/client": "^1.0.0"
+      }
+    },
+    "node_modules/@redis/search": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@redis/search/-/search-1.1.2.tgz",
+      "integrity": "sha512-/cMfstG/fOh/SsE+4/BQGeuH/JJloeWuH+qJzM8dbxuWvdWibWAOAHHCZTMPhV3xIlH4/cUEIA8OV5QnYpaVoA==",
+      "peerDependencies": {
+        "@redis/client": "^1.0.0"
+      }
+    },
+    "node_modules/@redis/time-series": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.0.4.tgz",
+      "integrity": "sha512-ThUIgo2U/g7cCuZavucQTQzA9g9JbDDY2f64u3AbAoz/8vE2lt2U37LamDUVChhaDA3IRT9R6VvJwqnUfTJzng==",
+      "peerDependencies": {
+        "@redis/client": "^1.0.0"
+      }
+    },
     "node_modules/@sinclair/typebox": {
       "version": "0.25.24",
       "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.25.24.tgz",
@@ -2718,6 +2777,14 @@
         "node": ">=12"
       }
     },
+    "node_modules/cluster-key-slot": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz",
+      "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/co": {
       "version": "4.6.0",
       "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz",
@@ -3678,6 +3745,14 @@
         "is-property": "^1.0.2"
       }
     },
+    "node_modules/generic-pool": {
+      "version": "3.9.0",
+      "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz",
+      "integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==",
+      "engines": {
+        "node": ">= 4"
+      }
+    },
     "node_modules/gensync": {
       "version": "1.0.0-beta.2",
       "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz",
@@ -6577,6 +6652,19 @@
       "integrity": "sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==",
       "dev": true
     },
+    "node_modules/redis": {
+      "version": "4.6.6",
+      "resolved": "https://registry.npmjs.org/redis/-/redis-4.6.6.tgz",
+      "integrity": "sha512-aLs2fuBFV/VJ28oLBqYykfnhGGkFxvx0HdCEBYdJ99FFbSEMZ7c1nVKwR6ZRv+7bb7JnC0mmCzaqu8frgOYhpA==",
+      "dependencies": {
+        "@redis/bloom": "1.2.0",
+        "@redis/client": "1.5.7",
+        "@redis/graph": "1.1.0",
+        "@redis/json": "1.0.4",
+        "@redis/search": "1.1.2",
+        "@redis/time-series": "1.0.4"
+      }
+    },
     "node_modules/require-directory": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
@@ -8704,6 +8792,53 @@
         "fastq": "^1.6.0"
       }
     },
+    "@redis/bloom": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz",
+      "integrity": "sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==",
+      "requires": {}
+    },
+    "@redis/client": {
+      "version": "1.5.7",
+      "resolved": "https://registry.npmjs.org/@redis/client/-/client-1.5.7.tgz",
+      "integrity": "sha512-gaOBOuJPjK5fGtxSseaKgSvjiZXQCdLlGg9WYQst+/GRUjmXaiB5kVkeQMRtPc7Q2t93XZcJfBMSwzs/XS9UZw==",
+      "requires": {
+        "cluster-key-slot": "1.1.2",
+        "generic-pool": "3.9.0",
+        "yallist": "4.0.0"
+      },
+      "dependencies": {
+        "yallist": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+          "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
+        }
+      }
+    },
+    "@redis/graph": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.0.tgz",
+      "integrity": "sha512-16yZWngxyXPd+MJxeSr0dqh2AIOi8j9yXKcKCwVaKDbH3HTuETpDVPcLujhFYVPtYrngSco31BUcSa9TH31Gqg==",
+      "requires": {}
+    },
+    "@redis/json": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.4.tgz",
+      "integrity": "sha512-LUZE2Gdrhg0Rx7AN+cZkb1e6HjoSKaeeW8rYnt89Tly13GBI5eP4CwDVr+MY8BAYfCg4/N15OUrtLoona9uSgw==",
+      "requires": {}
+    },
+    "@redis/search": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@redis/search/-/search-1.1.2.tgz",
+      "integrity": "sha512-/cMfstG/fOh/SsE+4/BQGeuH/JJloeWuH+qJzM8dbxuWvdWibWAOAHHCZTMPhV3xIlH4/cUEIA8OV5QnYpaVoA==",
+      "requires": {}
+    },
+    "@redis/time-series": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.0.4.tgz",
+      "integrity": "sha512-ThUIgo2U/g7cCuZavucQTQzA9g9JbDDY2f64u3AbAoz/8vE2lt2U37LamDUVChhaDA3IRT9R6VvJwqnUfTJzng==",
+      "requires": {}
+    },
     "@sinclair/typebox": {
       "version": "0.25.24",
       "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.25.24.tgz",
@@ -9604,6 +9739,11 @@
         "wrap-ansi": "^7.0.0"
       }
     },
+    "cluster-key-slot": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz",
+      "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA=="
+    },
     "co": {
       "version": "4.6.0",
       "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz",
@@ -10332,6 +10472,11 @@
         "is-property": "^1.0.2"
       }
     },
+    "generic-pool": {
+      "version": "3.9.0",
+      "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz",
+      "integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g=="
+    },
     "gensync": {
       "version": "1.0.0-beta.2",
       "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz",
@@ -12454,6 +12599,19 @@
       "integrity": "sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==",
       "dev": true
     },
+    "redis": {
+      "version": "4.6.6",
+      "resolved": "https://registry.npmjs.org/redis/-/redis-4.6.6.tgz",
+      "integrity": "sha512-aLs2fuBFV/VJ28oLBqYykfnhGGkFxvx0HdCEBYdJ99FFbSEMZ7c1nVKwR6ZRv+7bb7JnC0mmCzaqu8frgOYhpA==",
+      "requires": {
+        "@redis/bloom": "1.2.0",
+        "@redis/client": "1.5.7",
+        "@redis/graph": "1.1.0",
+        "@redis/json": "1.0.4",
+        "@redis/search": "1.1.2",
+        "@redis/time-series": "1.0.4"
+      }
+    },
     "require-directory": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",

backend/package.json

@@ -47,13 +47,14 @@
     "maxmind": "~4.3.11",
     "mysql2": "~3.5.2",
     "rust-gbt": "file:./rust-gbt",
+    "redis": "^4.6.6",
     "socks-proxy-agent": "~7.0.0",
     "typescript": "~4.9.3",
     "ws": "~8.13.0"
   },
   "devDependencies": {
-    "@babel/core": "^7.21.3",
     "@babel/code-frame": "^7.18.6",
+    "@babel/core": "^7.21.3",
     "@types/compression": "^1.7.2",
     "@types/crypto-js": "^4.1.1",
     "@types/express": "^4.17.17",

backend/src/__fixtures__/mempool-config.template.json

@@ -10,6 +10,7 @@
     "AUTOMATIC_BLOCK_REINDEXING": false,
     "POLL_RATE_MS": 3,
     "CACHE_DIR": "__MEMPOOL_CACHE_DIR__",
+    "CACHE_ENABLED": true,
     "CLEAR_PROTECTION_MINUTES": 4,
     "RECOMMENDED_FEE_PERCENTILE": 5,
     "BLOCK_WEIGHT_UNITS": 6,
@@ -127,5 +128,9 @@
     "AUDIT": false,
     "AUDIT_START_HEIGHT": 774000,
     "SERVERS": []
+  },
+  "REDIS": {
+    "ENABLED": false,
+    "UNIX_SOCKET_PATH": "/tmp/redis.sock"
   }
 }

backend/src/__tests__/config.test.ts

@@ -23,6 +23,7 @@ describe('Mempool Backend Config', () => {
         AUTOMATIC_BLOCK_REINDEXING: false,
         POLL_RATE_MS: 2000,
         CACHE_DIR: './cache',
+        CACHE_ENABLED: true,
         CLEAR_PROTECTION_MINUTES: 20,
         RECOMMENDED_FEE_PERCENTILE: 50,
         BLOCK_WEIGHT_UNITS: 4000000,
@@ -127,6 +128,11 @@ describe('Mempool Backend Config', () => {
         AUDIT_START_HEIGHT: 774000,
         SERVERS: []
       });
+
+      expect(config.REDIS).toStrictEqual({
+        ENABLED: false,
+        UNIX_SOCKET_PATH: ''
+      });
     });
   });
 
@@ -160,6 +166,8 @@ describe('Mempool Backend Config', () => {
       expect(config.PRICE_DATA_SERVER).toStrictEqual(fixture.PRICE_DATA_SERVER);
 
       expect(config.EXTERNAL_DATA_SERVER).toStrictEqual(fixture.EXTERNAL_DATA_SERVER);
+
+      expect(config.REDIS).toStrictEqual(fixture.REDIS);
     });
   });
 

backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts

@@ -3,10 +3,12 @@ import { IEsploraApi } from './esplora-api.interface';
 export interface AbstractBitcoinApi {
   $getRawMempool(): Promise<IEsploraApi.Transaction['txid'][]>;
   $getRawTransaction(txId: string, skipConversion?: boolean, addPrevout?: boolean, lazyPrevouts?: boolean): Promise<IEsploraApi.Transaction>;
+  $getMempoolTransactions(lastTxid: string);
   $getTransactionHex(txId: string): Promise<string>;
   $getBlockHeightTip(): Promise<number>;
   $getBlockHashTip(): Promise<string>;
   $getTxIdsForBlock(hash: string): Promise<string[]>;
+  $getTxsForBlock(hash: string): Promise<IEsploraApi.Transaction[]>;
   $getBlockHash(height: number): Promise<string>;
   $getBlockHeader(hash: string): Promise<string>;
   $getBlock(hash: string): Promise<IEsploraApi.Block>;

backend/src/api/bitcoin/bitcoin-api.ts

@@ -5,6 +5,7 @@ import { IEsploraApi } from './esplora-api.interface';
 import blocks from '../blocks';
 import mempool from '../mempool';
 import { TransactionExtended } from '../../mempool.interfaces';
+import transactionUtils from '../transaction-utils';
 
 class BitcoinApi implements AbstractBitcoinApi {
   private rawMempoolCache: IBitcoinApi.RawMempool | null = null;
@@ -59,9 +60,20 @@ class BitcoinApi implements AbstractBitcoinApi {
       });
   }
 
-  $getTransactionHex(txId: string): Promise<string> {
+  $getMempoolTransactions(lastTxid: string): Promise<IEsploraApi.Transaction[]> {
-    return this.$getRawTransaction(txId, true)
+    return Promise.resolve([]);
-      .then((tx) => tx.hex || '');
+  }
+
+  async $getTransactionHex(txId: string): Promise<string> {
+    const txInMempool = mempool.getMempool()[txId];
+    if (txInMempool && txInMempool.hex) {
+      return txInMempool.hex;
+    }
+
+    return this.bitcoindClient.getRawTransaction(txId, true)
+      .then((transaction: IBitcoinApi.Transaction) => {
+        return transaction.hex;
+      });
   }
 
   $getBlockHeightTip(): Promise<number> {
@@ -77,6 +89,10 @@ class BitcoinApi implements AbstractBitcoinApi {
       .then((rpcBlock: IBitcoinApi.Block) => rpcBlock.tx);
   }
 
+  $getTxsForBlock(hash: string): Promise<IEsploraApi.Transaction[]> {
+    throw new Error('Method getTxsForBlock not supported by the Bitcoin RPC API.');
+  }
+
   $getRawBlock(hash: string): Promise<Buffer> {
     return this.bitcoindClient.getBlock(hash, 0)
       .then((raw: string) => Buffer.from(raw, "hex"));
@@ -201,7 +217,7 @@ class BitcoinApi implements AbstractBitcoinApi {
         scriptpubkey: vout.scriptPubKey.hex,
         scriptpubkey_address: vout.scriptPubKey && vout.scriptPubKey.address ? vout.scriptPubKey.address
           : vout.scriptPubKey.addresses ? vout.scriptPubKey.addresses[0] : '',
-        scriptpubkey_asm: vout.scriptPubKey.asm ? this.convertScriptSigAsm(vout.scriptPubKey.hex) : '',
+        scriptpubkey_asm: vout.scriptPubKey.asm ? transactionUtils.convertScriptSigAsm(vout.scriptPubKey.hex) : '',
         scriptpubkey_type: this.translateScriptPubKeyType(vout.scriptPubKey.type),
       };
     });
@@ -211,7 +227,7 @@ class BitcoinApi implements AbstractBitcoinApi {
         is_coinbase: !!vin.coinbase,
         prevout: null,
         scriptsig: vin.scriptSig && vin.scriptSig.hex || vin.coinbase || '',
-        scriptsig_asm: vin.scriptSig && this.convertScriptSigAsm(vin.scriptSig.hex) || '',
+        scriptsig_asm: vin.scriptSig && transactionUtils.convertScriptSigAsm(vin.scriptSig.hex) || '',
         sequence: vin.sequence,
         txid: vin.txid || '',
         vout: vin.vout || 0,
@@ -283,7 +299,7 @@ class BitcoinApi implements AbstractBitcoinApi {
       }
       const innerTx = await this.$getRawTransaction(vin.txid, false, false);
       vin.prevout = innerTx.vout[vin.vout];
-      this.addInnerScriptsToVin(vin);
+      transactionUtils.addInnerScriptsToVin(vin);
     }
     return transaction;
   }
@@ -322,7 +338,7 @@ class BitcoinApi implements AbstractBitcoinApi {
       }
       const innerTx = await this.$getRawTransaction(transaction.vin[i].txid, false, false);
       transaction.vin[i].prevout = innerTx.vout[transaction.vin[i].vout];
-      this.addInnerScriptsToVin(transaction.vin[i]);
+      transactionUtils.addInnerScriptsToVin(transaction.vin[i]);
       totalIn += innerTx.vout[transaction.vin[i].vout].value;
     }
     if (lazyPrevouts && transaction.vin.length > 12) {
@@ -334,122 +350,6 @@ class BitcoinApi implements AbstractBitcoinApi {
     return transaction;
   }
 
-  private convertScriptSigAsm(hex: string): string {
-    const buf = Buffer.from(hex, 'hex');
-
-    const b: string[] = [];
-
-    let i = 0;
-    while (i < buf.length) {
-      const op = buf[i];
-      if (op >= 0x01 && op <= 0x4e) {
-        i++;
-        let push: number;
-        if (op === 0x4c) {
-          push = buf.readUInt8(i);
-          b.push('OP_PUSHDATA1');
-          i += 1;
-        } else if (op === 0x4d) {
-          push = buf.readUInt16LE(i);
-          b.push('OP_PUSHDATA2');
-          i += 2;
-        } else if (op === 0x4e) {
-          push = buf.readUInt32LE(i);
-          b.push('OP_PUSHDATA4');
-          i += 4;
-        } else {
-          push = op;
-          b.push('OP_PUSHBYTES_' + push);
-        }
-
-        const data = buf.slice(i, i + push);
-        if (data.length !== push) {
-          break;
-        }
-
-        b.push(data.toString('hex'));
-        i += data.length;
-      } else {
-        if (op === 0x00) {
-          b.push('OP_0');
-        } else if (op === 0x4f) {
-          b.push('OP_PUSHNUM_NEG1');
-        } else if (op === 0xb1) {
-          b.push('OP_CLTV');
-        } else if (op === 0xb2) {
-          b.push('OP_CSV');
-        } else if (op === 0xba) {
-          b.push('OP_CHECKSIGADD');
-        } else {
-          const opcode = bitcoinjs.script.toASM([ op ]);
-          if (opcode && op < 0xfd) {
-            if (/^OP_(\d+)$/.test(opcode)) {
-              b.push(opcode.replace(/^OP_(\d+)$/, 'OP_PUSHNUM_$1'));
-            } else {
-              b.push(opcode);
-            }
-          } else {
-            b.push('OP_RETURN_' + op);
-          }
-        }
-        i += 1;
-      }
-    }
-
-    return b.join(' ');
-  }
-
-  private addInnerScriptsToVin(vin: IEsploraApi.Vin): void {
-    if (!vin.prevout) {
-      return;
-    }
-
-    if (vin.prevout.scriptpubkey_type === 'p2sh') {
-      const redeemScript = vin.scriptsig_asm.split(' ').reverse()[0];
-      vin.inner_redeemscript_asm = this.convertScriptSigAsm(redeemScript);
-      if (vin.witness && vin.witness.length > 2) {
-        const witnessScript = vin.witness[vin.witness.length - 1];
-        vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript);
-      }
-    }
-
-    if (vin.prevout.scriptpubkey_type === 'v0_p2wsh' && vin.witness) {
-      const witnessScript = vin.witness[vin.witness.length - 1];
-      vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript);
-    }
-
-    if (vin.prevout.scriptpubkey_type === 'v1_p2tr' && vin.witness) {
-      const witnessScript = this.witnessToP2TRScript(vin.witness);
-      if (witnessScript !== null) {
-        vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript);
-      }
-    }
-  }
-
-  /**
-   * This function must only be called when we know the witness we are parsing
-   * is a taproot witness.
-   * @param witness An array of hex strings that represents the witness stack of
-   *                the input.
-   * @returns null if the witness is not a script spend, and the hex string of
-   *          the script item if it is a script spend.
-   */
-  private witnessToP2TRScript(witness: string[]): string | null {
-    if (witness.length < 2) return null;
-    // Note: see BIP341 for parsing details of witness stack
-
-    // If there are at least two witness elements, and the first byte of the
-    // last element is 0x50, this last element is called annex a and
-    // is removed from the witness stack.
-    const hasAnnex = witness[witness.length - 1].substring(0, 2) === '50';
-    // If there are at least two witness elements left, script path spending is used.
-    // Call the second-to-last stack element s, the script.
-    // (Note: this phrasing from BIP341 assumes we've *removed* the annex from the stack)
-    if (hasAnnex && witness.length < 3) return null;
-    const positionOfScript = hasAnnex ? witness.length - 3 : witness.length - 2;
-    return witness[positionOfScript];
-  }
-
 }
 
 export default BitcoinApi;

backend/src/api/bitcoin/esplora-api.ts

@@ -69,6 +69,10 @@ class ElectrsApi implements AbstractBitcoinApi {
     return this.$queryWrapper<IEsploraApi.Transaction>(config.ESPLORA.REST_API_URL + '/tx/' + txId);
   }
 
+  async $getMempoolTransactions(lastSeenTxid?: string): Promise<IEsploraApi.Transaction[]> {
+    return this.$queryWrapper<IEsploraApi.Transaction[]>(config.ESPLORA.REST_API_URL + '/mempool/txs' + (lastSeenTxid ? '/' + lastSeenTxid : ''));
+  }
+
   $getTransactionHex(txId: string): Promise<string> {
     return this.$queryWrapper<string>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/hex');
   }
@@ -85,6 +89,10 @@ class ElectrsApi implements AbstractBitcoinApi {
     return this.$queryWrapper<string[]>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/txids');
   }
 
+  $getTxsForBlock(hash: string): Promise<IEsploraApi.Transaction[]> {
+    return this.$queryWrapper<IEsploraApi.Transaction[]>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/txs');
+  }
+
   $getBlockHash(height: number): Promise<string> {
     return this.$queryWrapper<string>(config.ESPLORA.REST_API_URL + '/block-height/' + height);
   }

backend/src/api/blocks.ts

@@ -26,6 +26,8 @@ import PricesRepository from '../repositories/PricesRepository';
 import priceUpdater from '../tasks/price-updater';
 import chainTips from './chain-tips';
 import websocketHandler from './websocket-handler';
+import redisCache from './redis-cache';
+import rbfCache from './rbf-cache';
 
 class Blocks {
   private blocks: BlockExtended[] = [];
@@ -70,6 +72,9 @@ class Blocks {
    * @param blockHash
    * @param blockHeight
    * @param onlyCoinbase - Set to true if you only need the coinbase transaction
+   * @param txIds - optional ordered list of transaction ids if already known
+   * @param quiet - don't print non-essential logs
+   * @param addMempoolData - calculate sigops etc
    * @returns Promise<TransactionExtended[]>
    */
   private async $getTransactionsExtended(
@@ -80,62 +85,77 @@ class Blocks {
     quiet: boolean = false,
     addMempoolData: boolean = false,
   ): Promise<TransactionExtended[]> {
-    const transactions: TransactionExtended[] = [];
+    const isEsplora = config.MEMPOOL.BACKEND === 'esplora';
+    const transactionMap: { [txid: string]: TransactionExtended } = {};
+
     if (!txIds) {
       txIds = await bitcoinApi.$getTxIdsForBlock(blockHash);
     }
 
     const mempool = memPool.getMempool();
-    let transactionsFound = 0;
+    let foundInMempool = 0;
-    let transactionsFetched = 0;
+    let totalFound = 0;
 
-    for (let i = 0; i < txIds.length; i++) {
+    // Copy existing transactions from the mempool
-      if (mempool[txIds[i]]) {
+    if (!onlyCoinbase) {
-        // We update blocks before the mempool (index.ts), therefore we can
+      for (const txid of txIds) {
-        // optimize here by directly fetching txs in the "outdated" mempool
+        if (mempool[txid]) {
-        transactions.push(mempool[txIds[i]]);
+          transactionMap[txid] = mempool[txid];
-        transactionsFound++;
+          foundInMempool++;
-      } else if (config.MEMPOOL.BACKEND === 'esplora' || !memPool.hasPriority() || i === 0) {
+          totalFound++;
-        // Otherwise we fetch the tx data through backend services (esplora, electrum, core rpc...)
-        if (!quiet && (i % (Math.round((txIds.length) / 10)) === 0 || i + 1 === txIds.length)) { // Avoid log spam
-          logger.debug(`Indexing tx ${i + 1} of ${txIds.length} in block #${blockHeight}`);
-        }
-        try {
-          const tx = await transactionUtils.$getTransactionExtended(txIds[i], false, false, false, addMempoolData);
-          transactions.push(tx);
-          transactionsFetched++;
-        } catch (e) {
-          try {
-            if (config.MEMPOOL.BACKEND === 'esplora') {
-              // Try again with core
-              const tx = await transactionUtils.$getTransactionExtended(txIds[i], false, false, true, addMempoolData);
-              transactions.push(tx);
-              transactionsFetched++;
-            } else {
-              throw e;
-            }
-          } catch (e) {
-            if (i === 0) {
-              const msg = `Cannot fetch coinbase tx ${txIds[i]}. Reason: ` + (e instanceof Error ? e.message : e); 
-              logger.err(msg);
-              throw new Error(msg);
-            } else {
-              logger.err(`Cannot fetch tx ${txIds[i]}. Reason: ` + (e instanceof Error ? e.message : e));
-            }
-          }
         }
       }
+    }
 
-      if (onlyCoinbase === true) {
+    // Skip expensive lookups while mempool has priority
-        break; // Fetch the first transaction and exit
+    if (onlyCoinbase) {
+      try {
+        const coinbase = await transactionUtils.$getTransactionExtended(txIds[0], false, false, false, addMempoolData);
+        return [coinbase];
+      } catch (e) {
+        const msg = `Cannot fetch coinbase tx ${txIds[0]}. Reason: ` + (e instanceof Error ? e.message : e);
+        logger.err(msg);
+        throw new Error(msg);
+      }
+    }
+
+    // Fetch remaining txs in bulk
+    if (isEsplora && (txIds.length - totalFound > 500)) {
+      try {
+        const rawTransactions = await bitcoinApi.$getTxsForBlock(blockHash);
+        for (const tx of rawTransactions) {
+          if (!transactionMap[tx.txid]) {
+            transactionMap[tx.txid] = addMempoolData ? transactionUtils.extendMempoolTransaction(tx) : transactionUtils.extendTransaction(tx);
+            totalFound++;
+          }
+        }
+      } catch (e) {
+        logger.err(`Cannot fetch bulk txs for block ${blockHash}. Reason: ` + (e instanceof Error ? e.message : e));
+      }
+    }
+
+    // Fetch remaining txs individually
+    for (const txid of txIds.filter(txid => !transactionMap[txid])) {
+      if (!transactionMap[txid]) {
+        if (!quiet && (totalFound % (Math.round((txIds.length) / 10)) === 0 || totalFound + 1 === txIds.length)) { // Avoid log spam
+          logger.debug(`Indexing tx ${totalFound + 1} of ${txIds.length} in block #${blockHeight}`);
+        }
+        try {
+          const tx = await transactionUtils.$getTransactionExtended(txid, false, false, false, addMempoolData);
+          transactionMap[txid] = tx;
+          totalFound++;
+        } catch (e) {
+          logger.err(`Cannot fetch tx ${txid}. Reason: ` + (e instanceof Error ? e.message : e));
+        }
       }
     }
 
     if (!quiet) {
-      logger.debug(`${transactionsFound} of ${txIds.length} found in mempool. ${transactionsFetched} fetched through backend service.`);
+      logger.debug(`${foundInMempool} of ${txIds.length} found in mempool. ${totalFound - foundInMempool} fetched through backend service.`);
     }
 
-    return transactions;
+    // Return list of transactions, preserving block order
+    return txIds.map(txid => transactionMap[txid]).filter(tx => tx != null);
   }
 
   /**
@@ -765,10 +785,18 @@ class Blocks {
       if (this.newBlockCallbacks.length) {
         this.newBlockCallbacks.forEach((cb) => cb(blockExtended, txIds, transactions));
       }
-      if (!memPool.hasPriority() && (block.height % config.MEMPOOL.DISK_CACHE_BLOCK_INTERVAL === 0)) {
+      if (config.MEMPOOL.CACHE_ENABLED && !memPool.hasPriority() && (block.height % config.MEMPOOL.DISK_CACHE_BLOCK_INTERVAL === 0)) {
         diskCache.$saveCacheToDisk();
       }
 
+      // Update Redis cache
+      if (config.REDIS.ENABLED) {
+        await redisCache.$updateBlocks(this.blocks);
+        await redisCache.$updateBlockSummaries(this.blockSummaries);
+        await redisCache.$removeTransactions(txIds);
+        await rbfCache.updateCache();
+      }
+
       handledBlocks++;
     }
 

backend/src/api/disk-cache.ts

@@ -29,7 +29,7 @@ class DiskCache {
   };
 
   constructor() {
-    if (!cluster.isPrimary) {
+    if (!cluster.isPrimary || !config.MEMPOOL.CACHE_ENABLED) {
       return;
     }
     process.on('SIGINT', (e) => {
@@ -39,7 +39,7 @@ class DiskCache {
   }
 
   async $saveCacheToDisk(sync: boolean = false): Promise<void> {
-    if (!cluster.isPrimary) {
+    if (!cluster.isPrimary || !config.MEMPOOL.CACHE_ENABLED) {
       return;
     }
     if (this.isWritingCache) {
@@ -175,7 +175,7 @@ class DiskCache {
   }
 
   async $loadMempoolCache(): Promise<void> {
-    if (!fs.existsSync(DiskCache.FILE_NAME)) {
+    if (!config.MEMPOOL.CACHE_ENABLED || !fs.existsSync(DiskCache.FILE_NAME)) {
       return;
     }
     try {

backend/src/api/mempool.ts

@@ -1,5 +1,5 @@
 import config from '../config';
-import bitcoinApi from './bitcoin/bitcoin-api-factory';
+import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory';
 import { MempoolTransactionExtended, TransactionExtended, VbytesPerSecond } from '../mempool.interfaces';
 import logger from '../logger';
 import { Common } from './common';
@@ -9,6 +9,7 @@ import loadingIndicators from './loading-indicators';
 import bitcoinClient from './bitcoin/bitcoin-client';
 import bitcoinSecondClient from './bitcoin/bitcoin-second-client';
 import rbfCache from './rbf-cache';
+import redisCache from './redis-cache';
 
 class Mempool {
   private inSync: boolean = false;
@@ -85,6 +86,10 @@ class Mempool {
   public async $setMempool(mempoolData: { [txId: string]: MempoolTransactionExtended }) {
     this.mempoolCache = mempoolData;
     let count = 0;
+    const redisTimer = Date.now();
+    if (config.MEMPOOL.CACHE_ENABLED && config.REDIS.ENABLED) {
+      logger.debug(`Migrating ${Object.keys(this.mempoolCache).length} transactions from disk cache to Redis cache`);
+    }
     for (const txid of Object.keys(this.mempoolCache)) {
       if (!this.mempoolCache[txid].sigops || this.mempoolCache[txid].effectiveFeePerVsize == null) {
         this.mempoolCache[txid] = transactionUtils.extendMempoolTransaction(this.mempoolCache[txid]);
@@ -93,6 +98,13 @@ class Mempool {
         this.mempoolCache[txid].order = transactionUtils.txidToOrdering(txid);
       }
       count++;
+      if (config.MEMPOOL.CACHE_ENABLED && config.REDIS.ENABLED) {
+        await redisCache.$addTransaction(this.mempoolCache[txid]);
+      }
+    }
+    if (config.MEMPOOL.CACHE_ENABLED && config.REDIS.ENABLED) {
+      await redisCache.$flushTransactions();
+      logger.debug(`Finished migrating cache transactions in ${((Date.now() - redisTimer) / 1000).toFixed(2)} seconds`);
     }
     if (this.mempoolChangedCallback) {
       this.mempoolChangedCallback(this.mempoolCache, [], []);
@@ -103,6 +115,44 @@ class Mempool {
     this.addToSpendMap(Object.values(this.mempoolCache));
   }
 
+  public async $reloadMempool(expectedCount: number): Promise<MempoolTransactionExtended[]> {
+    let count = 0;
+    let done = false;
+    let last_txid;
+    const newTransactions: MempoolTransactionExtended[] = [];
+    loadingIndicators.setProgress('mempool', count / expectedCount * 100);
+    while (!done) {
+      try {
+        const result = await bitcoinApi.$getMempoolTransactions(last_txid);
+        if (result) {
+          for (const tx of result) {
+            const extendedTransaction = transactionUtils.extendMempoolTransaction(tx);
+            if (!this.mempoolCache[extendedTransaction.txid]) {
+              newTransactions.push(extendedTransaction);
+              this.mempoolCache[extendedTransaction.txid] = extendedTransaction;
+            }
+            count++;
+          }
+          logger.info(`Fetched ${count} of ${expectedCount} mempool transactions from esplora`);
+          if (result.length > 0) {
+            last_txid = result[result.length - 1].txid;
+          } else {
+            done = true;
+          }
+          if (Math.floor((count / expectedCount) * 100) < 100) {
+            loadingIndicators.setProgress('mempool', count / expectedCount * 100);
+          }
+        } else {
+          done = true;
+        }
+      } catch(err) {
+        logger.err('failed to fetch bulk mempool transactions from esplora');
+      }
+    }
+    logger.info(`Done inserting loaded mempool transactions into local cache`);
+    return newTransactions;
+  }
+
   public async $updateMemPoolInfo() {
     this.mempoolInfo = await this.$getMempoolInfo();
   }
@@ -143,7 +193,7 @@ class Mempool {
     const currentMempoolSize = Object.keys(this.mempoolCache).length;
     this.updateTimerProgress(timer, 'got raw mempool');
     const diff = transactions.length - currentMempoolSize;
-    const newTransactions: MempoolTransactionExtended[] = [];
+    let newTransactions: MempoolTransactionExtended[] = [];
 
     this.mempoolCacheDelta = Math.abs(diff);
 
@@ -162,41 +212,66 @@ class Mempool {
     };
 
     let intervalTimer = Date.now();
-    for (const txid of transactions) {
+
-      if (!this.mempoolCache[txid]) {
+    let loaded = false;
-        try {
+    if (config.MEMPOOL.BACKEND === 'esplora' && currentMempoolSize < transactions.length * 0.5 && transactions.length > 20_000) {
-          const transaction = await transactionUtils.$getMempoolTransactionExtended(txid, false, false, false);
+      this.inSync = false;
-          this.updateTimerProgress(timer, 'fetched new transaction');
+      logger.info(`Missing ${transactions.length - currentMempoolSize} mempool transactions, attempting to reload in bulk from esplora`);
-          this.mempoolCache[txid] = transaction;
+      try {
-          if (this.inSync) {
+        newTransactions = await this.$reloadMempool(transactions.length);
-            this.txPerSecondArray.push(new Date().getTime());
+        if (config.REDIS.ENABLED) {
-            this.vBytesPerSecondArray.push({
+          for (const tx of newTransactions) {
-              unixTime: new Date().getTime(),
+            await redisCache.$addTransaction(tx);
-              vSize: transaction.vsize,
-            });
           }
-          hasChange = true;
-          newTransactions.push(transaction);
-        } catch (e: any) {
-          if (config.MEMPOOL.BACKEND === 'esplora' && e.response?.status === 404) {
-            this.missingTxCount++;
-          }
-          logger.debug(`Error finding transaction '${txid}' in the mempool: ` + (e instanceof Error ? e.message : e));
         }
+        loaded = true;
+      } catch (e) {
+        logger.err('failed to load mempool in bulk from esplora, falling back to fetching individual transactions');
       }
+    }
 
-      if (Date.now() - intervalTimer > 5_000) {
+    if (!loaded) {
+      for (const txid of transactions) {
+        if (!this.mempoolCache[txid]) {
+          try {
+            const transaction = await transactionUtils.$getMempoolTransactionExtended(txid, false, false, false);
+            this.updateTimerProgress(timer, 'fetched new transaction');
+            this.mempoolCache[txid] = transaction;
+            if (this.inSync) {
+              this.txPerSecondArray.push(new Date().getTime());
+              this.vBytesPerSecondArray.push({
+                unixTime: new Date().getTime(),
+                vSize: transaction.vsize,
+              });
+            }
+            hasChange = true;
+            newTransactions.push(transaction);
 
-        if (this.inSync) {
+            if (config.REDIS.ENABLED) {
-          // Break and restart mempool loop if we spend too much time processing
+              await redisCache.$addTransaction(transaction);
-          // new transactions that may lead to falling behind on block height
+            }
-          logger.debug('Breaking mempool loop because the 5s time limit exceeded.');
+          } catch (e: any) {
-          break;
+            if (config.MEMPOOL.BACKEND === 'esplora' && e.response?.status === 404) {
-        } else {
+              this.missingTxCount++;
-          const progress = (currentMempoolSize + newTransactions.length) / transactions.length * 100;
+            }
-          logger.debug(`Mempool is synchronizing. Processed ${newTransactions.length}/${diff} txs (${Math.round(progress)}%)`);
+            logger.debug(`Error finding transaction '${txid}' in the mempool: ` + (e instanceof Error ? e.message : e));
-          loadingIndicators.setProgress('mempool', progress);
+          }
-          intervalTimer = Date.now()
+        }
+
+        if (Date.now() - intervalTimer > 5_000) {
+          if (this.inSync) {
+            // Break and restart mempool loop if we spend too much time processing
+            // new transactions that may lead to falling behind on block height
+            logger.debug('Breaking mempool loop because the 5s time limit exceeded.');
+            break;
+          } else {
+            const progress = (currentMempoolSize + newTransactions.length) / transactions.length * 100;
+            logger.debug(`Mempool is synchronizing. Processed ${newTransactions.length}/${diff} txs (${Math.round(progress)}%)`);
+            if (Math.floor(progress) < 100) {
+              loadingIndicators.setProgress('mempool', progress);
+            }
+            intervalTimer = Date.now()
+          }
         }
       }
     }
@@ -246,12 +321,6 @@ class Mempool {
     const newTransactionsStripped = newTransactions.map((tx) => Common.stripTransaction(tx));
     this.latestTransactions = newTransactionsStripped.concat(this.latestTransactions).slice(0, 6);
 
-    if (!this.inSync && transactions.length === newMempoolSize) {
-      this.inSync = true;
-      logger.notice('The mempool is now in sync!');
-      loadingIndicators.setProgress('mempool', 100);
-    }
-
     this.mempoolCacheDelta = Math.abs(transactions.length - newMempoolSize);
 
     if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) {
@@ -263,6 +332,19 @@ class Mempool {
       this.updateTimerProgress(timer, 'completed async mempool callback');
     }
 
+    if (!this.inSync && transactions.length === newMempoolSize) {
+      this.inSync = true;
+      logger.notice('The mempool is now in sync!');
+      loadingIndicators.setProgress('mempool', 100);
+    }
+
+    // Update Redis cache
+    if (config.REDIS.ENABLED) {
+      await redisCache.$flushTransactions();
+      await redisCache.$removeTransactions(deletedTransactions.map(tx => tx.txid));
+      await rbfCache.updateCache();
+    }
+
     const end = new Date().getTime();
     const time = end - start;
     logger.debug(`Mempool updated in ${time / 1000} seconds. New size: ${Object.keys(this.mempoolCache).length} (${diff > 0 ? '+' + diff : diff})`);

backend/src/api/rbf-cache.ts

@@ -1,15 +1,17 @@
+import config from "../config";
 import logger from "../logger";
 import { MempoolTransactionExtended, TransactionStripped } from "../mempool.interfaces";
 import bitcoinApi from './bitcoin/bitcoin-api-factory';
 import { Common } from "./common";
+import redisCache from "./redis-cache";
 
-interface RbfTransaction extends TransactionStripped {
+export interface RbfTransaction extends TransactionStripped {
   rbf?: boolean;
   mined?: boolean;
   fullRbf?: boolean;
 }
 
-interface RbfTree {
+export interface RbfTree {
   tx: RbfTransaction;
   time: number;
   interval?: number;
@@ -28,6 +30,19 @@ export interface ReplacementInfo {
   newVsize: number;
 }
 
+enum CacheOp {
+  Remove = 0,
+  Add = 1,
+  Change = 2,
+}
+
+interface CacheEvent {
+  op: CacheOp;
+  type: 'tx' | 'tree' | 'exp';
+  txid: string,
+  value?: any,
+}
+
 class RbfCache {
   private replacedBy: Map<string, string> = new Map();
   private replaces: Map<string, string[]> = new Map();
@@ -36,11 +51,43 @@ class RbfCache {
   private treeMap: Map<string, string> = new Map(); // map of txids to sequence ids
   private txs: Map<string, MempoolTransactionExtended> = new Map();
   private expiring: Map<string, number> = new Map();
+  private cacheQueue: CacheEvent[] = [];
 
   constructor() {
     setInterval(this.cleanup.bind(this), 1000 * 60 * 10);
   }
 
+  private addTx(txid: string, tx: MempoolTransactionExtended): void {
+    this.txs.set(txid, tx);
+    this.cacheQueue.push({ op: CacheOp.Add, type: 'tx', txid });
+  }
+
+  private addTree(txid: string, tree: RbfTree): void {
+    this.rbfTrees.set(txid, tree);
+    this.dirtyTrees.add(txid);
+    this.cacheQueue.push({ op: CacheOp.Add, type: 'tree', txid });
+  }
+
+  private addExpiration(txid: string, expiry: number): void {
+    this.expiring.set(txid, expiry);
+    this.cacheQueue.push({ op: CacheOp.Add, type: 'exp', txid, value: expiry });
+  }
+
+  private removeTx(txid: string): void {
+    this.txs.delete(txid);
+    this.cacheQueue.push({ op: CacheOp.Remove, type: 'tx', txid });
+  }
+
+  private removeTree(txid: string): void {
+    this.rbfTrees.delete(txid);
+    this.cacheQueue.push({ op: CacheOp.Remove, type: 'tree', txid });
+  }
+
+  private removeExpiration(txid: string): void {
+    this.expiring.delete(txid);
+    this.cacheQueue.push({ op: CacheOp.Remove, type: 'exp', txid });
+  }
+
   public add(replaced: MempoolTransactionExtended[], newTxExtended: MempoolTransactionExtended): void {
     if (!newTxExtended || !replaced?.length || this.txs.has(newTxExtended.txid)) {
       return;
@@ -49,7 +96,7 @@ class RbfCache {
     const newTx = Common.stripTransaction(newTxExtended) as RbfTransaction;
     const newTime = newTxExtended.firstSeen || (Date.now() / 1000);
     newTx.rbf = newTxExtended.vin.some((v) => v.sequence < 0xfffffffe);
-    this.txs.set(newTx.txid, newTxExtended);
+    this.addTx(newTx.txid, newTxExtended);
 
     // maintain rbf trees
     let txFullRbf = false;
@@ -66,7 +113,7 @@ class RbfCache {
         const treeId = this.treeMap.get(replacedTx.txid);
         if (treeId) {
           const tree = this.rbfTrees.get(treeId);
-          this.rbfTrees.delete(treeId);
+          this.removeTree(treeId);
           if (tree) {
             tree.interval = newTime - tree?.time;
             replacedTrees.push(tree);
@@ -83,7 +130,7 @@ class RbfCache {
           replaces: [],
         });
         treeFullRbf = treeFullRbf || !replacedTx.rbf;
-        this.txs.set(replacedTx.txid, replacedTxExtended);
+        this.addTx(replacedTx.txid, replacedTxExtended);
       }
     }
     newTx.fullRbf = txFullRbf;
@@ -94,10 +141,9 @@ class RbfCache {
       fullRbf: treeFullRbf,
       replaces: replacedTrees
     };
-    this.rbfTrees.set(treeId, newTree);
+    this.addTree(treeId, newTree);
     this.updateTreeMap(treeId, newTree);
     this.replaces.set(newTx.txid, replacedTrees.map(tree => tree.tx.txid));
-    this.dirtyTrees.add(treeId);
   }
 
   public getReplacedBy(txId: string): string | undefined {
@@ -173,6 +219,7 @@ class RbfCache {
         this.setTreeMined(tree, txid);
         tree.mined = true;
         this.dirtyTrees.add(treeId);
+        this.cacheQueue.push({ op: CacheOp.Change, type: 'tree', txid: treeId });
       }
     }
     this.evict(txid);
@@ -181,7 +228,8 @@ class RbfCache {
   // flag a transaction as removed from the mempool
   public evict(txid: string, fast: boolean = false): void {
     if (this.txs.has(txid) && (fast || !this.expiring.has(txid))) {
-      this.expiring.set(txid, fast ? Date.now() + (1000 * 60 * 10) : Date.now() + (1000 * 86400)); // 24 hours
+      const expiryTime = fast ? Date.now() + (1000 * 60 * 10) : Date.now() + (1000 * 86400); // 24 hours
+      this.addExpiration(txid, expiryTime);
     }
   }
 
@@ -202,11 +250,11 @@ class RbfCache {
     const now = Date.now();
     for (const txid of this.expiring.keys()) {
       if ((this.expiring.get(txid) || 0) < now) {
-        this.expiring.delete(txid);
+        this.removeExpiration(txid);
         this.remove(txid);
       }
     }
-    logger.debug(`rbf cache contains ${this.txs.size} txs, ${this.expiring.size} due to expire`);
+    logger.debug(`rbf cache contains ${this.txs.size} txs, ${this.rbfTrees.size} trees, ${this.expiring.size} due to expire`);
   }
 
   // remove a transaction & all previous versions from the cache
@@ -216,14 +264,14 @@ class RbfCache {
       const replaces = this.replaces.get(txid);
       this.replaces.delete(txid);
       this.treeMap.delete(txid);
-      this.txs.delete(txid);
+      this.removeTx(txid);
-      this.expiring.delete(txid);
+      this.removeExpiration(txid);
       for (const tx of (replaces || [])) {
         // recursively remove prior versions from the cache
         this.replacedBy.delete(tx);
         // if this is the id of a tree, remove that too
         if (this.treeMap.get(tx) === tx) {
-          this.rbfTrees.delete(tx);
+          this.removeTree(tx);
         }
         this.remove(tx);
       }
@@ -255,6 +303,33 @@ class RbfCache {
     }
   }
 
+  public async updateCache(): Promise<void> {
+    if (!config.REDIS.ENABLED) {
+      return;
+    }
+    // Update the Redis cache by replaying queued events
+    for (const e of this.cacheQueue) {
+      if (e.op === CacheOp.Add || e.op === CacheOp.Change) {
+        let value = e.value;
+          switch(e.type) {
+            case 'tx': {
+              value = this.txs.get(e.txid);
+            } break;
+            case 'tree': {
+              const tree = this.rbfTrees.get(e.txid);
+              value = tree ? this.exportTree(tree) : null;
+            } break;
+          }
+          if (value != null) {
+            await redisCache.$setRbfEntry(e.type, e.txid, value);
+          }
+      } else if (e.op === CacheOp.Remove) {
+        await redisCache.$removeRbfEntry(e.type, e.txid);
+      }
+    }
+    this.cacheQueue = [];
+  }
+
   public dump(): any {
     const trees = Array.from(this.rbfTrees.values()).map((tree: RbfTree) => { return this.exportTree(tree); });
 
@@ -360,8 +435,7 @@ class RbfCache {
     };
     this.treeMap.set(txid, root);
     if (root === txid) {
-      this.rbfTrees.set(root, tree);
+      this.addTree(root, tree);
-      this.dirtyTrees.add(root);
     }
     return tree;
   }

backend/src/api/redis-cache.ts

@@ -0,0 +1,253 @@
+import { createClient } from 'redis';
+import memPool from './mempool';
+import blocks from './blocks';
+import logger from '../logger';
+import config from '../config';
+import { BlockExtended, BlockSummary, MempoolTransactionExtended } from '../mempool.interfaces';
+import rbfCache from './rbf-cache';
+import transactionUtils from './transaction-utils';
+
+enum NetworkDB {
+  mainnet = 0,
+  testnet,
+  signet,
+  liquid,
+  liquidtestnet,
+}
+
+class RedisCache {
+  private client;
+  private connected = false;
+  private schemaVersion = 1;
+
+  private cacheQueue: MempoolTransactionExtended[] = [];
+  private txFlushLimit: number = 10000;
+
+  constructor() {
+    if (config.REDIS.ENABLED) {
+      const redisConfig = {
+        socket: {
+          path: config.REDIS.UNIX_SOCKET_PATH
+        },
+        database: NetworkDB[config.MEMPOOL.NETWORK],
+      };
+      this.client = createClient(redisConfig);
+      this.client.on('error', (e) => {
+        logger.err(`Error in Redis client: ${e instanceof Error ? e.message : e}`);
+      });
+      this.$ensureConnected();
+      this.client.exists('mempool:0').then((mempoolExists) => {
+        if (!mempoolExists) {
+          for (let i = 0; i < 16; i++) {
+            this.client.json.set(`mempool:${i.toString(16)}`, '$', {});
+          }
+        }
+      });
+    }
+  }
+
+  private async $ensureConnected(): Promise<void> {
+    if (!this.connected && config.REDIS.ENABLED) {
+      return this.client.connect().then(async () => {
+        this.connected = true;
+        logger.info(`Redis client connected`);
+        const version = await this.client.get('schema_version');
+        if (version !== this.schemaVersion) {
+          // schema changed
+          // perform migrations or flush DB if necessary
+          logger.info(`Redis schema version changed from ${version} to ${this.schemaVersion}`);
+          await this.client.set('schema_version', this.schemaVersion);
+        }
+      });
+    }
+  }
+
+  async $updateBlocks(blocks: BlockExtended[]) {
+    try {
+      await this.$ensureConnected();
+      await this.client.json.set('blocks', '$', blocks);
+    } catch (e) {
+      logger.warn(`Failed to update blocks in Redis cache: ${e instanceof Error ? e.message : e}`);
+    }
+  }
+
+  async $updateBlockSummaries(summaries: BlockSummary[]) {
+    try {
+      await this.$ensureConnected();
+      await this.client.json.set('block-summaries', '$', summaries);
+    } catch (e) {
+      logger.warn(`Failed to update blocks in Redis cache: ${e instanceof Error ? e.message : e}`);
+    }
+  }
+
+  async $addTransaction(tx: MempoolTransactionExtended) {
+    this.cacheQueue.push(tx);
+    if (this.cacheQueue.length >= this.txFlushLimit) {
+      await this.$flushTransactions();
+    }
+  }
+
+  async $flushTransactions() {
+    const success = await this.$addTransactions(this.cacheQueue);
+    if (success) {
+      logger.info(`Flushed ${this.cacheQueue.length} transactions to Redis cache`);
+      this.cacheQueue = [];
+    } else {
+      logger.err(`Failed to flush ${this.cacheQueue.length} transactions to Redis cache`);
+    }
+  }
+
+  private async $addTransactions(newTransactions: MempoolTransactionExtended[]): Promise<boolean> {
+    try {
+      await this.$ensureConnected();
+      await Promise.all(newTransactions.map(tx => {
+        const minified: any = { ...tx };
+        delete minified.hex;
+        for (const vin of minified.vin) {
+          delete vin.inner_redeemscript_asm;
+          delete vin.inner_witnessscript_asm;
+          delete vin.scriptsig_asm;
+        }
+        for (const vout of minified.vout) {
+          delete vout.scriptpubkey_asm;
+        }
+        return this.client.json.set(`mempool:${tx.txid.slice(0,1)}`, tx.txid, minified);
+      }));
+      return true;
+    } catch (e) {
+      logger.warn(`Failed to add ${newTransactions.length} transactions to Redis cache: ${e instanceof Error ? e.message : e}`);
+      return false;
+    }
+  }
+
+  async $removeTransactions(transactions: string[]) {
+    try {
+      await this.$ensureConnected();
+      await Promise.all(transactions.map(txid => {
+        return this.client.json.del(`mempool:${txid.slice(0,1)}`, txid);
+      }));
+    } catch (e) {
+      logger.warn(`Failed to remove ${transactions.length} transactions from Redis cache: ${e instanceof Error ? e.message : e}`);
+    }
+  }
+
+  async $setRbfEntry(type: string, txid: string, value: any): Promise<void> {
+    try {
+      await this.$ensureConnected();
+      await this.client.json.set(`rbf:${type}:${txid}`, '$', value);
+    } catch (e) {
+      logger.warn(`Failed to set RBF ${type} in Redis cache: ${e instanceof Error ? e.message : e}`);
+    }
+  }
+
+  async $removeRbfEntry(type: string, txid: string): Promise<void> {
+    try {
+      await this.$ensureConnected();
+      await this.client.del(`rbf:${type}:${txid}`);
+    } catch (e) {
+      logger.warn(`Failed to remove RBF ${type} from Redis cache: ${e instanceof Error ? e.message : e}`);
+    }
+  }
+
+  async $getBlocks(): Promise<BlockExtended[]> {
+    try {
+      await this.$ensureConnected();
+      return this.client.json.get('blocks');
+    } catch (e) {
+      logger.warn(`Failed to retrieve blocks from Redis cache: ${e instanceof Error ? e.message : e}`);
+      return [];
+    }
+  }
+
+  async $getBlockSummaries(): Promise<BlockSummary[]> {
+    try {
+      await this.$ensureConnected();
+      return this.client.json.get('block-summaries');
+    } catch (e) {
+      logger.warn(`Failed to retrieve blocks from Redis cache: ${e instanceof Error ? e.message : e}`);
+      return [];
+    }
+  }
+
+  async $getMempool(): Promise<{ [txid: string]: MempoolTransactionExtended }> {
+    const start = Date.now();
+    let mempool = {};
+    try {
+      await this.$ensureConnected();
+      for (let i = 0; i < 16; i++) {
+        const shard = await this.client.json.get(`mempool:${i.toString(16)}`);
+        logger.info(`Loaded ${Object.keys(shard).length} transactions from redis cache ${i.toString(16)}`);
+        mempool = Object.assign(mempool, shard);
+      }
+      logger.info(`Total ${Object.keys(mempool).length} transactions loaded from redis cache `);
+      logger.info(`Loaded redis cache in ${Date.now() - start} ms`);
+      return mempool || {};
+    } catch (e) {
+      logger.warn(`Failed to retrieve mempool from Redis cache: ${e instanceof Error ? e.message : e}`);
+    }
+    return {};
+  }
+
+  async $getRbfEntries(type: string): Promise<any[]> {
+    try {
+      await this.$ensureConnected();
+      const keys = await this.client.keys(`rbf:${type}:*`);
+      const promises: Promise<MempoolTransactionExtended[]>[] = [];
+      for (let i = 0; i < keys.length; i += 10000) {
+        const keySlice = keys.slice(i, i + 10000);
+        if (!keySlice.length) {
+          continue;
+        }
+        promises.push(this.client.json.mGet(keySlice, '$').then(chunk => chunk?.length ? chunk.flat().map((v, i) => [keySlice[i].slice(`rbf:${type}:`.length), v]) : [] ));
+      }
+      const entries = await Promise.all(promises);
+      return entries.flat();
+    } catch (e) {
+      logger.warn(`Failed to retrieve Rbf ${type}s from Redis cache: ${e instanceof Error ? e.message : e}`);
+      return [];
+    }
+  }
+
+  async $loadCache() {
+    logger.info('Restoring mempool and blocks data from Redis cache');
+    // Load block data
+    const loadedBlocks = await this.$getBlocks();
+    const loadedBlockSummaries = await this.$getBlockSummaries();
+    // Load mempool
+    const loadedMempool = await this.$getMempool();
+    this.inflateLoadedTxs(loadedMempool);
+    // Load rbf data
+    const rbfTxs = await this.$getRbfEntries('tx');
+    const rbfTrees = await this.$getRbfEntries('tree');
+    const rbfExpirations = await this.$getRbfEntries('exp');
+
+    // Set loaded data
+    blocks.setBlocks(loadedBlocks || []);
+    blocks.setBlockSummaries(loadedBlockSummaries || []);
+    await memPool.$setMempool(loadedMempool);
+    await rbfCache.load({
+      txs: rbfTxs,
+      trees: rbfTrees.map(loadedTree => loadedTree[1]),
+      expiring: rbfExpirations,
+    });
+  }
+
+  private inflateLoadedTxs(mempool: { [txid: string]: MempoolTransactionExtended }) {
+    for (const tx of Object.values(mempool)) {
+      for (const vin of tx.vin) {
+        if (vin.scriptsig) {
+          vin.scriptsig_asm = transactionUtils.convertScriptSigAsm(vin.scriptsig);
+          transactionUtils.addInnerScriptsToVin(vin);
+        }
+      }
+      for (const vout of tx.vout) {
+        if (vout.scriptpubkey) {
+          vout.scriptpubkey_asm = transactionUtils.convertScriptSigAsm(vout.scriptpubkey);
+        }
+      }
+    }
+  }
+
+}
+
+export default new RedisCache();

backend/src/api/transaction-utils.ts

@@ -53,7 +53,7 @@ class TransactionUtils {
     return (await this.$getTransactionExtended(txId, addPrevouts, lazyPrevouts, forceCore, true)) as MempoolTransactionExtended;
   }
 
-  private extendTransaction(transaction: IEsploraApi.Transaction): TransactionExtended {
+  public extendTransaction(transaction: IEsploraApi.Transaction): TransactionExtended {
     // @ts-ignore
     if (transaction.vsize) {
       // @ts-ignore
@@ -170,6 +170,122 @@ class TransactionUtils {
       16
     );
   }
+
+  public addInnerScriptsToVin(vin: IEsploraApi.Vin): void {
+    if (!vin.prevout) {
+      return;
+    }
+
+    if (vin.prevout.scriptpubkey_type === 'p2sh') {
+      const redeemScript = vin.scriptsig_asm.split(' ').reverse()[0];
+      vin.inner_redeemscript_asm = this.convertScriptSigAsm(redeemScript);
+      if (vin.witness && vin.witness.length > 2) {
+        const witnessScript = vin.witness[vin.witness.length - 1];
+        vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript);
+      }
+    }
+
+    if (vin.prevout.scriptpubkey_type === 'v0_p2wsh' && vin.witness) {
+      const witnessScript = vin.witness[vin.witness.length - 1];
+      vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript);
+    }
+
+    if (vin.prevout.scriptpubkey_type === 'v1_p2tr' && vin.witness) {
+      const witnessScript = this.witnessToP2TRScript(vin.witness);
+      if (witnessScript !== null) {
+        vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript);
+      }
+    }
+  }
+
+  public convertScriptSigAsm(hex: string): string {
+    const buf = Buffer.from(hex, 'hex');
+
+    const b: string[] = [];
+
+    let i = 0;
+    while (i < buf.length) {
+      const op = buf[i];
+      if (op >= 0x01 && op <= 0x4e) {
+        i++;
+        let push: number;
+        if (op === 0x4c) {
+          push = buf.readUInt8(i);
+          b.push('OP_PUSHDATA1');
+          i += 1;
+        } else if (op === 0x4d) {
+          push = buf.readUInt16LE(i);
+          b.push('OP_PUSHDATA2');
+          i += 2;
+        } else if (op === 0x4e) {
+          push = buf.readUInt32LE(i);
+          b.push('OP_PUSHDATA4');
+          i += 4;
+        } else {
+          push = op;
+          b.push('OP_PUSHBYTES_' + push);
+        }
+
+        const data = buf.slice(i, i + push);
+        if (data.length !== push) {
+          break;
+        }
+
+        b.push(data.toString('hex'));
+        i += data.length;
+      } else {
+        if (op === 0x00) {
+          b.push('OP_0');
+        } else if (op === 0x4f) {
+          b.push('OP_PUSHNUM_NEG1');
+        } else if (op === 0xb1) {
+          b.push('OP_CLTV');
+        } else if (op === 0xb2) {
+          b.push('OP_CSV');
+        } else if (op === 0xba) {
+          b.push('OP_CHECKSIGADD');
+        } else {
+          const opcode = bitcoinjs.script.toASM([ op ]);
+          if (opcode && op < 0xfd) {
+            if (/^OP_(\d+)$/.test(opcode)) {
+              b.push(opcode.replace(/^OP_(\d+)$/, 'OP_PUSHNUM_$1'));
+            } else {
+              b.push(opcode);
+            }
+          } else {
+            b.push('OP_RETURN_' + op);
+          }
+        }
+        i += 1;
+      }
+    }
+
+    return b.join(' ');
+  }
+
+  /**
+   * This function must only be called when we know the witness we are parsing
+   * is a taproot witness.
+   * @param witness An array of hex strings that represents the witness stack of
+   *                the input.
+   * @returns null if the witness is not a script spend, and the hex string of
+   *          the script item if it is a script spend.
+   */
+  public witnessToP2TRScript(witness: string[]): string | null {
+    if (witness.length < 2) return null;
+    // Note: see BIP341 for parsing details of witness stack
+
+    // If there are at least two witness elements, and the first byte of the
+    // last element is 0x50, this last element is called annex a and
+    // is removed from the witness stack.
+    const hasAnnex = witness[witness.length - 1].substring(0, 2) === '50';
+    // If there are at least two witness elements left, script path spending is used.
+    // Call the second-to-last stack element s, the script.
+    // (Note: this phrasing from BIP341 assumes we've *removed* the annex from the stack)
+    if (hasAnnex && witness.length < 3) return null;
+    const positionOfScript = hasAnnex ? witness.length - 3 : witness.length - 2;
+    return witness[positionOfScript];
+  }
 }
 
 export default new TransactionUtils();

backend/src/api/websocket-handler.ts

@@ -604,7 +604,7 @@ class WebsocketHandler {
         }
       }
 
-      if (client['track-mempool-block'] >= 0) {
+      if (client['track-mempool-block'] >= 0 && memPool.isInSync()) {
         const index = client['track-mempool-block'];
         if (mBlockDeltas[index]) {
           response['projected-block-transactions'] = getCachedResponse(`projected-block-transactions-${index}`, {
@@ -644,7 +644,7 @@ class WebsocketHandler {
     memPool.handleMinedRbfTransactions(rbfTransactions);
     memPool.removeFromSpendMap(transactions);
 
-    if (config.MEMPOOL.AUDIT) {
+    if (config.MEMPOOL.AUDIT && memPool.isInSync()) {
       let projectedBlocks;
       let auditMempool = _memPool;
       // template calculation functions have mempool side effects, so calculate audits using
@@ -665,7 +665,7 @@ class WebsocketHandler {
         projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
       }
 
-      if (Common.indexingEnabled() && memPool.isInSync()) {
+      if (Common.indexingEnabled()) {
         const { censored, added, fresh, sigop, fullrbf, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool);
         const matchRate = Math.round(score * 100 * 100) / 100;
 
@@ -858,7 +858,7 @@ class WebsocketHandler {
         }
       }
 
-      if (client['track-mempool-block'] >= 0) {
+      if (client['track-mempool-block'] >= 0 && memPool.isInSync()) {
         const index = client['track-mempool-block'];
         if (mBlockDeltas && mBlockDeltas[index]) {
           response['projected-block-transactions'] = getCachedResponse(`projected-block-transactions-${index}`, {

backend/src/config.ts

@@ -12,6 +12,7 @@ interface IConfig {
     API_URL_PREFIX: string;
     POLL_RATE_MS: number;
     CACHE_DIR: string;
+    CACHE_ENABLED: boolean;
     CLEAR_PROTECTION_MINUTES: number;
     RECOMMENDED_FEE_PERCENTILE: number;
     BLOCK_WEIGHT_UNITS: number;
@@ -137,7 +138,11 @@ interface IConfig {
     AUDIT: boolean;
     AUDIT_START_HEIGHT: number;
     SERVERS: string[];
-  }
+  },
+  REDIS: {
+    ENABLED: boolean;
+    UNIX_SOCKET_PATH: string;
+  },
 }
 
 const defaults: IConfig = {
@@ -150,6 +155,7 @@ const defaults: IConfig = {
     'API_URL_PREFIX': '/api/v1/',
     'POLL_RATE_MS': 2000,
     'CACHE_DIR': './cache',
+    'CACHE_ENABLED': true,
     'CLEAR_PROTECTION_MINUTES': 20,
     'RECOMMENDED_FEE_PERCENTILE': 50,
     'BLOCK_WEIGHT_UNITS': 4000000,
@@ -275,7 +281,11 @@ const defaults: IConfig = {
     'AUDIT': false,
     'AUDIT_START_HEIGHT': 774000,
     'SERVERS': [],
-  }
+  },
+  'REDIS': {
+    'ENABLED': false,
+    'UNIX_SOCKET_PATH': '',
+  },
 };
 
 class Config implements IConfig {
@@ -296,6 +306,7 @@ class Config implements IConfig {
   EXTERNAL_DATA_SERVER: IConfig['EXTERNAL_DATA_SERVER'];
   MAXMIND: IConfig['MAXMIND'];
   REPLICATION: IConfig['REPLICATION'];
+  REDIS: IConfig['REDIS'];
 
   constructor() {
     const configs = this.merge(configFromFile, defaults);
@@ -316,6 +327,7 @@ class Config implements IConfig {
     this.EXTERNAL_DATA_SERVER = configs.EXTERNAL_DATA_SERVER;
     this.MAXMIND = configs.MAXMIND;
     this.REPLICATION = configs.REPLICATION;
+    this.REDIS = configs.REDIS;
   }
 
   merge = (...objects: object[]): IConfig => {

backend/src/index.ts

@@ -41,6 +41,7 @@ import chainTips from './api/chain-tips';
 import { AxiosError } from 'axios';
 import v8 from 'v8';
 import { formatBytes, getBytesUnit } from './utils/format';
+import redisCache from './api/redis-cache';
 
 class Server {
   private wss: WebSocket.Server | undefined;
@@ -122,7 +123,11 @@ class Server {
     await poolsUpdater.updatePoolsJson(); // Needs to be done before loading the disk cache because we sometimes wipe it
     await syncAssets.syncAssets$();
     if (config.MEMPOOL.ENABLED) {
-      await diskCache.$loadMempoolCache();
+      if (config.MEMPOOL.CACHE_ENABLED) {
+        await diskCache.$loadMempoolCache();
+      } else if (config.REDIS.ENABLED) {
+        await redisCache.$loadCache();
+      }
     }
 
     if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED && cluster.isPrimary) {

frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts

@@ -117,7 +117,14 @@ export class MempoolBlocksComponent implements OnInit, OnChanges, OnDestroy {
     });
     this.reduceMempoolBlocksToFitScreen(this.mempoolBlocks);
     this.stateService.isTabHidden$.subscribe((tabHidden) => this.tabHidden = tabHidden);
-    this.loadingBlocks$ = this.stateService.isLoadingWebSocket$;
+    this.loadingBlocks$ = combineLatest([
+      this.stateService.isLoadingWebSocket$,
+      this.stateService.isLoadingMempool$
+    ]).pipe(
+      switchMap(([loadingBlocks, loadingMempool]) => {
+        return of(loadingBlocks || loadingMempool);
+      })
+    );
 
     this.mempoolBlocks$ = merge(
       of(true),

frontend/src/app/lightning/channel/channel-close-box/channel-close-box.component.html

@@ -1,19 +1,43 @@
 <div class="box">
-  <table class="table table-borderless table-striped">
+  <div class="starting-balance" *ngIf="showStartingBalance">
-    <tbody>
+    <h5 i18n="lightning.starting-balance|Channel starting balance">Starting balance</h5>
-      <tr></tr>
+    <div class="nodes">
-      <tr>
+      <h5 class="alias">{{ left.alias }}</h5>
-        <td i18n="lightning.starting-balance|Channel starting balance">Starting balance</td>
+      <h5 class="alias">{{ right.alias }}</h5>
-        <td *ngIf="showStartingBalance && minStartingBalance === maxStartingBalance"><app-sats [satoshis]="minStartingBalance"></app-sats></td>
+    </div>
-        <td *ngIf="showStartingBalance && minStartingBalance !== maxStartingBalance">{{ minStartingBalance | number : '1.0-0' }} - {{ maxStartingBalance | number : '1.0-0' }}<app-sats [valueOverride]=" "></app-sats></td>
+    <div class="balances">
-        <td *ngIf="!showStartingBalance">?</td>
+      <div class="balance left">
-      </tr>
+        <span class="value" *ngIf="minStartingBalance !== maxStartingBalance">{{ minStartingBalance | number : '1.0-0' }} - {{ maxStartingBalance | number : '1.0-0' }}<app-sats [valueOverride]=" "></app-sats></span>
-      <tr *ngIf="channel.status === 2">
+        <span class="value" *ngIf="minStartingBalance === maxStartingBalance">{{ minStartingBalance | number : '1.0-0' }}<app-sats [valueOverride]=" "></app-sats></span>
-        <td i18n="lightning.closing-balance|Channel closing balance">Closing balance</td>
+      </div>
-        <td *ngIf="showClosingBalance && minClosingBalance === maxClosingBalance"><app-sats [satoshis]="minClosingBalance"></app-sats></td>
+      <div class="balance right">
-        <td *ngIf="showClosingBalance && minClosingBalance !== maxClosingBalance">{{ minClosingBalance | number : '1.0-0' }} - {{ maxClosingBalance | number : '1.0-0' }}<app-sats [valueOverride]=" "></app-sats></td>
+        <span class="value" *ngIf="minStartingBalance !== maxStartingBalance">{{ channel.capacity - maxStartingBalance | number : '1.0-0' }} - {{ channel.capacity - minStartingBalance | number : '1.0-0' }}<app-sats [valueOverride]=" "></app-sats></span>
-        <td *ngIf="!showClosingBalance">?</td>
+        <span class="value" *ngIf="minStartingBalance === maxStartingBalance">{{ channel.capacity - maxStartingBalance | number : '1.0-0' }}<app-sats [valueOverride]=" "></app-sats></span>
-      </tr>
+      </div>
-    </tbody>
+    </div>
-  </table>
+    <div class="balance-bar">
+      <div class="bar left" [class.hide-value]="hideStartingLeft" [style]="startingBalanceStyle.left"></div>
+      <div class="bar center" [style]="startingBalanceStyle.center"></div>
+      <div class="bar right"  [class.hide-value]="hideStartingRight" [style]="startingBalanceStyle.right"></div>
+    </div>
+  </div>
+  <br>
+  <div class="closing-balance" *ngIf="showClosingBalance">
+    <h5 i18n="lightning.closing-balance|Channel closing balance">Closing balance</h5>
+    <div class="balances">
+      <div class="balance left">
+        <span class="value" *ngIf="minClosingBalance !== maxClosingBalance">{{ minClosingBalance | number : '1.0-0' }} - {{ maxClosingBalance | number : '1.0-0' }}<app-sats [valueOverride]=" "></app-sats></span>
+        <span class="value" *ngIf="minClosingBalance === maxClosingBalance">{{ minClosingBalance | number : '1.0-0' }}<app-sats [valueOverride]=" "></app-sats></span>
+      </div>
+      <div class="balance right">
+        <span class="value" *ngIf="minClosingBalance !== maxClosingBalance">{{ channel.capacity - maxClosingBalance | number : '1.0-0' }} - {{ channel.capacity - minClosingBalance | number : '1.0-0' }}<app-sats [valueOverride]=" "></app-sats></span>
+        <span class="value" *ngIf="minClosingBalance === maxClosingBalance">{{ channel.capacity - maxClosingBalance | number : '1.0-0' }}<app-sats [valueOverride]=" "></app-sats></span>
+      </div>
+    </div>
+    <div class="balance-bar">
+      <div class="bar left" [class.hide-value]="hideClosingLeft" [style]="closingBalanceStyle.left"></div>
+      <div class="bar center" [style]="closingBalanceStyle.center"></div>
+      <div class="bar right" [class.hide-value]="hideClosingRight" [style]="closingBalanceStyle.right"></div>
+    </div>
+  </div>
 </div>

frontend/src/app/lightning/channel/channel-close-box/channel-close-box.component.scss

@@ -7,3 +7,97 @@
     margin-bottom: 20px;
   }
 }
+
+.starting-balance, .closing-balance {
+  width: 100%;
+
+  h5 {
+    text-align: center;
+  }
+}
+
+.nodes {
+  display: none;
+  flex-direction: row;
+  align-items: baseline;
+  justify-content: space-between;
+
+  @media (max-width: 768px) {
+    display: flex;
+	}
+}
+
+.balances {
+  display: flex;
+  flex-direction: row;
+  align-items: baseline;
+  justify-content: space-between;
+  margin-bottom: 8px;
+
+  .balance {
+    &.left {
+      text-align: start;
+    }
+    &.right {
+      text-align: end;
+    }
+  }
+}
+
+.balance-bar {
+  width: 100%;
+  height: 2em;
+  position: relative;
+
+  .bar {
+    position: absolute;
+    top: 0;
+    bottom: 0;
+    height: 100%;
+    display: flex;
+    flex-direction: row;
+    align-items: center;
+    justify-content: center;
+
+    &.left {
+      background: #105fb0;
+    }
+    &.center {
+      background: repeating-linear-gradient(
+        60deg,
+        #105fb0 0,
+        #105fb0 12px,
+        #1a9436 12px,
+        #1a9436 24px
+      );
+    }
+    &.right {
+      background: #1a9436;
+    }
+
+    .value {
+      flex: 0;
+      white-space: nowrap;
+    }
+
+    &.hide-value {
+      .value {
+        display: none;
+      }
+    }
+  }
+
+  @media (max-width: 768px) {
+    height: 1em;
+
+    .bar.center {
+      background: repeating-linear-gradient(
+        60deg,
+        #105fb0 0,
+        #105fb0 8px,
+        #1a9436 8px,
+        #1a9436 16px
+      )
+    }
+	}
+}

frontend/src/app/lightning/channel/channel-close-box/channel-close-box.component.ts

@@ -8,8 +8,8 @@ import { ChangeDetectionStrategy, Component, Input, OnChanges, SimpleChanges } f
 })
 export class ChannelCloseBoxComponent implements OnChanges {
   @Input() channel: any;
-  @Input() local: any;
+  @Input() left: any;
-  @Input() remote: any;
+  @Input() right: any;
 
   showStartingBalance: boolean = false;
   showClosingBalance: boolean = false;
@@ -18,29 +18,55 @@ export class ChannelCloseBoxComponent implements OnChanges {
   minClosingBalance: number;
   maxClosingBalance: number;
 
+  startingBalanceStyle: {
+    left: string,
+    center: string,
+    right: string,
+  } = {
+    left: '',
+    center: '',
+    right: '',
+  };
+
+  closingBalanceStyle: {
+    left: string,
+    center: string,
+    right: string,
+  } = {
+    left: '',
+    center: '',
+    right: '',
+  };
+
+  hideStartingLeft: boolean = false;
+  hideStartingRight: boolean = false;
+  hideClosingLeft: boolean = false;
+  hideClosingRight: boolean = false;
+
   constructor() { }
 
   ngOnChanges(changes: SimpleChanges): void {
-    if (this.channel && this.local && this.remote) {
+    let closingCapacity;
-      this.showStartingBalance = (this.local.funding_balance || this.remote.funding_balance) && this.channel.funding_ratio;
+    if (this.channel && this.left && this.right) {
-      this.showClosingBalance = this.local.closing_balance || this.remote.closing_balance;
+      this.showStartingBalance = (this.left.funding_balance || this.right.funding_balance) && this.channel.funding_ratio;
+      this.showClosingBalance = this.left.closing_balance || this.right.closing_balance;
 
       if (this.channel.single_funded) {
-        if (this.local.funding_balance) {
+        if (this.left.funding_balance) {
           this.minStartingBalance = this.channel.capacity;
           this.maxStartingBalance = this.channel.capacity;
-        } else if (this.remote.funding_balance) {
+        } else if (this.right.funding_balance) {
           this.minStartingBalance = 0;
           this.maxStartingBalance = 0;
         }
       } else {
-        this.minStartingBalance = clampRound(0, this.channel.capacity, this.local.funding_balance * this.channel.funding_ratio);
+        this.minStartingBalance = clampRound(0, this.channel.capacity, this.left.funding_balance * this.channel.funding_ratio);
-        this.maxStartingBalance = clampRound(0, this.channel.capacity, this.channel.capacity - (this.remote.funding_balance * this.channel.funding_ratio));
+        this.maxStartingBalance = clampRound(0, this.channel.capacity, this.channel.capacity - (this.right.funding_balance * this.channel.funding_ratio));
       }
 
-      const closingCapacity = this.channel.capacity - this.channel.closing_fee;
+      closingCapacity = this.channel.capacity - this.channel.closing_fee;
-      this.minClosingBalance = clampRound(0, closingCapacity, this.local.closing_balance);
+      this.minClosingBalance = clampRound(0, closingCapacity, this.left.closing_balance);
-      this.maxClosingBalance = clampRound(0, closingCapacity, closingCapacity - this.remote.closing_balance);
+      this.maxClosingBalance = clampRound(0, closingCapacity, closingCapacity - this.right.closing_balance);
 
       // margin of error to account for 2 x 330 sat anchor outputs
       if (Math.abs(this.minClosingBalance - this.maxClosingBalance) <= 660) {
@@ -50,6 +76,26 @@ export class ChannelCloseBoxComponent implements OnChanges {
       this.showStartingBalance = false;
       this.showClosingBalance = false;
     }
+
+    const startingMinPc = (this.minStartingBalance / this.channel.capacity) * 100;
+    const startingMaxPc = (this.maxStartingBalance / this.channel.capacity) * 100;
+    this.startingBalanceStyle = {
+      left: `left: 0%; right: ${100 - startingMinPc}%;`,
+      center: `left: ${startingMinPc}%; right: ${100 -startingMaxPc}%;`,
+      right: `left: ${startingMaxPc}%; right: 0%;`,
+    };
+    this.hideStartingLeft = startingMinPc < 15;
+    this.hideStartingRight = startingMaxPc > 85;
+
+    const closingMinPc = (this.minClosingBalance / closingCapacity) * 100;
+    const closingMaxPc = (this.maxClosingBalance / closingCapacity) * 100;
+    this.closingBalanceStyle = {
+      left: `left: 0%; right: ${100 - closingMinPc}%;`,
+      center: `left: ${closingMinPc}%; right: ${100 - closingMaxPc}%;`,
+      right: `left: ${closingMaxPc}%; right: 0%;`,
+    };
+    this.hideClosingLeft = closingMinPc < 15;
+    this.hideClosingRight = closingMaxPc > 85;
   }
 }
 

frontend/src/app/lightning/channel/channel.component.html

@@ -75,14 +75,14 @@
   <div class="row row-cols-1 row-cols-md-2" *ngIf="!error">
     <div class="col">
       <app-channel-box [channel]="channel.node_left"></app-channel-box>
-      <app-channel-close-box *ngIf="showCloseBoxes(channel)" [channel]="channel" [local]="channel.node_left" [remote]="channel.node_right"></app-channel-close-box>
     </div>
     <div class="col">
       <app-channel-box [channel]="channel.node_right"></app-channel-box>
-      <app-channel-close-box *ngIf="showCloseBoxes(channel)" [channel]="channel" [local]="channel.node_right" [remote]="channel.node_left"></app-channel-close-box>
     </div>
   </div>
 
+  <app-channel-close-box *ngIf="showCloseBoxes(channel)" [channel]="channel" [left]="channel.node_left" [right]="channel.node_right"></app-channel-close-box>
+
   <br>
 
   <ng-container *ngIf="transactions$ | async as transactions">

frontend/src/app/services/state.service.ts

@@ -113,6 +113,7 @@ export class StateService {
   mempoolTxPosition$ = new Subject<{ txid: string, position: MempoolPosition}>();
   blockTransactions$ = new Subject<Transaction>();
   isLoadingWebSocket$ = new ReplaySubject<boolean>(1);
+  isLoadingMempool$ = new BehaviorSubject<boolean>(true);
   vbytesPerSecond$ = new ReplaySubject<number>(1);
   previousRetarget$ = new ReplaySubject<number>(1);
   backendInfo$ = new ReplaySubject<IBackendInfo>(1);

frontend/src/app/services/websocket.service.ts

@@ -368,6 +368,11 @@ export class WebsocketService {
 
     if (response.loadingIndicators) {
       this.stateService.loadingIndicators$.next(response.loadingIndicators);
+      if (response.loadingIndicators.mempool != null && response.loadingIndicators.mempool < 100) {
+        this.stateService.isLoadingMempool$.next(true);
+      } else {
+        this.stateService.isLoadingMempool$.next(false);
+      }
     }
 
     if (response.mempoolInfo) {