chimaera logic

[accelerator] add credit card provider fa icons

.github/workflows/ci.yml

@@ -35,7 +35,7 @@ jobs:
       - name: Install ${{ steps.gettoolchain.outputs.toolchain }} Rust toolchain
         # Latest version available on this commit is 1.71.1
         # Commit date is Aug 3, 2023
-        uses: dtolnay/rust-toolchain@bb45937a053e097f8591208d8e74c90db1873d07
+        uses: dtolnay/rust-toolchain@d8352f6b1d2e870bc5716e7a6d9b65c4cc244a1a
         with:
           toolchain: ${{ steps.gettoolchain.outputs.toolchain }}
 
@@ -251,17 +251,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        module: ["mempool", "liquid"]
-        include:
-          - module: "mempool"
-            spec: |
-              cypress/e2e/mainnet/*.spec.ts
-              cypress/e2e/signet/*.spec.ts
-              cypress/e2e/testnet/*.spec.ts
-          - module: "liquid"
-            spec: |
-              cypress/e2e/liquid/liquid.spec.ts
-              cypress/e2e/liquidtestnet/liquidtestnet.spec.ts
+        module: ["mempool", "liquid", "testnet4"]
 
     name: E2E tests for ${{ matrix.module }}
     steps:
@@ -310,8 +300,10 @@ jobs:
 
       - name: Unzip assets before building (src/resources)
         run: unzip -o promo-video-assets.zip -d ${{ matrix.module }}/frontend/src/resources/promo-video
-      
+
+      # mempool
       - name: Chrome browser tests (${{ matrix.module }})
+        if: ${{ matrix.module == 'mempool' }}
         uses: cypress-io/github-action@v5
         with:
           tag: ${{ github.event_name }}
@@ -322,7 +314,9 @@ jobs:
           wait-on-timeout: 120
           record: true
           parallel: true
-          spec: ${{ matrix.spec }}
+          spec: |
+            cypress/e2e/mainnet/*.spec.ts
+            cypress/e2e/signet/*.spec.ts
           group: Tests on Chrome (${{ matrix.module }})
           browser: "chrome"
           ci-build-id: "${{ github.sha }}-${{ github.workflow }}-${{ github.event_name }}"
@@ -332,6 +326,56 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}
 
+      # liquid
+      - name: Chrome browser tests (${{ matrix.module }})
+        if: ${{ matrix.module == 'liquid' }}
+        uses: cypress-io/github-action@v5
+        with:
+          tag: ${{ github.event_name }}
+          working-directory: ${{ matrix.module }}/frontend
+          build: npm run config:defaults:${{ matrix.module }}
+          start: npm run start:local-staging
+          wait-on: "http://localhost:4200"
+          wait-on-timeout: 120
+          record: true
+          parallel: true
+          spec: |
+            cypress/e2e/liquid/liquid.spec.ts
+            cypress/e2e/liquidtestnet/liquidtestnet.spec.ts
+          group: Tests on Chrome (${{ matrix.module }})
+          browser: "chrome"
+          ci-build-id: "${{ github.sha }}-${{ github.workflow }}-${{ github.event_name }}"
+        env:
+          COMMIT_INFO_MESSAGE: ${{ github.event.pull_request.title }}
+          CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}
+
+      # testnet
+      - name: Chrome browser tests (${{ matrix.module }})
+        if: ${{ matrix.module == 'testnet4' }}
+        uses: cypress-io/github-action@v5
+        with:
+          tag: ${{ github.event_name }}
+          working-directory: ${{ matrix.module }}/frontend
+          build: npm run config:defaults:mempool
+          start: npm run start:local-staging
+          wait-on: "http://localhost:4200"
+          wait-on-timeout: 120
+          record: true
+          parallel: true
+          spec: |
+            cypress/e2e/testnet4/*.spec.ts
+          group: Tests on Chrome (${{ matrix.module }})
+          browser: "chrome"
+          ci-build-id: "${{ github.sha }}-${{ github.workflow }}-${{ github.event_name }}"
+        env:
+          CYPRESS_REROUTE_TESTNET: true
+          COMMIT_INFO_MESSAGE: ${{ github.event.pull_request.title }}
+          CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}
+
   validate_docker_json:
     if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')"
     runs-on: "ubuntu-latest"
@@ -359,4 +403,4 @@ jobs:
       - name: Validate JSON syntax
         run: |
           cat mempool-config.json | jq
-        working-directory: docker/docker/backend
+        working-directory: docker/docker/backend

LICENSE

@@ -1,5 +1,5 @@
 The Mempool Open Source Project®
-Copyright (c) 2019-2023 Mempool Space K.K. and other shadowy super-coders
+Copyright (c) 2019-2024 Mempool Space K.K. and other shadowy super-coders
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU Affero General Public License as published by the Free
@@ -12,10 +12,12 @@ or any other contributor to The Mempool Open Source Project.
 
 The Mempool Open Source Project®, Mempool Accelerator™, Mempool Enterprise®, 
 Mempool Liquidity™, mempool.space®, Be your own explorer™, Explore the full 
-Bitcoin ecosystem™, Mempool Goggles™, the mempool Logo, the mempool Square logo, 
-the mempool Blocks logo, the mempool Blocks 3 | 2 logo, the mempool.space Vertical 
-Logo, and the mempool.space Horizontal logo are registered trademarks or trademarks 
-of Mempool Space K.K in Japan, the United States, and/or other countries.
+Bitcoin ecosystem™, Mempool Goggles™, the mempool Logo, the mempool Square Logo, 
+the mempool block visualization Logo, the mempool Blocks Logo, the mempool 
+transaction Logo, the mempool Blocks 3 | 2 Logo, the mempool research Logo, 
+the mempool.space Vertical Logo, and the mempool.space Horizontal Logo are 
+registered trademarks or trademarks of Mempool Space K.K in Japan, 
+the United States, and/or other countries.
 
 See our full Trademark Policy and Guidelines for more details, published on 
 <https://mempool.space/trademark-policy>.

backend/.eslintrc

@@ -20,6 +20,7 @@
     "@typescript-eslint/no-this-alias": 1,
     "@typescript-eslint/no-var-requires": 1,
     "@typescript-eslint/explicit-function-return-type": 1,
+    "@typescript-eslint/no-unused-vars": 1,
     "no-console": 1,
     "no-constant-condition": 1,
     "no-dupe-else-if": 1,
@@ -32,6 +33,8 @@
     "prefer-rest-params": 1,
     "quotes": [1, "single", { "allowTemplateLiterals": true }],
     "semi": 1,
-    "eqeqeq": 1
+    "curly": [1, "all"],
+    "eqeqeq": 1,
+    "no-trailing-spaces": 1
   }
 }

backend/README.md

@@ -77,7 +77,7 @@ Query OK, 0 rows affected (0.00 sec)
 
 #### Build
 
-_Make sure to use Node.js 16.10 and npm 7._
+_Make sure to use Node.js 20.x and npm 9.x or newer_
 
 _The build process requires [Rust](https://www.rust-lang.org/tools/install) to be installed._
 
@@ -181,7 +181,7 @@ Create a new wallet, if needed:
    bitcoin-cli -regtest createwallet test
    ```
 
-Load wallet (this command may take a while if you have lot of UTXOs):
+Load wallet (this command may take a while if you have a lot of UTXOs):
    ```
    bitcoin-cli -regtest loadwallet test
    ```
@@ -229,13 +229,13 @@ Generate block at regular interval (every 10 seconds in this example):
 
 ### Mining pools update
 
-By default, mining pools will be not automatically updated regularly (`config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING` is set to `false`). 
+By default, mining pools will be not automatically updated regularly (`config.MEMPOOL.AUTOMATIC_POOLS_UPDATE` is set to `false`). 
 
 To manually update your mining pools, you can use the `--update-pools` command line flag when you run the nodejs backend. For example `npm run start --update-pools`. This will trigger the mining pools update and automatically re-index appropriate blocks.
 
-You can enabled the automatic mining pools update by settings `config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING` to `true` in your `mempool-config.json`.
+You can enable the automatic mining pools update by settings `config.MEMPOOL.AUTOMATIC_POOLS_UPDATE` to `true` in your `mempool-config.json`.
 
-When a `coinbase tag` or `coinbase address` change is detected, all blocks tagged to the `unknown` mining pools (starting from height 130635) will be deleted from the `blocks` table. Additionaly, all blocks which were tagged to the pool which has been updated will also be deleted from the `blocks` table. Of course, those blocks will be automatically reindexed.
+When a `coinbase tag` or `coinbase address` change is detected, pool assignments for all relevant blocks (tagged to that pool or the `unknown` mining pool, starting from height 130635) are updated using the new criteria.
 
 ### Re-index tables
 

backend/jest.config.ts

@@ -7,7 +7,7 @@ const config: Config.InitialOptions = {
   automock: false,
   collectCoverage: true,
   collectCoverageFrom: ["./src/**/**.ts"],
-  coverageProvider: "babel",
+  coverageProvider: "v8",
   coverageThreshold: {
     global: {
       lines: 1

backend/mempool-config.sample.json

@@ -24,11 +24,12 @@
     "EXTERNAL_RETRY_INTERVAL": 0,
     "USER_AGENT": "mempool",
     "STDOUT_LOG_MIN_PRIORITY": "debug",
-    "AUTOMATIC_BLOCK_REINDEXING": false,
+    "AUTOMATIC_POOLS_UPDATE": false,
     "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json",
     "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
+    "POOLS_UPDATE_DELAY": 604800,
     "AUDIT": false,
-    "RUST_GBT": false,
+    "RUST_GBT": true,
     "LIMIT_GBT": false,
     "CPFP_INDEXING": false,
     "DISK_CACHE_BLOCK_INTERVAL": 6,
@@ -45,7 +46,8 @@
     "PASSWORD": "mempool",
     "TIMEOUT": 60000,
     "COOKIE": false,
-    "COOKIE_PATH": "/path/to/bitcoin/.cookie"
+    "COOKIE_PATH": "/path/to/bitcoin/.cookie",
+    "DEBUG_LOG_PATH": "/path/to/bitcoin/debug.log"
   },
   "ELECTRUM": {
     "HOST": "127.0.0.1",
@@ -59,7 +61,8 @@
     "RETRY_UNIX_SOCKET_AFTER": 30000,
     "REQUEST_TIMEOUT": 10000,
     "FALLBACK_TIMEOUT": 5000,
-    "FALLBACK": []
+    "FALLBACK": [],
+    "MAX_BEHIND_TIP": 2
   },
   "SECOND_CORE_RPC": {
     "HOST": "127.0.0.1",
@@ -139,6 +142,8 @@
     "ENABLED": false,
     "AUDIT": false,
     "AUDIT_START_HEIGHT": 774000,
+    "STATISTICS": false,
+    "STATISTICS_START_TIME": 1481932800,
     "SERVERS": [
       "list",
       "of",
@@ -150,6 +155,10 @@
     "API": "https://mempool.space/api/v1/services",
     "ACCELERATIONS": false
   },
+  "STRATUM": {
+    "ENABLED": false,
+    "API": "http://localhost:1234"
+  },
   "FIAT_PRICE": {
     "ENABLED": true,
     "PAID": false,

backend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mempool-backend",
-  "version": "3.0.0-dev",
+  "version": "3.1.0-dev",
   "description": "Bitcoin mempool visualizer and blockchain explorer backend",
   "license": "GNU Affero General Public License v3.0",
   "homepage": "https://mempool.space",
@@ -39,24 +39,21 @@
     "prettier": "./node_modules/.bin/prettier --write \"src/**/*.{js,ts}\""
   },
   "dependencies": {
-    "@babel/core": "^7.24.0",
     "@mempool/electrum-client": "1.1.9",
     "@types/node": "^18.15.3",
-    "axios": "~1.6.1",
+    "axios": "1.7.2",
     "bitcoinjs-lib": "~6.1.3",
     "crypto-js": "~4.2.0",
-    "express": "~4.19.2",
+    "express": "~4.21.1",
     "maxmind": "~4.3.11",
-    "mysql2": "~3.9.4",
+    "mysql2": "~3.12.0",
     "rust-gbt": "file:./rust-gbt",
-    "redis": "^4.6.6",
+    "redis": "^4.7.0",
     "socks-proxy-agent": "~7.0.0",
     "typescript": "~4.9.3",
-    "ws": "~8.16.0"
+    "ws": "~8.18.0"
   },
   "devDependencies": {
-    "@babel/code-frame": "^7.18.6",
-    "@babel/core": "^7.24.0",
     "@types/compression": "^1.7.2",
     "@types/crypto-js": "^4.1.1",
     "@types/express": "^4.17.17",

backend/src/__fixtures__/mempool-config.template.json

@@ -10,7 +10,7 @@
     "UNIX_SOCKET_PATH": "/mempool/socket/mempool-bitcoin-mainnet",
     "SPAWN_CLUSTER_PROCS": 2,
     "API_URL_PREFIX": "__MEMPOOL_API_URL_PREFIX__",
-    "AUTOMATIC_BLOCK_REINDEXING": false,
+    "AUTOMATIC_POOLS_UPDATE": false,
     "POLL_RATE_MS": 3,
     "CACHE_DIR": "__MEMPOOL_CACHE_DIR__",
     "CACHE_ENABLED": true,
@@ -28,6 +28,7 @@
     "INDEXING_BLOCKS_AMOUNT": 14,
     "POOLS_JSON_TREE_URL": "__MEMPOOL_POOLS_JSON_TREE_URL__",
     "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__",
+    "POOLS_UPDATE_DELAY": 604800,
     "AUDIT": true,
     "RUST_GBT": false,
     "LIMIT_GBT": false,
@@ -46,7 +47,8 @@
     "PASSWORD": "__CORE_RPC_PASSWORD__",
     "TIMEOUT": 1000,
     "COOKIE": false,
-    "COOKIE_PATH": "__CORE_RPC_COOKIE_PATH__"
+    "COOKIE_PATH": "__CORE_RPC_COOKIE_PATH__",
+    "DEBUG_LOG_PATH": "__CORE_RPC_DEBUG_LOG_PATH__"
   },
   "ELECTRUM": {
     "HOST": "__ELECTRUM_HOST__",
@@ -60,7 +62,8 @@
     "RETRY_UNIX_SOCKET_AFTER": 888,
     "REQUEST_TIMEOUT": 10000,
     "FALLBACK_TIMEOUT": 5000,
-    "FALLBACK": []
+    "FALLBACK": [],
+    "MAX_BEHIND_TIP": 2
   },
   "SECOND_CORE_RPC": {
     "HOST": "__SECOND_CORE_RPC_HOST__",
@@ -131,6 +134,8 @@
     "ENABLED": false,
     "AUDIT": false,
     "AUDIT_START_HEIGHT": 774000,
+    "STATISTICS": false,
+    "STATISTICS_START_TIME": 1481932800,
     "SERVERS": []
   },
   "MEMPOOL_SERVICES": {
@@ -146,5 +151,9 @@
     "ENABLED": true,
     "PAID": false,
     "API_KEY": "__MEMPOOL_CURRENCY_API_KEY__"
+  },
+  "STRATUM": {
+    "ENABLED": false,
+    "API": "http://localhost:1234"
   }
 }

backend/src/__tests__/api/common.ts

@@ -1,24 +1,40 @@
 import { Common } from '../../api/common';
-import { MempoolTransactionExtended } from '../../mempool.interfaces';
+import { MempoolTransactionExtended, TransactionExtended } from '../../mempool.interfaces';
 
 const randomTransactions = require('./test-data/transactions-random.json');
 const replacedTransactions = require('./test-data/transactions-replaced.json');
 const rbfTransactions = require('./test-data/transactions-rbfs.json');
+const nonStandardTransactions = require('./test-data/non-standard-txs.json');
 
-describe('Mempool Utils', () => {
-  test('should detect RBF transactions with fast method', () => {
+describe('Common', () => {
+  describe('RBF', () => {
     const newTransactions = rbfTransactions.concat(randomTransactions);
-    const result: { [txid: string]: MempoolTransactionExtended[] } = Common.findRbfTransactions(newTransactions, replacedTransactions);
-    expect(Object.values(result).length).toEqual(2);
-    expect(result).toHaveProperty('7219d95161f3718335991ac6d967d24eedec370908c9879bb1e192e6d797d0a6');
-    expect(result).toHaveProperty('5387881d695d4564d397026dc5f740f816f8390b4b2c5ec8c20309122712a875');
+    test('should detect RBF transactions with fast method', () => {
+      const result: { [txid: string]: { replaced: MempoolTransactionExtended[], replacedBy: TransactionExtended }} = Common.findRbfTransactions(newTransactions, replacedTransactions);
+      expect(Object.values(result).length).toEqual(2);
+      expect(result).toHaveProperty('7219d95161f3718335991ac6d967d24eedec370908c9879bb1e192e6d797d0a6');
+      expect(result).toHaveProperty('5387881d695d4564d397026dc5f740f816f8390b4b2c5ec8c20309122712a875');
+    });
+
+    test('should detect RBF transactions with scalable method', () => {
+      const result: { [txid: string]: { replaced: MempoolTransactionExtended[], replacedBy: TransactionExtended }} = Common.findRbfTransactions(newTransactions, replacedTransactions, true);
+      expect(Object.values(result).length).toEqual(2);
+      expect(result).toHaveProperty('7219d95161f3718335991ac6d967d24eedec370908c9879bb1e192e6d797d0a6');
+      expect(result).toHaveProperty('5387881d695d4564d397026dc5f740f816f8390b4b2c5ec8c20309122712a875');
+    });
   });
 
-  test.only('should detect RBF transactions with scalable method', () => {
-    const newTransactions = rbfTransactions.concat(randomTransactions);
-    const result: { [txid: string]: MempoolTransactionExtended[] } = Common.findRbfTransactions(newTransactions, replacedTransactions, true);
-    expect(Object.values(result).length).toEqual(2);
-    expect(result).toHaveProperty('7219d95161f3718335991ac6d967d24eedec370908c9879bb1e192e6d797d0a6');
-    expect(result).toHaveProperty('5387881d695d4564d397026dc5f740f816f8390b4b2c5ec8c20309122712a875');
+  describe('Mempool Goggles', () => {
+    test('should detect nonstandard transactions', () => {
+      nonStandardTransactions.forEach((tx) => {
+        expect(Common.isNonStandard(tx)).toEqual(true);
+      });
+    });
+  
+    test('should not misclassify as nonstandard transactions', () => {
+      randomTransactions.forEach((tx) => {
+        expect(Common.isNonStandard(tx)).toEqual(false);
+      });
+    });
   });
 });

backend/src/__tests__/api/test-data/non-standard-txs.json

@@ -0,0 +1,52 @@
+[
+  {
+    "txid": "50136231cb7eeeffb17fc41d1cca213426abe5bf3760e3d6421cad0c0edad367",
+    "version": 1,
+    "locktime": 0,
+    "vin": [
+        {
+            "txid": "c7f86fb7b830124057475b282809f3474ef3565daa3de0b599980fb9e84ab019",
+            "vout": 4217,
+            "prevout": {
+                "scriptpubkey": "001466197b5eadd8067ec194a457e1044b6d1fbdd3b3",
+                "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 66197b5eadd8067ec194a457e1044b6d1fbdd3b3",
+                "scriptpubkey_type": "v0_p2wpkh",
+                "scriptpubkey_address": "bc1qvcvhkh4dmqr8asv553t7zpztd50mm5ang4na33",
+                "value": 106
+            },
+            "scriptsig": "",
+            "scriptsig_asm": "",
+            "witness": [
+                "3043021f2af6060a142c6cfd7428adad6a50745d2424813d7ced5c0bbcca85e70de1be022021440ca1c8c3ed49ecd1b64dca6911adcd430c5d3dd60d77ffe0072953999f5b01",
+                "02ead5c34e3d2c506574b562f857576e11380b6ba15d9f0ad7b7303fdaa9c1513d"
+            ],
+            "is_coinbase": false,
+            "sequence": 4294967295
+        }
+    ],
+    "vout": [
+        {
+            "scriptpubkey": "6a023a29",
+            "scriptpubkey_asm": "OP_RETURN OP_PUSHBYTES_2 3a29",
+            "scriptpubkey_type": "op_return",
+            "value": 0
+        },
+        {
+            "scriptpubkey": "6a036d7648",
+            "scriptpubkey_asm": "OP_RETURN OP_PUSHBYTES_3 6d7648",
+            "scriptpubkey_type": "op_return",
+            "value": 0
+        }
+    ],
+    "size": 186,
+    "weight": 420,
+    "sigops": 1,
+    "fee": 106,
+    "status": {
+        "confirmed": true,
+        "block_height": 836361,
+        "block_hash": "0000000000000000000341cc26cda4af82cd25f7063c448772228cbf2836915b",
+        "block_time": 1711448028
+    }
+  }
+]

backend/src/__tests__/api/test-data/transactions-random.json

@@ -273,5 +273,328 @@
         },
         "bestDescendant": null,
         "cpfpChecked": true
+    },
+    {
+        "txid": "20b984492b5264162a4c92c9a34bc7fa08b67d669de7b4c5982ad3cb28aaecf6",
+        "version": 2,
+        "locktime": 0,
+        "vin": [
+            {
+                "txid": "3adda6afd547193793c248e667c2b7dbf26d705003de65e3a25e5be698286aef",
+                "vout": 2,
+                "prevout": {
+                    "scriptpubkey": "0014989cf12774fc705609610c7b9419f2d1c4807644",
+                    "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 989cf12774fc705609610c7b9419f2d1c4807644",
+                    "scriptpubkey_type": "v0_p2wpkh",
+                    "scriptpubkey_address": "bc1qnzw0zfm5l3c9vztpp3aegx0j68zgqajyffr2r6",
+                    "value": 27619
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "304402205d7f1e0d928982645c2bcc4c730c4545c382d6520c2a14eebc71594702cd06b302200511d452ce51c79017536f50acb115eefe7c04506ad12b9307d2b5d56b999beb01",
+                    "03716cb4f0430fe69c596a12c6680c55803150645989b406772838d548cde7cca5"
+                ],
+                "is_coinbase": false,
+                "sequence": 4294967295
+            }
+        ],
+        "vout": [
+            {
+                "scriptpubkey": "6a5d0614c0a2331441",
+                "scriptpubkey_asm": "OP_RETURN OP_PUSHNUM_13 OP_PUSHBYTES_6 14c0a2331441",
+                "scriptpubkey_type": "op_return",
+                "value": 0
+            },
+            {
+                "scriptpubkey": "5114d71c6c3ea7ba7e6ee477a0bfd82c20c78997882c",
+                "scriptpubkey_asm": "OP_PUSHNUM_1 OP_PUSHBYTES_20 d71c6c3ea7ba7e6ee477a0bfd82c20c78997882c",
+                "scriptpubkey_type": "unknown",
+                "scriptpubkey_address": "bc1p6uwxc048hflxaerh5zlastpqc7ye0zpvq7gq2a",
+                "value": 546
+            },
+            {
+                "scriptpubkey": "0014989cf12774fc705609610c7b9419f2d1c4807644",
+                "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 989cf12774fc705609610c7b9419f2d1c4807644",
+                "scriptpubkey_type": "v0_p2wpkh",
+                "scriptpubkey_address": "bc1qnzw0zfm5l3c9vztpp3aegx0j68zgqajyffr2r6",
+                "value": 23073
+            }
+        ],
+        "size": 240,
+        "weight": 633,
+        "sigops": 1,
+        "fee": 4000,
+        "status": {
+            "confirmed": true,
+            "block_height": 848136,
+            "block_hash": "00000000000000000002c69c7a3010fcd596c0c7451c23e7cd1f5e19ebf8ee6d",
+            "block_time": 1718517071
+        }
+    },
+    {
+        "txid": "b10c0000004da5a9d1d9b4ae32e09f0b3e62d21a5cce5428d4ad714fb444eb5d",
+        "version": 1,
+        "locktime": 1231006505,
+        "vin": [
+            {
+                "txid": "d46a24962c1d7bd6e87d80570c6a53413eaf30d7fde7f52347f13645ae53969b",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "41049434a2dd7c5b82df88f578f8d7fd14e8d36513aaa9c003eb5bd6cb56065e44b7e0227139e8a8e68e7de0a4ed32b8c90edc9673b8a7ea541b52f2a22196f7b8cfac",
+                    "scriptpubkey_asm": "OP_PUSHBYTES_65 049434a2dd7c5b82df88f578f8d7fd14e8d36513aaa9c003eb5bd6cb56065e44b7e0227139e8a8e68e7de0a4ed32b8c90edc9673b8a7ea541b52f2a22196f7b8cf OP_CHECKSIG",
+                    "scriptpubkey_type": "p2pk",
+                    "value": 6102
+                },
+                "scriptsig": "473044022004f027ae0b19bb7a7aa8fcdf135f1da769d087342020359ef4099a9f0f0ba4ec02206a83a9b78df3fed89a3b6052e69963e1fb08d8f6d17d945e43b51b5214aa41e601",
+                "scriptsig_asm": "OP_PUSHBYTES_71 3044022004f027ae0b19bb7a7aa8fcdf135f1da769d087342020359ef4099a9f0f0ba4ec02206a83a9b78df3fed89a3b6052e69963e1fb08d8f6d17d945e43b51b5214aa41e601",
+                "is_coinbase": false,
+                "sequence": 20090103
+            },
+            {
+                "txid": "cb9b47ac04023b29fb633a8ef04af351ac9fd74c57c9a2163f683516274767e3",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "76a914bbb1f7d0f7e15ac088af9bafe25aaac1a59832d088ac",
+                    "scriptpubkey_asm": "OP_DUP OP_HASH160 OP_PUSHBYTES_20 bbb1f7d0f7e15ac088af9bafe25aaac1a59832d0 OP_EQUALVERIFY OP_CHECKSIG",
+                    "scriptpubkey_type": "p2pkh",
+                    "scriptpubkey_address": "1J7SZJry7CX4zWdH3P8E8UJjZrhcLEjJ39",
+                    "value": 1913
+                },
+                "scriptsig": "46304302204dc2939be89ab6626457fff40aec2cc4e6213e64bcb4d2c43bf6b49358ff638c021f33d2f8fdf6d54a2c82bb7cddc62becc2cbbaca6fd7f3ec927ea975f29ad8510221028b98707adfd6f468d56c1a6067a6f0c7fef43afbacad45384017f8be93a18d40",
+                "scriptsig_asm": "OP_PUSHBYTES_70 304302204dc2939be89ab6626457fff40aec2cc4e6213e64bcb4d2c43bf6b49358ff638c021f33d2f8fdf6d54a2c82bb7cddc62becc2cbbaca6fd7f3ec927ea975f29ad85102 OP_PUSHBYTES_33 028b98707adfd6f468d56c1a6067a6f0c7fef43afbacad45384017f8be93a18d40",
+                "is_coinbase": false,
+                "sequence": 20081031
+            },
+            {
+                "txid": "cb9b47ac04023b29fb633a8ef04af351ac9fd74c57c9a2163f683516274767e3",
+                "vout": 1,
+                "prevout": {
+                    "scriptpubkey": "52210304e708d258a632ffb128a62ecf5eebd1904e505497d031619513afc8bca7858f2102b9dc03f1133e7cbc7eb311631acc2dbda908fb0f0fae095da2f4dd427f51308a4104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5f53ae",
+                    "scriptpubkey_asm": "OP_PUSHNUM_2 OP_PUSHBYTES_33 0304e708d258a632ffb128a62ecf5eebd1904e505497d031619513afc8bca7858f OP_PUSHBYTES_33 02b9dc03f1133e7cbc7eb311631acc2dbda908fb0f0fae095da2f4dd427f51308a OP_PUSHBYTES_65 04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5f OP_PUSHNUM_3 OP_CHECKMULTISIG",
+                    "scriptpubkey_type": "multisig",
+                    "value": 1971
+                },
+                "scriptsig": "00453042021e4f6ff73d7b304a5cbf3bb7738abb5f81a4af6335962134ce27a1cc45fec702201b95e3acb7db93257b20651cdcb79af66bf0bb86a8ae5b4e0a5df4e3f86787e2033b303802153b78ce563f89a0ed9414f5aa28ad0d96d6795f9c63021f34793e2878497561e7616291ebdda3024b681cdacc8b863b5b0804cd30c2a481",
+                "scriptsig_asm": "OP_0 OP_PUSHBYTES_69 3042021e4f6ff73d7b304a5cbf3bb7738abb5f81a4af6335962134ce27a1cc45fec702201b95e3acb7db93257b20651cdcb79af66bf0bb86a8ae5b4e0a5df4e3f86787e203 OP_PUSHBYTES_59 303802153b78ce563f89a0ed9414f5aa28ad0d96d6795f9c63021f34793e2878497561e7616291ebdda3024b681cdacc8b863b5b0804cd30c2a481",
+                "is_coinbase": false,
+                "sequence": 19750504
+            },
+            {
+                "txid": "45e1cb33599acb071810ccc801b71bd7610865f5b899492946ab1bfbcb61cad6",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "a91419f0b86f61606c6eb51b217698ca7e8bff1e398b87",
+                    "scriptpubkey_asm": "OP_HASH160 OP_PUSHBYTES_20 19f0b86f61606c6eb51b217698ca7e8bff1e398b OP_EQUAL",
+                    "scriptpubkey_type": "p2sh",
+                    "scriptpubkey_address": "344BBtYkhaCXgA7oYSXASUfh4bFieiponG",
+                    "value": 2140
+                },
+                "scriptsig": "00443041021d1313459a48bd1d0628eec635495f793e970729684394f9b814d2b24012022050be6d9918444e283da0136884f8311ec465d0fed2f8d24b75a8485ebdc13aea013a303702153b78ce563f89a0ed9414f5aa28ad0d96d6795f9c63021e78644ba72eab69fefb5fe50700671bfb91dda699f72ffbb325edc6a3c4ef8239303602153b78ce563f89a0ed9414f5aa28ad0d96d6795f9c63021d2c2db104e70720c39af43b6ba3edd930c26e0818aa59ff9c886281d8ba834ced532103e0a220d36f6f7ed5f3f58c279d055707c454135baf18fd00d798fec3cb52dfbc2103cf689db9313b9f7fc0b984dd9cac750be76041b392919b06f6bf94813da34cd421027f8af2eb6e904deddaa60d5af393d430575eb35e4dfd942a8a5882734b078906410411db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5cb2e0eaddfb84ccf9744464f82e160bfa9b8b64f9d4c03f999b8643f656b412a34104ae1a62fe09c5f51b13905f07f06b99a2f7159b2225f374cd378d71302fa28414e7aab37397f554a7df5f142c21c1b7303b8a0626f1baded5c72a704f7e6cd84c55ae",
+                "scriptsig_asm": "OP_0 OP_PUSHBYTES_68 3041021d1313459a48bd1d0628eec635495f793e970729684394f9b814d2b24012022050be6d9918444e283da0136884f8311ec465d0fed2f8d24b75a8485ebdc13aea01 OP_PUSHBYTES_58 303702153b78ce563f89a0ed9414f5aa28ad0d96d6795f9c63021e78644ba72eab69fefb5fe50700671bfb91dda699f72ffbb325edc6a3c4ef82 OP_PUSHBYTES_57 303602153b78ce563f89a0ed9414f5aa28ad0d96d6795f9c63021d2c2db104e70720c39af43b6ba3edd930c26e0818aa59ff9c886281d8ba83 OP_PUSHDATA1 532103e0a220d36f6f7ed5f3f58c279d055707c454135baf18fd00d798fec3cb52dfbc2103cf689db9313b9f7fc0b984dd9cac750be76041b392919b06f6bf94813da34cd421027f8af2eb6e904deddaa60d5af393d430575eb35e4dfd942a8a5882734b078906410411db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5cb2e0eaddfb84ccf9744464f82e160bfa9b8b64f9d4c03f999b8643f656b412a34104ae1a62fe09c5f51b13905f07f06b99a2f7159b2225f374cd378d71302fa28414e7aab37397f554a7df5f142c21c1b7303b8a0626f1baded5c72a704f7e6cd84c55ae",
+                "is_coinbase": false,
+                "sequence": 16,
+                "inner_redeemscript_asm": "OP_PUSHNUM_3 OP_PUSHBYTES_33 03e0a220d36f6f7ed5f3f58c279d055707c454135baf18fd00d798fec3cb52dfbc OP_PUSHBYTES_33 03cf689db9313b9f7fc0b984dd9cac750be76041b392919b06f6bf94813da34cd4 OP_PUSHBYTES_33 027f8af2eb6e904deddaa60d5af393d430575eb35e4dfd942a8a5882734b078906 OP_PUSHBYTES_65 0411db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5cb2e0eaddfb84ccf9744464f82e160bfa9b8b64f9d4c03f999b8643f656b412a3 OP_PUSHBYTES_65 04ae1a62fe09c5f51b13905f07f06b99a2f7159b2225f374cd378d71302fa28414e7aab37397f554a7df5f142c21c1b7303b8a0626f1baded5c72a704f7e6cd84c OP_PUSHNUM_5 OP_CHECKMULTISIG"
+            },
+            {
+                "txid": "cb9b47ac04023b29fb633a8ef04af351ac9fd74c57c9a2163f683516274767e3",
+                "vout": 2,
+                "prevout": {
+                    "scriptpubkey": "a9143b13a1f71c20c799d86bb624b3898c826d6c82da87",
+                    "scriptpubkey_asm": "OP_HASH160 OP_PUSHBYTES_20 3b13a1f71c20c799d86bb624b3898c826d6c82da OP_EQUAL",
+                    "scriptpubkey_type": "p2sh",
+                    "scriptpubkey_address": "375PJxsKRtAq4WoS6u82jvgZW94R8Wx3iH",
+                    "value": 5139
+                },
+                "scriptsig": "1600149b27f072e4b972927c445d1946162a550b0914d8",
+                "scriptsig_asm": "OP_PUSHBYTES_22 00149b27f072e4b972927c445d1946162a550b0914d8",
+                "witness": [
+                    "3040021c23902a01d4c5cff2c33c8bdb778a5aadea78a9a0d6d4db60aaa0fba1022069237d9dbf2db8cff9c260ba71250493682d01a746f4a45c5c7ea386e56d2bc902",
+                    "0240187acd3e2fd3d8e1acffefa85907b6550730c24f78dfd3301c829fc4daf3cc"
+                ],
+                "is_coinbase": false,
+                "sequence": 141,
+                "inner_redeemscript_asm": "OP_0 OP_PUSHBYTES_20 9b27f072e4b972927c445d1946162a550b0914d8"
+            },
+            {
+                "txid": "cb9b47ac04023b29fb633a8ef04af351ac9fd74c57c9a2163f683516274767e3",
+                "vout": 3,
+                "prevout": {
+                    "scriptpubkey": "a914a3c0698f2300c7b2e8107d4c9c988e642110039087",
+                    "scriptpubkey_asm": "OP_HASH160 OP_PUSHBYTES_20 a3c0698f2300c7b2e8107d4c9c988e6421100390 OP_EQUAL",
+                    "scriptpubkey_type": "p2sh",
+                    "scriptpubkey_address": "3GcrZrbUuvE4UtUdSbKTXcRnTqmfMdyMAC",
+                    "value": 3220
+                },
+                "scriptsig": "220020a18160de7291554f349c7d5cbee4ab97fb542e94cf302ce8d7e9747e4188ca75",
+                "scriptsig_asm": "OP_PUSHBYTES_34 0020a18160de7291554f349c7d5cbee4ab97fb542e94cf302ce8d7e9747e4188ca75",
+                "witness": [
+                    "303f021c65aee6696e80be6e14545cfd64b44f17b0514c150eefdb090c0f0bd9021f3fef4aa95c252a225622aba99e4d5af5a6fe40d177acd593e64cf2f8557ccc03",
+                    "03b55c6f0749e0f3e2caeca05f68e3699f1b3c62a550730f704985a6a9aae437a1",
+                    "76a914db865fd920959506111079995f1e4017b489bfe38763ac6721024d560f7f5d28aae5e1a8aa2b7ba615d7fc48e4ea27e5d27336e6a8f5fa0f5c8c7c820120876475527c2103443e8834fa7d79d7b5e95e0e9d0847f6b03ac3ea977979858b4104947fca87ca52ae67a91446c3747322b220fdb925c9802f0e949c1feab99988ac6868"
+                ],
+                "is_coinbase": false,
+                "sequence": 3735928559,
+                "inner_redeemscript_asm": "OP_0 OP_PUSHBYTES_32 a18160de7291554f349c7d5cbee4ab97fb542e94cf302ce8d7e9747e4188ca75",
+                "inner_witnessscript_asm": "OP_DUP OP_HASH160 OP_PUSHBYTES_20 db865fd920959506111079995f1e4017b489bfe3 OP_EQUAL OP_IF OP_CHECKSIG OP_ELSE OP_PUSHBYTES_33 024d560f7f5d28aae5e1a8aa2b7ba615d7fc48e4ea27e5d27336e6a8f5fa0f5c8c OP_SWAP OP_SIZE OP_PUSHBYTES_1 20 OP_EQUAL OP_NOTIF OP_DROP OP_PUSHNUM_2 OP_SWAP OP_PUSHBYTES_33 03443e8834fa7d79d7b5e95e0e9d0847f6b03ac3ea977979858b4104947fca87ca OP_PUSHNUM_2 OP_CHECKMULTISIG OP_ELSE OP_HASH160 OP_PUSHBYTES_20 46c3747322b220fdb925c9802f0e949c1feab999 OP_EQUALVERIFY OP_CHECKSIG OP_ENDIF OP_ENDIF"
+            },
+            {
+                "txid": "cb9b47ac04023b29fb633a8ef04af351ac9fd74c57c9a2163f683516274767e3",
+                "vout": 4,
+                "prevout": {
+                    "scriptpubkey": "0014c0ca6e754e65d3ba59112d7abc33e500c00ecfa7",
+                    "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 c0ca6e754e65d3ba59112d7abc33e500c00ecfa7",
+                    "scriptpubkey_type": "v0_p2wpkh",
+                    "scriptpubkey_address": "bc1qcr9xua2wvhfm5kg394atcvl9qrqqana8rrmy8h",
+                    "value": 17144
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "303e021c11f60486afd0f5d6573603fb2076ef2f676455b92ada257d2f25558a021e317719c946f951d49bf4df4285a618629cd9e554fcbf787c319a0c4dd22601",
+                    "032467f24cc31664f0cf34ff8d5cbb590888ddc1dcfec724a32ae3dd5338b8508e"
+                ],
+                "is_coinbase": false,
+                "sequence": 21000000
+            },
+            {
+                "txid": "637db3928a8fb1b22b81f92dc738ee7637e5b172d650363d0b327429578bd001",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "0020a9530a167fcada672c142ee636dcd171796e69ef8e37aa1f77f35c58edd7a357",
+                    "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_32 a9530a167fcada672c142ee636dcd171796e69ef8e37aa1f77f35c58edd7a357",
+                    "scriptpubkey_type": "v0_p2wsh",
+                    "scriptpubkey_address": "bc1q49fs59nletdxwtq59mnrdhx3w9uku6003cm658mh7dw93mwh5dts2w2kht",
+                    "value": 8149
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "303d021c32f9454db85cb1a4ca63a9883d4347c5e13f3654e884ae44e9efa3c8021d62f07fe452c06b084bc3e09afd3aac4039136549a465533bc1ca66967902",
+                    "01",
+                    "632102fd6db4de50399b2aa086edb23f8e140bbc823d6651e024a0eb871288068789cd67012ab27521034134a2bb35c3f83dab2489d96160741888b8b5589bb694dea6e7bc24486e9c6f68ac"
+                ],
+                "is_coinbase": false,
+                "sequence": 4190024921,
+                "inner_witnessscript_asm": "OP_IF OP_PUSHBYTES_33 02fd6db4de50399b2aa086edb23f8e140bbc823d6651e024a0eb871288068789cd OP_ELSE OP_PUSHBYTES_1 2a OP_CSV OP_DROP OP_PUSHBYTES_33 034134a2bb35c3f83dab2489d96160741888b8b5589bb694dea6e7bc24486e9c6f OP_ENDIF OP_CHECKSIG"
+            },
+            {
+                "txid": "0020db02df125062ebae5bacd189ebff22577b2817c1872be79a0d3ba3982c41",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "512071212ded0ff4c9b1b0c505d8012772e2dbe98a3cae7168377b950fb6b866a849",
+                    "scriptpubkey_asm": "OP_PUSHNUM_1 OP_PUSHBYTES_32 71212ded0ff4c9b1b0c505d8012772e2dbe98a3cae7168377b950fb6b866a849",
+                    "scriptpubkey_type": "v1_p2tr",
+                    "scriptpubkey_address": "bc1pwysjmmg07nymrvx9qhvqzfmjutd7nz3u4ecksdmmj58mdwrx4pysq6m68g",
+                    "value": 9001
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "d822f203827852998cad370232e8c57294540a5da51107fa26cf466bdd2b8b0b3d161999cc80aed8de7386a2bd5d5313aea159a231cc26fa53aaa702b7fa21ed"
+                ],
+                "is_coinbase": false,
+                "sequence": 341
+            },
+            {
+                "txid": "795741ecf9c431b14b1c8d2dd017d3978fd4f6452e91edf416f31ef9971206b4",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "512089ac120a490eee88db5588112f95f88093284c814f07c3ad943a7faefba2271a",
+                    "scriptpubkey_asm": "OP_PUSHNUM_1 OP_PUSHBYTES_32 89ac120a490eee88db5588112f95f88093284c814f07c3ad943a7faefba2271a",
+                    "scriptpubkey_type": "v1_p2tr",
+                    "scriptpubkey_address": "bc1p3xkpyzjfpmhg3k643qgjl90cszfjsnypfuru8tv58fl6a7azyudqkcu66k",
+                    "value": 19953
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "fe6eb715dceffefc067fdc787d250a9a9116682d216f6356ea38fc1f112bd74995faa90315e81981d2c2260b7eaca3c41a16b280362980f0d8faf4c05ebb82c5",
+                    "e34ad0ad33885a473831f8ba8d9339123cb19d0e642e156d8e0d6e2ab2691aedb30e55a35637a806927225e1aa72223d41e59f92c6579b819e7d331a7ada9d2e01",
+                    "2a4861fb4cb951c791bf6c93859ef65abccd90034f91b9b77abb918e13b6fce75d5fa3e2d2f6eeeae105315178c2cb9db2ef238fe89b282f691c06db43bc71ca02",
+                    "fc97bb2be673c3bf388aaf58178ef14d354caf83c92aca8ef1831d619b8511e928f4f5fdea3962067b11e7cecfe094cd0f66a4ea9af9ec836d70d18f2b37df0281",
+                    "a5781a0adaa80ab7f7f164172dd1a1cb127e523daa0d6949aba074a15c589f12dfb8183182afec9230cb7947b7422a4abc1bb78173550d66274ea19f6c9dd92c82",
+                    "",
+                    "",
+                    "205f4237bd7dae576b34abc8a9c6fa4f0e4787c04234ca963e9e96c8f9b67b56d1ac205f4237bd7f93c69403a30c6b641f27ccf5201090152fcf1596474221307831c3ba205ac8ff25ce63564963d1148b84627f614af1f3c77d7caa23adc61264fa5e4996ba20b210c83e6f5b3f866837112d023d9ae8da2a6412168d54968ab87860ab970690ba20d3ee3b7a8b8149122b3c886330b3241538ba4b935c4040f4a73ddab917241bc5ba20cdfabb9d0e5c8f09a83f19e36e100d8f5e882f1b60aa60dacd9e6d072c117bc0ba20aab038c238e95fb54cdd0a6705dc1b1f8d135a9e9b20ab9c7ff96eef0e9bf545ba559c",
+                    "c0b1674191a88ec5cdd733e4240a81803105dc412d6c6708d53ab94fc248f4f5534a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33bf4184fc596403b9d638783cf57adfe4c75c605f6356fbc91338530e9831e9e166f7cf9580f1c2dfb3c4d5d043cdbb128c640e3f20161245aa7372e9666168516a1075db55d416d3ca199f55b6084e2115b9345e16c5cf302fc80e9d5fbf5d48dd5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb46829a3efd3ef04f9153d47a990bd7b048a4b2d213daaa5fb8ed670fb85f13bdbcf54e48e5f5c656b26c3bca14a8c95aa583d07ebe84dde3b7dd4a78f4e4186e713d29c9c0e8e4d2a9790922af73f0b8d51f0bd4bb19940d9cf910ead8fbe85bc9bbb41a757f405890fb0f5856228e23b715702d714d59bf2b1feb70d8b2b4e3e089fdbcf0ef9d8d00f66e47917f67cc5d78aec1ac786e2abb8d2facb4e4790aad6cc455ae816e6cdafdb58d54e35d4f46d860047458eacf1c7405dc634631c570d8d31992805518fd62daa3bdd2a5c4fd2cd3054c9b3dca1d78055e9528cff6adc8f907925d2ebe48765103e6845c06f1f2bb77c6adc1cc002865865eb5cfd5c1cb10c007c60e14f9d087e0291d4d0c7869697c6681d979c6639dbd960792b4d4133e794d097969002ee05d336686fc03c9e15a597c1b9827669460fac9879903637777defed8717c581b4c0509329550e344bdc14ac38f71fc050096887e535c8fd456524104a6674693c29946543f8a0befccce5a352bda55ec8559fc630f5f37393096d97bfee8660f4100ffd61874d62f9a65de9fb6acf740c4c386990ef7373be398c4bdc43709db7398106609eea2a7841aaf3a4fa2000dc18184faa2a7eb5a2af5845a8d3796308ff9840e567b14cf6bb158ff26c999e6f9a1f5448f9aa"
+                ],
+                "is_coinbase": false,
+                "sequence": 342,
+                "inner_witnessscript_asm": "OP_PUSHBYTES_32 5f4237bd7dae576b34abc8a9c6fa4f0e4787c04234ca963e9e96c8f9b67b56d1 OP_CHECKSIG OP_PUSHBYTES_32 5f4237bd7f93c69403a30c6b641f27ccf5201090152fcf1596474221307831c3 OP_CHECKSIGADD OP_PUSHBYTES_32 5ac8ff25ce63564963d1148b84627f614af1f3c77d7caa23adc61264fa5e4996 OP_CHECKSIGADD OP_PUSHBYTES_32 b210c83e6f5b3f866837112d023d9ae8da2a6412168d54968ab87860ab970690 OP_CHECKSIGADD OP_PUSHBYTES_32 d3ee3b7a8b8149122b3c886330b3241538ba4b935c4040f4a73ddab917241bc5 OP_CHECKSIGADD OP_PUSHBYTES_32 cdfabb9d0e5c8f09a83f19e36e100d8f5e882f1b60aa60dacd9e6d072c117bc0 OP_CHECKSIGADD OP_PUSHBYTES_32 aab038c238e95fb54cdd0a6705dc1b1f8d135a9e9b20ab9c7ff96eef0e9bf545 OP_CHECKSIGADD OP_PUSHNUM_5 OP_NUMEQUAL"
+            }
+        ],
+        "vout": [
+            {
+                "scriptpubkey": "210261542eb020b36c1da48e2e607b90a8c1f2ccdbd06eaf5fb4bb0d7cc34293d32aac",
+                "scriptpubkey_asm": "OP_PUSHBYTES_33 0261542eb020b36c1da48e2e607b90a8c1f2ccdbd06eaf5fb4bb0d7cc34293d32a OP_CHECKSIG",
+                "scriptpubkey_type": "p2pk",
+                "value": 576
+            },
+            {
+                "scriptpubkey": "76a9140240539af6c68431e4ce9cc5ef464f12c1741b3c88ac",
+                "scriptpubkey_asm": "OP_DUP OP_HASH160 OP_PUSHBYTES_20 0240539af6c68431e4ce9cc5ef464f12c1741b3c OP_EQUALVERIFY OP_CHECKSIG",
+                "scriptpubkey_type": "p2pkh",
+                "scriptpubkey_address": "1CuQsdrcgcmPvugo3NqEwh1kDcpeEnuFC",
+                "value": 546
+            },
+            {
+                "scriptpubkey": "5121028b45a50f795be0413680036665d17a3eca099648ea80637bc3a70a7d2b52ae2851ae",
+                "scriptpubkey_asm": "OP_PUSHNUM_1 OP_PUSHBYTES_33 028b45a50f795be0413680036665d17a3eca099648ea80637bc3a70a7d2b52ae28 OP_PUSHNUM_1 OP_CHECKMULTISIG",
+                "scriptpubkey_type": "multisig",
+                "value": 582
+            },
+            {
+                "scriptpubkey": "a91449ed2c96e33b6134408af8484508bcc3248c8dbd87",
+                "scriptpubkey_asm": "OP_HASH160 OP_PUSHBYTES_20 49ed2c96e33b6134408af8484508bcc3248c8dbd OP_EQUAL",
+                "scriptpubkey_type": "p2sh",
+                "scriptpubkey_address": "38RuNhSiZiftB6WVnStu5aUz6jXtCDXQZk",
+                "value": 540
+            },
+            {
+                "scriptpubkey": "0014c8e51cf6891c0a2101aecea8cd5ce9bbbfaf7bba",
+                "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 c8e51cf6891c0a2101aecea8cd5ce9bbbfaf7bba",
+                "scriptpubkey_type": "v0_p2wpkh",
+                "scriptpubkey_address": "bc1qerj3ea5frs9zzqdwe65v6h8fhwl677a6s0hxhf",
+                "value": 294
+            },
+            {
+                "scriptpubkey": "0020c485bbb80c4be276e77eac3a983a391cc8b1a1b5f160995a36c3dff18296385a",
+                "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_32 c485bbb80c4be276e77eac3a983a391cc8b1a1b5f160995a36c3dff18296385a",
+                "scriptpubkey_type": "v0_p2wsh",
+                "scriptpubkey_address": "bc1qcjzmhwqvf038dem74safsw3ernytrgd479sfjk3kc00lrq5k8pdqczl83q",
+                "value": 330
+            },
+            {
+                "scriptpubkey": "5120a7a42b268957a06c9de4d7260f1df392ce4d6e7b743f5adc27415ce2afceb3b9",
+                "scriptpubkey_asm": "OP_PUSHNUM_1 OP_PUSHBYTES_32 a7a42b268957a06c9de4d7260f1df392ce4d6e7b743f5adc27415ce2afceb3b9",
+                "scriptpubkey_type": "v1_p2tr",
+                "scriptpubkey_address": "bc1p57jzkf5f27sxe80y6unq780njt8y6mnmwsl44hp8g9ww9t7wkwusv7av76",
+                "value": 330
+            },
+            {
+                "scriptpubkey": "51024e73",
+                "scriptpubkey_asm": "OP_PUSHNUM_1 OP_PUSHBYTES_2 4e73",
+                "scriptpubkey_type": "unknown",
+                "scriptpubkey_address": "bc1pfeessrawgf",
+                "value": 240
+            },
+            {
+                "scriptpubkey": "6a224e6f7420796f757220696e707574732c206e6f7420796f7572206f7574707574732e005152535455565758595a5b5c5d5e5f60",
+                "scriptpubkey_asm": "OP_RETURN OP_PUSHBYTES_34 4e6f7420796f757220696e707574732c206e6f7420796f7572206f7574707574732e OP_0 OP_PUSHNUM_1 OP_PUSHNUM_2 OP_PUSHNUM_3 OP_PUSHNUM_4 OP_PUSHNUM_5 OP_PUSHNUM_6 OP_PUSHNUM_7 OP_PUSHNUM_8 OP_PUSHNUM_9 OP_PUSHNUM_10 OP_PUSHNUM_11 OP_PUSHNUM_12 OP_PUSHNUM_13 OP_PUSHNUM_14 OP_PUSHNUM_15 OP_PUSHNUM_16",
+                "scriptpubkey_type": "op_return",
+                "value": 0
+            }
+        ],
+        "size": 3500,
+        "weight": 8186,
+        "sigops": 115,
+        "fee": 71294,
+        "status": {
+            "confirmed": true,
+            "block_height": 850000,
+            "block_hash": "00000000000000000002a0b5db2a7f8d9087464c2586b546be7bce8eb53b8187",
+            "block_time": 1719689674
+        }
     }
 ]

backend/src/__tests__/config.test.ts

@@ -23,7 +23,7 @@ describe('Mempool Backend Config', () => {
         UNIX_SOCKET_PATH: '',
         SPAWN_CLUSTER_PROCS: 0,
         API_URL_PREFIX: '/api/v1/',
-        AUTOMATIC_BLOCK_REINDEXING: false,
+        AUTOMATIC_POOLS_UPDATE: false,
         POLL_RATE_MS: 2000,
         CACHE_DIR: './cache',
         CACHE_ENABLED: true,
@@ -41,8 +41,9 @@ describe('Mempool Backend Config', () => {
         STDOUT_LOG_MIN_PRIORITY: 'debug',
         POOLS_JSON_TREE_URL: 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
         POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json',
+        POOLS_UPDATE_DELAY: 604800,
         AUDIT: false,
-        RUST_GBT: false,
+        RUST_GBT: true,
         LIMIT_GBT: false,
         CPFP_INDEXING: false,
         MAX_BLOCKS_BULK_QUERY: 0,
@@ -63,6 +64,7 @@ describe('Mempool Backend Config', () => {
         REQUEST_TIMEOUT: 10000,
         FALLBACK_TIMEOUT: 5000,
         FALLBACK: [],
+        MAX_BEHIND_TIP: 2,
        });
 
       expect(config.CORE_RPC).toStrictEqual({
@@ -72,7 +74,8 @@ describe('Mempool Backend Config', () => {
         PASSWORD: 'mempool',
         TIMEOUT: 60000,
         COOKIE: false,
-        COOKIE_PATH: '/bitcoin/.cookie'
+        COOKIE_PATH: '/bitcoin/.cookie',
+        DEBUG_LOG_PATH: '',
       });
 
       expect(config.SECOND_CORE_RPC).toStrictEqual({
@@ -135,6 +138,8 @@ describe('Mempool Backend Config', () => {
         ENABLED: false,
         AUDIT: false,
         AUDIT_START_HEIGHT: 774000,
+        STATISTICS: false,
+        STATISTICS_START_TIME: 1481932800,
         SERVERS: []
       });
 
@@ -154,6 +159,11 @@ describe('Mempool Backend Config', () => {
         PAID: false,
         API_KEY: '',
       });
+
+      expect(config.STRATUM).toStrictEqual({
+        ENABLED: false,
+        API: 'http://localhost:1234',
+      });
     });
   });
 

backend/src/__tests__/gbt/gbt-tests.ts

@@ -13,7 +13,7 @@ const vectorBuffer: Buffer = fs.readFileSync(path.join(__dirname, './', './test-
 
 describe('Rust GBT', () => {
   test('should produce the same template as getBlockTemplate from Bitcoin Core', async () => {
-    const rustGbt = new GbtGenerator();
+    const rustGbt = new GbtGenerator(4_000_000, 8);
     const { mempool, maxUid } = mempoolFromArrayBuffer(vectorBuffer.buffer);
     const result = await rustGbt.make(mempool, [], maxUid);
 

backend/src/api/about.routes.ts

@@ -70,7 +70,7 @@ class AboutRoutes {
           res.status(500).end();
         }
       })
-      .get(config.MEMPOOL.API_URL_PREFIX + 'services/account/images/:username', async (req, res) => {
+      .get(config.MEMPOOL.API_URL_PREFIX + 'services/account/images/:username/:md5', async (req, res) => {
         const url = `${config.MEMPOOL_SERVICES.API}/${req.originalUrl.replace('/api/v1/services/', '')}`;
         try {
           const response = await axios.get(url, { responseType: 'stream', timeout: 10000 });

backend/src/api/acceleration/acceleration.routes.ts

@@ -14,6 +14,7 @@ class AccelerationRoutes {
       .get(config.MEMPOOL.API_URL_PREFIX + 'services/accelerator/accelerations/history', this.$getAcceleratorAccelerationsHistory.bind(this))
       .get(config.MEMPOOL.API_URL_PREFIX + 'services/accelerator/accelerations/history/aggregated', this.$getAcceleratorAccelerationsHistoryAggregated.bind(this))
       .get(config.MEMPOOL.API_URL_PREFIX + 'services/accelerator/accelerations/stats', this.$getAcceleratorAccelerationsStats.bind(this))
+      .post(config.MEMPOOL.API_URL_PREFIX + 'services/accelerator/estimate', this.$getAcceleratorEstimate.bind(this))
     ;
   }
 
@@ -64,6 +65,20 @@ class AccelerationRoutes {
       res.status(500).end();
     }
   }
+
+  private async $getAcceleratorEstimate(req: Request, res: Response): Promise<void> {
+    const url = `${config.MEMPOOL_SERVICES.API}/${req.originalUrl.replace('/api/v1/services/', '')}`;
+    try {
+      const response = await axios.post(url, req.body, { responseType: 'stream', timeout: 10000 });
+      for (const key in response.headers) {
+        res.setHeader(key, response.headers[key]);
+      }
+      response.data.pipe(res);
+    } catch (e) {
+      logger.err(`Unable to get acceleration estimate from ${url} in $getAcceleratorEstimate(), ${e}`, this.tag);
+      res.status(500).end();
+    }
+  }
 }
 
 export default new AccelerationRoutes();

backend/src/api/acceleration/acceleration.ts

@@ -1,15 +1,14 @@
 import logger from '../../logger';
 import { MempoolTransactionExtended } from '../../mempool.interfaces';
-import { IEsploraApi } from '../bitcoin/esplora-api.interface';
+import { GraphTx, getSameBlockRelatives, initializeRelatives, makeBlockTemplate, mempoolComparator, removeAncestors, setAncestorScores } from '../mini-miner';
 
 const BLOCK_WEIGHT_UNITS = 4_000_000;
-const BLOCK_SIGOPS = 80_000;
 const MAX_RELATIVE_GRAPH_SIZE = 200;
 const BID_BOOST_WINDOW = 40_000;
 const BID_BOOST_MIN_OFFSET = 10_000;
 const BID_BOOST_MAX_OFFSET = 400_000;
 
-type Acceleration = {
+export type Acceleration = {
   txid: string;
   max_bid: number;
 };
@@ -28,31 +27,6 @@ export interface AccelerationInfo {
   cost: number; // additional cost to accelerate ((cost + txSummary.effectiveFee) / txSummary.effectiveVsize) >= targetFeeRate
 }
 
-interface GraphTx {
-  txid: string;
-  vsize: number;
-  weight: number;
-  fees: {
-    base: number; // in sats
-  };
-  depends: string[];
-  spentby: string[];
-}
-
-interface MempoolTx extends GraphTx {
-  ancestorcount: number;
-  ancestorsize: number;
-  fees: { // in sats
-    base: number;
-    ancestor: number;
-  };
-
-  ancestors: Map<string, MempoolTx>,
-  ancestorRate: number;
-  individualRate: number;
-  score: number;
-}
-
 class AccelerationCosts {
   /**
    * Takes a list of accelerations and verbose block data
@@ -61,7 +35,7 @@ class AccelerationCosts {
    * @param accelerationsx
    * @param verboseBlock
    */
-  public calculateBoostRate(accelerations: Acceleration[], blockTxs: IEsploraApi.Transaction[]): number {
+  public calculateBoostRate(accelerations: Acceleration[], blockTxs: MempoolTransactionExtended[]): number {
     // Run GBT ourselves to calculate accurate effective fee rates
     // the list of transactions comes from a mined block, so we already know everything fits within consensus limits
     const template = makeBlockTemplate(blockTxs, accelerations, 1, Infinity, Infinity);
@@ -170,108 +144,28 @@ class AccelerationCosts {
   /**
    * Takes an accelerated mined txid and a target rate
    * Returns the total vsize, fees and acceleration cost (in sats) of the tx and all same-block ancestors
-   * 
-   * @param txid 
-   * @param medianFeeRate 
+   *
+   * @param txid
+   * @param medianFeeRate
    */
   public getAccelerationInfo(tx: MempoolTransactionExtended, targetFeeRate: number, transactions: MempoolTransactionExtended[]): AccelerationInfo {
     // Get same-block transaction ancestors
-    const allRelatives = this.getSameBlockRelatives(tx, transactions);
-    const relativesMap = this.initializeRelatives(allRelatives);
-    const rootTx = relativesMap.get(tx.txid) as MempoolTx;
+    const allRelatives = getSameBlockRelatives(tx, transactions);
+    const relativesMap = initializeRelatives(allRelatives);
+    const rootTx = relativesMap.get(tx.txid) as GraphTx;
 
     // Calculate cost to boost
     return this.calculateAccelerationAncestors(rootTx, relativesMap, targetFeeRate);
   }
 
-  /**
-   * Takes a raw transaction, and builds a graph of same-block relatives,
-   * and returns as a MempoolTx
-   * 
-   * @param tx 
-   */
-  private getSameBlockRelatives(tx: MempoolTransactionExtended, transactions: MempoolTransactionExtended[]): Map<string, GraphTx> {
-    const blockTxs = new Map<string, MempoolTransactionExtended>(); // map of txs in this block
-    const spendMap = new Map<string, string>(); // map of outpoints to spending txids
-    for (const tx of transactions) {
-      blockTxs.set(tx.txid, tx);
-      for (const vin of tx.vin) {
-        spendMap.set(`${vin.txid}:${vin.vout}`, tx.txid);
-      }
-    }
-
-    const relatives: Map<string, GraphTx> = new Map();
-    const stack: string[] = [tx.txid];
-
-    // build set of same-block ancestors
-    while (stack.length > 0) {
-      const nextTxid = stack.pop();
-      const nextTx = nextTxid ? blockTxs.get(nextTxid) : null;
-      if (!nextTx || relatives.has(nextTx.txid)) {
-        continue;
-      }
-
-      const mempoolTx = this.convertToGraphTx(nextTx);
-
-      mempoolTx.fees.base = nextTx.fee || 0;
-      mempoolTx.depends = nextTx.vin.map(vin => vin.txid).filter(inTxid => inTxid && blockTxs.has(inTxid)) as string[];
-      mempoolTx.spentby = nextTx.vout.map((vout, index) => spendMap.get(`${nextTx.txid}:${index}`)).filter(outTxid => outTxid && blockTxs.has(outTxid)) as string[];
-
-      for (const txid of [...mempoolTx.depends, ...mempoolTx.spentby]) {
-        if (txid) {
-          stack.push(txid);
-        }
-      }
-
-      relatives.set(mempoolTx.txid, mempoolTx);
-    }
-
-    return relatives;
-  }
-
-  /**
-   * Takes a raw transaction and converts it to MempoolTx format
-   * fee and ancestor data is initialized with dummy/null values
-   * 
-   * @param tx 
-   */
-  private convertToGraphTx(tx: MempoolTransactionExtended): GraphTx {
-    return {
-      txid: tx.txid,
-      vsize: Math.ceil(tx.weight / 4),
-      weight: tx.weight,
-      fees: {
-        base: 0, // dummy
-      },
-      depends: [], // dummy
-      spentby: [], //dummy
-    };
-  }
-
-  private convertGraphToMempoolTx(tx: GraphTx): MempoolTx {
-    return {
-      ...tx,
-      fees: {
-        base: tx.fees.base,
-        ancestor: tx.fees.base,
-      },
-      ancestorcount: 1,
-      ancestorsize: Math.ceil(tx.weight / 4),
-      ancestors: new Map<string, MempoolTx>(),
-      ancestorRate: 0,
-      individualRate: 0,
-      score: 0,
-    };
-  }
-
   /**
    * Given a root transaction, a list of in-mempool ancestors, and a target fee rate,
    * Calculate the minimum set of transactions to fee-bump, their total vsize + fees
-   * 
+   *
    * @param tx
    * @param ancestors
    */
-  private calculateAccelerationAncestors(tx: MempoolTx, relatives: Map<string, MempoolTx>, targetFeeRate: number): AccelerationInfo {
+  private calculateAccelerationAncestors(tx: GraphTx, relatives: Map<string, GraphTx>, targetFeeRate: number): AccelerationInfo {
     // add root tx to the ancestor map
     relatives.set(tx.txid, tx);
 
@@ -283,12 +177,12 @@ class AccelerationCosts {
     });
 
     // Initialize individual & ancestor fee rates
-    relatives.forEach(entry => this.setAncestorScores(entry));
+    relatives.forEach(entry => setAncestorScores(entry));
 
     // Sort by descending ancestor score
-    let sortedRelatives = Array.from(relatives.values()).sort(this.mempoolComparator);
+    let sortedRelatives = Array.from(relatives.values()).sort(mempoolComparator);
 
-    let includedInCluster: Map<string, MempoolTx> | null = null;
+    let includedInCluster: Map<string, GraphTx> | null = null;
 
     // While highest score >= targetFeeRate
     let maxIterations = MAX_RELATIVE_GRAPH_SIZE;
@@ -297,17 +191,17 @@ class AccelerationCosts {
       // Grab the highest scoring entry
       const best = sortedRelatives.shift();
       if (best) {
-        const cluster = new Map<string, MempoolTx>(best.ancestors?.entries() || []);
+        const cluster = new Map<string, GraphTx>(best.ancestors?.entries() || []);
         if (best.ancestors.has(tx.txid)) {
           includedInCluster = cluster;
         }
         cluster.set(best.txid, best);
         // Remove this cluster (it already pays over the target rate, so doesn't need to be boosted)
         // and update scores, ancestor totals and dependencies for the survivors
-        this.removeAncestors(cluster, relatives);
+        removeAncestors(cluster, relatives);
 
         // re-sort
-        sortedRelatives = Array.from(relatives.values()).sort(this.mempoolComparator);
+        sortedRelatives = Array.from(relatives.values()).sort(mempoolComparator);
       }
     }
 
@@ -345,394 +239,6 @@ class AccelerationCosts {
       nextBlockFee: Math.ceil(tx.ancestorsize * targetFeeRate),
     };
   }
-
-  /**
-   * Recursively traverses an in-mempool dependency graph, and sets a Map of in-mempool ancestors
-   * for each transaction.
-   * 
-   * @param tx 
-   * @param all 
-   */
-  private setAncestors(tx: MempoolTx, all: Map<string, MempoolTx>, visited: Map<string, Map<string, MempoolTx>>, depth: number = 0): Map<string, MempoolTx> {
-    // sanity check for infinite recursion / too many ancestors (should never happen)
-    if (depth >= 100) {
-      logger.warn('acceleration dependency calculation failed: setAncestors reached depth of 100, unable to proceed', `Accelerator`);
-      throw new Error('invalid_tx_dependencies');
-    }
-    
-    // initialize the ancestor map for this tx
-    tx.ancestors = new Map<string, MempoolTx>();
-    tx.depends.forEach(parentId => {
-      const parent = all.get(parentId);
-      if (parent) {
-        // add the parent
-        tx.ancestors?.set(parentId, parent);
-        // check for a cached copy of this parent's ancestors
-        let ancestors = visited.get(parent.txid);
-        if (!ancestors) {
-          // recursively fetch the parent's ancestors
-          ancestors = this.setAncestors(parent, all, visited, depth + 1);
-        }
-        // and add to this tx's map
-        ancestors.forEach((ancestor, ancestorId) => {
-          tx.ancestors?.set(ancestorId, ancestor);
-        });
-      }
-    });
-    visited.set(tx.txid, tx.ancestors);
-
-    return tx.ancestors;
-  }
-
-  /**
-   * Efficiently sets a Map of in-mempool ancestors for each member of an expanded relative graph
-   * by running setAncestors on each leaf, and caching intermediate results.
-   * then initializes ancestor data for each transaction
-   * 
-   * @param all 
-   */
-  private initializeRelatives(all: Map<string, GraphTx>): Map<string, MempoolTx> {
-    const mempoolTxs = new Map<string, MempoolTx>();
-    all.forEach(entry => {
-      mempoolTxs.set(entry.txid, this.convertGraphToMempoolTx(entry));
-    });
-    const visited: Map<string, Map<string, MempoolTx>> = new Map();
-    const leaves: MempoolTx[] = Array.from(mempoolTxs.values()).filter(entry => entry.spentby.length === 0);
-    for (const leaf of leaves) {
-      this.setAncestors(leaf, mempoolTxs, visited);
-    }
-    mempoolTxs.forEach(entry => {
-      entry.ancestors?.forEach(ancestor => {
-        entry.ancestorcount++;
-        entry.ancestorsize += ancestor.vsize;
-        entry.fees.ancestor += ancestor.fees.base;
-      });
-      this.setAncestorScores(entry);
-    });
-    return mempoolTxs;
-  }
-
-  /**
-   * Remove a cluster of transactions from an in-mempool dependency graph
-   * and update the survivors' scores and ancestors
-   * 
-   * @param cluster 
-   * @param ancestors 
-   */
-  private removeAncestors(cluster: Map<string, MempoolTx>, all: Map<string, MempoolTx>): void {
-    // remove
-    cluster.forEach(tx => {
-      all.delete(tx.txid);
-    });
-
-    // update survivors
-    all.forEach(tx => {
-      cluster.forEach(remove => {
-        if (tx.ancestors?.has(remove.txid)) {
-          // remove as dependency
-          tx.ancestors.delete(remove.txid);
-          tx.depends = tx.depends.filter(parent => parent !== remove.txid);
-          // update ancestor sizes and fees
-          tx.ancestorsize -= remove.vsize;
-          tx.fees.ancestor -= remove.fees.base;
-        }
-      });
-      // recalculate fee rates
-      this.setAncestorScores(tx);
-    });
-  }
-
-  /**
-   * Take a mempool transaction, and set the fee rates and ancestor score
-   * 
-   * @param tx 
-   */
-  private setAncestorScores(tx: MempoolTx): void {
-    tx.individualRate = tx.fees.base / tx.vsize;
-    tx.ancestorRate = tx.fees.ancestor / tx.ancestorsize;
-    tx.score = Math.min(tx.individualRate, tx.ancestorRate);
-  }
-
-  // Sort by descending score
-  private mempoolComparator(a, b): number {
-    return b.score - a.score;
-  }
 }
 
-export default new AccelerationCosts;
-
-interface TemplateTransaction {
-  txid: string;
-  order: number;
-  weight: number;
-  adjustedVsize: number; // sigop-adjusted vsize, rounded up to the nearest integer
-  sigops: number;
-  fee: number;
-  feeDelta: number;
-  ancestors: string[];
-  cluster: string[];
-  effectiveFeePerVsize: number;
-}
-
-interface MinerTransaction extends TemplateTransaction {
-  inputs: string[];
-  feePerVsize: number;
-  relativesSet: boolean;
-  ancestorMap: Map<string, MinerTransaction>;
-  children: Set<MinerTransaction>;
-  ancestorFee: number;
-  ancestorVsize: number;
-  ancestorSigops: number;
-  score: number;
-  used: boolean;
-  modified: boolean;
-  dependencyRate: number;
-}
-
-/*
-* Build a block using an approximation of the transaction selection algorithm from Bitcoin Core
-* (see BlockAssembler in https://github.com/bitcoin/bitcoin/blob/master/src/node/miner.cpp)
-*/
-export function makeBlockTemplate(candidates: IEsploraApi.Transaction[], accelerations: Acceleration[], maxBlocks: number = 8, weightLimit: number = BLOCK_WEIGHT_UNITS, sigopLimit: number = BLOCK_SIGOPS): TemplateTransaction[] {
-  const auditPool: Map<string, MinerTransaction> = new Map();
-  const mempoolArray: MinerTransaction[] = [];
-  
-  candidates.forEach(tx => {
-    // initializing everything up front helps V8 optimize property access later
-    const adjustedVsize = Math.ceil(Math.max(tx.weight / 4, 5 * (tx.sigops || 0)));
-    const feePerVsize = (tx.fee / adjustedVsize);
-    auditPool.set(tx.txid, {
-      txid: tx.txid,
-      order: txidToOrdering(tx.txid),
-      fee: tx.fee,
-      feeDelta: 0,
-      weight: tx.weight,
-      adjustedVsize,
-      feePerVsize: feePerVsize,
-      effectiveFeePerVsize: feePerVsize,
-      dependencyRate: feePerVsize,
-      sigops: tx.sigops || 0,
-      inputs: (tx.vin?.map(vin => vin.txid) || []) as string[],
-      relativesSet: false,
-      ancestors: [],
-      cluster: [],
-      ancestorMap: new Map<string, MinerTransaction>(),
-      children: new Set<MinerTransaction>(),
-      ancestorFee: 0,
-      ancestorVsize: 0,
-      ancestorSigops: 0,
-      score: 0,
-      used: false,
-      modified: false,
-    });
-    mempoolArray.push(auditPool.get(tx.txid) as MinerTransaction);
-  });
-
-  // set accelerated effective fee
-  for (const acceleration of accelerations) {
-    const tx = auditPool.get(acceleration.txid);
-    if (tx) {
-      tx.feeDelta = acceleration.max_bid;
-      tx.feePerVsize = ((tx.fee + tx.feeDelta) / tx.adjustedVsize);
-      tx.effectiveFeePerVsize = tx.feePerVsize;
-      tx.dependencyRate = tx.feePerVsize;
-    }
-  }
-
-  // Build relatives graph & calculate ancestor scores
-  for (const tx of mempoolArray) {
-    if (!tx.relativesSet) {
-      setRelatives(tx, auditPool);
-    }
-  }
-
-  // Sort by descending ancestor score
-  mempoolArray.sort(priorityComparator);
-
-  // Build blocks by greedily choosing the highest feerate package
-  // (i.e. the package rooted in the transaction with the best ancestor score)
-  const blocks: number[][] = [];
-  let blockWeight = 0;
-  let blockSigops = 0;
-  const transactions: MinerTransaction[] = [];
-  let modified: MinerTransaction[] = [];
-  const overflow: MinerTransaction[] = [];
-  let failures = 0;
-  while (mempoolArray.length || modified.length) {
-    // skip invalid transactions
-    while (mempoolArray[0].used || mempoolArray[0].modified) {
-      mempoolArray.shift();
-    }
-
-    // Select best next package
-    let nextTx;
-    const nextPoolTx = mempoolArray[0];
-    const nextModifiedTx = modified[0];
-    if (nextPoolTx && (!nextModifiedTx || (nextPoolTx.score || 0) > (nextModifiedTx.score || 0))) {
-      nextTx = nextPoolTx;
-      mempoolArray.shift();
-    } else {
-      modified.shift();
-      if (nextModifiedTx) {
-        nextTx = nextModifiedTx;
-      }
-    }
-
-    if (nextTx && !nextTx?.used) {
-      // Check if the package fits into this block
-      if (blocks.length >= (maxBlocks - 1) || ((blockWeight + (4 * nextTx.ancestorVsize) < weightLimit) && (blockSigops + nextTx.ancestorSigops <= sigopLimit))) {
-        const ancestors: MinerTransaction[] = Array.from(nextTx.ancestorMap.values());
-        // sort ancestors by dependency graph (equivalent to sorting by ascending ancestor count)
-        const sortedTxSet = [...ancestors.sort((a, b) => { return (a.ancestorMap.size || 0) - (b.ancestorMap.size || 0); }), nextTx];
-        const clusterTxids = sortedTxSet.map(tx => tx.txid);
-        const effectiveFeeRate = Math.min(nextTx.dependencyRate || Infinity, nextTx.ancestorFee / nextTx.ancestorVsize);
-        const used: MinerTransaction[] = [];
-        while (sortedTxSet.length) {
-          const ancestor = sortedTxSet.pop();
-          if (!ancestor) {
-            continue;
-          }
-          ancestor.used = true;
-          ancestor.usedBy = nextTx.txid;
-          // update this tx with effective fee rate & relatives data
-          if (ancestor.effectiveFeePerVsize !== effectiveFeeRate) {
-            ancestor.effectiveFeePerVsize = effectiveFeeRate;
-          }
-          ancestor.cluster = clusterTxids;
-          transactions.push(ancestor);
-          blockWeight += ancestor.weight;
-          blockSigops += ancestor.sigops;
-          used.push(ancestor);
-        }
-
-        // remove these as valid package ancestors for any descendants remaining in the mempool
-        if (used.length) {
-          used.forEach(tx => {
-            modified = updateDescendants(tx, auditPool, modified, effectiveFeeRate);
-          });
-        }
-
-        failures = 0;
-      } else {
-        // hold this package in an overflow list while we check for smaller options
-        overflow.push(nextTx);
-        failures++;
-      }
-    }
-
-    // this block is full
-    const exceededPackageTries = failures > 1000 && blockWeight > (weightLimit - 4000);
-    const queueEmpty = !mempoolArray.length && !modified.length;
-
-    if (exceededPackageTries || queueEmpty) {
-      break;
-    }
-  }
-
-  for (const tx of transactions) {
-    tx.ancestors = Object.values(tx.ancestorMap);
-  }
-
-  return transactions;
-}
-
-// traverse in-mempool ancestors
-// recursion unavoidable, but should be limited to depth < 25 by mempool policy
-function setRelatives(
-  tx: MinerTransaction,
-  mempool: Map<string, MinerTransaction>,
-): void {
-  for (const parent of tx.inputs) {
-    const parentTx = mempool.get(parent);
-    if (parentTx && !tx.ancestorMap?.has(parent)) {
-      tx.ancestorMap.set(parent, parentTx);
-      parentTx.children.add(tx);
-      // visit each node only once
-      if (!parentTx.relativesSet) {
-        setRelatives(parentTx, mempool);
-      }
-      parentTx.ancestorMap.forEach((ancestor) => {
-        tx.ancestorMap.set(ancestor.txid, ancestor);
-      });
-    }
-  };
-  tx.ancestorFee = (tx.fee + tx.feeDelta);
-  tx.ancestorVsize = tx.adjustedVsize || 0;
-  tx.ancestorSigops = tx.sigops || 0;
-  tx.ancestorMap.forEach((ancestor) => {
-    tx.ancestorFee += (ancestor.fee + ancestor.feeDelta);
-    tx.ancestorVsize += ancestor.adjustedVsize;
-    tx.ancestorSigops += ancestor.sigops;
-  });
-  tx.score = tx.ancestorFee / tx.ancestorVsize;
-  tx.relativesSet = true;
-}
-
-// iterate over remaining descendants, removing the root as a valid ancestor & updating the ancestor score
-// avoids recursion to limit call stack depth
-function updateDescendants(
-  rootTx: MinerTransaction,
-  mempool: Map<string, MinerTransaction>,
-  modified: MinerTransaction[],
-  clusterRate: number,
-): MinerTransaction[] {
-  const descendantSet: Set<MinerTransaction> = new Set();
-  // stack of nodes left to visit
-  const descendants: MinerTransaction[] = [];
-  let descendantTx: MinerTransaction | undefined;
-  rootTx.children.forEach(childTx => {
-    if (!descendantSet.has(childTx)) {
-      descendants.push(childTx);
-      descendantSet.add(childTx);
-    }
-  });
-  while (descendants.length) {
-    descendantTx = descendants.pop();
-    if (descendantTx && descendantTx.ancestorMap && descendantTx.ancestorMap.has(rootTx.txid)) {
-      // remove tx as ancestor
-      descendantTx.ancestorMap.delete(rootTx.txid);
-      descendantTx.ancestorFee -= (rootTx.fee + rootTx.feeDelta);
-      descendantTx.ancestorVsize -= rootTx.adjustedVsize;
-      descendantTx.ancestorSigops -= rootTx.sigops;
-      descendantTx.score = descendantTx.ancestorFee / descendantTx.ancestorVsize;
-      descendantTx.dependencyRate = descendantTx.dependencyRate ? Math.min(descendantTx.dependencyRate, clusterRate) : clusterRate;
-
-      if (!descendantTx.modified) {
-        descendantTx.modified = true;
-        modified.push(descendantTx);
-      }
-
-      // add this node's children to the stack
-      descendantTx.children.forEach(childTx => {
-        // visit each node only once
-        if (!descendantSet.has(childTx)) {
-          descendants.push(childTx);
-          descendantSet.add(childTx);
-        }
-      });
-    }
-  }
-  // return new, resorted modified list
-  return modified.sort(priorityComparator);
-}
-
-// Used to sort an array of MinerTransactions by descending ancestor score
-function priorityComparator(a: MinerTransaction, b: MinerTransaction): number {
-  if (b.score === a.score) {
-    // tie-break by txid for stability
-    return a.order - b.order;
-  } else {
-    return b.score - a.score;
-  }
-}
-
-// returns the most significant 4 bytes of the txid as an integer
-function txidToOrdering(txid: string): number {
-  return parseInt(
-    txid.substring(62, 64) +
-      txid.substring(60, 62) +
-      txid.substring(58, 60) +
-      txid.substring(56, 58),
-    16
-  );
-}
+export default new AccelerationCosts;

backend/src/api/audit.ts

@@ -2,24 +2,28 @@ import config from '../config';
 import logger from '../logger';
 import { MempoolTransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces';
 import rbfCache from './rbf-cache';
+import transactionUtils from './transaction-utils';
 
 const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners
 
 class Audit {
-  auditBlock(transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended }, useAccelerations: boolean = false)
-   : { censored: string[], added: string[], prioritized: string[], fresh: string[], sigop: string[], fullrbf: string[], accelerated: string[], score: number, similarity: number } {
+  auditBlock(height: number, transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended })
+   : { unseen: string[], censored: string[], added: string[], prioritized: string[], fresh: string[], sigop: string[], fullrbf: string[], accelerated: string[], score: number, similarity: number } {
     if (!projectedBlocks?.[0]?.transactionIds || !mempool) {
-      return { censored: [], added: [], prioritized: [], fresh: [], sigop: [], fullrbf: [], accelerated: [], score: 1, similarity: 1 };
+      return { unseen: [], censored: [], added: [], prioritized: [], fresh: [], sigop: [], fullrbf: [], accelerated: [], score: 1, similarity: 1 };
     }
 
     const matches: string[] = []; // present in both mined block and template
     const added: string[] = []; // present in mined block, not in template
-    const prioritized: string[] = [] // present in the mined block, not in the template, but further down in the mempool
+    const unseen: string[] = []; // present in the mined block, not in our mempool
+    let prioritized: string[] = []; // higher in the block than would be expected by in-band feerate alone
+    let deprioritized: string[] = []; // lower in the block than would be expected by in-band feerate alone
     const fresh: string[] = []; // missing, but firstSeen or lastBoosted within PROPAGATION_MARGIN
     const rbf: string[] = []; // either missing or present, and either part of a full-rbf replacement, or a conflict with the mined block
     const accelerated: string[] = []; // prioritized by the mempool accelerator
     const isCensored = {}; // missing, without excuse
     const isDisplaced = {};
+    const isAccelerated = {};
     let displacedWeight = 0;
     let matchedWeight = 0;
     let projectedWeight = 0;
@@ -32,6 +36,7 @@ class Audit {
       inBlock[tx.txid] = tx;
       if (mempool[tx.txid] && mempool[tx.txid].acceleration) {
         accelerated.push(tx.txid);
+        isAccelerated[tx.txid] = true;
       }
     }
     // coinbase is always expected
@@ -75,10 +80,6 @@ class Audit {
     let failures = 0;
     let blockIndex = 1;
     while (projectedBlocks[blockIndex] && failures < 500) {
-      if (index >= projectedBlocks[blockIndex].transactionIds.length) {
-        index = 0;
-        blockIndex++;
-      }
       const txid = projectedBlocks[blockIndex].transactionIds[index];
       const tx = mempool[txid];
       if (tx) {
@@ -102,6 +103,10 @@ class Audit {
         logger.warn('projected transaction missing from mempool cache');
       }
       index++;
+      if (index >= projectedBlocks[blockIndex].transactionIds.length) {
+        index = 0;
+        blockIndex++;
+      }
     }
 
     // mark unexpected transactions in the mined block as 'added'
@@ -113,11 +118,16 @@ class Audit {
       } else {
         if (rbfCache.has(tx.txid)) {
           rbf.push(tx.txid);
-        } else if (!isDisplaced[tx.txid]) {
+          if (!mempool[tx.txid] && !rbfCache.getReplacedBy(tx.txid)) {
+            unseen.push(tx.txid);
+          }
+        } else {
           if (mempool[tx.txid]) {
-            prioritized.push(tx.txid);
+            if (isDisplaced[tx.txid]) {
+              added.push(tx.txid);
+            }
           } else {
-            added.push(tx.txid);
+            unseen.push(tx.txid);
           }
         }
         overflowWeight += tx.weight;
@@ -125,6 +135,8 @@ class Audit {
       totalWeight += tx.weight;
     }
 
+    ({ prioritized, deprioritized } = transactionUtils.identifyPrioritizedTransactions(transactions, 'effectiveFeePerVsize'));
+
     // transactions missing from near the end of our template are probably not being censored
     let overflowWeightRemaining = overflowWeight - (config.MEMPOOL.BLOCK_WEIGHT_UNITS - totalWeight);
     let maxOverflowRate = 0;
@@ -165,6 +177,7 @@ class Audit {
     const similarity = projectedWeight ? matchedWeight / projectedWeight : 1;
 
     return {
+      unseen,
       censored: Object.keys(isCensored),
       added,
       prioritized,

backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts

@@ -1,4 +1,4 @@
-import { IBitcoinApi } from './bitcoin-api.interface';
+import { IBitcoinApi, SubmitPackageResult, TestMempoolAcceptResult } from './bitcoin-api.interface';
 import { IEsploraApi } from './esplora-api.interface';
 
 export interface AbstractBitcoinApi {
@@ -22,11 +22,15 @@ export interface AbstractBitcoinApi {
   $getScriptHash(scripthash: string): Promise<IEsploraApi.ScriptHash>;
   $getScriptHashTransactions(address: string, lastSeenTxId: string): Promise<IEsploraApi.Transaction[]>;
   $sendRawTransaction(rawTransaction: string): Promise<string>;
+  $testMempoolAccept(rawTransactions: string[], maxfeerate?: number): Promise<TestMempoolAcceptResult[]>;
+  $submitPackage(rawTransactions: string[], maxfeerate?: number, maxburnamount?: number): Promise<SubmitPackageResult>;
   $getOutspend(txId: string, vout: number): Promise<IEsploraApi.Outspend>;
   $getOutspends(txId: string): Promise<IEsploraApi.Outspend[]>;
   $getBatchedOutspends(txId: string[]): Promise<IEsploraApi.Outspend[][]>;
   $getBatchedOutspendsInternal(txId: string[]): Promise<IEsploraApi.Outspend[][]>;
   $getOutSpendsByOutpoint(outpoints: { txid: string, vout: number }[]): Promise<IEsploraApi.Outspend[]>;
+  $getCoinbaseTx(blockhash: string): Promise<IEsploraApi.Transaction>;
+  $getAddressTransactionSummary(address: string): Promise<IEsploraApi.AddressTxSummary[]>;
 
   startHealthChecks(): void;
   getHealthStatus(): HealthCheckHost[];

backend/src/api/bitcoin/bitcoin-api.interface.ts

@@ -205,3 +205,34 @@ export namespace IBitcoinApi {
     "utxo_size_inc": number;
   }
 }
+
+export interface TestMempoolAcceptResult {
+  txid: string,
+  wtxid: string,
+  allowed?: boolean,
+  vsize?: number,
+  fees?: {
+    base: number,
+    "effective-feerate": number,
+    "effective-includes": string[],
+  },
+  ['reject-reason']?: string,
+}
+
+export interface SubmitPackageResult {
+  package_msg: string;
+  "tx-results": { [wtxid: string]: TxResult };
+  "replaced-transactions"?: string[];
+}
+
+export interface TxResult {
+  txid: string;
+  "other-wtxid"?: string;
+  vsize?: number;
+  fees?: {
+    base: number;
+    "effective-feerate"?: number;
+    "effective-includes"?: string[];
+  };
+  error?: string;
+}

backend/src/api/bitcoin/bitcoin-api.ts

@@ -1,6 +1,6 @@
 import * as bitcoinjs from 'bitcoinjs-lib';
 import { AbstractBitcoinApi, HealthCheckHost } from './bitcoin-api-abstract-factory';
-import { IBitcoinApi } from './bitcoin-api.interface';
+import { IBitcoinApi, SubmitPackageResult, TestMempoolAcceptResult } from './bitcoin-api.interface';
 import { IEsploraApi } from './esplora-api.interface';
 import blocks from '../blocks';
 import mempool from '../mempool';
@@ -107,8 +107,14 @@ class BitcoinApi implements AbstractBitcoinApi {
       .then((rpcBlock: IBitcoinApi.Block) => rpcBlock.tx);
   }
 
-  $getTxsForBlock(hash: string): Promise<IEsploraApi.Transaction[]> {
-    throw new Error('Method getTxsForBlock not supported by the Bitcoin RPC API.');
+  async $getTxsForBlock(hash: string): Promise<IEsploraApi.Transaction[]> {
+    const verboseBlock: IBitcoinApi.VerboseBlock = await this.bitcoindClient.getBlock(hash, 2);
+    const transactions: IEsploraApi.Transaction[] = [];
+    for (const tx of verboseBlock.tx) {
+      const converted = await this.$convertTransaction(tx, true);
+      transactions.push(converted);
+    }
+    return transactions;
   }
 
   $getRawBlock(hash: string): Promise<Buffer> {
@@ -159,13 +165,21 @@ class BitcoinApi implements AbstractBitcoinApi {
     const mp = mempool.getMempool();
     for (const tx in mp) {
       for (const vout of mp[tx].vout) {
-        if (vout.scriptpubkey_address.indexOf(prefix) === 0) {
+        if (vout.scriptpubkey_address?.indexOf(prefix) === 0) {
           found[vout.scriptpubkey_address] = '';
           if (Object.keys(found).length >= 10) {
             return Object.keys(found);
           }
         }
       }
+      for (const vin of mp[tx].vin) {
+        if (vin.prevout?.scriptpubkey_address?.indexOf(prefix) === 0) {
+          found[vin.prevout?.scriptpubkey_address] = '';
+          if (Object.keys(found).length >= 10) {
+            return Object.keys(found);
+          }
+        }
+      }
     }
     return Object.keys(found);
   }
@@ -174,6 +188,18 @@ class BitcoinApi implements AbstractBitcoinApi {
     return this.bitcoindClient.sendRawTransaction(rawTransaction);
   }
 
+  async $testMempoolAccept(rawTransactions: string[], maxfeerate?: number): Promise<TestMempoolAcceptResult[]> {
+    if (rawTransactions.length) {
+      return this.bitcoindClient.testMempoolAccept(rawTransactions, maxfeerate ?? undefined);
+    } else {
+      return [];
+    }
+  }
+
+  $submitPackage(rawTransactions: string[], maxfeerate?: number, maxburnamount?: number): Promise<SubmitPackageResult> {
+    return this.bitcoindClient.submitPackage(rawTransactions, maxfeerate ?? undefined, maxburnamount ?? undefined);
+  }
+
   async $getOutspend(txId: string, vout: number): Promise<IEsploraApi.Outspend> {
     const txOut = await this.bitcoindClient.getTxOut(txId, vout, false);
     return {
@@ -224,6 +250,15 @@ class BitcoinApi implements AbstractBitcoinApi {
     return outspends;
   }
 
+  async $getCoinbaseTx(blockhash: string): Promise<IEsploraApi.Transaction> {
+    const txids = await this.$getTxIdsForBlock(blockhash);
+    return this.$getRawTransaction(txids[0]);
+  }
+
+  async $getAddressTransactionSummary(address: string): Promise<IEsploraApi.AddressTxSummary[]> {
+    throw new Error('Method getAddressTransactionSummary not supported by the Bitcoin RPC API.');
+  }
+
   $getEstimatedHashrate(blockHeight: number): Promise<number> {
     // 120 is the default block span in Core
     return this.bitcoindClient.getNetworkHashPs(120, blockHeight);
@@ -296,6 +331,7 @@ class BitcoinApi implements AbstractBitcoinApi {
       'witness_v1_taproot': 'v1_p2tr',
       'nonstandard': 'nonstandard',
       'multisig': 'multisig',
+      'anchor': 'anchor',
       'nulldata': 'op_return'
     };
 

backend/src/api/bitcoin/bitcoin-core.routes.ts

@@ -1,6 +1,11 @@
 import { Application, NextFunction, Request, Response } from 'express';
 import logger from '../../logger';
 import bitcoinClient from './bitcoin-client';
+import config from '../../config';
+
+const BLOCKHASH_REGEX = /^[a-f0-9]{64}$/i;
+const TXID_REGEX = /^[a-f0-9]{64}$/i;
+const RAW_TX_REGEX = /^[a-f0-9]{2,}$/i;
 
 /**
  * Define a set of routes used by the accelerator server
@@ -9,26 +14,26 @@ import bitcoinClient from './bitcoin-client';
 class BitcoinBackendRoutes {
   private static tag = 'BitcoinBackendRoutes';
 
-  public initRoutes(app: Application) {
+  public initRoutes(app: Application): void {
     app
-      .get('/api/internal/bitcoin-core/' + 'get-mempool-entry', this.disableCache, this.$getMempoolEntry)
-      .post('/api/internal/bitcoin-core/' + 'decode-raw-transaction', this.disableCache, this.$decodeRawTransaction)
-      .get('/api/internal/bitcoin-core/' + 'get-raw-transaction', this.disableCache, this.$getRawTransaction)
-      .post('/api/internal/bitcoin-core/' + 'send-raw-transaction', this.disableCache, this.$sendRawTransaction)
-      .post('/api/internal/bitcoin-core/' + 'test-mempool-accept', this.disableCache, this.$testMempoolAccept)
-      .get('/api/internal/bitcoin-core/' + 'get-mempool-ancestors', this.disableCache, this.$getMempoolAncestors)
-      .get('/api/internal/bitcoin-core/' + 'get-block', this.disableCache, this.$getBlock)
-      .get('/api/internal/bitcoin-core/' + 'get-block-hash', this.disableCache, this.$getBlockHash)
-      .get('/api/internal/bitcoin-core/' + 'get-block-count', this.disableCache, this.$getBlockCount)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'internal/bitcoin-core/' + 'get-mempool-entry', this.disableCache, this.$getMempoolEntry)
+      .post(config.MEMPOOL.API_URL_PREFIX + 'internal/bitcoin-core/' + 'decode-raw-transaction', this.disableCache, this.$decodeRawTransaction)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'internal/bitcoin-core/' + 'get-raw-transaction', this.disableCache, this.$getRawTransaction)
+      .post(config.MEMPOOL.API_URL_PREFIX + 'internal/bitcoin-core/' + 'send-raw-transaction', this.disableCache, this.$sendRawTransaction)
+      .post(config.MEMPOOL.API_URL_PREFIX + 'internal/bitcoin-core/' + 'test-mempool-accept', this.disableCache, this.$testMempoolAccept)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'internal/bitcoin-core/' + 'get-mempool-ancestors', this.disableCache, this.$getMempoolAncestors)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'internal/bitcoin-core/' + 'get-block', this.disableCache, this.$getBlock)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'internal/bitcoin-core/' + 'get-block-hash', this.disableCache, this.$getBlockHash)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'internal/bitcoin-core/' + 'get-block-count', this.disableCache, this.$getBlockCount)
     ;
   }
 
   /**
    * Disable caching for bitcoin core routes
-   * 
-   * @param req 
-   * @param res 
-   * @param next 
+   *
+   * @param req
+   * @param res
+   * @param next
    */
   private disableCache(req: Request, res: Response, next: NextFunction): void  {
     res.setHeader('Pragma', 'no-cache');
@@ -39,16 +44,16 @@ class BitcoinBackendRoutes {
 
   /**
    * Exeption handler to return proper details to the accelerator server
-   * 
-   * @param e 
-   * @param fnName 
-   * @param res 
+   *
+   * @param e
+   * @param fnName
+   * @param res
    */
   private static handleException(e: any, fnName: string, res: Response): void {
     if (typeof(e.code) === 'number') {
-      res.status(400).send(JSON.stringify(e, ['code', 'message']));
-    } else {     
-      const err = `exception in ${fnName}. ${e}. Details: ${JSON.stringify(e, ['code', 'message'])}`; 
+      res.status(400).send(JSON.stringify(e, ['code']));
+    } else {
+      const err = `unknown exception in ${fnName}`;
       logger.err(err, BitcoinBackendRoutes.tag);
       res.status(500).send(err);
     }
@@ -57,13 +62,13 @@ class BitcoinBackendRoutes {
   private async $getMempoolEntry(req: Request, res: Response): Promise<void> {
     const txid = req.query.txid;
     try {
-      if (typeof(txid) !== 'string' || txid.length !== 64) {
-        res.status(400).send(`invalid param txid ${txid}. must be a string of 64 char`);
+      if (typeof(txid) !== 'string' || txid.length !== 64 || !TXID_REGEX.test(txid)) {
+        res.status(400).send(`invalid param txid. must be 64 hexadecimal characters`);
         return;
       }
       const mempoolEntry = await bitcoinClient.getMempoolEntry(txid);
       if (!mempoolEntry) {
-        res.status(404).send(`no mempool entry found for txid ${txid}`);
+        res.status(404).send();
         return;
       }
       res.status(200).send(mempoolEntry);
@@ -75,13 +80,13 @@ class BitcoinBackendRoutes {
   private async $decodeRawTransaction(req: Request, res: Response): Promise<void> {
     const rawTx = req.body.rawTx;
     try {
-      if (typeof(rawTx) !== 'string') {
-        res.status(400).send(`invalid param rawTx ${rawTx}. must be a string`);
+      if (typeof(rawTx) !== 'string' || !RAW_TX_REGEX.test(rawTx)) {
+        res.status(400).send(`invalid param rawTx. must be a string of hexadecimal characters`);
         return;
       }
       const decodedTx = await bitcoinClient.decodeRawTransaction(rawTx);
       if (!decodedTx) {
-        res.status(400).send(`unable to decode rawTx ${rawTx}`);
+        res.status(400).send(`unable to decode rawTx`);
         return;
       }
       res.status(200).send(decodedTx);
@@ -94,23 +99,23 @@ class BitcoinBackendRoutes {
     const txid = req.query.txid;
     const verbose = req.query.verbose;
     try {
-      if (typeof(txid) !== 'string' || txid.length !== 64) {
-        res.status(400).send(`invalid param txid ${txid}. must be a string of 64 char`);
+      if (typeof(txid) !== 'string' || txid.length !== 64 || !TXID_REGEX.test(txid)) {
+        res.status(400).send(`invalid param txid. must be 64 hexadecimal characters`);
         return;
       }
       if (typeof(verbose) !== 'string') {
-        res.status(400).send(`invalid param verbose ${verbose}. must be a string representing an integer`);
+        res.status(400).send(`invalid param verbose. must be a string representing an integer`);
         return;
       }
       const verboseNumber = parseInt(verbose, 10);
       if (typeof(verboseNumber) !== 'number') {
-        res.status(400).send(`invalid param verbose ${verbose}. must be a valid integer`);
+        res.status(400).send(`invalid param verbose. must be a valid integer`);
         return;
       }
 
       const decodedTx = await bitcoinClient.getRawTransaction(txid, verboseNumber);
       if (!decodedTx) {
-        res.status(400).send(`unable to get raw transaction for txid ${txid}`);
+        res.status(400).send(`unable to get raw transaction`);
         return;
       }
       res.status(200).send(decodedTx);
@@ -122,13 +127,13 @@ class BitcoinBackendRoutes {
   private async $sendRawTransaction(req: Request, res: Response): Promise<void> {
     const rawTx = req.body.rawTx;
     try {
-      if (typeof(rawTx) !== 'string') {
-        res.status(400).send(`invalid param rawTx ${rawTx}. must be a string`);
+      if (typeof(rawTx) !== 'string' || !RAW_TX_REGEX.test(rawTx)) {
+        res.status(400).send(`invalid param rawTx. must be a string of hexadecimal characters`);
         return;
       }
       const txHex = await bitcoinClient.sendRawTransaction(rawTx);
       if (!txHex) {
-        res.status(400).send(`unable to send rawTx ${rawTx}`);
+        res.status(400).send(`unable to send rawTx`);
         return;
       }
       res.status(200).send(txHex);
@@ -140,13 +145,13 @@ class BitcoinBackendRoutes {
   private async $testMempoolAccept(req: Request, res: Response): Promise<void> {
     const rawTxs = req.body.rawTxs;
     try {
-      if (typeof(rawTxs) !== 'object') {
-        res.status(400).send(`invalid param rawTxs ${JSON.stringify(rawTxs)}. must be an array of string`);
+      if (typeof(rawTxs) !== 'object' || !Array.isArray(rawTxs) || rawTxs.some((tx) => typeof(tx) !== 'string' || !RAW_TX_REGEX.test(tx))) {
+        res.status(400).send(`invalid param rawTxs. must be an array of strings of hexadecimal characters`);
         return;
       }
       const txHex = await bitcoinClient.testMempoolAccept(rawTxs);
       if (typeof(txHex) !== 'object' || txHex.length === 0) {
-        res.status(400).send(`testmempoolaccept failed for raw txs ${JSON.stringify(rawTxs)}, got an empty result`);
+        res.status(400).send(`testmempoolaccept failed for raw txs, got an empty result`);
         return;
       }
       res.status(200).send(txHex);
@@ -159,18 +164,18 @@ class BitcoinBackendRoutes {
     const txid = req.query.txid;
     const verbose = req.query.verbose;
     try {
-      if (typeof(txid) !== 'string' || txid.length !== 64) {
-        res.status(400).send(`invalid param txid ${txid}. must be a string of 64 char`);
+      if (typeof(txid) !== 'string' || txid.length !== 64 || !TXID_REGEX.test(txid)) {
+        res.status(400).send(`invalid param txid. must be 64 hexadecimal characters`);
         return;
       }
       if (typeof(verbose) !== 'string' || (verbose !== 'true' && verbose !== 'false')) {
-        res.status(400).send(`invalid param verbose ${verbose}. must be a string ('true' | 'false')`);
+        res.status(400).send(`invalid param verbose. must be a string ('true' | 'false')`);
         return;
       }
-    
+
       const ancestors = await bitcoinClient.getMempoolAncestors(txid, verbose === 'true' ? true : false);
       if (!ancestors) {
-        res.status(400).send(`unable to get mempool ancestors for txid ${txid}`);
+        res.status(400).send(`unable to get mempool ancestors`);
         return;
       }
       res.status(200).send(ancestors);
@@ -183,23 +188,23 @@ class BitcoinBackendRoutes {
     const blockHash = req.query.hash;
     const verbosity = req.query.verbosity;
     try {
-      if (typeof(blockHash) !== 'string' || blockHash.length !== 64) {
-        res.status(400).send(`invalid param blockHash ${blockHash}. must be a string of 64 char`);
+      if (typeof(blockHash) !== 'string' || blockHash.length !== 64 || !BLOCKHASH_REGEX.test(blockHash)) {
+        res.status(400).send(`invalid param blockHash. must be 64 hexadecimal characters`);
         return;
       }
       if (typeof(verbosity) !== 'string') {
-        res.status(400).send(`invalid param verbosity ${verbosity}. must be a string representing an integer`);
+        res.status(400).send(`invalid param verbosity. must be a string representing an integer`);
         return;
       }
       const verbosityNumber = parseInt(verbosity, 10);
       if (typeof(verbosityNumber) !== 'number') {
-        res.status(400).send(`invalid param verbosity ${verbosity}. must be a valid integer`);
+        res.status(400).send(`invalid param verbosity. must be a valid integer`);
         return;
       }
 
       const block = await bitcoinClient.getBlock(blockHash, verbosityNumber);
       if (!block) {
-        res.status(400).send(`unable to get block for block hash ${blockHash}`);
+        res.status(400).send(`unable to get block`);
         return;
       }
       res.status(200).send(block);
@@ -212,18 +217,18 @@ class BitcoinBackendRoutes {
     const blockHeight = req.query.height;
     try {
       if (typeof(blockHeight) !== 'string') {
-        res.status(400).send(`invalid param blockHeight ${blockHeight}, must be a string representing an integer`);
+        res.status(400).send(`invalid param blockHeight, must be a string representing an integer`);
         return;
       }
       const blockHeightNumber = parseInt(blockHeight, 10);
       if (typeof(blockHeightNumber) !== 'number') {
-        res.status(400).send(`invalid param blockHeight ${blockHeight}. must be a valid integer`);
+        res.status(400).send(`invalid param blockHeight. must be a valid integer`);
         return;
       }
 
       const block = await bitcoinClient.getBlockHash(blockHeightNumber);
       if (!block) {
-        res.status(400).send(`unable to get block hash for block height ${blockHeightNumber}`);
+        res.status(400).send(`unable to get block hash`);
         return;
       }
       res.status(200).send(block);
@@ -246,4 +251,4 @@ class BitcoinBackendRoutes {
   }
 }
 
-export default new BitcoinBackendRoutes
+export default new BitcoinBackendRoutes;

backend/src/api/bitcoin/bitcoin.routes.ts

@@ -19,7 +19,14 @@ import bitcoinClient from './bitcoin-client';
 import difficultyAdjustment from '../difficulty-adjustment';
 import transactionRepository from '../../repositories/TransactionRepository';
 import rbfCache from '../rbf-cache';
-import { calculateCpfp } from '../cpfp';
+import { calculateMempoolTxCpfp } from '../cpfp';
+import { handleError } from '../../utils/api';
+import poolsUpdater from '../../tasks/pools-updater';
+
+const TXID_REGEX = /^[a-f0-9]{64}$/i;
+const BLOCK_HASH_REGEX = /^[a-f0-9]{64}$/i;
+const ADDRESS_REGEX = /^[a-z0-9]{2,120}$/i;
+const SCRIPT_HASH_REGEX = /^([a-f0-9]{2})+$/i;
 
 class BitcoinRoutes {
   public initRoutes(app: Application) {
@@ -41,11 +48,19 @@ class BitcoinRoutes {
       .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/:height', this.getBlocks.bind(this))
       .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash', this.getBlock)
       .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/summary', this.getStrippedBlockTransactions)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/tx/:txid/summary', this.getStrippedBlockTransaction)
       .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/audit-summary', this.getBlockAuditSummary)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/tx/:txid/audit', this.$getBlockTxAuditSummary)
       .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/height', this.getBlockTipHeight)
       .post(config.MEMPOOL.API_URL_PREFIX + 'psbt/addparents', this.postPsbtCompletion)
       .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from', this.getBlocksByBulk.bind(this))
       .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from/:to', this.getBlocksByBulk.bind(this))
+      // Temporarily add txs/package endpoint for all backends until esplora supports it
+      .post(config.MEMPOOL.API_URL_PREFIX + 'txs/package', this.$submitPackage)
+      // Internal routes
+      .get(config.MEMPOOL.API_URL_PREFIX + 'internal/blocks/definition/list', this.getBlockDefinitionHashes)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'internal/blocks/definition/current', this.getCurrentBlockDefinitionHash)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'internal/blocks/:definitionHash', this.getBlocksByDefinitionHash)
       ;
 
       if (config.MEMPOOL.BACKEND !== 'esplora') {
@@ -55,6 +70,7 @@ class BitcoinRoutes {
           .get(config.MEMPOOL.API_URL_PREFIX + 'mempool/recent', this.getRecentMempoolTransactions)
           .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId', this.getTransaction)
           .post(config.MEMPOOL.API_URL_PREFIX + 'tx', this.$postTransaction)
+          .post(config.MEMPOOL.API_URL_PREFIX + 'txs/test', this.$testTransactions)
           .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/hex', this.getRawTransaction)
           .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/status', this.getTransactionStatus)
           .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/outspends', this.getTransactionOutspends)
@@ -84,7 +100,7 @@ class BitcoinRoutes {
       res.set('Content-Type', 'application/json');
       res.send(result);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get init data');
     }
   }
 
@@ -103,19 +119,22 @@ class BitcoinRoutes {
       const result = mempoolBlocks.getMempoolBlocks();
       res.json(result);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get mempool blocks');
     }
   }
 
   private getTransactionTimes(req: Request, res: Response) {
     if (!Array.isArray(req.query.txId)) {
-      res.status(500).send('Not an array');
+      handleError(req, res, 500, 'Not an array');
       return;
     }
     const txIds: string[] = [];
     for (const _txId in req.query.txId) {
       if (typeof req.query.txId[_txId] === 'string') {
-        txIds.push(req.query.txId[_txId].toString());
+        const txid = req.query.txId[_txId].toString();
+        if (TXID_REGEX.test(txid)) {
+          txIds.push(txid);
+        }
       }
     }
 
@@ -126,12 +145,16 @@ class BitcoinRoutes {
   private async $getBatchedOutspends(req: Request, res: Response): Promise<IEsploraApi.Outspend[][] | void> {
     const txids_csv = req.query.txids;
     if (!txids_csv || typeof txids_csv !== 'string') {
-      res.status(500).send('Invalid txids format');
+      handleError(req, res, 500, 'Invalid txids format');
       return;
     }
     const txids = txids_csv.split(',');
     if (txids.length > 50) {
-      res.status(400).send('Too many txids requested');
+      handleError(req, res, 400, 'Too many txids requested');
+      return;
+    }
+    if (txids.some((txid) => !TXID_REGEX.test(txid))) {
+      handleError(req, res, 400, 'Invalid txids format');
       return;
     }
 
@@ -139,13 +162,13 @@ class BitcoinRoutes {
       const batchedOutspends = await bitcoinApi.$getBatchedOutspends(txids);
       res.json(batchedOutspends);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get batched outspends');
     }
   }
 
   private async $getCpfpInfo(req: Request, res: Response) {
-    if (!/^[a-fA-F0-9]{64}$/.test(req.params.txId)) {
-      res.status(501).send(`Invalid transaction ID.`);
+    if (!TXID_REGEX.test(req.params.txId)) {
+      handleError(req, res, 501, `Invalid transaction ID`);
       return;
     }
 
@@ -158,13 +181,17 @@ class BitcoinRoutes {
           descendants: tx.descendants || null,
           effectiveFeePerVsize: tx.effectiveFeePerVsize || null,
           sigops: tx.sigops,
+          fee: tx.fee,
           adjustedVsize: tx.adjustedVsize,
-          acceleration: tx.acceleration
+          acceleration: tx.acceleration,
+          acceleratedBy: tx.acceleratedBy || undefined,
+          acceleratedAt: tx.acceleratedAt || undefined,
+          feeDelta: tx.feeDelta || undefined,
         });
         return;
       }
 
-      const cpfpInfo = calculateCpfp(tx, mempool.getMempool());
+      const cpfpInfo = calculateMempoolTxCpfp(tx, mempool.getMempool());
 
       res.json(cpfpInfo);
       return;
@@ -174,7 +201,7 @@ class BitcoinRoutes {
         try {
           cpfpInfo = await transactionRepository.$getCpfpInfo(req.params.txId);
         } catch (e) {
-          res.status(500).send('failed to get CPFP info');
+          handleError(req, res, 500, 'Failed to get CPFP info');
           return;
         }
       }
@@ -195,6 +222,10 @@ class BitcoinRoutes {
   }
 
   private async getTransaction(req: Request, res: Response) {
+    if (!TXID_REGEX.test(req.params.txId)) {
+      handleError(req, res, 501, `Invalid transaction ID`);
+      return;
+    }
     try {
       const transaction = await transactionUtils.$getTransactionExtended(req.params.txId, true, false, false, true);
       res.json(transaction);
@@ -202,12 +233,18 @@ class BitcoinRoutes {
       let statusCode = 500;
       if (e instanceof Error && e instanceof Error && e.message && e.message.indexOf('No such mempool or blockchain transaction') > -1) {
         statusCode = 404;
+        handleError(req, res, statusCode, 'No such mempool or blockchain transaction');
+        return;
       }
-      res.status(statusCode).send(e instanceof Error ? e.message : e);
+      handleError(req, res, statusCode, 'Failed to get transaction');
     }
   }
 
   private async getRawTransaction(req: Request, res: Response) {
+    if (!TXID_REGEX.test(req.params.txId)) {
+      handleError(req, res, 501, `Invalid transaction ID`);
+      return;
+    }
     try {
       const transaction: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(req.params.txId, true);
       res.setHeader('content-type', 'text/plain');
@@ -216,8 +253,10 @@ class BitcoinRoutes {
       let statusCode = 500;
       if (e instanceof Error && e.message && e.message.indexOf('No such mempool or blockchain transaction') > -1) {
         statusCode = 404;
+        handleError(req, res, statusCode, 'No such mempool or blockchain transaction');
+        return;
       }
-      res.status(statusCode).send(e instanceof Error ? e.message : e);
+      handleError(req, res, statusCode, 'Failed to get raw transaction');
     }
   }
 
@@ -278,18 +317,22 @@ class BitcoinRoutes {
         // Not modified
         // 422 Unprocessable Entity
         // https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/422
-        res.status(422).send(`Psbt had no missing nonWitnessUtxos.`);
+        handleError(req, res, 422, `Psbt had no missing nonWitnessUtxos.`);
       }
     } catch (e: any) {
       if (e instanceof Error && new RegExp(notFoundError).test(e.message)) {
-        res.status(404).send(e.message);
+        handleError(req, res, 404, notFoundError);
       } else {
-        res.status(500).send(e instanceof Error ? e.message : e);
+        handleError(req, res, 500, 'Failed to process PSBT');
       }
     }
   }
 
   private async getTransactionStatus(req: Request, res: Response) {
+    if (!TXID_REGEX.test(req.params.txId)) {
+      handleError(req, res, 501, `Invalid transaction ID`);
+      return;
+    }
     try {
       const transaction = await transactionUtils.$getTransactionExtended(req.params.txId, true);
       res.json(transaction.status);
@@ -297,22 +340,54 @@ class BitcoinRoutes {
       let statusCode = 500;
       if (e instanceof Error && e.message && e.message.indexOf('No such mempool or blockchain transaction') > -1) {
         statusCode = 404;
+        handleError(req, res, statusCode, 'No such mempool or blockchain transaction');
+        return;
       }
-      res.status(statusCode).send(e instanceof Error ? e.message : e);
+      handleError(req, res, statusCode, 'Failed to get transaction status');
     }
   }
 
   private async getStrippedBlockTransactions(req: Request, res: Response) {
+    if (!BLOCK_HASH_REGEX.test(req.params.hash)) {
+      handleError(req, res, 501, `Invalid block hash`);
+      return;
+    }
     try {
       const transactions = await blocks.$getStrippedBlockTransactions(req.params.hash);
       res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24 * 30).toUTCString());
       res.json(transactions);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get block summary');
+    }
+  }
+
+  private async getStrippedBlockTransaction(req: Request, res: Response) {
+    if (!BLOCK_HASH_REGEX.test(req.params.hash)) {
+      handleError(req, res, 501, `Invalid block hash`);
+      return;
+    }
+    if (!TXID_REGEX.test(req.params.txid)) {
+      handleError(req, res, 501, `Invalid transaction ID`);
+      return;
+    }
+    try {
+      const transaction = await blocks.$getSingleTxFromSummary(req.params.hash, req.params.txid);
+      if (!transaction) {
+        handleError(req, res, 404, `Transaction not found in summary`);
+        return;
+      }
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24 * 30).toUTCString());
+      res.json(transaction);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get transaction from summary');
     }
   }
 
   private async getBlock(req: Request, res: Response) {
+    if (!BLOCK_HASH_REGEX.test(req.params.hash)) {
+      handleError(req, res, 501, `Invalid block hash`);
+      return;
+    }
     try {
       const block = await blocks.$getBlock(req.params.hash);
 
@@ -324,37 +399,69 @@ class BitcoinRoutes {
       } else if (blockAge > 30 * day) {
         cacheDuration = 10 * day;
       } else {
-        cacheDuration = 600
+        cacheDuration = 600;
       }
 
       res.setHeader('Expires', new Date(Date.now() + 1000 * cacheDuration).toUTCString());
       res.json(block);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get block');
     }
   }
 
   private async getBlockHeader(req: Request, res: Response) {
+    if (!BLOCK_HASH_REGEX.test(req.params.hash)) {
+      handleError(req, res, 501, `Invalid block hash`);
+      return;
+    }
     try {
       const blockHeader = await bitcoinApi.$getBlockHeader(req.params.hash);
       res.setHeader('content-type', 'text/plain');
       res.send(blockHeader);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get block header');
     }
   }
 
   private async getBlockAuditSummary(req: Request, res: Response) {
+    if (!BLOCK_HASH_REGEX.test(req.params.hash)) {
+      handleError(req, res, 501, `Invalid block hash`);
+      return;
+    }
     try {
       const auditSummary = await blocks.$getBlockAuditSummary(req.params.hash);
       if (auditSummary) {
         res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24 * 30).toUTCString());
         res.json(auditSummary);
       } else {
-        return res.status(404).send(`audit not available`);
+        handleError(req, res, 404, `Audit not available`);
+        return;
       }
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get block audit summary');
+    }
+  }
+
+  private async $getBlockTxAuditSummary(req: Request, res: Response) {
+    if (!BLOCK_HASH_REGEX.test(req.params.hash)) {
+      handleError(req, res, 501, `Invalid block hash`);
+      return;
+    }
+    if (!TXID_REGEX.test(req.params.txid)) {
+      handleError(req, res, 501, `Invalid transaction ID`);
+      return;
+    }
+    try {
+      const auditSummary = await blocks.$getBlockTxAuditSummary(req.params.hash, req.params.txid);
+      if (auditSummary) {
+        res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24 * 30).toUTCString());
+        res.json(auditSummary);
+      } else {
+        handleError(req, res, 404, `Transaction audit not available`);
+        return;
+      }
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get transaction audit summary');
     }
   }
 
@@ -368,42 +475,49 @@ class BitcoinRoutes {
         return await this.getLegacyBlocks(req, res);
       }
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get blocks');
     }
   }
 
   private async getBlocksByBulk(req: Request, res: Response) {
     try {
       if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { // Liquid - Not implemented
-        return res.status(404).send(`This API is only available for Bitcoin networks`);
+        handleError(req, res, 404, `This API is only available for Bitcoin networks`);
+        return;
       }
       if (config.MEMPOOL.MAX_BLOCKS_BULK_QUERY <= 0) {
-        return res.status(404).send(`This API is disabled. Set config.MEMPOOL.MAX_BLOCKS_BULK_QUERY to a positive number to enable it.`);
+        handleError(req, res, 404, `This API is disabled. Set config.MEMPOOL.MAX_BLOCKS_BULK_QUERY to a positive number to enable it.`);
+        return;
       }
       if (!Common.indexingEnabled()) {
-        return res.status(404).send(`Indexing is required for this API`);
+        handleError(req, res, 404, `Indexing is required for this API`);
+        return;
       }
 
       const from = parseInt(req.params.from, 10);
       if (!req.params.from || from < 0) {
-        return res.status(400).send(`Parameter 'from' must be a block height (integer)`);
+        handleError(req, res, 400, `Parameter 'from' must be a block height (integer)`);
+        return;
       }
       const to = req.params.to === undefined ? await bitcoinApi.$getBlockHeightTip() : parseInt(req.params.to, 10);
       if (to < 0) {
-        return res.status(400).send(`Parameter 'to' must be a block height (integer)`);
+        handleError(req, res, 400, `Parameter 'to' must be a block height (integer)`);
+        return;
       }
       if (from > to) {
-        return res.status(400).send(`Parameter 'to' must be a higher block height than 'from'`);
+        handleError(req, res, 400, `Parameter 'to' must be a higher block height than 'from'`);
+        return;
       }
       if ((to - from + 1) > config.MEMPOOL.MAX_BLOCKS_BULK_QUERY) {
-        return res.status(400).send(`You can only query ${config.MEMPOOL.MAX_BLOCKS_BULK_QUERY} blocks at once.`);
+        handleError(req, res, 400, `You can only query ${config.MEMPOOL.MAX_BLOCKS_BULK_QUERY} blocks at once.`);
+        return;
       }
 
       res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       res.json(await blocks.$getBlocksBetweenHeight(from, to));
 
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get blocks');
     }
   }
 
@@ -438,11 +552,15 @@ class BitcoinRoutes {
       res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       res.json(returnBlocks);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get blocks');
     }
   }
-  
+
   private async getBlockTransactions(req: Request, res: Response) {
+    if (!BLOCK_HASH_REGEX.test(req.params.hash)) {
+      handleError(req, res, 501, `Invalid block hash`);
+      return;
+    }
     try {
       loadingIndicators.setProgress('blocktxs-' + req.params.hash, 0);
 
@@ -463,7 +581,7 @@ class BitcoinRoutes {
       res.json(transactions);
     } catch (e) {
       loadingIndicators.setProgress('blocktxs-' + req.params.hash, 100);
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get block transactions');
     }
   }
 
@@ -472,13 +590,17 @@ class BitcoinRoutes {
       const blockHash = await bitcoinApi.$getBlockHash(parseInt(req.params.height, 10));
       res.send(blockHash);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get block at height');
     }
   }
 
   private async getAddress(req: Request, res: Response) {
     if (config.MEMPOOL.BACKEND === 'none') {
-      res.status(405).send('Address lookups cannot be used with bitcoind as backend.');
+      handleError(req, res, 405, 'Address lookups cannot be used with bitcoind as backend.');
+      return;
+    }
+    if (!ADDRESS_REGEX.test(req.params.address)) {
+      handleError(req, res, 501, `Invalid address`);
       return;
     }
 
@@ -487,15 +609,20 @@ class BitcoinRoutes {
       res.json(addressData);
     } catch (e) {
       if (e instanceof Error && e.message && (e.message.indexOf('too long') > 0 || e.message.indexOf('confirmed status') > 0)) {
-        return res.status(413).send(e instanceof Error ? e.message : e);
+        handleError(req, res, 413, e.message);
+        return;
       }
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get address');
     }
   }
 
   private async getAddressTransactions(req: Request, res: Response): Promise<void> {
     if (config.MEMPOOL.BACKEND === 'none') {
-      res.status(405).send('Address lookups cannot be used with bitcoind as backend.');
+      handleError(req, res, 405, 'Address lookups cannot be used with bitcoind as backend.');
+      return;
+    }
+    if (!ADDRESS_REGEX.test(req.params.address)) {
+      handleError(req, res, 501, `Invalid address`);
       return;
     }
 
@@ -508,23 +635,27 @@ class BitcoinRoutes {
       res.json(transactions);
     } catch (e) {
       if (e instanceof Error && e.message && (e.message.indexOf('too long') > 0 || e.message.indexOf('confirmed status') > 0)) {
-        res.status(413).send(e instanceof Error ? e.message : e);
+        handleError(req, res, 413, e.message);
         return;
       }
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get address transactions');
     }
   }
 
   private async getAddressTransactionSummary(req: Request, res: Response): Promise<void> {
     if (config.MEMPOOL.BACKEND !== 'esplora') {
-      res.status(405).send('Address summary lookups require mempool/electrs backend.');
+      handleError(req, res, 405, 'Address summary lookups require mempool/electrs backend.');
       return;
     }
   }
 
   private async getScriptHash(req: Request, res: Response) {
     if (config.MEMPOOL.BACKEND === 'none') {
-      res.status(405).send('Address lookups cannot be used with bitcoind as backend.');
+      handleError(req, res, 405, 'Address lookups cannot be used with bitcoind as backend.');
+      return;
+    }
+    if (!SCRIPT_HASH_REGEX.test(req.params.scripthash)) {
+      handleError(req, res, 501, `Invalid scripthash`);
       return;
     }
 
@@ -535,15 +666,20 @@ class BitcoinRoutes {
       res.json(addressData);
     } catch (e) {
       if (e instanceof Error && e.message && (e.message.indexOf('too long') > 0 || e.message.indexOf('confirmed status') > 0)) {
-        return res.status(413).send(e instanceof Error ? e.message : e);
+        handleError(req, res, 413, e.message);
+        return;
       }
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get script hash');
     }
   }
 
   private async getScriptHashTransactions(req: Request, res: Response): Promise<void> {
     if (config.MEMPOOL.BACKEND === 'none') {
-      res.status(405).send('Address lookups cannot be used with bitcoind as backend.');
+      handleError(req, res, 405, 'Address lookups cannot be used with bitcoind as backend.');
+      return;
+    }
+    if (!SCRIPT_HASH_REGEX.test(req.params.scripthash)) {
+      handleError(req, res, 501, `Invalid scripthash`);
       return;
     }
 
@@ -558,26 +694,26 @@ class BitcoinRoutes {
       res.json(transactions);
     } catch (e) {
       if (e instanceof Error && e.message && (e.message.indexOf('too long') > 0 || e.message.indexOf('confirmed status') > 0)) {
-        res.status(413).send(e instanceof Error ? e.message : e);
+        handleError(req, res, 413, e.message);
         return;
       }
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get script hash transactions');
     }
   }
 
   private async getScriptHashTransactionSummary(req: Request, res: Response): Promise<void> {
     if (config.MEMPOOL.BACKEND !== 'esplora') {
-      res.status(405).send('Scripthash summary lookups require mempool/electrs backend.');
+      handleError(req, res, 405, 'Scripthash summary lookups require mempool/electrs backend.');
       return;
     }
   }
 
   private async getAddressPrefix(req: Request, res: Response) {
     try {
-      const blockHash = await bitcoinApi.$getAddressPrefix(req.params.prefix);
-      res.send(blockHash);
+      const addressPrefix = await bitcoinApi.$getAddressPrefix(req.params.prefix);
+      res.send(addressPrefix);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get address prefix');
     }
   }
 
@@ -604,7 +740,53 @@ class BitcoinRoutes {
       const rawMempool = await bitcoinApi.$getRawMempool();
       res.send(rawMempool);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getBlockDefinitionHashes(req: Request, res: Response): Promise<void> {
+    try {
+      const result = await blocks.$getBlockDefinitionHashes();
+      if (!result) {
+        handleError(req, res, 503, `Service Temporarily Unavailable`);
+        return;
+      }
+      res.setHeader('content-type', 'application/json');
+      res.send(result);
+    } catch (e) {
+      handleError(req, res, 500, e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getCurrentBlockDefinitionHash(req: Request, res: Response): Promise<void> {
+    try {
+      const currentSha = await poolsUpdater.getShaFromDb();
+      if (!currentSha) {
+        handleError(req, res, 503, `Service Temporarily Unavailable`);
+        return;
+      }
+      res.setHeader('content-type', 'text/plain');
+      res.send(currentSha);
+    } catch (e) {
+      handleError(req, res, 500, e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getBlocksByDefinitionHash(req: Request, res: Response): Promise<void> {
+    try {
+      if (typeof(req.params.definitionHash) !== 'string') {
+        res.status(400).send('Parameter "hash" must be a valid string');
+        return;
+      }
+      const blocksHash = await blocks.$getBlocksByDefinitionHash(req.params.definitionHash as string);
+      if (!blocksHash) {
+        handleError(req, res, 503, `Service Temporarily Unavailable`);
+        return;
+      }
+      res.setHeader('content-type', 'application/json');
+      res.send(blocksHash);
+    } catch (e) {
+      handleError(req, res, 500, e instanceof Error ? e.message : e);
     }
   }
 
@@ -612,12 +794,13 @@ class BitcoinRoutes {
     try {
       const result = blocks.getCurrentBlockHeight();
       if (!result) {
-        return res.status(503).send(`Service Temporarily Unavailable`);
+        handleError(req, res, 503, `Service Temporarily Unavailable`);
+        return;
       }
       res.setHeader('content-type', 'text/plain');
       res.send(result.toString());
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get height at tip');
     }
   }
 
@@ -627,39 +810,55 @@ class BitcoinRoutes {
       res.setHeader('content-type', 'text/plain');
       res.send(result);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get hash at tip');
     }
   }
 
   private async getRawBlock(req: Request, res: Response) {
+    if (!BLOCK_HASH_REGEX.test(req.params.hash)) {
+      handleError(req, res, 501, `Invalid block hash`);
+      return;
+    }
     try {
       const result = await bitcoinApi.$getRawBlock(req.params.hash);
       res.setHeader('content-type', 'application/octet-stream');
       res.send(result);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get raw block');
     }
   }
 
   private async getTxIdsForBlock(req: Request, res: Response) {
+    if (!BLOCK_HASH_REGEX.test(req.params.hash)) {
+      handleError(req, res, 501, `Invalid block hash`);
+      return;
+    }
     try {
       const result = await bitcoinApi.$getTxIdsForBlock(req.params.hash);
       res.json(result);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get txids for block');
     }
   }
 
   private async validateAddress(req: Request, res: Response) {
+    if (!ADDRESS_REGEX.test(req.params.address)) {
+      handleError(req, res, 501, `Invalid address`);
+      return;
+    }
     try {
       const result = await bitcoinClient.validateAddress(req.params.address);
       res.json(result);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to validate address');
     }
   }
 
   private async getRbfHistory(req: Request, res: Response) {
+    if (!TXID_REGEX.test(req.params.txId)) {
+      handleError(req, res, 501, `Invalid transaction ID`);
+      return;
+    }
     try {
       const replacements = rbfCache.getRbfTree(req.params.txId) || null;
       const replaces = rbfCache.getReplaces(req.params.txId) || null;
@@ -668,7 +867,7 @@ class BitcoinRoutes {
         replaces
       });
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get rbf history');
     }
   }
 
@@ -677,7 +876,7 @@ class BitcoinRoutes {
       const result = rbfCache.getRbfTrees(false);
       res.json(result);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get rbf trees');
     }
   }
 
@@ -686,11 +885,15 @@ class BitcoinRoutes {
       const result = rbfCache.getRbfTrees(true);
       res.json(result);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get full rbf replacements');
     }
   }
 
   private async getCachedTx(req: Request, res: Response) {
+    if (!TXID_REGEX.test(req.params.txId)) {
+      handleError(req, res, 501, `Invalid transaction ID`);
+      return;
+    }
     try {
       const result = rbfCache.getTx(req.params.txId);
       if (result) {
@@ -699,16 +902,20 @@ class BitcoinRoutes {
         res.status(204).send();
       }
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get cached tx');
     }
   }
 
   private async getTransactionOutspends(req: Request, res: Response) {
+    if (!TXID_REGEX.test(req.params.txId)) {
+      handleError(req, res, 501, `Invalid transaction ID`);
+      return;
+    }
     try {
       const result = await bitcoinApi.$getOutspends(req.params.txId);
       res.json(result);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get transaction outspends');
     }
   }
 
@@ -718,10 +925,10 @@ class BitcoinRoutes {
       if (da) {
         res.json(da);
       } else {
-        res.status(503).send(`Service Temporarily Unavailable`);
+        handleError(req, res, 503, `Service Temporarily Unavailable`);
       }
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get difficulty change');
     }
   }
 
@@ -732,8 +939,8 @@ class BitcoinRoutes {
       const txIdResult = await bitcoinApi.$sendRawTransaction(rawTx);
       res.send(txIdResult);
     } catch (e: any) {
-      res.status(400).send(e.message && e.code ? 'sendrawtransaction RPC error: ' + JSON.stringify({ code: e.code, message: e.message })
-        : (e.message || 'Error'));
+      handleError(req, res, 400, (e.message && e.code) ? 'sendrawtransaction RPC error: ' + JSON.stringify({ code: e.code })
+        : 'Failed to send raw transaction');
     }
   }
 
@@ -744,8 +951,33 @@ class BitcoinRoutes {
       const txIdResult = await bitcoinClient.sendRawTransaction(txHex);
       res.send(txIdResult);
     } catch (e: any) {
-      res.status(400).send(e.message && e.code ? 'sendrawtransaction RPC error: ' + JSON.stringify({ code: e.code, message: e.message })
-        : (e.message || 'Error'));
+      handleError(req, res, 400, (e.message && e.code) ? 'sendrawtransaction RPC error: ' + JSON.stringify({ code: e.code })
+        : 'Failed to send raw transaction');
+    }
+  }
+
+  private async $testTransactions(req: Request, res: Response) {
+    try {
+      const rawTxs = Common.getTransactionsFromRequest(req);
+      const maxfeerate = parseFloat(req.query.maxfeerate as string);
+      const result = await bitcoinApi.$testMempoolAccept(rawTxs, maxfeerate);
+      res.send(result);
+    } catch (e: any) {
+      handleError(req, res, 400, (e.message && e.code) ? 'testmempoolaccept RPC error: ' + JSON.stringify({ code: e.code })
+        : 'Failed to test transactions');
+    }
+  }
+
+  private async $submitPackage(req: Request, res: Response) {
+    try {
+      const rawTxs = Common.getTransactionsFromRequest(req);
+      const maxfeerate = parseFloat(req.query.maxfeerate as string);
+      const maxburnamount = parseFloat(req.query.maxburnamount as string);
+      const result = await bitcoinClient.submitPackage(rawTxs, maxfeerate ?? undefined, maxburnamount ?? undefined);
+      res.send(result);
+    } catch (e: any) {
+      handleError(req, res, 400, (e.message && e.code) ? 'submitpackage RPC error: ' + JSON.stringify({ code: e.code })
+        : 'Failed to submit package');
     }
   }
 

backend/src/api/bitcoin/esplora-api.interface.ts

@@ -54,7 +54,7 @@ export namespace IEsploraApi {
     scriptpubkey: string;
     scriptpubkey_asm: string;
     scriptpubkey_type: string;
-    scriptpubkey_address: string;
+    scriptpubkey_address?: string;
     value: number;
     // Elements
     valuecommitment?: number;
@@ -179,4 +179,11 @@ export namespace IEsploraApi {
     burn_count: number;
   }
 
+  export interface AddressTxSummary {
+    txid: string;
+    value: number;
+    height: number;
+    time: number;
+    tx_position?: number;
+  }
 }

backend/src/api/bitcoin/esplora-api.ts

@@ -1,11 +1,12 @@
 import config from '../../config';
-import axios, { AxiosResponse, isAxiosError } from 'axios';
+import axios, { isAxiosError } from 'axios';
 import http from 'http';
 import { AbstractBitcoinApi, HealthCheckHost } from './bitcoin-api-abstract-factory';
 import { IEsploraApi } from './esplora-api.interface';
 import logger from '../../logger';
 import { Common } from '../common';
-
+import { SubmitPackageResult, TestMempoolAcceptResult } from './bitcoin-api.interface';
+import os from 'os';
 interface FailoverHost {
   host: string,
   rtts: number[],
@@ -19,22 +20,37 @@ interface FailoverHost {
   preferred?: boolean,
   checked: boolean,
   lastChecked?: number,
+  publicDomain: string,
+  hashes: {
+    frontend?: string,
+    backend?: string,
+    electrs?: string,
+    lastUpdated: number,
+  }
 }
 
 class FailoverRouter {
   activeHost: FailoverHost;
   fallbackHost: FailoverHost;
+  maxSlippage: number = config.ESPLORA.MAX_BEHIND_TIP ?? 2;
   maxHeight: number = 0;
   hosts: FailoverHost[];
   multihost: boolean;
-  pollInterval: number = 60000;
+  gitHashInterval: number = 600000; // 10 minutes
+  pollInterval: number = 60000; // 1 minute
   pollTimer: NodeJS.Timeout | null = null;
   pollConnection = axios.create();
+  localHostname: string = 'localhost';
   requestConnection = axios.create({
     httpAgent: new http.Agent({ keepAlive: true })
   });
 
   constructor() {
+    try {
+      this.localHostname = os.hostname();
+    } catch (e) {
+      logger.warn('Failed to set local hostname, using "localhost"');
+    }
     // setup list of hosts
     this.hosts = (config.ESPLORA.FALLBACK || []).map(domain => {
       return {
@@ -43,6 +59,10 @@ class FailoverRouter {
         rtts: [],
         rtt: Infinity,
         failures: 0,
+        publicDomain: 'https://' + this.extractPublicDomain(domain),
+        hashes: {
+          lastUpdated: 0,
+        },
       };
     });
     this.activeHost = {
@@ -53,6 +73,10 @@ class FailoverRouter {
       socket: !!config.ESPLORA.UNIX_SOCKET_PATH,
       preferred: true,
       checked: false,
+      publicDomain: `http://${this.localHostname}`,
+      hashes: {
+        lastUpdated: 0,
+      },
     };
     this.fallbackHost = this.activeHost;
     this.hosts.unshift(this.activeHost);
@@ -92,18 +116,36 @@ class FailoverRouter {
         );
         if (result) {
           const height = result.data;
-          this.maxHeight = Math.max(height, this.maxHeight);
+          host.latestHeight = height;
+          this.maxHeight = Math.max(height || 0, ...this.hosts.map(h => (!(h.unreachable || h.timedOut || h.outOfSync) ? h.latestHeight || 0 : 0)));
           const rtt = result.config['meta'].rtt;
           host.rtts.unshift(rtt);
           host.rtts.slice(0, 5);
           host.rtt = host.rtts.reduce((acc, l) => acc + l, 0) / host.rtts.length;
-          host.latestHeight = height;
-          if (height == null || isNaN(height) || (this.maxHeight - height > 2)) {
+          if (height == null || isNaN(height) || (this.maxHeight - height > this.maxSlippage)) {
             host.outOfSync = true;
           } else {
             host.outOfSync = false;
           }
           host.unreachable = false;
+
+          // update esplora git hash using the x-powered-by header from the height check
+          const poweredBy = result.headers['x-powered-by'];
+          if (poweredBy) {
+            const match = poweredBy.match(/([a-fA-F0-9]{5,40})/);
+            if (match && match[1]?.length) {
+              host.hashes.electrs = match[1];
+            }
+          }
+
+          // Check front and backend git hashes less often
+          if (Date.now() - host.hashes.lastUpdated > this.gitHashInterval) {
+            await Promise.all([
+              this.$updateFrontendGitHash(host),
+              this.$updateBackendGitHash(host)
+            ]);
+            host.hashes.lastUpdated = Date.now();
+          }
         } else {
           host.outOfSync = true;
           host.unreachable = true;
@@ -125,7 +167,6 @@ class FailoverRouter {
       host.checked = true;
       host.lastChecked = Date.now();
 
-      // switch if the current host is out of sync or significantly slower than the next best alternative
       const rankOrder = this.sortHosts();
       // switch if the current host is out of sync or significantly slower than the next best alternative
       if (this.activeHost.outOfSync || this.activeHost.unreachable || (this.activeHost !== rankOrder[0] && rankOrder[0].preferred) || (!this.activeHost.preferred && this.activeHost.rtt > (rankOrder[0].rtt * 2) + 50)) {
@@ -183,7 +224,6 @@ class FailoverRouter {
 
   // depose the active host and choose the next best replacement
   private electHost(): void {
-    this.activeHost.outOfSync = true;
     this.activeHost.failures = 0;
     const rankOrder = this.sortHosts();
     this.activeHost = rankOrder[0];
@@ -194,6 +234,7 @@ class FailoverRouter {
     host.failures++;
     if (host.failures > 5 && this.multihost) {
       logger.warn(`🚨🚨🚨 Too many esplora failures on ${this.activeHost.host}, falling back to next best alternative 🚨🚨🚨`);
+      this.activeHost.unreachable = true;
       this.electHost();
       return this.activeHost;
     } else {
@@ -201,6 +242,47 @@ class FailoverRouter {
     }
   }
 
+  // methods for retrieving git hashes by host
+  private async $updateFrontendGitHash(host: FailoverHost): Promise<void> {
+    try {
+      const url = `${host.publicDomain}/resources/config.js`;
+      const response = await this.pollConnection.get<string>(url, { timeout: config.ESPLORA.FALLBACK_TIMEOUT });
+      const match = response.data.match(/GIT_COMMIT_HASH\s*=\s*['"](.*?)['"]/);
+      if (match && match[1]?.length) {
+        host.hashes.frontend = match[1];
+      }
+    } catch (e) {
+      // failed to get frontend build hash - do nothing
+    }
+  }
+
+  private async $updateBackendGitHash(host: FailoverHost): Promise<void> {
+    try {
+      const url = `${host.publicDomain}/api/v1/backend-info`;
+      const response = await this.pollConnection.get<any>(url, { timeout: config.ESPLORA.FALLBACK_TIMEOUT });
+      if (response.data?.gitCommit) {
+        host.hashes.backend = response.data.gitCommit;
+      }
+    } catch (e) {
+      // failed to get backend build hash - do nothing
+    }
+  }
+
+  // returns the public mempool domain corresponding to an esplora server url
+  // (a bit of a hack to avoid manually specifying frontend & backend URLs for each esplora server)
+  private extractPublicDomain(url: string): string {
+    // force the url to start with a valid protocol
+    const urlWithProtocol = url.startsWith('http') ? url : `https://${url}`;
+    // parse as URL and extract the hostname
+    try {
+      const parsed = new URL(urlWithProtocol);
+      return parsed.hostname;
+    } catch (e) {
+      // fallback to the original url
+      return url;
+    }
+  }
+
   private async $query<T>(method: 'get'| 'post', path, data: any, responseType = 'json', host = this.activeHost, retry: boolean = true): Promise<T> {
     let axiosConfig;
     let url;
@@ -304,7 +386,7 @@ class ElectrsApi implements AbstractBitcoinApi {
   }
 
   $getAddress(address: string): Promise<IEsploraApi.Address> {
-    throw new Error('Method getAddress not implemented.');
+    return this.failoverRouter.$get<IEsploraApi.Address>('/address/' + address);
   }
 
   $getAddressTransactions(address: string, txId?: string): Promise<IEsploraApi.Transaction[]> {
@@ -327,6 +409,14 @@ class ElectrsApi implements AbstractBitcoinApi {
     throw new Error('Method not implemented.');
   }
 
+  $testMempoolAccept(rawTransactions: string[], maxfeerate?: number): Promise<TestMempoolAcceptResult[]> {
+    throw new Error('Method not implemented.');
+  }
+
+  $submitPackage(rawTransactions: string[]): Promise<SubmitPackageResult> {
+    throw new Error('Method not implemented.');
+  }
+
   $getOutspend(txId: string, vout: number): Promise<IEsploraApi.Outspend> {
     return this.failoverRouter.$get<IEsploraApi.Outspend>('/tx/' + txId + '/outspend/' + vout);
   }
@@ -347,6 +437,15 @@ class ElectrsApi implements AbstractBitcoinApi {
     return this.failoverRouter.$post<IEsploraApi.Outspend[]>('/internal/txs/outspends/by-outpoint', outpoints.map(out => `${out.txid}:${out.vout}`), 'json');
   }
 
+  async $getCoinbaseTx(blockhash: string): Promise<IEsploraApi.Transaction> {
+    const txid = await this.failoverRouter.$get<string>(`/block/${blockhash}/txid/0`);
+    return this.failoverRouter.$get<IEsploraApi.Transaction>('/tx/' + txid);
+  }
+
+  async $getAddressTransactionSummary(address: string): Promise<IEsploraApi.AddressTxSummary[]> {
+    return this.failoverRouter.$get<IEsploraApi.AddressTxSummary[]>('/address/' + address + '/txs/summary');
+  }
+
   public startHealthChecks(): void {
     this.failoverRouter.startHealthChecks();
   }
@@ -363,6 +462,7 @@ class ElectrsApi implements AbstractBitcoinApi {
         unreachable: !!host.unreachable,
         checked: !!host.checked,
         lastChecked: host.lastChecked || 0,
+        hashes: host.hashes,
       }));
     } else {
       return [];

backend/src/api/blocks.ts

@@ -2,7 +2,7 @@ import config from '../config';
 import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory';
 import logger from '../logger';
 import memPool from './mempool';
-import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionMinerInfo, CpfpSummary, MempoolTransactionExtended, TransactionClassified, BlockAudit } from '../mempool.interfaces';
+import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionMinerInfo, CpfpSummary, MempoolTransactionExtended, TransactionClassified, BlockAudit, TransactionAudit } from '../mempool.interfaces';
 import { Common } from './common';
 import diskCache from './disk-cache';
 import transactionUtils from './transaction-utils';
@@ -30,6 +30,11 @@ import redisCache from './redis-cache';
 import rbfCache from './rbf-cache';
 import { calcBitsDifference } from './difficulty-adjustment';
 import AccelerationRepository from '../repositories/AccelerationRepository';
+import { calculateFastBlockCpfp, calculateGoodBlockCpfp } from './cpfp';
+import mempool from './mempool';
+import CpfpRepository from '../repositories/CpfpRepository';
+import { parseDATUMTemplateCreator } from '../utils/bitcoin-script';
+import database from '../database';
 
 class Blocks {
   private blocks: BlockExtended[] = [];
@@ -215,10 +220,10 @@ class Blocks {
     };
   }
 
-  public summarizeBlockTransactions(hash: string, transactions: TransactionExtended[]): BlockSummary {
+  public summarizeBlockTransactions(hash: string, height: number, transactions: TransactionExtended[]): BlockSummary {
     return {
       id: hash,
-      transactions: Common.classifyTransactions(transactions),
+      transactions: Common.classifyTransactions(transactions, height),
     };
   }
 
@@ -295,10 +300,12 @@ class Blocks {
     extras.virtualSize = block.weight / 4.0;
     if (coinbaseTx?.vout.length > 0) {
       extras.coinbaseAddress = coinbaseTx.vout[0].scriptpubkey_address ?? null;
+      extras.coinbaseAddresses = [...new Set<string>(coinbaseTx.vout.map(v => v.scriptpubkey_address).filter(a => a) as string[])];
       extras.coinbaseSignature = coinbaseTx.vout[0].scriptpubkey_asm ?? null;
       extras.coinbaseSignatureAscii = transactionUtils.hex2ascii(coinbaseTx.vin[0].scriptsig) ?? null;
     } else {
       extras.coinbaseAddress = null;
+      extras.coinbaseAddresses = null;
       extras.coinbaseSignature = null;
       extras.coinbaseSignatureAscii = null;
     }
@@ -336,7 +343,12 @@ class Blocks {
           id: pool.uniqueId,
           name: pool.name,
           slug: pool.slug,
+          minerNames: null,
         };
+
+        if (extras.pool.name === 'OCEAN') {
+          extras.pool.minerNames = parseDATUMTemplateCreator(extras.coinbaseRaw);
+        }
       }
 
       extras.matchRate = null;
@@ -370,8 +382,7 @@ class Blocks {
       }
     }
 
-    const asciiScriptSig = transactionUtils.hex2ascii(txMinerInfo.vin[0].scriptsig);
-    const addresses = txMinerInfo.vout.map((vout) => vout.scriptpubkey_address).filter((address) => address);
+    const addresses = txMinerInfo.vout.map((vout) => vout.scriptpubkey_address).filter(address => address) as string[];
 
     let pools: PoolTag[] = [];
     if (config.DATABASE.ENABLED === true) {
@@ -380,26 +391,9 @@ class Blocks {
       pools = poolsParser.miningPools;
     }
 
-    for (let i = 0; i < pools.length; ++i) {
-      if (addresses.length) {
-        const poolAddresses: string[] = typeof pools[i].addresses === 'string' ?
-          JSON.parse(pools[i].addresses) : pools[i].addresses;
-        for (let y = 0; y < poolAddresses.length; y++) {
-          if (addresses.indexOf(poolAddresses[y]) !== -1) {
-            return pools[i];
-          }
-        }
-      }
-
-      const regexes: string[] = typeof pools[i].regexes === 'string' ?
-        JSON.parse(pools[i].regexes) : pools[i].regexes;
-      for (let y = 0; y < regexes.length; ++y) {
-        const regex = new RegExp(regexes[y], 'i');
-        const match = asciiScriptSig.match(regex);
-        if (match !== null) {
-          return pools[i];
-        }
-      }
+    const pool = poolsParser.matchBlockMiner(txMinerInfo.vin[0].scriptsig, addresses || [], pools);
+    if (pool) {
+      return pool;
     }
 
     if (config.DATABASE.ENABLED === true) {
@@ -418,8 +412,16 @@ class Blocks {
     }
 
     try {
+      const blockchainInfo = await bitcoinClient.getBlockchainInfo();
+      const currentBlockHeight = blockchainInfo.blocks;
+      let indexingBlockAmount = Math.min(config.MEMPOOL.INDEXING_BLOCKS_AMOUNT, currentBlockHeight);
+      if (indexingBlockAmount <= -1) {
+        indexingBlockAmount = currentBlockHeight + 1;
+      }
+      const lastBlockToIndex = Math.max(0, currentBlockHeight - indexingBlockAmount + 1);
+
       // Get all indexed block hash
-      const indexedBlocks = await blocksRepository.$getIndexedBlocks();
+      const indexedBlocks = (await blocksRepository.$getIndexedBlocks()).filter(block => block.height >= lastBlockToIndex);
       const indexedBlockSummariesHashesArray = await BlocksSummariesRepository.$getIndexedSummariesId();
 
       const indexedBlockSummariesHashes = {}; // Use a map for faster seek during the indexing loop
@@ -452,7 +454,7 @@ class Blocks {
 
 
         if (config.MEMPOOL.BACKEND === 'esplora') {
-          const txs = (await bitcoinApi.$getTxsForBlock(block.hash)).map(tx => transactionUtils.extendTransaction(tx));
+          const txs = (await bitcoinApi.$getTxsForBlock(block.hash)).map(tx => transactionUtils.extendMempoolTransaction(tx));
           const cpfpSummary = await this.$indexCPFP(block.hash, block.height, txs);
           if (cpfpSummary) {
             await this.$getStrippedBlockTransactions(block.hash, true, true, cpfpSummary, block.height); // This will index the block summary
@@ -583,8 +585,11 @@ class Blocks {
     const blockchainInfo = await bitcoinClient.getBlockchainInfo();
     const currentBlockHeight = blockchainInfo.blocks;
 
-    const unclassifiedBlocksList = await BlocksSummariesRepository.$getSummariesWithVersion(0);
-    const unclassifiedTemplatesList = await BlocksSummariesRepository.$getTemplatesWithVersion(0);
+    const targetSummaryVersion: number = 1;
+    const targetTemplateVersion: number = 1;
+
+    const unclassifiedBlocksList = await BlocksSummariesRepository.$getSummariesBelowVersion(targetSummaryVersion);
+    const unclassifiedTemplatesList = await BlocksSummariesRepository.$getTemplatesBelowVersion(targetTemplateVersion);
 
     // nothing to do
     if (!unclassifiedBlocksList?.length && !unclassifiedTemplatesList?.length) {
@@ -617,16 +622,24 @@ class Blocks {
 
     for (let height = currentBlockHeight; height >= 0; height--) {
       try {
-        let txs: TransactionExtended[] | null = null;
+        let txs: MempoolTransactionExtended[] | null = null;
         if (unclassifiedBlocks[height]) {
           const blockHash = unclassifiedBlocks[height];
           // fetch transactions
-          txs = (await bitcoinApi.$getTxsForBlock(blockHash)).map(tx => transactionUtils.extendTransaction(tx)) || [];
+          txs = (await bitcoinApi.$getTxsForBlock(blockHash)).map(tx => transactionUtils.extendMempoolTransaction(tx)) || [];
           // add CPFP
-          const cpfpSummary = Common.calculateCpfp(height, txs, true);
+          const cpfpSummary = calculateGoodBlockCpfp(height, txs, []);
           // classify
-          const { transactions: classifiedTxs } = this.summarizeBlockTransactions(blockHash, cpfpSummary.transactions);
-          await BlocksSummariesRepository.$saveTransactions(height, blockHash, classifiedTxs, 1);
+          const { transactions: classifiedTxs } = this.summarizeBlockTransactions(blockHash, height, cpfpSummary.transactions);
+          await BlocksSummariesRepository.$saveTransactions(height, blockHash, classifiedTxs, 2);
+          if (unclassifiedBlocks[height].version < 2 && targetSummaryVersion === 2) {
+            const cpfpClusters = await CpfpRepository.$getClustersAt(height);
+            if (!cpfpRepository.compareClusters(cpfpClusters, cpfpSummary.clusters)) {
+              // CPFP clusters changed - update the compact_cpfp tables
+              await CpfpRepository.$deleteClustersAt(height);
+              await this.$saveCpfp(blockHash, height, cpfpSummary);
+            }
+          }
           await Common.sleep$(250);
         }
         if (unclassifiedTemplates[height]) {
@@ -652,9 +665,9 @@ class Blocks {
               }
               templateTxs.push(tx || templateTx);
             }
-            const cpfpSummary = Common.calculateCpfp(height, templateTxs?.filter(tx => tx['effectiveFeePerVsize'] != null) as TransactionExtended[], true);
+            const cpfpSummary = calculateGoodBlockCpfp(height, templateTxs?.filter(tx => tx['effectiveFeePerVsize'] != null) as MempoolTransactionExtended[], []);
             // classify
-            const { transactions: classifiedTxs } = this.summarizeBlockTransactions(blockHash, cpfpSummary.transactions);
+            const { transactions: classifiedTxs } = this.summarizeBlockTransactions(blockHash, height, cpfpSummary.transactions);
             const classifiedTxMap: { [txid: string]: TransactionClassified } = {};
             for (const tx of classifiedTxs) {
               classifiedTxMap[tx.txid] = tx;
@@ -690,6 +703,52 @@ class Blocks {
     this.classifyingBlocks = false;
   }
 
+  /**
+   * [INDEXING] Index missing coinbase addresses for all blocks
+   */
+  public async $indexCoinbaseAddresses(): Promise<void> {
+    try {
+      // Get all indexed block hash
+      const unindexedBlocks = await blocksRepository.$getBlocksWithoutCoinbaseAddresses();
+
+      if (!unindexedBlocks?.length) {
+        return;
+      }
+
+      logger.info(`Indexing missing coinbase addresses for ${unindexedBlocks.length} blocks`);
+
+      // Logging
+      let count = 0;
+      let countThisRun = 0;
+      let timer = Date.now() / 1000;
+      const startedAt = Date.now() / 1000;
+      for (const { height, hash } of unindexedBlocks) {
+        // Logging
+        const elapsedSeconds = (Date.now() / 1000) - timer;
+        if (elapsedSeconds > 5) {
+          const runningFor = (Date.now() / 1000) - startedAt;
+          const blockPerSeconds = countThisRun / elapsedSeconds;
+          const progress = Math.round(count / unindexedBlocks.length * 10000) / 100;
+          logger.debug(`Indexing coinbase addresses for #${height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${count}/${unindexedBlocks.length} (${progress}%) | elapsed: ${runningFor.toFixed(2)} seconds`);
+          timer = Date.now() / 1000;
+          countThisRun = 0;
+        }
+
+        const coinbaseTx = await bitcoinApi.$getCoinbaseTx(hash);
+        const addresses = new Set<string>(coinbaseTx.vout.map(v => v.scriptpubkey_address).filter(a => a) as string[]);
+        await blocksRepository.$saveCoinbaseAddresses(hash, [...addresses]);
+
+        // Logging
+        count++;
+        countThisRun++;
+      }
+      logger.notice(`coinbase addresses indexing completed: indexed ${count} blocks`);
+    } catch (e) {
+      logger.err(`coinbase addresses indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`);
+      throw e;
+    }
+  }
+
   /**
    * [INDEXING] Index all blocks metadata for the mining dashboard
    */
@@ -839,8 +898,11 @@ class Blocks {
       } else {
         this.currentBlockHeight++;
         logger.debug(`New block found (#${this.currentBlockHeight})!`);
-        this.updateTimerProgress(timer, `getting orphaned blocks for ${this.currentBlockHeight}`);
-        await chainTips.updateOrphanedBlocks();
+        // skip updating the orphan block cache if we've fallen behind the chain tip
+        if (this.currentBlockHeight >= blockHeightTip - 2) {
+          this.updateTimerProgress(timer, `getting orphaned blocks for ${this.currentBlockHeight}`);
+          await chainTips.updateOrphanedBlocks();
+        }
       }
 
       this.updateTimerProgress(timer, `getting block data for ${this.currentBlockHeight}`);
@@ -857,9 +919,14 @@ class Blocks {
         }
       }
 
-      const cpfpSummary: CpfpSummary = Common.calculateCpfp(block.height, transactions);
+      let accelerations = Object.values(mempool.getAccelerations());
+      if (accelerations?.length > 0) {
+        const pool = await this.$findBlockMiner(transactionUtils.stripCoinbaseTransaction(transactions[0]));
+        accelerations = accelerations.filter(a => a.pools.includes(pool.uniqueId));
+      }
+      const cpfpSummary: CpfpSummary = calculateGoodBlockCpfp(block.height, transactions, accelerations.map(a => ({ txid: a.txid, max_bid: a.feeDelta })));
       const blockExtended: BlockExtended = await this.$getBlockExtended(block, cpfpSummary.transactions);
-      const blockSummary: BlockSummary = this.summarizeBlockTransactions(block.id, cpfpSummary.transactions);
+      const blockSummary: BlockSummary = this.summarizeBlockTransactions(block.id, block.height, cpfpSummary.transactions);
       this.updateTimerProgress(timer, `got block data for ${this.currentBlockHeight}`);
 
       if (Common.indexingEnabled()) {
@@ -880,12 +947,12 @@ class Blocks {
               const newBlock = await this.$indexBlock(lastBlock.height - i);
               this.blocks.push(newBlock);
               this.updateTimerProgress(timer, `reindexed block`);
-              let cpfpSummary;
+              let newCpfpSummary;
               if (config.MEMPOOL.CPFP_INDEXING) {
-                cpfpSummary = await this.$indexCPFP(newBlock.id, lastBlock.height - i);
+                newCpfpSummary = await this.$indexCPFP(newBlock.id, lastBlock.height - i);
                 this.updateTimerProgress(timer, `reindexed block cpfp`);
               }
-              await this.$getStrippedBlockTransactions(newBlock.id, true, true, cpfpSummary, newBlock.height);
+              await this.$getStrippedBlockTransactions(newBlock.id, true, true, newCpfpSummary, newBlock.height);
               this.updateTimerProgress(timer, `reindexed block summary`);
             }
             await mining.$indexDifficultyAdjustments();
@@ -934,7 +1001,7 @@ class Blocks {
 
       // start async callbacks
       this.updateTimerProgress(timer, `starting async callbacks for ${this.currentBlockHeight}`);
-      const callbackPromises = this.newAsyncBlockCallbacks.map((cb) => cb(blockExtended, txIds, transactions));
+      const callbackPromises = this.newAsyncBlockCallbacks.map((cb) => cb(blockExtended, txIds, cpfpSummary.transactions));
 
       if (block.height % 2016 === 0) {
         if (Common.indexingEnabled()) {
@@ -1116,7 +1183,7 @@ class Blocks {
         transactions: cpfpSummary.transactions.map(tx => {
           let flags: number = 0;
           try {
-            flags = tx.flags || Common.getTransactionFlags(tx);
+            flags = Common.getTransactionFlags(tx, height);
           } catch (e) {
             logger.warn('Failed to classify transaction: ' + (e instanceof Error ? e.message : e));
           }
@@ -1131,11 +1198,11 @@ class Blocks {
           };
         }),
       };
-      summaryVersion = 1;
+      summaryVersion = cpfpSummary.version;
     } else {
       if (config.MEMPOOL.BACKEND === 'esplora') {
         const txs = (await bitcoinApi.$getTxsForBlock(hash)).map(tx => transactionUtils.extendTransaction(tx));
-        summary = this.summarizeBlockTransactions(hash, txs);
+        summary = this.summarizeBlockTransactions(hash, height || 0, txs);
         summaryVersion = 1;
       } else {
         // Call Core RPC
@@ -1157,6 +1224,11 @@ class Blocks {
     return summary.transactions;
   }
 
+  public async $getSingleTxFromSummary(hash: string, txid: string): Promise<TransactionClassified | null> {
+    const txs = await this.$getStrippedBlockTransactions(hash);
+    return txs.find(tx => tx.txid === txid) || null;
+  }
+
   /**
    * Get 15 blocks
    * 
@@ -1256,6 +1328,7 @@ class Blocks {
         utxoset_size: block.extras.utxoSetSize ?? null,
         coinbase_raw: block.extras.coinbaseRaw ?? null,
         coinbase_address: block.extras.coinbaseAddress ?? null,
+        coinbase_addresses: block.extras.coinbaseAddresses ?? null,
         coinbase_signature: block.extras.coinbaseSignature ?? null,
         coinbase_signature_ascii: block.extras.coinbaseSignatureAscii ?? null,
         pool_slug: block.extras.pool.slug ?? null,
@@ -1270,7 +1343,7 @@ class Blocks {
           let summaryVersion = 0;
           if (config.MEMPOOL.BACKEND === 'esplora') {
             const txs = (await bitcoinApi.$getTxsForBlock(cleanBlock.hash)).map(tx => transactionUtils.extendTransaction(tx));
-            summary = this.summarizeBlockTransactions(cleanBlock.hash, txs);
+            summary = this.summarizeBlockTransactions(cleanBlock.hash, cleanBlock.height, txs);
             summaryVersion = 1;
           } else {
             // Call Core RPC
@@ -1325,6 +1398,14 @@ class Blocks {
     }
   }
 
+  public async $getBlockTxAuditSummary(hash: string, txid: string): Promise<TransactionAudit | null> {
+    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
+      return BlocksAuditsRepository.$getBlockTxAudit(hash, txid);
+    } else {
+      return null;
+    }
+  }
+
   public getLastDifficultyAdjustmentTime(): number {
     return this.lastDifficultyAdjustmentTime;
   }
@@ -1341,11 +1422,11 @@ class Blocks {
     return this.currentBlockHeight;
   }
 
-  public async $indexCPFP(hash: string, height: number, txs?: TransactionExtended[]): Promise<CpfpSummary | null> {
+  public async $indexCPFP(hash: string, height: number, txs?: MempoolTransactionExtended[]): Promise<CpfpSummary | null> {
     let transactions = txs;
     if (!transactions) {
       if (config.MEMPOOL.BACKEND === 'esplora') {
-        transactions = (await bitcoinApi.$getTxsForBlock(hash)).map(tx => transactionUtils.extendTransaction(tx));
+        transactions = (await bitcoinApi.$getTxsForBlock(hash)).map(tx => transactionUtils.extendMempoolTransaction(tx));
       }
       if (!transactions) {
         const block = await bitcoinClient.getBlock(hash, 2);
@@ -1357,7 +1438,7 @@ class Blocks {
     }
 
     if (transactions?.length != null) {
-      const summary = Common.calculateCpfp(height, transactions as TransactionExtended[]);
+      const summary = calculateFastBlockCpfp(height, transactions);
 
       await this.$saveCpfp(hash, height, summary);
 
@@ -1381,6 +1462,36 @@ class Blocks {
       // not a fatal error, we'll try again next time the indexer runs
     }
   }
+
+  public async $getBlockDefinitionHashes(): Promise<string[] | null> {
+    try {
+      const [rows]: any = await database.query(`SELECT DISTINCT(definition_hash) FROM blocks`);
+      if (rows && Array.isArray(rows)) {
+        return rows.map(r => r.definition_hash);
+      } else {
+        logger.debug(`Unable to retreive list of blocks.definition_hash from db (no result)`);
+        return null;
+      }
+    } catch (e) {
+      logger.debug(`Unable to retreive list of blocks.definition_hash from db (exception: ${e})`);
+      return null;
+    }
+  }
+
+  public async $getBlocksByDefinitionHash(definitionHash: string): Promise<string[] | null> {
+    try {
+      const [rows]: any = await database.query(`SELECT hash FROM blocks WHERE definition_hash = ?`, [definitionHash]);
+      if (rows && Array.isArray(rows)) {
+        return rows.map(r => r.hash);
+      } else {
+        logger.debug(`Unable to retreive list of blocks for definition hash ${definitionHash} from db (no result)`);
+        return null;
+      }
+    } catch (e) {
+      logger.debug(`Unable to retreive list of blocks for definition hash ${definitionHash} from db (exception: ${e})`);
+      return null;
+    }
+  }
 }
 
 export default new Blocks();

backend/src/api/chain-tips.ts

@@ -12,32 +12,68 @@ export interface OrphanedBlock {
   height: number;
   hash: string;
   status: 'valid-fork' | 'valid-headers' | 'headers-only';
+  prevhash: string;
 }
 
 class ChainTips {
   private chainTips: ChainTip[] = [];
-  private orphanedBlocks: OrphanedBlock[] = [];
+  private orphanedBlocks: { [hash: string]: OrphanedBlock } = {};
+  private blockCache: { [hash: string]: OrphanedBlock } = {};
+  private orphansByHeight: { [height: number]: OrphanedBlock[] } = {};
 
   public async updateOrphanedBlocks(): Promise<void> {
     try {
       this.chainTips = await bitcoinClient.getChainTips();
-      this.orphanedBlocks = [];
+
+      const start = Date.now();
+      const breakAt = start + 10000;
+      let newOrphans = 0;
+      this.orphanedBlocks = {};
 
       for (const chain of this.chainTips) {
         if (chain.status === 'valid-fork' || chain.status === 'valid-headers') {
-          let block = await bitcoinClient.getBlock(chain.hash);
-          while (block && block.confirmations === -1) {
-            this.orphanedBlocks.push({
-              height: block.height,
-              hash: block.hash,
-              status: chain.status
-            });
-            block = await bitcoinClient.getBlock(block.previousblockhash);
+          const orphans: OrphanedBlock[] = [];
+          let hash = chain.hash;
+          do {
+            let orphan = this.blockCache[hash];
+            if (!orphan) {
+              const block = await bitcoinClient.getBlock(hash);
+              if (block && block.confirmations === -1) {
+                newOrphans++;
+                orphan = {
+                  height: block.height,
+                  hash: block.hash,
+                  status: chain.status,
+                  prevhash: block.previousblockhash,
+                };
+                this.blockCache[hash] = orphan;
+              }
+            }
+            if (orphan) {
+              orphans.push(orphan);
+            }
+            hash = orphan?.prevhash;
+          } while (hash && (Date.now() < breakAt));
+          for (const orphan of orphans) {
+            this.orphanedBlocks[orphan.hash] = orphan;
           }
         }
+        if (Date.now() >= breakAt) {
+          logger.debug(`Breaking orphaned blocks updater after 10s, will continue next block`);
+          break;
+        }
       }
 
-      logger.debug(`Updated orphaned blocks cache. Found ${this.orphanedBlocks.length} orphaned blocks`);
+      this.orphansByHeight = {};
+      const allOrphans = Object.values(this.orphanedBlocks);
+      for (const orphan of allOrphans) {
+        if (!this.orphansByHeight[orphan.height]) {
+          this.orphansByHeight[orphan.height] = [];
+        }
+        this.orphansByHeight[orphan.height].push(orphan);
+      }
+
+      logger.debug(`Updated orphaned blocks cache. Fetched ${newOrphans} new orphaned blocks. Total ${allOrphans.length}`);
     } catch (e) {
       logger.err(`Cannot get fetch orphaned blocks. Reason: ${e instanceof Error ? e.message : e}`);
     }
@@ -48,13 +84,7 @@ class ChainTips {
       return [];
     }
 
-    const orphans: OrphanedBlock[] = [];
-    for (const block of this.orphanedBlocks) {
-      if (block.height === height) {
-        orphans.push(block);
-      }
-    }
-    return orphans;
+    return this.orphansByHeight[height] || [];
   }
 }
 

backend/src/api/common.ts

@@ -1,6 +1,6 @@
 import * as bitcoinjs from 'bitcoinjs-lib';
 import { Request } from 'express';
-import { CpfpInfo, CpfpSummary, CpfpCluster, EffectiveFeeStats, MempoolBlockWithTransactions, TransactionExtended, MempoolTransactionExtended, TransactionStripped, WorkingEffectiveFeeStats, TransactionClassified, TransactionFlags } from '../mempool.interfaces';
+import { EffectiveFeeStats, MempoolBlockWithTransactions, TransactionExtended, MempoolTransactionExtended, TransactionStripped, WorkingEffectiveFeeStats, TransactionClassified, TransactionFlags } from '../mempool.interfaces';
 import config from '../config';
 import { NodeSocket } from '../repositories/NodesSocketsRepository';
 import { isIP } from 'net';
@@ -10,7 +10,6 @@ import logger from '../logger';
 import { getVarIntLength, opcodes, parseMultisigScript } from '../utils/bitcoin-script';
 
 // Bitcoin Core default policy settings
-const TX_MAX_STANDARD_VERSION = 2;
 const MAX_STANDARD_TX_WEIGHT = 400_000;
 const MAX_BLOCK_SIGOPS_COST = 80_000;
 const MAX_STANDARD_TX_SIGOPS_COST = (MAX_BLOCK_SIGOPS_COST / 5);
@@ -80,8 +79,8 @@ export class Common {
     return arr;
   }
 
-  static findRbfTransactions(added: MempoolTransactionExtended[], deleted: MempoolTransactionExtended[], forceScalable = false): { [txid: string]: MempoolTransactionExtended[] } {
-    const matches: { [txid: string]: MempoolTransactionExtended[] } = {};
+  static findRbfTransactions(added: MempoolTransactionExtended[], deleted: MempoolTransactionExtended[], forceScalable = false): { [txid: string]: { replaced: MempoolTransactionExtended[], replacedBy: TransactionExtended }} {
+    const matches: { [txid: string]: { replaced: MempoolTransactionExtended[], replacedBy: TransactionExtended }} = {};
 
     // For small N, a naive nested loop is extremely fast, but it doesn't scale
     if (added.length < 1000 && deleted.length < 50 && !forceScalable) {
@@ -96,7 +95,7 @@ export class Common {
               addedTx.vin.some((vin) => vin.txid === deletedVin.txid && vin.vout === deletedVin.vout));
             });
         if (foundMatches?.length) {
-          matches[addedTx.txid] = [...new Set(foundMatches)];
+          matches[addedTx.txid] = { replaced: [...new Set(foundMatches)], replacedBy: addedTx };
         }
       });
     } else {
@@ -124,7 +123,7 @@ export class Common {
             foundMatches.add(deletedTx);
           }
           if (foundMatches.size) {
-            matches[addedTx.txid] = [...foundMatches];
+            matches[addedTx.txid] = { replaced: [...foundMatches], replacedBy: addedTx };
           }
         }
       }
@@ -139,17 +138,17 @@ export class Common {
       const replaced: Set<MempoolTransactionExtended> = new Set();
       for (let i = 0; i < tx.vin.length; i++) {
         const vin = tx.vin[i];
-        const match = spendMap.get(`${vin.txid}:${vin.vout}`);
+        const key = `${vin.txid}:${vin.vout}`;
+        const match = spendMap.get(key);
         if (match && match.txid !== tx.txid) {
           replaced.add(match);
           // remove this tx from the spendMap
           // prevents the same tx being replaced more than once
           for (const replacedVin of match.vin) {
-            const key = `${replacedVin.txid}:${replacedVin.vout}`;
-            spendMap.delete(key);
+            const replacedKey = `${replacedVin.txid}:${replacedVin.vout}`;
+            spendMap.delete(replacedKey);
           }
         }
-        const key = `${vin.txid}:${vin.vout}`;
         spendMap.delete(key);
       }
       if (replaced.size) {
@@ -200,10 +199,13 @@ export class Common {
    *
    * returns true early if any standardness rule is violated, otherwise false
    * (except for non-mandatory-script-verify-flag and p2sh script evaluation rules which are *not* enforced)
+   *
+   * As standardness rules change, we'll need to apply the rules in force *at the time* to older blocks.
+   * For now, just pull out individual rules into versioned functions where necessary.
    */
-  static isNonStandard(tx: TransactionExtended): boolean {
+  static isNonStandard(tx: TransactionExtended, height?: number): boolean {
     // version
-    if (tx.version > TX_MAX_STANDARD_VERSION) {
+    if (this.isNonStandardVersion(tx, height)) {
       return true;
     }
 
@@ -250,6 +252,8 @@ export class Common {
         }
       } else if (['unknown', 'provably_unspendable', 'empty'].includes(vin.prevout?.scriptpubkey_type || '')) {
         return true;
+      } else if (this.isNonStandardAnchor(tx, height)) {
+        return true;
       }
       // TODO: bad-witness-nonstandard
     }
@@ -258,9 +262,15 @@ export class Common {
     let opreturnCount = 0;
     for (const vout of tx.vout) {
       // scriptpubkey
-      if (['unknown', 'provably_unspendable', 'empty'].includes(vout.scriptpubkey_type)) {
+      if (['nonstandard', 'provably_unspendable', 'empty'].includes(vout.scriptpubkey_type)) {
         // (non-standard output type)
         return true;
+      } else if (vout.scriptpubkey_type === 'unknown') {
+        // undefined segwit version/length combinations are actually standard in outputs
+        // https://github.com/bitcoin/bitcoin/blob/2c79abc7ad4850e9e3ba32a04c530155cda7f980/src/script/interpreter.cpp#L1950-L1951
+        if (vout.scriptpubkey.startsWith('00') || !this.isWitnessProgram(vout.scriptpubkey)) {
+          return true;
+        }
       } else if (vout.scriptpubkey_type === 'multisig') {
         if (!DEFAULT_PERMIT_BAREMULTISIG) {
           // bare-multisig
@@ -286,7 +296,7 @@ export class Common {
         dustSize += getVarIntLength(dustSize);
         // add value size
         dustSize += 8;
-        if (['v0_p2wpkh', 'v0_p2wsh', 'v1_p2tr'].includes(vout.scriptpubkey_type)) {
+        if (Common.isWitnessProgram(vout.scriptpubkey)) {
           dustSize += 67;
         } else {
           dustSize += 148;
@@ -308,6 +318,70 @@ export class Common {
     return false;
   }
 
+  // A witness program is any valid scriptpubkey that consists of a 1-byte push opcode
+  // followed by a data push between 2 and 40 bytes.
+  // https://github.com/bitcoin/bitcoin/blob/2c79abc7ad4850e9e3ba32a04c530155cda7f980/src/script/script.cpp#L224-L240
+  static isWitnessProgram(scriptpubkey: string): false | { version: number, program: string } {
+    if (scriptpubkey.length < 8 || scriptpubkey.length > 84) {
+      return false;
+    }
+    const version = parseInt(scriptpubkey.slice(0,2), 16);
+    if (version !== 0 && version < 0x51 || version > 0x60) {
+        return false;
+    }
+    const push = parseInt(scriptpubkey.slice(2,4), 16);
+    if (push + 2 === (scriptpubkey.length / 2)) {
+      return {
+        version: version ? version - 0x50 : 0,
+        program: scriptpubkey.slice(4),
+      };
+    }
+    return false;
+  }
+
+  // Individual versioned standardness rules
+
+  static V3_STANDARDNESS_ACTIVATION_HEIGHT = {
+    'testnet4': 42_000,
+    'testnet': 2_900_000,
+    'signet': 211_000,
+    '': 863_500,
+  };
+  static isNonStandardVersion(tx: TransactionExtended, height?: number): boolean {
+    let TX_MAX_STANDARD_VERSION = 3;
+    if (
+      height != null
+      && this.V3_STANDARDNESS_ACTIVATION_HEIGHT[config.MEMPOOL.NETWORK]
+      && height <= this.V3_STANDARDNESS_ACTIVATION_HEIGHT[config.MEMPOOL.NETWORK]
+    ) {
+      // V3 transactions were non-standard to spend before v28.x (scheduled for 2024/09/30 https://github.com/bitcoin/bitcoin/issues/29891)
+      TX_MAX_STANDARD_VERSION = 2;
+    }
+
+    if (tx.version > TX_MAX_STANDARD_VERSION) {
+      return true;
+    }
+    return false;
+  }
+
+  static ANCHOR_STANDARDNESS_ACTIVATION_HEIGHT = {
+    'testnet4': 42_000,
+    'testnet': 2_900_000,
+    'signet': 211_000,
+    '': 863_500,
+  };
+  static isNonStandardAnchor(tx: TransactionExtended, height?: number): boolean {
+    if (
+      height != null
+      && this.ANCHOR_STANDARDNESS_ACTIVATION_HEIGHT[config.MEMPOOL.NETWORK]
+      && height <= this.ANCHOR_STANDARDNESS_ACTIVATION_HEIGHT[config.MEMPOOL.NETWORK]
+    ) {
+      // anchor outputs were non-standard to spend before v28.x (scheduled for 2024/09/30 https://github.com/bitcoin/bitcoin/issues/29891)
+      return true;
+    }
+    return false;
+  }
+
   static getNonWitnessSize(tx: TransactionExtended): number {
     let weight = tx.weight;
     let hasWitness = false;
@@ -388,16 +462,19 @@ export class Common {
     return flags;
   }
 
-  static getTransactionFlags(tx: TransactionExtended): number {
+  static getTransactionFlags(tx: TransactionExtended, height?: number): number {
     let flags = tx.flags ? BigInt(tx.flags) : 0n;
 
     // Update variable flags (CPFP, RBF)
+    flags &= ~TransactionFlags.cpfp_child;
     if (tx.ancestors?.length) {
       flags |= TransactionFlags.cpfp_child;
     }
+    flags &= ~TransactionFlags.cpfp_parent;
     if (tx.descendants?.length) {
       flags |= TransactionFlags.cpfp_parent;
     }
+    flags &= ~TransactionFlags.replacement;
     if (tx.replacement) {
       flags |= TransactionFlags.replacement;
     }
@@ -433,11 +510,10 @@ export class Common {
           case 'v0_p2wpkh': flags |= TransactionFlags.p2wpkh; break;
           case 'v0_p2wsh': flags |= TransactionFlags.p2wsh; break;
           case 'v1_p2tr': {
-            if (!vin.witness?.length) {
-              throw new Error('Taproot input missing witness data');
-            }
             flags |= TransactionFlags.p2tr;
-            flags = Common.isInscription(vin, flags);
+            if (vin.witness?.length) {
+              flags = Common.isInscription(vin, flags);
+            }
           } break;
         }
       } else {
@@ -519,7 +595,7 @@ export class Common {
     if (hasFakePubkey) {
       flags |= TransactionFlags.fake_pubkey;
     }
-    
+
     // fast but bad heuristic to detect possible coinjoins
     // (at least 5 inputs and 5 outputs, less than half of which are unique amounts, with no address reuse)
     const addressReuse = Object.keys(reusedOutputAddresses).reduce((acc, key) => Math.max(acc, (reusedInputAddresses[key] || 0) + (reusedOutputAddresses[key] || 0)), 0) > 1;
@@ -535,17 +611,17 @@ export class Common {
       flags |= TransactionFlags.batch_payout;
     }
 
-    if (this.isNonStandard(tx)) {
+    if (this.isNonStandard(tx, height)) {
       flags |= TransactionFlags.nonstandard;
     }
 
     return Number(flags);
   }
 
-  static classifyTransaction(tx: TransactionExtended): TransactionClassified {
+  static classifyTransaction(tx: TransactionExtended, height?: number): TransactionClassified {
     let flags = 0;
     try {
-      flags = Common.getTransactionFlags(tx);
+      flags = Common.getTransactionFlags(tx, height);
     } catch (e) {
       logger.warn('Failed to add classification flags to transaction: ' + (e instanceof Error ? e.message : e));
     }
@@ -556,8 +632,8 @@ export class Common {
     };
   }
 
-  static classifyTransactions(txs: TransactionExtended[]): TransactionClassified[] {
-    return txs.map(Common.classifyTransaction);
+  static classifyTransactions(txs: TransactionExtended[], height?: number): TransactionClassified[] {
+    return txs.map(tx => Common.classifyTransaction(tx, height));
   }
 
   static stripTransaction(tx: TransactionExtended): TransactionStripped {
@@ -780,96 +856,6 @@ export class Common {
     }
   }
 
-  static calculateCpfp(height: number, transactions: TransactionExtended[], saveRelatives: boolean = false): CpfpSummary {
-    const clusters: CpfpCluster[] = []; // list of all cpfp clusters in this block
-    const clusterMap: { [txid: string]: CpfpCluster } = {}; // map transactions to their cpfp cluster
-    let clusterTxs: TransactionExtended[] = []; // working list of elements of the current cluster
-    let ancestors: { [txid: string]: boolean } = {}; // working set of ancestors of the current cluster root
-    const txMap: { [txid: string]: TransactionExtended } = {};
-    // initialize the txMap
-    for (const tx of transactions) {
-      txMap[tx.txid] = tx;
-    }
-    // reverse pass to identify CPFP clusters
-    for (let i = transactions.length - 1; i >= 0; i--) {
-      const tx = transactions[i];
-      if (!ancestors[tx.txid]) {
-        let totalFee = 0;
-        let totalVSize = 0;
-        clusterTxs.forEach(tx => {
-          totalFee += tx?.fee || 0;
-          totalVSize += (tx.weight / 4);
-        });
-        const effectiveFeePerVsize = totalFee / totalVSize;
-        let cluster: CpfpCluster;
-        if (clusterTxs.length > 1) {
-          cluster = {
-            root: clusterTxs[0].txid,
-            height,
-            txs: clusterTxs.map(tx => { return { txid: tx.txid, weight: tx.weight, fee: tx.fee || 0 }; }),
-            effectiveFeePerVsize,
-          };
-          clusters.push(cluster);
-        }
-        clusterTxs.forEach(tx => {
-          txMap[tx.txid].effectiveFeePerVsize = effectiveFeePerVsize;
-          if (cluster) {
-            clusterMap[tx.txid] = cluster;
-          }
-        });
-        // reset working vars
-        clusterTxs = [];
-        ancestors = {};
-      }
-      clusterTxs.push(tx);
-      tx.vin.forEach(vin => {
-        ancestors[vin.txid] = true;
-      });
-    }
-    // forward pass to enforce ancestor rate caps
-    for (const tx of transactions) {
-      let minAncestorRate = tx.effectiveFeePerVsize;
-      for (const vin of tx.vin) {
-        if (txMap[vin.txid]?.effectiveFeePerVsize) {
-          minAncestorRate = Math.min(minAncestorRate, txMap[vin.txid].effectiveFeePerVsize);
-        }
-      }
-      // check rounded values to skip cases with almost identical fees
-      const roundedMinAncestorRate = Math.ceil(minAncestorRate);
-      const roundedEffectiveFeeRate = Math.floor(tx.effectiveFeePerVsize);
-      if (roundedMinAncestorRate < roundedEffectiveFeeRate) {
-        tx.effectiveFeePerVsize = minAncestorRate;
-        if (!clusterMap[tx.txid]) {
-          // add a single-tx cluster to record the dependent rate
-          const cluster = {
-            root: tx.txid,
-            height,
-            txs: [{ txid: tx.txid, weight: tx.weight, fee: tx.fee || 0 }],
-            effectiveFeePerVsize: minAncestorRate,
-          };
-          clusterMap[tx.txid] = cluster;
-          clusters.push(cluster);
-        } else {
-          // update the existing cluster with the dependent rate
-          clusterMap[tx.txid].effectiveFeePerVsize = minAncestorRate;
-        }
-      }
-    }
-    if (saveRelatives) {
-      for (const cluster of clusters) {
-        cluster.txs.forEach((member, index) => {
-          txMap[member.txid].descendants = cluster.txs.slice(0, index).reverse();
-          txMap[member.txid].ancestors = cluster.txs.slice(index + 1).reverse();
-          txMap[member.txid].effectiveFeePerVsize = cluster.effectiveFeePerVsize;
-        });
-      }
-    }
-    return {
-      transactions,
-      clusters,
-    };
-  }
-
   static calcEffectiveFeeStatistics(transactions: { weight: number, fee: number, effectiveFeePerVsize?: number, txid: string, acceleration?: boolean }[]): EffectiveFeeStats {
     const sortedTxs = transactions.map(tx => { return { txid: tx.txid, weight: tx.weight, rate: tx.effectiveFeePerVsize || ((tx.fee || 0) / (tx.weight / 4)) }; }).sort((a, b) => a.rate - b.rate);
 
@@ -877,9 +863,10 @@ export class Common {
     let medianFee = 0;
     let medianWeight = 0;
 
-    // calculate the "medianFee" as the average fee rate of the middle 10000 weight units of transactions
-    const leftBound = 1995000;
-    const rightBound = 2005000;
+    // calculate the "medianFee" as the average fee rate of the middle 0.25% weight units of transactions
+    const halfWidth = config.MEMPOOL.BLOCK_WEIGHT_UNITS / 800;
+    const leftBound = Math.floor((config.MEMPOOL.BLOCK_WEIGHT_UNITS / 2) - halfWidth);
+    const rightBound = Math.ceil((config.MEMPOOL.BLOCK_WEIGHT_UNITS / 2) + halfWidth);
     for (let i = 0; i < sortedTxs.length && weightCount < rightBound; i++) {
       const left = weightCount;
       const right = weightCount + sortedTxs[i].weight;
@@ -946,6 +933,33 @@ export class Common {
     return this.validateTransactionHex(matches[1].toLowerCase());
   }
 
+  static getTransactionsFromRequest(req: Request, limit: number = 25): string[] {
+    if (!Array.isArray(req.body) || req.body.some(hex => typeof hex !== 'string')) {
+      throw Object.assign(new Error('Invalid request body (should be an array of hexadecimal strings)'), { code: -1 });
+    }
+
+    if (limit && req.body.length > limit) {
+      throw Object.assign(new Error('Exceeded maximum of 25 transactions'), { code: -1 });
+    }
+
+    const txs = req.body;
+
+    return txs.map(rawTx => {
+      // Support both upper and lower case hex
+      // Support both txHash= Form and direct API POST
+      const reg = /^((?:[a-fA-F0-9]{2})+)$/;
+      const matches = reg.exec(rawTx);
+      if (!matches || !matches[1]) {
+        throw Object.assign(new Error('Invalid hex string'), { code: -2 });
+      }
+
+      // Guaranteed to be a hex string of multiple of 2
+      // Guaranteed to be lower case
+      // Guaranteed to pass validation (see function below)
+      return this.validateTransactionHex(matches[1].toLowerCase());
+    });
+  }
+
   private static validateTransactionHex(txhex: string): string {
     // Do not mutate txhex
 

backend/src/api/cpfp.ts

@@ -1,29 +1,174 @@
-import { CpfpInfo, MempoolTransactionExtended } from '../mempool.interfaces';
+import { Ancestor, CpfpCluster, CpfpInfo, CpfpSummary, MempoolTransactionExtended, TransactionExtended } from '../mempool.interfaces';
+import { GraphTx, convertToGraphTx, expandRelativesGraph, initializeRelatives, makeBlockTemplate, mempoolComparator, removeAncestors, setAncestorScores } from './mini-miner';
 import memPool from './mempool';
+import { Acceleration } from './acceleration/acceleration';
 
 const CPFP_UPDATE_INTERVAL = 60_000; // update CPFP info at most once per 60s per transaction
-const MAX_GRAPH_SIZE = 50; // the maximum number of in-mempool relatives to consider
+const MAX_CLUSTER_ITERATIONS = 100;
 
-interface GraphTx extends MempoolTransactionExtended {
-  depends: string[];
-  spentby: string[];
-  ancestorMap: Map<string, GraphTx>;
-  fees: {
-    base: number;
-    ancestor: number;
+export function calculateFastBlockCpfp(height: number, transactions: MempoolTransactionExtended[], saveRelatives: boolean = false): CpfpSummary {
+  const clusters: CpfpCluster[] = []; // list of all cpfp clusters in this block
+  const clusterMap: { [txid: string]: CpfpCluster } = {}; // map transactions to their cpfp cluster
+  let clusterTxs: TransactionExtended[] = []; // working list of elements of the current cluster
+  let ancestors: { [txid: string]: boolean } = {}; // working set of ancestors of the current cluster root
+  const txMap: { [txid: string]: TransactionExtended } = {};
+  // initialize the txMap
+  for (const tx of transactions) {
+    txMap[tx.txid] = tx;
+  }
+  // reverse pass to identify CPFP clusters
+  for (let i = transactions.length - 1; i >= 0; i--) {
+    const tx = transactions[i];
+    if (!ancestors[tx.txid]) {
+      let totalFee = 0;
+      let totalVSize = 0;
+      clusterTxs.forEach(tx => {
+        totalFee += tx?.fee || 0;
+        totalVSize += (tx.weight / 4);
+      });
+      const effectiveFeePerVsize = totalFee / totalVSize;
+      let cluster: CpfpCluster;
+      if (clusterTxs.length > 1) {
+        cluster = {
+          root: clusterTxs[0].txid,
+          height,
+          txs: clusterTxs.map(tx => { return { txid: tx.txid, weight: tx.weight, fee: tx.fee || 0 }; }),
+          effectiveFeePerVsize,
+        };
+        clusters.push(cluster);
+      }
+      clusterTxs.forEach(tx => {
+        txMap[tx.txid].effectiveFeePerVsize = effectiveFeePerVsize;
+        if (cluster) {
+          clusterMap[tx.txid] = cluster;
+        }
+      });
+      // reset working vars
+      clusterTxs = [];
+      ancestors = {};
+    }
+    clusterTxs.push(tx);
+    tx.vin.forEach(vin => {
+      ancestors[vin.txid] = true;
+    });
+  }
+  // forward pass to enforce ancestor rate caps
+  for (const tx of transactions) {
+    let minAncestorRate = tx.effectiveFeePerVsize;
+    for (const vin of tx.vin) {
+      if (txMap[vin.txid]?.effectiveFeePerVsize) {
+        minAncestorRate = Math.min(minAncestorRate, txMap[vin.txid].effectiveFeePerVsize);
+      }
+    }
+    // check rounded values to skip cases with almost identical fees
+    const roundedMinAncestorRate = Math.ceil(minAncestorRate);
+    const roundedEffectiveFeeRate = Math.floor(tx.effectiveFeePerVsize);
+    if (roundedMinAncestorRate < roundedEffectiveFeeRate) {
+      tx.effectiveFeePerVsize = minAncestorRate;
+      if (!clusterMap[tx.txid]) {
+        // add a single-tx cluster to record the dependent rate
+        const cluster = {
+          root: tx.txid,
+          height,
+          txs: [{ txid: tx.txid, weight: tx.weight, fee: tx.fee || 0 }],
+          effectiveFeePerVsize: minAncestorRate,
+        };
+        clusterMap[tx.txid] = cluster;
+        clusters.push(cluster);
+      } else {
+        // update the existing cluster with the dependent rate
+        clusterMap[tx.txid].effectiveFeePerVsize = minAncestorRate;
+      }
+    }
+  }
+  if (saveRelatives) {
+    for (const cluster of clusters) {
+      cluster.txs.forEach((member, index) => {
+        txMap[member.txid].descendants = cluster.txs.slice(0, index).reverse();
+        txMap[member.txid].ancestors = cluster.txs.slice(index + 1).reverse();
+        txMap[member.txid].effectiveFeePerVsize = cluster.effectiveFeePerVsize;
+      });
+    }
+  }
+  return {
+    transactions,
+    clusters,
+    version: 1,
+  };
+}
+
+export function calculateGoodBlockCpfp(height: number, transactions: MempoolTransactionExtended[], accelerations: Acceleration[]): CpfpSummary {
+  const txMap: { [txid: string]: MempoolTransactionExtended } = {};
+  for (const tx of transactions) {
+    txMap[tx.txid] = tx;
+  }
+  const template = makeBlockTemplate(transactions, accelerations, 1, Infinity, Infinity);
+  const clusters = new Map<string, string[]>();
+  for (const tx of template) {
+    const cluster = tx.cluster || [];
+    const root = cluster.length ? cluster[cluster.length - 1] : null;
+    if (cluster.length > 1 && root && !clusters.has(root)) {
+      clusters.set(root, cluster);
+    }
+    txMap[tx.txid].effectiveFeePerVsize = tx.effectiveFeePerVsize;
+  }
+
+  const clusterArray: CpfpCluster[] = [];
+
+  for (const cluster of clusters.values()) {
+    for (const txid of cluster) {
+      const mempoolTx = txMap[txid];
+      if (mempoolTx) {
+        const ancestors: Ancestor[] = [];
+        const descendants: Ancestor[] = [];
+        let matched = false;
+        cluster.forEach(relativeTxid => {
+          if (relativeTxid === txid) {
+            matched = true;
+          } else {
+            const relative = {
+              txid: relativeTxid,
+              fee: txMap[relativeTxid].fee,
+              weight: (txMap[relativeTxid].adjustedVsize * 4) || txMap[relativeTxid].weight,
+            };
+            if (matched) {
+              descendants.push(relative);
+            } else {
+              ancestors.push(relative);
+            }
+          }
+        });
+        if (mempoolTx.ancestors?.length !== ancestors.length || mempoolTx.descendants?.length !== descendants.length) {
+          mempoolTx.cpfpDirty = true;
+        }
+        Object.assign(mempoolTx, { ancestors, descendants, bestDescendant: null, cpfpChecked: true });
+      }
+    }
+    const root = cluster[cluster.length - 1];
+    clusterArray.push({
+      root: root,
+      height,
+      txs: cluster.reverse().map(txid => ({
+        txid,
+        fee: txMap[txid].fee,
+        weight: (txMap[txid].adjustedVsize * 4) || txMap[txid].weight,
+      })),
+      effectiveFeePerVsize: txMap[root].effectiveFeePerVsize,
+    });
+  }
+
+  return {
+    transactions: transactions.map(tx => txMap[tx.txid]),
+    clusters: clusterArray,
+    version: 2,
   };
-  ancestorcount: number;
-  ancestorsize: number;
-  ancestorRate: number;
-  individualRate: number;
-  score: number;
 }
 
 /**
  * Takes a mempool transaction and a copy of the current mempool, and calculates the CPFP data for
  * that transaction (and all others in the same cluster)
  */
-export function calculateCpfp(tx: MempoolTransactionExtended, mempool: { [txid: string]: MempoolTransactionExtended }): CpfpInfo {
+export function calculateMempoolTxCpfp(tx: MempoolTransactionExtended, mempool: { [txid: string]: MempoolTransactionExtended }): CpfpInfo {
   if (tx.cpfpUpdated && Date.now() < (tx.cpfpUpdated + CPFP_UPDATE_INTERVAL)) {
     tx.cpfpDirty = false;
     return {
@@ -32,30 +177,31 @@ export function calculateCpfp(tx: MempoolTransactionExtended, mempool: { [txid:
       descendants: tx.descendants || [],
       effectiveFeePerVsize: tx.effectiveFeePerVsize || tx.adjustedFeePerVsize || tx.feePerVsize,
       sigops: tx.sigops,
+      fee: tx.fee,
       adjustedVsize: tx.adjustedVsize,
       acceleration: tx.acceleration
     };
   }
 
   const ancestorMap = new Map<string, GraphTx>();
-  const graphTx = mempoolToGraphTx(tx);
+  const graphTx = convertToGraphTx(tx, memPool.getSpendMap());
   ancestorMap.set(tx.txid, graphTx);
 
-  const allRelatives = expandRelativesGraph(mempool, ancestorMap);
+  const allRelatives = expandRelativesGraph(mempool, ancestorMap, memPool.getSpendMap());
   const relativesMap = initializeRelatives(allRelatives);
   const cluster = calculateCpfpCluster(tx.txid, relativesMap);
 
   let totalVsize = 0;
   let totalFee = 0;
   for (const tx of cluster.values()) {
-    totalVsize += tx.adjustedVsize;
-    totalFee += tx.fee;
+    totalVsize += tx.vsize;
+    totalFee += tx.fees.base;
   }
   const effectiveFeePerVsize = totalFee / totalVsize;
   for (const tx of cluster.values()) {
     mempool[tx.txid].effectiveFeePerVsize = effectiveFeePerVsize;
-    mempool[tx.txid].ancestors = Array.from(tx.ancestorMap.values()).map(tx => ({ txid: tx.txid, weight: tx.weight, fee: tx.fee }));
-    mempool[tx.txid].descendants = Array.from(cluster.values()).filter(entry => entry.txid !== tx.txid && !tx.ancestorMap.has(entry.txid)).map(tx => ({ txid: tx.txid, weight: tx.weight, fee: tx.fee }));
+    mempool[tx.txid].ancestors = Array.from(tx.ancestors.values()).map(tx => ({ txid: tx.txid, weight: tx.weight, fee: tx.fees.base }));
+    mempool[tx.txid].descendants = Array.from(cluster.values()).filter(entry => entry.txid !== tx.txid && !tx.ancestors.has(entry.txid)).map(tx => ({ txid: tx.txid, weight: tx.weight, fee: tx.fees.base }));
     mempool[tx.txid].bestDescendant = null;
     mempool[tx.txid].cpfpChecked = true;
     mempool[tx.txid].cpfpDirty = true;
@@ -70,88 +216,12 @@ export function calculateCpfp(tx: MempoolTransactionExtended, mempool: { [txid:
     descendants: tx.descendants || [],
     effectiveFeePerVsize: tx.effectiveFeePerVsize || tx.adjustedFeePerVsize || tx.feePerVsize,
     sigops: tx.sigops,
+    fee: tx.fee,
     adjustedVsize: tx.adjustedVsize,
     acceleration: tx.acceleration
   };
 }
 
-function mempoolToGraphTx(tx: MempoolTransactionExtended): GraphTx {
-  return {
-    ...tx,
-    depends: tx.vin.map(v => v.txid),
-    spentby: tx.vout.map((v, i) => memPool.getFromSpendMap(tx.txid, i)).map(tx => tx?.txid).filter(txid => txid != null) as string[],
-    ancestorMap: new Map(),
-    fees: {
-      base: tx.fee,
-      ancestor: tx.fee,
-    },
-    ancestorcount: 1,
-    ancestorsize: tx.adjustedVsize,
-    ancestorRate: 0,
-    individualRate: 0,
-    score: 0,
-  };
-}
-
-/**
- * Takes a map of transaction ancestors, and expands it into a full graph of up to MAX_GRAPH_SIZE in-mempool relatives
- */
-function expandRelativesGraph(mempool: { [txid: string]: MempoolTransactionExtended }, ancestors: Map<string, GraphTx>): Map<string, GraphTx> {
-  const relatives: Map<string, GraphTx> = new Map();
-  const stack: GraphTx[] = Array.from(ancestors.values());
-  while (stack.length > 0) {
-    if (relatives.size > MAX_GRAPH_SIZE) {
-      return relatives;
-    }
-
-    const nextTx = stack.pop();
-    if (!nextTx) {
-      continue;
-    }
-    relatives.set(nextTx.txid, nextTx);
-
-    for (const relativeTxid of [...nextTx.depends, ...nextTx.spentby]) {
-      if (relatives.has(relativeTxid)) {
-        // already processed this tx
-        continue;
-      }
-      let mempoolTx = ancestors.get(relativeTxid);
-      if (!mempoolTx && mempool[relativeTxid]) {
-        mempoolTx = mempoolToGraphTx(mempool[relativeTxid]);
-      }
-      if (mempoolTx) {
-        stack.push(mempoolTx);
-      }
-    }
-  }
-
-  return relatives;
-}
-
- /**
-   * Efficiently sets a Map of in-mempool ancestors for each member of an expanded relative graph
-   * by running setAncestors on each leaf, and caching intermediate results.
-   * then initializes ancestor data for each transaction
-   * 
-   * @param all 
-   */
- function initializeRelatives(mempoolTxs: Map<string, GraphTx>): Map<string, GraphTx> {
-  const visited: Map<string, Map<string, GraphTx>> = new Map();
-  const leaves: GraphTx[] = Array.from(mempoolTxs.values()).filter(entry => entry.spentby.length === 0);
-  for (const leaf of leaves) {
-    setAncestors(leaf, mempoolTxs, visited);
-  }
-  mempoolTxs.forEach(entry => {
-    entry.ancestorMap?.forEach(ancestor => {
-      entry.ancestorcount++;
-      entry.ancestorsize += ancestor.adjustedVsize;
-      entry.fees.ancestor += ancestor.fees.base;
-    });
-    setAncestorScores(entry);
-  });
-  return mempoolTxs;
-}
-
 /**
    * Given a root transaction and a list of in-mempool ancestors,
    * Calculate the CPFP cluster
@@ -172,10 +242,10 @@ function calculateCpfpCluster(txid: string, graph: Map<string, GraphTx>): Map<st
   let sortedRelatives = Array.from(graph.values()).sort(mempoolComparator);
 
   // Iterate until we reach a cluster that includes our target tx
-  let maxIterations = MAX_GRAPH_SIZE;
+  let maxIterations = MAX_CLUSTER_ITERATIONS;
   let best = sortedRelatives.shift();
-  let bestCluster = new Map<string, GraphTx>(best?.ancestorMap?.entries() || []);
-  while (sortedRelatives.length && best && (best.txid !== tx.txid && !best.ancestorMap.has(tx.txid)) && maxIterations > 0) {
+  let bestCluster = new Map<string, GraphTx>(best?.ancestors?.entries() || []);
+  while (sortedRelatives.length && best && (best.txid !== tx.txid && !best.ancestors.has(tx.txid)) && maxIterations > 0) {
     maxIterations--;
     if ((best && best.txid === tx.txid) || (bestCluster && bestCluster.has(tx.txid))) {
       break;
@@ -190,7 +260,7 @@ function calculateCpfpCluster(txid: string, graph: Map<string, GraphTx>): Map<st
       // Grab the next highest scoring entry
       best = sortedRelatives.shift();
       if (best) {
-        bestCluster = new Map<string, GraphTx>(best?.ancestorMap?.entries() || []);
+        bestCluster = new Map<string, GraphTx>(best?.ancestors?.entries() || []);
         bestCluster.set(best?.txid, best);
       }
     }
@@ -199,88 +269,4 @@ function calculateCpfpCluster(txid: string, graph: Map<string, GraphTx>): Map<st
   bestCluster.set(tx.txid, tx);
 
   return bestCluster;
-}
-
- /**
-   * Remove a cluster of transactions from an in-mempool dependency graph
-   * and update the survivors' scores and ancestors
-   * 
-   * @param cluster 
-   * @param ancestors 
-   */
- function removeAncestors(cluster: Map<string, GraphTx>, all: Map<string, GraphTx>): void {
-  // remove
-  cluster.forEach(tx => {
-    all.delete(tx.txid);
-  });
-
-  // update survivors
-  all.forEach(tx => {
-    cluster.forEach(remove => {
-      if (tx.ancestorMap?.has(remove.txid)) {
-        // remove as dependency
-        tx.ancestorMap.delete(remove.txid);
-        tx.depends = tx.depends.filter(parent => parent !== remove.txid);
-        // update ancestor sizes and fees
-        tx.ancestorsize -= remove.adjustedVsize;
-        tx.fees.ancestor -= remove.fees.base;
-      }
-    });
-    // recalculate fee rates
-    setAncestorScores(tx);
-  });
-}
-
-/**
-   * Recursively traverses an in-mempool dependency graph, and sets a Map of in-mempool ancestors
-   * for each transaction.
-   * 
-   * @param tx 
-   * @param all 
-   */
-function setAncestors(tx: GraphTx, all: Map<string, GraphTx>, visited: Map<string, Map<string, GraphTx>>, depth: number = 0): Map<string, GraphTx> {
-  // sanity check for infinite recursion / too many ancestors (should never happen)
-  if (depth > MAX_GRAPH_SIZE) {
-    return tx.ancestorMap;
-  }
-  
-  // initialize the ancestor map for this tx
-  tx.ancestorMap = new Map<string, GraphTx>();
-  tx.depends.forEach(parentId => {
-    const parent = all.get(parentId);
-    if (parent) {
-      // add the parent
-      tx.ancestorMap?.set(parentId, parent);
-      // check for a cached copy of this parent's ancestors
-      let ancestors = visited.get(parent.txid);
-      if (!ancestors) {
-        // recursively fetch the parent's ancestors
-        ancestors = setAncestors(parent, all, visited, depth + 1);
-      }
-      // and add to this tx's map
-      ancestors.forEach((ancestor, ancestorId) => {
-        tx.ancestorMap?.set(ancestorId, ancestor);
-      });
-    }
-  });
-  visited.set(tx.txid, tx.ancestorMap);
-
-  return tx.ancestorMap;
-}
-
-/**
-   * Take a mempool transaction, and set the fee rates and ancestor score
-   * 
-   * @param tx 
-   */
-function setAncestorScores(tx: GraphTx): GraphTx {
-  tx.individualRate = (tx.fees.base * 100_000_000) / tx.adjustedVsize;
-  tx.ancestorRate = (tx.fees.ancestor * 100_000_000) / tx.ancestorsize;
-  tx.score = Math.min(tx.individualRate, tx.ancestorRate);
-  return tx;
-}
-
-// Sort by descending score
-function mempoolComparator(a: GraphTx, b: GraphTx): number {
-  return b.score - a.score;
 }

backend/src/api/database-migration.ts

@@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
 import { RowDataPacket } from 'mysql2';
 
 class DatabaseMigration {
-  private static currentVersion = 78;
+  private static currentVersion = 95;
   private queryTimeout = 3600_000;
   private statisticsAddedIndexed = false;
   private uniqueLogs: string[] = [];
@@ -653,9 +653,11 @@ class DatabaseMigration {
       await this.$executeQuery('ALTER TABLE `prices` ADD `TRY` float DEFAULT "-1"');
       await this.$executeQuery('ALTER TABLE `prices` ADD `ZAR` float DEFAULT "-1"');
 
-      await this.$executeQuery('TRUNCATE hashrates');
-      await this.$executeQuery('TRUNCATE difficulty_adjustments');
-      await this.$executeQuery(`UPDATE state SET string = NULL WHERE name = 'pools_json_sha'`);
+      if (isBitcoin === true) {
+        await this.$executeQuery('TRUNCATE hashrates');
+        await this.$executeQuery('TRUNCATE difficulty_adjustments');
+        await this.$executeQuery(`UPDATE state SET string = NULL WHERE name = 'pools_json_sha'`);
+      }
 
       await this.updateToSchemaVersion(75);
     }
@@ -674,6 +676,460 @@ class DatabaseMigration {
       await this.$executeQuery('ALTER TABLE `prices` CHANGE `time` `time` datetime NOT NULL');
       await this.updateToSchemaVersion(78);
     }
+
+    if (databaseSchemaVersion < 79 && config.MEMPOOL.NETWORK === 'mainnet') {
+      // Clear bad data
+      await this.$executeQuery(`TRUNCATE accelerations`);
+      this.uniqueLog(logger.notice, `'accelerations' table has been truncated`);
+      await this.$executeQuery(`
+        UPDATE state
+        SET number = 0
+        WHERE name = 'last_acceleration_block'
+      `);
+      await this.updateToSchemaVersion(79);
+    }
+
+    if (databaseSchemaVersion < 80) {
+      await this.$executeQuery('ALTER TABLE `blocks` ADD coinbase_addresses JSON DEFAULT NULL');
+      await this.updateToSchemaVersion(80);
+    }
+
+    if (databaseSchemaVersion < 81 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `blocks_audits` ADD version INT NOT NULL DEFAULT 0');
+      await this.$executeQuery('ALTER TABLE `blocks_audits` ADD INDEX `version` (`version`)');
+      await this.$executeQuery('ALTER TABLE `blocks_audits` ADD unseen_txs JSON DEFAULT "[]"');
+      await this.updateToSchemaVersion(81);
+    }
+
+    if (databaseSchemaVersion < 82 && isBitcoin === true && config.MEMPOOL.NETWORK === 'mainnet') {
+      await this.$fixBadV1AuditBlocks();
+      await this.updateToSchemaVersion(82);
+    }
+
+    if (databaseSchemaVersion < 83 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `blocks` ADD first_seen datetime(6) DEFAULT NULL');
+      await this.updateToSchemaVersion(83);
+    }
+
+    // add new pools indexes
+    if (databaseSchemaVersion < 84 && isBitcoin === true) {
+      await this.$executeQuery(`
+        ALTER TABLE \`pools\`
+          ADD INDEX \`slug\` (\`slug\`),
+          ADD INDEX \`unique_id\` (\`unique_id\`)
+      `);
+      await this.updateToSchemaVersion(84);
+    }
+
+    // lightning channels indexes
+    if (databaseSchemaVersion < 85 && isBitcoin === true) {
+      await this.$executeQuery(`
+        ALTER TABLE \`channels\`
+          ADD INDEX \`created\` (\`created\`),
+          ADD INDEX \`capacity\` (\`capacity\`),
+          ADD INDEX \`closing_reason\` (\`closing_reason\`),
+          ADD INDEX \`closing_resolved\` (\`closing_resolved\`)
+      `);
+      await this.updateToSchemaVersion(85);
+    }
+
+    // lightning nodes indexes
+    if (databaseSchemaVersion < 86 && isBitcoin === true) {
+      await this.$executeQuery(`
+        ALTER TABLE \`nodes\`
+          ADD INDEX \`status\` (\`status\`),
+          ADD INDEX \`channels\` (\`channels\`),
+          ADD INDEX \`country_id\` (\`country_id\`),
+          ADD INDEX \`as_number\` (\`as_number\`),
+          ADD INDEX \`first_seen\` (\`first_seen\`)
+      `);
+      await this.updateToSchemaVersion(86);
+    }
+
+    // lightning node sockets indexes
+    if (databaseSchemaVersion < 87 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `nodes_sockets` ADD INDEX `type` (`type`)');
+      await this.updateToSchemaVersion(87);
+    }
+
+    // lightning stats indexes
+    if (databaseSchemaVersion < 88 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `lightning_stats` ADD INDEX `added` (`added`)');
+      await this.updateToSchemaVersion(88);
+    }
+
+    // geo names indexes
+    if (databaseSchemaVersion < 89 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `geo_names` ADD INDEX `names` (`names`)');
+      await this.updateToSchemaVersion(89);
+    }
+
+    // hashrates indexes
+    if (databaseSchemaVersion < 90 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `hashrates` ADD INDEX `type` (`type`)');
+      await this.updateToSchemaVersion(90);
+    }
+
+    // block audits indexes
+    if (databaseSchemaVersion < 91 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `blocks_audits` ADD INDEX `time` (`time`)');
+      await this.updateToSchemaVersion(91);
+    }
+
+    // elements_pegs indexes
+    if (databaseSchemaVersion < 92 && config.MEMPOOL.NETWORK === 'liquid') {
+      await this.$executeQuery(`
+        ALTER TABLE \`elements_pegs\`
+          ADD INDEX \`block\` (\`block\`),
+          ADD INDEX \`datetime\` (\`datetime\`),
+          ADD INDEX \`amount\` (\`amount\`),
+          ADD INDEX \`bitcoinaddress\` (\`bitcoinaddress\`),
+          ADD INDEX \`bitcointxid\` (\`bitcointxid\`)
+      `);
+      await this.updateToSchemaVersion(92);
+    }
+
+    // federation_txos indexes
+    if (databaseSchemaVersion < 93 && config.MEMPOOL.NETWORK === 'liquid') {
+      await this.$executeQuery(`
+        ALTER TABLE \`federation_txos\`
+          ADD INDEX \`unspent\` (\`unspent\`),
+          ADD INDEX \`lastblockupdate\` (\`lastblockupdate\`),
+          ADD INDEX \`blocktime\` (\`blocktime\`),
+          ADD INDEX \`emergencyKey\` (\`emergencyKey\`),
+          ADD INDEX \`expiredAt\` (\`expiredAt\`)
+      `);
+      await this.updateToSchemaVersion(93);
+    }
+
+    // Unify database schema for all mempool netwoks
+    // versions above 94 should not use network-specific flags
+    if (databaseSchemaVersion < 94) {
+
+      if (!isBitcoin) {
+        // Apply all the bitcoin specific migrations to non-bitcoin networks: liquid, liquidtestnet and testnet4 (!)
+        // Version 5
+        await this.$executeQuery('ALTER TABLE blocks ADD `reward` double unsigned NOT NULL DEFAULT "0"');
+
+        // Version 6
+        await this.$executeQuery('ALTER TABLE blocks MODIFY `height` integer unsigned NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE blocks MODIFY `tx_count` smallint unsigned NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE blocks MODIFY `size` integer unsigned NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE blocks MODIFY `weight` integer unsigned NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE blocks MODIFY `difficulty` double NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE blocks DROP FOREIGN KEY IF EXISTS `blocks_ibfk_1`');
+        await this.$executeQuery('ALTER TABLE pools MODIFY `id` smallint unsigned AUTO_INCREMENT');
+        await this.$executeQuery('ALTER TABLE blocks MODIFY `pool_id` smallint unsigned NULL');
+        await this.$executeQuery('ALTER TABLE blocks ADD FOREIGN KEY (`pool_id`) REFERENCES `pools` (`id`)');
+        await this.$executeQuery('ALTER TABLE blocks ADD `version` integer unsigned NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE blocks ADD `bits` integer unsigned NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE blocks ADD `nonce` bigint unsigned NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE blocks ADD `merkle_root` varchar(65) NOT NULL DEFAULT ""');
+        await this.$executeQuery('ALTER TABLE blocks ADD `previous_block_hash` varchar(65) NULL');
+
+        // Version 7
+        await this.$executeQuery('DROP table IF EXISTS hashrates;');
+        await this.$executeQuery(this.getCreateDailyStatsTableQuery(), await this.$checkIfTableExists('hashrates'));
+
+        // Version 8
+        await this.$executeQuery('ALTER TABLE `hashrates` DROP INDEX `PRIMARY`');
+        await this.$executeQuery('ALTER TABLE `hashrates` ADD `id` int NOT NULL AUTO_INCREMENT PRIMARY KEY FIRST');
+        await this.$executeQuery('ALTER TABLE `hashrates` ADD `share` float NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE `hashrates` ADD `type` enum("daily", "weekly") DEFAULT "daily"');
+
+        // Version 9
+        await this.$executeQuery('ALTER TABLE `state` CHANGE `name` `name` varchar(100)');
+        await this.$executeQuery('ALTER TABLE `hashrates` ADD UNIQUE `hashrate_timestamp_pool_id` (`hashrate_timestamp`, `pool_id`)');
+
+        // Version 10
+        await this.$executeQuery('ALTER TABLE `blocks` ADD INDEX `blockTimestamp` (`blockTimestamp`)');
+
+        // Version 11
+        await this.$executeQuery(`ALTER TABLE blocks
+          ADD avg_fee INT UNSIGNED NULL,
+          ADD avg_fee_rate INT UNSIGNED NULL
+        `);
+        await this.$executeQuery('ALTER TABLE blocks MODIFY `reward` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE blocks MODIFY `median_fee` INT UNSIGNED NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE blocks MODIFY `fees` INT UNSIGNED NOT NULL DEFAULT "0"');
+
+        // Version 12
+        await this.$executeQuery('ALTER TABLE blocks MODIFY `fees` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
+
+        // Version 13
+        await this.$executeQuery('ALTER TABLE blocks MODIFY `difficulty` DOUBLE UNSIGNED NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE blocks MODIFY `median_fee` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE blocks MODIFY `avg_fee` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE blocks MODIFY `avg_fee_rate` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
+
+        // Version 14
+        await this.$executeQuery('ALTER TABLE `hashrates` DROP FOREIGN KEY `hashrates_ibfk_1`');
+        await this.$executeQuery('ALTER TABLE `hashrates` MODIFY `pool_id` SMALLINT UNSIGNED NOT NULL DEFAULT "0"');
+
+        // Version 17
+        await this.$executeQuery('ALTER TABLE `pools` ADD `slug` CHAR(50) NULL');
+
+        // Version 18
+        await this.$executeQuery('ALTER TABLE `blocks` ADD INDEX `hash` (`hash`);');
+
+        // Version 20
+        await this.$executeQuery(this.getCreateBlocksSummariesTableQuery(), await this.$checkIfTableExists('blocks_summaries'));
+
+        // Version 22
+        await this.$executeQuery('DROP TABLE IF EXISTS `difficulty_adjustments`');
+        await this.$executeQuery(this.getCreateDifficultyAdjustmentsTableQuery(), await this.$checkIfTableExists('difficulty_adjustments'));
+
+        // Version 24
+        await this.$executeQuery('DROP TABLE IF EXISTS `blocks_audits`');
+        await this.$executeQuery(this.getCreateBlocksAuditsTableQuery(), await this.$checkIfTableExists('blocks_audits'));
+
+        // Version 25
+        await this.$executeQuery(this.getCreateLightningStatisticsQuery(), await this.$checkIfTableExists('lightning_stats'));
+        await this.$executeQuery(this.getCreateNodesQuery(), await this.$checkIfTableExists('nodes'));
+        await this.$executeQuery(this.getCreateChannelsQuery(), await this.$checkIfTableExists('channels'));
+        await this.$executeQuery(this.getCreateNodesStatsQuery(), await this.$checkIfTableExists('node_stats'));
+
+        // Version 26
+        await this.$executeQuery('ALTER TABLE `lightning_stats` ADD tor_nodes int(11) NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE `lightning_stats` ADD clearnet_nodes int(11) NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE `lightning_stats` ADD unannounced_nodes int(11) NOT NULL DEFAULT "0"');
+
+        // Version 27
+        await this.$executeQuery('ALTER TABLE `lightning_stats` ADD avg_capacity bigint(20) unsigned NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE `lightning_stats` ADD avg_fee_rate int(11) unsigned NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE `lightning_stats` ADD avg_base_fee_mtokens bigint(20) unsigned NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE `lightning_stats` ADD med_capacity bigint(20) unsigned NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE `lightning_stats` ADD med_fee_rate int(11) unsigned NOT NULL DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE `lightning_stats` ADD med_base_fee_mtokens bigint(20) unsigned NOT NULL DEFAULT "0"');
+
+        // Version 28
+        await this.$executeQuery(`ALTER TABLE lightning_stats MODIFY added DATE`);
+
+        // Version 29
+        await this.$executeQuery(this.getCreateGeoNamesTableQuery(), await this.$checkIfTableExists('geo_names'));
+        await this.$executeQuery('ALTER TABLE `nodes` ADD as_number int(11) unsigned NULL DEFAULT NULL');
+        await this.$executeQuery('ALTER TABLE `nodes` ADD city_id int(11) unsigned NULL DEFAULT NULL');
+        await this.$executeQuery('ALTER TABLE `nodes` ADD country_id int(11) unsigned NULL DEFAULT NULL');
+        await this.$executeQuery('ALTER TABLE `nodes` ADD accuracy_radius int(11) unsigned NULL DEFAULT NULL');
+        await this.$executeQuery('ALTER TABLE `nodes` ADD subdivision_id int(11) unsigned NULL DEFAULT NULL');
+        await this.$executeQuery('ALTER TABLE `nodes` ADD longitude double NULL DEFAULT NULL');
+        await this.$executeQuery('ALTER TABLE `nodes` ADD latitude double NULL DEFAULT NULL');
+
+        // Version 30
+        await this.$executeQuery('ALTER TABLE `geo_names` CHANGE `type` `type` enum("city","country","division","continent","as_organization") NOT NULL');
+
+        // Version 31
+        await this.$executeQuery('ALTER TABLE `prices` ADD `id` int NULL AUTO_INCREMENT UNIQUE');
+        await this.$executeQuery('DROP TABLE IF EXISTS `blocks_prices`');
+        await this.$executeQuery(this.getCreateBlocksPricesTableQuery(), await this.$checkIfTableExists('blocks_prices'));
+
+        // Version 32
+        await this.$executeQuery('ALTER TABLE `blocks_summaries` ADD `template` JSON DEFAULT "[]"');
+
+        // Version 33
+        await this.$executeQuery('ALTER TABLE `geo_names` CHANGE `type` `type` enum("city","country","division","continent","as_organization", "country_iso_code") NOT NULL');
+
+        // Version 34
+        await this.$executeQuery('ALTER TABLE `lightning_stats` ADD clearnet_tor_nodes int(11) NOT NULL DEFAULT "0"');
+    
+        // Version 35
+        await this.$executeQuery('DELETE from `lightning_stats` WHERE added > "2021-09-19"');
+        await this.$executeQuery('ALTER TABLE `lightning_stats` ADD CONSTRAINT added_unique UNIQUE (added);');
+
+        // Version 36
+        await this.$executeQuery('ALTER TABLE `nodes` ADD status TINYINT NOT NULL DEFAULT "1"');
+    
+        // Version 37
+        await this.$executeQuery(this.getCreateLNNodesSocketsTableQuery(), await this.$checkIfTableExists('nodes_sockets'));
+        
+        // Version 38
+        await this.$executeQuery(`TRUNCATE lightning_stats`);
+        await this.$executeQuery(`TRUNCATE node_stats`);
+        await this.$executeQuery('ALTER TABLE `lightning_stats` CHANGE `added` `added` timestamp NULL');
+        await this.$executeQuery('ALTER TABLE `node_stats` CHANGE `added` `added` timestamp NULL');
+        await this.updateToSchemaVersion(38);
+      
+        // Version 39
+        await this.$executeQuery('ALTER TABLE `nodes` ADD alias_search TEXT NULL DEFAULT NULL AFTER `alias`');
+        await this.$executeQuery('ALTER TABLE nodes ADD FULLTEXT(alias_search)');
+
+        // Version 40
+        await this.$executeQuery('ALTER TABLE `nodes` ADD capacity bigint(20) unsigned DEFAULT NULL');
+        await this.$executeQuery('ALTER TABLE `nodes` ADD channels int(11) unsigned DEFAULT NULL');
+        await this.$executeQuery('ALTER TABLE `nodes` ADD INDEX `capacity` (`capacity`);');
+
+        // Version 41
+        await this.$executeQuery('UPDATE channels SET closing_reason = NULL WHERE closing_reason = 1');
+
+        // Version 42
+        await this.$executeQuery('ALTER TABLE `channels` ADD closing_resolved tinyint(1) DEFAULT 0');
+      
+        // Version 43
+        await this.$executeQuery(this.getCreateLNNodeRecordsTableQuery(), await this.$checkIfTableExists('nodes_records'));
+
+        // Version 44
+        await this.$executeQuery('UPDATE blocks_summaries SET template = NULL');
+
+        // Version 45
+        await this.$executeQuery('ALTER TABLE `blocks_audits` ADD fresh_txs JSON DEFAULT "[]"');
+    
+        // Version 48
+        await this.$executeQuery('ALTER TABLE `channels` ADD source_checked tinyint(1) DEFAULT 0');
+        await this.$executeQuery('ALTER TABLE `channels` ADD closing_fee bigint(20) unsigned DEFAULT 0');
+        await this.$executeQuery('ALTER TABLE `channels` ADD node1_funding_balance bigint(20) unsigned DEFAULT 0');
+        await this.$executeQuery('ALTER TABLE `channels` ADD node2_funding_balance bigint(20) unsigned DEFAULT 0');
+        await this.$executeQuery('ALTER TABLE `channels` ADD node1_closing_balance bigint(20) unsigned DEFAULT 0');
+        await this.$executeQuery('ALTER TABLE `channels` ADD node2_closing_balance bigint(20) unsigned DEFAULT 0');
+        await this.$executeQuery('ALTER TABLE `channels` ADD funding_ratio float unsigned DEFAULT NULL');
+        await this.$executeQuery('ALTER TABLE `channels` ADD closed_by varchar(66) DEFAULT NULL');
+        await this.$executeQuery('ALTER TABLE `channels` ADD single_funded tinyint(1) DEFAULT 0');
+        await this.$executeQuery('ALTER TABLE `channels` ADD outputs JSON DEFAULT "[]"');
+
+        // Version 57
+        await this.$executeQuery(`ALTER TABLE nodes MODIFY updated_at datetime NULL`);
+
+        // Version 60
+        await this.$executeQuery('ALTER TABLE `blocks_audits` ADD sigop_txs JSON DEFAULT "[]"');
+
+        // Version 61
+        if (! await this.$checkIfTableExists('blocks_templates')) {
+          await this.$executeQuery('CREATE TABLE blocks_templates AS SELECT id, template FROM blocks_summaries WHERE template != "[]"');
+        }
+        await this.$executeQuery('ALTER TABLE blocks_templates MODIFY template JSON DEFAULT "[]"');
+        await this.$executeQuery('ALTER TABLE blocks_templates ADD PRIMARY KEY (id)');
+        await this.$executeQuery('ALTER TABLE blocks_summaries DROP COLUMN template');
+
+        // Version 62
+        await this.$executeQuery('ALTER TABLE `blocks_audits` ADD expected_fees BIGINT UNSIGNED DEFAULT NULL');
+        await this.$executeQuery('ALTER TABLE `blocks_audits` ADD expected_weight BIGINT UNSIGNED DEFAULT NULL');
+      
+        // Version 63
+        await this.$executeQuery('ALTER TABLE `blocks_audits` ADD fullrbf_txs JSON DEFAULT "[]"');
+    
+        // Version 64
+        await this.$executeQuery('ALTER TABLE `nodes` ADD features text NULL');
+    
+        // Version 65
+        await this.$executeQuery('ALTER TABLE `blocks_audits` ADD accelerated_txs JSON DEFAULT "[]"');
+
+        // Version 67
+        await this.$executeQuery('ALTER TABLE `blocks_summaries` ADD version INT NOT NULL DEFAULT 0');
+        await this.$executeQuery('ALTER TABLE `blocks_summaries` ADD INDEX `version` (`version`)');
+        await this.$executeQuery('ALTER TABLE `blocks_templates` ADD version INT NOT NULL DEFAULT 0');
+        await this.$executeQuery('ALTER TABLE `blocks_templates` ADD INDEX `version` (`version`)');
+
+        // Version 76
+        await this.$executeQuery('ALTER TABLE `blocks_audits` ADD prioritized_txs JSON DEFAULT "[]"');
+
+        // Version 81
+        await this.$executeQuery('ALTER TABLE `blocks_audits` ADD version INT NOT NULL DEFAULT 0');
+        await this.$executeQuery('ALTER TABLE `blocks_audits` ADD INDEX `version` (`version`)');
+        await this.$executeQuery('ALTER TABLE `blocks_audits` ADD unseen_txs JSON DEFAULT "[]"');
+
+        // Version 83
+        await this.$executeQuery('ALTER TABLE `blocks` ADD first_seen datetime(6) DEFAULT NULL');
+
+        // Version 84
+        await this.$executeQuery(`
+          ALTER TABLE \`pools\`
+            ADD INDEX \`slug\` (\`slug\`),
+            ADD INDEX \`unique_id\` (\`unique_id\`)
+        `);
+
+        // Version 85
+        await this.$executeQuery(`
+          ALTER TABLE \`channels\`
+            ADD INDEX \`created\` (\`created\`),
+            ADD INDEX \`capacity\` (\`capacity\`),
+            ADD INDEX \`closing_reason\` (\`closing_reason\`),
+            ADD INDEX \`closing_resolved\` (\`closing_resolved\`)
+        `);
+        
+        // Version 86        
+        await this.$executeQuery(`
+          ALTER TABLE \`nodes\`
+            ADD INDEX \`status\` (\`status\`),
+            ADD INDEX \`channels\` (\`channels\`),
+            ADD INDEX \`country_id\` (\`country_id\`),
+            ADD INDEX \`as_number\` (\`as_number\`),
+            ADD INDEX \`first_seen\` (\`first_seen\`)
+        `);
+
+        // Version 87
+        await this.$executeQuery('ALTER TABLE `nodes_sockets` ADD INDEX `type` (`type`)');
+        await this.updateToSchemaVersion(87);
+        
+        // Version 88
+        await this.$executeQuery('ALTER TABLE `lightning_stats` ADD INDEX `added` (`added`)');
+    
+        // Version 89
+        await this.$executeQuery('ALTER TABLE `geo_names` ADD INDEX `names` (`names`)');
+    
+        // Version 90
+        await this.$executeQuery('ALTER TABLE `hashrates` ADD INDEX `type` (`type`)');
+
+        // Version 91
+        await this.$executeQuery('ALTER TABLE `blocks_audits` ADD INDEX `time` (`time`)');
+      }
+      
+      if (config.MEMPOOL.NETWORK !== 'liquid') {
+        // Apply all the liquid specific migrations to all other networks
+        // Version 68
+        await this.$executeQuery('ALTER TABLE elements_pegs ADD PRIMARY KEY (txid, txindex);');
+        await this.$executeQuery(this.getCreateFederationAddressesTableQuery(), await this.$checkIfTableExists('federation_addresses'));
+        await this.$executeQuery(this.getCreateFederationTxosTableQuery(), await this.$checkIfTableExists('federation_txos'));
+
+        // Version 71
+        await this.$executeQuery('ALTER TABLE `federation_txos` ADD timelock INT NOT NULL DEFAULT 0');
+        await this.$executeQuery('ALTER TABLE `federation_txos` ADD expiredAt INT NOT NULL DEFAULT 0');
+        await this.$executeQuery('ALTER TABLE `federation_txos` ADD emergencyKey TINYINT NOT NULL DEFAULT 0');
+
+        // Version 92
+        await this.$executeQuery(`
+          ALTER TABLE \`elements_pegs\`
+            ADD INDEX \`block\` (\`block\`),
+            ADD INDEX \`datetime\` (\`datetime\`),
+            ADD INDEX \`amount\` (\`amount\`),
+            ADD INDEX \`bitcoinaddress\` (\`bitcoinaddress\`),
+            ADD INDEX \`bitcointxid\` (\`bitcointxid\`)
+        `);
+    
+        // Version 93
+        await this.$executeQuery(`
+          ALTER TABLE \`federation_txos\`
+            ADD INDEX \`unspent\` (\`unspent\`),
+            ADD INDEX \`lastblockupdate\` (\`lastblockupdate\`),
+            ADD INDEX \`blocktime\` (\`blocktime\`),
+            ADD INDEX \`emergencyKey\` (\`emergencyKey\`),
+            ADD INDEX \`expiredAt\` (\`expiredAt\`)
+        `);
+      }
+
+      if (config.MEMPOOL.NETWORK !== 'mainnet') {
+        // Apply all the mainnet specific migrations to all other networks
+        // Version 69
+        await this.$executeQuery(this.getCreateAccelerationsTableQuery(), await this.$checkIfTableExists('accelerations'));
+
+        // Version 70
+        await this.$executeQuery('ALTER TABLE accelerations MODIFY COLUMN added DATETIME;');
+
+        // Version 77
+        await this.$executeQuery('ALTER TABLE `accelerations` ADD requested datetime DEFAULT NULL');
+      }
+      await this.updateToSchemaVersion(94);
+    }
+
+    // blocks pools-v2.json hash
+    if (databaseSchemaVersion < 95) {
+      let poolJsonSha = 'f737d86571d190cf1a1a3cf5fd86b33ba9624254';
+      const [poolJsonShaDb]: any[] = await DB.query(`SELECT string FROM state WHERE name = 'pools_json_sha'`);
+      if (poolJsonShaDb?.length > 0) {
+        poolJsonSha = poolJsonShaDb[0].string;
+      }
+      await this.$executeQuery(`ALTER TABLE blocks ADD definition_hash varchar(255) NOT NULL DEFAULT "${poolJsonSha}"`);
+      await this.$executeQuery('ALTER TABLE blocks ADD INDEX `definition_hash` (`definition_hash`)');
+      await this.updateToSchemaVersion(95);
+    }
   }
 
   /**
@@ -1288,6 +1744,28 @@ class DatabaseMigration {
       logger.warn(`Failed to migrate cpfp transaction data`);
     }
   }
+
+  private async $fixBadV1AuditBlocks(): Promise<void> {
+    const badBlocks = [
+      '000000000000000000011ad49227fc8c9ba0ca96ad2ebce41a862f9a244478dc',
+      '000000000000000000010ac1f68b3080153f2826ffddc87ceffdd68ed97d6960',
+      '000000000000000000024cbdafeb2660ae8bd2947d166e7fe15d1689e86b2cf7',
+      '00000000000000000002e1dbfbf6ae057f331992a058b822644b368034f87286',
+      '0000000000000000000019973b2778f08ad6d21e083302ff0833d17066921ebb',
+    ];
+
+    for (const hash of badBlocks) {
+      try {
+        await this.$executeQuery(`
+          UPDATE blocks_audits
+          SET prioritized_txs = '[]'
+          WHERE hash = '${hash}'
+        `, true);
+      } catch (e) {
+        continue;
+      }
+    }
+  }
 }
 
 export default new DatabaseMigration();

backend/src/api/disk-cache.ts

@@ -257,6 +257,7 @@ class DiskCache {
           trees: rbfData.rbf.trees,
           expiring: rbfData.rbf.expiring.map(([txid, value]) => ({ key: txid, value })),
           mempool: memPool.getMempool(),
+          spendMap: memPool.getSpendMap(),
         });
       }
     } catch (e) {

backend/src/api/explorer/channels.routes.ts

@@ -1,6 +1,9 @@
 import config from '../../config';
 import { Application, Request, Response } from 'express';
 import channelsApi from './channels.api';
+import { handleError } from '../../utils/api';
+
+const TXID_REGEX = /^[a-f0-9]{64}$/i;
 
 class ChannelsRoutes {
   constructor() { }
@@ -22,7 +25,7 @@ class ChannelsRoutes {
       const channels = await channelsApi.$searchChannelsById(req.params.search);
       res.json(channels);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to search channels by id');
     }
   }
 
@@ -38,7 +41,7 @@ class ChannelsRoutes {
       res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       res.json(channel);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get channel');
     }
   }
 
@@ -53,10 +56,12 @@ class ChannelsRoutes {
       const status: string = typeof req.query.status === 'string' ? req.query.status : '';
 
       if (index < -1) {
-        res.status(400).send('Invalid index');
+        handleError(req, res, 400, 'Invalid index');
+        return;
       }
       if (['open', 'active', 'closed'].includes(status) === false) {
-        res.status(400).send('Invalid status');
+        handleError(req, res, 400, 'Invalid status');
+        return;
       }
 
       const channels = await channelsApi.$getChannelsForNode(req.query.public_key, index, 10, status);
@@ -67,20 +72,23 @@ class ChannelsRoutes {
       res.header('X-Total-Count', channelsCount.toString());
       res.json(channels);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get channels for node');
     }
   }
 
   private async $getChannelsByTransactionIds(req: Request, res: Response): Promise<void> {
     try {
       if (!Array.isArray(req.query.txId)) {
-        res.status(400).send('Not an array');
+        handleError(req, res, 400, 'Not an array');
         return;
       }
       const txIds: string[] = [];
       for (const _txId in req.query.txId) {
         if (typeof req.query.txId[_txId] === 'string') {
-          txIds.push(req.query.txId[_txId].toString());
+          const txid = req.query.txId[_txId].toString();
+          if (TXID_REGEX.test(txid)) {
+            txIds.push(txid);
+          }
         }
       }
       const channels = await channelsApi.$getChannelsByTransactionId(txIds);
@@ -105,7 +113,7 @@ class ChannelsRoutes {
 
       res.json(result);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get channels by transaction ids');
     }
   }
 
@@ -117,7 +125,7 @@ class ChannelsRoutes {
       res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       res.json(channels);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get penalty closed channels');
     }
   }
 
@@ -130,7 +138,7 @@ class ChannelsRoutes {
       res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       res.json(channels);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get channel geodata');
     }
   }
 

backend/src/api/explorer/general.routes.ts

@@ -3,6 +3,8 @@ import { Application, Request, Response } from 'express';
 import nodesApi from './nodes.api';
 import channelsApi from './channels.api';
 import statisticsApi from './statistics.api';
+import { handleError } from '../../utils/api';
+
 class GeneralLightningRoutes {
   constructor() { }
 
@@ -27,7 +29,7 @@ class GeneralLightningRoutes {
         channels: channels,
       });
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to search for nodes and channels');
     }
   }
 
@@ -41,7 +43,7 @@ class GeneralLightningRoutes {
       res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       res.json(statistics);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get lightning statistics');
     }
   }
 
@@ -50,7 +52,7 @@ class GeneralLightningRoutes {
       const statistics = await statisticsApi.$getLatestStatistics();
       res.json(statistics);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get lightning statistics');
     }
   }
 }

backend/src/api/explorer/nodes.api.ts

@@ -666,7 +666,9 @@ class NodesApi {
         node.last_update = null;
       }
   
-      const sockets = (node.addresses?.map(a => a.addr).join(',')) ?? '';
+      const uniqueAddr = [...new Set(node.addresses?.map(a => a.addr))];
+      const formattedSockets = (uniqueAddr.join(',')) ?? '';
+
       const query = `INSERT INTO nodes(
           public_key,
           first_seen,
@@ -695,13 +697,13 @@ class NodesApi {
         node.alias,
         this.aliasToSearchText(node.alias),
         node.color,
-        sockets,
+        formattedSockets,
         JSON.stringify(node.features),
         node.last_update,
         node.alias,
         this.aliasToSearchText(node.alias),
         node.color,
-        sockets,
+        formattedSockets,
         JSON.stringify(node.features),
       ]);
     } catch (e) {
@@ -713,7 +715,9 @@ class NodesApi {
    * Update node sockets
    */
   public async $updateNodeSockets(publicKey: string, sockets: {network: string; addr: string}[]): Promise<void> {
-    const formattedSockets = (sockets.map(a => a.addr).join(',')) ?? '';
+    const uniqueAddr = [...new Set(sockets.map(a => a.addr))];
+
+    const formattedSockets = (uniqueAddr.join(',')) ?? '';
     try {
       await DB.query(`UPDATE nodes SET sockets = ? WHERE public_key = ?`, [formattedSockets, publicKey]);
     } catch (e) {

backend/src/api/explorer/nodes.routes.ts

@@ -3,6 +3,7 @@ import { Application, Request, Response } from 'express';
 import nodesApi from './nodes.api';
 import DB from '../../database';
 import { INodesRanking } from '../../mempool.interfaces';
+import { handleError } from '../../utils/api';
 
 class NodesRoutes {
   constructor() { }
@@ -31,7 +32,7 @@ class NodesRoutes {
       const nodes = await nodesApi.$searchNodeByPublicKeyOrAlias(req.params.search);
       res.json(nodes);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to search for node');
     }
   }
 
@@ -181,13 +182,13 @@ class NodesRoutes {
           }
         } catch (e) {}
       }
-      
+
       res.header('Pragma', 'public');
       res.header('Cache-control', 'public');
       res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       res.json(nodes);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get node group');
     }
   }
 
@@ -195,7 +196,7 @@ class NodesRoutes {
     try {
       const node = await nodesApi.$getNode(req.params.public_key);
       if (!node) {
-        res.status(404).send('Node not found');
+        handleError(req, res, 404, 'Node not found');
         return;
       }
       res.header('Pragma', 'public');
@@ -203,7 +204,7 @@ class NodesRoutes {
       res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       res.json(node);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get node');
     }
   }
 
@@ -215,7 +216,7 @@ class NodesRoutes {
       res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       res.json(statistics);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get historical node stats');
     }
   }
 
@@ -223,7 +224,7 @@ class NodesRoutes {
     try {
       const node = await nodesApi.$getFeeHistogram(req.params.public_key);
       if (!node) {
-        res.status(404).send('Node not found');
+        handleError(req, res, 404, 'Node not found');
         return;
       }
       res.header('Pragma', 'public');
@@ -231,7 +232,7 @@ class NodesRoutes {
       res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       res.json(node);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get fee histogram');
     }
   }
 
@@ -247,7 +248,7 @@ class NodesRoutes {
         topByChannels: topChannelsNodes,
       });
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get nodes ranking');
     }
   }
 
@@ -259,7 +260,7 @@ class NodesRoutes {
       res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       res.json(topCapacityNodes);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get top nodes by capacity');
     }
   }
 
@@ -271,7 +272,7 @@ class NodesRoutes {
       res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       res.json(topCapacityNodes);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get top nodes by channels');
     }
   }
 
@@ -283,7 +284,7 @@ class NodesRoutes {
       res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       res.json(topCapacityNodes);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get oldest nodes');
     }
   }
 
@@ -295,7 +296,7 @@ class NodesRoutes {
       res.setHeader('Expires', new Date(Date.now() + 1000 * 600).toUTCString());
       res.json(nodesPerAs);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get ISP ranking');
     }
   }
 
@@ -307,7 +308,7 @@ class NodesRoutes {
       res.setHeader('Expires', new Date(Date.now() + 1000 * 600).toUTCString());
       res.json(worldNodes);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get world nodes');
     }
   }
 
@@ -322,7 +323,7 @@ class NodesRoutes {
       );
 
       if (country.length === 0) {
-        res.status(404).send(`This country does not exist or does not host any lightning nodes on clearnet`);
+        handleError(req, res, 404, `This country does not exist or does not host any lightning nodes on clearnet`);
         return;
       }
 
@@ -335,7 +336,7 @@ class NodesRoutes {
         nodes: nodes,
       });
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get nodes per country');
     }
   }
 
@@ -349,7 +350,7 @@ class NodesRoutes {
       );
 
       if (isp.length === 0) {
-        res.status(404).send(`This ISP does not exist or does not host any lightning nodes on clearnet`);
+        handleError(req, res, 404, `This ISP does not exist or does not host any lightning nodes on clearnet`);
         return;
       }
 
@@ -362,7 +363,7 @@ class NodesRoutes {
         nodes: nodes,
       });
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get nodes per ISP');
     }
   }
 
@@ -374,7 +375,7 @@ class NodesRoutes {
       res.setHeader('Expires', new Date(Date.now() + 1000 * 600).toUTCString());
       res.json(nodesPerAs);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get nodes per country');
     }
   }
 }

backend/src/api/liquid/liquid.routes.ts

@@ -3,6 +3,7 @@ import { Application, Request, Response } from 'express';
 import config from '../../config';
 import elementsParser from './elements-parser';
 import icons from './icons';
+import { handleError } from '../../utils/api';
 
 class LiquidRoutes {
   public initRoutes(app: Application) {
@@ -42,7 +43,7 @@ class LiquidRoutes {
       res.setHeader('content-length', result.length);
       res.send(result);
     } else {
-      res.status(404).send('Asset icon not found');
+      handleError(req, res, 404, 'Asset icon not found');
     }
   }
 
@@ -51,7 +52,7 @@ class LiquidRoutes {
     if (result) {
       res.json(result);
     } else {
-      res.status(404).send('Asset icons not found');
+      handleError(req, res, 404, 'Asset icons not found');
     }
   }
 
@@ -82,7 +83,7 @@ class LiquidRoutes {
       res.setHeader('Expires', new Date(Date.now() + 1000 * 60 * 60).toUTCString());
       res.json(pegs);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get pegs by month');
     }
   }
 
@@ -94,7 +95,7 @@ class LiquidRoutes {
       res.setHeader('Expires', new Date(Date.now() + 1000 * 60 * 60).toUTCString());
       res.json(reserves);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get reserves by month');
     }
   }
 
@@ -106,7 +107,7 @@ class LiquidRoutes {
       res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
       res.json(currentSupply);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get pegs');
     }
   }
 
@@ -118,7 +119,7 @@ class LiquidRoutes {
       res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
       res.json(currentReserves);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get reserves');
     }
   }
 
@@ -130,7 +131,7 @@ class LiquidRoutes {
       res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
       res.json(auditStatus);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get federation audit status');
     }
   }
 
@@ -142,7 +143,7 @@ class LiquidRoutes {
       res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
       res.json(federationAddresses);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get federation addresses');
     }
   }
 
@@ -154,7 +155,7 @@ class LiquidRoutes {
       res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
       res.json(federationAddresses);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get federation addresses');
     }
   }
 
@@ -166,7 +167,7 @@ class LiquidRoutes {
       res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
       res.json(federationUtxos);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get federation utxos');
     }
   }
 
@@ -178,7 +179,7 @@ class LiquidRoutes {
       res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
       res.json(expiredUtxos);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get expired utxos');
     }
   }
 
@@ -190,7 +191,7 @@ class LiquidRoutes {
       res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
       res.json(federationUtxos);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get federation utxos number');
     }
   }
 
@@ -202,7 +203,7 @@ class LiquidRoutes {
       res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
       res.json(emergencySpentUtxos);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get emergency spent utxos');
     }
   }
 
@@ -214,7 +215,7 @@ class LiquidRoutes {
       res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
       res.json(emergencySpentUtxos);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get emergency spent utxos stats');
     }
   }
 
@@ -226,7 +227,7 @@ class LiquidRoutes {
       res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
       res.json(recentPegs);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get pegs list');
     }
   }
 
@@ -238,7 +239,7 @@ class LiquidRoutes {
       res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
       res.json(pegsVolume);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get pegs volume daily');
     }
   }
 
@@ -250,7 +251,7 @@ class LiquidRoutes {
       res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
       res.json(pegsCount);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get pegs count');
     }
   }
 

backend/src/api/mempool-blocks.ts

@@ -1,11 +1,13 @@
 import { GbtGenerator, GbtResult, ThreadTransaction as RustThreadTransaction, ThreadAcceleration as RustThreadAcceleration } from 'rust-gbt';
 import logger from '../logger';
-import { MempoolBlock, MempoolTransactionExtended, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats, TransactionClassified, TransactionCompressed, MempoolDeltaChange, GbtCandidates } from '../mempool.interfaces';
+import { MempoolBlock, MempoolTransactionExtended, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats, TransactionClassified, TransactionCompressed, MempoolDeltaChange, GbtCandidates, PoolTag } from '../mempool.interfaces';
 import { Common, OnlineFeeStatsCalculator } from './common';
 import config from '../config';
 import { Worker } from 'worker_threads';
 import path from 'path';
 import mempool from './mempool';
+import { Acceleration } from './services/acceleration';
+import PoolsRepository from '../repositories/PoolsRepository';
 
 const MAX_UINT32 = Math.pow(2, 32) - 1;
 
@@ -14,12 +16,14 @@ class MempoolBlocks {
   private mempoolBlockDeltas: MempoolBlockDelta[] = [];
   private txSelectionWorker: Worker | null = null;
   private rustInitialized: boolean = false;
-  private rustGbtGenerator: GbtGenerator = new GbtGenerator();
+  private rustGbtGenerator: GbtGenerator = new GbtGenerator(config.MEMPOOL.BLOCK_WEIGHT_UNITS, config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT);
 
   private nextUid: number = 1;
   private uidMap: Map<number, string> = new Map(); // map short numerical uids to full txids
   private txidMap: Map<string, number> = new Map(); // map full txids back to short numerical uids
 
+  private pools: { [id: number]: PoolTag } = {};
+
   public getMempoolBlocks(): MempoolBlock[] {
     return this.mempoolBlocks.map((block) => {
       return {
@@ -41,6 +45,18 @@ class MempoolBlocks {
     return this.mempoolBlockDeltas;
   }
 
+  public async updatePools$(): Promise<void> {
+    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
+      this.pools = {};
+      return;
+    }
+    const allPools = await PoolsRepository.$getPools();
+    this.pools = {};
+    for (const pool of allPools) {
+      this.pools[pool.uniqueId] = pool;
+    }
+  }
+
   private calculateMempoolDeltas(prevBlocks: MempoolBlockWithTransactions[], mempoolBlocks: MempoolBlockWithTransactions[]): MempoolBlockDelta[] {
     const mempoolBlockDeltas: MempoolBlockDelta[] = [];
     for (let i = 0; i < Math.max(mempoolBlocks.length, prevBlocks.length); i++) {
@@ -214,7 +230,7 @@ class MempoolBlocks {
 
   private resetRustGbt(): void {
     this.rustInitialized = false;
-    this.rustGbtGenerator = new GbtGenerator();
+    this.rustGbtGenerator = new GbtGenerator(config.MEMPOOL.BLOCK_WEIGHT_UNITS, config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT);
   }
 
   public async $rustMakeBlockTemplates(txids: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, candidates: GbtCandidates | undefined, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
@@ -246,7 +262,7 @@ class MempoolBlocks {
     });
 
     // run the block construction algorithm in a separate thread, and wait for a result
-    const rustGbt = saveResults ? this.rustGbtGenerator : new GbtGenerator();
+    const rustGbt = saveResults ? this.rustGbtGenerator : new GbtGenerator(config.MEMPOOL.BLOCK_WEIGHT_UNITS, config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT);
     try {
       const { blocks, blockWeights, rates, clusters, overflow } = this.convertNapiResultTxids(
         await rustGbt.make(transactions as RustThreadTransaction[], convertedAccelerations as RustThreadAcceleration[], this.nextUid),
@@ -333,10 +349,13 @@ class MempoolBlocks {
     }
   }
 
-  private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], candidates: GbtCandidates | undefined, accelerations, accelerationPool, saveResults): MempoolBlockWithTransactions[] {
+  private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], candidates: GbtCandidates | undefined, accelerations: { [txid: string]: Acceleration }, accelerationPool, saveResults): MempoolBlockWithTransactions[] {
     for (const txid of Object.keys(candidates?.txs ?? mempool)) {
       if (txid in mempool) {
         mempool[txid].cpfpDirty = false;
+        mempool[txid].ancestors = [];
+        mempool[txid].descendants = [];
+        mempool[txid].bestDescendant = null;
       }
     }
     for (const [txid, rate] of rates) {
@@ -350,7 +369,7 @@ class MempoolBlocks {
     const lastBlockIndex = blocks.length - 1;
     let hasBlockStack = blocks.length >= 8;
     let stackWeight;
-    let feeStatsCalculator: OnlineFeeStatsCalculator | void;
+    let feeStatsCalculator: OnlineFeeStatsCalculator | null = null;
     if (hasBlockStack) {
       if (blockWeights && blockWeights[7] !== null) {
         stackWeight = blockWeights[7];
@@ -361,28 +380,36 @@ class MempoolBlocks {
       feeStatsCalculator = new OnlineFeeStatsCalculator(stackWeight, 0.5, [10, 20, 30, 40, 50, 60, 70, 80, 90]);
     }
 
+    const ancestors: Ancestor[] = [];
+    const descendants: Ancestor[] = [];
+    let ancestor: MempoolTransactionExtended;
     for (const cluster of clusters) {
       for (const memberTxid of cluster) {
         const mempoolTx = mempool[memberTxid];
         if (mempoolTx) {
-          const ancestors: Ancestor[] = [];
-          const descendants: Ancestor[] = [];
+          // ugly micro-optimization to avoid allocating new arrays
+          ancestors.length = 0;
+          descendants.length = 0;
           let matched = false;
           cluster.forEach(txid => {
+            ancestor = mempool[txid];
             if (txid === memberTxid) {
               matched = true;
             } else {
-              if (!mempool[txid]) {
+              if (!ancestor) {
                 console.log('txid missing from mempool! ', txid, candidates?.txs[txid]);
+                return;
               }
               const relative = {
                 txid: txid,
-                fee: mempool[txid].fee,
-                weight: (mempool[txid].adjustedVsize * 4),
+                fee: ancestor.fee,
+                weight: (ancestor.adjustedVsize * 4),
               };
               if (matched) {
                 descendants.push(relative);
-                mempoolTx.lastBoosted = Math.max(mempoolTx.lastBoosted || 0, mempool[txid].firstSeen || 0);
+                if (!mempoolTx.lastBoosted || (ancestor.firstSeen && ancestor.firstSeen > mempoolTx.lastBoosted)) {
+                  mempoolTx.lastBoosted = ancestor.firstSeen;
+                }
               } else {
                 ancestors.push(relative);
               }
@@ -391,17 +418,33 @@ class MempoolBlocks {
           if (mempoolTx.ancestors?.length !== ancestors.length || mempoolTx.descendants?.length !== descendants.length) {
             mempoolTx.cpfpDirty = true;
           }
-          Object.assign(mempoolTx, {ancestors, descendants, bestDescendant: null, cpfpChecked: true});
+          // ugly micro-optimization to avoid allocating new arrays or objects
+          if (mempoolTx.ancestors) {
+            mempoolTx.ancestors.length = 0;
+          } else {
+            mempoolTx.ancestors = [];
+          }
+          if (mempoolTx.descendants) {
+            mempoolTx.descendants.length = 0;
+          } else {
+            mempoolTx.descendants = [];
+          }
+          mempoolTx.ancestors.push(...ancestors);
+          mempoolTx.descendants.push(...descendants);
+          mempoolTx.cpfpChecked = true;
         }
       }
     }
 
-    const isAccelerated : { [txid: string]: boolean } = {};
+    const isAcceleratedBy : { [txid: string]: number[] | false } = {};
 
     const sizeLimit = (config.MEMPOOL.BLOCK_WEIGHT_UNITS / 4) * 1.2;
     // update this thread's mempool with the results
     let mempoolTx: MempoolTransactionExtended;
-    const mempoolBlocks: MempoolBlockWithTransactions[] = blocks.map((block, blockIndex) => {
+    let acceleration: Acceleration;
+    const mempoolBlocks: MempoolBlockWithTransactions[] = [];
+    for (let blockIndex = 0; blockIndex < blocks.length; blockIndex++) {
+      const block = blocks[blockIndex];
       let totalSize = 0;
       let totalVsize = 0;
       let totalWeight = 0;
@@ -417,8 +460,9 @@ class MempoolBlocks {
         }
       }
 
-      for (const txid of block) {
-        if (txid) {
+      for (let i = 0; i < block.length; i++) {
+        const txid = block[i];
+        if (txid in mempool) {
           mempoolTx = mempool[txid];
           // save position in projected blocks
           mempoolTx.position = {
@@ -426,24 +470,40 @@ class MempoolBlocks {
             vsize: totalVsize + (mempoolTx.vsize / 2),
           };
 
-          const acceleration = accelerations[txid];
-          if (isAccelerated[txid] || (acceleration && (!accelerationPool || acceleration.pools.includes(accelerationPool)))) {
-            if (!mempoolTx.acceleration) {
-              mempoolTx.cpfpDirty = true;
-            }
-            mempoolTx.acceleration = true;
-            for (const ancestor of mempoolTx.ancestors || []) {
-              if (!mempool[ancestor.txid].acceleration) {
-                mempool[ancestor.txid].cpfpDirty = true;
+          if (txid in accelerations) {
+            acceleration = accelerations[txid];
+            if (isAcceleratedBy[txid] || (acceleration && (!accelerationPool || acceleration.pools.includes(accelerationPool)))) {
+              if (!mempoolTx.acceleration) {
+                mempoolTx.cpfpDirty = true;
+              }
+              mempoolTx.acceleration = true;
+              mempoolTx.acceleratedBy = isAcceleratedBy[txid] || acceleration?.pools;
+              mempoolTx.acceleratedAt = acceleration?.added;
+              mempoolTx.feeDelta = acceleration?.feeDelta;
+              for (const ancestor of mempoolTx.ancestors || []) {
+                if (!(ancestor.txid in mempool)) {
+                  continue;
+                }
+                if (!mempool[ancestor.txid].acceleration) {
+                  mempool[ancestor.txid].cpfpDirty = true;
+                }
+                mempool[ancestor.txid].acceleration = true;
+                mempool[ancestor.txid].acceleratedBy = mempoolTx.acceleratedBy;
+                mempool[ancestor.txid].acceleratedAt = mempoolTx.acceleratedAt;
+                mempool[ancestor.txid].feeDelta = mempoolTx.feeDelta;
+                isAcceleratedBy[ancestor.txid] = mempoolTx.acceleratedBy;
+              }
+            } else {
+              if (mempoolTx.acceleration) {
+                mempoolTx.cpfpDirty = true;
+                delete mempoolTx.acceleration;
               }
-              mempool[ancestor.txid].acceleration = true;
-              isAccelerated[ancestor.txid] = true;
             }
           } else {
             if (mempoolTx.acceleration) {
               mempoolTx.cpfpDirty = true;
+              delete mempoolTx.acceleration;
             }
-            delete mempoolTx.acceleration;
           }
 
           // online calculation of stack-of-blocks fee stats
@@ -461,7 +521,7 @@ class MempoolBlocks {
           }
         }
       }
-      return this.dataToMempoolBlocks(
+      mempoolBlocks[blockIndex] = this.dataToMempoolBlocks(
         block,
         transactions,
         totalSize,
@@ -469,13 +529,13 @@ class MempoolBlocks {
         totalFees,
         (hasBlockStack && blockIndex === lastBlockIndex && feeStatsCalculator) ? feeStatsCalculator.getRawFeeStats() : undefined,
       );
-    });
+    };
 
     if (saveResults) {
       const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, mempoolBlocks);
       this.mempoolBlocks = mempoolBlocks;
       this.mempoolBlockDeltas = deltas;
-
+      this.updateAccelerationPositions(mempool, accelerations, mempoolBlocks);
     }
 
     return mempoolBlocks;
@@ -622,6 +682,124 @@ class MempoolBlocks {
       tx.acc ? 1 : 0,
     ];
   }
+
+  // estimates and saves positions of accelerations in mining partner mempools
+  private updateAccelerationPositions(mempoolCache: { [txid: string]: MempoolTransactionExtended }, accelerations: { [txid: string]: Acceleration }, mempoolBlocks: MempoolBlockWithTransactions[]): void {
+    const accelerationPositions: { [txid: string]: { poolId: number, pool: string, block: number, vsize: number }[] } = {};
+    // keep track of simulated mempool blocks for each active pool
+    const pools: {
+      [pool: string]: { name: string, block: number, vsize: number, accelerations: string[], complete: boolean };
+    } = {};
+    // prepare a list of accelerations in ascending order (we'll pop items off the end of the list)
+    const accQueue: { acceleration: Acceleration, rate: number, vsize: number }[] = Object.values(accelerations).filter(acc => acc.txid in mempoolCache).map(acc => {
+      let vsize = mempoolCache[acc.txid].vsize;
+      for (const ancestor of mempoolCache[acc.txid].ancestors || []) {
+        vsize += (ancestor.weight / 4);
+      }
+      return {
+        acceleration: acc,
+        rate: mempoolCache[acc.txid].effectiveFeePerVsize,
+        vsize
+      };
+    }).sort((a, b) => a.rate - b.rate);
+    // initialize the pool tracker
+    for (const { acceleration }  of accQueue) {
+      accelerationPositions[acceleration.txid] = [];
+      for (const pool of acceleration.pools) {
+        if (!pools[pool]) {
+          pools[pool] = {
+            name: this.pools[pool]?.name || 'unknown',
+            block: 0,
+            vsize: 0,
+            accelerations: [],
+            complete: false,
+          };
+        }
+        pools[pool].accelerations.push(acceleration.txid);
+      }
+      for (const ancestor of mempoolCache[acceleration.txid].ancestors || []) {
+        accelerationPositions[ancestor.txid] = [];
+      }
+    }
+
+    for (const pool of Object.keys(pools)) {
+      // if any pools accepted *every* acceleration, we can just use the GBT result positions directly
+      if (pools[pool].accelerations.length === Object.keys(accelerations).length) {
+        pools[pool].complete = true;
+      }
+    }
+
+    let block = 0;
+    let index = 0;
+    let next = accQueue.pop();
+    // build simulated blocks for each pool by taking the best option from
+    // either the mempool or the list of accelerations.
+    while (next && block < mempoolBlocks.length) {
+      while (next && index < mempoolBlocks[block].transactions.length) {
+        const nextTx = mempoolBlocks[block].transactions[index];
+        if (next.rate >= (nextTx.rate || (nextTx.fee / nextTx.vsize))) {
+          for (const pool of next.acceleration.pools) {
+            if (pools[pool].vsize + next.vsize <= 999_000) {
+              pools[pool].vsize += next.vsize;
+            } else {
+              pools[pool].block++;
+              pools[pool].vsize = next.vsize;
+            }
+            // insert the acceleration into matching pool's blocks
+            if (pools[pool].complete && mempoolCache[next.acceleration.txid]?.position !== undefined) {
+              accelerationPositions[next.acceleration.txid].push({
+                ...mempoolCache[next.acceleration.txid].position as { block: number, vsize: number },
+                poolId: pool,
+                pool: pools[pool].name
+              });
+            } else {
+              accelerationPositions[next.acceleration.txid].push({
+                poolId: pool,
+                pool: pools[pool].name,
+                block: pools[pool].block,
+                vsize: pools[pool].vsize - (next.vsize / 2),
+              });
+            }
+            // and any accelerated ancestors
+            for (const ancestor of mempoolCache[next.acceleration.txid].ancestors || []) {
+              if (pools[pool].complete && mempoolCache[ancestor.txid]?.position !== undefined) {
+                accelerationPositions[ancestor.txid].push({
+                  ...mempoolCache[ancestor.txid].position as { block: number, vsize: number },
+                  poolId: pool,
+                  pool: pools[pool].name,
+                });
+              } else {
+                accelerationPositions[ancestor.txid].push({
+                  poolId: pool,
+                  pool: pools[pool].name,
+                  block: pools[pool].block,
+                  vsize: pools[pool].vsize - (next.vsize / 2),
+                });
+              }
+            }
+          }
+          next = accQueue.pop();
+        } else {
+          // skip accelerated transactions and their CPFP ancestors
+          if (accelerationPositions[nextTx.txid] == null) {
+            // insert into all pools' blocks
+            for (const pool of Object.keys(pools)) {
+              if (pools[pool].vsize + nextTx.vsize <= 999_000) {
+                pools[pool].vsize += nextTx.vsize;
+              } else {
+                pools[pool].block++;
+                pools[pool].vsize = nextTx.vsize;
+              }
+            }
+          }
+          index++;
+        }
+      }
+      block++;
+      index = 0;
+    }
+    mempool.setAccelerationPositions(accelerationPositions);
+  }
 }
 
 export default new MempoolBlocks();

backend/src/api/mempool.ts

@@ -10,6 +10,7 @@ import bitcoinClient from './bitcoin/bitcoin-client';
 import bitcoinSecondClient from './bitcoin/bitcoin-second-client';
 import rbfCache from './rbf-cache';
 import { Acceleration } from './services/acceleration';
+import accelerationApi from './services/acceleration';
 import redisCache from './redis-cache';
 import blocks from './blocks';
 
@@ -19,14 +20,16 @@ class Mempool {
   private mempoolCache: { [txId: string]: MempoolTransactionExtended } = {};
   private mempoolCandidates: { [txid: string ]: boolean } = {};
   private spendMap = new Map<string, MempoolTransactionExtended>();
+  private recentlyDeleted: MempoolTransactionExtended[][] = []; // buffer of transactions deleted in recent mempool updates
   private mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0, total_fee: 0,
                                                     maxmempool: 300000000, mempoolminfee: Common.isLiquid() ? 0.00000100 : 0.00001000, minrelaytxfee: Common.isLiquid() ? 0.00000100 : 0.00001000 };
   private mempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, newTransactions: MempoolTransactionExtended[],
-    deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => void) | undefined;
+    deletedTransactions: MempoolTransactionExtended[][], accelerationDelta: string[]) => void) | undefined;
   private $asyncMempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, mempoolSize: number, newTransactions: MempoolTransactionExtended[],
-    deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[], candidates?: GbtCandidates) => Promise<void>) | undefined;
+    deletedTransactions: MempoolTransactionExtended[][], accelerationDelta: string[], candidates?: GbtCandidates) => Promise<void>) | undefined;
 
   private accelerations: { [txId: string]: Acceleration } = {};
+  private accelerationPositions: { [txid: string]: { poolId: number, pool: string, block: number, vsize: number }[] } = {};
 
   private txPerSecondArray: number[] = [];
   private txPerSecond: number = 0;
@@ -73,12 +76,12 @@ class Mempool {
   }
 
   public setMempoolChangedCallback(fn: (newMempool: { [txId: string]: MempoolTransactionExtended; },
-    newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => void): void {
+    newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[][], accelerationDelta: string[]) => void): void {
     this.mempoolChangedCallback = fn;
   }
 
   public setAsyncMempoolChangedCallback(fn: (newMempool: { [txId: string]: MempoolTransactionExtended; }, mempoolSize: number,
-    newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[],
+    newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[][], accelerationDelta: string[],
     candidates?: GbtCandidates) => Promise<void>): void {
     this.$asyncMempoolChangedCallback = fn;
   }
@@ -205,7 +208,7 @@ class Mempool {
     return txTimes;
   }
 
-  public async $updateMempool(transactions: string[], accelerations: Acceleration[] | null, minFeeMempool: string[], minFeeTip: number, pollRate: number): Promise<void> {
+  public async $updateMempool(transactions: string[], accelerations: Record<string, Acceleration> | null, minFeeMempool: string[], minFeeTip: number, pollRate: number): Promise<void> {
     logger.debug(`Updating mempool...`);
 
     // warn if this run stalls the main loop for more than 2 minutes
@@ -352,7 +355,7 @@ class Mempool {
     const newTransactionsStripped = newTransactions.map((tx) => Common.stripTransaction(tx));
     this.latestTransactions = newTransactionsStripped.concat(this.latestTransactions).slice(0, 6);
 
-    const accelerationDelta = accelerations != null ? await this.$updateAccelerations(accelerations) : [];
+    const accelerationDelta = accelerations != null ? await this.updateAccelerations(accelerations) : [];
     if (accelerationDelta.length) {
       hasChange = true;
     }
@@ -361,12 +364,15 @@ class Mempool {
 
     const candidatesChanged = candidates?.added?.length || candidates?.removed?.length;
 
-    if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) {
-      this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions, accelerationDelta);
+    this.recentlyDeleted.unshift(deletedTransactions);
+    this.recentlyDeleted.length = Math.min(this.recentlyDeleted.length, 10); // truncate to the last 10 mempool updates
+
+    if (this.mempoolChangedCallback && (hasChange || newTransactions.length || deletedTransactions.length)) {
+      this.mempoolChangedCallback(this.mempoolCache, newTransactions, this.recentlyDeleted, accelerationDelta);
     }
-    if (this.$asyncMempoolChangedCallback && (hasChange || deletedTransactions.length || candidatesChanged)) {
+    if (this.$asyncMempoolChangedCallback && (hasChange || newTransactions.length || deletedTransactions.length || candidatesChanged)) {
       this.updateTimerProgress(timer, 'running async mempool callback');
-      await this.$asyncMempoolChangedCallback(this.mempoolCache, newMempoolSize, newTransactions, deletedTransactions, accelerationDelta, candidates);
+      await this.$asyncMempoolChangedCallback(this.mempoolCache, newMempoolSize, newTransactions, this.recentlyDeleted, accelerationDelta, candidates);
       this.updateTimerProgress(timer, 'completed async mempool callback');
     }
 
@@ -394,58 +400,11 @@ class Mempool {
     return this.accelerations;
   }
 
-  public $updateAccelerations(newAccelerations: Acceleration[]): string[] {
-    if (!config.MEMPOOL_SERVICES.ACCELERATIONS) {
-      return [];
-    }
-
+  public updateAccelerations(newAccelerationMap: Record<string, Acceleration>): string[] {
     try {
-      const changed: string[] = [];
-
-      const newAccelerationMap: { [txid: string]: Acceleration } = {};
-      for (const acceleration of newAccelerations) {
-        newAccelerationMap[acceleration.txid] = acceleration;
-        if (this.accelerations[acceleration.txid] == null) {
-          // new acceleration
-          changed.push(acceleration.txid);
-        } else {
-          if (this.accelerations[acceleration.txid].feeDelta !== acceleration.feeDelta) {
-            // feeDelta changed
-            changed.push(acceleration.txid);
-          } else if (this.accelerations[acceleration.txid].pools?.length) {
-            let poolsChanged = false;
-            const pools = new Set();
-            this.accelerations[acceleration.txid].pools.forEach(pool => {
-              pools.add(pool);
-            });
-            acceleration.pools.forEach(pool => {
-              if (!pools.has(pool)) {
-                poolsChanged = true;
-              } else {
-                pools.delete(pool);
-              }
-            });
-            if (pools.size > 0) {
-              poolsChanged = true;
-            }
-            if (poolsChanged) {
-              // pools changed
-              changed.push(acceleration.txid);
-            }
-          }
-        }
-      }
-
-      for (const oldTxid of Object.keys(this.accelerations)) {
-        if (!newAccelerationMap[oldTxid]) {
-          // removed
-          changed.push(oldTxid);
-        }
-      }
-
+      const accelerationDelta = accelerationApi.getAccelerationDelta(this.accelerations, newAccelerationMap);
       this.accelerations = newAccelerationMap;
-
-      return changed;
+      return accelerationDelta;
     } catch (e: any) {
       logger.debug(`Failed to update accelerations: ` + (e instanceof Error ? e.message : e));
       return [];
@@ -510,6 +469,14 @@ class Mempool {
     }
   }
 
+  setAccelerationPositions(positions: { [txid: string]: { poolId: number, pool: string, block: number, vsize: number }[] }): void {
+    this.accelerationPositions = positions;
+  }
+
+  getAccelerationPositions(txid: string): { [pool: number]: { poolId: number, pool: string, block: number, vsize: number } } | undefined {
+    return this.accelerationPositions[txid];
+  }
+
   private startTimer() {
     const state: any = {
       start: Date.now(),
@@ -532,16 +499,7 @@ class Mempool {
     }
   }
 
-  public handleRbfTransactions(rbfTransactions: { [txid: string]: MempoolTransactionExtended[]; }): void {
-    for (const rbfTransaction in rbfTransactions) {
-      if (this.mempoolCache[rbfTransaction] && rbfTransactions[rbfTransaction]?.length) {
-        // Store replaced transactions
-        rbfCache.add(rbfTransactions[rbfTransaction], this.mempoolCache[rbfTransaction]);
-      }
-    }
-  }
-
-  public handleMinedRbfTransactions(rbfTransactions: { [txid: string]: { replaced: MempoolTransactionExtended[], replacedBy: TransactionExtended }}): void {
+  public handleRbfTransactions(rbfTransactions: { [txid: string]: { replaced: MempoolTransactionExtended[], replacedBy: TransactionExtended }}): void {
     for (const rbfTransaction in rbfTransactions) {
       if (rbfTransactions[rbfTransaction].replacedBy && rbfTransactions[rbfTransaction]?.replaced?.length) {
         // Store replaced transactions

backend/src/api/mini-miner.ts

@@ -0,0 +1,515 @@
+import { Acceleration } from './acceleration/acceleration';
+import { MempoolTransactionExtended } from '../mempool.interfaces';
+import logger from '../logger';
+
+const BLOCK_WEIGHT_UNITS = 4_000_000;
+const BLOCK_SIGOPS = 80_000;
+const MAX_RELATIVE_GRAPH_SIZE = 100;
+
+export interface GraphTx {
+  txid: string;
+  vsize: number;
+  weight: number;
+  depends: string[];
+  spentby: string[];
+
+  ancestorcount: number;
+  ancestorsize: number;
+  fees: { // in sats
+    base: number;
+    ancestor: number;
+  };
+
+  ancestors: Map<string, GraphTx>,
+  ancestorRate: number;
+  individualRate: number;
+  score: number;
+}
+
+interface TemplateTransaction {
+  txid: string;
+  order: number;
+  weight: number;
+  adjustedVsize: number; // sigop-adjusted vsize, rounded up to the nearest integer
+  sigops: number;
+  fee: number;
+  feeDelta: number;
+  ancestors: string[];
+  cluster: string[];
+  effectiveFeePerVsize: number;
+}
+
+interface MinerTransaction extends TemplateTransaction {
+  inputs: string[];
+  feePerVsize: number;
+  relativesSet: boolean;
+  ancestorMap: Map<string, MinerTransaction>;
+  children: Set<MinerTransaction>;
+  ancestorFee: number;
+  ancestorVsize: number;
+  ancestorSigops: number;
+  score: number;
+  used: boolean;
+  modified: boolean;
+  dependencyRate: number;
+}
+
+/**
+ * Takes a raw transaction, and builds a graph of same-block relatives,
+ * and returns as a GraphTx
+ *
+ * @param tx
+ */
+export function getSameBlockRelatives(tx: MempoolTransactionExtended, transactions: MempoolTransactionExtended[]): Map<string, GraphTx> {
+  const blockTxs = new Map<string, MempoolTransactionExtended>(); // map of txs in this block
+  const spendMap = new Map<string, string>(); // map of outpoints to spending txids
+  for (const tx of transactions) {
+    blockTxs.set(tx.txid, tx);
+    for (const vin of tx.vin) {
+      spendMap.set(`${vin.txid}:${vin.vout}`, tx.txid);
+    }
+  }
+
+  const relatives: Map<string, GraphTx> = new Map();
+  const stack: string[] = [tx.txid];
+
+  // build set of same-block ancestors
+  while (stack.length > 0) {
+    const nextTxid = stack.pop();
+    const nextTx = nextTxid ? blockTxs.get(nextTxid) : null;
+    if (!nextTx || relatives.has(nextTx.txid)) {
+      continue;
+    }
+
+    const mempoolTx = convertToGraphTx(nextTx, spendMap);
+
+    for (const txid of [...mempoolTx.depends, ...mempoolTx.spentby]) {
+      if (txid) {
+        stack.push(txid);
+      }
+    }
+
+    relatives.set(mempoolTx.txid, mempoolTx);
+  }
+
+  return relatives;
+}
+
+/**
+ * Takes a raw transaction and converts it to GraphTx format
+ * fee and ancestor data is initialized with dummy/null values
+ *
+ * @param tx
+ */
+export function convertToGraphTx(tx: MempoolTransactionExtended, spendMap?: Map<string, MempoolTransactionExtended | string>): GraphTx {
+  return {
+    txid: tx.txid,
+    vsize: Math.max(tx.sigops * 5, Math.ceil(tx.weight / 4)),
+    weight: tx.weight,
+    fees: {
+      base: tx.fee || 0,
+      ancestor: tx.fee || 0,
+    },
+    depends: (tx.vin.map(vin => vin.txid).filter(depend => depend) as string[]),
+    spentby: spendMap ? (tx.vout.map((vout, index) => { const spend = spendMap.get(`${tx.txid}:${index}`); return (spend?.['txid'] || spend); }).filter(spent => spent) as string[]) : [],
+
+    ancestorcount: 1,
+    ancestorsize: Math.max(tx.sigops * 5, Math.ceil(tx.weight / 4)),
+    ancestors: new Map<string, GraphTx>(),
+    ancestorRate: 0,
+    individualRate: 0,
+    score: 0,
+  };
+}
+
+/**
+ * Takes a map of transaction ancestors, and expands it into a full graph of up to MAX_GRAPH_SIZE in-mempool relatives
+ */
+export function expandRelativesGraph(mempool: { [txid: string]: MempoolTransactionExtended }, ancestors: Map<string, GraphTx>, spendMap: Map<string, MempoolTransactionExtended>): Map<string, GraphTx> {
+  const relatives: Map<string, GraphTx> = new Map();
+  const stack: GraphTx[] = Array.from(ancestors.values());
+  while (stack.length > 0) {
+    if (relatives.size > MAX_RELATIVE_GRAPH_SIZE) {
+      return relatives;
+    }
+
+    const nextTx = stack.pop();
+    if (!nextTx) {
+      continue;
+    }
+    relatives.set(nextTx.txid, nextTx);
+
+    for (const relativeTxid of [...nextTx.depends, ...nextTx.spentby]) {
+      if (relatives.has(relativeTxid)) {
+        // already processed this tx
+        continue;
+      }
+      let ancestorTx = ancestors.get(relativeTxid);
+      if (!ancestorTx && relativeTxid in mempool) {
+        const mempoolTx = mempool[relativeTxid];
+        ancestorTx = convertToGraphTx(mempoolTx, spendMap);
+      }
+      if (ancestorTx) {
+        stack.push(ancestorTx);
+      }
+    }
+  }
+
+  return relatives;
+}
+
+/**
+ * Recursively traverses an in-mempool dependency graph, and sets a Map of in-mempool ancestors
+ * for each transaction.
+ *
+ * @param tx
+ * @param all
+ */
+function setAncestors(tx: GraphTx, all: Map<string, GraphTx>, visited: Map<string, Map<string, GraphTx>>, depth: number = 0): Map<string, GraphTx> {
+  // sanity check for infinite recursion / too many ancestors (should never happen)
+  if (depth > MAX_RELATIVE_GRAPH_SIZE) {
+    logger.warn('cpfp dependency calculation failed: setAncestors reached depth of 100, unable to proceed');
+    return tx.ancestors;
+  }
+
+  // initialize the ancestor map for this tx
+  tx.ancestors = new Map<string, GraphTx>();
+  tx.depends.forEach(parentId => {
+    const parent = all.get(parentId);
+    if (parent) {
+      // add the parent
+      tx.ancestors?.set(parentId, parent);
+      // check for a cached copy of this parent's ancestors
+      let ancestors = visited.get(parent.txid);
+      if (!ancestors) {
+        // recursively fetch the parent's ancestors
+        ancestors = setAncestors(parent, all, visited, depth + 1);
+      }
+      // and add to this tx's map
+      ancestors.forEach((ancestor, ancestorId) => {
+        tx.ancestors?.set(ancestorId, ancestor);
+      });
+    }
+  });
+  visited.set(tx.txid, tx.ancestors);
+
+  return tx.ancestors;
+}
+
+/**
+   * Efficiently sets a Map of in-mempool ancestors for each member of an expanded relative graph
+   * by running setAncestors on each leaf, and caching intermediate results.
+   * then initializes ancestor data for each transaction
+   *
+   * @param all
+   */
+export function initializeRelatives(mempoolTxs: Map<string, GraphTx>): Map<string, GraphTx> {
+  const visited: Map<string, Map<string, GraphTx>> = new Map();
+  const leaves: GraphTx[] = Array.from(mempoolTxs.values()).filter(entry => entry.spentby.length === 0);
+  for (const leaf of leaves) {
+    setAncestors(leaf, mempoolTxs, visited);
+  }
+  mempoolTxs.forEach(entry => {
+    entry.ancestors?.forEach(ancestor => {
+      entry.ancestorcount++;
+      entry.ancestorsize += ancestor.vsize;
+      entry.fees.ancestor += ancestor.fees.base;
+    });
+    setAncestorScores(entry);
+  });
+  return mempoolTxs;
+}
+
+/**
+ * Remove a cluster of transactions from an in-mempool dependency graph
+ * and update the survivors' scores and ancestors
+ *
+ * @param cluster
+ * @param ancestors
+ */
+export function removeAncestors(cluster: Map<string, GraphTx>, all: Map<string, GraphTx>): void {
+  // remove
+  cluster.forEach(tx => {
+    all.delete(tx.txid);
+  });
+
+  // update survivors
+  all.forEach(tx => {
+    cluster.forEach(remove => {
+      if (tx.ancestors?.has(remove.txid)) {
+        // remove as dependency
+        tx.ancestors.delete(remove.txid);
+        tx.depends = tx.depends.filter(parent => parent !== remove.txid);
+        // update ancestor sizes and fees
+        tx.ancestorsize -= remove.vsize;
+        tx.fees.ancestor -= remove.fees.base;
+      }
+    });
+    // recalculate fee rates
+    setAncestorScores(tx);
+  });
+}
+
+/**
+ * Take a mempool transaction, and set the fee rates and ancestor score
+ *
+ * @param tx
+ */
+export function setAncestorScores(tx: GraphTx): void {
+  tx.individualRate = tx.fees.base / tx.vsize;
+  tx.ancestorRate = tx.fees.ancestor / tx.ancestorsize;
+  tx.score = Math.min(tx.individualRate, tx.ancestorRate);
+}
+
+// Sort by descending score
+export function mempoolComparator(a: GraphTx, b: GraphTx): number {
+  return b.score - a.score;
+}
+
+/*
+* Build a block using an approximation of the transaction selection algorithm from Bitcoin Core
+* (see BlockAssembler in https://github.com/bitcoin/bitcoin/blob/master/src/node/miner.cpp)
+*/
+export function makeBlockTemplate(candidates: MempoolTransactionExtended[], accelerations: Acceleration[], maxBlocks: number = 8, weightLimit: number = BLOCK_WEIGHT_UNITS, sigopLimit: number = BLOCK_SIGOPS): TemplateTransaction[] {
+  const auditPool: Map<string, MinerTransaction> = new Map();
+  const mempoolArray: MinerTransaction[] = [];
+
+  candidates.forEach(tx => {
+    // initializing everything up front helps V8 optimize property access later
+    const adjustedVsize = Math.ceil(Math.max(tx.weight / 4, 5 * (tx.sigops || 0)));
+    const feePerVsize = (tx.fee / adjustedVsize);
+    auditPool.set(tx.txid, {
+      txid: tx.txid,
+      order: txidToOrdering(tx.txid),
+      fee: tx.fee,
+      feeDelta: 0,
+      weight: tx.weight,
+      adjustedVsize,
+      feePerVsize: feePerVsize,
+      effectiveFeePerVsize: feePerVsize,
+      dependencyRate: feePerVsize,
+      sigops: tx.sigops || 0,
+      inputs: (tx.vin?.map(vin => vin.txid) || []) as string[],
+      relativesSet: false,
+      ancestors: [],
+      cluster: [],
+      ancestorMap: new Map<string, MinerTransaction>(),
+      children: new Set<MinerTransaction>(),
+      ancestorFee: 0,
+      ancestorVsize: 0,
+      ancestorSigops: 0,
+      score: 0,
+      used: false,
+      modified: false,
+    });
+    mempoolArray.push(auditPool.get(tx.txid) as MinerTransaction);
+  });
+
+  // set accelerated effective fee
+  for (const acceleration of accelerations) {
+    const tx = auditPool.get(acceleration.txid);
+    if (tx) {
+      tx.feeDelta = acceleration.max_bid;
+      tx.feePerVsize = ((tx.fee + tx.feeDelta) / tx.adjustedVsize);
+      tx.effectiveFeePerVsize = tx.feePerVsize;
+      tx.dependencyRate = tx.feePerVsize;
+    }
+  }
+
+  // Build relatives graph & calculate ancestor scores
+  for (const tx of mempoolArray) {
+    if (!tx.relativesSet) {
+      setRelatives(tx, auditPool);
+    }
+  }
+
+  // Sort by descending ancestor score
+  mempoolArray.sort(priorityComparator);
+
+  // Build blocks by greedily choosing the highest feerate package
+  // (i.e. the package rooted in the transaction with the best ancestor score)
+  const blocks: number[][] = [];
+  let blockWeight = 0;
+  let blockSigops = 0;
+  const transactions: MinerTransaction[] = [];
+  let modified: MinerTransaction[] = [];
+  const overflow: MinerTransaction[] = [];
+  let failures = 0;
+  while (mempoolArray.length || modified.length) {
+    // skip invalid transactions
+    while (mempoolArray[0]?.used || mempoolArray[0]?.modified) {
+      mempoolArray.shift();
+    }
+
+    // Select best next package
+    let nextTx;
+    const nextPoolTx = mempoolArray[0];
+    const nextModifiedTx = modified[0];
+    if (nextPoolTx && (!nextModifiedTx || (nextPoolTx.score || 0) > (nextModifiedTx.score || 0))) {
+      nextTx = nextPoolTx;
+      mempoolArray.shift();
+    } else {
+      modified.shift();
+      if (nextModifiedTx) {
+        nextTx = nextModifiedTx;
+      }
+    }
+
+    if (nextTx && !nextTx?.used) {
+      // Check if the package fits into this block
+      if (blocks.length >= (maxBlocks - 1) || ((blockWeight + (4 * nextTx.ancestorVsize) < weightLimit) && (blockSigops + nextTx.ancestorSigops <= sigopLimit))) {
+        const ancestors: MinerTransaction[] = Array.from(nextTx.ancestorMap.values());
+        // sort ancestors by dependency graph (equivalent to sorting by ascending ancestor count)
+        const sortedTxSet = [...ancestors.sort((a, b) => { return (a.ancestorMap.size || 0) - (b.ancestorMap.size || 0); }), nextTx];
+        const clusterTxids = sortedTxSet.map(tx => tx.txid);
+        const effectiveFeeRate = Math.min(nextTx.dependencyRate || Infinity, nextTx.ancestorFee / nextTx.ancestorVsize);
+        const used: MinerTransaction[] = [];
+        while (sortedTxSet.length) {
+          const ancestor = sortedTxSet.pop();
+          if (!ancestor) {
+            continue;
+          }
+          ancestor.used = true;
+          ancestor.usedBy = nextTx.txid;
+          // update this tx with effective fee rate & relatives data
+          if (ancestor.effectiveFeePerVsize !== effectiveFeeRate) {
+            ancestor.effectiveFeePerVsize = effectiveFeeRate;
+          }
+          ancestor.cluster = clusterTxids;
+          transactions.push(ancestor);
+          blockWeight += ancestor.weight;
+          blockSigops += ancestor.sigops;
+          used.push(ancestor);
+        }
+
+        // remove these as valid package ancestors for any descendants remaining in the mempool
+        if (used.length) {
+          used.forEach(tx => {
+            modified = updateDescendants(tx, auditPool, modified, effectiveFeeRate);
+          });
+        }
+
+        failures = 0;
+      } else {
+        // hold this package in an overflow list while we check for smaller options
+        overflow.push(nextTx);
+        failures++;
+      }
+    }
+
+    // this block is full
+    const exceededPackageTries = failures > 1000 && blockWeight > (weightLimit - 4000);
+    const queueEmpty = !mempoolArray.length && !modified.length;
+
+    if (exceededPackageTries || queueEmpty) {
+      break;
+    }
+  }
+
+  for (const tx of transactions) {
+    tx.ancestors = Object.values(tx.ancestorMap);
+  }
+
+  return transactions;
+}
+
+// traverse in-mempool ancestors
+// recursion unavoidable, but should be limited to depth < 25 by mempool policy
+function setRelatives(
+  tx: MinerTransaction,
+  mempool: Map<string, MinerTransaction>,
+): void {
+  for (const parent of tx.inputs) {
+    const parentTx = mempool.get(parent);
+    if (parentTx && !tx.ancestorMap?.has(parent)) {
+      tx.ancestorMap.set(parent, parentTx);
+      parentTx.children.add(tx);
+      // visit each node only once
+      if (!parentTx.relativesSet) {
+        setRelatives(parentTx, mempool);
+      }
+      parentTx.ancestorMap.forEach((ancestor) => {
+        tx.ancestorMap.set(ancestor.txid, ancestor);
+      });
+    }
+  };
+  tx.ancestorFee = (tx.fee + tx.feeDelta);
+  tx.ancestorVsize = tx.adjustedVsize || 0;
+  tx.ancestorSigops = tx.sigops || 0;
+  tx.ancestorMap.forEach((ancestor) => {
+    tx.ancestorFee += (ancestor.fee + ancestor.feeDelta);
+    tx.ancestorVsize += ancestor.adjustedVsize;
+    tx.ancestorSigops += ancestor.sigops;
+  });
+  tx.score = tx.ancestorFee / tx.ancestorVsize;
+  tx.relativesSet = true;
+}
+
+// iterate over remaining descendants, removing the root as a valid ancestor & updating the ancestor score
+// avoids recursion to limit call stack depth
+function updateDescendants(
+  rootTx: MinerTransaction,
+  mempool: Map<string, MinerTransaction>,
+  modified: MinerTransaction[],
+  clusterRate: number,
+): MinerTransaction[] {
+  const descendantSet: Set<MinerTransaction> = new Set();
+  // stack of nodes left to visit
+  const descendants: MinerTransaction[] = [];
+  let descendantTx: MinerTransaction | undefined;
+  rootTx.children.forEach(childTx => {
+    if (!descendantSet.has(childTx)) {
+      descendants.push(childTx);
+      descendantSet.add(childTx);
+    }
+  });
+  while (descendants.length) {
+    descendantTx = descendants.pop();
+    if (descendantTx && descendantTx.ancestorMap && descendantTx.ancestorMap.has(rootTx.txid)) {
+      // remove tx as ancestor
+      descendantTx.ancestorMap.delete(rootTx.txid);
+      descendantTx.ancestorFee -= (rootTx.fee + rootTx.feeDelta);
+      descendantTx.ancestorVsize -= rootTx.adjustedVsize;
+      descendantTx.ancestorSigops -= rootTx.sigops;
+      descendantTx.score = descendantTx.ancestorFee / descendantTx.ancestorVsize;
+      descendantTx.dependencyRate = descendantTx.dependencyRate ? Math.min(descendantTx.dependencyRate, clusterRate) : clusterRate;
+
+      if (!descendantTx.modified) {
+        descendantTx.modified = true;
+        modified.push(descendantTx);
+      }
+
+      // add this node's children to the stack
+      descendantTx.children.forEach(childTx => {
+        // visit each node only once
+        if (!descendantSet.has(childTx)) {
+          descendants.push(childTx);
+          descendantSet.add(childTx);
+        }
+      });
+    }
+  }
+  // return new, resorted modified list
+  return modified.sort(priorityComparator);
+}
+
+// Used to sort an array of MinerTransactions by descending ancestor score
+function priorityComparator(a: MinerTransaction, b: MinerTransaction): number {
+  if (b.score === a.score) {
+    // tie-break by txid for stability
+    return a.order - b.order;
+  } else {
+    return b.score - a.score;
+  }
+}
+
+// returns the most significant 4 bytes of the txid as an integer
+function txidToOrdering(txid: string): number {
+  return parseInt(
+    txid.substring(62, 64) +
+      txid.substring(60, 62) +
+      txid.substring(58, 60) +
+      txid.substring(56, 58),
+    16
+  );
+}

backend/src/api/mining/mining-routes.ts

@@ -9,6 +9,8 @@ import bitcoinClient from '../bitcoin/bitcoin-client';
 import mining from "./mining";
 import PricesRepository from '../../repositories/PricesRepository';
 import AccelerationRepository from '../../repositories/AccelerationRepository';
+import accelerationApi from '../services/acceleration';
+import { handleError } from '../../utils/api';
 
 class MiningRoutes {
   public initRoutes(app: Application) {
@@ -24,6 +26,7 @@ class MiningRoutes {
       .get(config.MEMPOOL.API_URL_PREFIX + 'mining/difficulty-adjustments', this.$getDifficultyAdjustments)
       .get(config.MEMPOOL.API_URL_PREFIX + 'mining/reward-stats/:blockCount', this.$getRewardStats)
       .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/fees/:interval', this.$getHistoricalBlockFees)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/fees', this.$getBlockFeesTimespan)
       .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/rewards/:interval', this.$getHistoricalBlockRewards)
       .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/fee-rates/:interval', this.$getHistoricalBlockFeeRates)
       .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/sizes-weights/:interval', this.$getHistoricalBlockSizeAndWeight)
@@ -40,6 +43,8 @@ class MiningRoutes {
       .get(config.MEMPOOL.API_URL_PREFIX + 'accelerations/block/:height', this.$getAccelerationsByHeight)
       .get(config.MEMPOOL.API_URL_PREFIX + 'accelerations/recent/:interval', this.$getRecentAccelerations)
       .get(config.MEMPOOL.API_URL_PREFIX + 'accelerations/total', this.$getAccelerationTotals)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'accelerations', this.$getActiveAccelerations)
+      .post(config.MEMPOOL.API_URL_PREFIX + 'acceleration/request/:txid', this.$requestAcceleration)
     ;
   }
 
@@ -49,12 +54,12 @@ class MiningRoutes {
       res.header('Cache-control', 'public');
       res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
       if (['testnet', 'signet', 'liquidtestnet'].includes(config.MEMPOOL.NETWORK)) {
-        res.status(400).send('Prices are not available on testnets.');
+        handleError(req, res, 400, 'Prices are not available on testnets.');
         return;
       }
       const timestamp = parseInt(req.query.timestamp as string, 10) || 0;
       const currency = req.query.currency as string;
-      
+
       let response;
       if (timestamp && currency) {
         response = await PricesRepository.$getNearestHistoricalPrice(timestamp, currency);
@@ -67,7 +72,7 @@ class MiningRoutes {
       }
       res.status(200).send(response);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get historical prices');
     }
   }
 
@@ -80,9 +85,9 @@ class MiningRoutes {
       res.json(stats);
     } catch (e) {
       if (e instanceof Error && e.message.indexOf('This mining pool does not exist') > -1) {
-        res.status(404).send(e.message);
+        handleError(req, res, 404, e.message);
       } else {
-        res.status(500).send(e instanceof Error ? e.message : e);
+        handleError(req, res, 500, 'Failed to get pool');
       }
     }
   }
@@ -99,9 +104,9 @@ class MiningRoutes {
       res.json(poolBlocks);
     } catch (e) {
       if (e instanceof Error && e.message.indexOf('This mining pool does not exist') > -1) {
-        res.status(404).send(e.message);
+        handleError(req, res, 404, e.message);
       } else {
-        res.status(500).send(e instanceof Error ? e.message : e);
+        handleError(req, res, 500, 'Failed to get blocks for pool');
       }
     }
   }
@@ -125,7 +130,7 @@ class MiningRoutes {
         res.json(pools);
       }
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get pools');
     }
   }
 
@@ -139,7 +144,7 @@ class MiningRoutes {
       res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       res.json(stats);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get pools');
     }
   }
 
@@ -153,7 +158,7 @@ class MiningRoutes {
       res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
       res.json(hashrates);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get pools historical hashrate');
     }
   }
 
@@ -168,9 +173,9 @@ class MiningRoutes {
       res.json(hashrates);
     } catch (e) {
       if (e instanceof Error && e.message.indexOf('This mining pool does not exist') > -1) {
-        res.status(404).send(e.message);
+        handleError(req, res, 404, e.message);
       } else {
-        res.status(500).send(e instanceof Error ? e.message : e);
+        handleError(req, res, 500, 'Failed to get pool historical hashrate');
       }
     }
   }
@@ -178,7 +183,7 @@ class MiningRoutes {
   private async $getHistoricalHashrate(req: Request, res: Response) {
     let currentHashrate = 0, currentDifficulty = 0;
     try {
-      currentHashrate = await bitcoinClient.getNetworkHashPs();
+      currentHashrate = await bitcoinClient.getNetworkHashPs(1008);
       currentDifficulty = await bitcoinClient.getDifficulty();
     } catch (e) {
       logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate and difficulty');
@@ -199,7 +204,7 @@ class MiningRoutes {
         currentDifficulty: currentDifficulty,
       });
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get historical hashrate');
     }
   }
 
@@ -213,7 +218,25 @@ class MiningRoutes {
       res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       res.json(blockFees);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get historical block fees');
+    }
+  }
+
+  private async $getBlockFeesTimespan(req: Request, res: Response) {
+    try {
+      if (!parseInt(req.query.from as string, 10) || !parseInt(req.query.to as string, 10)) {
+        throw new Error('Invalid timestamp range');
+      }
+      if (parseInt(req.query.from as string, 10) > parseInt(req.query.to as string, 10)) {
+        throw new Error('from must be less than to');
+      }
+      const blockFees = await mining.$getBlockFeesTimespan(parseInt(req.query.from as string, 10), parseInt(req.query.to as string, 10));
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(blockFees);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get historical block fees');
     }
   }
 
@@ -227,7 +250,7 @@ class MiningRoutes {
       res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       res.json(blockRewards);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get historical block rewards');
     }
   }
 
@@ -241,7 +264,7 @@ class MiningRoutes {
       res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       res.json(blockFeeRates);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get historical block fee rates');
     }
   }
 
@@ -259,7 +282,7 @@ class MiningRoutes {
         weights: blockWeights
       });
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get historical block size and weight');
     }
   }
 
@@ -271,7 +294,7 @@ class MiningRoutes {
       res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
       res.json(difficulty.map(adj => [adj.time, adj.height, adj.difficulty, adj.adjustment]));
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get historical difficulty adjustments');
     }
   }
 
@@ -281,7 +304,7 @@ class MiningRoutes {
       res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       res.json(response);
     } catch (e) {
-      res.status(500).end();
+      handleError(req, res, 500, 'Failed to get reward stats');
     }
   }
 
@@ -295,7 +318,7 @@ class MiningRoutes {
       res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       res.json(blocksHealth.map(health => [health.time, health.height, health.match_rate]));
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get historical blocks health');
     }
   }
 
@@ -304,7 +327,7 @@ class MiningRoutes {
       const audit = await BlocksAuditsRepository.$getBlockAudit(req.params.hash);
 
       if (!audit) {
-        res.status(204).send(`This block has not been audited.`);
+        handleError(req, res, 204, `This block has not been audited.`);
         return;
       }
 
@@ -313,7 +336,7 @@ class MiningRoutes {
       res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24).toUTCString());
       res.json(audit);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get block audit');
     }
   }
 
@@ -336,7 +359,7 @@ class MiningRoutes {
       res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
       res.json(result);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get height from timestamp');
     }
   }
 
@@ -349,7 +372,7 @@ class MiningRoutes {
       res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       res.json(await BlocksAuditsRepository.$getBlockAuditScores(height, height - 15));
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get block audit scores');
     }
   }
 
@@ -362,7 +385,7 @@ class MiningRoutes {
       res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24).toUTCString());
       res.json(audit || 'null');
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get block audit score');
     }
   }
 
@@ -372,12 +395,12 @@ class MiningRoutes {
       res.header('Cache-control', 'public');
       res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid'].includes(config.MEMPOOL.NETWORK)) {
-        res.status(400).send('Acceleration data is not available.');
+        handleError(req, res, 400, 'Acceleration data is not available.');
         return;
       }
       res.status(200).send(await AccelerationRepository.$getAccelerationInfo(req.params.slug));
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get accelerations by pool');
     }
   }
 
@@ -387,13 +410,13 @@ class MiningRoutes {
       res.header('Cache-control', 'public');
       res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24).toUTCString());
       if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid'].includes(config.MEMPOOL.NETWORK)) {
-        res.status(400).send('Acceleration data is not available.');
+        handleError(req, res, 400, 'Acceleration data is not available.');
         return;
       }
       const height = req.params.height === undefined ? undefined : parseInt(req.params.height, 10);
       res.status(200).send(await AccelerationRepository.$getAccelerationInfo(null, height));
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get accelerations by height');
     }
   }
 
@@ -403,12 +426,12 @@ class MiningRoutes {
       res.header('Cache-control', 'public');
       res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid'].includes(config.MEMPOOL.NETWORK)) {
-        res.status(400).send('Acceleration data is not available.');
+        handleError(req, res, 400, 'Acceleration data is not available.');
         return;
       }
       res.status(200).send(await AccelerationRepository.$getAccelerationInfo(null, null, req.params.interval));
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get recent accelerations');
     }
   }
 
@@ -418,12 +441,39 @@ class MiningRoutes {
       res.header('Cache-control', 'public');
       res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid'].includes(config.MEMPOOL.NETWORK)) {
-        res.status(400).send('Acceleration data is not available.');
+        handleError(req, res, 400, 'Acceleration data is not available.');
         return;
       }
       res.status(200).send(await AccelerationRepository.$getAccelerationTotals(<string>req.query.pool, <string>req.query.interval));
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get acceleration totals');
+    }
+  }
+
+  private async $getActiveAccelerations(req: Request, res: Response): Promise<void> {
+    try {
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid'].includes(config.MEMPOOL.NETWORK)) {
+        handleError(req, res, 400, 'Acceleration data is not available.');
+        return;
+      }
+      res.status(200).send(Object.values(accelerationApi.getAccelerations() || {}));
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get active accelerations');
+    }
+  }
+
+  private async $requestAcceleration(req: Request, res: Response): Promise<void> {
+    res.setHeader('Pragma', 'no-cache');
+    res.setHeader('Cache-control', 'private, no-store, no-cache, must-revalidate, proxy-revalidate, max-age=0');
+    res.setHeader('expires', -1);
+    try {
+      accelerationApi.accelerationRequested(req.params.txid);
+      res.status(200).send();
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to request acceleration');
     }
   }
 }

backend/src/api/mining/mining.ts

@@ -45,11 +45,22 @@ class Mining {
    */
   public async $getHistoricalBlockFees(interval: string | null = null): Promise<any> {
     return await BlocksRepository.$getHistoricalBlockFees(
-      this.getTimeRange(interval, 5),
+      this.getTimeRange(interval),
       Common.getSqlInterval(interval)
     );
   }
 
+  /**
+   * Get timespan block total fees
+   */
+  public async $getBlockFeesTimespan(from: number, to: number): Promise<number> {
+    return await BlocksRepository.$getHistoricalBlockFees(
+      this.getTimeRangeFromTimespan(from, to),
+      null,
+      {from, to}
+    );
+  }
+  
   /**
    * Get historical block rewards
    */
@@ -125,9 +136,13 @@ class Mining {
     poolsStatistics['blockCount'] = blockCount;
 
     const totalBlock24h: number = await BlocksRepository.$blockCount(null, '24h');
+    const totalBlock3d: number = await BlocksRepository.$blockCount(null, '3d');
+    const totalBlock1w: number = await BlocksRepository.$blockCount(null, '1w');
 
     try {
       poolsStatistics['lastEstimatedHashrate'] = await bitcoinClient.getNetworkHashPs(totalBlock24h);
+      poolsStatistics['lastEstimatedHashrate3d'] = await bitcoinClient.getNetworkHashPs(totalBlock3d);
+      poolsStatistics['lastEstimatedHashrate1w'] = await bitcoinClient.getNetworkHashPs(totalBlock1w);
     } catch (e) {
       poolsStatistics['lastEstimatedHashrate'] = 0;
       logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate', logger.tags.mining);
@@ -646,6 +661,24 @@ class Mining {
     }
   }
 
+  private getTimeRangeFromTimespan(from: number, to: number, scale = 1): number {
+    const timespan = to - from;
+    switch (true) {
+      case timespan > 3600 * 24 * 365 * 4: return 86400 * scale; // 24h
+      case timespan > 3600 * 24 * 365 * 3: return 43200 * scale; // 12h
+      case timespan > 3600 * 24 * 365 * 2: return 43200 * scale; // 12h
+      case timespan > 3600 * 24 * 365: return 28800 * scale; // 8h
+      case timespan > 3600 * 24 * 30 * 6: return 28800 * scale; // 8h
+      case timespan > 3600 * 24 * 30 * 3: return 10800 * scale; // 3h
+      case timespan > 3600 * 24 * 30: return 7200 * scale; // 2h
+      case timespan > 3600 * 24 * 7: return 1800 * scale; // 30min
+      case timespan > 3600 * 24 * 3: return 300 * scale; // 5min
+      case timespan > 3600 * 24: return 1 * scale;
+      default: return 1 * scale;
+    }
+  }
+  
+
   // Finds the oldest block in a consecutive chain back from the tip
   // assumes `blocks` is sorted in ascending height order
   private getOldestConsecutiveBlock(blocks: DifficultyBlock[]): DifficultyBlock {

backend/src/api/pools-parser.ts

@@ -5,6 +5,9 @@ import PoolsRepository from '../repositories/PoolsRepository';
 import { PoolTag } from '../mempool.interfaces';
 import diskCache from './disk-cache';
 import mining from './mining/mining';
+import transactionUtils from './transaction-utils';
+import BlocksRepository from '../repositories/BlocksRepository';
+import redisCache from './redis-cache';
 
 class PoolsParser {
   miningPools: any[] = [];
@@ -16,15 +19,6 @@ class PoolsParser {
     'addresses': '[]',
     'slug': 'unknown'
   };
-  private uniqueLogs: string[] = [];
-
-  private uniqueLog(loggerFunction: any, msg: string): void {
-    if (this.uniqueLogs.includes(msg)) {
-      return;
-    }
-    this.uniqueLogs.push(msg);
-    loggerFunction(msg);
-  }
 
   public setMiningPools(pools): void {
     for (const pool of pools) {
@@ -37,28 +31,53 @@ class PoolsParser {
 
   /**
    * Populate our db with updated mining pool definition
-   * @param pools 
+   * @param pools
    */
   public async migratePoolsJson(): Promise<void> {
     // We also need to wipe the backend cache to make sure we don't serve blocks with
     // the wrong mining pool (usually happen with unknown blocks)
     diskCache.setIgnoreBlocksCache();
+    redisCache.setIgnoreBlocksCache();
 
     await this.$insertUnknownPool();
 
+    let reindexUnknown = false;
+
     for (const pool of this.miningPools) {
       if (!pool.id) {
         logger.info(`Mining pool ${pool.name} has no unique 'id' defined. Skipping.`);
         continue;
       }
 
+      // One of the two fields 'addresses' or 'regexes' must be a non-empty array
+      if (!pool.addresses && !pool.regexes) {
+        logger.err(`Mining pool ${pool.name} must have at least one of the fields 'addresses' or 'regexes'. Skipping.`);
+        continue;
+      }
+
+      pool.addresses = pool.addresses || [];
+      pool.regexes = pool.regexes || [];
+
+      if (pool.addresses.length === 0 && pool.regexes.length === 0) {
+        logger.err(`Mining pool ${pool.name} has no 'addresses' nor 'regexes' defined. Skipping.`);
+        continue;
+      }
+
+      if (pool.addresses.length === 0) {
+        logger.warn(`Mining pool ${pool.name} has no 'addresses' defined.`);
+      }
+
+      if (pool.regexes.length === 0) {
+        logger.warn(`Mining pool ${pool.name} has no 'regexes' defined.`);
+      }
+
       const poolDB = await PoolsRepository.$getPoolByUniqueId(pool.id, false);
       if (!poolDB) {
         // New mining pool
         const slug = pool.name.replace(/[^a-z0-9]/gi, '').toLowerCase();
         logger.debug(`Inserting new mining pool ${pool.name}`);
         await PoolsRepository.$insertNewMiningPool(pool, slug);
-        await this.$deleteUnknownBlocks();
+        reindexUnknown = true;
       } else {
         if (poolDB.name !== pool.name) {
           // Pool has been renamed
@@ -76,7 +95,45 @@ class PoolsParser {
           // Pool addresses changed or coinbase tags changed
           logger.notice(`Updating addresses and/or coinbase tags for ${pool.name} mining pool.`);
           await PoolsRepository.$updateMiningPoolTags(poolDB.id, pool.addresses, pool.regexes);
-          await this.$deleteBlocksForPool(poolDB);
+          reindexUnknown = true;
+          await this.$reindexBlocksForPool(poolDB.id);
+        }
+      }
+    }
+
+    if (reindexUnknown) {
+      logger.notice(`Updating addresses and/or coinbase tags for unknown mining pool.`);
+      let unknownPool;
+      if (config.DATABASE.ENABLED === true) {
+        unknownPool = await PoolsRepository.$getUnknownPool();
+      } else {
+        unknownPool = this.unknownPool;
+      }
+      await this.$reindexBlocksForPool(unknownPool.id);
+    }
+  }
+
+  public matchBlockMiner(scriptsig: string, addresses: string[], pools: PoolTag[]): PoolTag | undefined {
+    const asciiScriptSig = transactionUtils.hex2ascii(scriptsig);
+
+    for (let i = 0; i < pools.length; ++i) {
+      if (addresses.length) {
+        const poolAddresses: string[] = typeof pools[i].addresses === 'string' ?
+          JSON.parse(pools[i].addresses) : pools[i].addresses;
+        for (let y = 0; y < poolAddresses.length; y++) {
+          if (addresses.indexOf(poolAddresses[y]) !== -1) {
+            return pools[i];
+          }
+        }
+      }
+
+      const regexes: string[] = typeof pools[i].regexes === 'string' ?
+        JSON.parse(pools[i].regexes) : pools[i].regexes;
+      for (let y = 0; y < regexes.length; ++y) {
+        const regex = new RegExp(regexes[y], 'i');
+        const match = asciiScriptSig.match(regex);
+        if (match !== null) {
+          return pools[i];
         }
       }
     }
@@ -112,68 +169,47 @@ class PoolsParser {
   }
 
   /**
-   * Delete indexed blocks for an updated mining pool
-   * 
-   * @param pool 
+   * re-index pool assignment for blocks previously associated with pool
+   *
+   * @param pool local id of existing pool to reindex
    */
-  private async $deleteBlocksForPool(pool: PoolTag): Promise<void> {
-    // Get oldest blocks mined by the pool and assume pools-v2.json updates only concern most recent years
-    // Ignore early days of Bitcoin as there were no mining pool yet
-    const [oldestPoolBlock]: any[] = await DB.query(`
-      SELECT height
+  private async $reindexBlocksForPool(poolId: number): Promise<void> {
+    let firstKnownBlockPool = 130635; // https://mempool.space/block/0000000000000a067d94ff753eec72830f1205ad3a4c216a08a80c832e551a52
+    if (config.MEMPOOL.NETWORK === 'testnet') {
+      firstKnownBlockPool = 21106; // https://mempool.space/testnet/block/0000000070b701a5b6a1b965f6a38e0472e70b2bb31b973e4638dec400877581
+    } else if (config.MEMPOOL.NETWORK === 'signet') {
+      firstKnownBlockPool = 0;
+    }
+
+    const [blocks]: any[] = await DB.query(`
+      SELECT height, hash, coinbase_raw, coinbase_addresses
       FROM blocks
       WHERE pool_id = ?
-      ORDER BY height
-      LIMIT 1`,
-      [pool.id]
-    );
+      AND height >= ?
+      ORDER BY height DESC
+    `, [poolId, firstKnownBlockPool]);
 
-    let firstKnownBlockPool = 130635; // https://mempool.space/block/0000000000000a067d94ff753eec72830f1205ad3a4c216a08a80c832e551a52
-    if (config.MEMPOOL.NETWORK === 'testnet') {
-      firstKnownBlockPool = 21106; // https://mempool.space/testnet/block/0000000070b701a5b6a1b965f6a38e0472e70b2bb31b973e4638dec400877581
-    } else if (config.MEMPOOL.NETWORK === 'signet') {
-      firstKnownBlockPool = 0;
+    let pools: PoolTag[] = [];
+    if (config.DATABASE.ENABLED === true) {
+      pools = await PoolsRepository.$getPools();
+    } else {
+      pools = this.miningPools;
     }
 
-    const oldestBlockHeight = oldestPoolBlock.length ?? 0 > 0 ? oldestPoolBlock[0].height : firstKnownBlockPool;
-    const [unknownPool] = await DB.query(`SELECT id from pools where slug = "unknown"`);
-    this.uniqueLog(logger.notice, `Deleting blocks with unknown mining pool from height ${oldestBlockHeight} for re-indexing`);
-    await DB.query(`
-      DELETE FROM blocks
-      WHERE pool_id = ? AND height >= ${oldestBlockHeight}`,
-      [unknownPool[0].id]
-    );
-    logger.notice(`Deleting blocks from ${pool.name} mining pool for re-indexing`);
-    await DB.query(`
-      DELETE FROM blocks
-      WHERE pool_id = ?`,
-      [pool.id]
-    );
+    let changed = 0;
+    for (const block of blocks) {
+      const addresses = JSON.parse(block.coinbase_addresses) || [];
+      const newPool = this.matchBlockMiner(block.coinbase_raw, addresses, pools);
+      if (newPool && newPool.id !== poolId) {
+        changed++;
+        await BlocksRepository.$savePool(block.hash, newPool.id);
+      }
+    }
+
+    logger.info(`${changed} blocks assigned to a new pool`, logger.tags.mining);
 
     // Re-index hashrates and difficulty adjustments later
     mining.reindexHashrateRequested = true;
-    mining.reindexDifficultyAdjustmentRequested = true;
-  }
-
-  private async $deleteUnknownBlocks(): Promise<void> {
-    let firstKnownBlockPool = 130635; // https://mempool.space/block/0000000000000a067d94ff753eec72830f1205ad3a4c216a08a80c832e551a52
-    if (config.MEMPOOL.NETWORK === 'testnet') {
-      firstKnownBlockPool = 21106; // https://mempool.space/testnet/block/0000000070b701a5b6a1b965f6a38e0472e70b2bb31b973e4638dec400877581
-    } else if (config.MEMPOOL.NETWORK === 'signet') {
-      firstKnownBlockPool = 0;
-    }
-
-    const [unknownPool] = await DB.query(`SELECT id from pools where slug = "unknown"`);
-    this.uniqueLog(logger.notice, `Deleting blocks with unknown mining pool from height ${firstKnownBlockPool} for re-indexing`);
-    await DB.query(`
-      DELETE FROM blocks
-      WHERE pool_id = ? AND height >= ${firstKnownBlockPool}`,
-      [unknownPool[0].id]
-    );
-
-    // Re-index hashrates and difficulty adjustments later
-    mining.reindexHashrateRequested = true;
-    mining.reindexDifficultyAdjustmentRequested = true;
   }
 }
 

backend/src/api/prices/prices.routes.ts

@@ -1,10 +1,15 @@
 import { Application, Request, Response } from 'express';
 import config from '../../config';
 import pricesUpdater from '../../tasks/price-updater';
+import logger from '../../logger';
+import PricesRepository from '../../repositories/PricesRepository';
 
 class PricesRoutes {
   public initRoutes(app: Application): void {
-    app.get(config.MEMPOOL.API_URL_PREFIX + 'prices', this.$getCurrentPrices.bind(this));
+    app
+      .get(config.MEMPOOL.API_URL_PREFIX + 'prices', this.$getCurrentPrices.bind(this))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'internal/usd-price-history', this.$getAllPrices.bind(this))
+    ;
   }
 
   private $getCurrentPrices(req: Request, res: Response): void {
@@ -14,6 +19,23 @@ class PricesRoutes {
 
     res.json(pricesUpdater.getLatestPrices());
   }
+
+  private async $getAllPrices(req: Request, res: Response): Promise<void> {
+    res.header('Pragma', 'public');
+    res.header('Cache-control', 'public');
+    res.setHeader('Expires', new Date(Date.now() + 360_0000 / config.MEMPOOL.PRICE_UPDATES_PER_HOUR).toUTCString());
+
+    try {
+      const usdPriceHistory = await PricesRepository.$getPricesTimesAndId();
+      const responseData = usdPriceHistory.map(p => {
+        return { time: p.time, USD: p.USD };
+      });
+      res.status(200).json(responseData);
+    } catch (e: any) {
+      logger.err(`Exception ${e} in PricesRoutes::$getAllPrices. Code: ${e.code}. Message: ${e.message}`);
+      res.status(403).send();
+    }
+  }
 }
 
 export default new PricesRoutes();

backend/src/api/rbf-cache.ts

@@ -44,6 +44,22 @@ interface CacheEvent {
   value?: any,
 }
 
+/**
+ * Singleton for tracking RBF trees
+ *
+ * Maintains a set of RBF trees, where each tree represents a sequence of
+ * consecutive RBF replacements.
+ *
+ * Trees are identified by the txid of the root transaction.
+ *
+ * To maintain consistency, the following invariants must be upheld:
+ *  - Symmetry: replacedBy(A) = B <=> A in replaces(B)
+ *  - Unique id: treeMap(treeMap(X)) = treeMap(X)
+ *  - Unique tree: A in replaces(B) => treeMap(A) == treeMap(B)
+ *  - Existence: X in treeMap => treeMap(X) in rbfTrees
+ *  - Completeness: X in replacedBy => X in treeMap, Y in replaces => Y in treeMap
+ */
+
 class RbfCache {
   private replacedBy: Map<string, string> = new Map();
   private replaces: Map<string, string[]> = new Map();
@@ -61,6 +77,10 @@ class RbfCache {
     setInterval(this.cleanup.bind(this), 1000 * 60 * 10);
   }
 
+  /**
+   * Low level cache operations
+   */
+
   private addTx(txid: string, tx: MempoolTransactionExtended): void {
     this.txs.set(txid, tx);
     this.cacheQueue.push({ op: CacheOp.Add, type: 'tx', txid });
@@ -92,8 +112,18 @@ class RbfCache {
     this.cacheQueue.push({ op: CacheOp.Remove, type: 'exp', txid });
   }
 
+  /**
+   * Basic data structure operations
+   * must uphold tree invariants
+   */
+
+
   public add(replaced: MempoolTransactionExtended[], newTxExtended: MempoolTransactionExtended): void {
-    if (!newTxExtended || !replaced?.length || this.txs.has(newTxExtended.txid)) {
+    if ( !newTxExtended
+      || !replaced?.length
+      || this.txs.has(newTxExtended.txid)
+      || !(replaced.some(tx => !this.replacedBy.has(tx.txid)))
+    ) {
       return;
     }
 
@@ -114,6 +144,10 @@ class RbfCache {
       if (!replacedTx.rbf) {
         txFullRbf = true;
       }
+      if (this.replacedBy.has(replacedTx.txid)) {
+        // should never happen
+        continue;
+      }
       this.replacedBy.set(replacedTx.txid, newTx.txid);
       if (this.treeMap.has(replacedTx.txid)) {
         const treeId = this.treeMap.get(replacedTx.txid);
@@ -140,18 +174,47 @@ class RbfCache {
       }
     }
     newTx.fullRbf = txFullRbf;
-    const treeId = replacedTrees[0].tx.txid;
     const newTree = {
       tx: newTx,
       time: newTime,
       fullRbf: treeFullRbf,
       replaces: replacedTrees
     };
-    this.addTree(treeId, newTree);
-    this.updateTreeMap(treeId, newTree);
+    this.addTree(newTree.tx.txid, newTree);
+    this.updateTreeMap(newTree.tx.txid, newTree);
     this.replaces.set(newTx.txid, replacedTrees.map(tree => tree.tx.txid));
   }
 
+  public mined(txid): void {
+    if (!this.txs.has(txid)) {
+      return;
+    }
+    const treeId = this.treeMap.get(txid);
+    if (treeId && this.rbfTrees.has(treeId)) {
+      const tree = this.rbfTrees.get(treeId);
+      if (tree) {
+        this.setTreeMined(tree, txid);
+        tree.mined = true;
+        this.dirtyTrees.add(treeId);
+        this.cacheQueue.push({ op: CacheOp.Change, type: 'tree', txid: treeId });
+      }
+    }
+    this.evict(txid);
+  }
+
+  // flag a transaction as removed from the mempool
+  public evict(txid: string, fast: boolean = false): void {
+    this.evictionCount++;
+    if (this.txs.has(txid) && (fast || !this.expiring.has(txid))) {
+      const expiryTime = fast ? Date.now() + (1000 * 60 * 10) : Date.now() + (1000 * 86400); // 24 hours
+      this.addExpiration(txid, expiryTime);
+    }
+  }
+
+  /**
+   * Read-only public interface
+   */
+
   public has(txId: string): boolean {
     return this.txs.has(txId);
   }
@@ -232,32 +295,6 @@ class RbfCache {
     return changes;
   }
 
-  public mined(txid): void {
-    if (!this.txs.has(txid)) {
-      return;
-    }
-    const treeId = this.treeMap.get(txid);
-    if (treeId && this.rbfTrees.has(treeId)) {
-      const tree = this.rbfTrees.get(treeId);
-      if (tree) {
-        this.setTreeMined(tree, txid);
-        tree.mined = true;
-        this.dirtyTrees.add(treeId);
-        this.cacheQueue.push({ op: CacheOp.Change, type: 'tree', txid: treeId });
-      }
-    }
-    this.evict(txid);
-  }
-
-  // flag a transaction as removed from the mempool
-  public evict(txid: string, fast: boolean = false): void {
-    this.evictionCount++;
-    if (this.txs.has(txid) && (fast || !this.expiring.has(txid))) {
-      const expiryTime = fast ? Date.now() + (1000 * 60 * 10) : Date.now() + (1000 * 86400); // 24 hours
-      this.addExpiration(txid, expiryTime);
-    }
-  }
-
   // is the transaction involved in a full rbf replacement?
   public isFullRbf(txid: string): boolean {
     const treeId = this.treeMap.get(txid);
@@ -271,6 +308,10 @@ class RbfCache {
     return tree?.fullRbf;
   }
 
+  /**
+   * Cache maintenance & utility functions
+   */
+
   private cleanup(): void {
     const now = Date.now();
     for (const txid of this.expiring.keys()) {
@@ -299,10 +340,6 @@ class RbfCache {
       for (const tx of (replaces || [])) {
         // recursively remove prior versions from the cache
         this.replacedBy.delete(tx);
-        // if this is the id of a tree, remove that too
-        if (this.treeMap.get(tx) === tx) {
-          this.removeTree(tx);
-        }
         this.remove(tx);
       }
     }
@@ -370,14 +407,21 @@ class RbfCache {
     };
   }
 
-  public async load({ txs, trees, expiring, mempool }): Promise<void> {
+  public async load({ txs, trees, expiring, mempool, spendMap }): Promise<void> {
     try {
       txs.forEach(txEntry => {
         this.txs.set(txEntry.value.txid, txEntry.value);
       });
       this.staleCount = 0;
-      for (const deflatedTree of trees) {
-        await this.importTree(mempool, deflatedTree.root, deflatedTree.root, deflatedTree, this.txs);
+      for (const deflatedTree of trees.sort((a, b) => Object.keys(b).length - Object.keys(a).length)) {
+        const tree = await this.importTree(mempool, deflatedTree.root, deflatedTree.root, deflatedTree, this.txs);
+        if (tree) {
+          this.addTree(tree.tx.txid, tree);
+          this.updateTreeMap(tree.tx.txid, tree);
+          if (tree.mined) {
+            this.evict(tree.tx.txid);
+          }
+        }
       }
       expiring.forEach(expiringEntry => {
         if (this.txs.has(expiringEntry.key)) {
@@ -385,6 +429,31 @@ class RbfCache {
         }
       });
       this.staleCount = 0;
+
+      // connect cached trees to current mempool transactions
+      const conflicts: Record<string, { replacedBy: MempoolTransactionExtended, replaces: Set<MempoolTransactionExtended> }> = {};
+      for (const tree of this.rbfTrees.values()) {
+        const tx = this.getTx(tree.tx.txid);
+        if (!tx || tree.mined) {
+          continue;
+        }
+        for (const vin of tx.vin) {
+          const conflict = spendMap.get(`${vin.txid}:${vin.vout}`);
+          if (conflict && conflict.txid !== tx.txid) {
+            if (!conflicts[conflict.txid]) {
+              conflicts[conflict.txid] = {
+                replacedBy: conflict,
+                replaces: new Set(),
+              };
+            }
+            conflicts[conflict.txid].replaces.add(tx);
+          }
+        }
+      }
+      for (const { replacedBy, replaces } of Object.values(conflicts)) {
+        this.add([...replaces.values()], replacedBy);
+      }
+
       await this.checkTrees();
       logger.debug(`loaded ${txs.length} txs, ${trees.length} trees into rbf cache, ${expiring.length} due to expire, ${this.staleCount} were stale`);
       this.cleanup();
@@ -426,6 +495,12 @@ class RbfCache {
       return;
     }
 
+    // if this tx is already in the cache, return early
+    if (this.treeMap.has(txid)) {
+      this.removeTree(deflated.key);
+      return;
+    }
+
     // recursively reconstruct child trees
     for (const childId of treeInfo.replaces) {
       const replaced = await this.importTree(mempool, root, childId, deflated, txs, mined);
@@ -457,10 +532,6 @@ class RbfCache {
       fullRbf: treeInfo.fullRbf,
       replaces,
     };
-    this.treeMap.set(txid, root);
-    if (root === txid) {
-      this.addTree(root, tree);
-    }
     return tree;
   }
 
@@ -511,6 +582,7 @@ class RbfCache {
       processTxs(txs);
     }
 
+    // evict missing transactions
     for (const txid of txids) {
       if (!found[txid]) {
         this.evict(txid, false);

backend/src/api/redis-cache.ts

@@ -27,6 +27,7 @@ class RedisCache {
   private rbfCacheQueue: { type: string, txid: string, value: any }[] = [];
   private rbfRemoveQueue: { type: string, txid: string }[] = [];
   private txFlushLimit: number = 10000;
+  private ignoreBlocksCache = false;
 
   constructor() {
     if (config.REDIS.ENABLED) {
@@ -155,7 +156,7 @@ class RedisCache {
     const toAdd = this.cacheQueue.slice(0, this.txFlushLimit);
     try {
       const msetData = toAdd.map(tx => {
-        const minified: any = { ...tx };
+        const minified: any = structuredClone(tx);
         delete minified.hex;
         for (const vin of minified.vin) {
           delete vin.inner_redeemscript_asm;
@@ -341,9 +342,7 @@ class RedisCache {
       return;
     }
     logger.info('Restoring mempool and blocks data from Redis cache');
-    // Load block data
-    const loadedBlocks = await this.$getBlocks();
-    const loadedBlockSummaries = await this.$getBlockSummaries();
+
     // Load mempool
     const loadedMempool = await this.$getMempool();
     this.inflateLoadedTxs(loadedMempool);
@@ -352,15 +351,21 @@ class RedisCache {
     const rbfTrees = await this.$getRbfEntries('tree');
     const rbfExpirations = await this.$getRbfEntries('exp');
 
-    // Set loaded data
-    blocks.setBlocks(loadedBlocks || []);
-    blocks.setBlockSummaries(loadedBlockSummaries || []);
+    // Load & set block data
+    if (!this.ignoreBlocksCache) {
+      const loadedBlocks = await this.$getBlocks();
+      const loadedBlockSummaries = await this.$getBlockSummaries();
+      blocks.setBlocks(loadedBlocks || []);
+      blocks.setBlockSummaries(loadedBlockSummaries || []);
+    }
+    // Set other data
     await memPool.$setMempool(loadedMempool);
     await rbfCache.load({
       txs: rbfTxs,
       trees: rbfTrees.map(loadedTree => { loadedTree.value.key = loadedTree.key; return loadedTree.value; }),
       expiring: rbfExpirations,
       mempool: memPool.getMempool(),
+      spendMap: memPool.getSpendMap(),
     });
   }
 
@@ -411,6 +416,10 @@ class RedisCache {
     }
     return result;
   }
+
+  public setIgnoreBlocksCache(): void {
+    this.ignoreBlocksCache = true;
+  }
 }
 
 export default new RedisCache();

backend/src/api/services/acceleration.ts

@@ -1,7 +1,12 @@
+import { WebSocket } from 'ws';
 import config from '../../config';
 import logger from '../../logger';
-import { BlockExtended, PoolTag } from '../../mempool.interfaces';
+import { BlockExtended } from '../../mempool.interfaces';
 import axios from 'axios';
+import mempool from '../mempool';
+import websocketHandler from '../websocket-handler';
+
+type MyAccelerationStatus = 'requested' | 'accelerating' | 'done';
 
 export interface Acceleration {
   txid: string,
@@ -10,6 +15,12 @@ export interface Acceleration {
   effectiveFee: number,
   feeDelta: number,
   pools: number[],
+  positions?: {
+    [pool: number]: {
+      block: number,
+      vbytes: number,
+    },
+  },
 };
 
 export interface AccelerationHistory {
@@ -25,25 +36,115 @@ export interface AccelerationHistory {
   feeDelta: number,
   blockHash: string,
   blockHeight: number,
-  pools: {
-    pool_unique_id: number,
-    username: string,
-  }[],
+  pools: number[];
 };
 
 class AccelerationApi {
-  public async $fetchAccelerations(): Promise<Acceleration[] | null> {
-    if (config.MEMPOOL_SERVICES.ACCELERATIONS) {
-      try {
-        const response = await axios.get(`${config.MEMPOOL_SERVICES.API}/accelerator/accelerations`, { responseType: 'json', timeout: 10000 });
-        return response.data as Acceleration[];
-      } catch (e) {
-        logger.warn('Failed to fetch current accelerations from the mempool services backend: ' + (e instanceof Error ? e.message : e));
-        return null;
+  private ws: WebSocket | null = null;
+  private useWebsocket: boolean = config.MEMPOOL.OFFICIAL && config.MEMPOOL_SERVICES.ACCELERATIONS;
+  private startedWebsocketLoop: boolean = false;
+  private websocketConnected: boolean = false;
+  private onDemandPollingEnabled = !config.MEMPOOL_SERVICES.ACCELERATIONS;
+  private apiPath = config.MEMPOOL.OFFICIAL ? (config.MEMPOOL_SERVICES.API + '/accelerator/accelerations') : (config.EXTERNAL_DATA_SERVER.MEMPOOL_API + '/accelerations');
+  private websocketPath = config.MEMPOOL_SERVICES?.API ? `${config.MEMPOOL_SERVICES.API.replace('https://', 'wss://').replace('http://', 'ws://')}/accelerator/ws` : '/';
+  private _accelerations: Record<string, Acceleration> = {};
+  private lastPoll = 0;
+  private lastPing = Date.now();
+  private lastPong = Date.now();
+  private forcePoll = false;
+  private myAccelerations: Record<string, { status: MyAccelerationStatus, added: number, acceleration?: Acceleration }> = {};
+
+  public constructor() {}
+
+  public getAccelerations(): Record<string, Acceleration> {
+    return this._accelerations;
+  }
+
+  public countMyAccelerationsWithStatus(filter: MyAccelerationStatus): number {
+    return Object.values(this.myAccelerations).reduce((count, {status}) => { return count + (status === filter ? 1 : 0); }, 0);
+  }
+
+  public accelerationRequested(txid: string): void {
+    if (this.onDemandPollingEnabled) {
+      this.myAccelerations[txid] = { status: 'requested', added: Date.now() };
+    }
+  }
+
+  public accelerationConfirmed(): void {
+    this.forcePoll = true;
+  }
+
+  private async $fetchAccelerations(): Promise<Acceleration[] | null> {
+    try {
+      const response = await axios.get(this.apiPath, { responseType: 'json', timeout: 10000 });
+      return response?.data || [];
+    } catch (e) {
+      logger.warn('Failed to fetch current accelerations from the mempool services backend: ' + (e instanceof Error ? e.message : e));
+      return null;
+    }
+  }
+
+  public async $updateAccelerations(): Promise<Record<string, Acceleration> | null> {
+    if (this.useWebsocket && this.websocketConnected) {
+      return this._accelerations;
+    }
+    if (!this.onDemandPollingEnabled) {
+      const accelerations = await this.$fetchAccelerations();
+      if (accelerations) {
+        const latestAccelerations = {};
+        for (const acc of accelerations) {
+          latestAccelerations[acc.txid] = acc;
+        }
+        this._accelerations = latestAccelerations;
+        return this._accelerations;
       }
     } else {
-      return [];
+      return this.$updateAccelerationsOnDemand();
     }
+    return null;
+  }
+
+  private async $updateAccelerationsOnDemand(): Promise<Record<string, Acceleration> | null> {
+    const shouldUpdate = this.forcePoll
+      || this.countMyAccelerationsWithStatus('requested') > 0
+      || (this.countMyAccelerationsWithStatus('accelerating') > 0 && this.lastPoll < (Date.now() - (10 * 60 * 1000)));
+
+    // update accelerations if necessary
+    if (shouldUpdate) {
+      const accelerations = await this.$fetchAccelerations();
+      this.lastPoll = Date.now();
+      this.forcePoll = false;
+      if (accelerations) {
+        const latestAccelerations: Record<string, Acceleration> = {};
+        // set relevant accelerations to 'accelerating'
+        for (const acc of accelerations) {
+          if (this.myAccelerations[acc.txid]) {
+            latestAccelerations[acc.txid] = acc;
+            this.myAccelerations[acc.txid] = { status: 'accelerating', added: Date.now(), acceleration: acc };
+          }
+        }
+        // txs that are no longer accelerating are either confirmed or canceled, so mark for expiry
+        for (const [txid, { status, acceleration }] of Object.entries(this.myAccelerations)) {
+          if (status === 'accelerating' && !latestAccelerations[txid]) {
+            this.myAccelerations[txid] = { status: 'done', added: Date.now(), acceleration };
+          }
+        }
+      }
+    }
+
+    // clear expired accelerations (confirmed / failed / not accepted) after 10 minutes
+    for (const [txid, { status, added }] of Object.entries(this.myAccelerations)) {
+      if (['requested', 'done'].includes(status) && added < (Date.now() - (1000 * 60 * 10))) {
+        delete this.myAccelerations[txid];
+      }
+    }
+
+    const latestAccelerations = {};
+    for (const acc of Object.values(this.myAccelerations).map(({ acceleration }) => acceleration).filter(acc => acc) as Acceleration[]) {
+      latestAccelerations[acc.txid] = acc;
+    }
+    this._accelerations = latestAccelerations;
+    return this._accelerations;
   }
 
   public async $fetchAccelerationHistory(page?: number, status?: string): Promise<AccelerationHistory[] | null> {
@@ -74,6 +175,148 @@ class AccelerationApi {
     }
     return anyAccelerated;
   }
+
+  // get a list of accelerations that have changed between two sets of accelerations
+  public getAccelerationDelta(oldAccelerationMap: Record<string, Acceleration>, newAccelerationMap: Record<string, Acceleration>): string[] {
+    const changed: string[] = [];
+    const mempoolCache = mempool.getMempool();
+
+    for (const acceleration of Object.values(newAccelerationMap)) {
+      // skip transactions we don't know about
+      if (!mempoolCache[acceleration.txid]) {
+        continue;
+      }
+      if (oldAccelerationMap[acceleration.txid] == null) {
+        // new acceleration
+        changed.push(acceleration.txid);
+      } else {
+        if (oldAccelerationMap[acceleration.txid].feeDelta !== acceleration.feeDelta) {
+          // feeDelta changed
+          changed.push(acceleration.txid);
+        } else if (oldAccelerationMap[acceleration.txid].pools?.length) {
+          let poolsChanged = false;
+          const pools = new Set();
+            oldAccelerationMap[acceleration.txid].pools.forEach(pool => {
+            pools.add(pool);
+          });
+          acceleration.pools.forEach(pool => {
+            if (!pools.has(pool)) {
+              poolsChanged = true;
+            } else {
+              pools.delete(pool);
+            }
+          });
+          if (pools.size > 0) {
+            poolsChanged = true;
+          }
+          if (poolsChanged) {
+            // pools changed
+            changed.push(acceleration.txid);
+          }
+        }
+      }
+    }
+
+    for (const oldTxid of Object.keys(oldAccelerationMap)) {
+      if (!newAccelerationMap[oldTxid]) {
+        // removed
+        changed.push(oldTxid);
+      }
+    }
+
+    return changed;
+  }
+
+  private handleWebsocketMessage(msg: any): void {
+    if (msg?.accelerations !== null) {
+      const latestAccelerations = {};
+      for (const acc of msg?.accelerations || []) {
+        latestAccelerations[acc.txid] = acc;
+      }
+      this._accelerations = latestAccelerations;
+      websocketHandler.handleAccelerationsChanged(this._accelerations);
+    }
+  }
+
+  public async connectWebsocket(): Promise<void> {
+    if (this.startedWebsocketLoop) {
+      return;
+    }
+    while (this.useWebsocket) {
+      this.startedWebsocketLoop = true;
+      if (!this.ws) {
+        this.ws = new WebSocket(this.websocketPath);
+        this.lastPing = 0;
+
+        this.ws.on('open', () => {
+          logger.info(`Acceleration websocket opened to ${this.websocketPath}`);
+          this.websocketConnected = true;
+          this.ws?.send(JSON.stringify({
+            'watch-accelerations': true
+          }));
+        });
+
+        this.ws.on('error', (error) => {
+          let errMsg = `Acceleration websocket error on ${this.websocketPath}: ${error['code']}`;
+          if (error['errors']) {
+            errMsg += ' - ' + error['errors'].join(' - ');
+          }
+          logger.err(errMsg);
+          this.ws = null;
+          this.websocketConnected = false;
+        });
+
+        this.ws.on('close', () => {
+          logger.info('Acceleration websocket closed');
+          this.ws = null;
+          this.websocketConnected = false;
+        });
+
+        this.ws.on('message', (data, isBinary) => {
+          try {
+            const msg = (isBinary ? data : data.toString()) as string;
+            const parsedMsg = msg?.length ? JSON.parse(msg) : null;
+            this.handleWebsocketMessage(parsedMsg);
+          } catch (e) {
+            logger.warn('Failed to parse acceleration websocket message: ' + (e instanceof Error ? e.message : e));
+          }
+        });
+
+        this.ws.on('ping', () => {
+          logger.debug('received ping from acceleration websocket server');
+        });
+
+        this.ws.on('pong', () => {
+          logger.debug('received pong from acceleration websocket server');
+          this.lastPong = Date.now();
+        });
+      } else if (this.websocketConnected) {
+        if (this.lastPing && this.lastPing > this.lastPong && (Date.now() - this.lastPing > 10000)) {
+          logger.warn('No pong received within 10 seconds, terminating connection');
+          try {
+            this.ws?.terminate();
+          } catch (e) {
+            logger.warn('failed to terminate acceleration websocket connection: ' + (e instanceof Error ? e.message : e));
+          } finally {
+            this.ws = null;
+            this.websocketConnected = false;
+            this.lastPing = 0;
+          }
+        } else if (!this.lastPing || (Date.now() - this.lastPing > 30000)) {
+          logger.debug('sending ping to acceleration websocket server');
+          if (this.ws?.readyState === WebSocket.OPEN) {
+            try {
+              this.ws?.ping();
+              this.lastPing = Date.now();
+            } catch (e) {
+              logger.warn('failed to send ping to acceleration websocket server: ' + (e instanceof Error ? e.message : e));
+            }
+          }
+        }
+      }
+      await new Promise(resolve => setTimeout(resolve, 5000));
+    }
+  }
 }
 
 export default new AccelerationApi();

backend/src/api/services/services-routes.ts

@@ -0,0 +1,27 @@
+import { Application, Request, Response } from 'express';
+import config from '../../config';
+import WalletApi from './wallets';
+import { handleError } from '../../utils/api';
+
+class ServicesRoutes {
+  public initRoutes(app: Application): void {
+    app
+      .get(config.MEMPOOL.API_URL_PREFIX + 'wallet/:walletId', this.$getWallet)
+    ;
+  }
+
+  private async $getWallet(req: Request, res: Response): Promise<void> {
+    try {
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 5).toUTCString());
+      const walletId = req.params.walletId;
+      const wallet = await WalletApi.getWallet(walletId);
+      res.status(200).send(wallet);
+    } catch (e) {
+      handleError(req, res, 500, 'Failed to get wallet');
+    }
+  }
+}
+
+export default new ServicesRoutes();

backend/src/api/services/stratum.ts

@@ -0,0 +1,105 @@
+import { WebSocket } from 'ws';
+import logger from '../../logger';
+import config from '../../config';
+import websocketHandler from '../websocket-handler';
+
+export interface StratumJob {
+  pool: number;
+  height: number;
+  coinbase: string;
+  scriptsig: string;
+  reward: number;
+  jobId: string;
+  extraNonce: string;
+  extraNonce2Size: number;
+  prevHash: string;
+  coinbase1: string;
+  coinbase2: string;
+  merkleBranches: string[];
+  version: string;
+  bits: string;
+  time: string;
+  timestamp: number;
+  cleanJobs: boolean;
+  received: number;
+}
+
+function isStratumJob(obj: any): obj is StratumJob {
+  return obj
+    && typeof obj === 'object'
+    && 'pool' in obj
+    && 'prevHash' in obj
+    && 'height' in obj
+    && 'received' in obj
+    && 'version' in obj
+    && 'timestamp' in obj
+    && 'bits' in obj
+    && 'merkleBranches' in obj
+    && 'cleanJobs' in obj;
+}
+
+class StratumApi {
+  private ws: WebSocket | null = null;
+  private runWebsocketLoop: boolean = false;
+  private startedWebsocketLoop: boolean = false;
+  private websocketConnected: boolean = false;
+  private jobs: Record<string, StratumJob> = {};
+
+  public constructor() {}
+
+  public getJobs(): Record<string, StratumJob> {
+    return this.jobs;
+  }
+
+  private handleWebsocketMessage(msg: any): void {
+    if (isStratumJob(msg)) {
+      this.jobs[msg.pool] = msg;
+      websocketHandler.handleNewStratumJob(this.jobs[msg.pool]);
+    }
+  }
+
+  public async connectWebsocket(): Promise<void> {
+    if (!config.STRATUM.ENABLED) {
+      return;
+    }
+    this.runWebsocketLoop = true;
+    if (this.startedWebsocketLoop) {
+      return;
+    }
+    while (this.runWebsocketLoop) {
+      this.startedWebsocketLoop = true;
+      if (!this.ws) {
+        this.ws = new WebSocket(`${config.STRATUM.API}`);
+        this.websocketConnected = true;
+
+        this.ws.on('open', () => {
+          logger.info('Stratum websocket opened');
+        });
+
+        this.ws.on('error', (error) => {
+          logger.err('Stratum websocket error: ' + error);
+          this.ws = null;
+          this.websocketConnected = false;
+        });
+
+        this.ws.on('close', () => {
+          logger.info('Stratum websocket closed');
+          this.ws = null;
+          this.websocketConnected = false;
+        });
+
+        this.ws.on('message', (data, isBinary) => {
+          try {
+            const parsedMsg = JSON.parse((isBinary ? data : data.toString()) as string);
+            this.handleWebsocketMessage(parsedMsg);
+          } catch (e) {
+            logger.warn('Failed to parse stratum websocket message: ' + (e instanceof Error ? e.message : e));
+          }
+        });
+      }
+      await new Promise(resolve => setTimeout(resolve, 5000));
+    }
+  }
+}
+
+export default new StratumApi();

backend/src/api/services/wallets.ts

@@ -0,0 +1,153 @@
+import config from '../../config';
+import logger from '../../logger';
+import { IEsploraApi } from '../bitcoin/esplora-api.interface';
+import bitcoinApi from '../bitcoin/bitcoin-api-factory';
+import axios from 'axios';
+import { TransactionExtended } from '../../mempool.interfaces';
+
+interface WalletAddress {
+  address: string;
+  active: boolean;
+  stats: {
+    funded_txo_count: number;
+    funded_txo_sum: number;
+    spent_txo_count: number;
+    spent_txo_sum: number;
+    tx_count: number;
+  };
+  transactions: IEsploraApi.AddressTxSummary[];
+  lastSync: number;
+}
+
+interface Wallet {
+  name: string;
+  addresses: Record<string, WalletAddress>;
+  lastPoll: number;
+}
+
+const POLL_FREQUENCY = 5 * 60 * 1000; // 5 minutes
+
+class WalletApi {
+  private wallets: Record<string, Wallet> = {};
+  private syncing = false;
+
+  constructor() {
+    this.wallets = config.WALLETS.ENABLED ? (config.WALLETS.WALLETS as string[]).reduce((acc, wallet) => {
+      acc[wallet] = { name: wallet, addresses: {}, lastPoll: 0 };
+      return acc;
+    }, {} as Record<string, Wallet>) : {};
+  }
+
+  public getWallet(wallet: string): Record<string, WalletAddress> {
+    return this.wallets?.[wallet]?.addresses || {};
+  }
+
+  // resync wallet addresses from the services backend
+  async $syncWallets(): Promise<void> {
+    if (!config.WALLETS.ENABLED || this.syncing) {
+      return;
+    }
+    this.syncing = true;
+    for (const walletKey of Object.keys(this.wallets)) {
+      const wallet = this.wallets[walletKey];
+      if (wallet.lastPoll < (Date.now() - POLL_FREQUENCY)) {
+        try {
+          const response = await axios.get(config.MEMPOOL_SERVICES.API + `/wallets/${wallet.name}`);
+          const addresses: Record<string, WalletAddress> = response.data;
+          const addressList: WalletAddress[] = Object.values(addresses);
+          // sync all current addresses
+          for (const address of addressList) {
+            await this.$syncWalletAddress(wallet, address);
+          }
+          // remove old addresses
+          for (const address of Object.keys(wallet.addresses)) {
+            if (!addresses[address]) {
+              delete wallet.addresses[address];
+            }
+          }
+          wallet.lastPoll = Date.now();
+          logger.debug(`Synced ${Object.keys(wallet.addresses).length} addresses for wallet ${wallet.name}`);
+        } catch (e) {
+          logger.err(`Error syncing wallet ${wallet.name}: ${(e instanceof Error ? e.message : e)}`);
+        }
+      }
+    }
+    this.syncing = false;
+  }
+
+  // resync address transactions from esplora
+  async $syncWalletAddress(wallet: Wallet, address: WalletAddress): Promise<void> {
+    // fetch full transaction data if the address is new or still active and hasn't been synced in the last hour
+    const refreshTransactions = !wallet.addresses[address.address] || (address.active && (Date.now() - wallet.addresses[address.address].lastSync) > 60 * 60 * 1000);
+    if (refreshTransactions) {
+      try {
+        const summary = await bitcoinApi.$getAddressTransactionSummary(address.address);
+        const addressInfo = await bitcoinApi.$getAddress(address.address);
+        const walletAddress: WalletAddress = {
+          address: address.address,
+          active: address.active,
+          transactions: summary,
+          stats: addressInfo.chain_stats,
+          lastSync: Date.now(),
+        };
+        wallet.addresses[address.address] = walletAddress;
+      } catch (e) {
+        logger.err(`Error syncing wallet address ${address.address}: ${(e instanceof Error ? e.message : e)}`);
+      }
+    }
+  }
+
+  // check a new block for transactions that affect wallet address balances, and add relevant transactions to wallets
+  processBlock(block: IEsploraApi.Block, blockTxs: TransactionExtended[]): Record<string, IEsploraApi.Transaction[]> {
+    const walletTransactions: Record<string, IEsploraApi.Transaction[]> = {};
+    for (const walletKey of Object.keys(this.wallets)) {
+      const wallet = this.wallets[walletKey];
+      walletTransactions[walletKey] = [];
+      for (const tx of blockTxs) {
+        const funded: Record<string, number> = {};
+        const spent: Record<string, number> = {};
+        const fundedCount: Record<string, number> = {};
+        const spentCount: Record<string, number> = {};
+        let anyMatch = false;
+        for (const vin of tx.vin) {
+          const address = vin.prevout?.scriptpubkey_address;
+          if (address && wallet.addresses[address]) {
+            anyMatch = true;
+            spent[address] = (spent[address] ?? 0) + (vin.prevout?.value ?? 0);
+            spentCount[address] = (spentCount[address] ?? 0) + 1;
+          }
+        }
+        for (const vout of tx.vout) {
+          const address = vout.scriptpubkey_address;
+          if (address && wallet.addresses[address]) {
+            anyMatch = true;
+            funded[address] = (funded[address] ?? 0) + (vout.value ?? 0);
+            fundedCount[address] = (fundedCount[address] ?? 0) + 1;
+          }
+        }
+        for (const address of Object.keys({ ...funded, ...spent })) {
+          // update address stats
+          wallet.addresses[address].stats.tx_count++;
+          wallet.addresses[address].stats.funded_txo_count += fundedCount[address] || 0;
+          wallet.addresses[address].stats.spent_txo_count += spentCount[address] || 0;
+          wallet.addresses[address].stats.funded_txo_sum += funded[address] || 0;
+          wallet.addresses[address].stats.spent_txo_sum += spent[address] || 0;
+          // add tx to summary
+          const txSummary: IEsploraApi.AddressTxSummary = {
+            txid: tx.txid,
+            value: (funded[address] ?? 0) - (spent[address] ?? 0),
+            height: block.height,
+            time: block.timestamp,
+          };
+          wallet.addresses[address].transactions?.push(txSummary);
+        }
+        if (anyMatch) {
+          walletTransactions[walletKey].push(tx);
+        }
+      }
+    }
+    return walletTransactions;
+  }
+}
+
+export default new WalletApi();

backend/src/api/statistics/statistics-api.ts

@@ -64,7 +64,7 @@ class StatisticsApi {
     }
   }
 
-  public async $create(statistics: Statistic): Promise<number | undefined> {
+  public async $create(statistics: Statistic, convertToDatetime = false): Promise<number | undefined> {
     try {
       const query = `INSERT INTO statistics(
               added,
@@ -114,7 +114,7 @@ class StatisticsApi {
               vsize_1800,
               vsize_2000
             )
-            VALUES (${statistics.added}, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
+            VALUES (${convertToDatetime ? `FROM_UNIXTIME(${statistics.added})` : statistics.added}, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
                ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
 
       const params: (string | number)[] = [
@@ -456,6 +456,59 @@ class StatisticsApi {
       };
     });
   }
+
+  public mapOptimizedStatisticToStatistic(statistic: OptimizedStatistic[]): Statistic[] {
+    return statistic.map((s) => {
+      return {
+        added: s.added,
+        unconfirmed_transactions: s.count,
+        tx_per_second: 0,
+        vbytes_per_second: s.vbytes_per_second,
+        mempool_byte_weight: s.mempool_byte_weight || 0,
+        total_fee: s.total_fee || 0,
+        min_fee: s.min_fee,
+        fee_data: '',
+        vsize_1: s.vsizes[0],
+        vsize_2: s.vsizes[1],
+        vsize_3: s.vsizes[2],
+        vsize_4: s.vsizes[3],
+        vsize_5: s.vsizes[4],
+        vsize_6: s.vsizes[5],
+        vsize_8: s.vsizes[6],
+        vsize_10: s.vsizes[7],
+        vsize_12: s.vsizes[8],
+        vsize_15: s.vsizes[9],
+        vsize_20: s.vsizes[10],
+        vsize_30: s.vsizes[11],
+        vsize_40: s.vsizes[12],
+        vsize_50: s.vsizes[13],
+        vsize_60: s.vsizes[14],
+        vsize_70: s.vsizes[15],
+        vsize_80: s.vsizes[16],
+        vsize_90: s.vsizes[17],
+        vsize_100: s.vsizes[18],
+        vsize_125: s.vsizes[19],
+        vsize_150: s.vsizes[20],
+        vsize_175: s.vsizes[21],
+        vsize_200: s.vsizes[22],
+        vsize_250: s.vsizes[23],
+        vsize_300: s.vsizes[24],
+        vsize_350: s.vsizes[25],
+        vsize_400: s.vsizes[26],
+        vsize_500: s.vsizes[27],
+        vsize_600: s.vsizes[28],
+        vsize_700: s.vsizes[29],
+        vsize_800: s.vsizes[30],
+        vsize_900: s.vsizes[31],
+        vsize_1000: s.vsizes[32],
+        vsize_1200: s.vsizes[33],
+        vsize_1400: s.vsizes[34],
+        vsize_1600: s.vsizes[35],
+        vsize_1800: s.vsizes[36],
+        vsize_2000: s.vsizes[37],
+      }
+    });
+  }
 }
 
 export default new StatisticsApi();

backend/src/api/statistics/statistics.routes.ts

@@ -1,7 +1,7 @@
 import { Application, Request, Response } from 'express';
 import config from '../../config';
 import statisticsApi from './statistics-api';
-
+import { handleError } from '../../utils/api';
 class StatisticsRoutes {
   public initRoutes(app: Application) {
     app
@@ -65,7 +65,7 @@ class StatisticsRoutes {
       }
       res.json(result);
     } catch (e) {
-      res.status(500).send(e instanceof Error ? e.message : e);
+      handleError(req, res, 500, 'Failed to get statistics');
     }
   }
 }

backend/src/api/transaction-utils.ts

@@ -103,7 +103,7 @@ class TransactionUtils {
     }
     const feePerVbytes = (transaction.fee || 0) / (transaction.weight / 4);
     const transactionExtended: TransactionExtended = Object.assign({
-      vsize: Math.round(transaction.weight / 4),
+      vsize: transaction.weight / 4,
       feePerVsize: feePerVbytes,
       effectiveFeePerVsize: feePerVbytes,
     }, transaction);
@@ -121,14 +121,15 @@ class TransactionUtils {
     const adjustedVsize = Math.max(fractionalVsize, sigops *  5); // adjusted vsize = Max(weight, sigops * bytes_per_sigop) / witness_scale_factor
     const feePerVbytes = (transaction.fee || 0) / fractionalVsize;
     const adjustedFeePerVsize = (transaction.fee || 0) / adjustedVsize;
+    const effectiveFeePerVsize = transaction['effectiveFeePerVsize'] || adjustedFeePerVsize || feePerVbytes;
     const transactionExtended: MempoolTransactionExtended = Object.assign(transaction, {
       order: this.txidToOrdering(transaction.txid),
-      vsize: Math.round(transaction.weight / 4),
+      vsize,
       adjustedVsize,
       sigops,
       feePerVsize: feePerVbytes,
       adjustedFeePerVsize: adjustedFeePerVsize,
-      effectiveFeePerVsize: adjustedFeePerVsize,
+      effectiveFeePerVsize: effectiveFeePerVsize,
     });
     if (!transactionExtended?.status?.confirmed && !transactionExtended.firstSeen) {
       transactionExtended.firstSeen = Math.round((Date.now() / 1000));
@@ -338,6 +339,87 @@ class TransactionUtils {
     const positionOfScript = hasAnnex ? witness.length - 3 : witness.length - 2;
     return witness[positionOfScript];
   }
+
+  // calculate the most parsimonious set of prioritizations given a list of block transactions
+  // (i.e. the most likely prioritizations and deprioritizations)
+  public identifyPrioritizedTransactions(transactions: any[], rateKey: string): { prioritized: string[], deprioritized: string[] } {
+    // find the longest increasing subsequence of transactions
+    // (adapted from https://en.wikipedia.org/wiki/Longest_increasing_subsequence#Efficient_algorithms)
+    // should be O(n log n)
+    const X = transactions.slice(1).reverse().map((tx) => ({ txid: tx.txid, rate: tx[rateKey] })); // standard block order is by *decreasing* effective fee rate, but we want to iterate in increasing order (and skip the coinbase)
+    if (X.length < 2) {
+      return { prioritized: [], deprioritized: [] };
+    }
+    const N = X.length;
+    const P: number[] = new Array(N);
+    const M: number[] = new Array(N + 1);
+    M[0] = -1; // undefined so can be set to any value
+
+    let L = 0;
+    for (let i = 0; i < N; i++) {
+      // Binary search for the smallest positive l ≤ L
+      // such that X[M[l]].effectiveFeePerVsize > X[i].effectiveFeePerVsize
+      let lo = 1;
+      let hi = L + 1;
+      while (lo < hi) {
+        const mid = lo + Math.floor((hi - lo) / 2); // lo <= mid < hi
+        if (X[M[mid]].rate > X[i].rate) {
+          hi = mid;
+        } else { // if X[M[mid]].effectiveFeePerVsize < X[i].effectiveFeePerVsize
+          lo = mid + 1;
+        }
+      }
+
+      // After searching, lo == hi is 1 greater than the
+      // length of the longest prefix of X[i]
+      const newL = lo;
+
+      // The predecessor of X[i] is the last index of
+      // the subsequence of length newL-1
+      P[i] = M[newL - 1];
+      M[newL] = i;
+
+      if (newL > L) {
+        // If we found a subsequence longer than any we've
+        // found yet, update L
+        L = newL;
+      }
+    }
+
+    // Reconstruct the longest increasing subsequence
+    // It consists of the values of X at the L indices:
+    // ..., P[P[M[L]]], P[M[L]], M[L]
+    const LIS: any[] = new Array(L);
+    let k = M[L];
+    for (let j = L - 1; j >= 0; j--) {
+      LIS[j] = X[k];
+      k = P[k];
+    }
+
+    const lisMap = new Map<string, number>();
+    LIS.forEach((tx, index) => lisMap.set(tx.txid, index));
+
+    const prioritized: string[] = [];
+    const deprioritized: string[] = [];
+
+    let lastRate = X[0].rate;
+
+    for (const tx of X) {
+      if (lisMap.has(tx.txid)) {
+        lastRate = tx.rate;
+      } else {
+        if (Math.abs(tx.rate - lastRate) < 0.1) {
+          // skip if the rate is almost the same as the previous transaction
+        } else if (tx.rate <= lastRate) {
+          prioritized.push(tx.txid);
+        } else {
+          deprioritized.push(tx.txid);
+        }
+      }
+    }
+
+    return { prioritized, deprioritized };
+  }
 }
 
 export default new TransactionUtils();

backend/src/api/websocket-handler.ts

@@ -3,6 +3,7 @@ import * as WebSocket from 'ws';
 import {
   BlockExtended, TransactionExtended, MempoolTransactionExtended, WebsocketResponse,
   OptimizedStatistic, ILoadingIndicators, GbtCandidates, TxTrackingInfo,
+  MempoolDelta, MempoolDeltaTxids
 } from '../mempool.interfaces';
 import blocks from './blocks';
 import memPool from './mempool';
@@ -15,16 +16,19 @@ import transactionUtils from './transaction-utils';
 import rbfCache, { ReplacementInfo } from './rbf-cache';
 import difficultyAdjustment from './difficulty-adjustment';
 import feeApi from './fee-api';
+import BlocksRepository from '../repositories/BlocksRepository';
 import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository';
 import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
 import Audit from './audit';
 import priceUpdater from '../tasks/price-updater';
 import { ApiPrice } from '../repositories/PricesRepository';
+import { Acceleration } from './services/acceleration';
 import accelerationApi from './services/acceleration';
 import mempool from './mempool';
 import statistics from './statistics/statistics';
 import accelerationRepository from '../repositories/AccelerationRepository';
 import bitcoinApi from './bitcoin/bitcoin-api-factory';
+import walletApi from './services/wallets';
 
 interface AddressTransactions {
   mempool: MempoolTransactionExtended[],
@@ -32,7 +36,9 @@ interface AddressTransactions {
   removed: MempoolTransactionExtended[],
 }
 import bitcoinSecondClient from './bitcoin/bitcoin-second-client';
-import { calculateCpfp } from './cpfp';
+import { calculateMempoolTxCpfp } from './cpfp';
+import { getRecentFirstSeen } from '../utils/file-read';
+import stratumApi, { StratumJob } from './services/stratum';
 
 // valid 'want' subscriptions
 const wantable = [
@@ -56,6 +62,8 @@ class WebsocketHandler {
   private lastRbfSummary: ReplacementInfo[] | null = null;
   private mempoolSequence: number = 0;
 
+  private accelerations: Record<string, Acceleration> = {};
+
   constructor() { }
 
   addWebsocketServer(wss: WebSocket.Server) {
@@ -205,7 +213,8 @@ class WebsocketHandler {
                 }
                 response['txPosition'] = JSON.stringify({
                   txid: trackTxid,
-                  position
+                  position,
+                  accelerationPositions: memPool.getAccelerationPositions(tx.txid),
                 });
               }
             } else {
@@ -303,6 +312,14 @@ class WebsocketHandler {
             }
           }
 
+          if (parsedMessage && parsedMessage['track-wallet']) {
+            if (parsedMessage['track-wallet'] === 'stop') {
+              client['track-wallet'] = null;
+            } else {
+              client['track-wallet'] = parsedMessage['track-wallet'];
+            }
+          }
+
           if (parsedMessage && parsedMessage['track-asset']) {
             if (/^[a-fA-F0-9]{64}$/.test(parsedMessage['track-asset'])) {
               client['track-asset'] = parsedMessage['track-asset'];
@@ -346,6 +363,17 @@ class WebsocketHandler {
             }
           }
 
+          if (parsedMessage && parsedMessage['track-accelerations'] != null) {
+            if (parsedMessage['track-accelerations']) {
+              client['track-accelerations'] = true;
+              response['accelerations'] = JSON.stringify({
+                accelerations: Object.values(memPool.getAccelerations()),
+              });
+            } else {
+              client['track-accelerations'] = false;
+            }
+          }
+
           if (parsedMessage.action === 'init') {
             if (!this.socketData['blocks']?.length || !this.socketData['da'] || !this.socketData['backendInfo'] || !this.socketData['conversions']) {
               this.updateSocketData();
@@ -364,6 +392,28 @@ class WebsocketHandler {
             client['track-donation'] = parsedMessage['track-donation'];
           }
 
+          if (parsedMessage['track-mempool-txids'] === true) {
+            client['track-mempool-txids'] = true;
+          } else if (parsedMessage['track-mempool-txids'] === false) {
+            delete client['track-mempool-txids'];
+          }
+
+          if (parsedMessage['track-mempool'] === true) {
+            client['track-mempool'] = true;
+          } else if (parsedMessage['track-mempool'] === false) {
+            delete client['track-mempool'];
+          }
+
+          if (parsedMessage && parsedMessage['track-stratum'] != null) {
+            if (parsedMessage['track-stratum']) {
+              const sub = parsedMessage['track-stratum'];
+              client['track-stratum'] = sub;
+              response['stratumJobs'] = this.socketData['stratumJobs'];
+            } else {
+              client['track-stratum'] = false;
+            }
+          }
+
           if (Object.keys(response).length) {
             client.send(this.serializeResponse(response));
           }
@@ -459,6 +509,42 @@ class WebsocketHandler {
     }
   }
 
+  handleAccelerationsChanged(accelerations: Record<string, Acceleration>): void {
+    if (!this.webSocketServers.length) {
+      throw new Error('No WebSocket.Server has been set');
+    }
+
+    const websocketAccelerationDelta = accelerationApi.getAccelerationDelta(this.accelerations, accelerations);
+    this.accelerations = accelerations;
+
+    if (!websocketAccelerationDelta.length) {
+      return;
+    }
+
+    // pre-compute acceleration delta
+    const accelerationUpdate = {
+      added: websocketAccelerationDelta.map(txid => accelerations[txid]).filter(acc => acc != null),
+      removed: websocketAccelerationDelta.filter(txid => !accelerations[txid]),
+    };
+
+    try {
+      const response = JSON.stringify({
+        accelerations: accelerationUpdate,
+      });
+
+      for (const server of this.webSocketServers) {
+        server.clients.forEach((client) => {
+          if (client.readyState !== WebSocket.OPEN) {
+            return;
+          }
+          client.send(response);
+        });
+      }
+    } catch (e) {
+      logger.debug(`Error sending acceleration update to websocket clients: ${e}`);
+    }
+  }
+
   handleReorg(): void {
     if (!this.webSocketServers.length) {
       throw new Error('No WebSocket.Server have been set');
@@ -495,8 +581,17 @@ class WebsocketHandler {
     }
   }
 
+  /**
+   *
+   * @param newMempool
+   * @param mempoolSize
+   * @param newTransactions  array of transactions added this mempool update.
+   * @param recentlyDeletedTransactions array of arrays of transactions removed in the last N mempool updates, most recent first.
+   * @param accelerationDelta
+   * @param candidates
+   */
   async $handleMempoolChange(newMempool: { [txid: string]: MempoolTransactionExtended }, mempoolSize: number,
-    newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[],
+    newTransactions: MempoolTransactionExtended[], recentlyDeletedTransactions: MempoolTransactionExtended[][], accelerationDelta: string[],
     candidates?: GbtCandidates): Promise<void> {
     if (!this.webSocketServers.length) {
       throw new Error('No WebSocket.Server have been set');
@@ -504,6 +599,8 @@ class WebsocketHandler {
 
     this.printLogs();
 
+    const deletedTransactions = recentlyDeletedTransactions.length ? recentlyDeletedTransactions[0] : [];
+
     const transactionIds = (memPool.limitGBT && candidates) ? Object.keys(candidates?.txs || {}) : Object.keys(newMempool);
     let added = newTransactions;
     let removed = deletedTransactions;
@@ -513,17 +610,18 @@ class WebsocketHandler {
     }
 
     if (config.MEMPOOL.RUST_GBT) {
-      await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, newMempool, added, removed, candidates, config.MEMPOOL_SERVICES.ACCELERATIONS);
+      await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, newMempool, added, removed, candidates, true);
     } else {
-      await mempoolBlocks.$updateBlockTemplates(transactionIds, newMempool, added, removed, candidates, accelerationDelta, true, config.MEMPOOL_SERVICES.ACCELERATIONS);
+      await mempoolBlocks.$updateBlockTemplates(transactionIds, newMempool, added, removed, candidates, accelerationDelta, true, true);
     }
 
     const mBlocks = mempoolBlocks.getMempoolBlocks();
     const mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();
     const mempoolInfo = memPool.getMempoolInfo();
     const vBytesPerSecond = memPool.getVBytesPerSecond();
-    const rbfTransactions = Common.findRbfTransactions(newTransactions, deletedTransactions);
+    const rbfTransactions = Common.findRbfTransactions(newTransactions, recentlyDeletedTransactions.flat());
     const da = difficultyAdjustment.getDifficultyAdjustment();
+    const accelerations = accelerationApi.getAccelerations();
     memPool.handleRbfTransactions(rbfTransactions);
     const rbfChanges = rbfCache.getRbfChanges();
     let rbfReplacements;
@@ -545,6 +643,33 @@ class WebsocketHandler {
 
     const latestTransactions = memPool.getLatestTransactions();
 
+    if (memPool.isInSync()) {
+      this.mempoolSequence++;
+    }
+
+    const replacedTransactions: { replaced: string, by: TransactionExtended }[] = [];
+    for (const tx of newTransactions) {
+      if (rbfTransactions[tx.txid]) {
+        for (const replaced of rbfTransactions[tx.txid].replaced) {
+          replacedTransactions.push({ replaced: replaced.txid, by: tx });
+        }
+      }
+    }
+    const mempoolDeltaTxids: MempoolDeltaTxids = {
+      sequence: this.mempoolSequence,
+      added: newTransactions.map(tx => tx.txid),
+      removed: deletedTransactions.map(tx => tx.txid),
+      mined: [],
+      replaced: replacedTransactions.map(replacement => ({ replaced: replacement.replaced, by: replacement.by.txid })),
+    };
+    const mempoolDelta: MempoolDelta = {
+      sequence: this.mempoolSequence,
+      added: newTransactions,
+      removed: deletedTransactions.map(tx => tx.txid),
+      mined: [],
+      replaced: replacedTransactions,
+    };
+
     // update init data
     const socketDataFields = {
       'mempoolInfo': mempoolInfo,
@@ -604,9 +729,14 @@ class WebsocketHandler {
     const addressCache = this.makeAddressCache(newTransactions);
     const removedAddressCache = this.makeAddressCache(deletedTransactions);
 
-    if (memPool.isInSync()) {
-      this.mempoolSequence++;
-    }
+    const websocketAccelerationDelta = accelerationApi.getAccelerationDelta(this.accelerations, accelerations);
+    this.accelerations = accelerations;
+
+    // pre-compute acceleration delta
+    const accelerationUpdate = {
+      added: websocketAccelerationDelta.map(txid => accelerations[txid]).filter(acc => acc != null),
+      removed: websocketAccelerationDelta.filter(txid => !accelerations[txid]),
+    };
 
     // TODO - Fix indentation after PR is merged
     for (const server of this.webSocketServers) {
@@ -766,10 +896,14 @@ class WebsocketHandler {
             position: {
               ...mempoolTx.position,
               accelerated: mempoolTx.acceleration || undefined,
-            }
+              acceleratedBy: mempoolTx.acceleratedBy || undefined,
+              acceleratedAt: mempoolTx.acceleratedAt || undefined,
+              feeDelta: mempoolTx.feeDelta || undefined,
+            },
+            accelerationPositions: memPool.getAccelerationPositions(mempoolTx.txid),
           };
           if (!mempoolTx.cpfpChecked && !mempoolTx.acceleration) {
-            calculateCpfp(mempoolTx, newMempool);
+            calculateMempoolTxCpfp(mempoolTx, newMempool);
           }
           if (mempoolTx.cpfpDirty) {
             positionData['cpfp'] = {
@@ -779,7 +913,7 @@ class WebsocketHandler {
               effectiveFeePerVsize: mempoolTx.effectiveFeePerVsize || null,
               sigops: mempoolTx.sigops,
               adjustedVsize: mempoolTx.adjustedVsize,
-              acceleration: mempoolTx.acceleration
+              acceleration: mempoolTx.acceleration,
             };
           }
           response['txPosition'] = JSON.stringify(positionData);
@@ -804,9 +938,12 @@ class WebsocketHandler {
             txInfo.position = {
               ...mempoolTx.position,
               accelerated: mempoolTx.acceleration || undefined,
+              acceleratedBy: mempoolTx.acceleratedBy || undefined,
+              acceleratedAt: mempoolTx.acceleratedAt || undefined,
+              feeDelta: mempoolTx.feeDelta || undefined,
             };
             if (!mempoolTx.cpfpChecked) {
-              calculateCpfp(mempoolTx, newMempool);
+              calculateMempoolTxCpfp(mempoolTx, newMempool);
             }
             if (mempoolTx.cpfpDirty) {
               txInfo.cpfp = {
@@ -847,6 +984,18 @@ class WebsocketHandler {
         response['rbfLatestSummary'] = getCachedResponse('rbfLatestSummary', rbfSummary);
       }
 
+      if (client['track-mempool-txids']) {
+        response['mempool-txids'] = getCachedResponse('mempool-txids', mempoolDeltaTxids);
+      }
+
+      if (client['track-mempool']) {
+        response['mempool-transactions'] = getCachedResponse('mempool-transactions', mempoolDelta);
+      }
+
+      if (client['track-accelerations'] && (accelerationUpdate.added.length || accelerationUpdate.removed.length)) {
+        response['accelerations'] = getCachedResponse('accelerations', accelerationUpdate);
+      }
+
       if (Object.keys(response).length) {
         client.send(this.serializeResponse(response));
       }
@@ -859,6 +1008,8 @@ class WebsocketHandler {
       throw new Error('No WebSocket.Server have been set');
     }
 
+    const blockTransactions = structuredClone(transactions);
+
     this.printLogs();
     await statistics.runStatistics();
 
@@ -868,31 +1019,27 @@ class WebsocketHandler {
     let transactionIds: string[] = (memPool.limitGBT) ? Object.keys(candidates?.txs || {}) : Object.keys(_memPool);
 
     const accelerations = Object.values(mempool.getAccelerations());
-    await accelerationRepository.$indexAccelerationsForBlock(block, accelerations, transactions);
+    await accelerationRepository.$indexAccelerationsForBlock(block, accelerations, structuredClone(transactions));
 
     const rbfTransactions = Common.findMinedRbfTransactions(transactions, memPool.getSpendMap());
-    memPool.handleMinedRbfTransactions(rbfTransactions);
+    memPool.handleRbfTransactions(rbfTransactions);
     memPool.removeFromSpendMap(transactions);
 
     if (config.MEMPOOL.AUDIT && memPool.isInSync()) {
       let projectedBlocks;
       const auditMempool = _memPool;
-      const isAccelerated = config.MEMPOOL_SERVICES.ACCELERATIONS && accelerationApi.isAcceleratedBlock(block, Object.values(mempool.getAccelerations()));
+      const isAccelerated = accelerationApi.isAcceleratedBlock(block, Object.values(mempool.getAccelerations()));
 
-      if ((config.MEMPOOL_SERVICES.ACCELERATIONS)) {
-        if (config.MEMPOOL.RUST_GBT) {
-          const added = memPool.limitGBT ? (candidates?.added || []) : [];
-          const removed = memPool.limitGBT ? (candidates?.removed || []) : [];
-          projectedBlocks = await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, auditMempool, added, removed, candidates, isAccelerated, block.extras.pool.id);
-        } else {
-          projectedBlocks = await mempoolBlocks.$makeBlockTemplates(transactionIds, auditMempool, candidates, false, isAccelerated, block.extras.pool.id);
-        }
+      if (config.MEMPOOL.RUST_GBT) {
+        const added = memPool.limitGBT ? (candidates?.added || []) : [];
+        const removed = memPool.limitGBT ? (candidates?.removed || []) : [];
+        projectedBlocks = await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, auditMempool, added, removed, candidates, isAccelerated, block.extras.pool.id);
       } else {
-        projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
+        projectedBlocks = await mempoolBlocks.$makeBlockTemplates(transactionIds, auditMempool, candidates, false, isAccelerated, block.extras.pool.id);
       }
 
       if (Common.indexingEnabled()) {
-        const { censored, added, prioritized, fresh, sigop, fullrbf, accelerated, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool);
+        const { unseen, censored, added, prioritized, fresh, sigop, fullrbf, accelerated, score, similarity } = Audit.auditBlock(block.height, blockTransactions, projectedBlocks, auditMempool);
         const matchRate = Math.round(score * 100 * 100) / 100;
 
         const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions : [];
@@ -914,9 +1061,11 @@ class WebsocketHandler {
         });
 
         BlocksAuditsRepository.$saveAudit({
+          version: 1,
           time: block.timestamp,
           height: block.height,
           hash: block.id,
+          unseenTxs: unseen,
           addedTxs: added,
           prioritizedTxs: prioritized,
           missingTxs: censored,
@@ -943,6 +1092,14 @@ class WebsocketHandler {
       }
     }
 
+    if (config.CORE_RPC.DEBUG_LOG_PATH && block.extras) {
+      const firstSeen = getRecentFirstSeen(block.id);
+      if (firstSeen) {
+        BlocksRepository.$saveFirstSeenTime(block.id, firstSeen);
+        block.extras.firstSeen = firstSeen;
+      }
+    }
+
     const confirmedTxids: { [txid: string]: boolean } = {};
 
     // Update mempool to remove transactions included in the new block
@@ -968,7 +1125,7 @@ class WebsocketHandler {
       const removed = memPool.limitGBT ? (candidates?.removed || []) : transactions;
       await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, _memPool, added, removed, candidates, true);
     } else {
-      await mempoolBlocks.$makeBlockTemplates(transactionIds, _memPool, candidates, true, config.MEMPOOL_SERVICES.ACCELERATIONS);
+      await mempoolBlocks.$makeBlockTemplates(transactionIds, _memPool, candidates, true, true);
     }
     const mBlocks = mempoolBlocks.getMempoolBlocks();
     const mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();
@@ -992,6 +1149,34 @@ class WebsocketHandler {
 
     const mBlocksWithTransactions = mempoolBlocks.getMempoolBlocksWithTransactions();
 
+    if (memPool.isInSync()) {
+      this.mempoolSequence++;
+    }
+
+    const replacedTransactions: { replaced: string, by: TransactionExtended }[] = [];
+    for (const txid of Object.keys(rbfTransactions)) {
+      for (const replaced of rbfTransactions[txid].replaced) {
+        replacedTransactions.push({ replaced: replaced.txid, by: rbfTransactions[txid].replacedBy });
+      }
+    }
+    const mempoolDeltaTxids: MempoolDeltaTxids = {
+      sequence: this.mempoolSequence,
+      added: [],
+      removed: [],
+      mined: transactions.map(tx => tx.txid),
+      replaced: replacedTransactions.map(replacement => ({ replaced: replacement.replaced, by: replacement.by.txid })),
+    };
+    const mempoolDelta: MempoolDelta = {
+      sequence: this.mempoolSequence,
+      added: [],
+      removed: [],
+      mined: transactions.map(tx => tx.txid),
+      replaced: replacedTransactions,
+    };
+
+    // check for wallet transactions
+    const walletTransactions = config.WALLETS.ENABLED ? walletApi.processBlock(block, transactions) : [];
+
     const responseCache = { ...this.socketData };
     function getCachedResponse(key, data): string {
       if (!responseCache[key]) {
@@ -1000,10 +1185,6 @@ class WebsocketHandler {
       return responseCache[key];
     }
 
-    if (memPool.isInSync()) {
-      this.mempoolSequence++;
-    }
-
     // TODO - Fix indentation after PR is merged
     for (const server of this.webSocketServers) {
     server.clients.forEach((client) => {
@@ -1047,7 +1228,11 @@ class WebsocketHandler {
               position: {
                 ...mempoolTx.position,
                 accelerated: mempoolTx.acceleration || undefined,
-              }
+                acceleratedBy: mempoolTx.acceleratedBy || undefined,
+                acceleratedAt: mempoolTx.acceleratedAt || undefined,
+                feeDelta: mempoolTx.feeDelta || undefined,
+              },
+              accelerationPositions: memPool.getAccelerationPositions(mempoolTx.txid),
             });
           }
         }
@@ -1066,6 +1251,9 @@ class WebsocketHandler {
                   ...mempoolTx.position,
                 },
                 accelerated: mempoolTx.acceleration || undefined,
+                acceleratedBy: mempoolTx.acceleratedBy || undefined,
+                acceleratedAt: mempoolTx.acceleratedAt || undefined,
+                feeDelta: mempoolTx.feeDelta || undefined,
               };
             }
           }
@@ -1185,6 +1373,19 @@ class WebsocketHandler {
         }
       }
 
+      if (client['track-mempool-txids']) {
+        response['mempool-txids'] = getCachedResponse('mempool-txids', mempoolDeltaTxids);
+      }
+
+      if (client['track-mempool']) {
+        response['mempool-transactions'] = getCachedResponse('mempool-transactions', mempoolDelta);
+      }
+
+      if (client['track-wallet']) {
+        const trackedWallet = client['track-wallet'];
+        response['wallet-transactions'] = getCachedResponse(`wallet-transactions-${trackedWallet}`, walletTransactions[trackedWallet] ?? {});
+      }
+
       if (Object.keys(response).length) {
         client.send(this.serializeResponse(response));
       }
@@ -1194,11 +1395,28 @@ class WebsocketHandler {
     await statistics.runStatistics();
   }
 
+  public handleNewStratumJob(job: StratumJob): void {
+    this.updateSocketDataFields({ 'stratumJobs': stratumApi.getJobs() });
+
+    for (const server of this.webSocketServers) {
+      server.clients.forEach((client) => {
+        if (client.readyState !== WebSocket.OPEN) {
+          return;
+        }
+        if (client['track-stratum'] && (client['track-stratum'] === 'all' || client['track-stratum'] === job.pool)) {
+          client.send(JSON.stringify({
+            'stratumJob': job
+        }));
+        }
+      });
+    }
+  }
+
   // takes a dictionary of JSON serialized values
   // and zips it together into a valid JSON object
   private serializeResponse(response): string {
     return '{'
-        + Object.keys(response).map(key => `"${key}": ${response[key]}`).join(', ')
+        + Object.keys(response).filter(key => response[key] != null).map(key => `"${key}": ${response[key]}`).join(', ')
         + '}';
   }
 

backend/src/config.ts

@@ -29,9 +29,10 @@ interface IConfig {
     EXTERNAL_RETRY_INTERVAL: number;
     USER_AGENT: string;
     STDOUT_LOG_MIN_PRIORITY: 'emerg' | 'alert' | 'crit' | 'err' | 'warn' | 'notice' | 'info' | 'debug';
-    AUTOMATIC_BLOCK_REINDEXING: boolean;
+    AUTOMATIC_POOLS_UPDATE: boolean;
     POOLS_JSON_URL: string,
     POOLS_JSON_TREE_URL: string,
+    POOLS_UPDATE_DELAY: number,
     AUDIT: boolean;
     RUST_GBT: boolean;
     LIMIT_GBT: boolean;
@@ -51,6 +52,7 @@ interface IConfig {
     REQUEST_TIMEOUT: number;
     FALLBACK_TIMEOUT: number;
     FALLBACK: string[];
+    MAX_BEHIND_TIP: number;
   };
   LIGHTNING: {
     ENABLED: boolean;
@@ -84,6 +86,7 @@ interface IConfig {
     TIMEOUT: number;
     COOKIE: boolean;
     COOKIE_PATH: string;
+    DEBUG_LOG_PATH: string;
   };
   SECOND_CORE_RPC: {
     HOST: string;
@@ -141,6 +144,8 @@ interface IConfig {
     ENABLED: boolean;
     AUDIT: boolean;
     AUDIT_START_HEIGHT: number;
+    STATISTICS: boolean;
+    STATISTICS_START_TIME: number | string;
     SERVERS: string[];
   },
   MEMPOOL_SERVICES: {
@@ -157,6 +162,14 @@ interface IConfig {
     PAID: boolean;
     API_KEY: string;
   },
+  WALLETS: {
+    ENABLED: boolean;
+    WALLETS: string[];
+  },
+  STRATUM: {
+    ENABLED: boolean;
+    API: string;
+  }
 }
 
 const defaults: IConfig = {
@@ -186,11 +199,12 @@ const defaults: IConfig = {
     'EXTERNAL_RETRY_INTERVAL': 0,
     'USER_AGENT': 'mempool',
     'STDOUT_LOG_MIN_PRIORITY': 'debug',
-    'AUTOMATIC_BLOCK_REINDEXING': false,
+    'AUTOMATIC_POOLS_UPDATE': false,
     'POOLS_JSON_URL': 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json',
     'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
+    'POOLS_UPDATE_DELAY': 604800, // in seconds, default is one week
     'AUDIT': false,
-    'RUST_GBT': false,
+    'RUST_GBT': true,
     'LIMIT_GBT': false,
     'CPFP_INDEXING': false,
     'MAX_BLOCKS_BULK_QUERY': 0,
@@ -208,6 +222,7 @@ const defaults: IConfig = {
     'REQUEST_TIMEOUT': 10000,
     'FALLBACK_TIMEOUT': 5000,
     'FALLBACK': [],
+    'MAX_BEHIND_TIP': 2,
   },
   'ELECTRUM': {
     'HOST': '127.0.0.1',
@@ -221,7 +236,8 @@ const defaults: IConfig = {
     'PASSWORD': 'mempool',
     'TIMEOUT': 60000,
     'COOKIE': false,
-    'COOKIE_PATH': '/bitcoin/.cookie'
+    'COOKIE_PATH': '/bitcoin/.cookie',
+    'DEBUG_LOG_PATH': '',
   },
   'SECOND_CORE_RPC': {
     'HOST': '127.0.0.1',
@@ -298,6 +314,8 @@ const defaults: IConfig = {
     'ENABLED': false,
     'AUDIT': false,
     'AUDIT_START_HEIGHT': 774000,
+    'STATISTICS': false,
+    'STATISTICS_START_TIME': 1481932800,
     'SERVERS': [],
   },
   'MEMPOOL_SERVICES': {
@@ -314,6 +332,14 @@ const defaults: IConfig = {
     'PAID': false,
     'API_KEY': '',
   },
+  'WALLETS': {
+    'ENABLED': false,
+    'WALLETS': [],
+  },
+  'STRATUM': {
+    'ENABLED': false,
+    'API': 'http://localhost:1234',
+  }
 };
 
 class Config implements IConfig {
@@ -335,6 +361,8 @@ class Config implements IConfig {
   MEMPOOL_SERVICES: IConfig['MEMPOOL_SERVICES'];
   REDIS: IConfig['REDIS'];
   FIAT_PRICE: IConfig['FIAT_PRICE'];
+  WALLETS: IConfig['WALLETS'];
+  STRATUM: IConfig['STRATUM'];
 
   constructor() {
     const configs = this.merge(configFromFile, defaults);
@@ -356,6 +384,8 @@ class Config implements IConfig {
     this.MEMPOOL_SERVICES = configs.MEMPOOL_SERVICES;
     this.REDIS = configs.REDIS;
     this.FIAT_PRICE = configs.FIAT_PRICE;
+    this.WALLETS = configs.WALLETS;
+    this.STRATUM = configs.STRATUM;
   }
 
   merge = (...objects: object[]): IConfig => {

backend/src/database.ts

@@ -2,8 +2,7 @@ import * as fs from 'fs';
 import path from 'path';
 import config from './config';
 import { createPool, Pool, PoolConnection } from 'mysql2/promise';
-import { LogLevel } from './logger';
-import logger from './logger';
+import logger, { LogLevel } from './logger';
 import { FieldPacket, OkPacket, PoolOptions, ResultSetHeader, RowDataPacket } from 'mysql2/typings/mysql';
 import { execSync } from 'child_process';
 

backend/src/index.ts

@@ -32,6 +32,7 @@ import pricesRoutes from './api/prices/prices.routes';
 import miningRoutes from './api/mining/mining-routes';
 import liquidRoutes from './api/liquid/liquid.routes';
 import bitcoinRoutes from './api/bitcoin/bitcoin.routes';
+import servicesRoutes from './api/services/services-routes';
 import fundingTxFetcher from './tasks/lightning/sync-tasks/funding-tx-fetcher';
 import forensicsService from './tasks/lightning/forensics.service';
 import priceUpdater from './tasks/price-updater';
@@ -45,6 +46,9 @@ import bitcoinCoreRoutes from './api/bitcoin/bitcoin-core.routes';
 import bitcoinSecondClient from './api/bitcoin/bitcoin-second-client';
 import accelerationRoutes from './api/acceleration/acceleration.routes';
 import aboutRoutes from './api/about.routes';
+import mempoolBlocks from './api/mempool-blocks';
+import walletApi from './api/services/wallets';
+import stratumApi from './api/services/stratum';
 
 class Server {
   private wss: WebSocket.Server | undefined;
@@ -131,6 +135,7 @@ class Server {
       })
       .use(express.urlencoded({ extended: true }))
       .use(express.text({ type: ['text/plain', 'application/base64'] }))
+      .use(express.json())
       ;
 
     if (config.DATABASE.ENABLED && config.FIAT_PRICE.ENABLED) {
@@ -148,6 +153,7 @@ class Server {
 
     await poolsUpdater.updatePoolsJson(); // Needs to be done before loading the disk cache because we sometimes wipe it
     await syncAssets.syncAssets$();
+    await mempoolBlocks.updatePools$();
     if (config.MEMPOOL.ENABLED) {
       if (config.MEMPOOL.CACHE_ENABLED) {
         await diskCache.$loadMempoolCache();
@@ -208,6 +214,8 @@ class Server {
         }
       });
     }
+
+    poolsUpdater.$startService();
   }
 
   async runMainUpdateLoop(): Promise<void> {
@@ -226,13 +234,17 @@ class Server {
       const newMempool = await bitcoinApi.$getRawMempool();
       const minFeeMempool = memPool.limitGBT ? await bitcoinSecondClient.getRawMemPool() : null;
       const minFeeTip = memPool.limitGBT ? await bitcoinSecondClient.getBlockCount() : -1;
-      const newAccelerations = await accelerationApi.$fetchAccelerations();
+      const latestAccelerations = await accelerationApi.$updateAccelerations();
       const numHandledBlocks = await blocks.$updateBlocks();
       const pollRate = config.MEMPOOL.POLL_RATE_MS * (indexer.indexerIsRunning() ? 10 : 1);
       if (numHandledBlocks === 0) {
-        await memPool.$updateMempool(newMempool, newAccelerations, minFeeMempool, minFeeTip, pollRate);
+        await memPool.$updateMempool(newMempool, latestAccelerations, minFeeMempool, minFeeTip, pollRate);
       }
       indexer.$run();
+      if (config.WALLETS.ENABLED) {
+        // might take a while, so run in the background
+        walletApi.$syncWallets();
+      }
       if (config.FIAT_PRICE.ENABLED) {
         priceUpdater.$run();
       }
@@ -307,11 +319,18 @@ class Server {
       priceUpdater.setRatesChangedCallback(websocketHandler.handleNewConversionRates.bind(websocketHandler));
     }
     loadingIndicators.setProgressChangedCallback(websocketHandler.handleLoadingChanged.bind(websocketHandler));
+
+    accelerationApi.connectWebsocket();
+    if (config.STRATUM.ENABLED) {
+      stratumApi.connectWebsocket();
+    }
   }
-  
+
   setUpHttpApiRoutes(): void {
     bitcoinRoutes.initRoutes(this.app);
-    bitcoinCoreRoutes.initRoutes(this.app);
+    if (config.MEMPOOL.OFFICIAL) {
+      bitcoinCoreRoutes.initRoutes(this.app);
+    }
     pricesRoutes.initRoutes(this.app);
     if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED && config.MEMPOOL.ENABLED) {
       statisticsRoutes.initRoutes(this.app);
@@ -330,7 +349,12 @@ class Server {
     if (config.MEMPOOL_SERVICES.ACCELERATIONS) {
       accelerationRoutes.initRoutes(this.app);
     }
-    aboutRoutes.initRoutes(this.app);
+    if (config.WALLETS.ENABLED) {
+      servicesRoutes.initRoutes(this.app);
+    }
+    if (!config.MEMPOOL.OFFICIAL) {
+      aboutRoutes.initRoutes(this.app);
+    }
   }
 
   healthCheck(): void {

backend/src/indexer.ts

@@ -8,7 +8,9 @@ import priceUpdater from './tasks/price-updater';
 import PricesRepository from './repositories/PricesRepository';
 import config from './config';
 import auditReplicator from './replication/AuditReplication';
+import statisticsReplicator from './replication/StatisticsReplication';
 import AccelerationRepository from './repositories/AccelerationRepository';
+import BlocksAuditsRepository from './repositories/BlocksAuditsRepository';
 
 export interface CoreIndex {
   name: string;
@@ -181,6 +183,7 @@ class Indexer {
       }
 
       this.runSingleTask('blocksPrices');
+      await blocks.$indexCoinbaseAddresses();
       await mining.$indexDifficultyAdjustments();
       await mining.$generateNetworkHashrateHistory();
       await mining.$generatePoolHashrateHistory();
@@ -188,7 +191,9 @@ class Indexer {
       await blocks.$generateCPFPDatabase();
       await blocks.$generateAuditStats();
       await auditReplicator.$sync();
+      await statisticsReplicator.$sync();
       await AccelerationRepository.$indexPastAccelerations();
+      await BlocksAuditsRepository.$migrateAuditsV0toV1();
       // do not wait for classify blocks to finish
       blocks.$classifyBlocks();
     } catch (e) {

backend/src/mempool.interfaces.ts

@@ -29,9 +29,11 @@ export interface PoolStats extends PoolInfo {
 }
 
 export interface BlockAudit {
+  version: number,
   time: number,
   height: number,
   hash: string,
+  unseenTxs: string[],
   missingTxs: string[],
   freshTxs: string[],
   sigopTxs: string[],
@@ -42,6 +44,19 @@ export interface BlockAudit {
   matchRate: number,
   expectedFees?: number,
   expectedWeight?: number,
+  template?: any[];
+}
+
+export interface TransactionAudit {
+  seen?: boolean;
+  expected?: boolean;
+  added?: boolean;
+  prioritized?: boolean;
+  delayed?: number;
+  accelerated?: boolean;
+  conflict?: boolean;
+  coinbase?: boolean;
+  firstSeen?: number;
 }
 
 export interface AuditScore {
@@ -71,6 +86,22 @@ export interface MempoolBlockDelta {
   changed: MempoolDeltaChange[];
 }
 
+export interface MempoolDeltaTxids {
+  sequence: number,
+  added: string[];
+  removed: string[];
+  mined: string[];
+  replaced: { replaced: string, by: string }[];
+}
+
+export interface MempoolDelta {
+  sequence: number,
+  added: MempoolTransactionExtended[];
+  removed: string[];
+  mined: string[];
+  replaced: { replaced: string, by: TransactionExtended }[];
+}
+
 interface VinStrippedToScriptsig {
   scriptsig: string;
 }
@@ -95,6 +126,9 @@ export interface TransactionExtended extends IEsploraApi.Transaction {
     vsize: number,
   };
   acceleration?: boolean;
+  acceleratedBy?: number[];
+  acceleratedAt?: number;
+  feeDelta?: number;
   replacement?: boolean;
   uid?: number;
   flags?: number;
@@ -192,6 +226,7 @@ export interface CpfpInfo {
   sigops?: number;
   adjustedVsize?: number,
   acceleration?: boolean,
+  fee?: number;
 }
 
 export interface TransactionStripped {
@@ -264,12 +299,14 @@ export interface BlockExtension {
     id: number; // Note - This is the `unique_id`, not to mix with the auto increment `id`
     name: string;
     slug: string;
+    minerNames: string[] | null;
   };
   avgFee: number;
   avgFeeRate: number;
   coinbaseRaw: string;
   orphans: OrphanedBlock[] | null;
   coinbaseAddress: string | null;
+  coinbaseAddresses: string[] | null;
   coinbaseSignature: string | null;
   coinbaseSignatureAscii: string | null;
   virtualSize: number;
@@ -283,10 +320,13 @@ export interface BlockExtension {
   segwitTotalSize: number;
   segwitTotalWeight: number;
   header: string;
+  firstSeen: number | null;
   utxoSetChange: number;
   // Requires coinstatsindex, will be set to NULL otherwise
   utxoSetSize: number | null;
   totalInputAmt: number | null;
+  // pools-v2.json git hash
+  definitionHash: string | undefined;
 }
 
 /**
@@ -349,8 +389,9 @@ export interface CpfpCluster {
 }
 
 export interface CpfpSummary {
-  transactions: TransactionExtended[];
+  transactions: MempoolTransactionExtended[];
   clusters: CpfpCluster[];
+  version: number;
 }
 
 export interface Statistic {
@@ -406,6 +447,7 @@ export interface Statistic {
 
 export interface OptimizedStatistic {
   added: string;
+  count: number;
   vbytes_per_second: number;
   total_fee: number;
   mempool_byte_weight: number;
@@ -415,7 +457,7 @@ export interface OptimizedStatistic {
 
 export interface TxTrackingInfo {
   replacedBy?: string,
-  position?: { block: number, vsize: number, accelerated?: boolean },
+  position?: { block: number, vsize: number, accelerated?: boolean, acceleratedBy?: number[], acceleratedAt?: number, feeDelta?: number },
   cpfp?: {
     ancestors?: Ancestor[],
     bestDescendant?: Ancestor | null,
@@ -426,6 +468,9 @@ export interface TxTrackingInfo {
   },
   utxoSpent?: { [vout: number]: { vin: number, txid: string } },
   accelerated?: boolean,
+  acceleratedBy?: number[],
+  acceleratedAt?: number,
+  feeDelta?: number,
   confirmed?: boolean
 }
 

backend/src/replication/AuditReplication.ts

@@ -31,11 +31,11 @@ class AuditReplication {
     const missingAudits = await this.$getMissingAuditBlocks();
 
     logger.debug(`Fetching missing audit data for ${missingAudits.length} blocks from trusted servers`, 'Replication');
-    
+
     let totalSynced = 0;
     let totalMissed = 0;
     let loggerTimer = Date.now();
-    // process missing audits in batches of 
+    // process missing audits in batches of BATCH_SIZE
     for (let i = 0; i < missingAudits.length; i += BATCH_SIZE) {
       const slice = missingAudits.slice(i, i + BATCH_SIZE);
       const results = await Promise.all(slice.map(hash => this.$syncAudit(hash)));
@@ -109,9 +109,11 @@ class AuditReplication {
       version: 1,
     });
     await blocksAuditsRepository.$saveAudit({
+      version: auditSummary.version || 0,
       hash: blockHash,
       height: auditSummary.height,
       time: auditSummary.timestamp || auditSummary.time,
+      unseenTxs: auditSummary.unseenTxs || [],
       missingTxs: auditSummary.missingTxs || [],
       addedTxs: auditSummary.addedTxs || [],
       prioritizedTxs: auditSummary.prioritizedTxs || [],

backend/src/replication/StatisticsReplication.ts

@@ -0,0 +1,237 @@
+import DB from '../database';
+import logger from '../logger';
+import { $sync } from './replicator';
+import config from '../config';
+import { Common } from '../api/common';
+import statistics from '../api/statistics/statistics-api';
+
+interface MissingStatistics {
+  '24h': Set<number>;
+  '1w': Set<number>;
+  '1m': Set<number>;
+  '3m': Set<number>;
+  '6m': Set<number>;
+  '2y': Set<number>;
+  'all': Set<number>;
+}
+
+const steps = {
+  '24h': 60,
+  '1w': 300,
+  '1m': 1800,
+  '3m': 7200,
+  '6m': 10800,
+  '2y': 28800,
+  'all': 43200,
+};
+
+/**
+ * Syncs missing statistics data from trusted servers
+ */
+class StatisticsReplication {
+  inProgress: boolean = false;
+
+  public async $sync(): Promise<void> {
+    if (!config.REPLICATION.ENABLED || !config.REPLICATION.STATISTICS || !config.STATISTICS.ENABLED) {
+      // replication not enabled, or statistics not enabled
+      return;
+    }
+    if (this.inProgress) {
+      logger.info(`StatisticsReplication sync already in progress`, 'Replication');
+      return;
+    }
+    this.inProgress = true;
+
+    const missingStatistics = await this.$getMissingStatistics();
+    const missingIntervals = Object.keys(missingStatistics).filter(key => missingStatistics[key].size > 0);
+    const totalMissing =  missingIntervals.reduce((total, key) => total + missingStatistics[key].size, 0);
+
+    if (totalMissing === 0) {
+      this.inProgress = false;
+      logger.info(`Statistics table is complete, no replication needed`, 'Replication');
+      return;
+    }
+    
+    for (const interval of missingIntervals) {
+      logger.debug(`Missing ${missingStatistics[interval].size} statistics rows in '${interval}' timespan`, 'Replication');
+    }
+    logger.debug(`Fetching ${missingIntervals.join(', ')} statistics endpoints from trusted servers to fill ${totalMissing} rows missing in statistics`, 'Replication');
+    
+    let totalSynced = 0;
+    let totalMissed = 0;
+
+    for (const interval of missingIntervals) {
+      const results = await this.$syncStatistics(interval, missingStatistics[interval]);
+      totalSynced += results.synced;
+      totalMissed += results.missed;
+
+      logger.info(`Found ${totalSynced} / ${totalSynced + totalMissed} of ${totalMissing} missing statistics rows`, 'Replication');
+      await Common.sleep$(3000);
+    }
+
+    logger.debug(`Synced ${totalSynced} statistics rows, ${totalMissed} still missing`, 'Replication');
+
+    this.inProgress = false;
+  }
+
+  private async $syncStatistics(interval: string, missingTimes: Set<number>): Promise<any> {
+  
+    let success = false;
+    let synced = 0;
+    let missed = new Set(missingTimes);
+    const syncResult = await $sync(`/api/v1/statistics/${interval}`);
+    if (syncResult && syncResult.data?.length) {
+      success = true;
+      logger.info(`Fetched /api/v1/statistics/${interval} from ${syncResult.server}`);     
+    
+      for (const stat of syncResult.data) {
+        const time = this.roundToNearestStep(stat.added, steps[interval]);
+        if (missingTimes.has(time)) {
+          try {
+            await statistics.$create(statistics.mapOptimizedStatisticToStatistic([stat])[0], true);
+            if (missed.delete(time)) {
+              synced++;
+            }
+          } catch (e: any) {
+            logger.err(`Failed to insert statistics row at ${stat.added} (${interval}) from ${syncResult.server}. Reason: ` + (e instanceof Error ? e.message : e));
+          }
+        }
+      }
+
+    } else {
+      logger.warn(`An error occured when trying to fetch /api/v1/statistics/${interval}`);
+    }
+
+    return { success, synced, missed: missed.size };
+  }
+
+  private async $getMissingStatistics(): Promise<MissingStatistics> {
+    try {
+      const now = Math.floor(Date.now() / 1000);
+      const day = 60 * 60 * 24;
+
+      const startTime = this.getStartTimeFromConfig();
+
+      const missingStatistics: MissingStatistics = {
+        '24h': new Set<number>(),
+        '1w': new Set<number>(),
+        '1m': new Set<number>(),
+        '3m': new Set<number>(),
+        '6m': new Set<number>(),
+        '2y': new Set<number>(),
+        'all': new Set<number>()
+      };
+
+      const intervals = [              // [start,               end,                 label ]
+                                          [now - day + 600,     now - 60,            '24h']       , // from 24 hours ago to now = 1 minute granularity
+        startTime < now - day ?           [now - day * 7,       now - day,           '1w' ] : null, // from 1 week ago to 24 hours ago = 5 minutes granularity
+        startTime < now - day * 7 ?       [now - day * 30,      now - day * 7,       '1m' ] : null, // from 1 month ago to 1 week ago = 30 minutes granularity
+        startTime < now - day * 30 ?      [now - day * 90,      now - day * 30,      '3m' ] : null, // from 3 months ago to 1 month ago = 2 hours granularity
+        startTime < now - day * 90 ?      [now - day * 180,     now - day * 90,      '6m' ] : null, // from 6 months ago to 3 months ago = 3 hours granularity
+        startTime < now - day * 180 ?     [now - day * 365 * 2, now - day * 180,     '2y' ] : null, // from 2 years ago to 6 months ago = 8 hours granularity
+        startTime < now - day * 365 * 2 ? [startTime,           now - day * 365 * 2, 'all'] : null, // from start of statistics to 2 years ago = 12 hours granularity   
+      ];
+
+      for (const interval of intervals) {
+        if (!interval) {
+          continue;
+        }
+        missingStatistics[interval[2] as string] = await this.$getMissingStatisticsInterval(interval, startTime);
+      }
+      
+      return missingStatistics;
+    } catch (e: any) {
+      logger.err(`Cannot fetch missing statistics times from db. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  private async $getMissingStatisticsInterval(interval: any, startTime: number): Promise<Set<number>> {
+    try {
+      const start = interval[0];
+      const end = interval[1];
+      const step = steps[interval[2]];
+
+      const [rows]: any[] = await DB.query(`
+        SELECT UNIX_TIMESTAMP(added) as added
+        FROM statistics
+        WHERE added >= FROM_UNIXTIME(?) AND added <= FROM_UNIXTIME(?)
+        GROUP BY UNIX_TIMESTAMP(added) DIV ${step} ORDER BY statistics.added DESC
+      `, [start, end]);
+
+      const startingTime = Math.max(startTime, start) - Math.max(startTime, start) % step;
+
+      const timeSteps: number[] = [];
+      for (let time = startingTime; time < end; time += step) {
+        timeSteps.push(time);
+      }
+
+      if (timeSteps.length === 0) {
+        return new Set<number>();
+      }
+      
+      const roundedTimesAlreadyHere: number[] = Array.from(new Set(rows.map(row => this.roundToNearestStep(row.added, step))));
+
+      const missingTimes = timeSteps.filter(time => !roundedTimesAlreadyHere.includes(time)).filter((time, i, arr) => {
+        // Remove outsiders
+        if (i === 0) {
+          return arr[i + 1] === time + step
+        } else if (i === arr.length - 1) {
+          return arr[i - 1] === time - step;
+        }
+        return (arr[i + 1] === time + step) && (arr[i - 1] === time - step)
+      });
+
+      // Don't bother fetching if very few rows are missing
+      if (missingTimes.length < timeSteps.length * 0.01) {
+        return new Set();
+      }
+
+      return new Set(missingTimes);
+    } catch (e: any) {
+      logger.err(`Cannot fetch missing statistics times from db. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  private roundToNearestStep(time: number, step: number): number {
+    const remainder = time % step;
+    if (remainder < step / 2) {
+      return time - remainder;
+    } else {
+      return time + (step - remainder);
+    }
+  }
+
+  private getStartTimeFromConfig(): number {
+    const now = Math.floor(Date.now() / 1000);
+    const day = 60 * 60 * 24;
+
+    let startTime: number;
+    if (typeof(config.REPLICATION.STATISTICS_START_TIME) === 'string' && ['24h', '1w', '1m', '3m', '6m', '2y', 'all'].includes(config.REPLICATION.STATISTICS_START_TIME)) {
+      if (config.REPLICATION.STATISTICS_START_TIME === 'all') {
+        startTime = 1481932800;
+      } else if (config.REPLICATION.STATISTICS_START_TIME === '2y') {
+        startTime = now - day * 365 * 2;
+      } else if (config.REPLICATION.STATISTICS_START_TIME === '6m') {
+        startTime = now - day * 180;
+      } else if (config.REPLICATION.STATISTICS_START_TIME === '3m') {
+        startTime = now - day * 90;
+      } else if (config.REPLICATION.STATISTICS_START_TIME === '1m') {
+        startTime = now - day * 30;
+      } else if (config.REPLICATION.STATISTICS_START_TIME === '1w') {
+        startTime = now - day * 7;
+      } else {
+        startTime = now - day;
+      }
+    } else {
+      startTime = Math.max(config.REPLICATION.STATISTICS_START_TIME as number || 1481932800, 1481932800);
+    }
+
+    return startTime;
+  }
+
+}
+
+export default new StatisticsReplication();
+

backend/src/repositories/AccelerationRepository.ts

@@ -1,4 +1,4 @@
-import { AccelerationInfo, makeBlockTemplate } from '../api/acceleration/acceleration';
+import { AccelerationInfo } from '../api/acceleration/acceleration';
 import { RowDataPacket } from 'mysql2';
 import DB from '../database';
 import logger from '../logger';
@@ -11,6 +11,7 @@ import accelerationCosts from '../api/acceleration/acceleration';
 import bitcoinApi from '../api/bitcoin/bitcoin-api-factory';
 import transactionUtils from '../api/transaction-utils';
 import { BlockExtended, MempoolTransactionExtended } from '../mempool.interfaces';
+import { makeBlockTemplate } from '../api/mini-miner';
 
 export interface PublicAcceleration {
   txid: string,
@@ -191,6 +192,7 @@ class AccelerationRepository {
     }
   }
 
+  // modifies block transactions
   public async $indexAccelerationsForBlock(block: BlockExtended, accelerations: Acceleration[], transactions: MempoolTransactionExtended[]): Promise<void> {
     const blockTxs: { [txid: string]: MempoolTransactionExtended } = {};
     for (const tx of transactions) {
@@ -212,6 +214,15 @@ class AccelerationRepository {
         this.$saveAcceleration(accelerationInfo, block, block.extras.pool.id, successfulAccelerations);
       }
     }
+    let anyConfirmed = false;
+    for (const acc of accelerations) {
+      if (blockTxs[acc.txid]) {
+        anyConfirmed = true;
+      }
+    }
+    if (anyConfirmed) {
+      accelerationApi.accelerationConfirmed();
+    }
     const lastSyncedHeight = await this.$getLastSyncedHeight();
     // if we've missed any blocks, let the indexer catch up from the last synced height on the next run
     if (block.height === lastSyncedHeight + 1) {
@@ -244,6 +255,8 @@ class AccelerationRepository {
     let count = 0;
     try {
       while (!done) {
+        // don't DDoS the services backend
+        Common.sleep$(500 + (Math.random() * 1000));
         const accelerations = await accelerationApi.$fetchAccelerationHistory(page);
         page++;
         if (!accelerations?.length) {
@@ -306,10 +319,10 @@ class AccelerationRepository {
         }
         const accelerationSummaries = accelerations.map(acc => ({
           ...acc,
-          pools: acc.pools.map(pool => pool.pool_unique_id),
+          pools: acc.pools,
         }))
         for (const acc of accelerations) {
-          if (blockTxs[acc.txid]) {
+          if (blockTxs[acc.txid] && acc.pools.includes(block.extras.pool.id)) {
             const tx = blockTxs[acc.txid];
             const accelerationInfo = accelerationCosts.getAccelerationInfo(tx, boostRate, transactions);
             accelerationInfo.cost = Math.max(0, Math.min(acc.feeDelta, accelerationInfo.cost));

backend/src/repositories/BlocksAuditsRepository.ts

@@ -1,13 +1,24 @@
-import blocks from '../api/blocks';
 import DB from '../database';
 import logger from '../logger';
-import { BlockAudit, AuditScore } from '../mempool.interfaces';
+import bitcoinApi from '../api/bitcoin/bitcoin-api-factory';
+import { BlockAudit, AuditScore, TransactionAudit, TransactionStripped } from '../mempool.interfaces';
+
+interface MigrationAudit {
+  version: number,
+  height: number,
+  id: string,
+  timestamp: number,
+  prioritizedTxs: string[],
+  acceleratedTxs: string[],
+  template: TransactionStripped[],
+  transactions: TransactionStripped[],
+}
 
 class BlocksAuditRepositories {
   public async $saveAudit(audit: BlockAudit): Promise<void> {
     try {
-      await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, prioritized_txs, fresh_txs, sigop_txs, fullrbf_txs, accelerated_txs, match_rate, expected_fees, expected_weight)
-        VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs),
+      await DB.query(`INSERT INTO blocks_audits(version, time, height, hash, unseen_txs, missing_txs, added_txs, prioritized_txs, fresh_txs, sigop_txs, fullrbf_txs, accelerated_txs, match_rate, expected_fees, expected_weight)
+        VALUE (?, FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [audit.version, audit.time, audit.height, audit.hash, JSON.stringify(audit.unseenTxs), JSON.stringify(audit.missingTxs),
           JSON.stringify(audit.addedTxs), JSON.stringify(audit.prioritizedTxs), JSON.stringify(audit.freshTxs), JSON.stringify(audit.sigopTxs), JSON.stringify(audit.fullrbfTxs), JSON.stringify(audit.acceleratedTxs), audit.matchRate, audit.expectedFees, audit.expectedWeight]);
     } catch (e: any) {
       if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
@@ -62,24 +73,30 @@ class BlocksAuditRepositories {
   public async $getBlockAudit(hash: string): Promise<BlockAudit | null> {
     try {
       const [rows]: any[] = await DB.query(
-        `SELECT blocks_audits.height, blocks_audits.hash as id, UNIX_TIMESTAMP(blocks_audits.time) as timestamp,
-        template,
-        missing_txs as missingTxs,
-        added_txs as addedTxs,
-        prioritized_txs as prioritizedTxs,
-        fresh_txs as freshTxs,
-        sigop_txs as sigopTxs,
-        fullrbf_txs as fullrbfTxs,
-        accelerated_txs as acceleratedTxs,
-        match_rate as matchRate,
-        expected_fees as expectedFees,
-        expected_weight as expectedWeight
+        `SELECT
+          blocks_audits.version,
+          blocks_audits.height,
+          blocks_audits.hash as id,
+          UNIX_TIMESTAMP(blocks_audits.time) as timestamp,
+          template,
+          unseen_txs as unseenTxs,
+          missing_txs as missingTxs,
+          added_txs as addedTxs,
+          prioritized_txs as prioritizedTxs,
+          fresh_txs as freshTxs,
+          sigop_txs as sigopTxs,
+          fullrbf_txs as fullrbfTxs,
+          accelerated_txs as acceleratedTxs,
+          match_rate as matchRate,
+          expected_fees as expectedFees,
+          expected_weight as expectedWeight
         FROM blocks_audits
         JOIN blocks_templates ON blocks_templates.id = blocks_audits.hash
         WHERE blocks_audits.hash = ?
       `, [hash]);
       
       if (rows.length) {
+        rows[0].unseenTxs = JSON.parse(rows[0].unseenTxs);
         rows[0].missingTxs = JSON.parse(rows[0].missingTxs);
         rows[0].addedTxs = JSON.parse(rows[0].addedTxs);
         rows[0].prioritizedTxs = JSON.parse(rows[0].prioritizedTxs);
@@ -98,6 +115,42 @@ class BlocksAuditRepositories {
     }
   }
 
+  public async $getBlockTxAudit(hash: string, txid: string): Promise<TransactionAudit | null> {
+    try {
+      const blockAudit = await this.$getBlockAudit(hash);
+
+      if (blockAudit) {
+        const isAdded = blockAudit.addedTxs.includes(txid);
+        const isPrioritized = blockAudit.prioritizedTxs.includes(txid);
+        const isAccelerated = blockAudit.acceleratedTxs.includes(txid);
+        const isConflict = blockAudit.fullrbfTxs.includes(txid);
+        let isExpected = false;
+        let firstSeen = undefined;
+        blockAudit.template?.forEach(tx => {
+          if (tx.txid === txid) {
+            isExpected = true;
+            firstSeen = tx.time;
+          }
+        });
+        const wasSeen = blockAudit.version === 1 ? !blockAudit.unseenTxs.includes(txid) : (isExpected || isPrioritized || isAccelerated);
+
+        return {
+          seen: wasSeen,
+          expected: isExpected,
+          added: isAdded && (blockAudit.version === 0 || !wasSeen),
+          prioritized: isPrioritized,
+          conflict: isConflict,
+          accelerated: isAccelerated,
+          firstSeen,
+        };
+      }
+      return null;
+    } catch (e: any) {
+      logger.err(`Cannot fetch block transaction audit from db. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
   public async $getBlockAuditScore(hash: string): Promise<AuditScore> {
     try {
       const [rows]: any[] = await DB.query(
@@ -151,6 +204,96 @@ class BlocksAuditRepositories {
       throw e;
     }
   }
+
+  /**
+   * [INDEXING] Migrate audits from v0 to v1
+   */
+  public async $migrateAuditsV0toV1(): Promise<void> {
+    try {
+      let done = false;
+      let processed = 0;
+      let lastHeight;
+      while (!done) {
+        const [toMigrate]: MigrationAudit[][] = await DB.query(
+          `SELECT
+            blocks_audits.height as height,
+            blocks_audits.hash as id,
+            UNIX_TIMESTAMP(blocks_audits.time) as timestamp,
+            blocks_summaries.transactions as transactions,
+            blocks_templates.template as template,
+            blocks_audits.prioritized_txs as prioritizedTxs,
+            blocks_audits.accelerated_txs as acceleratedTxs
+          FROM blocks_audits
+          JOIN blocks_summaries ON blocks_summaries.id = blocks_audits.hash
+          JOIN blocks_templates ON blocks_templates.id = blocks_audits.hash
+          WHERE blocks_audits.version = 0
+          AND blocks_summaries.version = 2
+          ORDER BY blocks_audits.height DESC
+          LIMIT 100
+        `) as any[];
+
+        if (toMigrate.length <= 0 || lastHeight === toMigrate[0].height) {
+          done = true;
+          break;
+        }
+        lastHeight = toMigrate[0].height;
+
+        logger.info(`migrating ${toMigrate.length} audits to version 1`);
+
+        for (const audit of toMigrate) {
+          // unpack JSON-serialized transaction lists
+          audit.transactions = JSON.parse((audit.transactions as any as string) || '[]');
+          audit.template = JSON.parse((audit.template as any as string) || '[]');
+
+          // we know transactions in the template, or marked "prioritized" or "accelerated"
+          // were seen in our mempool before the block was mined.
+          const isSeen = new Set<string>();
+          for (const tx of audit.template) {
+            isSeen.add(tx.txid);
+          }
+          for (const txid of audit.prioritizedTxs) {
+            isSeen.add(txid);
+          }
+          for (const txid of audit.acceleratedTxs) {
+            isSeen.add(txid);
+          }
+          const unseenTxs = audit.transactions.slice(0).map(tx => tx.txid).filter(txid => !isSeen.has(txid));
+
+          // identify "prioritized" transactions
+          const prioritizedTxs: string[] = [];
+          let lastEffectiveRate = 0;
+          // Iterate over the mined template from bottom to top (excluding the coinbase)
+          // Transactions should appear in ascending order of mining priority.
+          for (let i = audit.transactions.length - 1; i > 0; i--) {
+            const blockTx = audit.transactions[i];
+            // If a tx has a lower in-band effective fee rate than the previous tx,
+            // it must have been prioritized out-of-band (in order to have a higher mining priority)
+            // so exclude from the analysis.
+            if ((blockTx.rate || 0) < lastEffectiveRate) {
+              prioritizedTxs.push(blockTx.txid);
+            } else {
+              lastEffectiveRate = blockTx.rate || 0;
+            }
+          }
+
+          // Update audit in the database
+          await DB.query(`
+            UPDATE blocks_audits SET
+              version = ?,
+              unseen_txs = ?,
+              prioritized_txs = ?
+            WHERE hash = ?
+          `, [1, JSON.stringify(unseenTxs), JSON.stringify(prioritizedTxs), audit.id]);
+        }
+
+        processed += toMigrate.length;
+      }
+
+      logger.info(`migrated ${processed} audits to version 1`);
+    } catch (e: any) {
+      logger.err(`Error while migrating audits from v0 to v1. Will try again later. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+  }
 }
 
 export default new BlocksAuditRepositories();

backend/src/repositories/BlocksRepository.ts

@@ -5,7 +5,7 @@ import logger from '../logger';
 import { Common } from '../api/common';
 import PoolsRepository from './PoolsRepository';
 import HashratesRepository from './HashratesRepository';
-import { RowDataPacket, escape } from 'mysql2';
+import { RowDataPacket } from 'mysql2';
 import BlocksSummariesRepository from './BlocksSummariesRepository';
 import DifficultyAdjustmentsRepository from './DifficultyAdjustmentsRepository';
 import bitcoinClient from '../api/bitcoin/bitcoin-client';
@@ -14,6 +14,8 @@ import chainTips from '../api/chain-tips';
 import blocks from '../api/blocks';
 import BlocksAuditsRepository from './BlocksAuditsRepository';
 import transactionUtils from '../api/transaction-utils';
+import { parseDATUMTemplateCreator } from '../utils/bitcoin-script';
+import poolsUpdater from '../tasks/pools-updater';
 
 interface DatabaseBlock {
   id: string;
@@ -40,6 +42,7 @@ interface DatabaseBlock {
   avgFeeRate: number;
   coinbaseRaw: string;
   coinbaseAddress: string;
+  coinbaseAddresses: string;
   coinbaseSignature: string;
   coinbaseSignatureAscii: string;
   avgTxSize: number;
@@ -55,6 +58,7 @@ interface DatabaseBlock {
   utxoSetChange: number;
   utxoSetSize: number;
   totalInputAmt: number;
+  firstSeen: number;
 }
 
 const BLOCK_DB_FIELDS = `
@@ -82,6 +86,7 @@ const BLOCK_DB_FIELDS = `
   blocks.avg_fee_rate AS avgFeeRate,
   blocks.coinbase_raw AS coinbaseRaw,
   blocks.coinbase_address AS coinbaseAddress,
+  blocks.coinbase_addresses AS coinbaseAddresses,
   blocks.coinbase_signature AS coinbaseSignature,
   blocks.coinbase_signature_ascii AS coinbaseSignatureAscii,
   blocks.avg_tx_size AS avgTxSize,
@@ -96,7 +101,8 @@ const BLOCK_DB_FIELDS = `
   blocks.header,
   blocks.utxoset_change AS utxoSetChange,
   blocks.utxoset_size AS utxoSetSize,
-  blocks.total_input_amt AS totalInputAmt
+  blocks.total_input_amt AS totalInputAmt,
+  UNIX_TIMESTAMP(blocks.first_seen) AS firstSeen
 `;
 
 class BlocksRepository {
@@ -109,27 +115,27 @@ class BlocksRepository {
 
     try {
       const query = `INSERT INTO blocks(
-        height,             hash,                blockTimestamp,    size,
-        weight,             tx_count,            coinbase_raw,      difficulty,
-        pool_id,            fees,                fee_span,          median_fee,
-        reward,             version,             bits,              nonce,
-        merkle_root,        previous_block_hash, avg_fee,           avg_fee_rate,
-        median_timestamp,   header,              coinbase_address,
-        coinbase_signature, utxoset_size,        utxoset_change,    avg_tx_size,
-        total_inputs,       total_outputs,       total_input_amt,   total_output_amt,
-        fee_percentiles,    segwit_total_txs,    segwit_total_size, segwit_total_weight,
-        median_fee_amt,     coinbase_signature_ascii
+        height,             hash,                     blockTimestamp,    size,
+        weight,             tx_count,                 coinbase_raw,      difficulty,
+        pool_id,            fees,                     fee_span,          median_fee,
+        reward,             version,                  bits,              nonce,
+        merkle_root,        previous_block_hash,      avg_fee,           avg_fee_rate,
+        median_timestamp,   header,                   coinbase_address,  coinbase_addresses,
+        coinbase_signature, utxoset_size,             utxoset_change,    avg_tx_size,
+        total_inputs,       total_outputs,            total_input_amt,   total_output_amt,
+        fee_percentiles,    segwit_total_txs,         segwit_total_size, segwit_total_weight,
+        median_fee_amt,     coinbase_signature_ascii, definition_hash
       ) VALUE (
         ?, ?, FROM_UNIXTIME(?), ?,
         ?, ?, ?, ?,
         ?, ?, ?, ?,
         ?, ?, ?, ?,
         ?, ?, ?, ?,
-        FROM_UNIXTIME(?), ?, ?,
+        FROM_UNIXTIME(?), ?, ?, ?,
         ?, ?, ?, ?,
         ?, ?, ?, ?,
         ?, ?, ?, ?,
-        ?, ?
+        ?, ?, ?
       )`;
 
       const poolDbId = await PoolsRepository.$getPoolByUniqueId(block.extras.pool.id);
@@ -161,6 +167,7 @@ class BlocksRepository {
         block.mediantime,
         block.extras.header,
         block.extras.coinbaseAddress,
+        block.extras.coinbaseAddresses ? JSON.stringify(block.extras.coinbaseAddresses) : null,
         truncatedCoinbaseSignature,
         block.extras.utxoSetSize,
         block.extras.utxoSetChange,
@@ -175,6 +182,7 @@ class BlocksRepository {
         block.extras.segwitTotalWeight,
         block.extras.medianFeeAmt,
         truncatedCoinbaseSignatureAscii,
+        poolsUpdater.currentSha
       ];
 
       await DB.query(query, params);
@@ -495,7 +503,7 @@ class BlocksRepository {
     }
 
     query += ` ORDER BY height DESC
-      LIMIT 10`;
+      LIMIT 100`;
 
     try {
       const [rows]: any[] = await DB.query(query, params);
@@ -529,7 +537,7 @@ class BlocksRepository {
         return null;
       }
 
-      return await this.formatDbBlockIntoExtendedBlock(rows[0] as DatabaseBlock);  
+      return await this.formatDbBlockIntoExtendedBlock(rows[0] as DatabaseBlock);
     } catch (e) {
       logger.err(`Cannot get indexed block ${height}. Reason: ` + (e instanceof Error ? e.message : e));
       throw e;
@@ -663,7 +671,7 @@ class BlocksRepository {
   /**
    * Get the historical averaged block fees
    */
-  public async $getHistoricalBlockFees(div: number, interval: string | null): Promise<any> {
+  public async $getHistoricalBlockFees(div: number, interval: string | null, timespan?: {from: number, to: number}): Promise<any> {
     try {
       let query = `SELECT
         CAST(AVG(blocks.height) as INT) as avgHeight,
@@ -677,6 +685,8 @@ class BlocksRepository {
 
       if (interval !== null) {
         query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
+      } else if (timespan) {
+        query += ` WHERE blockTimestamp BETWEEN FROM_UNIXTIME(${timespan.from}) AND FROM_UNIXTIME(${timespan.to})`;
       }
 
       query += ` GROUP BY UNIX_TIMESTAMP(blockTimestamp) DIV ${div}`;
@@ -920,6 +930,25 @@ class BlocksRepository {
     }
   }
 
+  /**
+   * Get all indexed blocks with missing coinbase addresses
+   */
+  public async $getBlocksWithoutCoinbaseAddresses(): Promise<any> {
+    try {
+      const [blocks] = await DB.query(`
+        SELECT height, hash, coinbase_addresses
+        FROM blocks
+        WHERE coinbase_addresses IS NULL AND
+          coinbase_address IS NOT NULL
+        ORDER BY height DESC
+      `);
+      return blocks;
+    } catch (e) {
+      logger.err(`Cannot get blocks with missing coinbase addresses. Reason: ` + (e instanceof Error ? e.message : e));
+      return [];
+    }
+  }
+
   /**
    * Save indexed median fee to avoid recomputing it later
    * 
@@ -958,6 +987,62 @@ class BlocksRepository {
     }
   }
 
+  /**
+   * Save coinbase addresses
+   * 
+   * @param id
+   * @param addresses
+   */
+  public async $saveCoinbaseAddresses(id: string, addresses: string[]): Promise<void> {
+    try {
+      await DB.query(`
+        UPDATE blocks SET coinbase_addresses = ?
+        WHERE hash = ?`,
+        [JSON.stringify(addresses), id]
+      );
+    } catch (e) {
+      logger.err(`Cannot update block coinbase addresses. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Save pool
+   * 
+   * @param id
+   * @param poolId
+   */
+  public async $savePool(id: string, poolId: number): Promise<void> {
+    try {
+      await DB.query(`
+        UPDATE blocks SET pool_id = ?, definition_hash = ?
+        WHERE hash = ?`,
+        [poolId, poolsUpdater.currentSha, id]
+      );
+    } catch (e) {
+      logger.err(`Cannot update block pool. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Save block first seen time
+   * 
+   * @param id 
+   */
+  public async $saveFirstSeenTime(id: string, firstSeen: number): Promise<void> {
+    try {
+      await DB.query(`
+        UPDATE blocks SET first_seen = FROM_UNIXTIME(?)
+        WHERE hash = ?`,
+        [firstSeen, id]
+      );
+    } catch (e) {
+      logger.err(`Cannot update block first seen time. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
   /**
    * Convert a mysql row block into a BlockExtended. Note that you
    * must provide the correct field into dbBlk object param
@@ -992,11 +1077,13 @@ class BlocksRepository {
       id: dbBlk.poolId,
       name: dbBlk.poolName,
       slug: dbBlk.poolSlug,
+      minerNames: null,
     };
     extras.avgFee = dbBlk.avgFee;
     extras.avgFeeRate = dbBlk.avgFeeRate;
     extras.coinbaseRaw = dbBlk.coinbaseRaw;
     extras.coinbaseAddress = dbBlk.coinbaseAddress;
+    extras.coinbaseAddresses = dbBlk.coinbaseAddresses ? JSON.parse(dbBlk.coinbaseAddresses) : [];
     extras.coinbaseSignature = dbBlk.coinbaseSignature;
     extras.coinbaseSignatureAscii = dbBlk.coinbaseSignatureAscii;
     extras.avgTxSize = dbBlk.avgTxSize;
@@ -1013,6 +1100,7 @@ class BlocksRepository {
     extras.utxoSetSize = dbBlk.utxoSetSize;
     extras.totalInputAmt = dbBlk.totalInputAmt;
     extras.virtualSize = dbBlk.weight / 4.0;
+    extras.firstSeen = dbBlk.firstSeen;
 
     // Re-org can happen after indexing so we need to always get the
     // latest state from core
@@ -1043,7 +1131,7 @@ class BlocksRepository {
         let summaryVersion = 0;
         if (config.MEMPOOL.BACKEND === 'esplora') {
           const txs = (await bitcoinApi.$getTxsForBlock(dbBlk.id)).map(tx => transactionUtils.extendTransaction(tx));
-          summary = blocks.summarizeBlockTransactions(dbBlk.id, txs);
+          summary = blocks.summarizeBlockTransactions(dbBlk.id, dbBlk.height, txs);
           summaryVersion = 1;
         } else {
           // Call Core RPC
@@ -1060,6 +1148,10 @@ class BlocksRepository {
       }
     }
 
+    if (extras.pool.name === 'OCEAN') {
+      extras.pool.minerNames = parseDATUMTemplateCreator(extras.coinbaseRaw);
+    }
+
     blk.extras = <BlockExtension>extras;
     return <BlockExtended>blk;
   }

backend/src/repositories/BlocksSummariesRepository.ts

@@ -114,6 +114,43 @@ class BlocksSummariesRepository {
     return [];
   }
 
+  public async $getSummariesBelowVersion(version: number): Promise<{ height: number, id: string, version: number }[]> {
+    try {
+      const [rows]: any[] = await DB.query(`
+        SELECT
+          height,
+          id,
+          version
+        FROM blocks_summaries
+        WHERE version < ?
+        ORDER BY height DESC;`, [version]);
+      return rows;
+    } catch (e) {
+      logger.err(`Cannot get block summaries below version. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+
+    return [];
+  }
+
+  public async $getTemplatesBelowVersion(version: number): Promise<{ height: number, id: string, version: number }[]> {
+    try {
+      const [rows]: any[] = await DB.query(`
+        SELECT
+          blocks_summaries.height as height,
+          blocks_templates.id as id,
+          blocks_templates.version as version
+        FROM blocks_templates
+        JOIN blocks_summaries ON blocks_templates.id = blocks_summaries.id
+        WHERE blocks_templates.version < ?
+        ORDER BY height DESC;`, [version]);
+      return rows;
+    } catch (e) {
+      logger.err(`Cannot get block summaries below version. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+
+    return [];
+  }
+
   /**
    * Get the fee percentiles if the block has already been indexed, [] otherwise
    * 

backend/src/repositories/CpfpRepository.ts

@@ -91,6 +91,26 @@ class CpfpRepository {
     return;
   }
 
+  public async $getClustersAt(height: number): Promise<CpfpCluster[]> {
+    const [clusterRows]: any = await DB.query(
+      `
+        SELECT *
+        FROM compact_cpfp_clusters
+        WHERE height = ?
+      `,
+      [height]
+    );
+    return clusterRows.map(cluster => {
+      if (cluster?.txs) {
+        cluster.effectiveFeePerVsize = cluster.fee_rate;
+        cluster.txs = this.unpack(cluster.txs);
+        return cluster;
+      } else {
+        return null;
+      }
+    }).filter(cluster => cluster !== null);
+  }
+
   public async $deleteClustersFrom(height: number): Promise<void> {
     logger.info(`Delete newer cpfp clusters from height ${height} from the database`);
     try {
@@ -122,6 +142,37 @@ class CpfpRepository {
     }
   }
 
+  public async $deleteClustersAt(height: number): Promise<void> {
+    logger.info(`Delete cpfp clusters at height ${height} from the database`);
+    try {
+      const [rows] = await DB.query(
+        `
+          SELECT txs, height, root from compact_cpfp_clusters
+          WHERE height = ?
+        `,
+        [height]
+      ) as RowDataPacket[][];
+      if (rows?.length) {
+        for (const clusterToDelete of rows) {
+          const txs = this.unpack(clusterToDelete?.txs);
+          for (const tx of txs) {
+            await transactionRepository.$removeTransaction(tx.txid);
+          }
+        }
+      }
+      await DB.query(
+        `
+          DELETE from compact_cpfp_clusters
+          WHERE height = ?
+        `,
+        [height]
+      );
+    } catch (e: any) {
+      logger.err(`Cannot delete cpfp clusters from db. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
   // insert a dummy row to mark that we've indexed as far as this block
   public async $insertProgressMarker(height: number): Promise<void> {
     try {
@@ -190,6 +241,32 @@ class CpfpRepository {
       return [];
     }
   }
+
+  // returns `true` if two sets of CPFP clusters are deeply identical
+  public compareClusters(clustersA: CpfpCluster[], clustersB: CpfpCluster[]): boolean {
+    if (clustersA.length !== clustersB.length) {
+      return false;
+    }
+
+    clustersA = clustersA.sort((a,b) => a.root.localeCompare(b.root));
+    clustersB = clustersB.sort((a,b) => a.root.localeCompare(b.root));
+
+    for (let i = 0; i < clustersA.length; i++) {
+      if (clustersA[i].root !== clustersB[i].root) {
+        return false;
+      }
+      if (clustersA[i].txs.length !== clustersB[i].txs.length) {
+        return false;
+      }
+      for (let j = 0; j < clustersA[i].txs.length; j++) {
+        if (clustersA[i].txs[j].txid !== clustersB[i].txs[j].txid) {
+          return false;
+        }
+      }
+    }
+
+    return true;
+  }
 }
 
 export default new CpfpRepository();

backend/src/rpc-api/commands.ts

@@ -83,6 +83,7 @@ module.exports = {
   signRawTransaction: 'signrawtransaction', // bitcoind v0.7.0+
   stop: 'stop',
   submitBlock: 'submitblock', // bitcoind v0.7.0+
+  submitPackage: 'submitpackage',
   validateAddress: 'validateaddress',
   verifyChain: 'verifychain', // bitcoind v0.9.0+
   verifyMessage: 'verifymessage',

backend/src/tasks/pools-updater.ts

@@ -6,16 +6,30 @@ import backendInfo from '../api/backend-info';
 import logger from '../logger';
 import { SocksProxyAgent } from 'socks-proxy-agent';
 import * as https from 'https';
+import { Common } from '../api/common';
 
 /**
  * Maintain the most recent version of pools-v2.json
  */
 class PoolsUpdater {
+  tag = 'PoolsUpdater';
+
   lastRun: number = 0;
   currentSha: string | null = null;
   poolsUrl: string = config.MEMPOOL.POOLS_JSON_URL;
   treeUrl: string = config.MEMPOOL.POOLS_JSON_TREE_URL;
 
+  public async $startService(): Promise<void> {
+    while ('Bitcoin is still alive') {
+      try {
+        await this.updatePoolsJson();
+      } catch (e: any) {
+        logger.info(`Exception ${e} in PoolsUpdater::$startService. Code: ${e.code}. Message: ${e.message}`, this.tag);
+      }
+      await Common.sleep$(10000);
+    }
+  }
+
   public async updatePoolsJson(): Promise<void> {
     if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false ||
       config.MEMPOOL.ENABLED === false
@@ -23,11 +37,8 @@ class PoolsUpdater {
       return;
     }
 
-    const oneWeek = 604800;
-    const oneDay = 86400;
-
     const now = new Date().getTime() / 1000;
-    if (now - this.lastRun < oneWeek) { // Execute the PoolsUpdate only once a week, or upon restart
+    if (now - this.lastRun < config.MEMPOOL.POOLS_UPDATE_DELAY) { // Execute the PoolsUpdate only once a week, or upon restart
       return;
     }
 
@@ -43,26 +54,26 @@ class PoolsUpdater {
         this.currentSha = await this.getShaFromDb();
       }
 
-      logger.debug(`pools-v2.json sha | Current: ${this.currentSha} | Github: ${githubSha}`);
+      logger.debug(`pools-v2.json sha | Current: ${this.currentSha} | Github: ${githubSha}`, this.tag);
       if (this.currentSha !== null && this.currentSha === githubSha) {
         return;
       }
 
       // See backend README for more details about the mining pools update process
       if (this.currentSha !== null && // If we don't have any mining pool, download it at least once
-        config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING !== true && // Automatic pools update is disabled
+        config.MEMPOOL.AUTOMATIC_POOLS_UPDATE !== true && // Automatic pools update is disabled
         !process.env.npm_config_update_pools // We're not manually updating mining pool
       ) {
-        logger.warn(`Updated mining pools data is available (${githubSha}) but AUTOMATIC_BLOCK_REINDEXING is disabled`);
-        logger.info(`You can update your mining pools using the --update-pools command flag. You may want to clear your nginx cache as well if applicable`);
+        logger.warn(`Updated mining pools data is available (${githubSha}) but AUTOMATIC_POOLS_UPDATE is disabled`, this.tag);
+        logger.info(`You can update your mining pools using the --update-pools command flag. You may want to clear your nginx cache as well if applicable`, this.tag);
         return;
       }
 
       const network = config.SOCKS5PROXY.ENABLED ? 'tor' : 'clearnet';
       if (this.currentSha === null) {
-        logger.info(`Downloading pools-v2.json for the first time from ${this.poolsUrl} over ${network}`, logger.tags.mining);
+        logger.info(`Downloading pools-v2.json for the first time from ${this.poolsUrl} over ${network}`, this.tag);
       } else {
-        logger.warn(`pools-v2.json is outdated, fetching latest from ${this.poolsUrl} over ${network}`, logger.tags.mining);
+        logger.warn(`pools-v2.json is outdated, fetching latest from ${this.poolsUrl} over ${network}`, this.tag);
       }
       const poolsJson = await this.query(this.poolsUrl);
       if (poolsJson === undefined) {
@@ -71,24 +82,24 @@ class PoolsUpdater {
       poolsParser.setMiningPools(poolsJson);
 
       if (config.DATABASE.ENABLED === false) { // Don't run db operations
-        logger.info(`Mining pools-v2.json (${githubSha}) import completed (no database)`);
+        logger.info(`Mining pools-v2.json (${githubSha}) import completed (no database)`, this.tag);
         return;
       }
 
       try {
         await DB.query('START TRANSACTION;');
-        await poolsParser.migratePoolsJson();
         await this.updateDBSha(githubSha);
+        await poolsParser.migratePoolsJson();
         await DB.query('COMMIT;');
       } catch (e) {
-        logger.err(`Could not migrate mining pools, rolling back. Exception: ${JSON.stringify(e)}`, logger.tags.mining);
+        logger.err(`Could not migrate mining pools, rolling back. Exception: ${JSON.stringify(e)}`, this.tag);
         await DB.query('ROLLBACK;');
       }
-      logger.info(`Mining pools-v2.json (${githubSha}) import completed`);
+      logger.info(`Mining pools-v2.json (${githubSha}) import completed`, this.tag);
 
     } catch (e) {
-      this.lastRun = now - (oneWeek - oneDay); // Try again in 24h instead of waiting next week
-      logger.err(`PoolsUpdater failed. Will try again in 24h. Exception: ${JSON.stringify(e)}`, logger.tags.mining);
+      this.lastRun = now - 600; // Try again in 10 minutes
+      logger.err(`PoolsUpdater failed. Will try again in 10 minutes. Exception: ${JSON.stringify(e)}`, this.tag);
     }
   }
 
@@ -102,7 +113,7 @@ class PoolsUpdater {
         await DB.query('DELETE FROM state where name="pools_json_sha"');
         await DB.query(`INSERT INTO state VALUES('pools_json_sha', NULL, '${githubSha}')`);
       } catch (e) {
-        logger.err('Cannot save github pools-v2.json sha into the db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
+        logger.err('Cannot save github pools-v2.json sha into the db. Reason: ' + (e instanceof Error ? e.message : e), this.tag);
       }
     }
   }
@@ -110,12 +121,12 @@ class PoolsUpdater {
   /**
    * Fetch our latest pools-v2.json sha from the db
    */
-  private async getShaFromDb(): Promise<string | null> {
+  public async getShaFromDb(): Promise<string | null> {
     try {
       const [rows]: any[] = await DB.query('SELECT string FROM state WHERE name="pools_json_sha"');
       return (rows.length > 0 ? rows[0].string : null);
     } catch (e) {
-      logger.err('Cannot fetch pools-v2.json sha from db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
+      logger.err('Cannot fetch pools-v2.json sha from db. Reason: ' + (e instanceof Error ? e.message : e), this.tag);
       return null;
     }
   }
@@ -134,7 +145,7 @@ class PoolsUpdater {
       }
     }
 
-    logger.err(`Cannot find "pools-v2.json" in git tree (${this.treeUrl})`, logger.tags.mining);
+    logger.err(`Cannot find "pools-v2.json" in git tree (${this.treeUrl})`, this.tag);
     return null;
   }
 
@@ -186,7 +197,7 @@ class PoolsUpdater {
         }
         return data.data;
       } catch (e) {
-        logger.err('Could not connect to Github. Reason: ' + (e instanceof Error ? e.message : e));
+        logger.err('Could not connect to Github. Reason: ' + (e instanceof Error ? e.message : e), this.tag);
         retry++;
       }
       await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL);

backend/src/tasks/price-feeds/free-currency-api.ts

@@ -76,7 +76,7 @@ class FreeCurrencyApi implements ConversionFeed {
   }
 
   public async $fetchConversionRates(date: string): Promise<ConversionRates> {
-    const response = await query(`${this.API_URL_PREFIX}historical?date=${date}&apikey=${this.API_KEY}`);
+    const response = await query(`${this.API_URL_PREFIX}historical?date=${date}&apikey=${this.API_KEY}`, true);
     if (response && response['data'] && (response['data'][date] || this.PAID)) {
       if (this.PAID) {
         response['data'] = this.convertData(response['data']);

backend/src/tasks/price-updater.ts

@@ -59,7 +59,7 @@ class PriceUpdater {
   private currencyConversionFeed: ConversionFeed | undefined;
   private newCurrencies: string[] = ['BGN', 'BRL', 'CNY', 'CZK', 'DKK', 'HKD', 'HRK', 'HUF', 'IDR', 'ILS', 'INR', 'ISK', 'KRW', 'MXN', 'MYR', 'NOK', 'NZD', 'PHP', 'PLN', 'RON', 'RUB', 'SEK', 'SGD', 'THB', 'TRY', 'ZAR'];
   private lastTimeConversionsRatesFetched: number = 0;
-  private latestConversionsRatesFromFeed: ConversionRates = {};
+  private latestConversionsRatesFromFeed: ConversionRates = { USD: -1 };
   private ratesChangedCallback: ((rates: ApiPrice) => void) | undefined;
 
   constructor() {
@@ -157,9 +157,9 @@ class PriceUpdater {
       try {
         this.latestConversionsRatesFromFeed = await this.currencyConversionFeed.$fetchLatestConversionRates();
         this.lastTimeConversionsRatesFetched = Math.round(new Date().getTime() / 1000);
-        logger.debug(`Fetched currencies conversion rates from external API: ${JSON.stringify(this.latestConversionsRatesFromFeed)}`);
+        logger.debug(`Fetched currencies conversion rates from conversions API: ${JSON.stringify(this.latestConversionsRatesFromFeed)}`);
       } catch (e) {
-        logger.err(`Cannot fetch conversion rates from the API. Reason: ${(e instanceof Error ? e.message : e)}`);
+        logger.err(`Cannot fetch conversion rates from conversions API. Reason: ${(e instanceof Error ? e.message : e)}`);
       }
     }
 
@@ -408,17 +408,17 @@ class PriceUpdater {
     try {
       const remainingQuota = await this.currencyConversionFeed?.$getQuota();
       if (remainingQuota['month']['remaining'] < 500) { // We need some calls left for the daily updates
-        logger.debug(`Not enough currency API credit to insert missing prices in ${priceTimesToFill.length} rows (${remainingQuota['month']['remaining']} calls left).`, logger.tags.mining);
+        logger.debug(`Not enough conversions API credit to insert missing prices in ${priceTimesToFill.length} rows (${remainingQuota['month']['remaining']} calls left).`, logger.tags.mining);
         this.additionalCurrenciesHistoryInserted = true; // Do not try again until next day
         return;
       }
     } catch (e) {
-      logger.err(`Cannot fetch currency API credit, insertion of missing prices aborted. Reason: ${(e instanceof Error ? e.message : e)}`);
+      logger.err(`Cannot fetch conversions API credit, insertion of missing prices aborted. Reason: ${(e instanceof Error ? e.message : e)}`);
       return;
     }
 
     this.additionalCurrenciesHistoryRunning = true;
-    logger.debug(`Fetching missing conversion rates from external API to fill ${priceTimesToFill.length} rows`, logger.tags.mining);
+    logger.debug(`Inserting missing historical conversion rates using conversions API to fill ${priceTimesToFill.length} rows`, logger.tags.mining);
 
     let conversionRates: { [timestamp: number]: ConversionRates } = {};
     let totalInserted = 0;
@@ -430,10 +430,23 @@ class PriceUpdater {
       const month = new Date(priceTime.time * 1000).getMonth();
       const yearMonthTimestamp = new Date(year, month, 1).getTime() / 1000;
       if (conversionRates[yearMonthTimestamp] === undefined) {
-        conversionRates[yearMonthTimestamp] = await this.currencyConversionFeed?.$fetchConversionRates(`${year}-${month + 1 < 10 ? `0${month + 1}` : `${month + 1}`}-01`) || { USD: -1 };
-        if (conversionRates[yearMonthTimestamp]['USD'] < 0) {
-          logger.err(`Cannot fetch conversion rates from the API for ${year}-${month + 1 < 10 ? `0${month + 1}` : `${month + 1}`}-01. Aborting insertion of missing prices.`, logger.tags.mining);
-          this.lastFailedHistoricalRun = Math.round(new Date().getTime() / 1000);
+        try {
+          if (year === new Date().getFullYear() && month === new Date().getMonth()) { // For rows in the current month, we use the latest conversion rates
+            conversionRates[yearMonthTimestamp] = this.latestConversionsRatesFromFeed;
+          } else {
+            conversionRates[yearMonthTimestamp] = await this.currencyConversionFeed?.$fetchConversionRates(`${year}-${month + 1 < 10 ? `0${month + 1}` : `${month + 1}`}-15`) || { USD: -1 };
+          }
+
+          if (conversionRates[yearMonthTimestamp]['USD'] < 0) {
+            throw new Error('Incorrect USD conversion rate');
+          }
+        } catch (e) {
+          if ((e instanceof Error ? e.message : '').includes('429')) { // Continue 60 seconds later if and only if error is 429
+            this.lastFailedHistoricalRun = Math.round(new Date().getTime() / 1000);
+            logger.info(`Got a 429 error from conversions API. This is expected to happen a few times during the initial historical price insertion, process will resume in 60 seconds.`, logger.tags.mining);
+          } else {
+            logger.err(`Cannot fetch conversion rates from conversions API for ${year}-${month + 1 < 10 ? `0${month + 1}` : `${month + 1}`}-01, trying again next day. Error: ${(e instanceof Error ? e.message : e)}`, logger.tags.mining);
+          }
           break;
         }
       }

backend/src/utils/api.ts

@@ -0,0 +1,9 @@
+import { Request, Response } from 'express';
+
+export function handleError(req: Request, res: Response, statusCode: number, errorMessage: string | unknown): void {
+  if (req.accepts('json')) {
+    res.status(statusCode).json({ error: errorMessage });
+  } else {
+    res.status(statusCode).send(errorMessage);
+  }
+}

backend/src/utils/axios-query.ts

@@ -5,7 +5,7 @@ import config from '../config';
 import logger from '../logger';
 import * as https from 'https';
 
-export async function query(path): Promise<object | undefined> {
+export async function query(path, throwOnFail: boolean = false): Promise<object | undefined> {
  type axiosOptions = {
    headers: {
      'User-Agent': string
@@ -21,6 +21,7 @@ export async function query(path): Promise<object | undefined> {
    timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000
  };
  let retry = 0;
+ let lastError: any = null;
 
  while (retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
    try {
@@ -50,6 +51,7 @@ export async function query(path): Promise<object | undefined> {
      }
      return data.data;
    } catch (e) {
+     lastError = e;
      logger.warn(`Could not connect to ${path} (Attempt ${retry + 1}/${config.MEMPOOL.EXTERNAL_MAX_RETRY}). Reason: ` + (e instanceof Error ? e.message : e));
      retry++;
    }
@@ -59,5 +61,10 @@ export async function query(path): Promise<object | undefined> {
  }
 
  logger.err(`Could not connect to ${path}. All ${config.MEMPOOL.EXTERNAL_MAX_RETRY} attempts failed`);
+
+ if (throwOnFail && lastError) {
+    throw lastError;
+  }
+
  return undefined;
 }

backend/src/utils/bitcoin-script.ts

@@ -158,7 +158,7 @@ export function parseMultisigScript(script: string): void | { m: number, n: numb
   if (!opN) {
     return;
   }
-  if (!opN.startsWith('OP_PUSHNUM_')) {
+  if (opN !== 'OP_0' && !opN.startsWith('OP_PUSHNUM_')) {
     return;
   }
   const n = parseInt(opN.match(/[0-9]+/)?.[0] || '', 10);
@@ -178,7 +178,7 @@ export function parseMultisigScript(script: string): void | { m: number, n: numb
   if (!opM) {
     return;
   }
-  if (!opM.startsWith('OP_PUSHNUM_')) {
+  if (opM !== 'OP_0' && !opM.startsWith('OP_PUSHNUM_')) {
     return;
   }
   const m = parseInt(opM.match(/[0-9]+/)?.[0] || '', 10);
@@ -200,4 +200,28 @@ export function getVarIntLength(n: number): number {
   } else {
     return 9;
   }
+}
+
+/** Extracts miner names from a DATUM coinbase transaction */
+export function parseDATUMTemplateCreator(coinbaseRaw: string): string[] | null {
+  let bytes: number[] = [];
+  for (let c = 0; c < coinbaseRaw.length; c += 2) {
+      bytes.push(parseInt(coinbaseRaw.slice(c, c + 2), 16));
+  }
+
+  // Skip block height
+  let tagLengthByte = 1 + bytes[0];
+
+  let tagsLength = bytes[tagLengthByte];
+  if (tagsLength == 0x4c) {
+    tagLengthByte += 1;
+    tagsLength = bytes[tagLengthByte];
+  }
+
+  const tagStart = tagLengthByte + 1;
+  const tags = bytes.slice(tagStart, tagStart + tagsLength);
+  let tagString = String.fromCharCode(...tags);
+  tagString = tagString.replace('\x00', '');
+
+  return tagString.split('\x0f').map((name) => name.replace(/[^a-zA-Z0-9 ]/g, ''));
 }

backend/src/utils/file-read.ts

@@ -0,0 +1,58 @@
+import * as fs from 'fs';
+import logger from '../logger';
+import config from '../config';
+
+function readFile(filePath: string, bufferSize?: number): string[] {
+  const fileSize = fs.statSync(filePath).size;
+  const chunkSize = bufferSize || fileSize;
+  const fileDescriptor = fs.openSync(filePath, 'r');
+  const buffer = Buffer.alloc(chunkSize);
+
+  fs.readSync(fileDescriptor, buffer, 0, chunkSize, fileSize - chunkSize);
+  fs.closeSync(fileDescriptor);
+
+  const lines = buffer.toString('utf8', 0, chunkSize).split('\n');
+  return lines;
+}
+
+function extractDateFromLogLine(line: string): number | undefined {
+  // Extract time from log: "2021-08-31T12:34:56Z" or "2021-08-31T12:34:56.123456Z"
+  const dateMatch = line.match(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{6})?Z/);
+  if (!dateMatch) {
+    return undefined;
+  }
+
+  const dateStr = dateMatch[0];
+  const date = new Date(dateStr);
+  let timestamp = Math.floor(date.getTime() / 1000); // Remove decimal (microseconds are added later)
+
+  const timePart = dateStr.split('T')[1];
+  const microseconds = timePart.split('.')[1] || '';
+
+  if (!microseconds) {
+    return timestamp;
+  }
+
+  return parseFloat(timestamp + '.' + microseconds);
+}
+
+export function getRecentFirstSeen(hash: string): number | undefined {
+  const debugLogPath = config.CORE_RPC.DEBUG_LOG_PATH;
+  if (debugLogPath) {
+    try {
+      // Read the last few lines of debug.log
+      const lines = readFile(debugLogPath, 2048);
+
+      for (let i = lines.length - 1; i >= 0; i--) {
+        const line = lines[i];
+        if (line && line.includes(`Saw new header hash=${hash}`)) {
+          return extractDateFromLogLine(line);
+        }
+      }
+    } catch (e) {
+      logger.err(`Cannot parse block first seen time from Core logs. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  return undefined;
+}

contributors/bitcoinmechanic.txt

@@ -0,0 +1,3 @@
+I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of January 25, 2022.
+
+Signed: bitcoinmechanic

contributors/hans-crypto.txt

@@ -0,0 +1,3 @@
+I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of May 21, 2024.
+
+Signed: hans-crypto

contributors/henrialb.txt

@@ -0,0 +1,3 @@
+I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of April 12, 2024.
+
+Signed: henrialb

contributors/jlopp.txt

@@ -0,0 +1,3 @@
+I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of July 12, 2024.
+
+Signed: jlopp

contributors/mackalex.txt

@@ -0,0 +1,3 @@
+I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of June 18th, 2024.
+
+Signed: mackalex

contributors/svrgnty.txt

@@ -0,0 +1,3 @@
+I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of July 9, 2024.
+
+Signed: svrgnty

docker/README.md

@@ -106,9 +106,10 @@ Below we list all settings from `mempool-config.json` and the corresponding over
     "EXTERNAL_ASSETS": [],
     "STDOUT_LOG_MIN_PRIORITY": "info",
     "INDEXING_BLOCKS_AMOUNT": false,
-    "AUTOMATIC_BLOCK_REINDEXING": false,
+    "AUTOMATIC_POOLS_UPDATE": false,
     "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json",
     "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
+    "POOLS_UPDATE_DELAY": 604800,
     "CPFP_INDEXING": false,
     "MAX_BLOCKS_BULK_QUERY": 0,
     "DISK_CACHE_BLOCK_INTERVAL": 6,
@@ -137,9 +138,10 @@ Corresponding `docker-compose.yml` overrides:
       MEMPOOL_EXTERNAL_ASSETS: ""
       MEMPOOL_STDOUT_LOG_MIN_PRIORITY: ""
       MEMPOOL_INDEXING_BLOCKS_AMOUNT: ""
-      MEMPOOL_AUTOMATIC_BLOCK_REINDEXING: ""
+      MEMPOOL_AUTOMATIC_POOLS_UPDATE: ""
       MEMPOOL_POOLS_JSON_URL: ""
       MEMPOOL_POOLS_JSON_TREE_URL: ""
+      MEMPOOL_POOLS_UPDATE_DELAY: ""
       MEMPOOL_CPFP_INDEXING: ""
       MEMPOOL_MAX_BLOCKS_BULK_QUERY: ""
       MEMPOOL_DISK_CACHE_BLOCK_INTERVAL: ""

docker/backend/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:20.12.0-buster-slim AS builder
+FROM node:20.15.0-buster-slim AS builder
 
 ARG commitHash
 ENV MEMPOOL_COMMIT_HASH=${commitHash}
@@ -24,7 +24,7 @@ RUN npm install --omit=dev --omit=optional
 WORKDIR /build
 RUN npm run package
 
-FROM node:20.12.0-buster-slim
+FROM node:20.15.0-buster-slim
 
 WORKDIR /backend
 

docker/backend/mempool-config.json

@@ -25,7 +25,7 @@
     "INDEXING_BLOCKS_AMOUNT": __MEMPOOL_INDEXING_BLOCKS_AMOUNT__,
     "BLOCKS_SUMMARIES_INDEXING": __MEMPOOL_BLOCKS_SUMMARIES_INDEXING__,
     "GOGGLES_INDEXING": __MEMPOOL_GOGGLES_INDEXING__,
-    "AUTOMATIC_BLOCK_REINDEXING": __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__,
+    "AUTOMATIC_POOLS_UPDATE": __MEMPOOL_AUTOMATIC_POOLS_UPDATE__,
     "AUDIT": __MEMPOOL_AUDIT__,
     "RUST_GBT": __MEMPOOL_RUST_GBT__,
     "LIMIT_GBT": __MEMPOOL_LIMIT_GBT__,
@@ -36,6 +36,7 @@
     "ALLOW_UNREACHABLE": __MEMPOOL_ALLOW_UNREACHABLE__,
     "POOLS_JSON_TREE_URL": "__MEMPOOL_POOLS_JSON_TREE_URL__",
     "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__",
+    "POOLS_UPDATE_DELAY": __MEMPOOL_POOLS_UPDATE_DELAY__,
     "PRICE_UPDATES_PER_HOUR": __MEMPOOL_PRICE_UPDATES_PER_HOUR__,
     "MAX_TRACKED_ADDRESSES": __MEMPOOL_MAX_TRACKED_ADDRESSES__
   },
@@ -46,7 +47,8 @@
     "PASSWORD": "__CORE_RPC_PASSWORD__",
     "TIMEOUT": __CORE_RPC_TIMEOUT__,
     "COOKIE": __CORE_RPC_COOKIE__,
-    "COOKIE_PATH": "__CORE_RPC_COOKIE_PATH__"
+    "COOKIE_PATH": "__CORE_RPC_COOKIE_PATH__",
+    "DEBUG_LOG_PATH": "__CORE_RPC_DEBUG_LOG_PATH__"
   },
   "ELECTRUM": {
     "HOST": "__ELECTRUM_HOST__",
@@ -60,7 +62,8 @@
     "RETRY_UNIX_SOCKET_AFTER": __ESPLORA_RETRY_UNIX_SOCKET_AFTER__,
     "REQUEST_TIMEOUT": __ESPLORA_REQUEST_TIMEOUT__,
     "FALLBACK_TIMEOUT": __ESPLORA_FALLBACK_TIMEOUT__,
-    "FALLBACK": __ESPLORA_FALLBACK__
+    "FALLBACK": __ESPLORA_FALLBACK__,
+    "MAX_BEHIND_TIP": __ESPLORA_MAX_BEHIND_TIP__
   },
   "SECOND_CORE_RPC": {
     "HOST": "__SECOND_CORE_RPC_HOST__",
@@ -137,12 +140,18 @@
     "ENABLED": __REPLICATION_ENABLED__,
     "AUDIT": __REPLICATION_AUDIT__,
     "AUDIT_START_HEIGHT": __REPLICATION_AUDIT_START_HEIGHT__,
+    "STATISTICS": __REPLICATION_STATISTICS__,
+    "STATISTICS_START_TIME": __REPLICATION_STATISTICS_START_TIME__,
     "SERVERS": __REPLICATION_SERVERS__
   },
   "MEMPOOL_SERVICES": {
     "API": "__MEMPOOL_SERVICES_API__",
     "ACCELERATIONS": __MEMPOOL_SERVICES_ACCELERATIONS__
   },
+  "STRATUM": {
+    "ENABLED": __STRATUM_ENABLED__,
+    "API": "__STRATUM_API__"
+  },
   "REDIS": {
     "ENABLED": __REDIS_ENABLED__,
     "UNIX_SOCKET_PATH": "__REDIS_UNIX_SOCKET_PATH__",

docker/backend/start.sh

@@ -26,11 +26,12 @@ __MEMPOOL_EXTERNAL_MAX_RETRY__=${MEMPOOL_EXTERNAL_MAX_RETRY:=1}
 __MEMPOOL_EXTERNAL_RETRY_INTERVAL__=${MEMPOOL_EXTERNAL_RETRY_INTERVAL:=0}
 __MEMPOOL_USER_AGENT__=${MEMPOOL_USER_AGENT:=mempool}
 __MEMPOOL_STDOUT_LOG_MIN_PRIORITY__=${MEMPOOL_STDOUT_LOG_MIN_PRIORITY:=info}
-__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__=${MEMPOOL_AUTOMATIC_BLOCK_REINDEXING:=false}
+__MEMPOOL_AUTOMATIC_POOLS_UPDATE__=${MEMPOOL_AUTOMATIC_POOLS_UPDATE:=false}
 __MEMPOOL_POOLS_JSON_URL__=${MEMPOOL_POOLS_JSON_URL:=https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json}
 __MEMPOOL_POOLS_JSON_TREE_URL__=${MEMPOOL_POOLS_JSON_TREE_URL:=https://api.github.com/repos/mempool/mining-pools/git/trees/master}
+__MEMPOOL_POOLS_UPDATE_DELAY__=${MEMPOOL_POOLS_UPDATE_DELAY:=604800}
 __MEMPOOL_AUDIT__=${MEMPOOL_AUDIT:=false}
-__MEMPOOL_RUST_GBT__=${MEMPOOL_RUST_GBT:=false}
+__MEMPOOL_RUST_GBT__=${MEMPOOL_RUST_GBT:=true}
 __MEMPOOL_LIMIT_GBT__=${MEMPOOL_LIMIT_GBT:=false}
 __MEMPOOL_CPFP_INDEXING__=${MEMPOOL_CPFP_INDEXING:=false}
 __MEMPOOL_MAX_BLOCKS_BULK_QUERY__=${MEMPOOL_MAX_BLOCKS_BULK_QUERY:=0}
@@ -48,6 +49,7 @@ __CORE_RPC_PASSWORD__=${CORE_RPC_PASSWORD:=mempool}
 __CORE_RPC_TIMEOUT__=${CORE_RPC_TIMEOUT:=60000}
 __CORE_RPC_COOKIE__=${CORE_RPC_COOKIE:=false}
 __CORE_RPC_COOKIE_PATH__=${CORE_RPC_COOKIE_PATH:=""}
+__CORE_RPC_DEBUG_LOG_PATH__=${CORE_RPC_DEBUG_LOG_PATH:=""}
 
 # ELECTRUM
 __ELECTRUM_HOST__=${ELECTRUM_HOST:=127.0.0.1}
@@ -62,6 +64,7 @@ __ESPLORA_RETRY_UNIX_SOCKET_AFTER__=${ESPLORA_RETRY_UNIX_SOCKET_AFTER:=30000}
 __ESPLORA_REQUEST_TIMEOUT__=${ESPLORA_REQUEST_TIMEOUT:=5000}
 __ESPLORA_FALLBACK_TIMEOUT__=${ESPLORA_FALLBACK_TIMEOUT:=5000}
 __ESPLORA_FALLBACK__=${ESPLORA_FALLBACK:=[]}
+__ESPLORA_MAX_BEHIND_TIP__=${ESPLORA_MAX_BEHIND_TIP:=2}
 
 # SECOND_CORE_RPC
 __SECOND_CORE_RPC_HOST__=${SECOND_CORE_RPC_HOST:=127.0.0.1}
@@ -138,15 +141,21 @@ __MAXMIND_GEOIP2_ISP__=${MAXMIND_GEOIP2_ISP:=""}
 __REPLICATION_ENABLED__=${REPLICATION_ENABLED:=false}
 __REPLICATION_AUDIT__=${REPLICATION_AUDIT:=false}
 __REPLICATION_AUDIT_START_HEIGHT__=${REPLICATION_AUDIT_START_HEIGHT:=774000}
+__REPLICATION_STATISTICS__=${REPLICATION_STATISTICS:=false}
+__REPLICATION_STATISTICS_START_TIME__=${REPLICATION_STATISTICS_START_TIME:=1481932800}
 __REPLICATION_SERVERS__=${REPLICATION_SERVERS:=[]}
 
 # MEMPOOL_SERVICES
-__MEMPOOL_SERVICES_API__=${MEMPOOL_SERVICES_API:=""}
+__MEMPOOL_SERVICES_API__=${MEMPOOL_SERVICES_API:="https://mempool.space/api/v1/services"}
 __MEMPOOL_SERVICES_ACCELERATIONS__=${MEMPOOL_SERVICES_ACCELERATIONS:=false}
 
+# STRATUM
+__STRATUM_ENABLED__=${STRATUM_ENABLED:=false}
+__STRATUM_API__=${STRATUM_API:="http://localhost:1234"}
+
 # REDIS
 __REDIS_ENABLED__=${REDIS_ENABLED:=false}
-__REDIS_UNIX_SOCKET_PATH__=${REDIS_UNIX_SOCKET_PATH:=true}
+__REDIS_UNIX_SOCKET_PATH__=${REDIS_UNIX_SOCKET_PATH:=""}
 __REDIS_BATCH_QUERY_BASE_SIZE__=${REDIS_BATCH_QUERY_BASE_SIZE:=5000}
 
 # FIAT_PRICE
@@ -181,9 +190,10 @@ sed -i "s!__MEMPOOL_EXTERNAL_MAX_RETRY__!${__MEMPOOL_EXTERNAL_MAX_RETRY__}!g" me
 sed -i "s!__MEMPOOL_EXTERNAL_RETRY_INTERVAL__!${__MEMPOOL_EXTERNAL_RETRY_INTERVAL__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_USER_AGENT__!${__MEMPOOL_USER_AGENT__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__!${__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__}!g" mempool-config.json
-sed -i "s!__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__!${__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__}!g" mempool-config.json
+sed -i "s!__MEMPOOL_AUTOMATIC_POOLS_UPDATE__!${__MEMPOOL_AUTOMATIC_POOLS_UPDATE__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_POOLS_JSON_URL__!${__MEMPOOL_POOLS_JSON_URL__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_POOLS_JSON_TREE_URL__!${__MEMPOOL_POOLS_JSON_TREE_URL__}!g" mempool-config.json
+sed -i "s!__MEMPOOL_POOLS_UPDATE_DELAY__!${__MEMPOOL_POOLS_UPDATE_DELAY__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_AUDIT__!${__MEMPOOL_AUDIT__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_RUST_GBT__!${__MEMPOOL_RUST_GBT__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_LIMIT_GBT__!${__MEMPOOL_LIMIT_GBT__}!g" mempool-config.json
@@ -202,6 +212,7 @@ sed -i "s!__CORE_RPC_PASSWORD__!${__CORE_RPC_PASSWORD__}!g" mempool-config.json
 sed -i "s!__CORE_RPC_TIMEOUT__!${__CORE_RPC_TIMEOUT__}!g" mempool-config.json
 sed -i "s!__CORE_RPC_COOKIE__!${__CORE_RPC_COOKIE__}!g" mempool-config.json
 sed -i "s!__CORE_RPC_COOKIE_PATH__!${__CORE_RPC_COOKIE_PATH__}!g" mempool-config.json
+sed -i "s!__CORE_RPC_DEBUG_LOG_PATH__!${__CORE_RPC_DEBUG_LOG_PATH__}!g" mempool-config.json
 
 sed -i "s!__ELECTRUM_HOST__!${__ELECTRUM_HOST__}!g" mempool-config.json
 sed -i "s!__ELECTRUM_PORT__!${__ELECTRUM_PORT__}!g" mempool-config.json
@@ -214,6 +225,7 @@ sed -i "s!__ESPLORA_RETRY_UNIX_SOCKET_AFTER__!${__ESPLORA_RETRY_UNIX_SOCKET_AFTE
 sed -i "s!__ESPLORA_REQUEST_TIMEOUT__!${__ESPLORA_REQUEST_TIMEOUT__}!g" mempool-config.json
 sed -i "s!__ESPLORA_FALLBACK_TIMEOUT__!${__ESPLORA_FALLBACK_TIMEOUT__}!g" mempool-config.json
 sed -i "s!__ESPLORA_FALLBACK__!${__ESPLORA_FALLBACK__}!g" mempool-config.json
+sed -i "s!__ESPLORA_MAX_BEHIND_TIP__!${__ESPLORA_MAX_BEHIND_TIP__}!g" mempool-config.json
 
 sed -i "s!__SECOND_CORE_RPC_HOST__!${__SECOND_CORE_RPC_HOST__}!g" mempool-config.json
 sed -i "s!__SECOND_CORE_RPC_PORT__!${__SECOND_CORE_RPC_PORT__}!g" mempool-config.json
@@ -284,12 +296,18 @@ sed -i "s!__MAXMIND_GEOIP2_ISP__!${__MAXMIND_GEOIP2_ISP__}!g" mempool-config.jso
 sed -i "s!__REPLICATION_ENABLED__!${__REPLICATION_ENABLED__}!g" mempool-config.json
 sed -i "s!__REPLICATION_AUDIT__!${__REPLICATION_AUDIT__}!g" mempool-config.json
 sed -i "s!__REPLICATION_AUDIT_START_HEIGHT__!${__REPLICATION_AUDIT_START_HEIGHT__}!g" mempool-config.json
+sed -i "s!__REPLICATION_STATISTICS__!${__REPLICATION_STATISTICS__}!g" mempool-config.json
+sed -i "s!__REPLICATION_STATISTICS_START_TIME__!${__REPLICATION_STATISTICS_START_TIME__}!g" mempool-config.json
 sed -i "s!__REPLICATION_SERVERS__!${__REPLICATION_SERVERS__}!g" mempool-config.json
 
 # MEMPOOL_SERVICES
 sed -i "s!__MEMPOOL_SERVICES_API__!${__MEMPOOL_SERVICES_API__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_SERVICES_ACCELERATIONS__!${__MEMPOOL_SERVICES_ACCELERATIONS__}!g" mempool-config.json
 
+# STRATUM
+sed -i "s!__STRATUM_ENABLED__!${__STRATUM_ENABLED__}!g" mempool-config.json
+sed -i "s!__STRATUM_API__!${__STRATUM_API__}!g" mempool-config.json
+
 # REDIS
 sed -i "s!__REDIS_ENABLED__!${__REDIS_ENABLED__}!g" mempool-config.json
 sed -i "s!__REDIS_UNIX_SOCKET_PATH__!${__REDIS_UNIX_SOCKET_PATH__}!g" mempool-config.json

docker/docker-compose.yml

@@ -11,10 +11,14 @@ services:
     stop_grace_period: 1m
     command: "./wait-for db:3306 --timeout=720 -- nginx -g 'daemon off;'"
     ports:
-      - 80:8080
+      - 8080:8080
   api:
     environment:
-      MEMPOOL_BACKEND: "none"
+      MEMPOOL_BACKEND: "electrum"
+      ELECTRUM_HOST: "172.27.0.1"
+      ELECTRUM_PORT: "50001"
+      ELECTRUM_TLS_ENABLED: "false"
+
       CORE_RPC_HOST: "172.27.0.1"
       CORE_RPC_PORT: "8332"
       CORE_RPC_USERNAME: "mempool"

docker/frontend/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:20.12.0-buster-slim AS builder
+FROM node:20.15.0-buster-slim AS builder
 
 ARG commitHash
 ENV DOCKER_COMMIT_HASH=${commitHash}
@@ -13,7 +13,7 @@ RUN npm install --omit=dev --omit=optional
 
 RUN npm run build
 
-FROM nginx:1.25.4-alpine
+FROM nginx:1.27.0-alpine
 
 WORKDIR /patch
 

docker/frontend/entrypoint.sh

@@ -16,7 +16,9 @@ fi
 
 # Runtime overrides - read env vars defined in docker compose
 
+__MAINNET_ENABLED__=${MAINNET_ENABLED:=true}
 __TESTNET_ENABLED__=${TESTNET_ENABLED:=false}
+__TESTNET4_ENABLED__=${TESTNET_ENABLED:=false}
 __SIGNET_ENABLED__=${SIGNET_ENABLED:=false}
 __LIQUID_ENABLED__=${LIQUID_ENABLED:=false}
 __LIQUID_TESTNET_ENABLED__=${LIQUID_TESTNET_ENABLED:=false}
@@ -28,6 +30,7 @@ __NGINX_PORT__=${NGINX_PORT:=8999}
 __BLOCK_WEIGHT_UNITS__=${BLOCK_WEIGHT_UNITS:=4000000}
 __MEMPOOL_BLOCKS_AMOUNT__=${MEMPOOL_BLOCKS_AMOUNT:=8}
 __BASE_MODULE__=${BASE_MODULE:=mempool}
+__ROOT_NETWORK__=${ROOT_NETWORK:=}
 __MEMPOOL_WEBSITE_URL__=${MEMPOOL_WEBSITE_URL:=https://mempool.space}
 __LIQUID_WEBSITE_URL__=${LIQUID_WEBSITE_URL:=https://liquid.network}
 __MINING_DASHBOARD__=${MINING_DASHBOARD:=true}
@@ -37,12 +40,17 @@ __MAINNET_BLOCK_AUDIT_START_HEIGHT__=${MAINNET_BLOCK_AUDIT_START_HEIGHT:=0}
 __TESTNET_BLOCK_AUDIT_START_HEIGHT__=${TESTNET_BLOCK_AUDIT_START_HEIGHT:=0}
 __SIGNET_BLOCK_AUDIT_START_HEIGHT__=${SIGNET_BLOCK_AUDIT_START_HEIGHT:=0}
 __ACCELERATOR__=${ACCELERATOR:=false}
+__ACCELERATOR_BUTTON__=${ACCELERATOR_BUTTON:=true}
+__SERVICES_API__=${SERVICES_API:=https://mempool.space/api/v1/services}
 __PUBLIC_ACCELERATIONS__=${PUBLIC_ACCELERATIONS:=false}
 __HISTORICAL_PRICE__=${HISTORICAL_PRICE:=true}
 __ADDITIONAL_CURRENCIES__=${ADDITIONAL_CURRENCIES:=false}
+__STRATUM_ENABLED__=${STRATUM_ENABLED:=false}
 
 # Export as environment variables to be used by envsubst
+export __MAINNET_ENABLED__
 export __TESTNET_ENABLED__
+export __TESTNET4_ENABLED__
 export __SIGNET_ENABLED__
 export __LIQUID_ENABLED__
 export __LIQUID_TESTNET_ENABLED__
@@ -54,6 +62,7 @@ export __NGINX_PORT__
 export __BLOCK_WEIGHT_UNITS__
 export __MEMPOOL_BLOCKS_AMOUNT__
 export __BASE_MODULE__
+export __ROOT_NETWORK__
 export __MEMPOOL_WEBSITE_URL__
 export __LIQUID_WEBSITE_URL__
 export __MINING_DASHBOARD__
@@ -63,9 +72,12 @@ export __MAINNET_BLOCK_AUDIT_START_HEIGHT__
 export __TESTNET_BLOCK_AUDIT_START_HEIGHT__
 export __SIGNET_BLOCK_AUDIT_START_HEIGHT__
 export __ACCELERATOR__
+export __ACCELERATOR_BUTTON__
+export __SERVICES_API__
 export __PUBLIC_ACCELERATIONS__
 export __HISTORICAL_PRICE__
 export __ADDITIONAL_CURRENCIES__
+export __STRATUM_ENABLED__
 
 folder=$(find /var/www/mempool -name "config.js" | xargs dirname)
 echo ${folder}

docker/init.sh

@@ -13,8 +13,8 @@ localhostIP="127.0.0.1"
 cp ./docker/frontend/* ./frontend
 cp ./nginx.conf ./frontend/
 cp ./nginx-mempool.conf ./frontend/
-sed -i"" -e "s/${localhostIP}:80/0.0.0.0:__MEMPOOL_FRONTEND_HTTP_PORT__/g" ./frontend/nginx.conf
-sed -i"" -e "s/${localhostIP}/0.0.0.0/g" ./frontend/nginx.conf
+# sed -i"" -e "s/${localhostIP}:80/0.0.0.0:__MEMPOOL_FRONTEND_HTTP_PORT__/g" ./frontend/nginx.conf
+# sed -i"" -e "s/${localhostIP}/0.0.0.0/g" ./frontend/nginx.conf
 sed -i"" -e "s/user nobody;//g" ./frontend/nginx.conf
 sed -i"" -e "s!/etc/nginx/nginx-mempool.conf!/etc/nginx/conf.d/nginx-mempool.conf!g" ./frontend/nginx.conf
 sed -i"" -e "s/${localhostIP}:8999/__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__:__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__/g" ./frontend/nginx-mempool.conf

frontend/.eslintrc

@@ -34,6 +34,8 @@
     "prefer-rest-params": 1,
     "quotes": [1, "single", { "allowTemplateLiterals": true }],
     "semi": 1,
-    "eqeqeq": 1
+    "curly": [1, "all"],
+    "eqeqeq": 1,
+    "no-trailing-spaces": 1
   }
 }

frontend/.gitignore

@@ -63,6 +63,7 @@ src/resources/pools.json
 src/resources/mining-pools/*
 src/resources/**/*.mp4
 src/resources/**/*.vtt
+src/resources/customize.js
 
 # environment config
 mempool-frontend-config.json

frontend/README.md

@@ -33,7 +33,7 @@ $ npm run config:defaults:liquid
 
 ### 3. Run the Frontend
 
-_Make sure to use Node.js 16.10 and npm 7._
+_Make sure to use Node.js 20.x and npm 9.x or newer._
 
 Install project dependencies and run the frontend server:
 
@@ -70,7 +70,7 @@ Set up the [Mempool backend](../backend/) first, if you haven't already.
 
 ### 1. Build the Frontend
 
-_Make sure to use Node.js 16.10 and npm 7._
+_Make sure to use Node.js 20.x and npm 9.x or newer._
 
 Build the frontend:
 

frontend/angular.json

@@ -54,6 +54,10 @@
             "translation": "src/locale/messages.fr.xlf",
             "baseHref": "/fr/"
           },
+          "hr": {
+            "translation": "src/locale/messages.hr.xlf",
+            "baseHref": "/hr/"
+          },
           "ja": {
             "translation": "src/locale/messages.ja.xlf",
             "baseHref": "/ja/"
@@ -166,6 +170,7 @@
               "src/resources",
               "src/robots.txt",
               "src/config.js",
+              "src/customize.js",
               "src/config.template.js"
             ],
             "styles": [
@@ -180,6 +185,11 @@
                 "bundleName": "wiz",
                 "inject": false
               },
+              {
+                "input": "src/theme-bukele.scss",
+                "bundleName": "bukele",
+                "inject": false
+              },
               "node_modules/@fortawesome/fontawesome-svg-core/styles.css"
             ],
             "vendorChunk": true,

frontend/custom-bitb-config.json

@@ -0,0 +1,48 @@
+{
+  "theme": "wiz",
+  "enterprise": "bitb",
+  "branding": {
+    "name": "bitb",
+    "title": "BITB",
+    "site_id": 20,
+    "header_img": "/resources/bitblogo.svg",
+    "footer_img": "/resources/bitblogo.svg"
+  },
+  "dashboard": {
+    "widgets": [
+      {
+        "component": "fees",
+        "mobileOrder": 4
+      },
+      {
+        "component": "walletBalance",
+        "mobileOrder": 1,
+        "props": {
+          "wallet": "BITB"
+        }
+      },
+      {
+        "component": "goggles",
+        "mobileOrder": 5
+      },
+      {
+        "component": "wallet",
+        "mobileOrder": 2,
+        "props": {
+          "wallet": "BITB",
+          "period": "all"
+        }
+      },
+      {
+        "component": "blocks"
+      },
+      {
+        "component": "walletTransactions",
+        "mobileOrder": 3,
+        "props": {
+          "wallet": "BITB"
+        }
+      }
+    ]
+  }
+}

frontend/custom-meta-config.json

@@ -0,0 +1,51 @@
+{
+  "theme": "contrast",
+  "enterprise": "meta",
+  "branding": {
+    "name": "metaplanet",
+    "title": "Metaplanet",
+    "site_id": 21,
+    "header_img": "/resources/metalogo.svg",
+    "footer_img": "/resources/metalogo.svg"
+  },
+  "dashboard": {
+    "widgets": [
+      {
+        "component": "fees",
+        "mobileOrder": 4
+      },
+      {
+        "component": "walletBalance",
+        "mobileOrder": 1,
+        "props": {
+          "wallet": "3350"
+        }
+      },
+      {
+        "component": "twitter",
+        "mobileOrder": 5,
+        "props": {
+          "handle": "Metaplanet_JP"
+        }
+      },
+      {
+        "component": "wallet",
+        "mobileOrder": 2,
+        "props": {
+          "wallet": "3350",
+          "period": "all"
+        }
+      },
+      {
+        "component": "blocks"
+      },
+      {
+        "component": "walletTransactions",
+        "mobileOrder": 3,
+        "props": {
+          "wallet": "3350"
+        }
+      }
+    ]
+  }
+}

frontend/custom-sv-config.json

@@ -0,0 +1,52 @@
+{
+  "theme": "bukele",
+  "enterprise": "onbtc",
+  "branding": {
+    "name": "onbtc",
+    "title": "Bitcoin Office",
+    "site_id": 19,
+    "header_img": "/resources/onbtclogo.svg",
+    "footer_img": "/resources/onbtclogo.svg",
+    "rounded_corner": true
+  },
+  "dashboard": {
+    "widgets": [
+      {
+        "component": "fees",
+        "mobileOrder": 4
+      },
+      {
+        "component": "balance",
+        "mobileOrder": 1,
+        "props": {
+          "address": "32ixEdVJWo3kmvJGMTZq5jAQVZZeuwnqzo"
+        }
+      },
+      {
+        "component": "twitter",
+        "mobileOrder": 5,
+        "props": {
+          "handle": "nayibbukele"
+        }
+      },
+      {
+        "component": "address",
+        "mobileOrder": 2,
+        "props": {
+          "address": "32ixEdVJWo3kmvJGMTZq5jAQVZZeuwnqzo",
+          "period": "1m"
+        }
+      },
+      {
+        "component": "blocks"
+      },
+      {
+        "component": "addressTransactions",
+        "mobileOrder": 3,
+        "props": {
+          "address": "32ixEdVJWo3kmvJGMTZq5jAQVZZeuwnqzo"
+        }
+      }
+    ]
+  }
+}

frontend/cypress/e2e/liquid/liquid.spec.ts

@@ -45,6 +45,7 @@ describe('Liquid', () => {
 
     it('loads a specific block page', () => {
       cy.visit(`${basePath}/block/7e1369a23a5ab861e7bdede2aadcccae4ea873ffd9caf11c7c5541eb5bcdff54`);
+      cy.get('.pagination').scrollIntoView({ offset: { top: 200, left: 0 } });
       cy.waitForSkeletonGone();
     });
 
@@ -71,20 +72,6 @@ describe('Liquid', () => {
       });
     });
 
-    it('renders unconfidential addresses correctly on mobile', () => {
-      cy.viewport('iphone-6');
-      cy.visit(`${basePath}/address/ex1qqmmjdwrlg59c8q4l75sj6wedjx57tj5grt8pat`);
-      cy.waitForSkeletonGone();
-      //TODO: Add proper IDs for these selectors
-      const firstRowSelector = '.container-xl > :nth-child(3) > div > :nth-child(1) > .table > tbody';
-      const thirdRowSelector = '.container-xl > :nth-child(3) > div > :nth-child(3)';
-      cy.get(firstRowSelector).invoke('css', 'width').then(firstRowWidth => {
-        cy.get(thirdRowSelector).invoke('css', 'width').then(thirdRowWidth => {
-          expect(parseInt(firstRowWidth)).to.be.lessThan(parseInt(thirdRowWidth));
-        });
-      });
-    });
-
     describe('peg in/peg out', () => {
       it('loads peg in addresses', () => {
         cy.visit(`${basePath}/tx/fe764f7bedfc2a37b29d9c8aef67d64a57d253a6b11c5a55555cfd5826483a58`);

frontend/cypress/e2e/liquidtestnet/liquidtestnet.spec.ts

@@ -46,7 +46,8 @@ describe('Liquid Testnet', () => {
     });
 
     it('loads a specific block page', () => {
-      cy.visit(`${basePath}/block/7e1369a23a5ab861e7bdede2aadcccae4ea873ffd9caf11c7c5541eb5bcdff54`);
+      cy.visit(`${basePath}/block/fb4cbcbff3993ca4bf8caf657d55a23db5ed4ab1cfa33c489303c2e04e1c38e0`);
+      cy.get('.pagination').scrollIntoView({ offset: { top: 200, left: 0 } });
       cy.waitForSkeletonGone();
     });
 

frontend/cypress/e2e/mainnet/mainnet.spec.ts

@@ -103,6 +103,7 @@ describe('Mainnet', () => {
 
     it('check op_return tx tooltip', () => {
       cy.visit('/block/00000000000000000003c5f542bed265319c6cf64238cf1f1bb9bca3ebf686d2');
+      cy.get('.pagination').scrollIntoView({ offset: { top: 200, left: 0 } });
       cy.waitForSkeletonGone();
       cy.get('tbody > :nth-child(2) > :nth-child(1) > a').first().trigger('onmouseover');
       cy.get('tbody > :nth-child(2) > :nth-child(1) > a').first().trigger('mouseenter');
@@ -111,9 +112,10 @@ describe('Mainnet', () => {
 
     it('check op_return coinbase tooltip', () => {
       cy.visit('/block/00000000000000000003c5f542bed265319c6cf64238cf1f1bb9bca3ebf686d2');
+      cy.get('.pagination').scrollIntoView({ offset: { top: 200, left: 0 } });
       cy.waitForSkeletonGone();
-      cy.get('div > a > .badge').first().trigger('onmouseover');
-      cy.get('div > a > .badge').first().trigger('mouseenter');
+      cy.get('tbody > :nth-child(2) > :nth-child(1) > a').first().trigger('onmouseover');
+      cy.get('tbody > :nth-child(2) > :nth-child(1) > a').first().trigger('mouseenter');
       cy.get('.tooltip-inner').should('be.visible');
     });
 
@@ -142,13 +144,13 @@ describe('Mainnet', () => {
         });
       });
 
-      ['BC1PQYQSZQ', 'bc1PqYqSzQ'].forEach((searchTerm) => {
+      ['BC1PQYQS', 'bc1PqYqS'].forEach((searchTerm) => {
         it(`allows searching for partial case insensitive bech32m addresses: ${searchTerm}`, () => {
           cy.visit('/');
           cy.get('.search-box-container > .form-control').type(searchTerm).then(() => {
-            cy.get('app-search-results button.dropdown-item').should('have.length', 1);
+            cy.get('app-search-results button.dropdown-item').should('have.length', 10);
             cy.get('app-search-results button.dropdown-item.active').click().then(() => {
-              cy.url().should('include', '/address/bc1pqyqszqgpqyqszqgpqyqszqgpqyqszqgpqyqszqgpqyqszqgpqyqsyjer9e');
+              cy.url().should('include', '/address/bc1pqyqs26fs4gnyw4aqttyjqa5ta7075zzfjftyz98qa8vdr49dh7fqm2zkv3');
               cy.waitForSkeletonGone();
               cy.get('.text-center').should('not.have.text', 'Invalid Bitcoin address');
             });
@@ -156,13 +158,13 @@ describe('Mainnet', () => {
         });
       });
 
-      ['BC1Q000375VXCU', 'bC1q000375vXcU'].forEach((searchTerm) => {
+      ['BC1Q0003', 'bC1q0003'].forEach((searchTerm) => {
         it(`allows searching for partial case insensitive bech32 addresses: ${searchTerm}`, () => {
           cy.visit('/');
           cy.get('.search-box-container > .form-control').type(searchTerm).then(() => {
-            cy.get('app-search-results button.dropdown-item').should('have.length', 1);
+            cy.get('app-search-results button.dropdown-item').should('have.length', 10);
             cy.get('app-search-results button.dropdown-item.active').click().then(() => {
-              cy.url().should('include', '/address/bc1q000375vxcuf5v04lmwy22vy2thvhqkxghgq7dy');
+              cy.url().should('include', '/address/bc1q000303cgr9zazthut63kdktwtatfe206um8nyh');
               cy.waitForSkeletonGone();
               cy.get('.text-center').should('not.have.text', 'Invalid Bitcoin address');
             });
@@ -283,6 +285,7 @@ describe('Mainnet', () => {
         it('loads genesis block and keypress arrow right', () => {
           cy.viewport('macbook-16');
           cy.visit('/block/0');
+          cy.get('.pagination').scrollIntoView({ offset: { top: 200, left: 0 } });
           cy.waitForSkeletonGone();
           cy.waitForPageIdle();
 
@@ -295,6 +298,7 @@ describe('Mainnet', () => {
         it('loads genesis block and keypress arrow left', () => {
           cy.viewport('macbook-16');
           cy.visit('/block/0');
+          cy.get('.pagination').scrollIntoView({ offset: { top: 200, left: 0 } });
           cy.waitForSkeletonGone();
           cy.waitForPageIdle();
 
@@ -323,6 +327,7 @@ describe('Mainnet', () => {
         it('loads genesis block and click on the arrow left', () => {
           cy.viewport('macbook-16');
           cy.visit('/block/0');
+          cy.get('.pagination').scrollIntoView({ offset: { top: 200, left: 0 } });
           cy.waitForSkeletonGone();
           cy.waitForPageIdle();
           cy.get('[ngbtooltip="Next Block"] > .ng-fa-icon > .svg-inline--fa').should('be.visible');
@@ -339,7 +344,9 @@ describe('Mainnet', () => {
       cy.visit('/');
       cy.waitForSkeletonGone();
 
-      cy.changeNetwork('testnet');
+      //TODO(knorrium): add a check for the proxied server
+      // cy.changeNetwork('testnet4');
+
       cy.changeNetwork('signet');
       cy.changeNetwork('mainnet');
     });
@@ -439,6 +446,7 @@ describe('Mainnet', () => {
     describe('blocks', () => {
       it('shows empty blocks properly', () => {
         cy.visit('/block/0000000000000000000bd14f744ef2e006e61c32214670de7eb891a5732ee775');
+        cy.get('.pagination').scrollIntoView({ offset: { top: 200, left: 0 } });
         cy.waitForSkeletonGone();
         cy.waitForPageIdle();
         cy.get('h2').invoke('text').should('equal', '1 transaction');
@@ -446,6 +454,7 @@ describe('Mainnet', () => {
 
       it('expands and collapses the block details', () => {
         cy.visit('/block/0');
+        cy.get('.pagination').scrollIntoView({ offset: { top: 200, left: 0 } });
         cy.waitForSkeletonGone();
         cy.waitForPageIdle();
         cy.get('.btn.btn-outline-info').click().then(() => {
@@ -458,6 +467,7 @@ describe('Mainnet', () => {
       });
       it('shows blocks with no pagination', () => {
         cy.visit('/block/00000000000000000001ba40caf1ad4cec0ceb77692662315c151953bfd7c4c4');
+        cy.get('.pagination').scrollIntoView({ offset: { top: 200, left: 0 } });
         cy.waitForSkeletonGone();
         cy.waitForPageIdle();
         cy.get('.block-tx-title h2').invoke('text').should('equal', '19 transactions');
@@ -467,6 +477,7 @@ describe('Mainnet', () => {
       it('supports pagination on the block screen', () => {
         // 41 txs
         cy.visit('/block/00000000000000000009f9b7b0f63ad50053ad12ec3b7f5ca951332f134f83d8');
+        cy.get('.pagination').scrollIntoView({ offset: { top: 200, left: 0 } });
         cy.waitForSkeletonGone();
         cy.get('.pagination-container a').invoke('text').then((text1) => {
           cy.get('.active + li').first().click().then(() => {
@@ -482,6 +493,7 @@ describe('Mainnet', () => {
       it('shows blocks pagination with 5 pages (desktop)', () => {
         cy.viewport(760, 800);
         cy.visit('/block/000000000000000000049281946d26fcba7d99fdabc1feac524bc3a7003d69b3').then(() => {
+          cy.get('.pagination').scrollIntoView({ offset: { top: 200, left: 0 } });
           cy.waitForSkeletonGone();
           cy.waitForPageIdle();
         });
@@ -493,6 +505,7 @@ describe('Mainnet', () => {
       it('shows blocks pagination with 3 pages (mobile)', () => {
         cy.viewport(669, 800);
         cy.visit('/block/000000000000000000049281946d26fcba7d99fdabc1feac524bc3a7003d69b3').then(() => {
+          cy.get('.pagination').scrollIntoView({ offset: { top: 200, left: 0 } });
           cy.waitForSkeletonGone();
           cy.waitForPageIdle();
         });
@@ -532,16 +545,7 @@ describe('Mainnet', () => {
           }
         });
 
-        cy.get('.alert').should('be.visible');
-        cy.get('.alert').invoke('css', 'width').then((alertWidth) => {
-          cy.get('.container-xl > :nth-child(3)').invoke('css', 'width').should('equal', alertWidth);
-        });
-
-        cy.get('.btn-warning').then(getRectangle).then((rectA) => {
-          cy.get('.alert').then(getRectangle).then((rectB) => {
-            expect(areOverlapping(rectA, rectB), 'Confirmations box and RBF alert are overlapping').to.be.false;
-          });
-        });
+        cy.get('.alert-replaced').should('be.visible');
       });
 
       it('shows RBF transactions properly (desktop)', () => {

frontend/cypress/e2e/signet/signet.spec.ts

@@ -95,12 +95,14 @@ describe('Signet', () => {
     describe('blocks', () => {
       it('shows empty blocks properly', () => {
         cy.visit('/signet/block/00000133d54e4589f6436703b067ec23209e0a21b8a9b12f57d0592fd85f7a42');
+        cy.get('.pagination').scrollIntoView({ offset: { top: 200, left: 0 } });
         cy.waitForSkeletonGone();
         cy.get('h2').invoke('text').should('equal', '1 transaction');
       });
 
       it('expands and collapses the block details', () => {
         cy.visit('/signet/block/0');
+        cy.get('.pagination').scrollIntoView({ offset: { top: 200, left: 0 } });
         cy.waitForSkeletonGone();
         cy.get('.btn.btn-outline-info').click().then(() => {
           cy.get('#details').should('be.visible');
@@ -113,6 +115,7 @@ describe('Signet', () => {
 
       it('shows blocks with no pagination', () => {
         cy.visit('/signet/block/00000078f920a96a69089877b934ce7fd009ab55e3170920a021262cb258e7cc');
+        cy.get('.pagination').scrollIntoView({ offset: { top: 200, left: 0 } });
         cy.waitForSkeletonGone();
         cy.get('h2').invoke('text').should('equal', '13 transactions');
         cy.get('ul.pagination').first().children().should('have.length', 5);
@@ -121,6 +124,7 @@ describe('Signet', () => {
       it('supports pagination on the block screen', () => {
         // 43 txs
         cy.visit('/signet/block/00000094bd52f73bdbfc4bece3a94c21fec2dc968cd54210496e69e4059d66a6');
+        cy.get('.pagination').scrollIntoView({ offset: { top: 200, left: 0 } });
         cy.waitForSkeletonGone();
         cy.get('.header-bg.box > a').invoke('text').then((text1) => {
           cy.get('.active + li').first().click().then(() => {

frontend/cypress/e2e/testnet4/testnet4.spec.ts

@@ -2,7 +2,7 @@ import { emitMempoolInfo } from '../../support/websocket';
 
 const baseModule = Cypress.env('BASE_MODULE');
 
-describe('Testnet', () => {
+describe('Testnet4', () => {
   beforeEach(() => {
     cy.intercept('/api/block-height/*').as('block-height');
     cy.intercept('/api/block/*').as('block');
@@ -13,7 +13,7 @@ describe('Testnet', () => {
   if (baseModule === 'mempool') {
 
     it('loads the dashboard', () => {
-      cy.visit('/testnet');
+      cy.visit('/testnet4');
       cy.waitForSkeletonGone();
     });
 
@@ -25,7 +25,7 @@ describe('Testnet', () => {
 
     it.skip('loads the dashboard with the skeleton blocks', () => {
       cy.mockMempoolSocket();
-      cy.visit('/testnet');
+      cy.visit('/testnet4');
       cy.get(':nth-child(1) > #bitcoin-block-0').should('be.visible');
       cy.get(':nth-child(2) > #bitcoin-block-0').should('be.visible');
       cy.get(':nth-child(3) > #bitcoin-block-0').should('be.visible');
@@ -45,7 +45,7 @@ describe('Testnet', () => {
     });
 
     it('loads the pools screen', () => {
-      cy.visit('/testnet');
+      cy.visit('/testnet4');
       cy.waitForSkeletonGone();
       cy.get('#btn-pools').click().then(() => {
         cy.wait(1000);
@@ -53,7 +53,7 @@ describe('Testnet', () => {
     });
 
     it('loads the graphs screen', () => {
-      cy.visit('/testnet');
+      cy.visit('/testnet4');
       cy.waitForSkeletonGone();
       cy.get('#btn-graphs').click().then(() => {
         cy.wait(1000);
@@ -63,7 +63,7 @@ describe('Testnet', () => {
     describe('tv mode', () => {
       it('loads the tv screen - desktop', () => {
         cy.viewport('macbook-16');
-        cy.visit('/testnet/graphs');
+        cy.visit('/testnet4/graphs');
         cy.waitForSkeletonGone();
         cy.get('#btn-tv').click().then(() => {
           cy.wait(1000);
@@ -73,7 +73,7 @@ describe('Testnet', () => {
       });
 
       it('loads the tv screen - mobile', () => {
-        cy.visit('/testnet/graphs');
+        cy.visit('/testnet4/graphs');
         cy.waitForSkeletonGone();
         cy.get('#btn-tv').click().then(() => {
           cy.viewport('iphone-6');
@@ -85,7 +85,7 @@ describe('Testnet', () => {
 
 
     it('loads the api screen', () => {
-      cy.visit('/testnet');
+      cy.visit('/testnet4');
       cy.waitForSkeletonGone();
       cy.get('#btn-docs').click().then(() => {
         cy.wait(1000);
@@ -94,13 +94,15 @@ describe('Testnet', () => {
 
     describe('blocks', () => {
       it('shows empty blocks properly', () => {
-        cy.visit('/testnet/block/0');
+        cy.visit('/testnet4/block/0');
+        cy.get('.pagination').scrollIntoView({ offset: { top: 200, left: 0 } });
         cy.waitForSkeletonGone();
         cy.get('h2').invoke('text').should('equal', '1 transaction');
       });
 
       it('expands and collapses the block details', () => {
-        cy.visit('/testnet/block/0');
+        cy.visit('/testnet4/block/0');
+        cy.get('.pagination').scrollIntoView({ offset: { top: 200, left: 0 } });
         cy.waitForSkeletonGone();
         cy.get('.btn.btn-outline-info').click().then(() => {
           cy.get('#details').should('be.visible');
@@ -112,15 +114,17 @@ describe('Testnet', () => {
       });
 
       it('shows blocks with no pagination', () => {
-        cy.visit('/testnet/block/000000000000002f8ce27716e74ecc7ad9f7b5101fed12d09e28bb721b9460ea');
+        cy.visit('/testnet4/block/000000000066e8b6cc78a93f8989587f5819624bae2eb1c05f535cadded19f99');
+        cy.get('.pagination').scrollIntoView({ offset: { top: 200, left: 0 } });
         cy.waitForSkeletonGone();
-        cy.get('h2').invoke('text').should('equal', '11 transactions');
+        cy.get('h2').invoke('text').should('equal', '18 transactions');
         cy.get('ul.pagination').first().children().should('have.length', 5);
       });
 
       it('supports pagination on the block screen', () => {
         // 48 txs
-        cy.visit('/testnet/block/000000000000002ca3878ebd98b313a1c2d531f2e70a6575d232ca7564dea7a9');
+        cy.visit('/testnet4/block/000000000000006982d53f8273bdff21dafc380c292eabc669b5ab6d732311c3');
+        cy.get('.pagination').scrollIntoView({ offset: { top: 200, left: 0 } });
         cy.waitForSkeletonGone();
         cy.get('.header-bg.box > a').invoke('text').then((text1) => {
           cy.get('.active + li').first().click().then(() => {

frontend/cypress/fixtures/mainnet_mempoolInfo.json

@@ -750,7 +750,7 @@
     },
     "backendInfo": {
       "hostname": "node205.tk7.mempool.space",
-      "version": "3.0.0-dev",
+      "version": "3.1.0-dev",
       "gitCommit": "abbc8a134",
       "lightning": false
     },