Merge branch 'master' into knorrium/update_cypress_deps

backend/.gitignore

@@ -45,3 +45,6 @@ testem.log
 #System Files
 .DS_Store
 Thumbs.db
+
+# package folder (npm run package output)
+/package

backend/npm_package.sh

@@ -1,14 +1,17 @@
 #/bin/sh
 set -e
 
+# Remove previous dist folder
+rm -rf dist
+# Build new dist folder
 npm run build
 # Remove previous package folder
 rm -rf package
 # Move JS and deps
 mv dist package
-mv node_modules package
+cp -R node_modules package
 # Remove symlink for rust-gbt and insert real folder
 rm package/node_modules/rust-gbt
-mv rust-gbt package/node_modules
+cp -R rust-gbt package/node_modules
 # Clean up deps
 npm run package-rm-build-deps

backend/package.json

@@ -22,7 +22,7 @@
   "main": "index.ts",
   "scripts": {
     "tsc": "./node_modules/typescript/bin/tsc -p tsconfig.build.json",
-    "build": "npm run tsc && npm run create-resources",
+    "build": "npm run rust-build && npm run tsc && npm run create-resources",
     "create-resources": "cp ./src/tasks/price-feeds/mtgox-weekly.json ./dist/tasks && node dist/api/fetch-version.js",
     "package": "./npm_package.sh",
     "package-rm-build-deps": "./npm_package_rm_build_deps.sh",
@@ -33,7 +33,8 @@
     "test": "./node_modules/.bin/jest --coverage",
     "lint": "./node_modules/.bin/eslint . --ext .ts",
     "lint:fix": "./node_modules/.bin/eslint . --ext .ts --fix",
-    "prettier": "./node_modules/.bin/prettier --write \"src/**/*.{js,ts}\""
+    "prettier": "./node_modules/.bin/prettier --write \"src/**/*.{js,ts}\"",
+    "rust-build": "cd rust-gbt && npm run build-release"
   },
   "dependencies": {
     "@babel/core": "^7.21.3",

backend/src/__tests__/api/common.ts

@@ -0,0 +1,24 @@
+import { Common } from '../../api/common';
+import { MempoolTransactionExtended } from '../../mempool.interfaces';
+
+const randomTransactions = require('./test-data/transactions-random.json');
+const replacedTransactions = require('./test-data/transactions-replaced.json');
+const rbfTransactions = require('./test-data/transactions-rbfs.json');
+
+describe('Mempool Utils', () => {
+  test('should detect RBF transactions with fast method', () => {
+    const newTransactions = rbfTransactions.concat(randomTransactions);
+    const result: { [txid: string]: MempoolTransactionExtended[] } = Common.findRbfTransactions(newTransactions, replacedTransactions);
+    expect(Object.values(result).length).toEqual(2);
+    expect(result).toHaveProperty('7219d95161f3718335991ac6d967d24eedec370908c9879bb1e192e6d797d0a6');
+    expect(result).toHaveProperty('5387881d695d4564d397026dc5f740f816f8390b4b2c5ec8c20309122712a875');
+  });
+
+  test.only('should detect RBF transactions with scalable method', () => {
+    const newTransactions = rbfTransactions.concat(randomTransactions);
+    const result: { [txid: string]: MempoolTransactionExtended[] } = Common.findRbfTransactions(newTransactions, replacedTransactions, true);
+    expect(Object.values(result).length).toEqual(2);
+    expect(result).toHaveProperty('7219d95161f3718335991ac6d967d24eedec370908c9879bb1e192e6d797d0a6');
+    expect(result).toHaveProperty('5387881d695d4564d397026dc5f740f816f8390b4b2c5ec8c20309122712a875');
+  });
+});

backend/src/__tests__/api/test-data/transactions-random.json

@@ -0,0 +1,277 @@
+[
+    {
+        "txid": "13f007241d78e8b0b4e57d2ae3fd37bcfe3226534d7cadeba5a549860d960db0",
+        "version": 2,
+        "locktime": 0,
+        "vin": [
+            {
+                "txid": "cb8f206f4e88bec97107089f3e9e61d50cde53d4541992ae19759b71103cf75c",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "0014fd6d15ff832c12f1ff04a5ccd5039f7227b260bd",
+                    "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 fd6d15ff832c12f1ff04a5ccd5039f7227b260bd",
+                    "scriptpubkey_type": "v0_p2wpkh",
+                    "scriptpubkey_address": "bc1ql4k3tlur9sf0rlcy5hxd2qulwgnmyc9akehvth",
+                    "value": 610677
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "304302205c430b36ebd2bb327951d83440af1f58f127871b2baada4c4dde2bc0b6721f56021f3445099f1a40e35baeda32e8e3727b505ffba0d882b11f498c7762f4184e9901",
+                    "0236b5edd4fbbcfb045960e42ec8a9968944084785932e32940e8cd2583b37da67"
+                ],
+                "is_coinbase": false,
+                "sequence": 2147483648
+            }
+        ],
+        "vout": [
+            {
+                "scriptpubkey": "76a9149d32ef812385f3811634e0c0117dd153a5de10a488ac",
+                "scriptpubkey_asm": "OP_DUP OP_HASH160 OP_PUSHBYTES_20 9d32ef812385f3811634e0c0117dd153a5de10a4 OP_EQUALVERIFY OP_CHECKSIG",
+                "scriptpubkey_type": "p2pkh",
+                "scriptpubkey_address": "1FLC7Bag7okAkKPCyZbgZZg3Hh1EuGZ5Rd",
+                "value": 344697
+            },
+            {
+                "scriptpubkey": "00147dee8a7a38abbfb00dbfba365c8d6712934cc491",
+                "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 7dee8a7a38abbfb00dbfba365c8d6712934cc491",
+                "scriptpubkey_type": "v0_p2wpkh",
+                "scriptpubkey_address": "bc1q0hhg573c4wlmqrdlhgm9ert8z2f5e3y3lf9hvx",
+                "value": 265396
+            }
+        ],
+        "size": 224,
+        "weight": 572,
+        "fee": 584,
+        "status": {
+            "confirmed": false
+        },
+        "order": 2953680397,
+        "vsize": 143,
+        "adjustedVsize": 143,
+        "sigops": 5,
+        "feePerVsize": 4.083916083916084,
+        "adjustedFeePerVsize": 4.083916083916084,
+        "effectiveFeePerVsize": 4.083916083916084,
+        "firstSeen": 1691222538,
+        "uid": 526973,
+        "inputs": [
+            526728
+        ],
+        "position": {
+            "block": 7,
+            "vsize": 21429708.5
+        },
+        "bestDescendant": null,
+        "cpfpChecked": true
+    },
+    {
+        "txid": "8e89b20f8a7fadb0e4cdbe57a00eee224f5076bac5387fc276916724e7c4a16a",
+        "version": 2,
+        "locktime": 800571,
+        "vin": [
+            {
+                "txid": "35e16762459539f3a8e52c5dee6a9ccaa9e9268efed33aa2c6e1b7805e849f24",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "0014d4f16ef275b3e1c4a4ecbef55a164933e0f6460f",
+                    "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 d4f16ef275b3e1c4a4ecbef55a164933e0f6460f",
+                    "scriptpubkey_type": "v0_p2wpkh",
+                    "scriptpubkey_address": "bc1q6nckaun4k0suff8vhm6459jfx0s0v3s0ff4ukl",
+                    "value": 1528924
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "3044022019008b26e885bb43da25a11ffac147a057722072eedb68411f114f6e7eb82ebc02201b618264bb97756b88fc3bbc365b73044ac18b33b1067e31cfd5bcd0f50ed2c701",
+                    "039b71145070bd3e8af28e27fa577f2e12ab6bb4e212d3eeaef08b4bc39e8cbc13"
+                ],
+                "is_coinbase": false,
+                "sequence": 4294967293
+            },
+            {
+                "txid": "67c27ed0f767526234bcd5f795a31fab8ec4d0251bf12c68f2746951f4110d90",
+                "vout": 3,
+                "prevout": {
+                    "scriptpubkey": "0014a7c3d613b321375054b2ac9b6114367bc034ad6f",
+                    "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 a7c3d613b321375054b2ac9b6114367bc034ad6f",
+                    "scriptpubkey_type": "v0_p2wpkh",
+                    "scriptpubkey_address": "bc1q5lpavyanyym4q49j4jdkz9pk00qrftt0yqzvk3",
+                    "value": 436523
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "304402204e67285fc656bc45ed082499b076d5dba2fa21d0d7e64a0ae52b19d69a11760002200f037d81ee540b74397844513b72b08ed92b06db76bd20b08f7a0a3b36ab13d501",
+                    "02a3ebae85f0225b6fbb5ff060afce683a4683507a57544605a29ee7d287e591b4"
+                ],
+                "is_coinbase": false,
+                "sequence": 4294967293
+            },
+            {
+                "txid": "21c38fb9a2521e438c614f53b19ddd7a5594bcc4b77480e762fd4b702fad3374",
+                "vout": 1,
+                "prevout": {
+                    "scriptpubkey": "00149660e34ef88106536c816c037b5b28dd64a812e2",
+                    "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 9660e34ef88106536c816c037b5b28dd64a812e2",
+                    "scriptpubkey_type": "v0_p2wpkh",
+                    "scriptpubkey_address": "bc1qjeswxnhcsyr9xmypdsphkkegm4j2syhztgzxv4",
+                    "value": 758149
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "3044022021b556f0aa99329076bcc435338aceaf534963efcab306931b1b2b0461e16e0c02203a78942a3745c4da656bddfd8cf16b85dc04d652904e88682127cdd9ca63339001",
+                    "0298963be4a8f66aca9fcf1c6dc95547aeaa82347543190c91e094c2321142b9f0"
+                ],
+                "is_coinbase": false,
+                "sequence": 4294967293
+            },
+            {
+                "txid": "aa998dbae65240a7386bf7d468459551d99c3de8e2f9057ff5f2d38e17daf788",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "00147bb7413a39943b21ded98ad5e6ad7a222d273e17",
+                    "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 7bb7413a39943b21ded98ad5e6ad7a222d273e17",
+                    "scriptpubkey_type": "v0_p2wpkh",
+                    "scriptpubkey_address": "bc1q0wm5zw3ejsajrhke3t27dtt6ygkjw0sh9lltg6",
+                    "value": 1067200
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "304402205e2269f7d4ee0513b34354c38e920aef2dabac6f4350afb2dd105ff3ee43ae7b02202870322f2cb85cb0b2b0e38152f018bfff271dc3ec5aed0515854d0b259aaf3d01",
+                    "03b87320cf3263a644a0d3f89c1b4a7304d9dfda9eb8c891560716abcb73e88b99"
+                ],
+                "is_coinbase": false,
+                "sequence": 4294967293
+            },
+            {
+                "txid": "230253d195d779d4688ba16993985cd27b2e7a687d8b889b3bc63f19ece36f20",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "001439647bd997819d12dfc72b0fb9ff9ffcb84946f8",
+                    "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 39647bd997819d12dfc72b0fb9ff9ffcb84946f8",
+                    "scriptpubkey_type": "v0_p2wpkh",
+                    "scriptpubkey_address": "bc1q89j8hkvhsxw39h789v8mnlulljuyj3hc9zve97",
+                    "value": 361950
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "304402204f7ca868bb9b92a07fecdc6b9dd56e4e1d007ca1405952f98ed6bc416345b5f2022055320a97791417abf6628fcf6513ac5785b06c630f854d8595e96ea06c3841d301",
+                    "03a3ffe8e3ef2eea129b227e9658164bae0a6d21c17da6de9973ba34d9e04b21a0"
+                ],
+                "is_coinbase": false,
+                "sequence": 4294967293
+            },
+            {
+                "txid": "670771e265a0b62dbd3c1fec2b865177eaf0bafd0ae49dd40a1c9fcd9a847a81",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "0014d45d1b0022c7387e42c5452ced561bdb8fd4b521",
+                    "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 d45d1b0022c7387e42c5452ced561bdb8fd4b521",
+                    "scriptpubkey_type": "v0_p2wpkh",
+                    "scriptpubkey_address": "bc1q63w3kqpzcuu8usk9g5kw64smmw8afdfpxmc2m0",
+                    "value": 453275
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "3044022071312921800441903b2099e723add8702dd0f92ec11526ff87acf6967ec64cbd02203deabe7ed56d5daaa9a95c5a607b1ab705ff1c46bc6984a6dca120e63a91768601",
+                    "0257302ac8d9c4c8f9b1744f19bb432359326b9cc7bdddeeab9202749a6d92be58"
+                ],
+                "is_coinbase": false,
+                "sequence": 4294967293
+            },
+            {
+                "txid": "0af82159eee2b69242f2ff032636e410b67ec1ace52e55fb0d20ed814cd64803",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "001459e4d6bfefc6b45f955a69c4aeca26348e9d54ed",
+                    "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 59e4d6bfefc6b45f955a69c4aeca26348e9d54ed",
+                    "scriptpubkey_type": "v0_p2wpkh",
+                    "scriptpubkey_address": "bc1qt8jdd0l0c669l926d8z2aj3xxj8f648dtyn7tc",
+                    "value": 439961
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "3044022027540322e92c23c5513aa2587e7feb56a8ce82f879269d6b3cbd425634b44f8e022045572dee7262b02130bfe32d8aa8abbfaa64e101abfc819bba5380c78876692d01",
+                    "03fe02262d87f4a5289d3dd66e3d9a74cd49fa1cad0249284a7451896a827249a5"
+                ],
+                "is_coinbase": false,
+                "sequence": 4294967293
+            },
+            {
+                "txid": "68cf9c784870a4f888f044755f7ce318557f652461db8ef887d279672f186018",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "001454822b2d5d52597a78b630921cf439a41e32f2f9",
+                    "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 54822b2d5d52597a78b630921cf439a41e32f2f9",
+                    "scriptpubkey_type": "v0_p2wpkh",
+                    "scriptpubkey_address": "bc1q2jpzkt2a2fvh579kxzfpeape5s0r9uhewhl5n4",
+                    "value": 227639
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "304402203ad511d6a8730748b8828bc38897d360451adf620ebdc1d229c08c097c80bef202202f50c793d95b5200cf2258e03896a3be7720df0eb3b8c810c86db74341a7e83e01",
+                    "0294992e9f4546e6e119741f908411ae531e9d1ff732d69b4dff8172aaf2a4b216"
+                ],
+                "is_coinbase": false,
+                "sequence": 4294967293
+            },
+            {
+                "txid": "793f01dfdb19bf41f958fd917c16d9c4dd5d5e1a5c0434bfdb367212659d1b5b",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "0014f54edf8ae647b5300e2674523254e923d93d169f",
+                    "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 f54edf8ae647b5300e2674523254e923d93d169f",
+                    "scriptpubkey_type": "v0_p2wpkh",
+                    "scriptpubkey_address": "bc1q748dlzhxg76nqr3xw3fry48fy0vn695lvhlkxv",
+                    "value": 227070
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "304402206e807ab616f4f2887ba703ae744d856142d9aca8128698419bbb67fb4fad8177022060fc65c7cd66baa88ad1e1d317a6edd5f6cb52fe8bff6e5405ffa1acf9d945d901",
+                    "02a0ad0167c6e9edf62677404d74d3b80ea276e47e758ffaa6ca17bd65ac79f7aa"
+                ],
+                "is_coinbase": false,
+                "sequence": 4294967293
+            }
+        ],
+        "vout": [
+            {
+                "scriptpubkey": "00148a5c45ccfc29d209940d94525e2edb7743a1ad8a",
+                "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 8a5c45ccfc29d209940d94525e2edb7743a1ad8a",
+                "scriptpubkey_type": "v0_p2wpkh",
+                "scriptpubkey_address": "bc1q3fwytn8u98fqn9qdj3f9utkmwap6rtv2ym33zm",
+                "value": 5500000
+            }
+        ],
+        "size": 1375,
+        "weight": 2605,
+        "fee": 691,
+        "status": {
+            "confirmed": false
+        },
+        "order": 1788986599,
+        "vsize": 651,
+        "adjustedVsize": 651.25,
+        "sigops": 9,
+        "feePerVsize": 1.0610364683301343,
+        "adjustedFeePerVsize": 1.0610364683301343,
+        "effectiveFeePerVsize": 1.0610364683301343,
+        "firstSeen": 1691163298,
+        "uid": 120494,
+        "inputs": [],
+        "position": {
+            "block": 7,
+            "vsize": 93780091.5
+        },
+        "bestDescendant": null,
+        "cpfpChecked": true
+    }
+]

backend/src/__tests__/api/test-data/transactions-rbfs.json

@@ -0,0 +1,121 @@
+[
+    {
+        "txid": "7219d95161f3718335991ac6d967d24eedec370908c9879bb1e192e6d797d0a6",
+        "version": 1,
+        "locktime": 0,
+        "vin": [
+            {
+                "txid": "d863deb706de5a611028f7547e16ea81d7819e44beb640fb30a9ba30c585140f",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "76a914cd5b6566b455d043558829f6932edaae5d8f0ad388ac",
+                    "scriptpubkey_asm": "OP_DUP OP_HASH160 OP_PUSHBYTES_20 cd5b6566b455d043558829f6932edaae5d8f0ad3 OP_EQUALVERIFY OP_CHECKSIG",
+                    "scriptpubkey_type": "p2pkh",
+                    "scriptpubkey_address": "1Kiq1dyVBzYLWGrBPWjChvKyzB2H95x5RJ",
+                    "value": 799995000
+                },
+                "scriptsig": "483045022100aeeddfb9785c5a4b70e90d0445785c68b7a44e28853441134a70ddc4da39527602203dfe1ec1a377aaacb64ae65c7c944caf1398d2dc063f712251b4cf696d44d3cb01210314338e3e191aea3ac9e9292611faeedf0379bbe62c30fd76c7450722a1ac47c6",
+                "scriptsig_asm": "OP_PUSHBYTES_72 3045022100aeeddfb9785c5a4b70e90d0445785c68b7a44e28853441134a70ddc4da39527602203dfe1ec1a377aaacb64ae65c7c944caf1398d2dc063f712251b4cf696d44d3cb01 OP_PUSHBYTES_33 0314338e3e191aea3ac9e9292611faeedf0379bbe62c30fd76c7450722a1ac47c6",
+                "is_coinbase": false,
+                "sequence": 4294967293
+            }
+        ],
+        "vout": [
+            {
+                "scriptpubkey": "6a4c5058325b8669baa9259e082f064005bc92274b559337ac317798f5d76f2d0577ed5a96042fce8c33d841b6c47a99f9597000ab04a10b34cd419fc19784d9e36f1a33fd7b000c3bce00b6000c1d1e00614b",
+                "scriptpubkey_asm": "OP_RETURN OP_PUSHDATA1 58325b8669baa9259e082f064005bc92274b559337ac317798f5d76f2d0577ed5a96042fce8c33d841b6c47a99f9597000ab04a10b34cd419fc19784d9e36f1a33fd7b000c3bce00b6000c1d1e00614b",
+                "scriptpubkey_type": "op_return",
+                "value": 0
+            },
+            {
+                "scriptpubkey": "a9144890aae025c84cb72a9730b49ca12595d6f6088d87",
+                "scriptpubkey_asm": "OP_HASH160 OP_PUSHBYTES_20 4890aae025c84cb72a9730b49ca12595d6f6088d OP_EQUAL",
+                "scriptpubkey_type": "p2sh",
+                "scriptpubkey_address": "38Jht2bzmJL4EwoFvvyFzejhfEb4J7KxLb",
+                "value": 155000
+            },
+            {
+                "scriptpubkey": "76a91486e7dad6617303942a448b7f8afe9653e5624a5e88ac",
+                "scriptpubkey_asm": "OP_DUP OP_HASH160 OP_PUSHBYTES_20 86e7dad6617303942a448b7f8afe9653e5624a5e OP_EQUALVERIFY OP_CHECKSIG",
+                "scriptpubkey_type": "p2pkh",
+                "scriptpubkey_address": "1DJKJGApgX4W8BSQ8FRPLqX78UaCskT4r2",
+                "value": 155000
+            },
+            {
+                "scriptpubkey": "76a914cd5b6566b455d043558829f6932edaae5d8f0ad388ac",
+                "scriptpubkey_asm": "OP_DUP OP_HASH160 OP_PUSHBYTES_20 cd5b6566b455d043558829f6932edaae5d8f0ad3 OP_EQUALVERIFY OP_CHECKSIG",
+                "scriptpubkey_type": "p2pkh",
+                "scriptpubkey_address": "1Kiq1dyVBzYLWGrBPWjChvKyzB2H95x5RJ",
+                "value": 799675549
+            }
+        ],
+        "size": 350,
+        "weight": 1400,
+        "fee": 9451,
+        "status": {
+            "confirmed": false
+        },
+        "order": 2798688215,
+        "vsize": 350,
+        "adjustedVsize": 350,
+        "sigops": 8,
+        "feePerVsize": 27.002857142857142,
+        "adjustedFeePerVsize": 27.002857142857142,
+        "effectiveFeePerVsize": 27.002857142857142,
+        "firstSeen": 1691218536,
+        "uid": 513598,
+        "inputs": [],
+        "position": {
+            "block": 0,
+            "vsize": 22166
+        },
+        "bestDescendant": null,
+        "cpfpChecked": true
+    },
+    {
+        "txid": "5387881d695d4564d397026dc5f740f816f8390b4b2c5ec8c20309122712a875",
+        "version": 2,
+        "locktime": 0,
+        "vin": [
+            {
+                "txid": "b50225a04a1d6fbbfa7a2122bc0580396f614027b3957f476229633576f06130",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "0014a24f913f8a9c30a4c302c2c78f2fd7addb08fd07",
+                    "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 a24f913f8a9c30a4c302c2c78f2fd7addb08fd07",
+                    "scriptpubkey_type": "v0_p2wpkh",
+                    "scriptpubkey_address": "bc1q5f8ez0u2nsc2fsczctrc7t7h4hds3lg82ewqhz",
+                    "value": 612917
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "3045022100a0c23953ace5d022b7a6d45d1ae1730bf20a4d594bb5d4fa7aa80e4881b44d320220008f9b144805bb91995fc0f452a56e09f4ad16fa149d71ae9b5d57c742e8e2cc01",
+                    "03dc2c7b687019b40a68d713322675206cc266e34e5340ec982c13ff0222c3b2b6"
+                ],
+                "is_coinbase": false,
+                "sequence": 2147483649
+            }
+        ],
+        "vout": [
+            {
+                "scriptpubkey": "0014199a98f9589364ffe5ef5bbae45ce5dfcbb873bd",
+                "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 199a98f9589364ffe5ef5bbae45ce5dfcbb873bd",
+                "scriptpubkey_type": "v0_p2wpkh",
+                "scriptpubkey_address": "bc1qrxdf372cjdj0le00twawgh89ml9msuaau62gk4",
+                "value": 611909
+            }
+        ],
+        "size": 192,
+        "weight": 438,
+        "fee": 1008,
+        "status": {
+            "confirmed": false
+        },
+        "bestDescendant": null,
+        "descendants": null,
+        "adjustedFeePerVsize": 10.2283,
+        "sigops": 1,
+        "adjustedVsize": 109.5
+    }
+]

backend/src/__tests__/api/test-data/transactions-replaced.json

@@ -0,0 +1,139 @@
+[
+    {
+        "txid": "008592364e21c1e3d62ba9538ac78a81779897b52100af5707ab063df98964f2",
+        "version": 1,
+        "locktime": 0,
+        "vin": [
+            {
+                "txid": "d863deb706de5a611028f7547e16ea81d7819e44beb640fb30a9ba30c585140f",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "76a914cd5b6566b455d043558829f6932edaae5d8f0ad388ac",
+                    "scriptpubkey_asm": "OP_DUP OP_HASH160 OP_PUSHBYTES_20 cd5b6566b455d043558829f6932edaae5d8f0ad3 OP_EQUALVERIFY OP_CHECKSIG",
+                    "scriptpubkey_type": "p2pkh",
+                    "scriptpubkey_address": "1Kiq1dyVBzYLWGrBPWjChvKyzB2H95x5RJ",
+                    "value": 799995000
+                },
+                "scriptsig": "483045022100c1fb331d155a7d299a0451d14fa1122b328e0e239afc9ba8dc2aff449ddc5a3a02201c1e19030d1efa432f5069cd369d7ad09a67f68501345e4db35f7b799605f55601210314338e3e191aea3ac9e9292611faeedf0379bbe62c30fd76c7450722a1ac47c6",
+                "scriptsig_asm": "OP_PUSHBYTES_72 3045022100c1fb331d155a7d299a0451d14fa1122b328e0e239afc9ba8dc2aff449ddc5a3a02201c1e19030d1efa432f5069cd369d7ad09a67f68501345e4db35f7b799605f55601 OP_PUSHBYTES_33 0314338e3e191aea3ac9e9292611faeedf0379bbe62c30fd76c7450722a1ac47c6",
+                "is_coinbase": false,
+                "sequence": 4294967293
+            }
+        ],
+        "vout": [
+            {
+                "scriptpubkey": "6a4c5058325b78064160b631b5a15d9078d99c0db066449fb4c59bbfa4d987ba906e2990088b2fce8c33d841b6c47a99f9597000ab04a10b34cd419fc19784d9e36f1a33fd7b000c3bce00b6000c1d1e00614b",
+                "scriptpubkey_asm": "OP_RETURN OP_PUSHDATA1 58325b78064160b631b5a15d9078d99c0db066449fb4c59bbfa4d987ba906e2990088b2fce8c33d841b6c47a99f9597000ab04a10b34cd419fc19784d9e36f1a33fd7b000c3bce00b6000c1d1e00614b",
+                "scriptpubkey_type": "op_return",
+                "value": 0
+            },
+            {
+                "scriptpubkey": "a9144890aae025c84cb72a9730b49ca12595d6f6088d87",
+                "scriptpubkey_asm": "OP_HASH160 OP_PUSHBYTES_20 4890aae025c84cb72a9730b49ca12595d6f6088d OP_EQUAL",
+                "scriptpubkey_type": "p2sh",
+                "scriptpubkey_address": "38Jht2bzmJL4EwoFvvyFzejhfEb4J7KxLb",
+                "value": 155000
+            },
+            {
+                "scriptpubkey": "76a91486e7dad6617303942a448b7f8afe9653e5624a5e88ac",
+                "scriptpubkey_asm": "OP_DUP OP_HASH160 OP_PUSHBYTES_20 86e7dad6617303942a448b7f8afe9653e5624a5e OP_EQUALVERIFY OP_CHECKSIG",
+                "scriptpubkey_type": "p2pkh",
+                "scriptpubkey_address": "1DJKJGApgX4W8BSQ8FRPLqX78UaCskT4r2",
+                "value": 155000
+            },
+            {
+                "scriptpubkey": "76a914cd5b6566b455d043558829f6932edaae5d8f0ad388ac",
+                "scriptpubkey_asm": "OP_DUP OP_HASH160 OP_PUSHBYTES_20 cd5b6566b455d043558829f6932edaae5d8f0ad3 OP_EQUALVERIFY OP_CHECKSIG",
+                "scriptpubkey_type": "p2pkh",
+                "scriptpubkey_address": "1Kiq1dyVBzYLWGrBPWjChvKyzB2H95x5RJ",
+                "value": 799676250
+            }
+        ],
+        "size": 350,
+        "weight": 1400,
+        "fee": 8750,
+        "status": {
+            "confirmed": false
+        },
+        "order": 4066675193,
+        "vsize": 350,
+        "adjustedVsize": 350,
+        "sigops": 8,
+        "feePerVsize": 25,
+        "adjustedFeePerVsize": 25,
+        "effectiveFeePerVsize": 25,
+        "firstSeen": 1691218516,
+        "uid": 512584,
+        "inputs": [],
+        "position": {
+            "block": 0,
+            "vsize": 13846
+        },
+        "bestDescendant": null,
+        "cpfpChecked": true
+    },
+    {
+        "txid": "b7981a624e4261c11f1246314d41e74be56af82eb557bcd054a5e0f94c023668",
+        "version": 2,
+        "locktime": 0,
+        "vin": [
+            {
+                "txid": "b50225a04a1d6fbbfa7a2122bc0580396f614027b3957f476229633576f06130",
+                "vout": 0,
+                "prevout": {
+                    "scriptpubkey": "0014a24f913f8a9c30a4c302c2c78f2fd7addb08fd07",
+                    "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 a24f913f8a9c30a4c302c2c78f2fd7addb08fd07",
+                    "scriptpubkey_type": "v0_p2wpkh",
+                    "scriptpubkey_address": "bc1q5f8ez0u2nsc2fsczctrc7t7h4hds3lg82ewqhz",
+                    "value": 612917
+                },
+                "scriptsig": "",
+                "scriptsig_asm": "",
+                "witness": [
+                    "304402204dd10f14afa41bc76d8278140ff1ec3d3f87f2c207bbb5418cc76dab30d7f6a402207877cc9c6a2c724b6ea7a1c24ac00022469f194fd1a4bd8030bbca1787d3f5f301",
+                    "03dc2c7b687019b40a68d713322675206cc266e34e5340ec982c13ff0222c3b2b6"
+                ],
+                "is_coinbase": false,
+                "sequence": 2147483648
+            }
+        ],
+        "vout": [
+            {
+                "scriptpubkey": "76a9149d32ef812385f3811634e0c0117dd153a5de10a488ac",
+                "scriptpubkey_asm": "OP_DUP OP_HASH160 OP_PUSHBYTES_20 9d32ef812385f3811634e0c0117dd153a5de10a4 OP_EQUALVERIFY OP_CHECKSIG",
+                "scriptpubkey_type": "p2pkh",
+                "scriptpubkey_address": "1FLC7Bag7okAkKPCyZbgZZg3Hh1EuGZ5Rd",
+                "value": 344697
+            },
+            {
+                "scriptpubkey": "00144c2671336ca8761863b4c68d64d4672491fec1b9",
+                "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 4c2671336ca8761863b4c68d64d4672491fec1b9",
+                "scriptpubkey_type": "v0_p2wpkh",
+                "scriptpubkey_address": "bc1qfsn8zvmv4pmpsca5c6xkf4r8yjglasdesrawcx",
+                "value": 267636
+            }
+        ],
+        "size": 225,
+        "weight": 573,
+        "fee": 584,
+        "status": {
+            "confirmed": false
+        },
+        "order": 1748369996,
+        "vsize": 143,
+        "adjustedVsize": 143.25,
+        "sigops": 5,
+        "feePerVsize": 4.076788830715532,
+        "adjustedFeePerVsize": 4.076788830715532,
+        "effectiveFeePerVsize": 4.076788830715532,
+        "firstSeen": 1691222376,
+        "uid": 526515,
+        "inputs": [],
+        "position": {
+            "block": 7,
+            "vsize": 22021095.5
+        },
+        "bestDescendant": null,
+        "cpfpChecked": true
+    }
+]

backend/src/api/blocks.ts

@@ -674,7 +674,11 @@ class Blocks {
           this.updateTimerProgress(timer, 'got previous block hash for initial difficulty adjustment');
           const previousPeriodBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(previousPeriodBlockHash);
           this.updateTimerProgress(timer, 'got previous block for initial difficulty adjustment');
-          this.previousDifficultyRetarget = calcBitsDifference(previousPeriodBlock.bits, block.bits);
+          if (['liquid', 'liquidtestnet'].includes(config.MEMPOOL.NETWORK)) {
+            this.previousDifficultyRetarget = NaN;
+          } else {
+            this.previousDifficultyRetarget = calcBitsDifference(previousPeriodBlock.bits, block.bits);
+          }
           logger.debug(`Initial difficulty adjustment data set.`);
         }
       } else {
@@ -783,20 +787,31 @@ class Blocks {
 
       if (block.height % 2016 === 0) {
         if (Common.indexingEnabled()) {
+          let adjustment;
+          if (['liquid', 'liquidtestnet'].includes(config.MEMPOOL.NETWORK)) {
+            adjustment = NaN;
+          } else {
+            adjustment = Math.round(
+              // calcBitsDifference returns +- percentage, +100 returns to positive, /100 returns to ratio.
+              // Instead of actually doing /100, just reduce the multiplier.
+              (calcBitsDifference(this.currentBits, block.bits) + 100) * 10000
+            ) / 1000000; // Remove float point noise
+          }
+
           await DifficultyAdjustmentsRepository.$saveAdjustments({
             time: block.timestamp,
             height: block.height,
             difficulty: block.difficulty,
-            adjustment: Math.round(
+            adjustment,
-              // calcBitsDifference returns +- percentage, +100 returns to positive, /100 returns to ratio.
-              // Instead of actually doing /100, just reduce the multiplier.
-              (calcBitsDifference(this.currentBits, block.bits) + 100) * 10000
-            ) / 1000000, // Remove float point noise
           });
           this.updateTimerProgress(timer, `saved difficulty adjustment for ${this.currentBlockHeight}`);
         }
 
-        this.previousDifficultyRetarget = calcBitsDifference(this.currentBits, block.bits);
+        if (['liquid', 'liquidtestnet'].includes(config.MEMPOOL.NETWORK)) {
+          this.previousDifficultyRetarget = NaN;
+        } else {
+          this.previousDifficultyRetarget = calcBitsDifference(this.currentBits, block.bits);
+        }
         this.lastDifficultyAdjustmentTime = block.timestamp;
         this.currentBits = block.bits;
       }

backend/src/api/common.ts

@@ -59,10 +59,12 @@ export class Common {
     return arr;
   }
 
-  static findRbfTransactions(added: MempoolTransactionExtended[], deleted: MempoolTransactionExtended[]): { [txid: string]: MempoolTransactionExtended[] } {
+  static findRbfTransactions(added: MempoolTransactionExtended[], deleted: MempoolTransactionExtended[], forceScalable = false): { [txid: string]: MempoolTransactionExtended[] } {
     const matches: { [txid: string]: MempoolTransactionExtended[] } = {};
-    added
+
-      .forEach((addedTx) => {
+    // For small N, a naive nested loop is extremely fast, but it doesn't scale
+    if (added.length < 1000 && deleted.length < 50 && !forceScalable) {
+      added.forEach((addedTx) => {
         const foundMatches = deleted.filter((deletedTx) => {
           // The new tx must, absolutely speaking, pay at least as much fee as the replaced tx.
           return addedTx.fee > deletedTx.fee
@@ -73,9 +75,40 @@ export class Common {
               addedTx.vin.some((vin) => vin.txid === deletedVin.txid && vin.vout === deletedVin.vout));
             });
         if (foundMatches?.length) {
-          matches[addedTx.txid] = foundMatches;
+          matches[addedTx.txid] = [...new Set(foundMatches)];
         }
       });
+    } else {
+      // for large N, build a lookup table of prevouts we can check in ~constant time
+      const deletedSpendMap: { [txid: string]: { [vout: number]: MempoolTransactionExtended } } = {};
+      for (const tx of deleted) {
+        for (const vin of tx.vin) {
+          if (!deletedSpendMap[vin.txid]) {
+            deletedSpendMap[vin.txid] = {};
+          }
+          deletedSpendMap[vin.txid][vin.vout] = tx;
+        }
+      }
+
+      for (const addedTx of added) {
+        const foundMatches = new Set<MempoolTransactionExtended>();
+        for (const vin of addedTx.vin) {
+          const deletedTx = deletedSpendMap[vin.txid]?.[vin.vout];
+          if (deletedTx && deletedTx.txid !== addedTx.txid
+              // The new tx must, absolutely speaking, pay at least as much fee as the replaced tx.
+              && addedTx.fee > deletedTx.fee
+              // The new transaction must pay more fee per kB than the replaced tx.
+              && addedTx.adjustedFeePerVsize > deletedTx.adjustedFeePerVsize
+          ) {
+            foundMatches.add(deletedTx);
+          }
+          if (foundMatches.size) {
+            matches[addedTx.txid] = [...foundMatches];
+          }
+        }
+      }
+    }
+
     return matches;
   }
 

frontend/src/app/bisq/bisq-address/bisq-address.component.ts

@@ -47,6 +47,7 @@ export class BisqAddressComponent implements OnInit, OnDestroy {
               catchError((err) => {
                 this.isLoadingAddress = false;
                 this.error = err;
+                this.seoService.logSoft404();
                 console.log(err);
                 return of(null);
               })
@@ -62,6 +63,7 @@ export class BisqAddressComponent implements OnInit, OnDestroy {
       (error) => {
         console.log(error);
         this.error = error;
+        this.seoService.logSoft404();
         this.isLoadingAddress = false;
       });
   }

frontend/src/app/bisq/bisq-block/bisq-block.component.ts

@@ -82,6 +82,7 @@ export class BisqBlockComponent implements OnInit, OnDestroy {
       )
       .subscribe((block: BisqBlock) => {
         if (!block) {
+          this.seoService.logSoft404();
           return;
         }
         this.isLoading = false;
@@ -97,6 +98,7 @@ export class BisqBlockComponent implements OnInit, OnDestroy {
 
   caughtHttpError(err: HttpErrorResponse){
     this.error = err;
+    this.seoService.logSoft404();
     return of(null);
   }
 }

frontend/src/app/bisq/bisq-transaction/bisq-transaction.component.ts

@@ -70,11 +70,13 @@ export class BisqTransactionComponent implements OnInit, OnDestroy {
                     catchError((txError: HttpErrorResponse) => {
                       console.log(txError);
                       this.error = txError;
+                      this.seoService.logSoft404();
                       return of(null);
                     })
                   );
               }
               this.error = bisqTxError;
+              this.seoService.logSoft404();
               return of(null);
             })
           );
@@ -103,6 +105,7 @@ export class BisqTransactionComponent implements OnInit, OnDestroy {
       this.isLoadingTx = false;
 
       if (!tx) {
+        this.seoService.logSoft404();
         return;
       }
 

frontend/src/app/components/address/address.component.ts

@@ -91,6 +91,7 @@ export class AddressComponent implements OnInit, OnDestroy {
                 catchError((err) => {
                   this.isLoadingAddress = false;
                   this.error = err;
+                  this.seoService.logSoft404();
                   console.log(err);
                   return of(null);
                 })
@@ -162,6 +163,7 @@ export class AddressComponent implements OnInit, OnDestroy {
       (error) => {
         console.log(error);
         this.error = error;
+        this.seoService.logSoft404();
         this.isLoadingAddress = false;
       });
 

frontend/src/app/components/asset/asset.component.ts

@@ -86,6 +86,7 @@ export class AssetComponent implements OnInit, OnDestroy {
                   catchError((err) => {
                     this.isLoadingAsset = false;
                     this.error = err;
+                    this.seoService.logSoft404();
                     console.log(err);
                     return of(null);
                   })
@@ -153,6 +154,7 @@ export class AssetComponent implements OnInit, OnDestroy {
       (error) => {
         console.log(error);
         this.error = error;
+        this.seoService.logSoft404();
         this.isLoadingAsset = false;
       });
 

frontend/src/app/components/block/block-preview.component.ts

@@ -82,6 +82,7 @@ export class BlockPreviewComponent implements OnInit, OnDestroy {
               }),
               catchError((err) => {
                 this.error = err;
+                this.seoService.logSoft404();
                 this.openGraphService.fail('block-data-' + this.rawId);
                 this.openGraphService.fail('block-viz-' + this.rawId);
                 return of(null);
@@ -138,6 +139,7 @@ export class BlockPreviewComponent implements OnInit, OnDestroy {
     (error) => {
       this.error = error;
       this.isLoadingOverview = false;
+      this.seoService.logSoft404();
       this.openGraphService.fail('block-viz-' + this.rawId);
       this.openGraphService.fail('block-data-' + this.rawId);
       if (this.blockGraph) {

frontend/src/app/components/block/block.component.ts

@@ -206,6 +206,7 @@ export class BlockComponent implements OnInit, OnDestroy {
                       this.error = err;
                       this.isLoadingBlock = false;
                       this.isLoadingOverview = false;
+                      this.seoService.logSoft404();
                       return EMPTY;
                     })
                   );
@@ -214,6 +215,7 @@ export class BlockComponent implements OnInit, OnDestroy {
                   this.error = err;
                   this.isLoadingBlock = false;
                   this.isLoadingOverview = false;
+                  this.seoService.logSoft404();
                   return EMPTY;
                 }),
               );
@@ -229,6 +231,7 @@ export class BlockComponent implements OnInit, OnDestroy {
               this.error = err;
               this.isLoadingBlock = false;
               this.isLoadingOverview = false;
+              this.seoService.logSoft404();
               return EMPTY;
             })
           );

frontend/src/app/components/pool/pool-preview.component.ts

@@ -61,6 +61,7 @@ export class PoolPreviewComponent implements OnInit {
               }),
               catchError(() => {
                 this.isLoading = false;
+                this.seoService.logSoft404();
                 this.openGraphService.fail('pool-hash-' + this.slug);
                 return of([slug]);
               })
@@ -70,6 +71,7 @@ export class PoolPreviewComponent implements OnInit {
           return this.apiService.getPoolStats$(slug).pipe(
             catchError(() => {
               this.isLoading = false;
+              this.seoService.logSoft404();
               this.openGraphService.fail('pool-stats-' + this.slug);
               return of(null);
             })

frontend/src/app/components/pool/pool.component.ts

@@ -1,8 +1,8 @@
 import { ChangeDetectionStrategy, Component, Inject, Input, LOCALE_ID, OnInit } from '@angular/core';
 import { ActivatedRoute } from '@angular/router';
 import { EChartsOption, graphic } from 'echarts';
-import { BehaviorSubject, Observable } from 'rxjs';
+import { BehaviorSubject, Observable, of, timer } from 'rxjs';
-import { distinctUntilChanged, map, share, switchMap, tap } from 'rxjs/operators';
+import { catchError, distinctUntilChanged, map, share, switchMap, tap } from 'rxjs/operators';
 import { BlockExtended, PoolStat } from '../../interfaces/node-api.interface';
 import { ApiService } from '../../services/api.service';
 import { StateService } from '../../services/state.service';
@@ -62,10 +62,21 @@ export class PoolComponent implements OnInit {
                 this.prepareChartOptions(data.map(val => [val.timestamp * 1000, val.avgHashrate]));
                 return [slug];
               }),
+              catchError(() => {
+                this.isLoading = false;
+                this.seoService.logSoft404();
+                return of([slug]);
+              })
             );
         }),
         switchMap((slug) => {
-          return this.apiService.getPoolStats$(slug);
+          return this.apiService.getPoolStats$(slug).pipe(
+            catchError(() => {
+              this.isLoading = false;
+              this.seoService.logSoft404();
+              return of(null);
+            })
+          );
         }),
         tap(() => {
           this.loadMoreSubject.next(this.blocks[0]?.height);

frontend/src/app/components/transaction/transaction-preview.component.ts

@@ -133,6 +133,7 @@ export class TransactionPreviewComponent implements OnInit, OnDestroy {
       )
       .subscribe((tx: Transaction) => {
           if (!tx) {
+            this.seoService.logSoft404();
             this.openGraphService.fail('tx-data-' + this.txId);
             return;
           }
@@ -182,6 +183,7 @@ export class TransactionPreviewComponent implements OnInit, OnDestroy {
           this.openGraphService.waitOver('tx-data-' + this.txId);
         },
         (error) => {
+          this.seoService.logSoft404();
           this.openGraphService.fail('tx-data-' + this.txId);
           this.error = error;
           this.isLoadingTx = false;

frontend/src/app/components/transaction/transaction.component.ts

@@ -220,8 +220,10 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
     ).subscribe((tx) => {
       this.loadingCachedTx = false;
       if (!tx) {
+        this.seoService.logSoft404();
         return;
       }
+      this.seoService.clearSoft404();
 
       if (!this.tx) {
         this.tx = tx;
@@ -338,8 +340,10 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
       .subscribe((tx: Transaction) => {
           if (!tx) {
             this.fetchCachedTx$.next(this.txId);
+            this.seoService.logSoft404();
             return;
           }
+          this.seoService.clearSoft404();
 
           this.tx = tx;
           this.setFeatures();
@@ -400,6 +404,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
         },
         (error) => {
           this.error = error;
+          this.seoService.logSoft404();
           this.isLoadingTx = false;
         }
       );
@@ -487,6 +492,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
       this.waitingForTransaction = true;
     }
     this.error = error;
+    this.seoService.logSoft404();
     this.isLoadingTx = false;
     return of(false);
   }

frontend/src/app/lightning/channel/channel-preview.component.ts

@@ -54,6 +54,7 @@ export class ChannelPreviewComponent implements OnInit {
               }),
               catchError((err) => {
                 this.error = err;
+                this.seoService.logSoft404();
                 this.openGraphService.fail('channel-map-' + this.shortId);
                 this.openGraphService.fail('channel-data-' + this.shortId);
                 return of(null);

frontend/src/app/lightning/channel/channel.component.ts

@@ -38,6 +38,7 @@ export class ChannelComponent implements OnInit {
               }),
               catchError((err) => {
                 this.error = err;
+                this.seoService.logSoft404();
                 return [{
                   short_id: params.get('short_id')
                 }];

frontend/src/app/lightning/group/group-preview.component.ts

@@ -50,6 +50,7 @@ export class GroupPreviewComponent implements OnInit {
               name: this.slug.replace(/-/gi, ' '),
               description: '',
             };
+            this.seoService.logSoft404();
             this.openGraphService.fail('ln-group-map-' + this.slug);
             this.openGraphService.fail('ln-group-data-' + this.slug);
             return of(null);
@@ -106,6 +107,7 @@ export class GroupPreviewComponent implements OnInit {
           };
         }),
         catchError(() => {
+          this.seoService.logSoft404();
           this.openGraphService.fail('ln-group-map-' + this.slug);
           this.openGraphService.fail('ln-group-data-' + this.slug);
           return of({

frontend/src/app/lightning/node/node-preview.component.ts

@@ -81,6 +81,7 @@ export class NodePreviewComponent implements OnInit {
         }),
         catchError(err => {
           this.error = err;
+          this.seoService.logSoft404();
           this.openGraphService.fail('node-map-' + this.publicKey);
           this.openGraphService.fail('node-data-' + this.publicKey);
           return [{

frontend/src/app/lightning/node/node.component.ts

@@ -123,6 +123,7 @@ export class NodeComponent implements OnInit {
         }),
         catchError(err => {
           this.error = err;
+          this.seoService.logSoft404();
           return [{
             alias: this.publicKey,
             public_key: this.publicKey,

frontend/src/app/lightning/nodes-per-isp/nodes-per-isp-preview.component.ts

@@ -85,6 +85,7 @@ export class NodesPerISPPreview implements OnInit {
         }),
         catchError(err => {
           this.error = err;
+          this.seoService.logSoft404();
           this.openGraphService.fail('isp-map-' + this.id);
           this.openGraphService.fail('isp-data-' + this.id);
           return of({

frontend/src/app/services/seo.service.ts

@@ -1,5 +1,7 @@
 import { Injectable } from '@angular/core';
 import { Title, Meta } from '@angular/platform-browser';
+import { ActivatedRoute, NavigationEnd, Router } from '@angular/router';
+import { filter, map, switchMap } from 'rxjs';
 import { StateService } from './state.service';
 
 @Injectable({
@@ -13,8 +15,22 @@ export class SeoService {
     private titleService: Title,
     private metaService: Meta,
     private stateService: StateService,
+    private router: Router,
+    private activatedRoute: ActivatedRoute,
   ) {
     this.stateService.networkChanged$.subscribe((network) => this.network = network);
+    this.router.events.pipe(
+      filter(event => event instanceof NavigationEnd),
+      map(() => this.activatedRoute),
+      map(route => {
+        while (route.firstChild) route = route.firstChild;
+        return route;
+      }),
+      filter(route => route.outlet === 'primary'),
+      switchMap(route => route.data),
+    ).subscribe((data) => {
+      this.clearSoft404();
+    });
   }
 
   setTitle(newTitle: string): void {
@@ -53,4 +69,14 @@ export class SeoService {
   ucfirst(str: string) {
     return str.charAt(0).toUpperCase() + str.slice(1);
   }
+
+  clearSoft404() {
+    window['soft404'] = false;
+    console.log('cleared soft 404');
+  }
+
+  logSoft404() {
+    window['soft404'] = true;
+    console.log('set soft 404');
+  }
 }

unfurler/src/concurrency/ReusableSSRPage.ts

@@ -0,0 +1,65 @@
+import * as puppeteer from 'puppeteer';
+import { timeoutExecute } from 'puppeteer-cluster/dist/util';
+import logger from '../logger';
+import config from '../config';
+import ReusablePage from './ReusablePage';
+const mempoolHost = config.MEMPOOL.HTTP_HOST + (config.MEMPOOL.HTTP_PORT ? ':' + config.MEMPOOL.HTTP_PORT : '');
+
+const mockImageBuffer = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVQYV2NgYAAAAAMAAWgmWQ0AAAAASUVORK5CYII=", 'base64');
+
+interface RepairablePage extends puppeteer.Page {
+  repairRequested?: boolean;
+  language?: string | null;
+  createdAt?: number;
+  free?: boolean;
+  index?: number;
+}
+
+export default class ReusableSSRPage extends ReusablePage {
+
+  public constructor(options: puppeteer.LaunchOptions, puppeteer: any) {
+    super(options, puppeteer);
+  }
+
+  public async close() {
+    await (this.browser as puppeteer.Browser).close();
+  }
+
+  protected async initPage(): Promise<RepairablePage> {
+    const page = await (this.browser as puppeteer.Browser).newPage() as RepairablePage;
+    page.language = null;
+    page.createdAt = Date.now();
+    const defaultUrl = mempoolHost + '/about';
+
+    page.on('pageerror', (err) => {
+      console.log(err);
+      // page.repairRequested = true;
+    });
+    await page.setRequestInterception(true);
+      page.on('request', req => {
+        if (req.isInterceptResolutionHandled()) {
+          return req.continue();
+        }
+        if (req.resourceType() === 'image') {
+          return req.respond({
+            contentType: 'image/png',
+            headers: {"Access-Control-Allow-Origin": "*"},
+            body: mockImageBuffer
+          });
+        } else if (!['document', 'script', 'xhr', 'fetch'].includes(req.resourceType())) {
+          return req.abort();
+        } else {
+          return req.continue();
+        }
+      });
+    try {
+      await page.goto(defaultUrl, { waitUntil: "networkidle0" });
+      await page.waitForSelector('meta[property="og:meta:ready"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 });
+    } catch (e) {
+      logger.err(`failed to load frontend during ssr page initialization: ` + (e instanceof Error ? e.message : `${e}`));
+      page.repairRequested = true;
+    }
+    page.free = true;
+    return page
+  }
+}

unfurler/src/index.ts

@@ -5,9 +5,11 @@ import * as https from 'https';
 import config from './config';
 import { Cluster } from 'puppeteer-cluster';
 import ReusablePage from './concurrency/ReusablePage';
+import ReusableSSRPage from './concurrency/ReusablePage';
 import { parseLanguageUrl } from './language/lang';
 import { matchRoute } from './routes';
 import logger from './logger';
+import { TimeoutError } from "puppeteer";
 const puppeteerConfig = require('../puppeteer.config.json');
 
 if (config.PUPPETEER.EXEC_PATH) {
@@ -20,15 +22,33 @@ class Server {
   private server: http.Server | undefined;
   private app: Application;
   cluster?: Cluster;
+  ssrCluster?: Cluster;
   mempoolHost: string;
+  mempoolUrl: URL;
   network: string;
   secureHost = true;
+  canonicalHost: string;
 
   constructor() {
     this.app = express();
     this.mempoolHost = config.MEMPOOL.HTTP_HOST + (config.MEMPOOL.HTTP_PORT ? ':' + config.MEMPOOL.HTTP_PORT : '');
+    this.mempoolUrl = new URL(this.mempoolHost);
     this.secureHost = config.SERVER.HOST.startsWith('https');
     this.network = config.MEMPOOL.NETWORK || 'bitcoin';
+
+    let canonical;
+    switch(config.MEMPOOL.NETWORK) {
+      case "liquid":
+        canonical = "https://liquid.network"
+        break;
+      case "bisq":
+        canonical = "https://bisq.markets"
+        break;
+      default:
+        canonical = "https://mempool.space"
+    }
+    this.canonicalHost = canonical;
+
     this.startServer();
   }
 
@@ -49,6 +69,12 @@ class Server {
           puppeteerOptions: puppeteerConfig,
       });
       await this.cluster?.task(async (args) => { return this.clusterTask(args) });
+      this.ssrCluster = await Cluster.launch({
+        concurrency: ReusableSSRPage,
+        maxConcurrency: config.PUPPETEER.CLUSTER_SIZE,
+        puppeteerOptions: puppeteerConfig,
+      });
+      await this.ssrCluster?.task(async (args) => { return this.ssrClusterTask(args) });
     }
 
     this.setUpRoutes();
@@ -65,6 +91,10 @@ class Server {
       await this.cluster.idle();
       await this.cluster.close();
     }
+    if (this.ssrCluster) {
+      await this.ssrCluster.idle();
+      await this.ssrCluster.close();
+    }
     if (this.server) {
       await this.server.close();
     }
@@ -102,8 +132,8 @@ class Server {
       }
 
       // wait for preview component to initialize
-      await page.waitForSelector('meta[property="og:preview:loading"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 })
       let success;
+      await page.waitForSelector('meta[property="og:preview:loading"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 })
       success = await Promise.race([
         page.waitForSelector('meta[property="og:preview:ready"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 }).then(() => true),
         page.waitForSelector('meta[property="og:preview:fail"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 }).then(() => false)
@@ -127,6 +157,51 @@ class Server {
     }
   }
 
+  async ssrClusterTask({ page, data: { url, path, action } }) {
+    try {
+      const urlParts = parseLanguageUrl(path);
+      if (page.language !== urlParts.lang) {
+        // switch language
+        page.language = urlParts.lang;
+        const localizedUrl = urlParts.lang ? `${this.mempoolHost}/${urlParts.lang}${urlParts.path}` : `${this.mempoolHost}${urlParts.path}` ;
+        await page.goto(localizedUrl, { waitUntil: "load" });
+      } else {
+        const loaded = await page.evaluate(async (path) => {
+          if (window['ogService']) {
+            window['ogService'].loadPage(path);
+            return true;
+          } else {
+            return false;
+          }
+        }, urlParts.path);
+        if (!loaded) {
+          throw new Error('failed to access open graph service');
+        }
+      }
+
+      await page.waitForNetworkIdle({
+        timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000,
+      });
+      const is404 = await page.evaluate(async () => {
+        return !!window['soft404'];
+      });
+      if (is404) {
+        return '404';
+      } else {
+        let html = await page.content();
+        return html;
+      }
+    } catch (e) {
+      if (e instanceof TimeoutError) {
+        let html = await page.content();
+        return html;
+      } else {
+        logger.err(`failed to render ${path} for ${action}: ` + (e instanceof Error ? e.message : `${e}`));
+        page.repairRequested = true;
+      }
+    }
+  }
+
   async renderDisabled(req, res) {
     res.status(500).send("preview rendering disabled");
   }
@@ -166,43 +241,92 @@ class Server {
     // drop requests for static files
     const rawPath = req.params[0];
     const match = rawPath.match(/\.[\w]+$/);
-    if (match?.length && match[0] !== '.html') {
+    if (match?.length && match[0] !== '.html'
-      res.status(404).send();
+      || rawPath.startsWith('/api/v1/donations/images')
-      return;
+      || rawPath.startsWith('/api/v1/contributors/images')
+      || rawPath.startsWith('/api/v1/translators/images')
+      || rawPath.startsWith('/resources/profile')
+    ) {
+      if (isSearchCrawler(req.headers['user-agent'])) {
+        if (this.secureHost) {
+          https.get(config.SERVER.HOST + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => got.pipe(res));
+        } else {
+          http.get(config.SERVER.HOST + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => got.pipe(res));
+        }
+        return;
+      } else {
+        res.status(404).send();
+        return;
+      }
     }
 
+    let result = '';
+    try {
+      if (isSearchCrawler(req.headers['user-agent'])) {
+        result = await this.renderSEOPage(rawPath);
+      } else {
+        result = await this.renderUnfurlMeta(rawPath);
+      }
+      if (result && result.length) {
+        if (result === '404') {
+          res.status(404).send();
+        } else {
+          res.send(result);
+        }
+      } else {
+        res.status(500).send();
+      }
+    } catch (e) {
+      logger.err(e instanceof Error ? e.message : `${e} ${req.params[0]}`);
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  async renderUnfurlMeta(rawPath: string): Promise<string> {
     const { lang, path } = parseLanguageUrl(rawPath);
     const matchedRoute = matchRoute(this.network, path);
     let ogImageUrl = config.SERVER.HOST + (matchedRoute.staticImg || matchedRoute.fallbackImg);
     let ogTitle = 'The Mempool Open Source Project®';
 
+    const canonical = this.canonicalHost + rawPath;
+
     if (matchedRoute.render) {
       ogImageUrl = `${config.SERVER.HOST}/render/${lang || 'en'}/preview${path}`;
       ogTitle = `${this.network ? capitalize(this.network) + ' ' : ''}${matchedRoute.networkMode !== 'mainnet' ? capitalize(matchedRoute.networkMode) + ' ' : ''}${matchedRoute.title}`;
     }
 
-    res.send(`
+    return `<!doctype html>
-      <!doctype html>
+<html lang="en-US" dir="ltr">
-      <html lang="en-US" dir="ltr">
+  <head>
-      <head>
+    <meta charset="utf-8">
-        <meta charset="utf-8">
+    <title>${ogTitle}</title>
-        <title>${ogTitle}</title>
+    <link rel="canonical" href="${canonical}" />
-        <meta name="description" content="The Mempool Open Source Project® - Explore the full Bitcoin ecosystem with mempool.space™"/>
+    <meta name="description" content="The Mempool Open Source Project® - Explore the full Bitcoin ecosystem with mempool.space™"/>
-        <meta property="og:image" content="${ogImageUrl}"/>
+    <meta property="og:image" content="${ogImageUrl}"/>
-        <meta property="og:image:type" content="image/png"/>
+    <meta property="og:image:type" content="image/png"/>
-        <meta property="og:image:width" content="${matchedRoute.render ? 1200 : 1000}"/>
+    <meta property="og:image:width" content="${matchedRoute.render ? 1200 : 1000}"/>
-        <meta property="og:image:height" content="${matchedRoute.render ? 600 : 500}"/>
+    <meta property="og:image:height" content="${matchedRoute.render ? 600 : 500}"/>
-        <meta property="og:title" content="${ogTitle}">
+    <meta property="og:title" content="${ogTitle}">
-        <meta property="twitter:card" content="summary_large_image">
+    <meta property="twitter:card" content="summary_large_image">
-        <meta property="twitter:site" content="@mempool">
+    <meta property="twitter:site" content="@mempool">
-        <meta property="twitter:creator" content="@mempool">
+    <meta property="twitter:creator" content="@mempool">
-        <meta property="twitter:title" content="${ogTitle}">
+    <meta property="twitter:title" content="${ogTitle}">
-        <meta property="twitter:description" content="Explore the full Bitcoin ecosystem with mempool.space"/>
+    <meta property="twitter:description" content="Explore the full Bitcoin ecosystem with mempool.space"/>
-        <meta property="twitter:image:src" content="${ogImageUrl}"/>
+    <meta property="twitter:image:src" content="${ogImageUrl}"/>
-        <meta property="twitter:domain" content="mempool.space">
+    <meta property="twitter:domain" content="mempool.space">
-      <body></body>
+  </head>
-      </html>
+  <body></body>
-    `);
+</html>`;
+  }
+
+  async renderSEOPage(rawPath: string): Promise<string> {
+    let html = await this.ssrCluster?.execute({ url: this.mempoolHost + rawPath, path: rawPath, action: 'ssr' });
+    // remove javascript to prevent double hydration
+    if (html && html.length) {
+      html = html.replaceAll(/<script.*<\/script>/g, "");
+      html = html.replaceAll(this.mempoolHost, this.canonicalHost);
+    }
+    return html;
   }
 }
 
@@ -221,3 +345,7 @@ function capitalize(str) {
     return str;
   }
 }
+
+function isSearchCrawler(useragent: string): boolean {
+  return /googlebot|applebot|bingbot/i.test(useragent);
+}