diff --git a/backend/.gitignore b/backend/.gitignore index 7610c445c..b4393c2f0 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -45,3 +45,6 @@ testem.log #System Files .DS_Store Thumbs.db + +# package folder (npm run package output) +/package diff --git a/backend/npm_package.sh b/backend/npm_package.sh index 627a77360..db305f381 100755 --- a/backend/npm_package.sh +++ b/backend/npm_package.sh @@ -1,14 +1,17 @@ #/bin/sh set -e +# Remove previous dist folder +rm -rf dist +# Build new dist folder npm run build # Remove previous package folder rm -rf package # Move JS and deps mv dist package -mv node_modules package +cp -R node_modules package # Remove symlink for rust-gbt and insert real folder rm package/node_modules/rust-gbt -mv rust-gbt package/node_modules +cp -R rust-gbt package/node_modules # Clean up deps npm run package-rm-build-deps diff --git a/backend/package.json b/backend/package.json index e21924207..d1cdce286 100644 --- a/backend/package.json +++ b/backend/package.json @@ -22,7 +22,7 @@ "main": "index.ts", "scripts": { "tsc": "./node_modules/typescript/bin/tsc -p tsconfig.build.json", - "build": "npm run tsc && npm run create-resources", + "build": "npm run rust-build && npm run tsc && npm run create-resources", "create-resources": "cp ./src/tasks/price-feeds/mtgox-weekly.json ./dist/tasks && node dist/api/fetch-version.js", "package": "./npm_package.sh", "package-rm-build-deps": "./npm_package_rm_build_deps.sh", @@ -33,7 +33,8 @@ "test": "./node_modules/.bin/jest --coverage", "lint": "./node_modules/.bin/eslint . --ext .ts", "lint:fix": "./node_modules/.bin/eslint . --ext .ts --fix", - "prettier": "./node_modules/.bin/prettier --write \"src/**/*.{js,ts}\"" + "prettier": "./node_modules/.bin/prettier --write \"src/**/*.{js,ts}\"", + "rust-build": "cd rust-gbt && npm run build-release" }, "dependencies": { "@babel/core": "^7.21.3", diff --git a/backend/src/__tests__/api/common.ts b/backend/src/__tests__/api/common.ts new file mode 100644 index 000000000..10d6df868 --- /dev/null +++ b/backend/src/__tests__/api/common.ts @@ -0,0 +1,24 @@ +import { Common } from '../../api/common'; +import { MempoolTransactionExtended } from '../../mempool.interfaces'; + +const randomTransactions = require('./test-data/transactions-random.json'); +const replacedTransactions = require('./test-data/transactions-replaced.json'); +const rbfTransactions = require('./test-data/transactions-rbfs.json'); + +describe('Mempool Utils', () => { + test('should detect RBF transactions with fast method', () => { + const newTransactions = rbfTransactions.concat(randomTransactions); + const result: { [txid: string]: MempoolTransactionExtended[] } = Common.findRbfTransactions(newTransactions, replacedTransactions); + expect(Object.values(result).length).toEqual(2); + expect(result).toHaveProperty('7219d95161f3718335991ac6d967d24eedec370908c9879bb1e192e6d797d0a6'); + expect(result).toHaveProperty('5387881d695d4564d397026dc5f740f816f8390b4b2c5ec8c20309122712a875'); + }); + + test.only('should detect RBF transactions with scalable method', () => { + const newTransactions = rbfTransactions.concat(randomTransactions); + const result: { [txid: string]: MempoolTransactionExtended[] } = Common.findRbfTransactions(newTransactions, replacedTransactions, true); + expect(Object.values(result).length).toEqual(2); + expect(result).toHaveProperty('7219d95161f3718335991ac6d967d24eedec370908c9879bb1e192e6d797d0a6'); + expect(result).toHaveProperty('5387881d695d4564d397026dc5f740f816f8390b4b2c5ec8c20309122712a875'); + }); +}); diff --git a/backend/src/__tests__/api/test-data/transactions-random.json b/backend/src/__tests__/api/test-data/transactions-random.json new file mode 100644 index 000000000..6bf4a0edd --- /dev/null +++ b/backend/src/__tests__/api/test-data/transactions-random.json @@ -0,0 +1,277 @@ +[ + { + "txid": "13f007241d78e8b0b4e57d2ae3fd37bcfe3226534d7cadeba5a549860d960db0", + "version": 2, + "locktime": 0, + "vin": [ + { + "txid": "cb8f206f4e88bec97107089f3e9e61d50cde53d4541992ae19759b71103cf75c", + "vout": 0, + "prevout": { + "scriptpubkey": "0014fd6d15ff832c12f1ff04a5ccd5039f7227b260bd", + "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 fd6d15ff832c12f1ff04a5ccd5039f7227b260bd", + "scriptpubkey_type": "v0_p2wpkh", + "scriptpubkey_address": "bc1ql4k3tlur9sf0rlcy5hxd2qulwgnmyc9akehvth", + "value": 610677 + }, + "scriptsig": "", + "scriptsig_asm": "", + "witness": [ + "304302205c430b36ebd2bb327951d83440af1f58f127871b2baada4c4dde2bc0b6721f56021f3445099f1a40e35baeda32e8e3727b505ffba0d882b11f498c7762f4184e9901", + "0236b5edd4fbbcfb045960e42ec8a9968944084785932e32940e8cd2583b37da67" + ], + "is_coinbase": false, + "sequence": 2147483648 + } + ], + "vout": [ + { + "scriptpubkey": "76a9149d32ef812385f3811634e0c0117dd153a5de10a488ac", + "scriptpubkey_asm": "OP_DUP OP_HASH160 OP_PUSHBYTES_20 9d32ef812385f3811634e0c0117dd153a5de10a4 OP_EQUALVERIFY OP_CHECKSIG", + "scriptpubkey_type": "p2pkh", + "scriptpubkey_address": "1FLC7Bag7okAkKPCyZbgZZg3Hh1EuGZ5Rd", + "value": 344697 + }, + { + "scriptpubkey": "00147dee8a7a38abbfb00dbfba365c8d6712934cc491", + "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 7dee8a7a38abbfb00dbfba365c8d6712934cc491", + "scriptpubkey_type": "v0_p2wpkh", + "scriptpubkey_address": "bc1q0hhg573c4wlmqrdlhgm9ert8z2f5e3y3lf9hvx", + "value": 265396 + } + ], + "size": 224, + "weight": 572, + "fee": 584, + "status": { + "confirmed": false + }, + "order": 2953680397, + "vsize": 143, + "adjustedVsize": 143, + "sigops": 5, + "feePerVsize": 4.083916083916084, + "adjustedFeePerVsize": 4.083916083916084, + "effectiveFeePerVsize": 4.083916083916084, + "firstSeen": 1691222538, + "uid": 526973, + "inputs": [ + 526728 + ], + "position": { + "block": 7, + "vsize": 21429708.5 + }, + "bestDescendant": null, + "cpfpChecked": true + }, + { + "txid": "8e89b20f8a7fadb0e4cdbe57a00eee224f5076bac5387fc276916724e7c4a16a", + "version": 2, + "locktime": 800571, + "vin": [ + { + "txid": "35e16762459539f3a8e52c5dee6a9ccaa9e9268efed33aa2c6e1b7805e849f24", + "vout": 0, + "prevout": { + "scriptpubkey": "0014d4f16ef275b3e1c4a4ecbef55a164933e0f6460f", + "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 d4f16ef275b3e1c4a4ecbef55a164933e0f6460f", + "scriptpubkey_type": "v0_p2wpkh", + "scriptpubkey_address": "bc1q6nckaun4k0suff8vhm6459jfx0s0v3s0ff4ukl", + "value": 1528924 + }, + "scriptsig": "", + "scriptsig_asm": "", + "witness": [ + "3044022019008b26e885bb43da25a11ffac147a057722072eedb68411f114f6e7eb82ebc02201b618264bb97756b88fc3bbc365b73044ac18b33b1067e31cfd5bcd0f50ed2c701", + "039b71145070bd3e8af28e27fa577f2e12ab6bb4e212d3eeaef08b4bc39e8cbc13" + ], + "is_coinbase": false, + "sequence": 4294967293 + }, + { + "txid": "67c27ed0f767526234bcd5f795a31fab8ec4d0251bf12c68f2746951f4110d90", + "vout": 3, + "prevout": { + "scriptpubkey": "0014a7c3d613b321375054b2ac9b6114367bc034ad6f", + "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 a7c3d613b321375054b2ac9b6114367bc034ad6f", + "scriptpubkey_type": "v0_p2wpkh", + "scriptpubkey_address": "bc1q5lpavyanyym4q49j4jdkz9pk00qrftt0yqzvk3", + "value": 436523 + }, + "scriptsig": "", + "scriptsig_asm": "", + "witness": [ + "304402204e67285fc656bc45ed082499b076d5dba2fa21d0d7e64a0ae52b19d69a11760002200f037d81ee540b74397844513b72b08ed92b06db76bd20b08f7a0a3b36ab13d501", + "02a3ebae85f0225b6fbb5ff060afce683a4683507a57544605a29ee7d287e591b4" + ], + "is_coinbase": false, + "sequence": 4294967293 + }, + { + "txid": "21c38fb9a2521e438c614f53b19ddd7a5594bcc4b77480e762fd4b702fad3374", + "vout": 1, + "prevout": { + "scriptpubkey": "00149660e34ef88106536c816c037b5b28dd64a812e2", + "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 9660e34ef88106536c816c037b5b28dd64a812e2", + "scriptpubkey_type": "v0_p2wpkh", + "scriptpubkey_address": "bc1qjeswxnhcsyr9xmypdsphkkegm4j2syhztgzxv4", + "value": 758149 + }, + "scriptsig": "", + "scriptsig_asm": "", + "witness": [ + "3044022021b556f0aa99329076bcc435338aceaf534963efcab306931b1b2b0461e16e0c02203a78942a3745c4da656bddfd8cf16b85dc04d652904e88682127cdd9ca63339001", + "0298963be4a8f66aca9fcf1c6dc95547aeaa82347543190c91e094c2321142b9f0" + ], + "is_coinbase": false, + "sequence": 4294967293 + }, + { + "txid": "aa998dbae65240a7386bf7d468459551d99c3de8e2f9057ff5f2d38e17daf788", + "vout": 0, + "prevout": { + "scriptpubkey": "00147bb7413a39943b21ded98ad5e6ad7a222d273e17", + "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 7bb7413a39943b21ded98ad5e6ad7a222d273e17", + "scriptpubkey_type": "v0_p2wpkh", + "scriptpubkey_address": "bc1q0wm5zw3ejsajrhke3t27dtt6ygkjw0sh9lltg6", + "value": 1067200 + }, + "scriptsig": "", + "scriptsig_asm": "", + "witness": [ + "304402205e2269f7d4ee0513b34354c38e920aef2dabac6f4350afb2dd105ff3ee43ae7b02202870322f2cb85cb0b2b0e38152f018bfff271dc3ec5aed0515854d0b259aaf3d01", + "03b87320cf3263a644a0d3f89c1b4a7304d9dfda9eb8c891560716abcb73e88b99" + ], + "is_coinbase": false, + "sequence": 4294967293 + }, + { + "txid": "230253d195d779d4688ba16993985cd27b2e7a687d8b889b3bc63f19ece36f20", + "vout": 0, + "prevout": { + "scriptpubkey": "001439647bd997819d12dfc72b0fb9ff9ffcb84946f8", + "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 39647bd997819d12dfc72b0fb9ff9ffcb84946f8", + "scriptpubkey_type": "v0_p2wpkh", + "scriptpubkey_address": "bc1q89j8hkvhsxw39h789v8mnlulljuyj3hc9zve97", + "value": 361950 + }, + "scriptsig": "", + "scriptsig_asm": "", + "witness": [ + "304402204f7ca868bb9b92a07fecdc6b9dd56e4e1d007ca1405952f98ed6bc416345b5f2022055320a97791417abf6628fcf6513ac5785b06c630f854d8595e96ea06c3841d301", + "03a3ffe8e3ef2eea129b227e9658164bae0a6d21c17da6de9973ba34d9e04b21a0" + ], + "is_coinbase": false, + "sequence": 4294967293 + }, + { + "txid": "670771e265a0b62dbd3c1fec2b865177eaf0bafd0ae49dd40a1c9fcd9a847a81", + "vout": 0, + "prevout": { + "scriptpubkey": "0014d45d1b0022c7387e42c5452ced561bdb8fd4b521", + "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 d45d1b0022c7387e42c5452ced561bdb8fd4b521", + "scriptpubkey_type": "v0_p2wpkh", + "scriptpubkey_address": "bc1q63w3kqpzcuu8usk9g5kw64smmw8afdfpxmc2m0", + "value": 453275 + }, + "scriptsig": "", + "scriptsig_asm": "", + "witness": [ + "3044022071312921800441903b2099e723add8702dd0f92ec11526ff87acf6967ec64cbd02203deabe7ed56d5daaa9a95c5a607b1ab705ff1c46bc6984a6dca120e63a91768601", + "0257302ac8d9c4c8f9b1744f19bb432359326b9cc7bdddeeab9202749a6d92be58" + ], + "is_coinbase": false, + "sequence": 4294967293 + }, + { + "txid": "0af82159eee2b69242f2ff032636e410b67ec1ace52e55fb0d20ed814cd64803", + "vout": 0, + "prevout": { + "scriptpubkey": "001459e4d6bfefc6b45f955a69c4aeca26348e9d54ed", + "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 59e4d6bfefc6b45f955a69c4aeca26348e9d54ed", + "scriptpubkey_type": "v0_p2wpkh", + "scriptpubkey_address": "bc1qt8jdd0l0c669l926d8z2aj3xxj8f648dtyn7tc", + "value": 439961 + }, + "scriptsig": "", + "scriptsig_asm": "", + "witness": [ + "3044022027540322e92c23c5513aa2587e7feb56a8ce82f879269d6b3cbd425634b44f8e022045572dee7262b02130bfe32d8aa8abbfaa64e101abfc819bba5380c78876692d01", + "03fe02262d87f4a5289d3dd66e3d9a74cd49fa1cad0249284a7451896a827249a5" + ], + "is_coinbase": false, + "sequence": 4294967293 + }, + { + "txid": "68cf9c784870a4f888f044755f7ce318557f652461db8ef887d279672f186018", + "vout": 0, + "prevout": { + "scriptpubkey": "001454822b2d5d52597a78b630921cf439a41e32f2f9", + "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 54822b2d5d52597a78b630921cf439a41e32f2f9", + "scriptpubkey_type": "v0_p2wpkh", + "scriptpubkey_address": "bc1q2jpzkt2a2fvh579kxzfpeape5s0r9uhewhl5n4", + "value": 227639 + }, + "scriptsig": "", + "scriptsig_asm": "", + "witness": [ + "304402203ad511d6a8730748b8828bc38897d360451adf620ebdc1d229c08c097c80bef202202f50c793d95b5200cf2258e03896a3be7720df0eb3b8c810c86db74341a7e83e01", + "0294992e9f4546e6e119741f908411ae531e9d1ff732d69b4dff8172aaf2a4b216" + ], + "is_coinbase": false, + "sequence": 4294967293 + }, + { + "txid": "793f01dfdb19bf41f958fd917c16d9c4dd5d5e1a5c0434bfdb367212659d1b5b", + "vout": 0, + "prevout": { + "scriptpubkey": "0014f54edf8ae647b5300e2674523254e923d93d169f", + "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 f54edf8ae647b5300e2674523254e923d93d169f", + "scriptpubkey_type": "v0_p2wpkh", + "scriptpubkey_address": "bc1q748dlzhxg76nqr3xw3fry48fy0vn695lvhlkxv", + "value": 227070 + }, + "scriptsig": "", + "scriptsig_asm": "", + "witness": [ + "304402206e807ab616f4f2887ba703ae744d856142d9aca8128698419bbb67fb4fad8177022060fc65c7cd66baa88ad1e1d317a6edd5f6cb52fe8bff6e5405ffa1acf9d945d901", + "02a0ad0167c6e9edf62677404d74d3b80ea276e47e758ffaa6ca17bd65ac79f7aa" + ], + "is_coinbase": false, + "sequence": 4294967293 + } + ], + "vout": [ + { + "scriptpubkey": "00148a5c45ccfc29d209940d94525e2edb7743a1ad8a", + "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 8a5c45ccfc29d209940d94525e2edb7743a1ad8a", + "scriptpubkey_type": "v0_p2wpkh", + "scriptpubkey_address": "bc1q3fwytn8u98fqn9qdj3f9utkmwap6rtv2ym33zm", + "value": 5500000 + } + ], + "size": 1375, + "weight": 2605, + "fee": 691, + "status": { + "confirmed": false + }, + "order": 1788986599, + "vsize": 651, + "adjustedVsize": 651.25, + "sigops": 9, + "feePerVsize": 1.0610364683301343, + "adjustedFeePerVsize": 1.0610364683301343, + "effectiveFeePerVsize": 1.0610364683301343, + "firstSeen": 1691163298, + "uid": 120494, + "inputs": [], + "position": { + "block": 7, + "vsize": 93780091.5 + }, + "bestDescendant": null, + "cpfpChecked": true + } +] \ No newline at end of file diff --git a/backend/src/__tests__/api/test-data/transactions-rbfs.json b/backend/src/__tests__/api/test-data/transactions-rbfs.json new file mode 100644 index 000000000..4e5989cc9 --- /dev/null +++ b/backend/src/__tests__/api/test-data/transactions-rbfs.json @@ -0,0 +1,121 @@ +[ + { + "txid": "7219d95161f3718335991ac6d967d24eedec370908c9879bb1e192e6d797d0a6", + "version": 1, + "locktime": 0, + "vin": [ + { + "txid": "d863deb706de5a611028f7547e16ea81d7819e44beb640fb30a9ba30c585140f", + "vout": 0, + "prevout": { + "scriptpubkey": "76a914cd5b6566b455d043558829f6932edaae5d8f0ad388ac", + "scriptpubkey_asm": "OP_DUP OP_HASH160 OP_PUSHBYTES_20 cd5b6566b455d043558829f6932edaae5d8f0ad3 OP_EQUALVERIFY OP_CHECKSIG", + "scriptpubkey_type": "p2pkh", + "scriptpubkey_address": "1Kiq1dyVBzYLWGrBPWjChvKyzB2H95x5RJ", + "value": 799995000 + }, + "scriptsig": "483045022100aeeddfb9785c5a4b70e90d0445785c68b7a44e28853441134a70ddc4da39527602203dfe1ec1a377aaacb64ae65c7c944caf1398d2dc063f712251b4cf696d44d3cb01210314338e3e191aea3ac9e9292611faeedf0379bbe62c30fd76c7450722a1ac47c6", + "scriptsig_asm": "OP_PUSHBYTES_72 3045022100aeeddfb9785c5a4b70e90d0445785c68b7a44e28853441134a70ddc4da39527602203dfe1ec1a377aaacb64ae65c7c944caf1398d2dc063f712251b4cf696d44d3cb01 OP_PUSHBYTES_33 0314338e3e191aea3ac9e9292611faeedf0379bbe62c30fd76c7450722a1ac47c6", + "is_coinbase": false, + "sequence": 4294967293 + } + ], + "vout": [ + { + "scriptpubkey": "6a4c5058325b8669baa9259e082f064005bc92274b559337ac317798f5d76f2d0577ed5a96042fce8c33d841b6c47a99f9597000ab04a10b34cd419fc19784d9e36f1a33fd7b000c3bce00b6000c1d1e00614b", + "scriptpubkey_asm": "OP_RETURN OP_PUSHDATA1 58325b8669baa9259e082f064005bc92274b559337ac317798f5d76f2d0577ed5a96042fce8c33d841b6c47a99f9597000ab04a10b34cd419fc19784d9e36f1a33fd7b000c3bce00b6000c1d1e00614b", + "scriptpubkey_type": "op_return", + "value": 0 + }, + { + "scriptpubkey": "a9144890aae025c84cb72a9730b49ca12595d6f6088d87", + "scriptpubkey_asm": "OP_HASH160 OP_PUSHBYTES_20 4890aae025c84cb72a9730b49ca12595d6f6088d OP_EQUAL", + "scriptpubkey_type": "p2sh", + "scriptpubkey_address": "38Jht2bzmJL4EwoFvvyFzejhfEb4J7KxLb", + "value": 155000 + }, + { + "scriptpubkey": "76a91486e7dad6617303942a448b7f8afe9653e5624a5e88ac", + "scriptpubkey_asm": "OP_DUP OP_HASH160 OP_PUSHBYTES_20 86e7dad6617303942a448b7f8afe9653e5624a5e OP_EQUALVERIFY OP_CHECKSIG", + "scriptpubkey_type": "p2pkh", + "scriptpubkey_address": "1DJKJGApgX4W8BSQ8FRPLqX78UaCskT4r2", + "value": 155000 + }, + { + "scriptpubkey": "76a914cd5b6566b455d043558829f6932edaae5d8f0ad388ac", + "scriptpubkey_asm": "OP_DUP OP_HASH160 OP_PUSHBYTES_20 cd5b6566b455d043558829f6932edaae5d8f0ad3 OP_EQUALVERIFY OP_CHECKSIG", + "scriptpubkey_type": "p2pkh", + "scriptpubkey_address": "1Kiq1dyVBzYLWGrBPWjChvKyzB2H95x5RJ", + "value": 799675549 + } + ], + "size": 350, + "weight": 1400, + "fee": 9451, + "status": { + "confirmed": false + }, + "order": 2798688215, + "vsize": 350, + "adjustedVsize": 350, + "sigops": 8, + "feePerVsize": 27.002857142857142, + "adjustedFeePerVsize": 27.002857142857142, + "effectiveFeePerVsize": 27.002857142857142, + "firstSeen": 1691218536, + "uid": 513598, + "inputs": [], + "position": { + "block": 0, + "vsize": 22166 + }, + "bestDescendant": null, + "cpfpChecked": true + }, + { + "txid": "5387881d695d4564d397026dc5f740f816f8390b4b2c5ec8c20309122712a875", + "version": 2, + "locktime": 0, + "vin": [ + { + "txid": "b50225a04a1d6fbbfa7a2122bc0580396f614027b3957f476229633576f06130", + "vout": 0, + "prevout": { + "scriptpubkey": "0014a24f913f8a9c30a4c302c2c78f2fd7addb08fd07", + "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 a24f913f8a9c30a4c302c2c78f2fd7addb08fd07", + "scriptpubkey_type": "v0_p2wpkh", + "scriptpubkey_address": "bc1q5f8ez0u2nsc2fsczctrc7t7h4hds3lg82ewqhz", + "value": 612917 + }, + "scriptsig": "", + "scriptsig_asm": "", + "witness": [ + "3045022100a0c23953ace5d022b7a6d45d1ae1730bf20a4d594bb5d4fa7aa80e4881b44d320220008f9b144805bb91995fc0f452a56e09f4ad16fa149d71ae9b5d57c742e8e2cc01", + "03dc2c7b687019b40a68d713322675206cc266e34e5340ec982c13ff0222c3b2b6" + ], + "is_coinbase": false, + "sequence": 2147483649 + } + ], + "vout": [ + { + "scriptpubkey": "0014199a98f9589364ffe5ef5bbae45ce5dfcbb873bd", + "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 199a98f9589364ffe5ef5bbae45ce5dfcbb873bd", + "scriptpubkey_type": "v0_p2wpkh", + "scriptpubkey_address": "bc1qrxdf372cjdj0le00twawgh89ml9msuaau62gk4", + "value": 611909 + } + ], + "size": 192, + "weight": 438, + "fee": 1008, + "status": { + "confirmed": false + }, + "bestDescendant": null, + "descendants": null, + "adjustedFeePerVsize": 10.2283, + "sigops": 1, + "adjustedVsize": 109.5 + } +] \ No newline at end of file diff --git a/backend/src/__tests__/api/test-data/transactions-replaced.json b/backend/src/__tests__/api/test-data/transactions-replaced.json new file mode 100644 index 000000000..e235d83e1 --- /dev/null +++ b/backend/src/__tests__/api/test-data/transactions-replaced.json @@ -0,0 +1,139 @@ +[ + { + "txid": "008592364e21c1e3d62ba9538ac78a81779897b52100af5707ab063df98964f2", + "version": 1, + "locktime": 0, + "vin": [ + { + "txid": "d863deb706de5a611028f7547e16ea81d7819e44beb640fb30a9ba30c585140f", + "vout": 0, + "prevout": { + "scriptpubkey": "76a914cd5b6566b455d043558829f6932edaae5d8f0ad388ac", + "scriptpubkey_asm": "OP_DUP OP_HASH160 OP_PUSHBYTES_20 cd5b6566b455d043558829f6932edaae5d8f0ad3 OP_EQUALVERIFY OP_CHECKSIG", + "scriptpubkey_type": "p2pkh", + "scriptpubkey_address": "1Kiq1dyVBzYLWGrBPWjChvKyzB2H95x5RJ", + "value": 799995000 + }, + "scriptsig": "483045022100c1fb331d155a7d299a0451d14fa1122b328e0e239afc9ba8dc2aff449ddc5a3a02201c1e19030d1efa432f5069cd369d7ad09a67f68501345e4db35f7b799605f55601210314338e3e191aea3ac9e9292611faeedf0379bbe62c30fd76c7450722a1ac47c6", + "scriptsig_asm": "OP_PUSHBYTES_72 3045022100c1fb331d155a7d299a0451d14fa1122b328e0e239afc9ba8dc2aff449ddc5a3a02201c1e19030d1efa432f5069cd369d7ad09a67f68501345e4db35f7b799605f55601 OP_PUSHBYTES_33 0314338e3e191aea3ac9e9292611faeedf0379bbe62c30fd76c7450722a1ac47c6", + "is_coinbase": false, + "sequence": 4294967293 + } + ], + "vout": [ + { + "scriptpubkey": "6a4c5058325b78064160b631b5a15d9078d99c0db066449fb4c59bbfa4d987ba906e2990088b2fce8c33d841b6c47a99f9597000ab04a10b34cd419fc19784d9e36f1a33fd7b000c3bce00b6000c1d1e00614b", + "scriptpubkey_asm": "OP_RETURN OP_PUSHDATA1 58325b78064160b631b5a15d9078d99c0db066449fb4c59bbfa4d987ba906e2990088b2fce8c33d841b6c47a99f9597000ab04a10b34cd419fc19784d9e36f1a33fd7b000c3bce00b6000c1d1e00614b", + "scriptpubkey_type": "op_return", + "value": 0 + }, + { + "scriptpubkey": "a9144890aae025c84cb72a9730b49ca12595d6f6088d87", + "scriptpubkey_asm": "OP_HASH160 OP_PUSHBYTES_20 4890aae025c84cb72a9730b49ca12595d6f6088d OP_EQUAL", + "scriptpubkey_type": "p2sh", + "scriptpubkey_address": "38Jht2bzmJL4EwoFvvyFzejhfEb4J7KxLb", + "value": 155000 + }, + { + "scriptpubkey": "76a91486e7dad6617303942a448b7f8afe9653e5624a5e88ac", + "scriptpubkey_asm": "OP_DUP OP_HASH160 OP_PUSHBYTES_20 86e7dad6617303942a448b7f8afe9653e5624a5e OP_EQUALVERIFY OP_CHECKSIG", + "scriptpubkey_type": "p2pkh", + "scriptpubkey_address": "1DJKJGApgX4W8BSQ8FRPLqX78UaCskT4r2", + "value": 155000 + }, + { + "scriptpubkey": "76a914cd5b6566b455d043558829f6932edaae5d8f0ad388ac", + "scriptpubkey_asm": "OP_DUP OP_HASH160 OP_PUSHBYTES_20 cd5b6566b455d043558829f6932edaae5d8f0ad3 OP_EQUALVERIFY OP_CHECKSIG", + "scriptpubkey_type": "p2pkh", + "scriptpubkey_address": "1Kiq1dyVBzYLWGrBPWjChvKyzB2H95x5RJ", + "value": 799676250 + } + ], + "size": 350, + "weight": 1400, + "fee": 8750, + "status": { + "confirmed": false + }, + "order": 4066675193, + "vsize": 350, + "adjustedVsize": 350, + "sigops": 8, + "feePerVsize": 25, + "adjustedFeePerVsize": 25, + "effectiveFeePerVsize": 25, + "firstSeen": 1691218516, + "uid": 512584, + "inputs": [], + "position": { + "block": 0, + "vsize": 13846 + }, + "bestDescendant": null, + "cpfpChecked": true + }, + { + "txid": "b7981a624e4261c11f1246314d41e74be56af82eb557bcd054a5e0f94c023668", + "version": 2, + "locktime": 0, + "vin": [ + { + "txid": "b50225a04a1d6fbbfa7a2122bc0580396f614027b3957f476229633576f06130", + "vout": 0, + "prevout": { + "scriptpubkey": "0014a24f913f8a9c30a4c302c2c78f2fd7addb08fd07", + "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 a24f913f8a9c30a4c302c2c78f2fd7addb08fd07", + "scriptpubkey_type": "v0_p2wpkh", + "scriptpubkey_address": "bc1q5f8ez0u2nsc2fsczctrc7t7h4hds3lg82ewqhz", + "value": 612917 + }, + "scriptsig": "", + "scriptsig_asm": "", + "witness": [ + "304402204dd10f14afa41bc76d8278140ff1ec3d3f87f2c207bbb5418cc76dab30d7f6a402207877cc9c6a2c724b6ea7a1c24ac00022469f194fd1a4bd8030bbca1787d3f5f301", + "03dc2c7b687019b40a68d713322675206cc266e34e5340ec982c13ff0222c3b2b6" + ], + "is_coinbase": false, + "sequence": 2147483648 + } + ], + "vout": [ + { + "scriptpubkey": "76a9149d32ef812385f3811634e0c0117dd153a5de10a488ac", + "scriptpubkey_asm": "OP_DUP OP_HASH160 OP_PUSHBYTES_20 9d32ef812385f3811634e0c0117dd153a5de10a4 OP_EQUALVERIFY OP_CHECKSIG", + "scriptpubkey_type": "p2pkh", + "scriptpubkey_address": "1FLC7Bag7okAkKPCyZbgZZg3Hh1EuGZ5Rd", + "value": 344697 + }, + { + "scriptpubkey": "00144c2671336ca8761863b4c68d64d4672491fec1b9", + "scriptpubkey_asm": "OP_0 OP_PUSHBYTES_20 4c2671336ca8761863b4c68d64d4672491fec1b9", + "scriptpubkey_type": "v0_p2wpkh", + "scriptpubkey_address": "bc1qfsn8zvmv4pmpsca5c6xkf4r8yjglasdesrawcx", + "value": 267636 + } + ], + "size": 225, + "weight": 573, + "fee": 584, + "status": { + "confirmed": false + }, + "order": 1748369996, + "vsize": 143, + "adjustedVsize": 143.25, + "sigops": 5, + "feePerVsize": 4.076788830715532, + "adjustedFeePerVsize": 4.076788830715532, + "effectiveFeePerVsize": 4.076788830715532, + "firstSeen": 1691222376, + "uid": 526515, + "inputs": [], + "position": { + "block": 7, + "vsize": 22021095.5 + }, + "bestDescendant": null, + "cpfpChecked": true + } +] \ No newline at end of file diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 33196b052..73b010b91 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -674,7 +674,11 @@ class Blocks { this.updateTimerProgress(timer, 'got previous block hash for initial difficulty adjustment'); const previousPeriodBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(previousPeriodBlockHash); this.updateTimerProgress(timer, 'got previous block for initial difficulty adjustment'); - this.previousDifficultyRetarget = calcBitsDifference(previousPeriodBlock.bits, block.bits); + if (['liquid', 'liquidtestnet'].includes(config.MEMPOOL.NETWORK)) { + this.previousDifficultyRetarget = NaN; + } else { + this.previousDifficultyRetarget = calcBitsDifference(previousPeriodBlock.bits, block.bits); + } logger.debug(`Initial difficulty adjustment data set.`); } } else { @@ -783,20 +787,31 @@ class Blocks { if (block.height % 2016 === 0) { if (Common.indexingEnabled()) { + let adjustment; + if (['liquid', 'liquidtestnet'].includes(config.MEMPOOL.NETWORK)) { + adjustment = NaN; + } else { + adjustment = Math.round( + // calcBitsDifference returns +- percentage, +100 returns to positive, /100 returns to ratio. + // Instead of actually doing /100, just reduce the multiplier. + (calcBitsDifference(this.currentBits, block.bits) + 100) * 10000 + ) / 1000000; // Remove float point noise + } + await DifficultyAdjustmentsRepository.$saveAdjustments({ time: block.timestamp, height: block.height, difficulty: block.difficulty, - adjustment: Math.round( - // calcBitsDifference returns +- percentage, +100 returns to positive, /100 returns to ratio. - // Instead of actually doing /100, just reduce the multiplier. - (calcBitsDifference(this.currentBits, block.bits) + 100) * 10000 - ) / 1000000, // Remove float point noise + adjustment, }); this.updateTimerProgress(timer, `saved difficulty adjustment for ${this.currentBlockHeight}`); } - this.previousDifficultyRetarget = calcBitsDifference(this.currentBits, block.bits); + if (['liquid', 'liquidtestnet'].includes(config.MEMPOOL.NETWORK)) { + this.previousDifficultyRetarget = NaN; + } else { + this.previousDifficultyRetarget = calcBitsDifference(this.currentBits, block.bits); + } this.lastDifficultyAdjustmentTime = block.timestamp; this.currentBits = block.bits; } diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index 64bac2036..b6f8ab657 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -59,10 +59,12 @@ export class Common { return arr; } - static findRbfTransactions(added: MempoolTransactionExtended[], deleted: MempoolTransactionExtended[]): { [txid: string]: MempoolTransactionExtended[] } { + static findRbfTransactions(added: MempoolTransactionExtended[], deleted: MempoolTransactionExtended[], forceScalable = false): { [txid: string]: MempoolTransactionExtended[] } { const matches: { [txid: string]: MempoolTransactionExtended[] } = {}; - added - .forEach((addedTx) => { + + // For small N, a naive nested loop is extremely fast, but it doesn't scale + if (added.length < 1000 && deleted.length < 50 && !forceScalable) { + added.forEach((addedTx) => { const foundMatches = deleted.filter((deletedTx) => { // The new tx must, absolutely speaking, pay at least as much fee as the replaced tx. return addedTx.fee > deletedTx.fee @@ -73,9 +75,40 @@ export class Common { addedTx.vin.some((vin) => vin.txid === deletedVin.txid && vin.vout === deletedVin.vout)); }); if (foundMatches?.length) { - matches[addedTx.txid] = foundMatches; + matches[addedTx.txid] = [...new Set(foundMatches)]; } }); + } else { + // for large N, build a lookup table of prevouts we can check in ~constant time + const deletedSpendMap: { [txid: string]: { [vout: number]: MempoolTransactionExtended } } = {}; + for (const tx of deleted) { + for (const vin of tx.vin) { + if (!deletedSpendMap[vin.txid]) { + deletedSpendMap[vin.txid] = {}; + } + deletedSpendMap[vin.txid][vin.vout] = tx; + } + } + + for (const addedTx of added) { + const foundMatches = new Set(); + for (const vin of addedTx.vin) { + const deletedTx = deletedSpendMap[vin.txid]?.[vin.vout]; + if (deletedTx && deletedTx.txid !== addedTx.txid + // The new tx must, absolutely speaking, pay at least as much fee as the replaced tx. + && addedTx.fee > deletedTx.fee + // The new transaction must pay more fee per kB than the replaced tx. + && addedTx.adjustedFeePerVsize > deletedTx.adjustedFeePerVsize + ) { + foundMatches.add(deletedTx); + } + if (foundMatches.size) { + matches[addedTx.txid] = [...foundMatches]; + } + } + } + } + return matches; } diff --git a/frontend/src/app/bisq/bisq-address/bisq-address.component.ts b/frontend/src/app/bisq/bisq-address/bisq-address.component.ts index eccc88bc7..7711c4d3c 100644 --- a/frontend/src/app/bisq/bisq-address/bisq-address.component.ts +++ b/frontend/src/app/bisq/bisq-address/bisq-address.component.ts @@ -47,6 +47,7 @@ export class BisqAddressComponent implements OnInit, OnDestroy { catchError((err) => { this.isLoadingAddress = false; this.error = err; + this.seoService.logSoft404(); console.log(err); return of(null); }) @@ -62,6 +63,7 @@ export class BisqAddressComponent implements OnInit, OnDestroy { (error) => { console.log(error); this.error = error; + this.seoService.logSoft404(); this.isLoadingAddress = false; }); } diff --git a/frontend/src/app/bisq/bisq-block/bisq-block.component.ts b/frontend/src/app/bisq/bisq-block/bisq-block.component.ts index 1bb3a24ab..206a18031 100644 --- a/frontend/src/app/bisq/bisq-block/bisq-block.component.ts +++ b/frontend/src/app/bisq/bisq-block/bisq-block.component.ts @@ -82,6 +82,7 @@ export class BisqBlockComponent implements OnInit, OnDestroy { ) .subscribe((block: BisqBlock) => { if (!block) { + this.seoService.logSoft404(); return; } this.isLoading = false; @@ -97,6 +98,7 @@ export class BisqBlockComponent implements OnInit, OnDestroy { caughtHttpError(err: HttpErrorResponse){ this.error = err; + this.seoService.logSoft404(); return of(null); } } diff --git a/frontend/src/app/bisq/bisq-transaction/bisq-transaction.component.ts b/frontend/src/app/bisq/bisq-transaction/bisq-transaction.component.ts index 47ac0d6db..4951643c8 100644 --- a/frontend/src/app/bisq/bisq-transaction/bisq-transaction.component.ts +++ b/frontend/src/app/bisq/bisq-transaction/bisq-transaction.component.ts @@ -70,11 +70,13 @@ export class BisqTransactionComponent implements OnInit, OnDestroy { catchError((txError: HttpErrorResponse) => { console.log(txError); this.error = txError; + this.seoService.logSoft404(); return of(null); }) ); } this.error = bisqTxError; + this.seoService.logSoft404(); return of(null); }) ); @@ -103,6 +105,7 @@ export class BisqTransactionComponent implements OnInit, OnDestroy { this.isLoadingTx = false; if (!tx) { + this.seoService.logSoft404(); return; } diff --git a/frontend/src/app/components/address/address.component.ts b/frontend/src/app/components/address/address.component.ts index e9bfaaa1b..3c79f2823 100644 --- a/frontend/src/app/components/address/address.component.ts +++ b/frontend/src/app/components/address/address.component.ts @@ -91,6 +91,7 @@ export class AddressComponent implements OnInit, OnDestroy { catchError((err) => { this.isLoadingAddress = false; this.error = err; + this.seoService.logSoft404(); console.log(err); return of(null); }) @@ -162,6 +163,7 @@ export class AddressComponent implements OnInit, OnDestroy { (error) => { console.log(error); this.error = error; + this.seoService.logSoft404(); this.isLoadingAddress = false; }); diff --git a/frontend/src/app/components/asset/asset.component.ts b/frontend/src/app/components/asset/asset.component.ts index 0e642063a..562ebff53 100644 --- a/frontend/src/app/components/asset/asset.component.ts +++ b/frontend/src/app/components/asset/asset.component.ts @@ -86,6 +86,7 @@ export class AssetComponent implements OnInit, OnDestroy { catchError((err) => { this.isLoadingAsset = false; this.error = err; + this.seoService.logSoft404(); console.log(err); return of(null); }) @@ -153,6 +154,7 @@ export class AssetComponent implements OnInit, OnDestroy { (error) => { console.log(error); this.error = error; + this.seoService.logSoft404(); this.isLoadingAsset = false; }); diff --git a/frontend/src/app/components/block/block-preview.component.ts b/frontend/src/app/components/block/block-preview.component.ts index d0fec960a..7c10dab6f 100644 --- a/frontend/src/app/components/block/block-preview.component.ts +++ b/frontend/src/app/components/block/block-preview.component.ts @@ -82,6 +82,7 @@ export class BlockPreviewComponent implements OnInit, OnDestroy { }), catchError((err) => { this.error = err; + this.seoService.logSoft404(); this.openGraphService.fail('block-data-' + this.rawId); this.openGraphService.fail('block-viz-' + this.rawId); return of(null); @@ -138,6 +139,7 @@ export class BlockPreviewComponent implements OnInit, OnDestroy { (error) => { this.error = error; this.isLoadingOverview = false; + this.seoService.logSoft404(); this.openGraphService.fail('block-viz-' + this.rawId); this.openGraphService.fail('block-data-' + this.rawId); if (this.blockGraph) { diff --git a/frontend/src/app/components/block/block.component.ts b/frontend/src/app/components/block/block.component.ts index 1345717bd..5d5233512 100644 --- a/frontend/src/app/components/block/block.component.ts +++ b/frontend/src/app/components/block/block.component.ts @@ -206,6 +206,7 @@ export class BlockComponent implements OnInit, OnDestroy { this.error = err; this.isLoadingBlock = false; this.isLoadingOverview = false; + this.seoService.logSoft404(); return EMPTY; }) ); @@ -214,6 +215,7 @@ export class BlockComponent implements OnInit, OnDestroy { this.error = err; this.isLoadingBlock = false; this.isLoadingOverview = false; + this.seoService.logSoft404(); return EMPTY; }), ); @@ -229,6 +231,7 @@ export class BlockComponent implements OnInit, OnDestroy { this.error = err; this.isLoadingBlock = false; this.isLoadingOverview = false; + this.seoService.logSoft404(); return EMPTY; }) ); diff --git a/frontend/src/app/components/pool/pool-preview.component.ts b/frontend/src/app/components/pool/pool-preview.component.ts index e03b73665..927291f07 100644 --- a/frontend/src/app/components/pool/pool-preview.component.ts +++ b/frontend/src/app/components/pool/pool-preview.component.ts @@ -61,6 +61,7 @@ export class PoolPreviewComponent implements OnInit { }), catchError(() => { this.isLoading = false; + this.seoService.logSoft404(); this.openGraphService.fail('pool-hash-' + this.slug); return of([slug]); }) @@ -70,6 +71,7 @@ export class PoolPreviewComponent implements OnInit { return this.apiService.getPoolStats$(slug).pipe( catchError(() => { this.isLoading = false; + this.seoService.logSoft404(); this.openGraphService.fail('pool-stats-' + this.slug); return of(null); }) diff --git a/frontend/src/app/components/pool/pool.component.ts b/frontend/src/app/components/pool/pool.component.ts index edd5801fe..a14d2a9b0 100644 --- a/frontend/src/app/components/pool/pool.component.ts +++ b/frontend/src/app/components/pool/pool.component.ts @@ -1,8 +1,8 @@ import { ChangeDetectionStrategy, Component, Inject, Input, LOCALE_ID, OnInit } from '@angular/core'; import { ActivatedRoute } from '@angular/router'; import { EChartsOption, graphic } from 'echarts'; -import { BehaviorSubject, Observable } from 'rxjs'; -import { distinctUntilChanged, map, share, switchMap, tap } from 'rxjs/operators'; +import { BehaviorSubject, Observable, of, timer } from 'rxjs'; +import { catchError, distinctUntilChanged, map, share, switchMap, tap } from 'rxjs/operators'; import { BlockExtended, PoolStat } from '../../interfaces/node-api.interface'; import { ApiService } from '../../services/api.service'; import { StateService } from '../../services/state.service'; @@ -62,10 +62,21 @@ export class PoolComponent implements OnInit { this.prepareChartOptions(data.map(val => [val.timestamp * 1000, val.avgHashrate])); return [slug]; }), + catchError(() => { + this.isLoading = false; + this.seoService.logSoft404(); + return of([slug]); + }) ); }), switchMap((slug) => { - return this.apiService.getPoolStats$(slug); + return this.apiService.getPoolStats$(slug).pipe( + catchError(() => { + this.isLoading = false; + this.seoService.logSoft404(); + return of(null); + }) + ); }), tap(() => { this.loadMoreSubject.next(this.blocks[0]?.height); diff --git a/frontend/src/app/components/transaction/transaction-preview.component.ts b/frontend/src/app/components/transaction/transaction-preview.component.ts index 6db0e588c..07843cc57 100644 --- a/frontend/src/app/components/transaction/transaction-preview.component.ts +++ b/frontend/src/app/components/transaction/transaction-preview.component.ts @@ -133,6 +133,7 @@ export class TransactionPreviewComponent implements OnInit, OnDestroy { ) .subscribe((tx: Transaction) => { if (!tx) { + this.seoService.logSoft404(); this.openGraphService.fail('tx-data-' + this.txId); return; } @@ -182,6 +183,7 @@ export class TransactionPreviewComponent implements OnInit, OnDestroy { this.openGraphService.waitOver('tx-data-' + this.txId); }, (error) => { + this.seoService.logSoft404(); this.openGraphService.fail('tx-data-' + this.txId); this.error = error; this.isLoadingTx = false; diff --git a/frontend/src/app/components/transaction/transaction.component.ts b/frontend/src/app/components/transaction/transaction.component.ts index 26e39515e..f1f3850e4 100644 --- a/frontend/src/app/components/transaction/transaction.component.ts +++ b/frontend/src/app/components/transaction/transaction.component.ts @@ -220,8 +220,10 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { ).subscribe((tx) => { this.loadingCachedTx = false; if (!tx) { + this.seoService.logSoft404(); return; } + this.seoService.clearSoft404(); if (!this.tx) { this.tx = tx; @@ -338,8 +340,10 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { .subscribe((tx: Transaction) => { if (!tx) { this.fetchCachedTx$.next(this.txId); + this.seoService.logSoft404(); return; } + this.seoService.clearSoft404(); this.tx = tx; this.setFeatures(); @@ -400,6 +404,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { }, (error) => { this.error = error; + this.seoService.logSoft404(); this.isLoadingTx = false; } ); @@ -487,6 +492,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { this.waitingForTransaction = true; } this.error = error; + this.seoService.logSoft404(); this.isLoadingTx = false; return of(false); } diff --git a/frontend/src/app/lightning/channel/channel-preview.component.ts b/frontend/src/app/lightning/channel/channel-preview.component.ts index 9f1bea4b8..9c7fdc1d6 100644 --- a/frontend/src/app/lightning/channel/channel-preview.component.ts +++ b/frontend/src/app/lightning/channel/channel-preview.component.ts @@ -54,6 +54,7 @@ export class ChannelPreviewComponent implements OnInit { }), catchError((err) => { this.error = err; + this.seoService.logSoft404(); this.openGraphService.fail('channel-map-' + this.shortId); this.openGraphService.fail('channel-data-' + this.shortId); return of(null); diff --git a/frontend/src/app/lightning/channel/channel.component.ts b/frontend/src/app/lightning/channel/channel.component.ts index d57aa3f01..052225cc3 100644 --- a/frontend/src/app/lightning/channel/channel.component.ts +++ b/frontend/src/app/lightning/channel/channel.component.ts @@ -38,6 +38,7 @@ export class ChannelComponent implements OnInit { }), catchError((err) => { this.error = err; + this.seoService.logSoft404(); return [{ short_id: params.get('short_id') }]; diff --git a/frontend/src/app/lightning/group/group-preview.component.ts b/frontend/src/app/lightning/group/group-preview.component.ts index be23e6178..5fd730931 100644 --- a/frontend/src/app/lightning/group/group-preview.component.ts +++ b/frontend/src/app/lightning/group/group-preview.component.ts @@ -50,6 +50,7 @@ export class GroupPreviewComponent implements OnInit { name: this.slug.replace(/-/gi, ' '), description: '', }; + this.seoService.logSoft404(); this.openGraphService.fail('ln-group-map-' + this.slug); this.openGraphService.fail('ln-group-data-' + this.slug); return of(null); @@ -106,6 +107,7 @@ export class GroupPreviewComponent implements OnInit { }; }), catchError(() => { + this.seoService.logSoft404(); this.openGraphService.fail('ln-group-map-' + this.slug); this.openGraphService.fail('ln-group-data-' + this.slug); return of({ diff --git a/frontend/src/app/lightning/node/node-preview.component.ts b/frontend/src/app/lightning/node/node-preview.component.ts index 4c2782c2d..56753b18b 100644 --- a/frontend/src/app/lightning/node/node-preview.component.ts +++ b/frontend/src/app/lightning/node/node-preview.component.ts @@ -81,6 +81,7 @@ export class NodePreviewComponent implements OnInit { }), catchError(err => { this.error = err; + this.seoService.logSoft404(); this.openGraphService.fail('node-map-' + this.publicKey); this.openGraphService.fail('node-data-' + this.publicKey); return [{ diff --git a/frontend/src/app/lightning/node/node.component.ts b/frontend/src/app/lightning/node/node.component.ts index 6447eb6bd..06ae50df2 100644 --- a/frontend/src/app/lightning/node/node.component.ts +++ b/frontend/src/app/lightning/node/node.component.ts @@ -123,6 +123,7 @@ export class NodeComponent implements OnInit { }), catchError(err => { this.error = err; + this.seoService.logSoft404(); return [{ alias: this.publicKey, public_key: this.publicKey, diff --git a/frontend/src/app/lightning/nodes-per-isp/nodes-per-isp-preview.component.ts b/frontend/src/app/lightning/nodes-per-isp/nodes-per-isp-preview.component.ts index 759606372..b823a5188 100644 --- a/frontend/src/app/lightning/nodes-per-isp/nodes-per-isp-preview.component.ts +++ b/frontend/src/app/lightning/nodes-per-isp/nodes-per-isp-preview.component.ts @@ -85,6 +85,7 @@ export class NodesPerISPPreview implements OnInit { }), catchError(err => { this.error = err; + this.seoService.logSoft404(); this.openGraphService.fail('isp-map-' + this.id); this.openGraphService.fail('isp-data-' + this.id); return of({ diff --git a/frontend/src/app/services/seo.service.ts b/frontend/src/app/services/seo.service.ts index 5f5d15c89..d536b4938 100644 --- a/frontend/src/app/services/seo.service.ts +++ b/frontend/src/app/services/seo.service.ts @@ -1,5 +1,7 @@ import { Injectable } from '@angular/core'; import { Title, Meta } from '@angular/platform-browser'; +import { ActivatedRoute, NavigationEnd, Router } from '@angular/router'; +import { filter, map, switchMap } from 'rxjs'; import { StateService } from './state.service'; @Injectable({ @@ -13,8 +15,22 @@ export class SeoService { private titleService: Title, private metaService: Meta, private stateService: StateService, + private router: Router, + private activatedRoute: ActivatedRoute, ) { this.stateService.networkChanged$.subscribe((network) => this.network = network); + this.router.events.pipe( + filter(event => event instanceof NavigationEnd), + map(() => this.activatedRoute), + map(route => { + while (route.firstChild) route = route.firstChild; + return route; + }), + filter(route => route.outlet === 'primary'), + switchMap(route => route.data), + ).subscribe((data) => { + this.clearSoft404(); + }); } setTitle(newTitle: string): void { @@ -53,4 +69,14 @@ export class SeoService { ucfirst(str: string) { return str.charAt(0).toUpperCase() + str.slice(1); } + + clearSoft404() { + window['soft404'] = false; + console.log('cleared soft 404'); + } + + logSoft404() { + window['soft404'] = true; + console.log('set soft 404'); + } } diff --git a/unfurler/src/concurrency/ReusableSSRPage.ts b/unfurler/src/concurrency/ReusableSSRPage.ts new file mode 100644 index 000000000..c68514a16 --- /dev/null +++ b/unfurler/src/concurrency/ReusableSSRPage.ts @@ -0,0 +1,65 @@ +import * as puppeteer from 'puppeteer'; +import { timeoutExecute } from 'puppeteer-cluster/dist/util'; +import logger from '../logger'; +import config from '../config'; +import ReusablePage from './ReusablePage'; +const mempoolHost = config.MEMPOOL.HTTP_HOST + (config.MEMPOOL.HTTP_PORT ? ':' + config.MEMPOOL.HTTP_PORT : ''); + +const mockImageBuffer = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVQYV2NgYAAAAAMAAWgmWQ0AAAAASUVORK5CYII=", 'base64'); + +interface RepairablePage extends puppeteer.Page { + repairRequested?: boolean; + language?: string | null; + createdAt?: number; + free?: boolean; + index?: number; +} + +export default class ReusableSSRPage extends ReusablePage { + + public constructor(options: puppeteer.LaunchOptions, puppeteer: any) { + super(options, puppeteer); + } + + public async close() { + await (this.browser as puppeteer.Browser).close(); + } + + protected async initPage(): Promise { + const page = await (this.browser as puppeteer.Browser).newPage() as RepairablePage; + page.language = null; + page.createdAt = Date.now(); + const defaultUrl = mempoolHost + '/about'; + + page.on('pageerror', (err) => { + console.log(err); + // page.repairRequested = true; + }); + await page.setRequestInterception(true); + page.on('request', req => { + if (req.isInterceptResolutionHandled()) { + return req.continue(); + } + if (req.resourceType() === 'image') { + return req.respond({ + contentType: 'image/png', + headers: {"Access-Control-Allow-Origin": "*"}, + body: mockImageBuffer + }); + } else if (!['document', 'script', 'xhr', 'fetch'].includes(req.resourceType())) { + return req.abort(); + } else { + return req.continue(); + } + }); + try { + await page.goto(defaultUrl, { waitUntil: "networkidle0" }); + await page.waitForSelector('meta[property="og:meta:ready"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 }); + } catch (e) { + logger.err(`failed to load frontend during ssr page initialization: ` + (e instanceof Error ? e.message : `${e}`)); + page.repairRequested = true; + } + page.free = true; + return page + } +} diff --git a/unfurler/src/index.ts b/unfurler/src/index.ts index 199d77c12..de13215e4 100644 --- a/unfurler/src/index.ts +++ b/unfurler/src/index.ts @@ -5,9 +5,11 @@ import * as https from 'https'; import config from './config'; import { Cluster } from 'puppeteer-cluster'; import ReusablePage from './concurrency/ReusablePage'; +import ReusableSSRPage from './concurrency/ReusablePage'; import { parseLanguageUrl } from './language/lang'; import { matchRoute } from './routes'; import logger from './logger'; +import { TimeoutError } from "puppeteer"; const puppeteerConfig = require('../puppeteer.config.json'); if (config.PUPPETEER.EXEC_PATH) { @@ -20,15 +22,33 @@ class Server { private server: http.Server | undefined; private app: Application; cluster?: Cluster; + ssrCluster?: Cluster; mempoolHost: string; + mempoolUrl: URL; network: string; secureHost = true; + canonicalHost: string; constructor() { this.app = express(); this.mempoolHost = config.MEMPOOL.HTTP_HOST + (config.MEMPOOL.HTTP_PORT ? ':' + config.MEMPOOL.HTTP_PORT : ''); + this.mempoolUrl = new URL(this.mempoolHost); this.secureHost = config.SERVER.HOST.startsWith('https'); this.network = config.MEMPOOL.NETWORK || 'bitcoin'; + + let canonical; + switch(config.MEMPOOL.NETWORK) { + case "liquid": + canonical = "https://liquid.network" + break; + case "bisq": + canonical = "https://bisq.markets" + break; + default: + canonical = "https://mempool.space" + } + this.canonicalHost = canonical; + this.startServer(); } @@ -49,6 +69,12 @@ class Server { puppeteerOptions: puppeteerConfig, }); await this.cluster?.task(async (args) => { return this.clusterTask(args) }); + this.ssrCluster = await Cluster.launch({ + concurrency: ReusableSSRPage, + maxConcurrency: config.PUPPETEER.CLUSTER_SIZE, + puppeteerOptions: puppeteerConfig, + }); + await this.ssrCluster?.task(async (args) => { return this.ssrClusterTask(args) }); } this.setUpRoutes(); @@ -65,6 +91,10 @@ class Server { await this.cluster.idle(); await this.cluster.close(); } + if (this.ssrCluster) { + await this.ssrCluster.idle(); + await this.ssrCluster.close(); + } if (this.server) { await this.server.close(); } @@ -102,8 +132,8 @@ class Server { } // wait for preview component to initialize - await page.waitForSelector('meta[property="og:preview:loading"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 }) let success; + await page.waitForSelector('meta[property="og:preview:loading"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 }) success = await Promise.race([ page.waitForSelector('meta[property="og:preview:ready"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 }).then(() => true), page.waitForSelector('meta[property="og:preview:fail"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 }).then(() => false) @@ -127,6 +157,51 @@ class Server { } } + async ssrClusterTask({ page, data: { url, path, action } }) { + try { + const urlParts = parseLanguageUrl(path); + if (page.language !== urlParts.lang) { + // switch language + page.language = urlParts.lang; + const localizedUrl = urlParts.lang ? `${this.mempoolHost}/${urlParts.lang}${urlParts.path}` : `${this.mempoolHost}${urlParts.path}` ; + await page.goto(localizedUrl, { waitUntil: "load" }); + } else { + const loaded = await page.evaluate(async (path) => { + if (window['ogService']) { + window['ogService'].loadPage(path); + return true; + } else { + return false; + } + }, urlParts.path); + if (!loaded) { + throw new Error('failed to access open graph service'); + } + } + + await page.waitForNetworkIdle({ + timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000, + }); + const is404 = await page.evaluate(async () => { + return !!window['soft404']; + }); + if (is404) { + return '404'; + } else { + let html = await page.content(); + return html; + } + } catch (e) { + if (e instanceof TimeoutError) { + let html = await page.content(); + return html; + } else { + logger.err(`failed to render ${path} for ${action}: ` + (e instanceof Error ? e.message : `${e}`)); + page.repairRequested = true; + } + } + } + async renderDisabled(req, res) { res.status(500).send("preview rendering disabled"); } @@ -166,43 +241,92 @@ class Server { // drop requests for static files const rawPath = req.params[0]; const match = rawPath.match(/\.[\w]+$/); - if (match?.length && match[0] !== '.html') { - res.status(404).send(); - return; + if (match?.length && match[0] !== '.html' + || rawPath.startsWith('/api/v1/donations/images') + || rawPath.startsWith('/api/v1/contributors/images') + || rawPath.startsWith('/api/v1/translators/images') + || rawPath.startsWith('/resources/profile') + ) { + if (isSearchCrawler(req.headers['user-agent'])) { + if (this.secureHost) { + https.get(config.SERVER.HOST + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => got.pipe(res)); + } else { + http.get(config.SERVER.HOST + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => got.pipe(res)); + } + return; + } else { + res.status(404).send(); + return; + } } + let result = ''; + try { + if (isSearchCrawler(req.headers['user-agent'])) { + result = await this.renderSEOPage(rawPath); + } else { + result = await this.renderUnfurlMeta(rawPath); + } + if (result && result.length) { + if (result === '404') { + res.status(404).send(); + } else { + res.send(result); + } + } else { + res.status(500).send(); + } + } catch (e) { + logger.err(e instanceof Error ? e.message : `${e} ${req.params[0]}`); + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + async renderUnfurlMeta(rawPath: string): Promise { const { lang, path } = parseLanguageUrl(rawPath); const matchedRoute = matchRoute(this.network, path); let ogImageUrl = config.SERVER.HOST + (matchedRoute.staticImg || matchedRoute.fallbackImg); let ogTitle = 'The Mempool Open Source Project®'; + const canonical = this.canonicalHost + rawPath; + if (matchedRoute.render) { ogImageUrl = `${config.SERVER.HOST}/render/${lang || 'en'}/preview${path}`; ogTitle = `${this.network ? capitalize(this.network) + ' ' : ''}${matchedRoute.networkMode !== 'mainnet' ? capitalize(matchedRoute.networkMode) + ' ' : ''}${matchedRoute.title}`; } - res.send(` - - - - - ${ogTitle} - - - - - - - - - - - - - - - - `); + return ` + + + + ${ogTitle} + + + + + + + + + + + + + + + + +`; + } + + async renderSEOPage(rawPath: string): Promise { + let html = await this.ssrCluster?.execute({ url: this.mempoolHost + rawPath, path: rawPath, action: 'ssr' }); + // remove javascript to prevent double hydration + if (html && html.length) { + html = html.replaceAll(//g, ""); + html = html.replaceAll(this.mempoolHost, this.canonicalHost); + } + return html; } } @@ -221,3 +345,7 @@ function capitalize(str) { return str; } } + +function isSearchCrawler(useragent: string): boolean { + return /googlebot|applebot|bingbot/i.test(useragent); +}