machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into junderw/fix-armv7-docker

Browse Source
This commit is contained in:
Felipe Knorr Kuhn
2023-07-18 14:30:20 -07:00
committed by GitHub
parent 6d6266c28a 0346822124
commit 1c4ec20bf2
169 changed files with 9338 additions and 9155 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
backend/mempool-config.sample.json
View File

@@ -29,7 +29,9 @@
"ADVANCED_GBT_MEMPOOL": false,
"RUST_GBT": false,
"CPFP_INDEXING": false,
"DISK_CACHE_BLOCK_INTERVAL": 6
"DISK_CACHE_BLOCK_INTERVAL": 6,
"MAX_PUSH_TX_SIZE_WEIGHT": 4000000,
"ALLOW_UNREACHABLE": true
},
"CORE_RPC": {
"HOST": "127.0.0.1",
@@ -123,5 +125,16 @@
"LIQUID_ONION": "http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1",
"BISQ_URL": "https://bisq.markets/api",
"BISQ_ONION": "http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api"
},
"REPLICATION": {
"ENABLED": false,
"AUDIT": false,
"AUDIT_START_HEIGHT": 774000,
"SERVERS": [
"list",
"of",
"trusted",
"servers"
]
}
}

325
backend/package-lock.json generated
View File

@@ -1,27 +1,26 @@
{
"name": "mempool-backend",
"version": "2.6.0-dev",
"version": "3.0.0-dev",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "mempool-backend",
"version": "2.6.0-dev",
"version": "3.0.0-dev",
"license": "GNU Affero General Public License v3.0",
"dependencies": {
"@babel/core": "^7.21.3",
"@mempool/electrum-client": "1.1.9",
"@types/node": "^18.15.3",
"axios": "~0.27.2",
"bitcoinjs-lib": "~6.1.0",
"axios": "~1.4.0",
"bitcoinjs-lib": "~6.1.3",
"crypto-js": "~4.1.1",
"express": "~4.18.2",
"maxmind": "~4.3.8",
"mysql2": "~3.2.0",
"node-worker-threads-pool": "~1.5.1",
"maxmind": "~4.3.11",
"mysql2": "~3.5.2",
"rust-gbt": "file:./rust-gbt",
"socks-proxy-agent": "~7.0.0",
"typescript": "~4.7.4",
"typescript": "~4.9.3",
"ws": "~8.13.0"
},
"devDependencies": {
@@ -29,19 +28,28 @@
"@babel/core": "^7.21.3",
"@types/compression": "^1.7.2",
"@types/crypto-js": "^4.1.1",
"@types/express": "^4.17.15",
"@types/express": "^4.17.17",
"@types/jest": "^29.5.0",
"@types/ws": "~8.5.4",
"@types/ws": "~8.5.5",
"@typescript-eslint/eslint-plugin": "^5.55.0",
"@typescript-eslint/parser": "^5.55.0",
"eslint": "^8.36.0",
"eslint-config-prettier": "^8.7.0",
"eslint-config-prettier": "^8.8.0",
"jest": "^29.5.0",
"prettier": "^2.8.4",
"ts-jest": "^29.0.5",
"prettier": "^3.0.0",
"ts-jest": "^29.1.1",
"ts-node": "^10.9.1"
}
},
"node_modules/@aashutoshrathi/word-wrap": {
"version": "1.2.6",
"resolved": "https://registry.npmjs.org/@aashutoshrathi/word-wrap/-/word-wrap-1.2.6.tgz",
"integrity": "sha512-1Yjs2SvM8TflER/OD3cOjhWWOZb58A2t7wpE2S9XfBYTiIl+XFhQG2bjy4Pu1I+EAlCNUzRDYDdFwFYUKvXcIA==",
"dev": true,
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/@ampproject/remapping": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.2.0.tgz",
@@ -1490,7 +1498,6 @@
"version": "2.16.1",
"resolved": "https://registry.npmjs.org/@napi-rs/cli/-/cli-2.16.1.tgz",
"integrity": "sha512-L0Gr5iEQIDEbvWdDr1HUaBOxBSHL1VZhWSk1oryawoT8qJIY+KGfLFelU+Qma64ivCPbxYpkfPoKYVG3rcoGIA==",
"dev": true,
"bin": {
"napi": "scripts/index.js"
},
@@ -1795,9 +1802,9 @@
"dev": true
},
"node_modules/@types/ws": {
"version": "8.5.4",
"resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.4.tgz",
"integrity": "sha512-zdQDHKUgcX/zBc4GrwsE/7dVdAD8JR4EuiAXiiUhhfyIJXXb2+PrGshFyeXWQPMmmZ2XxgaqclgpIC7eTXc1mg==",
"version": "8.5.5",
"resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.5.tgz",
"integrity": "sha512-lwhs8hktwxSjf9UaZ9tG5M03PGogvFaH8gUgLNbN9HKIg0dvv6q+gkSuJ8HN4/VbyxkuLzCjlN7GquQ0gUJfIg==",
"dev": true,
"dependencies": {
"@types/node": "*"
@@ -1865,9 +1872,9 @@
}
},
"node_modules/@typescript-eslint/eslint-plugin/node_modules/semver": {
"version": "7.3.8",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz",
"integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==",
"version": "7.5.4",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
"integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
"dev": true,
"dependencies": {
"lru-cache": "^6.0.0"
@@ -2009,9 +2016,9 @@
}
},
"node_modules/@typescript-eslint/typescript-estree/node_modules/semver": {
"version": "7.3.8",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz",
"integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==",
"version": "7.5.4",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
"integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
"dev": true,
"dependencies": {
"lru-cache": "^6.0.0"
@@ -2068,9 +2075,9 @@
}
},
"node_modules/@typescript-eslint/utils/node_modules/semver": {
"version": "7.3.8",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz",
"integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==",
"version": "7.5.4",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
"integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
"dev": true,
"dependencies": {
"lru-cache": "^6.0.0"
@@ -2255,12 +2262,13 @@
"integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
},
"node_modules/axios": {
"version": "0.27.2",
"resolved": "https://registry.npmjs.org/axios/-/axios-0.27.2.tgz",
"integrity": "sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ==",
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/axios/-/axios-1.4.0.tgz",
"integrity": "sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA==",
"dependencies": {
"follow-redirects": "^1.14.9",
"form-data": "^4.0.0"
"follow-redirects": "^1.15.0",
"form-data": "^4.0.0",
"proxy-from-env": "^1.1.0"
}
},
"node_modules/babel-jest": {
@@ -2449,9 +2457,9 @@
}
},
"node_modules/bitcoinjs-lib": {
"version": "6.1.1",
"resolved": "https://registry.npmjs.org/bitcoinjs-lib/-/bitcoinjs-lib-6.1.1.tgz",
"integrity": "sha512-FYihfgTk29lt1eK2y48OtuarEDUnTprNBW3ctT8yHiOhvmeS3DzAVG6gI0VCvMkydz6UdlXlYNWIPqGD0SUYRQ==",
"version": "6.1.3",
"resolved": "https://registry.npmjs.org/bitcoinjs-lib/-/bitcoinjs-lib-6.1.3.tgz",
"integrity": "sha512-TYXs/Qf+GNk2nnsB9HrXWqzFuEgCg0Gx+v3UW3B8VuceFHXVvhT+7hRnTSvwkX0i8rz2rtujeU6gFaDcFqYFDw==",
"dependencies": {
"@noble/hashes": "^1.2.0",
"bech32": "^2.0.0",
@@ -5367,9 +5375,9 @@
}
},
"node_modules/jest-snapshot/node_modules/semver": {
"version": "7.3.8",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz",
"integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==",
"version": "7.5.4",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
"integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
"dev": true,
"dependencies": {
"lru-cache": "^6.0.0"
@@ -5892,12 +5900,12 @@
}
},
"node_modules/maxmind": {
"version": "4.3.9",
"resolved": "https://registry.npmjs.org/maxmind/-/maxmind-4.3.9.tgz",
"integrity": "sha512-rEfIxZ9M2P7CWQQzN5/LapCawpf2DLh+LWD/cA7lNfCbFL6dNJOKgtynp8QbRsxExutn7Ofz1P1tXEdL3gnukw==",
"version": "4.3.11",
"resolved": "https://registry.npmjs.org/maxmind/-/maxmind-4.3.11.tgz",
"integrity": "sha512-tJDrKbUzN6PSA88tWgg0L2R4Ln00XwecYQJPFI+RvlF2k1sx6VQYtuQ1SVxm8+bw5tF7GWV4xyb+3/KyzEpPUw==",
"dependencies": {
"mmdb-lib": "2.0.2",
"tiny-lru": "10.3.0"
"tiny-lru": "11.0.1"
},
"engines": {
"node": ">=12",
@@ -6019,15 +6027,15 @@
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
},
"node_modules/mysql2": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.2.0.tgz",
"integrity": "sha512-0Vn6a9WSrq6fWwvPgrvIwnOCldiEcgbzapVRDAtDZ4cMTxN7pnGqCTx8EG32S/NYXl6AXkdO+9hV1tSIi/LigA==",
"version": "3.5.2",
"resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.5.2.tgz",
"integrity": "sha512-cptobmhYkYeTBIFp2c0piw2+gElpioga1rUw5UidHvo8yaHijMZoo8A3zyBVoo/K71f7ZFvrShA9iMIy9dCzCA==",
"dependencies": {
"denque": "^2.1.0",
"generate-function": "^2.3.1",
"iconv-lite": "^0.6.3",
"long": "^5.2.1",
"lru-cache": "^7.14.1",
"lru-cache": "^8.0.0",
"named-placeholders": "^1.1.3",
"seq-queue": "^0.0.5",
"sqlstring": "^2.3.2"
@@ -6048,11 +6056,11 @@
}
},
"node_modules/mysql2/node_modules/lru-cache": {
"version": "7.18.3",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz",
"integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==",
"version": "8.0.5",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-8.0.5.tgz",
"integrity": "sha512-MhWWlVnuab1RG5/zMRRcVGXZLCXrZTgfwMikgzCegsPnG62yDQo5JnqKkrK4jO5iKqDAZGItAqN5CtKBCBWRUA==",
"engines": {
"node": ">=12"
"node": ">=16.14"
}
},
"node_modules/named-placeholders": {
@@ -6106,11 +6114,6 @@
"integrity": "sha512-5GFldHPXVG/YZmFzJvKK2zDSzPKhEp0+ZR5SVaoSag9fsL5YgHbUHDfnG5494ISANDcK4KwPXAx2xqVEydmd7w==",
"dev": true
},
"node_modules/node-worker-threads-pool": {
"version": "1.5.1",
"resolved": "https://registry.npmjs.org/node-worker-threads-pool/-/node-worker-threads-pool-1.5.1.tgz",
"integrity": "sha512-7TXAhpMm+jO4MfESxYLtMGSnJWv+itdNHMdaFmeZuPXxwFGU90mtEB42BciUULXOUAxYBfXILAuvrSG3rQZ7mw=="
},
"node_modules/normalize-path": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz",
@@ -6176,17 +6179,17 @@
}
},
"node_modules/optionator": {
"version": "0.9.1",
"resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.1.tgz",
"integrity": "sha512-74RlY5FCnhq4jRxVUPKDaRwrVNXMqsGsiW6AJw4XK8hmtm10wC0ypZBLw5IIp85NZMr91+qd1RvvENwg7jjRFw==",
"version": "0.9.3",
"resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.3.tgz",
"integrity": "sha512-JjCoypp+jKn1ttEFExxhetCKeJt9zhAgAve5FXHixTvFDW/5aEktX9bufBKLRRMdU7bNtpLfcGu94B3cdEJgjg==",
"dev": true,
"dependencies": {
"@aashutoshrathi/word-wrap": "^1.2.3",
"deep-is": "^0.1.3",
"fast-levenshtein": "^2.0.6",
"levn": "^0.4.1",
"prelude-ls": "^1.2.1",
"type-check": "^0.4.0",
"word-wrap": "^1.2.3"
"type-check": "^0.4.0"
},
"engines": {
"node": ">= 0.8.0"
@@ -6417,15 +6420,15 @@
}
},
"node_modules/prettier": {
"version": "2.8.7",
"resolved": "https://registry.npmjs.org/prettier/-/prettier-2.8.7.tgz",
"integrity": "sha512-yPngTo3aXUUmyuTjeTUT75txrf+aMh9FiD7q9ZE/i6r0bPb22g4FsE6Y338PQX1bmfy08i9QQCB7/rcUAVntfw==",
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/prettier/-/prettier-3.0.0.tgz",
"integrity": "sha512-zBf5eHpwHOGPC47h0zrPyNn+eAEIdEzfywMoYn2XPi0P44Zp0tSq64rq0xAREh4auw2cJZHo9QUob+NqCQky4g==",
"dev": true,
"bin": {
"prettier": "bin-prettier.js"
"prettier": "bin/prettier.cjs"
},
"engines": {
"node": ">=10.13.0"
"node": ">=14"
},
"funding": {
"url": "https://github.com/prettier/prettier?sponsor=1"
@@ -6482,6 +6485,11 @@
"node": ">= 0.10"
}
},
"node_modules/proxy-from-env": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
"integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
},
"node_modules/punycode": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz",
@@ -6711,9 +6719,9 @@
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
},
"node_modules/semver": {
"version": "6.3.0",
"resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
"integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==",
"version": "6.3.1",
"resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz",
"integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==",
"dev": true,
"bin": {
"semver": "bin/semver.js"
@@ -7050,9 +7058,9 @@
"dev": true
},
"node_modules/tiny-lru": {
"version": "10.3.0",
"resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-10.3.0.tgz",
"integrity": "sha512-vTKRT2AEO1sViFDWAIzZVpV8KURCaMtnHa4RZB3XqtYLbrTO/fLDXKPEX9kVWq9u+nZREkwakbcmzGgvJm8QKA==",
"version": "11.0.1",
"resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-11.0.1.tgz",
"integrity": "sha512-iNgFugVuQgBKrqeO/mpiTTgmBsTP0WL6yeuLfLs/Ctf0pI/ixGqIRm8sDCwMcXGe9WWvt2sGXI5mNqZbValmJg==",
"engines": {
"node": ">=12"
}
@@ -7093,9 +7101,9 @@
}
},
"node_modules/ts-jest": {
"version": "29.0.5",
"resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-29.0.5.tgz",
"integrity": "sha512-PL3UciSgIpQ7f6XjVOmbi96vmDHUqAyqDr8YxzopDqX3kfgYtX1cuNeBjP+L9sFXi6nzsGGA6R3fP3DDDJyrxA==",
"version": "29.1.1",
"resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-29.1.1.tgz",
"integrity": "sha512-D6xjnnbP17cC85nliwGiL+tpoKN0StpgE0TeOjXQTU6MVCfsB4v7aW05CgQ/1OywGb0x/oy9hHFnN+sczTiRaA==",
"dev": true,
"dependencies": {
"bs-logger": "0.x",
@@ -7104,7 +7112,7 @@
"json5": "^2.2.3",
"lodash.memoize": "4.x",
"make-error": "1.x",
"semver": "7.x",
"semver": "^7.5.3",
"yargs-parser": "^21.0.1"
},
"bin": {
@@ -7118,7 +7126,7 @@
"@jest/types": "^29.0.0",
"babel-jest": "^29.0.0",
"jest": "^29.0.0",
"typescript": ">=4.3"
"typescript": ">=4.3 <6"
},
"peerDependenciesMeta": {
"@babel/core": {
@@ -7148,9 +7156,9 @@
}
},
"node_modules/ts-jest/node_modules/semver": {
"version": "7.3.8",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz",
"integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==",
"version": "7.5.4",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
"integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
"dev": true,
"dependencies": {
"lru-cache": "^6.0.0"
@@ -7283,9 +7291,9 @@
"integrity": "sha512-7uc1O8h1M1g0rArakJdf0uLRSSgFcYexrVoKo+bzJd32gd4gDy2L/Z+8/FjPnU9ydY3pEnVPtr9FyscYY60K1g=="
},
"node_modules/typescript": {
"version": "4.7.4",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-4.7.4.tgz",
"integrity": "sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ==",
"version": "4.9.5",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz",
"integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==",
"bin": {
"tsc": "bin/tsc",
"tsserver": "bin/tsserver"
@@ -7405,15 +7413,6 @@
"node": ">= 8"
}
},
"node_modules/word-wrap": {
"version": "1.2.3",
"resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz",
"integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==",
"dev": true,
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/wrap-ansi": {
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
@@ -7568,9 +7567,9 @@
},
"rust-gbt": {
"name": "gbt",
"version": "0.1.0",
"version": "3.0.0-dev",
"hasInstallScript": true,
"devDependencies": {
"dependencies": {
"@napi-rs/cli": "^2.16.1"
},
"engines": {
@@ -7579,6 +7578,12 @@
}
},
"dependencies": {
"@aashutoshrathi/word-wrap": {
"version": "1.2.6",
"resolved": "https://registry.npmjs.org/@aashutoshrathi/word-wrap/-/word-wrap-1.2.6.tgz",
"integrity": "sha512-1Yjs2SvM8TflER/OD3cOjhWWOZb58A2t7wpE2S9XfBYTiIl+XFhQG2bjy4Pu1I+EAlCNUzRDYDdFwFYUKvXcIA==",
"dev": true
},
"@ampproject/remapping": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.2.0.tgz",
@@ -8666,8 +8671,7 @@
"@napi-rs/cli": {
"version": "2.16.1",
"resolved": "https://registry.npmjs.org/@napi-rs/cli/-/cli-2.16.1.tgz",
"integrity": "sha512-L0Gr5iEQIDEbvWdDr1HUaBOxBSHL1VZhWSk1oryawoT8qJIY+KGfLFelU+Qma64ivCPbxYpkfPoKYVG3rcoGIA==",
"dev": true
"integrity": "sha512-L0Gr5iEQIDEbvWdDr1HUaBOxBSHL1VZhWSk1oryawoT8qJIY+KGfLFelU+Qma64ivCPbxYpkfPoKYVG3rcoGIA=="
},
"@noble/hashes": {
"version": "1.3.0",
@@ -8947,9 +8951,9 @@
"dev": true
},
"@types/ws": {
"version": "8.5.4",
"resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.4.tgz",
"integrity": "sha512-zdQDHKUgcX/zBc4GrwsE/7dVdAD8JR4EuiAXiiUhhfyIJXXb2+PrGshFyeXWQPMmmZ2XxgaqclgpIC7eTXc1mg==",
"version": "8.5.5",
"resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.5.tgz",
"integrity": "sha512-lwhs8hktwxSjf9UaZ9tG5M03PGogvFaH8gUgLNbN9HKIg0dvv6q+gkSuJ8HN4/VbyxkuLzCjlN7GquQ0gUJfIg==",
"dev": true,
"requires": {
"@types/node": "*"
@@ -8998,9 +9002,9 @@
}
},
"semver": {
"version": "7.3.8",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz",
"integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==",
"version": "7.5.4",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
"integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
"dev": true,
"requires": {
"lru-cache": "^6.0.0"
@@ -9079,9 +9083,9 @@
}
},
"semver": {
"version": "7.3.8",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz",
"integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==",
"version": "7.5.4",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
"integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
"dev": true,
"requires": {
"lru-cache": "^6.0.0"
@@ -9121,9 +9125,9 @@
}
},
"semver": {
"version": "7.3.8",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz",
"integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==",
"version": "7.5.4",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
"integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
"dev": true,
"requires": {
"lru-cache": "^6.0.0"
@@ -9258,12 +9262,13 @@
"integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
},
"axios": {
"version": "0.27.2",
"resolved": "https://registry.npmjs.org/axios/-/axios-0.27.2.tgz",
"integrity": "sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ==",
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/axios/-/axios-1.4.0.tgz",
"integrity": "sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA==",
"requires": {
"follow-redirects": "^1.14.9",
"form-data": "^4.0.0"
"follow-redirects": "^1.15.0",
"form-data": "^4.0.0",
"proxy-from-env": "^1.1.0"
}
},
"babel-jest": {
@@ -9409,9 +9414,9 @@
"integrity": "sha512-lkc0XyiX9E9KiVAS1ZiOqK1xfiwvf4FXDDdkDq5crcDzOq+xGytY+14qCsqz7kCiy8rpN1CRNfacRhf9G3JNSA=="
},
"bitcoinjs-lib": {
"version": "6.1.1",
"resolved": "https://registry.npmjs.org/bitcoinjs-lib/-/bitcoinjs-lib-6.1.1.tgz",
"integrity": "sha512-FYihfgTk29lt1eK2y48OtuarEDUnTprNBW3ctT8yHiOhvmeS3DzAVG6gI0VCvMkydz6UdlXlYNWIPqGD0SUYRQ==",
"version": "6.1.3",
"resolved": "https://registry.npmjs.org/bitcoinjs-lib/-/bitcoinjs-lib-6.1.3.tgz",
"integrity": "sha512-TYXs/Qf+GNk2nnsB9HrXWqzFuEgCg0Gx+v3UW3B8VuceFHXVvhT+7hRnTSvwkX0i8rz2rtujeU6gFaDcFqYFDw==",
"requires": {
"@noble/hashes": "^1.2.0",
"bech32": "^2.0.0",
@@ -11577,9 +11582,9 @@
}
},
"semver": {
"version": "7.3.8",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz",
"integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==",
"version": "7.5.4",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
"integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
"dev": true,
"requires": {
"lru-cache": "^6.0.0"
@@ -11973,12 +11978,12 @@
}
},
"maxmind": {
"version": "4.3.9",
"resolved": "https://registry.npmjs.org/maxmind/-/maxmind-4.3.9.tgz",
"integrity": "sha512-rEfIxZ9M2P7CWQQzN5/LapCawpf2DLh+LWD/cA7lNfCbFL6dNJOKgtynp8QbRsxExutn7Ofz1P1tXEdL3gnukw==",
"version": "4.3.11",
"resolved": "https://registry.npmjs.org/maxmind/-/maxmind-4.3.11.tgz",
"integrity": "sha512-tJDrKbUzN6PSA88tWgg0L2R4Ln00XwecYQJPFI+RvlF2k1sx6VQYtuQ1SVxm8+bw5tF7GWV4xyb+3/KyzEpPUw==",
"requires": {
"mmdb-lib": "2.0.2",
"tiny-lru": "10.3.0"
"tiny-lru": "11.0.1"
}
},
"media-typer": {
@@ -12062,15 +12067,15 @@
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
},
"mysql2": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.2.0.tgz",
"integrity": "sha512-0Vn6a9WSrq6fWwvPgrvIwnOCldiEcgbzapVRDAtDZ4cMTxN7pnGqCTx8EG32S/NYXl6AXkdO+9hV1tSIi/LigA==",
"version": "3.5.2",
"resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.5.2.tgz",
"integrity": "sha512-cptobmhYkYeTBIFp2c0piw2+gElpioga1rUw5UidHvo8yaHijMZoo8A3zyBVoo/K71f7ZFvrShA9iMIy9dCzCA==",
"requires": {
"denque": "^2.1.0",
"generate-function": "^2.3.1",
"iconv-lite": "^0.6.3",
"long": "^5.2.1",
"lru-cache": "^7.14.1",
"lru-cache": "^8.0.0",
"named-placeholders": "^1.1.3",
"seq-queue": "^0.0.5",
"sqlstring": "^2.3.2"
@@ -12085,9 +12090,9 @@
}
},
"lru-cache": {
"version": "7.18.3",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz",
"integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA=="
"version": "8.0.5",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-8.0.5.tgz",
"integrity": "sha512-MhWWlVnuab1RG5/zMRRcVGXZLCXrZTgfwMikgzCegsPnG62yDQo5JnqKkrK4jO5iKqDAZGItAqN5CtKBCBWRUA=="
}
}
},
@@ -12135,11 +12140,6 @@
"integrity": "sha512-5GFldHPXVG/YZmFzJvKK2zDSzPKhEp0+ZR5SVaoSag9fsL5YgHbUHDfnG5494ISANDcK4KwPXAx2xqVEydmd7w==",
"dev": true
},
"node-worker-threads-pool": {
"version": "1.5.1",
"resolved": "https://registry.npmjs.org/node-worker-threads-pool/-/node-worker-threads-pool-1.5.1.tgz",
"integrity": "sha512-7TXAhpMm+jO4MfESxYLtMGSnJWv+itdNHMdaFmeZuPXxwFGU90mtEB42BciUULXOUAxYBfXILAuvrSG3rQZ7mw=="
},
"normalize-path": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz",
@@ -12187,17 +12187,17 @@
}
},
"optionator": {
"version": "0.9.1",
"resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.1.tgz",
"integrity": "sha512-74RlY5FCnhq4jRxVUPKDaRwrVNXMqsGsiW6AJw4XK8hmtm10wC0ypZBLw5IIp85NZMr91+qd1RvvENwg7jjRFw==",
"version": "0.9.3",
"resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.3.tgz",
"integrity": "sha512-JjCoypp+jKn1ttEFExxhetCKeJt9zhAgAve5FXHixTvFDW/5aEktX9bufBKLRRMdU7bNtpLfcGu94B3cdEJgjg==",
"dev": true,
"requires": {
"@aashutoshrathi/word-wrap": "^1.2.3",
"deep-is": "^0.1.3",
"fast-levenshtein": "^2.0.6",
"levn": "^0.4.1",
"prelude-ls": "^1.2.1",
"type-check": "^0.4.0",
"word-wrap": "^1.2.3"
"type-check": "^0.4.0"
}
},
"p-limit": {
@@ -12358,9 +12358,9 @@
"dev": true
},
"prettier": {
"version": "2.8.7",
"resolved": "https://registry.npmjs.org/prettier/-/prettier-2.8.7.tgz",
"integrity": "sha512-yPngTo3aXUUmyuTjeTUT75txrf+aMh9FiD7q9ZE/i6r0bPb22g4FsE6Y338PQX1bmfy08i9QQCB7/rcUAVntfw==",
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/prettier/-/prettier-3.0.0.tgz",
"integrity": "sha512-zBf5eHpwHOGPC47h0zrPyNn+eAEIdEzfywMoYn2XPi0P44Zp0tSq64rq0xAREh4auw2cJZHo9QUob+NqCQky4g==",
"dev": true
},
"pretty-format": {
@@ -12401,6 +12401,11 @@
"ipaddr.js": "1.9.1"
}
},
"proxy-from-env": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
"integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
},
"punycode": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz",
@@ -12536,9 +12541,9 @@
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
},
"semver": {
"version": "6.3.0",
"resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
"integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==",
"version": "6.3.1",
"resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz",
"integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==",
"dev": true
},
"send": {
@@ -12801,9 +12806,9 @@
"dev": true
},
"tiny-lru": {
"version": "10.3.0",
"resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-10.3.0.tgz",
"integrity": "sha512-vTKRT2AEO1sViFDWAIzZVpV8KURCaMtnHa4RZB3XqtYLbrTO/fLDXKPEX9kVWq9u+nZREkwakbcmzGgvJm8QKA=="
"version": "11.0.1",
"resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-11.0.1.tgz",
"integrity": "sha512-iNgFugVuQgBKrqeO/mpiTTgmBsTP0WL6yeuLfLs/Ctf0pI/ixGqIRm8sDCwMcXGe9WWvt2sGXI5mNqZbValmJg=="
},
"tmpl": {
"version": "1.0.5",
@@ -12832,9 +12837,9 @@
"integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="
},
"ts-jest": {
"version": "29.0.5",
"resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-29.0.5.tgz",
"integrity": "sha512-PL3UciSgIpQ7f6XjVOmbi96vmDHUqAyqDr8YxzopDqX3kfgYtX1cuNeBjP+L9sFXi6nzsGGA6R3fP3DDDJyrxA==",
"version": "29.1.1",
"resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-29.1.1.tgz",
"integrity": "sha512-D6xjnnbP17cC85nliwGiL+tpoKN0StpgE0TeOjXQTU6MVCfsB4v7aW05CgQ/1OywGb0x/oy9hHFnN+sczTiRaA==",
"dev": true,
"requires": {
"bs-logger": "0.x",
@@ -12843,7 +12848,7 @@
"json5": "^2.2.3",
"lodash.memoize": "4.x",
"make-error": "1.x",
"semver": "7.x",
"semver": "^7.5.3",
"yargs-parser": "^21.0.1"
},
"dependencies": {
@@ -12857,9 +12862,9 @@
}
},
"semver": {
"version": "7.3.8",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz",
"integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==",
"version": "7.5.4",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
"integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
"dev": true,
"requires": {
"lru-cache": "^6.0.0"
@@ -12945,9 +12950,9 @@
"integrity": "sha512-7uc1O8h1M1g0rArakJdf0uLRSSgFcYexrVoKo+bzJd32gd4gDy2L/Z+8/FjPnU9ydY3pEnVPtr9FyscYY60K1g=="
},
"typescript": {
"version": "4.7.4",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-4.7.4.tgz",
"integrity": "sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ=="
"version": "4.9.5",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz",
"integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g=="
},
"unpipe": {
"version": "1.0.0",
@@ -13026,12 +13031,6 @@
"isexe": "^2.0.0"
}
},
"word-wrap": {
"version": "1.2.3",
"resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz",
"integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==",
"dev": true
},
"wrap-ansi": {
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",

23
backend/package.json
View File

@@ -1,6 +1,6 @@
{
"name": "mempool-backend",
"version": "2.6.0-dev",
"version": "3.0.0-dev",
"description": "Bitcoin mempool visualizer and blockchain explorer backend",
"license": "GNU Affero General Public License v3.0",
"homepage": "https://mempool.space",
@@ -40,16 +40,15 @@
"@babel/core": "^7.21.3",
"@mempool/electrum-client": "1.1.9",
"@types/node": "^18.15.3",
"axios": "~0.27.2",
"bitcoinjs-lib": "~6.1.0",
"axios": "~1.4.0",
"bitcoinjs-lib": "~6.1.3",
"crypto-js": "~4.1.1",
"express": "~4.18.2",
"maxmind": "~4.3.8",
"mysql2": "~3.2.0",
"node-worker-threads-pool": "~1.5.1",
"maxmind": "~4.3.11",
"mysql2": "~3.5.2",
"rust-gbt": "file:./rust-gbt",
"socks-proxy-agent": "~7.0.0",
"typescript": "~4.7.4",
"typescript": "~4.9.3",
"ws": "~8.13.0"
},
"devDependencies": {
@@ -57,16 +56,16 @@
"@babel/code-frame": "^7.18.6",
"@types/compression": "^1.7.2",
"@types/crypto-js": "^4.1.1",
"@types/express": "^4.17.15",
"@types/express": "^4.17.17",
"@types/jest": "^29.5.0",
"@types/ws": "~8.5.4",
"@types/ws": "~8.5.5",
"@typescript-eslint/eslint-plugin": "^5.55.0",
"@typescript-eslint/parser": "^5.55.0",
"eslint": "^8.36.0",
"eslint-config-prettier": "^8.7.0",
"eslint-config-prettier": "^8.8.0",
"jest": "^29.5.0",
"prettier": "^2.8.4",
"ts-jest": "^29.0.5",
"prettier": "^3.0.0",
"ts-jest": "^29.1.1",
"ts-node": "^10.9.1"
}
}

4
backend/rust-gbt/package-lock.json generated
View File

@@ -1,12 +1,12 @@
{
"name": "gbt",
"version": "0.1.0",
"version": "3.0.0-dev",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "gbt",
"version": "0.1.0",
"version": "3.0.0-dev",
"hasInstallScript": true,
"dependencies": {
"@napi-rs/cli": "^2.16.1"

4
backend/rust-gbt/package.json
View File

@@ -1,6 +1,6 @@
{
"name": "gbt",
"version": "0.1.0",
"version": "3.0.0-dev",
"description": "An inefficient re-implementation of the getBlockTemplate algorithm in Rust",
"main": "index.js",
"types": "index.d.ts",
@@ -30,4 +30,4 @@
"engines": {
"node": ">= 12"
}
}
}

12
backend/src/__fixtures__/mempool-config.template.json
View File

@@ -30,7 +30,9 @@
"RUST_GBT": false,
"CPFP_INDEXING": true,
"MAX_BLOCKS_BULK_QUERY": 999,
"DISK_CACHE_BLOCK_INTERVAL": 999
"DISK_CACHE_BLOCK_INTERVAL": 999,
"MAX_PUSH_TX_SIZE_WEIGHT": 4000000,
"ALLOW_UNREACHABLE": true
},
"CORE_RPC": {
"HOST": "__CORE_RPC_HOST__",
@@ -119,5 +121,11 @@
},
"CLIGHTNING": {
"SOCKET": "__CLIGHTNING_SOCKET__"
},
"REPLICATION": {
"ENABLED": false,
"AUDIT": false,
"AUDIT_START_HEIGHT": 774000,
"SERVERS": []
}
}
}

9
backend/src/__tests__/config.test.ts
View File

@@ -44,6 +44,8 @@ describe('Mempool Backend Config', () => {
CPFP_INDEXING: false,
MAX_BLOCKS_BULK_QUERY: 0,
DISK_CACHE_BLOCK_INTERVAL: 6,
MAX_PUSH_TX_SIZE_WEIGHT: 400000,
ALLOW_UNREACHABLE: true,
});
expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true });
@@ -118,6 +120,13 @@ describe('Mempool Backend Config', () => {
GEOLITE2_ASN: '/usr/local/share/GeoIP/GeoLite2-ASN.mmdb',
GEOIP2_ISP: '/usr/local/share/GeoIP/GeoIP2-ISP.mmdb'
});
expect(config.REPLICATION).toStrictEqual({
ENABLED: false,
AUDIT: false,
AUDIT_START_HEIGHT: 774000,
SERVERS: []
});
});
});

26
backend/src/api/audit.ts
View File

@@ -1,19 +1,21 @@
import config from '../config';
import logger from '../logger';
import { TransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces';
import { MempoolTransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces';
import rbfCache from './rbf-cache';
const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners
class Audit {
auditBlock(transactions: TransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: TransactionExtended })
: { censored: string[], added: string[], fresh: string[], sigop: string[], score: number, similarity: number } {
auditBlock(transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended })
: { censored: string[], added: string[], fresh: string[], sigop: string[], fullrbf: string[], score: number, similarity: number } {
if (!projectedBlocks?.[0]?.transactionIds || !mempool) {
return { censored: [], added: [], fresh: [], sigop: [], score: 0, similarity: 1 };
return { censored: [], added: [], fresh: [], sigop: [], fullrbf: [], score: 0, similarity: 1 };
}
const matches: string[] = []; // present in both mined block and template
const added: string[] = []; // present in mined block, not in template
const fresh: string[] = []; // missing, but firstSeen within PROPAGATION_MARGIN
const fresh: string[] = []; // missing, but firstSeen or lastBoosted within PROPAGATION_MARGIN
const fullrbf: string[] = []; // either missing or present, and part of a fullrbf replacement
const isCensored = {}; // missing, without excuse
const isDisplaced = {};
let displacedWeight = 0;
@@ -34,8 +36,13 @@ class Audit {
// look for transactions that were expected in the template, but missing from the mined block
for (const txid of projectedBlocks[0].transactionIds) {
if (!inBlock[txid]) {
// tx is recent, may have reached the miner too late for inclusion
if (mempool[txid]?.firstSeen != null && (now - (mempool[txid]?.firstSeen || 0)) <= PROPAGATION_MARGIN) {
if (rbfCache.isFullRbf(txid)) {
fullrbf.push(txid);
} else if (mempool[txid]?.firstSeen != null && (now - (mempool[txid]?.firstSeen || 0)) <= PROPAGATION_MARGIN) {
// tx is recent, may have reached the miner too late for inclusion
fresh.push(txid);
} else if (mempool[txid]?.lastBoosted != null && (now - (mempool[txid]?.lastBoosted || 0)) <= PROPAGATION_MARGIN) {
// tx was recently cpfp'd, miner may not have the latest effective rate
fresh.push(txid);
} else {
isCensored[txid] = true;
@@ -91,7 +98,9 @@ class Audit {
if (inTemplate[tx.txid]) {
matches.push(tx.txid);
} else {
if (!isDisplaced[tx.txid]) {
if (rbfCache.isFullRbf(tx.txid)) {
fullrbf.push(tx.txid);
} else if (!isDisplaced[tx.txid]) {
added.push(tx.txid);
}
overflowWeight += tx.weight;
@@ -138,6 +147,7 @@ class Audit {
added,
fresh,
sigop: [],
fullrbf,
score,
similarity,
};

5
backend/src/api/bisq/bisq.ts
View File

@@ -7,7 +7,6 @@ import { SocksProxyAgent } from 'socks-proxy-agent';
import { BisqBlocks, BisqBlock, BisqTransaction, BisqStats, BisqTrade } from './interfaces';
import { Common } from '../common';
import { BlockExtended } from '../../mempool.interfaces';
import { StaticPool } from 'node-worker-threads-pool';
import backendInfo from '../backend-info';
import logger from '../../logger';
@@ -31,10 +30,6 @@ class Bisq {
private priceUpdateCallbackFunction: ((price: number) => void) | undefined;
private topDirectoryWatcher: fs.FSWatcher | undefined;
private subdirectoryWatcher: fs.FSWatcher | undefined;
private jsonParsePool = new StaticPool({
size: 4,
task: (blob: string) => JSON.parse(blob),
});
constructor() {}

11
backend/src/api/bitcoin/bitcoin-api.ts
View File

@@ -29,6 +29,7 @@ class BitcoinApi implements AbstractBitcoinApi {
weight: block.weight,
previousblockhash: block.previousblockhash,
mediantime: block.mediantime,
stale: block.confirmations === -1,
};
}
@@ -64,17 +65,11 @@ class BitcoinApi implements AbstractBitcoinApi {
}
$getBlockHeightTip(): Promise<number> {
return this.bitcoindClient.getChainTips()
.then((result: IBitcoinApi.ChainTips[]) => {
return result.find(tip => tip.status === 'active')!.height;
});
return this.bitcoindClient.getBlockCount();
}
$getBlockHashTip(): Promise<string> {
return this.bitcoindClient.getChainTips()
.then((result: IBitcoinApi.ChainTips[]) => {
return result.find(tip => tip.status === 'active')!.hash;
});
return this.bitcoindClient.getBestBlockHash();
}
$getTxIdsForBlock(hash: string): Promise<string[]> {

29
backend/src/api/bitcoin/bitcoin.routes.ts
View File

@@ -121,7 +121,6 @@ class BitcoinRoutes {
.get(config.MEMPOOL.API_URL_PREFIX + 'block-height/:height', this.getBlockHeight)
.get(config.MEMPOOL.API_URL_PREFIX + 'address/:address', this.getAddress)
.get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs', this.getAddressTransactions)
.get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs/chain/:txId', this.getAddressTransactions)
.get(config.MEMPOOL.API_URL_PREFIX + 'address-prefix/:prefix', this.getAddressPrefix)
;
}
@@ -546,27 +545,28 @@ class BitcoinRoutes {
}
}
private async getAddressTransactions(req: Request, res: Response) {
private async getAddressTransactions(req: Request, res: Response): Promise<void> {
if (config.MEMPOOL.BACKEND === 'none') {
res.status(405).send('Address lookups cannot be used with bitcoind as backend.');
return;
}
try {
const transactions = await bitcoinApi.$getAddressTransactions(req.params.address, req.params.txId);
let lastTxId: string = '';
if (req.query.after_txid && typeof req.query.after_txid === 'string') {
lastTxId = req.query.after_txid;
}
const transactions = await bitcoinApi.$getAddressTransactions(req.params.address, lastTxId);
res.json(transactions);
} catch (e) {
if (e instanceof Error && e.message && (e.message.indexOf('too long') > 0 || e.message.indexOf('confirmed status') > 0)) {
return res.status(413).send(e instanceof Error ? e.message : e);
res.status(413).send(e instanceof Error ? e.message : e);
return;
}
res.status(500).send(e instanceof Error ? e.message : e);
}
}
private async getAdressTxChain(req: Request, res: Response) {
res.status(501).send('Not implemented');
}
private async getAddressPrefix(req: Request, res: Response) {
try {
const blockHash = await bitcoinApi.$getAddressPrefix(req.params.prefix);
@@ -723,12 +723,7 @@ class BitcoinRoutes {
private async $postTransaction(req: Request, res: Response) {
res.setHeader('content-type', 'text/plain');
try {
let rawTx;
if (typeof req.body === 'object') {
rawTx = Object.keys(req.body)[0];
} else {
rawTx = req.body;
}
const rawTx = Common.getTransactionFromRequest(req, false);
const txIdResult = await bitcoinApi.$sendRawTransaction(rawTx);
res.send(txIdResult);
} catch (e: any) {
@@ -739,12 +734,8 @@ class BitcoinRoutes {
private async $postTransactionForm(req: Request, res: Response) {
res.setHeader('content-type', 'text/plain');
const matches = /tx=([a-z0-9]+)/.exec(req.body);
let txHex = '';
if (matches && matches[1]) {
txHex = matches[1];
}
try {
const txHex = Common.getTransactionFromRequest(req, true);
const txIdResult = await bitcoinClient.sendRawTransaction(txHex);
res.send(txIdResult);
} catch (e: any) {

1
backend/src/api/bitcoin/esplora-api.interface.ts
View File

@@ -89,6 +89,7 @@ export namespace IEsploraApi {
weight: number;
previousblockhash: string;
mediantime: number;
stale: boolean;
}
export interface Address {

44
backend/src/api/blocks.ts
View File

@@ -25,6 +25,7 @@ import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmen
import PricesRepository from '../repositories/PricesRepository';
import priceUpdater from '../tasks/price-updater';
import chainTips from './chain-tips';
import websocketHandler from './websocket-handler';
class Blocks {
private blocks: BlockExtended[] = [];
@@ -75,11 +76,14 @@ class Blocks {
blockHash: string,
blockHeight: number,
onlyCoinbase: boolean,
txIds: string[] | null = null,
quiet: boolean = false,
addMempoolData: boolean = false,
): Promise<TransactionExtended[]> {
const transactions: TransactionExtended[] = [];
const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash);
if (!txIds) {
txIds = await bitcoinApi.$getTxIdsForBlock(blockHash);
}
const mempool = memPool.getMempool();
let transactionsFound = 0;
@@ -553,7 +557,7 @@ class Blocks {
}
const blockHash = await bitcoinApi.$getBlockHash(blockHeight);
const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash);
const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, true);
const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, null, true);
const blockExtended = await this.$getBlockExtended(block, transactions);
newlyIndexed++;
@@ -585,7 +589,7 @@ class Blocks {
let fastForwarded = false;
let handledBlocks = 0;
const blockHeightTip = await bitcoinApi.$getBlockHeightTip();
const blockHeightTip = await bitcoinCoreApi.$getBlockHeightTip();
this.updateTimerProgress(timer, 'got block height tip');
if (this.blocks.length === 0) {
@@ -638,11 +642,11 @@ class Blocks {
}
this.updateTimerProgress(timer, `getting block data for ${this.currentBlockHeight}`);
const blockHash = await bitcoinApi.$getBlockHash(this.currentBlockHeight);
const blockHash = await bitcoinCoreApi.$getBlockHash(this.currentBlockHeight);
const verboseBlock = await bitcoinClient.getBlock(blockHash, 2);
const block = BitcoinApi.convertBlock(verboseBlock);
const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash);
const transactions = await this.$getTransactionsExtended(blockHash, block.height, false, false, true) as MempoolTransactionExtended[];
const txIds: string[] = verboseBlock.tx.map(tx => tx.txid);
const transactions = await this.$getTransactionsExtended(blockHash, block.height, false, txIds, false, true) as MempoolTransactionExtended[];
if (config.MEMPOOL.BACKEND !== 'esplora') {
// fill in missing transaction fee data from verboseBlock
for (let i = 0; i < transactions.length; i++) {
@@ -656,10 +660,6 @@ class Blocks {
const blockSummary: BlockSummary = this.summarizeBlockTransactions(block.id, cpfpSummary.transactions);
this.updateTimerProgress(timer, `got block data for ${this.currentBlockHeight}`);
// start async callbacks
this.updateTimerProgress(timer, `starting async callbacks for ${this.currentBlockHeight}`);
const callbackPromises = this.newAsyncBlockCallbacks.map((cb) => cb(blockExtended, txIds, transactions));
if (Common.indexingEnabled()) {
if (!fastForwarded) {
const lastBlock = await blocksRepository.$getBlockByHeight(blockExtended.height - 1);
@@ -671,9 +671,11 @@ class Blocks {
await BlocksRepository.$deleteBlocksFrom(lastBlock.height - 10);
await HashratesRepository.$deleteLastEntries();
await cpfpRepository.$deleteClustersFrom(lastBlock.height - 10);
this.blocks = this.blocks.slice(0, -10);
this.updateTimerProgress(timer, `rolled back chain divergence from ${this.currentBlockHeight}`);
for (let i = 10; i >= 0; --i) {
const newBlock = await this.$indexBlock(lastBlock.height - i);
this.blocks.push(newBlock);
this.updateTimerProgress(timer, `reindexed block`);
let cpfpSummary;
if (config.MEMPOOL.CPFP_INDEXING) {
@@ -688,6 +690,8 @@ class Blocks {
this.updateTimerProgress(timer, `reindexed difficulty adjustments`);
logger.info(`Re-indexed 10 blocks and summaries. Also re-indexed the last difficulty adjustments. Will re-index latest hashrates in a few seconds.`, logger.tags.mining);
indexer.reindex();
websocketHandler.handleReorg();
}
}
@@ -722,6 +726,10 @@ class Blocks {
}
}
// start async callbacks
this.updateTimerProgress(timer, `starting async callbacks for ${this.currentBlockHeight}`);
const callbackPromises = this.newAsyncBlockCallbacks.map((cb) => cb(blockExtended, txIds, transactions));
if (block.height % 2016 === 0) {
if (Common.indexingEnabled()) {
await DifficultyAdjustmentsRepository.$saveAdjustments({
@@ -814,6 +822,16 @@ class Blocks {
return blockExtended;
}
public async $indexStaleBlock(hash: string): Promise<BlockExtended> {
const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(hash);
const transactions = await this.$getTransactionsExtended(hash, block.height, true);
const blockExtended = await this.$getBlockExtended(block, transactions);
blockExtended.canonical = await bitcoinApi.$getBlockHash(block.height);
return blockExtended;
}
/**
* Get one block by its hash
*/
@@ -831,7 +849,11 @@ class Blocks {
// Bitcoin network, add our custom data on top
const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(hash);
return await this.$indexBlock(block.height);
if (block.stale) {
return await this.$indexStaleBlock(hash);
} else {
return await this.$indexBlock(block.height);
}
}
public async $getStrippedBlockTransactions(hash: string, skipMemoryCache = false,

127
backend/src/api/common.ts
View File

@@ -1,3 +1,5 @@
import * as bitcoinjs from 'bitcoinjs-lib';
import { Request } from 'express';
import { Ancestor, CpfpInfo, CpfpSummary, CpfpCluster, EffectiveFeeStats, MempoolBlockWithTransactions, TransactionExtended, MempoolTransactionExtended, TransactionStripped, WorkingEffectiveFeeStats } from '../mempool.interfaces';
import config from '../config';
import { NodeSocket } from '../repositories/NodesSocketsRepository';
@@ -86,19 +88,19 @@ export class Common {
const match = spendMap.get(`${vin.txid}:${vin.vout}`);
if (match && match.txid !== tx.txid) {
replaced.add(match);
// remove this tx from the spendMap
// prevents the same tx being replaced more than once
for (const replacedVin of match.vin) {
const key = `${replacedVin.txid}:${replacedVin.vout}`;
spendMap.delete(key);
}
}
const key = `${vin.txid}:${vin.vout}`;
spendMap.delete(key);
}
if (replaced.size) {
matches[tx.txid] = { replaced: Array.from(replaced), replacedBy: tx };
}
// remove this tx from the spendMap
// prevents the same tx being replaced more than once
for (const vin of tx.vin) {
const key = `${vin.txid}:${vin.vout}`;
if (spendMap.get(key)?.txid === tx.txid) {
spendMap.delete(key);
}
}
}
return matches;
}
@@ -511,6 +513,115 @@ export class Common {
static getNthPercentile(n: number, sortedDistribution: any[]): any {
return sortedDistribution[Math.floor((sortedDistribution.length - 1) * (n / 100))];
}
static getTransactionFromRequest(req: Request, form: boolean): string {
let rawTx: any = typeof req.body === 'object' && form
? Object.values(req.body)[0] as any
: req.body;
if (typeof rawTx !== 'string') {
throw Object.assign(new Error('Non-string request body'), { code: -1 });
}
// Support both upper and lower case hex
// Support both txHash= Form and direct API POST
const reg = form ? /^txHash=((?:[a-fA-F0-9]{2})+)$/ : /^((?:[a-fA-F0-9]{2})+)$/;
const matches = reg.exec(rawTx);
if (!matches || !matches[1]) {
throw Object.assign(new Error('Non-hex request body'), { code: -2 });
}
// Guaranteed to be a hex string of multiple of 2
// Guaranteed to be lower case
// Guaranteed to pass validation (see function below)
return this.validateTransactionHex(matches[1].toLowerCase());
}
private static validateTransactionHex(txhex: string): string {
// Do not mutate txhex
// We assume txhex to be valid hex (output of getTransactionFromRequest above)
// Check 1: Valid transaction parse
let tx: bitcoinjs.Transaction;
try {
tx = bitcoinjs.Transaction.fromHex(txhex);
} catch(e) {
throw Object.assign(new Error('Invalid transaction (could not parse)'), { code: -4 });
}
// Check 2: Simple size check
if (tx.weight() > config.MEMPOOL.MAX_PUSH_TX_SIZE_WEIGHT) {
throw Object.assign(new Error(`Transaction too large (max ${config.MEMPOOL.MAX_PUSH_TX_SIZE_WEIGHT} weight units)`), { code: -3 });
}
// Check 3: Check unreachable script in taproot (if not allowed)
if (!config.MEMPOOL.ALLOW_UNREACHABLE) {
tx.ins.forEach(input => {
const witness = input.witness;
// See BIP 341: Script validation rules
const hasAnnex = witness.length >= 2 &&
witness[witness.length - 1][0] === 0x50;
const scriptSpendMinLength = hasAnnex ? 3 : 2;
const maybeScriptSpend = witness.length >= scriptSpendMinLength;
if (maybeScriptSpend) {
const controlBlock = witness[witness.length - scriptSpendMinLength + 1];
if (controlBlock.length === 0 || !this.isValidLeafVersion(controlBlock[0])) {
// Skip this input, it's not taproot
return;
}
// Definitely taproot. Get script
const script = witness[witness.length - scriptSpendMinLength];
const decompiled = bitcoinjs.script.decompile(script);
if (!decompiled || decompiled.length < 2) {
// Skip this input
return;
}
// Iterate up to second last (will look ahead 1 item)
for (let i = 0; i < decompiled.length - 1; i++) {
const first = decompiled[i];
const second = decompiled[i + 1];
if (
first === bitcoinjs.opcodes.OP_FALSE &&
second === bitcoinjs.opcodes.OP_IF
) {
throw Object.assign(new Error('Unreachable taproot scripts not allowed'), { code: -5 });
}
}
}
})
}
// Pass through the input string untouched
return txhex;
}
private static isValidLeafVersion(leafVersion: number): boolean {
// See Note 7 in BIP341
// https://github.com/bitcoin/bips/blob/66a1a8151021913047934ebab3f8883f2f8ca75b/bip-0341.mediawiki#cite_note-7
// "What constraints are there on the leaf version?"
// Must be an integer between 0 and 255
// Since we're parsing a byte
if (Math.floor(leafVersion) !== leafVersion || leafVersion < 0 || leafVersion > 255) {
return false;
}
// "the leaf version cannot be odd"
if ((leafVersion & 0x01) === 1) {
return false;
}
// "The values that comply to this rule are
// the 32 even values between 0xc0 and 0xfe
if (leafVersion >= 0xc0 && leafVersion <= 0xfe) {
return true;
}
// and also 0x66, 0x7e, 0x80, 0x84, 0x96, 0x98, 0xba, 0xbc, 0xbe."
if ([0x66, 0x7e, 0x80, 0x84, 0x96, 0x98, 0xba, 0xbc, 0xbe].includes(leafVersion)) {
return true;
}
// Otherwise, invalid
return false;
}
}
/**

11
backend/src/api/database-migration.ts
View File

@@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
import { RowDataPacket } from 'mysql2';
class DatabaseMigration {
private static currentVersion = 62;
private static currentVersion = 64;
private queryTimeout = 3600_000;
private statisticsAddedIndexed = false;
private uniqueLogs: string[] = [];
@@ -539,6 +539,15 @@ class DatabaseMigration {
await this.updateToSchemaVersion(62);
}
if (databaseSchemaVersion < 63 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `blocks_audits` ADD fullrbf_txs JSON DEFAULT "[]"');
await this.updateToSchemaVersion(63);
}
if (databaseSchemaVersion < 64 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `nodes` ADD features text NULL');
await this.updateToSchemaVersion(64);
}
}
/**

20
backend/src/api/explorer/channels.api.ts
View File

@@ -117,6 +117,26 @@ class ChannelsApi {
}
}
public async $getPenaltyClosedChannels(): Promise<any[]> {
try {
const query = `
SELECT n1.alias AS alias_left,
n2.alias AS alias_right,
channels.*
FROM channels
LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key
LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key
WHERE channels.status = 2 AND channels.closing_reason = 3
ORDER BY closing_date DESC
`;
const [rows]: any = await DB.query(query);
return rows;
} catch (e) {
logger.err('$getPenaltyClosedChannels error: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
public async $getUnresolvedClosedChannels(): Promise<any[]> {
try {
const query = `SELECT * FROM channels WHERE status = 2 AND closing_reason = 2 AND closing_resolved = 0 AND closing_transaction_id != ''`;

13
backend/src/api/explorer/channels.routes.ts
View File

@@ -11,6 +11,7 @@ class ChannelsRoutes {
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels/search/:search', this.$searchChannelsById)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels/:short_id', this.$getChannel)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels', this.$getChannelsForNode)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/penalties', this.$getPenaltyClosedChannels)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels-geo', this.$getAllChannelsGeo)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels-geo/:publicKey', this.$getAllChannelsGeo)
;
@@ -108,6 +109,18 @@ class ChannelsRoutes {
}
}
private async $getPenaltyClosedChannels(req: Request, res: Response): Promise<void> {
try {
const channels = await channelsApi.$getPenaltyClosedChannels();
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(channels);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
private async $getAllChannelsGeo(req: Request, res: Response) {
try {
const style: string = typeof req.query.style === 'string' ? req.query.style : '';

38
backend/src/api/explorer/nodes.api.ts
View File

@@ -3,6 +3,7 @@ import DB from '../../database';
import { ResultSetHeader } from 'mysql2';
import { ILightningApi } from '../lightning/lightning-api.interface';
import { ITopNodesPerCapacity, ITopNodesPerChannels } from '../../mempool.interfaces';
import { bin2hex } from '../../utils/format';
class NodesApi {
public async $getWorldNodes(): Promise<any> {
@@ -56,7 +57,8 @@ class NodesApi {
UNIX_TIMESTAMP(updated_at) AS updated_at, color, sockets as sockets,
as_number, city_id, country_id, subdivision_id, longitude, latitude,
geo_names_iso.names as iso_code, geo_names_as.names as as_organization, geo_names_city.names as city,
geo_names_country.names as country, geo_names_subdivision.names as subdivision
geo_names_country.names as country, geo_names_subdivision.names as subdivision,
features
FROM nodes
LEFT JOIN geo_names geo_names_as on geo_names_as.id = as_number
LEFT JOIN geo_names geo_names_city on geo_names_city.id = city_id
@@ -76,6 +78,23 @@ class NodesApi {
node.city = JSON.parse(node.city);
node.country = JSON.parse(node.country);
// Features
node.features = JSON.parse(node.features);
node.featuresBits = null;
if (node.features) {
let maxBit = 0;
for (const feature of node.features) {
maxBit = Math.max(maxBit, feature.bit);
}
maxBit = Math.ceil(maxBit / 4) * 4 - 1;
node.featuresBits = new Array(maxBit + 1).fill(0);
for (const feature of node.features) {
node.featuresBits[feature.bit] = 1;
}
node.featuresBits = bin2hex(node.featuresBits.reverse().join(''));
}
// Active channels and capacity
const activeChannelsStats: any = await this.$getActiveChannelsStats(public_key);
node.active_channel_count = activeChannelsStats.active_channel_count ?? 0;
@@ -656,10 +675,19 @@ class NodesApi {
alias_search,
color,
sockets,
status
status,
features
)
VALUES (?, NOW(), FROM_UNIXTIME(?), ?, ?, ?, ?, 1)
ON DUPLICATE KEY UPDATE updated_at = FROM_UNIXTIME(?), alias = ?, alias_search = ?, color = ?, sockets = ?, status = 1`;
VALUES (?, NOW(), FROM_UNIXTIME(?), ?, ?, ?, ?, 1, ?)
ON DUPLICATE KEY UPDATE
updated_at = FROM_UNIXTIME(?),
alias = ?,
alias_search = ?,
color = ?,
sockets = ?,
status = 1,
features = ?
`;
await DB.query(query, [
node.pub_key,
@@ -668,11 +696,13 @@ class NodesApi {
this.aliasToSearchText(node.alias),
node.color,
sockets,
JSON.stringify(node.features),
node.last_update,
node.alias,
this.aliasToSearchText(node.alias),
node.color,
sockets,
JSON.stringify(node.features),
]);
} catch (e) {
logger.err('$saveNode() error: ' + (e instanceof Error ? e.message : e));

111
backend/src/api/lightning/clightning/clightning-convert.ts
View File

@@ -2,8 +2,91 @@ import { ILightningApi } from '../lightning-api.interface';
import FundingTxFetcher from '../../../tasks/lightning/sync-tasks/funding-tx-fetcher';
import logger from '../../../logger';
import { Common } from '../../common';
import { hex2bin } from '../../../utils/format';
import config from '../../../config';
// https://github.com/lightningnetwork/lnd/blob/master/lnwire/features.go
export enum FeatureBits {
DataLossProtectRequired = 0,
DataLossProtectOptional = 1,
InitialRoutingSync = 3,
UpfrontShutdownScriptRequired = 4,
UpfrontShutdownScriptOptional = 5,
GossipQueriesRequired = 6,
GossipQueriesOptional = 7,
TLVOnionPayloadRequired = 8,
TLVOnionPayloadOptional = 9,
StaticRemoteKeyRequired = 12,
StaticRemoteKeyOptional = 13,
PaymentAddrRequired = 14,
PaymentAddrOptional = 15,
MPPRequired = 16,
MPPOptional = 17,
WumboChannelsRequired = 18,
WumboChannelsOptional = 19,
AnchorsRequired = 20,
AnchorsOptional = 21,
AnchorsZeroFeeHtlcTxRequired = 22,
AnchorsZeroFeeHtlcTxOptional = 23,
ShutdownAnySegwitRequired = 26,
ShutdownAnySegwitOptional = 27,
AMPRequired = 30,
AMPOptional = 31,
ExplicitChannelTypeRequired = 44,
ExplicitChannelTypeOptional = 45,
ScidAliasRequired = 46,
ScidAliasOptional = 47,
PaymentMetadataRequired = 48,
PaymentMetadataOptional = 49,
ZeroConfRequired = 50,
ZeroConfOptional = 51,
KeysendRequired = 54,
KeysendOptional = 55,
ScriptEnforcedLeaseRequired = 2022,
ScriptEnforcedLeaseOptional = 2023,
MaxBolt11Feature = 5114,
};
export const FeaturesMap = new Map<FeatureBits, string>([
[FeatureBits.DataLossProtectRequired, 'data-loss-protect'],
[FeatureBits.DataLossProtectOptional, 'data-loss-protect'],
[FeatureBits.InitialRoutingSync, 'initial-routing-sync'],
[FeatureBits.UpfrontShutdownScriptRequired, 'upfront-shutdown-script'],
[FeatureBits.UpfrontShutdownScriptOptional, 'upfront-shutdown-script'],
[FeatureBits.GossipQueriesRequired, 'gossip-queries'],
[FeatureBits.GossipQueriesOptional, 'gossip-queries'],
[FeatureBits.TLVOnionPayloadRequired, 'tlv-onion'],
[FeatureBits.TLVOnionPayloadOptional, 'tlv-onion'],
[FeatureBits.StaticRemoteKeyOptional, 'static-remote-key'],
[FeatureBits.StaticRemoteKeyRequired, 'static-remote-key'],
[FeatureBits.PaymentAddrOptional, 'payment-addr'],
[FeatureBits.PaymentAddrRequired, 'payment-addr'],
[FeatureBits.MPPOptional, 'multi-path-payments'],
[FeatureBits.MPPRequired, 'multi-path-payments'],
[FeatureBits.AnchorsRequired, 'anchor-commitments'],
[FeatureBits.AnchorsOptional, 'anchor-commitments'],
[FeatureBits.AnchorsZeroFeeHtlcTxRequired, 'anchors-zero-fee-htlc-tx'],
[FeatureBits.AnchorsZeroFeeHtlcTxOptional, 'anchors-zero-fee-htlc-tx'],
[FeatureBits.WumboChannelsRequired, 'wumbo-channels'],
[FeatureBits.WumboChannelsOptional, 'wumbo-channels'],
[FeatureBits.AMPRequired, 'amp'],
[FeatureBits.AMPOptional, 'amp'],
[FeatureBits.PaymentMetadataOptional, 'payment-metadata'],
[FeatureBits.PaymentMetadataRequired, 'payment-metadata'],
[FeatureBits.ExplicitChannelTypeOptional, 'explicit-commitment-type'],
[FeatureBits.ExplicitChannelTypeRequired, 'explicit-commitment-type'],
[FeatureBits.KeysendOptional, 'keysend'],
[FeatureBits.KeysendRequired, 'keysend'],
[FeatureBits.ScriptEnforcedLeaseRequired, 'script-enforced-lease'],
[FeatureBits.ScriptEnforcedLeaseOptional, 'script-enforced-lease'],
[FeatureBits.ScidAliasRequired, 'scid-alias'],
[FeatureBits.ScidAliasOptional, 'scid-alias'],
[FeatureBits.ZeroConfRequired, 'zero-conf'],
[FeatureBits.ZeroConfOptional, 'zero-conf'],
[FeatureBits.ShutdownAnySegwitRequired, 'shutdown-any-segwit'],
[FeatureBits.ShutdownAnySegwitOptional, 'shutdown-any-segwit'],
]);
/**
* Convert a clightning "listnode" entry to a lnd node entry
*/
@@ -17,10 +100,36 @@ export function convertNode(clNode: any): ILightningApi.Node {
custom_records = undefined;
}
}
const nodeFeatures: ILightningApi.Feature[] = [];
const nodeFeaturesBinary = hex2bin(clNode.features).split('').reverse().join('');
for (let i = 0; i < nodeFeaturesBinary.length; i++) {
if (nodeFeaturesBinary[i] === '0') {
continue;
}
const feature = FeaturesMap.get(i);
if (!feature) {
nodeFeatures.push({
bit: i,
name: 'unknown',
is_required: i % 2 === 0,
is_known: false
});
} else {
nodeFeatures.push({
bit: i,
name: feature,
is_required: i % 2 === 0,
is_known: true
});
}
}
return {
alias: clNode.alias ?? '',
color: `#${clNode.color ?? ''}`,
features: [], // TODO parse and return clNode.feature
features: nodeFeatures,
pub_key: clNode.nodeid,
addresses: clNode.addresses?.map((addr) => {
let address = addr.address;

1
backend/src/api/lightning/lightning-api.interface.ts
View File

@@ -79,6 +79,7 @@ export namespace ILightningApi {
}
export interface Feature {
bit: number;
name: string;
is_required: boolean;
is_known: boolean;

17
backend/src/api/lightning/lnd/lnd-api.ts
View File

@@ -41,8 +41,23 @@ class LndApi implements AbstractLightningApi {
}
async $getNetworkGraph(): Promise<ILightningApi.NetworkGraph> {
return axios.get<ILightningApi.NetworkGraph>(config.LND.REST_API_URL + '/v1/graph', this.axiosConfig)
const graph = await axios.get<ILightningApi.NetworkGraph>(config.LND.REST_API_URL + '/v1/graph', this.axiosConfig)
.then((response) => response.data);
for (const node of graph.nodes) {
const nodeFeatures: ILightningApi.Feature[] = [];
for (const bit in node.features) {
nodeFeatures.push({
bit: parseInt(bit, 10),
name: node.features[bit].name,
is_required: node.features[bit].is_required,
is_known: node.features[bit].is_known,
});
}
node.features = nodeFeatures;
}
return graph;
}
}

1
backend/src/api/mempool-blocks.ts
View File

@@ -457,6 +457,7 @@ class MempoolBlocks {
};
if (matched) {
descendants.push(relative);
mempoolTx.lastBoosted = Math.max(mempoolTx.lastBoosted || 0, mempool[txid].firstSeen || 0);
} else {
ancestors.push(relative);
}

58
backend/src/api/rbf-cache.ts
View File

@@ -6,6 +6,7 @@ import { Common } from "./common";
interface RbfTransaction extends TransactionStripped {
rbf?: boolean;
mined?: boolean;
fullRbf?: boolean;
}
interface RbfTree {
@@ -17,6 +18,16 @@ interface RbfTree {
replaces: RbfTree[];
}
export interface ReplacementInfo {
mined: boolean;
fullRbf: boolean;
txid: string;
oldFee: number;
oldVsize: number;
newFee: number;
newVsize: number;
}
class RbfCache {
private replacedBy: Map<string, string> = new Map();
private replaces: Map<string, string[]> = new Map();
@@ -41,11 +52,15 @@ class RbfCache {
this.txs.set(newTx.txid, newTxExtended);
// maintain rbf trees
let fullRbf = false;
let txFullRbf = false;
let treeFullRbf = false;
const replacedTrees: RbfTree[] = [];
for (const replacedTxExtended of replaced) {
const replacedTx = Common.stripTransaction(replacedTxExtended) as RbfTransaction;
replacedTx.rbf = replacedTxExtended.vin.some((v) => v.sequence < 0xfffffffe);
if (!replacedTx.rbf) {
txFullRbf = true;
}
this.replacedBy.set(replacedTx.txid, newTx.txid);
if (this.treeMap.has(replacedTx.txid)) {
const treeId = this.treeMap.get(replacedTx.txid);
@@ -55,7 +70,7 @@ class RbfCache {
if (tree) {
tree.interval = newTime - tree?.time;
replacedTrees.push(tree);
fullRbf = fullRbf || tree.fullRbf;
treeFullRbf = treeFullRbf || tree.fullRbf || !tree.tx.rbf;
}
}
} else {
@@ -67,15 +82,16 @@ class RbfCache {
fullRbf: !replacedTx.rbf,
replaces: [],
});
fullRbf = fullRbf || !replacedTx.rbf;
treeFullRbf = treeFullRbf || !replacedTx.rbf;
this.txs.set(replacedTx.txid, replacedTxExtended);
}
}
newTx.fullRbf = txFullRbf;
const treeId = replacedTrees[0].tx.txid;
const newTree = {
tx: newTx,
time: newTime,
fullRbf,
fullRbf: treeFullRbf,
replaces: replacedTrees
};
this.rbfTrees.set(treeId, newTree);
@@ -169,6 +185,19 @@ class RbfCache {
}
}
// is the transaction involved in a full rbf replacement?
public isFullRbf(txid: string): boolean {
const treeId = this.treeMap.get(txid);
if (!treeId) {
return false;
}
const tree = this.rbfTrees.get(treeId);
if (!tree) {
return false;
}
return tree?.fullRbf;
}
private cleanup(): void {
const now = Date.now();
for (const txid of this.expiring.keys()) {
@@ -336,6 +365,27 @@ class RbfCache {
}
return tree;
}
public getLatestRbfSummary(): ReplacementInfo[] {
const rbfList = this.getRbfTrees(false);
return rbfList.slice(0, 6).map(rbfTree => {
let oldFee = 0;
let oldVsize = 0;
for (const replaced of rbfTree.replaces) {
oldFee += replaced.tx.fee;
oldVsize += replaced.tx.vsize;
}
return {
txid: rbfTree.tx.txid,
mined: !!rbfTree.tx.mined,
fullRbf: !!rbfTree.tx.fullRbf,
oldFee,
oldVsize,
newFee: rbfTree.tx.fee,
newVsize: rbfTree.tx.vsize,
};
});
}
}
export default new RbfCache();

78
backend/src/api/websocket-handler.ts
View File

@@ -12,7 +12,7 @@ import { Common } from './common';
import loadingIndicators from './loading-indicators';
import config from '../config';
import transactionUtils from './transaction-utils';
import rbfCache from './rbf-cache';
import rbfCache, { ReplacementInfo } from './rbf-cache';
import difficultyAdjustment from './difficulty-adjustment';
import feeApi from './fee-api';
import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository';
@@ -40,6 +40,7 @@ class WebsocketHandler {
private socketData: { [key: string]: string } = {};
private serializedInitData: string = '{}';
private lastRbfSummary: ReplacementInfo | null = null;
constructor() { }
@@ -225,8 +226,19 @@ class WebsocketHandler {
}
}
if (parsedMessage && parsedMessage['track-rbf-summary'] != null) {
if (parsedMessage['track-rbf-summary']) {
client['track-rbf-summary'] = true;
if (this.socketData['rbfSummary'] != null) {
response['rbfLatestSummary'] = this.socketData['rbfSummary'];
}
} else {
client['track-rbf-summary'] = false;
}
}
if (parsedMessage.action === 'init') {
if (!this.socketData['blocks']?.length || !this.socketData['da']) {
if (!this.socketData['blocks']?.length || !this.socketData['da'] || !this.socketData['backendInfo'] || !this.socketData['conversions']) {
this.updateSocketData();
}
if (!this.socketData['blocks']?.length) {
@@ -333,6 +345,40 @@ class WebsocketHandler {
});
}
handleReorg(): void {
if (!this.wss) {
throw new Error('WebSocket.Server is not set');
}
const da = difficultyAdjustment.getDifficultyAdjustment();
// update init data
this.updateSocketDataFields({
'blocks': blocks.getBlocks(),
'da': da?.previousTime ? da : undefined,
});
this.wss.clients.forEach((client) => {
if (client.readyState !== WebSocket.OPEN) {
return;
}
const response = {};
if (client['want-blocks']) {
response['blocks'] = this.socketData['blocks'];
}
if (client['want-stats']) {
response['da'] = this.socketData['da'];
}
if (Object.keys(response).length) {
const serializedResponse = this.serializeResponse(response);
client.send(serializedResponse);
}
});
}
async $handleMempoolChange(newMempool: { [txid: string]: MempoolTransactionExtended }, mempoolSize: number,
newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[]): Promise<void> {
if (!this.wss) {
@@ -361,10 +407,13 @@ class WebsocketHandler {
const rbfChanges = rbfCache.getRbfChanges();
let rbfReplacements;
let fullRbfReplacements;
let rbfSummary;
if (Object.keys(rbfChanges.trees).length) {
rbfReplacements = rbfCache.getRbfTrees(false);
fullRbfReplacements = rbfCache.getRbfTrees(true);
rbfSummary = rbfCache.getLatestRbfSummary();
}
for (const deletedTx of deletedTransactions) {
rbfCache.evict(deletedTx.txid);
}
@@ -372,10 +421,10 @@ class WebsocketHandler {
memPool.addToSpendMap(newTransactions);
const recommendedFees = feeApi.getRecommendedFee();
const latestTransactions = newTransactions.slice(0, 6).map((tx) => Common.stripTransaction(tx));
const latestTransactions = memPool.getLatestTransactions();
// update init data
this.updateSocketDataFields({
const socketDataFields = {
'mempoolInfo': mempoolInfo,
'vBytesPerSecond': vBytesPerSecond,
'mempool-blocks': mBlocks,
@@ -383,7 +432,11 @@ class WebsocketHandler {
'loadingIndicators': loadingIndicators.getLoadingIndicators(),
'da': da?.previousTime ? da : undefined,
'fees': recommendedFees,
});
};
if (rbfSummary) {
socketDataFields['rbfSummary'] = rbfSummary;
}
this.updateSocketDataFields(socketDataFields);
// cache serialized objects to avoid stringify-ing the same thing for every client
const responseCache = { ...this.socketData };
@@ -567,6 +620,10 @@ class WebsocketHandler {
response['rbfLatest'] = getCachedResponse('fullrbfLatest', fullRbfReplacements);
}
if (client['track-rbf-summary'] && rbfSummary) {
response['rbfLatestSummary'] = getCachedResponse('rbfLatestSummary', rbfSummary);
}
if (Object.keys(response).length) {
const serializedResponse = this.serializeResponse(response);
client.send(serializedResponse);
@@ -583,6 +640,10 @@ class WebsocketHandler {
const _memPool = memPool.getMempool();
const rbfTransactions = Common.findMinedRbfTransactions(transactions, memPool.getSpendMap());
memPool.handleMinedRbfTransactions(rbfTransactions);
memPool.removeFromSpendMap(transactions);
if (config.MEMPOOL.AUDIT) {
let projectedBlocks;
let auditMempool = _memPool;
@@ -605,7 +666,7 @@ class WebsocketHandler {
}
if (Common.indexingEnabled() && memPool.isInSync()) {
const { censored, added, fresh, sigop, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool);
const { censored, added, fresh, sigop, fullrbf, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool);
const matchRate = Math.round(score * 100 * 100) / 100;
const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions : [];
@@ -633,6 +694,7 @@ class WebsocketHandler {
missingTxs: censored,
freshTxs: fresh,
sigopTxs: sigop,
fullrbfTxs: fullrbf,
matchRate: matchRate,
expectedFees: totalFees,
expectedWeight: totalWeight,
@@ -652,10 +714,6 @@ class WebsocketHandler {
}
}
const rbfTransactions = Common.findMinedRbfTransactions(transactions, memPool.getSpendMap());
memPool.handleMinedRbfTransactions(rbfTransactions);
memPool.removeFromSpendMap(transactions);
// Update mempool to remove transactions included in the new block
for (const txId of txIds) {
delete _memPool[txId];

18
backend/src/config.ts
View File

@@ -35,6 +35,8 @@ interface IConfig {
CPFP_INDEXING: boolean;
MAX_BLOCKS_BULK_QUERY: number;
DISK_CACHE_BLOCK_INTERVAL: number;
MAX_PUSH_TX_SIZE_WEIGHT: number;
ALLOW_UNREACHABLE: boolean;
};
ESPLORA: {
REST_API_URL: string;
@@ -130,6 +132,12 @@ interface IConfig {
GEOLITE2_ASN: string;
GEOIP2_ISP: string;
},
REPLICATION: {
ENABLED: boolean;
AUDIT: boolean;
AUDIT_START_HEIGHT: number;
SERVERS: string[];
}
}
const defaults: IConfig = {
@@ -165,6 +173,8 @@ const defaults: IConfig = {
'CPFP_INDEXING': false,
'MAX_BLOCKS_BULK_QUERY': 0,
'DISK_CACHE_BLOCK_INTERVAL': 6,
'MAX_PUSH_TX_SIZE_WEIGHT': 400000,
'ALLOW_UNREACHABLE': true,
},
'ESPLORA': {
'REST_API_URL': 'http://127.0.0.1:3000',
@@ -260,6 +270,12 @@ const defaults: IConfig = {
'GEOLITE2_ASN': '/usr/local/share/GeoIP/GeoLite2-ASN.mmdb',
'GEOIP2_ISP': '/usr/local/share/GeoIP/GeoIP2-ISP.mmdb'
},
'REPLICATION': {
'ENABLED': false,
'AUDIT': false,
'AUDIT_START_HEIGHT': 774000,
'SERVERS': [],
}
};
class Config implements IConfig {
@@ -279,6 +295,7 @@ class Config implements IConfig {
PRICE_DATA_SERVER: IConfig['PRICE_DATA_SERVER'];
EXTERNAL_DATA_SERVER: IConfig['EXTERNAL_DATA_SERVER'];
MAXMIND: IConfig['MAXMIND'];
REPLICATION: IConfig['REPLICATION'];
constructor() {
const configs = this.merge(configFromFile, defaults);
@@ -298,6 +315,7 @@ class Config implements IConfig {
this.PRICE_DATA_SERVER = configs.PRICE_DATA_SERVER;
this.EXTERNAL_DATA_SERVER = configs.EXTERNAL_DATA_SERVER;
this.MAXMIND = configs.MAXMIND;
this.REPLICATION = configs.REPLICATION;
}
merge = (...objects: object[]): IConfig => {

5
backend/src/index.ts
View File

@@ -169,6 +169,7 @@ class Server {
}
async runMainUpdateLoop(): Promise<void> {
const start = Date.now();
try {
try {
await memPool.$updateMemPoolInfo();
@@ -188,7 +189,9 @@ class Server {
indexer.$run();
// rerun immediately if we skipped the mempool update, otherwise wait POLL_RATE_MS
setTimeout(this.runMainUpdateLoop.bind(this), numHandledBlocks > 0 ? 1 : config.MEMPOOL.POLL_RATE_MS);
const elapsed = Date.now() - start;
const remainingTime = Math.max(0, config.MEMPOOL.POLL_RATE_MS - elapsed)
setTimeout(this.runMainUpdateLoop.bind(this), numHandledBlocks > 0 ? 0 : remainingTime);
this.backendRetryCount = 0;
} catch (e: any) {
this.backendRetryCount++;

5
backend/src/indexer.ts
View File

@@ -6,6 +6,8 @@ import logger from './logger';
import bitcoinClient from './api/bitcoin/bitcoin-client';
import priceUpdater from './tasks/price-updater';
import PricesRepository from './repositories/PricesRepository';
import config from './config';
import auditReplicator from './replication/AuditReplication';
export interface CoreIndex {
name: string;
@@ -72,7 +74,7 @@ class Indexer {
return;
}
if (task === 'blocksPrices' && !this.tasksRunning.includes(task)) {
if (task === 'blocksPrices' && !this.tasksRunning.includes(task) && !['testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
this.tasksRunning.push(task);
const lastestPriceId = await PricesRepository.$getLatestPriceId();
if (priceUpdater.historyInserted === false || lastestPriceId === null) {
@@ -135,6 +137,7 @@ class Indexer {
await blocks.$generateBlocksSummariesDatabase();
await blocks.$generateCPFPDatabase();
await blocks.$generateAuditStats();
await auditReplicator.$sync();
} catch (e) {
this.indexerRunning = false;
logger.err(`Indexer failed, trying again in 10 seconds. Reason: ` + (e instanceof Error ? e.message : e));

12
backend/src/mempool.interfaces.ts
View File

@@ -34,6 +34,7 @@ export interface BlockAudit {
missingTxs: string[],
freshTxs: string[],
sigopTxs: string[],
fullrbfTxs: string[],
addedTxs: string[],
matchRate: number,
expectedFees?: number,
@@ -99,6 +100,7 @@ export interface MempoolTransactionExtended extends TransactionExtended {
adjustedVsize: number;
adjustedFeePerVsize: number;
inputs?: number[];
lastBoosted?: number;
}
export interface AuditTransaction {
@@ -227,6 +229,7 @@ export interface BlockExtension {
*/
export interface BlockExtended extends IEsploraApi.Block {
extras: BlockExtension;
canonical?: string;
}
export interface BlockSummary {
@@ -234,6 +237,15 @@ export interface BlockSummary {
transactions: TransactionStripped[];
}
export interface AuditSummary extends BlockAudit {
timestamp?: number,
size?: number,
weight?: number,
tx_count?: number,
transactions: TransactionStripped[];
template?: TransactionStripped[];
}
export interface BlockPrice {
height: number;
priceId: number;

134
backend/src/replication/AuditReplication.ts Normal file
View File

@@ -0,0 +1,134 @@
import DB from '../database';
import logger from '../logger';
import { AuditSummary } from '../mempool.interfaces';
import blocksAuditsRepository from '../repositories/BlocksAuditsRepository';
import blocksSummariesRepository from '../repositories/BlocksSummariesRepository';
import { $sync } from './replicator';
import config from '../config';
import { Common } from '../api/common';
import blocks from '../api/blocks';
const BATCH_SIZE = 16;
/**
* Syncs missing block template and audit data from trusted servers
*/
class AuditReplication {
inProgress: boolean = false;
skip: Set<string> = new Set();
public async $sync(): Promise<void> {
if (!config.REPLICATION.ENABLED || !config.REPLICATION.AUDIT) {
// replication not enabled
return;
}
if (this.inProgress) {
logger.info(`AuditReplication sync already in progress`, 'Replication');
return;
}
this.inProgress = true;
const missingAudits = await this.$getMissingAuditBlocks();
logger.debug(`Fetching missing audit data for ${missingAudits.length} blocks from trusted servers`, 'Replication');
let totalSynced = 0;
let totalMissed = 0;
let loggerTimer = Date.now();
// process missing audits in batches of
for (let i = 0; i < missingAudits.length; i += BATCH_SIZE) {
const slice = missingAudits.slice(i, i + BATCH_SIZE);
const results = await Promise.all(slice.map(hash => this.$syncAudit(hash)));
const synced = results.reduce((total, status) => status ? total + 1 : total, 0);
totalSynced += synced;
totalMissed += (slice.length - synced);
if (Date.now() - loggerTimer > 10000) {
loggerTimer = Date.now();
logger.info(`Found ${totalSynced} / ${totalSynced + totalMissed} of ${missingAudits.length} missing audits`, 'Replication');
}
await Common.sleep$(1000);
}
logger.debug(`Fetched ${totalSynced} audits, ${totalMissed} still missing`, 'Replication');
this.inProgress = false;
}
private async $syncAudit(hash: string): Promise<boolean> {
if (this.skip.has(hash)) {
// we already know none of our trusted servers have this audit
return false;
}
let success = false;
// start with a random server so load is uniformly spread
const syncResult = await $sync(`/api/v1/block/${hash}/audit-summary`);
if (syncResult) {
if (syncResult.data?.template?.length) {
await this.$saveAuditData(hash, syncResult.data);
logger.info(`Imported audit data from ${syncResult.server} for block ${syncResult.data.height} (${hash})`);
success = true;
}
if (!syncResult.data && !syncResult.exists) {
this.skip.add(hash);
}
}
return success;
}
private async $getMissingAuditBlocks(): Promise<string[]> {
try {
const startHeight = config.REPLICATION.AUDIT_START_HEIGHT || 0;
const [rows]: any[] = await DB.query(`
SELECT auditable.hash, auditable.height
FROM (
SELECT hash, height
FROM blocks
WHERE height >= ?
) AS auditable
LEFT JOIN blocks_audits ON auditable.hash = blocks_audits.hash
WHERE blocks_audits.hash IS NULL
ORDER BY auditable.height DESC
`, [startHeight]);
return rows.map(row => row.hash);
} catch (e: any) {
logger.err(`Cannot fetch missing audit blocks from db. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
private async $saveAuditData(blockHash: string, auditSummary: AuditSummary): Promise<void> {
// save audit & template to DB
await blocksSummariesRepository.$saveTemplate({
height: auditSummary.height,
template: {
id: blockHash,
transactions: auditSummary.template || []
}
});
await blocksAuditsRepository.$saveAudit({
hash: blockHash,
height: auditSummary.height,
time: auditSummary.timestamp || auditSummary.time,
missingTxs: auditSummary.missingTxs || [],
addedTxs: auditSummary.addedTxs || [],
freshTxs: auditSummary.freshTxs || [],
sigopTxs: auditSummary.sigopTxs || [],
fullrbfTxs: auditSummary.fullrbfTxs || [],
matchRate: auditSummary.matchRate,
expectedFees: auditSummary.expectedFees,
expectedWeight: auditSummary.expectedWeight,
});
// add missing data to cached blocks
const cachedBlock = blocks.getBlocks().find(block => block.id === blockHash);
if (cachedBlock) {
cachedBlock.extras.matchRate = auditSummary.matchRate;
cachedBlock.extras.expectedFees = auditSummary.expectedFees || null;
cachedBlock.extras.expectedWeight = auditSummary.expectedWeight || null;
}
}
}
export default new AuditReplication();

70
backend/src/replication/replicator.ts Normal file
View File

@@ -0,0 +1,70 @@
import config from '../config';
import backendInfo from '../api/backend-info';
import axios, { AxiosResponse } from 'axios';
import { SocksProxyAgent } from 'socks-proxy-agent';
import * as https from 'https';
export async function $sync(path): Promise<{ data?: any, exists: boolean, server?: string }> {
// start with a random server so load is uniformly spread
let allMissing = true;
const offset = Math.floor(Math.random() * config.REPLICATION.SERVERS.length);
for (let i = 0; i < config.REPLICATION.SERVERS.length; i++) {
const server = config.REPLICATION.SERVERS[(i + offset) % config.REPLICATION.SERVERS.length];
// don't query ourself
if (server === backendInfo.getBackendInfo().hostname) {
continue;
}
try {
const result = await query(`https://${server}${path}`);
if (result) {
return { data: result, exists: true, server };
}
} catch (e: any) {
if (e?.response?.status === 404) {
// this server is also missing this data
} else {
// something else went wrong
allMissing = false;
}
}
}
return { exists: !allMissing };
}
export async function query(path): Promise<object> {
type axiosOptions = {
headers: {
'User-Agent': string
};
timeout: number;
httpsAgent?: https.Agent;
};
const axiosOptions: axiosOptions = {
headers: {
'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
},
timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000
};
if (config.SOCKS5PROXY.ENABLED) {
const socksOptions = {
agentOptions: {
keepAlive: true,
},
hostname: config.SOCKS5PROXY.HOST,
port: config.SOCKS5PROXY.PORT,
username: config.SOCKS5PROXY.USERNAME || 'circuit0',
password: config.SOCKS5PROXY.PASSWORD,
};
axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions);
}
const data: AxiosResponse = await axios.get(path, axiosOptions);
if (data.statusText === 'error' || !data.data) {
throw new Error(`${data.status}`);
}
return data.data;
}

12
backend/src/repositories/BlocksAuditsRepository.ts
View File

@@ -6,9 +6,9 @@ import { BlockAudit, AuditScore } from '../mempool.interfaces';
class BlocksAuditRepositories {
public async $saveAudit(audit: BlockAudit): Promise<void> {
try {
await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, fresh_txs, sigop_txs, match_rate, expected_fees, expected_weight)
VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs),
JSON.stringify(audit.addedTxs), JSON.stringify(audit.freshTxs), JSON.stringify(audit.sigopTxs), audit.matchRate, audit.expectedFees, audit.expectedWeight]);
await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, fresh_txs, sigop_txs, fullrbf_txs, match_rate, expected_fees, expected_weight)
VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs),
JSON.stringify(audit.addedTxs), JSON.stringify(audit.freshTxs), JSON.stringify(audit.sigopTxs), JSON.stringify(audit.fullrbfTxs), audit.matchRate, audit.expectedFees, audit.expectedWeight]);
} catch (e: any) {
if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
logger.debug(`Cannot save block audit for block ${audit.hash} because it has already been indexed, ignoring`);
@@ -62,18 +62,17 @@ class BlocksAuditRepositories {
public async $getBlockAudit(hash: string): Promise<any> {
try {
const [rows]: any[] = await DB.query(
`SELECT blocks.height, blocks.hash as id, UNIX_TIMESTAMP(blocks.blockTimestamp) as timestamp, blocks.size,
blocks.weight, blocks.tx_count,
`SELECT blocks_audits.height, blocks_audits.hash as id, UNIX_TIMESTAMP(blocks_audits.time) as timestamp,
template,
missing_txs as missingTxs,
added_txs as addedTxs,
fresh_txs as freshTxs,
sigop_txs as sigopTxs,
fullrbf_txs as fullrbfTxs,
match_rate as matchRate,
expected_fees as expectedFees,
expected_weight as expectedWeight
FROM blocks_audits
JOIN blocks ON blocks.hash = blocks_audits.hash
JOIN blocks_templates ON blocks_templates.id = blocks_audits.hash
WHERE blocks_audits.hash = "${hash}"
`);
@@ -83,6 +82,7 @@ class BlocksAuditRepositories {
rows[0].addedTxs = JSON.parse(rows[0].addedTxs);
rows[0].freshTxs = JSON.parse(rows[0].freshTxs);
rows[0].sigopTxs = JSON.parse(rows[0].sigopTxs);
rows[0].fullrbfTxs = JSON.parse(rows[0].fullrbfTxs);
rows[0].template = JSON.parse(rows[0].template);
return rows[0];

6
backend/src/repositories/BlocksRepository.ts
View File

@@ -401,7 +401,7 @@ class BlocksRepository {
/**
* Get average block health for all blocks for a single pool
*/
public async $getAvgBlockHealthPerPoolId(poolId: number): Promise<number> {
public async $getAvgBlockHealthPerPoolId(poolId: number): Promise<number | null> {
const params: any[] = [];
const query = `
SELECT AVG(blocks_audits.match_rate) AS avg_match_rate
@@ -413,8 +413,8 @@ class BlocksRepository {
try {
const [rows] = await DB.query(query, params);
if (!rows[0] || !rows[0].avg_match_rate) {
return 0;
if (!rows[0] || rows[0].avg_match_rate == null) {
return null;
}
return Math.round(rows[0].avg_match_rate * 100) / 100;
} catch (e) {

9
backend/src/tasks/lightning/network-sync.service.ts
View File

@@ -3,7 +3,6 @@ import logger from '../../logger';
import channelsApi from '../../api/explorer/channels.api';
import bitcoinApi from '../../api/bitcoin/bitcoin-api-factory';
import config from '../../config';
import { IEsploraApi } from '../../api/bitcoin/esplora-api.interface';
import { ILightningApi } from '../../api/lightning/lightning-api.interface';
import { $lookupNodeLocation } from './sync-tasks/node-locations';
import lightningApi from '../../api/lightning/lightning-api-factory';
@@ -269,7 +268,11 @@ class NetworkSyncService {
}
private async $scanForClosedChannels(): Promise<void> {
if (this.closedChannelsScanBlock === blocks.getCurrentBlockHeight()) {
let currentBlockHeight = blocks.getCurrentBlockHeight();
if (config.MEMPOOL.ENABLED === false) { // https://github.com/mempool/mempool/issues/3582
currentBlockHeight = await bitcoinApi.$getBlockHeightTip();
}
if (this.closedChannelsScanBlock === currentBlockHeight) {
logger.debug(`We've already scan closed channels for this block, skipping.`);
return;
}
@@ -305,7 +308,7 @@ class NetworkSyncService {
}
}
this.closedChannelsScanBlock = blocks.getCurrentBlockHeight();
this.closedChannelsScanBlock = currentBlockHeight;
logger.debug(`Closed channels scan completed at block ${this.closedChannelsScanBlock}`, logger.tags.ln);
} catch (e) {
logger.err(`$scanForClosedChannels() error: ${e instanceof Error ? e.message : e}`, logger.tags.ln);

1
backend/src/tasks/price-updater.ts
View File

@@ -153,6 +153,7 @@ class PriceUpdater {
try {
const p = 60 * 60 * 1000; // milliseconds in an hour
const nowRounded = new Date(Math.round(new Date().getTime() / p) * p); // https://stackoverflow.com/a/28037042
this.latestPrices.time = nowRounded.getTime() / 1000;
await PricesRepository.$savePrices(nowRounded.getTime() / 1000, this.latestPrices);
} catch (e) {
this.lastRun = previousRun + 5 * 60;

66
backend/src/utils/format.ts
View File

@@ -26,4 +26,70 @@ export function formatBytes(bytes: number, toUnit: string, skipUnit = false): st
}
return `${bytes.toFixed(2)}${skipUnit ? '' : ' ' + byteUnits[unitIndex]}`;
}
// https://stackoverflow.com/a/64235212
export function hex2bin(hex: string): string {
if (!hex) {
return '';
}
hex = hex.replace('0x', '').toLowerCase();
let out = '';
for (const c of hex) {
switch (c) {
case '0': out += '0000'; break;
case '1': out += '0001'; break;
case '2': out += '0010'; break;
case '3': out += '0011'; break;
case '4': out += '0100'; break;
case '5': out += '0101'; break;
case '6': out += '0110'; break;
case '7': out += '0111'; break;
case '8': out += '1000'; break;
case '9': out += '1001'; break;
case 'a': out += '1010'; break;
case 'b': out += '1011'; break;
case 'c': out += '1100'; break;
case 'd': out += '1101'; break;
case 'e': out += '1110'; break;
case 'f': out += '1111'; break;
default: return '';
}
}
return out;
}
export function bin2hex(bin: string): string {
if (!bin) {
return '';
}
let out = '';
for (let i = 0; i < bin.length; i += 4) {
const c = bin.substring(i, i + 4);
switch (c) {
case '0000': out += '0'; break;
case '0001': out += '1'; break;
case '0010': out += '2'; break;
case '0011': out += '3'; break;
case '0100': out += '4'; break;
case '0101': out += '5'; break;
case '0110': out += '6'; break;
case '0111': out += '7'; break;
case '1000': out += '8'; break;
case '1001': out += '9'; break;
case '1010': out += 'a'; break;
case '1011': out += 'b'; break;
case '1100': out += 'c'; break;
case '1101': out += 'd'; break;
case '1110': out += 'e'; break;
case '1111': out += 'f'; break;
default: return '';
}
}
return out;
}