diff --git a/.github/workflows/get_backend_block_height.yml b/.github/workflows/get_backend_block_height.yml new file mode 100644 index 000000000..52f3b038c --- /dev/null +++ b/.github/workflows/get_backend_block_height.yml @@ -0,0 +1,19 @@ +name: 'Check if servers are in sync' + +on: [workflow_dispatch] + +jobs: + print-backend-sha: + runs-on: 'ubuntu-latest' + name: Get block height + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + path: repo + + - name: Run script + working-directory: repo + run: | + chmod +x ./scripts/get_block_tip_height.sh + sh ./scripts/get_block_tip_height.sh diff --git a/Cargo.lock b/Cargo.lock index 30a0d97ab..0b51ea544 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -57,9 +57,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1586fa608b1dab41f667475b4a41faec5ba680aee428bfa5de4ea520fdc6e901" dependencies = [ "quote", - "syn 2.0.20", + "syn", ] +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "gbt" version = "1.0.0" @@ -71,15 +77,15 @@ dependencies = [ "napi-derive", "priority-queue", "tracing", - "tracing-log", + "tracing-log 0.2.0", "tracing-subscriber", ] [[package]] name = "hashbrown" -version = "0.12.3" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" [[package]] name = "hermit-abi" @@ -92,11 +98,11 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.9.3" +version = "2.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +checksum = "7b0b929d511467233429c45a44ac1dcaa21ba0f5ba11e4879e6ed28ddb4f9df4" dependencies = [ - "autocfg", + "equivalent", "hashbrown", ] @@ -114,12 +120,12 @@ checksum = "f92be4933c13fd498862a9e02a3055f8a8d9c039ce33db97306fd5a6caa7f29b" [[package]] name = "libloading" -version = "0.7.4" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" +checksum = "2caa5afb8bf9f3a2652760ce7d4f62d21c4d5a423e68466fca30df82f2330164" dependencies = [ "cfg-if", - "winapi", + "windows-targets", ] [[package]] @@ -145,9 +151,9 @@ checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "napi" -version = "2.13.2" +version = "2.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ede2d12cd6fce44da537a4be1f5510c73be2506c2e32dfaaafd1f36968f3a0e" +checksum = "54a63d0570e4c3e0daf7a8d380563610e159f538e20448d6c911337246f40e84" dependencies = [ "bitflags", "ctor", @@ -159,29 +165,29 @@ dependencies = [ [[package]] name = "napi-build" -version = "2.0.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "882a73d9ef23e8dc2ebbffb6a6ae2ef467c0f18ac10711e4cc59c5485d41df0e" +checksum = "2f9130fccc5f763cf2069b34a089a18f0d0883c66aceb81f2fad541a3d823c43" [[package]] name = "napi-derive" -version = "2.13.0" +version = "2.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da1c6a8fa84d549aa8708fcd062372bf8ec6e849de39016ab921067d21bde367" +checksum = "05bb7c37e3c1dda9312fdbe4a9fc7507fca72288ba154ec093e2d49114e727ce" dependencies = [ "cfg-if", "convert_case", "napi-derive-backend", "proc-macro2", "quote", - "syn 1.0.109", + "syn", ] [[package]] name = "napi-derive-backend" -version = "1.0.52" +version = "1.0.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20bbc7c69168d06a848f925ec5f0e0997f98e8c8d4f2cc30157f0da51c009e17" +checksum = "f785a8b8d7b83e925f5aa6d2ae3c159d17fe137ac368dc185bef410e7acdaeb4" dependencies = [ "convert_case", "once_cell", @@ -189,14 +195,14 @@ dependencies = [ "quote", "regex", "semver", - "syn 1.0.109", + "syn", ] [[package]] name = "napi-sys" -version = "2.2.3" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "166b5ef52a3ab5575047a9fe8d4a030cdd0f63c96f071cd6907674453b07bae3" +checksum = "2503fa6af34dc83fb74888df8b22afe933b58d37daf7d80424b1c60c68196b8b" dependencies = [ "libloading", ] @@ -223,9 +229,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "overload" @@ -241,11 +247,12 @@ checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" [[package]] name = "priority-queue" -version = "1.3.2" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fff39edfcaec0d64e8d0da38564fad195d2d51b680940295fcc307366e101e61" +checksum = "509354d8a769e8d0b567d6821b84495c60213162761a732d68ce87c964bd347f" dependencies = [ "autocfg", + "equivalent", "indexmap", ] @@ -320,17 +327,6 @@ version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - [[package]] name = "syn" version = "2.0.20" @@ -384,7 +380,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.20", + "syn", ] [[package]] @@ -408,6 +404,17 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + [[package]] name = "tracing-subscriber" version = "0.3.17" @@ -423,7 +430,7 @@ dependencies = [ "thread_local", "tracing", "tracing-core", - "tracing-log", + "tracing-log 0.1.3", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 59562297c..2f70699f4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,4 +1,5 @@ [workspace] +resolver = "2" members = [ "./backend/rust-gbt", ] diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 3c2fccfb7..5d2cf1fba 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -1,5 +1,6 @@ { "MEMPOOL": { + "OFFICIAL": false, "NETWORK": "mainnet", "BACKEND": "electrum", "ENABLED": true, diff --git a/backend/package-lock.json b/backend/package-lock.json index 8c18dc8c2..95a949ef5 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -9,6 +9,7 @@ "version": "3.0.0-dev", "license": "GNU Affero General Public License v3.0", "dependencies": { + "@babel/core": "^7.24.0", "@mempool/electrum-client": "1.1.9", "@types/node": "^18.15.3", "axios": "~1.6.1", @@ -1499,9 +1500,9 @@ } }, "node_modules/@napi-rs/cli": { - "version": "2.16.1", - "resolved": "https://registry.npmjs.org/@napi-rs/cli/-/cli-2.16.1.tgz", - "integrity": "sha512-L0Gr5iEQIDEbvWdDr1HUaBOxBSHL1VZhWSk1oryawoT8qJIY+KGfLFelU+Qma64ivCPbxYpkfPoKYVG3rcoGIA==", + "version": "2.18.0", + "resolved": "https://registry.npmjs.org/@napi-rs/cli/-/cli-2.18.0.tgz", + "integrity": "sha512-lfSRT7cs3iC4L+kv9suGYQEezn5Nii7Kpu+THsYVI0tA1Vh59LH45p4QADaD7hvIkmOz79eEGtoKQ9nAkAPkzA==", "bin": { "napi": "scripts/index.js" }, @@ -7669,7 +7670,7 @@ "version": "3.0.1", "hasInstallScript": true, "dependencies": { - "@napi-rs/cli": "2.16.1" + "@napi-rs/cli": "2.18.0" }, "engines": { "node": ">= 12" @@ -8774,9 +8775,9 @@ "integrity": "sha512-mlvPiCzUlaETpYW3i6V87A24jjMYgsebaXtUo3WQyyLnYUuxs0KiXQ2mnKh3h15j8Xg/hfxeGIi+5OC9u0nftQ==" }, "@napi-rs/cli": { - "version": "2.16.1", - "resolved": "https://registry.npmjs.org/@napi-rs/cli/-/cli-2.16.1.tgz", - "integrity": "sha512-L0Gr5iEQIDEbvWdDr1HUaBOxBSHL1VZhWSk1oryawoT8qJIY+KGfLFelU+Qma64ivCPbxYpkfPoKYVG3rcoGIA==" + "version": "2.18.0", + "resolved": "https://registry.npmjs.org/@napi-rs/cli/-/cli-2.18.0.tgz", + "integrity": "sha512-lfSRT7cs3iC4L+kv9suGYQEezn5Nii7Kpu+THsYVI0tA1Vh59LH45p4QADaD7hvIkmOz79eEGtoKQ9nAkAPkzA==" }, "@noble/hashes": { "version": "1.3.0", @@ -12702,7 +12703,7 @@ "rust-gbt": { "version": "file:rust-gbt", "requires": { - "@napi-rs/cli": "2.16.1" + "@napi-rs/cli": "2.18.0" } }, "safe-buffer": { diff --git a/backend/rust-gbt/Cargo.toml b/backend/rust-gbt/Cargo.toml index 4d0a5b45d..10c572bf9 100644 --- a/backend/rust-gbt/Cargo.toml +++ b/backend/rust-gbt/Cargo.toml @@ -12,14 +12,14 @@ crate-type = ["cdylib"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -priority-queue = "1.3.2" +priority-queue = "2.0.2" bytes = "1.4.0" -napi = { version = "2.13.2", features = ["napi8", "tokio_rt"] } -napi-derive = "2.13.0" +napi = { version = "2.16.0", features = ["napi8", "tokio_rt"] } +napi-derive = "2.16.0" bytemuck = "1.13.1" tracing = "0.1.36" -tracing-log = "0.1.3" +tracing-log = "0.2.0" tracing-subscriber = { version = "0.3.15", features = ["env-filter"]} [build-dependencies] -napi-build = "2.0.1" +napi-build = "2.1.2" diff --git a/backend/rust-gbt/index.js b/backend/rust-gbt/index.js index 8680501d1..dd58a8b76 100644 --- a/backend/rust-gbt/index.js +++ b/backend/rust-gbt/index.js @@ -237,6 +237,49 @@ switch (platform) { loadError = e } break + case 'riscv64': + if (isMusl()) { + localFileExisted = existsSync( + join(__dirname, 'gbt.linux-riscv64-musl.node') + ) + try { + if (localFileExisted) { + nativeBinding = require('./gbt.linux-riscv64-musl.node') + } else { + nativeBinding = require('gbt-linux-riscv64-musl') + } + } catch (e) { + loadError = e + } + } else { + localFileExisted = existsSync( + join(__dirname, 'gbt.linux-riscv64-gnu.node') + ) + try { + if (localFileExisted) { + nativeBinding = require('./gbt.linux-riscv64-gnu.node') + } else { + nativeBinding = require('gbt-linux-riscv64-gnu') + } + } catch (e) { + loadError = e + } + } + break + case 's390x': + localFileExisted = existsSync( + join(__dirname, 'gbt.linux-s390x-gnu.node') + ) + try { + if (localFileExisted) { + nativeBinding = require('./gbt.linux-s390x-gnu.node') + } else { + nativeBinding = require('gbt-linux-s390x-gnu') + } + } catch (e) { + loadError = e + } + break default: throw new Error(`Unsupported architecture on Linux: ${arch}`) } diff --git a/backend/rust-gbt/package-lock.json b/backend/rust-gbt/package-lock.json index ab3d72e52..e351c82f8 100644 --- a/backend/rust-gbt/package-lock.json +++ b/backend/rust-gbt/package-lock.json @@ -9,16 +9,16 @@ "version": "3.0.1", "hasInstallScript": true, "dependencies": { - "@napi-rs/cli": "2.16.1" + "@napi-rs/cli": "2.18.0" }, "engines": { "node": ">= 12" } }, "node_modules/@napi-rs/cli": { - "version": "2.16.1", - "resolved": "https://registry.npmjs.org/@napi-rs/cli/-/cli-2.16.1.tgz", - "integrity": "sha512-L0Gr5iEQIDEbvWdDr1HUaBOxBSHL1VZhWSk1oryawoT8qJIY+KGfLFelU+Qma64ivCPbxYpkfPoKYVG3rcoGIA==", + "version": "2.18.0", + "resolved": "https://registry.npmjs.org/@napi-rs/cli/-/cli-2.18.0.tgz", + "integrity": "sha512-lfSRT7cs3iC4L+kv9suGYQEezn5Nii7Kpu+THsYVI0tA1Vh59LH45p4QADaD7hvIkmOz79eEGtoKQ9nAkAPkzA==", "bin": { "napi": "scripts/index.js" }, diff --git a/backend/rust-gbt/package.json b/backend/rust-gbt/package.json index aa98313ed..b0dd96698 100644 --- a/backend/rust-gbt/package.json +++ b/backend/rust-gbt/package.json @@ -25,7 +25,7 @@ } }, "dependencies": { - "@napi-rs/cli": "2.16.1" + "@napi-rs/cli": "2.18.0" }, "engines": { "node": ">= 12" diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index 9ee2bd0bc..26ae6fb28 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -1,6 +1,7 @@ { "MEMPOOL": { "ENABLED": true, + "OFFICIAL": false, "NETWORK": "__MEMPOOL_NETWORK__", "BACKEND": "__MEMPOOL_BACKEND__", "BLOCKS_SUMMARIES_INDEXING": true, @@ -79,7 +80,8 @@ "USERNAME": "__DATABASE_USERNAME__", "PASSWORD": "__DATABASE_PASSWORD__", "PID_DIR": "__DATABASE_PID_FILE__", - "TIMEOUT": 3000 + "TIMEOUT": 3000, + "POOL_SIZE": 100 }, "SYSLOG": { "ENABLED": false, diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index 6af0ce32f..5066e0ef7 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -14,6 +14,7 @@ describe('Mempool Backend Config', () => { expect(config.MEMPOOL).toStrictEqual({ ENABLED: true, + OFFICIAL: false, NETWORK: 'mainnet', BACKEND: 'none', BLOCKS_SUMMARIES_INDEXING: false, @@ -93,7 +94,8 @@ describe('Mempool Backend Config', () => { USERNAME: 'mempool', PASSWORD: 'mempool', TIMEOUT: 180000, - PID_DIR: '' + PID_DIR: '', + POOL_SIZE: 100, }); expect(config.SYSLOG).toStrictEqual({ diff --git a/backend/src/api/acceleration.ts b/backend/src/api/acceleration.ts new file mode 100644 index 000000000..2b032d09f --- /dev/null +++ b/backend/src/api/acceleration.ts @@ -0,0 +1,738 @@ +import logger from '../logger'; +import { MempoolTransactionExtended } from '../mempool.interfaces'; +import { IEsploraApi } from './bitcoin/esplora-api.interface'; + +const BLOCK_WEIGHT_UNITS = 4_000_000; +const BLOCK_SIGOPS = 80_000; +const MAX_RELATIVE_GRAPH_SIZE = 200; +const BID_BOOST_WINDOW = 40_000; +const BID_BOOST_MIN_OFFSET = 10_000; +const BID_BOOST_MAX_OFFSET = 400_000; + +type Acceleration = { + txid: string; + max_bid: number; +}; + +interface TxSummary { + txid: string; // txid of the current transaction + effectiveVsize: number; // Total vsize of the dependency tree + effectiveFee: number; // Total fee of the dependency tree in sats + ancestorCount: number; // Number of ancestors +} + +export interface AccelerationInfo { + txSummary: TxSummary; + targetFeeRate: number; // target fee rate (recommended next block fee, or median fee for mined block) + nextBlockFee: number; // fee in sats required to be in the next block (using recommended next block fee, or median fee for mined block) + cost: number; // additional cost to accelerate ((cost + txSummary.effectiveFee) / txSummary.effectiveVsize) >= targetFeeRate +} + +interface GraphTx { + txid: string; + vsize: number; + weight: number; + fees: { + base: number; + }; + depends: string[]; + spentby: string[]; +} + +interface MempoolTx extends GraphTx { + ancestorcount: number; + ancestorsize: number; + fees: { + base: number; + ancestor: number; + }; + + ancestors: Map, + ancestorRate: number; + individualRate: number; + score: number; +} + +class AccelerationCosts { + /** + * Takes a list of accelerations and verbose block data + * Returns the "fair" boost rate to charge accelerations + * + * @param accelerationsx + * @param verboseBlock + */ + public calculateBoostRate(accelerations: Acceleration[], blockTxs: IEsploraApi.Transaction[]): number { + // Run GBT ourselves to calculate accurate effective fee rates + // the list of transactions comes from a mined block, so we already know everything fits within consensus limits + const template = makeBlockTemplate(blockTxs, accelerations, 1, Infinity, Infinity); + + // initialize working maps for fast tx lookups + const accMap = {}; + const txMap = {}; + for (const acceleration of accelerations) { + accMap[acceleration.txid] = acceleration; + } + for (const tx of template) { + txMap[tx.txid] = tx; + } + + // Identify and exclude accelerated and otherwise prioritized transactions + const excludeMap = {}; + let totalWeight = 0; + let minAcceleratedPackage = Infinity; + let lastEffectiveRate = 0; + // Iterate over the mined template from bottom to top. + // Transactions should appear in ascending order of mining priority. + for (const blockTx of [...blockTxs].reverse()) { + const txid = blockTx.txid; + const tx = txMap[txid]; + totalWeight += tx.weight; + const isAccelerated = accMap[txid] != null; + // If a cluster has a in-band effective fee rate than the previous cluster, + // it must have been prioritized out-of-band (in order to have a higher mining priority) + // so exclude from the analysis. + const isPrioritized = tx.effectiveFeePerVsize < lastEffectiveRate; + if (isPrioritized || isAccelerated) { + let packageWeight = 0; + // exclude this whole CPFP cluster + for (const clusterTxid of tx.cluster) { + packageWeight += txMap[clusterTxid].weight; + if (!excludeMap[clusterTxid]) { + excludeMap[clusterTxid] = true; + } + } + // keep track of the smallest accelerated CPFP cluster for later + if (isAccelerated) { + minAcceleratedPackage = Math.min(minAcceleratedPackage, packageWeight); + } + } + if (!isPrioritized) { + if (!isAccelerated || !lastEffectiveRate) { + lastEffectiveRate = tx.effectiveFeePerVsize; + } + } + } + + // The Bid Boost Rate is calculated by disregarding the bottom X weight units of the block, + // where X is the larger of BID_BOOST_MIN_OFFSET or the smallest accelerated package weight (the "offset"), + // then taking the average fee rate of the following BID_BOOST_WINDOW weight units + // (ignoring accelerated transactions and their ancestors). + // + // Transactions within the offset might pay less than the fair rate due to bin-packing effects + // But the average rate paid by the next chunk of non-accelerated transactions provides a good + // upper bound on the "next best rate" of alternatives to including the accelerated transactions + // (since, if there were any better options, they would have been included instead) + const spareWeight = BLOCK_WEIGHT_UNITS - totalWeight; + const windowOffset = Math.min(Math.max(minAcceleratedPackage, BID_BOOST_MIN_OFFSET, spareWeight), BID_BOOST_MAX_OFFSET); + const leftBound = windowOffset; + const rightBound = windowOffset + BID_BOOST_WINDOW; + let totalFeeInWindow = 0; + let totalWeightInWindow = Math.max(0, spareWeight - leftBound); + let txIndex = blockTxs.length - 1; + for (let offset = spareWeight; offset < BLOCK_WEIGHT_UNITS && txIndex >= 0; txIndex--) { + const txid = blockTxs[txIndex].txid; + const tx = txMap[txid]; + if (excludeMap[txid]) { + // skip prioritized transactions and their ancestors + continue; + } + + const left = offset; + const right = offset + tx.weight; + offset += tx.weight; + if (right < leftBound) { + // not within window yet + continue; + } + if (left > rightBound) { + // past window + break; + } + // count fees for weight units within the window + const overlapLeft = Math.max(leftBound, left); + const overlapRight = Math.min(rightBound, right); + const overlapUnits = overlapRight - overlapLeft; + totalFeeInWindow += (tx.effectiveFeePerVsize * (overlapUnits / 4)); + totalWeightInWindow += overlapUnits; + } + + if (totalWeightInWindow < BID_BOOST_WINDOW) { + // not enough un-prioritized transactions to calculate a fair rate + // just charge everyone their max bids + return Infinity; + } + // Divide the total fee by the size of the BID_BOOST_WINDOW in vbytes + const averageRate = totalFeeInWindow / (BID_BOOST_WINDOW / 4); + return averageRate; + } + + + /** + * Takes an accelerated mined txid and a target rate + * Returns the total vsize, fees and acceleration cost (in sats) of the tx and all same-block ancestors + * + * @param txid + * @param medianFeeRate + */ + public getAccelerationInfo(tx: MempoolTransactionExtended, targetFeeRate: number, transactions: MempoolTransactionExtended[]): AccelerationInfo { + // Get same-block transaction ancestors + const allRelatives = this.getSameBlockRelatives(tx, transactions); + const relativesMap = this.initializeRelatives(allRelatives); + const rootTx = relativesMap.get(tx.txid) as MempoolTx; + + // Calculate cost to boost + return this.calculateAccelerationAncestors(rootTx, relativesMap, targetFeeRate); + } + + /** + * Takes a raw transaction, and builds a graph of same-block relatives, + * and returns as a MempoolTx + * + * @param tx + */ + private getSameBlockRelatives(tx: MempoolTransactionExtended, transactions: MempoolTransactionExtended[]): Map { + const blockTxs = new Map(); // map of txs in this block + const spendMap = new Map(); // map of outpoints to spending txids + for (const tx of transactions) { + blockTxs.set(tx.txid, tx); + for (const vin of tx.vin) { + spendMap.set(`${vin.txid}:${vin.vout}`, tx.txid); + } + } + + const relatives: Map = new Map(); + const stack: string[] = [tx.txid]; + + // build set of same-block ancestors + while (stack.length > 0) { + const nextTxid = stack.pop(); + const nextTx = nextTxid ? blockTxs.get(nextTxid) : null; + if (!nextTx || relatives.has(nextTx.txid)) { + continue; + } + + const mempoolTx = this.convertToGraphTx(nextTx); + + mempoolTx.fees.base = nextTx.fee || 0; + mempoolTx.depends = nextTx.vin.map(vin => vin.txid).filter(inTxid => inTxid && blockTxs.has(inTxid)) as string[]; + mempoolTx.spentby = nextTx.vout.map((vout, index) => spendMap.get(`${nextTx.txid}:${index}`)).filter(outTxid => outTxid && blockTxs.has(outTxid)) as string[]; + + for (const txid of [...mempoolTx.depends, ...mempoolTx.spentby]) { + if (txid) { + stack.push(txid); + } + } + + relatives.set(mempoolTx.txid, mempoolTx); + } + + return relatives; + } + + /** + * Takes a raw transaction and converts it to MempoolTx format + * fee and ancestor data is initialized with dummy/null values + * + * @param tx + */ + private convertToGraphTx(tx: MempoolTransactionExtended): GraphTx { + return { + txid: tx.txid, + vsize: tx.vsize, + weight: tx.weight, + fees: { + base: 0, // dummy + }, + depends: [], // dummy + spentby: [], //dummy + }; + } + + private convertGraphToMempoolTx(tx: GraphTx): MempoolTx { + return { + ...tx, + fees: { + base: tx.fees.base, + ancestor: tx.fees.base, + }, + ancestorcount: 1, + ancestorsize: tx.vsize, + ancestors: new Map(), + ancestorRate: 0, + individualRate: 0, + score: 0, + }; + } + + /** + * Given a root transaction, a list of in-mempool ancestors, and a target fee rate, + * Calculate the minimum set of transactions to fee-bump, their total vsize + fees + * + * @param tx + * @param ancestors + */ + private calculateAccelerationAncestors(tx: MempoolTx, relatives: Map, targetFeeRate: number): AccelerationInfo { + // add root tx to the ancestor map + relatives.set(tx.txid, tx); + + // Check for high-sigop transactions (not supported) + relatives.forEach(entry => { + if (entry.vsize > Math.ceil(entry.weight / 4)) { + throw new Error(`high_sigop_tx`); + } + }); + + // Initialize individual & ancestor fee rates + relatives.forEach(entry => this.setAncestorScores(entry)); + + // Sort by descending ancestor score + let sortedRelatives = Array.from(relatives.values()).sort(this.mempoolComparator); + + let includedInCluster: Map | null = null; + + // While highest score >= targetFeeRate + let maxIterations = MAX_RELATIVE_GRAPH_SIZE; + while (sortedRelatives.length && sortedRelatives[0].score && sortedRelatives[0].score >= targetFeeRate && maxIterations > 0) { + maxIterations--; + // Grab the highest scoring entry + const best = sortedRelatives.shift(); + if (best) { + const cluster = new Map(best.ancestors?.entries() || []); + if (best.ancestors.has(tx.txid)) { + includedInCluster = cluster; + } + cluster.set(best.txid, best); + // Remove this cluster (it already pays over the target rate, so doesn't need to be boosted) + // and update scores, ancestor totals and dependencies for the survivors + this.removeAncestors(cluster, relatives); + + // re-sort + sortedRelatives = Array.from(relatives.values()).sort(this.mempoolComparator); + } + } + + // sanity check for infinite loops / too many ancestors (should never happen) + if (maxIterations <= 0) { + logger.warn(`acceleration dependency calculation failed: calculateAccelerationAncestors loop exceeded ${MAX_RELATIVE_GRAPH_SIZE} iterations, unable to proceed`); + throw new Error('invalid_tx_dependencies'); + } + + let totalFee = Math.round(tx.fees.ancestor * 100_000_000); + + // transaction is already CPFP-d above the target rate by some descendant + if (includedInCluster) { + let clusterSize = 0; + let clusterFee = 0; + includedInCluster.forEach(entry => { + clusterSize += entry.vsize; + clusterFee += (entry.fees.base * 100_000_000); + }); + const clusterRate = clusterFee / clusterSize; + totalFee = Math.ceil(tx.ancestorsize * clusterRate); + } + + // Whatever remains in the accelerated tx's dependencies needs to be boosted to the targetFeeRate + // Cost = (totalVsize * targetFeeRate) - totalFee + return { + txSummary: { + txid: tx.txid, + effectiveVsize: tx.ancestorsize, + effectiveFee: totalFee, + ancestorCount: tx.ancestorcount, + }, + cost: Math.max(0, Math.ceil(tx.ancestorsize * targetFeeRate) - totalFee), + targetFeeRate, + nextBlockFee: Math.ceil(tx.ancestorsize * targetFeeRate), + }; + } + + /** + * Recursively traverses an in-mempool dependency graph, and sets a Map of in-mempool ancestors + * for each transaction. + * + * @param tx + * @param all + */ + private setAncestors(tx: MempoolTx, all: Map, visited: Map>, depth: number = 0): Map { + // sanity check for infinite recursion / too many ancestors (should never happen) + if (depth >= 100) { + logger.warn('acceleration dependency calculation failed: setAncestors reached depth of 100, unable to proceed', `Accelerator`); + throw new Error('invalid_tx_dependencies'); + } + + // initialize the ancestor map for this tx + tx.ancestors = new Map(); + tx.depends.forEach(parentId => { + const parent = all.get(parentId); + if (parent) { + // add the parent + tx.ancestors?.set(parentId, parent); + // check for a cached copy of this parent's ancestors + let ancestors = visited.get(parent.txid); + if (!ancestors) { + // recursively fetch the parent's ancestors + ancestors = this.setAncestors(parent, all, visited, depth + 1); + } + // and add to this tx's map + ancestors.forEach((ancestor, ancestorId) => { + tx.ancestors?.set(ancestorId, ancestor); + }); + } + }); + visited.set(tx.txid, tx.ancestors); + + return tx.ancestors; + } + + /** + * Efficiently sets a Map of in-mempool ancestors for each member of an expanded relative graph + * by running setAncestors on each leaf, and caching intermediate results. + * then initializes ancestor data for each transaction + * + * @param all + */ + private initializeRelatives(all: Map): Map { + const mempoolTxs = new Map(); + all.forEach(entry => { + mempoolTxs.set(entry.txid, this.convertGraphToMempoolTx(entry)); + }); + const visited: Map> = new Map(); + const leaves: MempoolTx[] = Array.from(mempoolTxs.values()).filter(entry => entry.spentby.length === 0); + for (const leaf of leaves) { + this.setAncestors(leaf, mempoolTxs, visited); + } + mempoolTxs.forEach(entry => { + entry.ancestors?.forEach(ancestor => { + entry.ancestorcount++; + entry.ancestorsize += ancestor.vsize; + entry.fees.ancestor += ancestor.fees.base; + }); + this.setAncestorScores(entry); + }); + return mempoolTxs; + } + + /** + * Remove a cluster of transactions from an in-mempool dependency graph + * and update the survivors' scores and ancestors + * + * @param cluster + * @param ancestors + */ + private removeAncestors(cluster: Map, all: Map): void { + // remove + cluster.forEach(tx => { + all.delete(tx.txid); + }); + + // update survivors + all.forEach(tx => { + cluster.forEach(remove => { + if (tx.ancestors?.has(remove.txid)) { + // remove as dependency + tx.ancestors.delete(remove.txid); + tx.depends = tx.depends.filter(parent => parent !== remove.txid); + // update ancestor sizes and fees + tx.ancestorsize -= remove.vsize; + tx.fees.ancestor -= remove.fees.base; + } + }); + // recalculate fee rates + this.setAncestorScores(tx); + }); + } + + /** + * Take a mempool transaction, and set the fee rates and ancestor score + * + * @param tx + */ + private setAncestorScores(tx: MempoolTx): void { + tx.individualRate = (tx.fees.base * 100_000_000) / tx.vsize; + tx.ancestorRate = (tx.fees.ancestor * 100_000_000) / tx.ancestorsize; + tx.score = Math.min(tx.individualRate, tx.ancestorRate); + } + + // Sort by descending score + private mempoolComparator(a, b): number { + return b.score - a.score; + } +} + +export default new AccelerationCosts; + +interface TemplateTransaction { + txid: string; + order: number; + weight: number; + adjustedVsize: number; // sigop-adjusted vsize, rounded up to the nearest integer + sigops: number; + fee: number; + feeDelta: number; + ancestors: string[]; + cluster: string[]; + effectiveFeePerVsize: number; +} + +interface MinerTransaction extends TemplateTransaction { + inputs: string[]; + feePerVsize: number; + relativesSet: boolean; + ancestorMap: Map; + children: Set; + ancestorFee: number; + ancestorVsize: number; + ancestorSigops: number; + score: number; + used: boolean; + modified: boolean; + dependencyRate: number; +} + +/* +* Build a block using an approximation of the transaction selection algorithm from Bitcoin Core +* (see BlockAssembler in https://github.com/bitcoin/bitcoin/blob/master/src/node/miner.cpp) +*/ +function makeBlockTemplate(candidates: IEsploraApi.Transaction[], accelerations: Acceleration[], maxBlocks: number = 8, weightLimit: number = BLOCK_WEIGHT_UNITS, sigopLimit: number = BLOCK_SIGOPS): TemplateTransaction[] { + const auditPool: Map = new Map(); + const mempoolArray: MinerTransaction[] = []; + + candidates.forEach(tx => { + // initializing everything up front helps V8 optimize property access later + const adjustedVsize = Math.ceil(Math.max(tx.weight / 4, 5 * (tx.sigops || 0))); + const feePerVsize = (tx.fee / adjustedVsize); + auditPool.set(tx.txid, { + txid: tx.txid, + order: txidToOrdering(tx.txid), + fee: tx.fee, + feeDelta: 0, + weight: tx.weight, + adjustedVsize, + feePerVsize: feePerVsize, + effectiveFeePerVsize: feePerVsize, + dependencyRate: feePerVsize, + sigops: tx.sigops || 0, + inputs: (tx.vin?.map(vin => vin.txid) || []) as string[], + relativesSet: false, + ancestors: [], + cluster: [], + ancestorMap: new Map(), + children: new Set(), + ancestorFee: 0, + ancestorVsize: 0, + ancestorSigops: 0, + score: 0, + used: false, + modified: false, + }); + mempoolArray.push(auditPool.get(tx.txid) as MinerTransaction); + }); + + // set accelerated effective fee + for (const acceleration of accelerations) { + const tx = auditPool.get(acceleration.txid); + if (tx) { + tx.feeDelta = acceleration.max_bid; + tx.feePerVsize = ((tx.fee + tx.feeDelta) / tx.adjustedVsize); + tx.effectiveFeePerVsize = tx.feePerVsize; + tx.dependencyRate = tx.feePerVsize; + } + } + + // Build relatives graph & calculate ancestor scores + for (const tx of mempoolArray) { + if (!tx.relativesSet) { + setRelatives(tx, auditPool); + } + } + + // Sort by descending ancestor score + mempoolArray.sort(priorityComparator); + + // Build blocks by greedily choosing the highest feerate package + // (i.e. the package rooted in the transaction with the best ancestor score) + const blocks: number[][] = []; + let blockWeight = 0; + let blockSigops = 0; + const transactions: MinerTransaction[] = []; + let modified: MinerTransaction[] = []; + const overflow: MinerTransaction[] = []; + let failures = 0; + while (mempoolArray.length || modified.length) { + // skip invalid transactions + while (mempoolArray[0].used || mempoolArray[0].modified) { + mempoolArray.shift(); + } + + // Select best next package + let nextTx; + const nextPoolTx = mempoolArray[0]; + const nextModifiedTx = modified[0]; + if (nextPoolTx && (!nextModifiedTx || (nextPoolTx.score || 0) > (nextModifiedTx.score || 0))) { + nextTx = nextPoolTx; + mempoolArray.shift(); + } else { + modified.shift(); + if (nextModifiedTx) { + nextTx = nextModifiedTx; + } + } + + if (nextTx && !nextTx?.used) { + // Check if the package fits into this block + if (blocks.length >= (maxBlocks - 1) || ((blockWeight + (4 * nextTx.ancestorVsize) < weightLimit) && (blockSigops + nextTx.ancestorSigops <= sigopLimit))) { + const ancestors: MinerTransaction[] = Array.from(nextTx.ancestorMap.values()); + // sort ancestors by dependency graph (equivalent to sorting by ascending ancestor count) + const sortedTxSet = [...ancestors.sort((a, b) => { return (a.ancestorMap.size || 0) - (b.ancestorMap.size || 0); }), nextTx]; + const clusterTxids = sortedTxSet.map(tx => tx.txid); + const effectiveFeeRate = Math.min(nextTx.dependencyRate || Infinity, nextTx.ancestorFee / nextTx.ancestorVsize); + const used: MinerTransaction[] = []; + while (sortedTxSet.length) { + const ancestor = sortedTxSet.pop(); + if (!ancestor) { + continue; + } + ancestor.used = true; + ancestor.usedBy = nextTx.txid; + // update this tx with effective fee rate & relatives data + if (ancestor.effectiveFeePerVsize !== effectiveFeeRate) { + ancestor.effectiveFeePerVsize = effectiveFeeRate; + } + ancestor.cluster = clusterTxids; + transactions.push(ancestor); + blockWeight += ancestor.weight; + blockSigops += ancestor.sigops; + used.push(ancestor); + } + + // remove these as valid package ancestors for any descendants remaining in the mempool + if (used.length) { + used.forEach(tx => { + modified = updateDescendants(tx, auditPool, modified, effectiveFeeRate); + }); + } + + failures = 0; + } else { + // hold this package in an overflow list while we check for smaller options + overflow.push(nextTx); + failures++; + } + } + + // this block is full + const exceededPackageTries = failures > 1000 && blockWeight > (weightLimit - 4000); + const queueEmpty = !mempoolArray.length && !modified.length; + + if (exceededPackageTries || queueEmpty) { + break; + } + } + + for (const tx of transactions) { + tx.ancestors = Object.values(tx.ancestorMap); + } + + return transactions; +} + +// traverse in-mempool ancestors +// recursion unavoidable, but should be limited to depth < 25 by mempool policy +function setRelatives( + tx: MinerTransaction, + mempool: Map, +): void { + for (const parent of tx.inputs) { + const parentTx = mempool.get(parent); + if (parentTx && !tx.ancestorMap?.has(parent)) { + tx.ancestorMap.set(parent, parentTx); + parentTx.children.add(tx); + // visit each node only once + if (!parentTx.relativesSet) { + setRelatives(parentTx, mempool); + } + parentTx.ancestorMap.forEach((ancestor) => { + tx.ancestorMap.set(ancestor.txid, ancestor); + }); + } + }; + tx.ancestorFee = (tx.fee + tx.feeDelta); + tx.ancestorVsize = tx.adjustedVsize || 0; + tx.ancestorSigops = tx.sigops || 0; + tx.ancestorMap.forEach((ancestor) => { + tx.ancestorFee += (ancestor.fee + ancestor.feeDelta); + tx.ancestorVsize += ancestor.adjustedVsize; + tx.ancestorSigops += ancestor.sigops; + }); + tx.score = tx.ancestorFee / tx.ancestorVsize; + tx.relativesSet = true; +} + +// iterate over remaining descendants, removing the root as a valid ancestor & updating the ancestor score +// avoids recursion to limit call stack depth +function updateDescendants( + rootTx: MinerTransaction, + mempool: Map, + modified: MinerTransaction[], + clusterRate: number, +): MinerTransaction[] { + const descendantSet: Set = new Set(); + // stack of nodes left to visit + const descendants: MinerTransaction[] = []; + let descendantTx: MinerTransaction | undefined; + rootTx.children.forEach(childTx => { + if (!descendantSet.has(childTx)) { + descendants.push(childTx); + descendantSet.add(childTx); + } + }); + while (descendants.length) { + descendantTx = descendants.pop(); + if (descendantTx && descendantTx.ancestorMap && descendantTx.ancestorMap.has(rootTx.txid)) { + // remove tx as ancestor + descendantTx.ancestorMap.delete(rootTx.txid); + descendantTx.ancestorFee -= (rootTx.fee + rootTx.feeDelta); + descendantTx.ancestorVsize -= rootTx.adjustedVsize; + descendantTx.ancestorSigops -= rootTx.sigops; + descendantTx.score = descendantTx.ancestorFee / descendantTx.ancestorVsize; + descendantTx.dependencyRate = descendantTx.dependencyRate ? Math.min(descendantTx.dependencyRate, clusterRate) : clusterRate; + + if (!descendantTx.modified) { + descendantTx.modified = true; + modified.push(descendantTx); + } + + // add this node's children to the stack + descendantTx.children.forEach(childTx => { + // visit each node only once + if (!descendantSet.has(childTx)) { + descendants.push(childTx); + descendantSet.add(childTx); + } + }); + } + } + // return new, resorted modified list + return modified.sort(priorityComparator); +} + +// Used to sort an array of MinerTransactions by descending ancestor score +function priorityComparator(a: MinerTransaction, b: MinerTransaction): number { + if (b.score === a.score) { + // tie-break by txid for stability + return a.order - b.order; + } else { + return b.score - a.score; + } +} + +// returns the most significant 4 bytes of the txid as an integer +function txidToOrdering(txid: string): number { + return parseInt( + txid.substring(62, 64) + + txid.substring(60, 62) + + txid.substring(58, 60) + + txid.substring(56, 58), + 16 + ); +} diff --git a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts index 02640efc0..abd4c47a5 100644 --- a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts +++ b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts @@ -29,6 +29,7 @@ export interface AbstractBitcoinApi { $getOutSpendsByOutpoint(outpoints: { txid: string, vout: number }[]): Promise; startHealthChecks(): void; + getHealthStatus(): HealthCheckHost[]; } export interface BitcoinRpcCredentials { host: string; @@ -38,3 +39,14 @@ export interface BitcoinRpcCredentials { timeout: number; cookie?: string; } + +export interface HealthCheckHost { + host: string; + active: boolean; + rtt: number; + latestHeight: number; + socket: boolean; + outOfSync: boolean; + unreachable: boolean; + checked: boolean; +} diff --git a/backend/src/api/bitcoin/bitcoin-api.ts b/backend/src/api/bitcoin/bitcoin-api.ts index f54c836f8..d19eb06ac 100644 --- a/backend/src/api/bitcoin/bitcoin-api.ts +++ b/backend/src/api/bitcoin/bitcoin-api.ts @@ -1,5 +1,5 @@ import * as bitcoinjs from 'bitcoinjs-lib'; -import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory'; +import { AbstractBitcoinApi, HealthCheckHost } from './bitcoin-api-abstract-factory'; import { IBitcoinApi } from './bitcoin-api.interface'; import { IEsploraApi } from './esplora-api.interface'; import blocks from '../blocks'; @@ -382,6 +382,10 @@ class BitcoinApi implements AbstractBitcoinApi { } public startHealthChecks(): void {}; + + public getHealthStatus() { + return []; + } } export default BitcoinApi; diff --git a/backend/src/api/bitcoin/esplora-api.ts b/backend/src/api/bitcoin/esplora-api.ts index b8e3784c7..9289ab4d1 100644 --- a/backend/src/api/bitcoin/esplora-api.ts +++ b/backend/src/api/bitcoin/esplora-api.ts @@ -1,7 +1,7 @@ import config from '../../config'; import axios, { AxiosResponse, isAxiosError } from 'axios'; import http from 'http'; -import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory'; +import { AbstractBitcoinApi, HealthCheckHost } from './bitcoin-api-abstract-factory'; import { IEsploraApi } from './esplora-api.interface'; import logger from '../../logger'; import { Common } from '../common'; @@ -164,7 +164,7 @@ class FailoverRouter { } // sort hosts by connection quality, and update default fallback - private sortHosts(): FailoverHost[] { + public sortHosts(): FailoverHost[] { // sort by connection quality return this.hosts.slice().sort((a, b) => { if ((a.unreachable || a.outOfSync) === (b.unreachable || b.outOfSync)) { @@ -349,6 +349,23 @@ class ElectrsApi implements AbstractBitcoinApi { public startHealthChecks(): void { this.failoverRouter.startHealthChecks(); } + + public getHealthStatus(): HealthCheckHost[] { + if (config.MEMPOOL.OFFICIAL) { + return this.failoverRouter.sortHosts().map(host => ({ + host: host.host, + active: host === this.failoverRouter.activeHost, + rtt: host.rtt, + latestHeight: host.latestHeight || 0, + socket: !!host.socket, + outOfSync: !!host.outOfSync, + unreachable: !!host.unreachable, + checked: !!host.checked, + })); + } else { + return []; + } + } } export default ElectrsApi; diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 9a5eb310a..861830226 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository'; import { RowDataPacket } from 'mysql2'; class DatabaseMigration { - private static currentVersion = 68; + private static currentVersion = 70; private queryTimeout = 3600_000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -580,6 +580,16 @@ class DatabaseMigration { await this.$executeQuery(`INSERT INTO state VALUES('last_bitcoin_block_audit', 0, NULL);`); await this.updateToSchemaVersion(68); } + + if (databaseSchemaVersion < 69 && config.MEMPOOL.NETWORK === 'mainnet') { + await this.$executeQuery(this.getCreateAccelerationsTableQuery(), await this.$checkIfTableExists('accelerations')); + await this.updateToSchemaVersion(69); + } + + if (databaseSchemaVersion < 70 && config.MEMPOOL.NETWORK === 'mainnet') { + await this.$executeQuery('ALTER TABLE accelerations MODIFY COLUMN added DATETIME;'); + await this.updateToSchemaVersion(70); + } } /** @@ -1123,6 +1133,23 @@ class DatabaseMigration { ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; } + private getCreateAccelerationsTableQuery(): string { + return `CREATE TABLE IF NOT EXISTS accelerations ( + txid varchar(65) NOT NULL, + added datetime NOT NULL, + height int(10) NOT NULL, + pool smallint unsigned NULL, + effective_vsize int(10) NOT NULL, + effective_fee bigint(20) unsigned NOT NULL, + boost_rate float unsigned, + boost_cost bigint(20) unsigned NOT NULL, + PRIMARY KEY (txid), + INDEX (added), + INDEX (height), + INDEX (pool) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; + } + public async $blocksReindexingTruncate(): Promise { logger.warn(`Truncating pools, blocks, hashrates and difficulty_adjustments tables for re-indexing (using '--reindex-blocks'). You can cancel this command within 5 seconds`); await Common.sleep$(5000); diff --git a/backend/src/api/mining/mining-routes.ts b/backend/src/api/mining/mining-routes.ts index bb78de44a..6b9ef7b9f 100644 --- a/backend/src/api/mining/mining-routes.ts +++ b/backend/src/api/mining/mining-routes.ts @@ -8,6 +8,7 @@ import HashratesRepository from '../../repositories/HashratesRepository'; import bitcoinClient from '../bitcoin/bitcoin-client'; import mining from "./mining"; import PricesRepository from '../../repositories/PricesRepository'; +import AccelerationRepository from '../../repositories/AccelerationRepository'; class MiningRoutes { public initRoutes(app: Application) { @@ -34,6 +35,10 @@ class MiningRoutes { .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/:hash', this.$getBlockAudit) .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/timestamp/:timestamp', this.$getHeightFromTimestamp) .get(config.MEMPOOL.API_URL_PREFIX + 'historical-price', this.$getHistoricalPrice) + + .get(config.MEMPOOL.API_URL_PREFIX + 'accelerations/pool/:slug', this.$getAccelerationsByPool) + .get(config.MEMPOOL.API_URL_PREFIX + 'accelerations/block/:height', this.$getAccelerationsByHeight) + .get(config.MEMPOOL.API_URL_PREFIX + 'accelerations/recent/:interval', this.$getRecentAccelerations) ; } @@ -352,6 +357,52 @@ class MiningRoutes { res.status(500).send(e instanceof Error ? e.message : e); } } + + private async $getAccelerationsByPool(req: Request, res: Response): Promise { + try { + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); + if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid'].includes(config.MEMPOOL.NETWORK)) { + res.status(400).send('Acceleration data is not available.'); + return; + } + res.status(200).send(await AccelerationRepository.$getAccelerationInfo(req.params.slug)); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async $getAccelerationsByHeight(req: Request, res: Response): Promise { + try { + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24).toUTCString()); + if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid'].includes(config.MEMPOOL.NETWORK)) { + res.status(400).send('Acceleration data is not available.'); + return; + } + const height = req.params.height === undefined ? undefined : parseInt(req.params.height, 10); + res.status(200).send(await AccelerationRepository.$getAccelerationInfo(null, height)); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async $getRecentAccelerations(req: Request, res: Response): Promise { + try { + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); + if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid'].includes(config.MEMPOOL.NETWORK)) { + res.status(400).send('Acceleration data is not available.'); + return; + } + res.status(200).send(await AccelerationRepository.$getAccelerationInfo(null, null, req.params.interval)); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } } export default new MiningRoutes(); diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index b78389b64..6711c88fb 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -24,6 +24,9 @@ import { ApiPrice } from '../repositories/PricesRepository'; import accelerationApi from './services/acceleration'; import mempool from './mempool'; import statistics from './statistics/statistics'; +import accelerationCosts from './acceleration'; +import accelerationRepository from '../repositories/AccelerationRepository'; +import bitcoinApi from './bitcoin/bitcoin-api-factory'; interface AddressTransactions { mempool: MempoolTransactionExtended[], @@ -37,6 +40,7 @@ const wantable = [ 'mempool-blocks', 'live-2h-chart', 'stats', + 'tomahawk', ]; class WebsocketHandler { @@ -121,7 +125,7 @@ class WebsocketHandler { for (const sub of wantable) { const key = `want-${sub}`; const wants = parsedMessage.data.includes(sub); - if (wants && client['wants'] && !client[key]) { + if (wants && !client[key]) { wantNow[key] = true; } client[key] = wants; @@ -145,6 +149,10 @@ class WebsocketHandler { response['da'] = this.socketData['da']; } + if (wantNow['want-tomahawk']) { + response['tomahawk'] = JSON.stringify(bitcoinApi.getHealthStatus()); + } + if (parsedMessage && parsedMessage['track-tx']) { if (/^[a-fA-F0-9]{64}$/.test(parsedMessage['track-tx'])) { client['track-tx'] = parsedMessage['track-tx']; @@ -544,6 +552,10 @@ class WebsocketHandler { response['mempool-blocks'] = getCachedResponse('mempool-blocks', mBlocks); } + if (client['want-tomahawk']) { + response['tomahawk'] = getCachedResponse('tomahawk', bitcoinApi.getHealthStatus()); + } + if (client['track-mempool-tx']) { const tx = newTransactions.find((t) => t.txid === client['track-mempool-tx']); if (tx) { @@ -728,6 +740,28 @@ class WebsocketHandler { const _memPool = memPool.getMempool(); + const isAccelerated = config.MEMPOOL_SERVICES.ACCELERATIONS && accelerationApi.isAcceleratedBlock(block, Object.values(mempool.getAccelerations())); + + + if (isAccelerated) { + const blockTxs: { [txid: string]: MempoolTransactionExtended } = {}; + for (const tx of transactions) { + blockTxs[tx.txid] = tx; + } + const accelerations = Object.values(mempool.getAccelerations()); + const boostRate = accelerationCosts.calculateBoostRate( + accelerations.map(acc => ({ txid: acc.txid, max_bid: acc.feeDelta })), + transactions + ); + for (const acc of accelerations) { + if (blockTxs[acc.txid]) { + const tx = blockTxs[acc.txid]; + const accelerationInfo = accelerationCosts.getAccelerationInfo(tx, boostRate, transactions); + accelerationRepository.$saveAcceleration(accelerationInfo, block, block.extras.pool.id); + } + } + } + const rbfTransactions = Common.findMinedRbfTransactions(transactions, memPool.getSpendMap()); memPool.handleMinedRbfTransactions(rbfTransactions); memPool.removeFromSpendMap(transactions); @@ -735,7 +769,6 @@ class WebsocketHandler { if (config.MEMPOOL.AUDIT && memPool.isInSync()) { let projectedBlocks; let auditMempool = _memPool; - const isAccelerated = config.MEMPOOL_SERVICES.ACCELERATIONS && accelerationApi.isAcceleratedBlock(block, Object.values(mempool.getAccelerations())); // template calculation functions have mempool side effects, so calculate audits using // a cloned copy of the mempool if we're running a different algorithm for mempool updates const separateAudit = config.MEMPOOL.ADVANCED_GBT_AUDIT !== config.MEMPOOL.ADVANCED_GBT_MEMPOOL; @@ -886,6 +919,10 @@ class WebsocketHandler { response['mempool-blocks'] = getCachedResponse('mempool-blocks', mBlocks); } + if (client['want-tomahawk']) { + response['tomahawk'] = getCachedResponse('tomahawk', bitcoinApi.getHealthStatus()); + } + if (client['track-tx']) { const trackTxid = client['track-tx']; if (trackTxid && confirmedTxids[trackTxid]) { diff --git a/backend/src/config.ts b/backend/src/config.ts index 32a7af3df..3330adca0 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -5,6 +5,7 @@ const configFromFile = require( interface IConfig { MEMPOOL: { ENABLED: boolean; + OFFICIAL: boolean; NETWORK: 'mainnet' | 'testnet' | 'signet' | 'liquid' | 'liquidtestnet'; BACKEND: 'esplora' | 'electrum' | 'none'; HTTP_PORT: number; @@ -103,6 +104,7 @@ interface IConfig { PASSWORD: string; TIMEOUT: number; PID_DIR: string; + POOL_SIZE: number; }; SYSLOG: { ENABLED: boolean; @@ -161,6 +163,7 @@ interface IConfig { const defaults: IConfig = { 'MEMPOOL': { 'ENABLED': true, + 'OFFICIAL': false, 'NETWORK': 'mainnet', 'BACKEND': 'none', 'HTTP_PORT': 8999, @@ -240,6 +243,7 @@ const defaults: IConfig = { 'PASSWORD': 'mempool', 'TIMEOUT': 180000, 'PID_DIR': '', + 'POOL_SIZE': 100, }, 'SYSLOG': { 'ENABLED': true, diff --git a/backend/src/database.ts b/backend/src/database.ts index dc543bbbc..05f624ff4 100644 --- a/backend/src/database.ts +++ b/backend/src/database.ts @@ -21,7 +21,7 @@ import { execSync } from 'child_process'; database: config.DATABASE.DATABASE, user: config.DATABASE.USERNAME, password: config.DATABASE.PASSWORD, - connectionLimit: 10, + connectionLimit: config.DATABASE.POOL_SIZE, supportBigNumbers: true, timezone: '+00:00', }; diff --git a/backend/src/repositories/AccelerationRepository.ts b/backend/src/repositories/AccelerationRepository.ts new file mode 100644 index 000000000..868f8526f --- /dev/null +++ b/backend/src/repositories/AccelerationRepository.ts @@ -0,0 +1,109 @@ +import { AccelerationInfo } from '../api/acceleration'; +import { ResultSetHeader, RowDataPacket } from 'mysql2'; +import DB from '../database'; +import logger from '../logger'; +import { IEsploraApi } from '../api/bitcoin/esplora-api.interface'; +import { Common } from '../api/common'; +import config from '../config'; + +export interface PublicAcceleration { + txid: string, + height: number, + pool: { + id: number, + slug: string, + name: string, + }, + effective_vsize: number, + effective_fee: number, + boost_rate: number, + boost_cost: number, +} + +class AccelerationRepository { + public async $saveAcceleration(acceleration: AccelerationInfo, block: IEsploraApi.Block, pool_id: number): Promise { + try { + await DB.query(` + INSERT INTO accelerations(txid, added, height, pool, effective_vsize, effective_fee, boost_rate, boost_cost) + VALUE (?, FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?) + ON DUPLICATE KEY UPDATE + added = FROM_UNIXTIME(?), + height = ?, + pool = ?, + effective_vsize = ?, + effective_fee = ?, + boost_rate = ?, + boost_cost = ? + `, [ + acceleration.txSummary.txid, + block.timestamp, + block.height, + pool_id, + acceleration.txSummary.effectiveVsize, + acceleration.txSummary.effectiveFee, + acceleration.targetFeeRate, acceleration.cost, + block.timestamp, + block.height, + pool_id, + acceleration.txSummary.effectiveVsize, + acceleration.txSummary.effectiveFee, + acceleration.targetFeeRate, acceleration.cost, + ]); + } catch (e: any) { + logger.err(`Cannot save acceleration (${acceleration.txSummary.txid}) into db. Reason: ` + (e instanceof Error ? e.message : e)); + // We don't throw, not a critical issue if we miss some accelerations + } + } + + public async $getAccelerationInfo(poolSlug: string | null = null, height: number | null = null, interval: string | null = null): Promise { + interval = Common.getSqlInterval(interval); + + if (!config.MEMPOOL_SERVICES.ACCELERATIONS || (interval == null && poolSlug == null && height == null)) { + return []; + } + + let query = ` + SELECT * FROM accelerations + JOIN pools on pools.unique_id = accelerations.pool + `; + let params: any[] = []; + + if (interval) { + query += ` WHERE accelerations.added BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW() `; + } else if (height != null) { + query += ` WHERE accelerations.height = ? `; + params.push(height); + } else if (poolSlug != null) { + query += ` WHERE pools.slug = ? `; + params.push(poolSlug); + } + + query += ` ORDER BY accelerations.added DESC `; + + try { + const [rows] = await DB.query(query, params) as RowDataPacket[][]; + if (rows?.length) { + return rows.map(row => ({ + txid: row.txid, + height: row.height, + pool: { + id: row.id, + slug: row.slug, + name: row.name, + }, + effective_vsize: row.effective_vsize, + effective_fee: row.effective_fee, + boost_rate: row.boost_rate, + boost_cost: row.boost_cost, + })); + } else { + return []; + } + } catch (e) { + logger.err(`Cannot query acceleration info. Reason: ` + (e instanceof Error ? e.message : e)); + throw e; + } + } +} + +export default new AccelerationRepository(); diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index 8f69fd0c1..eca4cf14c 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -3,6 +3,7 @@ "NETWORK": "__MEMPOOL_NETWORK__", "BACKEND": "__MEMPOOL_BACKEND__", "ENABLED": __MEMPOOL_ENABLED__, + "OFFICIAL": __MEMPOOL_OFFICIAL__, "HTTP_PORT": __MEMPOOL_HTTP_PORT__, "SPAWN_CLUSTER_PROCS": __MEMPOOL_SPAWN_CLUSTER_PROCS__, "API_URL_PREFIX": "__MEMPOOL_API_URL_PREFIX__", @@ -79,7 +80,8 @@ "USERNAME": "__DATABASE_USERNAME__", "PASSWORD": "__DATABASE_PASSWORD__", "TIMEOUT": __DATABASE_TIMEOUT__, - "PID_DIR": "__DATABASE_PID_DIR__" + "PID_DIR": "__DATABASE_PID_DIR__", + "POOL_SIZE": __DATABASE_POOL_SIZE__ }, "SYSLOG": { "ENABLED": __SYSLOG_ENABLED__, diff --git a/docker/backend/start.sh b/docker/backend/start.sh index ba9b99233..b700bba32 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -4,6 +4,7 @@ __MEMPOOL_NETWORK__=${MEMPOOL_NETWORK:=mainnet} __MEMPOOL_BACKEND__=${MEMPOOL_BACKEND:=electrum} __MEMPOOL_ENABLED__=${MEMPOOL_ENABLED:=true} +__MEMPOOL_OFFICIAL__=${MEMPOOL_OFFICIAL:=false} __MEMPOOL_HTTP_PORT__=${BACKEND_HTTP_PORT:=8999} __MEMPOOL_SPAWN_CLUSTER_PROCS__=${MEMPOOL_SPAWN_CLUSTER_PROCS:=0} __MEMPOOL_API_URL_PREFIX__=${MEMPOOL_API_URL_PREFIX:=/api/v1/} @@ -81,6 +82,7 @@ __DATABASE_USERNAME__=${DATABASE_USERNAME:=mempool} __DATABASE_PASSWORD__=${DATABASE_PASSWORD:=mempool} __DATABASE_TIMEOUT__=${DATABASE_TIMEOUT:=180000} __DATABASE_PID_DIR__=${DATABASE_PID_DIR:=""} +__DATABASE_POOL_SIZE__=${DATABASE_POOL_SIZE:=100} # SYSLOG __SYSLOG_ENABLED__=${SYSLOG_ENABLED:=false} @@ -158,6 +160,7 @@ mkdir -p "${__MEMPOOL_CACHE_DIR__}" sed -i "s!__MEMPOOL_NETWORK__!${__MEMPOOL_NETWORK__}!g" mempool-config.json sed -i "s!__MEMPOOL_BACKEND__!${__MEMPOOL_BACKEND__}!g" mempool-config.json sed -i "s!__MEMPOOL_ENABLED__!${__MEMPOOL_ENABLED__}!g" mempool-config.json +sed -i "s!__MEMPOOL_OFFICIAL__!${__MEMPOOL_OFFICIAL__}!g" mempool-config.json sed -i "s!__MEMPOOL_HTTP_PORT__!${__MEMPOOL_HTTP_PORT__}!g" mempool-config.json sed -i "s!__MEMPOOL_SPAWN_CLUSTER_PROCS__!${__MEMPOOL_SPAWN_CLUSTER_PROCS__}!g" mempool-config.json sed -i "s!__MEMPOOL_API_URL_PREFIX__!${__MEMPOOL_API_URL_PREFIX__}!g" mempool-config.json @@ -230,6 +233,7 @@ sed -i "s!__DATABASE_USERNAME__!${__DATABASE_USERNAME__}!g" mempool-config.json sed -i "s!__DATABASE_PASSWORD__!${__DATABASE_PASSWORD__}!g" mempool-config.json sed -i "s!__DATABASE_TIMEOUT__!${__DATABASE_TIMEOUT__}!g" mempool-config.json sed -i "s!__DATABASE_PID_DIR__!${__DATABASE_PID_DIR__}!g" mempool-config.json +sed -i "s!__DATABASE_POOL_SIZE__!${__DATABASE_POOL_SIZE__}!g" mempool-config.json sed -i "s!__SYSLOG_ENABLED__!${__SYSLOG_ENABLED__}!g" mempool-config.json sed -i "s!__SYSLOG_HOST__!${__SYSLOG_HOST__}!g" mempool-config.json diff --git a/docker/init.sh b/docker/init.sh index ee9ac9542..3c5ec6aa3 100755 --- a/docker/init.sh +++ b/docker/init.sh @@ -1,7 +1,7 @@ #!/bin/sh #backend -cp ./docker/backend/* ./backend/ +cp -r ./docker/backend/* ./backend/ #geoip-data mkdir -p ./backend/GeoIP/ @@ -13,8 +13,8 @@ localhostIP="127.0.0.1" cp ./docker/frontend/* ./frontend cp ./nginx.conf ./frontend/ cp ./nginx-mempool.conf ./frontend/ -sed -i "s/${localhostIP}:80/0.0.0.0:__MEMPOOL_FRONTEND_HTTP_PORT__/g" ./frontend/nginx.conf -sed -i "s/${localhostIP}/0.0.0.0/g" ./frontend/nginx.conf -sed -i "s/user nobody;//g" ./frontend/nginx.conf -sed -i "s!/etc/nginx/nginx-mempool.conf!/etc/nginx/conf.d/nginx-mempool.conf!g" ./frontend/nginx.conf -sed -i "s/${localhostIP}:8999/__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__:__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__/g" ./frontend/nginx-mempool.conf +sed -i"" -e "s/${localhostIP}:80/0.0.0.0:__MEMPOOL_FRONTEND_HTTP_PORT__/g" ./frontend/nginx.conf +sed -i"" -e "s/${localhostIP}/0.0.0.0/g" ./frontend/nginx.conf +sed -i"" -e "s/user nobody;//g" ./frontend/nginx.conf +sed -i"" -e "s!/etc/nginx/nginx-mempool.conf!/etc/nginx/conf.d/nginx-mempool.conf!g" ./frontend/nginx.conf +sed -i"" -e "s/${localhostIP}:8999/__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__:__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__/g" ./frontend/nginx-mempool.conf diff --git a/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html b/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html index 443fc1946..e9f64b9b8 100644 --- a/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html +++ b/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html @@ -56,7 +56,7 @@ -
+
{{ block.extras.pool.name}} diff --git a/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.scss b/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.scss index 795e1f4df..c1cc6809d 100644 --- a/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.scss +++ b/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.scss @@ -166,7 +166,7 @@ opacity: 1; } .hide { - opacity: 0; + opacity: 0.4; pointer-events : none; } diff --git a/frontend/src/app/components/server-health/server-health.component.html b/frontend/src/app/components/server-health/server-health.component.html new file mode 100644 index 000000000..0350997e1 --- /dev/null +++ b/frontend/src/app/components/server-health/server-health.component.html @@ -0,0 +1,32 @@ +
+ +

Node Status

+ + +
+ + + + + + + + + + + + + + + + + + + +
HostRTTRTTHeight
{{ i + 1 }}{{ host.active ? '⭐️' : host.flag }}{{ host.link }}{{ (host.rtt / 1000) | number : '1.1-1' }} {{ host.rtt == null ? '' : 's'}} {{ !host.checked ? '⏳' : (host.unreachable ? 'πŸ”₯' : 'βœ…') }}{{ host.rtt | number : '1.0-0' }} {{ host.rtt == null ? '' : 'ms'}} {{ !host.checked ? '⏳' : (host.unreachable ? 'πŸ”₯' : 'βœ…') }}{{ host.latestHeight }} {{ !host.checked ? '⏳' : (host.outOfSync ? '🚫' : (host.latestHeight && host.latestHeight < tip ? '🟧' : 'βœ…')) }}
+
+
+
diff --git a/frontend/src/app/components/server-health/server-health.component.scss b/frontend/src/app/components/server-health/server-health.component.scss new file mode 100644 index 000000000..572628a9d --- /dev/null +++ b/frontend/src/app/components/server-health/server-health.component.scss @@ -0,0 +1,72 @@ +.tomahawk { + .links { + float: right; + text-align: right; + margin-top: 1em; + + a, span { + margin-left: 1em; + } + } + + .dashboard-title { + text-align: left; + } + + .status-panel { + max-width: 720px; + margin: auto; + margin-top: 2em; + padding: 1em; + background: #24273e; + } + + .status-table { + width: 100%; + + td, th { + padding: 0.25em; + + &.rank, &.flag { + width: 28px; + text-align: right; + } + &.rtt, &.height { + width: 92px; + text-align: right; + } + &.only-small { + display: table-cell; + &.rtt { + width: 60px; + } + } + &.only-large { + display: none; + } + &.height { + padding-right: 0.5em; + } + &.host { + width: auto; + overflow: hidden; + text-overflow: ellipsis; + } + + @media (min-width: 576px) { + &.rank, &.flag { + width: 32px; + } + &.rtt, &.height { + width: 96px; + } + &.only-small { + display: none; + } + &.only-large { + display: table-cell; + } + } + } + } +} \ No newline at end of file diff --git a/frontend/src/app/components/server-health/server-health.component.ts b/frontend/src/app/components/server-health/server-health.component.ts new file mode 100644 index 000000000..363b28111 --- /dev/null +++ b/frontend/src/app/components/server-health/server-health.component.ts @@ -0,0 +1,67 @@ +import { Component, OnInit, ChangeDetectionStrategy, SecurityContext } from '@angular/core'; +import { WebsocketService } from '../../services/websocket.service'; +import { Observable, Subject, map } from 'rxjs'; +import { StateService } from '../../services/state.service'; +import { HealthCheckHost } from '../../interfaces/websocket.interface'; +import { DomSanitizer } from '@angular/platform-browser'; + +@Component({ + selector: 'app-server-health', + templateUrl: './server-health.component.html', + styleUrls: ['./server-health.component.scss'], + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class ServerHealthComponent implements OnInit { + hosts$: Observable; + tip$: Subject; + + constructor( + private websocketService: WebsocketService, + private stateService: StateService, + public sanitizer: DomSanitizer, + ) {} + + ngOnInit(): void { + this.hosts$ = this.stateService.serverHealth$.pipe( + map((hosts) => { + const subpath = window.location.pathname.slice(0, -6); + for (const host of hosts) { + let statusUrl = ''; + let linkHost = ''; + if (host.socket) { + statusUrl = 'https://' + window.location.hostname + subpath + '/status'; + linkHost = window.location.hostname + subpath; + } else { + const hostUrl = new URL(host.host); + statusUrl = 'https://' + hostUrl.hostname + subpath + '/status'; + linkHost = hostUrl.hostname + subpath; + } + host.statusPage = this.sanitizer.bypassSecurityTrustResourceUrl(this.sanitizer.sanitize(SecurityContext.URL, statusUrl)); + host.link = linkHost; + host.flag = this.parseFlag(host.host); + } + return hosts; + }) + ); + this.tip$ = this.stateService.chainTip$; + this.websocketService.want(['blocks', 'tomahawk']); + } + + trackByFn(index: number, host: HealthCheckHost): string { + return host.host; + } + + private parseFlag(host: string): string { + if (host.includes('.fra.')) { + return 'πŸ‡©πŸ‡ͺ'; + } else if (host.includes('.tk7.')) { + return 'πŸ‡―πŸ‡΅'; + } else if (host.includes('.fmt.')) { + return 'πŸ‡ΊπŸ‡Έ'; + } else if (host.includes('.va1.')) { + return 'πŸ‡ΊπŸ‡Έ'; + } else { + return ''; + } + } +} diff --git a/frontend/src/app/components/server-health/server-status.component.html b/frontend/src/app/components/server-health/server-status.component.html new file mode 100644 index 000000000..23ca16a61 --- /dev/null +++ b/frontend/src/app/components/server-health/server-status.component.html @@ -0,0 +1,16 @@ +
+
+ +

Live Network

+
+ + + + + +
diff --git a/frontend/src/app/components/server-health/server-status.component.scss b/frontend/src/app/components/server-health/server-status.component.scss new file mode 100644 index 000000000..09bebe040 --- /dev/null +++ b/frontend/src/app/components/server-health/server-status.component.scss @@ -0,0 +1,26 @@ +.tomahawk { + .links { + float: right; + text-align: right; + margin-top: 1em; + + a, span { + margin-left: 1em; + } + } + + .dashboard-title { + text-align: left; + } + + .mempoolStatus { + width: 100%; + height: 270px; + } + + .hostLink { + text-align: center; + margin: auto; + margin-top: 1em; + } +} \ No newline at end of file diff --git a/frontend/src/app/components/server-health/server-status.component.ts b/frontend/src/app/components/server-health/server-status.component.ts new file mode 100644 index 000000000..2e522b805 --- /dev/null +++ b/frontend/src/app/components/server-health/server-status.component.ts @@ -0,0 +1,80 @@ +import { Component, OnInit, ChangeDetectionStrategy, SecurityContext, OnDestroy, ChangeDetectorRef } from '@angular/core'; +import { WebsocketService } from '../../services/websocket.service'; +import { Observable, Subject, Subscription, map, tap } from 'rxjs'; +import { StateService } from '../../services/state.service'; +import { HealthCheckHost } from '../../interfaces/websocket.interface'; +import { DomSanitizer } from '@angular/platform-browser'; + +@Component({ + selector: 'app-server-status', + templateUrl: './server-status.component.html', + styleUrls: ['./server-status.component.scss'], + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class ServerStatusComponent implements OnInit, OnDestroy { + tip$: Subject; + hosts: HealthCheckHost[] = []; + hostSubscription: Subscription; + + constructor( + private websocketService: WebsocketService, + private stateService: StateService, + private cd: ChangeDetectorRef, + public sanitizer: DomSanitizer, + ) {} + + ngOnInit(): void { + this.hostSubscription = this.stateService.serverHealth$.pipe( + map((hosts) => { + const subpath = window.location.pathname.slice(0, -8); + for (const host of hosts) { + let statusUrl = ''; + let linkHost = ''; + if (host.socket) { + statusUrl = 'https://' + window.location.hostname + subpath + '/status'; + linkHost = window.location.hostname + subpath; + } else { + const hostUrl = new URL(host.host); + statusUrl = 'https://' + hostUrl.hostname + subpath + '/status'; + linkHost = hostUrl.hostname + subpath; + } + host.statusPage = this.sanitizer.bypassSecurityTrustResourceUrl(this.sanitizer.sanitize(SecurityContext.URL, statusUrl)); + host.link = linkHost; + } + return hosts; + }), + tap((hosts) => { + if (this.hosts.length !== hosts.length) { + this.hosts = hosts.sort((a,b) => { + const aParts = (a.host?.split('.') || []).reverse(); + const bParts = (b.host?.split('.') || []).reverse(); + let i = 0; + while (i < Math.max(aParts.length, bParts.length)) { + if (aParts[i] && !bParts[i]) { + return 1; + } else if (bParts[i] && !aParts[i]) { + return -1; + } else if (aParts[i] !== bParts[i]) { + return aParts[i].localeCompare(bParts[i]); + } + i++; + } + return 0; + }); + } + this.cd.markForCheck(); + }) + ).subscribe(); + this.tip$ = this.stateService.chainTip$; + this.websocketService.want(['blocks', 'tomahawk']); + } + + trackByFn(index: number, host: HealthCheckHost): string { + return host.host; + } + + ngOnDestroy(): void { + this.hosts = []; + this.hostSubscription.unsubscribe(); + } +} diff --git a/frontend/src/app/interfaces/websocket.interface.ts b/frontend/src/app/interfaces/websocket.interface.ts index ff5977332..d085c22fa 100644 --- a/frontend/src/app/interfaces/websocket.interface.ts +++ b/frontend/src/app/interfaces/websocket.interface.ts @@ -1,3 +1,4 @@ +import { SafeResourceUrl } from '@angular/platform-browser'; import { ILoadingIndicators } from '../services/state.service'; import { Transaction } from './electrs.interface'; import { BlockExtended, DifficultyAdjustment, RbfTree } from './node-api.interface'; @@ -120,4 +121,18 @@ export interface Recommendedfees { hourFee: number; minimumFee: number; economyFee: number; +} + +export interface HealthCheckHost { + host: string; + active: boolean; + rtt: number; + latestHeight: number; + socket: boolean; + outOfSync: boolean; + unreachable: boolean; + checked: boolean; + link?: string; + statusPage?: SafeResourceUrl; + flag?: string; } \ No newline at end of file diff --git a/frontend/src/app/liquid/liquid-master-page.module.ts b/frontend/src/app/liquid/liquid-master-page.module.ts index 8988cb05c..4b8364ad5 100644 --- a/frontend/src/app/liquid/liquid-master-page.module.ts +++ b/frontend/src/app/liquid/liquid-master-page.module.ts @@ -19,6 +19,8 @@ import { RecentPegsListComponent } from '../components/liquid-reserves-audit/rec import { FederationWalletComponent } from '../components/liquid-reserves-audit/federation-wallet/federation-wallet.component'; import { FederationUtxosListComponent } from '../components/liquid-reserves-audit/federation-utxos-list/federation-utxos-list.component'; import { FederationAddressesListComponent } from '../components/liquid-reserves-audit/federation-addresses-list/federation-addresses-list.component'; +import { ServerHealthComponent } from '../components/server-health/server-health.component'; +import { ServerStatusComponent } from '../components/server-health/server-status.component'; const routes: Routes = [ { @@ -140,6 +142,19 @@ const routes: Routes = [ }, ]; +if (window['__env']?.OFFICIAL_MEMPOOL_SPACE) { + routes[0].children.push({ + path: 'nodes', + data: { networks: ['bitcoin', 'liquid'] }, + component: ServerHealthComponent + }); + routes[0].children.push({ + path: 'network', + data: { networks: ['bitcoin', 'liquid'] }, + component: ServerStatusComponent + }); +} + @NgModule({ imports: [ RouterModule.forChild(routes) diff --git a/frontend/src/app/master-page.module.ts b/frontend/src/app/master-page.module.ts index d7ec87030..ec3e08674 100644 --- a/frontend/src/app/master-page.module.ts +++ b/frontend/src/app/master-page.module.ts @@ -10,6 +10,8 @@ import { PushTransactionComponent } from './components/push-transaction/push-tra import { CalculatorComponent } from './components/calculator/calculator.component'; import { BlocksList } from './components/blocks-list/blocks-list.component'; import { RbfList } from './components/rbf-list/rbf-list.component'; +import { ServerHealthComponent } from './components/server-health/server-health.component'; +import { ServerStatusComponent } from './components/server-health/server-status.component'; const browserWindow = window || {}; // @ts-ignore @@ -96,6 +98,19 @@ const routes: Routes = [ } ]; +if (window['__env']?.OFFICIAL_MEMPOOL_SPACE) { + routes[0].children.push({ + path: 'nodes', + data: { networks: ['bitcoin', 'liquid'] }, + component: ServerHealthComponent + }); + routes[0].children.push({ + path: 'network', + data: { networks: ['bitcoin', 'liquid'] }, + component: ServerStatusComponent + }); +} + @NgModule({ imports: [ RouterModule.forChild(routes) diff --git a/frontend/src/app/services/state.service.ts b/frontend/src/app/services/state.service.ts index dc1365baa..83cd449c8 100644 --- a/frontend/src/app/services/state.service.ts +++ b/frontend/src/app/services/state.service.ts @@ -1,14 +1,13 @@ import { Inject, Injectable, PLATFORM_ID, LOCALE_ID } from '@angular/core'; import { ReplaySubject, BehaviorSubject, Subject, fromEvent, Observable, merge } from 'rxjs'; import { Transaction } from '../interfaces/electrs.interface'; -import { IBackendInfo, MempoolBlock, MempoolBlockDelta, MempoolInfo, Recommendedfees, ReplacedTransaction, ReplacementInfo, TransactionCompressed, TransactionStripped } from '../interfaces/websocket.interface'; +import { HealthCheckHost, IBackendInfo, MempoolBlock, MempoolBlockDelta, MempoolInfo, Recommendedfees, ReplacedTransaction, ReplacementInfo, TransactionStripped } from '../interfaces/websocket.interface'; import { BlockExtended, CpfpInfo, DifficultyAdjustment, MempoolPosition, OptimizedMempoolStats, RbfTree } from '../interfaces/node-api.interface'; import { Router, NavigationStart } from '@angular/router'; import { isPlatformBrowser } from '@angular/common'; import { filter, map, scan, shareReplay } from 'rxjs/operators'; import { StorageService } from './storage.service'; import { hasTouchScreen } from '../shared/pipes/bytes-pipe/utils'; -import { ApiService } from './api.service'; import { ActiveFilter } from '../shared/filters.utils'; export interface MarkBlockState { @@ -129,6 +128,7 @@ export class StateService { loadingIndicators$ = new ReplaySubject(1); recommendedFees$ = new ReplaySubject(1); chainTip$ = new ReplaySubject(-1); + serverHealth$ = new Subject(); live2Chart$ = new Subject(); diff --git a/frontend/src/app/services/websocket.service.ts b/frontend/src/app/services/websocket.service.ts index 11e24ef71..f4dcc4037 100644 --- a/frontend/src/app/services/websocket.service.ts +++ b/frontend/src/app/services/websocket.service.ts @@ -415,6 +415,10 @@ export class WebsocketService { this.stateService.previousRetarget$.next(response.previousRetarget); } + if (response['tomahawk']) { + this.stateService.serverHealth$.next(response['tomahawk']); + } + if (response['git-commit']) { this.stateService.backendInfo$.next(response['git-commit']); } diff --git a/frontend/src/app/shared/shared.module.ts b/frontend/src/app/shared/shared.module.ts index 36e7e79b8..2f6496559 100644 --- a/frontend/src/app/shared/shared.module.ts +++ b/frontend/src/app/shared/shared.module.ts @@ -53,6 +53,8 @@ import { AssetComponent } from '../components/asset/asset.component'; import { AssetsComponent } from '../components/assets/assets.component'; import { AssetsNavComponent } from '../components/assets/assets-nav/assets-nav.component'; import { StatusViewComponent } from '../components/status-view/status-view.component'; +import { ServerHealthComponent } from '../components/server-health/server-health.component'; +import { ServerStatusComponent } from '../components/server-health/server-status.component'; import { FeesBoxComponent } from '../components/fees-box/fees-box.component'; import { DifficultyComponent } from '../components/difficulty/difficulty.component'; import { DifficultyTooltipComponent } from '../components/difficulty/difficulty-tooltip.component'; @@ -151,6 +153,8 @@ import { OnlyVsizeDirective, OnlyWeightDirective } from './components/weight-dir AssetComponent, AssetsComponent, StatusViewComponent, + ServerHealthComponent, + ServerStatusComponent, FeesBoxComponent, DifficultyComponent, DifficultyMiningComponent, @@ -277,6 +281,8 @@ import { OnlyVsizeDirective, OnlyWeightDirective } from './components/weight-dir AssetComponent, AssetsComponent, StatusViewComponent, + ServerHealthComponent, + ServerStatusComponent, FeesBoxComponent, DifficultyComponent, DifficultyMiningComponent, diff --git a/production/check b/production/check new file mode 100755 index 000000000..bbae2824b --- /dev/null +++ b/production/check @@ -0,0 +1,23 @@ +#!/usr/bin/env zsh + +check_frontend_git_commit_hash() { + echo -n $(curl -s https://node$1.$2.mempool.space/resources/config.js|grep GIT_COMMIT_HASH|cut -d "'" -f2|cut -c1-8) +} + +check_html_md5_hash() { + echo -n $(curl -s https://node$1.$2.mempool.space|md5|cut -c1-8) +} + +for site in fmt va1 fra tk7;do + echo "${site}" + for node in 201 202 203 204 205 206 207 208 209 210 211 212 213 214;do + [ "${site}" = "fmt" ] && [ "${node}" -gt 206 ] && continue + [ "${site}" = "tk7" ] && [ "${node}" -gt 206 ] && continue + echo -n "node${node}.${site}: " + check_frontend_git_commit_hash $node $site + echo -n " " + check_html_md5_hash $node $site + echo + done +done + diff --git a/production/mempool-config.bisq.json b/production/mempool-config.bisq.json index 26024f8a3..4913cb986 100644 --- a/production/mempool-config.bisq.json +++ b/production/mempool-config.bisq.json @@ -1,5 +1,6 @@ { "MEMPOOL": { + "OFFICIAL": true, "NETWORK": "bisq", "BACKEND": "esplora", "HTTP_PORT": 8996, diff --git a/production/mempool-config.liquid.json b/production/mempool-config.liquid.json index b852be3ae..9051bba74 100644 --- a/production/mempool-config.liquid.json +++ b/production/mempool-config.liquid.json @@ -1,5 +1,6 @@ { "MEMPOOL": { + "OFFICIAL": true, "NETWORK": "liquid", "BACKEND": "esplora", "HTTP_PORT": 8998, diff --git a/production/mempool-config.liquidtestnet.json b/production/mempool-config.liquidtestnet.json index 71c094dc6..ae6d7b1ac 100644 --- a/production/mempool-config.liquidtestnet.json +++ b/production/mempool-config.liquidtestnet.json @@ -1,5 +1,6 @@ { "MEMPOOL": { + "OFFICIAL": true, "NETWORK": "liquid", "BACKEND": "esplora", "HTTP_PORT": 8994, diff --git a/production/mempool-config.mainnet-lightning.json b/production/mempool-config.mainnet-lightning.json index bef04e7a6..8dea10b4a 100644 --- a/production/mempool-config.mainnet-lightning.json +++ b/production/mempool-config.mainnet-lightning.json @@ -1,5 +1,6 @@ { "MEMPOOL": { + "OFFICIAL": true, "ENABLED": false, "NETWORK": "mainnet", "BACKEND": "esplora", diff --git a/production/mempool-config.mainnet.json b/production/mempool-config.mainnet.json index 1bb9e35a5..5038d9bfb 100644 --- a/production/mempool-config.mainnet.json +++ b/production/mempool-config.mainnet.json @@ -1,5 +1,6 @@ { "MEMPOOL": { + "OFFICIAL": true, "NETWORK": "mainnet", "BACKEND": "esplora", "HTTP_PORT": 8999, @@ -21,7 +22,8 @@ "DISK_CACHE_BLOCK_INTERVAL": 1, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, "ALLOW_UNREACHABLE": true, - "PRICE_UPDATES_PER_HOUR": 12 + "PRICE_UPDATES_PER_HOUR": 12, + "MAX_TRACKED_ADDRESSES": 10 }, "SYSLOG" : { "MIN_PRIORITY": "debug" diff --git a/production/mempool-config.signet-lightning.json b/production/mempool-config.signet-lightning.json index 9cffe55ca..f90b18f50 100644 --- a/production/mempool-config.signet-lightning.json +++ b/production/mempool-config.signet-lightning.json @@ -1,5 +1,6 @@ { "MEMPOOL": { + "OFFICIAL": true, "ENABLED": false, "NETWORK": "signet", "BACKEND": "esplora", diff --git a/production/mempool-config.signet.json b/production/mempool-config.signet.json index 71b52be5b..0a711d16f 100644 --- a/production/mempool-config.signet.json +++ b/production/mempool-config.signet.json @@ -1,5 +1,6 @@ { "MEMPOOL": { + "OFFICIAL": true, "NETWORK": "signet", "BACKEND": "esplora", "HTTP_PORT": 8995, @@ -14,7 +15,8 @@ "POLL_RATE_MS": 1000, "DISK_CACHE_BLOCK_INTERVAL": 1, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, - "ALLOW_UNREACHABLE": true + "ALLOW_UNREACHABLE": true, + "MAX_TRACKED_ADDRESSES": 10 }, "SYSLOG" : { "MIN_PRIORITY": "debug" diff --git a/production/mempool-config.testnet-lightning.json b/production/mempool-config.testnet-lightning.json index f7bdebbeb..59a858cbf 100644 --- a/production/mempool-config.testnet-lightning.json +++ b/production/mempool-config.testnet-lightning.json @@ -1,5 +1,6 @@ { "MEMPOOL": { + "OFFICIAL": true, "ENABLED": false, "NETWORK": "testnet", "BACKEND": "esplora", diff --git a/production/mempool-config.testnet.json b/production/mempool-config.testnet.json index e35dfe78e..adc93c0e9 100644 --- a/production/mempool-config.testnet.json +++ b/production/mempool-config.testnet.json @@ -1,5 +1,6 @@ { "MEMPOOL": { + "OFFICIAL": true, "NETWORK": "testnet", "BACKEND": "esplora", "HTTP_PORT": 8997, @@ -14,7 +15,8 @@ "POLL_RATE_MS": 1000, "DISK_CACHE_BLOCK_INTERVAL": 1, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, - "ALLOW_UNREACHABLE": true + "ALLOW_UNREACHABLE": true, + "MAX_TRACKED_ADDRESSES": 10 }, "SYSLOG" : { "MIN_PRIORITY": "debug" diff --git a/production/nginx/http-proxy-cache.conf b/production/nginx/http-proxy-cache.conf index 0024a3b30..dcc17208b 100644 --- a/production/nginx/http-proxy-cache.conf +++ b/production/nginx/http-proxy-cache.conf @@ -8,5 +8,9 @@ proxy_cache_path /var/cache/nginx/apicold keys_zone=apicold:200m levels=1:2 inac proxy_cache_path /var/cache/nginx/unfurler keys_zone=unfurler:200m levels=1:2 inactive=30d max_size=2000m; proxy_cache_path /var/cache/nginx/slurper keys_zone=slurper:500m levels=1:2 inactive=365d max_size=5000m; proxy_cache_path /var/cache/nginx/markets keys_zone=markets:20m levels=1:2 inactive=365d max_size=100m; -types_hash_max_size 4096; -proxy_buffer_size 8k; + +types_hash_max_size 8192; + +proxy_busy_buffers_size 256k; +proxy_buffer_size 128k; +proxy_buffers 4 256k; diff --git a/scripts/get_block_tip_height.sh b/scripts/get_block_tip_height.sh new file mode 100644 index 000000000..603193bb8 --- /dev/null +++ b/scripts/get_block_tip_height.sh @@ -0,0 +1,25 @@ +BASE_HEIGHT=$(curl -sk https://node202.tk7.mempool.space/api/v1/blocks/tip/height) +IN_SYNC=true +echo "Base height (node202.tk7): $BASE_HEIGHT" + +for LOCATION in fmt va1 fra tk7 +do + for NODE in 201 202 203 204 205 206 + do + NODE_HEIGHT=$(curl -sk https://node$NODE.$LOCATION.mempool.space/api/v1/blocks/tip/height) + echo $(echo node$NODE.$LOCATION.mempool.space) - $NODE_HEIGHT + if [ "$NODE_HEIGHT" -ne "$BASE_HEIGHT" ]; then + COUNT=$((BASE_HEIGHT-NODE_HEIGHT)) + echo $(echo node$NODE.$LOCATION.mempool.space) is not in sync. delta: $COUNT + IN_SYNC=false + fi + done +done + +if [ "$IN_SYNC" = false ]; then + echo "One or more servers are out of sync. Check the logs." + exit -1 +else + echo "All servers are in sync." +fi +