machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into mononaut/tomahawk-timeouts

Browse Source
This commit is contained in:
wiz 2024-03-06 16:24:35 +09:00 committed by GitHub
commit faa5475ab7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
50 changed files with 1575 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
.github/workflows/get_backend_block_height.yml vendored Normal file
View File

@ -0,0 +1,19 @@
name: 'Check if servers are in sync'
on: [workflow_dispatch]
jobs:
print-backend-sha:
runs-on: 'ubuntu-latest'
name: Get block height
steps:
- name: Checkout
uses: actions/checkout@v3
with:
path: repo
- name: Run script
working-directory: repo
run: |
chmod +x ./scripts/get_block_tip_height.sh
sh ./scripts/get_block_tip_height.sh

85
Cargo.lock generated
View File

@ -57,9 +57,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1586fa608b1dab41f667475b4a41faec5ba680aee428bfa5de4ea520fdc6e901"
dependencies = [
"quote",
"syn 2.0.20",
"syn",
]
[[package]]
name = "equivalent"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
[[package]]
name = "gbt"
version = "1.0.0"
@ -71,15 +77,15 @@ dependencies = [
"napi-derive",
"priority-queue",
"tracing",
"tracing-log",
"tracing-log 0.2.0",
"tracing-subscriber",
]
[[package]]
name = "hashbrown"
version = "0.12.3"
version = "0.14.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604"
[[package]]
name = "hermit-abi"
@ -92,11 +98,11 @@ dependencies = [
[[package]]
name = "indexmap"
version = "1.9.3"
version = "2.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
checksum = "7b0b929d511467233429c45a44ac1dcaa21ba0f5ba11e4879e6ed28ddb4f9df4"
dependencies = [
"autocfg",
"equivalent",
"hashbrown",
]
@ -114,12 +120,12 @@ checksum = "f92be4933c13fd498862a9e02a3055f8a8d9c039ce33db97306fd5a6caa7f29b"
[[package]]
name = "libloading"
version = "0.7.4"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f"
checksum = "2caa5afb8bf9f3a2652760ce7d4f62d21c4d5a423e68466fca30df82f2330164"
dependencies = [
"cfg-if",
"winapi",
"windows-targets",
]
[[package]]
@ -145,9 +151,9 @@ checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
[[package]]
name = "napi"
version = "2.13.2"
version = "2.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ede2d12cd6fce44da537a4be1f5510c73be2506c2e32dfaaafd1f36968f3a0e"
checksum = "54a63d0570e4c3e0daf7a8d380563610e159f538e20448d6c911337246f40e84"
dependencies = [
"bitflags",
"ctor",
@ -159,29 +165,29 @@ dependencies = [
[[package]]
name = "napi-build"
version = "2.0.1"
version = "2.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "882a73d9ef23e8dc2ebbffb6a6ae2ef467c0f18ac10711e4cc59c5485d41df0e"
checksum = "2f9130fccc5f763cf2069b34a089a18f0d0883c66aceb81f2fad541a3d823c43"
[[package]]
name = "napi-derive"
version = "2.13.0"
version = "2.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da1c6a8fa84d549aa8708fcd062372bf8ec6e849de39016ab921067d21bde367"
checksum = "05bb7c37e3c1dda9312fdbe4a9fc7507fca72288ba154ec093e2d49114e727ce"
dependencies = [
"cfg-if",
"convert_case",
"napi-derive-backend",
"proc-macro2",
"quote",
"syn 1.0.109",
"syn",
]
[[package]]
name = "napi-derive-backend"
version = "1.0.52"
version = "1.0.62"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20bbc7c69168d06a848f925ec5f0e0997f98e8c8d4f2cc30157f0da51c009e17"
checksum = "f785a8b8d7b83e925f5aa6d2ae3c159d17fe137ac368dc185bef410e7acdaeb4"
dependencies = [
"convert_case",
"once_cell",
@ -189,14 +195,14 @@ dependencies = [
"quote",
"regex",
"semver",
"syn 1.0.109",
"syn",
]
[[package]]
name = "napi-sys"
version = "2.2.3"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "166b5ef52a3ab5575047a9fe8d4a030cdd0f63c96f071cd6907674453b07bae3"
checksum = "2503fa6af34dc83fb74888df8b22afe933b58d37daf7d80424b1c60c68196b8b"
dependencies = [
"libloading",
]
@ -223,9 +229,9 @@ dependencies = [
[[package]]
name = "once_cell"
version = "1.18.0"
version = "1.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
[[package]]
name = "overload"
@ -241,11 +247,12 @@ checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116"
[[package]]
name = "priority-queue"
version = "1.3.2"
version = "2.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fff39edfcaec0d64e8d0da38564fad195d2d51b680940295fcc307366e101e61"
checksum = "509354d8a769e8d0b567d6821b84495c60213162761a732d68ce87c964bd347f"
dependencies = [
"autocfg",
"equivalent",
"indexmap",
]
@ -320,17 +327,6 @@ version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"
[[package]]
name = "syn"
version = "1.0.109"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "syn"
version = "2.0.20"
@ -384,7 +380,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.20",
"syn",
]
[[package]]
@ -408,6 +404,17 @@ dependencies = [
"tracing-core",
]
[[package]]
name = "tracing-log"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3"
dependencies = [
"log",
"once_cell",
"tracing-core",
]
[[package]]
name = "tracing-subscriber"
version = "0.3.17"
@ -423,7 +430,7 @@ dependencies = [
"thread_local",
"tracing",
"tracing-core",
"tracing-log",
"tracing-log 0.1.3",
]
[[package]]

1
Cargo.toml
View File

@ -1,4 +1,5 @@
[workspace]
resolver = "2"
members = [
"./backend/rust-gbt",
]

1
backend/mempool-config.sample.json
View File

@ -1,5 +1,6 @@
{
"MEMPOOL": {
"OFFICIAL": false,
"NETWORK": "mainnet",
"BACKEND": "electrum",
"ENABLED": true,

17
backend/package-lock.json generated
View File

@ -9,6 +9,7 @@
"version": "3.0.0-dev",
"license": "GNU Affero General Public License v3.0",
"dependencies": {
"@babel/core": "^7.24.0",
"@mempool/electrum-client": "1.1.9",
"@types/node": "^18.15.3",
"axios": "~1.6.1",
@ -1499,9 +1500,9 @@
}
},
"node_modules/@napi-rs/cli": {
"version": "2.16.1",
"resolved": "https://registry.npmjs.org/@napi-rs/cli/-/cli-2.16.1.tgz",
"integrity": "sha512-L0Gr5iEQIDEbvWdDr1HUaBOxBSHL1VZhWSk1oryawoT8qJIY+KGfLFelU+Qma64ivCPbxYpkfPoKYVG3rcoGIA==",
"version": "2.18.0",
"resolved": "https://registry.npmjs.org/@napi-rs/cli/-/cli-2.18.0.tgz",
"integrity": "sha512-lfSRT7cs3iC4L+kv9suGYQEezn5Nii7Kpu+THsYVI0tA1Vh59LH45p4QADaD7hvIkmOz79eEGtoKQ9nAkAPkzA==",
"bin": {
"napi": "scripts/index.js"
},
@ -7669,7 +7670,7 @@
"version": "3.0.1",
"hasInstallScript": true,
"dependencies": {
"@napi-rs/cli": "2.16.1"
"@napi-rs/cli": "2.18.0"
},
"engines": {
"node": ">= 12"
@ -8774,9 +8775,9 @@
"integrity": "sha512-mlvPiCzUlaETpYW3i6V87A24jjMYgsebaXtUo3WQyyLnYUuxs0KiXQ2mnKh3h15j8Xg/hfxeGIi+5OC9u0nftQ=="
},
"@napi-rs/cli": {
"version": "2.16.1",
"resolved": "https://registry.npmjs.org/@napi-rs/cli/-/cli-2.16.1.tgz",
"integrity": "sha512-L0Gr5iEQIDEbvWdDr1HUaBOxBSHL1VZhWSk1oryawoT8qJIY+KGfLFelU+Qma64ivCPbxYpkfPoKYVG3rcoGIA=="
"version": "2.18.0",
"resolved": "https://registry.npmjs.org/@napi-rs/cli/-/cli-2.18.0.tgz",
"integrity": "sha512-lfSRT7cs3iC4L+kv9suGYQEezn5Nii7Kpu+THsYVI0tA1Vh59LH45p4QADaD7hvIkmOz79eEGtoKQ9nAkAPkzA=="
},
"@noble/hashes": {
"version": "1.3.0",
@ -12702,7 +12703,7 @@
"rust-gbt": {
"version": "file:rust-gbt",
"requires": {
"@napi-rs/cli": "2.16.1"
"@napi-rs/cli": "2.18.0"
}
},
"safe-buffer": {

10
backend/rust-gbt/Cargo.toml
View File

@ -12,14 +12,14 @@ crate-type = ["cdylib"]
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
priority-queue = "1.3.2"
priority-queue = "2.0.2"
bytes = "1.4.0"
napi = { version = "2.13.2", features = ["napi8", "tokio_rt"] }
napi-derive = "2.13.0"
napi = { version = "2.16.0", features = ["napi8", "tokio_rt"] }
napi-derive = "2.16.0"
bytemuck = "1.13.1"
tracing = "0.1.36"
tracing-log = "0.1.3"
tracing-log = "0.2.0"
tracing-subscriber = { version = "0.3.15", features = ["env-filter"]}
[build-dependencies]
napi-build = "2.0.1"
napi-build = "2.1.2"

43
backend/rust-gbt/index.js
View File

@ -237,6 +237,49 @@ switch (platform) {
loadError = e
}
break
case 'riscv64':
if (isMusl()) {
localFileExisted = existsSync(
join(__dirname, 'gbt.linux-riscv64-musl.node')
)
try {
if (localFileExisted) {
nativeBinding = require('./gbt.linux-riscv64-musl.node')
} else {
nativeBinding = require('gbt-linux-riscv64-musl')
}
} catch (e) {
loadError = e
}
} else {
localFileExisted = existsSync(
join(__dirname, 'gbt.linux-riscv64-gnu.node')
)
try {
if (localFileExisted) {
nativeBinding = require('./gbt.linux-riscv64-gnu.node')
} else {
nativeBinding = require('gbt-linux-riscv64-gnu')
}
} catch (e) {
loadError = e
}
}
break
case 's390x':
localFileExisted = existsSync(
join(__dirname, 'gbt.linux-s390x-gnu.node')
)
try {
if (localFileExisted) {
nativeBinding = require('./gbt.linux-s390x-gnu.node')
} else {
nativeBinding = require('gbt-linux-s390x-gnu')
}
} catch (e) {
loadError = e
}
break
default:
throw new Error(`Unsupported architecture on Linux: ${arch}`)
}

8
backend/rust-gbt/package-lock.json generated
View File

@ -9,16 +9,16 @@
"version": "3.0.1",
"hasInstallScript": true,
"dependencies": {
"@napi-rs/cli": "2.16.1"
"@napi-rs/cli": "2.18.0"
},
"engines": {
"node": ">= 12"
}
},
"node_modules/@napi-rs/cli": {
"version": "2.16.1",
"resolved": "https://registry.npmjs.org/@napi-rs/cli/-/cli-2.16.1.tgz",
"integrity": "sha512-L0Gr5iEQIDEbvWdDr1HUaBOxBSHL1VZhWSk1oryawoT8qJIY+KGfLFelU+Qma64ivCPbxYpkfPoKYVG3rcoGIA==",
"version": "2.18.0",
"resolved": "https://registry.npmjs.org/@napi-rs/cli/-/cli-2.18.0.tgz",
"integrity": "sha512-lfSRT7cs3iC4L+kv9suGYQEezn5Nii7Kpu+THsYVI0tA1Vh59LH45p4QADaD7hvIkmOz79eEGtoKQ9nAkAPkzA==",
"bin": {
"napi": "scripts/index.js"
},

2
backend/rust-gbt/package.json
View File

@ -25,7 +25,7 @@
}
},
"dependencies": {
"@napi-rs/cli": "2.16.1"
"@napi-rs/cli": "2.18.0"
},
"engines": {
"node": ">= 12"

4
backend/src/__fixtures__/mempool-config.template.json
View File

@ -1,6 +1,7 @@
{
"MEMPOOL": {
"ENABLED": true,
"OFFICIAL": false,
"NETWORK": "__MEMPOOL_NETWORK__",
"BACKEND": "__MEMPOOL_BACKEND__",
"BLOCKS_SUMMARIES_INDEXING": true,
@ -79,7 +80,8 @@
"USERNAME": "__DATABASE_USERNAME__",
"PASSWORD": "__DATABASE_PASSWORD__",
"PID_DIR": "__DATABASE_PID_FILE__",
"TIMEOUT": 3000
"TIMEOUT": 3000,
"POOL_SIZE": 100
},
"SYSLOG": {
"ENABLED": false,

4
backend/src/__tests__/config.test.ts
View File

@ -14,6 +14,7 @@ describe('Mempool Backend Config', () => {
expect(config.MEMPOOL).toStrictEqual({
ENABLED: true,
OFFICIAL: false,
NETWORK: 'mainnet',
BACKEND: 'none',
BLOCKS_SUMMARIES_INDEXING: false,
@ -93,7 +94,8 @@ describe('Mempool Backend Config', () => {
USERNAME: 'mempool',
PASSWORD: 'mempool',
TIMEOUT: 180000,
PID_DIR: ''
PID_DIR: '',
POOL_SIZE: 100,
});
expect(config.SYSLOG).toStrictEqual({

738
backend/src/api/acceleration.ts Normal file
View File

@ -0,0 +1,738 @@
import logger from '../logger';
import { MempoolTransactionExtended } from '../mempool.interfaces';
import { IEsploraApi } from './bitcoin/esplora-api.interface';
const BLOCK_WEIGHT_UNITS = 4_000_000;
const BLOCK_SIGOPS = 80_000;
const MAX_RELATIVE_GRAPH_SIZE = 200;
const BID_BOOST_WINDOW = 40_000;
const BID_BOOST_MIN_OFFSET = 10_000;
const BID_BOOST_MAX_OFFSET = 400_000;
type Acceleration = {
txid: string;
max_bid: number;
};
interface TxSummary {
txid: string; // txid of the current transaction
effectiveVsize: number; // Total vsize of the dependency tree
effectiveFee: number; // Total fee of the dependency tree in sats
ancestorCount: number; // Number of ancestors
}
export interface AccelerationInfo {
txSummary: TxSummary;
targetFeeRate: number; // target fee rate (recommended next block fee, or median fee for mined block)
nextBlockFee: number; // fee in sats required to be in the next block (using recommended next block fee, or median fee for mined block)
cost: number; // additional cost to accelerate ((cost + txSummary.effectiveFee) / txSummary.effectiveVsize) >= targetFeeRate
}
interface GraphTx {
txid: string;
vsize: number;
weight: number;
fees: {
base: number;
};
depends: string[];
spentby: string[];
}
interface MempoolTx extends GraphTx {
ancestorcount: number;
ancestorsize: number;
fees: {
base: number;
ancestor: number;
};
ancestors: Map<string, MempoolTx>,
ancestorRate: number;
individualRate: number;
score: number;
}
class AccelerationCosts {
/**
* Takes a list of accelerations and verbose block data
* Returns the "fair" boost rate to charge accelerations
*
* @param accelerationsx
* @param verboseBlock
*/
public calculateBoostRate(accelerations: Acceleration[], blockTxs: IEsploraApi.Transaction[]): number {
// Run GBT ourselves to calculate accurate effective fee rates
// the list of transactions comes from a mined block, so we already know everything fits within consensus limits
const template = makeBlockTemplate(blockTxs, accelerations, 1, Infinity, Infinity);
// initialize working maps for fast tx lookups
const accMap = {};
const txMap = {};
for (const acceleration of accelerations) {
accMap[acceleration.txid] = acceleration;
}
for (const tx of template) {
txMap[tx.txid] = tx;
}
// Identify and exclude accelerated and otherwise prioritized transactions
const excludeMap = {};
let totalWeight = 0;
let minAcceleratedPackage = Infinity;
let lastEffectiveRate = 0;
// Iterate over the mined template from bottom to top.
// Transactions should appear in ascending order of mining priority.
for (const blockTx of [...blockTxs].reverse()) {
const txid = blockTx.txid;
const tx = txMap[txid];
totalWeight += tx.weight;
const isAccelerated = accMap[txid] != null;
// If a cluster has a in-band effective fee rate than the previous cluster,
// it must have been prioritized out-of-band (in order to have a higher mining priority)
// so exclude from the analysis.
const isPrioritized = tx.effectiveFeePerVsize < lastEffectiveRate;
if (isPrioritized || isAccelerated) {
let packageWeight = 0;
// exclude this whole CPFP cluster
for (const clusterTxid of tx.cluster) {
packageWeight += txMap[clusterTxid].weight;
if (!excludeMap[clusterTxid]) {
excludeMap[clusterTxid] = true;
}
}
// keep track of the smallest accelerated CPFP cluster for later
if (isAccelerated) {
minAcceleratedPackage = Math.min(minAcceleratedPackage, packageWeight);
}
}
if (!isPrioritized) {
if (!isAccelerated || !lastEffectiveRate) {
lastEffectiveRate = tx.effectiveFeePerVsize;
}
}
}
// The Bid Boost Rate is calculated by disregarding the bottom X weight units of the block,
// where X is the larger of BID_BOOST_MIN_OFFSET or the smallest accelerated package weight (the "offset"),
// then taking the average fee rate of the following BID_BOOST_WINDOW weight units
// (ignoring accelerated transactions and their ancestors).
//
// Transactions within the offset might pay less than the fair rate due to bin-packing effects
// But the average rate paid by the next chunk of non-accelerated transactions provides a good
// upper bound on the "next best rate" of alternatives to including the accelerated transactions
// (since, if there were any better options, they would have been included instead)
const spareWeight = BLOCK_WEIGHT_UNITS - totalWeight;
const windowOffset = Math.min(Math.max(minAcceleratedPackage, BID_BOOST_MIN_OFFSET, spareWeight), BID_BOOST_MAX_OFFSET);
const leftBound = windowOffset;
const rightBound = windowOffset + BID_BOOST_WINDOW;
let totalFeeInWindow = 0;
let totalWeightInWindow = Math.max(0, spareWeight - leftBound);
let txIndex = blockTxs.length - 1;
for (let offset = spareWeight; offset < BLOCK_WEIGHT_UNITS && txIndex >= 0; txIndex--) {
const txid = blockTxs[txIndex].txid;
const tx = txMap[txid];
if (excludeMap[txid]) {
// skip prioritized transactions and their ancestors
continue;
}
const left = offset;
const right = offset + tx.weight;
offset += tx.weight;
if (right < leftBound) {
// not within window yet
continue;
}
if (left > rightBound) {
// past window
break;
}
// count fees for weight units within the window
const overlapLeft = Math.max(leftBound, left);
const overlapRight = Math.min(rightBound, right);
const overlapUnits = overlapRight - overlapLeft;
totalFeeInWindow += (tx.effectiveFeePerVsize * (overlapUnits / 4));
totalWeightInWindow += overlapUnits;
}
if (totalWeightInWindow < BID_BOOST_WINDOW) {
// not enough un-prioritized transactions to calculate a fair rate
// just charge everyone their max bids
return Infinity;
}
// Divide the total fee by the size of the BID_BOOST_WINDOW in vbytes
const averageRate = totalFeeInWindow / (BID_BOOST_WINDOW / 4);
return averageRate;
}
/**
* Takes an accelerated mined txid and a target rate
* Returns the total vsize, fees and acceleration cost (in sats) of the tx and all same-block ancestors
*
* @param txid
* @param medianFeeRate
*/
public getAccelerationInfo(tx: MempoolTransactionExtended, targetFeeRate: number, transactions: MempoolTransactionExtended[]): AccelerationInfo {
// Get same-block transaction ancestors
const allRelatives = this.getSameBlockRelatives(tx, transactions);
const relativesMap = this.initializeRelatives(allRelatives);
const rootTx = relativesMap.get(tx.txid) as MempoolTx;
// Calculate cost to boost
return this.calculateAccelerationAncestors(rootTx, relativesMap, targetFeeRate);
}
/**
* Takes a raw transaction, and builds a graph of same-block relatives,
* and returns as a MempoolTx
*
* @param tx
*/
private getSameBlockRelatives(tx: MempoolTransactionExtended, transactions: MempoolTransactionExtended[]): Map<string, GraphTx> {
const blockTxs = new Map<string, MempoolTransactionExtended>(); // map of txs in this block
const spendMap = new Map<string, string>(); // map of outpoints to spending txids
for (const tx of transactions) {
blockTxs.set(tx.txid, tx);
for (const vin of tx.vin) {
spendMap.set(`${vin.txid}:${vin.vout}`, tx.txid);
}
}
const relatives: Map<string, GraphTx> = new Map();
const stack: string[] = [tx.txid];
// build set of same-block ancestors
while (stack.length > 0) {
const nextTxid = stack.pop();
const nextTx = nextTxid ? blockTxs.get(nextTxid) : null;
if (!nextTx || relatives.has(nextTx.txid)) {
continue;
}
const mempoolTx = this.convertToGraphTx(nextTx);
mempoolTx.fees.base = nextTx.fee || 0;
mempoolTx.depends = nextTx.vin.map(vin => vin.txid).filter(inTxid => inTxid && blockTxs.has(inTxid)) as string[];
mempoolTx.spentby = nextTx.vout.map((vout, index) => spendMap.get(`${nextTx.txid}:${index}`)).filter(outTxid => outTxid && blockTxs.has(outTxid)) as string[];
for (const txid of [...mempoolTx.depends, ...mempoolTx.spentby]) {
if (txid) {
stack.push(txid);
}
}
relatives.set(mempoolTx.txid, mempoolTx);
}
return relatives;
}
/**
* Takes a raw transaction and converts it to MempoolTx format
* fee and ancestor data is initialized with dummy/null values
*
* @param tx
*/
private convertToGraphTx(tx: MempoolTransactionExtended): GraphTx {
return {
txid: tx.txid,
vsize: tx.vsize,
weight: tx.weight,
fees: {
base: 0, // dummy
},
depends: [], // dummy
spentby: [], //dummy
};
}
private convertGraphToMempoolTx(tx: GraphTx): MempoolTx {
return {
...tx,
fees: {
base: tx.fees.base,
ancestor: tx.fees.base,
},
ancestorcount: 1,
ancestorsize: tx.vsize,
ancestors: new Map<string, MempoolTx>(),
ancestorRate: 0,
individualRate: 0,
score: 0,
};
}
/**
* Given a root transaction, a list of in-mempool ancestors, and a target fee rate,
* Calculate the minimum set of transactions to fee-bump, their total vsize + fees
*
* @param tx
* @param ancestors
*/
private calculateAccelerationAncestors(tx: MempoolTx, relatives: Map<string, MempoolTx>, targetFeeRate: number): AccelerationInfo {
// add root tx to the ancestor map
relatives.set(tx.txid, tx);
// Check for high-sigop transactions (not supported)
relatives.forEach(entry => {
if (entry.vsize > Math.ceil(entry.weight / 4)) {
throw new Error(`high_sigop_tx`);
}
});
// Initialize individual & ancestor fee rates
relatives.forEach(entry => this.setAncestorScores(entry));
// Sort by descending ancestor score
let sortedRelatives = Array.from(relatives.values()).sort(this.mempoolComparator);
let includedInCluster: Map<string, MempoolTx> | null = null;
// While highest score >= targetFeeRate
let maxIterations = MAX_RELATIVE_GRAPH_SIZE;
while (sortedRelatives.length && sortedRelatives[0].score && sortedRelatives[0].score >= targetFeeRate && maxIterations > 0) {
maxIterations--;
// Grab the highest scoring entry
const best = sortedRelatives.shift();
if (best) {
const cluster = new Map<string, MempoolTx>(best.ancestors?.entries() || []);
if (best.ancestors.has(tx.txid)) {
includedInCluster = cluster;
}
cluster.set(best.txid, best);
// Remove this cluster (it already pays over the target rate, so doesn't need to be boosted)
// and update scores, ancestor totals and dependencies for the survivors
this.removeAncestors(cluster, relatives);
// re-sort
sortedRelatives = Array.from(relatives.values()).sort(this.mempoolComparator);
}
}
// sanity check for infinite loops / too many ancestors (should never happen)
if (maxIterations <= 0) {
logger.warn(`acceleration dependency calculation failed: calculateAccelerationAncestors loop exceeded ${MAX_RELATIVE_GRAPH_SIZE} iterations, unable to proceed`);
throw new Error('invalid_tx_dependencies');
}
let totalFee = Math.round(tx.fees.ancestor * 100_000_000);
// transaction is already CPFP-d above the target rate by some descendant
if (includedInCluster) {
let clusterSize = 0;
let clusterFee = 0;
includedInCluster.forEach(entry => {
clusterSize += entry.vsize;
clusterFee += (entry.fees.base * 100_000_000);
});
const clusterRate = clusterFee / clusterSize;
totalFee = Math.ceil(tx.ancestorsize * clusterRate);
}
// Whatever remains in the accelerated tx's dependencies needs to be boosted to the targetFeeRate
// Cost = (totalVsize * targetFeeRate) - totalFee
return {
txSummary: {
txid: tx.txid,
effectiveVsize: tx.ancestorsize,
effectiveFee: totalFee,
ancestorCount: tx.ancestorcount,
},
cost: Math.max(0, Math.ceil(tx.ancestorsize * targetFeeRate) - totalFee),
targetFeeRate,
nextBlockFee: Math.ceil(tx.ancestorsize * targetFeeRate),
};
}
/**
* Recursively traverses an in-mempool dependency graph, and sets a Map of in-mempool ancestors
* for each transaction.
*
* @param tx
* @param all
*/
private setAncestors(tx: MempoolTx, all: Map<string, MempoolTx>, visited: Map<string, Map<string, MempoolTx>>, depth: number = 0): Map<string, MempoolTx> {
// sanity check for infinite recursion / too many ancestors (should never happen)
if (depth >= 100) {
logger.warn('acceleration dependency calculation failed: setAncestors reached depth of 100, unable to proceed', `Accelerator`);
throw new Error('invalid_tx_dependencies');
}
// initialize the ancestor map for this tx
tx.ancestors = new Map<string, MempoolTx>();
tx.depends.forEach(parentId => {
const parent = all.get(parentId);
if (parent) {
// add the parent
tx.ancestors?.set(parentId, parent);
// check for a cached copy of this parent's ancestors
let ancestors = visited.get(parent.txid);
if (!ancestors) {
// recursively fetch the parent's ancestors
ancestors = this.setAncestors(parent, all, visited, depth + 1);
}
// and add to this tx's map
ancestors.forEach((ancestor, ancestorId) => {
tx.ancestors?.set(ancestorId, ancestor);
});
}
});
visited.set(tx.txid, tx.ancestors);
return tx.ancestors;
}
/**
* Efficiently sets a Map of in-mempool ancestors for each member of an expanded relative graph
* by running setAncestors on each leaf, and caching intermediate results.
* then initializes ancestor data for each transaction
*
* @param all
*/
private initializeRelatives(all: Map<string, GraphTx>): Map<string, MempoolTx> {
const mempoolTxs = new Map<string, MempoolTx>();
all.forEach(entry => {
mempoolTxs.set(entry.txid, this.convertGraphToMempoolTx(entry));
});
const visited: Map<string, Map<string, MempoolTx>> = new Map();
const leaves: MempoolTx[] = Array.from(mempoolTxs.values()).filter(entry => entry.spentby.length === 0);
for (const leaf of leaves) {
this.setAncestors(leaf, mempoolTxs, visited);
}
mempoolTxs.forEach(entry => {
entry.ancestors?.forEach(ancestor => {
entry.ancestorcount++;
entry.ancestorsize += ancestor.vsize;
entry.fees.ancestor += ancestor.fees.base;
});
this.setAncestorScores(entry);
});
return mempoolTxs;
}
/**
* Remove a cluster of transactions from an in-mempool dependency graph
* and update the survivors' scores and ancestors
*
* @param cluster
* @param ancestors
*/
private removeAncestors(cluster: Map<string, MempoolTx>, all: Map<string, MempoolTx>): void {
// remove
cluster.forEach(tx => {
all.delete(tx.txid);
});
// update survivors
all.forEach(tx => {
cluster.forEach(remove => {
if (tx.ancestors?.has(remove.txid)) {
// remove as dependency
tx.ancestors.delete(remove.txid);
tx.depends = tx.depends.filter(parent => parent !== remove.txid);
// update ancestor sizes and fees
tx.ancestorsize -= remove.vsize;
tx.fees.ancestor -= remove.fees.base;
}
});
// recalculate fee rates
this.setAncestorScores(tx);
});
}
/**
* Take a mempool transaction, and set the fee rates and ancestor score
*
* @param tx
*/
private setAncestorScores(tx: MempoolTx): void {
tx.individualRate = (tx.fees.base * 100_000_000) / tx.vsize;
tx.ancestorRate = (tx.fees.ancestor * 100_000_000) / tx.ancestorsize;
tx.score = Math.min(tx.individualRate, tx.ancestorRate);
}
// Sort by descending score
private mempoolComparator(a, b): number {
return b.score - a.score;
}
}
export default new AccelerationCosts;
interface TemplateTransaction {
txid: string;
order: number;
weight: number;
adjustedVsize: number; // sigop-adjusted vsize, rounded up to the nearest integer
sigops: number;
fee: number;
feeDelta: number;
ancestors: string[];
cluster: string[];
effectiveFeePerVsize: number;
}
interface MinerTransaction extends TemplateTransaction {
inputs: string[];
feePerVsize: number;
relativesSet: boolean;
ancestorMap: Map<string, MinerTransaction>;
children: Set<MinerTransaction>;
ancestorFee: number;
ancestorVsize: number;
ancestorSigops: number;
score: number;
used: boolean;
modified: boolean;
dependencyRate: number;
}
/*
* Build a block using an approximation of the transaction selection algorithm from Bitcoin Core
* (see BlockAssembler in https://github.com/bitcoin/bitcoin/blob/master/src/node/miner.cpp)
*/
function makeBlockTemplate(candidates: IEsploraApi.Transaction[], accelerations: Acceleration[], maxBlocks: number = 8, weightLimit: number = BLOCK_WEIGHT_UNITS, sigopLimit: number = BLOCK_SIGOPS): TemplateTransaction[] {
const auditPool: Map<string, MinerTransaction> = new Map();
const mempoolArray: MinerTransaction[] = [];
candidates.forEach(tx => {
// initializing everything up front helps V8 optimize property access later
const adjustedVsize = Math.ceil(Math.max(tx.weight / 4, 5 * (tx.sigops || 0)));
const feePerVsize = (tx.fee / adjustedVsize);
auditPool.set(tx.txid, {
txid: tx.txid,
order: txidToOrdering(tx.txid),
fee: tx.fee,
feeDelta: 0,
weight: tx.weight,
adjustedVsize,
feePerVsize: feePerVsize,
effectiveFeePerVsize: feePerVsize,
dependencyRate: feePerVsize,
sigops: tx.sigops || 0,
inputs: (tx.vin?.map(vin => vin.txid) || []) as string[],
relativesSet: false,
ancestors: [],
cluster: [],
ancestorMap: new Map<string, MinerTransaction>(),
children: new Set<MinerTransaction>(),
ancestorFee: 0,
ancestorVsize: 0,
ancestorSigops: 0,
score: 0,
used: false,
modified: false,
});
mempoolArray.push(auditPool.get(tx.txid) as MinerTransaction);
});
// set accelerated effective fee
for (const acceleration of accelerations) {
const tx = auditPool.get(acceleration.txid);
if (tx) {
tx.feeDelta = acceleration.max_bid;
tx.feePerVsize = ((tx.fee + tx.feeDelta) / tx.adjustedVsize);
tx.effectiveFeePerVsize = tx.feePerVsize;
tx.dependencyRate = tx.feePerVsize;
}
}
// Build relatives graph & calculate ancestor scores
for (const tx of mempoolArray) {
if (!tx.relativesSet) {
setRelatives(tx, auditPool);
}
}
// Sort by descending ancestor score
mempoolArray.sort(priorityComparator);
// Build blocks by greedily choosing the highest feerate package
// (i.e. the package rooted in the transaction with the best ancestor score)
const blocks: number[][] = [];
let blockWeight = 0;
let blockSigops = 0;
const transactions: MinerTransaction[] = [];
let modified: MinerTransaction[] = [];
const overflow: MinerTransaction[] = [];
let failures = 0;
while (mempoolArray.length || modified.length) {
// skip invalid transactions
while (mempoolArray[0].used || mempoolArray[0].modified) {
mempoolArray.shift();
}
// Select best next package
let nextTx;
const nextPoolTx = mempoolArray[0];
const nextModifiedTx = modified[0];
if (nextPoolTx && (!nextModifiedTx || (nextPoolTx.score || 0) > (nextModifiedTx.score || 0))) {
nextTx = nextPoolTx;
mempoolArray.shift();
} else {
modified.shift();
if (nextModifiedTx) {
nextTx = nextModifiedTx;
}
}
if (nextTx && !nextTx?.used) {
// Check if the package fits into this block
if (blocks.length >= (maxBlocks - 1) || ((blockWeight + (4 * nextTx.ancestorVsize) < weightLimit) && (blockSigops + nextTx.ancestorSigops <= sigopLimit))) {
const ancestors: MinerTransaction[] = Array.from(nextTx.ancestorMap.values());
// sort ancestors by dependency graph (equivalent to sorting by ascending ancestor count)
const sortedTxSet = [...ancestors.sort((a, b) => { return (a.ancestorMap.size || 0) - (b.ancestorMap.size || 0); }), nextTx];
const clusterTxids = sortedTxSet.map(tx => tx.txid);
const effectiveFeeRate = Math.min(nextTx.dependencyRate || Infinity, nextTx.ancestorFee / nextTx.ancestorVsize);
const used: MinerTransaction[] = [];
while (sortedTxSet.length) {
const ancestor = sortedTxSet.pop();
if (!ancestor) {
continue;
}
ancestor.used = true;
ancestor.usedBy = nextTx.txid;
// update this tx with effective fee rate & relatives data
if (ancestor.effectiveFeePerVsize !== effectiveFeeRate) {
ancestor.effectiveFeePerVsize = effectiveFeeRate;
}
ancestor.cluster = clusterTxids;
transactions.push(ancestor);
blockWeight += ancestor.weight;
blockSigops += ancestor.sigops;
used.push(ancestor);
}
// remove these as valid package ancestors for any descendants remaining in the mempool
if (used.length) {
used.forEach(tx => {
modified = updateDescendants(tx, auditPool, modified, effectiveFeeRate);
});
}
failures = 0;
} else {
// hold this package in an overflow list while we check for smaller options
overflow.push(nextTx);
failures++;
}
}
// this block is full
const exceededPackageTries = failures > 1000 && blockWeight > (weightLimit - 4000);
const queueEmpty = !mempoolArray.length && !modified.length;
if (exceededPackageTries || queueEmpty) {
break;
}
}
for (const tx of transactions) {
tx.ancestors = Object.values(tx.ancestorMap);
}
return transactions;
}
// traverse in-mempool ancestors
// recursion unavoidable, but should be limited to depth < 25 by mempool policy
function setRelatives(
tx: MinerTransaction,
mempool: Map<string, MinerTransaction>,
): void {
for (const parent of tx.inputs) {
const parentTx = mempool.get(parent);
if (parentTx && !tx.ancestorMap?.has(parent)) {
tx.ancestorMap.set(parent, parentTx);
parentTx.children.add(tx);
// visit each node only once
if (!parentTx.relativesSet) {
setRelatives(parentTx, mempool);
}
parentTx.ancestorMap.forEach((ancestor) => {
tx.ancestorMap.set(ancestor.txid, ancestor);
});
}
};
tx.ancestorFee = (tx.fee + tx.feeDelta);
tx.ancestorVsize = tx.adjustedVsize || 0;
tx.ancestorSigops = tx.sigops || 0;
tx.ancestorMap.forEach((ancestor) => {
tx.ancestorFee += (ancestor.fee + ancestor.feeDelta);
tx.ancestorVsize += ancestor.adjustedVsize;
tx.ancestorSigops += ancestor.sigops;
});
tx.score = tx.ancestorFee / tx.ancestorVsize;
tx.relativesSet = true;
}
// iterate over remaining descendants, removing the root as a valid ancestor & updating the ancestor score
// avoids recursion to limit call stack depth
function updateDescendants(
rootTx: MinerTransaction,
mempool: Map<string, MinerTransaction>,
modified: MinerTransaction[],
clusterRate: number,
): MinerTransaction[] {
const descendantSet: Set<MinerTransaction> = new Set();
// stack of nodes left to visit
const descendants: MinerTransaction[] = [];
let descendantTx: MinerTransaction | undefined;
rootTx.children.forEach(childTx => {
if (!descendantSet.has(childTx)) {
descendants.push(childTx);
descendantSet.add(childTx);
}
});
while (descendants.length) {
descendantTx = descendants.pop();
if (descendantTx && descendantTx.ancestorMap && descendantTx.ancestorMap.has(rootTx.txid)) {
// remove tx as ancestor
descendantTx.ancestorMap.delete(rootTx.txid);
descendantTx.ancestorFee -= (rootTx.fee + rootTx.feeDelta);
descendantTx.ancestorVsize -= rootTx.adjustedVsize;
descendantTx.ancestorSigops -= rootTx.sigops;
descendantTx.score = descendantTx.ancestorFee / descendantTx.ancestorVsize;
descendantTx.dependencyRate = descendantTx.dependencyRate ? Math.min(descendantTx.dependencyRate, clusterRate) : clusterRate;
if (!descendantTx.modified) {
descendantTx.modified = true;
modified.push(descendantTx);
}
// add this node's children to the stack
descendantTx.children.forEach(childTx => {
// visit each node only once
if (!descendantSet.has(childTx)) {
descendants.push(childTx);
descendantSet.add(childTx);
}
});
}
}
// return new, resorted modified list
return modified.sort(priorityComparator);
}
// Used to sort an array of MinerTransactions by descending ancestor score
function priorityComparator(a: MinerTransaction, b: MinerTransaction): number {
if (b.score === a.score) {
// tie-break by txid for stability
return a.order - b.order;
} else {
return b.score - a.score;
}
}
// returns the most significant 4 bytes of the txid as an integer
function txidToOrdering(txid: string): number {
return parseInt(
txid.substring(62, 64) +
txid.substring(60, 62) +
txid.substring(58, 60) +
txid.substring(56, 58),
16
);
}

12
backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts
View File

@ -29,6 +29,7 @@ export interface AbstractBitcoinApi {
$getOutSpendsByOutpoint(outpoints: { txid: string, vout: number }[]): Promise<IEsploraApi.Outspend[]>;
startHealthChecks(): void;
getHealthStatus(): HealthCheckHost[];
}
export interface BitcoinRpcCredentials {
host: string;
@ -38,3 +39,14 @@ export interface BitcoinRpcCredentials {
timeout: number;
cookie?: string;
}
export interface HealthCheckHost {
host: string;
active: boolean;
rtt: number;
latestHeight: number;
socket: boolean;
outOfSync: boolean;
unreachable: boolean;
checked: boolean;
}

6
backend/src/api/bitcoin/bitcoin-api.ts
View File

@ -1,5 +1,5 @@
import * as bitcoinjs from 'bitcoinjs-lib';
import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
import { AbstractBitcoinApi, HealthCheckHost } from './bitcoin-api-abstract-factory';
import { IBitcoinApi } from './bitcoin-api.interface';
import { IEsploraApi } from './esplora-api.interface';
import blocks from '../blocks';
@ -382,6 +382,10 @@ class BitcoinApi implements AbstractBitcoinApi {
}
public startHealthChecks(): void {};
public getHealthStatus() {
return [];
}
}
export default BitcoinApi;

21
backend/src/api/bitcoin/esplora-api.ts
View File

@ -1,7 +1,7 @@
import config from '../../config';
import axios, { AxiosResponse, isAxiosError } from 'axios';
import http from 'http';
import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
import { AbstractBitcoinApi, HealthCheckHost } from './bitcoin-api-abstract-factory';
import { IEsploraApi } from './esplora-api.interface';
import logger from '../../logger';
import { Common } from '../common';
@ -164,7 +164,7 @@ class FailoverRouter {
}
// sort hosts by connection quality, and update default fallback
private sortHosts(): FailoverHost[] {
public sortHosts(): FailoverHost[] {
// sort by connection quality
return this.hosts.slice().sort((a, b) => {
if ((a.unreachable || a.outOfSync) === (b.unreachable || b.outOfSync)) {
@ -349,6 +349,23 @@ class ElectrsApi implements AbstractBitcoinApi {
public startHealthChecks(): void {
this.failoverRouter.startHealthChecks();
}
public getHealthStatus(): HealthCheckHost[] {
if (config.MEMPOOL.OFFICIAL) {
return this.failoverRouter.sortHosts().map(host => ({
host: host.host,
active: host === this.failoverRouter.activeHost,
rtt: host.rtt,
latestHeight: host.latestHeight || 0,
socket: !!host.socket,
outOfSync: !!host.outOfSync,
unreachable: !!host.unreachable,
checked: !!host.checked,
}));
} else {
return [];
}
}
}
export default ElectrsApi;

29
backend/src/api/database-migration.ts
View File

@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
import { RowDataPacket } from 'mysql2';
class DatabaseMigration {
private static currentVersion = 68;
private static currentVersion = 70;
private queryTimeout = 3600_000;
private statisticsAddedIndexed = false;
private uniqueLogs: string[] = [];
@ -580,6 +580,16 @@ class DatabaseMigration {
await this.$executeQuery(`INSERT INTO state VALUES('last_bitcoin_block_audit', 0, NULL);`);
await this.updateToSchemaVersion(68);
}
if (databaseSchemaVersion < 69 && config.MEMPOOL.NETWORK === 'mainnet') {
await this.$executeQuery(this.getCreateAccelerationsTableQuery(), await this.$checkIfTableExists('accelerations'));
await this.updateToSchemaVersion(69);
}
if (databaseSchemaVersion < 70 && config.MEMPOOL.NETWORK === 'mainnet') {
await this.$executeQuery('ALTER TABLE accelerations MODIFY COLUMN added DATETIME;');
await this.updateToSchemaVersion(70);
}
}
/**
@ -1123,6 +1133,23 @@ class DatabaseMigration {
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}
private getCreateAccelerationsTableQuery(): string {
return `CREATE TABLE IF NOT EXISTS accelerations (
txid varchar(65) NOT NULL,
added datetime NOT NULL,
height int(10) NOT NULL,
pool smallint unsigned NULL,
effective_vsize int(10) NOT NULL,
effective_fee bigint(20) unsigned NOT NULL,
boost_rate float unsigned,
boost_cost bigint(20) unsigned NOT NULL,
PRIMARY KEY (txid),
INDEX (added),
INDEX (height),
INDEX (pool)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}
public async $blocksReindexingTruncate(): Promise<void> {
logger.warn(`Truncating pools, blocks, hashrates and difficulty_adjustments tables for re-indexing (using '--reindex-blocks'). You can cancel this command within 5 seconds`);
await Common.sleep$(5000);

51
backend/src/api/mining/mining-routes.ts
View File

@ -8,6 +8,7 @@ import HashratesRepository from '../../repositories/HashratesRepository';
import bitcoinClient from '../bitcoin/bitcoin-client';
import mining from "./mining";
import PricesRepository from '../../repositories/PricesRepository';
import AccelerationRepository from '../../repositories/AccelerationRepository';
class MiningRoutes {
public initRoutes(app: Application) {
@ -34,6 +35,10 @@ class MiningRoutes {
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/:hash', this.$getBlockAudit)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/timestamp/:timestamp', this.$getHeightFromTimestamp)
.get(config.MEMPOOL.API_URL_PREFIX + 'historical-price', this.$getHistoricalPrice)
.get(config.MEMPOOL.API_URL_PREFIX + 'accelerations/pool/:slug', this.$getAccelerationsByPool)
.get(config.MEMPOOL.API_URL_PREFIX + 'accelerations/block/:height', this.$getAccelerationsByHeight)
.get(config.MEMPOOL.API_URL_PREFIX + 'accelerations/recent/:interval', this.$getRecentAccelerations)
;
}
@ -352,6 +357,52 @@ class MiningRoutes {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
private async $getAccelerationsByPool(req: Request, res: Response): Promise<void> {
try {
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid'].includes(config.MEMPOOL.NETWORK)) {
res.status(400).send('Acceleration data is not available.');
return;
}
res.status(200).send(await AccelerationRepository.$getAccelerationInfo(req.params.slug));
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
private async $getAccelerationsByHeight(req: Request, res: Response): Promise<void> {
try {
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24).toUTCString());
if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid'].includes(config.MEMPOOL.NETWORK)) {
res.status(400).send('Acceleration data is not available.');
return;
}
const height = req.params.height === undefined ? undefined : parseInt(req.params.height, 10);
res.status(200).send(await AccelerationRepository.$getAccelerationInfo(null, height));
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
private async $getRecentAccelerations(req: Request, res: Response): Promise<void> {
try {
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
if (!config.MEMPOOL_SERVICES.ACCELERATIONS || ['testnet', 'signet', 'liquidtestnet', 'liquid'].includes(config.MEMPOOL.NETWORK)) {
res.status(400).send('Acceleration data is not available.');
return;
}
res.status(200).send(await AccelerationRepository.$getAccelerationInfo(null, null, req.params.interval));
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
}
export default new MiningRoutes();

41
backend/src/api/websocket-handler.ts
View File

@ -24,6 +24,9 @@ import { ApiPrice } from '../repositories/PricesRepository';
import accelerationApi from './services/acceleration';
import mempool from './mempool';
import statistics from './statistics/statistics';
import accelerationCosts from './acceleration';
import accelerationRepository from '../repositories/AccelerationRepository';
import bitcoinApi from './bitcoin/bitcoin-api-factory';
interface AddressTransactions {
mempool: MempoolTransactionExtended[],
@ -37,6 +40,7 @@ const wantable = [
'mempool-blocks',
'live-2h-chart',
'stats',
'tomahawk',
];
class WebsocketHandler {
@ -121,7 +125,7 @@ class WebsocketHandler {
for (const sub of wantable) {
const key = `want-${sub}`;
const wants = parsedMessage.data.includes(sub);
if (wants && client['wants'] && !client[key]) {
if (wants && !client[key]) {
wantNow[key] = true;
}
client[key] = wants;
@ -145,6 +149,10 @@ class WebsocketHandler {
response['da'] = this.socketData['da'];
}
if (wantNow['want-tomahawk']) {
response['tomahawk'] = JSON.stringify(bitcoinApi.getHealthStatus());
}
if (parsedMessage && parsedMessage['track-tx']) {
if (/^[a-fA-F0-9]{64}$/.test(parsedMessage['track-tx'])) {
client['track-tx'] = parsedMessage['track-tx'];
@ -544,6 +552,10 @@ class WebsocketHandler {
response['mempool-blocks'] = getCachedResponse('mempool-blocks', mBlocks);
}
if (client['want-tomahawk']) {
response['tomahawk'] = getCachedResponse('tomahawk', bitcoinApi.getHealthStatus());
}
if (client['track-mempool-tx']) {
const tx = newTransactions.find((t) => t.txid === client['track-mempool-tx']);
if (tx) {
@ -728,6 +740,28 @@ class WebsocketHandler {
const _memPool = memPool.getMempool();
const isAccelerated = config.MEMPOOL_SERVICES.ACCELERATIONS && accelerationApi.isAcceleratedBlock(block, Object.values(mempool.getAccelerations()));
if (isAccelerated) {
const blockTxs: { [txid: string]: MempoolTransactionExtended } = {};
for (const tx of transactions) {
blockTxs[tx.txid] = tx;
}
const accelerations = Object.values(mempool.getAccelerations());
const boostRate = accelerationCosts.calculateBoostRate(
accelerations.map(acc => ({ txid: acc.txid, max_bid: acc.feeDelta })),
transactions
);
for (const acc of accelerations) {
if (blockTxs[acc.txid]) {
const tx = blockTxs[acc.txid];
const accelerationInfo = accelerationCosts.getAccelerationInfo(tx, boostRate, transactions);
accelerationRepository.$saveAcceleration(accelerationInfo, block, block.extras.pool.id);
}
}
}
const rbfTransactions = Common.findMinedRbfTransactions(transactions, memPool.getSpendMap());
memPool.handleMinedRbfTransactions(rbfTransactions);
memPool.removeFromSpendMap(transactions);
@ -735,7 +769,6 @@ class WebsocketHandler {
if (config.MEMPOOL.AUDIT && memPool.isInSync()) {
let projectedBlocks;
let auditMempool = _memPool;
const isAccelerated = config.MEMPOOL_SERVICES.ACCELERATIONS && accelerationApi.isAcceleratedBlock(block, Object.values(mempool.getAccelerations()));
// template calculation functions have mempool side effects, so calculate audits using
// a cloned copy of the mempool if we're running a different algorithm for mempool updates
const separateAudit = config.MEMPOOL.ADVANCED_GBT_AUDIT !== config.MEMPOOL.ADVANCED_GBT_MEMPOOL;
@ -886,6 +919,10 @@ class WebsocketHandler {
response['mempool-blocks'] = getCachedResponse('mempool-blocks', mBlocks);
}
if (client['want-tomahawk']) {
response['tomahawk'] = getCachedResponse('tomahawk', bitcoinApi.getHealthStatus());
}
if (client['track-tx']) {
const trackTxid = client['track-tx'];
if (trackTxid && confirmedTxids[trackTxid]) {

4
backend/src/config.ts
View File

@ -5,6 +5,7 @@ const configFromFile = require(
interface IConfig {
MEMPOOL: {
ENABLED: boolean;
OFFICIAL: boolean;
NETWORK: 'mainnet' | 'testnet' | 'signet' | 'liquid' | 'liquidtestnet';
BACKEND: 'esplora' | 'electrum' | 'none';
HTTP_PORT: number;
@ -103,6 +104,7 @@ interface IConfig {
PASSWORD: string;
TIMEOUT: number;
PID_DIR: string;
POOL_SIZE: number;
};
SYSLOG: {
ENABLED: boolean;
@ -161,6 +163,7 @@ interface IConfig {
const defaults: IConfig = {
'MEMPOOL': {
'ENABLED': true,
'OFFICIAL': false,
'NETWORK': 'mainnet',
'BACKEND': 'none',
'HTTP_PORT': 8999,
@ -240,6 +243,7 @@ const defaults: IConfig = {
'PASSWORD': 'mempool',
'TIMEOUT': 180000,
'PID_DIR': '',
'POOL_SIZE': 100,
},
'SYSLOG': {
'ENABLED': true,

2
backend/src/database.ts
View File

@ -21,7 +21,7 @@ import { execSync } from 'child_process';
database: config.DATABASE.DATABASE,
user: config.DATABASE.USERNAME,
password: config.DATABASE.PASSWORD,
connectionLimit: 10,
connectionLimit: config.DATABASE.POOL_SIZE,
supportBigNumbers: true,
timezone: '+00:00',
};

109
backend/src/repositories/AccelerationRepository.ts Normal file
View File

@ -0,0 +1,109 @@
import { AccelerationInfo } from '../api/acceleration';
import { ResultSetHeader, RowDataPacket } from 'mysql2';
import DB from '../database';
import logger from '../logger';
import { IEsploraApi } from '../api/bitcoin/esplora-api.interface';
import { Common } from '../api/common';
import config from '../config';
export interface PublicAcceleration {
txid: string,
height: number,
pool: {
id: number,
slug: string,
name: string,
},
effective_vsize: number,
effective_fee: number,
boost_rate: number,
boost_cost: number,
}
class AccelerationRepository {
public async $saveAcceleration(acceleration: AccelerationInfo, block: IEsploraApi.Block, pool_id: number): Promise<void> {
try {
await DB.query(`
INSERT INTO accelerations(txid, added, height, pool, effective_vsize, effective_fee, boost_rate, boost_cost)
VALUE (?, FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
added = FROM_UNIXTIME(?),
height = ?,
pool = ?,
effective_vsize = ?,
effective_fee = ?,
boost_rate = ?,
boost_cost = ?
`, [
acceleration.txSummary.txid,
block.timestamp,
block.height,
pool_id,
acceleration.txSummary.effectiveVsize,
acceleration.txSummary.effectiveFee,
acceleration.targetFeeRate, acceleration.cost,
block.timestamp,
block.height,
pool_id,
acceleration.txSummary.effectiveVsize,
acceleration.txSummary.effectiveFee,
acceleration.targetFeeRate, acceleration.cost,
]);
} catch (e: any) {
logger.err(`Cannot save acceleration (${acceleration.txSummary.txid}) into db. Reason: ` + (e instanceof Error ? e.message : e));
// We don't throw, not a critical issue if we miss some accelerations
}
}
public async $getAccelerationInfo(poolSlug: string | null = null, height: number | null = null, interval: string | null = null): Promise<PublicAcceleration[]> {
interval = Common.getSqlInterval(interval);
if (!config.MEMPOOL_SERVICES.ACCELERATIONS || (interval == null && poolSlug == null && height == null)) {
return [];
}
let query = `
SELECT * FROM accelerations
JOIN pools on pools.unique_id = accelerations.pool
`;
let params: any[] = [];
if (interval) {
query += ` WHERE accelerations.added BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW() `;
} else if (height != null) {
query += ` WHERE accelerations.height = ? `;
params.push(height);
} else if (poolSlug != null) {
query += ` WHERE pools.slug = ? `;
params.push(poolSlug);
}
query += ` ORDER BY accelerations.added DESC `;
try {
const [rows] = await DB.query(query, params) as RowDataPacket[][];
if (rows?.length) {
return rows.map(row => ({
txid: row.txid,
height: row.height,
pool: {
id: row.id,
slug: row.slug,
name: row.name,
},
effective_vsize: row.effective_vsize,
effective_fee: row.effective_fee,
boost_rate: row.boost_rate,
boost_cost: row.boost_cost,
}));
} else {
return [];
}
} catch (e) {
logger.err(`Cannot query acceleration info. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
}
export default new AccelerationRepository();

4
docker/backend/mempool-config.json
View File

@ -3,6 +3,7 @@
"NETWORK": "__MEMPOOL_NETWORK__",
"BACKEND": "__MEMPOOL_BACKEND__",
"ENABLED": __MEMPOOL_ENABLED__,
"OFFICIAL": __MEMPOOL_OFFICIAL__,
"HTTP_PORT": __MEMPOOL_HTTP_PORT__,
"SPAWN_CLUSTER_PROCS": __MEMPOOL_SPAWN_CLUSTER_PROCS__,
"API_URL_PREFIX": "__MEMPOOL_API_URL_PREFIX__",
@ -79,7 +80,8 @@
"USERNAME": "__DATABASE_USERNAME__",
"PASSWORD": "__DATABASE_PASSWORD__",
"TIMEOUT": __DATABASE_TIMEOUT__,
"PID_DIR": "__DATABASE_PID_DIR__"
"PID_DIR": "__DATABASE_PID_DIR__",
"POOL_SIZE": __DATABASE_POOL_SIZE__
},
"SYSLOG": {
"ENABLED": __SYSLOG_ENABLED__,

4
docker/backend/start.sh
View File

@ -4,6 +4,7 @@
__MEMPOOL_NETWORK__=${MEMPOOL_NETWORK:=mainnet}
__MEMPOOL_BACKEND__=${MEMPOOL_BACKEND:=electrum}
__MEMPOOL_ENABLED__=${MEMPOOL_ENABLED:=true}
__MEMPOOL_OFFICIAL__=${MEMPOOL_OFFICIAL:=false}
__MEMPOOL_HTTP_PORT__=${BACKEND_HTTP_PORT:=8999}
__MEMPOOL_SPAWN_CLUSTER_PROCS__=${MEMPOOL_SPAWN_CLUSTER_PROCS:=0}
__MEMPOOL_API_URL_PREFIX__=${MEMPOOL_API_URL_PREFIX:=/api/v1/}
@ -81,6 +82,7 @@ __DATABASE_USERNAME__=${DATABASE_USERNAME:=mempool}
__DATABASE_PASSWORD__=${DATABASE_PASSWORD:=mempool}
__DATABASE_TIMEOUT__=${DATABASE_TIMEOUT:=180000}
__DATABASE_PID_DIR__=${DATABASE_PID_DIR:=""}
__DATABASE_POOL_SIZE__=${DATABASE_POOL_SIZE:=100}
# SYSLOG
__SYSLOG_ENABLED__=${SYSLOG_ENABLED:=false}
@ -158,6 +160,7 @@ mkdir -p "${__MEMPOOL_CACHE_DIR__}"
sed -i "s!__MEMPOOL_NETWORK__!${__MEMPOOL_NETWORK__}!g" mempool-config.json
sed -i "s!__MEMPOOL_BACKEND__!${__MEMPOOL_BACKEND__}!g" mempool-config.json
sed -i "s!__MEMPOOL_ENABLED__!${__MEMPOOL_ENABLED__}!g" mempool-config.json
sed -i "s!__MEMPOOL_OFFICIAL__!${__MEMPOOL_OFFICIAL__}!g" mempool-config.json
sed -i "s!__MEMPOOL_HTTP_PORT__!${__MEMPOOL_HTTP_PORT__}!g" mempool-config.json
sed -i "s!__MEMPOOL_SPAWN_CLUSTER_PROCS__!${__MEMPOOL_SPAWN_CLUSTER_PROCS__}!g" mempool-config.json
sed -i "s!__MEMPOOL_API_URL_PREFIX__!${__MEMPOOL_API_URL_PREFIX__}!g" mempool-config.json
@ -230,6 +233,7 @@ sed -i "s!__DATABASE_USERNAME__!${__DATABASE_USERNAME__}!g" mempool-config.json
sed -i "s!__DATABASE_PASSWORD__!${__DATABASE_PASSWORD__}!g" mempool-config.json
sed -i "s!__DATABASE_TIMEOUT__!${__DATABASE_TIMEOUT__}!g" mempool-config.json
sed -i "s!__DATABASE_PID_DIR__!${__DATABASE_PID_DIR__}!g" mempool-config.json
sed -i "s!__DATABASE_POOL_SIZE__!${__DATABASE_POOL_SIZE__}!g" mempool-config.json
sed -i "s!__SYSLOG_ENABLED__!${__SYSLOG_ENABLED__}!g" mempool-config.json
sed -i "s!__SYSLOG_HOST__!${__SYSLOG_HOST__}!g" mempool-config.json

12
docker/init.sh
View File

@ -1,7 +1,7 @@
#!/bin/sh
#backend
cp ./docker/backend/* ./backend/
cp -r ./docker/backend/* ./backend/
#geoip-data
mkdir -p ./backend/GeoIP/
@ -13,8 +13,8 @@ localhostIP="127.0.0.1"
cp ./docker/frontend/* ./frontend
cp ./nginx.conf ./frontend/
cp ./nginx-mempool.conf ./frontend/
sed -i "s/${localhostIP}:80/0.0.0.0:__MEMPOOL_FRONTEND_HTTP_PORT__/g" ./frontend/nginx.conf
sed -i "s/${localhostIP}/0.0.0.0/g" ./frontend/nginx.conf
sed -i "s/user nobody;//g" ./frontend/nginx.conf
sed -i "s!/etc/nginx/nginx-mempool.conf!/etc/nginx/conf.d/nginx-mempool.conf!g" ./frontend/nginx.conf
sed -i "s/${localhostIP}:8999/__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__:__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__/g" ./frontend/nginx-mempool.conf
sed -i"" -e "s/${localhostIP}:80/0.0.0.0:__MEMPOOL_FRONTEND_HTTP_PORT__/g" ./frontend/nginx.conf
sed -i"" -e "s/${localhostIP}/0.0.0.0/g" ./frontend/nginx.conf
sed -i"" -e "s/user nobody;//g" ./frontend/nginx.conf
sed -i"" -e "s!/etc/nginx/nginx-mempool.conf!/etc/nginx/conf.d/nginx-mempool.conf!g" ./frontend/nginx.conf
sed -i"" -e "s/${localhostIP}:8999/__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__:__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__/g" ./frontend/nginx-mempool.conf

2
frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html
View File

@ -56,7 +56,7 @@
<app-time kind="since" [time]="block.timestamp" [fastRender]="true" [precision]="1" minUnit="minute"></app-time></div>
</ng-container>
</div>
<div class="animated" [class]="showMiningInfo ? 'show' : 'hide'" *ngIf="block.extras?.pool != undefined">
<div class="animated" [class]="markHeight === block.height ? 'hide' : 'show'" *ngIf="block.extras?.pool != undefined">
<a [attr.data-cy]="'bitcoin-block-' + offset + '-index-' + i + '-pool'" class="badge badge-primary"
[routerLink]="[('/mining/pool/' + block.extras.pool.slug) | relativeUrl]">
{{ block.extras.pool.name}}</a>

2
frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.scss
View File

@ -166,7 +166,7 @@
opacity: 1;
}
.hide {
opacity: 0;
opacity: 0.4;
pointer-events : none;
}

32
frontend/src/app/components/server-health/server-health.component.html Normal file
View File

@ -0,0 +1,32 @@
<div class="tomahawk container-xl dashboard-container">
<div class="links">
<span>Status</span>
<a [routerLink]='"/network"'>Live</a>
</div>
<h1 class="dashboard-title">Node Status</h1>
<ng-container *ngIf="(hosts$ | async) as hosts">
<div class="status-panel">
<table class="status-table table table-fixed table-borderless table-striped" *ngIf="(tip$ | async) as tip">
<tbody>
<tr>
<th class="rank"></th>
<th class="flag"></th>
<th class="host">Host</th>
<th class="rtt only-small">RTT</th>
<th class="rtt only-large">RTT</th>
<th class="height">Height</th>
</tr>
<tr *ngFor="let host of hosts; let i = index; trackBy: trackByFn">
<td class="rank">{{ i + 1 }}</td>
<td class="flag">{{ host.active ? '⭐️' : host.flag }}</td>
<td class="host">{{ host.link }}</td>
<td class="rtt only-small">{{ (host.rtt / 1000) | number : '1.1-1' }} {{ host.rtt == null ? '' : 's'}} {{ !host.checked ? '⏳' : (host.unreachable ? '🔥' : '✅') }}</td>
<td class="rtt only-large">{{ host.rtt | number : '1.0-0' }} {{ host.rtt == null ? '' : 'ms'}} {{ !host.checked ? '⏳' : (host.unreachable ? '🔥' : '✅') }}</td>
<td class="height">{{ host.latestHeight }} {{ !host.checked ? '⏳' : (host.outOfSync ? '🚫' : (host.latestHeight && host.latestHeight < tip ? '🟧' : '✅')) }}</td>
</tr>
</tbody>
</table>
</div>
</ng-container>
</div>

72
frontend/src/app/components/server-health/server-health.component.scss Normal file
View File

@ -0,0 +1,72 @@
.tomahawk {
.links {
float: right;
text-align: right;
margin-top: 1em;
a, span {
margin-left: 1em;
}
}
.dashboard-title {
text-align: left;
}
.status-panel {
max-width: 720px;
margin: auto;
margin-top: 2em;
padding: 1em;
background: #24273e;
}
.status-table {
width: 100%;
td, th {
padding: 0.25em;
&.rank, &.flag {
width: 28px;
text-align: right;
}
&.rtt, &.height {
width: 92px;
text-align: right;
}
&.only-small {
display: table-cell;
&.rtt {
width: 60px;
}
}
&.only-large {
display: none;
}
&.height {
padding-right: 0.5em;
}
&.host {
width: auto;
overflow: hidden;
text-overflow: ellipsis;
}
@media (min-width: 576px) {
&.rank, &.flag {
width: 32px;
}
&.rtt, &.height {
width: 96px;
}
&.only-small {
display: none;
}
&.only-large {
display: table-cell;
}
}
}
}
}

67
frontend/src/app/components/server-health/server-health.component.ts Normal file
View File

@ -0,0 +1,67 @@
import { Component, OnInit, ChangeDetectionStrategy, SecurityContext } from '@angular/core';
import { WebsocketService } from '../../services/websocket.service';
import { Observable, Subject, map } from 'rxjs';
import { StateService } from '../../services/state.service';
import { HealthCheckHost } from '../../interfaces/websocket.interface';
import { DomSanitizer } from '@angular/platform-browser';
@Component({
selector: 'app-server-health',
templateUrl: './server-health.component.html',
styleUrls: ['./server-health.component.scss'],
changeDetection: ChangeDetectionStrategy.OnPush,
})
export class ServerHealthComponent implements OnInit {
hosts$: Observable<HealthCheckHost[]>;
tip$: Subject<number>;
constructor(
private websocketService: WebsocketService,
private stateService: StateService,
public sanitizer: DomSanitizer,
) {}
ngOnInit(): void {
this.hosts$ = this.stateService.serverHealth$.pipe(
map((hosts) => {
const subpath = window.location.pathname.slice(0, -6);
for (const host of hosts) {
let statusUrl = '';
let linkHost = '';
if (host.socket) {
statusUrl = 'https://' + window.location.hostname + subpath + '/status';
linkHost = window.location.hostname + subpath;
} else {
const hostUrl = new URL(host.host);
statusUrl = 'https://' + hostUrl.hostname + subpath + '/status';
linkHost = hostUrl.hostname + subpath;
}
host.statusPage = this.sanitizer.bypassSecurityTrustResourceUrl(this.sanitizer.sanitize(SecurityContext.URL, statusUrl));
host.link = linkHost;
host.flag = this.parseFlag(host.host);
}
return hosts;
})
);
this.tip$ = this.stateService.chainTip$;
this.websocketService.want(['blocks', 'tomahawk']);
}
trackByFn(index: number, host: HealthCheckHost): string {
return host.host;
}
private parseFlag(host: string): string {
if (host.includes('.fra.')) {
return '🇩🇪';
} else if (host.includes('.tk7.')) {
return '🇯🇵';
} else if (host.includes('.fmt.')) {
return '🇺🇸';
} else if (host.includes('.va1.')) {
return '🇺🇸';
} else {
return '';
}
}
}

16
frontend/src/app/components/server-health/server-status.component.html Normal file
View File

@ -0,0 +1,16 @@
<div class="tomahawk">
<div class="container-xl dashboard-container">
<div class="links">
<a [routerLink]='"/nodes"'>Status</a>
<span>Live</span>
</div>
<h1 class="dashboard-title">Live Network</h1>
</div>
<ng-container *ngFor="let host of hosts; trackBy: trackByFn">
<h5 [id]="host.host" class="hostLink">
<a [href]="'https://' + host.link">{{ host.link }}</a>
</h5>
<iframe class="mempoolStatus" [src]="host.statusPage"></iframe>
</ng-container>
</div>

26
frontend/src/app/components/server-health/server-status.component.scss Normal file
View File

@ -0,0 +1,26 @@
.tomahawk {
.links {
float: right;
text-align: right;
margin-top: 1em;
a, span {
margin-left: 1em;
}
}
.dashboard-title {
text-align: left;
}
.mempoolStatus {
width: 100%;
height: 270px;
}
.hostLink {
text-align: center;
margin: auto;
margin-top: 1em;
}
}

80
frontend/src/app/components/server-health/server-status.component.ts Normal file
View File

@ -0,0 +1,80 @@
import { Component, OnInit, ChangeDetectionStrategy, SecurityContext, OnDestroy, ChangeDetectorRef } from '@angular/core';
import { WebsocketService } from '../../services/websocket.service';
import { Observable, Subject, Subscription, map, tap } from 'rxjs';
import { StateService } from '../../services/state.service';
import { HealthCheckHost } from '../../interfaces/websocket.interface';
import { DomSanitizer } from '@angular/platform-browser';
@Component({
selector: 'app-server-status',
templateUrl: './server-status.component.html',
styleUrls: ['./server-status.component.scss'],
changeDetection: ChangeDetectionStrategy.OnPush,
})
export class ServerStatusComponent implements OnInit, OnDestroy {
tip$: Subject<number>;
hosts: HealthCheckHost[] = [];
hostSubscription: Subscription;
constructor(
private websocketService: WebsocketService,
private stateService: StateService,
private cd: ChangeDetectorRef,
public sanitizer: DomSanitizer,
) {}
ngOnInit(): void {
this.hostSubscription = this.stateService.serverHealth$.pipe(
map((hosts) => {
const subpath = window.location.pathname.slice(0, -8);
for (const host of hosts) {
let statusUrl = '';
let linkHost = '';
if (host.socket) {
statusUrl = 'https://' + window.location.hostname + subpath + '/status';
linkHost = window.location.hostname + subpath;
} else {
const hostUrl = new URL(host.host);
statusUrl = 'https://' + hostUrl.hostname + subpath + '/status';
linkHost = hostUrl.hostname + subpath;
}
host.statusPage = this.sanitizer.bypassSecurityTrustResourceUrl(this.sanitizer.sanitize(SecurityContext.URL, statusUrl));
host.link = linkHost;
}
return hosts;
}),
tap((hosts) => {
if (this.hosts.length !== hosts.length) {
this.hosts = hosts.sort((a,b) => {
const aParts = (a.host?.split('.') || []).reverse();
const bParts = (b.host?.split('.') || []).reverse();
let i = 0;
while (i < Math.max(aParts.length, bParts.length)) {
if (aParts[i] && !bParts[i]) {
return 1;
} else if (bParts[i] && !aParts[i]) {
return -1;
} else if (aParts[i] !== bParts[i]) {
return aParts[i].localeCompare(bParts[i]);
}
i++;
}
return 0;
});
}
this.cd.markForCheck();
})
).subscribe();
this.tip$ = this.stateService.chainTip$;
this.websocketService.want(['blocks', 'tomahawk']);
}
trackByFn(index: number, host: HealthCheckHost): string {
return host.host;
}
ngOnDestroy(): void {
this.hosts = [];
this.hostSubscription.unsubscribe();
}
}

15
frontend/src/app/interfaces/websocket.interface.ts
View File

@ -1,3 +1,4 @@
import { SafeResourceUrl } from '@angular/platform-browser';
import { ILoadingIndicators } from '../services/state.service';
import { Transaction } from './electrs.interface';
import { BlockExtended, DifficultyAdjustment, RbfTree } from './node-api.interface';
@ -120,4 +121,18 @@ export interface Recommendedfees {
hourFee: number;
minimumFee: number;
economyFee: number;
}
export interface HealthCheckHost {
host: string;
active: boolean;
rtt: number;
latestHeight: number;
socket: boolean;
outOfSync: boolean;
unreachable: boolean;
checked: boolean;
link?: string;
statusPage?: SafeResourceUrl;
flag?: string;
}

15
frontend/src/app/liquid/liquid-master-page.module.ts
View File

@ -19,6 +19,8 @@ import { RecentPegsListComponent } from '../components/liquid-reserves-audit/rec
import { FederationWalletComponent } from '../components/liquid-reserves-audit/federation-wallet/federation-wallet.component';
import { FederationUtxosListComponent } from '../components/liquid-reserves-audit/federation-utxos-list/federation-utxos-list.component';
import { FederationAddressesListComponent } from '../components/liquid-reserves-audit/federation-addresses-list/federation-addresses-list.component';
import { ServerHealthComponent } from '../components/server-health/server-health.component';
import { ServerStatusComponent } from '../components/server-health/server-status.component';
const routes: Routes = [
{
@ -140,6 +142,19 @@ const routes: Routes = [
},
];
if (window['__env']?.OFFICIAL_MEMPOOL_SPACE) {
routes[0].children.push({
path: 'nodes',
data: { networks: ['bitcoin', 'liquid'] },
component: ServerHealthComponent
});
routes[0].children.push({
path: 'network',
data: { networks: ['bitcoin', 'liquid'] },
component: ServerStatusComponent
});
}
@NgModule({
imports: [
RouterModule.forChild(routes)

15
frontend/src/app/master-page.module.ts
View File

@ -10,6 +10,8 @@ import { PushTransactionComponent } from './components/push-transaction/push-tra
import { CalculatorComponent } from './components/calculator/calculator.component';
import { BlocksList } from './components/blocks-list/blocks-list.component';
import { RbfList } from './components/rbf-list/rbf-list.component';
import { ServerHealthComponent } from './components/server-health/server-health.component';
import { ServerStatusComponent } from './components/server-health/server-status.component';
const browserWindow = window || {};
// @ts-ignore
@ -96,6 +98,19 @@ const routes: Routes = [
}
];
if (window['__env']?.OFFICIAL_MEMPOOL_SPACE) {
routes[0].children.push({
path: 'nodes',
data: { networks: ['bitcoin', 'liquid'] },
component: ServerHealthComponent
});
routes[0].children.push({
path: 'network',
data: { networks: ['bitcoin', 'liquid'] },
component: ServerStatusComponent
});
}
@NgModule({
imports: [
RouterModule.forChild(routes)

4
frontend/src/app/services/state.service.ts
View File

@ -1,14 +1,13 @@
import { Inject, Injectable, PLATFORM_ID, LOCALE_ID } from '@angular/core';
import { ReplaySubject, BehaviorSubject, Subject, fromEvent, Observable, merge } from 'rxjs';
import { Transaction } from '../interfaces/electrs.interface';
import { IBackendInfo, MempoolBlock, MempoolBlockDelta, MempoolInfo, Recommendedfees, ReplacedTransaction, ReplacementInfo, TransactionCompressed, TransactionStripped } from '../interfaces/websocket.interface';
import { HealthCheckHost, IBackendInfo, MempoolBlock, MempoolBlockDelta, MempoolInfo, Recommendedfees, ReplacedTransaction, ReplacementInfo, TransactionStripped } from '../interfaces/websocket.interface';
import { BlockExtended, CpfpInfo, DifficultyAdjustment, MempoolPosition, OptimizedMempoolStats, RbfTree } from '../interfaces/node-api.interface';
import { Router, NavigationStart } from '@angular/router';
import { isPlatformBrowser } from '@angular/common';
import { filter, map, scan, shareReplay } from 'rxjs/operators';
import { StorageService } from './storage.service';
import { hasTouchScreen } from '../shared/pipes/bytes-pipe/utils';
import { ApiService } from './api.service';
import { ActiveFilter } from '../shared/filters.utils';
export interface MarkBlockState {
@ -129,6 +128,7 @@ export class StateService {
loadingIndicators$ = new ReplaySubject<ILoadingIndicators>(1);
recommendedFees$ = new ReplaySubject<Recommendedfees>(1);
chainTip$ = new ReplaySubject<number>(-1);
serverHealth$ = new Subject<HealthCheckHost[]>();
live2Chart$ = new Subject<OptimizedMempoolStats>();

4
frontend/src/app/services/websocket.service.ts
View File

@ -415,6 +415,10 @@ export class WebsocketService {
this.stateService.previousRetarget$.next(response.previousRetarget);
}
if (response['tomahawk']) {
this.stateService.serverHealth$.next(response['tomahawk']);
}
if (response['git-commit']) {
this.stateService.backendInfo$.next(response['git-commit']);
}

6
frontend/src/app/shared/shared.module.ts
View File

@ -53,6 +53,8 @@ import { AssetComponent } from '../components/asset/asset.component';
import { AssetsComponent } from '../components/assets/assets.component';
import { AssetsNavComponent } from '../components/assets/assets-nav/assets-nav.component';
import { StatusViewComponent } from '../components/status-view/status-view.component';
import { ServerHealthComponent } from '../components/server-health/server-health.component';
import { ServerStatusComponent } from '../components/server-health/server-status.component';
import { FeesBoxComponent } from '../components/fees-box/fees-box.component';
import { DifficultyComponent } from '../components/difficulty/difficulty.component';
import { DifficultyTooltipComponent } from '../components/difficulty/difficulty-tooltip.component';
@ -151,6 +153,8 @@ import { OnlyVsizeDirective, OnlyWeightDirective } from './components/weight-dir
AssetComponent,
AssetsComponent,
StatusViewComponent,
ServerHealthComponent,
ServerStatusComponent,
FeesBoxComponent,
DifficultyComponent,
DifficultyMiningComponent,
@ -277,6 +281,8 @@ import { OnlyVsizeDirective, OnlyWeightDirective } from './components/weight-dir
AssetComponent,
AssetsComponent,
StatusViewComponent,
ServerHealthComponent,
ServerStatusComponent,
FeesBoxComponent,
DifficultyComponent,
DifficultyMiningComponent,

23
production/check Executable file
View File

@ -0,0 +1,23 @@
#!/usr/bin/env zsh
check_frontend_git_commit_hash() {
echo -n $(curl -s https://node$1.$2.mempool.space/resources/config.js|grep GIT_COMMIT_HASH|cut -d "'" -f2|cut -c1-8)
}
check_html_md5_hash() {
echo -n $(curl -s https://node$1.$2.mempool.space|md5|cut -c1-8)
}
for site in fmt va1 fra tk7;do
echo "${site}"
for node in 201 202 203 204 205 206 207 208 209 210 211 212 213 214;do
[ "${site}" = "fmt" ] && [ "${node}" -gt 206 ] && continue
[ "${site}" = "tk7" ] && [ "${node}" -gt 206 ] && continue
echo -n "node${node}.${site}: "
check_frontend_git_commit_hash $node $site
echo -n " "
check_html_md5_hash $node $site
echo
done
done

1
production/mempool-config.bisq.json
View File

@ -1,5 +1,6 @@
{
"MEMPOOL": {
"OFFICIAL": true,
"NETWORK": "bisq",
"BACKEND": "esplora",
"HTTP_PORT": 8996,

1
production/mempool-config.liquid.json
View File

@ -1,5 +1,6 @@
{
"MEMPOOL": {
"OFFICIAL": true,
"NETWORK": "liquid",
"BACKEND": "esplora",
"HTTP_PORT": 8998,

1
production/mempool-config.liquidtestnet.json
View File

@ -1,5 +1,6 @@
{
"MEMPOOL": {
"OFFICIAL": true,
"NETWORK": "liquid",
"BACKEND": "esplora",
"HTTP_PORT": 8994,

1
production/mempool-config.mainnet-lightning.json
View File

@ -1,5 +1,6 @@
{
"MEMPOOL": {
"OFFICIAL": true,
"ENABLED": false,
"NETWORK": "mainnet",
"BACKEND": "esplora",

4
production/mempool-config.mainnet.json
View File

@ -1,5 +1,6 @@
{
"MEMPOOL": {
"OFFICIAL": true,
"NETWORK": "mainnet",
"BACKEND": "esplora",
"HTTP_PORT": 8999,
@ -21,7 +22,8 @@
"DISK_CACHE_BLOCK_INTERVAL": 1,
"MAX_PUSH_TX_SIZE_WEIGHT": 4000000,
"ALLOW_UNREACHABLE": true,
"PRICE_UPDATES_PER_HOUR": 12
"PRICE_UPDATES_PER_HOUR": 12,
"MAX_TRACKED_ADDRESSES": 10
},
"SYSLOG" : {
"MIN_PRIORITY": "debug"

1
production/mempool-config.signet-lightning.json
View File

@ -1,5 +1,6 @@
{
"MEMPOOL": {
"OFFICIAL": true,
"ENABLED": false,
"NETWORK": "signet",
"BACKEND": "esplora",

4
production/mempool-config.signet.json
View File

@ -1,5 +1,6 @@
{
"MEMPOOL": {
"OFFICIAL": true,
"NETWORK": "signet",
"BACKEND": "esplora",
"HTTP_PORT": 8995,
@ -14,7 +15,8 @@
"POLL_RATE_MS": 1000,
"DISK_CACHE_BLOCK_INTERVAL": 1,
"MAX_PUSH_TX_SIZE_WEIGHT": 4000000,
"ALLOW_UNREACHABLE": true
"ALLOW_UNREACHABLE": true,
"MAX_TRACKED_ADDRESSES": 10
},
"SYSLOG" : {
"MIN_PRIORITY": "debug"

1
production/mempool-config.testnet-lightning.json
View File

@ -1,5 +1,6 @@
{
"MEMPOOL": {
"OFFICIAL": true,
"ENABLED": false,
"NETWORK": "testnet",
"BACKEND": "esplora",

4
production/mempool-config.testnet.json
View File

@ -1,5 +1,6 @@
{
"MEMPOOL": {
"OFFICIAL": true,
"NETWORK": "testnet",
"BACKEND": "esplora",
"HTTP_PORT": 8997,
@ -14,7 +15,8 @@
"POLL_RATE_MS": 1000,
"DISK_CACHE_BLOCK_INTERVAL": 1,
"MAX_PUSH_TX_SIZE_WEIGHT": 4000000,
"ALLOW_UNREACHABLE": true
"ALLOW_UNREACHABLE": true,
"MAX_TRACKED_ADDRESSES": 10
},
"SYSLOG" : {
"MIN_PRIORITY": "debug"

8
production/nginx/http-proxy-cache.conf
View File

@ -8,5 +8,9 @@ proxy_cache_path /var/cache/nginx/apicold keys_zone=apicold:200m levels=1:2 inac
proxy_cache_path /var/cache/nginx/unfurler keys_zone=unfurler:200m levels=1:2 inactive=30d max_size=2000m;
proxy_cache_path /var/cache/nginx/slurper keys_zone=slurper:500m levels=1:2 inactive=365d max_size=5000m;
proxy_cache_path /var/cache/nginx/markets keys_zone=markets:20m levels=1:2 inactive=365d max_size=100m;
types_hash_max_size 4096;
proxy_buffer_size 8k;
types_hash_max_size 8192;
proxy_busy_buffers_size 256k;
proxy_buffer_size 128k;
proxy_buffers 4 256k;

25
scripts/get_block_tip_height.sh Normal file
View File

@ -0,0 +1,25 @@
BASE_HEIGHT=$(curl -sk https://node202.tk7.mempool.space/api/v1/blocks/tip/height)
IN_SYNC=true
echo "Base height (node202.tk7): $BASE_HEIGHT"
for LOCATION in fmt va1 fra tk7
do
for NODE in 201 202 203 204 205 206
do
NODE_HEIGHT=$(curl -sk https://node$NODE.$LOCATION.mempool.space/api/v1/blocks/tip/height)
echo $(echo node$NODE.$LOCATION.mempool.space) - $NODE_HEIGHT
if [ "$NODE_HEIGHT" -ne "$BASE_HEIGHT" ]; then
COUNT=$((BASE_HEIGHT-NODE_HEIGHT))
echo $(echo node$NODE.$LOCATION.mempool.space) is not in sync. delta: $COUNT
IN_SYNC=false
fi
done
done
if [ "$IN_SYNC" = false ]; then
echo "One or more servers are out of sync. Check the logs."
exit -1
else
echo "All servers are in sync."
fi