machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into docker-bugfix

Browse Source
This commit is contained in:
James Blacklock
2024-01-30 15:50:03 -05:00
committed by GitHub
parent 5f1180118a e4d92c8fe0
commit 770b6dfd19
168 changed files with 27219 additions and 10818 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
backend/mempool-config.sample.json
View File

@@ -16,6 +16,7 @@
"MEMPOOL_BLOCKS_AMOUNT": 8,
"INDEXING_BLOCKS_AMOUNT": 11000,
"BLOCKS_SUMMARIES_INDEXING": false,
"GOGGLES_INDEXING": false,
"USE_SECOND_NODE_FOR_MINFEE": false,
"EXTERNAL_ASSETS": [],
"EXTERNAL_MAX_RETRY": 1,
@@ -33,7 +34,8 @@
"DISK_CACHE_BLOCK_INTERVAL": 6,
"MAX_PUSH_TX_SIZE_WEIGHT": 4000000,
"ALLOW_UNREACHABLE": true,
"PRICE_UPDATES_PER_HOUR": 1
"PRICE_UPDATES_PER_HOUR": 1,
"MAX_TRACKED_ADDRESSES": 100
},
"CORE_RPC": {
"HOST": "127.0.0.1",

20
backend/package-lock.json generated
View File

@@ -17,7 +17,7 @@
"crypto-js": "~4.2.0",
"express": "~4.18.2",
"maxmind": "~4.3.11",
"mysql2": "~3.6.0",
"mysql2": "~3.7.0",
"redis": "^4.6.6",
"rust-gbt": "file:./rust-gbt",
"socks-proxy-agent": "~7.0.0",
@@ -6110,9 +6110,9 @@
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
},
"node_modules/mysql2": {
"version": "3.6.0",
"resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.6.0.tgz",
"integrity": "sha512-EWUGAhv6SphezurlfI2Fpt0uJEWLmirrtQR7SkbTHFC+4/mJBrPiSzHESHKAWKG7ALVD6xaG/NBjjd1DGJGQQQ==",
"version": "3.7.0",
"resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.7.0.tgz",
"integrity": "sha512-c45jA3Jc1X8yJKzrWu1GpplBKGwv/wIV6ITZTlCSY7npF2YfJR+6nMP5e+NTQhUeJPSyOQAbGDCGEHbAl8HN9w==",
"dependencies": {
"denque": "^2.1.0",
"generate-function": "^2.3.1",
@@ -7667,10 +7667,10 @@
},
"rust-gbt": {
"name": "gbt",
"version": "3.0.0-dev",
"version": "3.0.1",
"hasInstallScript": true,
"dependencies": {
"@napi-rs/cli": "^2.16.1"
"@napi-rs/cli": "2.16.1"
},
"engines": {
"node": ">= 12"
@@ -12230,9 +12230,9 @@
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
},
"mysql2": {
"version": "3.6.0",
"resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.6.0.tgz",
"integrity": "sha512-EWUGAhv6SphezurlfI2Fpt0uJEWLmirrtQR7SkbTHFC+4/mJBrPiSzHESHKAWKG7ALVD6xaG/NBjjd1DGJGQQQ==",
"version": "3.7.0",
"resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.7.0.tgz",
"integrity": "sha512-c45jA3Jc1X8yJKzrWu1GpplBKGwv/wIV6ITZTlCSY7npF2YfJR+6nMP5e+NTQhUeJPSyOQAbGDCGEHbAl8HN9w==",
"requires": {
"denque": "^2.1.0",
"generate-function": "^2.3.1",
@@ -12703,7 +12703,7 @@
"rust-gbt": {
"version": "file:rust-gbt",
"requires": {
"@napi-rs/cli": "^2.16.1"
"@napi-rs/cli": "2.16.1"
}
},
"safe-buffer": {

5
backend/package.json
View File

@@ -35,7 +35,8 @@
"lint": "./node_modules/.bin/eslint . --ext .ts",
"lint:fix": "./node_modules/.bin/eslint . --ext .ts --fix",
"prettier": "./node_modules/.bin/prettier --write \"src/**/*.{js,ts}\"",
"rust-build": "cd rust-gbt && npm run build-release"
"rust-clean": "cd rust-gbt && rm -f *.node index.d.ts index.js && rm -rf target && cd ../",
"rust-build": "npm run rust-clean && cd rust-gbt && npm run build-release"
},
"dependencies": {
"@babel/core": "^7.23.2",
@@ -46,7 +47,7 @@
"crypto-js": "~4.2.0",
"express": "~4.18.2",
"maxmind": "~4.3.11",
"mysql2": "~3.6.0",
"mysql2": "~3.7.0",
"rust-gbt": "file:./rust-gbt",
"redis": "^4.6.6",
"socks-proxy-agent": "~7.0.0",

4
backend/rust-gbt/Cargo.toml
View File

@@ -1,7 +1,7 @@
[package]
name = "gbt"
version = "0.1.0"
description = "An inefficient re-implementation of the getBlockTemplate algorithm in Rust"
version = "1.0.0"
description = "An efficient re-implementation of the getBlockTemplate algorithm in Rust"
authors = ["mononaut"]
edition = "2021"
publish = false

3
backend/rust-gbt/index.d.ts vendored
View File

@@ -45,5 +45,6 @@ export class GbtResult {
blockWeights: Array<number>
clusters: Array<Array<number>>
rates: Array<Array<number>>
constructor(blocks: Array<Array<number>>, blockWeights: Array<number>, clusters: Array<Array<number>>, rates: Array<Array<number>>)
overflow: Array<number>
constructor(blocks: Array<Array<number>>, blockWeights: Array<number>, clusters: Array<Array<number>>, rates: Array<Array<number>>, overflow: Array<number>)
}

6
backend/rust-gbt/package-lock.json generated
View File

@@ -1,15 +1,15 @@
{
"name": "gbt",
"version": "3.0.0-dev",
"version": "3.0.1",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "gbt",
"version": "3.0.0-dev",
"version": "3.0.1",
"hasInstallScript": true,
"dependencies": {
"@napi-rs/cli": "^2.16.1"
"@napi-rs/cli": "2.16.1"
},
"engines": {
"node": ">= 12"

6
backend/rust-gbt/package.json
View File

@@ -1,7 +1,7 @@
{
"name": "gbt",
"version": "3.0.0-dev",
"description": "An inefficient re-implementation of the getBlockTemplate algorithm in Rust",
"version": "3.0.1",
"description": "An efficient re-implementation of the getBlockTemplate algorithm in Rust",
"main": "index.js",
"types": "index.d.ts",
"scripts": {
@@ -25,7 +25,7 @@
}
},
"dependencies": {
"@napi-rs/cli": "^2.16.1"
"@napi-rs/cli": "2.16.1"
},
"engines": {
"node": ">= 12"

139
backend/rust-gbt/src/gbt.rs
View File

@@ -60,6 +60,7 @@ pub fn gbt(mempool: &mut ThreadTransactionsMap, accelerations: &[ThreadAccelerat
indexed_accelerations[acceleration.uid as usize] = Some(acceleration);
}
info!("Initializing working vecs with uid capacity for {}", max_uid + 1);
let mempool_len = mempool.len();
let mut audit_pool: AuditPool = Vec::with_capacity(max_uid + 1);
audit_pool.resize(max_uid + 1, None);
@@ -127,74 +128,75 @@ pub fn gbt(mempool: &mut ThreadTransactionsMap, accelerations: &[ThreadAccelerat
let next_from_stack = next_valid_from_stack(&mut mempool_stack, &audit_pool);
let next_from_queue = next_valid_from_queue(&mut modified, &audit_pool);
if next_from_stack.is_none() && next_from_queue.is_none() {
continue;
}
let (next_tx, from_stack) = match (next_from_stack, next_from_queue) {
(Some(stack_tx), Some(queue_tx)) => match queue_tx.cmp(stack_tx) {
std::cmp::Ordering::Less => (stack_tx, true),
_ => (queue_tx, false),
},
(Some(stack_tx), None) => (stack_tx, true),
(None, Some(queue_tx)) => (queue_tx, false),
(None, None) => unreachable!(),
};
if from_stack {
mempool_stack.pop();
info!("No transactions left! {:#?} in overflow", overflow.len());
} else {
modified.pop();
}
let (next_tx, from_stack) = match (next_from_stack, next_from_queue) {
(Some(stack_tx), Some(queue_tx)) => match queue_tx.cmp(stack_tx) {
std::cmp::Ordering::Less => (stack_tx, true),
_ => (queue_tx, false),
},
(Some(stack_tx), None) => (stack_tx, true),
(None, Some(queue_tx)) => (queue_tx, false),
(None, None) => unreachable!(),
};
if blocks.len() < (MAX_BLOCKS - 1)
&& ((block_weight + (4 * next_tx.ancestor_sigop_adjusted_vsize())
>= MAX_BLOCK_WEIGHT_UNITS)
|| (block_sigops + next_tx.ancestor_sigops() > BLOCK_SIGOPS))
{
// hold this package in an overflow list while we check for smaller options
overflow.push(next_tx.uid);
failures += 1;
} else {
let mut package: Vec<(u32, u32, usize)> = Vec::new();
let mut cluster: Vec<u32> = Vec::new();
let is_cluster: bool = !next_tx.ancestors.is_empty();
for ancestor_id in &next_tx.ancestors {
if let Some(Some(ancestor)) = audit_pool.get(*ancestor_id as usize) {
package.push((*ancestor_id, ancestor.order(), ancestor.ancestors.len()));
}
}
package.sort_unstable_by(|a, b| -> Ordering {
if a.2 != b.2 {
// order by ascending ancestor count
a.2.cmp(&b.2)
} else if a.1 != b.1 {
// tie-break by ascending partial txid
a.1.cmp(&b.1)
} else {
// tie-break partial txid collisions by ascending uid
a.0.cmp(&b.0)
}
});
package.push((next_tx.uid, next_tx.order(), next_tx.ancestors.len()));
let cluster_rate = next_tx.cluster_rate();
for (txid, _, _) in &package {
cluster.push(*txid);
if let Some(Some(tx)) = audit_pool.get_mut(*txid as usize) {
tx.used = true;
tx.set_dirty_if_different(cluster_rate);
transactions.push(tx.uid);
block_weight += tx.weight;
block_sigops += tx.sigops;
}
update_descendants(*txid, &mut audit_pool, &mut modified, cluster_rate);
if from_stack {
mempool_stack.pop();
} else {
modified.pop();
}
if is_cluster {
clusters.push(cluster);
}
if blocks.len() < (MAX_BLOCKS - 1)
&& ((block_weight + (4 * next_tx.ancestor_sigop_adjusted_vsize())
>= MAX_BLOCK_WEIGHT_UNITS)
|| (block_sigops + next_tx.ancestor_sigops() > BLOCK_SIGOPS))
{
// hold this package in an overflow list while we check for smaller options
overflow.push(next_tx.uid);
failures += 1;
} else {
let mut package: Vec<(u32, u32, usize)> = Vec::new();
let mut cluster: Vec<u32> = Vec::new();
let is_cluster: bool = !next_tx.ancestors.is_empty();
for ancestor_id in &next_tx.ancestors {
if let Some(Some(ancestor)) = audit_pool.get(*ancestor_id as usize) {
package.push((*ancestor_id, ancestor.order(), ancestor.ancestors.len()));
}
}
package.sort_unstable_by(|a, b| -> Ordering {
if a.2 != b.2 {
// order by ascending ancestor count
a.2.cmp(&b.2)
} else if a.1 != b.1 {
// tie-break by ascending partial txid
a.1.cmp(&b.1)
} else {
// tie-break partial txid collisions by ascending uid
a.0.cmp(&b.0)
}
});
package.push((next_tx.uid, next_tx.order(), next_tx.ancestors.len()));
failures = 0;
let cluster_rate = next_tx.cluster_rate();
for (txid, _, _) in &package {
cluster.push(*txid);
if let Some(Some(tx)) = audit_pool.get_mut(*txid as usize) {
tx.used = true;
tx.set_dirty_if_different(cluster_rate);
transactions.push(tx.uid);
block_weight += tx.weight;
block_sigops += tx.sigops;
}
update_descendants(*txid, &mut audit_pool, &mut modified, cluster_rate);
}
if is_cluster {
clusters.push(cluster);
}
failures = 0;
}
}
// this block is full
@@ -203,10 +205,14 @@ pub fn gbt(mempool: &mut ThreadTransactionsMap, accelerations: &[ThreadAccelerat
let queue_is_empty = mempool_stack.is_empty() && modified.is_empty();
if (exceeded_package_tries || queue_is_empty) && blocks.len() < (MAX_BLOCKS - 1) {
// finalize this block
if !transactions.is_empty() {
blocks.push(transactions);
block_weights.push(block_weight);
if transactions.is_empty() {
info!("trying to push an empty block! breaking loop! mempool {:#?} | modified {:#?} | overflow {:#?}", mempool_stack.len(), modified.len(), overflow.len());
break;
}
blocks.push(transactions);
block_weights.push(block_weight);
// reset for the next block
transactions = Vec::with_capacity(initial_txes_per_block);
block_weight = BLOCK_RESERVED_WEIGHT;
@@ -265,6 +271,7 @@ pub fn gbt(mempool: &mut ThreadTransactionsMap, accelerations: &[ThreadAccelerat
block_weights,
clusters,
rates,
overflow,
}
}

1
backend/rust-gbt/src/lib.rs
View File

@@ -133,6 +133,7 @@ pub struct GbtResult {
pub block_weights: Vec<u32>,
pub clusters: Vec<Vec<u32>>,
pub rates: Vec<Vec<f64>>, // Tuples not supported. u32 fits inside f64
pub overflow: Vec<u32>,
}
/// All on another thread, this runs an arbitrary task in between

4
backend/src/__fixtures__/mempool-config.template.json
View File

@@ -4,6 +4,7 @@
"NETWORK": "__MEMPOOL_NETWORK__",
"BACKEND": "__MEMPOOL_BACKEND__",
"BLOCKS_SUMMARIES_INDEXING": true,
"GOGGLES_INDEXING": false,
"HTTP_PORT": 1,
"SPAWN_CLUSTER_PROCS": 2,
"API_URL_PREFIX": "__MEMPOOL_API_URL_PREFIX__",
@@ -34,7 +35,8 @@
"DISK_CACHE_BLOCK_INTERVAL": 999,
"MAX_PUSH_TX_SIZE_WEIGHT": 4000000,
"ALLOW_UNREACHABLE": true,
"PRICE_UPDATES_PER_HOUR": 1
"PRICE_UPDATES_PER_HOUR": 1,
"MAX_TRACKED_ADDRESSES": 1
},
"CORE_RPC": {
"HOST": "__CORE_RPC_HOST__",

2
backend/src/__tests__/config.test.ts
View File

@@ -17,6 +17,7 @@ describe('Mempool Backend Config', () => {
NETWORK: 'mainnet',
BACKEND: 'none',
BLOCKS_SUMMARIES_INDEXING: false,
GOGGLES_INDEXING: false,
HTTP_PORT: 8999,
SPAWN_CLUSTER_PROCS: 0,
API_URL_PREFIX: '/api/v1/',
@@ -48,6 +49,7 @@ describe('Mempool Backend Config', () => {
MAX_PUSH_TX_SIZE_WEIGHT: 400000,
ALLOW_UNREACHABLE: true,
PRICE_UPDATES_PER_HOUR: 1,
MAX_TRACKED_ADDRESSES: 1,
});
expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true });

1
backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts
View File

@@ -1,3 +1,4 @@
import { IBitcoinApi } from './bitcoin-api.interface';
import { IEsploraApi } from './esplora-api.interface';
export interface AbstractBitcoinApi {

249
backend/src/api/bitcoin/bitcoin-core.routes.ts Normal file
View File

@@ -0,0 +1,249 @@
import { Application, NextFunction, Request, Response } from 'express';
import logger from '../../logger';
import bitcoinClient from './bitcoin-client';
/**
* Define a set of routes used by the accelerator server
* Those routes are not designed to be public
*/
class BitcoinBackendRoutes {
private static tag = 'BitcoinBackendRoutes';
public initRoutes(app: Application) {
app
.get('/api/internal/bitcoin-core/' + 'get-mempool-entry', this.disableCache, this.$getMempoolEntry)
.post('/api/internal/bitcoin-core/' + 'decode-raw-transaction', this.disableCache, this.$decodeRawTransaction)
.get('/api/internal/bitcoin-core/' + 'get-raw-transaction', this.disableCache, this.$getRawTransaction)
.post('/api/internal/bitcoin-core/' + 'send-raw-transaction', this.disableCache, this.$sendRawTransaction)
.post('/api/internal/bitcoin-core/' + 'test-mempool-accept', this.disableCache, this.$testMempoolAccept)
.get('/api/internal/bitcoin-core/' + 'get-mempool-ancestors', this.disableCache, this.$getMempoolAncestors)
.get('/api/internal/bitcoin-core/' + 'get-block', this.disableCache, this.$getBlock)
.get('/api/internal/bitcoin-core/' + 'get-block-hash', this.disableCache, this.$getBlockHash)
.get('/api/internal/bitcoin-core/' + 'get-block-count', this.disableCache, this.$getBlockCount)
;
}
/**
* Disable caching for bitcoin core routes
*
* @param req
* @param res
* @param next
*/
private disableCache(req: Request, res: Response, next: NextFunction): void {
res.setHeader('Pragma', 'no-cache');
res.setHeader('Cache-control', 'private, no-store, no-cache, must-revalidate, proxy-revalidate, max-age=0');
res.setHeader('expires', -1);
next();
}
/**
* Exeption handler to return proper details to the accelerator server
*
* @param e
* @param fnName
* @param res
*/
private static handleException(e: any, fnName: string, res: Response): void {
if (typeof(e.code) === 'number') {
res.status(400).send(JSON.stringify(e, ['code', 'message']));
} else {
const err = `exception in ${fnName}. ${e}. Details: ${JSON.stringify(e, ['code', 'message'])}`;
logger.err(err, BitcoinBackendRoutes.tag);
res.status(500).send(err);
}
}
private async $getMempoolEntry(req: Request, res: Response): Promise<void> {
const txid = req.query.txid;
try {
if (typeof(txid) !== 'string' || txid.length !== 64) {
res.status(400).send(`invalid param txid ${txid}. must be a string of 64 char`);
return;
}
const mempoolEntry = await bitcoinClient.getMempoolEntry(txid);
if (!mempoolEntry) {
res.status(404).send(`no mempool entry found for txid ${txid}`);
return;
}
res.status(200).send(mempoolEntry);
} catch (e: any) {
BitcoinBackendRoutes.handleException(e, 'getMempoolEntry', res);
}
}
private async $decodeRawTransaction(req: Request, res: Response): Promise<void> {
const rawTx = req.body.rawTx;
try {
if (typeof(rawTx) !== 'string') {
res.status(400).send(`invalid param rawTx ${rawTx}. must be a string`);
return;
}
const decodedTx = await bitcoinClient.decodeRawTransaction(rawTx);
if (!decodedTx) {
res.status(400).send(`unable to decode rawTx ${rawTx}`);
return;
}
res.status(200).send(decodedTx);
} catch (e: any) {
BitcoinBackendRoutes.handleException(e, 'decodeRawTransaction', res);
}
}
private async $getRawTransaction(req: Request, res: Response): Promise<void> {
const txid = req.query.txid;
const verbose = req.query.verbose;
try {
if (typeof(txid) !== 'string' || txid.length !== 64) {
res.status(400).send(`invalid param txid ${txid}. must be a string of 64 char`);
return;
}
if (typeof(verbose) !== 'string') {
res.status(400).send(`invalid param verbose ${verbose}. must be a string representing an integer`);
return;
}
const verboseNumber = parseInt(verbose, 10);
if (typeof(verboseNumber) !== 'number') {
res.status(400).send(`invalid param verbose ${verbose}. must be a valid integer`);
return;
}
const decodedTx = await bitcoinClient.getRawTransaction(txid, verboseNumber);
if (!decodedTx) {
res.status(400).send(`unable to get raw transaction for txid ${txid}`);
return;
}
res.status(200).send(decodedTx);
} catch (e: any) {
BitcoinBackendRoutes.handleException(e, 'decodeRawTransaction', res);
}
}
private async $sendRawTransaction(req: Request, res: Response): Promise<void> {
const rawTx = req.body.rawTx;
try {
if (typeof(rawTx) !== 'string') {
res.status(400).send(`invalid param rawTx ${rawTx}. must be a string`);
return;
}
const txHex = await bitcoinClient.sendRawTransaction(rawTx);
if (!txHex) {
res.status(400).send(`unable to send rawTx ${rawTx}`);
return;
}
res.status(200).send(txHex);
} catch (e: any) {
BitcoinBackendRoutes.handleException(e, 'sendRawTransaction', res);
}
}
private async $testMempoolAccept(req: Request, res: Response): Promise<void> {
const rawTxs = req.body.rawTxs;
try {
if (typeof(rawTxs) !== 'object') {
res.status(400).send(`invalid param rawTxs ${JSON.stringify(rawTxs)}. must be an array of string`);
return;
}
const txHex = await bitcoinClient.testMempoolAccept(rawTxs);
if (typeof(txHex) !== 'object' || txHex.length === 0) {
res.status(400).send(`testmempoolaccept failed for raw txs ${JSON.stringify(rawTxs)}, got an empty result`);
return;
}
res.status(200).send(txHex);
} catch (e: any) {
BitcoinBackendRoutes.handleException(e, 'testMempoolAccept', res);
}
}
private async $getMempoolAncestors(req: Request, res: Response): Promise<void> {
const txid = req.query.txid;
const verbose = req.query.verbose;
try {
if (typeof(txid) !== 'string' || txid.length !== 64) {
res.status(400).send(`invalid param txid ${txid}. must be a string of 64 char`);
return;
}
if (typeof(verbose) !== 'string' || (verbose !== 'true' && verbose !== 'false')) {
res.status(400).send(`invalid param verbose ${verbose}. must be a string ('true' | 'false')`);
return;
}
const ancestors = await bitcoinClient.getMempoolAncestors(txid, verbose === 'true' ? true : false);
if (!ancestors) {
res.status(400).send(`unable to get mempool ancestors for txid ${txid}`);
return;
}
res.status(200).send(ancestors);
} catch (e: any) {
BitcoinBackendRoutes.handleException(e, 'getMempoolAncestors', res);
}
}
private async $getBlock(req: Request, res: Response): Promise<void> {
const blockHash = req.query.hash;
const verbosity = req.query.verbosity;
try {
if (typeof(blockHash) !== 'string' || blockHash.length !== 64) {
res.status(400).send(`invalid param blockHash ${blockHash}. must be a string of 64 char`);
return;
}
if (typeof(verbosity) !== 'string') {
res.status(400).send(`invalid param verbosity ${verbosity}. must be a string representing an integer`);
return;
}
const verbosityNumber = parseInt(verbosity, 10);
if (typeof(verbosityNumber) !== 'number') {
res.status(400).send(`invalid param verbosity ${verbosity}. must be a valid integer`);
return;
}
const block = await bitcoinClient.getBlock(blockHash, verbosityNumber);
if (!block) {
res.status(400).send(`unable to get block for block hash ${blockHash}`);
return;
}
res.status(200).send(block);
} catch (e: any) {
BitcoinBackendRoutes.handleException(e, 'getBlock', res);
}
}
private async $getBlockHash(req: Request, res: Response): Promise<void> {
const blockHeight = req.query.height;
try {
if (typeof(blockHeight) !== 'string') {
res.status(400).send(`invalid param blockHeight ${blockHeight}, must be a string representing an integer`);
return;
}
const blockHeightNumber = parseInt(blockHeight, 10);
if (typeof(blockHeightNumber) !== 'number') {
res.status(400).send(`invalid param blockHeight ${blockHeight}. must be a valid integer`);
return;
}
const block = await bitcoinClient.getBlockHash(blockHeightNumber);
if (!block) {
res.status(400).send(`unable to get block hash for block height ${blockHeightNumber}`);
return;
}
res.status(200).send(block);
} catch (e: any) {
BitcoinBackendRoutes.handleException(e, 'getBlockHash', res);
}
}
private async $getBlockCount(req: Request, res: Response): Promise<void> {
try {
const count = await bitcoinClient.getBlockCount();
if (!count) {
res.status(400).send(`unable to get block count`);
return;
}
res.status(200).send(`${count}`);
} catch (e: any) {
BitcoinBackendRoutes.handleException(e, 'getBlockCount', res);
}
}
}
export default new BitcoinBackendRoutes

138
backend/src/api/blocks.ts
View File

@@ -2,7 +2,7 @@ import config from '../config';
import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory';
import logger from '../logger';
import memPool from './mempool';
import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionStripped, TransactionMinerInfo, CpfpSummary, MempoolTransactionExtended } from '../mempool.interfaces';
import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionMinerInfo, CpfpSummary, MempoolTransactionExtended, TransactionClassified } from '../mempool.interfaces';
import { Common } from './common';
import diskCache from './disk-cache';
import transactionUtils from './transaction-utils';
@@ -201,7 +201,8 @@ class Blocks {
txid: tx.txid,
vsize: tx.weight / 4,
fee: tx.fee ? Math.round(tx.fee * 100000000) : 0,
value: Math.round(tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0) * 100000000)
value: Math.round(tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0) * 100000000),
flags: 0,
};
});
@@ -214,7 +215,7 @@ class Blocks {
public summarizeBlockTransactions(hash: string, transactions: TransactionExtended[]): BlockSummary {
return {
id: hash,
transactions: Common.stripTransactions(transactions),
transactions: Common.classifyTransactions(transactions),
};
}
@@ -560,6 +561,123 @@ class Blocks {
logger.debug(`Indexing block audit details completed`);
}
/**
* [INDEXING] Index transaction classification flags for Goggles
*/
public async $classifyBlocks(): Promise<void> {
// classification requires an esplora backend
if (!Common.gogglesIndexingEnabled() || config.MEMPOOL.BACKEND !== 'esplora') {
return;
}
const blockchainInfo = await bitcoinClient.getBlockchainInfo();
const currentBlockHeight = blockchainInfo.blocks;
const unclassifiedBlocksList = await BlocksSummariesRepository.$getSummariesWithVersion(0);
const unclassifiedTemplatesList = await BlocksSummariesRepository.$getTemplatesWithVersion(0);
// nothing to do
if (!unclassifiedBlocksList?.length && !unclassifiedTemplatesList?.length) {
return;
}
let timer = Date.now();
let indexedThisRun = 0;
let indexedTotal = 0;
const minHeight = Math.min(
unclassifiedBlocksList[unclassifiedBlocksList.length - 1]?.height ?? Infinity,
unclassifiedTemplatesList[unclassifiedTemplatesList.length - 1]?.height ?? Infinity,
);
const numToIndex = Math.max(
unclassifiedBlocksList.length,
unclassifiedTemplatesList.length,
);
const unclassifiedBlocks = {};
const unclassifiedTemplates = {};
for (const block of unclassifiedBlocksList) {
unclassifiedBlocks[block.height] = block.id;
}
for (const template of unclassifiedTemplatesList) {
unclassifiedTemplates[template.height] = template.id;
}
logger.debug(`Classifying blocks and templates from #${currentBlockHeight} to #${minHeight}`, logger.tags.goggles);
for (let height = currentBlockHeight; height >= 0; height--) {
try {
let txs: TransactionExtended[] | null = null;
if (unclassifiedBlocks[height]) {
const blockHash = unclassifiedBlocks[height];
// fetch transactions
txs = (await bitcoinApi.$getTxsForBlock(blockHash)).map(tx => transactionUtils.extendTransaction(tx)) || [];
// add CPFP
const cpfpSummary = Common.calculateCpfp(height, txs, true);
// classify
const { transactions: classifiedTxs } = this.summarizeBlockTransactions(blockHash, cpfpSummary.transactions);
await BlocksSummariesRepository.$saveTransactions(height, blockHash, classifiedTxs, 1);
await Common.sleep$(250);
}
if (unclassifiedTemplates[height]) {
// classify template
const blockHash = unclassifiedTemplates[height];
const template = await BlocksSummariesRepository.$getTemplate(blockHash);
const alreadyClassified = template?.transactions?.reduce((classified, tx) => (classified || tx.flags > 0), false);
let classifiedTemplate = template?.transactions || [];
if (!alreadyClassified) {
const templateTxs: (TransactionExtended | TransactionClassified)[] = [];
const blockTxMap: { [txid: string]: TransactionExtended } = {};
for (const tx of (txs || [])) {
blockTxMap[tx.txid] = tx;
}
for (const templateTx of (template?.transactions || [])) {
let tx: TransactionExtended | null = blockTxMap[templateTx.txid];
if (!tx) {
try {
tx = await transactionUtils.$getTransactionExtended(templateTx.txid, false, true, false);
} catch (e) {
// transaction probably not found
}
}
templateTxs.push(tx || templateTx);
}
const cpfpSummary = Common.calculateCpfp(height, templateTxs?.filter(tx => tx['effectiveFeePerVsize'] != null) as TransactionExtended[], true);
// classify
const { transactions: classifiedTxs } = this.summarizeBlockTransactions(blockHash, cpfpSummary.transactions);
const classifiedTxMap: { [txid: string]: TransactionClassified } = {};
for (const tx of classifiedTxs) {
classifiedTxMap[tx.txid] = tx;
}
classifiedTemplate = classifiedTemplate.map(tx => {
if (classifiedTxMap[tx.txid]) {
tx.flags = classifiedTxMap[tx.txid].flags || 0;
}
return tx;
});
}
await BlocksSummariesRepository.$saveTemplate({ height, template: { id: blockHash, transactions: classifiedTemplate }, version: 1 });
await Common.sleep$(250);
}
} catch (e) {
logger.warn(`Failed to classify template or block summary at ${height}`, logger.tags.goggles);
}
// timing & logging
if (unclassifiedBlocks[height] || unclassifiedTemplates[height]) {
indexedThisRun++;
indexedTotal++;
}
const elapsedSeconds = (Date.now() - timer) / 1000;
if (elapsedSeconds > 5) {
const perSecond = indexedThisRun / elapsedSeconds;
logger.debug(`Classified #${height}: ${indexedTotal} / ${numToIndex} blocks (${perSecond.toFixed(1)}/s)`);
timer = Date.now();
indexedThisRun = 0;
}
}
}
/**
* [INDEXING] Index all blocks metadata for the mining dashboard
*/
@@ -945,7 +1063,7 @@ class Blocks {
}
public async $getStrippedBlockTransactions(hash: string, skipMemoryCache = false,
skipDBLookup = false, cpfpSummary?: CpfpSummary, blockHeight?: number): Promise<TransactionStripped[]>
skipDBLookup = false, cpfpSummary?: CpfpSummary, blockHeight?: number): Promise<TransactionClassified[]>
{
if (skipMemoryCache === false) {
// Check the memory cache
@@ -965,6 +1083,7 @@ class Blocks {
let height = blockHeight;
let summary: BlockSummary;
let summaryVersion = 0;
if (cpfpSummary && !Common.isLiquid()) {
summary = {
id: hash,
@@ -974,14 +1093,17 @@ class Blocks {
fee: tx.fee || 0,
vsize: tx.vsize,
value: Math.round(tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0)),
rate: tx.effectiveFeePerVsize
rate: tx.effectiveFeePerVsize,
flags: tx.flags || Common.getTransactionFlags(tx),
};
}),
};
summaryVersion = 1;
} else {
if (config.MEMPOOL.BACKEND === 'esplora') {
const txs = (await bitcoinApi.$getTxsForBlock(hash)).map(tx => transactionUtils.extendTransaction(tx));
summary = this.summarizeBlockTransactions(hash, txs);
summaryVersion = 1;
} else {
// Call Core RPC
const block = await bitcoinClient.getBlock(hash, 2);
@@ -996,7 +1118,7 @@ class Blocks {
// Index the response if needed
if (Common.blocksSummariesIndexingEnabled() === true) {
await BlocksSummariesRepository.$saveTransactions(height, hash, summary.transactions);
await BlocksSummariesRepository.$saveTransactions(height, hash, summary.transactions, summaryVersion);
}
return summary.transactions;
@@ -1112,16 +1234,18 @@ class Blocks {
if (cleanBlock.fee_amt_percentiles === null) {
let summary;
let summaryVersion = 0;
if (config.MEMPOOL.BACKEND === 'esplora') {
const txs = (await bitcoinApi.$getTxsForBlock(cleanBlock.hash)).map(tx => transactionUtils.extendTransaction(tx));
summary = this.summarizeBlockTransactions(cleanBlock.hash, txs);
summaryVersion = 1;
} else {
// Call Core RPC
const block = await bitcoinClient.getBlock(cleanBlock.hash, 2);
summary = this.summarizeBlock(block);
}
await BlocksSummariesRepository.$saveTransactions(cleanBlock.height, cleanBlock.hash, summary.transactions);
await BlocksSummariesRepository.$saveTransactions(cleanBlock.height, cleanBlock.hash, summary.transactions, summaryVersion);
cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash);
}
if (cleanBlock.fee_amt_percentiles !== null) {

42
backend/src/api/common.ts
View File

@@ -1,10 +1,9 @@
import * as bitcoinjs from 'bitcoinjs-lib';
import { Request } from 'express';
import { Ancestor, CpfpInfo, CpfpSummary, CpfpCluster, EffectiveFeeStats, MempoolBlockWithTransactions, TransactionExtended, MempoolTransactionExtended, TransactionStripped, WorkingEffectiveFeeStats, TransactionClassified, TransactionFlags } from '../mempool.interfaces';
import { CpfpInfo, CpfpSummary, CpfpCluster, EffectiveFeeStats, MempoolBlockWithTransactions, TransactionExtended, MempoolTransactionExtended, TransactionStripped, WorkingEffectiveFeeStats, TransactionClassified, TransactionFlags } from '../mempool.interfaces';
import config from '../config';
import { NodeSocket } from '../repositories/NodesSocketsRepository';
import { isIP } from 'net';
import rbfCache from './rbf-cache';
import transactionUtils from './transaction-utils';
import { isPoint } from '../utils/secp256k1';
export class Common {
@@ -263,8 +262,13 @@ export class Common {
case 'v0_p2wsh': flags |= TransactionFlags.p2wsh; break;
case 'v1_p2tr': {
flags |= TransactionFlags.p2tr;
if (vin.witness.length > 2) {
const asm = vin.inner_witnessscript_asm || transactionUtils.convertScriptSigAsm(vin.witness[vin.witness.length - 2]);
// in taproot, if the last witness item begins with 0x50, it's an annex
const hasAnnex = vin.witness?.[vin.witness.length - 1].startsWith('50');
// script spends have more than one witness item, not counting the annex (if present)
if (vin.witness.length > (hasAnnex ? 2 : 1)) {
// the script itself is the second-to-last witness item, not counting the annex
const asm = vin.inner_witnessscript_asm || transactionUtils.convertScriptSigAsm(vin.witness[vin.witness.length - (hasAnnex ? 3 : 2)]);
// inscriptions smuggle data within an 'OP_0 OP_IF ... OP_ENDIF' envelope
if (asm?.includes('OP_0 OP_IF')) {
flags |= TransactionFlags.inscription;
}
@@ -344,14 +348,18 @@ export class Common {
}
static classifyTransaction(tx: TransactionExtended): TransactionClassified {
const flags = this.getTransactionFlags(tx);
const flags = Common.getTransactionFlags(tx);
tx.flags = flags;
return {
...this.stripTransaction(tx),
...Common.stripTransaction(tx),
flags,
};
}
static classifyTransactions(txs: TransactionExtended[]): TransactionClassified[] {
return txs.map(Common.classifyTransaction);
}
static stripTransaction(tx: TransactionExtended): TransactionStripped {
return {
txid: tx.txid,
@@ -364,7 +372,7 @@ export class Common {
}
static stripTransactions(txs: TransactionExtended[]): TransactionStripped[] {
return txs.map(this.stripTransaction);
return txs.map(Common.stripTransaction);
}
static sleep$(ms: number): Promise<void> {
@@ -500,6 +508,13 @@ export class Common {
);
}
static gogglesIndexingEnabled(): boolean {
return (
Common.blocksSummariesIndexingEnabled() &&
config.MEMPOOL.GOGGLES_INDEXING === true
);
}
static cpfpIndexingEnabled(): boolean {
return (
Common.indexingEnabled() &&
@@ -627,12 +642,12 @@ export class Common {
}
}
static calculateCpfp(height: number, transactions: TransactionExtended[]): CpfpSummary {
static calculateCpfp(height: number, transactions: TransactionExtended[], saveRelatives: boolean = false): CpfpSummary {
const clusters: CpfpCluster[] = []; // list of all cpfp clusters in this block
const clusterMap: { [txid: string]: CpfpCluster } = {}; // map transactions to their cpfp cluster
let clusterTxs: TransactionExtended[] = []; // working list of elements of the current cluster
let ancestors: { [txid: string]: boolean } = {}; // working set of ancestors of the current cluster root
const txMap = {};
const txMap: { [txid: string]: TransactionExtended } = {};
// initialize the txMap
for (const tx of transactions) {
txMap[tx.txid] = tx;
@@ -702,6 +717,15 @@ export class Common {
}
}
}
if (saveRelatives) {
for (const cluster of clusters) {
cluster.txs.forEach((member, index) => {
txMap[member.txid].descendants = cluster.txs.slice(0, index).reverse();
txMap[member.txid].ancestors = cluster.txs.slice(index + 1).reverse();
txMap[member.txid].effectiveFeePerVsize = cluster.effectiveFeePerVsize;
});
}
}
return {
transactions,
clusters,

10
backend/src/api/database-migration.ts
View File

@@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
import { RowDataPacket } from 'mysql2';
class DatabaseMigration {
private static currentVersion = 66;
private static currentVersion = 67;
private queryTimeout = 3600_000;
private statisticsAddedIndexed = false;
private uniqueLogs: string[] = [];
@@ -558,6 +558,14 @@ class DatabaseMigration {
await this.$executeQuery('ALTER TABLE `statistics` ADD min_fee FLOAT UNSIGNED DEFAULT NULL');
await this.updateToSchemaVersion(66);
}
if (databaseSchemaVersion < 67 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `blocks_summaries` ADD version INT NOT NULL DEFAULT 0');
await this.$executeQuery('ALTER TABLE `blocks_summaries` ADD INDEX `version` (`version`)');
await this.$executeQuery('ALTER TABLE `blocks_templates` ADD version INT NOT NULL DEFAULT 0');
await this.$executeQuery('ALTER TABLE `blocks_templates` ADD INDEX `version` (`version`)');
await this.updateToSchemaVersion(67);
}
}
/**

2
backend/src/api/explorer/channels.api.ts
View File

@@ -81,7 +81,7 @@ class ChannelsApi {
public async $searchChannelsById(search: string): Promise<any[]> {
try {
// restrict search to valid id/short_id prefix formats
let searchStripped = search.match(/[0-9]+[0-9x]*/)?.[0] || '';
let searchStripped = search.match(/^[0-9]+[0-9x]*$/)?.[0] || '';
if (!searchStripped.length) {
return [];
}

18
backend/src/api/fee-api.ts
View File

@@ -39,15 +39,25 @@ class FeeApi {
const secondMedianFee = pBlocks[1] ? this.optimizeMedianFee(pBlocks[1], pBlocks[2], firstMedianFee) : this.defaultFee;
const thirdMedianFee = pBlocks[2] ? this.optimizeMedianFee(pBlocks[2], pBlocks[3], secondMedianFee) : this.defaultFee;
let fastestFee = Math.max(minimumFee, firstMedianFee);
let halfHourFee = Math.max(minimumFee, secondMedianFee);
let hourFee = Math.max(minimumFee, thirdMedianFee);
const economyFee = Math.max(minimumFee, Math.min(2 * minimumFee, thirdMedianFee));
// ensure recommendations always increase w/ priority
fastestFee = Math.max(fastestFee, halfHourFee, hourFee, economyFee);
halfHourFee = Math.max(halfHourFee, hourFee, economyFee);
hourFee = Math.max(hourFee, economyFee);
// explicitly enforce a minimum of ceil(mempoolminfee) on all recommendations.
// simply rounding up recommended rates is insufficient, as the purging rate
// can exceed the median rate of projected blocks in some extreme scenarios
// (see https://bitcoin.stackexchange.com/a/120024)
return {
'fastestFee': Math.max(minimumFee, firstMedianFee),
'halfHourFee': Math.max(minimumFee, secondMedianFee),
'hourFee': Math.max(minimumFee, thirdMedianFee),
'economyFee': Math.max(minimumFee, Math.min(2 * minimumFee, thirdMedianFee)),
'fastestFee': fastestFee,
'halfHourFee': halfHourFee,
'hourFee': hourFee,
'economyFee': economyFee,
'minimumFee': minimumFee,
};
}

28
backend/src/api/mempool-blocks.ts
View File

@@ -1,6 +1,6 @@
import { GbtGenerator, GbtResult, ThreadTransaction as RustThreadTransaction, ThreadAcceleration as RustThreadAcceleration } from 'rust-gbt';
import logger from '../logger';
import { MempoolBlock, MempoolTransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats, PoolTag, TransactionClassified } from '../mempool.interfaces';
import { MempoolBlock, MempoolTransactionExtended, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats, PoolTag, TransactionClassified } from '../mempool.interfaces';
import { Common, OnlineFeeStatsCalculator } from './common';
import config from '../config';
import { Worker } from 'worker_threads';
@@ -368,12 +368,15 @@ class MempoolBlocks {
// run the block construction algorithm in a separate thread, and wait for a result
const rustGbt = saveResults ? this.rustGbtGenerator : new GbtGenerator();
try {
const { blocks, blockWeights, rates, clusters } = this.convertNapiResultTxids(
const { blocks, blockWeights, rates, clusters, overflow } = this.convertNapiResultTxids(
await rustGbt.make(Object.values(newMempool) as RustThreadTransaction[], convertedAccelerations as RustThreadAcceleration[], this.nextUid),
);
if (saveResults) {
this.rustInitialized = true;
}
const mempoolSize = Object.keys(newMempool).length;
const resultMempoolSize = blocks.reduce((total, block) => total + block.length, 0) + overflow.length;
logger.debug(`RUST updateBlockTemplates returned ${resultMempoolSize} txs out of ${mempoolSize} in the mempool, ${overflow.length} were unmineable`);
const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, accelerations, accelerationPool, saveResults);
logger.debug(`RUST makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`);
return processed;
@@ -424,7 +427,7 @@ class MempoolBlocks {
// run the block construction algorithm in a separate thread, and wait for a result
try {
const { blocks, blockWeights, rates, clusters } = this.convertNapiResultTxids(
const { blocks, blockWeights, rates, clusters, overflow } = this.convertNapiResultTxids(
await this.rustGbtGenerator.update(
added as RustThreadTransaction[],
removedUids,
@@ -432,9 +435,10 @@ class MempoolBlocks {
this.nextUid,
),
);
const resultMempoolSize = blocks.reduce((total, block) => total + block.length, 0);
const resultMempoolSize = blocks.reduce((total, block) => total + block.length, 0) + overflow.length;
logger.debug(`RUST updateBlockTemplates returned ${resultMempoolSize} txs out of ${mempoolSize} in the mempool, ${overflow.length} were unmineable`);
if (mempoolSize !== resultMempoolSize) {
throw new Error('GBT returned wrong number of transactions, cache is probably out of sync');
throw new Error('GBT returned wrong number of transactions , cache is probably out of sync');
} else {
const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, accelerations, accelerationPool, true);
this.removeUids(removedUids);
@@ -658,8 +662,8 @@ class MempoolBlocks {
return { blocks: convertedBlocks, rates: convertedRates, clusters: convertedClusters } as { blocks: string[][], rates: { [root: string]: number }, clusters: { [root: string]: string[] }};
}
private convertNapiResultTxids({ blocks, blockWeights, rates, clusters }: GbtResult)
: { blocks: string[][], blockWeights: number[], rates: [string, number][], clusters: string[][] } {
private convertNapiResultTxids({ blocks, blockWeights, rates, clusters, overflow }: GbtResult)
: { blocks: string[][], blockWeights: number[], rates: [string, number][], clusters: string[][], overflow: string[] } {
const convertedBlocks: string[][] = blocks.map(block => block.map(uid => {
const txid = this.uidMap.get(uid);
if (txid !== undefined) {
@@ -677,7 +681,15 @@ class MempoolBlocks {
for (const cluster of clusters) {
convertedClusters.push(cluster.map(uid => this.uidMap.get(uid)) as string[]);
}
return { blocks: convertedBlocks, blockWeights, rates: convertedRates, clusters: convertedClusters };
const convertedOverflow: string[] = overflow.map(uid => {
const txid = this.uidMap.get(uid);
if (txid !== undefined) {
return txid;
} else {
throw new Error('GBT returned an unmineable transaction with unknown uid');
}
});
return { blocks: convertedBlocks, blockWeights, rates: convertedRates, clusters: convertedClusters, overflow: convertedOverflow };
}
}

3
backend/src/api/tx-selection-worker.ts
View File

@@ -173,10 +173,13 @@ function makeBlockTemplates(mempool: Map<number, CompactThreadTransaction>)
// this block is full
const exceededPackageTries = failures > 1000 && blockWeight > (config.MEMPOOL.BLOCK_WEIGHT_UNITS - 4000);
const queueEmpty = top >= mempoolArray.length && modified.isEmpty();
if ((exceededPackageTries || queueEmpty) && blocks.length < 7) {
// construct this block
if (transactions.length) {
blocks.push(transactions.map(t => t.uid));
} else {
break;
}
// reset for the next block
transactions = [];

164
backend/src/api/websocket-handler.ts
View File

@@ -24,6 +24,12 @@ import { ApiPrice } from '../repositories/PricesRepository';
import accelerationApi from './services/acceleration';
import mempool from './mempool';
interface AddressTransactions {
mempool: MempoolTransactionExtended[],
confirmed: MempoolTransactionExtended[],
removed: MempoolTransactionExtended[],
}
// valid 'want' subscriptions
const wantable = [
'blocks',
@@ -195,24 +201,49 @@ class WebsocketHandler {
}
if (parsedMessage && parsedMessage['track-address']) {
if (/^([a-km-zA-HJ-NP-Z1-9]{26,35}|[a-km-zA-HJ-NP-Z1-9]{80}|[a-z]{2,5}1[ac-hj-np-z02-9]{8,100}|[A-Z]{2,5}1[AC-HJ-NP-Z02-9]{8,100}|04[a-fA-F0-9]{128}|(02|03)[a-fA-F0-9]{64})$/
.test(parsedMessage['track-address'])) {
let matchedAddress = parsedMessage['track-address'];
if (/^[A-Z]{2,5}1[AC-HJ-NP-Z02-9]{8,100}$/.test(parsedMessage['track-address'])) {
matchedAddress = matchedAddress.toLowerCase();
}
if (/^04[a-fA-F0-9]{128}$/.test(parsedMessage['track-address'])) {
client['track-address'] = '41' + matchedAddress + 'ac';
} else if (/^(02|03)[a-fA-F0-9]{64}$/.test(parsedMessage['track-address'])) {
client['track-address'] = '21' + matchedAddress + 'ac';
} else {
client['track-address'] = matchedAddress;
}
const validAddress = this.testAddress(parsedMessage['track-address']);
if (validAddress) {
client['track-address'] = validAddress;
} else {
client['track-address'] = null;
}
}
if (parsedMessage && parsedMessage['track-addresses'] && Array.isArray(parsedMessage['track-addresses'])) {
const addressMap: { [address: string]: string } = {};
for (const address of parsedMessage['track-addresses']) {
const validAddress = this.testAddress(address);
if (validAddress) {
addressMap[address] = validAddress;
}
}
if (Object.keys(addressMap).length > config.MEMPOOL.MAX_TRACKED_ADDRESSES) {
response['track-addresses-error'] = `"too many addresses requested, this connection supports tracking a maximum of ${config.MEMPOOL.MAX_TRACKED_ADDRESSES} addresses"`;
client['track-addresses'] = null;
} else if (Object.keys(addressMap).length > 0) {
client['track-addresses'] = addressMap;
} else {
client['track-addresses'] = null;
}
}
if (parsedMessage && parsedMessage['track-scriptpubkeys'] && Array.isArray(parsedMessage['track-scriptpubkeys'])) {
const spks: string[] = [];
for (const spk of parsedMessage['track-scriptpubkeys']) {
if (/^[a-fA-F0-9]+$/.test(spk)) {
spks.push(spk.toLowerCase());
}
}
if (spks.length > config.MEMPOOL.MAX_TRACKED_ADDRESSES) {
response['track-scriptpubkeys-error'] = `"too many scriptpubkeys requested, this connection supports tracking a maximum of ${config.MEMPOOL.MAX_TRACKED_ADDRESSES} scriptpubkeys"`;
client['track-scriptpubkeys'] = null;
} else if (spks.length) {
client['track-scriptpubkeys'] = spks;
} else {
client['track-scriptpubkeys'] = null;
}
}
if (parsedMessage && parsedMessage['track-asset']) {
if (/^[a-fA-F0-9]{64}$/.test(parsedMessage['track-asset'])) {
client['track-asset'] = parsedMessage['track-asset'];
@@ -544,6 +575,50 @@ class WebsocketHandler {
}
}
if (client['track-addresses']) {
const addressMap: { [address: string]: AddressTransactions } = {};
for (const [address, key] of Object.entries(client['track-addresses'] || {})) {
const newTransactions = Array.from(addressCache[key as string]?.values() || []);
const removedTransactions = Array.from(removedAddressCache[key as string]?.values() || []);
// txs may be missing prevouts in non-esplora backends
// so fetch the full transactions now
const fullTransactions = (config.MEMPOOL.BACKEND !== 'esplora') ? await this.getFullTransactions(newTransactions) : newTransactions;
if (fullTransactions?.length) {
addressMap[address] = {
mempool: fullTransactions,
confirmed: [],
removed: removedTransactions,
};
}
}
if (Object.keys(addressMap).length > 0) {
response['multi-address-transactions'] = JSON.stringify(addressMap);
}
}
if (client['track-scriptpubkeys']) {
const spkMap: { [spk: string]: AddressTransactions } = {};
for (const spk of client['track-scriptpubkeys'] || []) {
const newTransactions = Array.from(addressCache[spk as string]?.values() || []);
const removedTransactions = Array.from(removedAddressCache[spk as string]?.values() || []);
// txs may be missing prevouts in non-esplora backends
// so fetch the full transactions now
const fullTransactions = (config.MEMPOOL.BACKEND !== 'esplora') ? await this.getFullTransactions(newTransactions) : newTransactions;
if (fullTransactions?.length) {
spkMap[spk] = {
mempool: fullTransactions,
confirmed: [],
removed: removedTransactions,
};
}
}
if (Object.keys(spkMap).length > 0) {
response['multi-scriptpubkey-transactions'] = JSON.stringify(spkMap);
}
}
if (client['track-asset']) {
const foundTransactions: TransactionExtended[] = [];
@@ -703,7 +778,8 @@ class WebsocketHandler {
template: {
id: block.id,
transactions: stripped,
}
},
version: 1,
});
BlocksAuditsRepository.$saveAudit({
@@ -843,6 +919,42 @@ class WebsocketHandler {
}
}
if (client['track-addresses']) {
const addressMap: { [address: string]: AddressTransactions } = {};
for (const [address, key] of Object.entries(client['track-addresses'] || {})) {
const fullTransactions = Array.from(addressCache[key as string]?.values() || []);
if (fullTransactions?.length) {
addressMap[address] = {
mempool: [],
confirmed: fullTransactions,
removed: [],
};
}
}
if (Object.keys(addressMap).length > 0) {
response['multi-address-transactions'] = JSON.stringify(addressMap);
}
}
if (client['track-scriptpubkeys']) {
const spkMap: { [spk: string]: AddressTransactions } = {};
for (const spk of client['track-scriptpubkeys'] || []) {
const fullTransactions = Array.from(addressCache[spk as string]?.values() || []);
if (fullTransactions?.length) {
spkMap[spk] = {
mempool: [],
confirmed: fullTransactions,
removed: [],
};
}
}
if (Object.keys(spkMap).length > 0) {
response['multi-scriptpubkey-transactions'] = JSON.stringify(spkMap);
}
}
if (client['track-asset']) {
const foundTransactions: TransactionExtended[] = [];
@@ -912,6 +1024,28 @@ class WebsocketHandler {
+ '}';
}
// checks if an address conforms to a valid format
// returns the canonical form:
// - lowercase for bech32(m)
// - lowercase scriptpubkey for P2PK
// or false if invalid
private testAddress(address): string | false {
if (/^([a-km-zA-HJ-NP-Z1-9]{26,35}|[a-km-zA-HJ-NP-Z1-9]{80}|[a-z]{2,5}1[ac-hj-np-z02-9]{8,100}|[A-Z]{2,5}1[AC-HJ-NP-Z02-9]{8,100}|04[a-fA-F0-9]{128}|(02|03)[a-fA-F0-9]{64})$/.test(address)) {
if (/^[A-Z]{2,5}1[AC-HJ-NP-Z02-9]{8,100}|04[a-fA-F0-9]{128}|(02|03)[a-fA-F0-9]{64}$/.test(address)) {
address = address.toLowerCase();
}
if (/^04[a-fA-F0-9]{128}$/.test(address)) {
return '41' + address + 'ac';
} else if (/^(02|03)[a-fA-F0-9]{64}$/.test(address)) {
return '21' + address + 'ac';
} else {
return address;
}
} else {
return false;
}
}
private makeAddressCache(transactions: MempoolTransactionExtended[]): { [address: string]: Set<MempoolTransactionExtended> } {
const addressCache: { [address: string]: Set<MempoolTransactionExtended> } = {};
for (const tx of transactions) {
@@ -968,7 +1102,7 @@ class WebsocketHandler {
if (client['track-tx']) {
numTxSubs++;
}
if (client['track-mempool-block'] >= 0) {
if (client['track-mempool-block'] != null && client['track-mempool-block'] >= 0) {
numProjectedSubs++;
}
if (client['track-rbf']) {

4
backend/src/config.ts
View File

@@ -20,6 +20,7 @@ interface IConfig {
MEMPOOL_BLOCKS_AMOUNT: number;
INDEXING_BLOCKS_AMOUNT: number;
BLOCKS_SUMMARIES_INDEXING: boolean;
GOGGLES_INDEXING: boolean;
USE_SECOND_NODE_FOR_MINFEE: boolean;
EXTERNAL_ASSETS: string[];
EXTERNAL_MAX_RETRY: number;
@@ -39,6 +40,7 @@ interface IConfig {
MAX_PUSH_TX_SIZE_WEIGHT: number;
ALLOW_UNREACHABLE: boolean;
PRICE_UPDATES_PER_HOUR: number;
MAX_TRACKED_ADDRESSES: number;
};
ESPLORA: {
REST_API_URL: string;
@@ -174,6 +176,7 @@ const defaults: IConfig = {
'MEMPOOL_BLOCKS_AMOUNT': 8,
'INDEXING_BLOCKS_AMOUNT': 11000, // 0 = disable indexing, -1 = index all blocks
'BLOCKS_SUMMARIES_INDEXING': false,
'GOGGLES_INDEXING': false,
'USE_SECOND_NODE_FOR_MINFEE': false,
'EXTERNAL_ASSETS': [],
'EXTERNAL_MAX_RETRY': 1,
@@ -193,6 +196,7 @@ const defaults: IConfig = {
'MAX_PUSH_TX_SIZE_WEIGHT': 400000,
'ALLOW_UNREACHABLE': true,
'PRICE_UPDATES_PER_HOUR': 1,
'MAX_TRACKED_ADDRESSES': 1,
},
'ESPLORA': {
'REST_API_URL': 'http://127.0.0.1:3000',

2
backend/src/index.ts
View File

@@ -44,6 +44,7 @@ import v8 from 'v8';
import { formatBytes, getBytesUnit } from './utils/format';
import redisCache from './api/redis-cache';
import accelerationApi from './api/services/acceleration';
import bitcoinCoreRoutes from './api/bitcoin/bitcoin-core.routes';
class Server {
private wss: WebSocket.Server | undefined;
@@ -282,6 +283,7 @@ class Server {
setUpHttpApiRoutes(): void {
bitcoinRoutes.initRoutes(this.app);
bitcoinCoreRoutes.initRoutes(this.app);
pricesRoutes.initRoutes(this.app);
if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED && config.MEMPOOL.ENABLED) {
statisticsRoutes.initRoutes(this.app);

1
backend/src/indexer.ts
View File

@@ -185,6 +185,7 @@ class Indexer {
await blocks.$generateCPFPDatabase();
await blocks.$generateAuditStats();
await auditReplicator.$sync();
await blocks.$classifyBlocks();
} catch (e) {
this.indexerRunning = false;
logger.err(`Indexer failed, trying again in 10 seconds. Reason: ` + (e instanceof Error ? e.message : e));

1
backend/src/logger.ts
View File

@@ -35,6 +35,7 @@ class Logger {
public tags = {
mining: 'Mining',
ln: 'Lightning',
goggles: 'Goggles',
};
// @ts-ignore

7
backend/src/mempool.interfaces.ts
View File

@@ -280,7 +280,8 @@ export interface BlockExtended extends IEsploraApi.Block {
export interface BlockSummary {
id: string;
transactions: TransactionStripped[];
transactions: TransactionClassified[];
version?: number;
}
export interface AuditSummary extends BlockAudit {
@@ -288,8 +289,8 @@ export interface AuditSummary extends BlockAudit {
size?: number,
weight?: number,
tx_count?: number,
transactions: TransactionStripped[];
template?: TransactionStripped[];
transactions: TransactionClassified[];
template?: TransactionClassified[];
}
export interface BlockPrice {

3
backend/src/replication/AuditReplication.ts
View File

@@ -105,7 +105,8 @@ class AuditReplication {
template: {
id: blockHash,
transactions: auditSummary.template || []
}
},
version: 1,
});
await blocksAuditsRepository.$saveAudit({
hash: blockHash,

4
backend/src/repositories/BlocksRepository.ts
View File

@@ -1040,16 +1040,18 @@ class BlocksRepository {
if (extras.feePercentiles === null) {
let summary;
let summaryVersion = 0;
if (config.MEMPOOL.BACKEND === 'esplora') {
const txs = (await bitcoinApi.$getTxsForBlock(dbBlk.id)).map(tx => transactionUtils.extendTransaction(tx));
summary = blocks.summarizeBlockTransactions(dbBlk.id, txs);
summaryVersion = 1;
} else {
// Call Core RPC
const block = await bitcoinClient.getBlock(dbBlk.id, 2);
summary = blocks.summarizeBlock(block);
}
await BlocksSummariesRepository.$saveTransactions(dbBlk.height, dbBlk.id, summary.transactions);
await BlocksSummariesRepository.$saveTransactions(dbBlk.height, dbBlk.id, summary.transactions, summaryVersion);
extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id);
}
if (extras.feePercentiles !== null) {

57
backend/src/repositories/BlocksSummariesRepository.ts
View File

@@ -1,6 +1,6 @@
import DB from '../database';
import logger from '../logger';
import { BlockSummary, TransactionStripped } from '../mempool.interfaces';
import { BlockSummary, TransactionClassified } from '../mempool.interfaces';
class BlocksSummariesRepository {
public async $getByBlockId(id: string): Promise<BlockSummary | undefined> {
@@ -17,30 +17,31 @@ class BlocksSummariesRepository {
return undefined;
}
public async $saveTransactions(blockHeight: number, blockId: string, transactions: TransactionStripped[]): Promise<void> {
public async $saveTransactions(blockHeight: number, blockId: string, transactions: TransactionClassified[], version: number): Promise<void> {
try {
const transactionsStr = JSON.stringify(transactions);
await DB.query(`
INSERT INTO blocks_summaries
SET height = ?, transactions = ?, id = ?
ON DUPLICATE KEY UPDATE transactions = ?`,
[blockHeight, transactionsStr, blockId, transactionsStr]);
SET height = ?, transactions = ?, id = ?, version = ?
ON DUPLICATE KEY UPDATE transactions = ?, version = ?`,
[blockHeight, transactionsStr, blockId, version, transactionsStr, version]);
} catch (e: any) {
logger.debug(`Cannot save block summary transactions for ${blockId}. Reason: ${e instanceof Error ? e.message : e}`);
throw e;
}
}
public async $saveTemplate(params: { height: number, template: BlockSummary}) {
public async $saveTemplate(params: { height: number, template: BlockSummary, version: number}): Promise<void> {
const blockId = params.template?.id;
try {
const transactions = JSON.stringify(params.template?.transactions || []);
await DB.query(`
INSERT INTO blocks_templates (id, template)
VALUE (?, ?)
INSERT INTO blocks_templates (id, template, version)
VALUE (?, ?, ?)
ON DUPLICATE KEY UPDATE
template = ?
`, [blockId, transactions, transactions]);
template = ?,
version = ?
`, [blockId, transactions, params.version, transactions, params.version]);
} catch (e: any) {
if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
logger.debug(`Cannot save block template for ${blockId} because it has already been indexed, ignoring`);
@@ -57,6 +58,7 @@ class BlocksSummariesRepository {
return {
id: templates[0].id,
transactions: JSON.parse(templates[0].template),
version: templates[0].version,
};
}
} catch (e) {
@@ -76,6 +78,41 @@ class BlocksSummariesRepository {
return [];
}
public async $getSummariesWithVersion(version: number): Promise<{ height: number, id: string }[]> {
try {
const [rows]: any[] = await DB.query(`
SELECT
height,
id
FROM blocks_summaries
WHERE version = ?
ORDER BY height DESC;`, [version]);
return rows;
} catch (e) {
logger.err(`Cannot get block summaries with version. Reason: ` + (e instanceof Error ? e.message : e));
}
return [];
}
public async $getTemplatesWithVersion(version: number): Promise<{ height: number, id: string }[]> {
try {
const [rows]: any[] = await DB.query(`
SELECT
blocks_summaries.height as height,
blocks_templates.id as id
FROM blocks_templates
JOIN blocks_summaries ON blocks_templates.id = blocks_summaries.id
WHERE blocks_templates.version = ?
ORDER BY height DESC;`, [version]);
return rows;
} catch (e) {
logger.err(`Cannot get block summaries with version. Reason: ` + (e instanceof Error ? e.message : e));
}
return [];
}
/**
* Get the fee percentiles if the block has already been indexed, [] otherwise
*

1
backend/src/rpc-api/commands.ts
View File

@@ -91,4 +91,5 @@ module.exports = {
walletPassphraseChange: 'walletpassphrasechange',
getTxoutSetinfo: 'gettxoutsetinfo',
getIndexInfo: 'getindexinfo',
testMempoolAccept: 'testmempoolaccept',
};

4
backend/src/tasks/price-feeds/coinbase-api.ts
View File

@@ -5,14 +5,14 @@ class CoinbaseApi implements PriceFeed {
public name: string = 'Coinbase';
public currencies: string[] = ['USD', 'EUR', 'GBP'];
public url: string = 'https://api.coinbase.com/v2/prices/spot?currency=';
public url: string = 'https://api.coinbase.com/v2/prices/BTC-{CURRENCY}/spot';
public urlHist: string = 'https://api.exchange.coinbase.com/products/BTC-{CURRENCY}/candles?granularity={GRANULARITY}';
constructor() {
}
public async $fetchPrice(currency): Promise<number> {
const response = await query(this.url + currency);
const response = await query(this.url.replace('{CURRENCY}', currency));
if (response && response['data'] && response['data']['amount']) {
return parseInt(response['data']['amount'], 10);
} else {

14
backend/src/tasks/price-updater.ts
View File

@@ -23,6 +23,14 @@ export interface PriceHistory {
[timestamp: number]: ApiPrice;
}
function getMedian(arr: number[]): number {
const sortedArr = arr.slice().sort((a, b) => a - b);
const mid = Math.floor(sortedArr.length / 2);
return sortedArr.length % 2 !== 0
? sortedArr[mid]
: (sortedArr[mid - 1] + sortedArr[mid]) / 2;
}
class PriceUpdater {
public historyInserted = false;
private timeBetweenUpdatesMs = 360_0000 / config.MEMPOOL.PRICE_UPDATES_PER_HOUR;
@@ -173,7 +181,7 @@ class PriceUpdater {
if (prices.length === 0) {
this.latestPrices[currency] = -1;
} else {
this.latestPrices[currency] = Math.round((prices.reduce((partialSum, a) => partialSum + a, 0)) / prices.length);
this.latestPrices[currency] = Math.round(getMedian(prices));
}
}
@@ -300,9 +308,7 @@ class PriceUpdater {
if (grouped[time][currency].length === 0) {
continue;
}
prices[currency] = Math.round((grouped[time][currency].reduce(
(partialSum, a) => partialSum + a, 0)
) / grouped[time][currency].length);
prices[currency] = Math.round(getMedian(grouped[time][currency]));
}
await PricesRepository.$savePrices(parseInt(time, 10), prices);
++totalInserted;