machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into cookie

Browse Source
This commit is contained in:
softsimon
2023-11-11 12:57:49 +09:00
committed by GitHub
parent 0d69ad43ab 87b034fa8c
commit 06d1bbdf03
454 changed files with 30472 additions and 12477 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
backend/src/api/audit.ts
View File

@@ -1,19 +1,22 @@
import config from '../config';
import logger from '../logger';
import { TransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces';
import { MempoolTransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces';
import rbfCache from './rbf-cache';
const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners
class Audit {
auditBlock(transactions: TransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: TransactionExtended })
: { censored: string[], added: string[], fresh: string[], sigop: string[], score: number, similarity: number } {
auditBlock(transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended }, useAccelerations: boolean = false)
: { censored: string[], added: string[], fresh: string[], sigop: string[], fullrbf: string[], accelerated: string[], score: number, similarity: number } {
if (!projectedBlocks?.[0]?.transactionIds || !mempool) {
return { censored: [], added: [], fresh: [], sigop: [], score: 0, similarity: 1 };
return { censored: [], added: [], fresh: [], sigop: [], fullrbf: [], accelerated: [], score: 0, similarity: 1 };
}
const matches: string[] = []; // present in both mined block and template
const added: string[] = []; // present in mined block, not in template
const fresh: string[] = []; // missing, but firstSeen within PROPAGATION_MARGIN
const fresh: string[] = []; // missing, but firstSeen or lastBoosted within PROPAGATION_MARGIN
const rbf: string[] = []; // either missing or present, and either part of a full-rbf replacement, or a conflict with the mined block
const accelerated: string[] = []; // prioritized by the mempool accelerator
const isCensored = {}; // missing, without excuse
const isDisplaced = {};
let displacedWeight = 0;
@@ -26,6 +29,9 @@ class Audit {
const now = Math.round((Date.now() / 1000));
for (const tx of transactions) {
inBlock[tx.txid] = tx;
if (mempool[tx.txid] && mempool[tx.txid].acceleration) {
accelerated.push(tx.txid);
}
}
// coinbase is always expected
if (transactions[0]) {
@@ -34,8 +40,14 @@ class Audit {
// look for transactions that were expected in the template, but missing from the mined block
for (const txid of projectedBlocks[0].transactionIds) {
if (!inBlock[txid]) {
// tx is recent, may have reached the miner too late for inclusion
if (mempool[txid]?.firstSeen != null && (now - (mempool[txid]?.firstSeen || 0)) <= PROPAGATION_MARGIN) {
// allow missing transactions which either belong to a full rbf tree, or conflict with any transaction in the mined block
if (rbfCache.has(txid) && (rbfCache.isFullRbf(txid) || rbfCache.anyInSameTree(txid, (tx) => inBlock[tx.txid]))) {
rbf.push(txid);
} else if (mempool[txid]?.firstSeen != null && (now - (mempool[txid]?.firstSeen || 0)) <= PROPAGATION_MARGIN) {
// tx is recent, may have reached the miner too late for inclusion
fresh.push(txid);
} else if (mempool[txid]?.lastBoosted != null && (now - (mempool[txid]?.lastBoosted || 0)) <= PROPAGATION_MARGIN) {
// tx was recently cpfp'd, miner may not have the latest effective rate
fresh.push(txid);
} else {
isCensored[txid] = true;
@@ -91,7 +103,9 @@ class Audit {
if (inTemplate[tx.txid]) {
matches.push(tx.txid);
} else {
if (!isDisplaced[tx.txid]) {
if (rbfCache.has(tx.txid)) {
rbf.push(tx.txid);
} else if (!isDisplaced[tx.txid]) {
added.push(tx.txid);
}
overflowWeight += tx.weight;
@@ -138,6 +152,8 @@ class Audit {
added,
fresh,
sigop: [],
fullrbf: rbf,
accelerated,
score,
similarity,
};

5
backend/src/api/bisq/bisq.ts
View File

@@ -7,7 +7,6 @@ import { SocksProxyAgent } from 'socks-proxy-agent';
import { BisqBlocks, BisqBlock, BisqTransaction, BisqStats, BisqTrade } from './interfaces';
import { Common } from '../common';
import { BlockExtended } from '../../mempool.interfaces';
import { StaticPool } from 'node-worker-threads-pool';
import backendInfo from '../backend-info';
import logger from '../../logger';
@@ -31,10 +30,6 @@ class Bisq {
private priceUpdateCallbackFunction: ((price: number) => void) | undefined;
private topDirectoryWatcher: fs.FSWatcher | undefined;
private subdirectoryWatcher: fs.FSWatcher | undefined;
private jsonParsePool = new StaticPool({
size: 4,
task: (blob: string) => JSON.parse(blob),
});
constructor() {}

7
backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts
View File

@@ -3,10 +3,13 @@ import { IEsploraApi } from './esplora-api.interface';
export interface AbstractBitcoinApi {
$getRawMempool(): Promise<IEsploraApi.Transaction['txid'][]>;
$getRawTransaction(txId: string, skipConversion?: boolean, addPrevout?: boolean, lazyPrevouts?: boolean): Promise<IEsploraApi.Transaction>;
$getMempoolTransactions(txids: string[]): Promise<IEsploraApi.Transaction[]>;
$getAllMempoolTransactions(lastTxid: string);
$getTransactionHex(txId: string): Promise<string>;
$getBlockHeightTip(): Promise<number>;
$getBlockHashTip(): Promise<string>;
$getTxIdsForBlock(hash: string): Promise<string[]>;
$getTxsForBlock(hash: string): Promise<IEsploraApi.Transaction[]>;
$getBlockHash(height: number): Promise<string>;
$getBlockHeader(hash: string): Promise<string>;
$getBlock(hash: string): Promise<IEsploraApi.Block>;
@@ -14,10 +17,14 @@ export interface AbstractBitcoinApi {
$getAddress(address: string): Promise<IEsploraApi.Address>;
$getAddressTransactions(address: string, lastSeenTxId: string): Promise<IEsploraApi.Transaction[]>;
$getAddressPrefix(prefix: string): string[];
$getScriptHash(scripthash: string): Promise<IEsploraApi.ScriptHash>;
$getScriptHashTransactions(address: string, lastSeenTxId: string): Promise<IEsploraApi.Transaction[]>;
$sendRawTransaction(rawTransaction: string): Promise<string>;
$getOutspend(txId: string, vout: number): Promise<IEsploraApi.Outspend>;
$getOutspends(txId: string): Promise<IEsploraApi.Outspend[]>;
$getBatchedOutspends(txId: string[]): Promise<IEsploraApi.Outspend[][]>;
startHealthChecks(): void;
}
export interface BitcoinRpcCredentials {
host: string;

171
backend/src/api/bitcoin/bitcoin-api.ts
View File

@@ -5,6 +5,7 @@ import { IEsploraApi } from './esplora-api.interface';
import blocks from '../blocks';
import mempool from '../mempool';
import { TransactionExtended } from '../../mempool.interfaces';
import transactionUtils from '../transaction-utils';
class BitcoinApi implements AbstractBitcoinApi {
private rawMempoolCache: IBitcoinApi.RawMempool | null = null;
@@ -29,6 +30,7 @@ class BitcoinApi implements AbstractBitcoinApi {
weight: block.weight,
previousblockhash: block.previousblockhash,
mediantime: block.mediantime,
stale: block.confirmations === -1,
};
}
@@ -58,23 +60,33 @@ class BitcoinApi implements AbstractBitcoinApi {
});
}
$getTransactionHex(txId: string): Promise<string> {
return this.$getRawTransaction(txId, true)
.then((tx) => tx.hex || '');
$getMempoolTransactions(txids: string[]): Promise<IEsploraApi.Transaction[]> {
throw new Error('Method getMempoolTransactions not supported by the Bitcoin RPC API.');
}
$getAllMempoolTransactions(lastTxid: string): Promise<IEsploraApi.Transaction[]> {
throw new Error('Method getAllMempoolTransactions not supported by the Bitcoin RPC API.');
}
async $getTransactionHex(txId: string): Promise<string> {
const txInMempool = mempool.getMempool()[txId];
if (txInMempool && txInMempool.hex) {
return txInMempool.hex;
}
return this.bitcoindClient.getRawTransaction(txId, true)
.then((transaction: IBitcoinApi.Transaction) => {
return transaction.hex;
});
}
$getBlockHeightTip(): Promise<number> {
return this.bitcoindClient.getChainTips()
.then((result: IBitcoinApi.ChainTips[]) => {
return result.find(tip => tip.status === 'active')!.height;
});
return this.bitcoindClient.getBlockCount();
}
$getBlockHashTip(): Promise<string> {
return this.bitcoindClient.getChainTips()
.then((result: IBitcoinApi.ChainTips[]) => {
return result.find(tip => tip.status === 'active')!.hash;
});
return this.bitcoindClient.getBestBlockHash();
}
$getTxIdsForBlock(hash: string): Promise<string[]> {
@@ -82,6 +94,10 @@ class BitcoinApi implements AbstractBitcoinApi {
.then((rpcBlock: IBitcoinApi.Block) => rpcBlock.tx);
}
$getTxsForBlock(hash: string): Promise<IEsploraApi.Transaction[]> {
throw new Error('Method getTxsForBlock not supported by the Bitcoin RPC API.');
}
$getRawBlock(hash: string): Promise<Buffer> {
return this.bitcoindClient.getBlock(hash, 0)
.then((raw: string) => Buffer.from(raw, "hex"));
@@ -113,6 +129,14 @@ class BitcoinApi implements AbstractBitcoinApi {
throw new Error('Method getAddressTransactions not supported by the Bitcoin RPC API.');
}
$getScriptHash(scripthash: string): Promise<IEsploraApi.ScriptHash> {
throw new Error('Method getScriptHash not supported by the Bitcoin RPC API.');
}
$getScriptHashTransactions(scripthash: string, lastSeenTxId: string): Promise<IEsploraApi.Transaction[]> {
throw new Error('Method getScriptHashTransactions not supported by the Bitcoin RPC API.');
}
$getRawMempool(): Promise<IEsploraApi.Transaction['txid'][]> {
return this.bitcoindClient.getRawMemPool();
}
@@ -198,7 +222,7 @@ class BitcoinApi implements AbstractBitcoinApi {
scriptpubkey: vout.scriptPubKey.hex,
scriptpubkey_address: vout.scriptPubKey && vout.scriptPubKey.address ? vout.scriptPubKey.address
: vout.scriptPubKey.addresses ? vout.scriptPubKey.addresses[0] : '',
scriptpubkey_asm: vout.scriptPubKey.asm ? this.convertScriptSigAsm(vout.scriptPubKey.hex) : '',
scriptpubkey_asm: vout.scriptPubKey.asm ? transactionUtils.convertScriptSigAsm(vout.scriptPubKey.hex) : '',
scriptpubkey_type: this.translateScriptPubKeyType(vout.scriptPubKey.type),
};
});
@@ -208,7 +232,7 @@ class BitcoinApi implements AbstractBitcoinApi {
is_coinbase: !!vin.coinbase,
prevout: null,
scriptsig: vin.scriptSig && vin.scriptSig.hex || vin.coinbase || '',
scriptsig_asm: vin.scriptSig && this.convertScriptSigAsm(vin.scriptSig.hex) || '',
scriptsig_asm: vin.scriptSig && transactionUtils.convertScriptSigAsm(vin.scriptSig.hex) || '',
sequence: vin.sequence,
txid: vin.txid || '',
vout: vin.vout || 0,
@@ -280,7 +304,7 @@ class BitcoinApi implements AbstractBitcoinApi {
}
const innerTx = await this.$getRawTransaction(vin.txid, false, false);
vin.prevout = innerTx.vout[vin.vout];
this.addInnerScriptsToVin(vin);
transactionUtils.addInnerScriptsToVin(vin);
}
return transaction;
}
@@ -319,7 +343,7 @@ class BitcoinApi implements AbstractBitcoinApi {
}
const innerTx = await this.$getRawTransaction(transaction.vin[i].txid, false, false);
transaction.vin[i].prevout = innerTx.vout[transaction.vin[i].vout];
this.addInnerScriptsToVin(transaction.vin[i]);
transactionUtils.addInnerScriptsToVin(transaction.vin[i]);
totalIn += innerTx.vout[transaction.vin[i].vout].value;
}
if (lazyPrevouts && transaction.vin.length > 12) {
@@ -331,122 +355,7 @@ class BitcoinApi implements AbstractBitcoinApi {
return transaction;
}
private convertScriptSigAsm(hex: string): string {
const buf = Buffer.from(hex, 'hex');
const b: string[] = [];
let i = 0;
while (i < buf.length) {
const op = buf[i];
if (op >= 0x01 && op <= 0x4e) {
i++;
let push: number;
if (op === 0x4c) {
push = buf.readUInt8(i);
b.push('OP_PUSHDATA1');
i += 1;
} else if (op === 0x4d) {
push = buf.readUInt16LE(i);
b.push('OP_PUSHDATA2');
i += 2;
} else if (op === 0x4e) {
push = buf.readUInt32LE(i);
b.push('OP_PUSHDATA4');
i += 4;
} else {
push = op;
b.push('OP_PUSHBYTES_' + push);
}
const data = buf.slice(i, i + push);
if (data.length !== push) {
break;
}
b.push(data.toString('hex'));
i += data.length;
} else {
if (op === 0x00) {
b.push('OP_0');
} else if (op === 0x4f) {
b.push('OP_PUSHNUM_NEG1');
} else if (op === 0xb1) {
b.push('OP_CLTV');
} else if (op === 0xb2) {
b.push('OP_CSV');
} else if (op === 0xba) {
b.push('OP_CHECKSIGADD');
} else {
const opcode = bitcoinjs.script.toASM([ op ]);
if (opcode && op < 0xfd) {
if (/^OP_(\d+)$/.test(opcode)) {
b.push(opcode.replace(/^OP_(\d+)$/, 'OP_PUSHNUM_$1'));
} else {
b.push(opcode);
}
} else {
b.push('OP_RETURN_' + op);
}
}
i += 1;
}
}
return b.join(' ');
}
private addInnerScriptsToVin(vin: IEsploraApi.Vin): void {
if (!vin.prevout) {
return;
}
if (vin.prevout.scriptpubkey_type === 'p2sh') {
const redeemScript = vin.scriptsig_asm.split(' ').reverse()[0];
vin.inner_redeemscript_asm = this.convertScriptSigAsm(redeemScript);
if (vin.witness && vin.witness.length > 2) {
const witnessScript = vin.witness[vin.witness.length - 1];
vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript);
}
}
if (vin.prevout.scriptpubkey_type === 'v0_p2wsh' && vin.witness) {
const witnessScript = vin.witness[vin.witness.length - 1];
vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript);
}
if (vin.prevout.scriptpubkey_type === 'v1_p2tr' && vin.witness) {
const witnessScript = this.witnessToP2TRScript(vin.witness);
if (witnessScript !== null) {
vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript);
}
}
}
/**
* This function must only be called when we know the witness we are parsing
* is a taproot witness.
* @param witness An array of hex strings that represents the witness stack of
* the input.
* @returns null if the witness is not a script spend, and the hex string of
* the script item if it is a script spend.
*/
private witnessToP2TRScript(witness: string[]): string | null {
if (witness.length < 2) return null;
// Note: see BIP341 for parsing details of witness stack
// If there are at least two witness elements, and the first byte of the
// last element is 0x50, this last element is called annex a and
// is removed from the witness stack.
const hasAnnex = witness[witness.length - 1].substring(0, 2) === '50';
// If there are at least two witness elements left, script path spending is used.
// Call the second-to-last stack element s, the script.
// (Note: this phrasing from BIP341 assumes we've *removed* the annex from the stack)
if (hasAnnex && witness.length < 3) return null;
const positionOfScript = hasAnnex ? witness.length - 3 : witness.length - 2;
return witness[positionOfScript];
}
public startHealthChecks(): void {};
}
export default BitcoinApi;

88
backend/src/api/bitcoin/bitcoin.routes.ts
View File

@@ -6,7 +6,7 @@ import websocketHandler from '../websocket-handler';
import mempool from '../mempool';
import feeApi from '../fee-api';
import mempoolBlocks from '../mempool-blocks';
import bitcoinApi, { bitcoinCoreApi } from './bitcoin-api-factory';
import bitcoinApi from './bitcoin-api-factory';
import { Common } from '../common';
import backendInfo from '../backend-info';
import transactionUtils from '../transaction-utils';
@@ -121,7 +121,8 @@ class BitcoinRoutes {
.get(config.MEMPOOL.API_URL_PREFIX + 'block-height/:height', this.getBlockHeight)
.get(config.MEMPOOL.API_URL_PREFIX + 'address/:address', this.getAddress)
.get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs', this.getAddressTransactions)
.get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs/chain/:txId', this.getAddressTransactions)
.get(config.MEMPOOL.API_URL_PREFIX + 'scripthash/:scripthash', this.getScriptHash)
.get(config.MEMPOOL.API_URL_PREFIX + 'scripthash/:scripthash/txs', this.getScriptHashTransactions)
.get(config.MEMPOOL.API_URL_PREFIX + 'address-prefix/:prefix', this.getAddressPrefix)
;
}
@@ -213,6 +214,7 @@ class BitcoinRoutes {
effectiveFeePerVsize: tx.effectiveFeePerVsize || null,
sigops: tx.sigops,
adjustedVsize: tx.adjustedVsize,
acceleration: tx.acceleration
});
return;
}
@@ -224,7 +226,12 @@ class BitcoinRoutes {
} else {
let cpfpInfo;
if (config.DATABASE.ENABLED) {
cpfpInfo = await transactionRepository.$getCpfpInfo(req.params.txId);
try {
cpfpInfo = await transactionRepository.$getCpfpInfo(req.params.txId);
} catch (e) {
res.status(500).send('failed to get CPFP info');
return;
}
}
if (cpfpInfo) {
res.json(cpfpInfo);
@@ -394,9 +401,13 @@ class BitcoinRoutes {
private async getBlockAuditSummary(req: Request, res: Response) {
try {
const transactions = await blocks.$getBlockAuditSummary(req.params.hash);
res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24 * 30).toUTCString());
res.json(transactions);
const auditSummary = await blocks.$getBlockAuditSummary(req.params.hash);
if (auditSummary) {
res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24 * 30).toUTCString());
res.json(auditSummary);
} else {
return res.status(404).send(`audit not available`);
}
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
@@ -467,13 +478,13 @@ class BitcoinRoutes {
}
let nextHash = startFromHash;
for (let i = 0; i < 10 && nextHash; i++) {
for (let i = 0; i < 15 && nextHash; i++) {
const localBlock = blocks.getBlocks().find((b) => b.id === nextHash);
if (localBlock) {
returnBlocks.push(localBlock);
nextHash = localBlock.previousblockhash;
} else {
const block = await bitcoinCoreApi.$getBlock(nextHash);
const block = await bitcoinApi.$getBlock(nextHash);
returnBlocks.push(block);
nextHash = block.previousblockhash;
}
@@ -537,15 +548,37 @@ class BitcoinRoutes {
}
}
private async getAddressTransactions(req: Request, res: Response) {
private async getAddressTransactions(req: Request, res: Response): Promise<void> {
if (config.MEMPOOL.BACKEND === 'none') {
res.status(405).send('Address lookups cannot be used with bitcoind as backend.');
return;
}
try {
const transactions = await bitcoinApi.$getAddressTransactions(req.params.address, req.params.txId);
let lastTxId: string = '';
if (req.query.after_txid && typeof req.query.after_txid === 'string') {
lastTxId = req.query.after_txid;
}
const transactions = await bitcoinApi.$getAddressTransactions(req.params.address, lastTxId);
res.json(transactions);
} catch (e) {
if (e instanceof Error && e.message && (e.message.indexOf('too long') > 0 || e.message.indexOf('confirmed status') > 0)) {
res.status(413).send(e instanceof Error ? e.message : e);
return;
}
res.status(500).send(e instanceof Error ? e.message : e);
}
}
private async getScriptHash(req: Request, res: Response) {
if (config.MEMPOOL.BACKEND === 'none') {
res.status(405).send('Address lookups cannot be used with bitcoind as backend.');
return;
}
try {
const addressData = await bitcoinApi.$getScriptHash(req.params.scripthash);
res.json(addressData);
} catch (e) {
if (e instanceof Error && e.message && (e.message.indexOf('too long') > 0 || e.message.indexOf('confirmed status') > 0)) {
return res.status(413).send(e instanceof Error ? e.message : e);
@@ -554,8 +587,26 @@ class BitcoinRoutes {
}
}
private async getAdressTxChain(req: Request, res: Response) {
res.status(501).send('Not implemented');
private async getScriptHashTransactions(req: Request, res: Response): Promise<void> {
if (config.MEMPOOL.BACKEND === 'none') {
res.status(405).send('Address lookups cannot be used with bitcoind as backend.');
return;
}
try {
let lastTxId: string = '';
if (req.query.after_txid && typeof req.query.after_txid === 'string') {
lastTxId = req.query.after_txid;
}
const transactions = await bitcoinApi.$getScriptHashTransactions(req.params.scripthash, lastTxId);
res.json(transactions);
} catch (e) {
if (e instanceof Error && e.message && (e.message.indexOf('too long') > 0 || e.message.indexOf('confirmed status') > 0)) {
res.status(413).send(e instanceof Error ? e.message : e);
return;
}
res.status(500).send(e instanceof Error ? e.message : e);
}
}
private async getAddressPrefix(req: Request, res: Response) {
@@ -714,12 +765,7 @@ class BitcoinRoutes {
private async $postTransaction(req: Request, res: Response) {
res.setHeader('content-type', 'text/plain');
try {
let rawTx;
if (typeof req.body === 'object') {
rawTx = Object.keys(req.body)[0];
} else {
rawTx = req.body;
}
const rawTx = Common.getTransactionFromRequest(req, false);
const txIdResult = await bitcoinApi.$sendRawTransaction(rawTx);
res.send(txIdResult);
} catch (e: any) {
@@ -730,12 +776,8 @@ class BitcoinRoutes {
private async $postTransactionForm(req: Request, res: Response) {
res.setHeader('content-type', 'text/plain');
const matches = /tx=([a-z0-9]+)/.exec(req.body);
let txHex = '';
if (matches && matches[1]) {
txHex = matches[1];
}
try {
const txHex = Common.getTransactionFromRequest(req, true);
const txIdResult = await bitcoinClient.sendRawTransaction(txHex);
res.send(txIdResult);
} catch (e: any) {

71
backend/src/api/bitcoin/electrum-api.ts
View File

@@ -126,6 +126,77 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
}
}
async $getScriptHash(scripthash: string): Promise<IEsploraApi.ScriptHash> {
try {
const balance = await this.electrumClient.blockchainScripthash_getBalance(scripthash);
let history = memoryCache.get<IElectrumApi.ScriptHashHistory[]>('Scripthash_getHistory', scripthash);
if (!history) {
history = await this.electrumClient.blockchainScripthash_getHistory(scripthash);
memoryCache.set('Scripthash_getHistory', scripthash, history, 2);
}
const unconfirmed = history ? history.filter((h) => h.fee).length : 0;
return {
'scripthash': scripthash,
'chain_stats': {
'funded_txo_count': 0,
'funded_txo_sum': balance.confirmed ? balance.confirmed : 0,
'spent_txo_count': 0,
'spent_txo_sum': balance.confirmed < 0 ? balance.confirmed : 0,
'tx_count': (history?.length || 0) - unconfirmed,
},
'mempool_stats': {
'funded_txo_count': 0,
'funded_txo_sum': balance.unconfirmed > 0 ? balance.unconfirmed : 0,
'spent_txo_count': 0,
'spent_txo_sum': balance.unconfirmed < 0 ? -balance.unconfirmed : 0,
'tx_count': unconfirmed,
},
'electrum': true,
};
} catch (e: any) {
throw new Error(typeof e === 'string' ? e : e && e.message || e);
}
}
async $getScriptHashTransactions(scripthash: string, lastSeenTxId?: string): Promise<IEsploraApi.Transaction[]> {
try {
loadingIndicators.setProgress('address-' + scripthash, 0);
const transactions: IEsploraApi.Transaction[] = [];
let history = memoryCache.get<IElectrumApi.ScriptHashHistory[]>('Scripthash_getHistory', scripthash);
if (!history) {
history = await this.electrumClient.blockchainScripthash_getHistory(scripthash);
memoryCache.set('Scripthash_getHistory', scripthash, history, 2);
}
if (!history) {
throw new Error('failed to get scripthash history');
}
history.sort((a, b) => (b.height || 9999999) - (a.height || 9999999));
let startingIndex = 0;
if (lastSeenTxId) {
const pos = history.findIndex((historicalTx) => historicalTx.tx_hash === lastSeenTxId);
if (pos) {
startingIndex = pos + 1;
}
}
const endIndex = Math.min(startingIndex + 10, history.length);
for (let i = startingIndex; i < endIndex; i++) {
const tx = await this.$getRawTransaction(history[i].tx_hash, false, true);
transactions.push(tx);
loadingIndicators.setProgress('address-' + scripthash, (i + 1) / endIndex * 100);
}
return transactions;
} catch (e: any) {
loadingIndicators.setProgress('address-' + scripthash, 100);
throw new Error(typeof e === 'string' ? e : e && e.message || e);
}
}
private $getScriptHashBalance(scriptHash: string): Promise<IElectrumApi.ScriptHashBalance> {
return this.electrumClient.blockchainScripthash_getBalance(this.encodeScriptHash(scriptHash));
}

8
backend/src/api/bitcoin/esplora-api.interface.ts
View File

@@ -89,6 +89,7 @@ export namespace IEsploraApi {
weight: number;
previousblockhash: string;
mediantime: number;
stale: boolean;
}
export interface Address {
@@ -98,6 +99,13 @@ export namespace IEsploraApi {
electrum?: boolean;
}
export interface ScriptHash {
scripthash: string;
chain_stats: ChainStats;
mempool_stats: MempoolStats;
electrum?: boolean;
}
export interface ChainStats {
funded_txo_count: number;
funded_txo_sum: number;

272
backend/src/api/bitcoin/esplora-api.ts
View File

@@ -1,104 +1,260 @@
import config from '../../config';
import axios, { AxiosRequestConfig } from 'axios';
import axios, { AxiosResponse } from 'axios';
import http from 'http';
import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
import { IEsploraApi } from './esplora-api.interface';
import logger from '../../logger';
const axiosConnection = axios.create({
httpAgent: new http.Agent({ keepAlive: true, })
});
interface FailoverHost {
host: string,
rtts: number[],
rtt: number
failures: number,
socket?: boolean,
outOfSync?: boolean,
unreachable?: boolean,
preferred?: boolean,
}
class ElectrsApi implements AbstractBitcoinApi {
private axiosConfigWithUnixSocket: AxiosRequestConfig = config.ESPLORA.UNIX_SOCKET_PATH ? {
socketPath: config.ESPLORA.UNIX_SOCKET_PATH,
timeout: 10000,
} : {
timeout: 10000,
};
private axiosConfigTcpSocketOnly: AxiosRequestConfig = {
timeout: 10000,
};
unixSocketRetryTimeout;
activeAxiosConfig;
class FailoverRouter {
activeHost: FailoverHost;
fallbackHost: FailoverHost;
hosts: FailoverHost[];
multihost: boolean;
pollInterval: number = 60000;
pollTimer: NodeJS.Timeout | null = null;
pollConnection = axios.create();
requestConnection = axios.create({
httpAgent: new http.Agent({ keepAlive: true })
});
constructor() {
this.activeAxiosConfig = this.axiosConfigWithUnixSocket;
// setup list of hosts
this.hosts = (config.ESPLORA.FALLBACK || []).map(domain => {
return {
host: domain,
rtts: [],
rtt: Infinity,
failures: 0,
};
});
this.activeHost = {
host: config.ESPLORA.UNIX_SOCKET_PATH || config.ESPLORA.REST_API_URL,
rtts: [],
rtt: 0,
failures: 0,
socket: !!config.ESPLORA.UNIX_SOCKET_PATH,
preferred: true,
};
this.fallbackHost = this.activeHost;
this.hosts.unshift(this.activeHost);
this.multihost = this.hosts.length > 1;
}
fallbackToTcpSocket() {
if (!this.unixSocketRetryTimeout) {
logger.err(`Unable to connect to esplora unix socket. Falling back to tcp socket. Retrying unix socket in ${config.ESPLORA.RETRY_UNIX_SOCKET_AFTER / 1000} seconds`);
// Retry the unix socket after a few seconds
this.unixSocketRetryTimeout = setTimeout(() => {
logger.info(`Retrying to use unix socket for esplora now (applied for the next query)`);
this.activeAxiosConfig = this.axiosConfigWithUnixSocket;
this.unixSocketRetryTimeout = undefined;
}, config.ESPLORA.RETRY_UNIX_SOCKET_AFTER);
public startHealthChecks(): void {
// use axios interceptors to measure request rtt
this.pollConnection.interceptors.request.use((config) => {
config['meta'] = { startTime: Date.now() };
return config;
});
this.pollConnection.interceptors.response.use((response) => {
response.config['meta'].rtt = Date.now() - response.config['meta'].startTime;
return response;
});
if (this.multihost) {
this.pollHosts();
}
}
// start polling hosts to measure availability & rtt
private async pollHosts(): Promise<void> {
if (this.pollTimer) {
clearTimeout(this.pollTimer);
}
// Use the TCP socket (reach a different esplora instance through nginx)
this.activeAxiosConfig = this.axiosConfigTcpSocketOnly;
const results = await Promise.allSettled(this.hosts.map(async (host) => {
if (host.socket) {
return this.pollConnection.get<number>('/blocks/tip/height', { socketPath: host.host, timeout: config.ESPLORA.FALLBACK_TIMEOUT });
} else {
return this.pollConnection.get<number>(host.host + '/blocks/tip/height', { timeout: config.ESPLORA.FALLBACK_TIMEOUT });
}
}));
const maxHeight = results.reduce((max, result) => Math.max(max, result.status === 'fulfilled' ? result.value?.data || 0 : 0), 0);
// update rtts & sync status
for (let i = 0; i < results.length; i++) {
const host = this.hosts[i];
const result = results[i].status === 'fulfilled' ? (results[i] as PromiseFulfilledResult<AxiosResponse<number, any>>).value : null;
if (result) {
const height = result.data;
const rtt = result.config['meta'].rtt;
host.rtts.unshift(rtt);
host.rtts.slice(0, 5);
host.rtt = host.rtts.reduce((acc, l) => acc + l, 0) / host.rtts.length;
if (height == null || isNaN(height) || (maxHeight - height > 2)) {
host.outOfSync = true;
} else {
host.outOfSync = false;
}
host.unreachable = false;
} else {
host.unreachable = true;
}
}
this.sortHosts();
logger.debug(`Tomahawk ranking: ${this.hosts.map(host => '\navg rtt ' + Math.round(host.rtt).toString().padStart(5, ' ') + ' | reachable? ' + (!host.unreachable || false).toString().padStart(5, ' ') + ' | in sync? ' + (!host.outOfSync || false).toString().padStart(5, ' ') + ` | ${host.host}`).join('')}`);
// switch if the current host is out of sync or significantly slower than the next best alternative
if (this.activeHost.outOfSync || this.activeHost.unreachable || (this.activeHost !== this.hosts[0] && this.hosts[0].preferred) || (!this.activeHost.preferred && this.activeHost.rtt > (this.hosts[0].rtt * 2) + 50)) {
if (this.activeHost.unreachable) {
logger.warn(`Unable to reach ${this.activeHost.host}, failing over to next best alternative`);
} else if (this.activeHost.outOfSync) {
logger.warn(`${this.activeHost.host} has fallen behind, failing over to next best alternative`);
} else {
logger.debug(`${this.activeHost.host} is no longer the best esplora host`);
}
this.electHost();
}
this.pollTimer = setTimeout(() => { this.pollHosts(); }, this.pollInterval);
}
$queryWrapper<T>(url, responseType = 'json'): Promise<T> {
return axiosConnection.get<T>(url, { ...this.activeAxiosConfig, responseType: responseType })
.then((response) => response.data)
// sort hosts by connection quality, and update default fallback
private sortHosts(): void {
// sort by connection quality
this.hosts.sort((a, b) => {
if ((a.unreachable || a.outOfSync) === (b.unreachable || b.outOfSync)) {
if (a.preferred === b.preferred) {
// lower rtt is best
return a.rtt - b.rtt;
} else { // unless we have a preferred host
return a.preferred ? -1 : 1;
}
} else { // or the host is out of sync
return (a.unreachable || a.outOfSync) ? 1 : -1;
}
});
if (this.hosts.length > 1 && this.hosts[0] === this.activeHost) {
this.fallbackHost = this.hosts[1];
} else {
this.fallbackHost = this.hosts[0];
}
}
// depose the active host and choose the next best replacement
private electHost(): void {
this.activeHost.outOfSync = true;
this.activeHost.failures = 0;
this.sortHosts();
this.activeHost = this.hosts[0];
logger.warn(`Switching esplora host to ${this.activeHost.host}`);
}
private addFailure(host: FailoverHost): FailoverHost {
host.failures++;
if (host.failures > 5 && this.multihost) {
logger.warn(`Too many esplora failures on ${this.activeHost.host}, falling back to next best alternative`);
this.electHost();
return this.activeHost;
} else {
return this.fallbackHost;
}
}
private async $query<T>(method: 'get'| 'post', path, data: any, responseType = 'json', host = this.activeHost, retry: boolean = true): Promise<T> {
let axiosConfig;
let url;
if (host.socket) {
axiosConfig = { socketPath: host.host, timeout: config.ESPLORA.REQUEST_TIMEOUT, responseType };
url = path;
} else {
axiosConfig = { timeout: config.ESPLORA.REQUEST_TIMEOUT, responseType };
url = host.host + path;
}
return (method === 'post'
? this.requestConnection.post<T>(url, data, axiosConfig)
: this.requestConnection.get<T>(url, axiosConfig)
).then((response) => { host.failures = Math.max(0, host.failures - 1); return response.data; })
.catch((e) => {
if (e?.code === 'ECONNREFUSED') {
this.fallbackToTcpSocket();
let fallbackHost = this.fallbackHost;
if (e?.response?.status !== 404) {
logger.warn(`esplora request failed ${e?.response?.status || 500} ${host.host}${path}`);
fallbackHost = this.addFailure(host);
}
if (retry && e?.code === 'ECONNREFUSED' && this.multihost) {
// Retry immediately
return axiosConnection.get<T>(url, this.activeAxiosConfig)
.then((response) => response.data)
.catch((e) => {
logger.warn(`Cannot query esplora through the unix socket nor the tcp socket. Exception ${e}`);
throw e;
});
return this.$query(method, path, data, responseType, fallbackHost, false);
} else {
throw e;
}
});
}
public async $get<T>(path, responseType = 'json'): Promise<T> {
return this.$query<T>('get', path, null, responseType);
}
public async $post<T>(path, data: any, responseType = 'json'): Promise<T> {
return this.$query<T>('post', path, data, responseType);
}
}
class ElectrsApi implements AbstractBitcoinApi {
private failoverRouter = new FailoverRouter();
$getRawMempool(): Promise<IEsploraApi.Transaction['txid'][]> {
return this.$queryWrapper<IEsploraApi.Transaction['txid'][]>(config.ESPLORA.REST_API_URL + '/mempool/txids');
return this.failoverRouter.$get<IEsploraApi.Transaction['txid'][]>('/mempool/txids');
}
$getRawTransaction(txId: string): Promise<IEsploraApi.Transaction> {
return this.$queryWrapper<IEsploraApi.Transaction>(config.ESPLORA.REST_API_URL + '/tx/' + txId);
return this.failoverRouter.$get<IEsploraApi.Transaction>('/tx/' + txId);
}
async $getMempoolTransactions(txids: string[]): Promise<IEsploraApi.Transaction[]> {
return this.failoverRouter.$post<IEsploraApi.Transaction[]>('/mempool/txs', txids, 'json');
}
async $getAllMempoolTransactions(lastSeenTxid?: string): Promise<IEsploraApi.Transaction[]> {
return this.failoverRouter.$get<IEsploraApi.Transaction[]>('/mempool/txs' + (lastSeenTxid ? '/' + lastSeenTxid : ''));
}
$getTransactionHex(txId: string): Promise<string> {
return this.$queryWrapper<string>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/hex');
return this.failoverRouter.$get<string>('/tx/' + txId + '/hex');
}
$getBlockHeightTip(): Promise<number> {
return this.$queryWrapper<number>(config.ESPLORA.REST_API_URL + '/blocks/tip/height');
return this.failoverRouter.$get<number>('/blocks/tip/height');
}
$getBlockHashTip(): Promise<string> {
return this.$queryWrapper<string>(config.ESPLORA.REST_API_URL + '/blocks/tip/hash');
return this.failoverRouter.$get<string>('/blocks/tip/hash');
}
$getTxIdsForBlock(hash: string): Promise<string[]> {
return this.$queryWrapper<string[]>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/txids');
return this.failoverRouter.$get<string[]>('/block/' + hash + '/txids');
}
$getTxsForBlock(hash: string): Promise<IEsploraApi.Transaction[]> {
return this.failoverRouter.$get<IEsploraApi.Transaction[]>('/block/' + hash + '/txs');
}
$getBlockHash(height: number): Promise<string> {
return this.$queryWrapper<string>(config.ESPLORA.REST_API_URL + '/block-height/' + height);
return this.failoverRouter.$get<string>('/block-height/' + height);
}
$getBlockHeader(hash: string): Promise<string> {
return this.$queryWrapper<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/header');
return this.failoverRouter.$get<string>('/block/' + hash + '/header');
}
$getBlock(hash: string): Promise<IEsploraApi.Block> {
return this.$queryWrapper<IEsploraApi.Block>(config.ESPLORA.REST_API_URL + '/block/' + hash);
return this.failoverRouter.$get<IEsploraApi.Block>('/block/' + hash);
}
$getRawBlock(hash: string): Promise<Buffer> {
return this.$queryWrapper<any>(config.ESPLORA.REST_API_URL + '/block/' + hash + "/raw", 'arraybuffer')
return this.failoverRouter.$get<any>('/block/' + hash + '/raw', 'arraybuffer')
.then((response) => { return Buffer.from(response.data); });
}
@@ -110,6 +266,14 @@ class ElectrsApi implements AbstractBitcoinApi {
throw new Error('Method getAddressTransactions not implemented.');
}
$getScriptHash(scripthash: string): Promise<IEsploraApi.ScriptHash> {
throw new Error('Method getScriptHash not implemented.');
}
$getScriptHashTransactions(scripthash: string, txId?: string): Promise<IEsploraApi.Transaction[]> {
throw new Error('Method getScriptHashTransactions not implemented.');
}
$getAddressPrefix(prefix: string): string[] {
throw new Error('Method not implemented.');
}
@@ -119,11 +283,11 @@ class ElectrsApi implements AbstractBitcoinApi {
}
$getOutspend(txId: string, vout: number): Promise<IEsploraApi.Outspend> {
return this.$queryWrapper<IEsploraApi.Outspend>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspend/' + vout);
return this.failoverRouter.$get<IEsploraApi.Outspend>('/tx/' + txId + '/outspend/' + vout);
}
$getOutspends(txId: string): Promise<IEsploraApi.Outspend[]> {
return this.$queryWrapper<IEsploraApi.Outspend[]>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspends');
return this.failoverRouter.$get<IEsploraApi.Outspend[]>('/tx/' + txId + '/outspends');
}
async $getBatchedOutspends(txId: string[]): Promise<IEsploraApi.Outspend[][]> {
@@ -134,6 +298,10 @@ class ElectrsApi implements AbstractBitcoinApi {
}
return outspends;
}
public startHealthChecks(): void {
this.failoverRouter.startHealthChecks();
}
}
export default ElectrsApi;

414
backend/src/api/blocks.ts
View File

@@ -25,16 +25,20 @@ import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmen
import PricesRepository from '../repositories/PricesRepository';
import priceUpdater from '../tasks/price-updater';
import chainTips from './chain-tips';
import websocketHandler from './websocket-handler';
import redisCache from './redis-cache';
import rbfCache from './rbf-cache';
import { calcBitsDifference } from './difficulty-adjustment';
class Blocks {
private blocks: BlockExtended[] = [];
private blockSummaries: BlockSummary[] = [];
private currentBlockHeight = 0;
private currentDifficulty = 0;
private currentBits = 0;
private lastDifficultyAdjustmentTime = 0;
private previousDifficultyRetarget = 0;
private newBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => void)[] = [];
private newAsyncBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => Promise<void>)[] = [];
private newAsyncBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: MempoolTransactionExtended[]) => Promise<void>)[] = [];
private mainLoopTimeout: number = 120000;
@@ -60,7 +64,7 @@ class Blocks {
this.newBlockCallbacks.push(fn);
}
public setNewAsyncBlockCallback(fn: (block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => Promise<void>) {
public setNewAsyncBlockCallback(fn: (block: BlockExtended, txIds: string[], transactions: MempoolTransactionExtended[]) => Promise<void>) {
this.newAsyncBlockCallbacks.push(fn);
}
@@ -69,69 +73,111 @@ class Blocks {
* @param blockHash
* @param blockHeight
* @param onlyCoinbase - Set to true if you only need the coinbase transaction
* @param txIds - optional ordered list of transaction ids if already known
* @param quiet - don't print non-essential logs
* @param addMempoolData - calculate sigops etc
* @returns Promise<TransactionExtended[]>
*/
private async $getTransactionsExtended(
blockHash: string,
blockHeight: number,
onlyCoinbase: boolean,
txIds: string[] | null = null,
quiet: boolean = false,
addMempoolData: boolean = false,
): Promise<TransactionExtended[]> {
const transactions: TransactionExtended[] = [];
const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash);
const isEsplora = config.MEMPOOL.BACKEND === 'esplora';
const transactionMap: { [txid: string]: TransactionExtended } = {};
if (!txIds) {
txIds = await bitcoinApi.$getTxIdsForBlock(blockHash);
}
const mempool = memPool.getMempool();
let transactionsFound = 0;
let transactionsFetched = 0;
let foundInMempool = 0;
let totalFound = 0;
for (let i = 0; i < txIds.length; i++) {
if (mempool[txIds[i]]) {
// We update blocks before the mempool (index.ts), therefore we can
// optimize here by directly fetching txs in the "outdated" mempool
transactions.push(mempool[txIds[i]]);
transactionsFound++;
} else if (config.MEMPOOL.BACKEND === 'esplora' || !memPool.hasPriority() || i === 0) {
// Otherwise we fetch the tx data through backend services (esplora, electrum, core rpc...)
if (!quiet && (i % (Math.round((txIds.length) / 10)) === 0 || i + 1 === txIds.length)) { // Avoid log spam
logger.debug(`Indexing tx ${i + 1} of ${txIds.length} in block #${blockHeight}`);
}
try {
const tx = await transactionUtils.$getTransactionExtended(txIds[i], false, false, false, addMempoolData);
transactions.push(tx);
transactionsFetched++;
} catch (e) {
try {
if (config.MEMPOOL.BACKEND === 'esplora') {
// Try again with core
const tx = await transactionUtils.$getTransactionExtended(txIds[i], false, false, true, addMempoolData);
transactions.push(tx);
transactionsFetched++;
} else {
throw e;
}
} catch (e) {
if (i === 0) {
const msg = `Cannot fetch coinbase tx ${txIds[i]}. Reason: ` + (e instanceof Error ? e.message : e);
logger.err(msg);
throw new Error(msg);
} else {
logger.err(`Cannot fetch tx ${txIds[i]}. Reason: ` + (e instanceof Error ? e.message : e));
}
}
// Copy existing transactions from the mempool
if (!onlyCoinbase) {
for (const txid of txIds) {
if (mempool[txid]) {
transactionMap[txid] = mempool[txid];
foundInMempool++;
totalFound++;
}
}
}
if (onlyCoinbase === true) {
break; // Fetch the first transaction and exit
if (onlyCoinbase) {
try {
const coinbase = await transactionUtils.$getTransactionExtendedRetry(txIds[0], false, false, false, addMempoolData);
if (coinbase && coinbase.vin[0].is_coinbase) {
return [coinbase];
} else {
const msg = `Expected a coinbase tx, but the backend API returned something else`;
logger.err(msg);
throw new Error(msg);
}
} catch (e) {
const msg = `Cannot fetch coinbase tx ${txIds[0]}. Reason: ` + (e instanceof Error ? e.message : e);
logger.err(msg);
throw new Error(msg);
}
}
// Fetch remaining txs in bulk
if (isEsplora && (txIds.length - totalFound > 500)) {
try {
const rawTransactions = await bitcoinApi.$getTxsForBlock(blockHash);
for (const tx of rawTransactions) {
if (!transactionMap[tx.txid]) {
transactionMap[tx.txid] = addMempoolData ? transactionUtils.extendMempoolTransaction(tx) : transactionUtils.extendTransaction(tx);
totalFound++;
}
}
} catch (e) {
logger.err(`Cannot fetch bulk txs for block ${blockHash}. Reason: ` + (e instanceof Error ? e.message : e));
}
}
// Fetch remaining txs individually
for (const txid of txIds.filter(txid => !transactionMap[txid])) {
if (!quiet && (totalFound % (Math.round((txIds.length) / 10)) === 0 || totalFound + 1 === txIds.length)) { // Avoid log spam
logger.debug(`Indexing tx ${totalFound + 1} of ${txIds.length} in block #${blockHeight}`);
}
try {
const tx = await transactionUtils.$getTransactionExtendedRetry(txid, false, false, false, addMempoolData);
transactionMap[txid] = tx;
totalFound++;
} catch (e) {
const msg = `Cannot fetch tx ${txid}. Reason: ` + (e instanceof Error ? e.message : e);
logger.err(msg);
throw new Error(msg);
}
}
if (!quiet) {
logger.debug(`${transactionsFound} of ${txIds.length} found in mempool. ${transactionsFetched} fetched through backend service.`);
logger.debug(`${foundInMempool} of ${txIds.length} found in mempool. ${totalFound - foundInMempool} fetched through backend service.`);
}
return transactions;
// Require the first transaction to be a coinbase
const coinbase = transactionMap[txIds[0]];
if (!coinbase || !coinbase.vin[0].is_coinbase) {
const msg = `Expected first tx in a block to be a coinbase, but found something else`;
logger.err(msg);
throw new Error(msg);
}
// Require all transactions to be present
// (we should have thrown an error already if a tx request failed)
if (txIds.some(txid => !transactionMap[txid])) {
const msg = `Failed to fetch ${txIds.length - totalFound} transactions from block`;
logger.err(msg);
throw new Error(msg);
}
// Return list of transactions, preserving block order
return txIds.map(txid => transactionMap[txid]);
}
/**
@@ -158,9 +204,18 @@ class Blocks {
};
}
public summarizeBlockTransactions(hash: string, transactions: TransactionExtended[]): BlockSummary {
return {
id: hash,
transactions: Common.stripTransactions(transactions),
};
}
private convertLiquidFees(block: IBitcoinApi.VerboseBlock): IBitcoinApi.VerboseBlock {
block.tx.forEach(tx => {
tx.fee = Object.values(tx.fee || {}).reduce((total, output) => total + output, 0);
if (!isFinite(Number(tx.fee))) {
tx.fee = Object.values(tx.fee || {}).reduce((total, output) => total + output, 0);
}
});
return block;
}
@@ -365,8 +420,8 @@ class Blocks {
let newlyIndexed = 0;
let totalIndexed = indexedBlockSummariesHashesArray.length;
let indexedThisRun = 0;
let timer = new Date().getTime() / 1000;
const startedAt = new Date().getTime() / 1000;
let timer = Date.now() / 1000;
const startedAt = Date.now() / 1000;
for (const block of indexedBlocks) {
if (indexedBlockSummariesHashes[block.hash] === true) {
@@ -374,17 +429,24 @@ class Blocks {
}
// Logging
const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
const elapsedSeconds = (Date.now() / 1000) - timer;
if (elapsedSeconds > 5) {
const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
const blockPerSeconds = Math.max(1, indexedThisRun / elapsedSeconds);
const runningFor = (Date.now() / 1000) - startedAt;
const blockPerSeconds = indexedThisRun / elapsedSeconds;
const progress = Math.round(totalIndexed / indexedBlocks.length * 10000) / 100;
logger.debug(`Indexing block summary for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexedBlocks.length} (${progress}%) | elapsed: ${runningFor} seconds`, logger.tags.mining);
timer = new Date().getTime() / 1000;
logger.debug(`Indexing block summary for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexedBlocks.length} (${progress}%) | elapsed: ${runningFor.toFixed(2)} seconds`, logger.tags.mining);
timer = Date.now() / 1000;
indexedThisRun = 0;
}
await this.$getStrippedBlockTransactions(block.hash, true, true); // This will index the block summary
if (config.MEMPOOL.BACKEND === 'esplora') {
const txs = (await bitcoinApi.$getTxsForBlock(block.hash)).map(tx => transactionUtils.extendTransaction(tx));
const cpfpSummary = await this.$indexCPFP(block.hash, block.height, txs);
await this.$getStrippedBlockTransactions(block.hash, true, true, cpfpSummary, block.height); // This will index the block summary
} else {
await this.$getStrippedBlockTransactions(block.hash, true, true); // This will index the block summary
}
// Logging
indexedThisRun++;
@@ -423,18 +485,18 @@ class Blocks {
// Logging
let count = 0;
let countThisRun = 0;
let timer = new Date().getTime() / 1000;
const startedAt = new Date().getTime() / 1000;
let timer = Date.now() / 1000;
const startedAt = Date.now() / 1000;
for (const height of unindexedBlockHeights) {
// Logging
const hash = await bitcoinApi.$getBlockHash(height);
const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer);
const elapsedSeconds = (Date.now() / 1000) - timer;
if (elapsedSeconds > 5) {
const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
const blockPerSeconds = (countThisRun / elapsedSeconds);
const runningFor = (Date.now() / 1000) - startedAt;
const blockPerSeconds = countThisRun / elapsedSeconds;
const progress = Math.round(count / unindexedBlockHeights.length * 10000) / 100;
logger.debug(`Indexing cpfp clusters for #${height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${count}/${unindexedBlockHeights.length} (${progress}%) | elapsed: ${runningFor} seconds`);
timer = new Date().getTime() / 1000;
logger.debug(`Indexing cpfp clusters for #${height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${count}/${unindexedBlockHeights.length} (${progress}%) | elapsed: ${runningFor.toFixed(2)} seconds`);
timer = Date.now() / 1000;
countThisRun = 0;
}
@@ -513,8 +575,8 @@ class Blocks {
let totalIndexed = await blocksRepository.$blockCountBetweenHeight(currentBlockHeight, lastBlockToIndex);
let indexedThisRun = 0;
let newlyIndexed = 0;
const startedAt = new Date().getTime() / 1000;
let timer = new Date().getTime() / 1000;
const startedAt = Date.now() / 1000;
let timer = Date.now() / 1000;
while (currentBlockHeight >= lastBlockToIndex) {
const endBlock = Math.max(0, lastBlockToIndex, currentBlockHeight - chunkSize + 1);
@@ -534,19 +596,19 @@ class Blocks {
}
++indexedThisRun;
++totalIndexed;
const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer);
const elapsedSeconds = (Date.now() / 1000) - timer;
if (elapsedSeconds > 5 || blockHeight === lastBlockToIndex) {
const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
const blockPerSeconds = Math.max(1, indexedThisRun / elapsedSeconds);
const runningFor = (Date.now() / 1000) - startedAt;
const blockPerSeconds = indexedThisRun / elapsedSeconds;
const progress = Math.round(totalIndexed / indexingBlockAmount * 10000) / 100;
logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${runningFor} seconds`, logger.tags.mining);
timer = new Date().getTime() / 1000;
logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexingBlockAmount} (${progress.toFixed(2)}%) | elapsed: ${runningFor.toFixed(2)} seconds`, logger.tags.mining);
timer = Date.now() / 1000;
indexedThisRun = 0;
loadingIndicators.setProgress('block-indexing', progress, false);
}
const blockHash = await bitcoinApi.$getBlockHash(blockHeight);
const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash);
const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, true);
const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash);
const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, null, true);
const blockExtended = await this.$getBlockExtended(block, transactions);
newlyIndexed++;
@@ -578,7 +640,7 @@ class Blocks {
let fastForwarded = false;
let handledBlocks = 0;
const blockHeightTip = await bitcoinApi.$getBlockHeightTip();
const blockHeightTip = await bitcoinCoreApi.$getBlockHeightTip();
this.updateTimerProgress(timer, 'got block height tip');
if (this.blocks.length === 0) {
@@ -602,17 +664,21 @@ class Blocks {
const heightDiff = blockHeightTip % 2016;
const blockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff);
this.updateTimerProgress(timer, 'got block hash for initial difficulty adjustment');
const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash);
const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash);
this.updateTimerProgress(timer, 'got block for initial difficulty adjustment');
this.lastDifficultyAdjustmentTime = block.timestamp;
this.currentDifficulty = block.difficulty;
this.currentBits = block.bits;
if (blockHeightTip >= 2016) {
const previousPeriodBlockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff - 2016);
this.updateTimerProgress(timer, 'got previous block hash for initial difficulty adjustment');
const previousPeriodBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(previousPeriodBlockHash);
const previousPeriodBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(previousPeriodBlockHash);
this.updateTimerProgress(timer, 'got previous block for initial difficulty adjustment');
this.previousDifficultyRetarget = (block.difficulty - previousPeriodBlock.difficulty) / previousPeriodBlock.difficulty * 100;
if (['liquid', 'liquidtestnet'].includes(config.MEMPOOL.NETWORK)) {
this.previousDifficultyRetarget = NaN;
} else {
this.previousDifficultyRetarget = calcBitsDifference(previousPeriodBlock.bits, block.bits);
}
logger.debug(`Initial difficulty adjustment data set.`);
}
} else {
@@ -631,28 +697,24 @@ class Blocks {
}
this.updateTimerProgress(timer, `getting block data for ${this.currentBlockHeight}`);
const blockHash = await bitcoinApi.$getBlockHash(this.currentBlockHeight);
const blockHash = await bitcoinCoreApi.$getBlockHash(this.currentBlockHeight);
const verboseBlock = await bitcoinClient.getBlock(blockHash, 2);
const block = BitcoinApi.convertBlock(verboseBlock);
const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash);
const transactions = await this.$getTransactionsExtended(blockHash, block.height, false, false, true);
if (config.MEMPOOL.BACKEND !== 'esplora') {
// fill in missing transaction fee data from verboseBlock
for (let i = 0; i < transactions.length; i++) {
if (!transactions[i].fee && transactions[i].txid === verboseBlock.tx[i].txid) {
transactions[i].fee = verboseBlock.tx[i].fee * 100_000_000;
}
const txIds: string[] = verboseBlock.tx.map(tx => tx.txid);
const transactions = await this.$getTransactionsExtended(blockHash, block.height, false, txIds, false, true) as MempoolTransactionExtended[];
// fill in missing transaction fee data from verboseBlock
for (let i = 0; i < transactions.length; i++) {
if (!transactions[i].fee && transactions[i].txid === verboseBlock.tx[i].txid) {
transactions[i].fee = (verboseBlock.tx[i].fee * 100_000_000) || 0;
}
}
const cpfpSummary: CpfpSummary = Common.calculateCpfp(block.height, transactions);
const blockExtended: BlockExtended = await this.$getBlockExtended(block, cpfpSummary.transactions);
const blockSummary: BlockSummary = this.summarizeBlock(verboseBlock);
const blockSummary: BlockSummary = this.summarizeBlockTransactions(block.id, cpfpSummary.transactions);
this.updateTimerProgress(timer, `got block data for ${this.currentBlockHeight}`);
// start async callbacks
this.updateTimerProgress(timer, `starting async callbacks for ${this.currentBlockHeight}`);
const callbackPromises = this.newAsyncBlockCallbacks.map((cb) => cb(blockExtended, txIds, transactions));
if (Common.indexingEnabled()) {
if (!fastForwarded) {
const lastBlock = await blocksRepository.$getBlockByHeight(blockExtended.height - 1);
@@ -664,22 +726,27 @@ class Blocks {
await BlocksRepository.$deleteBlocksFrom(lastBlock.height - 10);
await HashratesRepository.$deleteLastEntries();
await cpfpRepository.$deleteClustersFrom(lastBlock.height - 10);
this.blocks = this.blocks.slice(0, -10);
this.updateTimerProgress(timer, `rolled back chain divergence from ${this.currentBlockHeight}`);
for (let i = 10; i >= 0; --i) {
const newBlock = await this.$indexBlock(lastBlock.height - i);
this.blocks.push(newBlock);
this.updateTimerProgress(timer, `reindexed block`);
await this.$getStrippedBlockTransactions(newBlock.id, true, true);
this.updateTimerProgress(timer, `reindexed block summary`);
let cpfpSummary;
if (config.MEMPOOL.CPFP_INDEXING) {
await this.$indexCPFP(newBlock.id, lastBlock.height - i);
cpfpSummary = await this.$indexCPFP(newBlock.id, lastBlock.height - i);
this.updateTimerProgress(timer, `reindexed block cpfp`);
}
await this.$getStrippedBlockTransactions(newBlock.id, true, true, cpfpSummary, newBlock.height);
this.updateTimerProgress(timer, `reindexed block summary`);
}
await mining.$indexDifficultyAdjustments();
await DifficultyAdjustmentsRepository.$deleteLastAdjustment();
this.updateTimerProgress(timer, `reindexed difficulty adjustments`);
logger.info(`Re-indexed 10 blocks and summaries. Also re-indexed the last difficulty adjustments. Will re-index latest hashrates in a few seconds.`, logger.tags.mining);
indexer.reindex();
websocketHandler.handleReorg();
}
}
@@ -704,7 +771,7 @@ class Blocks {
// Save blocks summary for visualization if it's enabled
if (Common.blocksSummariesIndexingEnabled() === true) {
await this.$getStrippedBlockTransactions(blockExtended.id, true);
await this.$getStrippedBlockTransactions(blockExtended.id, true, false, cpfpSummary, blockExtended.height);
this.updateTimerProgress(timer, `saved block summary for ${this.currentBlockHeight}`);
}
if (config.MEMPOOL.CPFP_INDEXING) {
@@ -714,22 +781,46 @@ class Blocks {
}
}
// start async callbacks
this.updateTimerProgress(timer, `starting async callbacks for ${this.currentBlockHeight}`);
const callbackPromises = this.newAsyncBlockCallbacks.map((cb) => cb(blockExtended, txIds, transactions));
if (block.height % 2016 === 0) {
if (Common.indexingEnabled()) {
let adjustment;
if (['liquid', 'liquidtestnet'].includes(config.MEMPOOL.NETWORK)) {
adjustment = NaN;
} else {
adjustment = Math.round(
// calcBitsDifference returns +- percentage, +100 returns to positive, /100 returns to ratio.
// Instead of actually doing /100, just reduce the multiplier.
(calcBitsDifference(this.currentBits, block.bits) + 100) * 10000
) / 1000000; // Remove float point noise
}
await DifficultyAdjustmentsRepository.$saveAdjustments({
time: block.timestamp,
height: block.height,
difficulty: block.difficulty,
adjustment: Math.round((block.difficulty / this.currentDifficulty) * 1000000) / 1000000, // Remove float point noise
adjustment,
});
this.updateTimerProgress(timer, `saved difficulty adjustment for ${this.currentBlockHeight}`);
}
this.previousDifficultyRetarget = (block.difficulty - this.currentDifficulty) / this.currentDifficulty * 100;
if (['liquid', 'liquidtestnet'].includes(config.MEMPOOL.NETWORK)) {
this.previousDifficultyRetarget = NaN;
} else {
this.previousDifficultyRetarget = calcBitsDifference(this.currentBits, block.bits);
}
this.lastDifficultyAdjustmentTime = block.timestamp;
this.currentDifficulty = block.difficulty;
this.currentBits = block.bits;
}
// wait for pending async callbacks to finish
this.updateTimerProgress(timer, `waiting for async callbacks to complete for ${this.currentBlockHeight}`);
await Promise.all(callbackPromises);
this.updateTimerProgress(timer, `async callbacks completed for ${this.currentBlockHeight}`);
this.blocks.push(blockExtended);
if (this.blocks.length > config.MEMPOOL.INITIAL_BLOCKS_AMOUNT * 4) {
this.blocks = this.blocks.slice(-config.MEMPOOL.INITIAL_BLOCKS_AMOUNT * 4);
@@ -742,14 +833,17 @@ class Blocks {
if (this.newBlockCallbacks.length) {
this.newBlockCallbacks.forEach((cb) => cb(blockExtended, txIds, transactions));
}
if (!memPool.hasPriority() && (block.height % config.MEMPOOL.DISK_CACHE_BLOCK_INTERVAL === 0)) {
if (config.MEMPOOL.CACHE_ENABLED && !memPool.hasPriority() && (block.height % config.MEMPOOL.DISK_CACHE_BLOCK_INTERVAL === 0)) {
diskCache.$saveCacheToDisk();
}
// wait for pending async callbacks to finish
this.updateTimerProgress(timer, `waiting for async callbacks to complete for ${this.currentBlockHeight}`);
await Promise.all(callbackPromises);
this.updateTimerProgress(timer, `async callbacks completed for ${this.currentBlockHeight}`);
// Update Redis cache
if (config.REDIS.ENABLED) {
await redisCache.$updateBlocks(this.blocks);
await redisCache.$updateBlockSummaries(this.blockSummaries);
await redisCache.$removeTransactions(txIds);
await rbfCache.updateCache();
}
handledBlocks++;
}
@@ -795,7 +889,7 @@ class Blocks {
}
const blockHash = await bitcoinApi.$getBlockHash(height);
const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash);
const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash);
const transactions = await this.$getTransactionsExtended(blockHash, block.height, true);
const blockExtended = await this.$getBlockExtended(block, transactions);
@@ -806,6 +900,16 @@ class Blocks {
return blockExtended;
}
public async $indexStaleBlock(hash: string): Promise<BlockExtended> {
const block: IEsploraApi.Block = await bitcoinApi.$getBlock(hash);
const transactions = await this.$getTransactionsExtended(hash, block.height, true);
const blockExtended = await this.$getBlockExtended(block, transactions);
blockExtended.canonical = await bitcoinApi.$getBlockHash(block.height);
return blockExtended;
}
/**
* Get one block by its hash
*/
@@ -822,12 +926,16 @@ class Blocks {
}
// Bitcoin network, add our custom data on top
const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(hash);
return await this.$indexBlock(block.height);
const block: IEsploraApi.Block = await bitcoinApi.$getBlock(hash);
if (block.stale) {
return await this.$indexStaleBlock(hash);
} else {
return await this.$indexBlock(block.height);
}
}
public async $getStrippedBlockTransactions(hash: string, skipMemoryCache = false,
skipDBLookup = false): Promise<TransactionStripped[]>
skipDBLookup = false, cpfpSummary?: CpfpSummary, blockHeight?: number): Promise<TransactionStripped[]>
{
if (skipMemoryCache === false) {
// Check the memory cache
@@ -845,13 +953,40 @@ class Blocks {
}
}
// Call Core RPC
const block = await bitcoinClient.getBlock(hash, 2);
const summary = this.summarizeBlock(block);
let height = blockHeight;
let summary: BlockSummary;
if (cpfpSummary && !Common.isLiquid()) {
summary = {
id: hash,
transactions: cpfpSummary.transactions.map(tx => {
return {
txid: tx.txid,
fee: tx.fee || 0,
vsize: tx.vsize,
value: Math.round(tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0)),
rate: tx.effectiveFeePerVsize
};
}),
};
} else {
if (config.MEMPOOL.BACKEND === 'esplora') {
const txs = (await bitcoinApi.$getTxsForBlock(hash)).map(tx => transactionUtils.extendTransaction(tx));
summary = this.summarizeBlockTransactions(hash, txs);
} else {
// Call Core RPC
const block = await bitcoinClient.getBlock(hash, 2);
summary = this.summarizeBlock(block);
height = block.height;
}
}
if (height == null) {
const block = await bitcoinApi.$getBlock(hash);
height = block.height;
}
// Index the response if needed
if (Common.blocksSummariesIndexingEnabled() === true) {
await BlocksSummariesRepository.$saveTransactions(block.height, block.hash, summary.transactions);
await BlocksSummariesRepository.$saveTransactions(height, hash, summary.transactions);
}
return summary.transactions;
@@ -965,8 +1100,17 @@ class Blocks {
if (Common.blocksSummariesIndexingEnabled() && cleanBlock.fee_amt_percentiles === null) {
cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash);
if (cleanBlock.fee_amt_percentiles === null) {
const block = await bitcoinClient.getBlock(cleanBlock.hash, 2);
const summary = this.summarizeBlock(block);
let summary;
if (config.MEMPOOL.BACKEND === 'esplora') {
const txs = (await bitcoinApi.$getTxsForBlock(cleanBlock.hash)).map(tx => transactionUtils.extendTransaction(tx));
summary = this.summarizeBlockTransactions(cleanBlock.hash, txs);
} else {
// Call Core RPC
const block = await bitcoinClient.getBlock(cleanBlock.hash, 2);
summary = this.summarizeBlock(block);
}
await BlocksSummariesRepository.$saveTransactions(cleanBlock.height, cleanBlock.hash, summary.transactions);
cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash);
}
@@ -1007,19 +1151,11 @@ class Blocks {
}
public async $getBlockAuditSummary(hash: string): Promise<any> {
let summary;
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
summary = await BlocksAuditsRepository.$getBlockAudit(hash);
return BlocksAuditsRepository.$getBlockAudit(hash);
} else {
return null;
}
// fallback to non-audited transaction summary
if (!summary?.transactions?.length) {
const strippedTransactions = await this.$getStrippedBlockTransactions(hash);
summary = {
transactions: strippedTransactions
};
}
return summary;
}
public getLastDifficultyAdjustmentTime(): number {
@@ -1034,25 +1170,39 @@ class Blocks {
return this.currentBlockHeight;
}
public async $indexCPFP(hash: string, height: number): Promise<void> {
const block = await bitcoinClient.getBlock(hash, 2);
const transactions = block.tx.map(tx => {
tx.fee *= 100_000_000;
return tx;
});
public async $indexCPFP(hash: string, height: number, txs?: TransactionExtended[]): Promise<CpfpSummary> {
let transactions = txs;
if (!transactions) {
if (config.MEMPOOL.BACKEND === 'esplora') {
transactions = (await bitcoinApi.$getTxsForBlock(hash)).map(tx => transactionUtils.extendTransaction(tx));
}
if (!transactions) {
const block = await bitcoinClient.getBlock(hash, 2);
transactions = block.tx.map(tx => {
tx.fee *= 100_000_000;
return tx;
});
}
}
const summary = Common.calculateCpfp(height, transactions);
const summary = Common.calculateCpfp(height, transactions as TransactionExtended[]);
await this.$saveCpfp(hash, height, summary);
const effectiveFeeStats = Common.calcEffectiveFeeStatistics(summary.transactions);
await blocksRepository.$saveEffectiveFeeStats(hash, effectiveFeeStats);
return summary;
}
public async $saveCpfp(hash: string, height: number, cpfpSummary: CpfpSummary): Promise<void> {
const result = await cpfpRepository.$batchSaveClusters(cpfpSummary.clusters);
if (!result) {
await cpfpRepository.$insertProgressMarker(height);
try {
const result = await cpfpRepository.$batchSaveClusters(cpfpSummary.clusters);
if (!result) {
await cpfpRepository.$insertProgressMarker(height);
}
} catch (e) {
// not a fatal error, we'll try again next time the indexer runs
}
}
}

177
backend/src/api/common.ts
View File

@@ -1,3 +1,5 @@
import * as bitcoinjs from 'bitcoinjs-lib';
import { Request } from 'express';
import { Ancestor, CpfpInfo, CpfpSummary, CpfpCluster, EffectiveFeeStats, MempoolBlockWithTransactions, TransactionExtended, MempoolTransactionExtended, TransactionStripped, WorkingEffectiveFeeStats } from '../mempool.interfaces';
import config from '../config';
import { NodeSocket } from '../repositories/NodesSocketsRepository';
@@ -57,10 +59,12 @@ export class Common {
return arr;
}
static findRbfTransactions(added: MempoolTransactionExtended[], deleted: MempoolTransactionExtended[]): { [txid: string]: MempoolTransactionExtended[] } {
static findRbfTransactions(added: MempoolTransactionExtended[], deleted: MempoolTransactionExtended[], forceScalable = false): { [txid: string]: MempoolTransactionExtended[] } {
const matches: { [txid: string]: MempoolTransactionExtended[] } = {};
added
.forEach((addedTx) => {
// For small N, a naive nested loop is extremely fast, but it doesn't scale
if (added.length < 1000 && deleted.length < 50 && !forceScalable) {
added.forEach((addedTx) => {
const foundMatches = deleted.filter((deletedTx) => {
// The new tx must, absolutely speaking, pay at least as much fee as the replaced tx.
return addedTx.fee > deletedTx.fee
@@ -71,9 +75,40 @@ export class Common {
addedTx.vin.some((vin) => vin.txid === deletedVin.txid && vin.vout === deletedVin.vout));
});
if (foundMatches?.length) {
matches[addedTx.txid] = foundMatches;
matches[addedTx.txid] = [...new Set(foundMatches)];
}
});
} else {
// for large N, build a lookup table of prevouts we can check in ~constant time
const deletedSpendMap: { [txid: string]: { [vout: number]: MempoolTransactionExtended } } = {};
for (const tx of deleted) {
for (const vin of tx.vin) {
if (!deletedSpendMap[vin.txid]) {
deletedSpendMap[vin.txid] = {};
}
deletedSpendMap[vin.txid][vin.vout] = tx;
}
}
for (const addedTx of added) {
const foundMatches = new Set<MempoolTransactionExtended>();
for (const vin of addedTx.vin) {
const deletedTx = deletedSpendMap[vin.txid]?.[vin.vout];
if (deletedTx && deletedTx.txid !== addedTx.txid
// The new tx must, absolutely speaking, pay at least as much fee as the replaced tx.
&& addedTx.fee > deletedTx.fee
// The new transaction must pay more fee per kB than the replaced tx.
&& addedTx.adjustedFeePerVsize > deletedTx.adjustedFeePerVsize
) {
foundMatches.add(deletedTx);
}
if (foundMatches.size) {
matches[addedTx.txid] = [...foundMatches];
}
}
}
}
return matches;
}
@@ -86,19 +121,19 @@ export class Common {
const match = spendMap.get(`${vin.txid}:${vin.vout}`);
if (match && match.txid !== tx.txid) {
replaced.add(match);
// remove this tx from the spendMap
// prevents the same tx being replaced more than once
for (const replacedVin of match.vin) {
const key = `${replacedVin.txid}:${replacedVin.vout}`;
spendMap.delete(key);
}
}
const key = `${vin.txid}:${vin.vout}`;
spendMap.delete(key);
}
if (replaced.size) {
matches[tx.txid] = { replaced: Array.from(replaced), replacedBy: tx };
}
// remove this tx from the spendMap
// prevents the same tx being replaced more than once
for (const vin of tx.vin) {
const key = `${vin.txid}:${vin.vout}`;
if (spendMap.get(key)?.txid === tx.txid) {
spendMap.delete(key);
}
}
}
return matches;
}
@@ -106,13 +141,18 @@ export class Common {
static stripTransaction(tx: TransactionExtended): TransactionStripped {
return {
txid: tx.txid,
fee: tx.fee,
fee: tx.fee || 0,
vsize: tx.weight / 4,
value: tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0),
acc: tx.acceleration || undefined,
rate: tx.effectiveFeePerVsize,
};
}
static stripTransactions(txs: TransactionExtended[]): TransactionStripped[] {
return txs.map(this.stripTransaction);
}
static sleep$(ms: number): Promise<void> {
return new Promise((resolve) => {
setTimeout(() => {
@@ -454,7 +494,7 @@ export class Common {
};
}
static calcEffectiveFeeStatistics(transactions: { weight: number, fee: number, effectiveFeePerVsize?: number, txid: string }[]): EffectiveFeeStats {
static calcEffectiveFeeStatistics(transactions: { weight: number, fee: number, effectiveFeePerVsize?: number, txid: string, acceleration?: boolean }[]): EffectiveFeeStats {
const sortedTxs = transactions.map(tx => { return { txid: tx.txid, weight: tx.weight, rate: tx.effectiveFeePerVsize || ((tx.fee || 0) / (tx.weight / 4)) }; }).sort((a, b) => a.rate - b.rate);
let weightCount = 0;
@@ -507,6 +547,115 @@ export class Common {
static getNthPercentile(n: number, sortedDistribution: any[]): any {
return sortedDistribution[Math.floor((sortedDistribution.length - 1) * (n / 100))];
}
static getTransactionFromRequest(req: Request, form: boolean): string {
let rawTx: any = typeof req.body === 'object' && form
? Object.values(req.body)[0] as any
: req.body;
if (typeof rawTx !== 'string') {
throw Object.assign(new Error('Non-string request body'), { code: -1 });
}
// Support both upper and lower case hex
// Support both txHash= Form and direct API POST
const reg = form ? /^txHash=((?:[a-fA-F0-9]{2})+)$/ : /^((?:[a-fA-F0-9]{2})+)$/;
const matches = reg.exec(rawTx);
if (!matches || !matches[1]) {
throw Object.assign(new Error('Non-hex request body'), { code: -2 });
}
// Guaranteed to be a hex string of multiple of 2
// Guaranteed to be lower case
// Guaranteed to pass validation (see function below)
return this.validateTransactionHex(matches[1].toLowerCase());
}
private static validateTransactionHex(txhex: string): string {
// Do not mutate txhex
// We assume txhex to be valid hex (output of getTransactionFromRequest above)
// Check 1: Valid transaction parse
let tx: bitcoinjs.Transaction;
try {
tx = bitcoinjs.Transaction.fromHex(txhex);
} catch(e) {
throw Object.assign(new Error('Invalid transaction (could not parse)'), { code: -4 });
}
// Check 2: Simple size check
if (tx.weight() > config.MEMPOOL.MAX_PUSH_TX_SIZE_WEIGHT) {
throw Object.assign(new Error(`Transaction too large (max ${config.MEMPOOL.MAX_PUSH_TX_SIZE_WEIGHT} weight units)`), { code: -3 });
}
// Check 3: Check unreachable script in taproot (if not allowed)
if (!config.MEMPOOL.ALLOW_UNREACHABLE) {
tx.ins.forEach(input => {
const witness = input.witness;
// See BIP 341: Script validation rules
const hasAnnex = witness.length >= 2 &&
witness[witness.length - 1][0] === 0x50;
const scriptSpendMinLength = hasAnnex ? 3 : 2;
const maybeScriptSpend = witness.length >= scriptSpendMinLength;
if (maybeScriptSpend) {
const controlBlock = witness[witness.length - scriptSpendMinLength + 1];
if (controlBlock.length === 0 || !this.isValidLeafVersion(controlBlock[0])) {
// Skip this input, it's not taproot
return;
}
// Definitely taproot. Get script
const script = witness[witness.length - scriptSpendMinLength];
const decompiled = bitcoinjs.script.decompile(script);
if (!decompiled || decompiled.length < 2) {
// Skip this input
return;
}
// Iterate up to second last (will look ahead 1 item)
for (let i = 0; i < decompiled.length - 1; i++) {
const first = decompiled[i];
const second = decompiled[i + 1];
if (
first === bitcoinjs.opcodes.OP_FALSE &&
second === bitcoinjs.opcodes.OP_IF
) {
throw Object.assign(new Error('Unreachable taproot scripts not allowed'), { code: -5 });
}
}
}
})
}
// Pass through the input string untouched
return txhex;
}
private static isValidLeafVersion(leafVersion: number): boolean {
// See Note 7 in BIP341
// https://github.com/bitcoin/bips/blob/66a1a8151021913047934ebab3f8883f2f8ca75b/bip-0341.mediawiki#cite_note-7
// "What constraints are there on the leaf version?"
// Must be an integer between 0 and 255
// Since we're parsing a byte
if (Math.floor(leafVersion) !== leafVersion || leafVersion < 0 || leafVersion > 255) {
return false;
}
// "the leaf version cannot be odd"
if ((leafVersion & 0x01) === 1) {
return false;
}
// "The values that comply to this rule are
// the 32 even values between 0xc0 and 0xfe
if (leafVersion >= 0xc0 && leafVersion <= 0xfe) {
return true;
}
// and also 0x66, 0x7e, 0x80, 0x84, 0x96, 0x98, 0xba, 0xbc, 0xbe."
if ([0x66, 0x7e, 0x80, 0x84, 0x96, 0x98, 0xba, 0xbc, 0xbe].includes(leafVersion)) {
return true;
}
// Otherwise, invalid
return false;
}
}
/**

21
backend/src/api/database-migration.ts
View File

@@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
import { RowDataPacket } from 'mysql2';
class DatabaseMigration {
private static currentVersion = 62;
private static currentVersion = 66;
private queryTimeout = 3600_000;
private statisticsAddedIndexed = false;
private uniqueLogs: string[] = [];
@@ -539,6 +539,25 @@ class DatabaseMigration {
await this.updateToSchemaVersion(62);
}
if (databaseSchemaVersion < 63 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `blocks_audits` ADD fullrbf_txs JSON DEFAULT "[]"');
await this.updateToSchemaVersion(63);
}
if (databaseSchemaVersion < 64 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `nodes` ADD features text NULL');
await this.updateToSchemaVersion(64);
}
if (databaseSchemaVersion < 65 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `blocks_audits` ADD accelerated_txs JSON DEFAULT "[]"');
await this.updateToSchemaVersion(65);
}
if (databaseSchemaVersion < 66) {
await this.$executeQuery('ALTER TABLE `statistics` ADD min_fee FLOAT UNSIGNED DEFAULT NULL');
await this.updateToSchemaVersion(66);
}
}
/**

62
backend/src/api/difficulty-adjustment.ts
View File

@@ -16,6 +16,68 @@ export interface DifficultyAdjustment {
expectedBlocks: number; // Block count
}
/**
* Calculate the difficulty increase/decrease by using the `bits` integer contained in two
* block headers.
*
* Warning: Only compare `bits` from blocks in two adjacent difficulty periods. This code
* assumes the maximum difference is x4 or /4 (as per the protocol) and will throw an
* error if an exponent difference of 2 or more is seen.
*
* @param {number} oldBits The 32 bit `bits` integer from a block header.
* @param {number} newBits The 32 bit `bits` integer from a block header in the next difficulty period.
* @returns {number} A floating point decimal of the difficulty change from old to new.
* (ie. 21.3 means 21.3% increase in difficulty, -21.3 is a 21.3% decrease in difficulty)
*/
export function calcBitsDifference(oldBits: number, newBits: number): number {
// Must be
// - integer
// - highest exponent is 0x20, so max value (as integer) is 0x207fffff
// - min value is 1 (exponent = 0)
// - highest bit of the number-part is +- sign, it must not be 1
const verifyBits = (bits: number): void => {
if (
Math.floor(bits) !== bits ||
bits > 0x207fffff ||
bits < 1 ||
(bits & 0x00800000) !== 0 ||
(bits & 0x007fffff) === 0
) {
throw new Error('Invalid bits');
}
};
verifyBits(oldBits);
verifyBits(newBits);
// No need to mask exponents because we checked the bounds above
const oldExp = oldBits >> 24;
const newExp = newBits >> 24;
const oldNum = oldBits & 0x007fffff;
const newNum = newBits & 0x007fffff;
// The diff can only possibly be 1, 0, -1
// (because maximum difficulty change is x4 or /4 (2 bits up or down))
let result: number;
switch (newExp - oldExp) {
// New less than old, target lowered, difficulty increased
case -1:
result = ((oldNum << 8) * 100) / newNum - 100;
break;
// Same exponent, compare numbers as is.
case 0:
result = (oldNum * 100) / newNum - 100;
break;
// Old less than new, target raised, difficulty decreased
case 1:
result = (oldNum * 100) / (newNum << 8) - 100;
break;
default:
throw new Error('Impossible exponent difference');
}
// Min/Max values
return result > 300 ? 300 : result < -75 ? -75 : result;
}
export function calcDifficultyAdjustment(
DATime: number,
nowSeconds: number,

11
backend/src/api/disk-cache.ts
View File

@@ -29,7 +29,7 @@ class DiskCache {
};
constructor() {
if (!cluster.isPrimary) {
if (!cluster.isPrimary || !config.MEMPOOL.CACHE_ENABLED) {
return;
}
process.on('SIGINT', (e) => {
@@ -39,7 +39,7 @@ class DiskCache {
}
async $saveCacheToDisk(sync: boolean = false): Promise<void> {
if (!cluster.isPrimary) {
if (!cluster.isPrimary || !config.MEMPOOL.CACHE_ENABLED) {
return;
}
if (this.isWritingCache) {
@@ -175,10 +175,11 @@ class DiskCache {
}
async $loadMempoolCache(): Promise<void> {
if (!fs.existsSync(DiskCache.FILE_NAME)) {
if (!config.MEMPOOL.CACHE_ENABLED || !fs.existsSync(DiskCache.FILE_NAME)) {
return;
}
try {
const start = Date.now();
let data: any = {};
const cacheData = fs.readFileSync(DiskCache.FILE_NAME, 'utf8');
if (cacheData) {
@@ -195,6 +196,7 @@ class DiskCache {
if (data.mempoolArray) {
for (const tx of data.mempoolArray) {
delete tx.uid;
data.mempool[tx.txid] = tx;
}
}
@@ -207,6 +209,7 @@ class DiskCache {
const cacheData2 = JSON.parse(fs.readFileSync(fileName, 'utf8'));
if (cacheData2.mempoolArray) {
for (const tx of cacheData2.mempoolArray) {
delete tx.uid;
data.mempool[tx.txid] = tx;
}
} else {
@@ -218,6 +221,8 @@ class DiskCache {
}
}
logger.info(`Loaded mempool from disk cache in ${Date.now() - start} ms`);
await memPool.$setMempool(data.mempool);
if (!this.ignoreBlocksCache) {
blocks.setBlocks(data.blocks);

22
backend/src/api/explorer/channels.api.ts
View File

@@ -80,7 +80,7 @@ class ChannelsApi {
public async $searchChannelsById(search: string): Promise<any[]> {
try {
const searchStripped = search.replace('%', '') + '%';
const searchStripped = search.replace(/[^0-9x]/g, '') + '%';
const query = `SELECT id, short_id, capacity, status FROM channels WHERE id LIKE ? OR short_id LIKE ? LIMIT 10`;
const [rows]: any = await DB.query(query, [searchStripped, searchStripped]);
return rows;
@@ -117,6 +117,26 @@ class ChannelsApi {
}
}
public async $getPenaltyClosedChannels(): Promise<any[]> {
try {
const query = `
SELECT n1.alias AS alias_left,
n2.alias AS alias_right,
channels.*
FROM channels
LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key
LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key
WHERE channels.status = 2 AND channels.closing_reason = 3
ORDER BY closing_date DESC
`;
const [rows]: any = await DB.query(query);
return rows;
} catch (e) {
logger.err('$getPenaltyClosedChannels error: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
public async $getUnresolvedClosedChannels(): Promise<any[]> {
try {
const query = `SELECT * FROM channels WHERE status = 2 AND closing_reason = 2 AND closing_resolved = 0 AND closing_transaction_id != ''`;

13
backend/src/api/explorer/channels.routes.ts
View File

@@ -11,6 +11,7 @@ class ChannelsRoutes {
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels/search/:search', this.$searchChannelsById)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels/:short_id', this.$getChannel)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels', this.$getChannelsForNode)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/penalties', this.$getPenaltyClosedChannels)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels-geo', this.$getAllChannelsGeo)
.get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels-geo/:publicKey', this.$getAllChannelsGeo)
;
@@ -108,6 +109,18 @@ class ChannelsRoutes {
}
}
private async $getPenaltyClosedChannels(req: Request, res: Response): Promise<void> {
try {
const channels = await channelsApi.$getPenaltyClosedChannels();
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
res.json(channels);
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
private async $getAllChannelsGeo(req: Request, res: Response) {
try {
const style: string = typeof req.query.style === 'string' ? req.query.style : '';

38
backend/src/api/explorer/nodes.api.ts
View File

@@ -3,6 +3,7 @@ import DB from '../../database';
import { ResultSetHeader } from 'mysql2';
import { ILightningApi } from '../lightning/lightning-api.interface';
import { ITopNodesPerCapacity, ITopNodesPerChannels } from '../../mempool.interfaces';
import { bin2hex } from '../../utils/format';
class NodesApi {
public async $getWorldNodes(): Promise<any> {
@@ -56,7 +57,8 @@ class NodesApi {
UNIX_TIMESTAMP(updated_at) AS updated_at, color, sockets as sockets,
as_number, city_id, country_id, subdivision_id, longitude, latitude,
geo_names_iso.names as iso_code, geo_names_as.names as as_organization, geo_names_city.names as city,
geo_names_country.names as country, geo_names_subdivision.names as subdivision
geo_names_country.names as country, geo_names_subdivision.names as subdivision,
features
FROM nodes
LEFT JOIN geo_names geo_names_as on geo_names_as.id = as_number
LEFT JOIN geo_names geo_names_city on geo_names_city.id = city_id
@@ -76,6 +78,23 @@ class NodesApi {
node.city = JSON.parse(node.city);
node.country = JSON.parse(node.country);
// Features
node.features = JSON.parse(node.features);
node.featuresBits = null;
if (node.features) {
let maxBit = 0;
for (const feature of node.features) {
maxBit = Math.max(maxBit, feature.bit);
}
maxBit = Math.ceil(maxBit / 4) * 4 - 1;
node.featuresBits = new Array(maxBit + 1).fill(0);
for (const feature of node.features) {
node.featuresBits[feature.bit] = 1;
}
node.featuresBits = bin2hex(node.featuresBits.reverse().join(''));
}
// Active channels and capacity
const activeChannelsStats: any = await this.$getActiveChannelsStats(public_key);
node.active_channel_count = activeChannelsStats.active_channel_count ?? 0;
@@ -656,10 +675,19 @@ class NodesApi {
alias_search,
color,
sockets,
status
status,
features
)
VALUES (?, NOW(), FROM_UNIXTIME(?), ?, ?, ?, ?, 1)
ON DUPLICATE KEY UPDATE updated_at = FROM_UNIXTIME(?), alias = ?, alias_search = ?, color = ?, sockets = ?, status = 1`;
VALUES (?, NOW(), FROM_UNIXTIME(?), ?, ?, ?, ?, 1, ?)
ON DUPLICATE KEY UPDATE
updated_at = FROM_UNIXTIME(?),
alias = ?,
alias_search = ?,
color = ?,
sockets = ?,
status = 1,
features = ?
`;
await DB.query(query, [
node.pub_key,
@@ -668,11 +696,13 @@ class NodesApi {
this.aliasToSearchText(node.alias),
node.color,
sockets,
JSON.stringify(node.features),
node.last_update,
node.alias,
this.aliasToSearchText(node.alias),
node.color,
sockets,
JSON.stringify(node.features),
]);
} catch (e) {
logger.err('$saveNode() error: ' + (e instanceof Error ? e.message : e));

18
backend/src/api/explorer/nodes.routes.ts
View File

@@ -42,6 +42,12 @@ class NodesRoutes {
switch (config.MEMPOOL.NETWORK) {
case 'testnet':
nodesList = [
'0259db43b4e4ac0ff12a805f2d81e521253ba2317f6739bc611d8e2fa156d64256',
'0352b9944b9a52bd2116c91f1ba70c4ef851ac5ba27e1b20f1d92da3ade010dd10',
'03424f5a7601eaa47482cb17100b31a84a04d14fb44b83a57eeceffd8e299878e3',
'032850492ee61a5f7006a2fda6925e4b4ec3782f2b6de2ff0e439ef5a38c3b2470',
'022c80bace98831c44c32fb69755f2b353434e0ee9e7fbda29507f7ef8abea1421',
'02c3559c833e6f99f9ca05fe503e0b4e7524dea9121344edfd3e811101e0c28680',
'032c7c7819276c4f706a04df1a0f1e10a5495994a7be4c1d3d28ca766e5a2b957b',
'025a7e38c2834dd843591a4d23d5f09cdeb77ddca85f673c2d944a14220ff14cf7',
'0395e2731a1673ef21d7a16a727c4fc4d4c35a861c428ce2c819c53d2b81c8bd55',
@@ -64,6 +70,12 @@ class NodesRoutes {
break;
case 'signet':
nodesList = [
'029fe3621fc0c6e08056a14b868f8fb9acca1aa28a129512f6cea0f0d7654d9f92',
'02f60cd7a3a4f1c953dd9554a6ebd51a34f8b10b8124b7fc43a0b381139b55c883',
'03cbbf581774700865eebd1be42d022bc004ba30881274ab304e088a25d70e773d',
'0243348cb3741cfe2d8485fa8375c29c7bc7cbb67577c363cb6987a5e5fd0052cc',
'02cb73e631af44bee600d80f8488a9194c9dc5c7590e575c421a070d1be05bc8e9',
'0306f55ee631aa1e2cd4d9b2bfcbc14404faec5c541cef8b2e6f779061029d09c4',
'03ddab321b760433cbf561b615ef62ac7d318630c5f51d523aaf5395b90b751956',
'033d92c7bfd213ef1b34c90e985fb5dc77f9ec2409d391492484e57a44c4aca1de',
'02ad010dda54253c1eb9efe38b0760657a3b43ecad62198c359c051c9d99d45781',
@@ -86,6 +98,12 @@ class NodesRoutes {
break;
default:
nodesList = [
'02b12b889fe3c943cb05645921040ef13d6d397a2e7a4ad000e28500c505ff26d6',
'0302240ac9d71b39617cbde2764837ec3d6198bd6074b15b75d2ff33108e89d2e1',
'03364a8ace313376e5e4b68c954e287c6388e16df9e9fdbaf0363ecac41105cbf6',
'03229ab4b7f692753e094b93df90530150680f86b535b5183b0cffd75b3df583fc',
'03a696eb7acde991c1be97a58a9daef416659539ae462b897f5e9ae361f990228e',
'0248bf26cf3a63ab8870f34dc0ec9e6c8c6288cdba96ba3f026f34ec0f13ac4055',
'03fbc17549ec667bccf397ababbcb4cdc0e3394345e4773079ab2774612ec9be61',
'03da9a8623241ccf95f19cd645c6cecd4019ac91570e976eb0a128bebbc4d8a437',
'03ca5340cf85cb2e7cf076e489f785410838de174e40be62723e8a60972ad75144',

29
backend/src/api/fee-api.ts
View File

@@ -3,21 +3,30 @@ import { Common } from './common';
import mempool from './mempool';
import projectedBlocks from './mempool-blocks';
interface RecommendedFees {
fastestFee: number,
halfHourFee: number,
hourFee: number,
economyFee: number,
minimumFee: number,
}
class FeeApi {
constructor() { }
defaultFee = Common.isLiquid() ? 0.1 : 1;
public getRecommendedFee() {
public getRecommendedFee(): RecommendedFees {
const pBlocks = projectedBlocks.getMempoolBlocks();
const mPool = mempool.getMempoolInfo();
const minimumFee = Math.ceil(mPool.mempoolminfee * 100000);
const defaultMinFee = Math.max(minimumFee, this.defaultFee);
if (!pBlocks.length) {
return {
'fastestFee': this.defaultFee,
'halfHourFee': this.defaultFee,
'hourFee': this.defaultFee,
'fastestFee': defaultMinFee,
'halfHourFee': defaultMinFee,
'hourFee': defaultMinFee,
'economyFee': minimumFee,
'minimumFee': minimumFee,
};
@@ -27,11 +36,15 @@ class FeeApi {
const secondMedianFee = pBlocks[1] ? this.optimizeMedianFee(pBlocks[1], pBlocks[2], firstMedianFee) : this.defaultFee;
const thirdMedianFee = pBlocks[2] ? this.optimizeMedianFee(pBlocks[2], pBlocks[3], secondMedianFee) : this.defaultFee;
// explicitly enforce a minimum of ceil(mempoolminfee) on all recommendations.
// simply rounding up recommended rates is insufficient, as the purging rate
// can exceed the median rate of projected blocks in some extreme scenarios
// (see https://bitcoin.stackexchange.com/a/120024)
return {
'fastestFee': firstMedianFee,
'halfHourFee': secondMedianFee,
'hourFee': thirdMedianFee,
'economyFee': Math.min(2 * minimumFee, thirdMedianFee),
'fastestFee': Math.max(minimumFee, firstMedianFee),
'halfHourFee': Math.max(minimumFee, secondMedianFee),
'hourFee': Math.max(minimumFee, thirdMedianFee),
'economyFee': Math.max(minimumFee, Math.min(2 * minimumFee, thirdMedianFee)),
'minimumFee': minimumFee,
};
}

119
backend/src/api/lightning/clightning/clightning-convert.ts
View File

@@ -2,8 +2,91 @@ import { ILightningApi } from '../lightning-api.interface';
import FundingTxFetcher from '../../../tasks/lightning/sync-tasks/funding-tx-fetcher';
import logger from '../../../logger';
import { Common } from '../../common';
import { hex2bin } from '../../../utils/format';
import config from '../../../config';
// https://github.com/lightningnetwork/lnd/blob/master/lnwire/features.go
export enum FeatureBits {
DataLossProtectRequired = 0,
DataLossProtectOptional = 1,
InitialRoutingSync = 3,
UpfrontShutdownScriptRequired = 4,
UpfrontShutdownScriptOptional = 5,
GossipQueriesRequired = 6,
GossipQueriesOptional = 7,
TLVOnionPayloadRequired = 8,
TLVOnionPayloadOptional = 9,
StaticRemoteKeyRequired = 12,
StaticRemoteKeyOptional = 13,
PaymentAddrRequired = 14,
PaymentAddrOptional = 15,
MPPRequired = 16,
MPPOptional = 17,
WumboChannelsRequired = 18,
WumboChannelsOptional = 19,
AnchorsRequired = 20,
AnchorsOptional = 21,
AnchorsZeroFeeHtlcTxRequired = 22,
AnchorsZeroFeeHtlcTxOptional = 23,
ShutdownAnySegwitRequired = 26,
ShutdownAnySegwitOptional = 27,
AMPRequired = 30,
AMPOptional = 31,
ExplicitChannelTypeRequired = 44,
ExplicitChannelTypeOptional = 45,
ScidAliasRequired = 46,
ScidAliasOptional = 47,
PaymentMetadataRequired = 48,
PaymentMetadataOptional = 49,
ZeroConfRequired = 50,
ZeroConfOptional = 51,
KeysendRequired = 54,
KeysendOptional = 55,
ScriptEnforcedLeaseRequired = 2022,
ScriptEnforcedLeaseOptional = 2023,
MaxBolt11Feature = 5114,
};
export const FeaturesMap = new Map<FeatureBits, string>([
[FeatureBits.DataLossProtectRequired, 'data-loss-protect'],
[FeatureBits.DataLossProtectOptional, 'data-loss-protect'],
[FeatureBits.InitialRoutingSync, 'initial-routing-sync'],
[FeatureBits.UpfrontShutdownScriptRequired, 'upfront-shutdown-script'],
[FeatureBits.UpfrontShutdownScriptOptional, 'upfront-shutdown-script'],
[FeatureBits.GossipQueriesRequired, 'gossip-queries'],
[FeatureBits.GossipQueriesOptional, 'gossip-queries'],
[FeatureBits.TLVOnionPayloadRequired, 'tlv-onion'],
[FeatureBits.TLVOnionPayloadOptional, 'tlv-onion'],
[FeatureBits.StaticRemoteKeyOptional, 'static-remote-key'],
[FeatureBits.StaticRemoteKeyRequired, 'static-remote-key'],
[FeatureBits.PaymentAddrOptional, 'payment-addr'],
[FeatureBits.PaymentAddrRequired, 'payment-addr'],
[FeatureBits.MPPOptional, 'multi-path-payments'],
[FeatureBits.MPPRequired, 'multi-path-payments'],
[FeatureBits.AnchorsRequired, 'anchor-commitments'],
[FeatureBits.AnchorsOptional, 'anchor-commitments'],
[FeatureBits.AnchorsZeroFeeHtlcTxRequired, 'anchors-zero-fee-htlc-tx'],
[FeatureBits.AnchorsZeroFeeHtlcTxOptional, 'anchors-zero-fee-htlc-tx'],
[FeatureBits.WumboChannelsRequired, 'wumbo-channels'],
[FeatureBits.WumboChannelsOptional, 'wumbo-channels'],
[FeatureBits.AMPRequired, 'amp'],
[FeatureBits.AMPOptional, 'amp'],
[FeatureBits.PaymentMetadataOptional, 'payment-metadata'],
[FeatureBits.PaymentMetadataRequired, 'payment-metadata'],
[FeatureBits.ExplicitChannelTypeOptional, 'explicit-commitment-type'],
[FeatureBits.ExplicitChannelTypeRequired, 'explicit-commitment-type'],
[FeatureBits.KeysendOptional, 'keysend'],
[FeatureBits.KeysendRequired, 'keysend'],
[FeatureBits.ScriptEnforcedLeaseRequired, 'script-enforced-lease'],
[FeatureBits.ScriptEnforcedLeaseOptional, 'script-enforced-lease'],
[FeatureBits.ScidAliasRequired, 'scid-alias'],
[FeatureBits.ScidAliasOptional, 'scid-alias'],
[FeatureBits.ZeroConfRequired, 'zero-conf'],
[FeatureBits.ZeroConfOptional, 'zero-conf'],
[FeatureBits.ShutdownAnySegwitRequired, 'shutdown-any-segwit'],
[FeatureBits.ShutdownAnySegwitOptional, 'shutdown-any-segwit'],
]);
/**
* Convert a clightning "listnode" entry to a lnd node entry
*/
@@ -17,10 +100,36 @@ export function convertNode(clNode: any): ILightningApi.Node {
custom_records = undefined;
}
}
const nodeFeatures: ILightningApi.Feature[] = [];
const nodeFeaturesBinary = hex2bin(clNode.features).split('').reverse().join('');
for (let i = 0; i < nodeFeaturesBinary.length; i++) {
if (nodeFeaturesBinary[i] === '0') {
continue;
}
const feature = FeaturesMap.get(i);
if (!feature) {
nodeFeatures.push({
bit: i,
name: 'unknown',
is_required: i % 2 === 0,
is_known: false
});
} else {
nodeFeatures.push({
bit: i,
name: feature,
is_required: i % 2 === 0,
is_known: true
});
}
}
return {
alias: clNode.alias ?? '',
color: `#${clNode.color ?? ''}`,
features: [], // TODO parse and return clNode.feature
features: nodeFeatures,
pub_key: clNode.nodeid,
addresses: clNode.addresses?.map((addr) => {
let address = addr.address;
@@ -108,7 +217,7 @@ async function buildFullChannel(clChannelA: any, clChannelB: any): Promise<ILigh
return {
channel_id: Common.channelShortIdToIntegerId(clChannelA.short_channel_id),
capacity: clChannelA.satoshis,
capacity: (clChannelA.amount_msat / 1000).toString(),
last_update: lastUpdate,
node1_policy: convertPolicy(clChannelA),
node2_policy: convertPolicy(clChannelB),
@@ -132,7 +241,7 @@ async function buildIncompleteChannel(clChannel: any): Promise<ILightningApi.Cha
return {
channel_id: Common.channelShortIdToIntegerId(clChannel.short_channel_id),
capacity: clChannel.satoshis,
capacity: (clChannel.amount_msat / 1000).toString(),
last_update: clChannel.last_update ?? 0,
node1_policy: convertPolicy(clChannel),
node2_policy: null,
@@ -148,8 +257,8 @@ async function buildIncompleteChannel(clChannel: any): Promise<ILightningApi.Cha
function convertPolicy(clChannel: any): ILightningApi.RoutingPolicy {
return {
time_lock_delta: clChannel.delay,
min_htlc: clChannel.htlc_minimum_msat.slice(0, -4),
max_htlc_msat: clChannel.htlc_maximum_msat.slice(0, -4),
min_htlc: clChannel.htlc_minimum_msat.toString(),
max_htlc_msat: clChannel.htlc_maximum_msat.toString(),
fee_base_msat: clChannel.base_fee_millisatoshi,
fee_rate_milli_msat: clChannel.fee_per_millionth,
disabled: !clChannel.active,

1
backend/src/api/lightning/lightning-api.interface.ts
View File

@@ -79,6 +79,7 @@ export namespace ILightningApi {
}
export interface Feature {
bit: number;
name: string;
is_required: boolean;
is_known: boolean;

17
backend/src/api/lightning/lnd/lnd-api.ts
View File

@@ -41,8 +41,23 @@ class LndApi implements AbstractLightningApi {
}
async $getNetworkGraph(): Promise<ILightningApi.NetworkGraph> {
return axios.get<ILightningApi.NetworkGraph>(config.LND.REST_API_URL + '/v1/graph', this.axiosConfig)
const graph = await axios.get<ILightningApi.NetworkGraph>(config.LND.REST_API_URL + '/v1/graph', this.axiosConfig)
.then((response) => response.data);
for (const node of graph.nodes) {
const nodeFeatures: ILightningApi.Feature[] = [];
for (const bit in node.features) {
nodeFeatures.push({
bit: parseInt(bit, 10),
name: node.features[bit].name,
is_required: node.features[bit].is_required,
is_known: node.features[bit].is_known,
});
}
node.features = nodeFeatures;
}
return graph;
}
}

2
backend/src/api/memory-cache.ts
View File

@@ -31,7 +31,7 @@ class MemoryCache {
}
private cleanup() {
this.cache = this.cache.filter((cache) => cache.expires < (new Date()));
this.cache = this.cache.filter((cache) => cache.expires > (new Date()));
}
}

382
backend/src/api/mempool-blocks.ts
View File

@@ -1,20 +1,24 @@
import { GbtGenerator, GbtResult, ThreadTransaction as RustThreadTransaction, ThreadAcceleration as RustThreadAcceleration } from 'rust-gbt';
import logger from '../logger';
import { MempoolBlock, MempoolTransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats } from '../mempool.interfaces';
import { MempoolBlock, MempoolTransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats, PoolTag } from '../mempool.interfaces';
import { Common, OnlineFeeStatsCalculator } from './common';
import config from '../config';
import { Worker } from 'worker_threads';
import path from 'path';
import mempool from './mempool';
const MAX_UINT32 = Math.pow(2, 32) - 1;
class MempoolBlocks {
private mempoolBlocks: MempoolBlockWithTransactions[] = [];
private mempoolBlockDeltas: MempoolBlockDelta[] = [];
private txSelectionWorker: Worker | null = null;
private rustInitialized: boolean = false;
private rustGbtGenerator: GbtGenerator = new GbtGenerator();
private nextUid: number = 1;
private uidMap: Map<number, string> = new Map(); // map short numerical uids to full txids
constructor() {}
public getMempoolBlocks(): MempoolBlock[] {
return this.mempoolBlocks.map((block) => {
return {
@@ -40,9 +44,7 @@ class MempoolBlocks {
const latestMempool = memPool;
const memPoolArray: MempoolTransactionExtended[] = [];
for (const i in latestMempool) {
if (latestMempool.hasOwnProperty(i)) {
memPoolArray.push(latestMempool[i]);
}
memPoolArray.push(latestMempool[i]);
}
const start = new Date().getTime();
@@ -143,7 +145,7 @@ class MempoolBlocks {
const stackWeight = transactionsSorted.slice(index).reduce((total, tx) => total + (tx.weight || 0), 0);
if (stackWeight > config.MEMPOOL.BLOCK_WEIGHT_UNITS) {
onlineStats = true;
feeStatsCalculator = new OnlineFeeStatsCalculator(stackWeight, 0.5);
feeStatsCalculator = new OnlineFeeStatsCalculator(stackWeight, 0.5, [10, 20, 30, 40, 50, 60, 70, 80, 90]);
feeStatsCalculator.processNext(tx);
}
}
@@ -169,7 +171,7 @@ class MempoolBlocks {
for (let i = 0; i < Math.max(mempoolBlocks.length, prevBlocks.length); i++) {
let added: TransactionStripped[] = [];
let removed: string[] = [];
const changed: { txid: string, rate: number | undefined }[] = [];
const changed: { txid: string, rate: number | undefined, acc: boolean | undefined }[] = [];
if (mempoolBlocks[i] && !prevBlocks[i]) {
added = mempoolBlocks[i].transactions;
} else if (!mempoolBlocks[i] && prevBlocks[i]) {
@@ -191,8 +193,8 @@ class MempoolBlocks {
mempoolBlocks[i].transactions.forEach(tx => {
if (!prevIds[tx.txid]) {
added.push(tx);
} else if (tx.rate !== prevIds[tx.txid].rate) {
changed.push({ txid: tx.txid, rate: tx.rate });
} else if (tx.rate !== prevIds[tx.txid].rate || tx.acc !== prevIds[tx.txid].acc) {
changed.push({ txid: tx.txid, rate: tx.rate, acc: tx.acc });
}
});
}
@@ -205,29 +207,35 @@ class MempoolBlocks {
return mempoolBlockDeltas;
}
public async $makeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false): Promise<MempoolBlockWithTransactions[]> {
public async $makeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
const start = Date.now();
// reset mempool short ids
this.resetUids();
for (const tx of Object.values(newMempool)) {
this.setUid(tx);
if (saveResults) {
this.resetUids();
}
// set missing short ids
for (const tx of Object.values(newMempool)) {
this.setUid(tx, !saveResults);
}
const accelerations = useAccelerations ? mempool.getAccelerations() : {};
// prepare a stripped down version of the mempool with only the minimum necessary data
// to reduce the overhead of passing this data to the worker thread
const strippedMempool: Map<number, CompactThreadTransaction> = new Map();
Object.values(newMempool).forEach(entry => {
if (entry.uid != null) {
strippedMempool.set(entry.uid, {
if (entry.uid !== null && entry.uid !== undefined) {
const stripped = {
uid: entry.uid,
fee: entry.fee,
fee: entry.fee + (useAccelerations && (!accelerationPool || accelerations[entry.txid]?.pools?.includes(accelerationPool)) ? (accelerations[entry.txid]?.feeDelta || 0) : 0),
weight: (entry.adjustedVsize * 4),
sigops: entry.sigops,
feePerVsize: entry.adjustedFeePerVsize || entry.feePerVsize,
effectiveFeePerVsize: entry.effectiveFeePerVsize || entry.adjustedFeePerVsize || entry.feePerVsize,
inputs: entry.vin.map(v => this.getUid(newMempool[v.txid])).filter(uid => uid != null) as number[],
});
inputs: entry.vin.map(v => this.getUid(newMempool[v.txid])).filter(uid => (uid !== null && uid !== undefined)) as number[],
};
strippedMempool.set(entry.uid, stripped);
}
});
@@ -260,8 +268,10 @@ class MempoolBlocks {
// clean up thread error listener
this.txSelectionWorker?.removeListener('error', threadErrorListener);
const processed = this.processBlockTemplates(newMempool, blocks, rates, clusters, saveResults);
const processed = this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), accelerations, accelerationPool, saveResults);
logger.debug(`makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`);
return processed;
} catch (e) {
logger.err('makeBlockTemplates failed. ' + (e instanceof Error ? e.message : e));
@@ -269,30 +279,34 @@ class MempoolBlocks {
return this.mempoolBlocks;
}
public async $updateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], saveResults: boolean = false): Promise<void> {
public async $updateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], accelerationDelta: string[] = [], saveResults: boolean = false, useAccelerations: boolean = false): Promise<void> {
if (!this.txSelectionWorker) {
// need to reset the worker
await this.$makeBlockTemplates(newMempool, saveResults);
await this.$makeBlockTemplates(newMempool, saveResults, useAccelerations);
return;
}
const start = Date.now();
for (const tx of Object.values(added)) {
this.setUid(tx);
const accelerations = useAccelerations ? mempool.getAccelerations() : {};
const addedAndChanged: MempoolTransactionExtended[] = useAccelerations ? accelerationDelta.map(txid => newMempool[txid]).filter(tx => tx != null).concat(added) : added;
for (const tx of addedAndChanged) {
this.setUid(tx, true);
}
const removedUids = removed.map(tx => this.getUid(tx)).filter(uid => uid != null) as number[];
// prepare a stripped down version of the mempool with only the minimum necessary data
// to reduce the overhead of passing this data to the worker thread
const addedStripped: CompactThreadTransaction[] = added.filter(entry => entry.uid != null).map(entry => {
const addedStripped: CompactThreadTransaction[] = addedAndChanged.filter(entry => entry.uid != null).map(entry => {
return {
uid: entry.uid || 0,
fee: entry.fee,
fee: entry.fee + (useAccelerations ? (accelerations[entry.txid]?.feeDelta || 0) : 0),
weight: (entry.adjustedVsize * 4),
sigops: entry.sigops,
feePerVsize: entry.adjustedFeePerVsize || entry.feePerVsize,
effectiveFeePerVsize: entry.effectiveFeePerVsize || entry.adjustedFeePerVsize || entry.feePerVsize,
inputs: entry.vin.map(v => this.getUid(newMempool[v.txid])).filter(uid => uid != null) as number[],
inputs: entry.vin.map(v => this.getUid(newMempool[v.txid])).filter(uid => (uid !== null && uid !== undefined)) as number[],
};
});
@@ -314,84 +328,153 @@ class MempoolBlocks {
// clean up thread error listener
this.txSelectionWorker?.removeListener('error', threadErrorListener);
this.processBlockTemplates(newMempool, blocks, rates, clusters, saveResults);
this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), accelerations, null, saveResults);
logger.debug(`updateBlockTemplates completed in ${(Date.now() - start) / 1000} seconds`);
} catch (e) {
logger.err('updateBlockTemplates failed. ' + (e instanceof Error ? e.message : e));
}
}
private processBlockTemplates(mempool, blocks: string[][], rates: { [root: string]: number }, clusters: { [root: string]: string[] }, saveResults): MempoolBlockWithTransactions[] {
for (const txid of Object.keys(rates)) {
private resetRustGbt(): void {
this.rustInitialized = false;
this.rustGbtGenerator = new GbtGenerator();
}
public async $rustMakeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
const start = Date.now();
// reset mempool short ids
if (saveResults) {
this.resetUids();
}
// set missing short ids
for (const tx of Object.values(newMempool)) {
this.setUid(tx, !saveResults);
}
// set short ids for transaction inputs
for (const tx of Object.values(newMempool)) {
tx.inputs = tx.vin.map(v => this.getUid(newMempool[v.txid])).filter(uid => (uid !== null && uid !== undefined)) as number[];
}
const accelerations = useAccelerations ? mempool.getAccelerations() : {};
const acceleratedList = accelerationPool ? Object.values(accelerations).filter(acc => newMempool[acc.txid] && acc.pools.includes(accelerationPool)) : Object.values(accelerations).filter(acc => newMempool[acc.txid]);
const convertedAccelerations = acceleratedList.map(acc => {
return {
uid: this.getUid(newMempool[acc.txid]),
delta: acc.feeDelta,
};
});
// run the block construction algorithm in a separate thread, and wait for a result
const rustGbt = saveResults ? this.rustGbtGenerator : new GbtGenerator();
try {
const { blocks, blockWeights, rates, clusters } = this.convertNapiResultTxids(
await rustGbt.make(Object.values(newMempool) as RustThreadTransaction[], convertedAccelerations as RustThreadAcceleration[], this.nextUid),
);
if (saveResults) {
this.rustInitialized = true;
}
const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, accelerations, accelerationPool, saveResults);
logger.debug(`RUST makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`);
return processed;
} catch (e) {
logger.err('RUST makeBlockTemplates failed. ' + (e instanceof Error ? e.message : e));
if (saveResults) {
this.resetRustGbt();
}
}
return this.mempoolBlocks;
}
public async $oneOffRustBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, useAccelerations: boolean, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
return this.$rustMakeBlockTemplates(newMempool, false, useAccelerations, accelerationPool);
}
public async $rustUpdateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, mempoolSize: number, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], useAccelerations: boolean, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
// GBT optimization requires that uids never get too sparse
// as a sanity check, we should also explicitly prevent uint32 uid overflow
if (this.nextUid + added.length >= Math.min(Math.max(262144, 2 * mempoolSize), MAX_UINT32)) {
this.resetRustGbt();
}
if (!this.rustInitialized) {
// need to reset the worker
return this.$rustMakeBlockTemplates(newMempool, true, useAccelerations, accelerationPool);
}
const start = Date.now();
// set missing short ids
for (const tx of added) {
this.setUid(tx, true);
}
// set short ids for transaction inputs
for (const tx of added) {
tx.inputs = tx.vin.map(v => this.getUid(newMempool[v.txid])).filter(uid => (uid !== null && uid !== undefined)) as number[];
}
const removedUids = removed.map(tx => this.getUid(tx)).filter(uid => (uid !== null && uid !== undefined)) as number[];
const accelerations = useAccelerations ? mempool.getAccelerations() : {};
const acceleratedList = accelerationPool ? Object.values(accelerations).filter(acc => newMempool[acc.txid] && acc.pools.includes(accelerationPool)) : Object.values(accelerations).filter(acc => newMempool[acc.txid]);
const convertedAccelerations = acceleratedList.map(acc => {
return {
uid: this.getUid(newMempool[acc.txid]),
delta: acc.feeDelta,
};
});
// run the block construction algorithm in a separate thread, and wait for a result
try {
const { blocks, blockWeights, rates, clusters } = this.convertNapiResultTxids(
await this.rustGbtGenerator.update(
added as RustThreadTransaction[],
removedUids,
convertedAccelerations as RustThreadAcceleration[],
this.nextUid,
),
);
const resultMempoolSize = blocks.reduce((total, block) => total + block.length, 0);
if (mempoolSize !== resultMempoolSize) {
throw new Error('GBT returned wrong number of transactions, cache is probably out of sync');
} else {
const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, accelerations, accelerationPool, true);
this.removeUids(removedUids);
logger.debug(`RUST updateBlockTemplates completed in ${(Date.now() - start)/1000} seconds`);
return processed;
}
} catch (e) {
logger.err('RUST updateBlockTemplates failed. ' + (e instanceof Error ? e.message : e));
this.resetRustGbt();
return this.mempoolBlocks;
}
}
private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], accelerations, accelerationPool, saveResults): MempoolBlockWithTransactions[] {
for (const [txid, rate] of rates) {
if (txid in mempool) {
mempool[txid].effectiveFeePerVsize = rates[txid];
mempool[txid].cpfpDirty = (rate !== mempool[txid].effectiveFeePerVsize);
mempool[txid].effectiveFeePerVsize = rate;
mempool[txid].cpfpChecked = false;
}
}
const lastBlockIndex = blocks.length - 1;
let hasBlockStack = blocks.length >= 8;
let stackWeight;
let feeStatsCalculator: OnlineFeeStatsCalculator | void;
if (hasBlockStack) {
stackWeight = blocks[blocks.length - 1].reduce((total, tx) => total + (mempool[tx]?.weight || 0), 0);
hasBlockStack = stackWeight > config.MEMPOOL.BLOCK_WEIGHT_UNITS;
feeStatsCalculator = new OnlineFeeStatsCalculator(stackWeight, 0.5);
}
const readyBlocks: { transactionIds, transactions, totalSize, totalWeight, totalFees, feeStats }[] = [];
const sizeLimit = (config.MEMPOOL.BLOCK_WEIGHT_UNITS / 4) * 1.2;
// update this thread's mempool with the results
for (let blockIndex = 0; blockIndex < blocks.length; blockIndex++) {
const block: string[] = blocks[blockIndex];
let txid: string;
let mempoolTx: MempoolTransactionExtended;
let totalSize = 0;
let totalVsize = 0;
let totalWeight = 0;
let totalFees = 0;
const transactions: MempoolTransactionExtended[] = [];
for (let txIndex = 0; txIndex < block.length; txIndex++) {
txid = block[txIndex];
if (txid) {
mempoolTx = mempool[txid];
// save position in projected blocks
mempoolTx.position = {
block: blockIndex,
vsize: totalVsize + (mempoolTx.vsize / 2),
};
mempoolTx.ancestors = [];
mempoolTx.descendants = [];
mempoolTx.bestDescendant = null;
mempoolTx.cpfpChecked = true;
// online calculation of stack-of-blocks fee stats
if (hasBlockStack && blockIndex === blocks.length - 1 && feeStatsCalculator) {
feeStatsCalculator.processNext(mempoolTx);
}
totalSize += mempoolTx.size;
totalVsize += mempoolTx.vsize;
totalWeight += mempoolTx.weight;
totalFees += mempoolTx.fee;
if (totalVsize <= sizeLimit) {
transactions.push(mempoolTx);
}
}
if (blockWeights && blockWeights[7] !== null) {
stackWeight = blockWeights[7];
} else {
stackWeight = blocks[lastBlockIndex].reduce((total, tx) => total + (mempool[tx]?.weight || 0), 0);
}
readyBlocks.push({
transactionIds: block,
transactions,
totalSize,
totalWeight,
totalFees,
feeStats: (hasBlockStack && blockIndex === blocks.length - 1 && feeStatsCalculator) ? feeStatsCalculator.getRawFeeStats() : undefined,
});
hasBlockStack = stackWeight > config.MEMPOOL.BLOCK_WEIGHT_UNITS;
feeStatsCalculator = new OnlineFeeStatsCalculator(stackWeight, 0.5, [10, 20, 30, 40, 50, 60, 70, 80, 90]);
}
for (const cluster of Object.values(clusters)) {
for (const cluster of clusters) {
for (const memberTxid of cluster) {
if (memberTxid in mempool) {
const mempoolTx = mempool[memberTxid];
const mempoolTx = mempool[memberTxid];
if (mempoolTx) {
const ancestors: Ancestor[] = [];
const descendants: Ancestor[] = [];
let matched = false;
@@ -406,20 +489,93 @@ class MempoolBlocks {
};
if (matched) {
descendants.push(relative);
mempoolTx.lastBoosted = Math.max(mempoolTx.lastBoosted || 0, mempool[txid].firstSeen || 0);
} else {
ancestors.push(relative);
}
}
});
mempoolTx.ancestors = ancestors;
mempoolTx.descendants = descendants;
mempoolTx.bestDescendant = null;
if (mempoolTx.ancestors?.length !== ancestors.length || mempoolTx.descendants?.length !== descendants.length) {
mempoolTx.cpfpDirty = true;
}
Object.assign(mempoolTx, {ancestors, descendants, bestDescendant: null, cpfpChecked: true});
}
}
}
const mempoolBlocks = readyBlocks.map((b, index) => {
return this.dataToMempoolBlocks(b.transactionIds, b.transactions, b.totalSize, b.totalWeight, b.totalFees, b.feeStats);
const isAccelerated : { [txid: string]: boolean } = {};
const sizeLimit = (config.MEMPOOL.BLOCK_WEIGHT_UNITS / 4) * 1.2;
// update this thread's mempool with the results
let mempoolTx: MempoolTransactionExtended;
const mempoolBlocks: MempoolBlockWithTransactions[] = blocks.map((block, blockIndex) => {
let totalSize = 0;
let totalVsize = 0;
let totalWeight = 0;
let totalFees = 0;
const transactions: MempoolTransactionExtended[] = [];
for (const txid of block) {
if (txid) {
mempoolTx = mempool[txid];
// save position in projected blocks
mempoolTx.position = {
block: blockIndex,
vsize: totalVsize + (mempoolTx.vsize / 2),
};
if (!mempoolTx.cpfpChecked) {
if (mempoolTx.ancestors?.length) {
mempoolTx.ancestors = [];
}
if (mempoolTx.descendants?.length) {
mempoolTx.descendants = [];
}
mempoolTx.bestDescendant = null;
mempoolTx.cpfpChecked = true;
}
const acceleration = accelerations[txid];
if (isAccelerated[txid] || (acceleration && (!accelerationPool || acceleration.pools.includes(accelerationPool)))) {
if (!mempoolTx.acceleration) {
mempoolTx.cpfpDirty = true;
}
mempoolTx.acceleration = true;
for (const ancestor of mempoolTx.ancestors || []) {
if (!mempool[ancestor.txid].acceleration) {
mempool[ancestor.txid].cpfpDirty = true;
}
mempool[ancestor.txid].acceleration = true;
isAccelerated[ancestor.txid] = true;
}
} else {
if (mempoolTx.acceleration) {
mempoolTx.cpfpDirty = true;
}
delete mempoolTx.acceleration;
}
// online calculation of stack-of-blocks fee stats
if (hasBlockStack && blockIndex === lastBlockIndex && feeStatsCalculator) {
feeStatsCalculator.processNext(mempoolTx);
}
totalSize += mempoolTx.size;
totalVsize += mempoolTx.vsize;
totalWeight += mempoolTx.weight;
totalFees += mempoolTx.fee;
if (totalVsize <= sizeLimit) {
transactions.push(mempoolTx);
}
}
}
return this.dataToMempoolBlocks(
block,
transactions,
totalSize,
totalWeight,
totalFees,
(hasBlockStack && blockIndex === lastBlockIndex && feeStatsCalculator) ? feeStatsCalculator.getRawFeeStats() : undefined,
);
});
if (saveResults) {
@@ -433,7 +589,7 @@ class MempoolBlocks {
private dataToMempoolBlocks(transactionIds: string[], transactions: MempoolTransactionExtended[], totalSize: number, totalWeight: number, totalFees: number, feeStats?: EffectiveFeeStats ): MempoolBlockWithTransactions {
if (!feeStats) {
feeStats = Common.calcEffectiveFeeStatistics(transactions);
feeStats = Common.calcEffectiveFeeStatistics(transactions.filter(tx => !tx.acceleration));
}
return {
blockSize: totalSize,
@@ -452,16 +608,20 @@ class MempoolBlocks {
this.nextUid = 1;
}
private setUid(tx: MempoolTransactionExtended): number {
const uid = this.nextUid;
this.nextUid++;
this.uidMap.set(uid, tx.txid);
tx.uid = uid;
return uid;
private setUid(tx: MempoolTransactionExtended, skipSet = false): number {
if (tx.uid === null || tx.uid === undefined || !skipSet) {
const uid = this.nextUid;
this.nextUid++;
this.uidMap.set(uid, tx.txid);
tx.uid = uid;
return uid;
} else {
return tx.uid;
}
}
private getUid(tx: MempoolTransactionExtended): number | void {
if (tx?.uid != null && this.uidMap.has(tx.uid)) {
if (tx?.uid !== null && tx?.uid !== undefined && this.uidMap.has(tx.uid)) {
return tx.uid;
}
}
@@ -496,6 +656,28 @@ class MempoolBlocks {
}
return { blocks: convertedBlocks, rates: convertedRates, clusters: convertedClusters } as { blocks: string[][], rates: { [root: string]: number }, clusters: { [root: string]: string[] }};
}
private convertNapiResultTxids({ blocks, blockWeights, rates, clusters }: GbtResult)
: { blocks: string[][], blockWeights: number[], rates: [string, number][], clusters: string[][] } {
const convertedBlocks: string[][] = blocks.map(block => block.map(uid => {
const txid = this.uidMap.get(uid);
if (txid !== undefined) {
return txid;
} else {
throw new Error('GBT returned a block containing a transaction with unknown uid');
}
}));
const convertedRates: [string, number][] = [];
for (const [rateUid, rate] of rates) {
const rateTxid = this.uidMap.get(rateUid) as string;
convertedRates.push([rateTxid, rate]);
}
const convertedClusters: string[][] = [];
for (const cluster of clusters) {
convertedClusters.push(cluster.map(uid => this.uidMap.get(uid)) as string[]);
}
return { blocks: convertedBlocks, blockWeights, rates: convertedRates, clusters: convertedClusters };
}
}
export default new MempoolBlocks();

260
backend/src/api/mempool.ts
View File

@@ -9,6 +9,8 @@ import loadingIndicators from './loading-indicators';
import bitcoinClient from './bitcoin/bitcoin-client';
import bitcoinSecondClient from './bitcoin/bitcoin-second-client';
import rbfCache from './rbf-cache';
import accelerationApi, { Acceleration } from './services/acceleration';
import redisCache from './redis-cache';
class Mempool {
private inSync: boolean = false;
@@ -18,9 +20,11 @@ class Mempool {
private mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0, total_fee: 0,
maxmempool: 300000000, mempoolminfee: 0.00001000, minrelaytxfee: 0.00001000 };
private mempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, newTransactions: MempoolTransactionExtended[],
deletedTransactions: MempoolTransactionExtended[]) => void) | undefined;
private $asyncMempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, newTransactions: MempoolTransactionExtended[],
deletedTransactions: MempoolTransactionExtended[]) => Promise<void>) | undefined;
deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => void) | undefined;
private $asyncMempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, mempoolSize: number, newTransactions: MempoolTransactionExtended[],
deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => Promise<void>) | undefined;
private accelerations: { [txId: string]: Acceleration } = {};
private txPerSecondArray: number[] = [];
private txPerSecond: number = 0;
@@ -65,12 +69,12 @@ class Mempool {
}
public setMempoolChangedCallback(fn: (newMempool: { [txId: string]: MempoolTransactionExtended; },
newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[]) => void): void {
newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => void): void {
this.mempoolChangedCallback = fn;
}
public setAsyncMempoolChangedCallback(fn: (newMempool: { [txId: string]: MempoolTransactionExtended; },
newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[]) => Promise<void>): void {
public setAsyncMempoolChangedCallback(fn: (newMempool: { [txId: string]: MempoolTransactionExtended; }, mempoolSize: number,
newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => Promise<void>): void {
this.$asyncMempoolChangedCallback = fn;
}
@@ -84,20 +88,74 @@ class Mempool {
public async $setMempool(mempoolData: { [txId: string]: MempoolTransactionExtended }) {
this.mempoolCache = mempoolData;
let count = 0;
const redisTimer = Date.now();
if (config.MEMPOOL.CACHE_ENABLED && config.REDIS.ENABLED) {
logger.debug(`Migrating ${Object.keys(this.mempoolCache).length} transactions from disk cache to Redis cache`);
}
for (const txid of Object.keys(this.mempoolCache)) {
if (this.mempoolCache[txid].sigops == null || this.mempoolCache[txid].effectiveFeePerVsize == null) {
if (!this.mempoolCache[txid].sigops || this.mempoolCache[txid].effectiveFeePerVsize == null) {
this.mempoolCache[txid] = transactionUtils.extendMempoolTransaction(this.mempoolCache[txid]);
}
if (this.mempoolCache[txid].order == null) {
this.mempoolCache[txid].order = transactionUtils.txidToOrdering(txid);
}
count++;
if (config.MEMPOOL.CACHE_ENABLED && config.REDIS.ENABLED) {
await redisCache.$addTransaction(this.mempoolCache[txid]);
}
}
if (config.MEMPOOL.CACHE_ENABLED && config.REDIS.ENABLED) {
await redisCache.$flushTransactions();
logger.debug(`Finished migrating cache transactions in ${((Date.now() - redisTimer) / 1000).toFixed(2)} seconds`);
}
if (this.mempoolChangedCallback) {
this.mempoolChangedCallback(this.mempoolCache, [], []);
this.mempoolChangedCallback(this.mempoolCache, [], [], []);
}
if (this.$asyncMempoolChangedCallback) {
await this.$asyncMempoolChangedCallback(this.mempoolCache, [], []);
await this.$asyncMempoolChangedCallback(this.mempoolCache, count, [], [], []);
}
this.addToSpendMap(Object.values(this.mempoolCache));
}
public async $reloadMempool(expectedCount: number): Promise<MempoolTransactionExtended[]> {
let count = 0;
let done = false;
let last_txid;
const newTransactions: MempoolTransactionExtended[] = [];
loadingIndicators.setProgress('mempool', count / expectedCount * 100);
while (!done) {
try {
const result = await bitcoinApi.$getAllMempoolTransactions(last_txid);
if (result) {
for (const tx of result) {
const extendedTransaction = transactionUtils.extendMempoolTransaction(tx);
if (!this.mempoolCache[extendedTransaction.txid]) {
newTransactions.push(extendedTransaction);
this.mempoolCache[extendedTransaction.txid] = extendedTransaction;
}
count++;
}
logger.info(`Fetched ${count} of ${expectedCount} mempool transactions from esplora`);
if (result.length > 0) {
last_txid = result[result.length - 1].txid;
} else {
done = true;
}
if (Math.floor((count / expectedCount) * 100) < 100) {
loadingIndicators.setProgress('mempool', count / expectedCount * 100);
}
} else {
done = true;
}
} catch(err) {
logger.err('failed to fetch bulk mempool transactions from esplora');
}
}
logger.info(`Done inserting loaded mempool transactions into local cache`);
return newTransactions;
}
public async $updateMemPoolInfo() {
this.mempoolInfo = await this.$getMempoolInfo();
}
@@ -127,7 +185,7 @@ class Mempool {
return txTimes;
}
public async $updateMempool(transactions: string[]): Promise<void> {
public async $updateMempool(transactions: string[], pollRate: number): Promise<void> {
logger.debug(`Updating mempool...`);
// warn if this run stalls the main loop for more than 2 minutes
@@ -138,7 +196,7 @@ class Mempool {
const currentMempoolSize = Object.keys(this.mempoolCache).length;
this.updateTimerProgress(timer, 'got raw mempool');
const diff = transactions.length - currentMempoolSize;
const newTransactions: MempoolTransactionExtended[] = [];
let newTransactions: MempoolTransactionExtended[] = [];
this.mempoolCacheDelta = Math.abs(diff);
@@ -156,13 +214,36 @@ class Mempool {
}
};
let loggerTimer = new Date().getTime() / 1000;
for (const txid of transactions) {
if (!this.mempoolCache[txid]) {
try {
const transaction = await transactionUtils.$getMempoolTransactionExtended(txid, false, false, false);
this.updateTimerProgress(timer, 'fetched new transaction');
this.mempoolCache[txid] = transaction;
let intervalTimer = Date.now();
let loaded = false;
if (config.MEMPOOL.BACKEND === 'esplora' && currentMempoolSize < transactions.length * 0.5 && transactions.length > 20_000) {
this.inSync = false;
logger.info(`Missing ${transactions.length - currentMempoolSize} mempool transactions, attempting to reload in bulk from esplora`);
try {
newTransactions = await this.$reloadMempool(transactions.length);
if (config.REDIS.ENABLED) {
for (const tx of newTransactions) {
await redisCache.$addTransaction(tx);
}
}
loaded = true;
} catch (e) {
logger.err('failed to load mempool in bulk from esplora, falling back to fetching individual transactions');
}
}
if (!loaded) {
const remainingTxids = transactions.filter(txid => !this.mempoolCache[txid]);
const sliceLength = 10000;
for (let i = 0; i < Math.ceil(remainingTxids.length / sliceLength); i++) {
const slice = remainingTxids.slice(i * sliceLength, (i + 1) * sliceLength);
const txs = await transactionUtils.$getMempoolTransactionsExtended(slice, false, false, false);
logger.debug(`fetched ${txs.length} transactions`);
this.updateTimerProgress(timer, 'fetched new transactions');
for (const transaction of txs) {
this.mempoolCache[transaction.txid] = transaction;
if (this.inSync) {
this.txPerSecondArray.push(new Date().getTime());
this.vBytesPerSecondArray.push({
@@ -172,25 +253,35 @@ class Mempool {
}
hasChange = true;
newTransactions.push(transaction);
} catch (e: any) {
if (config.MEMPOOL.BACKEND === 'esplora' && e.response?.status === 404) {
this.missingTxCount++;
if (config.REDIS.ENABLED) {
await redisCache.$addTransaction(transaction);
}
}
if (txs.length < slice.length) {
const missing = slice.length - txs.length;
if (config.MEMPOOL.BACKEND === 'esplora') {
this.missingTxCount += missing;
}
logger.debug(`Error finding ${missing} transactions in the mempool: `);
}
if (Date.now() - intervalTimer > Math.max(pollRate * 2, 5_000)) {
if (this.inSync) {
// Break and restart mempool loop if we spend too much time processing
// new transactions that may lead to falling behind on block height
logger.debug('Breaking mempool loop because the 5s time limit exceeded.');
break;
} else {
const progress = (currentMempoolSize + newTransactions.length) / transactions.length * 100;
logger.debug(`Mempool is synchronizing. Processed ${newTransactions.length}/${diff} txs (${Math.round(progress)}%)`);
if (Math.floor(progress) < 100) {
loadingIndicators.setProgress('mempool', progress);
}
intervalTimer = Date.now();
}
logger.debug(`Error finding transaction '${txid}' in the mempool: ` + (e instanceof Error ? e.message : e));
}
}
const elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer);
if (elapsedSeconds > 4) {
const progress = (currentMempoolSize + newTransactions.length) / transactions.length * 100;
logger.debug(`Mempool is synchronizing. Processed ${newTransactions.length}/${diff} txs (${Math.round(progress)}%)`);
loadingIndicators.setProgress('mempool', progress);
loggerTimer = new Date().getTime() / 1000;
}
// Break and restart mempool loop if we spend too much time processing
// new transactions that may lead to falling behind on block height
if (this.inSync && (new Date().getTime()) - start > 10_000) {
logger.debug('Breaking mempool loop because the 10s time limit exceeded.');
break;
}
}
@@ -212,7 +303,7 @@ class Mempool {
logger.warn(`Mempool clear protection triggered because transactions.length: ${transactions.length} and currentMempoolSize: ${currentMempoolSize}.`);
setTimeout(() => {
this.mempoolProtection = 2;
logger.warn('Mempool clear protection resumed.');
logger.warn('Mempool clear protection ended, normal operation resumed.');
}, 1000 * 60 * config.MEMPOOL.CLEAR_PROTECTION_MINUTES);
}
@@ -235,24 +326,37 @@ class Mempool {
}
}
const newMempoolSize = currentMempoolSize + newTransactions.length - deletedTransactions.length;
const newTransactionsStripped = newTransactions.map((tx) => Common.stripTransaction(tx));
this.latestTransactions = newTransactionsStripped.concat(this.latestTransactions).slice(0, 6);
if (!this.inSync && transactions.length === Object.keys(this.mempoolCache).length) {
const accelerationDelta = await this.$updateAccelerations();
if (accelerationDelta.length) {
hasChange = true;
}
this.mempoolCacheDelta = Math.abs(transactions.length - newMempoolSize);
if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) {
this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions, accelerationDelta);
}
if (this.$asyncMempoolChangedCallback && (hasChange || deletedTransactions.length)) {
this.updateTimerProgress(timer, 'running async mempool callback');
await this.$asyncMempoolChangedCallback(this.mempoolCache, newMempoolSize, newTransactions, deletedTransactions, accelerationDelta);
this.updateTimerProgress(timer, 'completed async mempool callback');
}
if (!this.inSync && transactions.length === newMempoolSize) {
this.inSync = true;
logger.notice('The mempool is now in sync!');
loadingIndicators.setProgress('mempool', 100);
}
this.mempoolCacheDelta = Math.abs(transactions.length - Object.keys(this.mempoolCache).length);
if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) {
this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions);
}
if (this.$asyncMempoolChangedCallback && (hasChange || deletedTransactions.length)) {
this.updateTimerProgress(timer, 'running async mempool callback');
await this.$asyncMempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions);
this.updateTimerProgress(timer, 'completed async mempool callback');
// Update Redis cache
if (config.REDIS.ENABLED) {
await redisCache.$flushTransactions();
await redisCache.$removeTransactions(deletedTransactions.map(tx => tx.txid));
await rbfCache.updateCache();
}
const end = new Date().getTime();
@@ -262,6 +366,70 @@ class Mempool {
this.clearTimer(timer);
}
public getAccelerations(): { [txid: string]: Acceleration } {
return this.accelerations;
}
public async $updateAccelerations(): Promise<string[]> {
if (!config.MEMPOOL_SERVICES.ACCELERATIONS) {
return [];
}
try {
const newAccelerations = await accelerationApi.$fetchAccelerations();
const changed: string[] = [];
const newAccelerationMap: { [txid: string]: Acceleration } = {};
for (const acceleration of newAccelerations) {
newAccelerationMap[acceleration.txid] = acceleration;
if (this.accelerations[acceleration.txid] == null) {
// new acceleration
changed.push(acceleration.txid);
} else {
if (this.accelerations[acceleration.txid].feeDelta !== acceleration.feeDelta) {
// feeDelta changed
changed.push(acceleration.txid);
} else if (this.accelerations[acceleration.txid].pools?.length) {
let poolsChanged = false;
const pools = new Set();
this.accelerations[acceleration.txid].pools.forEach(pool => {
pools.add(pool);
});
acceleration.pools.forEach(pool => {
if (!pools.has(pool)) {
poolsChanged = true;
} else {
pools.delete(pool);
}
});
if (pools.size > 0) {
poolsChanged = true;
}
if (poolsChanged) {
// pools changed
changed.push(acceleration.txid);
}
}
}
}
for (const oldTxid of Object.keys(this.accelerations)) {
if (!newAccelerationMap[oldTxid]) {
// removed
changed.push(oldTxid);
}
}
this.accelerations = newAccelerationMap;
return changed;
} catch (e: any) {
logger.debug(`Failed to update accelerations: ` + (e instanceof Error ? e.message : e));
return [];
}
}
private startTimer() {
const state: any = {
start: Date.now(),

28
backend/src/api/mining/mining-routes.ts
View File

@@ -12,6 +12,7 @@ import PricesRepository from '../../repositories/PricesRepository';
class MiningRoutes {
public initRoutes(app: Application) {
app
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools', this.$listPools)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/:interval', this.$getPools)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug/hashrate', this.$getPoolHistoricalHashrate)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug/blocks', this.$getPoolBlocks)
@@ -41,6 +42,10 @@ class MiningRoutes {
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
if (['testnet', 'signet', 'liquidtestnet'].includes(config.MEMPOOL.NETWORK)) {
res.status(400).send('Prices are not available on testnets.');
return;
}
if (req.query.timestamp) {
res.status(200).send(await PricesRepository.$getNearestHistoricalPrice(
parseInt(<string>req.query.timestamp ?? 0, 10)
@@ -88,6 +93,29 @@ class MiningRoutes {
}
}
private async $listPools(req: Request, res: Response): Promise<void> {
try {
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
const pools = await mining.$listPools();
if (!pools) {
res.status(500).end();
return;
}
res.header('X-total-count', pools.length.toString());
if (pools.length === 0) {
res.status(204).send();
} else {
res.json(pools);
}
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
private async $getPools(req: Request, res: Response) {
try {
const stats = await mining.$getPoolsStats(req.params.interval);

38
backend/src/api/mining/mining.ts
View File

@@ -11,7 +11,7 @@ import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjust
import config from '../../config';
import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository';
import PricesRepository from '../../repositories/PricesRepository';
import { bitcoinCoreApi } from '../bitcoin/bitcoin-api-factory';
import bitcoinApi from '../bitcoin/bitcoin-api-factory';
import { IEsploraApi } from '../bitcoin/esplora-api.interface';
import database from '../../database';
@@ -26,7 +26,7 @@ class Mining {
/**
* Get historical blocks health
*/
public async $getBlocksHealthHistory(interval: string | null = null): Promise<any> {
public async $getBlocksHealthHistory(interval: string | null = null): Promise<any> {
return await BlocksAuditsRepository.$getBlocksHealthHistory(
this.getTimeRange(interval),
Common.getSqlInterval(interval)
@@ -56,7 +56,7 @@ class Mining {
/**
* Get historical block fee rates percentiles
*/
public async $getHistoricalBlockFeeRates(interval: string | null = null): Promise<any> {
public async $getHistoricalBlockFeeRates(interval: string | null = null): Promise<any> {
return await BlocksRepository.$getHistoricalBlockFeeRates(
this.getTimeRange(interval),
Common.getSqlInterval(interval)
@@ -66,7 +66,7 @@ class Mining {
/**
* Get historical block sizes
*/
public async $getHistoricalBlockSizes(interval: string | null = null): Promise<any> {
public async $getHistoricalBlockSizes(interval: string | null = null): Promise<any> {
return await BlocksRepository.$getHistoricalBlockSizes(
this.getTimeRange(interval),
Common.getSqlInterval(interval)
@@ -76,7 +76,7 @@ class Mining {
/**
* Get historical block weights
*/
public async $getHistoricalBlockWeights(interval: string | null = null): Promise<any> {
public async $getHistoricalBlockWeights(interval: string | null = null): Promise<any> {
return await BlocksRepository.$getHistoricalBlockWeights(
this.getTimeRange(interval),
Common.getSqlInterval(interval)
@@ -106,6 +106,8 @@ class Mining {
emptyBlocks: emptyBlocksCount.length > 0 ? emptyBlocksCount[0]['count'] : 0,
slug: poolInfo.slug,
avgMatchRate: poolInfo.avgMatchRate !== null ? Math.round(100 * poolInfo.avgMatchRate) / 100 : null,
avgFeeDelta: poolInfo.avgFeeDelta,
poolUniqueId: poolInfo.poolUniqueId
};
poolsStats.push(poolStat);
});
@@ -200,7 +202,7 @@ class Mining {
try {
const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp;
const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0));
const genesisBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(await bitcoinApi.$getBlockHash(0));
const genesisTimestamp = genesisBlock.timestamp * 1000;
const indexedTimestamp = await HashratesRepository.$getWeeklyHashrateTimestamps();
@@ -311,7 +313,7 @@ class Mining {
const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp;
try {
const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0));
const genesisBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(await bitcoinApi.$getBlockHash(0));
const genesisTimestamp = genesisBlock.timestamp * 1000;
const indexedTimestamp = (await HashratesRepository.$getRawNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp);
const lastMidnight = this.getDateMidnight(new Date());
@@ -420,8 +422,9 @@ class Mining {
}
const blocks: any = await BlocksRepository.$getBlocksDifficulty();
const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0));
const genesisBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(await bitcoinApi.$getBlockHash(0));
let currentDifficulty = genesisBlock.difficulty;
let currentBits = genesisBlock.bits;
let totalIndexed = 0;
if (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT === -1 && indexedHeights[0] !== true) {
@@ -435,6 +438,7 @@ class Mining {
const oldestConsecutiveBlock = await BlocksRepository.$getOldestConsecutiveBlock();
if (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT !== -1) {
currentBits = oldestConsecutiveBlock.bits;
currentDifficulty = oldestConsecutiveBlock.difficulty;
}
@@ -442,10 +446,11 @@ class Mining {
let timer = new Date().getTime() / 1000;
for (const block of blocks) {
if (block.difficulty !== currentDifficulty) {
if (block.bits !== currentBits) {
if (indexedHeights[block.height] === true) { // Already indexed
if (block.height >= oldestConsecutiveBlock.height) {
currentDifficulty = block.difficulty;
currentBits = block.bits;
}
continue;
}
@@ -463,6 +468,7 @@ class Mining {
totalIndexed++;
if (block.height >= oldestConsecutiveBlock.height) {
currentDifficulty = block.difficulty;
currentBits = block.bits;
}
}
@@ -589,6 +595,20 @@ class Mining {
}
}
/**
* List existing mining pools
*/
public async $listPools(): Promise<{name: string, slug: string, unique_id: number}[] | null> {
const [rows] = await database.query(`
SELECT
name,
slug,
unique_id
FROM pools`
);
return rows as {name: string, slug: string, unique_id: number}[];
}
private getDateMidnight(date: Date): Date {
date.setUTCHours(0);
date.setUTCMinutes(0);

19
backend/src/api/prices/prices.routes.ts Normal file
View File

@@ -0,0 +1,19 @@
import { Application, Request, Response } from 'express';
import config from '../../config';
import pricesUpdater from '../../tasks/price-updater';
class PricesRoutes {
public initRoutes(app: Application): void {
app.get(config.MEMPOOL.API_URL_PREFIX + 'prices', this.$getCurrentPrices.bind(this));
}
private $getCurrentPrices(req: Request, res: Response): void {
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 360_0000 / config.MEMPOOL.PRICE_UPDATES_PER_HOUR).toUTCString());
res.json(pricesUpdater.getLatestPrices());
}
}
export default new PricesRoutes();

186
backend/src/api/rbf-cache.ts
View File

@@ -1,14 +1,17 @@
import config from "../config";
import logger from "../logger";
import { MempoolTransactionExtended, TransactionStripped } from "../mempool.interfaces";
import bitcoinApi from './bitcoin/bitcoin-api-factory';
import { Common } from "./common";
import redisCache from "./redis-cache";
interface RbfTransaction extends TransactionStripped {
export interface RbfTransaction extends TransactionStripped {
rbf?: boolean;
mined?: boolean;
fullRbf?: boolean;
}
interface RbfTree {
export interface RbfTree {
tx: RbfTransaction;
time: number;
interval?: number;
@@ -17,6 +20,29 @@ interface RbfTree {
replaces: RbfTree[];
}
export interface ReplacementInfo {
mined: boolean;
fullRbf: boolean;
txid: string;
oldFee: number;
oldVsize: number;
newFee: number;
newVsize: number;
}
enum CacheOp {
Remove = 0,
Add = 1,
Change = 2,
}
interface CacheEvent {
op: CacheOp;
type: 'tx' | 'tree' | 'exp';
txid: string,
value?: any,
}
class RbfCache {
private replacedBy: Map<string, string> = new Map();
private replaces: Map<string, string[]> = new Map();
@@ -25,11 +51,43 @@ class RbfCache {
private treeMap: Map<string, string> = new Map(); // map of txids to sequence ids
private txs: Map<string, MempoolTransactionExtended> = new Map();
private expiring: Map<string, number> = new Map();
private cacheQueue: CacheEvent[] = [];
constructor() {
setInterval(this.cleanup.bind(this), 1000 * 60 * 10);
}
private addTx(txid: string, tx: MempoolTransactionExtended): void {
this.txs.set(txid, tx);
this.cacheQueue.push({ op: CacheOp.Add, type: 'tx', txid });
}
private addTree(txid: string, tree: RbfTree): void {
this.rbfTrees.set(txid, tree);
this.dirtyTrees.add(txid);
this.cacheQueue.push({ op: CacheOp.Add, type: 'tree', txid });
}
private addExpiration(txid: string, expiry: number): void {
this.expiring.set(txid, expiry);
this.cacheQueue.push({ op: CacheOp.Add, type: 'exp', txid, value: expiry });
}
private removeTx(txid: string): void {
this.txs.delete(txid);
this.cacheQueue.push({ op: CacheOp.Remove, type: 'tx', txid });
}
private removeTree(txid: string): void {
this.rbfTrees.delete(txid);
this.cacheQueue.push({ op: CacheOp.Remove, type: 'tree', txid });
}
private removeExpiration(txid: string): void {
this.expiring.delete(txid);
this.cacheQueue.push({ op: CacheOp.Remove, type: 'exp', txid });
}
public add(replaced: MempoolTransactionExtended[], newTxExtended: MempoolTransactionExtended): void {
if (!newTxExtended || !replaced?.length || this.txs.has(newTxExtended.txid)) {
return;
@@ -38,24 +96,28 @@ class RbfCache {
const newTx = Common.stripTransaction(newTxExtended) as RbfTransaction;
const newTime = newTxExtended.firstSeen || (Date.now() / 1000);
newTx.rbf = newTxExtended.vin.some((v) => v.sequence < 0xfffffffe);
this.txs.set(newTx.txid, newTxExtended);
this.addTx(newTx.txid, newTxExtended);
// maintain rbf trees
let fullRbf = false;
let txFullRbf = false;
let treeFullRbf = false;
const replacedTrees: RbfTree[] = [];
for (const replacedTxExtended of replaced) {
const replacedTx = Common.stripTransaction(replacedTxExtended) as RbfTransaction;
replacedTx.rbf = replacedTxExtended.vin.some((v) => v.sequence < 0xfffffffe);
if (!replacedTx.rbf) {
txFullRbf = true;
}
this.replacedBy.set(replacedTx.txid, newTx.txid);
if (this.treeMap.has(replacedTx.txid)) {
const treeId = this.treeMap.get(replacedTx.txid);
if (treeId) {
const tree = this.rbfTrees.get(treeId);
this.rbfTrees.delete(treeId);
this.removeTree(treeId);
if (tree) {
tree.interval = newTime - tree?.time;
replacedTrees.push(tree);
fullRbf = fullRbf || tree.fullRbf;
treeFullRbf = treeFullRbf || tree.fullRbf || !tree.tx.rbf;
}
}
} else {
@@ -67,21 +129,39 @@ class RbfCache {
fullRbf: !replacedTx.rbf,
replaces: [],
});
fullRbf = fullRbf || !replacedTx.rbf;
this.txs.set(replacedTx.txid, replacedTxExtended);
treeFullRbf = treeFullRbf || !replacedTx.rbf;
this.addTx(replacedTx.txid, replacedTxExtended);
}
}
newTx.fullRbf = txFullRbf;
const treeId = replacedTrees[0].tx.txid;
const newTree = {
tx: newTx,
time: newTime,
fullRbf,
fullRbf: treeFullRbf,
replaces: replacedTrees
};
this.rbfTrees.set(treeId, newTree);
this.addTree(treeId, newTree);
this.updateTreeMap(treeId, newTree);
this.replaces.set(newTx.txid, replacedTrees.map(tree => tree.tx.txid));
this.dirtyTrees.add(treeId);
}
public has(txId: string): boolean {
return this.txs.has(txId);
}
public anyInSameTree(txId: string, predicate: (tx: RbfTransaction) => boolean): boolean {
const tree = this.getRbfTree(txId);
if (!tree) {
return false;
}
const txs = this.getTransactionsInTree(tree);
for (const tx of txs) {
if (predicate(tx)) {
return true;
}
}
return false;
}
public getReplacedBy(txId: string): string | undefined {
@@ -157,6 +237,7 @@ class RbfCache {
this.setTreeMined(tree, txid);
tree.mined = true;
this.dirtyTrees.add(treeId);
this.cacheQueue.push({ op: CacheOp.Change, type: 'tree', txid: treeId });
}
}
this.evict(txid);
@@ -165,19 +246,33 @@ class RbfCache {
// flag a transaction as removed from the mempool
public evict(txid: string, fast: boolean = false): void {
if (this.txs.has(txid) && (fast || !this.expiring.has(txid))) {
this.expiring.set(txid, fast ? Date.now() + (1000 * 60 * 10) : Date.now() + (1000 * 86400)); // 24 hours
const expiryTime = fast ? Date.now() + (1000 * 60 * 10) : Date.now() + (1000 * 86400); // 24 hours
this.addExpiration(txid, expiryTime);
}
}
// is the transaction involved in a full rbf replacement?
public isFullRbf(txid: string): boolean {
const treeId = this.treeMap.get(txid);
if (!treeId) {
return false;
}
const tree = this.rbfTrees.get(treeId);
if (!tree) {
return false;
}
return tree?.fullRbf;
}
private cleanup(): void {
const now = Date.now();
for (const txid of this.expiring.keys()) {
if ((this.expiring.get(txid) || 0) < now) {
this.expiring.delete(txid);
this.removeExpiration(txid);
this.remove(txid);
}
}
logger.debug(`rbf cache contains ${this.txs.size} txs, ${this.expiring.size} due to expire`);
logger.debug(`rbf cache contains ${this.txs.size} txs, ${this.rbfTrees.size} trees, ${this.expiring.size} due to expire`);
}
// remove a transaction & all previous versions from the cache
@@ -187,14 +282,14 @@ class RbfCache {
const replaces = this.replaces.get(txid);
this.replaces.delete(txid);
this.treeMap.delete(txid);
this.txs.delete(txid);
this.expiring.delete(txid);
this.removeTx(txid);
this.removeExpiration(txid);
for (const tx of (replaces || [])) {
// recursively remove prior versions from the cache
this.replacedBy.delete(tx);
// if this is the id of a tree, remove that too
if (this.treeMap.get(tx) === tx) {
this.rbfTrees.delete(tx);
this.removeTree(tx);
}
this.remove(tx);
}
@@ -226,6 +321,33 @@ class RbfCache {
}
}
public async updateCache(): Promise<void> {
if (!config.REDIS.ENABLED) {
return;
}
// Update the Redis cache by replaying queued events
for (const e of this.cacheQueue) {
if (e.op === CacheOp.Add || e.op === CacheOp.Change) {
let value = e.value;
switch(e.type) {
case 'tx': {
value = this.txs.get(e.txid);
} break;
case 'tree': {
const tree = this.rbfTrees.get(e.txid);
value = tree ? this.exportTree(tree) : null;
} break;
}
if (value != null) {
await redisCache.$setRbfEntry(e.type, e.txid, value);
}
} else if (e.op === CacheOp.Remove) {
await redisCache.$removeRbfEntry(e.type, e.txid);
}
}
this.cacheQueue = [];
}
public dump(): any {
const trees = Array.from(this.rbfTrees.values()).map((tree: RbfTree) => { return this.exportTree(tree); });
@@ -238,14 +360,14 @@ class RbfCache {
public async load({ txs, trees, expiring }): Promise<void> {
txs.forEach(txEntry => {
this.txs.set(txEntry[0], txEntry[1]);
this.txs.set(txEntry.key, txEntry.value);
});
for (const deflatedTree of trees) {
await this.importTree(deflatedTree.root, deflatedTree.root, deflatedTree, this.txs);
}
expiring.forEach(expiringEntry => {
if (this.txs.has(expiringEntry[0])) {
this.expiring.set(expiringEntry[0], new Date(expiringEntry[1]).getTime());
if (this.txs.has(expiringEntry.key)) {
this.expiring.set(expiringEntry.key, new Date(expiringEntry.value).getTime());
}
});
this.cleanup();
@@ -331,11 +453,31 @@ class RbfCache {
};
this.treeMap.set(txid, root);
if (root === txid) {
this.rbfTrees.set(root, tree);
this.dirtyTrees.add(root);
this.addTree(root, tree);
}
return tree;
}
public getLatestRbfSummary(): ReplacementInfo[] {
const rbfList = this.getRbfTrees(false);
return rbfList.slice(0, 6).map(rbfTree => {
let oldFee = 0;
let oldVsize = 0;
for (const replaced of rbfTree.replaces) {
oldFee += replaced.tx.fee;
oldVsize += replaced.tx.vsize;
}
return {
txid: rbfTree.tx.txid,
mined: !!rbfTree.tx.mined,
fullRbf: !!rbfTree.tx.fullRbf,
oldFee,
oldVsize,
newFee: rbfTree.tx.fee,
newVsize: rbfTree.tx.vsize,
};
});
}
}
export default new RbfCache();

276
backend/src/api/redis-cache.ts Normal file
View File

@@ -0,0 +1,276 @@
import { createClient } from 'redis';
import memPool from './mempool';
import blocks from './blocks';
import logger from '../logger';
import config from '../config';
import { BlockExtended, BlockSummary, MempoolTransactionExtended } from '../mempool.interfaces';
import rbfCache from './rbf-cache';
import transactionUtils from './transaction-utils';
enum NetworkDB {
mainnet = 0,
testnet,
signet,
liquid,
liquidtestnet,
}
class RedisCache {
private client;
private connected = false;
private schemaVersion = 1;
private cacheQueue: MempoolTransactionExtended[] = [];
private txFlushLimit: number = 10000;
constructor() {
if (config.REDIS.ENABLED) {
const redisConfig = {
socket: {
path: config.REDIS.UNIX_SOCKET_PATH
},
database: NetworkDB[config.MEMPOOL.NETWORK],
};
this.client = createClient(redisConfig);
this.client.on('error', (e) => {
logger.err(`Error in Redis client: ${e instanceof Error ? e.message : e}`);
});
this.$ensureConnected();
}
}
private async $ensureConnected(): Promise<void> {
if (!this.connected && config.REDIS.ENABLED) {
return this.client.connect().then(async () => {
this.connected = true;
logger.info(`Redis client connected`);
const version = await this.client.get('schema_version');
if (version !== this.schemaVersion) {
// schema changed
// perform migrations or flush DB if necessary
logger.info(`Redis schema version changed from ${version} to ${this.schemaVersion}`);
await this.client.set('schema_version', this.schemaVersion);
}
});
}
}
async $updateBlocks(blocks: BlockExtended[]) {
try {
await this.$ensureConnected();
await this.client.set('blocks', JSON.stringify(blocks));
logger.debug(`Saved latest blocks to Redis cache`);
} catch (e) {
logger.warn(`Failed to update blocks in Redis cache: ${e instanceof Error ? e.message : e}`);
}
}
async $updateBlockSummaries(summaries: BlockSummary[]) {
try {
await this.$ensureConnected();
await this.client.set('block-summaries', JSON.stringify(summaries));
logger.debug(`Saved latest block summaries to Redis cache`);
} catch (e) {
logger.warn(`Failed to update block summaries in Redis cache: ${e instanceof Error ? e.message : e}`);
}
}
async $addTransaction(tx: MempoolTransactionExtended) {
this.cacheQueue.push(tx);
if (this.cacheQueue.length >= this.txFlushLimit) {
await this.$flushTransactions();
}
}
async $flushTransactions() {
const success = await this.$addTransactions(this.cacheQueue);
if (success) {
logger.debug(`Saved ${this.cacheQueue.length} transactions to Redis cache`);
this.cacheQueue = [];
} else {
logger.err(`Failed to save ${this.cacheQueue.length} transactions to Redis cache`);
}
}
private async $addTransactions(newTransactions: MempoolTransactionExtended[]): Promise<boolean> {
if (!newTransactions.length) {
return true;
}
try {
await this.$ensureConnected();
const msetData = newTransactions.map(tx => {
const minified: any = { ...tx };
delete minified.hex;
for (const vin of minified.vin) {
delete vin.inner_redeemscript_asm;
delete vin.inner_witnessscript_asm;
delete vin.scriptsig_asm;
}
for (const vout of minified.vout) {
delete vout.scriptpubkey_asm;
}
return [`mempool:tx:${tx.txid}`, JSON.stringify(minified)];
});
await this.client.MSET(msetData);
return true;
} catch (e) {
logger.warn(`Failed to add ${newTransactions.length} transactions to Redis cache: ${e instanceof Error ? e.message : e}`);
return false;
}
}
async $removeTransactions(transactions: string[]) {
try {
await this.$ensureConnected();
for (let i = 0; i < Math.ceil(transactions.length / 10000); i++) {
const slice = transactions.slice(i * 10000, (i + 1) * 10000);
await this.client.unlink(slice.map(txid => `mempool:tx:${txid}`));
logger.debug(`Deleted ${slice.length} transactions from the Redis cache`);
}
} catch (e) {
logger.warn(`Failed to remove ${transactions.length} transactions from Redis cache: ${e instanceof Error ? e.message : e}`);
}
}
async $setRbfEntry(type: string, txid: string, value: any): Promise<void> {
try {
await this.$ensureConnected();
await this.client.set(`rbf:${type}:${txid}`, JSON.stringify(value));
} catch (e) {
logger.warn(`Failed to set RBF ${type} in Redis cache: ${e instanceof Error ? e.message : e}`);
}
}
async $removeRbfEntry(type: string, txid: string): Promise<void> {
try {
await this.$ensureConnected();
await this.client.unlink(`rbf:${type}:${txid}`);
} catch (e) {
logger.warn(`Failed to remove RBF ${type} from Redis cache: ${e instanceof Error ? e.message : e}`);
}
}
async $getBlocks(): Promise<BlockExtended[]> {
try {
await this.$ensureConnected();
const json = await this.client.get('blocks');
return JSON.parse(json);
} catch (e) {
logger.warn(`Failed to retrieve blocks from Redis cache: ${e instanceof Error ? e.message : e}`);
return [];
}
}
async $getBlockSummaries(): Promise<BlockSummary[]> {
try {
await this.$ensureConnected();
const json = await this.client.get('block-summaries');
return JSON.parse(json);
} catch (e) {
logger.warn(`Failed to retrieve blocks from Redis cache: ${e instanceof Error ? e.message : e}`);
return [];
}
}
async $getMempool(): Promise<{ [txid: string]: MempoolTransactionExtended }> {
const start = Date.now();
const mempool = {};
try {
await this.$ensureConnected();
const mempoolList = await this.scanKeys<MempoolTransactionExtended>('mempool:tx:*');
for (const tx of mempoolList) {
mempool[tx.key] = tx.value;
}
logger.info(`Loaded mempool from Redis cache in ${Date.now() - start} ms`);
return mempool || {};
} catch (e) {
logger.warn(`Failed to retrieve mempool from Redis cache: ${e instanceof Error ? e.message : e}`);
}
return {};
}
async $getRbfEntries(type: string): Promise<any[]> {
try {
await this.$ensureConnected();
const rbfEntries = await this.scanKeys<MempoolTransactionExtended[]>(`rbf:${type}:*`);
return rbfEntries;
} catch (e) {
logger.warn(`Failed to retrieve Rbf ${type}s from Redis cache: ${e instanceof Error ? e.message : e}`);
return [];
}
}
async $loadCache() {
logger.info('Restoring mempool and blocks data from Redis cache');
// Load block data
const loadedBlocks = await this.$getBlocks();
const loadedBlockSummaries = await this.$getBlockSummaries();
// Load mempool
const loadedMempool = await this.$getMempool();
this.inflateLoadedTxs(loadedMempool);
// Load rbf data
const rbfTxs = await this.$getRbfEntries('tx');
const rbfTrees = await this.$getRbfEntries('tree');
const rbfExpirations = await this.$getRbfEntries('exp');
// Set loaded data
blocks.setBlocks(loadedBlocks || []);
blocks.setBlockSummaries(loadedBlockSummaries || []);
await memPool.$setMempool(loadedMempool);
await rbfCache.load({
txs: rbfTxs,
trees: rbfTrees.map(loadedTree => loadedTree.value),
expiring: rbfExpirations,
});
}
private inflateLoadedTxs(mempool: { [txid: string]: MempoolTransactionExtended }) {
for (const tx of Object.values(mempool)) {
for (const vin of tx.vin) {
if (vin.scriptsig) {
vin.scriptsig_asm = transactionUtils.convertScriptSigAsm(vin.scriptsig);
transactionUtils.addInnerScriptsToVin(vin);
}
}
for (const vout of tx.vout) {
if (vout.scriptpubkey) {
vout.scriptpubkey_asm = transactionUtils.convertScriptSigAsm(vout.scriptpubkey);
}
}
}
}
private async scanKeys<T>(pattern): Promise<{ key: string, value: T }[]> {
logger.info(`loading Redis entries for ${pattern}`);
let keys: string[] = [];
const result: { key: string, value: T }[] = [];
const patternLength = pattern.length - 1;
let count = 0;
const processValues = async (keys): Promise<void> => {
const values = await this.client.MGET(keys);
for (let i = 0; i < values.length; i++) {
if (values[i]) {
result.push({ key: keys[i].slice(patternLength), value: JSON.parse(values[i]) });
count++;
}
}
logger.info(`loaded ${count} entries from Redis cache`);
};
for await (const key of this.client.scanIterator({
MATCH: pattern,
COUNT: 100
})) {
keys.push(key);
if (keys.length >= 10000) {
await processValues(keys);
keys = [];
}
}
if (keys.length) {
await processValues(keys);
}
return result;
}
}
export default new RedisCache();

30
backend/src/api/services/acceleration.ts Normal file
View File

@@ -0,0 +1,30 @@
import { query } from '../../utils/axios-query';
import config from '../../config';
import { BlockExtended, PoolTag } from '../../mempool.interfaces';
export interface Acceleration {
txid: string,
feeDelta: number,
pools: number[],
}
class AccelerationApi {
public async $fetchAccelerations(): Promise<Acceleration[]> {
if (config.MEMPOOL_SERVICES.ACCELERATIONS) {
const response = await query(`${config.MEMPOOL_SERVICES.API}/accelerator/accelerations`);
return (response as Acceleration[]) || [];
} else {
return [];
}
}
public isAcceleratedBlock(block: BlockExtended, accelerations: Acceleration[]): boolean {
let anyAccelerated = false;
for (let i = 0; i < accelerations.length && !anyAccelerated; i++) {
anyAccelerated = anyAccelerated || accelerations[i].pools?.includes(block.extras.pool.id);
}
return anyAccelerated;
}
}
export default new AccelerationApi();

28
backend/src/api/statistics/statistics-api.ts
View File

@@ -15,6 +15,7 @@ class StatisticsApi {
mempool_byte_weight,
fee_data,
total_fee,
min_fee,
vsize_1,
vsize_2,
vsize_3,
@@ -54,7 +55,7 @@ class StatisticsApi {
vsize_1800,
vsize_2000
)
VALUES (NOW(), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
VALUES (NOW(), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)`;
const [result]: any = await DB.query(query);
return result.insertId;
@@ -73,6 +74,7 @@ class StatisticsApi {
mempool_byte_weight,
fee_data,
total_fee,
min_fee,
vsize_1,
vsize_2,
vsize_3,
@@ -112,7 +114,7 @@ class StatisticsApi {
vsize_1800,
vsize_2000
)
VALUES (${statistics.added}, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
VALUES (${statistics.added}, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
const params: (string | number)[] = [
@@ -122,6 +124,7 @@ class StatisticsApi {
statistics.mempool_byte_weight,
statistics.fee_data,
statistics.total_fee,
statistics.min_fee,
statistics.vsize_1,
statistics.vsize_2,
statistics.vsize_3,
@@ -171,7 +174,9 @@ class StatisticsApi {
private getQueryForDaysAvg(div: number, interval: string) {
return `SELECT
UNIX_TIMESTAMP(added) as added,
CAST(avg(unconfirmed_transactions) as DOUBLE) as unconfirmed_transactions,
CAST(avg(vbytes_per_second) as DOUBLE) as vbytes_per_second,
CAST(avg(min_fee) as DOUBLE) as min_fee,
CAST(avg(vsize_1) as DOUBLE) as vsize_1,
CAST(avg(vsize_2) as DOUBLE) as vsize_2,
CAST(avg(vsize_3) as DOUBLE) as vsize_3,
@@ -211,7 +216,7 @@ class StatisticsApi {
CAST(avg(vsize_1800) as DOUBLE) as vsize_1800,
CAST(avg(vsize_2000) as DOUBLE) as vsize_2000 \
FROM statistics \
WHERE added BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW() \
${interval === 'all' ? '' : `WHERE added BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`} \
GROUP BY UNIX_TIMESTAMP(added) DIV ${div} \
ORDER BY statistics.added DESC;`;
}
@@ -219,7 +224,9 @@ class StatisticsApi {
private getQueryForDays(div: number, interval: string) {
return `SELECT
UNIX_TIMESTAMP(added) as added,
CAST(avg(unconfirmed_transactions) as DOUBLE) as unconfirmed_transactions,
CAST(avg(vbytes_per_second) as DOUBLE) as vbytes_per_second,
CAST(avg(min_fee) as DOUBLE) as min_fee,
vsize_1,
vsize_2,
vsize_3,
@@ -259,7 +266,7 @@ class StatisticsApi {
vsize_1800,
vsize_2000 \
FROM statistics \
WHERE added BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW() \
${interval === 'all' ? '' : `WHERE added BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`} \
GROUP BY UNIX_TIMESTAMP(added) DIV ${div} \
ORDER BY statistics.added DESC;`;
}
@@ -386,13 +393,26 @@ class StatisticsApi {
}
}
public async $listAll(): Promise<OptimizedStatistic[]> {
try {
const query = this.getQueryForDays(43200, 'all'); // 12h interval
const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
} catch (e) {
logger.err('$listAll() error' + (e instanceof Error ? e.message : e));
return [];
}
}
private mapStatisticToOptimizedStatistic(statistic: Statistic[]): OptimizedStatistic[] {
return statistic.map((s) => {
return {
added: s.added,
count: s.unconfirmed_transactions,
vbytes_per_second: s.vbytes_per_second,
mempool_byte_weight: s.mempool_byte_weight,
total_fee: s.total_fee,
min_fee: s.min_fee,
vsizes: [
s.vsize_1,
s.vsize_2,

12
backend/src/api/statistics/statistics.routes.ts
View File

@@ -15,10 +15,11 @@ class StatisticsRoutes {
.get(config.MEMPOOL.API_URL_PREFIX + 'statistics/2y', this.$getStatisticsByTime.bind(this, '2y'))
.get(config.MEMPOOL.API_URL_PREFIX + 'statistics/3y', this.$getStatisticsByTime.bind(this, '3y'))
.get(config.MEMPOOL.API_URL_PREFIX + 'statistics/4y', this.$getStatisticsByTime.bind(this, '4y'))
.get(config.MEMPOOL.API_URL_PREFIX + 'statistics/all', this.$getStatisticsByTime.bind(this, 'all'))
;
}
private async $getStatisticsByTime(time: '2h' | '24h' | '1w' | '1m' | '3m' | '6m' | '1y' | '2y' | '3y' | '4y', req: Request, res: Response) {
private async $getStatisticsByTime(time: '2h' | '24h' | '1w' | '1m' | '3m' | '6m' | '1y' | '2y' | '3y' | '4y' | 'all', req: Request, res: Response) {
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
@@ -26,10 +27,6 @@ class StatisticsRoutes {
try {
let result;
switch (time as string) {
case '2h':
result = await statisticsApi.$list2H();
res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
break;
case '24h':
result = await statisticsApi.$list24H();
res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
@@ -58,8 +55,13 @@ class StatisticsRoutes {
case '4y':
result = await statisticsApi.$list4Y();
break;
case 'all':
result = await statisticsApi.$listAll();
break;
default:
result = await statisticsApi.$list2H();
res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
break;
}
res.json(result);
} catch (e) {

4
backend/src/api/statistics/statistics.ts
View File

@@ -89,6 +89,9 @@ class Statistics {
}
});
// get minFee and convert to sats/vb
const minFee = memPool.getMempoolInfo().mempoolminfee * 100000;
try {
const insertId = await statisticsApi.$create({
added: 'NOW()',
@@ -98,6 +101,7 @@ class Statistics {
mempool_byte_weight: totalWeight,
total_fee: totalFee,
fee_data: '',
min_fee: minFee,
vsize_1: weightVsizeFees['1'] || 0,
vsize_2: weightVsizeFees['2'] || 0,
vsize_3: weightVsizeFees['3'] || 0,

192
backend/src/api/transaction-utils.ts
View File

@@ -3,6 +3,9 @@ import { IEsploraApi } from './bitcoin/esplora-api.interface';
import { Common } from './common';
import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory';
import * as bitcoinjs from 'bitcoinjs-lib';
import logger from '../logger';
import config from '../config';
import pLimit from '../utils/p-limit';
class TransactionUtils {
constructor() { }
@@ -22,6 +25,23 @@ class TransactionUtils {
};
}
// Wrapper for $getTransactionExtended with an automatic retry direct to Core if the first API request fails.
// Propagates any error from the retry request.
public async $getTransactionExtendedRetry(txid: string, addPrevouts = false, lazyPrevouts = false, forceCore = false, addMempoolData = false): Promise<TransactionExtended> {
try {
const result = await this.$getTransactionExtended(txid, addPrevouts, lazyPrevouts, forceCore, addMempoolData);
if (result) {
return result;
} else {
logger.err(`Cannot fetch tx ${txid}. Reason: backend returned null data`);
}
} catch (e) {
logger.err(`Cannot fetch tx ${txid}. Reason: ` + (e instanceof Error ? e.message : e));
}
// retry direct from Core if first request failed
return this.$getTransactionExtended(txid, addPrevouts, lazyPrevouts, true, addMempoolData);
}
/**
* @param txId
* @param addPrevouts
@@ -31,10 +51,17 @@ class TransactionUtils {
public async $getTransactionExtended(txId: string, addPrevouts = false, lazyPrevouts = false, forceCore = false, addMempoolData = false): Promise<TransactionExtended> {
let transaction: IEsploraApi.Transaction;
if (forceCore === true) {
transaction = await bitcoinCoreApi.$getRawTransaction(txId, true);
transaction = await bitcoinCoreApi.$getRawTransaction(txId, false, addPrevouts, lazyPrevouts);
} else {
transaction = await bitcoinApi.$getRawTransaction(txId, false, addPrevouts, lazyPrevouts);
}
if (Common.isLiquid()) {
if (!isFinite(Number(transaction.fee))) {
transaction.fee = Object.values(transaction.fee || {}).reduce((total, output) => total + output, 0);
}
}
if (addMempoolData || !transaction?.status?.confirmed) {
return this.extendMempoolTransaction(transaction);
} else {
@@ -46,14 +73,35 @@ class TransactionUtils {
return (await this.$getTransactionExtended(txId, addPrevouts, lazyPrevouts, forceCore, true)) as MempoolTransactionExtended;
}
private extendTransaction(transaction: IEsploraApi.Transaction): TransactionExtended {
public async $getMempoolTransactionsExtended(txids: string[], addPrevouts = false, lazyPrevouts = false, forceCore = false): Promise<MempoolTransactionExtended[]> {
if (forceCore || config.MEMPOOL.BACKEND !== 'esplora') {
const limiter = pLimit(8); // Run 8 requests at a time
const results = await Promise.allSettled(txids.map(
txid => limiter(() => this.$getMempoolTransactionExtended(txid, addPrevouts, lazyPrevouts, forceCore))
));
return results.filter(reply => reply.status === 'fulfilled')
.map(r => (r as PromiseFulfilledResult<MempoolTransactionExtended>).value);
} else {
const transactions = await bitcoinApi.$getMempoolTransactions(txids);
return transactions.map(transaction => {
if (Common.isLiquid()) {
if (!isFinite(Number(transaction.fee))) {
transaction.fee = Object.values(transaction.fee || {}).reduce((total, output) => total + output, 0);
}
}
return this.extendMempoolTransaction(transaction);
});
}
}
public extendTransaction(transaction: IEsploraApi.Transaction): TransactionExtended {
// @ts-ignore
if (transaction.vsize) {
// @ts-ignore
return transaction;
}
const feePerVbytes = Math.max(Common.isLiquid() ? 0.1 : 1,
(transaction.fee || 0) / (transaction.weight / 4));
const feePerVbytes = (transaction.fee || 0) / (transaction.weight / 4);
const transactionExtended: TransactionExtended = Object.assign({
vsize: Math.round(transaction.weight / 4),
feePerVsize: feePerVbytes,
@@ -68,14 +116,13 @@ class TransactionUtils {
public extendMempoolTransaction(transaction: IEsploraApi.Transaction): MempoolTransactionExtended {
const vsize = Math.ceil(transaction.weight / 4);
const fractionalVsize = (transaction.weight / 4);
const sigops = this.countSigops(transaction);
const sigops = !Common.isLiquid() ? this.countSigops(transaction) : 0;
// https://github.com/bitcoin/bitcoin/blob/e9262ea32a6e1d364fb7974844fadc36f931f8c6/src/policy/policy.cpp#L295-L298
const adjustedVsize = Math.max(fractionalVsize, sigops * 5); // adjusted vsize = Max(weight, sigops * bytes_per_sigop) / witness_scale_factor
const feePerVbytes = Math.max(Common.isLiquid() ? 0.1 : 1,
(transaction.fee || 0) / fractionalVsize);
const adjustedFeePerVsize = Math.max(Common.isLiquid() ? 0.1 : 1,
(transaction.fee || 0) / adjustedVsize);
const feePerVbytes = (transaction.fee || 0) / fractionalVsize;
const adjustedFeePerVsize = (transaction.fee || 0) / adjustedVsize;
const transactionExtended: MempoolTransactionExtended = Object.assign(transaction, {
order: this.txidToOrdering(transaction.txid),
vsize: Math.round(transaction.weight / 4),
adjustedVsize,
sigops,
@@ -154,6 +201,133 @@ class TransactionUtils {
return sigops;
}
// returns the most significant 4 bytes of the txid as an integer
public txidToOrdering(txid: string): number {
return parseInt(
txid.substr(62, 2) +
txid.substr(60, 2) +
txid.substr(58, 2) +
txid.substr(56, 2),
16
);
}
public addInnerScriptsToVin(vin: IEsploraApi.Vin): void {
if (!vin.prevout) {
return;
}
if (vin.prevout.scriptpubkey_type === 'p2sh') {
const redeemScript = vin.scriptsig_asm.split(' ').reverse()[0];
vin.inner_redeemscript_asm = this.convertScriptSigAsm(redeemScript);
if (vin.witness && vin.witness.length > 2) {
const witnessScript = vin.witness[vin.witness.length - 1];
vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript);
}
}
if (vin.prevout.scriptpubkey_type === 'v0_p2wsh' && vin.witness) {
const witnessScript = vin.witness[vin.witness.length - 1];
vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript);
}
if (vin.prevout.scriptpubkey_type === 'v1_p2tr' && vin.witness) {
const witnessScript = this.witnessToP2TRScript(vin.witness);
if (witnessScript !== null) {
vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript);
}
}
}
public convertScriptSigAsm(hex: string): string {
const buf = Buffer.from(hex, 'hex');
const b: string[] = [];
let i = 0;
while (i < buf.length) {
const op = buf[i];
if (op >= 0x01 && op <= 0x4e) {
i++;
let push: number;
if (op === 0x4c) {
push = buf.readUInt8(i);
b.push('OP_PUSHDATA1');
i += 1;
} else if (op === 0x4d) {
push = buf.readUInt16LE(i);
b.push('OP_PUSHDATA2');
i += 2;
} else if (op === 0x4e) {
push = buf.readUInt32LE(i);
b.push('OP_PUSHDATA4');
i += 4;
} else {
push = op;
b.push('OP_PUSHBYTES_' + push);
}
const data = buf.slice(i, i + push);
if (data.length !== push) {
break;
}
b.push(data.toString('hex'));
i += data.length;
} else {
if (op === 0x00) {
b.push('OP_0');
} else if (op === 0x4f) {
b.push('OP_PUSHNUM_NEG1');
} else if (op === 0xb1) {
b.push('OP_CLTV');
} else if (op === 0xb2) {
b.push('OP_CSV');
} else if (op === 0xba) {
b.push('OP_CHECKSIGADD');
} else {
const opcode = bitcoinjs.script.toASM([ op ]);
if (opcode && op < 0xfd) {
if (/^OP_(\d+)$/.test(opcode)) {
b.push(opcode.replace(/^OP_(\d+)$/, 'OP_PUSHNUM_$1'));
} else {
b.push(opcode);
}
} else {
b.push('OP_RETURN_' + op);
}
}
i += 1;
}
}
return b.join(' ');
}
/**
* This function must only be called when we know the witness we are parsing
* is a taproot witness.
* @param witness An array of hex strings that represents the witness stack of
* the input.
* @returns null if the witness is not a script spend, and the hex string of
* the script item if it is a script spend.
*/
public witnessToP2TRScript(witness: string[]): string | null {
if (witness.length < 2) return null;
// Note: see BIP341 for parsing details of witness stack
// If there are at least two witness elements, and the first byte of the
// last element is 0x50, this last element is called annex a and
// is removed from the witness stack.
const hasAnnex = witness[witness.length - 1].substring(0, 2) === '50';
// If there are at least two witness elements left, script path spending is used.
// Call the second-to-last stack element s, the script.
// (Note: this phrasing from BIP341 assumes we've *removed* the annex from the stack)
if (hasAnnex && witness.length < 3) return null;
const positionOfScript = hasAnnex ? witness.length - 3 : witness.length - 2;
return witness[positionOfScript];
}
}
export default new TransactionUtils();

442
backend/src/api/websocket-handler.ts
View File

@@ -12,7 +12,7 @@ import { Common } from './common';
import loadingIndicators from './loading-indicators';
import config from '../config';
import transactionUtils from './transaction-utils';
import rbfCache from './rbf-cache';
import rbfCache, { ReplacementInfo } from './rbf-cache';
import difficultyAdjustment from './difficulty-adjustment';
import feeApi from './fee-api';
import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository';
@@ -21,6 +21,16 @@ import Audit from './audit';
import { deepClone } from '../utils/clone';
import priceUpdater from '../tasks/price-updater';
import { ApiPrice } from '../repositories/PricesRepository';
import accelerationApi from './services/acceleration';
import mempool from './mempool';
// valid 'want' subscriptions
const wantable = [
'blocks',
'mempool-blocks',
'live-2h-chart',
'stats',
];
class WebsocketHandler {
private wss: WebSocket.Server | undefined;
@@ -30,8 +40,9 @@ class WebsocketHandler {
private numConnected = 0;
private numDisconnected = 0;
private initData: { [key: string]: string } = {};
private socketData: { [key: string]: string } = {};
private serializedInitData: string = '{}';
private lastRbfSummary: ReplacementInfo | null = null;
constructor() { }
@@ -39,28 +50,28 @@ class WebsocketHandler {
this.wss = wss;
}
setExtraInitProperties(property: string, value: any) {
setExtraInitData(property: string, value: any) {
this.extraInitProperties[property] = value;
this.setInitDataFields(this.extraInitProperties);
this.updateSocketDataFields(this.extraInitProperties);
}
private setInitDataFields(data: { [property: string]: any }): void {
private updateSocketDataFields(data: { [property: string]: any }): void {
for (const property of Object.keys(data)) {
if (data[property] != null) {
this.initData[property] = JSON.stringify(data[property]);
this.socketData[property] = JSON.stringify(data[property]);
} else {
delete this.initData[property];
delete this.socketData[property];
}
}
this.serializedInitData = '{'
+ Object.keys(this.initData).map(key => `"${key}": ${this.initData[key]}`).join(', ')
+ '}';
+ Object.keys(this.socketData).map(key => `"${key}": ${this.socketData[key]}`).join(', ')
+ '}';
}
private updateInitData(): void {
private updateSocketData(): void {
const _blocks = blocks.getBlocks().slice(-config.MEMPOOL.INITIAL_BLOCKS_AMOUNT);
const da = difficultyAdjustment.getDifficultyAdjustment();
this.setInitDataFields({
this.updateSocketDataFields({
'mempoolInfo': memPool.getMempoolInfo(),
'vBytesPerSecond': memPool.getVBytesPerSecond(),
'blocks': _blocks,
@@ -94,11 +105,33 @@ class WebsocketHandler {
const parsedMessage: WebsocketResponse = JSON.parse(message);
const response = {};
if (parsedMessage.action === 'want') {
client['want-blocks'] = parsedMessage.data.indexOf('blocks') > -1;
client['want-mempool-blocks'] = parsedMessage.data.indexOf('mempool-blocks') > -1;
client['want-live-2h-chart'] = parsedMessage.data.indexOf('live-2h-chart') > -1;
client['want-stats'] = parsedMessage.data.indexOf('stats') > -1;
const wantNow = {};
if (parsedMessage && parsedMessage.action === 'want' && Array.isArray(parsedMessage.data)) {
for (const sub of wantable) {
const key = `want-${sub}`;
const wants = parsedMessage.data.includes(sub);
if (wants && client['wants'] && !client[key]) {
wantNow[key] = true;
}
client[key] = wants;
}
client['wants'] = true;
}
// send initial data when a client first starts a subscription
if (wantNow['want-blocks'] || (parsedMessage && parsedMessage['refresh-blocks'])) {
response['blocks'] = this.socketData['blocks'];
}
if (wantNow['want-mempool-blocks']) {
response['mempool-blocks'] = this.socketData['mempool-blocks'];
}
if (wantNow['want-stats']) {
response['mempoolInfo'] = this.socketData['mempoolInfo'];
response['vBytesPerSecond'] = this.socketData['vBytesPerSecond'];
response['fees'] = this.socketData['fees'];
response['da'] = this.socketData['da'];
}
if (parsedMessage && parsedMessage['track-tx']) {
@@ -109,21 +142,21 @@ class WebsocketHandler {
if (parsedMessage['watch-mempool']) {
const rbfCacheTxid = rbfCache.getReplacedBy(trackTxid);
if (rbfCacheTxid) {
response['txReplaced'] = {
response['txReplaced'] = JSON.stringify({
txid: rbfCacheTxid,
};
});
client['track-tx'] = null;
} else {
// It might have appeared before we had the time to start watching for it
const tx = memPool.getMempool()[trackTxid];
if (tx) {
if (config.MEMPOOL.BACKEND === 'esplora') {
response['tx'] = tx;
response['tx'] = JSON.stringify(tx);
} else {
// tx.prevout is missing from transactions when in bitcoind mode
try {
const fullTx = await transactionUtils.$getMempoolTransactionExtended(tx.txid, true);
response['tx'] = fullTx;
response['tx'] = JSON.stringify(fullTx);
} catch (e) {
logger.debug('Error finding transaction: ' + (e instanceof Error ? e.message : e));
}
@@ -131,7 +164,7 @@ class WebsocketHandler {
} else {
try {
const fullTx = await transactionUtils.$getMempoolTransactionExtended(client['track-tx'], true);
response['tx'] = fullTx;
response['tx'] = JSON.stringify(fullTx);
} catch (e) {
logger.debug('Error finding transaction. ' + (e instanceof Error ? e.message : e));
client['track-mempool-tx'] = parsedMessage['track-tx'];
@@ -141,10 +174,16 @@ class WebsocketHandler {
}
const tx = memPool.getMempool()[trackTxid];
if (tx && tx.position) {
response['txPosition'] = {
txid: trackTxid,
position: tx.position,
const position: { block: number, vsize: number, accelerated?: boolean } = {
...tx.position
};
if (tx.acceleration) {
position.accelerated = tx.acceleration;
}
response['txPosition'] = JSON.stringify({
txid: trackTxid,
position
});
}
} else {
client['track-tx'] = null;
@@ -152,13 +191,19 @@ class WebsocketHandler {
}
if (parsedMessage && parsedMessage['track-address']) {
if (/^([a-km-zA-HJ-NP-Z1-9]{26,35}|[a-km-zA-HJ-NP-Z1-9]{80}|[a-z]{2,5}1[ac-hj-np-z02-9]{8,100}|[A-Z]{2,5}1[AC-HJ-NP-Z02-9]{8,100})$/
if (/^([a-km-zA-HJ-NP-Z1-9]{26,35}|[a-km-zA-HJ-NP-Z1-9]{80}|[a-z]{2,5}1[ac-hj-np-z02-9]{8,100}|[A-Z]{2,5}1[AC-HJ-NP-Z02-9]{8,100}|04[a-fA-F0-9]{128}|(02|03)[a-fA-F0-9]{64})$/
.test(parsedMessage['track-address'])) {
let matchedAddress = parsedMessage['track-address'];
if (/^[A-Z]{2,5}1[AC-HJ-NP-Z02-9]{8,100}$/.test(parsedMessage['track-address'])) {
matchedAddress = matchedAddress.toLowerCase();
}
client['track-address'] = matchedAddress;
if (/^04[a-fA-F0-9]{128}$/.test(parsedMessage['track-address'])) {
client['track-address'] = '41' + matchedAddress + 'ac';
} else if (/^(02|03)[a-fA-F0-9]{64}$/.test(parsedMessage['track-address'])) {
client['track-address'] = '21' + matchedAddress + 'ac';
} else {
client['track-address'] = matchedAddress;
}
} else {
client['track-address'] = null;
}
@@ -177,10 +222,10 @@ class WebsocketHandler {
const index = parsedMessage['track-mempool-block'];
client['track-mempool-block'] = index;
const mBlocksWithTransactions = mempoolBlocks.getMempoolBlocksWithTransactions();
response['projected-block-transactions'] = {
response['projected-block-transactions'] = JSON.stringify({
index: index,
blockTransactions: mBlocksWithTransactions[index]?.transactions || [],
};
});
} else {
client['track-mempool-block'] = null;
}
@@ -189,23 +234,35 @@ class WebsocketHandler {
if (parsedMessage && parsedMessage['track-rbf'] !== undefined) {
if (['all', 'fullRbf'].includes(parsedMessage['track-rbf'])) {
client['track-rbf'] = parsedMessage['track-rbf'];
response['rbfLatest'] = JSON.stringify(rbfCache.getRbfTrees(parsedMessage['track-rbf'] === 'fullRbf'));
} else {
client['track-rbf'] = false;
}
}
if (parsedMessage.action === 'init') {
if (!this.initData['blocks']?.length || !this.initData['da']) {
this.updateInitData();
if (parsedMessage && parsedMessage['track-rbf-summary'] != null) {
if (parsedMessage['track-rbf-summary']) {
client['track-rbf-summary'] = true;
if (this.socketData['rbfSummary'] != null) {
response['rbfLatestSummary'] = this.socketData['rbfSummary'];
}
} else {
client['track-rbf-summary'] = false;
}
if (!this.initData['blocks']?.length) {
}
if (parsedMessage.action === 'init') {
if (!this.socketData['blocks']?.length || !this.socketData['da'] || !this.socketData['backendInfo'] || !this.socketData['conversions']) {
this.updateSocketData();
}
if (!this.socketData['blocks']?.length) {
return;
}
client.send(this.serializedInitData);
}
if (parsedMessage.action === 'ping') {
response['pong'] = true;
response['pong'] = JSON.stringify(true);
}
if (parsedMessage['track-donation'] && parsedMessage['track-donation'].length === 22) {
@@ -221,7 +278,8 @@ class WebsocketHandler {
}
if (Object.keys(response).length) {
client.send(JSON.stringify(response));
const serializedResponse = this.serializeResponse(response);
client.send(serializedResponse);
}
} catch (e) {
logger.debug('Error parsing websocket message: ' + (e instanceof Error ? e.message : e));
@@ -250,7 +308,7 @@ class WebsocketHandler {
throw new Error('WebSocket.Server is not set');
}
this.setInitDataFields({ 'loadingIndicators': indicators });
this.updateSocketDataFields({ 'loadingIndicators': indicators });
const response = JSON.stringify({ loadingIndicators: indicators });
this.wss.clients.forEach((client) => {
@@ -266,7 +324,7 @@ class WebsocketHandler {
throw new Error('WebSocket.Server is not set');
}
this.setInitDataFields({ 'conversions': conversionRates });
this.updateSocketDataFields({ 'conversions': conversionRates });
const response = JSON.stringify({ conversions: conversionRates });
this.wss.clients.forEach((client) => {
@@ -301,8 +359,42 @@ class WebsocketHandler {
});
}
async $handleMempoolChange(newMempool: { [txid: string]: MempoolTransactionExtended },
newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[]): Promise<void> {
handleReorg(): void {
if (!this.wss) {
throw new Error('WebSocket.Server is not set');
}
const da = difficultyAdjustment.getDifficultyAdjustment();
// update init data
this.updateSocketDataFields({
'blocks': blocks.getBlocks(),
'da': da?.previousTime ? da : undefined,
});
this.wss.clients.forEach((client) => {
if (client.readyState !== WebSocket.OPEN) {
return;
}
const response = {};
if (client['want-blocks']) {
response['blocks'] = this.socketData['blocks'];
}
if (client['want-stats']) {
response['da'] = this.socketData['da'];
}
if (Object.keys(response).length) {
const serializedResponse = this.serializeResponse(response);
client.send(serializedResponse);
}
});
}
async $handleMempoolChange(newMempool: { [txid: string]: MempoolTransactionExtended }, mempoolSize: number,
newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]): Promise<void> {
if (!this.wss) {
throw new Error('WebSocket.Server is not set');
}
@@ -310,7 +402,11 @@ class WebsocketHandler {
this.printLogs();
if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) {
await mempoolBlocks.$updateBlockTemplates(newMempool, newTransactions, deletedTransactions, true);
if (config.MEMPOOL.RUST_GBT) {
await mempoolBlocks.$rustUpdateBlockTemplates(newMempool, mempoolSize, newTransactions, deletedTransactions, config.MEMPOOL_SERVICES.ACCELERATIONS);
} else {
await mempoolBlocks.$updateBlockTemplates(newMempool, newTransactions, deletedTransactions, accelerationDelta, true, config.MEMPOOL_SERVICES.ACCELERATIONS);
}
} else {
mempoolBlocks.updateMempoolBlocks(newMempool, true);
}
@@ -325,10 +421,13 @@ class WebsocketHandler {
const rbfChanges = rbfCache.getRbfChanges();
let rbfReplacements;
let fullRbfReplacements;
let rbfSummary;
if (Object.keys(rbfChanges.trees).length) {
rbfReplacements = rbfCache.getRbfTrees(false);
fullRbfReplacements = rbfCache.getRbfTrees(true);
rbfSummary = rbfCache.getLatestRbfSummary();
}
for (const deletedTx of deletedTransactions) {
rbfCache.evict(deletedTx.txid);
}
@@ -336,11 +435,25 @@ class WebsocketHandler {
memPool.addToSpendMap(newTransactions);
const recommendedFees = feeApi.getRecommendedFee();
const latestTransactions = memPool.getLatestTransactions();
// update init data
this.updateInitData();
const socketDataFields = {
'mempoolInfo': mempoolInfo,
'vBytesPerSecond': vBytesPerSecond,
'mempool-blocks': mBlocks,
'transactions': latestTransactions,
'loadingIndicators': loadingIndicators.getLoadingIndicators(),
'da': da?.previousTime ? da : undefined,
'fees': recommendedFees,
};
if (rbfSummary) {
socketDataFields['rbfSummary'] = rbfSummary;
}
this.updateSocketDataFields(socketDataFields);
// cache serialized objects to avoid stringify-ing the same thing for every client
const responseCache = { ...this.initData };
const responseCache = { ...this.socketData };
function getCachedResponse(key: string, data): string {
if (!responseCache[key]) {
responseCache[key] = JSON.stringify(data);
@@ -371,7 +484,9 @@ class WebsocketHandler {
}
}
const latestTransactions = newTransactions.slice(0, 6).map((tx) => Common.stripTransaction(tx));
// pre-compute address transactions
const addressCache = this.makeAddressCache(newTransactions);
const removedAddressCache = this.makeAddressCache(deletedTransactions);
this.wss.clients.forEach(async (client) => {
if (client.readyState !== WebSocket.OPEN) {
@@ -412,40 +527,17 @@ class WebsocketHandler {
}
if (client['track-address']) {
const foundTransactions: TransactionExtended[] = [];
const newTransactions = Array.from(addressCache[client['track-address']]?.values() || []);
const removedTransactions = Array.from(removedAddressCache[client['track-address']]?.values() || []);
// txs may be missing prevouts in non-esplora backends
// so fetch the full transactions now
const fullTransactions = (config.MEMPOOL.BACKEND !== 'esplora') ? await this.getFullTransactions(newTransactions) : newTransactions;
for (const tx of newTransactions) {
const someVin = tx.vin.some((vin) => !!vin.prevout && vin.prevout.scriptpubkey_address === client['track-address']);
if (someVin) {
if (config.MEMPOOL.BACKEND !== 'esplora') {
try {
const fullTx = await transactionUtils.$getMempoolTransactionExtended(tx.txid, true);
foundTransactions.push(fullTx);
} catch (e) {
logger.debug('Error finding transaction in mempool: ' + (e instanceof Error ? e.message : e));
}
} else {
foundTransactions.push(tx);
}
return;
}
const someVout = tx.vout.some((vout) => vout.scriptpubkey_address === client['track-address']);
if (someVout) {
if (config.MEMPOOL.BACKEND !== 'esplora') {
try {
const fullTx = await transactionUtils.$getMempoolTransactionExtended(tx.txid, true);
foundTransactions.push(fullTx);
} catch (e) {
logger.debug('Error finding transaction in mempool: ' + (e instanceof Error ? e.message : e));
}
} else {
foundTransactions.push(tx);
}
}
if (removedTransactions.length) {
response['address-removed-transactions'] = JSON.stringify(removedTransactions);
}
if (foundTransactions.length) {
response['address-transactions'] = JSON.stringify(foundTransactions);
if (fullTransactions.length) {
response['address-transactions'] = JSON.stringify(fullTransactions);
}
}
@@ -453,7 +545,6 @@ class WebsocketHandler {
const foundTransactions: TransactionExtended[] = [];
newTransactions.forEach((tx) => {
if (client['track-asset'] === Common.nativeAssetId) {
if (tx.vin.some((vin) => !!vin.is_pegin)) {
foundTransactions.push(tx);
@@ -486,11 +577,11 @@ class WebsocketHandler {
response['utxoSpent'] = JSON.stringify(outspends);
}
const rbfReplacedBy = rbfCache.getReplacedBy(client['track-tx']);
const rbfReplacedBy = rbfChanges.map[client['track-tx']] ? rbfCache.getReplacedBy(client['track-tx']) : false;
if (rbfReplacedBy) {
response['rbfTransaction'] = JSON.stringify({
txid: rbfReplacedBy,
})
});
}
const rbfChange = rbfChanges.map[client['track-tx']];
@@ -500,14 +591,29 @@ class WebsocketHandler {
const mempoolTx = newMempool[trackTxid];
if (mempoolTx && mempoolTx.position) {
response['txPosition'] = JSON.stringify({
const positionData = {
txid: trackTxid,
position: mempoolTx.position,
});
position: {
...mempoolTx.position,
accelerated: mempoolTx.acceleration || undefined,
}
};
if (mempoolTx.cpfpDirty) {
positionData['cpfp'] = {
ancestors: mempoolTx.ancestors,
bestDescendant: mempoolTx.bestDescendant || null,
descendants: mempoolTx.descendants || null,
effectiveFeePerVsize: mempoolTx.effectiveFeePerVsize || null,
sigops: mempoolTx.sigops,
adjustedVsize: mempoolTx.adjustedVsize,
acceleration: mempoolTx.acceleration
};
}
response['txPosition'] = JSON.stringify(positionData);
}
}
if (client['track-mempool-block'] >= 0) {
if (client['track-mempool-block'] >= 0 && memPool.isInSync()) {
const index = client['track-mempool-block'];
if (mBlockDeltas[index]) {
response['projected-block-transactions'] = getCachedResponse(`projected-block-transactions-${index}`, {
@@ -523,16 +629,18 @@ class WebsocketHandler {
response['rbfLatest'] = getCachedResponse('fullrbfLatest', fullRbfReplacements);
}
if (client['track-rbf-summary'] && rbfSummary) {
response['rbfLatestSummary'] = getCachedResponse('rbfLatestSummary', rbfSummary);
}
if (Object.keys(response).length) {
const serializedResponse = '{'
+ Object.keys(response).map(key => `"${key}": ${response[key]}`).join(', ')
+ '}';
const serializedResponse = this.serializeResponse(response);
client.send(serializedResponse);
}
});
}
async handleNewBlock(block: BlockExtended, txIds: string[], transactions: TransactionExtended[]): Promise<void> {
async handleNewBlock(block: BlockExtended, txIds: string[], transactions: MempoolTransactionExtended[]): Promise<void> {
if (!this.wss) {
throw new Error('WebSocket.Server is not set');
}
@@ -541,35 +649,45 @@ class WebsocketHandler {
const _memPool = memPool.getMempool();
if (config.MEMPOOL.AUDIT) {
const rbfTransactions = Common.findMinedRbfTransactions(transactions, memPool.getSpendMap());
memPool.handleMinedRbfTransactions(rbfTransactions);
memPool.removeFromSpendMap(transactions);
if (config.MEMPOOL.AUDIT && memPool.isInSync()) {
let projectedBlocks;
let auditMempool = _memPool;
const isAccelerated = config.MEMPOOL_SERVICES.ACCELERATIONS && accelerationApi.isAcceleratedBlock(block, Object.values(mempool.getAccelerations()));
// template calculation functions have mempool side effects, so calculate audits using
// a cloned copy of the mempool if we're running a different algorithm for mempool updates
const separateAudit = config.MEMPOOL.ADVANCED_GBT_AUDIT !== config.MEMPOOL.ADVANCED_GBT_MEMPOOL;
if (separateAudit) {
auditMempool = deepClone(_memPool);
if (config.MEMPOOL.ADVANCED_GBT_AUDIT) {
projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false);
if (config.MEMPOOL.RUST_GBT) {
projectedBlocks = await mempoolBlocks.$oneOffRustBlockTemplates(auditMempool, isAccelerated, block.extras.pool.id);
} else {
projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, isAccelerated, block.extras.pool.id);
}
} else {
projectedBlocks = mempoolBlocks.updateMempoolBlocks(auditMempool, false);
}
} else {
projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
if ((config.MEMPOOL_SERVICES.ACCELERATIONS)) {
if (config.MEMPOOL.RUST_GBT) {
projectedBlocks = await mempoolBlocks.$rustUpdateBlockTemplates(auditMempool, Object.keys(auditMempool).length, [], [], isAccelerated, block.extras.pool.id);
} else {
projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, isAccelerated, block.extras.pool.id);
}
} else {
projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
}
}
if (Common.indexingEnabled() && memPool.isInSync()) {
const { censored, added, fresh, sigop, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool);
if (Common.indexingEnabled()) {
const { censored, added, fresh, sigop, fullrbf, accelerated, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool);
const matchRate = Math.round(score * 100 * 100) / 100;
const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions.map((tx) => {
return {
txid: tx.txid,
vsize: tx.vsize,
fee: tx.fee ? Math.round(tx.fee) : 0,
value: tx.value,
};
}) : [];
const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions : [];
let totalFees = 0;
let totalWeight = 0;
@@ -594,6 +712,8 @@ class WebsocketHandler {
missingTxs: censored,
freshTxs: fresh,
sigopTxs: sigop,
fullrbfTxs: fullrbf,
acceleratedTxs: accelerated,
matchRate: matchRate,
expectedFees: totalFees,
expectedWeight: totalWeight,
@@ -613,10 +733,6 @@ class WebsocketHandler {
}
}
const rbfTransactions = Common.findMinedRbfTransactions(transactions, memPool.getSpendMap());
memPool.handleMinedRbfTransactions(rbfTransactions);
memPool.removeFromSpendMap(transactions);
// Update mempool to remove transactions included in the new block
for (const txId of txIds) {
delete _memPool[txId];
@@ -624,7 +740,11 @@ class WebsocketHandler {
}
if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) {
await mempoolBlocks.$makeBlockTemplates(_memPool, true);
if (config.MEMPOOL.RUST_GBT) {
await mempoolBlocks.$rustUpdateBlockTemplates(_memPool, Object.keys(_memPool).length, [], transactions, true);
} else {
await mempoolBlocks.$makeBlockTemplates(_memPool, true, config.MEMPOOL_SERVICES.ACCELERATIONS);
}
} else {
mempoolBlocks.updateMempoolBlocks(_memPool, true);
}
@@ -633,11 +753,22 @@ class WebsocketHandler {
const da = difficultyAdjustment.getDifficultyAdjustment();
const fees = feeApi.getRecommendedFee();
const mempoolInfo = memPool.getMempoolInfo();
// pre-compute address transactions
const addressCache = this.makeAddressCache(transactions);
// update init data
this.updateInitData();
this.updateSocketDataFields({
'mempoolInfo': mempoolInfo,
'blocks': [...blocks.getBlocks(), block].slice(-config.MEMPOOL.INITIAL_BLOCKS_AMOUNT),
'mempool-blocks': mBlocks,
'loadingIndicators': loadingIndicators.getLoadingIndicators(),
'da': da?.previousTime ? da : undefined,
'fees': fees,
});
const responseCache = { ...this.initData };
const responseCache = { ...this.socketData };
function getCachedResponse(key, data): string {
if (!responseCache[key]) {
responseCache[key] = JSON.stringify(data);
@@ -645,22 +776,26 @@ class WebsocketHandler {
return responseCache[key];
}
const mempoolInfo = memPool.getMempoolInfo();
this.wss.clients.forEach((client) => {
if (client.readyState !== WebSocket.OPEN) {
return;
}
if (!client['want-blocks']) {
return;
const response = {};
if (client['want-blocks']) {
response['block'] = getCachedResponse('block', block);
}
const response = {};
response['block'] = getCachedResponse('block', block);
response['mempoolInfo'] = getCachedResponse('mempoolInfo', mempoolInfo);
response['da'] = getCachedResponse('da', da?.previousTime ? da : undefined);
response['fees'] = getCachedResponse('fees', fees);
if (client['want-stats']) {
response['mempoolInfo'] = getCachedResponse('mempoolInfo', mempoolInfo);
response['vBytesPerSecond'] = getCachedResponse('vBytesPerSecond', memPool.getVBytesPerSecond());
response['fees'] = getCachedResponse('fees', fees);
if (da?.previousTime) {
response['da'] = getCachedResponse('da', da);
}
}
if (mBlocks && client['want-mempool-blocks']) {
response['mempool-blocks'] = getCachedResponse('mempool-blocks', mBlocks);
@@ -675,24 +810,17 @@ class WebsocketHandler {
if (mempoolTx && mempoolTx.position) {
response['txPosition'] = JSON.stringify({
txid: trackTxid,
position: mempoolTx.position,
position: {
...mempoolTx.position,
accelerated: mempoolTx.acceleration || undefined,
}
});
}
}
}
if (client['track-address']) {
const foundTransactions: TransactionExtended[] = [];
transactions.forEach((tx) => {
if (tx.vin && tx.vin.some((vin) => !!vin.prevout && vin.prevout.scriptpubkey_address === client['track-address'])) {
foundTransactions.push(tx);
return;
}
if (tx.vout && tx.vout.some((vout) => vout.scriptpubkey_address === client['track-address'])) {
foundTransactions.push(tx);
}
});
const foundTransactions: TransactionExtended[] = Array.from(addressCache[client['track-address']]?.values() || []);
if (foundTransactions.length) {
foundTransactions.forEach((tx) => {
@@ -745,7 +873,7 @@ class WebsocketHandler {
}
}
if (client['track-mempool-block'] >= 0) {
if (client['track-mempool-block'] >= 0 && memPool.isInSync()) {
const index = client['track-mempool-block'];
if (mBlockDeltas && mBlockDeltas[index]) {
response['projected-block-transactions'] = getCachedResponse(`projected-block-transactions-${index}`, {
@@ -755,11 +883,65 @@ class WebsocketHandler {
}
}
const serializedResponse = '{'
if (Object.keys(response).length) {
const serializedResponse = this.serializeResponse(response);
client.send(serializedResponse);
}
});
}
// takes a dictionary of JSON serialized values
// and zips it together into a valid JSON object
private serializeResponse(response): string {
return '{'
+ Object.keys(response).map(key => `"${key}": ${response[key]}`).join(', ')
+ '}';
client.send(serializedResponse);
});
}
private makeAddressCache(transactions: MempoolTransactionExtended[]): { [address: string]: Set<MempoolTransactionExtended> } {
const addressCache: { [address: string]: Set<MempoolTransactionExtended> } = {};
for (const tx of transactions) {
for (const vin of tx.vin) {
if (vin?.prevout?.scriptpubkey_address) {
if (!addressCache[vin.prevout.scriptpubkey_address]) {
addressCache[vin.prevout.scriptpubkey_address] = new Set();
}
addressCache[vin.prevout.scriptpubkey_address].add(tx);
}
if (vin?.prevout?.scriptpubkey) {
if (!addressCache[vin.prevout.scriptpubkey]) {
addressCache[vin.prevout.scriptpubkey] = new Set();
}
addressCache[vin.prevout.scriptpubkey].add(tx);
}
}
for (const vout of tx.vout) {
if (vout?.scriptpubkey_address) {
if (!addressCache[vout?.scriptpubkey_address]) {
addressCache[vout?.scriptpubkey_address] = new Set();
}
addressCache[vout?.scriptpubkey_address].add(tx);
}
if (vout?.scriptpubkey) {
if (!addressCache[vout.scriptpubkey]) {
addressCache[vout.scriptpubkey] = new Set();
}
addressCache[vout.scriptpubkey].add(tx);
}
}
}
return addressCache;
}
private async getFullTransactions(transactions: MempoolTransactionExtended[]): Promise<MempoolTransactionExtended[]> {
for (let i = 0; i < transactions.length; i++) {
try {
transactions[i] = await transactionUtils.$getMempoolTransactionExtended(transactions[i].txid, true);
} catch (e) {
logger.debug('Error finding transaction in mempool: ' + (e instanceof Error ? e.message : e));
}
}
return transactions;
}
private printLogs(): void {