machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #4555 from mempool/mononaut/limit-gbt-input-redux

Browse Source 
Limited GBT
This commit is contained in:
softsimon 2024-03-29 21:26:33 +09:00 committed by GitHub
commit ba8d4ccc02
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
19 changed files with 515 additions and 317 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
backend/mempool-config.sample.json
View File

@ -28,9 +28,8 @@
"POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json",
"POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
"AUDIT": false,
"ADVANCED_GBT_AUDIT": false,
"ADVANCED_GBT_MEMPOOL": false,
"RUST_GBT": false,
"LIMIT_GBT": false,
"CPFP_INDEXING": false,
"DISK_CACHE_BLOCK_INTERVAL": 6,
"MAX_PUSH_TX_SIZE_WEIGHT": 4000000,

3
backend/src/__fixtures__/mempool-config.template.json
View File

@ -28,9 +28,8 @@
"POOLS_JSON_TREE_URL": "__MEMPOOL_POOLS_JSON_TREE_URL__",
"POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__",
"AUDIT": true,
"ADVANCED_GBT_AUDIT": true,
"ADVANCED_GBT_MEMPOOL": true,
"RUST_GBT": false,
"LIMIT_GBT": false,
"CPFP_INDEXING": true,
"MAX_BLOCKS_BULK_QUERY": 999,
"DISK_CACHE_BLOCK_INTERVAL": 999,

3
backend/src/__tests__/config.test.ts
View File

@ -41,9 +41,8 @@ describe('Mempool Backend Config', () => {
POOLS_JSON_TREE_URL: 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json',
AUDIT: false,
ADVANCED_GBT_AUDIT: false,
ADVANCED_GBT_MEMPOOL: false,
RUST_GBT: false,
LIMIT_GBT: false,
CPFP_INDEXING: false,
MAX_BLOCKS_BULK_QUERY: 0,
DISK_CACHE_BLOCK_INTERVAL: 6,

3
backend/src/api/bitcoin/bitcoin.routes.ts
View File

@ -19,6 +19,7 @@ import bitcoinClient from './bitcoin-client';
import difficultyAdjustment from '../difficulty-adjustment';
import transactionRepository from '../../repositories/TransactionRepository';
import rbfCache from '../rbf-cache';
import { calculateCpfp } from '../cpfp';
class BitcoinRoutes {
public initRoutes(app: Application) {
@ -217,7 +218,7 @@ class BitcoinRoutes {
return;
}
const cpfpInfo = Common.setRelativesAndGetCpfpInfo(tx, mempool.getMempool());
const cpfpInfo = calculateCpfp(tx, mempool.getMempool());
res.json(cpfpInfo);
return;

63
backend/src/api/common.ts
View File

@ -574,69 +574,6 @@ export class Common {
}
}
static setRelativesAndGetCpfpInfo(tx: MempoolTransactionExtended, memPool: { [txid: string]: MempoolTransactionExtended }): CpfpInfo {
const parents = this.findAllParents(tx, memPool);
const lowerFeeParents = parents.filter((parent) => parent.adjustedFeePerVsize < tx.effectiveFeePerVsize);
let totalWeight = (tx.adjustedVsize * 4) + lowerFeeParents.reduce((prev, val) => prev + (val.adjustedVsize * 4), 0);
let totalFees = tx.fee + lowerFeeParents.reduce((prev, val) => prev + val.fee, 0);
tx.ancestors = parents
.map((t) => {
return {
txid: t.txid,
weight: (t.adjustedVsize * 4),
fee: t.fee,
};
});
// Add high (high fee) decendant weight and fees
if (tx.bestDescendant) {
totalWeight += tx.bestDescendant.weight;
totalFees += tx.bestDescendant.fee;
}
tx.effectiveFeePerVsize = Math.max(0, totalFees / (totalWeight / 4));
tx.cpfpChecked = true;
return {
ancestors: tx.ancestors,
bestDescendant: tx.bestDescendant || null,
};
}
private static findAllParents(tx: MempoolTransactionExtended, memPool: { [txid: string]: MempoolTransactionExtended }): MempoolTransactionExtended[] {
let parents: MempoolTransactionExtended[] = [];
tx.vin.forEach((parent) => {
if (parents.find((p) => p.txid === parent.txid)) {
return;
}
const parentTx = memPool[parent.txid];
if (parentTx) {
if (tx.bestDescendant && tx.bestDescendant.fee / (tx.bestDescendant.weight / 4) > parentTx.adjustedFeePerVsize) {
if (parentTx.bestDescendant && parentTx.bestDescendant.fee < tx.fee + tx.bestDescendant.fee) {
parentTx.bestDescendant = {
weight: (tx.adjustedVsize * 4) + tx.bestDescendant.weight,
fee: tx.fee + tx.bestDescendant.fee,
txid: tx.txid,
};
}
} else if (tx.adjustedFeePerVsize > parentTx.adjustedFeePerVsize) {
parentTx.bestDescendant = {
weight: (tx.adjustedVsize * 4),
fee: tx.fee,
txid: tx.txid
};
}
parents.push(parentTx);
parents = parents.concat(this.findAllParents(parentTx, memPool));
}
});
return parents;
}
// calculates the ratio of matched transactions to projected transactions by weight
static getSimilarity(projectedBlock: MempoolBlockWithTransactions, transactions: TransactionExtended[]): number {
let matchedWeight = 0;

286
backend/src/api/cpfp.ts Normal file
View File

@ -0,0 +1,286 @@
import { CpfpInfo, MempoolTransactionExtended } from '../mempool.interfaces';
import memPool from './mempool';
const CPFP_UPDATE_INTERVAL = 60_000; // update CPFP info at most once per 60s per transaction
const MAX_GRAPH_SIZE = 50; // the maximum number of in-mempool relatives to consider
interface GraphTx extends MempoolTransactionExtended {
depends: string[];
spentby: string[];
ancestorMap: Map<string, GraphTx>;
fees: {
base: number;
ancestor: number;
};
ancestorcount: number;
ancestorsize: number;
ancestorRate: number;
individualRate: number;
score: number;
}
/**
* Takes a mempool transaction and a copy of the current mempool, and calculates the CPFP data for
* that transaction (and all others in the same cluster)
*/
export function calculateCpfp(tx: MempoolTransactionExtended, mempool: { [txid: string]: MempoolTransactionExtended }): CpfpInfo {
if (tx.cpfpUpdated && Date.now() < (tx.cpfpUpdated + CPFP_UPDATE_INTERVAL)) {
tx.cpfpDirty = false;
return {
ancestors: tx.ancestors || [],
bestDescendant: tx.bestDescendant || null,
descendants: tx.descendants || [],
effectiveFeePerVsize: tx.effectiveFeePerVsize || tx.adjustedFeePerVsize || tx.feePerVsize,
sigops: tx.sigops,
adjustedVsize: tx.adjustedVsize,
acceleration: tx.acceleration
};
}
const ancestorMap = new Map<string, GraphTx>();
const graphTx = mempoolToGraphTx(tx);
ancestorMap.set(tx.txid, graphTx);
const allRelatives = expandRelativesGraph(mempool, ancestorMap);
const relativesMap = initializeRelatives(allRelatives);
const cluster = calculateCpfpCluster(tx.txid, relativesMap);
let totalVsize = 0;
let totalFee = 0;
for (const tx of cluster.values()) {
totalVsize += tx.adjustedVsize;
totalFee += tx.fee;
}
const effectiveFeePerVsize = totalFee / totalVsize;
for (const tx of cluster.values()) {
mempool[tx.txid].effectiveFeePerVsize = effectiveFeePerVsize;
mempool[tx.txid].ancestors = Array.from(tx.ancestorMap.values()).map(tx => ({ txid: tx.txid, weight: tx.weight, fee: tx.fee }));
mempool[tx.txid].descendants = Array.from(cluster.values()).filter(entry => entry.txid !== tx.txid && !tx.ancestorMap.has(entry.txid)).map(tx => ({ txid: tx.txid, weight: tx.weight, fee: tx.fee }));
mempool[tx.txid].bestDescendant = null;
mempool[tx.txid].cpfpChecked = true;
mempool[tx.txid].cpfpDirty = true;
mempool[tx.txid].cpfpUpdated = Date.now();
}
tx = mempool[tx.txid];
return {
ancestors: tx.ancestors || [],
bestDescendant: tx.bestDescendant || null,
descendants: tx.descendants || [],
effectiveFeePerVsize: tx.effectiveFeePerVsize || tx.adjustedFeePerVsize || tx.feePerVsize,
sigops: tx.sigops,
adjustedVsize: tx.adjustedVsize,
acceleration: tx.acceleration
};
}
function mempoolToGraphTx(tx: MempoolTransactionExtended): GraphTx {
return {
...tx,
depends: tx.vin.map(v => v.txid),
spentby: tx.vout.map((v, i) => memPool.getFromSpendMap(tx.txid, i)).map(tx => tx?.txid).filter(txid => txid != null) as string[],
ancestorMap: new Map(),
fees: {
base: tx.fee,
ancestor: tx.fee,
},
ancestorcount: 1,
ancestorsize: tx.adjustedVsize,
ancestorRate: 0,
individualRate: 0,
score: 0,
};
}
/**
* Takes a map of transaction ancestors, and expands it into a full graph of up to MAX_GRAPH_SIZE in-mempool relatives
*/
function expandRelativesGraph(mempool: { [txid: string]: MempoolTransactionExtended }, ancestors: Map<string, GraphTx>): Map<string, GraphTx> {
const relatives: Map<string, GraphTx> = new Map();
const stack: GraphTx[] = Array.from(ancestors.values());
while (stack.length > 0) {
if (relatives.size > MAX_GRAPH_SIZE) {
return relatives;
}
const nextTx = stack.pop();
if (!nextTx) {
continue;
}
relatives.set(nextTx.txid, nextTx);
for (const relativeTxid of [...nextTx.depends, ...nextTx.spentby]) {
if (relatives.has(relativeTxid)) {
// already processed this tx
continue;
}
let mempoolTx = ancestors.get(relativeTxid);
if (!mempoolTx && mempool[relativeTxid]) {
mempoolTx = mempoolToGraphTx(mempool[relativeTxid]);
}
if (mempoolTx) {
stack.push(mempoolTx);
}
}
}
return relatives;
}
/**
* Efficiently sets a Map of in-mempool ancestors for each member of an expanded relative graph
* by running setAncestors on each leaf, and caching intermediate results.
* then initializes ancestor data for each transaction
*
* @param all
*/
function initializeRelatives(mempoolTxs: Map<string, GraphTx>): Map<string, GraphTx> {
const visited: Map<string, Map<string, GraphTx>> = new Map();
const leaves: GraphTx[] = Array.from(mempoolTxs.values()).filter(entry => entry.spentby.length === 0);
for (const leaf of leaves) {
setAncestors(leaf, mempoolTxs, visited);
}
mempoolTxs.forEach(entry => {
entry.ancestorMap?.forEach(ancestor => {
entry.ancestorcount++;
entry.ancestorsize += ancestor.adjustedVsize;
entry.fees.ancestor += ancestor.fees.base;
});
setAncestorScores(entry);
});
return mempoolTxs;
}
/**
* Given a root transaction and a list of in-mempool ancestors,
* Calculate the CPFP cluster
*
* @param tx
* @param ancestors
*/
function calculateCpfpCluster(txid: string, graph: Map<string, GraphTx>): Map<string, GraphTx> {
const tx = graph.get(txid);
if (!tx) {
return new Map<string, GraphTx>([]);
}
// Initialize individual & ancestor fee rates
graph.forEach(entry => setAncestorScores(entry));
// Sort by descending ancestor score
let sortedRelatives = Array.from(graph.values()).sort(mempoolComparator);
// Iterate until we reach a cluster that includes our target tx
let maxIterations = MAX_GRAPH_SIZE;
let best = sortedRelatives.shift();
let bestCluster = new Map<string, GraphTx>(best?.ancestorMap?.entries() || []);
while (sortedRelatives.length && best && (best.txid !== tx.txid && !best.ancestorMap.has(tx.txid)) && maxIterations > 0) {
maxIterations--;
if ((best && best.txid === tx.txid) || (bestCluster && bestCluster.has(tx.txid))) {
break;
} else {
// Remove this cluster (it doesn't include our target tx)
// and update scores, ancestor totals and dependencies for the survivors
removeAncestors(bestCluster, graph);
// re-sort
sortedRelatives = Array.from(graph.values()).sort(mempoolComparator);
// Grab the next highest scoring entry
best = sortedRelatives.shift();
if (best) {
bestCluster = new Map<string, GraphTx>(best?.ancestorMap?.entries() || []);
bestCluster.set(best?.txid, best);
}
}
}
bestCluster.set(tx.txid, tx);
return bestCluster;
}
/**
* Remove a cluster of transactions from an in-mempool dependency graph
* and update the survivors' scores and ancestors
*
* @param cluster
* @param ancestors
*/
function removeAncestors(cluster: Map<string, GraphTx>, all: Map<string, GraphTx>): void {
// remove
cluster.forEach(tx => {
all.delete(tx.txid);
});
// update survivors
all.forEach(tx => {
cluster.forEach(remove => {
if (tx.ancestorMap?.has(remove.txid)) {
// remove as dependency
tx.ancestorMap.delete(remove.txid);
tx.depends = tx.depends.filter(parent => parent !== remove.txid);
// update ancestor sizes and fees
tx.ancestorsize -= remove.adjustedVsize;
tx.fees.ancestor -= remove.fees.base;
}
});
// recalculate fee rates
setAncestorScores(tx);
});
}
/**
* Recursively traverses an in-mempool dependency graph, and sets a Map of in-mempool ancestors
* for each transaction.
*
* @param tx
* @param all
*/
function setAncestors(tx: GraphTx, all: Map<string, GraphTx>, visited: Map<string, Map<string, GraphTx>>, depth: number = 0): Map<string, GraphTx> {
// sanity check for infinite recursion / too many ancestors (should never happen)
if (depth > MAX_GRAPH_SIZE) {
return tx.ancestorMap;
}
// initialize the ancestor map for this tx
tx.ancestorMap = new Map<string, GraphTx>();
tx.depends.forEach(parentId => {
const parent = all.get(parentId);
if (parent) {
// add the parent
tx.ancestorMap?.set(parentId, parent);
// check for a cached copy of this parent's ancestors
let ancestors = visited.get(parent.txid);
if (!ancestors) {
// recursively fetch the parent's ancestors
ancestors = setAncestors(parent, all, visited, depth + 1);
}
// and add to this tx's map
ancestors.forEach((ancestor, ancestorId) => {
tx.ancestorMap?.set(ancestorId, ancestor);
});
}
});
visited.set(tx.txid, tx.ancestorMap);
return tx.ancestorMap;
}
/**
* Take a mempool transaction, and set the fee rates and ancestor score
*
* @param tx
*/
function setAncestorScores(tx: GraphTx): GraphTx {
tx.individualRate = (tx.fees.base * 100_000_000) / tx.adjustedVsize;
tx.ancestorRate = (tx.fees.ancestor * 100_000_000) / tx.ancestorsize;
tx.score = Math.min(tx.individualRate, tx.ancestorRate);
return tx;
}
// Sort by descending score
function mempoolComparator(a: GraphTx, b: GraphTx): number {
return b.score - a.score;
}

244
backend/src/api/mempool-blocks.ts
View File

@ -1,6 +1,6 @@
import { GbtGenerator, GbtResult, ThreadTransaction as RustThreadTransaction, ThreadAcceleration as RustThreadAcceleration } from 'rust-gbt';
import logger from '../logger';
import { MempoolBlock, MempoolTransactionExtended, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats, PoolTag, TransactionClassified, TransactionCompressed, MempoolDeltaChange } from '../mempool.interfaces';
import { MempoolBlock, MempoolTransactionExtended, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats, TransactionClassified, TransactionCompressed, MempoolDeltaChange, GbtCandidates } from '../mempool.interfaces';
import { Common, OnlineFeeStatsCalculator } from './common';
import config from '../config';
import { Worker } from 'worker_threads';
@ -18,6 +18,7 @@ class MempoolBlocks {
private nextUid: number = 1;
private uidMap: Map<number, string> = new Map(); // map short numerical uids to full txids
private txidMap: Map<string, number> = new Map(); // map full txids back to short numerical uids
public getMempoolBlocks(): MempoolBlock[] {
return this.mempoolBlocks.map((block) => {
@ -40,132 +41,6 @@ class MempoolBlocks {
return this.mempoolBlockDeltas;
}
public updateMempoolBlocks(memPool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false): MempoolBlockWithTransactions[] {
const latestMempool = memPool;
const memPoolArray: MempoolTransactionExtended[] = [];
for (const i in latestMempool) {
memPoolArray.push(latestMempool[i]);
}
const start = new Date().getTime();
// Clear bestDescendants & ancestors
memPoolArray.forEach((tx) => {
tx.bestDescendant = null;
tx.ancestors = [];
tx.cpfpChecked = false;
if (!tx.effectiveFeePerVsize) {
tx.effectiveFeePerVsize = tx.adjustedFeePerVsize;
}
});
// First sort
memPoolArray.sort((a, b) => {
if (a.adjustedFeePerVsize === b.adjustedFeePerVsize) {
// tie-break by lexicographic txid order for stability
return a.txid < b.txid ? -1 : 1;
} else {
return b.adjustedFeePerVsize - a.adjustedFeePerVsize;
}
});
// Loop through and traverse all ancestors and sum up all the sizes + fees
// Pass down size + fee to all unconfirmed children
let sizes = 0;
memPoolArray.forEach((tx) => {
sizes += tx.weight;
if (sizes > 4000000 * 8) {
return;
}
Common.setRelativesAndGetCpfpInfo(tx, memPool);
});
// Final sort, by effective fee
memPoolArray.sort((a, b) => {
if (a.effectiveFeePerVsize === b.effectiveFeePerVsize) {
// tie-break by lexicographic txid order for stability
return a.txid < b.txid ? -1 : 1;
} else {
return b.effectiveFeePerVsize - a.effectiveFeePerVsize;
}
});
const end = new Date().getTime();
const time = end - start;
logger.debug('Mempool blocks calculated in ' + time / 1000 + ' seconds');
const blocks = this.calculateMempoolBlocks(memPoolArray);
if (saveResults) {
const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, blocks);
this.mempoolBlocks = blocks;
this.mempoolBlockDeltas = deltas;
}
return blocks;
}
private calculateMempoolBlocks(transactionsSorted: MempoolTransactionExtended[]): MempoolBlockWithTransactions[] {
const mempoolBlocks: MempoolBlockWithTransactions[] = [];
let feeStatsCalculator: OnlineFeeStatsCalculator = new OnlineFeeStatsCalculator(config.MEMPOOL.BLOCK_WEIGHT_UNITS);
let onlineStats = false;
let blockSize = 0;
let blockWeight = 0;
let blockVsize = 0;
let blockFees = 0;
const sizeLimit = (config.MEMPOOL.BLOCK_WEIGHT_UNITS / 4) * 1.2;
let transactionIds: string[] = [];
let transactions: MempoolTransactionExtended[] = [];
transactionsSorted.forEach((tx, index) => {
if (blockWeight + tx.weight <= config.MEMPOOL.BLOCK_WEIGHT_UNITS
|| mempoolBlocks.length === config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT - 1) {
tx.position = {
block: mempoolBlocks.length,
vsize: blockVsize + (tx.vsize / 2),
};
blockWeight += tx.weight;
blockVsize += tx.vsize;
blockSize += tx.size;
blockFees += tx.fee;
if (blockVsize <= sizeLimit) {
transactions.push(tx);
}
transactionIds.push(tx.txid);
if (onlineStats) {
feeStatsCalculator.processNext(tx);
}
} else {
mempoolBlocks.push(this.dataToMempoolBlocks(transactionIds, transactions, blockSize, blockWeight, blockFees));
blockVsize = 0;
tx.position = {
block: mempoolBlocks.length,
vsize: blockVsize + (tx.vsize / 2),
};
if (mempoolBlocks.length === config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT - 1) {
const stackWeight = transactionsSorted.slice(index).reduce((total, tx) => total + (tx.weight || 0), 0);
if (stackWeight > config.MEMPOOL.BLOCK_WEIGHT_UNITS) {
onlineStats = true;
feeStatsCalculator = new OnlineFeeStatsCalculator(stackWeight, 0.5, [10, 20, 30, 40, 50, 60, 70, 80, 90]);
feeStatsCalculator.processNext(tx);
}
}
blockVsize += tx.vsize;
blockWeight = tx.weight;
blockSize = tx.size;
blockFees = tx.fee;
transactionIds = [tx.txid];
transactions = [tx];
}
});
if (transactions.length) {
const feeStats = onlineStats ? feeStatsCalculator.getRawFeeStats() : undefined;
mempoolBlocks.push(this.dataToMempoolBlocks(transactionIds, transactions, blockSize, blockWeight, blockFees, feeStats));
}
return mempoolBlocks;
}
private calculateMempoolDeltas(prevBlocks: MempoolBlockWithTransactions[], mempoolBlocks: MempoolBlockWithTransactions[]): MempoolBlockDelta[] {
const mempoolBlockDeltas: MempoolBlockDelta[] = [];
for (let i = 0; i < Math.max(mempoolBlocks.length, prevBlocks.length); i++) {
@ -207,7 +82,7 @@ class MempoolBlocks {
return mempoolBlockDeltas;
}
public async $makeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
public async $makeBlockTemplates(transactions: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, candidates: GbtCandidates | undefined, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
const start = Date.now();
// reset mempool short ids
@ -215,7 +90,8 @@ class MempoolBlocks {
this.resetUids();
}
// set missing short ids
for (const tx of Object.values(newMempool)) {
for (const txid of transactions) {
const tx = newMempool[txid];
this.setUid(tx, !saveResults);
}
@ -224,7 +100,8 @@ class MempoolBlocks {
// prepare a stripped down version of the mempool with only the minimum necessary data
// to reduce the overhead of passing this data to the worker thread
const strippedMempool: Map<number, CompactThreadTransaction> = new Map();
Object.values(newMempool).forEach(entry => {
for (const txid of transactions) {
const entry = newMempool[txid];
if (entry.uid !== null && entry.uid !== undefined) {
const stripped = {
uid: entry.uid,
@ -237,7 +114,7 @@ class MempoolBlocks {
};
strippedMempool.set(entry.uid, stripped);
}
});
}
// (re)initialize tx selection worker thread
if (!this.txSelectionWorker) {
@ -268,7 +145,7 @@ class MempoolBlocks {
// clean up thread error listener
this.txSelectionWorker?.removeListener('error', threadErrorListener);
const processed = this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), accelerations, accelerationPool, saveResults);
const processed = this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), candidates, accelerations, accelerationPool, saveResults);
logger.debug(`makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`);
@ -279,10 +156,10 @@ class MempoolBlocks {
return this.mempoolBlocks;
}
public async $updateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], accelerationDelta: string[] = [], saveResults: boolean = false, useAccelerations: boolean = false): Promise<void> {
public async $updateBlockTemplates(transactions: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], candidates: GbtCandidates | undefined, accelerationDelta: string[] = [], saveResults: boolean = false, useAccelerations: boolean = false): Promise<void> {
if (!this.txSelectionWorker) {
// need to reset the worker
await this.$makeBlockTemplates(newMempool, saveResults, useAccelerations);
await this.$makeBlockTemplates(transactions, newMempool, candidates, saveResults, useAccelerations);
return;
}
@ -292,9 +169,9 @@ class MempoolBlocks {
const addedAndChanged: MempoolTransactionExtended[] = useAccelerations ? accelerationDelta.map(txid => newMempool[txid]).filter(tx => tx != null).concat(added) : added;
for (const tx of addedAndChanged) {
this.setUid(tx, true);
this.setUid(tx, false);
}
const removedUids = removed.map(tx => this.getUid(tx)).filter(uid => uid != null) as number[];
const removedTxs = removed.filter(tx => tx.uid != null) as MempoolTransactionExtended[];
// prepare a stripped down version of the mempool with only the minimum necessary data
// to reduce the overhead of passing this data to the worker thread
@ -320,15 +197,15 @@ class MempoolBlocks {
});
this.txSelectionWorker?.once('error', reject);
});
this.txSelectionWorker.postMessage({ type: 'update', added: addedStripped, removed: removedUids });
this.txSelectionWorker.postMessage({ type: 'update', added: addedStripped, removed: removedTxs.map(tx => tx.uid) as number[] });
const { blocks, rates, clusters } = this.convertResultTxids(await workerResultPromise);
this.removeUids(removedUids);
this.removeUids(removedTxs);
// clean up thread error listener
this.txSelectionWorker?.removeListener('error', threadErrorListener);
this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), accelerations, null, saveResults);
this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), candidates, accelerations, null, saveResults);
logger.debug(`updateBlockTemplates completed in ${(Date.now() - start) / 1000} seconds`);
} catch (e) {
logger.err('updateBlockTemplates failed. ' + (e instanceof Error ? e.message : e));
@ -340,25 +217,28 @@ class MempoolBlocks {
this.rustGbtGenerator = new GbtGenerator();
}
public async $rustMakeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
public async $rustMakeBlockTemplates(txids: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, candidates: GbtCandidates | undefined, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
const start = Date.now();
// reset mempool short ids
if (saveResults) {
this.resetUids();
}
const transactions = txids.map(txid => newMempool[txid]).filter(tx => tx != null);
// set missing short ids
for (const tx of Object.values(newMempool)) {
for (const tx of transactions) {
this.setUid(tx, !saveResults);
}
// set short ids for transaction inputs
for (const tx of Object.values(newMempool)) {
for (const tx of transactions) {
tx.inputs = tx.vin.map(v => this.getUid(newMempool[v.txid])).filter(uid => (uid !== null && uid !== undefined)) as number[];
}
const accelerations = useAccelerations ? mempool.getAccelerations() : {};
const acceleratedList = accelerationPool ? Object.values(accelerations).filter(acc => newMempool[acc.txid] && acc.pools.includes(accelerationPool)) : Object.values(accelerations).filter(acc => newMempool[acc.txid]);
const convertedAccelerations = acceleratedList.map(acc => {
this.setUid(newMempool[acc.txid], true);
return {
uid: this.getUid(newMempool[acc.txid]),
delta: acc.feeDelta,
@ -369,15 +249,15 @@ class MempoolBlocks {
const rustGbt = saveResults ? this.rustGbtGenerator : new GbtGenerator();
try {
const { blocks, blockWeights, rates, clusters, overflow } = this.convertNapiResultTxids(
await rustGbt.make(Object.values(newMempool) as RustThreadTransaction[], convertedAccelerations as RustThreadAcceleration[], this.nextUid),
await rustGbt.make(transactions as RustThreadTransaction[], convertedAccelerations as RustThreadAcceleration[], this.nextUid),
);
if (saveResults) {
this.rustInitialized = true;
}
const mempoolSize = Object.keys(newMempool).length;
const expectedSize = transactions.length;
const resultMempoolSize = blocks.reduce((total, block) => total + block.length, 0) + overflow.length;
logger.debug(`RUST updateBlockTemplates returned ${resultMempoolSize} txs out of ${mempoolSize} in the mempool, ${overflow.length} were unmineable`);
const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, accelerations, accelerationPool, saveResults);
logger.debug(`RUST updateBlockTemplates returned ${resultMempoolSize} txs out of ${expectedSize} in the mempool, ${overflow.length} were unmineable`);
const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, candidates, accelerations, accelerationPool, saveResults);
logger.debug(`RUST makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`);
return processed;
} catch (e) {
@ -389,36 +269,37 @@ class MempoolBlocks {
return this.mempoolBlocks;
}
public async $oneOffRustBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, useAccelerations: boolean, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
return this.$rustMakeBlockTemplates(newMempool, false, useAccelerations, accelerationPool);
public async $oneOffRustBlockTemplates(transactions: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, candidates: GbtCandidates | undefined, useAccelerations: boolean, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
return this.$rustMakeBlockTemplates(transactions, newMempool, candidates, false, useAccelerations, accelerationPool);
}
public async $rustUpdateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, mempoolSize: number, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], useAccelerations: boolean, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
public async $rustUpdateBlockTemplates(transactions: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], candidates: GbtCandidates | undefined, useAccelerations: boolean, accelerationPool?: number): Promise<MempoolBlockWithTransactions[]> {
// GBT optimization requires that uids never get too sparse
// as a sanity check, we should also explicitly prevent uint32 uid overflow
if (this.nextUid + added.length >= Math.min(Math.max(262144, 2 * mempoolSize), MAX_UINT32)) {
if (this.nextUid + added.length >= Math.min(Math.max(262144, 2 * transactions.length), MAX_UINT32)) {
this.resetRustGbt();
}
if (!this.rustInitialized) {
// need to reset the worker
return this.$rustMakeBlockTemplates(newMempool, true, useAccelerations, accelerationPool);
return this.$rustMakeBlockTemplates(transactions, newMempool, candidates, true, useAccelerations, accelerationPool);
}
const start = Date.now();
// set missing short ids
for (const tx of added) {
this.setUid(tx, true);
this.setUid(tx, false);
}
// set short ids for transaction inputs
for (const tx of added) {
tx.inputs = tx.vin.map(v => this.getUid(newMempool[v.txid])).filter(uid => (uid !== null && uid !== undefined)) as number[];
}
const removedUids = removed.map(tx => this.getUid(tx)).filter(uid => (uid !== null && uid !== undefined)) as number[];
const removedTxs = removed.filter(tx => tx.uid != null) as MempoolTransactionExtended[];
const accelerations = useAccelerations ? mempool.getAccelerations() : {};
const acceleratedList = accelerationPool ? Object.values(accelerations).filter(acc => newMempool[acc.txid] && acc.pools.includes(accelerationPool)) : Object.values(accelerations).filter(acc => newMempool[acc.txid]);
const convertedAccelerations = acceleratedList.map(acc => {
this.setUid(newMempool[acc.txid], true);
return {
uid: this.getUid(newMempool[acc.txid]),
delta: acc.feeDelta,
@ -430,18 +311,18 @@ class MempoolBlocks {
const { blocks, blockWeights, rates, clusters, overflow } = this.convertNapiResultTxids(
await this.rustGbtGenerator.update(
added as RustThreadTransaction[],
removedUids,
removedTxs.map(tx => tx.uid) as number[],
convertedAccelerations as RustThreadAcceleration[],
this.nextUid,
),
);
const resultMempoolSize = blocks.reduce((total, block) => total + block.length, 0) + overflow.length;
logger.debug(`RUST updateBlockTemplates returned ${resultMempoolSize} txs out of ${mempoolSize} in the mempool, ${overflow.length} were unmineable`);
if (mempoolSize !== resultMempoolSize) {
throw new Error('GBT returned wrong number of transactions , cache is probably out of sync');
logger.debug(`RUST updateBlockTemplates returned ${resultMempoolSize} txs out of ${transactions.length} candidates, ${overflow.length} were unmineable`);
if (transactions.length !== resultMempoolSize) {
throw new Error(`GBT returned wrong number of transactions ${transactions.length} vs ${resultMempoolSize}, cache is probably out of sync`);
} else {
const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, accelerations, accelerationPool, true);
this.removeUids(removedUids);
const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, candidates, accelerations, accelerationPool, true);
this.removeUids(removedTxs);
logger.debug(`RUST updateBlockTemplates completed in ${(Date.now() - start)/1000} seconds`);
return processed;
}
@ -452,7 +333,7 @@ class MempoolBlocks {
}
}
private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], accelerations, accelerationPool, saveResults): MempoolBlockWithTransactions[] {
private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], candidates: GbtCandidates | undefined, accelerations, accelerationPool, saveResults): MempoolBlockWithTransactions[] {
for (const [txid, rate] of rates) {
if (txid in mempool) {
mempool[txid].cpfpDirty = (rate !== mempool[txid].effectiveFeePerVsize);
@ -486,6 +367,9 @@ class MempoolBlocks {
if (txid === memberTxid) {
matched = true;
} else {
if (!mempool[txid]) {
console.log('txid missing from mempool! ', txid, candidates?.txs[txid]);
}
const relative = {
txid: txid,
fee: mempool[txid].fee,
@ -518,6 +402,16 @@ class MempoolBlocks {
let totalWeight = 0;
let totalFees = 0;
const transactions: MempoolTransactionExtended[] = [];
// backfill purged transactions
if (candidates?.txs && blockIndex === blocks.length - 1) {
for (const txid of Object.keys(mempool)) {
if (!candidates.txs[txid]) {
block.push(txid);
}
}
}
for (const txid of block) {
if (txid) {
mempoolTx = mempool[txid];
@ -526,16 +420,6 @@ class MempoolBlocks {
block: blockIndex,
vsize: totalVsize + (mempoolTx.vsize / 2),
};
if (!mempoolTx.cpfpChecked) {
if (mempoolTx.ancestors?.length) {
mempoolTx.ancestors = [];
}
if (mempoolTx.descendants?.length) {
mempoolTx.descendants = [];
}
mempoolTx.bestDescendant = null;
mempoolTx.cpfpChecked = true;
}
const acceleration = accelerations[txid];
if (isAccelerated[txid] || (acceleration && (!accelerationPool || acceleration.pools.includes(accelerationPool)))) {
@ -610,30 +494,38 @@ class MempoolBlocks {
private resetUids(): void {
this.uidMap.clear();
this.txidMap.clear();
this.nextUid = 1;
}
private setUid(tx: MempoolTransactionExtended, skipSet = false): number {
if (tx.uid === null || tx.uid === undefined || !skipSet) {
if (!this.txidMap.has(tx.txid) || !skipSet) {
const uid = this.nextUid;
this.nextUid++;
this.uidMap.set(uid, tx.txid);
this.txidMap.set(tx.txid, uid);
tx.uid = uid;
return uid;
} else {
tx.uid = this.txidMap.get(tx.txid) as number;
return tx.uid;
}
}
private getUid(tx: MempoolTransactionExtended): number | void {
if (tx?.uid !== null && tx?.uid !== undefined && this.uidMap.has(tx.uid)) {
return tx.uid;
if (tx) {
return this.txidMap.get(tx.txid);
}
}
private removeUids(uids: number[]): void {
for (const uid of uids) {
this.uidMap.delete(uid);
private removeUids(txs: MempoolTransactionExtended[]): void {
for (const tx of txs) {
const uid = this.txidMap.get(tx.txid);
if (uid != null) {
this.uidMap.delete(uid);
this.txidMap.delete(tx.txid);
}
tx.uid = undefined;
}
}

92
backend/src/api/mempool.ts
View File

@ -1,6 +1,6 @@
import config from '../config';
import bitcoinApi from './bitcoin/bitcoin-api-factory';
import { MempoolTransactionExtended, TransactionExtended, VbytesPerSecond } from '../mempool.interfaces';
import { MempoolTransactionExtended, TransactionExtended, VbytesPerSecond, GbtCandidates } from '../mempool.interfaces';
import logger from '../logger';
import { Common } from './common';
import transactionUtils from './transaction-utils';
@ -11,18 +11,20 @@ import bitcoinSecondClient from './bitcoin/bitcoin-second-client';
import rbfCache from './rbf-cache';
import { Acceleration } from './services/acceleration';
import redisCache from './redis-cache';
import blocks from './blocks';
class Mempool {
private inSync: boolean = false;
private mempoolCacheDelta: number = -1;
private mempoolCache: { [txId: string]: MempoolTransactionExtended } = {};
private mempoolCandidates: { [txid: string ]: boolean } = {};
private spendMap = new Map<string, MempoolTransactionExtended>();
private mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0, total_fee: 0,
maxmempool: 300000000, mempoolminfee: Common.isLiquid() ? 0.00000100 : 0.00001000, minrelaytxfee: Common.isLiquid() ? 0.00000100 : 0.00001000 };
private mempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, newTransactions: MempoolTransactionExtended[],
deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => void) | undefined;
private $asyncMempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, mempoolSize: number, newTransactions: MempoolTransactionExtended[],
deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => Promise<void>) | undefined;
deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[], candidates?: GbtCandidates) => Promise<void>) | undefined;
private accelerations: { [txId: string]: Acceleration } = {};
@ -40,6 +42,8 @@ class Mempool {
private missingTxCount = 0;
private mainLoopTimeout: number = 120000;
public limitGBT = config.MEMPOOL.USE_SECOND_NODE_FOR_MINFEE && config.MEMPOOL.LIMIT_GBT;
constructor() {
setInterval(this.updateTxPerSecond.bind(this), 1000);
}
@ -74,7 +78,8 @@ class Mempool {
}
public setAsyncMempoolChangedCallback(fn: (newMempool: { [txId: string]: MempoolTransactionExtended; }, mempoolSize: number,
newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => Promise<void>): void {
newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[],
candidates?: GbtCandidates) => Promise<void>): void {
this.$asyncMempoolChangedCallback = fn;
}
@ -86,6 +91,10 @@ class Mempool {
return this.spendMap;
}
public getFromSpendMap(txid, index): MempoolTransactionExtended | void {
return this.spendMap.get(`${txid}:${index}`);
}
public async $setMempool(mempoolData: { [txId: string]: MempoolTransactionExtended }) {
this.mempoolCache = mempoolData;
let count = 0;
@ -108,6 +117,9 @@ class Mempool {
await redisCache.$addTransaction(this.mempoolCache[txid]);
}
this.mempoolCache[txid].flags = Common.getTransactionFlags(this.mempoolCache[txid]);
this.mempoolCache[txid].cpfpChecked = false;
this.mempoolCache[txid].cpfpDirty = true;
this.mempoolCache[txid].cpfpUpdated = undefined;
}
if (config.MEMPOOL.CACHE_ENABLED && config.REDIS.ENABLED) {
await redisCache.$flushTransactions();
@ -117,7 +129,7 @@ class Mempool {
this.mempoolChangedCallback(this.mempoolCache, [], [], []);
}
if (this.$asyncMempoolChangedCallback) {
await this.$asyncMempoolChangedCallback(this.mempoolCache, count, [], [], []);
await this.$asyncMempoolChangedCallback(this.mempoolCache, count, [], [], [], this.limitGBT ? { txs: {}, added: [], removed: [] } : undefined);
}
this.addToSpendMap(Object.values(this.mempoolCache));
}
@ -160,6 +172,10 @@ class Mempool {
return newTransactions;
}
public getMempoolCandidates(): { [txid: string]: boolean } {
return this.mempoolCandidates;
}
public async $updateMemPoolInfo() {
this.mempoolInfo = await this.$getMempoolInfo();
}
@ -189,7 +205,7 @@ class Mempool {
return txTimes;
}
public async $updateMempool(transactions: string[], accelerations: Acceleration[] | null, pollRate: number): Promise<void> {
public async $updateMempool(transactions: string[], accelerations: Acceleration[] | null, minFeeMempool: string[], minFeeTip: number, pollRate: number): Promise<void> {
logger.debug(`Updating mempool...`);
// warn if this run stalls the main loop for more than 2 minutes
@ -330,6 +346,8 @@ class Mempool {
}
}
const candidates = await this.getNextCandidates(minFeeMempool, minFeeTip, deletedTransactions);
const newMempoolSize = currentMempoolSize + newTransactions.length - deletedTransactions.length;
const newTransactionsStripped = newTransactions.map((tx) => Common.stripTransaction(tx));
this.latestTransactions = newTransactionsStripped.concat(this.latestTransactions).slice(0, 6);
@ -341,12 +359,14 @@ class Mempool {
this.mempoolCacheDelta = Math.abs(transactions.length - newMempoolSize);
const candidatesChanged = candidates?.added?.length || candidates?.removed?.length;
if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) {
this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions, accelerationDelta);
}
if (this.$asyncMempoolChangedCallback && (hasChange || deletedTransactions.length)) {
if (this.$asyncMempoolChangedCallback && (hasChange || deletedTransactions.length || candidatesChanged)) {
this.updateTimerProgress(timer, 'running async mempool callback');
await this.$asyncMempoolChangedCallback(this.mempoolCache, newMempoolSize, newTransactions, deletedTransactions, accelerationDelta);
await this.$asyncMempoolChangedCallback(this.mempoolCache, newMempoolSize, newTransactions, deletedTransactions, accelerationDelta, candidates);
this.updateTimerProgress(timer, 'completed async mempool callback');
}
@ -432,6 +452,64 @@ class Mempool {
}
}
public async getNextCandidates(minFeeTransactions: string[], blockHeight: number, deletedTransactions: MempoolTransactionExtended[]): Promise<GbtCandidates | undefined> {
if (this.limitGBT) {
const deletedTxsMap = {};
for (const tx of deletedTransactions) {
deletedTxsMap[tx.txid] = tx;
}
const newCandidateTxMap = {};
for (const txid of minFeeTransactions) {
if (this.mempoolCache[txid]) {
newCandidateTxMap[txid] = true;
}
}
const accelerations = this.getAccelerations();
for (const txid of Object.keys(accelerations)) {
if (this.mempoolCache[txid]) {
newCandidateTxMap[txid] = true;
}
}
const removed: MempoolTransactionExtended[] = [];
const added: MempoolTransactionExtended[] = [];
// don't prematurely remove txs included in a new block
if (blockHeight > blocks.getCurrentBlockHeight()) {
for (const txid of Object.keys(this.mempoolCandidates)) {
newCandidateTxMap[txid] = true;
}
} else {
for (const txid of Object.keys(this.mempoolCandidates)) {
if (!newCandidateTxMap[txid]) {
if (this.mempoolCache[txid]) {
removed.push(this.mempoolCache[txid]);
this.mempoolCache[txid].effectiveFeePerVsize = this.mempoolCache[txid].adjustedFeePerVsize;
this.mempoolCache[txid].ancestors = [];
this.mempoolCache[txid].descendants = [];
this.mempoolCache[txid].bestDescendant = null;
this.mempoolCache[txid].cpfpChecked = false;
this.mempoolCache[txid].cpfpUpdated = undefined;
} else if (deletedTxsMap[txid]) {
removed.push(deletedTxsMap[txid]);
}
}
}
}
for (const txid of Object.keys(newCandidateTxMap)) {
if (!this.mempoolCandidates[txid]) {
added.push(this.mempoolCache[txid]);
}
}
this.mempoolCandidates = newCandidateTxMap;
return {
txs: this.mempoolCandidates,
added,
removed
};
}
}
private startTimer() {
const state: any = {
start: Date.now(),

89
backend/src/api/websocket-handler.ts
View File

@ -2,7 +2,7 @@ import logger from '../logger';
import * as WebSocket from 'ws';
import {
BlockExtended, TransactionExtended, MempoolTransactionExtended, WebsocketResponse,
OptimizedStatistic, ILoadingIndicators
OptimizedStatistic, ILoadingIndicators, GbtCandidates,
} from '../mempool.interfaces';
import blocks from './blocks';
import memPool from './mempool';
@ -18,7 +18,6 @@ import feeApi from './fee-api';
import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository';
import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
import Audit from './audit';
import { deepClone } from '../utils/clone';
import priceUpdater from '../tasks/price-updater';
import { ApiPrice } from '../repositories/PricesRepository';
import accelerationApi from './services/acceleration';
@ -32,6 +31,8 @@ interface AddressTransactions {
confirmed: MempoolTransactionExtended[],
removed: MempoolTransactionExtended[],
}
import bitcoinSecondClient from './bitcoin/bitcoin-second-client';
import { calculateCpfp } from './cpfp';
// valid 'want' subscriptions
const wantable = [
@ -436,21 +437,26 @@ class WebsocketHandler {
}
async $handleMempoolChange(newMempool: { [txid: string]: MempoolTransactionExtended }, mempoolSize: number,
newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]): Promise<void> {
newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[],
candidates?: GbtCandidates): Promise<void> {
if (!this.wss) {
throw new Error('WebSocket.Server is not set');
}
this.printLogs();
if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) {
if (config.MEMPOOL.RUST_GBT) {
await mempoolBlocks.$rustUpdateBlockTemplates(newMempool, mempoolSize, newTransactions, deletedTransactions, config.MEMPOOL_SERVICES.ACCELERATIONS);
} else {
await mempoolBlocks.$updateBlockTemplates(newMempool, newTransactions, deletedTransactions, accelerationDelta, true, config.MEMPOOL_SERVICES.ACCELERATIONS);
}
const transactionIds = (memPool.limitGBT && candidates) ? Object.keys(candidates?.txs || {}) : Object.keys(newMempool);
let added = newTransactions;
let removed = deletedTransactions;
if (memPool.limitGBT) {
added = candidates?.added || [];
removed = candidates?.removed || [];
}
if (config.MEMPOOL.RUST_GBT) {
await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, newMempool, added, removed, candidates, config.MEMPOOL_SERVICES.ACCELERATIONS);
} else {
mempoolBlocks.updateMempoolBlocks(newMempool, true);
await mempoolBlocks.$updateBlockTemplates(transactionIds, newMempool, added, removed, candidates, accelerationDelta, true, config.MEMPOOL_SERVICES.ACCELERATIONS);
}
const mBlocks = mempoolBlocks.getMempoolBlocks();
@ -689,6 +695,9 @@ class WebsocketHandler {
accelerated: mempoolTx.acceleration || undefined,
}
};
if (!mempoolTx.cpfpChecked) {
calculateCpfp(mempoolTx, newMempool);
}
if (mempoolTx.cpfpDirty) {
positionData['cpfp'] = {
ancestors: mempoolTx.ancestors,
@ -739,8 +748,9 @@ class WebsocketHandler {
await statistics.runStatistics();
const _memPool = memPool.getMempool();
const isAccelerated = config.MEMPOOL_SERVICES.ACCELERATIONS && accelerationApi.isAcceleratedBlock(block, Object.values(mempool.getAccelerations()));
const candidateTxs = await memPool.getMempoolCandidates();
let candidates: GbtCandidates | undefined = (memPool.limitGBT && candidateTxs) ? { txs: candidateTxs, added: [], removed: [] } : undefined;
let transactionIds: string[] = (memPool.limitGBT) ? Object.keys(candidates?.txs || {}) : Object.keys(_memPool);
const accelerations = Object.values(mempool.getAccelerations());
await accelerationRepository.$indexAccelerationsForBlock(block, accelerations, transactions);
@ -751,31 +761,19 @@ class WebsocketHandler {
if (config.MEMPOOL.AUDIT && memPool.isInSync()) {
let projectedBlocks;
let auditMempool = _memPool;
// template calculation functions have mempool side effects, so calculate audits using
// a cloned copy of the mempool if we're running a different algorithm for mempool updates
const separateAudit = config.MEMPOOL.ADVANCED_GBT_AUDIT !== config.MEMPOOL.ADVANCED_GBT_MEMPOOL;
if (separateAudit) {
auditMempool = deepClone(_memPool);
if (config.MEMPOOL.ADVANCED_GBT_AUDIT) {
if (config.MEMPOOL.RUST_GBT) {
projectedBlocks = await mempoolBlocks.$oneOffRustBlockTemplates(auditMempool, isAccelerated, block.extras.pool.id);
} else {
projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, isAccelerated, block.extras.pool.id);
}
const auditMempool = _memPool;
const isAccelerated = config.MEMPOOL_SERVICES.ACCELERATIONS && accelerationApi.isAcceleratedBlock(block, Object.values(mempool.getAccelerations()));
if ((config.MEMPOOL_SERVICES.ACCELERATIONS)) {
if (config.MEMPOOL.RUST_GBT) {
const added = memPool.limitGBT ? (candidates?.added || []) : [];
const removed = memPool.limitGBT ? (candidates?.removed || []) : [];
projectedBlocks = await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, auditMempool, added, removed, candidates, isAccelerated, block.extras.pool.id);
} else {
projectedBlocks = mempoolBlocks.updateMempoolBlocks(auditMempool, false);
projectedBlocks = await mempoolBlocks.$makeBlockTemplates(transactionIds, auditMempool, candidates, false, isAccelerated, block.extras.pool.id);
}
} else {
if ((config.MEMPOOL_SERVICES.ACCELERATIONS)) {
if (config.MEMPOOL.RUST_GBT) {
projectedBlocks = await mempoolBlocks.$rustUpdateBlockTemplates(auditMempool, Object.keys(auditMempool).length, [], [], isAccelerated, block.extras.pool.id);
} else {
projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, isAccelerated, block.extras.pool.id);
}
} else {
projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
}
projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
}
if (Common.indexingEnabled()) {
@ -838,14 +836,23 @@ class WebsocketHandler {
confirmedTxids[txId] = true;
}
if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) {
if (config.MEMPOOL.RUST_GBT) {
await mempoolBlocks.$rustUpdateBlockTemplates(_memPool, Object.keys(_memPool).length, [], transactions, true);
} else {
await mempoolBlocks.$makeBlockTemplates(_memPool, true, config.MEMPOOL_SERVICES.ACCELERATIONS);
}
if (memPool.limitGBT) {
const minFeeMempool = memPool.limitGBT ? await bitcoinSecondClient.getRawMemPool() : null;
const minFeeTip = memPool.limitGBT ? await bitcoinSecondClient.getBlockCount() : -1;
candidates = await memPool.getNextCandidates(minFeeMempool, minFeeTip, transactions);
transactionIds = Object.keys(candidates?.txs || {});
} else {
mempoolBlocks.updateMempoolBlocks(_memPool, true);
candidates = undefined;
transactionIds = Object.keys(memPool.getMempool());
}
if (config.MEMPOOL.RUST_GBT) {
const added = memPool.limitGBT ? (candidates?.added || []) : [];
const removed = memPool.limitGBT ? (candidates?.removed || []) : transactions;
await mempoolBlocks.$rustUpdateBlockTemplates(transactionIds, _memPool, added, removed, candidates, true);
} else {
await mempoolBlocks.$makeBlockTemplates(transactionIds, _memPool, candidates, true, config.MEMPOOL_SERVICES.ACCELERATIONS);
}
const mBlocks = mempoolBlocks.getMempoolBlocks();
const mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();

6
backend/src/config.ts
View File

@ -32,9 +32,8 @@ interface IConfig {
POOLS_JSON_URL: string,
POOLS_JSON_TREE_URL: string,
AUDIT: boolean;
ADVANCED_GBT_AUDIT: boolean;
ADVANCED_GBT_MEMPOOL: boolean;
RUST_GBT: boolean;
LIMIT_GBT: boolean;
CPFP_INDEXING: boolean;
MAX_BLOCKS_BULK_QUERY: number;
DISK_CACHE_BLOCK_INTERVAL: number;
@ -194,9 +193,8 @@ const defaults: IConfig = {
'POOLS_JSON_URL': 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json',
'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
'AUDIT': false,
'ADVANCED_GBT_AUDIT': false,
'ADVANCED_GBT_MEMPOOL': false,
'RUST_GBT': false,
'LIMIT_GBT': false,
'CPFP_INDEXING': false,
'MAX_BLOCKS_BULK_QUERY': 0,
'DISK_CACHE_BLOCK_INTERVAL': 6,

5
backend/src/index.ts
View File

@ -45,6 +45,7 @@ import { formatBytes, getBytesUnit } from './utils/format';
import redisCache from './api/redis-cache';
import accelerationApi from './api/services/acceleration';
import bitcoinCoreRoutes from './api/bitcoin/bitcoin-core.routes';
import bitcoinSecondClient from './api/bitcoin/bitcoin-second-client';
class Server {
private wss: WebSocket.Server | undefined;
@ -215,11 +216,13 @@ class Server {
}
}
const newMempool = await bitcoinApi.$getRawMempool();
const minFeeMempool = memPool.limitGBT ? await bitcoinSecondClient.getRawMemPool() : null;
const minFeeTip = memPool.limitGBT ? await bitcoinSecondClient.getBlockCount() : -1;
const newAccelerations = await accelerationApi.$fetchAccelerations();
const numHandledBlocks = await blocks.$updateBlocks();
const pollRate = config.MEMPOOL.POLL_RATE_MS * (indexer.indexerIsRunning() ? 10 : 1);
if (numHandledBlocks === 0) {
await memPool.$updateMempool(newMempool, newAccelerations, pollRate);
await memPool.$updateMempool(newMempool, newAccelerations, minFeeMempool, minFeeTip, pollRate);
}
indexer.$run();
if (config.FIAT_PRICE.ENABLED) {

10
backend/src/mempool.interfaces.ts
View File

@ -107,6 +107,7 @@ export interface MempoolTransactionExtended extends TransactionExtended {
inputs?: number[];
lastBoosted?: number;
cpfpDirty?: boolean;
cpfpUpdated?: number;
}
export interface AuditTransaction {
@ -143,6 +144,12 @@ export interface CompactThreadTransaction {
dirty?: boolean;
}
export interface GbtCandidates {
txs: { [txid: string ]: boolean },
added: MempoolTransactionExtended[];
removed: MempoolTransactionExtended[];
}
export interface ThreadTransaction {
txid: string;
fee: number;
@ -181,6 +188,9 @@ export interface CpfpInfo {
bestDescendant?: BestDescendant | null;
descendants?: Ancestor[];
effectiveFeePerVsize?: number;
sigops?: number;
adjustedVsize?: number,
acceleration?: boolean,
}
export interface TransactionStripped {

6
docker/README.md
View File

@ -109,8 +109,6 @@ Below we list all settings from `mempool-config.json` and the corresponding over
"AUTOMATIC_BLOCK_REINDEXING": false,
"POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json",
"POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
"ADVANCED_GBT_AUDIT": false,
"ADVANCED_GBT_MEMPOOL": false,
"CPFP_INDEXING": false,
"MAX_BLOCKS_BULK_QUERY": 0,
"DISK_CACHE_BLOCK_INTERVAL": 6,
@ -142,8 +140,6 @@ Corresponding `docker-compose.yml` overrides:
MEMPOOL_AUTOMATIC_BLOCK_REINDEXING: ""
MEMPOOL_POOLS_JSON_URL: ""
MEMPOOL_POOLS_JSON_TREE_URL: ""
MEMPOOL_ADVANCED_GBT_AUDIT: ""
MEMPOOL_ADVANCED_GBT_MEMPOOL: ""
MEMPOOL_CPFP_INDEXING: ""
MEMPOOL_MAX_BLOCKS_BULK_QUERY: ""
MEMPOOL_DISK_CACHE_BLOCK_INTERVAL: ""
@ -151,8 +147,6 @@ Corresponding `docker-compose.yml` overrides:
...
```
`ADVANCED_GBT_AUDIT` AND `ADVANCED_GBT_MEMPOOL` enable a more accurate (but slower) block prediction algorithm for the block audit feature and the projected mempool-blocks respectively.
`CPFP_INDEXING` enables indexing CPFP (Child Pays For Parent) information for the last `INDEXING_BLOCKS_AMOUNT` blocks.
<br/>

3
docker/backend/mempool-config.json
View File

@ -26,9 +26,8 @@
"GOGGLES_INDEXING": __MEMPOOL_GOGGLES_INDEXING__,
"AUTOMATIC_BLOCK_REINDEXING": __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__,
"AUDIT": __MEMPOOL_AUDIT__,
"ADVANCED_GBT_AUDIT": __MEMPOOL_ADVANCED_GBT_AUDIT__,
"ADVANCED_GBT_MEMPOOL": __MEMPOOL_ADVANCED_GBT_MEMPOOL__,
"RUST_GBT": __MEMPOOL_RUST_GBT__,
"LIMIT_GBT": __MEMPOOL_LIMIT_GBT__,
"CPFP_INDEXING": __MEMPOOL_CPFP_INDEXING__,
"MAX_BLOCKS_BULK_QUERY": __MEMPOOL_MAX_BLOCKS_BULK_QUERY__,
"DISK_CACHE_BLOCK_INTERVAL": __MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__,

6
docker/backend/start.sh
View File

@ -29,9 +29,8 @@ __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__=${MEMPOOL_AUTOMATIC_BLOCK_REINDEXING:=fal
__MEMPOOL_POOLS_JSON_URL__=${MEMPOOL_POOLS_JSON_URL:=https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json}
__MEMPOOL_POOLS_JSON_TREE_URL__=${MEMPOOL_POOLS_JSON_TREE_URL:=https://api.github.com/repos/mempool/mining-pools/git/trees/master}
__MEMPOOL_AUDIT__=${MEMPOOL_AUDIT:=false}
__MEMPOOL_ADVANCED_GBT_AUDIT__=${MEMPOOL_ADVANCED_GBT_AUDIT:=false}
__MEMPOOL_ADVANCED_GBT_MEMPOOL__=${MEMPOOL_ADVANCED_GBT_MEMPOOL:=false}
__MEMPOOL_RUST_GBT__=${MEMPOOL_RUST_GBT:=false}
__MEMPOOL_LIMIT_GBT__=${MEMPOOL_LIMIT_GBT:=false}
__MEMPOOL_CPFP_INDEXING__=${MEMPOOL_CPFP_INDEXING:=false}
__MEMPOOL_MAX_BLOCKS_BULK_QUERY__=${MEMPOOL_MAX_BLOCKS_BULK_QUERY:=0}
__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__=${MEMPOOL_DISK_CACHE_BLOCK_INTERVAL:=6}
@ -189,9 +188,8 @@ sed -i "s!__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__!${__MEMPOOL_AUTOMATIC_BLOCK_REI
sed -i "s!__MEMPOOL_POOLS_JSON_URL__!${__MEMPOOL_POOLS_JSON_URL__}!g" mempool-config.json
sed -i "s!__MEMPOOL_POOLS_JSON_TREE_URL__!${__MEMPOOL_POOLS_JSON_TREE_URL__}!g" mempool-config.json
sed -i "s!__MEMPOOL_AUDIT__!${__MEMPOOL_AUDIT__}!g" mempool-config.json
sed -i "s!__MEMPOOL_ADVANCED_GBT_MEMPOOL__!${__MEMPOOL_ADVANCED_GBT_MEMPOOL__}!g" mempool-config.json
sed -i "s!__MEMPOOL_RUST_GBT__!${__MEMPOOL_RUST_GBT__}!g" mempool-config.json
sed -i "s!__MEMPOOL_ADVANCED_GBT_AUDIT__!${__MEMPOOL_ADVANCED_GBT_AUDIT__}!g" mempool-config.json
sed -i "s!__MEMPOOL_LIMIT_GBT__!${__MEMPOOL_LIMIT_GBT__}!g" mempool-config.json
sed -i "s!__MEMPOOL_CPFP_INDEXING__!${__MEMPOOL_CPFP_INDEXING__}!g" mempool-config.json
sed -i "s!__MEMPOOL_MAX_BLOCKS_BULK_QUERY__!${__MEMPOOL_MAX_BLOCKS_BULK_QUERY__}!g" mempool-config.json
sed -i "s!__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__!${__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__}!g" mempool-config.json

4
frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts
View File

@ -206,6 +206,10 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On
update(add: TransactionStripped[], remove: string[], change: { txid: string, rate: number | undefined, acc: boolean | undefined }[], direction: string = 'left', resetLayout: boolean = false): void {
if (this.scene) {
add = add.filter(tx => !this.scene.txs[tx.txid]);
remove = remove.filter(txid => this.scene.txs[txid]);
change = change.filter(tx => this.scene.txs[tx.txid]);
this.scene.update(add, remove, change, direction, resetLayout);
this.start();
this.updateSearchHighlight();

2
production/mempool-config.mainnet.json
View File

@ -15,8 +15,6 @@
"GOGGLES_INDEXING": true,
"AUDIT": true,
"CPFP_INDEXING": true,
"ADVANCED_GBT_AUDIT": true,
"ADVANCED_GBT_MEMPOOL": true,
"RUST_GBT": true,
"USE_SECOND_NODE_FOR_MINFEE": true,
"DISK_CACHE_BLOCK_INTERVAL": 1,

2
production/mempool-config.signet.json
View File

@ -9,8 +9,6 @@
"API_URL_PREFIX": "/api/v1/",
"INDEXING_BLOCKS_AMOUNT": -1,
"AUDIT": true,
"ADVANCED_GBT_AUDIT": true,
"ADVANCED_GBT_MEMPOOL": true,
"RUST_GBT": true,
"POLL_RATE_MS": 1000,
"DISK_CACHE_BLOCK_INTERVAL": 1,

2
production/mempool-config.testnet.json
View File

@ -9,8 +9,6 @@
"API_URL_PREFIX": "/api/v1/",
"INDEXING_BLOCKS_AMOUNT": -1,
"AUDIT": true,
"ADVANCED_GBT_AUDIT": true,
"ADVANCED_GBT_MEMPOOL": true,
"RUST_GBT": true,
"POLL_RATE_MS": 1000,
"DISK_CACHE_BLOCK_INTERVAL": 1,