machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

calculate total block weights inside rust gbt

Browse Source
This commit is contained in:
Mononaut 2023-06-26 14:22:10 -04:00
parent 2838b068f7
commit 5065fa42d0
No known key found for this signature in database
GPG Key ID: A3F058E41374C04E
4 changed files with 87 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
backend/rust-gbt/index.d.ts vendored
View File

@ -22,13 +22,15 @@ export class GbtGenerator {
* The result from calling the gbt function.
*
* This tuple contains the following:
* blocks: A 2D Vector of transaction IDs (u32), the inner Vecs each represent a block.
* clusters: A 2D Vector of transaction IDs representing clusters of dependent mempool transactions
* rates: A Vector of tuples containing transaction IDs (u32) and effective fee per vsize (f64)
* blocks: A 2D Vector of transaction IDs (u32), the inner Vecs each represent a block.
* block_weights: A Vector of total weights per block.
* clusters: A 2D Vector of transaction IDs representing clusters of dependent mempool transactions
* rates: A Vector of tuples containing transaction IDs (u32) and effective fee per vsize (f64)
*/
export class GbtResult {
blocks: Array<Array<number>>
blockWeights: Array<number>
clusters: Array<Array<number>>
rates: Array<Array<number>>
constructor(blocks: Array<Array<number>>, clusters: Array<Array<number>>, rates: Array<Array<number>>)
constructor(blocks: Array<Array<number>>, blockWeights: Array<number>, clusters: Array<Array<number>>, rates: Array<Array<number>>)
}

4
backend/rust-gbt/src/gbt.rs
View File

@ -62,6 +62,7 @@ pub fn gbt(mempool: &mut ThreadTransactionsMap) -> GbtResult {
let mut audit_pool: AuditPool = u32hashmap_with_capacity(STARTING_CAPACITY);
let mut mempool_stack: Vec<u32> = Vec::with_capacity(STARTING_CAPACITY);
let mut clusters: Vec<Vec<u32>> = Vec::new();
let mut block_weights: Vec<u32> = Vec::new();
info!("Initializing working structs");
for (uid, tx) in mempool {
@ -192,6 +193,7 @@ pub fn gbt(mempool: &mut ThreadTransactionsMap) -> GbtResult {
// finalize this block
if !transactions.is_empty() {
blocks.push(transactions);
block_weights.push(block_weight);
}
// reset for the next block
transactions = Vec::with_capacity(STARTING_CAPACITY);
@ -221,6 +223,7 @@ pub fn gbt(mempool: &mut ThreadTransactionsMap) -> GbtResult {
info!("add the final unbounded block if it contains any transactions");
if !transactions.is_empty() {
blocks.push(transactions);
block_weights.push(block_weight);
}
info!("make a list of dirty transactions and their new rates");
@ -238,6 +241,7 @@ pub fn gbt(mempool: &mut ThreadTransactionsMap) -> GbtResult {
GbtResult {
blocks,
block_weights,
clusters,
rates,
}

8
backend/rust-gbt/src/lib.rs
View File

@ -109,12 +109,14 @@ impl GbtGenerator {
/// The result from calling the gbt function.
///
/// This tuple contains the following:
/// blocks: A 2D Vector of transaction IDs (u32), the inner Vecs each represent a block.
/// clusters: A 2D Vector of transaction IDs representing clusters of dependent mempool transactions
/// rates: A Vector of tuples containing transaction IDs (u32) and effective fee per vsize (f64)
/// blocks: A 2D Vector of transaction IDs (u32), the inner Vecs each represent a block.
/// block_weights: A Vector of total weights per block.
/// clusters: A 2D Vector of transaction IDs representing clusters of dependent mempool transactions
/// rates: A Vector of tuples containing transaction IDs (u32) and effective fee per vsize (f64)
#[napi(constructor)]
pub struct GbtResult {
pub blocks: Vec<Vec<u32>>,
pub block_weights: Vec<u32>,
pub clusters: Vec<Vec<u32>>,
pub rates: Vec<Vec<f64>>, // Tuples not supported. u32 fits inside f64
}

134
backend/src/api/mempool-blocks.ts
View File

@ -262,7 +262,7 @@ class MempoolBlocks {
// clean up thread error listener
this.txSelectionWorker?.removeListener('error', threadErrorListener);
const processed = this.processBlockTemplates(newMempool, blocks, rates, clusters, saveResults);
const processed = this.processBlockTemplates(newMempool, blocks, null, rates, clusters, saveResults);
logger.debug(`makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`);
@ -318,7 +318,7 @@ class MempoolBlocks {
// clean up thread error listener
this.txSelectionWorker?.removeListener('error', threadErrorListener);
this.processBlockTemplates(newMempool, blocks, rates, clusters, saveResults);
this.processBlockTemplates(newMempool, blocks, null, rates, clusters, saveResults);
logger.debug(`updateBlockTemplates completed in ${(Date.now() - start) / 1000} seconds`);
} catch (e) {
logger.err('updateBlockTemplates failed. ' + (e instanceof Error ? e.message : e));
@ -349,13 +349,13 @@ class MempoolBlocks {
// run the block construction algorithm in a separate thread, and wait for a result
const rustGbt = saveResults ? this.rustGbtGenerator : new GbtGenerator();
try {
const { blocks, rates, clusters } = this.convertNapiResultTxids(
const { blocks, blockWeights, rates, clusters } = this.convertNapiResultTxids(
await rustGbt.make(new Uint8Array(mempoolBuffer)),
);
if (saveResults) {
this.rustInitialized = true;
}
const processed = this.processBlockTemplates(newMempool, blocks, rates, clusters, saveResults);
const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, saveResults);
logger.debug(`RUST makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`);
return processed;
} catch (e) {
@ -395,7 +395,7 @@ class MempoolBlocks {
// run the block construction algorithm in a separate thread, and wait for a result
try {
const { blocks, rates, clusters } = this.convertNapiResultTxids(
const { blocks, blockWeights, rates, clusters } = this.convertNapiResultTxids(
await this.rustGbtGenerator.update(
new Uint8Array(addedBuffer),
new Uint8Array(removedBuffer),
@ -406,7 +406,7 @@ class MempoolBlocks {
if (expectedMempoolSize !== actualMempoolSize) {
throw new Error('GBT returned wrong number of transactions, cache is probably out of sync');
} else {
this.processBlockTemplates(newMempool, blocks, rates, clusters, true);
this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, true);
}
logger.debug(`RUST updateBlockTemplates completed in ${(Date.now() - start)/1000} seconds`);
} catch (e) {
@ -415,10 +415,11 @@ class MempoolBlocks {
}
}
private processBlockTemplates(mempool, blocks: string[][], rates: { [root: string]: number }, clusters: { [root: string]: string[] }, saveResults): MempoolBlockWithTransactions[] {
private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: { [root: string]: number }, clusters: { [root: string]: string[] }, saveResults): MempoolBlockWithTransactions[] {
for (const txid of Object.keys(rates)) {
if (txid in mempool) {
mempool[txid].effectiveFeePerVsize = rates[txid];
mempool[txid].cpfpChecked = false;
}
}
@ -426,62 +427,15 @@ class MempoolBlocks {
let stackWeight;
let feeStatsCalculator: OnlineFeeStatsCalculator | void;
if (hasBlockStack) {
stackWeight = blocks[blocks.length - 1].reduce((total, tx) => total + (mempool[tx]?.weight || 0), 0);
if (blockWeights && blockWeights[7] !== null) {
stackWeight = blockWeights[7];
} else {
stackWeight = blocks[blocks.length - 1].reduce((total, tx) => total + (mempool[tx]?.weight || 0), 0);
}
hasBlockStack = stackWeight > config.MEMPOOL.BLOCK_WEIGHT_UNITS;
feeStatsCalculator = new OnlineFeeStatsCalculator(stackWeight, 0.5, [10, 20, 30, 40, 50, 60, 70, 80, 90]);
}
const readyBlocks: { transactionIds, transactions, totalSize, totalWeight, totalFees, feeStats }[] = [];
const sizeLimit = (config.MEMPOOL.BLOCK_WEIGHT_UNITS / 4) * 1.2;
// update this thread's mempool with the results
for (let blockIndex = 0; blockIndex < blocks.length; blockIndex++) {
const block: string[] = blocks[blockIndex];
let txid: string;
let mempoolTx: MempoolTransactionExtended;
let totalSize = 0;
let totalVsize = 0;
let totalWeight = 0;
let totalFees = 0;
const transactions: MempoolTransactionExtended[] = [];
for (let txIndex = 0; txIndex < block.length; txIndex++) {
txid = block[txIndex];
if (txid) {
mempoolTx = mempool[txid];
// save position in projected blocks
mempoolTx.position = {
block: blockIndex,
vsize: totalVsize + (mempoolTx.vsize / 2),
};
mempoolTx.ancestors = [];
mempoolTx.descendants = [];
mempoolTx.bestDescendant = null;
mempoolTx.cpfpChecked = true;
// online calculation of stack-of-blocks fee stats
if (hasBlockStack && blockIndex === blocks.length - 1 && feeStatsCalculator) {
feeStatsCalculator.processNext(mempoolTx);
}
totalSize += mempoolTx.size;
totalVsize += mempoolTx.vsize;
totalWeight += mempoolTx.weight;
totalFees += mempoolTx.fee;
if (totalVsize <= sizeLimit) {
transactions.push(mempoolTx);
}
}
}
readyBlocks.push({
transactionIds: block,
transactions,
totalSize,
totalWeight,
totalFees,
feeStats: (hasBlockStack && blockIndex === blocks.length - 1 && feeStatsCalculator) ? feeStatsCalculator.getRawFeeStats() : undefined,
});
}
for (const cluster of Object.values(clusters)) {
for (const memberTxid of cluster) {
if (memberTxid in mempool) {
@ -508,10 +462,66 @@ class MempoolBlocks {
mempoolTx.ancestors = ancestors;
mempoolTx.descendants = descendants;
mempoolTx.bestDescendant = null;
mempoolTx.cpfpChecked = true;
}
}
}
const readyBlocks: { transactionIds, transactions, totalSize, totalWeight, totalFees, feeStats }[] = [];
const sizeLimit = (config.MEMPOOL.BLOCK_WEIGHT_UNITS / 4) * 1.2;
// update this thread's mempool with the results
for (let blockIndex = 0; blockIndex < blocks.length; blockIndex++) {
const block: string[] = blocks[blockIndex];
let mempoolTx: MempoolTransactionExtended;
let totalSize = 0;
let totalVsize = 0;
let totalWeight = 0;
let totalFees = 0;
const transactions: MempoolTransactionExtended[] = [];
for (const txid of block) {
if (txid) {
mempoolTx = mempool[txid];
// save position in projected blocks
mempoolTx.position = {
block: blockIndex,
vsize: totalVsize + (mempoolTx.vsize / 2),
};
if (!mempoolTx.cpfpChecked) {
if (mempoolTx.ancestors?.length) {
mempoolTx.ancestors = [];
}
if (mempoolTx.descendants?.length) {
mempoolTx.descendants = [];
}
mempoolTx.bestDescendant = null;
mempoolTx.cpfpChecked = true;
}
// online calculation of stack-of-blocks fee stats
if (hasBlockStack && blockIndex === blocks.length - 1 && feeStatsCalculator) {
feeStatsCalculator.processNext(mempoolTx);
}
totalSize += mempoolTx.size;
totalVsize += mempoolTx.vsize;
totalWeight += mempoolTx.weight;
totalFees += mempoolTx.fee;
if (totalVsize <= sizeLimit) {
transactions.push(mempoolTx);
}
}
}
readyBlocks.push({
transactionIds: block,
transactions,
totalSize,
totalWeight,
totalFees,
feeStats: (hasBlockStack && blockIndex === blocks.length - 1 && feeStatsCalculator) ? feeStatsCalculator.getRawFeeStats() : undefined,
});
}
const mempoolBlocks = readyBlocks.map((b) => {
return this.dataToMempoolBlocks(b.transactionIds, b.transactions, b.totalSize, b.totalWeight, b.totalFees, b.feeStats);
});
@ -595,8 +605,8 @@ class MempoolBlocks {
return { blocks: convertedBlocks, rates: convertedRates, clusters: convertedClusters } as { blocks: string[][], rates: { [root: string]: number }, clusters: { [root: string]: string[] }};
}
private convertNapiResultTxids({ blocks, rates, clusters }: { blocks: number[][], rates: number[][], clusters: number[][]})
: { blocks: string[][], rates: { [root: string]: number }, clusters: { [root: string]: string[] }} {
private convertNapiResultTxids({ blocks, blockWeights, rates, clusters }: { blocks: number[][], blockWeights: number[], rates: number[][], clusters: number[][]})
: { blocks: string[][], blockWeights: number[], rates: { [root: string]: number }, clusters: { [root: string]: string[] }} {
const rateMap = new Map<number, number>();
const clusterMap = new Map<number, number[]>();
for (const rate of rates) {
@ -634,7 +644,7 @@ class MempoolBlocks {
throw new Error('GBT returned a cluster rooted in a transaction with unknown uid');
}
}
return { blocks: convertedBlocks, rates: convertedRates, clusters: convertedClusters } as { blocks: string[][], rates: { [root: string]: number }, clusters: { [root: string]: string[] }};
return { blocks: convertedBlocks, blockWeights, rates: convertedRates, clusters: convertedClusters } as { blocks: string[][], blockWeights: number[], rates: { [root: string]: number }, clusters: { [root: string]: string[] }};
}
private mempoolToArrayBuffer(txs: MempoolTransactionExtended[], mempool: { [txid: string]: MempoolTransactionExtended }): ArrayBuffer {