Merge pull request #4554 from mempool/mononaut/unmineable-txs

Handle unmineable transactions in GBT implementations
This commit is contained in:
softsimon 2024-01-10 00:18:41 +07:00 committed by GitHub
commit aad288c0d9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 98 additions and 74 deletions

View File

@ -45,5 +45,6 @@ export class GbtResult {
blockWeights: Array<number> blockWeights: Array<number>
clusters: Array<Array<number>> clusters: Array<Array<number>>
rates: Array<Array<number>> rates: Array<Array<number>>
constructor(blocks: Array<Array<number>>, blockWeights: Array<number>, clusters: Array<Array<number>>, rates: Array<Array<number>>) overflow: Array<number>
constructor(blocks: Array<Array<number>>, blockWeights: Array<number>, clusters: Array<Array<number>>, rates: Array<Array<number>>, overflow: Array<number>)
} }

View File

@ -60,6 +60,7 @@ pub fn gbt(mempool: &mut ThreadTransactionsMap, accelerations: &[ThreadAccelerat
indexed_accelerations[acceleration.uid as usize] = Some(acceleration); indexed_accelerations[acceleration.uid as usize] = Some(acceleration);
} }
info!("Initializing working vecs with uid capacity for {}", max_uid + 1);
let mempool_len = mempool.len(); let mempool_len = mempool.len();
let mut audit_pool: AuditPool = Vec::with_capacity(max_uid + 1); let mut audit_pool: AuditPool = Vec::with_capacity(max_uid + 1);
audit_pool.resize(max_uid + 1, None); audit_pool.resize(max_uid + 1, None);
@ -127,8 +128,8 @@ pub fn gbt(mempool: &mut ThreadTransactionsMap, accelerations: &[ThreadAccelerat
let next_from_stack = next_valid_from_stack(&mut mempool_stack, &audit_pool); let next_from_stack = next_valid_from_stack(&mut mempool_stack, &audit_pool);
let next_from_queue = next_valid_from_queue(&mut modified, &audit_pool); let next_from_queue = next_valid_from_queue(&mut modified, &audit_pool);
if next_from_stack.is_none() && next_from_queue.is_none() { if next_from_stack.is_none() && next_from_queue.is_none() {
continue; info!("No transactions left! {:#?} in overflow", overflow.len());
} } else {
let (next_tx, from_stack) = match (next_from_stack, next_from_queue) { let (next_tx, from_stack) = match (next_from_stack, next_from_queue) {
(Some(stack_tx), Some(queue_tx)) => match queue_tx.cmp(stack_tx) { (Some(stack_tx), Some(queue_tx)) => match queue_tx.cmp(stack_tx) {
std::cmp::Ordering::Less => (stack_tx, true), std::cmp::Ordering::Less => (stack_tx, true),
@ -196,6 +197,7 @@ pub fn gbt(mempool: &mut ThreadTransactionsMap, accelerations: &[ThreadAccelerat
failures = 0; failures = 0;
} }
}
// this block is full // this block is full
let exceeded_package_tries = let exceeded_package_tries =
@ -203,10 +205,14 @@ pub fn gbt(mempool: &mut ThreadTransactionsMap, accelerations: &[ThreadAccelerat
let queue_is_empty = mempool_stack.is_empty() && modified.is_empty(); let queue_is_empty = mempool_stack.is_empty() && modified.is_empty();
if (exceeded_package_tries || queue_is_empty) && blocks.len() < (MAX_BLOCKS - 1) { if (exceeded_package_tries || queue_is_empty) && blocks.len() < (MAX_BLOCKS - 1) {
// finalize this block // finalize this block
if !transactions.is_empty() { if transactions.is_empty() {
info!("trying to push an empty block! breaking loop! mempool {:#?} | modified {:#?} | overflow {:#?}", mempool_stack.len(), modified.len(), overflow.len());
break;
}
blocks.push(transactions); blocks.push(transactions);
block_weights.push(block_weight); block_weights.push(block_weight);
}
// reset for the next block // reset for the next block
transactions = Vec::with_capacity(initial_txes_per_block); transactions = Vec::with_capacity(initial_txes_per_block);
block_weight = BLOCK_RESERVED_WEIGHT; block_weight = BLOCK_RESERVED_WEIGHT;
@ -265,6 +271,7 @@ pub fn gbt(mempool: &mut ThreadTransactionsMap, accelerations: &[ThreadAccelerat
block_weights, block_weights,
clusters, clusters,
rates, rates,
overflow,
} }
} }

View File

@ -133,6 +133,7 @@ pub struct GbtResult {
pub block_weights: Vec<u32>, pub block_weights: Vec<u32>,
pub clusters: Vec<Vec<u32>>, pub clusters: Vec<Vec<u32>>,
pub rates: Vec<Vec<f64>>, // Tuples not supported. u32 fits inside f64 pub rates: Vec<Vec<f64>>, // Tuples not supported. u32 fits inside f64
pub overflow: Vec<u32>,
} }
/// All on another thread, this runs an arbitrary task in between /// All on another thread, this runs an arbitrary task in between

View File

@ -368,12 +368,15 @@ class MempoolBlocks {
// run the block construction algorithm in a separate thread, and wait for a result // run the block construction algorithm in a separate thread, and wait for a result
const rustGbt = saveResults ? this.rustGbtGenerator : new GbtGenerator(); const rustGbt = saveResults ? this.rustGbtGenerator : new GbtGenerator();
try { try {
const { blocks, blockWeights, rates, clusters } = this.convertNapiResultTxids( const { blocks, blockWeights, rates, clusters, overflow } = this.convertNapiResultTxids(
await rustGbt.make(Object.values(newMempool) as RustThreadTransaction[], convertedAccelerations as RustThreadAcceleration[], this.nextUid), await rustGbt.make(Object.values(newMempool) as RustThreadTransaction[], convertedAccelerations as RustThreadAcceleration[], this.nextUid),
); );
if (saveResults) { if (saveResults) {
this.rustInitialized = true; this.rustInitialized = true;
} }
const mempoolSize = Object.keys(newMempool).length;
const resultMempoolSize = blocks.reduce((total, block) => total + block.length, 0) + overflow.length;
logger.debug(`RUST updateBlockTemplates returned ${resultMempoolSize} txs out of ${mempoolSize} in the mempool, ${overflow.length} were unmineable`);
const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, accelerations, accelerationPool, saveResults); const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, accelerations, accelerationPool, saveResults);
logger.debug(`RUST makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); logger.debug(`RUST makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`);
return processed; return processed;
@ -424,7 +427,7 @@ class MempoolBlocks {
// run the block construction algorithm in a separate thread, and wait for a result // run the block construction algorithm in a separate thread, and wait for a result
try { try {
const { blocks, blockWeights, rates, clusters } = this.convertNapiResultTxids( const { blocks, blockWeights, rates, clusters, overflow } = this.convertNapiResultTxids(
await this.rustGbtGenerator.update( await this.rustGbtGenerator.update(
added as RustThreadTransaction[], added as RustThreadTransaction[],
removedUids, removedUids,
@ -432,7 +435,8 @@ class MempoolBlocks {
this.nextUid, this.nextUid,
), ),
); );
const resultMempoolSize = blocks.reduce((total, block) => total + block.length, 0); const resultMempoolSize = blocks.reduce((total, block) => total + block.length, 0) + overflow.length;
logger.debug(`RUST updateBlockTemplates returned ${resultMempoolSize} txs out of ${mempoolSize} in the mempool, ${overflow.length} were unmineable`);
if (mempoolSize !== resultMempoolSize) { if (mempoolSize !== resultMempoolSize) {
throw new Error('GBT returned wrong number of transactions , cache is probably out of sync'); throw new Error('GBT returned wrong number of transactions , cache is probably out of sync');
} else { } else {
@ -658,8 +662,8 @@ class MempoolBlocks {
return { blocks: convertedBlocks, rates: convertedRates, clusters: convertedClusters } as { blocks: string[][], rates: { [root: string]: number }, clusters: { [root: string]: string[] }}; return { blocks: convertedBlocks, rates: convertedRates, clusters: convertedClusters } as { blocks: string[][], rates: { [root: string]: number }, clusters: { [root: string]: string[] }};
} }
private convertNapiResultTxids({ blocks, blockWeights, rates, clusters }: GbtResult) private convertNapiResultTxids({ blocks, blockWeights, rates, clusters, overflow }: GbtResult)
: { blocks: string[][], blockWeights: number[], rates: [string, number][], clusters: string[][] } { : { blocks: string[][], blockWeights: number[], rates: [string, number][], clusters: string[][], overflow: string[] } {
const convertedBlocks: string[][] = blocks.map(block => block.map(uid => { const convertedBlocks: string[][] = blocks.map(block => block.map(uid => {
const txid = this.uidMap.get(uid); const txid = this.uidMap.get(uid);
if (txid !== undefined) { if (txid !== undefined) {
@ -677,7 +681,15 @@ class MempoolBlocks {
for (const cluster of clusters) { for (const cluster of clusters) {
convertedClusters.push(cluster.map(uid => this.uidMap.get(uid)) as string[]); convertedClusters.push(cluster.map(uid => this.uidMap.get(uid)) as string[]);
} }
return { blocks: convertedBlocks, blockWeights, rates: convertedRates, clusters: convertedClusters }; const convertedOverflow: string[] = overflow.map(uid => {
const txid = this.uidMap.get(uid);
if (txid !== undefined) {
return txid;
} else {
throw new Error('GBT returned an unmineable transaction with unknown uid');
}
});
return { blocks: convertedBlocks, blockWeights, rates: convertedRates, clusters: convertedClusters, overflow: convertedOverflow };
} }
} }

View File

@ -173,10 +173,13 @@ function makeBlockTemplates(mempool: Map<number, CompactThreadTransaction>)
// this block is full // this block is full
const exceededPackageTries = failures > 1000 && blockWeight > (config.MEMPOOL.BLOCK_WEIGHT_UNITS - 4000); const exceededPackageTries = failures > 1000 && blockWeight > (config.MEMPOOL.BLOCK_WEIGHT_UNITS - 4000);
const queueEmpty = top >= mempoolArray.length && modified.isEmpty(); const queueEmpty = top >= mempoolArray.length && modified.isEmpty();
if ((exceededPackageTries || queueEmpty) && blocks.length < 7) { if ((exceededPackageTries || queueEmpty) && blocks.length < 7) {
// construct this block // construct this block
if (transactions.length) { if (transactions.length) {
blocks.push(transactions.map(t => t.uid)); blocks.push(transactions.map(t => t.uid));
} else {
break;
} }
// reset for the next block // reset for the next block
transactions = []; transactions = [];