From b44b790e28bd0b1cf2f61a49881cde2b544b2922 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sat, 22 Jun 2024 04:10:32 +0000 Subject: [PATCH 1/2] configurable block size & count in rust gbt --- backend/src/__tests__/gbt/gbt-tests.ts | 2 +- backend/src/api/mempool-blocks.ts | 6 +++--- rust/gbt/src/gbt.rs | 18 ++++++++++------- rust/gbt/src/lib.rs | 27 +++++++++++++++++++++++--- 4 files changed, 39 insertions(+), 14 deletions(-) diff --git a/backend/src/__tests__/gbt/gbt-tests.ts b/backend/src/__tests__/gbt/gbt-tests.ts index 8a3995f71..561963aaa 100644 --- a/backend/src/__tests__/gbt/gbt-tests.ts +++ b/backend/src/__tests__/gbt/gbt-tests.ts @@ -13,7 +13,7 @@ const vectorBuffer: Buffer = fs.readFileSync(path.join(__dirname, './', './test- describe('Rust GBT', () => { test('should produce the same template as getBlockTemplate from Bitcoin Core', async () => { - const rustGbt = new GbtGenerator(); + const rustGbt = new GbtGenerator(4_000_000, 8); const { mempool, maxUid } = mempoolFromArrayBuffer(vectorBuffer.buffer); const result = await rustGbt.make(mempool, [], maxUid); diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 55f613c0d..3c1c434bd 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -16,7 +16,7 @@ class MempoolBlocks { private mempoolBlockDeltas: MempoolBlockDelta[] = []; private txSelectionWorker: Worker | null = null; private rustInitialized: boolean = false; - private rustGbtGenerator: GbtGenerator = new GbtGenerator(); + private rustGbtGenerator: GbtGenerator = new GbtGenerator(config.MEMPOOL.BLOCK_WEIGHT_UNITS, config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT); private nextUid: number = 1; private uidMap: Map = new Map(); // map short numerical uids to full txids @@ -230,7 +230,7 @@ class MempoolBlocks { private resetRustGbt(): void { this.rustInitialized = false; - this.rustGbtGenerator = new GbtGenerator(); + this.rustGbtGenerator = new GbtGenerator(config.MEMPOOL.BLOCK_WEIGHT_UNITS, config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT); } public async $rustMakeBlockTemplates(txids: string[], newMempool: { [txid: string]: MempoolTransactionExtended }, candidates: GbtCandidates | undefined, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise { @@ -262,7 +262,7 @@ class MempoolBlocks { }); // run the block construction algorithm in a separate thread, and wait for a result - const rustGbt = saveResults ? this.rustGbtGenerator : new GbtGenerator(); + const rustGbt = saveResults ? this.rustGbtGenerator : new GbtGenerator(config.MEMPOOL.BLOCK_WEIGHT_UNITS, config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT); try { const { blocks, blockWeights, rates, clusters, overflow } = this.convertNapiResultTxids( await rustGbt.make(transactions as RustThreadTransaction[], convertedAccelerations as RustThreadAcceleration[], this.nextUid), diff --git a/rust/gbt/src/gbt.rs b/rust/gbt/src/gbt.rs index 38bf826a6..b8d3eb65e 100644 --- a/rust/gbt/src/gbt.rs +++ b/rust/gbt/src/gbt.rs @@ -8,11 +8,9 @@ use crate::{ GbtResult, ThreadTransactionsMap, thread_acceleration::ThreadAcceleration, }; -const MAX_BLOCK_WEIGHT_UNITS: u32 = 4_000_000 - 4_000; const BLOCK_SIGOPS: u32 = 80_000; const BLOCK_RESERVED_WEIGHT: u32 = 4_000; const BLOCK_RESERVED_SIGOPS: u32 = 400; -const MAX_BLOCKS: usize = 8; type AuditPool = Vec>>; type ModifiedQueue = PriorityQueue; @@ -53,7 +51,13 @@ impl Ord for TxPriority { // TODO: Make gbt smaller to fix these lints. #[allow(clippy::too_many_lines)] #[allow(clippy::cognitive_complexity)] -pub fn gbt(mempool: &mut ThreadTransactionsMap, accelerations: &[ThreadAcceleration], max_uid: usize) -> GbtResult { +pub fn gbt( + mempool: &mut ThreadTransactionsMap, + accelerations: &[ThreadAcceleration], + max_uid: usize, + max_block_weight: u32, + max_blocks: usize, +) -> GbtResult { let mut indexed_accelerations = Vec::with_capacity(max_uid + 1); indexed_accelerations.resize(max_uid + 1, None); for acceleration in accelerations { @@ -146,9 +150,9 @@ pub fn gbt(mempool: &mut ThreadTransactionsMap, accelerations: &[ThreadAccelerat modified.pop(); } - if blocks.len() < (MAX_BLOCKS - 1) + if blocks.len() < (max_blocks - 1) && ((block_weight + (4 * next_tx.ancestor_sigop_adjusted_vsize()) - >= MAX_BLOCK_WEIGHT_UNITS) + >= max_block_weight) || (block_sigops + next_tx.ancestor_sigops() > BLOCK_SIGOPS)) { // hold this package in an overflow list while we check for smaller options @@ -201,9 +205,9 @@ pub fn gbt(mempool: &mut ThreadTransactionsMap, accelerations: &[ThreadAccelerat // this block is full let exceeded_package_tries = - failures > 1000 && block_weight > (MAX_BLOCK_WEIGHT_UNITS - BLOCK_RESERVED_WEIGHT); + failures > 1000 && block_weight > (max_block_weight - BLOCK_RESERVED_WEIGHT); let queue_is_empty = mempool_stack.is_empty() && modified.is_empty(); - if (exceeded_package_tries || queue_is_empty) && blocks.len() < (MAX_BLOCKS - 1) { + if (exceeded_package_tries || queue_is_empty) && blocks.len() < (max_blocks - 1) { // finalize this block if transactions.is_empty() { info!("trying to push an empty block! breaking loop! mempool {:#?} | modified {:#?} | overflow {:#?}", mempool_stack.len(), modified.len(), overflow.len()); diff --git a/rust/gbt/src/lib.rs b/rust/gbt/src/lib.rs index edc9714ee..1568990e4 100644 --- a/rust/gbt/src/lib.rs +++ b/rust/gbt/src/lib.rs @@ -35,6 +35,8 @@ type ThreadTransactionsMap = HashMap; #[napi] pub struct GbtGenerator { thread_transactions: Arc>, + max_block_weight: u32, + max_blocks: usize, } #[napi::module_init] @@ -65,10 +67,12 @@ impl GbtGenerator { #[napi(constructor)] #[allow(clippy::new_without_default)] #[must_use] - pub fn new() -> Self { + pub fn new(max_block_weight: u32, max_blocks: u32) -> Self { debug!("Created new GbtGenerator"); Self { thread_transactions: Arc::new(Mutex::new(u32hashmap_with_capacity(STARTING_CAPACITY))), + max_block_weight, + max_blocks: max_blocks as usize, } } @@ -76,12 +80,19 @@ impl GbtGenerator { /// /// Rejects if the thread panics or if the Mutex is poisoned. #[napi] - pub async fn make(&self, mempool: Vec, accelerations: Vec, max_uid: u32) -> Result { + pub async fn make( + &self, + mempool: Vec, + accelerations: Vec, + max_uid: u32, + ) -> Result { trace!("make: Current State {:#?}", self.thread_transactions); run_task( Arc::clone(&self.thread_transactions), accelerations, max_uid as usize, + self.max_block_weight, + self.max_blocks, move |map| { for tx in mempool { map.insert(tx.uid, tx); @@ -107,6 +118,8 @@ impl GbtGenerator { Arc::clone(&self.thread_transactions), accelerations, max_uid as usize, + self.max_block_weight, + self.max_blocks, move |map| { for tx in new_txs { map.insert(tx.uid, tx); @@ -149,6 +162,8 @@ async fn run_task( thread_transactions: Arc>, accelerations: Vec, max_uid: usize, + max_block_weight: u32, + max_blocks: usize, callback: F, ) -> Result where @@ -166,7 +181,13 @@ where callback(&mut map); info!("Starting gbt algorithm for {} elements...", map.len()); - let result = gbt::gbt(&mut map, &accelerations, max_uid); + let result = gbt::gbt( + &mut map, + &accelerations, + max_uid, + max_block_weight, + max_blocks as usize, + ); info!("Finished gbt algorithm for {} elements...", map.len()); debug!( From fa9a8bdba808de9e8390a148433d8ceabfc66742 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sat, 22 Jun 2024 04:30:36 +0000 Subject: [PATCH 2/2] rust gbt restore 4kWU reserve --- rust/gbt/src/gbt.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/gbt/src/gbt.rs b/rust/gbt/src/gbt.rs index b8d3eb65e..6c9e81f5f 100644 --- a/rust/gbt/src/gbt.rs +++ b/rust/gbt/src/gbt.rs @@ -152,7 +152,7 @@ pub fn gbt( if blocks.len() < (max_blocks - 1) && ((block_weight + (4 * next_tx.ancestor_sigop_adjusted_vsize()) - >= max_block_weight) + >= max_block_weight - 4_000) || (block_sigops + next_tx.ancestor_sigops() > BLOCK_SIGOPS)) { // hold this package in an overflow list while we check for smaller options @@ -205,7 +205,7 @@ pub fn gbt( // this block is full let exceeded_package_tries = - failures > 1000 && block_weight > (max_block_weight - BLOCK_RESERVED_WEIGHT); + failures > 1000 && block_weight > (max_block_weight - 4_000 - BLOCK_RESERVED_WEIGHT); let queue_is_empty = mempool_stack.is_empty() && modified.is_empty(); if (exceeded_package_tries || queue_is_empty) && blocks.len() < (max_blocks - 1) { // finalize this block