From 52bb8b4a4d2a8d03da22b997f6e49c0827c0cd8a Mon Sep 17 00:00:00 2001 From: Mononaut Date: Fri, 23 Jun 2023 16:42:58 -0400 Subject: [PATCH] Rust GBT proof of concept --- backend/package.json | 5 +- backend/rust-gbt/.gitignore | 5 + backend/rust-gbt/Cargo.lock | 156 +++++++++++ backend/rust-gbt/Cargo.toml | 21 ++ backend/rust-gbt/README.md | 121 ++++++++ backend/rust-gbt/package-lock.json | 25 ++ backend/rust-gbt/package.json | 17 ++ backend/rust-gbt/src/audit_transaction.rs | 53 ++++ backend/rust-gbt/src/gbt.rs | 312 +++++++++++++++++++++ backend/rust-gbt/src/lib.rs | 75 +++++ backend/rust-gbt/src/thread_transaction.rs | 45 +++ backend/src/api/mempool-blocks.ts | 109 ++++++- backend/src/api/websocket-handler.ts | 6 +- backend/src/mempool.interfaces.ts | 2 +- 14 files changed, 943 insertions(+), 9 deletions(-) create mode 100644 backend/rust-gbt/.gitignore create mode 100644 backend/rust-gbt/Cargo.lock create mode 100644 backend/rust-gbt/Cargo.toml create mode 100644 backend/rust-gbt/README.md create mode 100644 backend/rust-gbt/package-lock.json create mode 100644 backend/rust-gbt/package.json create mode 100644 backend/rust-gbt/src/audit_transaction.rs create mode 100644 backend/rust-gbt/src/gbt.rs create mode 100644 backend/rust-gbt/src/lib.rs create mode 100644 backend/rust-gbt/src/thread_transaction.rs diff --git a/backend/package.json b/backend/package.json index 42b82717f..5337ff508 100644 --- a/backend/package.json +++ b/backend/package.json @@ -22,7 +22,7 @@ "main": "index.ts", "scripts": { "tsc": "./node_modules/typescript/bin/tsc -p tsconfig.build.json", - "build": "npm run tsc && npm run create-resources", + "build": "npm run build-rust && npm run tsc && npm run create-resources", "create-resources": "cp ./src/tasks/price-feeds/mtgox-weekly.json ./dist/tasks && node dist/api/fetch-version.js", "package": "npm run build && rm -rf package && mv dist package && mv node_modules package && npm run package-rm-build-deps", "package-rm-build-deps": "(cd package/node_modules; rm -r typescript @typescript-eslint)", @@ -33,7 +33,8 @@ "test": "./node_modules/.bin/jest --coverage", "lint": "./node_modules/.bin/eslint . --ext .ts", "lint:fix": "./node_modules/.bin/eslint . --ext .ts --fix", - "prettier": "./node_modules/.bin/prettier --write \"src/**/*.{js,ts}\"" + "prettier": "./node_modules/.bin/prettier --write \"src/**/*.{js,ts}\"", + "build-rust": "cd rust-gbt && npm install" }, "dependencies": { "@babel/core": "^7.21.3", diff --git a/backend/rust-gbt/.gitignore b/backend/rust-gbt/.gitignore new file mode 100644 index 000000000..6ca71fb5f --- /dev/null +++ b/backend/rust-gbt/.gitignore @@ -0,0 +1,5 @@ +target +index.node +**/node_modules +**/.DS_Store +npm-debug.log* diff --git a/backend/rust-gbt/Cargo.lock b/backend/rust-gbt/Cargo.lock new file mode 100644 index 000000000..0407bcc14 --- /dev/null +++ b/backend/rust-gbt/Cargo.lock @@ -0,0 +1,156 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "gbt" +version = "0.1.0" +dependencies = [ + "neon", +] + +[[package]] +name = "libloading" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "351a32417a12d5f7e82c368a66781e307834dae04c6ce0cd4456d52989229883" +dependencies = [ + "cfg-if", + "winapi", +] + +[[package]] +name = "neon" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28e15415261d880aed48122e917a45e87bb82cf0260bb6db48bbab44b7464373" +dependencies = [ + "neon-build", + "neon-macros", + "neon-runtime", + "semver", + "smallvec", +] + +[[package]] +name = "neon-build" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bac98a702e71804af3dacfde41edde4a16076a7bbe889ae61e56e18c5b1c811" + +[[package]] +name = "neon-macros" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7288eac8b54af7913c60e0eb0e2a7683020dffa342ab3fd15e28f035ba897cf" +dependencies = [ + "quote", + "syn", + "syn-mid", +] + +[[package]] +name = "neon-runtime" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4676720fa8bb32c64c3d9f49c47a47289239ec46b4bdb66d0913cc512cb0daca" +dependencies = [ + "cfg-if", + "libloading", + "smallvec", +] + +[[package]] +name = "proc-macro2" +version = "1.0.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec2b086b7a862cf4de201096214fa870344cf922b2b30c167badb3af3195406" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + +[[package]] +name = "smallvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn-mid" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baa8e7560a164edb1621a55d18a0c59abf49d360f47aa7b821061dd7eea7fac9" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "unicode-ident" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/backend/rust-gbt/Cargo.toml b/backend/rust-gbt/Cargo.toml new file mode 100644 index 000000000..abd55fbd8 --- /dev/null +++ b/backend/rust-gbt/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "gbt" +version = "0.1.0" +description = "An inefficient re-implementation of the getBlockTemplate algorithm in Rust" +authors = ["mononaut"] +edition = "2018" +exclude = ["index.node"] + +[lib] +crate-type = ["cdylib"] + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +priority-queue = "1.3.2" +bytes = "1.4.0" + +[dependencies.neon] +version = "0.10" +default-features = false +features = ["napi-6","channel-api"] diff --git a/backend/rust-gbt/README.md b/backend/rust-gbt/README.md new file mode 100644 index 000000000..f437527d8 --- /dev/null +++ b/backend/rust-gbt/README.md @@ -0,0 +1,121 @@ +# gbt + +**gbt:** rust implementation of the getBlockTemplate algorithm + +This project was bootstrapped by [create-neon](https://www.npmjs.com/package/create-neon). + +## Installing gbt + +Installing gbt requires a [supported version of Node and Rust](https://github.com/neon-bindings/neon#platform-support). + +You can install the project with npm. In the project directory, run: + +```sh +$ npm install +``` + +This fully installs the project, including installing any dependencies and running the build. + +## Building gbt + +If you have already installed the project and only want to run the build, run: + +```sh +$ npm run build +``` + +This command uses the [cargo-cp-artifact](https://github.com/neon-bindings/cargo-cp-artifact) utility to run the Rust build and copy the built library into `./index.node`. + +## Exploring gbt + +After building gbt, you can explore its exports at the Node REPL: + +```sh +$ npm install +$ node +> require('.').hello() +"hello node" +``` + +## Available Scripts + +In the project directory, you can run: + +### `npm install` + +Installs the project, including running `npm run build`. + +### `npm build` + +Builds the Node addon (`index.node`) from source. + +Additional [`cargo build`](https://doc.rust-lang.org/cargo/commands/cargo-build.html) arguments may be passed to `npm build` and `npm build-*` commands. For example, to enable a [cargo feature](https://doc.rust-lang.org/cargo/reference/features.html): + +``` +npm run build -- --feature=beetle +``` + +#### `npm build-debug` + +Alias for `npm build`. + +#### `npm build-release` + +Same as [`npm build`](#npm-build) but, builds the module with the [`release`](https://doc.rust-lang.org/cargo/reference/profiles.html#release) profile. Release builds will compile slower, but run faster. + +### `npm test` + +Runs the unit tests by calling `cargo test`. You can learn more about [adding tests to your Rust code](https://doc.rust-lang.org/book/ch11-01-writing-tests.html) from the [Rust book](https://doc.rust-lang.org/book/). + +## Project Layout + +The directory structure of this project is: + +``` +gbt/ +├── Cargo.toml +├── README.md +├── index.node +├── package.json +├── src/ +| └── lib.rs +└── target/ +``` + +### Cargo.toml + +The Cargo [manifest file](https://doc.rust-lang.org/cargo/reference/manifest.html), which informs the `cargo` command. + +### README.md + +This file. + +### index.node + +The Node addon—i.e., a binary Node module—generated by building the project. This is the main module for this package, as dictated by the `"main"` key in `package.json`. + +Under the hood, a [Node addon](https://nodejs.org/api/addons.html) is a [dynamically-linked shared object](https://en.wikipedia.org/wiki/Library_(computing)#Shared_libraries). The `"build"` script produces this file by copying it from within the `target/` directory, which is where the Rust build produces the shared object. + +### package.json + +The npm [manifest file](https://docs.npmjs.com/cli/v7/configuring-npm/package-json), which informs the `npm` command. + +### src/ + +The directory tree containing the Rust source code for the project. + +### src/lib.rs + +The Rust library's main module. + +### target/ + +Binary artifacts generated by the Rust build. + +## Learn More + +To learn more about Neon, see the [Neon documentation](https://neon-bindings.com). + +To learn more about Rust, see the [Rust documentation](https://www.rust-lang.org). + +To learn more about Node, see the [Node documentation](https://nodejs.org). diff --git a/backend/rust-gbt/package-lock.json b/backend/rust-gbt/package-lock.json new file mode 100644 index 000000000..6a6cc5fb7 --- /dev/null +++ b/backend/rust-gbt/package-lock.json @@ -0,0 +1,25 @@ +{ + "name": "gbt", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "gbt", + "version": "0.1.0", + "hasInstallScript": true, + "devDependencies": { + "cargo-cp-artifact": "^0.1" + } + }, + "node_modules/cargo-cp-artifact": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/cargo-cp-artifact/-/cargo-cp-artifact-0.1.8.tgz", + "integrity": "sha512-3j4DaoTrsCD1MRkTF2Soacii0Nx7UHCce0EwUf4fHnggwiE4fbmF2AbnfzayR36DF8KGadfh7M/Yfy625kgPlA==", + "dev": true, + "bin": { + "cargo-cp-artifact": "bin/cargo-cp-artifact.js" + } + } + } +} diff --git a/backend/rust-gbt/package.json b/backend/rust-gbt/package.json new file mode 100644 index 000000000..bb45af85e --- /dev/null +++ b/backend/rust-gbt/package.json @@ -0,0 +1,17 @@ +{ + "name": "gbt", + "version": "0.1.0", + "description": "An inefficient re-implementation of the getBlockTemplate algorithm in Rust", + "main": "index.node", + "scripts": { + "build": "cargo-cp-artifact -nc index.node -- cargo build --message-format=json-render-diagnostics", + "build-debug": "npm run build --", + "build-release": "npm run build -- --release", + "install": "npm run build-release", + "test": "cargo test" + }, + "author": "mononaut", + "devDependencies": { + "cargo-cp-artifact": "^0.1" + } +} \ No newline at end of file diff --git a/backend/rust-gbt/src/audit_transaction.rs b/backend/rust-gbt/src/audit_transaction.rs new file mode 100644 index 000000000..4b23b9a21 --- /dev/null +++ b/backend/rust-gbt/src/audit_transaction.rs @@ -0,0 +1,53 @@ +use std::{collections::{HashSet}, hash::{Hash, Hasher}, cmp::Ordering}; + +#[derive(Clone)] +pub struct AuditTransaction { + pub uid: u32, + pub fee: u64, + pub weight: u32, + pub sigops: u32, + pub fee_per_vsize: f64, + pub effective_fee_per_vsize: f64, + pub dependency_rate: f64, + pub inputs: Vec, + pub is_relatives_set: bool, + pub ancestors: HashSet, + pub children: HashSet, + pub ancestor_fee: u64, + pub ancestor_weight: u32, + pub ancestor_sigops: u32, + pub score: f64, + pub used: bool, + pub modified: bool, + pub dirty: bool, +} + +impl Hash for AuditTransaction { + fn hash(&self, state: &mut H) { + self.uid.hash(state); + } +} + +impl PartialEq for AuditTransaction { + fn eq(&self, other: &Self) -> bool { + self.uid == other.uid + } +} + +impl Eq for AuditTransaction {} + +impl PartialOrd for AuditTransaction { + fn partial_cmp(&self, other: &AuditTransaction) -> Option { + if self.score == other.score { + return Some(self.uid.cmp(&other.uid)); + } else { + return self.score.partial_cmp(&other.score); + } + } +} + +impl Ord for AuditTransaction { + fn cmp(&self, other: &AuditTransaction) -> Ordering { + self.partial_cmp(other).unwrap() + } +} \ No newline at end of file diff --git a/backend/rust-gbt/src/gbt.rs b/backend/rust-gbt/src/gbt.rs new file mode 100644 index 000000000..98c0d9178 --- /dev/null +++ b/backend/rust-gbt/src/gbt.rs @@ -0,0 +1,312 @@ +use std::cmp::Ordering; +use std::collections::{HashMap, HashSet, VecDeque}; +use std::f64::INFINITY; +use priority_queue::PriorityQueue; + +use crate::thread_transaction::{ThreadTransaction}; +use crate::audit_transaction::{AuditTransaction}; + +const BLOCK_WEIGHT_UNITS: u32 = 4_000_000; +const BLOCK_SIGOPS: u32 = 80_000; + +struct TxPriority { + uid: u32, + score: f64, +} +impl PartialEq for TxPriority { + fn eq(&self, other: &Self) -> bool { + self.uid == other.uid + } +} +impl Eq for TxPriority {} +impl PartialOrd for TxPriority { + fn partial_cmp(&self, other: &TxPriority) -> Option { + if self.score == other.score { + return Some(self.uid.cmp(&other.uid)); + } else { + return other.score.partial_cmp(&self.score); + } + } +} +impl Ord for TxPriority { + fn cmp(&self, other: &Self) -> Ordering { + self.partial_cmp(other).unwrap() + } +} + +pub fn gbt(mempool_array: Vec) -> (Vec>, Vec<(u32, f64)>, Vec>) { + let mut mempool: HashMap = HashMap::new(); + for transaction in mempool_array { + mempool.insert(transaction.uid, transaction); + } + + + return make_block_templates(mempool); +} + +/* +* Build projected mempool blocks using an approximation of the transaction selection algorithm from Bitcoin Core +* (see BlockAssembler in https://github.com/bitcoin/bitcoin/blob/master/src/node/miner.cpp) +* Ported from https://github.com/mempool/mempool/blob/master/backend/src/api/tx-selection-worker.ts +*/ +fn make_block_templates(mempool: HashMap) -> (Vec>, Vec<(u32, f64)>, Vec>) { + let mut audit_pool: HashMap = HashMap::new(); + let mut mempool_array: VecDeque = VecDeque::new(); + let mut cluster_array: Vec> = Vec::new(); + + // Initialize working structs + for (uid, tx) in &mempool { + let audit_tx = AuditTransaction { + uid: tx.uid, + fee: tx.fee, + weight: tx.weight, + sigops: tx.sigops, + fee_per_vsize: tx.fee_per_vsize, + effective_fee_per_vsize: tx.effective_fee_per_vsize, + dependency_rate: INFINITY, + inputs: tx.inputs.clone(), + is_relatives_set: false, + ancestors: HashSet::new(), + children: HashSet::new(), + ancestor_fee: tx.fee, + ancestor_weight: tx.weight, + ancestor_sigops: tx.sigops, + score: 0.0, + used: false, + modified: false, + dirty: false, + }; + audit_pool.insert(audit_tx.uid, audit_tx); + mempool_array.push_back(*uid); + } + + // Build relatives graph & calculate ancestor scores + for txid in &mempool_array { + set_relatives(*txid, &mut audit_pool); + } + + // Sort by descending ancestor score + mempool_array.make_contiguous().sort_unstable_by(|a, b| { + let a_tx = audit_pool.get(a).unwrap(); + let b_tx = audit_pool.get(b).unwrap(); + b_tx.cmp(a_tx) + }); + + // Build blocks by greedily choosing the highest feerate package + // (i.e. the package rooted in the transaction with the best ancestor score) + let mut blocks: Vec> = Vec::new(); + let mut block_weight: u32 = 4000; + let mut block_sigops: u32 = 0; + let mut transactions: Vec = Vec::new(); + let mut modified: PriorityQueue = PriorityQueue::new(); + let mut overflow: Vec = Vec::new(); + let mut failures = 0; + while mempool_array.len() > 0 || !modified.is_empty() { + let next_txid: u32; + if modified.is_empty() { + next_txid = mempool_array.pop_front().unwrap(); + } else if mempool_array.len() == 0 { + next_txid = modified.pop().unwrap().0; + } else { + let next_array_txid = mempool_array.front().unwrap(); + let next_modified_txid = modified.peek().unwrap().0; + let array_tx: &AuditTransaction = audit_pool.get(next_array_txid).unwrap(); + let modified_tx: &AuditTransaction = audit_pool.get(next_modified_txid).unwrap(); + match array_tx.cmp(&modified_tx) { + std::cmp::Ordering::Equal | std::cmp::Ordering::Greater => { + next_txid = mempool_array.pop_front().unwrap(); + } + std::cmp::Ordering::Less => { + next_txid = modified.pop().unwrap().0; + } + } + } + let next_tx: AuditTransaction = audit_pool.get(&next_txid).unwrap().clone(); + + if next_tx.used { + continue; + } + + if blocks.len() < 7 && ((block_weight + next_tx.ancestor_weight >= BLOCK_WEIGHT_UNITS) || (block_sigops + next_tx.ancestor_sigops > BLOCK_SIGOPS)) { + // hold this package in an overflow list while we check for smaller options + overflow.push(next_txid); + failures += 1; + } else { + let mut package: Vec<(u32, usize, u32)> = Vec::new(); + let mut cluster: Vec = Vec::new(); + let is_cluster: bool = next_tx.ancestors.len() > 0; + package.push((next_tx.uid, next_tx.ancestors.len(), next_tx.weight)); + cluster.push(next_tx.uid); + for ancestor_id in &next_tx.ancestors { + if let Some(ancestor) = audit_pool.get(ancestor_id) { + package.push((*ancestor_id, ancestor.ancestors.len(), ancestor.weight)); + cluster.push(*ancestor_id); + } + } + package.sort_unstable_by_key(|a| 0 - a.1); + + if is_cluster { + cluster_array.push(cluster); + } + + let cluster_rate = next_tx.dependency_rate.min(next_tx.ancestor_fee as f64 / (next_tx.ancestor_weight as f64 / 4.0)); + + for package_entry in &package { + if let Some(tx) = audit_pool.get_mut(&package_entry.0) { + tx.used = true; + if tx.effective_fee_per_vsize != cluster_rate { + tx.effective_fee_per_vsize = cluster_rate; + tx.dirty = true; + } + transactions.push(tx.uid); + block_weight += tx.weight; + block_sigops += tx.sigops; + } + update_descendants(package_entry.0, &mut audit_pool, &mut modified, cluster_rate); + } + + failures = 0; + } + + // this block is full + let exceeded_package_tries = failures > 1000 && block_weight > (BLOCK_WEIGHT_UNITS - 4000); + let queue_is_empty = mempool_array.len() == 0 && modified.is_empty(); + if (exceeded_package_tries || queue_is_empty) && blocks.len() < 7 { + // finalize this block + if transactions.len() > 0 { + blocks.push(transactions); + } + // reset for the next block + transactions = Vec::new(); + block_weight = 4000; + // 'overflow' packages didn't fit in this block, but are valid candidates for the next + overflow.reverse(); + for overflowed in &overflow { + if let Some(overflowed_tx) = audit_pool.get(overflowed) { + if overflowed_tx.modified { + modified.push(*overflowed, TxPriority{ uid: *overflowed, score: overflowed_tx.score}); + } else { + mempool_array.push_front(*overflowed); + } + } + } + overflow = Vec::new(); + } + } + // add the final unbounded block if it contains any transactions + if transactions.len() > 0 { + blocks.push(transactions); + } + + // make a list of dirty transactions and their new rates + let mut rates: Vec<(u32, f64)> = Vec::new(); + for (txid, tx) in audit_pool { + if tx.dirty { + rates.push((txid, tx.effective_fee_per_vsize)); + } + } + + return (blocks, rates, cluster_array); +} + +fn set_relatives(txid: u32, audit_pool: &mut HashMap) { + let mut parents: HashSet = HashSet::new(); + if let Some(tx) = audit_pool.get(&txid) { + if tx.is_relatives_set { + return; + } + for input in &tx.inputs { + parents.insert(*input); + } + } else { + return; + } + + let mut ancestors: HashSet = HashSet::new(); + for parent_id in &parents { + set_relatives(*parent_id, audit_pool); + + let parent_entry: Option<&mut AuditTransaction> = audit_pool.get_mut(&parent_id); + match parent_entry { + Some(parent) => { + ancestors.insert(*parent_id); + parent.children.insert(txid); + for ancestor in &parent.ancestors { + ancestors.insert(*ancestor); + } + } + + None => {} + } + } + + let mut total_fee: u64 = 0; + let mut total_weight: u32 = 0; + let mut total_sigops: u32 = 0; + + for ancestor_id in &ancestors { + let ancestor = audit_pool.get(&ancestor_id).unwrap(); + total_fee += ancestor.fee; + total_weight += ancestor.weight; + total_sigops += ancestor.sigops; + } + + if let Some(tx) = audit_pool.get_mut(&txid) { + tx.ancestors = ancestors; + tx.ancestor_fee = tx.fee + total_fee; + tx.ancestor_weight = tx.weight + total_weight; + tx.ancestor_sigops = tx.sigops + total_sigops; + tx.score = (tx.ancestor_fee as f64) / (if tx.ancestor_weight != 0 {tx.ancestor_weight as f64 / 4.0} else { 1.0 }); + tx.is_relatives_set = true; + } +} + +// iterate over remaining descendants, removing the root as a valid ancestor & updating the ancestor score +fn update_descendants(root_txid: u32, audit_pool: &mut HashMap, modified: &mut PriorityQueue, cluster_rate: f64) { + let mut visited: HashSet = HashSet::new(); + let mut descendant_stack: Vec = Vec::new(); + let root_fee: u64; + let root_weight: u32; + let root_sigops: u32; + if let Some(root_tx) = audit_pool.get(&root_txid) { + for descendant_id in &root_tx.children { + if !visited.contains(descendant_id) { + descendant_stack.push(*descendant_id); + visited.insert(*descendant_id); + } + } + root_fee = root_tx.fee; + root_weight = root_tx.weight; + root_sigops = root_tx.sigops; + } else { + return; + } + while descendant_stack.len() > 0 { + let next_txid: u32 = descendant_stack.pop().unwrap(); + if let Some(descendant) = audit_pool.get_mut(&next_txid) { + // remove root tx as ancestor + descendant.ancestors.remove(&root_txid); + descendant.ancestor_fee -= root_fee; + descendant.ancestor_weight -= root_weight; + descendant.ancestor_sigops -= root_sigops; + let current_score = descendant.score; + descendant.score = (descendant.ancestor_fee as f64) / (if descendant.ancestor_weight != 0 {descendant.ancestor_weight as f64 / 4.0} else { 1.0 }); + descendant.dependency_rate = descendant.dependency_rate.min(cluster_rate); + descendant.modified = true; + // update modified priority if score has changed + if !descendant.modified || descendant.score < current_score { + modified.push_decrease(descendant.uid, TxPriority { uid: descendant.uid, score: descendant.score}); + } else if descendant.score > current_score { + modified.push_increase(descendant.uid, TxPriority { uid: descendant.uid, score: descendant.score}); + } + + // add this node's children to the stack + for child_id in &descendant.children { + if !visited.contains(child_id) { + descendant_stack.push(*child_id); + visited.insert(*child_id); + } + } + } + } +} \ No newline at end of file diff --git a/backend/rust-gbt/src/lib.rs b/backend/rust-gbt/src/lib.rs new file mode 100644 index 000000000..ff7d6c208 --- /dev/null +++ b/backend/rust-gbt/src/lib.rs @@ -0,0 +1,75 @@ +use neon::{prelude::*, types::buffer::TypedArray}; + +mod gbt; +mod thread_transaction; +mod audit_transaction; +use thread_transaction::{ThreadTransaction}; + +fn go(mut cx: FunctionContext) -> JsResult { + let mempool_arg = cx.argument::(0)?.root(&mut cx).into_inner(&mut cx); + let callback = cx.argument::(1)?.root(&mut cx); + let channel = cx.channel(); + + let buffer = mempool_arg.as_slice(&mut cx); + let thread_transactions = ThreadTransaction::batch_from_buffer(buffer); + + std::thread::spawn(move || { + let (blocks, rates, clusters) = gbt::gbt(thread_transactions); + + channel.send(move |mut cx| { + let result = JsObject::new(&mut cx); + + let js_blocks = JsArray::new(&mut cx, blocks.len() as u32); + for (i, block) in blocks.iter().enumerate() { + let inner = JsArray::new(&mut cx, block.len() as u32); + for (j, uid) in block.iter().enumerate() { + let v = cx.number(*uid); + inner.set(&mut cx, j as u32, v)?; + } + js_blocks.set(&mut cx, i as u32, inner)?; + } + + let js_clusters = JsArray::new(&mut cx, clusters.len() as u32); + for (i, cluster) in clusters.iter().enumerate() { + let inner = JsArray::new(&mut cx, cluster.len() as u32); + for (j, uid) in cluster.iter().enumerate() { + let v = cx.number(*uid); + inner.set(&mut cx, j as u32, v)?; + } + js_clusters.set(&mut cx, i as u32, inner)?; + } + + let js_rates = JsArray::new(&mut cx, rates.len() as u32); + for (i, (uid, rate)) in rates.iter().enumerate() { + let inner = JsArray::new(&mut cx, 2); + let js_uid = cx.number(*uid); + let js_rate = cx.number(*rate); + inner.set(&mut cx, 0, js_uid)?; + inner.set(&mut cx, 1, js_rate)?; + js_rates.set(&mut cx, i as u32, inner)?; + } + + result.set(&mut cx, "blocks", js_blocks)?; + result.set(&mut cx, "clusters", js_clusters)?; + result.set(&mut cx, "rates", js_rates)?; + + let callback = callback.into_inner(&mut cx); + let this = cx.undefined(); + let args = vec![ + result.upcast() + ]; + + callback.call(&mut cx, this, args)?; + + Ok(()) + }); + }); + + Ok(cx.undefined()) +} + +#[neon::main] +fn main(mut cx: ModuleContext) -> NeonResult<()> { + cx.export_function("go", go)?; + Ok(()) +} diff --git a/backend/rust-gbt/src/thread_transaction.rs b/backend/rust-gbt/src/thread_transaction.rs new file mode 100644 index 000000000..fa977c3e9 --- /dev/null +++ b/backend/rust-gbt/src/thread_transaction.rs @@ -0,0 +1,45 @@ +// use neon::{types::{JsObject, JsNumber, JsArray, JsValue, JsBoolean, JsArrayBuffer, buffer::TypedArray}, prelude::{Object, FunctionContext, Handle}}; +extern crate bytes; +use std::io::Cursor; +use bytes::buf::Buf; +pub struct ThreadTransaction { + pub uid: u32, + pub fee: u64, + pub weight: u32, + pub sigops: u32, + pub fee_per_vsize: f64, + pub effective_fee_per_vsize: f64, + pub inputs: Vec, +} + +impl ThreadTransaction { + pub fn batch_from_buffer(buffer: &[u8]) -> Vec { + let mut transactions: Vec = Vec::new(); + let mut cursor = Cursor::new(buffer); + let size = cursor.get_u32(); + for _ in 0..size { + let uid = cursor.get_u32(); + let fee = cursor.get_f64() as u64; + let weight = cursor.get_u32(); + let sigops = cursor.get_u32(); + let fee_per_vsize = cursor.get_f64(); + let effective_fee_per_vsize = cursor.get_f64(); + let input_count = cursor.get_u32(); + let mut inputs: Vec = Vec::new(); + for _ in 0..input_count { + inputs.push(cursor.get_u32()); + } + transactions.push(ThreadTransaction { + uid, + fee, + weight, + sigops, + fee_per_vsize, + effective_fee_per_vsize, + inputs, + }) + } + + return transactions; + } +} \ No newline at end of file diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 57d1a393f..7f4490477 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -1,10 +1,12 @@ import logger from '../logger'; -import { MempoolBlock, MempoolTransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats } from '../mempool.interfaces'; +import { MempoolBlock, MempoolTransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats, AuditTransaction } from '../mempool.interfaces'; import { Common, OnlineFeeStatsCalculator } from './common'; import config from '../config'; import { Worker } from 'worker_threads'; import path from 'path'; +const neonAddon = require('../../rust-gbt'); + class MempoolBlocks { private mempoolBlocks: MempoolBlockWithTransactions[] = []; private mempoolBlockDeltas: MempoolBlockDelta[] = []; @@ -219,7 +221,7 @@ class MempoolBlocks { const strippedMempool: Map = new Map(); Object.values(newMempool).forEach(entry => { if (entry.uid != null) { - strippedMempool.set(entry.uid, { + const stripped = { uid: entry.uid, fee: entry.fee, weight: (entry.adjustedVsize * 4), @@ -227,7 +229,8 @@ class MempoolBlocks { feePerVsize: entry.adjustedFeePerVsize || entry.feePerVsize, effectiveFeePerVsize: entry.effectiveFeePerVsize || entry.adjustedFeePerVsize || entry.feePerVsize, inputs: entry.vin.map(v => this.getUid(newMempool[v.txid])).filter(uid => uid != null) as number[], - }); + }; + strippedMempool.set(entry.uid, stripped); } }); @@ -261,7 +264,9 @@ class MempoolBlocks { this.txSelectionWorker?.removeListener('error', threadErrorListener); const processed = this.processBlockTemplates(newMempool, blocks, rates, clusters, saveResults); + logger.debug(`makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); + return processed; } catch (e) { logger.err('makeBlockTemplates failed. ' + (e instanceof Error ? e.message : e)); @@ -321,6 +326,36 @@ class MempoolBlocks { } } + public async $rustMakeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false): Promise { + const start = Date.now(); + + // reset mempool short ids + this.resetUids(); + for (const tx of Object.values(newMempool)) { + this.setUid(tx); + } + + // serialize relevant mempool data into an ArrayBuffer + // to reduce the overhead of passing this data to the rust thread + const mempoolBuffer = this.mempoolToArrayBuffer(newMempool); + + // run the block construction algorithm in a separate thread, and wait for a result + try { + const { blocks, rates, clusters } = this.convertNeonResultTxids(await new Promise((resolve) => { neonAddon.go(mempoolBuffer, resolve); })); + const processed = this.processBlockTemplates(newMempool, blocks, rates, clusters, saveResults); + logger.debug(`RUST makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); + return processed; + } catch (e) { + logger.err('RUST makeBlockTemplates failed. ' + (e instanceof Error ? e.message : e)); + } + return this.mempoolBlocks; + } + + public async $rustUpdateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], saveResults: boolean = false): Promise { + await this.$rustMakeBlockTemplates(newMempool, saveResults); + return; + } + private processBlockTemplates(mempool, blocks: string[][], rates: { [root: string]: number }, clusters: { [root: string]: string[] }, saveResults): MempoolBlockWithTransactions[] { for (const txid of Object.keys(rates)) { if (txid in mempool) { @@ -496,6 +531,74 @@ class MempoolBlocks { } return { blocks: convertedBlocks, rates: convertedRates, clusters: convertedClusters } as { blocks: string[][], rates: { [root: string]: number }, clusters: { [root: string]: string[] }}; } + + private convertNeonResultTxids({ blocks, rates, clusters }: { blocks: number[][], rates: number[][], clusters: number[][]}) + : { blocks: string[][], rates: { [root: string]: number }, clusters: { [root: string]: string[] }} { + const rateMap = new Map(); + const clusterMap = new Map(); + for (const rate of rates) { + rateMap.set(rate[0], rate[1]); + } + for (const cluster of clusters) { + clusterMap.set(cluster[0], cluster); + } + const convertedBlocks: string[][] = blocks.map(block => block.map(uid => { + return this.uidMap.get(uid) || ''; + })); + const convertedRates = {}; + for (const rateUid of rateMap.keys()) { + const rateTxid = this.uidMap.get(rateUid); + if (rateTxid) { + convertedRates[rateTxid] = rateMap.get(rateUid); + } + } + const convertedClusters = {}; + for (const rootUid of clusterMap.keys()) { + const rootTxid = this.uidMap.get(rootUid); + if (rootTxid) { + const members = clusterMap.get(rootUid)?.map(uid => { + return this.uidMap.get(uid); + }); + convertedClusters[rootTxid] = members; + } + } + return { blocks: convertedBlocks, rates: convertedRates, clusters: convertedClusters } as { blocks: string[][], rates: { [root: string]: number }, clusters: { [root: string]: string[] }}; + } + + private mempoolToArrayBuffer(mempool: { [txid: string]: MempoolTransactionExtended }): ArrayBuffer { + let len = 4; + const inputs: { [uid: number]: number[] } = {}; + let validCount = 0; + for (const tx of Object.values(mempool)) { + if (tx.uid != null) { + validCount++; + const txInputs = tx.vin.map(v => this.getUid(mempool[v.txid])).filter(uid => uid != null) as number[] + inputs[tx.uid] = txInputs; + len += (10 + txInputs.length) * 4; + } + } + const buf = new ArrayBuffer(len); + const view = new DataView(buf); + view.setUint32(0, validCount, false); + let offset = 4; + for (const tx of Object.values(mempool)) { + if (tx.uid != null) { + view.setUint32(offset, tx.uid, false); + view.setFloat64(offset + 4, tx.fee, false); + view.setUint32(offset + 12, (tx.adjustedVsize * 4), false); + view.setUint32(offset + 16, tx.sigops, false); + view.setFloat64(offset + 20, (tx.adjustedFeePerVsize || tx.feePerVsize), false); + view.setFloat64(offset + 28, (tx.effectiveFeePerVsize || tx.adjustedFeePerVsize || tx.feePerVsize), false); + view.setUint32(offset + 36, inputs[tx.uid].length, false); + offset += 40; + for (const input of inputs[tx.uid]) { + view.setUint32(offset, input, false); + offset += 4; + } + } + } + return buf; + } } export default new MempoolBlocks(); diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 041c7e767..c32aee4cd 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -342,7 +342,7 @@ class WebsocketHandler { this.printLogs(); if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) { - await mempoolBlocks.$updateBlockTemplates(newMempool, newTransactions, deletedTransactions, true); + await mempoolBlocks.$rustUpdateBlockTemplates(newMempool, newTransactions, deletedTransactions, true); } else { mempoolBlocks.updateMempoolBlocks(newMempool, true); } @@ -588,7 +588,7 @@ class WebsocketHandler { if (separateAudit) { auditMempool = deepClone(_memPool); if (config.MEMPOOL.ADVANCED_GBT_AUDIT) { - projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false); + projectedBlocks = await mempoolBlocks.$rustMakeBlockTemplates(auditMempool, false); } else { projectedBlocks = mempoolBlocks.updateMempoolBlocks(auditMempool, false); } @@ -655,7 +655,7 @@ class WebsocketHandler { } if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) { - await mempoolBlocks.$makeBlockTemplates(_memPool, true); + await mempoolBlocks.$rustMakeBlockTemplates(_memPool, true); } else { mempoolBlocks.updateMempoolBlocks(_memPool, true); } diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index 3edd84cde..5958f464f 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -127,7 +127,7 @@ export interface CompactThreadTransaction { feePerVsize: number; effectiveFeePerVsize?: number; inputs: number[]; - cpfpRoot?: string; + cpfpRoot?: number; cpfpChecked?: boolean; dirty?: boolean; }