diff --git a/Cargo.toml b/Cargo.toml index 5896b05f..71af2dea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,7 +40,14 @@ key-value-db = ["sled"] cli-utils = ["clap", "base64"] async-interface = ["async-trait"] +# Debug/Test features +debug-proc-macros = ["magical-macros/debug", "testutils-macros/debug"] +test-electrum = ["electrum"] + [dev-dependencies] +testutils = { path = "./testutils" } +testutils-macros = { path = "./testutils-macros" } +serial_test = "0.4" lazy_static = "1.4" rustyline = "6.0" dirs = "2.0" @@ -67,3 +74,5 @@ name = "magic" path = "examples/repl.rs" required-features = ["cli-utils"] +[workspace] +members = ["macros", "testutils", "testutils-macros"] diff --git a/macros/Cargo.toml b/macros/Cargo.toml index 8c581e6b..64d9ed02 100644 --- a/macros/Cargo.toml +++ b/macros/Cargo.toml @@ -11,5 +11,8 @@ syn = { version = "1.0", features = ["parsing"] } proc-macro2 = "1.0" quote = "1.0" +[features] +debug = ["syn/extra-traits"] + [lib] proc-macro = true diff --git a/macros/src/lib.rs b/macros/src/lib.rs index bd74a521..67cf781f 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -85,7 +85,8 @@ pub fn maybe_async(_attr: TokenStream, item: TokenStream) -> TokenStream { } else { (quote! { compile_error!("#[maybe_async] can only be used on methods, trait or trait impl blocks") - }).into() + }) + .into() } } diff --git a/src/blockchain/electrum.rs b/src/blockchain/electrum.rs index 8913feb8..44331e2c 100644 --- a/src/blockchain/electrum.rs +++ b/src/blockchain/electrum.rs @@ -15,6 +15,13 @@ use crate::FeeRate; pub struct ElectrumBlockchain(Option); +#[cfg(test)] +#[cfg(feature = "test-electrum")] +#[magical_blockchain_tests(crate)] +fn local_electrs() -> ElectrumBlockchain { + ElectrumBlockchain::from(Client::new(&testutils::get_electrum_url(), None).unwrap()) +} + impl std::convert::From for ElectrumBlockchain { fn from(client: Client) -> Self { ElectrumBlockchain(Some(client)) diff --git a/src/blockchain/utils.rs b/src/blockchain/utils.rs index 6e52ed93..8b63dde3 100644 --- a/src/blockchain/utils.rs +++ b/src/blockchain/utils.rs @@ -186,7 +186,6 @@ pub trait ElectrumLikeSync { ); let mut updates = database.begin_batch(); let tx = match database.get_tx(&txid, true)? { - // TODO: do we need the raw? Some(mut saved_tx) => { // update the height if it's different (in case of reorg) if saved_tx.height != height { @@ -204,12 +203,20 @@ pub trait ElectrumLikeSync { // went wrong saved_tx.transaction.unwrap() } - None => maybe_await!(self.els_transaction_get(&txid))?, + None => { + let fetched_tx = maybe_await!(self.els_transaction_get(&txid))?; + database.set_raw_tx(&fetched_tx)?; + + fetched_tx + } }; let mut incoming: u64 = 0; let mut outgoing: u64 = 0; + let mut inputs_sum: u64 = 0; + let mut outputs_sum: u64 = 0; + // look for our own inputs for (i, input) in tx.input.iter().enumerate() { // the fact that we visit addresses in a BFS fashion starting from the external addresses @@ -217,17 +224,37 @@ pub trait ElectrumLikeSync { // the transactions at a lower depth have already been indexed, so if an outpoint is ours // we are guaranteed to have it in the db). if let Some(previous_output) = database.get_previous_output(&input.previous_output)? { + inputs_sum += previous_output.value; + if database.is_mine(&previous_output.script_pubkey)? { outgoing += previous_output.value; debug!("{} input #{} is mine, removing from utxo", txid, i); updates.del_utxo(&input.previous_output)?; } + } else { + // The input is not ours, but we still need to count it for the fees. so fetch the + // tx (from the database or from network) and check it + let tx = match database.get_tx(&input.previous_output.txid, true)? { + Some(saved_tx) => saved_tx.transaction.unwrap(), + None => { + let fetched_tx = + maybe_await!(self.els_transaction_get(&input.previous_output.txid))?; + database.set_raw_tx(&fetched_tx)?; + + fetched_tx + } + }; + + inputs_sum += tx.output[input.previous_output.vout as usize].value; } } let mut to_check_later = vec![]; for (i, output) in tx.output.iter().enumerate() { + // to compute the fees later + outputs_sum += output.value; + // this output is ours, we have a path to derive it if let Some((script_type, child)) = database.get_path_from_script_pubkey(&output.script_pubkey)? @@ -259,6 +286,7 @@ pub trait ElectrumLikeSync { sent: outgoing, height, timestamp: 0, + fees: inputs_sum - outputs_sum, }; info!("Saving tx {}", txid); updates.set_tx(&tx)?; diff --git a/src/database/mod.rs b/src/database/mod.rs index 9c0b5235..4ffb1347 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -8,6 +8,8 @@ use crate::types::*; pub mod keyvalue; pub mod memory; +pub use memory::MemoryDatabase; + pub trait BatchOperations { fn set_script_pubkey( &mut self, @@ -235,6 +237,7 @@ pub mod test { timestamp: 123456, received: 1337, sent: 420420, + fees: 140, height: Some(1000), }; diff --git a/src/error.rs b/src/error.rs index 72dfe944..16f67f1d 100644 --- a/src/error.rs +++ b/src/error.rs @@ -10,7 +10,7 @@ pub enum Error { SendAllMultipleOutputs, OutputBelowDustLimit(usize), InsufficientFunds, - InvalidAddressNetork(Address), + InvalidAddressNetwork(Address), UnknownUTXO, DifferentTransactions, diff --git a/src/lib.rs b/src/lib.rs index 4629ec77..8143052d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -31,6 +31,16 @@ pub extern crate sled; #[cfg(feature = "cli-utils")] pub mod cli; +#[cfg(test)] +#[macro_use] +extern crate testutils; +#[cfg(test)] +#[macro_use] +extern crate testutils_macros; +#[cfg(test)] +#[macro_use] +extern crate serial_test; + #[macro_use] pub mod error; pub mod blockchain; diff --git a/src/types.rs b/src/types.rs index dff48f96..70679096 100644 --- a/src/types.rs +++ b/src/types.rs @@ -6,7 +6,7 @@ use bitcoin::hash_types::Txid; use serde::{Deserialize, Serialize}; // TODO serde flatten? -#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum ScriptType { External = 0, Internal = 1, @@ -48,5 +48,6 @@ pub struct TransactionDetails { pub timestamp: u64, pub received: u64, pub sent: u64, + pub fees: u64, pub height: Option, } diff --git a/src/wallet/export.rs b/src/wallet/export.rs index f90200fa..67347990 100644 --- a/src/wallet/export.rs +++ b/src/wallet/export.rs @@ -129,6 +129,7 @@ mod test { timestamp: 12345678, received: 100_000, sent: 0, + fees: 500, height: Some(5000), }) .unwrap(); diff --git a/src/wallet/mod.rs b/src/wallet/mod.rs index cd2bfbc8..4d3636a9 100644 --- a/src/wallet/mod.rs +++ b/src/wallet/mod.rs @@ -24,7 +24,7 @@ pub mod tx_builder; pub mod utils; use tx_builder::TxBuilder; -use utils::{FeeRate, IsDust}; +use utils::IsDust; use crate::blockchain::{noop_progress, Blockchain, OfflineBlockchain, OnlineBlockchain}; use crate::database::{BatchDatabase, BatchOperations, DatabaseUtils}; @@ -190,8 +190,9 @@ where false => *satoshi, }; - if address.network != self.network { - return Err(Error::InvalidAddressNetork(address.clone())); + // TODO: proper checks for testnet/regtest p2sh/p2pkh + if address.network != self.network && self.network != Network::Regtest { + return Err(Error::InvalidAddressNetwork(address.clone())); } else if self.is_mine(&address.script_pubkey())? { received += value; } @@ -263,7 +264,8 @@ where } }; - let change_val = total_amount - outgoing - (fee_amount.ceil() as u64); + let mut fee_amount = fee_amount.ceil() as u64; + let change_val = total_amount - outgoing - fee_amount; if !builder.send_all && !change_val.is_dust() { let mut change_output = change_output.unwrap(); change_output.value = change_val; @@ -271,8 +273,6 @@ where tx.output.push(change_output); } else if builder.send_all && !change_val.is_dust() { - // set the outgoing value to whatever we've put in - outgoing = total_amount; // there's only one output, send everything to it tx.output[0].value = change_val; @@ -280,6 +280,9 @@ where if self.is_mine(&tx.output[0].script_pubkey)? { received = change_val; } + } else if !builder.send_all && change_val.is_dust() { + // skip the change output because it's dust, this adds up to the fees + fee_amount += change_val; } else if builder.send_all { // send_all but the only output would be below dust limit return Err(Error::InsufficientFunds); // TODO: or OutputBelowDustLimit? @@ -339,7 +342,8 @@ where txid, timestamp: time::get_timestamp(), received, - sent: outgoing, + sent: total_amount, + fees: fee_amount, height: None, }; @@ -750,6 +754,8 @@ where pub fn sync(&self, max_address_param: Option) -> Result<(), Error> { debug!("Begin sync..."); + let mut run_setup = false; + let max_address = match self.descriptor.is_fixed() { true => 0, false => max_address_param.unwrap_or(CACHE_ADDR_BATCH_SIZE), @@ -760,6 +766,7 @@ where .get_script_pubkey_from_path(ScriptType::External, max_address)? .is_none() { + run_setup = true; self.cache_addresses(ScriptType::External, 0, max_address)?; } @@ -775,15 +782,24 @@ where .get_script_pubkey_from_path(ScriptType::Internal, max_address)? .is_none() { + run_setup = true; self.cache_addresses(ScriptType::Internal, 0, max_address)?; } } - maybe_await!(self.client.sync( - None, - self.database.borrow_mut().deref_mut(), - noop_progress(), - )) + if run_setup { + maybe_await!(self.client.setup( + None, + self.database.borrow_mut().deref_mut(), + noop_progress(), + )) + } else { + maybe_await!(self.client.sync( + None, + self.database.borrow_mut().deref_mut(), + noop_progress(), + )) + } } pub fn client(&self) -> &B { diff --git a/testutils-macros/Cargo.toml b/testutils-macros/Cargo.toml new file mode 100644 index 00000000..f699d1d7 --- /dev/null +++ b/testutils-macros/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "testutils-macros" +version = "0.1.0" +authors = ["Alekos Filini "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +syn = { version = "1.0", features = ["parsing"] } +proc-macro2 = "1.0" +quote = "1.0" + +[features] +debug = ["syn/extra-traits"] + +[lib] +proc-macro = true diff --git a/testutils-macros/src/lib.rs b/testutils-macros/src/lib.rs new file mode 100644 index 00000000..1a8d42a8 --- /dev/null +++ b/testutils-macros/src/lib.rs @@ -0,0 +1,373 @@ +#[macro_use] +extern crate quote; + +use proc_macro::TokenStream; + +use syn::spanned::Spanned; +use syn::{parse, parse2, Ident, ReturnType}; + +#[proc_macro_attribute] +pub fn magical_blockchain_tests(attr: TokenStream, item: TokenStream) -> TokenStream { + let root_ident = if !attr.is_empty() { + match parse::(attr) { + Ok(parsed) => parsed, + Err(e) => { + let error_string = e.to_string(); + return (quote! { + compile_error!("Invalid crate path: {:?}", #error_string) + }) + .into(); + } + } + } else { + parse2::(quote! { magical_bitcoin_wallet }).unwrap() + }; + + match parse::(item) { + Err(_) => (quote! { + compile_error!("#[magical_blockchain_tests] can only be used on `fn`s") + }) + .into(), + Ok(parsed) => { + let parsed_sig_ident = parsed.sig.ident.clone(); + let mod_name = Ident::new( + &format!("generated_tests_{}", parsed_sig_ident.to_string()), + parsed.span(), + ); + + let return_type = match parsed.sig.output { + ReturnType::Type(_, ref t) => t.clone(), + ReturnType::Default => { + return (quote! { + compile_error!("The tagged function must return a type that impl `OnlineBlockchain`") + }).into(); + } + }; + + let output = quote! { + + #parsed + + mod #mod_name { + use bitcoin::Network; + + use miniscript::Descriptor; + + use testutils::{TestClient, serial}; + + use #root_ident::blockchain::OnlineBlockchain; + use #root_ident::descriptor::ExtendedDescriptor; + use #root_ident::database::MemoryDatabase; + use #root_ident::types::ScriptType; + use #root_ident::{Wallet, TxBuilder}; + + use super::*; + + fn get_blockchain() -> #return_type { + #parsed_sig_ident() + } + + fn get_wallet_from_descriptors(descriptors: &(ExtendedDescriptor, Option)) -> Wallet<#return_type, MemoryDatabase> { + Wallet::new(&descriptors.0.to_string(), descriptors.1.as_ref().map(|d| d.to_string()).as_deref(), Network::Regtest, MemoryDatabase::new(), get_blockchain()).unwrap() + } + + fn init_single_sig() -> (Wallet<#return_type, MemoryDatabase>, (ExtendedDescriptor, Option), TestClient) { + let descriptors = testutils! { + @descriptors ( "wpkh(Alice)" ) ( "wpkh(Alice)" ) ( @keys ( "Alice" => (@generate_xprv "/44'/0'/0'/0/*", "/44'/0'/0'/1/*") ) ) + }; + + let test_client = TestClient::new(); + let wallet = get_wallet_from_descriptors(&descriptors); + + (wallet, descriptors, test_client) + } + + #[test] + #[serial] + fn test_sync_simple() { + let (wallet, descriptors, mut test_client) = init_single_sig(); + + let tx = testutils! { + @tx ( (@external descriptors, 0) => 50_000 ) + }; + let txid = test_client.receive(tx); + + wallet.sync(None).unwrap(); + + assert_eq!(wallet.get_balance().unwrap(), 50_000); + assert_eq!(wallet.list_unspent().unwrap()[0].is_internal, false); + + let list_tx_item = &wallet.list_transactions(false).unwrap()[0]; + assert_eq!(list_tx_item.txid, txid); + assert_eq!(list_tx_item.received, 50_000); + assert_eq!(list_tx_item.sent, 0); + assert_eq!(list_tx_item.height, None); + } + + #[test] + #[serial] + fn test_sync_stop_gap_20() { + let (wallet, descriptors, mut test_client) = init_single_sig(); + + test_client.receive(testutils! { + @tx ( (@external descriptors, 5) => 50_000 ) + }); + test_client.receive(testutils! { + @tx ( (@external descriptors, 25) => 50_000 ) + }); + + wallet.sync(None).unwrap(); + + assert_eq!(wallet.get_balance().unwrap(), 100_000); + assert_eq!(wallet.list_transactions(false).unwrap().len(), 2); + } + + #[test] + #[serial] + fn test_sync_before_and_after_receive() { + let (wallet, descriptors, mut test_client) = init_single_sig(); + + wallet.sync(None).unwrap(); + assert_eq!(wallet.get_balance().unwrap(), 0); + + test_client.receive(testutils! { + @tx ( (@external descriptors, 0) => 50_000 ) + }); + + wallet.sync(None).unwrap(); + + assert_eq!(wallet.get_balance().unwrap(), 50_000); + assert_eq!(wallet.list_transactions(false).unwrap().len(), 1); + } + + #[test] + #[serial] + fn test_sync_multiple_outputs_same_tx() { + let (wallet, descriptors, mut test_client) = init_single_sig(); + + let txid = test_client.receive(testutils! { + @tx ( (@external descriptors, 0) => 50_000, (@external descriptors, 1) => 25_000, (@external descriptors, 5) => 30_000 ) + }); + + wallet.sync(None).unwrap(); + + assert_eq!(wallet.get_balance().unwrap(), 105_000); + assert_eq!(wallet.list_transactions(false).unwrap().len(), 1); + assert_eq!(wallet.list_unspent().unwrap().len(), 3); + + let list_tx_item = &wallet.list_transactions(false).unwrap()[0]; + assert_eq!(list_tx_item.txid, txid); + assert_eq!(list_tx_item.received, 105_000); + assert_eq!(list_tx_item.sent, 0); + assert_eq!(list_tx_item.height, None); + } + + #[test] + #[serial] + fn test_sync_receive_multi() { + let (wallet, descriptors, mut test_client) = init_single_sig(); + + test_client.receive(testutils! { + @tx ( (@external descriptors, 0) => 50_000 ) + }); + test_client.receive(testutils! { + @tx ( (@external descriptors, 5) => 25_000 ) + }); + + wallet.sync(None).unwrap(); + + assert_eq!(wallet.get_balance().unwrap(), 75_000); + assert_eq!(wallet.list_transactions(false).unwrap().len(), 2); + assert_eq!(wallet.list_unspent().unwrap().len(), 2); + } + + #[test] + #[serial] + fn test_sync_address_reuse() { + let (wallet, descriptors, mut test_client) = init_single_sig(); + + test_client.receive(testutils! { + @tx ( (@external descriptors, 0) => 50_000 ) + }); + + wallet.sync(None).unwrap(); + assert_eq!(wallet.get_balance().unwrap(), 50_000); + + test_client.receive(testutils! { + @tx ( (@external descriptors, 0) => 25_000 ) + }); + + wallet.sync(None).unwrap(); + assert_eq!(wallet.get_balance().unwrap(), 75_000); + } + + #[test] + #[serial] + fn test_sync_receive_rbf_replaced() { + let (wallet, descriptors, mut test_client) = init_single_sig(); + + let txid = test_client.receive(testutils! { + @tx ( (@external descriptors, 0) => 50_000 ) ( @replaceable true ) + }); + + wallet.sync(None).unwrap(); + + assert_eq!(wallet.get_balance().unwrap(), 50_000); + assert_eq!(wallet.list_transactions(false).unwrap().len(), 1); + assert_eq!(wallet.list_unspent().unwrap().len(), 1); + + let list_tx_item = &wallet.list_transactions(false).unwrap()[0]; + assert_eq!(list_tx_item.txid, txid); + assert_eq!(list_tx_item.received, 50_000); + assert_eq!(list_tx_item.sent, 0); + assert_eq!(list_tx_item.height, None); + + let new_txid = test_client.bump_fee(&txid); + + wallet.sync(None).unwrap(); + + assert_eq!(wallet.get_balance().unwrap(), 50_000); + assert_eq!(wallet.list_transactions(false).unwrap().len(), 1); + assert_eq!(wallet.list_unspent().unwrap().len(), 1); + + let list_tx_item = &wallet.list_transactions(false).unwrap()[0]; + assert_eq!(list_tx_item.txid, new_txid); + assert_eq!(list_tx_item.received, 50_000); + assert_eq!(list_tx_item.sent, 0); + assert_eq!(list_tx_item.height, None); + } + + #[test] + #[serial] + fn test_sync_reorg_block() { + let (wallet, descriptors, mut test_client) = init_single_sig(); + + let txid = test_client.receive(testutils! { + @tx ( (@external descriptors, 0) => 50_000 ) ( @confirmations 1 ) ( @replaceable true ) + }); + + wallet.sync(None).unwrap(); + + assert_eq!(wallet.get_balance().unwrap(), 50_000); + assert_eq!(wallet.list_transactions(false).unwrap().len(), 1); + assert_eq!(wallet.list_unspent().unwrap().len(), 1); + + let list_tx_item = &wallet.list_transactions(false).unwrap()[0]; + assert_eq!(list_tx_item.txid, txid); + assert!(list_tx_item.height.is_some()); + + // Invalidate 1 block + test_client.invalidate(1); + + wallet.sync(None).unwrap(); + + assert_eq!(wallet.get_balance().unwrap(), 50_000); + + let list_tx_item = &wallet.list_transactions(false).unwrap()[0]; + assert_eq!(list_tx_item.txid, txid); + assert_eq!(list_tx_item.height, None); + } + + #[test] + #[serial] + fn test_sync_after_send() { + let (wallet, descriptors, mut test_client) = init_single_sig(); + let node_addr = test_client.get_node_address(None); + + test_client.receive(testutils! { + @tx ( (@external descriptors, 0) => 50_000 ) + }); + + wallet.sync(None).unwrap(); + assert_eq!(wallet.get_balance().unwrap(), 50_000); + + let (psbt, details) = wallet.create_tx(TxBuilder::from_addressees(vec![(node_addr, 25_000)])).unwrap(); + let (psbt, finalized) = wallet.sign(psbt, None).unwrap(); + assert!(finalized, "Cannot finalize transaction"); + wallet.broadcast(psbt.extract_tx()).unwrap(); + + wallet.sync(None).unwrap(); + assert_eq!(wallet.get_balance().unwrap(), details.received); + + assert_eq!(wallet.list_transactions(false).unwrap().len(), 2); + assert_eq!(wallet.list_unspent().unwrap().len(), 1); + } + + #[test] + #[serial] + fn test_sync_outgoing_from_scratch() { + let (wallet, descriptors, mut test_client) = init_single_sig(); + let node_addr = test_client.get_node_address(None); + + let received_txid = test_client.receive(testutils! { + @tx ( (@external descriptors, 0) => 50_000 ) + }); + + wallet.sync(None).unwrap(); + assert_eq!(wallet.get_balance().unwrap(), 50_000); + + let (psbt, details) = wallet.create_tx(TxBuilder::from_addressees(vec![(node_addr, 25_000)])).unwrap(); + let (psbt, finalized) = wallet.sign(psbt, None).unwrap(); + assert!(finalized, "Cannot finalize transaction"); + let sent_txid = wallet.broadcast(psbt.extract_tx()).unwrap(); + + wallet.sync(None).unwrap(); + assert_eq!(wallet.get_balance().unwrap(), details.received); + + // empty wallet + let wallet = get_wallet_from_descriptors(&descriptors); + wallet.sync(None).unwrap(); + + let tx_map = wallet.list_transactions(false).unwrap().into_iter().map(|tx| (tx.txid, tx)).collect::>(); + + let received = tx_map.get(&received_txid).unwrap(); + assert_eq!(received.received, 50_000); + assert_eq!(received.sent, 0); + + let sent = tx_map.get(&sent_txid).unwrap(); + assert_eq!(sent.received, details.received); + assert_eq!(sent.sent, details.sent); + assert_eq!(sent.fees, details.fees); + } + + #[test] + #[serial] + fn test_sync_long_change_chain() { + let (wallet, descriptors, mut test_client) = init_single_sig(); + let node_addr = test_client.get_node_address(None); + + test_client.receive(testutils! { + @tx ( (@external descriptors, 0) => 50_000 ) + }); + + wallet.sync(None).unwrap(); + assert_eq!(wallet.get_balance().unwrap(), 50_000); + + let mut total_sent = 0; + for _ in 0..5 { + let (psbt, details) = wallet.create_tx(TxBuilder::from_addressees(vec![(node_addr.clone(), 5_000)])).unwrap(); + let (psbt, finalized) = wallet.sign(psbt, None).unwrap(); + assert!(finalized, "Cannot finalize transaction"); + wallet.broadcast(psbt.extract_tx()).unwrap(); + + wallet.sync(None).unwrap(); + + total_sent += 5_000 + details.fees; + } + + wallet.sync(None).unwrap(); + assert_eq!(wallet.get_balance().unwrap(), 50_000 - total_sent); + + // empty wallet + let wallet = get_wallet_from_descriptors(&descriptors); + wallet.sync(None).unwrap(); + assert_eq!(wallet.get_balance().unwrap(), 50_000 - total_sent); + } + } + + }; + + output.into() + } + } +} diff --git a/testutils/.gitignore b/testutils/.gitignore new file mode 100644 index 00000000..2c96eb1b --- /dev/null +++ b/testutils/.gitignore @@ -0,0 +1,2 @@ +target/ +Cargo.lock diff --git a/testutils/Cargo.toml b/testutils/Cargo.toml new file mode 100644 index 00000000..43c5c8eb --- /dev/null +++ b/testutils/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "testutils" +version = "0.1.0" +authors = ["Alekos Filini "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +# The latest bitcoincore-rpc depends on an older version of bitcoin, which in turns depends on an +# older version of secp256k1, which causes conflicts during linking. Use my fork right now, we can +# switch back to crates.io as soon as rust-bitcoin is updated in rust-bitcoincore-rpc. +# +# Tracking issue: https://github.com/rust-bitcoin/rust-bitcoincore-rpc/pull/80 + +[dependencies] +log = "0.4.8" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +serial_test = "0.4" +bitcoin = "0.23" +bitcoincore-rpc = "0.11" +electrum-client = "0.2.0-beta.1" diff --git a/testutils/src/lib.rs b/testutils/src/lib.rs new file mode 100644 index 00000000..093711cf --- /dev/null +++ b/testutils/src/lib.rs @@ -0,0 +1,507 @@ +#[macro_use] +extern crate serde_json; +#[macro_use] +extern crate serial_test; + +pub use serial_test::serial; + +use std::collections::HashMap; +use std::env; +use std::ops::Deref; +use std::path::PathBuf; +use std::str::FromStr; +use std::sync::Mutex; +use std::time::Duration; + +#[allow(unused_imports)] +use log::{debug, error, info, trace}; + +use bitcoin::consensus::encode::{deserialize, serialize}; +use bitcoin::hashes::hex::{FromHex, ToHex}; +use bitcoin::hashes::sha256d; +use bitcoin::{Address, Amount, Script, Transaction, Txid}; + +pub use bitcoincore_rpc::bitcoincore_rpc_json::AddressType; +pub use bitcoincore_rpc::{Auth, Client as RpcClient, RpcApi}; + +pub use electrum_client::{Client as ElectrumClient, ElectrumApi}; + +lazy_static! { + static ref SYNC_TESTS_MUTEX: Mutex<()> = Mutex::new(()); +} + +pub fn test_init() {} + +// TODO: we currently only support env vars, we could also parse a toml file +fn get_auth() -> Auth { + match env::var("MAGICAL_RPC_AUTH").as_ref().map(String::as_ref) { + Ok("USER_PASS") => Auth::UserPass( + env::var("MAGICAL_RPC_USER").unwrap(), + env::var("MAGICAL_RPC_PASS").unwrap(), + ), + _ => Auth::CookieFile(PathBuf::from( + env::var("MAGICAL_RPC_COOKIEFILE") + .unwrap_or("/home/user/.bitcoin/regtest/.cookie".to_string()), + )), + } +} + +pub fn get_electrum_url() -> String { + env::var("MAGICAL_ELECTRUM_URL").unwrap_or("tcp://127.0.0.1:50001".to_string()) +} + +pub struct TestClient { + client: RpcClient, + electrum: ElectrumClient, +} + +#[derive(Clone, Debug)] +pub struct TestIncomingOutput { + pub value: u64, + pub to_address: String, +} + +impl TestIncomingOutput { + pub fn new(value: u64, to_address: Address) -> Self { + Self { + value, + to_address: to_address.to_string(), + } + } +} + +#[derive(Clone, Debug)] +pub struct TestIncomingTx { + pub output: Vec, + pub min_confirmations: Option, + pub locktime: Option, + pub replaceable: Option, +} + +impl TestIncomingTx { + pub fn new( + output: Vec, + min_confirmations: Option, + locktime: Option, + replaceable: Option, + ) -> Self { + Self { + output, + min_confirmations, + locktime, + replaceable, + } + } + + pub fn add_output(&mut self, output: TestIncomingOutput) { + self.output.push(output); + } +} + +#[macro_export] +macro_rules! testutils { + ( @external $descriptors:expr, $child:expr ) => ({ + $descriptors.0.derive($child).expect("Derivation error").address(bitcoin::Network::Regtest).expect("No address form") + }); + ( @internal $descriptors:expr, $child:expr ) => ({ + $descriptors.1.expect("Missing internal descriptor").derive($child).expect("Derivation error").address(bitcoin::Network::Regtest).expect("No address form") + }); + ( @e $descriptors:expr, $child:expr ) => ({ testutils!(@external $descriptors, $child) }); + ( @i $descriptors:expr, $child:expr ) => ({ testutils!(@internal $descriptors, $child) }); + + ( @tx ( $( ( $( $addr:tt )* ) => $amount:expr ),+ ) $( ( @locktime $locktime:expr ) )* $( ( @confirmations $confirmations:expr ) )* $( ( @replaceable $replaceable:expr ) )* ) => ({ + let mut outs = Vec::new(); + $( outs.push(testutils::TestIncomingOutput::new($amount, testutils!( $($addr)* ))); )+ + + let mut locktime = None::; + $( locktime = Some($locktime); )* + + let mut min_confirmations = None::; + $( min_confirmations = Some($confirmations); )* + + let mut replaceable = None::; + $( replaceable = Some($replaceable); )* + + testutils::TestIncomingTx::new(outs, min_confirmations, locktime, replaceable) + }); + + ( @literal $key:expr ) => ({ + let key = $key.to_string(); + (key, None::, None::) + }); + ( @generate_xprv $( $external_path:expr )* $( ,$internal_path:expr )* ) => ({ + use rand::Rng; + + let mut seed = [0u8; 32]; + rand::thread_rng().fill(&mut seed[..]); + + let key = bitcoin::util::bip32::ExtendedPrivKey::new_master( + bitcoin::Network::Testnet, + &seed, + ); + + let mut external_path = None::; + $( external_path = Some($external_path.to_string()); )* + + let mut internal_path = None::; + $( internal_path = Some($internal_path.to_string()); )* + + (key.unwrap().to_string(), external_path, internal_path) + }); + ( @generate_wif ) => ({ + use rand::Rng; + + let mut key = [0u8; bitcoin::secp256k1::constants::SECRET_KEY_SIZE]; + rand::thread_rng().fill(&mut key[..]); + + (bitcoin::PrivateKey { + compressed: true, + network: bitcoin::Network::Testnet, + key: bitcoin::secp256k1::SecretKey::from_slice(&key).unwrap(), + }.to_string(), None::, None::) + }); + + ( @keys ( $( $alias:expr => ( $( $key_type:tt )* ) ),+ ) ) => ({ + let mut map = std::collections::HashMap::new(); + $( + let alias: &str = $alias; + map.insert(alias, testutils!( $($key_type)* )); + )+ + + map + }); + + ( @descriptors ( $external_descriptor:expr ) $( ( $internal_descriptor:expr ) )* $( ( @keys $( $keys:tt )* ) )* ) => ({ + use std::str::FromStr; + use std::collections::HashMap; + use std::convert::TryInto; + + let mut keys: HashMap<&'static str, (String, Option, Option)> = HashMap::new(); + $( + keys = testutils!{ @keys $( $keys )* }; + )* + + let external: Descriptor = FromStr::from_str($external_descriptor).unwrap(); + let external: Descriptor = external.translate_pk::<_, _, _, &'static str>(|k| { + if let Some((key, ext_path, _)) = keys.get(&k.as_str()) { + Ok(format!("{}{}", key, ext_path.as_ref().unwrap_or(&"".into()))) + } else { + Ok(k.clone()) + } + }, |kh| { + if let Some((key, ext_path, _)) = keys.get(&kh.as_str()) { + Ok(format!("{}{}", key, ext_path.as_ref().unwrap_or(&"".into()))) + } else { + Ok(kh.clone()) + } + + }).unwrap(); + let external: ExtendedDescriptor = external.try_into().unwrap(); + + let mut internal = None::; + $( + let string_internal: Descriptor = FromStr::from_str($internal_descriptor).unwrap(); + + let string_internal: Descriptor = string_internal.translate_pk::<_, _, _, &'static str>(|k| { + if let Some((key, _, int_path)) = keys.get(&k.as_str()) { + Ok(format!("{}{}", key, int_path.as_ref().unwrap_or(&"".into()))) + } else { + Ok(k.clone()) + } + }, |kh| { + if let Some((key, _, int_path)) = keys.get(&kh.as_str()) { + Ok(format!("{}{}", key, int_path.as_ref().unwrap_or(&"".into()))) + } else { + Ok(kh.clone()) + } + + }).unwrap(); + internal = Some(string_internal.try_into().unwrap()); + + )* + + (external, internal) + }) +} + +fn exponential_backoff_poll(mut poll: F) -> T +where + F: FnMut() -> Option, +{ + let mut delay = Duration::from_millis(64); + loop { + match poll() { + Some(data) => break data, + None if delay.as_millis() < 512 => delay = delay.mul_f32(2.0), + None => {} + } + + std::thread::sleep(delay); + } +} + +impl TestClient { + pub fn new() -> Self { + let url = env::var("MAGICAL_RPC_URL").unwrap_or("127.0.0.1:18443".to_string()); + let client = RpcClient::new(format!("http://{}", url), get_auth()).unwrap(); + let electrum = ElectrumClient::new(&get_electrum_url(), None).unwrap(); + + TestClient { client, electrum } + } + + fn wait_for_tx(&mut self, txid: Txid, monitor_script: &Script) { + // wait for electrs to index the tx + exponential_backoff_poll(|| { + trace!("wait_for_tx {}", txid); + + self.electrum + .script_get_history(monitor_script) + .unwrap() + .iter() + .position(|entry| entry.tx_hash == txid) + }); + } + + fn wait_for_block(&mut self, min_height: usize) { + self.electrum.block_headers_subscribe().unwrap(); + + loop { + let header = exponential_backoff_poll(|| { + self.electrum.ping().unwrap(); + self.electrum.block_headers_pop().unwrap() + }); + if header.height >= min_height { + break; + } + } + } + + pub fn receive(&mut self, meta_tx: TestIncomingTx) -> Txid { + assert!( + meta_tx.output.len() > 0, + "can't create a transaction with no outputs" + ); + + let mut map = HashMap::new(); + + let mut required_balance = 0; + for out in &meta_tx.output { + required_balance += out.value; + map.insert(out.to_address.clone(), Amount::from_sat(out.value)); + } + + if self.get_balance(None, None).unwrap() < Amount::from_sat(required_balance) { + panic!("Insufficient funds in bitcoind. Plase generate a few blocks with: `bitcoin-cli generatetoaddress 10 {}`", self.get_new_address(None, None).unwrap()); + } + + // FIXME: core can't create a tx with two outputs to the same address + let tx = self + .create_raw_transaction_hex(&[], &map, meta_tx.locktime, meta_tx.replaceable) + .unwrap(); + let tx = self.fund_raw_transaction(tx, None, None).unwrap(); + let mut tx: Transaction = deserialize(&tx.hex).unwrap(); + + if let Some(true) = meta_tx.replaceable { + // for some reason core doesn't set this field right + for input in &mut tx.input { + input.sequence = 0xFFFFFFFD; + } + } + + let tx = self + .sign_raw_transaction_with_wallet(&serialize(&tx), None, None) + .unwrap(); + + // broadcast through electrum so that it caches the tx immediately + let txid = self + .electrum + .transaction_broadcast(&deserialize(&tx.hex).unwrap()) + .unwrap(); + + if let Some(num) = meta_tx.min_confirmations { + self.generate(num); + } + + let monitor_script = Address::from_str(&meta_tx.output[0].to_address) + .unwrap() + .script_pubkey(); + self.wait_for_tx(txid, &monitor_script); + + debug!("Sent tx: {}", txid); + + txid + } + + pub fn bump_fee(&mut self, txid: &Txid) -> Txid { + let tx = self.get_raw_transaction_info(txid, None).unwrap(); + assert!( + tx.confirmations.is_none(), + "Can't bump tx {} because it's already confirmed", + txid + ); + + let bumped: serde_json::Value = self.call("bumpfee", &[txid.to_string().into()]).unwrap(); + let new_txid = Txid::from_str(&bumped["txid"].as_str().unwrap().to_string()).unwrap(); + + let monitor_script = + tx.vout[0].script_pub_key.addresses.as_ref().unwrap()[0].script_pubkey(); + self.wait_for_tx(new_txid, &monitor_script); + + debug!("Bumped {}, new txid {}", txid, new_txid); + + new_txid + } + + pub fn generate_manually(&mut self, txs: Vec) -> String { + use bitcoin::blockdata::block::{Block, BlockHeader}; + use bitcoin::blockdata::script::Builder; + use bitcoin::blockdata::transaction::{OutPoint, TxIn, TxOut}; + use bitcoin::hash_types::{BlockHash, TxMerkleNode}; + use bitcoin::util::hash::BitcoinHash; + + let block_template: serde_json::Value = self + .call("getblocktemplate", &[json!({"rules": ["segwit"]})]) + .unwrap(); + trace!("getblocktemplate: {:#?}", block_template); + + let header = BlockHeader { + version: block_template["version"].as_u64().unwrap() as u32, + prev_blockhash: BlockHash::from_hex( + block_template["previousblockhash"].as_str().unwrap(), + ) + .unwrap(), + merkle_root: TxMerkleNode::default(), + time: block_template["curtime"].as_u64().unwrap() as u32, + bits: u32::from_str_radix(block_template["bits"].as_str().unwrap(), 16).unwrap(), + nonce: 0, + }; + debug!("header: {:#?}", header); + + let height = block_template["height"].as_u64().unwrap() as i64; + let witness_reserved_value: Vec = sha256d::Hash::default().as_ref().into(); + // burn block subsidy and fees, not a big deal + let mut coinbase_tx = Transaction { + version: 1, + lock_time: 0, + input: vec![TxIn { + previous_output: OutPoint::null(), + script_sig: Builder::new().push_int(height).into_script(), + sequence: 0xFFFFFFFF, + witness: vec![witness_reserved_value], + }], + output: vec![], + }; + + let mut txdata = vec![coinbase_tx.clone()]; + txdata.extend_from_slice(&txs); + + let mut block = Block { header, txdata }; + + let witness_root = block.witness_root(); + let witness_commitment = + Block::compute_witness_commitment(&witness_root, &coinbase_tx.input[0].witness[0]); + + // now update and replace the coinbase tx + let mut coinbase_witness_commitment_script = vec![0x6a, 0x24, 0xaa, 0x21, 0xa9, 0xed]; + coinbase_witness_commitment_script.extend_from_slice(&witness_commitment); + + coinbase_tx.output.push(TxOut { + value: 0, + script_pubkey: coinbase_witness_commitment_script.into(), + }); + block.txdata[0] = coinbase_tx; + + // set merkle root + let merkle_root = block.merkle_root(); + block.header.merkle_root = merkle_root; + + assert!(block.check_merkle_root()); + assert!(block.check_witness_commitment()); + + // now do PoW :) + let target = block.header.target(); + while block.header.validate_pow(&target).is_err() { + block.header.nonce = block.header.nonce.checked_add(1).unwrap(); // panic if we run out of nonces + } + + let block_hex: String = serialize(&block).to_hex(); + debug!("generated block hex: {}", block_hex); + + self.electrum.block_headers_subscribe().unwrap(); + + let submit_result: serde_json::Value = + self.call("submitblock", &[block_hex.into()]).unwrap(); + debug!("submitblock: {:?}", submit_result); + assert!( + submit_result.is_null(), + "submitblock error: {:?}", + submit_result.as_str() + ); + + self.wait_for_block(height as usize); + + block.header.bitcoin_hash().to_hex() + } + + pub fn generate(&mut self, num_blocks: u64) { + let our_addr = self.get_new_address(None, None).unwrap(); + let hashes = self.generate_to_address(num_blocks, &our_addr).unwrap(); + let best_hash = hashes.last().unwrap(); + let height = self.get_block_info(best_hash).unwrap().height; + + self.wait_for_block(height); + + debug!("Generated blocks to new height {}", height); + } + + pub fn invalidate(&mut self, num_blocks: u64) { + self.electrum.block_headers_subscribe().unwrap(); + + let best_hash = self.get_best_block_hash().unwrap(); + let initial_height = self.get_block_info(&best_hash).unwrap().height; + + let mut to_invalidate = best_hash; + for i in 1..=num_blocks { + trace!( + "Invalidating block {}/{} ({})", + i, + num_blocks, + to_invalidate + ); + + self.invalidate_block(&to_invalidate).unwrap(); + to_invalidate = self.get_best_block_hash().unwrap(); + } + + self.wait_for_block(initial_height - num_blocks as usize); + + debug!( + "Invalidated {} blocks to new height of {}", + num_blocks, + initial_height - num_blocks as usize + ); + } + + pub fn reorg(&mut self, num_blocks: u64) { + self.invalidate(num_blocks); + self.generate(num_blocks); + } + + pub fn get_node_address(&self, address_type: Option) -> Address { + Address::from_str( + &self + .get_new_address(None, address_type) + .unwrap() + .to_string(), + ) + .unwrap() + } +} + +impl Deref for TestClient { + type Target = RpcClient; + + fn deref(&self) -> &Self::Target { + &self.client + } +}