[tests] Add a proc macro to generate tests for OnlineBlockchain types

Cargo.toml

@@ -40,7 +40,14 @@ key-value-db = ["sled"]
 cli-utils = ["clap", "base64"]
 async-interface = ["async-trait"]
 
+# Debug/Test features
+debug-proc-macros = ["magical-macros/debug", "testutils-macros/debug"]
+test-electrum = ["electrum"]
+
 [dev-dependencies]
+testutils = { path = "./testutils" }
+testutils-macros = { path = "./testutils-macros" }
+serial_test = "0.4"
 lazy_static = "1.4"
 rustyline = "6.0"
 dirs = "2.0"
@@ -67,3 +74,5 @@ name = "magic"
 path = "examples/repl.rs"
 required-features = ["cli-utils"]
 
+[workspace]
+members = ["macros", "testutils", "testutils-macros"]

macros/Cargo.toml

@@ -11,5 +11,8 @@ syn = { version = "1.0", features = ["parsing"] }
 proc-macro2 = "1.0"
 quote = "1.0"
 
+[features]
+debug = ["syn/extra-traits"]
+
 [lib]
 proc-macro = true

macros/src/lib.rs

@@ -85,7 +85,8 @@ pub fn maybe_async(_attr: TokenStream, item: TokenStream) -> TokenStream {
     } else {
         (quote! {
             compile_error!("#[maybe_async] can only be used on methods, trait or trait impl blocks")
-        }).into()
+        })
+        .into()
     }
 }
 

src/blockchain/electrum.rs

@@ -15,6 +15,13 @@ use crate::FeeRate;
 
 pub struct ElectrumBlockchain(Option<Client>);
 
+#[cfg(test)]
+#[cfg(feature = "test-electrum")]
+#[magical_blockchain_tests(crate)]
+fn local_electrs() -> ElectrumBlockchain {
+    ElectrumBlockchain::from(Client::new(&testutils::get_electrum_url(), None).unwrap())
+}
+
 impl std::convert::From<Client> for ElectrumBlockchain {
     fn from(client: Client) -> Self {
         ElectrumBlockchain(Some(client))

src/blockchain/utils.rs

@@ -186,7 +186,6 @@ pub trait ElectrumLikeSync {
         );
         let mut updates = database.begin_batch();
         let tx = match database.get_tx(&txid, true)? {
-            // TODO: do we need the raw?
             Some(mut saved_tx) => {
                 // update the height if it's different (in case of reorg)
                 if saved_tx.height != height {
@@ -204,12 +203,20 @@ pub trait ElectrumLikeSync {
                 // went wrong
                 saved_tx.transaction.unwrap()
             }
-            None => maybe_await!(self.els_transaction_get(&txid))?,
+            None => {
+                let fetched_tx = maybe_await!(self.els_transaction_get(&txid))?;
+                database.set_raw_tx(&fetched_tx)?;
+
+                fetched_tx
+            }
         };
 
         let mut incoming: u64 = 0;
         let mut outgoing: u64 = 0;
 
+        let mut inputs_sum: u64 = 0;
+        let mut outputs_sum: u64 = 0;
+
         // look for our own inputs
         for (i, input) in tx.input.iter().enumerate() {
             // the fact that we visit addresses in a BFS fashion starting from the external addresses
@@ -217,17 +224,37 @@ pub trait ElectrumLikeSync {
             // the transactions at a lower depth have already been indexed, so if an outpoint is ours
             // we are guaranteed to have it in the db).
             if let Some(previous_output) = database.get_previous_output(&input.previous_output)? {
+                inputs_sum += previous_output.value;
+
                 if database.is_mine(&previous_output.script_pubkey)? {
                     outgoing += previous_output.value;
 
                     debug!("{} input #{} is mine, removing from utxo", txid, i);
                     updates.del_utxo(&input.previous_output)?;
                 }
+            } else {
+                // The input is not ours, but we still need to count it for the fees. so fetch the
+                // tx (from the database or from network) and check it
+                let tx = match database.get_tx(&input.previous_output.txid, true)? {
+                    Some(saved_tx) => saved_tx.transaction.unwrap(),
+                    None => {
+                        let fetched_tx =
+                            maybe_await!(self.els_transaction_get(&input.previous_output.txid))?;
+                        database.set_raw_tx(&fetched_tx)?;
+
+                        fetched_tx
+                    }
+                };
+
+                inputs_sum += tx.output[input.previous_output.vout as usize].value;
             }
         }
 
         let mut to_check_later = vec![];
         for (i, output) in tx.output.iter().enumerate() {
+            // to compute the fees later
+            outputs_sum += output.value;
+
             // this output is ours, we have a path to derive it
             if let Some((script_type, child)) =
                 database.get_path_from_script_pubkey(&output.script_pubkey)?
@@ -259,6 +286,7 @@ pub trait ElectrumLikeSync {
             sent: outgoing,
             height,
             timestamp: 0,
+            fees: inputs_sum - outputs_sum,
         };
         info!("Saving tx {}", txid);
         updates.set_tx(&tx)?;

src/database/mod.rs

@@ -8,6 +8,8 @@ use crate::types::*;
 pub mod keyvalue;
 pub mod memory;
 
+pub use memory::MemoryDatabase;
+
 pub trait BatchOperations {
     fn set_script_pubkey(
         &mut self,
@@ -235,6 +237,7 @@ pub mod test {
             timestamp: 123456,
             received: 1337,
             sent: 420420,
+            fees: 140,
             height: Some(1000),
         };
 

src/error.rs

@@ -10,7 +10,7 @@ pub enum Error {
     SendAllMultipleOutputs,
     OutputBelowDustLimit(usize),
     InsufficientFunds,
-    InvalidAddressNetork(Address),
+    InvalidAddressNetwork(Address),
     UnknownUTXO,
     DifferentTransactions,
 

src/lib.rs

@@ -31,6 +31,16 @@ pub extern crate sled;
 #[cfg(feature = "cli-utils")]
 pub mod cli;
 
+#[cfg(test)]
+#[macro_use]
+extern crate testutils;
+#[cfg(test)]
+#[macro_use]
+extern crate testutils_macros;
+#[cfg(test)]
+#[macro_use]
+extern crate serial_test;
+
 #[macro_use]
 pub mod error;
 pub mod blockchain;

src/types.rs

@@ -6,7 +6,7 @@ use bitcoin::hash_types::Txid;
 use serde::{Deserialize, Serialize};
 
 // TODO serde flatten?
-#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)]
+#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Hash)]
 pub enum ScriptType {
     External = 0,
     Internal = 1,
@@ -48,5 +48,6 @@ pub struct TransactionDetails {
     pub timestamp: u64,
     pub received: u64,
     pub sent: u64,
+    pub fees: u64,
     pub height: Option<u32>,
 }

src/wallet/export.rs

@@ -129,6 +129,7 @@ mod test {
             timestamp: 12345678,
             received: 100_000,
             sent: 0,
+            fees: 500,
             height: Some(5000),
         })
         .unwrap();

src/wallet/mod.rs

@@ -24,7 +24,7 @@ pub mod tx_builder;
 pub mod utils;
 
 use tx_builder::TxBuilder;
-use utils::{FeeRate, IsDust};
+use utils::IsDust;
 
 use crate::blockchain::{noop_progress, Blockchain, OfflineBlockchain, OnlineBlockchain};
 use crate::database::{BatchDatabase, BatchOperations, DatabaseUtils};
@@ -190,8 +190,9 @@ where
                 false => *satoshi,
             };
 
-            if address.network != self.network {
+            // TODO: proper checks for testnet/regtest p2sh/p2pkh
-                return Err(Error::InvalidAddressNetork(address.clone()));
+            if address.network != self.network && self.network != Network::Regtest {
+                return Err(Error::InvalidAddressNetwork(address.clone()));
             } else if self.is_mine(&address.script_pubkey())? {
                 received += value;
             }
@@ -263,7 +264,8 @@ where
             }
         };
 
-        let change_val = total_amount - outgoing - (fee_amount.ceil() as u64);
+        let mut fee_amount = fee_amount.ceil() as u64;
+        let change_val = total_amount - outgoing - fee_amount;
         if !builder.send_all && !change_val.is_dust() {
             let mut change_output = change_output.unwrap();
             change_output.value = change_val;
@@ -271,8 +273,6 @@ where
 
             tx.output.push(change_output);
         } else if builder.send_all && !change_val.is_dust() {
-            // set the outgoing value to whatever we've put in
-            outgoing = total_amount;
             // there's only one output, send everything to it
             tx.output[0].value = change_val;
 
@@ -280,6 +280,9 @@ where
             if self.is_mine(&tx.output[0].script_pubkey)? {
                 received = change_val;
             }
+        } else if !builder.send_all && change_val.is_dust() {
+            // skip the change output because it's dust, this adds up to the fees
+            fee_amount += change_val;
         } else if builder.send_all {
             // send_all but the only output would be below dust limit
             return Err(Error::InsufficientFunds); // TODO: or OutputBelowDustLimit?
@@ -339,7 +342,8 @@ where
             txid,
             timestamp: time::get_timestamp(),
             received,
-            sent: outgoing,
+            sent: total_amount,
+            fees: fee_amount,
             height: None,
         };
 
@@ -750,6 +754,8 @@ where
     pub fn sync(&self, max_address_param: Option<u32>) -> Result<(), Error> {
         debug!("Begin sync...");
 
+        let mut run_setup = false;
+
         let max_address = match self.descriptor.is_fixed() {
             true => 0,
             false => max_address_param.unwrap_or(CACHE_ADDR_BATCH_SIZE),
@@ -760,6 +766,7 @@ where
             .get_script_pubkey_from_path(ScriptType::External, max_address)?
             .is_none()
         {
+            run_setup = true;
             self.cache_addresses(ScriptType::External, 0, max_address)?;
         }
 
@@ -775,15 +782,24 @@ where
                 .get_script_pubkey_from_path(ScriptType::Internal, max_address)?
                 .is_none()
             {
+                run_setup = true;
                 self.cache_addresses(ScriptType::Internal, 0, max_address)?;
             }
         }
 
-        maybe_await!(self.client.sync(
+        if run_setup {
-            None,
+            maybe_await!(self.client.setup(
-            self.database.borrow_mut().deref_mut(),
+                None,
-            noop_progress(),
+                self.database.borrow_mut().deref_mut(),
-        ))
+                noop_progress(),
+            ))
+        } else {
+            maybe_await!(self.client.sync(
+                None,
+                self.database.borrow_mut().deref_mut(),
+                noop_progress(),
+            ))
+        }
     }
 
     pub fn client(&self) -> &B {

testutils-macros/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "testutils-macros"
+version = "0.1.0"
+authors = ["Alekos Filini <alekos.filini@gmail.com>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+syn = { version = "1.0", features = ["parsing"] }
+proc-macro2 = "1.0"
+quote = "1.0"
+
+[features]
+debug = ["syn/extra-traits"]
+
+[lib]
+proc-macro = true

testutils-macros/src/lib.rs

@@ -0,0 +1,373 @@
+#[macro_use]
+extern crate quote;
+
+use proc_macro::TokenStream;
+
+use syn::spanned::Spanned;
+use syn::{parse, parse2, Ident, ReturnType};
+
+#[proc_macro_attribute]
+pub fn magical_blockchain_tests(attr: TokenStream, item: TokenStream) -> TokenStream {
+    let root_ident = if !attr.is_empty() {
+        match parse::<syn::ExprPath>(attr) {
+            Ok(parsed) => parsed,
+            Err(e) => {
+                let error_string = e.to_string();
+                return (quote! {
+                    compile_error!("Invalid crate path: {:?}", #error_string)
+                })
+                .into();
+            }
+        }
+    } else {
+        parse2::<syn::ExprPath>(quote! { magical_bitcoin_wallet }).unwrap()
+    };
+
+    match parse::<syn::ItemFn>(item) {
+        Err(_) => (quote! {
+            compile_error!("#[magical_blockchain_tests] can only be used on `fn`s")
+        })
+        .into(),
+        Ok(parsed) => {
+            let parsed_sig_ident = parsed.sig.ident.clone();
+            let mod_name = Ident::new(
+                &format!("generated_tests_{}", parsed_sig_ident.to_string()),
+                parsed.span(),
+            );
+
+            let return_type = match parsed.sig.output {
+                ReturnType::Type(_, ref t) => t.clone(),
+                ReturnType::Default => {
+                    return (quote! {
+                        compile_error!("The tagged function must return a type that impl `OnlineBlockchain`")
+                    }).into();
+                }
+            };
+
+            let output = quote! {
+
+            #parsed
+
+            mod #mod_name {
+                use bitcoin::Network;
+
+                use miniscript::Descriptor;
+
+                use testutils::{TestClient, serial};
+
+                use #root_ident::blockchain::OnlineBlockchain;
+                use #root_ident::descriptor::ExtendedDescriptor;
+                use #root_ident::database::MemoryDatabase;
+                use #root_ident::types::ScriptType;
+                use #root_ident::{Wallet, TxBuilder};
+
+                use super::*;
+
+                fn get_blockchain() -> #return_type {
+                    #parsed_sig_ident()
+                }
+
+                fn get_wallet_from_descriptors(descriptors: &(ExtendedDescriptor, Option<ExtendedDescriptor>)) -> Wallet<#return_type, MemoryDatabase> {
+                    Wallet::new(&descriptors.0.to_string(), descriptors.1.as_ref().map(|d| d.to_string()).as_deref(), Network::Regtest, MemoryDatabase::new(), get_blockchain()).unwrap()
+                }
+
+                fn init_single_sig() -> (Wallet<#return_type, MemoryDatabase>, (ExtendedDescriptor, Option<ExtendedDescriptor>), TestClient) {
+                    let descriptors = testutils! {
+                        @descriptors ( "wpkh(Alice)" ) ( "wpkh(Alice)" ) ( @keys ( "Alice" => (@generate_xprv "/44'/0'/0'/0/*", "/44'/0'/0'/1/*") ) )
+                    };
+
+                    let test_client = TestClient::new();
+                    let wallet = get_wallet_from_descriptors(&descriptors);
+
+                    (wallet, descriptors, test_client)
+                }
+
+                #[test]
+                #[serial]
+                fn test_sync_simple() {
+                    let (wallet, descriptors, mut test_client) = init_single_sig();
+
+                    let tx = testutils! {
+                        @tx ( (@external descriptors, 0) => 50_000 )
+                    };
+                    let txid = test_client.receive(tx);
+
+                    wallet.sync(None).unwrap();
+
+                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
+                    assert_eq!(wallet.list_unspent().unwrap()[0].is_internal, false);
+
+                    let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
+                    assert_eq!(list_tx_item.txid, txid);
+                    assert_eq!(list_tx_item.received, 50_000);
+                    assert_eq!(list_tx_item.sent, 0);
+                    assert_eq!(list_tx_item.height, None);
+                }
+
+                #[test]
+                #[serial]
+                fn test_sync_stop_gap_20() {
+                    let (wallet, descriptors, mut test_client) = init_single_sig();
+
+                    test_client.receive(testutils! {
+                        @tx ( (@external descriptors, 5) => 50_000 )
+                    });
+                    test_client.receive(testutils! {
+                        @tx ( (@external descriptors, 25) => 50_000 )
+                    });
+
+                    wallet.sync(None).unwrap();
+
+                    assert_eq!(wallet.get_balance().unwrap(), 100_000);
+                    assert_eq!(wallet.list_transactions(false).unwrap().len(), 2);
+                }
+
+                #[test]
+                #[serial]
+                fn test_sync_before_and_after_receive() {
+                    let (wallet, descriptors, mut test_client) = init_single_sig();
+
+                    wallet.sync(None).unwrap();
+                    assert_eq!(wallet.get_balance().unwrap(), 0);
+
+                    test_client.receive(testutils! {
+                        @tx ( (@external descriptors, 0) => 50_000 )
+                    });
+
+                    wallet.sync(None).unwrap();
+
+                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
+                    assert_eq!(wallet.list_transactions(false).unwrap().len(), 1);
+                }
+
+                #[test]
+                #[serial]
+                fn test_sync_multiple_outputs_same_tx() {
+                    let (wallet, descriptors, mut test_client) = init_single_sig();
+
+                    let txid = test_client.receive(testutils! {
+                        @tx ( (@external descriptors, 0) => 50_000, (@external descriptors, 1) => 25_000, (@external descriptors, 5) => 30_000 )
+                    });
+
+                    wallet.sync(None).unwrap();
+
+                    assert_eq!(wallet.get_balance().unwrap(), 105_000);
+                    assert_eq!(wallet.list_transactions(false).unwrap().len(), 1);
+                    assert_eq!(wallet.list_unspent().unwrap().len(), 3);
+
+                    let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
+                    assert_eq!(list_tx_item.txid, txid);
+                    assert_eq!(list_tx_item.received, 105_000);
+                    assert_eq!(list_tx_item.sent, 0);
+                    assert_eq!(list_tx_item.height, None);
+                }
+
+                #[test]
+                #[serial]
+                fn test_sync_receive_multi() {
+                    let (wallet, descriptors, mut test_client) = init_single_sig();
+
+                    test_client.receive(testutils! {
+                        @tx ( (@external descriptors, 0) => 50_000 )
+                    });
+                    test_client.receive(testutils! {
+                        @tx ( (@external descriptors, 5) => 25_000 )
+                    });
+
+                    wallet.sync(None).unwrap();
+
+                    assert_eq!(wallet.get_balance().unwrap(), 75_000);
+                    assert_eq!(wallet.list_transactions(false).unwrap().len(), 2);
+                    assert_eq!(wallet.list_unspent().unwrap().len(), 2);
+                }
+
+                #[test]
+                #[serial]
+                fn test_sync_address_reuse() {
+                    let (wallet, descriptors, mut test_client) = init_single_sig();
+
+                    test_client.receive(testutils! {
+                        @tx ( (@external descriptors, 0) => 50_000 )
+                    });
+
+                    wallet.sync(None).unwrap();
+                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
+
+                    test_client.receive(testutils! {
+                        @tx ( (@external descriptors, 0) => 25_000 )
+                    });
+
+                    wallet.sync(None).unwrap();
+                    assert_eq!(wallet.get_balance().unwrap(), 75_000);
+                }
+
+                #[test]
+                #[serial]
+                fn test_sync_receive_rbf_replaced() {
+                    let (wallet, descriptors, mut test_client) = init_single_sig();
+
+                    let txid = test_client.receive(testutils! {
+                        @tx ( (@external descriptors, 0) => 50_000 ) ( @replaceable true )
+                    });
+
+                    wallet.sync(None).unwrap();
+
+                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
+                    assert_eq!(wallet.list_transactions(false).unwrap().len(), 1);
+                    assert_eq!(wallet.list_unspent().unwrap().len(), 1);
+
+                    let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
+                    assert_eq!(list_tx_item.txid, txid);
+                    assert_eq!(list_tx_item.received, 50_000);
+                    assert_eq!(list_tx_item.sent, 0);
+                    assert_eq!(list_tx_item.height, None);
+
+                    let new_txid = test_client.bump_fee(&txid);
+
+                    wallet.sync(None).unwrap();
+
+                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
+                    assert_eq!(wallet.list_transactions(false).unwrap().len(), 1);
+                    assert_eq!(wallet.list_unspent().unwrap().len(), 1);
+
+                    let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
+                    assert_eq!(list_tx_item.txid, new_txid);
+                    assert_eq!(list_tx_item.received, 50_000);
+                    assert_eq!(list_tx_item.sent, 0);
+                    assert_eq!(list_tx_item.height, None);
+                }
+
+                #[test]
+                #[serial]
+                fn test_sync_reorg_block() {
+                    let (wallet, descriptors, mut test_client) = init_single_sig();
+
+                    let txid = test_client.receive(testutils! {
+                        @tx ( (@external descriptors, 0) => 50_000 ) ( @confirmations 1 ) ( @replaceable true )
+                    });
+
+                    wallet.sync(None).unwrap();
+
+                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
+                    assert_eq!(wallet.list_transactions(false).unwrap().len(), 1);
+                    assert_eq!(wallet.list_unspent().unwrap().len(), 1);
+
+                    let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
+                    assert_eq!(list_tx_item.txid, txid);
+                    assert!(list_tx_item.height.is_some());
+
+                    // Invalidate 1 block
+                    test_client.invalidate(1);
+
+                    wallet.sync(None).unwrap();
+
+                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
+
+                    let list_tx_item = &wallet.list_transactions(false).unwrap()[0];
+                    assert_eq!(list_tx_item.txid, txid);
+                    assert_eq!(list_tx_item.height, None);
+                }
+
+                #[test]
+                #[serial]
+                fn test_sync_after_send() {
+                    let (wallet, descriptors, mut test_client) = init_single_sig();
+                    let node_addr = test_client.get_node_address(None);
+
+                    test_client.receive(testutils! {
+                        @tx ( (@external descriptors, 0) => 50_000 )
+                    });
+
+                    wallet.sync(None).unwrap();
+                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
+
+                    let (psbt, details) = wallet.create_tx(TxBuilder::from_addressees(vec![(node_addr, 25_000)])).unwrap();
+                    let (psbt, finalized) = wallet.sign(psbt, None).unwrap();
+                    assert!(finalized, "Cannot finalize transaction");
+                    wallet.broadcast(psbt.extract_tx()).unwrap();
+
+                    wallet.sync(None).unwrap();
+                    assert_eq!(wallet.get_balance().unwrap(), details.received);
+
+                    assert_eq!(wallet.list_transactions(false).unwrap().len(), 2);
+                    assert_eq!(wallet.list_unspent().unwrap().len(), 1);
+                }
+
+                #[test]
+                #[serial]
+                fn test_sync_outgoing_from_scratch() {
+                    let (wallet, descriptors, mut test_client) = init_single_sig();
+                    let node_addr = test_client.get_node_address(None);
+
+                    let received_txid = test_client.receive(testutils! {
+                        @tx ( (@external descriptors, 0) => 50_000 )
+                    });
+
+                    wallet.sync(None).unwrap();
+                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
+
+                    let (psbt, details) = wallet.create_tx(TxBuilder::from_addressees(vec![(node_addr, 25_000)])).unwrap();
+                    let (psbt, finalized) = wallet.sign(psbt, None).unwrap();
+                    assert!(finalized, "Cannot finalize transaction");
+                    let sent_txid = wallet.broadcast(psbt.extract_tx()).unwrap();
+
+                    wallet.sync(None).unwrap();
+                    assert_eq!(wallet.get_balance().unwrap(), details.received);
+
+                    // empty wallet
+                    let wallet = get_wallet_from_descriptors(&descriptors);
+                    wallet.sync(None).unwrap();
+
+                    let tx_map = wallet.list_transactions(false).unwrap().into_iter().map(|tx| (tx.txid, tx)).collect::<std::collections::HashMap<_, _>>();
+
+                    let received = tx_map.get(&received_txid).unwrap();
+                    assert_eq!(received.received, 50_000);
+                    assert_eq!(received.sent, 0);
+
+                    let sent = tx_map.get(&sent_txid).unwrap();
+                    assert_eq!(sent.received, details.received);
+                    assert_eq!(sent.sent, details.sent);
+                    assert_eq!(sent.fees, details.fees);
+                }
+
+                #[test]
+                #[serial]
+                fn test_sync_long_change_chain() {
+                    let (wallet, descriptors, mut test_client) = init_single_sig();
+                    let node_addr = test_client.get_node_address(None);
+
+                    test_client.receive(testutils! {
+                        @tx ( (@external descriptors, 0) => 50_000 )
+                    });
+
+                    wallet.sync(None).unwrap();
+                    assert_eq!(wallet.get_balance().unwrap(), 50_000);
+
+                    let mut total_sent = 0;
+                    for _ in 0..5 {
+                        let (psbt, details) = wallet.create_tx(TxBuilder::from_addressees(vec![(node_addr.clone(), 5_000)])).unwrap();
+                        let (psbt, finalized) = wallet.sign(psbt, None).unwrap();
+                        assert!(finalized, "Cannot finalize transaction");
+                        wallet.broadcast(psbt.extract_tx()).unwrap();
+
+                        wallet.sync(None).unwrap();
+
+                        total_sent += 5_000 + details.fees;
+                    }
+
+                    wallet.sync(None).unwrap();
+                    assert_eq!(wallet.get_balance().unwrap(), 50_000 - total_sent);
+
+                    // empty wallet
+                    let wallet = get_wallet_from_descriptors(&descriptors);
+                    wallet.sync(None).unwrap();
+                    assert_eq!(wallet.get_balance().unwrap(), 50_000 - total_sent);
+                }
+            }
+
+                        };
+
+            output.into()
+        }
+    }
+}

testutils/.gitignore

@@ -0,0 +1,2 @@
+target/
+Cargo.lock

testutils/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "testutils"
+version = "0.1.0"
+authors = ["Alekos Filini <alekos.filini@gmail.com>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+# The latest bitcoincore-rpc depends on an older version of bitcoin, which in turns depends on an
+# older version of secp256k1, which causes conflicts during linking. Use my fork right now, we can
+# switch back to crates.io as soon as rust-bitcoin is updated in rust-bitcoincore-rpc.
+#
+# Tracking issue: https://github.com/rust-bitcoin/rust-bitcoincore-rpc/pull/80
+
+[dependencies]
+log = "0.4.8"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+serial_test = "0.4"
+bitcoin = "0.23"
+bitcoincore-rpc = "0.11"
+electrum-client = "0.2.0-beta.1"

testutils/src/lib.rs

@@ -0,0 +1,507 @@
+#[macro_use]
+extern crate serde_json;
+#[macro_use]
+extern crate serial_test;
+
+pub use serial_test::serial;
+
+use std::collections::HashMap;
+use std::env;
+use std::ops::Deref;
+use std::path::PathBuf;
+use std::str::FromStr;
+use std::sync::Mutex;
+use std::time::Duration;
+
+#[allow(unused_imports)]
+use log::{debug, error, info, trace};
+
+use bitcoin::consensus::encode::{deserialize, serialize};
+use bitcoin::hashes::hex::{FromHex, ToHex};
+use bitcoin::hashes::sha256d;
+use bitcoin::{Address, Amount, Script, Transaction, Txid};
+
+pub use bitcoincore_rpc::bitcoincore_rpc_json::AddressType;
+pub use bitcoincore_rpc::{Auth, Client as RpcClient, RpcApi};
+
+pub use electrum_client::{Client as ElectrumClient, ElectrumApi};
+
+lazy_static! {
+    static ref SYNC_TESTS_MUTEX: Mutex<()> = Mutex::new(());
+}
+
+pub fn test_init() {}
+
+// TODO: we currently only support env vars, we could also parse a toml file
+fn get_auth() -> Auth {
+    match env::var("MAGICAL_RPC_AUTH").as_ref().map(String::as_ref) {
+        Ok("USER_PASS") => Auth::UserPass(
+            env::var("MAGICAL_RPC_USER").unwrap(),
+            env::var("MAGICAL_RPC_PASS").unwrap(),
+        ),
+        _ => Auth::CookieFile(PathBuf::from(
+            env::var("MAGICAL_RPC_COOKIEFILE")
+                .unwrap_or("/home/user/.bitcoin/regtest/.cookie".to_string()),
+        )),
+    }
+}
+
+pub fn get_electrum_url() -> String {
+    env::var("MAGICAL_ELECTRUM_URL").unwrap_or("tcp://127.0.0.1:50001".to_string())
+}
+
+pub struct TestClient {
+    client: RpcClient,
+    electrum: ElectrumClient,
+}
+
+#[derive(Clone, Debug)]
+pub struct TestIncomingOutput {
+    pub value: u64,
+    pub to_address: String,
+}
+
+impl TestIncomingOutput {
+    pub fn new(value: u64, to_address: Address) -> Self {
+        Self {
+            value,
+            to_address: to_address.to_string(),
+        }
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct TestIncomingTx {
+    pub output: Vec<TestIncomingOutput>,
+    pub min_confirmations: Option<u64>,
+    pub locktime: Option<i64>,
+    pub replaceable: Option<bool>,
+}
+
+impl TestIncomingTx {
+    pub fn new(
+        output: Vec<TestIncomingOutput>,
+        min_confirmations: Option<u64>,
+        locktime: Option<i64>,
+        replaceable: Option<bool>,
+    ) -> Self {
+        Self {
+            output,
+            min_confirmations,
+            locktime,
+            replaceable,
+        }
+    }
+
+    pub fn add_output(&mut self, output: TestIncomingOutput) {
+        self.output.push(output);
+    }
+}
+
+#[macro_export]
+macro_rules! testutils {
+    ( @external $descriptors:expr, $child:expr ) => ({
+        $descriptors.0.derive($child).expect("Derivation error").address(bitcoin::Network::Regtest).expect("No address form")
+    });
+    ( @internal $descriptors:expr, $child:expr ) => ({
+        $descriptors.1.expect("Missing internal descriptor").derive($child).expect("Derivation error").address(bitcoin::Network::Regtest).expect("No address form")
+    });
+    ( @e $descriptors:expr, $child:expr ) => ({ testutils!(@external $descriptors, $child) });
+    ( @i $descriptors:expr, $child:expr ) => ({ testutils!(@internal $descriptors, $child) });
+
+    ( @tx ( $( ( $( $addr:tt )* ) => $amount:expr ),+ ) $( ( @locktime $locktime:expr ) )* $( ( @confirmations $confirmations:expr ) )* $( ( @replaceable $replaceable:expr ) )* ) => ({
+        let mut outs = Vec::new();
+        $( outs.push(testutils::TestIncomingOutput::new($amount, testutils!( $($addr)* ))); )+
+
+        let mut locktime = None::<i64>;
+        $( locktime = Some($locktime); )*
+
+        let mut min_confirmations = None::<u64>;
+        $( min_confirmations = Some($confirmations); )*
+
+        let mut replaceable = None::<bool>;
+        $( replaceable = Some($replaceable); )*
+
+        testutils::TestIncomingTx::new(outs, min_confirmations, locktime, replaceable)
+    });
+
+    ( @literal $key:expr ) => ({
+        let key = $key.to_string();
+        (key, None::<String>, None::<String>)
+    });
+    ( @generate_xprv $( $external_path:expr )* $( ,$internal_path:expr )* ) => ({
+        use rand::Rng;
+
+        let mut seed = [0u8; 32];
+        rand::thread_rng().fill(&mut seed[..]);
+
+        let key = bitcoin::util::bip32::ExtendedPrivKey::new_master(
+            bitcoin::Network::Testnet,
+            &seed,
+        );
+
+        let mut external_path = None::<String>;
+        $( external_path = Some($external_path.to_string()); )*
+
+        let mut internal_path = None::<String>;
+        $( internal_path = Some($internal_path.to_string()); )*
+
+        (key.unwrap().to_string(), external_path, internal_path)
+    });
+    ( @generate_wif ) => ({
+        use rand::Rng;
+
+        let mut key = [0u8; bitcoin::secp256k1::constants::SECRET_KEY_SIZE];
+        rand::thread_rng().fill(&mut key[..]);
+
+        (bitcoin::PrivateKey {
+            compressed: true,
+            network: bitcoin::Network::Testnet,
+            key: bitcoin::secp256k1::SecretKey::from_slice(&key).unwrap(),
+        }.to_string(), None::<String>, None::<String>)
+    });
+
+    ( @keys ( $( $alias:expr => ( $( $key_type:tt )* ) ),+ ) ) => ({
+        let mut map = std::collections::HashMap::new();
+        $(
+            let alias: &str = $alias;
+            map.insert(alias, testutils!( $($key_type)* ));
+        )+
+
+        map
+    });
+
+    ( @descriptors ( $external_descriptor:expr ) $( ( $internal_descriptor:expr ) )* $( ( @keys $( $keys:tt )* ) )* ) => ({
+        use std::str::FromStr;
+        use std::collections::HashMap;
+        use std::convert::TryInto;
+
+        let mut keys: HashMap<&'static str, (String, Option<String>, Option<String>)> = HashMap::new();
+        $(
+            keys = testutils!{ @keys $( $keys )* };
+        )*
+
+        let external: Descriptor<String> = FromStr::from_str($external_descriptor).unwrap();
+        let external: Descriptor<String> = external.translate_pk::<_, _, _, &'static str>(|k| {
+            if let Some((key, ext_path, _)) = keys.get(&k.as_str()) {
+                Ok(format!("{}{}", key, ext_path.as_ref().unwrap_or(&"".into())))
+            } else {
+                Ok(k.clone())
+            }
+        }, |kh| {
+            if let Some((key, ext_path, _)) = keys.get(&kh.as_str()) {
+                Ok(format!("{}{}", key, ext_path.as_ref().unwrap_or(&"".into())))
+            } else {
+                Ok(kh.clone())
+            }
+
+        }).unwrap();
+        let external: ExtendedDescriptor = external.try_into().unwrap();
+
+        let mut internal = None::<ExtendedDescriptor>;
+        $(
+            let string_internal: Descriptor<String> = FromStr::from_str($internal_descriptor).unwrap();
+
+            let string_internal: Descriptor<String> = string_internal.translate_pk::<_, _, _, &'static str>(|k| {
+                if let Some((key, _, int_path)) = keys.get(&k.as_str()) {
+                    Ok(format!("{}{}", key, int_path.as_ref().unwrap_or(&"".into())))
+                } else {
+                    Ok(k.clone())
+                }
+            }, |kh| {
+                if let Some((key, _, int_path)) = keys.get(&kh.as_str()) {
+                    Ok(format!("{}{}", key, int_path.as_ref().unwrap_or(&"".into())))
+                } else {
+                    Ok(kh.clone())
+                }
+
+            }).unwrap();
+            internal = Some(string_internal.try_into().unwrap());
+
+        )*
+
+        (external, internal)
+    })
+}
+
+fn exponential_backoff_poll<T, F>(mut poll: F) -> T
+where
+    F: FnMut() -> Option<T>,
+{
+    let mut delay = Duration::from_millis(64);
+    loop {
+        match poll() {
+            Some(data) => break data,
+            None if delay.as_millis() < 512 => delay = delay.mul_f32(2.0),
+            None => {}
+        }
+
+        std::thread::sleep(delay);
+    }
+}
+
+impl TestClient {
+    pub fn new() -> Self {
+        let url = env::var("MAGICAL_RPC_URL").unwrap_or("127.0.0.1:18443".to_string());
+        let client = RpcClient::new(format!("http://{}", url), get_auth()).unwrap();
+        let electrum = ElectrumClient::new(&get_electrum_url(), None).unwrap();
+
+        TestClient { client, electrum }
+    }
+
+    fn wait_for_tx(&mut self, txid: Txid, monitor_script: &Script) {
+        // wait for electrs to index the tx
+        exponential_backoff_poll(|| {
+            trace!("wait_for_tx {}", txid);
+
+            self.electrum
+                .script_get_history(monitor_script)
+                .unwrap()
+                .iter()
+                .position(|entry| entry.tx_hash == txid)
+        });
+    }
+
+    fn wait_for_block(&mut self, min_height: usize) {
+        self.electrum.block_headers_subscribe().unwrap();
+
+        loop {
+            let header = exponential_backoff_poll(|| {
+                self.electrum.ping().unwrap();
+                self.electrum.block_headers_pop().unwrap()
+            });
+            if header.height >= min_height {
+                break;
+            }
+        }
+    }
+
+    pub fn receive(&mut self, meta_tx: TestIncomingTx) -> Txid {
+        assert!(
+            meta_tx.output.len() > 0,
+            "can't create a transaction with no outputs"
+        );
+
+        let mut map = HashMap::new();
+
+        let mut required_balance = 0;
+        for out in &meta_tx.output {
+            required_balance += out.value;
+            map.insert(out.to_address.clone(), Amount::from_sat(out.value));
+        }
+
+        if self.get_balance(None, None).unwrap() < Amount::from_sat(required_balance) {
+            panic!("Insufficient funds in bitcoind. Plase generate a few blocks with: `bitcoin-cli generatetoaddress 10 {}`", self.get_new_address(None, None).unwrap());
+        }
+
+        // FIXME: core can't create a tx with two outputs to the same address
+        let tx = self
+            .create_raw_transaction_hex(&[], &map, meta_tx.locktime, meta_tx.replaceable)
+            .unwrap();
+        let tx = self.fund_raw_transaction(tx, None, None).unwrap();
+        let mut tx: Transaction = deserialize(&tx.hex).unwrap();
+
+        if let Some(true) = meta_tx.replaceable {
+            // for some reason core doesn't set this field right
+            for input in &mut tx.input {
+                input.sequence = 0xFFFFFFFD;
+            }
+        }
+
+        let tx = self
+            .sign_raw_transaction_with_wallet(&serialize(&tx), None, None)
+            .unwrap();
+
+        // broadcast through electrum so that it caches the tx immediately
+        let txid = self
+            .electrum
+            .transaction_broadcast(&deserialize(&tx.hex).unwrap())
+            .unwrap();
+
+        if let Some(num) = meta_tx.min_confirmations {
+            self.generate(num);
+        }
+
+        let monitor_script = Address::from_str(&meta_tx.output[0].to_address)
+            .unwrap()
+            .script_pubkey();
+        self.wait_for_tx(txid, &monitor_script);
+
+        debug!("Sent tx: {}", txid);
+
+        txid
+    }
+
+    pub fn bump_fee(&mut self, txid: &Txid) -> Txid {
+        let tx = self.get_raw_transaction_info(txid, None).unwrap();
+        assert!(
+            tx.confirmations.is_none(),
+            "Can't bump tx {} because it's already confirmed",
+            txid
+        );
+
+        let bumped: serde_json::Value = self.call("bumpfee", &[txid.to_string().into()]).unwrap();
+        let new_txid = Txid::from_str(&bumped["txid"].as_str().unwrap().to_string()).unwrap();
+
+        let monitor_script =
+            tx.vout[0].script_pub_key.addresses.as_ref().unwrap()[0].script_pubkey();
+        self.wait_for_tx(new_txid, &monitor_script);
+
+        debug!("Bumped {}, new txid {}", txid, new_txid);
+
+        new_txid
+    }
+
+    pub fn generate_manually(&mut self, txs: Vec<Transaction>) -> String {
+        use bitcoin::blockdata::block::{Block, BlockHeader};
+        use bitcoin::blockdata::script::Builder;
+        use bitcoin::blockdata::transaction::{OutPoint, TxIn, TxOut};
+        use bitcoin::hash_types::{BlockHash, TxMerkleNode};
+        use bitcoin::util::hash::BitcoinHash;
+
+        let block_template: serde_json::Value = self
+            .call("getblocktemplate", &[json!({"rules": ["segwit"]})])
+            .unwrap();
+        trace!("getblocktemplate: {:#?}", block_template);
+
+        let header = BlockHeader {
+            version: block_template["version"].as_u64().unwrap() as u32,
+            prev_blockhash: BlockHash::from_hex(
+                block_template["previousblockhash"].as_str().unwrap(),
+            )
+            .unwrap(),
+            merkle_root: TxMerkleNode::default(),
+            time: block_template["curtime"].as_u64().unwrap() as u32,
+            bits: u32::from_str_radix(block_template["bits"].as_str().unwrap(), 16).unwrap(),
+            nonce: 0,
+        };
+        debug!("header: {:#?}", header);
+
+        let height = block_template["height"].as_u64().unwrap() as i64;
+        let witness_reserved_value: Vec<u8> = sha256d::Hash::default().as_ref().into();
+        // burn block subsidy and fees, not a big deal
+        let mut coinbase_tx = Transaction {
+            version: 1,
+            lock_time: 0,
+            input: vec![TxIn {
+                previous_output: OutPoint::null(),
+                script_sig: Builder::new().push_int(height).into_script(),
+                sequence: 0xFFFFFFFF,
+                witness: vec![witness_reserved_value],
+            }],
+            output: vec![],
+        };
+
+        let mut txdata = vec![coinbase_tx.clone()];
+        txdata.extend_from_slice(&txs);
+
+        let mut block = Block { header, txdata };
+
+        let witness_root = block.witness_root();
+        let witness_commitment =
+            Block::compute_witness_commitment(&witness_root, &coinbase_tx.input[0].witness[0]);
+
+        // now update and replace the coinbase tx
+        let mut coinbase_witness_commitment_script = vec![0x6a, 0x24, 0xaa, 0x21, 0xa9, 0xed];
+        coinbase_witness_commitment_script.extend_from_slice(&witness_commitment);
+
+        coinbase_tx.output.push(TxOut {
+            value: 0,
+            script_pubkey: coinbase_witness_commitment_script.into(),
+        });
+        block.txdata[0] = coinbase_tx;
+
+        // set merkle root
+        let merkle_root = block.merkle_root();
+        block.header.merkle_root = merkle_root;
+
+        assert!(block.check_merkle_root());
+        assert!(block.check_witness_commitment());
+
+        // now do PoW :)
+        let target = block.header.target();
+        while block.header.validate_pow(&target).is_err() {
+            block.header.nonce = block.header.nonce.checked_add(1).unwrap(); // panic if we run out of nonces
+        }
+
+        let block_hex: String = serialize(&block).to_hex();
+        debug!("generated block hex: {}", block_hex);
+
+        self.electrum.block_headers_subscribe().unwrap();
+
+        let submit_result: serde_json::Value =
+            self.call("submitblock", &[block_hex.into()]).unwrap();
+        debug!("submitblock: {:?}", submit_result);
+        assert!(
+            submit_result.is_null(),
+            "submitblock error: {:?}",
+            submit_result.as_str()
+        );
+
+        self.wait_for_block(height as usize);
+
+        block.header.bitcoin_hash().to_hex()
+    }
+
+    pub fn generate(&mut self, num_blocks: u64) {
+        let our_addr = self.get_new_address(None, None).unwrap();
+        let hashes = self.generate_to_address(num_blocks, &our_addr).unwrap();
+        let best_hash = hashes.last().unwrap();
+        let height = self.get_block_info(best_hash).unwrap().height;
+
+        self.wait_for_block(height);
+
+        debug!("Generated blocks to new height {}", height);
+    }
+
+    pub fn invalidate(&mut self, num_blocks: u64) {
+        self.electrum.block_headers_subscribe().unwrap();
+
+        let best_hash = self.get_best_block_hash().unwrap();
+        let initial_height = self.get_block_info(&best_hash).unwrap().height;
+
+        let mut to_invalidate = best_hash;
+        for i in 1..=num_blocks {
+            trace!(
+                "Invalidating block {}/{} ({})",
+                i,
+                num_blocks,
+                to_invalidate
+            );
+
+            self.invalidate_block(&to_invalidate).unwrap();
+            to_invalidate = self.get_best_block_hash().unwrap();
+        }
+
+        self.wait_for_block(initial_height - num_blocks as usize);
+
+        debug!(
+            "Invalidated {} blocks to new height of {}",
+            num_blocks,
+            initial_height - num_blocks as usize
+        );
+    }
+
+    pub fn reorg(&mut self, num_blocks: u64) {
+        self.invalidate(num_blocks);
+        self.generate(num_blocks);
+    }
+
+    pub fn get_node_address(&self, address_type: Option<AddressType>) -> Address {
+        Address::from_str(
+            &self
+                .get_new_address(None, address_type)
+                .unwrap()
+                .to_string(),
+        )
+        .unwrap()
+    }
+}
+
+impl Deref for TestClient {
+    type Target = RpcClient;
+
+    fn deref(&self) -> &Self::Target {
+        &self.client
+    }
+}