From 4498e0f7f8dde96d8da54d4fe654554289099ead Mon Sep 17 00:00:00 2001
From: Alekos Filini <alekos.filini@gmail.com>
Date: Wed, 16 Jun 2021 12:39:59 +0200
Subject: [PATCH 01/11] [testutils] Allow the generated blockchain tests to
 access `test_client`

---
 src/testutils/blockchain_tests.rs | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/src/testutils/blockchain_tests.rs b/src/testutils/blockchain_tests.rs
index c8e12d5f..776d0f5e 100644
--- a/src/testutils/blockchain_tests.rs
+++ b/src/testutils/blockchain_tests.rs
@@ -337,7 +337,7 @@ fn get_auth() -> Auth {
 #[macro_export]
 macro_rules! bdk_blockchain_tests {
     (
-     fn test_instance() -> $blockchain:ty $block:block) => {
+     fn $_fn_name:ident ( $( $test_client:ident : &TestClient )? $(,)? ) -> $blockchain:ty $block:block) => {
         #[cfg(test)]
         mod bdk_blockchain_tests {
             use $crate::bitcoin::Network;
@@ -351,12 +351,14 @@ macro_rules! bdk_blockchain_tests {
 
             use super::*;
 
-            fn get_blockchain() -> $blockchain {
+            #[allow(unused_variables)]
+            fn get_blockchain(test_client: &TestClient) -> $blockchain {
+                $( let $test_client = test_client; )?
                 $block
             }
 
-            fn get_wallet_from_descriptors(descriptors: &(String, Option<String>)) -> Wallet<$blockchain, MemoryDatabase> {
-                Wallet::new(&descriptors.0.to_string(), descriptors.1.as_ref(), Network::Regtest, MemoryDatabase::new(), get_blockchain()).unwrap()
+            fn get_wallet_from_descriptors(descriptors: &(String, Option<String>), test_client: &TestClient) -> Wallet<$blockchain, MemoryDatabase> {
+                Wallet::new(&descriptors.0.to_string(), descriptors.1.as_ref(), Network::Regtest, MemoryDatabase::new(), get_blockchain(test_client)).unwrap()
             }
 
             fn init_single_sig() -> (Wallet<$blockchain, MemoryDatabase>, (String, Option<String>), TestClient) {
@@ -367,7 +369,7 @@ macro_rules! bdk_blockchain_tests {
                 };
 
                 let test_client = TestClient::default();
-                let wallet = get_wallet_from_descriptors(&descriptors);
+                let wallet = get_wallet_from_descriptors(&descriptors, &test_client);
 
                 // rpc need to call import_multi before receiving any tx, otherwise will not see tx in the mempool
                 #[cfg(feature = "test-rpc")]
@@ -648,7 +650,7 @@ macro_rules! bdk_blockchain_tests {
                 assert_eq!(wallet.get_balance().unwrap(), details.received, "incorrect balance after receive");
 
                 // empty wallet
-                let wallet = get_wallet_from_descriptors(&descriptors);
+                let wallet = get_wallet_from_descriptors(&descriptors, &test_client);
 
                 #[cfg(feature = "rpc")]  // rpc cannot see mempool tx before importmulti
                 test_client.generate(1, Some(node_addr));
@@ -698,7 +700,7 @@ macro_rules! bdk_blockchain_tests {
 
                 // empty wallet
 
-                let wallet = get_wallet_from_descriptors(&descriptors);
+                let wallet = get_wallet_from_descriptors(&descriptors, &test_client);
 
                 #[cfg(feature = "rpc")]  // rpc cannot see mempool tx before importmulti
                 test_client.generate(1, Some(node_addr));
@@ -875,5 +877,10 @@ macro_rules! bdk_blockchain_tests {
                 assert!(wallet.get_balance().unwrap() > 0, "incorrect balance after receiving coinbase");
             }
         }
-    }
+    };
+
+    ( fn $fn_name:ident ($( $tt:tt )+) -> $blockchain:ty $block:block) => {
+        compile_error!(concat!("Invalid arguments `", stringify!($($tt)*), "` in the blockchain tests fn."));
+        compile_error!("Only the exact `&TestClient` type is supported, **without** any leading path items.");
+    };
 }

From 42adad7dbdb7c22e3fadd2161e515592462c260e Mon Sep 17 00:00:00 2001
From: Riccardo Casatta <riccardo@casatta.it>
Date: Thu, 17 Jun 2021 11:44:49 +0200
Subject: [PATCH 02/11] bump bitcoind dep to 0.11.0

---
 Cargo.toml            |  2 +-
 src/blockchain/rpc.rs | 16 ++++++++++------
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 67cb6bad..df6d39f6 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -71,7 +71,7 @@ lazy_static = "1.4"
 env_logger = "0.7"
 clap = "2.33"
 serial_test = "0.4"
-bitcoind = "0.10.0"
+bitcoind = "0.11.0"
 
 [[example]]
 name = "address_validator"
diff --git a/src/blockchain/rpc.rs b/src/blockchain/rpc.rs
index ab55cc49..fbd204cb 100644
--- a/src/blockchain/rpc.rs
+++ b/src/blockchain/rpc.rs
@@ -464,7 +464,7 @@ mod test {
     use bitcoincore_rpc::json::CreateRawTransactionInput;
     use bitcoincore_rpc::RawTx;
     use bitcoincore_rpc::{Auth, RpcApi};
-    use bitcoind::BitcoinD;
+    use bitcoind::{BitcoinD, Conf};
     use std::collections::HashMap;
 
     fn create_rpc(
@@ -477,16 +477,20 @@ mod test {
 
         let config = RpcConfig {
             url: bitcoind.rpc_url(),
-            auth: Auth::CookieFile(bitcoind.config.cookie_file.clone()),
+            auth: Auth::CookieFile(bitcoind.params.cookie_file.clone()),
             network,
             wallet_name,
             skip_blocks: None,
         };
         RpcBlockchain::from_config(&config)
     }
-    fn create_bitcoind(args: Vec<String>) -> BitcoinD {
+    fn create_bitcoind(args: Vec<&str>) -> BitcoinD {
         let exe = std::env::var("BITCOIND_EXE").unwrap();
-        bitcoind::BitcoinD::with_args(exe, args, false, bitcoind::P2P::No).unwrap()
+        let conf = Conf {
+            args,
+            ..Default::default()
+        };
+        bitcoind::BitcoinD::with_conf(exe, &conf).unwrap()
     }
 
     const DESCRIPTOR_PUB: &'static str = "wpkh(tpubD6NzVbkrYhZ4X2yy78HWrr1M9NT8dKeWfzNiQqDdMqqa9UmmGztGGz6TaLFGsLfdft5iu32gxq1T4eMNxExNNWzVCpf9Y6JZi5TnqoC9wJq/*)";
@@ -529,7 +533,7 @@ mod test {
         generate(&bitcoind, 5);
         let config = RpcConfig {
             url: bitcoind.rpc_url(),
-            auth: Auth::CookieFile(bitcoind.config.cookie_file.clone()),
+            auth: Auth::CookieFile(bitcoind.params.cookie_file.clone()),
             network: Network::Regtest,
             wallet_name: "another-name".to_string(),
             skip_blocks: Some(103),
@@ -559,7 +563,7 @@ mod test {
         let rpc = create_rpc(&bitcoind, DESCRIPTOR_PUB, Network::Regtest).unwrap();
         let capabilities = rpc.get_capabilities();
         assert!(capabilities.contains(&Capability::FullHistory) && capabilities.len() == 1);
-        let bitcoind_indexed = create_bitcoind(vec!["-txindex".to_string()]);
+        let bitcoind_indexed = create_bitcoind(vec!["-txindex"]);
         let rpc_indexed = create_rpc(&bitcoind_indexed, DESCRIPTOR_PUB, Network::Regtest).unwrap();
         assert_eq!(rpc_indexed.get_capabilities().len(), 3);
         let address = generate(&bitcoind_indexed, 101);

From 6394c3e2091839b24b36a7d36db7efa3729e2945 Mon Sep 17 00:00:00 2001
From: Riccardo Casatta <riccardo@casatta.it>
Date: Thu, 17 Jun 2021 13:48:46 +0200
Subject: [PATCH 03/11] use bitcoind and electrsd crate to launch daemons

---
 .github/workflows/cont_integration.yml | 21 ++------
 Cargo.toml                             |  1 +
 src/blockchain/electrum.rs             |  5 +-
 src/blockchain/esplora.rs              |  5 +-
 src/blockchain/rpc.rs                  | 29 +++--------
 src/testutils/blockchain_tests.rs      | 71 ++++++++++++--------------
 src/testutils/mod.rs                   |  1 +
 7 files changed, 52 insertions(+), 81 deletions(-)

diff --git a/.github/workflows/cont_integration.yml b/.github/workflows/cont_integration.yml
index 71a49cab..b2fee531 100644
--- a/.github/workflows/cont_integration.yml
+++ b/.github/workflows/cont_integration.yml
@@ -82,22 +82,13 @@ jobs:
       matrix:
         blockchain:
           - name: electrum
-            container: bitcoindevkit/electrs:0.4.0
-            start: /root/electrs --network regtest --cookie-file $GITHUB_WORKSPACE/.bitcoin/regtest/.cookie --jsonrpc-import
-          - name: esplora
-            container: bitcoindevkit/esplora:0.4.0
-            start: /root/electrs --network regtest -vvv --daemon-dir $GITHUB_WORKSPACE/.bitcoin --jsonrpc-import --electrum-rpc-addr=0.0.0.0:60401 --http-addr 0.0.0.0:3002
+            container: bitcoindevkit/electrs
           - name: rpc
-            container: bitcoindevkit/electrs:0.4.0
-            start: /root/electrs --network regtest --cookie-file $GITHUB_WORKSPACE/.bitcoin/regtest/.cookie --jsonrpc-import
+            container: bitcoindevkit/electrs
     container: ${{ matrix.blockchain.container }}
     env:
-      BDK_RPC_AUTH: COOKIEFILE
-      BDK_RPC_COOKIEFILE: ${{ github.workspace }}/.bitcoin/regtest/.cookie
-      BDK_RPC_URL: 127.0.0.1:18443
-      BDK_RPC_WALLET: bdk-test
-      BDK_ELECTRUM_URL: tcp://127.0.0.1:60401
-      BDK_ESPLORA_URL: http://127.0.0.1:3002
+      ELECTRS_EXE: /root/electrs
+      BITCOIND_EXE: /root/bitcoind
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -119,10 +110,6 @@ jobs:
         run: $HOME/.cargo/bin/rustup set profile minimal
       - name: Update toolchain
         run: $HOME/.cargo/bin/rustup update
-      - name: Start core
-        run: ./ci/start-core.sh
-      - name: start ${{ matrix.blockchain.name }}
-        run: nohup ${{ matrix.blockchain.start }} & sleep 5
       - name: Test
         run: $HOME/.cargo/bin/cargo test --features test-${{ matrix.blockchain.name }},test-blockchains --no-default-features ${{ matrix.blockchain.name }}::bdk_blockchain_tests
 
diff --git a/Cargo.toml b/Cargo.toml
index df6d39f6..d30587bb 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -72,6 +72,7 @@ env_logger = "0.7"
 clap = "2.33"
 serial_test = "0.4"
 bitcoind = "0.11.0"
+electrsd = "0.3.0"
 
 [[example]]
 name = "address_validator"
diff --git a/src/blockchain/electrum.rs b/src/blockchain/electrum.rs
index 4d8926af..45deb3f7 100644
--- a/src/blockchain/electrum.rs
+++ b/src/blockchain/electrum.rs
@@ -169,9 +169,10 @@ impl ConfigurableBlockchain for ElectrumBlockchain {
     }
 }
 
+#[cfg(test)]
 #[cfg(feature = "test-blockchains")]
 crate::bdk_blockchain_tests! {
-    fn test_instance() -> ElectrumBlockchain {
-        ElectrumBlockchain::from(Client::new(&testutils::blockchain_tests::get_electrum_url()).unwrap())
+    fn test_instance(test_client: &TestClient) -> ElectrumBlockchain {
+        ElectrumBlockchain::from(Client::new(&test_client.electrsd.electrum_url).unwrap())
     }
 }
diff --git a/src/blockchain/esplora.rs b/src/blockchain/esplora.rs
index ff85f22f..d866992d 100644
--- a/src/blockchain/esplora.rs
+++ b/src/blockchain/esplora.rs
@@ -415,9 +415,10 @@ impl_error!(std::num::ParseIntError, Parsing, EsploraError);
 impl_error!(consensus::encode::Error, BitcoinEncoding, EsploraError);
 impl_error!(bitcoin::hashes::hex::Error, Hex, EsploraError);
 
+#[cfg(test)]
 #[cfg(feature = "test-blockchains")]
 crate::bdk_blockchain_tests! {
-    fn test_instance() -> EsploraBlockchain {
-        EsploraBlockchain::new(std::env::var("BDK_ESPLORA_URL").unwrap_or("127.0.0.1:3002".into()).as_str(), None)
+    fn test_instance(test_client: &TestClient) -> EsploraBlockchain {
+        EsploraBlockchain::new(test_client.electrsd.esplora_url.as_ref().unwrap(), None)
     }
 }
diff --git a/src/blockchain/rpc.rs b/src/blockchain/rpc.rs
index fbd204cb..63664c68 100644
--- a/src/blockchain/rpc.rs
+++ b/src/blockchain/rpc.rs
@@ -419,27 +419,14 @@ fn list_wallet_dir(client: &Client) -> Result<Vec<String>, Error> {
     Ok(result.wallets.into_iter().map(|n| n.name).collect())
 }
 
+#[cfg(test)]
 #[cfg(feature = "test-blockchains")]
 crate::bdk_blockchain_tests! {
 
-    fn test_instance() -> RpcBlockchain {
-        let url = std::env::var("BDK_RPC_URL").unwrap_or_else(|_| "127.0.0.1:18443".to_string());
-        let url = format!("http://{}", url);
-
-        // TODO same code in `fn get_auth` in testutils, make it public there
-        let auth = match std::env::var("BDK_RPC_AUTH").as_ref().map(String::as_ref) {
-            Ok("USER_PASS") => Auth::UserPass(
-                std::env::var("BDK_RPC_USER").unwrap(),
-                std::env::var("BDK_RPC_PASS").unwrap(),
-            ),
-            _ => Auth::CookieFile(std::path::PathBuf::from(
-                std::env::var("BDK_RPC_COOKIEFILE")
-                    .unwrap_or_else(|_| "/home/user/.bitcoin/regtest/.cookie".to_string()),
-            )),
-        };
+    fn test_instance(test_client: &TestClient) -> RpcBlockchain {
         let config = RpcConfig {
-            url,
-            auth,
+            url: test_client.bitcoind.rpc_url(),
+            auth: Auth::CookieFile(test_client.bitcoind.params.cookie_file.clone()),
             network: Network::Regtest,
             wallet_name: format!("client-wallet-test-{:?}", std::time::SystemTime::now() ),
             skip_blocks: None,
@@ -486,10 +473,8 @@ mod test {
     }
     fn create_bitcoind(args: Vec<&str>) -> BitcoinD {
         let exe = std::env::var("BITCOIND_EXE").unwrap();
-        let conf = Conf {
-            args,
-            ..Default::default()
-        };
+        let mut conf = Conf::default();
+        conf.args.extend(args);
         bitcoind::BitcoinD::with_conf(exe, &conf).unwrap()
     }
 
@@ -498,7 +483,7 @@ mod test {
 
     #[test]
     fn test_rpc_wallet_setup() {
-        env_logger::try_init().unwrap();
+        let _ = env_logger::try_init();
         let bitcoind = create_bitcoind(vec![]);
         let node_address = bitcoind.client.get_new_address(None, None).unwrap();
         let blockchain = create_rpc(&bitcoind, DESCRIPTOR_PUB, Network::Regtest).unwrap();
diff --git a/src/testutils/blockchain_tests.rs b/src/testutils/blockchain_tests.rs
index 776d0f5e..713c0ef6 100644
--- a/src/testutils/blockchain_tests.rs
+++ b/src/testutils/blockchain_tests.rs
@@ -5,31 +5,37 @@ use bitcoin::hashes::sha256d;
 use bitcoin::{Address, Amount, Script, Transaction, Txid};
 pub use bitcoincore_rpc::bitcoincore_rpc_json::AddressType;
 pub use bitcoincore_rpc::{Auth, Client as RpcClient, RpcApi};
+use bitcoind::BitcoinD;
 use core::str::FromStr;
+use electrsd::ElectrsD;
 pub use electrum_client::{Client as ElectrumClient, ElectrumApi};
 #[allow(unused_imports)]
 use log::{debug, error, info, trace};
 use std::collections::HashMap;
 use std::env;
 use std::ops::Deref;
-use std::path::PathBuf;
 use std::time::Duration;
 
 pub struct TestClient {
-    client: RpcClient,
-    electrum: ElectrumClient,
+    pub bitcoind: BitcoinD,
+    pub electrsd: ElectrsD,
 }
 
 impl TestClient {
-    pub fn new(rpc_host_and_wallet: String, rpc_wallet_name: String) -> Self {
-        let client = RpcClient::new(
-            format!("http://{}/wallet/{}", rpc_host_and_wallet, rpc_wallet_name),
-            get_auth(),
-        )
-        .unwrap();
-        let electrum = ElectrumClient::new(&get_electrum_url()).unwrap();
+    pub fn new(bitcoind_exe: String, electrs_exe: String) -> Self {
+        debug!("launching {} and {}", &bitcoind_exe, &electrs_exe);
+        let bitcoind = BitcoinD::new(bitcoind_exe).unwrap();
+        let electrsd = ElectrsD::new(electrs_exe, &bitcoind, false, false).unwrap(); // TODO http_enabled should be true only for esplora
 
-        TestClient { client, electrum }
+        let node_address = bitcoind.client.get_new_address(None, None).unwrap();
+        bitcoind
+            .client
+            .generate_to_address(101, &node_address)
+            .unwrap();
+
+        let mut test_client = TestClient { bitcoind, electrsd };
+        TestClient::wait_for_block(&mut test_client, 101);
+        test_client
     }
 
     fn wait_for_tx(&mut self, txid: Txid, monitor_script: &Script) {
@@ -37,7 +43,8 @@ impl TestClient {
         exponential_backoff_poll(|| {
             trace!("wait_for_tx {}", txid);
 
-            self.electrum
+            self.electrsd
+                .client
                 .script_get_history(monitor_script)
                 .unwrap()
                 .iter()
@@ -46,12 +53,12 @@ impl TestClient {
     }
 
     fn wait_for_block(&mut self, min_height: usize) {
-        self.electrum.block_headers_subscribe().unwrap();
+        self.electrsd.client.block_headers_subscribe().unwrap();
 
         loop {
             let header = exponential_backoff_poll(|| {
-                self.electrum.ping().unwrap();
-                self.electrum.block_headers_pop().unwrap()
+                self.electrsd.client.ping().unwrap();
+                self.electrsd.client.block_headers_pop().unwrap()
             });
             if header.height >= min_height {
                 break;
@@ -96,10 +103,13 @@ impl TestClient {
             .unwrap();
 
         // broadcast through electrum so that it caches the tx immediately
+
         let txid = self
-            .electrum
+            .electrsd
+            .client
             .transaction_broadcast(&deserialize(&tx.hex).unwrap())
             .unwrap();
+        debug!("broadcasted to electrum {}", txid);
 
         if let Some(num) = meta_tx.min_confirmations {
             self.generate(num, None);
@@ -209,7 +219,7 @@ impl TestClient {
         let block_hex: String = serialize(&block).to_hex();
         debug!("generated block hex: {}", block_hex);
 
-        self.electrum.block_headers_subscribe().unwrap();
+        self.electrsd.client.block_headers_subscribe().unwrap();
 
         let submit_result: serde_json::Value =
             self.call("submitblock", &[block_hex.into()]).unwrap();
@@ -237,7 +247,7 @@ impl TestClient {
     }
 
     pub fn invalidate(&mut self, num_blocks: u64) {
-        self.electrum.block_headers_subscribe().unwrap();
+        self.electrsd.client.block_headers_subscribe().unwrap();
 
         let best_hash = self.get_best_block_hash().unwrap();
         let initial_height = self.get_block_info(&best_hash).unwrap().height;
@@ -288,16 +298,16 @@ impl Deref for TestClient {
     type Target = RpcClient;
 
     fn deref(&self) -> &Self::Target {
-        &self.client
+        &self.bitcoind.client
     }
 }
 
 impl Default for TestClient {
     fn default() -> Self {
-        let rpc_host_and_port =
-            env::var("BDK_RPC_URL").unwrap_or_else(|_| "127.0.0.1:18443".to_string());
-        let wallet = env::var("BDK_RPC_WALLET").unwrap_or_else(|_| "bdk-test".to_string());
-        Self::new(rpc_host_and_port, wallet)
+        let bitcoind_exe =
+            env::var("BITCOIND_EXE").unwrap_or_else(|_| "/root/bitcoind".to_string());
+        let electrs_exe = env::var("ELECTRS_EXE").unwrap_or_else(|_| "/root/electrs".to_string());
+        Self::new(bitcoind_exe, electrs_exe)
     }
 }
 
@@ -317,20 +327,6 @@ where
     }
 }
 
-// TODO: we currently only support env vars, we could also parse a toml file
-fn get_auth() -> Auth {
-    match env::var("BDK_RPC_AUTH").as_ref().map(String::as_ref) {
-        Ok("USER_PASS") => Auth::UserPass(
-            env::var("BDK_RPC_USER").unwrap(),
-            env::var("BDK_RPC_PASS").unwrap(),
-        ),
-        _ => Auth::CookieFile(PathBuf::from(
-            env::var("BDK_RPC_COOKIEFILE")
-                .unwrap_or_else(|_| "/home/user/.bitcoin/regtest/.cookie".to_string()),
-        )),
-    }
-}
-
 /// This macro runs blockchain tests against a `Blockchain` implementation. It requires access to a
 /// Bitcoin core wallet via RPC. At the moment you have to dig into the code yourself and look at
 /// the setup required to run the tests yourself.
@@ -590,7 +586,6 @@ macro_rules! bdk_blockchain_tests {
                 let tx = psbt.extract_tx();
                 println!("{}", bitcoin::consensus::encode::serialize_hex(&tx));
                 wallet.broadcast(tx).unwrap();
-
                 wallet.sync(noop_progress(), None).unwrap();
                 assert_eq!(wallet.get_balance().unwrap(), details.received, "incorrect balance after send");
 
diff --git a/src/testutils/mod.rs b/src/testutils/mod.rs
index 5d7146bd..3e4dfec2 100644
--- a/src/testutils/mod.rs
+++ b/src/testutils/mod.rs
@@ -10,6 +10,7 @@
 // licenses.
 #![allow(missing_docs)]
 
+#[cfg(test)]
 #[cfg(feature = "test-blockchains")]
 pub mod blockchain_tests;
 

From 18dcda844f8c6cbe01231e88dfe7cff932bdd203 Mon Sep 17 00:00:00 2001
From: Riccardo Casatta <riccardo@casatta.it>
Date: Thu, 17 Jun 2021 14:13:17 +0200
Subject: [PATCH 04/11] remove serial_test

---
 Cargo.toml                        |  2 --
 src/database/keyvalue.rs          |  1 +
 src/lib.rs                        |  7 -------
 src/testutils/blockchain_tests.rs | 19 -------------------
 4 files changed, 1 insertion(+), 28 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index d30587bb..c89f6f95 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -34,7 +34,6 @@ tiny-bip39 = { version = "^0.8", optional = true }
 
 # Needed by bdk_blockchain_tests macro
 bitcoincore-rpc = { version = "0.13", optional = true }
-serial_test = { version = "0.4", optional = true }
 
 # Platform-specific dependencies
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
@@ -70,7 +69,6 @@ test-md-docs = ["electrum"]
 lazy_static = "1.4"
 env_logger = "0.7"
 clap = "2.33"
-serial_test = "0.4"
 bitcoind = "0.11.0"
 electrsd = "0.3.0"
 
diff --git a/src/database/keyvalue.rs b/src/database/keyvalue.rs
index 201fc256..ecf9f4d7 100644
--- a/src/database/keyvalue.rs
+++ b/src/database/keyvalue.rs
@@ -385,6 +385,7 @@ impl BatchDatabase for Tree {
 mod test {
     use std::sync::{Arc, Condvar, Mutex, Once};
     use std::time::{SystemTime, UNIX_EPOCH};
+    use lazy_static::lazy_static;
 
     use sled::{Db, Tree};
 
diff --git a/src/lib.rs b/src/lib.rs
index c2ab473b..b5548bae 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -234,13 +234,6 @@ pub extern crate reqwest;
 #[cfg(feature = "key-value-db")]
 pub extern crate sled;
 
-#[allow(unused_imports)]
-#[cfg(test)]
-#[allow(unused_imports)]
-#[cfg(test)]
-#[macro_use]
-pub extern crate serial_test;
-
 #[macro_use]
 pub(crate) mod error;
 pub mod blockchain;
diff --git a/src/testutils/blockchain_tests.rs b/src/testutils/blockchain_tests.rs
index 713c0ef6..041e1a16 100644
--- a/src/testutils/blockchain_tests.rs
+++ b/src/testutils/blockchain_tests.rs
@@ -343,7 +343,6 @@ macro_rules! bdk_blockchain_tests {
             use $crate::types::KeychainKind;
             use $crate::{Wallet, FeeRate};
             use $crate::testutils;
-            use $crate::serial_test::serial;
 
             use super::*;
 
@@ -375,7 +374,6 @@ macro_rules! bdk_blockchain_tests {
             }
 
             #[test]
-            #[serial]
             fn test_sync_simple() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
 
@@ -398,7 +396,6 @@ macro_rules! bdk_blockchain_tests {
             }
 
             #[test]
-            #[serial]
             fn test_sync_stop_gap_20() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
 
@@ -416,7 +413,6 @@ macro_rules! bdk_blockchain_tests {
             }
 
             #[test]
-            #[serial]
             fn test_sync_before_and_after_receive() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
 
@@ -434,7 +430,6 @@ macro_rules! bdk_blockchain_tests {
             }
 
             #[test]
-            #[serial]
             fn test_sync_multiple_outputs_same_tx() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
 
@@ -456,7 +451,6 @@ macro_rules! bdk_blockchain_tests {
             }
 
             #[test]
-            #[serial]
             fn test_sync_receive_multi() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
 
@@ -475,7 +469,6 @@ macro_rules! bdk_blockchain_tests {
             }
 
             #[test]
-            #[serial]
             fn test_sync_address_reuse() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
 
@@ -495,7 +488,6 @@ macro_rules! bdk_blockchain_tests {
             }
 
             #[test]
-            #[serial]
             fn test_sync_receive_rbf_replaced() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
 
@@ -534,7 +526,6 @@ macro_rules! bdk_blockchain_tests {
             // doesn't work for some reason.
             #[cfg(not(feature = "esplora"))]
             #[test]
-            #[serial]
             fn test_sync_reorg_block() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
 
@@ -565,7 +556,6 @@ macro_rules! bdk_blockchain_tests {
             }
 
             #[test]
-            #[serial]
             fn test_sync_after_send() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
                 println!("{}", descriptors.0);
@@ -594,7 +584,6 @@ macro_rules! bdk_blockchain_tests {
             }
 
             #[test]
-            #[serial]
             fn test_update_confirmation_time_after_generate() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
                 println!("{}", descriptors.0);
@@ -620,9 +609,7 @@ macro_rules! bdk_blockchain_tests {
 
             }
 
-
             #[test]
-            #[serial]
             fn test_sync_outgoing_from_scratch() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
                 let node_addr = test_client.get_node_address(None);
@@ -664,7 +651,6 @@ macro_rules! bdk_blockchain_tests {
             }
 
             #[test]
-            #[serial]
             fn test_sync_long_change_chain() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
                 let node_addr = test_client.get_node_address(None);
@@ -706,7 +692,6 @@ macro_rules! bdk_blockchain_tests {
             }
 
             #[test]
-            #[serial]
             fn test_sync_bump_fee_basic() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
                 let node_addr = test_client.get_node_address(None);
@@ -742,7 +727,6 @@ macro_rules! bdk_blockchain_tests {
             }
 
             #[test]
-            #[serial]
             fn test_sync_bump_fee_remove_change() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
                 let node_addr = test_client.get_node_address(None);
@@ -778,7 +762,6 @@ macro_rules! bdk_blockchain_tests {
             }
 
             #[test]
-            #[serial]
             fn test_sync_bump_fee_add_input_simple() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
                 let node_addr = test_client.get_node_address(None);
@@ -812,7 +795,6 @@ macro_rules! bdk_blockchain_tests {
             }
 
             #[test]
-            #[serial]
             fn test_sync_bump_fee_add_input_no_change() {
                 let (wallet, descriptors, mut test_client) = init_single_sig();
                 let node_addr = test_client.get_node_address(None);
@@ -849,7 +831,6 @@ macro_rules! bdk_blockchain_tests {
             }
 
             #[test]
-            #[serial]
             fn test_sync_receive_coinbase() {
                 let (wallet, _, mut test_client) = init_single_sig();
 

From 696647b8930893c4eba67fef94f72c417c20bd4e Mon Sep 17 00:00:00 2001
From: Riccardo Casatta <riccardo@casatta.it>
Date: Thu, 17 Jun 2021 16:03:33 +0200
Subject: [PATCH 05/11] trigger electrs when polling

---
 Cargo.toml                        |   2 +-
 src/blockchain/rpc.rs             | 226 ------------------------------
 src/database/keyvalue.rs          |   2 +-
 src/lib.rs                        |   1 +
 src/testutils/blockchain_tests.rs |   2 +
 5 files changed, 5 insertions(+), 228 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index c89f6f95..af9dc597 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -70,7 +70,7 @@ lazy_static = "1.4"
 env_logger = "0.7"
 clap = "2.33"
 bitcoind = "0.11.0"
-electrsd = "0.3.0"
+electrsd = { version="0.3.0", features = ["trigger"] }
 
 [[example]]
 name = "address_validator"
diff --git a/src/blockchain/rpc.rs b/src/blockchain/rpc.rs
index 63664c68..0165313c 100644
--- a/src/blockchain/rpc.rs
+++ b/src/blockchain/rpc.rs
@@ -434,229 +434,3 @@ crate::bdk_blockchain_tests! {
         RpcBlockchain::from_config(&config).unwrap()
     }
 }
-
-#[cfg(feature = "test-rpc")]
-#[cfg(test)]
-mod test {
-    use super::{RpcBlockchain, RpcConfig};
-    use crate::bitcoin::consensus::deserialize;
-    use crate::bitcoin::{Address, Amount, Network, Transaction};
-    use crate::blockchain::rpc::wallet_name_from_descriptor;
-    use crate::blockchain::{noop_progress, Blockchain, Capability, ConfigurableBlockchain};
-    use crate::database::MemoryDatabase;
-    use crate::wallet::AddressIndex;
-    use crate::Wallet;
-    use bitcoin::secp256k1::Secp256k1;
-    use bitcoin::Txid;
-    use bitcoincore_rpc::json::CreateRawTransactionInput;
-    use bitcoincore_rpc::RawTx;
-    use bitcoincore_rpc::{Auth, RpcApi};
-    use bitcoind::{BitcoinD, Conf};
-    use std::collections::HashMap;
-
-    fn create_rpc(
-        bitcoind: &BitcoinD,
-        desc: &str,
-        network: Network,
-    ) -> Result<RpcBlockchain, crate::Error> {
-        let secp = Secp256k1::new();
-        let wallet_name = wallet_name_from_descriptor(desc, None, network, &secp).unwrap();
-
-        let config = RpcConfig {
-            url: bitcoind.rpc_url(),
-            auth: Auth::CookieFile(bitcoind.params.cookie_file.clone()),
-            network,
-            wallet_name,
-            skip_blocks: None,
-        };
-        RpcBlockchain::from_config(&config)
-    }
-    fn create_bitcoind(args: Vec<&str>) -> BitcoinD {
-        let exe = std::env::var("BITCOIND_EXE").unwrap();
-        let mut conf = Conf::default();
-        conf.args.extend(args);
-        bitcoind::BitcoinD::with_conf(exe, &conf).unwrap()
-    }
-
-    const DESCRIPTOR_PUB: &'static str = "wpkh(tpubD6NzVbkrYhZ4X2yy78HWrr1M9NT8dKeWfzNiQqDdMqqa9UmmGztGGz6TaLFGsLfdft5iu32gxq1T4eMNxExNNWzVCpf9Y6JZi5TnqoC9wJq/*)";
-    const DESCRIPTOR_PRIV: &'static str = "wpkh(tprv8ZgxMBicQKsPdZxBDUcvTSMEaLwCTzTc6gmw8KBKwa3BJzWzec4g6VUbQBHJcutDH6mMEmBeVyN27H1NF3Nu8isZ1Sts4SufWyfLE6Mf1MB/*)";
-
-    #[test]
-    fn test_rpc_wallet_setup() {
-        let _ = env_logger::try_init();
-        let bitcoind = create_bitcoind(vec![]);
-        let node_address = bitcoind.client.get_new_address(None, None).unwrap();
-        let blockchain = create_rpc(&bitcoind, DESCRIPTOR_PUB, Network::Regtest).unwrap();
-        let db = MemoryDatabase::new();
-        let wallet = Wallet::new(DESCRIPTOR_PRIV, None, Network::Regtest, db, blockchain).unwrap();
-
-        wallet.sync(noop_progress(), None).unwrap();
-        generate(&bitcoind, 101);
-        wallet.sync(noop_progress(), None).unwrap();
-        let address = wallet.get_address(AddressIndex::New).unwrap();
-        let expected_address = "bcrt1q8dyvgt4vhr8ald4xuwewcxhdjha9a5k78wxm5t";
-        assert_eq!(expected_address, address.to_string());
-        send_to_address(&bitcoind, &address, 100_000);
-        wallet.sync(noop_progress(), None).unwrap();
-        assert_eq!(wallet.get_balance().unwrap(), 100_000);
-
-        let mut builder = wallet.build_tx();
-        builder.add_recipient(node_address.script_pubkey(), 50_000);
-        let (mut psbt, details) = builder.finish().unwrap();
-        let finalized = wallet.sign(&mut psbt, Default::default()).unwrap();
-        assert!(finalized, "Cannot finalize transaction");
-        let tx = psbt.extract_tx();
-        wallet.broadcast(tx).unwrap();
-        wallet.sync(noop_progress(), None).unwrap();
-        assert_eq!(
-            wallet.get_balance().unwrap(),
-            100_000 - 50_000 - details.fee.unwrap_or(0)
-        );
-        drop(wallet);
-
-        // test skip_blocks
-        generate(&bitcoind, 5);
-        let config = RpcConfig {
-            url: bitcoind.rpc_url(),
-            auth: Auth::CookieFile(bitcoind.params.cookie_file.clone()),
-            network: Network::Regtest,
-            wallet_name: "another-name".to_string(),
-            skip_blocks: Some(103),
-        };
-        let blockchain_skip = RpcBlockchain::from_config(&config).unwrap();
-        let db = MemoryDatabase::new();
-        let wallet_skip =
-            Wallet::new(DESCRIPTOR_PRIV, None, Network::Regtest, db, blockchain_skip).unwrap();
-        wallet_skip.sync(noop_progress(), None).unwrap();
-        send_to_address(&bitcoind, &address, 100_000);
-        wallet_skip.sync(noop_progress(), None).unwrap();
-        assert_eq!(wallet_skip.get_balance().unwrap(), 100_000);
-    }
-
-    #[test]
-    fn test_rpc_from_config() {
-        let bitcoind = create_bitcoind(vec![]);
-        let blockchain = create_rpc(&bitcoind, DESCRIPTOR_PUB, Network::Regtest);
-        assert!(blockchain.is_ok());
-        let blockchain = create_rpc(&bitcoind, DESCRIPTOR_PUB, Network::Testnet);
-        assert!(blockchain.is_err(), "wrong network doesn't error");
-    }
-
-    #[test]
-    fn test_rpc_capabilities_get_tx() {
-        let bitcoind = create_bitcoind(vec![]);
-        let rpc = create_rpc(&bitcoind, DESCRIPTOR_PUB, Network::Regtest).unwrap();
-        let capabilities = rpc.get_capabilities();
-        assert!(capabilities.contains(&Capability::FullHistory) && capabilities.len() == 1);
-        let bitcoind_indexed = create_bitcoind(vec!["-txindex"]);
-        let rpc_indexed = create_rpc(&bitcoind_indexed, DESCRIPTOR_PUB, Network::Regtest).unwrap();
-        assert_eq!(rpc_indexed.get_capabilities().len(), 3);
-        let address = generate(&bitcoind_indexed, 101);
-        let txid = send_to_address(&bitcoind_indexed, &address, 100_000);
-        assert!(rpc_indexed.get_tx(&txid).unwrap().is_some());
-        assert!(rpc.get_tx(&txid).is_err());
-    }
-
-    #[test]
-    fn test_rpc_estimate_fee_get_height() {
-        let bitcoind = create_bitcoind(vec![]);
-        let rpc = create_rpc(&bitcoind, DESCRIPTOR_PUB, Network::Regtest).unwrap();
-        let result = rpc.estimate_fee(2);
-        assert!(result.is_err());
-        let address = generate(&bitcoind, 100);
-        // create enough tx so that core give some fee estimation
-        for _ in 0..15 {
-            let _ = bitcoind.client.generate_to_address(1, &address).unwrap();
-            for _ in 0..2 {
-                send_to_address(&bitcoind, &address, 100_000);
-            }
-        }
-        let result = rpc.estimate_fee(2);
-        assert!(result.is_ok());
-        assert_eq!(rpc.get_height().unwrap(), 115);
-    }
-
-    #[test]
-    fn test_rpc_node_synced_height() {
-        let bitcoind = create_bitcoind(vec![]);
-        let rpc = create_rpc(&bitcoind, DESCRIPTOR_PUB, Network::Regtest).unwrap();
-        let synced_height = rpc.get_node_synced_height().unwrap();
-
-        assert_eq!(synced_height, 0);
-        rpc.set_node_synced_height(1).unwrap();
-
-        let synced_height = rpc.get_node_synced_height().unwrap();
-        assert_eq!(synced_height, 1);
-    }
-
-    #[test]
-    fn test_rpc_broadcast() {
-        let bitcoind = create_bitcoind(vec![]);
-        let rpc = create_rpc(&bitcoind, DESCRIPTOR_PUB, Network::Regtest).unwrap();
-        let address = generate(&bitcoind, 101);
-        let utxo = bitcoind
-            .client
-            .list_unspent(None, None, None, None, None)
-            .unwrap();
-        let input = CreateRawTransactionInput {
-            txid: utxo[0].txid,
-            vout: utxo[0].vout,
-            sequence: None,
-        };
-
-        let out: HashMap<_, _> = vec![(
-            address.to_string(),
-            utxo[0].amount - Amount::from_sat(100_000),
-        )]
-        .into_iter()
-        .collect();
-        let tx = bitcoind
-            .client
-            .create_raw_transaction(&[input], &out, None, None)
-            .unwrap();
-        let signed_tx = bitcoind
-            .client
-            .sign_raw_transaction_with_wallet(tx.raw_hex(), None, None)
-            .unwrap();
-        let parsed_tx: Transaction = deserialize(&signed_tx.hex).unwrap();
-        rpc.broadcast(&parsed_tx).unwrap();
-        assert!(bitcoind
-            .client
-            .get_raw_mempool()
-            .unwrap()
-            .contains(&tx.txid()));
-    }
-
-    #[test]
-    fn test_rpc_wallet_name() {
-        let secp = Secp256k1::new();
-        let name =
-            wallet_name_from_descriptor(DESCRIPTOR_PUB, None, Network::Regtest, &secp).unwrap();
-        assert_eq!("tmg7aqay", name);
-    }
-
-    fn generate(bitcoind: &BitcoinD, blocks: u64) -> Address {
-        let address = bitcoind.client.get_new_address(None, None).unwrap();
-        bitcoind
-            .client
-            .generate_to_address(blocks, &address)
-            .unwrap();
-        address
-    }
-
-    fn send_to_address(bitcoind: &BitcoinD, address: &Address, amount: u64) -> Txid {
-        bitcoind
-            .client
-            .send_to_address(
-                &address,
-                Amount::from_sat(amount),
-                None,
-                None,
-                None,
-                None,
-                None,
-                None,
-            )
-            .unwrap()
-    }
-}
diff --git a/src/database/keyvalue.rs b/src/database/keyvalue.rs
index ecf9f4d7..9978497d 100644
--- a/src/database/keyvalue.rs
+++ b/src/database/keyvalue.rs
@@ -383,9 +383,9 @@ impl BatchDatabase for Tree {
 
 #[cfg(test)]
 mod test {
+    use lazy_static::lazy_static;
     use std::sync::{Arc, Condvar, Mutex, Once};
     use std::time::{SystemTime, UNIX_EPOCH};
-    use lazy_static::lazy_static;
 
     use sled::{Db, Tree};
 
diff --git a/src/lib.rs b/src/lib.rs
index b5548bae..2e261368 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -234,6 +234,7 @@ pub extern crate reqwest;
 #[cfg(feature = "key-value-db")]
 pub extern crate sled;
 
+#[allow(unused_imports)]
 #[macro_use]
 pub(crate) mod error;
 pub mod blockchain;
diff --git a/src/testutils/blockchain_tests.rs b/src/testutils/blockchain_tests.rs
index 041e1a16..588e0e65 100644
--- a/src/testutils/blockchain_tests.rs
+++ b/src/testutils/blockchain_tests.rs
@@ -41,6 +41,7 @@ impl TestClient {
     fn wait_for_tx(&mut self, txid: Txid, monitor_script: &Script) {
         // wait for electrs to index the tx
         exponential_backoff_poll(|| {
+            self.electrsd.trigger().unwrap();
             trace!("wait_for_tx {}", txid);
 
             self.electrsd
@@ -57,6 +58,7 @@ impl TestClient {
 
         loop {
             let header = exponential_backoff_poll(|| {
+                self.electrsd.trigger().unwrap();
                 self.electrsd.client.ping().unwrap();
                 self.electrsd.client.block_headers_pop().unwrap()
             });

From f4ecfa0d493c1dbdbc6b3b2069243af5b7a90729 Mon Sep 17 00:00:00 2001
From: Riccardo Casatta <riccardo@casatta.it>
Date: Fri, 18 Jun 2021 14:28:40 +0200
Subject: [PATCH 06/11] Remove container and test blockchains downloading
 backends executables

---
 .github/workflows/cont_integration.yml | 41 ++++++++++++++++----------
 Cargo.toml                             |  5 ++--
 src/blockchain/esplora.rs              |  2 +-
 src/testutils/blockchain_tests.rs      | 10 +++++--
 4 files changed, 36 insertions(+), 22 deletions(-)

diff --git a/.github/workflows/cont_integration.yml b/.github/workflows/cont_integration.yml
index b2fee531..e78e152c 100644
--- a/.github/workflows/cont_integration.yml
+++ b/.github/workflows/cont_integration.yml
@@ -76,19 +76,19 @@ jobs:
 
   test-blockchains:
     name: Test ${{ matrix.blockchain.name }}
-    runs-on: ubuntu-16.04
+    runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
       matrix:
         blockchain:
           - name: electrum
-            container: bitcoindevkit/electrs
           - name: rpc
-            container: bitcoindevkit/electrs
-    container: ${{ matrix.blockchain.container }}
+          - name: esplora
     env:
-      ELECTRS_EXE: /root/electrs
-      BITCOIND_EXE: /root/bitcoind
+      BITCOIN_VER: 0.21.0
+      ELECTRS_RELEASE_URL: https://github.com/RCasatta/electrsd/releases/download/release_0.3.0
+      ELECTRS_VER: electrs_ubuntu-20.04_v0.8.10.zip
+      ELECTRS_ESPLORA_VER: electrs_esplora_ubuntu-20.04_a33e97e1a1fc63fa9c20a116bb92579bbf43b254.zip
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -100,16 +100,25 @@ jobs:
             ~/.cargo/git
             target
           key: ${{ runner.os }}-cargo-${{ github.job }}-${{ hashFiles('**/Cargo.toml','**/Cargo.lock') }}
-      - name: get pkg-config # running esplora tests seems to need this
-        run: apt update && apt install -y --fix-missing pkg-config libssl-dev
-      - name: Install rustup
-        run: curl https://sh.rustup.rs -sSf | sh -s -- -y
-      - name: Set default toolchain
-        run: $HOME/.cargo/bin/rustup default 1.53.0 # STABLE
-      - name: Set profile
-        run: $HOME/.cargo/bin/rustup set profile minimal
-      - name: Update toolchain
-        run: $HOME/.cargo/bin/rustup update
+      - name: Setup rust toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          override: true
+      - name: Download electrs
+        run: wget ${{ env.ELECTRS_RELEASE_URL }}/${{ env.ELECTRS_VER }} && unzip ${{ env.ELECTRS_VER }} && chmod +x ./electrs
+        if: ${{ matrix.blockchain.name != 'esplora' }}
+      - name: Download electrs esplora
+        run: wget ${{ env.ELECTRS_RELEASE_URL }}/${{ env.ELECTRS_ESPLORA_VER }} && unzip ${{ env.ELECTRS_ESPLORA_VER }} && chmod +x ./electrs
+        if: ${{ matrix.blockchain.name == 'esplora' }}
+      - name: Set ELECTRS_EXE env
+        run: echo "ELECTRS_EXE=${{ github.workspace }}/electrs" >> $GITHUB_ENV
+      - name: Show electrs options
+        run: ${{ env.ELECTRS_EXE }} --help
+      - name: Set BITCOIND_EXE env
+        run: echo "BITCOIND_EXE=${{ github.workspace }}/bitcoin-${{ env.BITCOIN_VER }}/bin/bitcoind" >> $GITHUB_ENV
+      - name: Install bitcoind
+        run: curl https://bitcoincore.org/bin/bitcoin-core-$BITCOIN_VER/bitcoin-$BITCOIN_VER-x86_64-linux-gnu.tar.gz | tar -xvz bitcoin-$BITCOIN_VER/bin/bitcoind
       - name: Test
         run: $HOME/.cargo/bin/cargo test --features test-${{ matrix.blockchain.name }},test-blockchains --no-default-features ${{ matrix.blockchain.name }}::bdk_blockchain_tests
 
diff --git a/Cargo.toml b/Cargo.toml
index af9dc597..133c0a69 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -62,15 +62,14 @@ rpc = ["bitcoincore-rpc"]
 test-blockchains = ["bitcoincore-rpc", "electrum-client"]
 test-electrum = ["electrum"]
 test-rpc = ["rpc"]
-test-esplora = ["esplora"]
+test-esplora = ["esplora", "electrsd/legacy"]
 test-md-docs = ["electrum"]
 
 [dev-dependencies]
 lazy_static = "1.4"
 env_logger = "0.7"
 clap = "2.33"
-bitcoind = "0.11.0"
-electrsd = { version="0.3.0", features = ["trigger"] }
+electrsd = { version="0.4.0", features = ["trigger"] }
 
 [[example]]
 name = "address_validator"
diff --git a/src/blockchain/esplora.rs b/src/blockchain/esplora.rs
index d866992d..dce7f836 100644
--- a/src/blockchain/esplora.rs
+++ b/src/blockchain/esplora.rs
@@ -419,6 +419,6 @@ impl_error!(bitcoin::hashes::hex::Error, Hex, EsploraError);
 #[cfg(feature = "test-blockchains")]
 crate::bdk_blockchain_tests! {
     fn test_instance(test_client: &TestClient) -> EsploraBlockchain {
-        EsploraBlockchain::new(test_client.electrsd.esplora_url.as_ref().unwrap(), None)
+        EsploraBlockchain::new(&format!("http://{}",test_client.electrsd.esplora_url.as_ref().unwrap()), None)
     }
 }
diff --git a/src/testutils/blockchain_tests.rs b/src/testutils/blockchain_tests.rs
index 588e0e65..12f96dd9 100644
--- a/src/testutils/blockchain_tests.rs
+++ b/src/testutils/blockchain_tests.rs
@@ -5,8 +5,8 @@ use bitcoin::hashes::sha256d;
 use bitcoin::{Address, Amount, Script, Transaction, Txid};
 pub use bitcoincore_rpc::bitcoincore_rpc_json::AddressType;
 pub use bitcoincore_rpc::{Auth, Client as RpcClient, RpcApi};
-use bitcoind::BitcoinD;
 use core::str::FromStr;
+use electrsd::bitcoind::BitcoinD;
 use electrsd::ElectrsD;
 pub use electrum_client::{Client as ElectrumClient, ElectrumApi};
 #[allow(unused_imports)]
@@ -25,7 +25,13 @@ impl TestClient {
     pub fn new(bitcoind_exe: String, electrs_exe: String) -> Self {
         debug!("launching {} and {}", &bitcoind_exe, &electrs_exe);
         let bitcoind = BitcoinD::new(bitcoind_exe).unwrap();
-        let electrsd = ElectrsD::new(electrs_exe, &bitcoind, false, false).unwrap(); // TODO http_enabled should be true only for esplora
+
+        #[cfg(feature = "test-esplora")]
+        let http_enabled = true;
+        #[cfg(not(feature = "test-esplora"))]
+        let http_enabled = false;
+
+        let electrsd = ElectrsD::new(electrs_exe, &bitcoind, false, http_enabled).unwrap();
 
         let node_address = bitcoind.client.get_new_address(None, None).unwrap();
         bitcoind

From 2ecae348ea864fb0e86bd706ba1f4b4a41d15636 Mon Sep 17 00:00:00 2001
From: Riccardo Casatta <riccardo@casatta.it>
Date: Mon, 21 Jun 2021 13:56:09 +0200
Subject: [PATCH 07/11] use cfg! instead of #[cfg] and use semver

---
 Cargo.toml                        | 2 +-
 src/testutils/blockchain_tests.rs | 5 +----
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 133c0a69..fb45a762 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -69,7 +69,7 @@ test-md-docs = ["electrum"]
 lazy_static = "1.4"
 env_logger = "0.7"
 clap = "2.33"
-electrsd = { version="0.4.0", features = ["trigger"] }
+electrsd = { version="0.4", features = ["trigger"] }
 
 [[example]]
 name = "address_validator"
diff --git a/src/testutils/blockchain_tests.rs b/src/testutils/blockchain_tests.rs
index 12f96dd9..a2174802 100644
--- a/src/testutils/blockchain_tests.rs
+++ b/src/testutils/blockchain_tests.rs
@@ -26,10 +26,7 @@ impl TestClient {
         debug!("launching {} and {}", &bitcoind_exe, &electrs_exe);
         let bitcoind = BitcoinD::new(bitcoind_exe).unwrap();
 
-        #[cfg(feature = "test-esplora")]
-        let http_enabled = true;
-        #[cfg(not(feature = "test-esplora"))]
-        let http_enabled = false;
+        let http_enabled = cfg!(feature = "test-esplora");
 
         let electrsd = ElectrsD::new(electrs_exe, &bitcoind, false, http_enabled).unwrap();
 

From 8e41c4587daf291a9e8eab8c1a57524a7d913e16 Mon Sep 17 00:00:00 2001
From: Riccardo Casatta <riccardo@casatta.it>
Date: Wed, 23 Jun 2021 14:47:45 +0200
Subject: [PATCH 08/11] use bitcoind with feature to download the binary

---
 .github/workflows/cont_integration.yml |  5 -----
 Cargo.toml                             |  2 +-
 src/testutils/blockchain_tests.rs      | 10 +++++++---
 3 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/cont_integration.yml b/.github/workflows/cont_integration.yml
index e78e152c..9e5be26f 100644
--- a/.github/workflows/cont_integration.yml
+++ b/.github/workflows/cont_integration.yml
@@ -85,7 +85,6 @@ jobs:
           - name: rpc
           - name: esplora
     env:
-      BITCOIN_VER: 0.21.0
       ELECTRS_RELEASE_URL: https://github.com/RCasatta/electrsd/releases/download/release_0.3.0
       ELECTRS_VER: electrs_ubuntu-20.04_v0.8.10.zip
       ELECTRS_ESPLORA_VER: electrs_esplora_ubuntu-20.04_a33e97e1a1fc63fa9c20a116bb92579bbf43b254.zip
@@ -115,10 +114,6 @@ jobs:
         run: echo "ELECTRS_EXE=${{ github.workspace }}/electrs" >> $GITHUB_ENV
       - name: Show electrs options
         run: ${{ env.ELECTRS_EXE }} --help
-      - name: Set BITCOIND_EXE env
-        run: echo "BITCOIND_EXE=${{ github.workspace }}/bitcoin-${{ env.BITCOIN_VER }}/bin/bitcoind" >> $GITHUB_ENV
-      - name: Install bitcoind
-        run: curl https://bitcoincore.org/bin/bitcoin-core-$BITCOIN_VER/bitcoin-$BITCOIN_VER-x86_64-linux-gnu.tar.gz | tar -xvz bitcoin-$BITCOIN_VER/bin/bitcoind
       - name: Test
         run: $HOME/.cargo/bin/cargo test --features test-${{ matrix.blockchain.name }},test-blockchains --no-default-features ${{ matrix.blockchain.name }}::bdk_blockchain_tests
 
diff --git a/Cargo.toml b/Cargo.toml
index fb45a762..ebfff97f 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -69,7 +69,7 @@ test-md-docs = ["electrum"]
 lazy_static = "1.4"
 env_logger = "0.7"
 clap = "2.33"
-electrsd = { version="0.4", features = ["trigger"] }
+electrsd = { version="0.5", features = ["trigger", "bitcoind_0_21_1"] }
 
 [[example]]
 name = "address_validator"
diff --git a/src/testutils/blockchain_tests.rs b/src/testutils/blockchain_tests.rs
index a2174802..85cc0194 100644
--- a/src/testutils/blockchain_tests.rs
+++ b/src/testutils/blockchain_tests.rs
@@ -6,7 +6,7 @@ use bitcoin::{Address, Amount, Script, Transaction, Txid};
 pub use bitcoincore_rpc::bitcoincore_rpc_json::AddressType;
 pub use bitcoincore_rpc::{Auth, Client as RpcClient, RpcApi};
 use core::str::FromStr;
-use electrsd::bitcoind::BitcoinD;
+use electrsd::bitcoind::{downloaded_exe_path, BitcoinD};
 use electrsd::ElectrsD;
 pub use electrum_client::{Client as ElectrumClient, ElectrumApi};
 #[allow(unused_imports)]
@@ -309,8 +309,12 @@ impl Deref for TestClient {
 
 impl Default for TestClient {
     fn default() -> Self {
-        let bitcoind_exe =
-            env::var("BITCOIND_EXE").unwrap_or_else(|_| "/root/bitcoind".to_string());
+        let bitcoind_exe = env::var("BITCOIND_EXE")
+            .ok()
+            .or(downloaded_exe_path())
+            .expect(
+                "you should provide env var BITCOIND_EXE or specifiy a bitcoind version feature",
+            );
         let electrs_exe = env::var("ELECTRS_EXE").unwrap_or_else(|_| "/root/electrs".to_string());
         Self::new(bitcoind_exe, electrs_exe)
     }

From a6be470fe4ae9f12747cdbee21b3fb498c9b09f8 Mon Sep 17 00:00:00 2001
From: Riccardo Casatta <riccardo@casatta.it>
Date: Mon, 28 Jun 2021 15:04:17 +0200
Subject: [PATCH 09/11] use electrsd with feature to download the binary

---
 Cargo.toml                        |  8 ++--
 README.md                         | 19 +++++++++
 run_blockchain_tests.sh           | 71 -------------------------------
 src/blockchain/electrum.rs        |  2 +-
 src/blockchain/esplora.rs         |  2 +-
 src/blockchain/rpc.rs             |  2 +-
 src/testutils/blockchain_tests.rs | 13 ++++--
 7 files changed, 35 insertions(+), 82 deletions(-)
 delete mode 100755 run_blockchain_tests.sh

diff --git a/Cargo.toml b/Cargo.toml
index ebfff97f..8a5bfb96 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -60,16 +60,16 @@ rpc = ["bitcoincore-rpc"]
 
 # Debug/Test features
 test-blockchains = ["bitcoincore-rpc", "electrum-client"]
-test-electrum = ["electrum"]
-test-rpc = ["rpc"]
-test-esplora = ["esplora", "electrsd/legacy"]
+test-electrum = ["electrum", "electrsd/electrs_0_8_10", "test-blockchains"]
+test-rpc = ["rpc", "electrsd/electrs_0_8_10", "test-blockchains"]
+test-esplora = ["esplora", "electrsd/legacy", "electrsd/esplora_a33e97e1", "test-blockchains"]
 test-md-docs = ["electrum"]
 
 [dev-dependencies]
 lazy_static = "1.4"
 env_logger = "0.7"
 clap = "2.33"
-electrsd = { version="0.5", features = ["trigger", "bitcoind_0_21_1"] }
+electrsd = { version= "0.6", features = ["trigger", "bitcoind_0_21_1"] }
 
 [[example]]
 name = "address_validator"
diff --git a/README.md b/README.md
index 2d8c3c21..0181e775 100644
--- a/README.md
+++ b/README.md
@@ -151,6 +151,25 @@ fn main() -> Result<(), bdk::Error> {
 }
 ```
 
+## Testing
+
+### Unit testing
+
+```
+cargo test
+```
+
+### Integration testing
+
+Integration testing require testing features, for example:
+
+```
+cargo test --features test-electrum
+```
+
+The other options are `test-esplora` or `test-rpc`.
+Note that `electrs` and `bitcoind` binaries are automatically downloaded (on mac and linux), to specify your own one must use `--no-default-features` and provide `BITCOIND_EXE` and `ELECTRS_EXE` as environment variables.
+
 ## License
 
 Licensed under either of
diff --git a/run_blockchain_tests.sh b/run_blockchain_tests.sh
deleted file mode 100755
index 646e79cd..00000000
--- a/run_blockchain_tests.sh
+++ /dev/null
@@ -1,71 +0,0 @@
-#!/bin/sh
-
-usage() {
-    cat <<'EOF'
-Script for running the bdk blockchain tests for a specific blockchain by starting up the backend in docker.
-
-Usage: ./run_blockchain_tests.sh [esplora|electrum|rpc] [test name].
-
-EOF
-}
-
-eprintln(){
-    echo "$@" >&2
-}
-
-cleanup() {
-    if test "$id"; then
-        eprintln "cleaning up $blockchain docker container $id";
-        docker rm -fv "$id" > /dev/null;
-        rm /tmp/regtest-"$id".cookie;
-    fi
-    trap - EXIT INT
-}
-
-# Makes sure we clean up the container at the end or if ^C
-trap 'rc=$?; cleanup; exit $rc' EXIT INT
-
-blockchain="$1"
-test_name="$2"
-
-case "$blockchain" in
-    electrum)
-        eprintln "starting electrs docker container"
-        id="$(docker run --detach -p 127.0.0.1:18443-18444:18443-18444/tcp -p 127.0.0.1:60401:60401/tcp bitcoindevkit/electrs:0.4.0)"
-        ;;
-    esplora)
-        eprintln "starting esplora docker container"
-        id="$(docker run --detach -p 127.0.0.1:18443-18444:18443-18444/tcp -p 127.0.0.1:60401:60401/tcp -p 127.0.0.1:3002:3002/tcp bitcoindevkit/esplora:0.4.0)"
-        export BDK_ESPLORA_URL=http://127.0.0.1:3002
-        ;;
-    rpc)
-        eprintln "starting bitcoind docker container (via electrs container)"
-        id="$(docker run --detach -p 127.0.0.1:18443-18444:18443-18444/tcp -p 127.0.0.1:60401:60401/tcp bitcoindevkit/electrs:0.4.0)"      
-        ;;
-    *)
-        usage;
-        exit 1;
-        ;;
-    esac
-
-# taken from https://github.com/bitcoindevkit/bitcoin-regtest-box
-export BDK_RPC_AUTH=COOKIEFILE
-export BDK_RPC_COOKIEFILE=/tmp/regtest-"$id".cookie
-export BDK_RPC_URL=127.0.0.1:18443
-export BDK_RPC_WALLET=bdk-test
-export BDK_ELECTRUM_URL=tcp://127.0.0.1:60401
-
-cli(){
-    docker exec -it "$id" /root/bitcoin-cli -regtest -datadir=/root/.bitcoin $@
-}
-
-#eprintln "running getwalletinfo until bitcoind seems to be alive"
-while ! cli getwalletinfo >/dev/null; do sleep 1; done
-
-# sleep again for good measure!
-sleep 1;
-
-# copy bitcoind cookie file to /tmp
-docker cp "$id":/root/.bitcoin/regtest/.cookie /tmp/regtest-"$id".cookie
-
-cargo test --features "test-blockchains,test-$blockchain" --no-default-features "$blockchain::bdk_blockchain_tests::$test_name"
diff --git a/src/blockchain/electrum.rs b/src/blockchain/electrum.rs
index 45deb3f7..cba210a9 100644
--- a/src/blockchain/electrum.rs
+++ b/src/blockchain/electrum.rs
@@ -170,7 +170,7 @@ impl ConfigurableBlockchain for ElectrumBlockchain {
 }
 
 #[cfg(test)]
-#[cfg(feature = "test-blockchains")]
+#[cfg(feature = "test-electrum")]
 crate::bdk_blockchain_tests! {
     fn test_instance(test_client: &TestClient) -> ElectrumBlockchain {
         ElectrumBlockchain::from(Client::new(&test_client.electrsd.electrum_url).unwrap())
diff --git a/src/blockchain/esplora.rs b/src/blockchain/esplora.rs
index dce7f836..7f49fd83 100644
--- a/src/blockchain/esplora.rs
+++ b/src/blockchain/esplora.rs
@@ -416,7 +416,7 @@ impl_error!(consensus::encode::Error, BitcoinEncoding, EsploraError);
 impl_error!(bitcoin::hashes::hex::Error, Hex, EsploraError);
 
 #[cfg(test)]
-#[cfg(feature = "test-blockchains")]
+#[cfg(feature = "test-esplora")]
 crate::bdk_blockchain_tests! {
     fn test_instance(test_client: &TestClient) -> EsploraBlockchain {
         EsploraBlockchain::new(&format!("http://{}",test_client.electrsd.esplora_url.as_ref().unwrap()), None)
diff --git a/src/blockchain/rpc.rs b/src/blockchain/rpc.rs
index 0165313c..276d23d4 100644
--- a/src/blockchain/rpc.rs
+++ b/src/blockchain/rpc.rs
@@ -420,7 +420,7 @@ fn list_wallet_dir(client: &Client) -> Result<Vec<String>, Error> {
 }
 
 #[cfg(test)]
-#[cfg(feature = "test-blockchains")]
+#[cfg(feature = "test-rpc")]
 crate::bdk_blockchain_tests! {
 
     fn test_instance(test_client: &TestClient) -> RpcBlockchain {
diff --git a/src/testutils/blockchain_tests.rs b/src/testutils/blockchain_tests.rs
index 85cc0194..abde5877 100644
--- a/src/testutils/blockchain_tests.rs
+++ b/src/testutils/blockchain_tests.rs
@@ -6,8 +6,8 @@ use bitcoin::{Address, Amount, Script, Transaction, Txid};
 pub use bitcoincore_rpc::bitcoincore_rpc_json::AddressType;
 pub use bitcoincore_rpc::{Auth, Client as RpcClient, RpcApi};
 use core::str::FromStr;
-use electrsd::bitcoind::{downloaded_exe_path, BitcoinD};
-use electrsd::ElectrsD;
+use electrsd::bitcoind::BitcoinD;
+use electrsd::{bitcoind, ElectrsD};
 pub use electrum_client::{Client as ElectrumClient, ElectrumApi};
 #[allow(unused_imports)]
 use log::{debug, error, info, trace};
@@ -311,11 +311,16 @@ impl Default for TestClient {
     fn default() -> Self {
         let bitcoind_exe = env::var("BITCOIND_EXE")
             .ok()
-            .or(downloaded_exe_path())
+            .or(bitcoind::downloaded_exe_path())
             .expect(
                 "you should provide env var BITCOIND_EXE or specifiy a bitcoind version feature",
             );
-        let electrs_exe = env::var("ELECTRS_EXE").unwrap_or_else(|_| "/root/electrs".to_string());
+        let electrs_exe = env::var("ELECTRS_EXE")
+            .ok()
+            .or(electrsd::downloaded_exe_path())
+            .expect(
+                "you should provide env var ELECTRS_EXE or specifiy a electrsd version feature",
+            );
         Self::new(bitcoind_exe, electrs_exe)
     }
 }

From f52fda4b4b72ef25f9c061c01d55bd856cd97e70 Mon Sep 17 00:00:00 2001
From: Riccardo Casatta <riccardo@casatta.it>
Date: Mon, 28 Jun 2021 16:12:25 +0200
Subject: [PATCH 10/11] update github ci removing electrs download and fixing
 cache

---
 .github/workflows/cont_integration.yml | 18 +++---------------
 1 file changed, 3 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/cont_integration.yml b/.github/workflows/cont_integration.yml
index 9e5be26f..04847357 100644
--- a/.github/workflows/cont_integration.yml
+++ b/.github/workflows/cont_integration.yml
@@ -84,10 +84,6 @@ jobs:
           - name: electrum
           - name: rpc
           - name: esplora
-    env:
-      ELECTRS_RELEASE_URL: https://github.com/RCasatta/electrsd/releases/download/release_0.3.0
-      ELECTRS_VER: electrs_ubuntu-20.04_v0.8.10.zip
-      ELECTRS_ESPLORA_VER: electrs_esplora_ubuntu-20.04_a33e97e1a1fc63fa9c20a116bb92579bbf43b254.zip
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -96,6 +92,8 @@ jobs:
         with:
           path: |
             ~/.cargo/registry
+            ~/.cargo/bitcoin
+            ~/.cargo/electrs
             ~/.cargo/git
             target
           key: ${{ runner.os }}-cargo-${{ github.job }}-${{ hashFiles('**/Cargo.toml','**/Cargo.lock') }}
@@ -104,18 +102,8 @@ jobs:
         with:
           toolchain: stable
           override: true
-      - name: Download electrs
-        run: wget ${{ env.ELECTRS_RELEASE_URL }}/${{ env.ELECTRS_VER }} && unzip ${{ env.ELECTRS_VER }} && chmod +x ./electrs
-        if: ${{ matrix.blockchain.name != 'esplora' }}
-      - name: Download electrs esplora
-        run: wget ${{ env.ELECTRS_RELEASE_URL }}/${{ env.ELECTRS_ESPLORA_VER }} && unzip ${{ env.ELECTRS_ESPLORA_VER }} && chmod +x ./electrs
-        if: ${{ matrix.blockchain.name == 'esplora' }}
-      - name: Set ELECTRS_EXE env
-        run: echo "ELECTRS_EXE=${{ github.workspace }}/electrs" >> $GITHUB_ENV
-      - name: Show electrs options
-        run: ${{ env.ELECTRS_EXE }} --help
       - name: Test
-        run: $HOME/.cargo/bin/cargo test --features test-${{ matrix.blockchain.name }},test-blockchains --no-default-features ${{ matrix.blockchain.name }}::bdk_blockchain_tests
+        run: cargo test --features test-${{ matrix.blockchain.name }} ${{ matrix.blockchain.name }}::bdk_blockchain_tests
 
   check-wasm:
     name: Check WASM

From 7109f7d9b47e876df7f93c9345c59b621a4f295e Mon Sep 17 00:00:00 2001
From: Riccardo Casatta <riccardo@casatta.it>
Date: Tue, 29 Jun 2021 11:30:45 +0200
Subject: [PATCH 11/11] fix readme

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 0181e775..c2a74189 100644
--- a/README.md
+++ b/README.md
@@ -168,7 +168,7 @@ cargo test --features test-electrum
 ```
 
 The other options are `test-esplora` or `test-rpc`.
-Note that `electrs` and `bitcoind` binaries are automatically downloaded (on mac and linux), to specify your own one must use `--no-default-features` and provide `BITCOIND_EXE` and `ELECTRS_EXE` as environment variables.
+Note that `electrs` and `bitcoind` binaries are automatically downloaded (on mac and linux), to specify you already have installed binaries you must use `--no-default-features` and provide `BITCOIND_EXE` and `ELECTRS_EXE` as environment variables.
 
 ## License