From 4f99c77abee714e6c4e5b7182d8d06531a6d74d7 Mon Sep 17 00:00:00 2001
From: Riccardo Casatta <riccardo@casatta.it>
Date: Mon, 16 Nov 2020 12:06:48 +0100
Subject: [PATCH 01/16] [sync] check last derivation in cache to avoid
 recomputation

---
 src/wallet/mod.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/wallet/mod.rs b/src/wallet/mod.rs
index 79ce7789..94c5a122 100644
--- a/src/wallet/mod.rs
+++ b/src/wallet/mod.rs
@@ -1268,7 +1268,7 @@ where
         if self
             .database
             .borrow()
-            .get_script_pubkey_from_path(ScriptType::External, max_address)?
+            .get_script_pubkey_from_path(ScriptType::External, max_address.saturating_sub(1))?
             .is_none()
         {
             run_setup = true;

From 25da54d5ec64345980ec46a15d309fba8072889c Mon Sep 17 00:00:00 2001
From: Riccardo Casatta <riccardo@casatta.it>
Date: Mon, 16 Nov 2020 12:09:14 +0100
Subject: [PATCH 02/16] ignore .idea

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index 8b75e7bb..4dc9a2ff 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,3 +2,4 @@
 Cargo.lock
 
 *.swp
+.idea

From 755d76bf544e88fae7342f35eee97f2629183776 Mon Sep 17 00:00:00 2001
From: Riccardo Casatta <riccardo@casatta.it>
Date: Mon, 16 Nov 2020 12:11:37 +0100
Subject: [PATCH 03/16] remove unneeded pub modifier

---
 src/database/memory.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/database/memory.rs b/src/database/memory.rs
index eeacd131..707cd951 100644
--- a/src/database/memory.rs
+++ b/src/database/memory.rs
@@ -57,7 +57,7 @@ pub(crate) enum MapKey<'a> {
 }
 
 impl MapKey<'_> {
-    pub fn as_prefix(&self) -> Vec<u8> {
+    fn as_prefix(&self) -> Vec<u8> {
         match self {
             MapKey::Path((st, _)) => {
                 let mut v = b"p".to_vec();

From c5dba115a0b76906a1c60cbf38e398fa0d03864f Mon Sep 17 00:00:00 2001
From: Riccardo Casatta <riccardo@casatta.it>
Date: Mon, 16 Nov 2020 12:18:34 +0100
Subject: [PATCH 04/16] =?UTF-8?q?[sync]=C2=A0Improve=20sync?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Make every request in batch, to save round trip times
Fetch timestamp of blockheader to populate timestamp field in transaction
Remove listunspent requests because we can compute it from our history
---
 src/blockchain/electrum.rs |  40 +--
 src/blockchain/esplora.rs  | 152 +++++++---
 src/blockchain/mod.rs      |   1 +
 src/blockchain/utils.rs    | 592 ++++++++++++++++++++-----------------
 4 files changed, 440 insertions(+), 345 deletions(-)

diff --git a/src/blockchain/electrum.rs b/src/blockchain/electrum.rs
index 9655ebbb..a134158c 100644
--- a/src/blockchain/electrum.rs
+++ b/src/blockchain/electrum.rs
@@ -42,11 +42,11 @@ use std::collections::HashSet;
 #[allow(unused_imports)]
 use log::{debug, error, info, trace};
 
-use bitcoin::{Script, Transaction, Txid};
+use bitcoin::{BlockHeader, Script, Transaction, Txid};
 
 use electrum_client::{Client, ElectrumApi};
 
-use self::utils::{ELSGetHistoryRes, ELSListUnspentRes, ElectrumLikeSync};
+use self::utils::{ELSGetHistoryRes, ElectrumLikeSync};
 use super::*;
 use crate::database::BatchDatabase;
 use crate::error::Error;
@@ -141,36 +141,18 @@ impl ElectrumLikeSync for Client {
             .map_err(Error::Electrum)
     }
 
-    fn els_batch_script_list_unspent<'s, I: IntoIterator<Item = &'s Script>>(
+    fn els_batch_transaction_get<'s, I: IntoIterator<Item = &'s Txid>>(
         &self,
-        scripts: I,
-    ) -> Result<Vec<Vec<ELSListUnspentRes>>, Error> {
-        self.batch_script_list_unspent(scripts)
-            .map(|v| {
-                v.into_iter()
-                    .map(|v| {
-                        v.into_iter()
-                            .map(
-                                |electrum_client::ListUnspentRes {
-                                     height,
-                                     tx_hash,
-                                     tx_pos,
-                                     ..
-                                 }| ELSListUnspentRes {
-                                    height,
-                                    tx_hash,
-                                    tx_pos,
-                                },
-                            )
-                            .collect()
-                    })
-                    .collect()
-            })
-            .map_err(Error::Electrum)
+        txids: I,
+    ) -> Result<Vec<Transaction>, Error> {
+        self.batch_transaction_get(txids).map_err(Error::Electrum)
     }
 
-    fn els_transaction_get(&self, txid: &Txid) -> Result<Transaction, Error> {
-        self.transaction_get(txid).map_err(Error::Electrum)
+    fn els_batch_block_header<I: IntoIterator<Item = u32>>(
+        &self,
+        heights: I,
+    ) -> Result<Vec<BlockHeader>, Error> {
+        self.batch_block_header(heights).map_err(Error::Electrum)
     }
 }
 
diff --git a/src/blockchain/esplora.rs b/src/blockchain/esplora.rs
index ef649d0b..2b69d523 100644
--- a/src/blockchain/esplora.rs
+++ b/src/blockchain/esplora.rs
@@ -48,15 +48,16 @@ use serde::Deserialize;
 use reqwest::{Client, StatusCode};
 
 use bitcoin::consensus::{deserialize, serialize};
-use bitcoin::hashes::hex::ToHex;
+use bitcoin::hashes::hex::{FromHex, ToHex};
 use bitcoin::hashes::{sha256, Hash};
-use bitcoin::{Script, Transaction, Txid};
+use bitcoin::{BlockHash, BlockHeader, Script, Transaction, TxMerkleNode, Txid};
 
-use self::utils::{ELSGetHistoryRes, ELSListUnspentRes, ElectrumLikeSync};
+use self::utils::{ELSGetHistoryRes, ElectrumLikeSync};
 use super::*;
 use crate::database::BatchDatabase;
 use crate::error::Error;
 use crate::FeeRate;
+use std::convert::TryInto;
 
 #[derive(Debug)]
 struct UrlClient {
@@ -161,6 +162,39 @@ impl UrlClient {
         Ok(Some(deserialize(&resp.error_for_status()?.bytes().await?)?))
     }
 
+    async fn _get_tx_no_opt(&self, txid: &Txid) -> Result<Transaction, EsploraError> {
+        match self._get_tx(txid).await {
+            Ok(Some(tx)) => Ok(tx),
+            Ok(None) => Err(EsploraError::TransactionNotFound(*txid)),
+            Err(e) => Err(e),
+        }
+    }
+
+    async fn _get_header(&self, block_height: u32) -> Result<BlockHeader, EsploraError> {
+        let resp = self
+            .client
+            .get(&format!("{}/block-height/{}", self.url, block_height))
+            .send()
+            .await?;
+
+        if let StatusCode::NOT_FOUND = resp.status() {
+            return Err(EsploraError::HeaderHeightNotFound(block_height));
+        }
+        let bytes = resp.bytes().await?;
+        let hash = std::str::from_utf8(&bytes)
+            .map_err(|_| EsploraError::HeaderHeightNotFound(block_height))?;
+
+        let resp = self
+            .client
+            .get(&format!("{}/block/{}", self.url, hash))
+            .send()
+            .await?;
+
+        let esplora_header = resp.json::<EsploraHeader>().await?;
+
+        Ok(esplora_header.try_into()?)
+    }
+
     async fn _broadcast(&self, transaction: &Transaction) -> Result<(), EsploraError> {
         self.client
             .post(&format!("{}/tx", self.url))
@@ -249,31 +283,6 @@ impl UrlClient {
         Ok(result)
     }
 
-    async fn _script_list_unspent(
-        &self,
-        script: &Script,
-    ) -> Result<Vec<ELSListUnspentRes>, EsploraError> {
-        Ok(self
-            .client
-            .get(&format!(
-                "{}/scripthash/{}/utxo",
-                self.url,
-                Self::script_to_scripthash(script)
-            ))
-            .send()
-            .await?
-            .error_for_status()?
-            .json::<Vec<EsploraListUnspent>>()
-            .await?
-            .into_iter()
-            .map(|x| ELSListUnspentRes {
-                tx_hash: x.txid,
-                height: x.status.block_height.unwrap_or(0),
-                tx_pos: x.vout,
-            })
-            .collect())
-    }
-
     async fn _get_fee_estimates(&self) -> Result<HashMap<String, f64>, EsploraError> {
         Ok(self
             .client
@@ -302,13 +311,13 @@ impl ElectrumLikeSync for UrlClient {
         await_or_block!(future)
     }
 
-    fn els_batch_script_list_unspent<'s, I: IntoIterator<Item = &'s Script>>(
+    fn els_batch_transaction_get<'s, I: IntoIterator<Item = &'s Txid>>(
         &self,
-        scripts: I,
-    ) -> Result<Vec<Vec<ELSListUnspentRes>>, Error> {
+        txids: I,
+    ) -> Result<Vec<Transaction>, Error> {
         let future = async {
-            Ok(stream::iter(scripts)
-                .then(|script| self._script_list_unspent(&script))
+            Ok(stream::iter(txids)
+                .then(|txid| self._get_tx_no_opt(&txid))
                 .try_collect()
                 .await?)
         };
@@ -316,9 +325,18 @@ impl ElectrumLikeSync for UrlClient {
         await_or_block!(future)
     }
 
-    fn els_transaction_get(&self, txid: &Txid) -> Result<Transaction, Error> {
-        Ok(await_or_block!(self._get_tx(txid))?
-            .ok_or_else(|| EsploraError::TransactionNotFound(*txid))?)
+    fn els_batch_block_header<I: IntoIterator<Item = u32>>(
+        &self,
+        heights: I,
+    ) -> Result<Vec<BlockHeader>, Error> {
+        let future = async {
+            Ok(stream::iter(heights)
+                .then(|h| self._get_header(h))
+                .try_collect()
+                .await?)
+        };
+
+        await_or_block!(future)
     }
 }
 
@@ -333,11 +351,37 @@ struct EsploraGetHistory {
     status: EsploraGetHistoryStatus,
 }
 
-#[derive(Deserialize)]
-struct EsploraListUnspent {
-    txid: Txid,
-    vout: usize,
-    status: EsploraGetHistoryStatus,
+#[derive(Default, Debug, Clone, PartialEq, Deserialize)]
+pub struct EsploraHeader {
+    pub id: String,
+    pub height: u32,
+    pub version: i32,
+    pub timestamp: u32,
+    pub tx_count: u32,
+    pub size: u32,
+    pub weight: u32,
+    pub merkle_root: String,
+    pub previousblockhash: String,
+    pub nonce: u32,
+    pub bits: u32,
+    pub difficulty: u32,
+}
+
+impl TryInto<BlockHeader> for EsploraHeader {
+    type Error = EsploraError;
+
+    fn try_into(self) -> Result<BlockHeader, esplora::EsploraError> {
+        Ok(BlockHeader {
+            version: self.version,
+            prev_blockhash: BlockHash::from_hex(&self.previousblockhash)
+                .map_err(|_| EsploraError::HeaderParseFail)?,
+            merkle_root: TxMerkleNode::from_hex(&self.merkle_root)
+                .map_err(|_| EsploraError::HeaderParseFail)?,
+            time: self.timestamp,
+            bits: self.bits,
+            nonce: self.nonce,
+        })
+    }
 }
 
 /// Configuration for an [`EsploraBlockchain`]
@@ -366,6 +410,12 @@ pub enum EsploraError {
 
     /// Transaction not found
     TransactionNotFound(Txid),
+    /// Header height not found
+    HeaderHeightNotFound(u32),
+    /// Header hash not found
+    HeaderHashNotFound(BlockHash),
+    /// EsploraHeader cannot be converted in BlockHeader
+    HeaderParseFail,
 }
 
 impl fmt::Display for EsploraError {
@@ -393,3 +443,23 @@ impl From<bitcoin::consensus::encode::Error> for EsploraError {
         EsploraError::BitcoinEncoding(other)
     }
 }
+
+#[cfg(test)]
+mod test {
+    use crate::blockchain::esplora::EsploraHeader;
+    use bitcoin::hashes::hex::FromHex;
+    use bitcoin::{BlockHash, BlockHeader};
+    use std::convert::TryInto;
+
+    #[test]
+    fn test_esplora_header() {
+        let json_str = r#"{"id":"00000000b873e79784647a6c82962c70d228557d24a747ea4d1b8bbe878e1206","height":1,"version":1,"timestamp":1296688928,"tx_count":1,"size":190,"weight":760,"merkle_root":"f0315ffc38709d70ad5647e22048358dd3745f3ce3874223c80a7c92fab0c8ba","previousblockhash":"000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943","nonce":1924588547,"bits":486604799,"difficulty":1}"#;
+        let json: EsploraHeader = serde_json::from_str(&json_str).unwrap();
+        let header: BlockHeader = json.try_into().unwrap();
+        assert_eq!(
+            header.block_hash(),
+            BlockHash::from_hex("00000000b873e79784647a6c82962c70d228557d24a747ea4d1b8bbe878e1206")
+                .unwrap()
+        );
+    }
+}
diff --git a/src/blockchain/mod.rs b/src/blockchain/mod.rs
index 83b6903d..6711e51e 100644
--- a/src/blockchain/mod.rs
+++ b/src/blockchain/mod.rs
@@ -40,6 +40,7 @@ use crate::database::BatchDatabase;
 use crate::error::Error;
 use crate::FeeRate;
 
+#[cfg(any(feature = "electrum", feature = "esplora"))]
 pub(crate) mod utils;
 
 #[cfg(any(feature = "electrum", feature = "esplora", feature = "compact_filters"))]
diff --git a/src/blockchain/utils.rs b/src/blockchain/utils.rs
index bf8ff6cb..c18b5fa4 100644
--- a/src/blockchain/utils.rs
+++ b/src/blockchain/utils.rs
@@ -22,20 +22,21 @@
 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 // SOFTWARE.
 
-use std::cmp;
-use std::collections::{HashSet, VecDeque};
-use std::convert::TryFrom;
+use std::collections::{HashMap, HashSet};
 
 #[allow(unused_imports)]
 use log::{debug, error, info, trace};
 
-use bitcoin::{Address, Network, OutPoint, Script, Transaction, Txid};
+use bitcoin::{BlockHeader, OutPoint, Script, Transaction, Txid};
 
 use super::*;
 use crate::database::{BatchDatabase, BatchOperations, DatabaseUtils};
 use crate::error::Error;
 use crate::types::{ScriptType, TransactionDetails, UTXO};
 use crate::wallet::utils::ChunksIterator;
+use rand::seq::SliceRandom;
+use rand::thread_rng;
+use std::time::Instant;
 
 #[derive(Debug)]
 pub struct ELSGetHistoryRes {
@@ -43,13 +44,6 @@ pub struct ELSGetHistoryRes {
     pub tx_hash: Txid,
 }
 
-#[derive(Debug)]
-pub struct ELSListUnspentRes {
-    pub height: usize,
-    pub tx_hash: Txid,
-    pub tx_pos: usize,
-}
-
 /// Implements the synchronization logic for an Electrum-like client.
 #[maybe_async]
 pub trait ElectrumLikeSync {
@@ -58,306 +52,354 @@ pub trait ElectrumLikeSync {
         scripts: I,
     ) -> Result<Vec<Vec<ELSGetHistoryRes>>, Error>;
 
-    fn els_batch_script_list_unspent<'s, I: IntoIterator<Item = &'s Script>>(
+    fn els_batch_transaction_get<'s, I: IntoIterator<Item = &'s Txid>>(
         &self,
-        scripts: I,
-    ) -> Result<Vec<Vec<ELSListUnspentRes>>, Error>;
+        txids: I,
+    ) -> Result<Vec<Transaction>, Error>;
 
-    fn els_transaction_get(&self, txid: &Txid) -> Result<Transaction, Error>;
+    fn els_batch_block_header<I: IntoIterator<Item = u32>>(
+        &self,
+        heights: I,
+    ) -> Result<Vec<BlockHeader>, Error>;
 
     // Provided methods down here...
 
     fn electrum_like_setup<D: BatchDatabase, P: Progress>(
         &self,
         stop_gap: Option<usize>,
-        database: &mut D,
+        db: &mut D,
         _progress_update: P,
     ) -> Result<(), Error> {
         // TODO: progress
+        let start = Instant::now();
+        debug!("start setup");
 
         let stop_gap = stop_gap.unwrap_or(20);
-        let batch_query_size = 20;
+        let chunk_size = stop_gap;
 
-        // check unconfirmed tx, delete so they are retrieved later
-        let mut del_batch = database.begin_batch();
-        for tx in database.iter_txs(false)? {
-            if tx.height.is_none() {
-                del_batch.del_tx(&tx.txid, false)?;
-            }
-        }
-        database.commit_batch(del_batch)?;
+        let mut history_txs_id = HashSet::new();
+        let mut txid_height = HashMap::new();
+        let mut max_indexes = HashMap::new();
 
-        // maximum derivation index for a change address that we've seen during sync
-        let mut change_max_deriv = None;
+        let mut wallet_chains = vec![ScriptType::Internal, ScriptType::External];
+        // shuffling improve privacy, the server doesn't know my first request is from my internal or external addresses
+        wallet_chains.shuffle(&mut thread_rng());
+        // download history of our internal and external script_pubkeys
+        for script_type in wallet_chains.iter() {
+            let script_iter = db.iter_script_pubkeys(Some(*script_type))?.into_iter();
 
-        let mut already_checked: HashSet<Script> = HashSet::new();
-        let mut to_check_later = VecDeque::with_capacity(batch_query_size);
-
-        // insert the first chunk
-        let mut iter_scriptpubkeys = database
-            .iter_script_pubkeys(Some(ScriptType::External))?
-            .into_iter();
-        let chunk: Vec<Script> = iter_scriptpubkeys.by_ref().take(batch_query_size).collect();
-        for item in chunk.into_iter().rev() {
-            to_check_later.push_front(item);
-        }
-
-        let mut iterating_external = true;
-        let mut index = 0;
-        let mut last_found = None;
-        while !to_check_later.is_empty() {
-            trace!("to_check_later size {}", to_check_later.len());
-
-            let until = cmp::min(to_check_later.len(), batch_query_size);
-            let chunk: Vec<Script> = to_check_later.drain(..until).collect();
-            let call_result = maybe_await!(self.els_batch_script_get_history(chunk.iter()))?;
-
-            for (script, history) in chunk.into_iter().zip(call_result.into_iter()) {
-                trace!("received history for {:?}, size {}", script, history.len());
-
-                if !history.is_empty() {
-                    last_found = Some(index);
-
-                    let mut check_later_scripts = maybe_await!(self.check_history(
-                        database,
-                        script,
-                        history,
-                        &mut change_max_deriv
-                    ))?
-                    .into_iter()
-                    .filter(|x| already_checked.insert(x.clone()))
-                    .collect();
-                    to_check_later.append(&mut check_later_scripts);
+            for (i, chunk) in ChunksIterator::new(script_iter, stop_gap).enumerate() {
+                // TODO if i == last, should create another chunk of addresses in db
+                let call_result: Vec<Vec<ELSGetHistoryRes>> =
+                    maybe_await!(self.els_batch_script_get_history(chunk.iter()))?;
+                let max_index = call_result
+                    .iter()
+                    .enumerate()
+                    .filter(|(_, v)| !v.is_empty())
+                    .map(|(i, _)| i as u32)
+                    .max();
+                if let Some(max) = max_index {
+                    max_indexes.insert(script_type, max + (i * chunk_size) as u32);
+                }
+                let flattened: Vec<ELSGetHistoryRes> = call_result.into_iter().flatten().collect();
+                debug!("#{} of {:?} results:{}", i, script_type, flattened.len());
+                if flattened.is_empty() {
+                    // Didn't find anything in the last `stop_gap` script_pubkeys, breaking
+                    break;
                 }
 
-                index += 1;
-            }
-
-            match iterating_external {
-                true if index - last_found.unwrap_or(0) >= stop_gap => iterating_external = false,
-                true => {
-                    trace!("pushing one more batch from `iter_scriptpubkeys`. index = {}, last_found = {:?}, stop_gap = {}", index, last_found, stop_gap);
-
-                    let chunk: Vec<Script> =
-                        iter_scriptpubkeys.by_ref().take(batch_query_size).collect();
-                    for item in chunk.into_iter().rev() {
-                        to_check_later.push_front(item);
+                for el in flattened {
+                    // el.height = -1 means unconfirmed with unconfirmed parents
+                    // el.height =  0 means unconfirmed with confirmed parents
+                    // but we threat those tx the same
+                    let height = el.height.max(0);
+                    if height == 0 {
+                        txid_height.insert(el.tx_hash, None);
+                    } else {
+                        txid_height.insert(el.tx_hash, Some(height as u32));
                     }
-                }
-                _ => {}
-            }
-        }
-
-        // check utxo
-        // TODO: try to minimize network requests and re-use scripts if possible
-        let mut batch = database.begin_batch();
-        for chunk in ChunksIterator::new(database.iter_utxos()?.into_iter(), batch_query_size) {
-            let scripts: Vec<_> = chunk.iter().map(|u| &u.txout.script_pubkey).collect();
-            let call_result = maybe_await!(self.els_batch_script_list_unspent(scripts))?;
-
-            // check which utxos are actually still unspent
-            for (utxo, list_unspent) in chunk.into_iter().zip(call_result.iter()) {
-                debug!(
-                    "outpoint {:?} is unspent for me, list unspent is {:?}",
-                    utxo.outpoint, list_unspent
-                );
-
-                let mut spent = true;
-                for unspent in list_unspent {
-                    let res_outpoint = OutPoint::new(unspent.tx_hash, unspent.tx_pos as u32);
-                    if utxo.outpoint == res_outpoint {
-                        spent = false;
-                        break;
-                    }
-                }
-                if spent {
-                    info!("{} not anymore unspent, removing", utxo.outpoint);
-                    batch.del_utxo(&utxo.outpoint)?;
+                    history_txs_id.insert(el.tx_hash);
                 }
             }
         }
 
-        let current_ext = database.get_last_index(ScriptType::External)?.unwrap_or(0);
-        let first_ext_new = last_found.map(|x| x + 1).unwrap_or(0) as u32;
-        if first_ext_new > current_ext {
-            info!("Setting external index to {}", first_ext_new);
-            database.set_last_index(ScriptType::External, first_ext_new)?;
+        // saving max indexes
+        info!("max indexes are: {:?}", max_indexes);
+        for script_type in wallet_chains.iter() {
+            if let Some(index) = max_indexes.get(script_type) {
+                db.set_last_index(*script_type, *index)?;
+            }
         }
 
-        let current_int = database.get_last_index(ScriptType::Internal)?.unwrap_or(0);
-        let first_int_new = change_max_deriv.map(|x| x + 1).unwrap_or(0);
-        if first_int_new > current_int {
-            info!("Setting internal index to {}", first_int_new);
-            database.set_last_index(ScriptType::Internal, first_int_new)?;
+        // get db status
+        let txs_details_in_db: HashMap<Txid, TransactionDetails> = db
+            .iter_txs(false)?
+            .into_iter()
+            .map(|tx| (tx.txid, tx))
+            .collect();
+        let txs_raw_in_db: HashMap<Txid, Transaction> = db
+            .iter_raw_txs()?
+            .into_iter()
+            .map(|tx| (tx.txid(), tx))
+            .collect();
+        let utxos_deps = utxos_deps(db, &txs_raw_in_db)?;
+
+        // download new txs and headers
+        let new_txs = maybe_await!(self.download_and_save_needed_raw_txs(
+            &history_txs_id,
+            &txs_raw_in_db,
+            chunk_size,
+            db
+        ))?;
+        let new_timestamps = maybe_await!(self.download_needed_headers(
+            &txid_height,
+            &txs_details_in_db,
+            chunk_size
+        ))?;
+
+        let mut batch = db.begin_batch();
+
+        // save any tx details not in db but in history_txs_id or with different height/timestamp
+        for txid in history_txs_id.iter() {
+            let height = *txid_height.get(txid).unwrap_or(&None);
+            let timestamp = *new_timestamps.get(txid).unwrap_or(&0u64);
+            if let Some(tx_details) = txs_details_in_db.get(txid) {
+                // check if height matches, otherwise updates it
+                if tx_details.height != height {
+                    let mut new_tx_details = tx_details.clone();
+                    new_tx_details.height = height;
+                    new_tx_details.timestamp = timestamp;
+                    batch.set_tx(&new_tx_details)?;
+                }
+            } else {
+                save_transaction_details_and_utxos(
+                    &txid,
+                    db,
+                    timestamp,
+                    height,
+                    &mut batch,
+                    &utxos_deps,
+                )?;
+            }
         }
 
-        database.commit_batch(batch)?;
+        // remove any tx details in db but not in history_txs_id
+        for txid in txs_details_in_db.keys() {
+            if !history_txs_id.contains(txid) {
+                batch.del_tx(&txid, false)?;
+            }
+        }
+
+        // remove any spent utxo
+        for new_tx in new_txs.iter() {
+            for input in new_tx.input.iter() {
+                batch.del_utxo(&input.previous_output)?;
+            }
+        }
+
+        db.commit_batch(batch)?;
+        info!("finish setup, elapsed {:?}ms", start.elapsed().as_millis());
 
         Ok(())
     }
 
-    fn check_tx_and_descendant<D: BatchDatabase>(
+    /// download txs identified by `history_txs_id` and theirs previous outputs if not already present in db
+    fn download_and_save_needed_raw_txs<D: BatchDatabase>(
         &self,
-        database: &mut D,
-        txid: &Txid,
-        height: Option<u32>,
-        cur_script: &Script,
-        change_max_deriv: &mut Option<u32>,
-    ) -> Result<Vec<Script>, Error> {
-        debug!(
-            "check_tx_and_descendant of {}, height: {:?}, script: {}",
-            txid, height, cur_script
-        );
-        let mut updates = database.begin_batch();
-        let tx = match database.get_tx(&txid, true)? {
-            Some(mut saved_tx) => {
-                // update the height if it's different (in case of reorg)
-                if saved_tx.height != height {
-                    info!(
-                        "updating height from {:?} to {:?} for tx {}",
-                        saved_tx.height, height, txid
-                    );
-                    saved_tx.height = height;
-                    updates.set_tx(&saved_tx)?;
-                }
-
-                debug!("already have {} in db, returning the cached version", txid);
-
-                // unwrap since we explicitly ask for the raw_tx, if it's not present something
-                // went wrong
-                saved_tx.transaction.unwrap()
-            }
-            None => {
-                let fetched_tx = maybe_await!(self.els_transaction_get(&txid))?;
-                database.set_raw_tx(&fetched_tx)?;
-
-                fetched_tx
-            }
-        };
-
-        let mut incoming: u64 = 0;
-        let mut outgoing: u64 = 0;
-
-        let mut inputs_sum: u64 = 0;
-        let mut outputs_sum: u64 = 0;
-
-        // look for our own inputs
-        for (i, input) in tx.input.iter().enumerate() {
-            // skip coinbase inputs
-            if input.previous_output.is_null() {
-                continue;
-            }
-
-            // the fact that we visit addresses in a BFS fashion starting from the external addresses
-            // should ensure that this query is always consistent (i.e. when we get to call this all
-            // the transactions at a lower depth have already been indexed, so if an outpoint is ours
-            // we are guaranteed to have it in the db).
-            if let Some(previous_output) = database.get_previous_output(&input.previous_output)? {
-                inputs_sum += previous_output.value;
-
-                if database.is_mine(&previous_output.script_pubkey)? {
-                    outgoing += previous_output.value;
-
-                    debug!("{} input #{} is mine, removing from utxo", txid, i);
-                    updates.del_utxo(&input.previous_output)?;
-                }
-            } else {
-                // The input is not ours, but we still need to count it for the fees. so fetch the
-                // tx (from the database or from network) and check it
-                let tx = match database.get_tx(&input.previous_output.txid, true)? {
-                    Some(saved_tx) => saved_tx.transaction.unwrap(),
-                    None => {
-                        let fetched_tx =
-                            maybe_await!(self.els_transaction_get(&input.previous_output.txid))?;
-                        database.set_raw_tx(&fetched_tx)?;
-
-                        fetched_tx
-                    }
-                };
-
-                inputs_sum += tx.output[input.previous_output.vout as usize].value;
-            }
-        }
-
-        let mut to_check_later = vec![];
-        for (i, output) in tx.output.iter().enumerate() {
-            // to compute the fees later
-            outputs_sum += output.value;
-
-            // this output is ours, we have a path to derive it
-            if let Some((script_type, child)) =
-                database.get_path_from_script_pubkey(&output.script_pubkey)?
-            {
-                debug!("{} output #{} is mine, adding utxo", txid, i);
-                updates.set_utxo(&UTXO {
-                    outpoint: OutPoint::new(tx.txid(), i as u32),
-                    txout: output.clone(),
-                    is_internal: script_type.is_internal(),
-                })?;
-                incoming += output.value;
-
-                if output.script_pubkey != *cur_script {
-                    debug!("{} output #{} script {} was not current script, adding script to be checked later", txid, i, output.script_pubkey);
-                    to_check_later.push(output.script_pubkey.clone())
-                }
-
-                // derive as many change addrs as external addresses that we've seen
-                if script_type == ScriptType::Internal
-                    && (change_max_deriv.is_none() || child > change_max_deriv.unwrap_or(0))
-                {
-                    *change_max_deriv = Some(child);
+        history_txs_id: &HashSet<Txid>,
+        txs_raw_in_db: &HashMap<Txid, Transaction>,
+        chunk_size: usize,
+        db: &mut D,
+    ) -> Result<Vec<Transaction>, Error> {
+        let mut txs_downloaded = vec![];
+        let txids_raw_in_db: HashSet<Txid> = txs_raw_in_db.keys().cloned().collect();
+        let txids_to_download: Vec<&Txid> = history_txs_id.difference(&txids_raw_in_db).collect();
+        if !txids_to_download.is_empty() {
+            info!("got {} txs to download", txids_to_download.len());
+            txs_downloaded.extend(maybe_await!(self.download_and_save_in_chunks(
+                txids_to_download,
+                chunk_size,
+                db,
+            ))?);
+            let mut prev_txids = HashSet::new();
+            let mut txids_downloaded = HashSet::new();
+            for tx in txs_downloaded.iter() {
+                txids_downloaded.insert(tx.txid());
+                // add every previous input tx, but skip coinbase
+                for input in tx.input.iter().filter(|i| !i.previous_output.is_null()) {
+                    prev_txids.insert(input.previous_output.txid);
                 }
             }
-        }
-
-        let tx = TransactionDetails {
-            txid: tx.txid(),
-            transaction: Some(tx),
-            received: incoming,
-            sent: outgoing,
-            height,
-            timestamp: 0,
-            fees: inputs_sum.saturating_sub(outputs_sum), // if the tx is a coinbase, fees would be negative
-        };
-        info!("Saving tx {}", txid);
-        updates.set_tx(&tx)?;
-
-        database.commit_batch(updates)?;
-
-        Ok(to_check_later)
-    }
-
-    fn check_history<D: BatchDatabase>(
-        &self,
-        database: &mut D,
-        script_pubkey: Script,
-        txs: Vec<ELSGetHistoryRes>,
-        change_max_deriv: &mut Option<u32>,
-    ) -> Result<Vec<Script>, Error> {
-        let mut to_check_later = Vec::new();
-
-        debug!(
-            "history of {} script {} has {} tx",
-            Address::from_script(&script_pubkey, Network::Testnet).unwrap(),
-            script_pubkey,
-            txs.len()
-        );
-
-        for tx in txs {
-            let height: Option<u32> = match tx.height {
-                0 | -1 => None,
-                x => u32::try_from(x).ok(),
-            };
-
-            to_check_later.extend_from_slice(&maybe_await!(self.check_tx_and_descendant(
-                database,
-                &tx.tx_hash,
-                height,
-                &script_pubkey,
-                change_max_deriv,
+            let already_present: HashSet<Txid> =
+                txids_downloaded.union(&txids_raw_in_db).cloned().collect();
+            let prev_txs_to_download: Vec<&Txid> =
+                prev_txids.difference(&already_present).collect();
+            info!("{} previous txs to download", prev_txs_to_download.len());
+            txs_downloaded.extend(maybe_await!(self.download_and_save_in_chunks(
+                prev_txs_to_download,
+                chunk_size,
+                db,
             ))?);
         }
 
-        Ok(to_check_later)
+        Ok(txs_downloaded)
+    }
+
+    /// download headers at heights in `txid_height` if tx details not already present, returns a map Txid -> timestamp
+    fn download_needed_headers(
+        &self,
+        txid_height: &HashMap<Txid, Option<u32>>,
+        txs_details_in_db: &HashMap<Txid, TransactionDetails>,
+        chunk_size: usize,
+    ) -> Result<HashMap<Txid, u64>, Error> {
+        let mut txid_timestamp = HashMap::new();
+        let needed_txid_height: HashMap<&Txid, &Option<u32>> = txid_height
+            .iter()
+            .filter(|(txid, _)| txs_details_in_db.get(*txid).is_none())
+            .collect();
+        let needed_heights: HashSet<u32> =
+            needed_txid_height.iter().filter_map(|(_, b)| **b).collect();
+        if !needed_heights.is_empty() {
+            info!("{} headers to download for timestamp", needed_heights.len());
+            let mut height_timestamp: HashMap<u32, u64> = HashMap::new();
+            for chunk in ChunksIterator::new(needed_heights.into_iter(), chunk_size) {
+                let call_result: Vec<BlockHeader> =
+                    maybe_await!(self.els_batch_block_header(chunk.clone()))?;
+                let vec: Vec<(u32, u64)> = chunk
+                    .into_iter()
+                    .zip(call_result.iter().map(|h| h.time as u64))
+                    .collect();
+                height_timestamp.extend(vec);
+            }
+            for (txid, height_opt) in needed_txid_height {
+                if let Some(height) = height_opt {
+                    let timestamp = height_timestamp
+                        .get(height)
+                        .ok_or_else(|| Error::Generic("timestamp missing".to_string()))?;
+                    txid_timestamp.insert(*txid, *timestamp);
+                }
+            }
+        }
+
+        Ok(txid_timestamp)
+    }
+
+    fn download_and_save_in_chunks<D: BatchDatabase>(
+        &self,
+        to_download: Vec<&Txid>,
+        chunk_size: usize,
+        db: &mut D,
+    ) -> Result<Vec<Transaction>, Error> {
+        let mut txs_downloaded = vec![];
+        for chunk in ChunksIterator::new(to_download.into_iter(), chunk_size) {
+            let call_result: Vec<Transaction> =
+                maybe_await!(self.els_batch_transaction_get(chunk))?;
+            let mut batch = db.begin_batch();
+            for new_tx in call_result.iter() {
+                batch.set_raw_tx(new_tx)?;
+            }
+            db.commit_batch(batch)?;
+            txs_downloaded.extend(call_result);
+        }
+
+        Ok(txs_downloaded)
     }
 }
+
+fn save_transaction_details_and_utxos<D: BatchDatabase>(
+    txid: &Txid,
+    db: &mut D,
+    timestamp: u64,
+    height: Option<u32>,
+    updates: &mut dyn BatchOperations,
+    utxo_deps: &HashMap<OutPoint, UTXO>,
+) -> Result<(), Error> {
+    let tx = db
+        .get_raw_tx(txid)?
+        .ok_or_else(|| Error::TransactionNotFound)?;
+
+    let mut incoming: u64 = 0;
+    let mut outgoing: u64 = 0;
+
+    let mut inputs_sum: u64 = 0;
+    let mut outputs_sum: u64 = 0;
+
+    // look for our own inputs
+    for input in tx.input.iter() {
+        // skip coinbase inputs
+        if input.previous_output.is_null() {
+            continue;
+        }
+
+        // We already downloaded all previous output txs in the previous step
+        if let Some(previous_output) = db.get_previous_output(&input.previous_output)? {
+            inputs_sum += previous_output.value;
+
+            if db.is_mine(&previous_output.script_pubkey)? {
+                outgoing += previous_output.value;
+            }
+        } else {
+            // The input is not ours, but we still need to count it for the fees
+            let tx = db
+                .get_raw_tx(&input.previous_output.txid)?
+                .ok_or_else(|| Error::TransactionNotFound)?;
+            inputs_sum += tx.output[input.previous_output.vout as usize].value;
+        }
+
+        // removes conflicting UTXO if any (generated from same inputs, like for example RBF)
+        if let Some(utxo) = utxo_deps.get(&input.previous_output) {
+            updates.del_utxo(&utxo.outpoint)?;
+        }
+    }
+
+    for (i, output) in tx.output.iter().enumerate() {
+        // to compute the fees later
+        outputs_sum += output.value;
+
+        // this output is ours, we have a path to derive it
+        if let Some((script_type, _child)) =
+            db.get_path_from_script_pubkey(&output.script_pubkey)?
+        {
+            debug!("{} output #{} is mine, adding utxo", txid, i);
+            updates.set_utxo(&UTXO {
+                outpoint: OutPoint::new(tx.txid(), i as u32),
+                txout: output.clone(),
+                is_internal: script_type.is_internal(),
+            })?;
+
+            incoming += output.value;
+        }
+    }
+
+    let tx_details = TransactionDetails {
+        txid: tx.txid(),
+        transaction: Some(tx),
+        received: incoming,
+        sent: outgoing,
+        height,
+        timestamp,
+        fees: inputs_sum.saturating_sub(outputs_sum), // if the tx is a coinbase, fees would be negative
+    };
+    updates.set_tx(&tx_details)?;
+
+    Ok(())
+}
+
+/// returns utxo dependency as the inputs needed for the utxo to exist
+/// `tx_raw_in_db` must contains utxo's generating txs or errors witt [crate::Error::TransactionNotFound]
+fn utxos_deps<D: BatchDatabase>(
+    db: &mut D,
+    tx_raw_in_db: &HashMap<Txid, Transaction>,
+) -> Result<HashMap<OutPoint, UTXO>, Error> {
+    let utxos = db.iter_utxos()?;
+    let mut utxos_deps = HashMap::new();
+    for utxo in utxos {
+        let from_tx = tx_raw_in_db
+            .get(&utxo.outpoint.txid)
+            .ok_or_else(|| Error::TransactionNotFound)?;
+        for input in from_tx.input.iter() {
+            utxos_deps.insert(input.previous_output, utxo.clone());
+        }
+    }
+    Ok(utxos_deps)
+}

From d9985c4bbb4d746269b28c5cac7beb584343b3e4 Mon Sep 17 00:00:00 2001
From: Riccardo Casatta <riccardo@casatta.it>
Date: Mon, 16 Nov 2020 12:23:02 +0100
Subject: [PATCH 05/16] [examples] support esplora blockchain source in repl

---
 examples/repl.rs | 38 +++++++++++++++++++++++---------------
 src/cli.rs       |  8 ++++++++
 2 files changed, 31 insertions(+), 15 deletions(-)

diff --git a/examples/repl.rs b/examples/repl.rs
index dc6a688e..69eca920 100644
--- a/examples/repl.rs
+++ b/examples/repl.rs
@@ -37,13 +37,15 @@ use log::{debug, error, info, trace, LevelFilter};
 use bitcoin::Network;
 
 use bdk::bitcoin;
-use bdk::blockchain::ConfigurableBlockchain;
-use bdk::blockchain::ElectrumBlockchain;
-use bdk::blockchain::ElectrumBlockchainConfig;
+use bdk::blockchain::{
+    AnyBlockchain, AnyBlockchainConfig, ConfigurableBlockchain, ElectrumBlockchainConfig,
+};
 use bdk::cli;
 use bdk::sled;
 use bdk::Wallet;
 
+use bdk::blockchain::esplora::EsploraBlockchainConfig;
+
 fn prepare_home_dir() -> PathBuf {
     let mut dir = PathBuf::new();
     dir.push(&dirs::home_dir().unwrap());
@@ -90,19 +92,25 @@ fn main() {
         .unwrap();
     debug!("database opened successfully");
 
-    let blockchain_config = ElectrumBlockchainConfig {
-        url: matches.value_of("server").unwrap().to_string(),
-        socks5: matches.value_of("proxy").map(ToString::to_string),
+    let config = match matches.value_of("esplora") {
+        Some(base_url) => AnyBlockchainConfig::Esplora(EsploraBlockchainConfig {
+            base_url: base_url.to_string(),
+        }),
+        None => AnyBlockchainConfig::Electrum(ElectrumBlockchainConfig {
+            url: matches.value_of("server").unwrap().to_string(),
+            socks5: matches.value_of("proxy").map(ToString::to_string),
+        }),
     };
-    let wallet = Wallet::new(
-        descriptor,
-        change_descriptor,
-        network,
-        tree,
-        ElectrumBlockchain::from_config(&blockchain_config).unwrap(),
-    )
-    .unwrap();
-    let wallet = Arc::new(wallet);
+    let wallet = Arc::new(
+        Wallet::new(
+            descriptor,
+            change_descriptor,
+            network,
+            tree,
+            AnyBlockchain::from_config(&config).unwrap(),
+        )
+        .unwrap(),
+    );
 
     if let Some(_sub_matches) = matches.subcommand_matches("repl") {
         let mut rl = Editor::<()>::new();
diff --git a/src/cli.rs b/src/cli.rs
index ada49cce..f60f7932 100644
--- a/src/cli.rs
+++ b/src/cli.rs
@@ -337,6 +337,14 @@ pub fn add_global_flags<'a, 'b>(app: App<'a, 'b>) -> App<'a, 'b> {
             .takes_value(true)
             .default_value("ssl://electrum.blockstream.info:60002"),
     )
+    .arg(
+        Arg::with_name("esplora")
+            .short("e")
+            .long("esplora")
+            .value_name("ESPLORA")
+            .help("Use the esplora server if given as parameter")
+            .takes_value(true),
+    )
     .arg(
         Arg::with_name("proxy")
             .short("p")

From 2f39a19b0182b669fb76553d75a32510f02421f6 Mon Sep 17 00:00:00 2001
From: Riccardo Casatta <riccardo@casatta.it>
Date: Tue, 17 Nov 2020 13:46:10 +0100
Subject: [PATCH 06/16] Use our Instant struct to be compatible with wasm

---
 src/blockchain/utils.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/blockchain/utils.rs b/src/blockchain/utils.rs
index c18b5fa4..bcfa19ba 100644
--- a/src/blockchain/utils.rs
+++ b/src/blockchain/utils.rs
@@ -26,6 +26,8 @@ use std::collections::{HashMap, HashSet};
 
 #[allow(unused_imports)]
 use log::{debug, error, info, trace};
+use rand::seq::SliceRandom;
+use rand::thread_rng;
 
 use bitcoin::{BlockHeader, OutPoint, Script, Transaction, Txid};
 
@@ -33,10 +35,8 @@ use super::*;
 use crate::database::{BatchDatabase, BatchOperations, DatabaseUtils};
 use crate::error::Error;
 use crate::types::{ScriptType, TransactionDetails, UTXO};
+use crate::wallet::time::Instant;
 use crate::wallet::utils::ChunksIterator;
-use rand::seq::SliceRandom;
-use rand::thread_rng;
-use std::time::Instant;
 
 #[derive(Debug)]
 pub struct ELSGetHistoryRes {
@@ -71,7 +71,7 @@ pub trait ElectrumLikeSync {
         _progress_update: P,
     ) -> Result<(), Error> {
         // TODO: progress
-        let start = Instant::now();
+        let start = Instant::new();
         debug!("start setup");
 
         let stop_gap = stop_gap.unwrap_or(20);

From 3c8b8e4fca89c5341824ddda6ee55a5d9cf90717 Mon Sep 17 00:00:00 2001
From: Riccardo Casatta <riccardo@casatta.it>
Date: Tue, 17 Nov 2020 14:27:32 +0100
Subject: [PATCH 07/16] conditionally remove cli args according to enabled
 feature

---
 src/cli.rs | 47 +++++++++++++++++++++++++++--------------------
 1 file changed, 27 insertions(+), 20 deletions(-)

diff --git a/src/cli.rs b/src/cli.rs
index f60f7932..de56e87e 100644
--- a/src/cli.rs
+++ b/src/cli.rs
@@ -309,7 +309,7 @@ pub fn make_cli_subcommands<'a, 'b>() -> App<'a, 'b> {
 }
 
 pub fn add_global_flags<'a, 'b>(app: App<'a, 'b>) -> App<'a, 'b> {
-    app.arg(
+    let mut app = app.arg(
         Arg::with_name("network")
             .short("n")
             .long("network")
@@ -328,23 +328,6 @@ pub fn add_global_flags<'a, 'b>(app: App<'a, 'b>) -> App<'a, 'b> {
             .takes_value(true)
             .default_value("main"),
     )
-    .arg(
-        Arg::with_name("server")
-            .short("s")
-            .long("server")
-            .value_name("SERVER:PORT")
-            .help("Sets the Electrum server to use")
-            .takes_value(true)
-            .default_value("ssl://electrum.blockstream.info:60002"),
-    )
-    .arg(
-        Arg::with_name("esplora")
-            .short("e")
-            .long("esplora")
-            .value_name("ESPLORA")
-            .help("Use the esplora server if given as parameter")
-            .takes_value(true),
-    )
     .arg(
         Arg::with_name("proxy")
             .short("p")
@@ -375,8 +358,32 @@ pub fn add_global_flags<'a, 'b>(app: App<'a, 'b>) -> App<'a, 'b> {
             .short("v")
             .multiple(true)
             .help("Sets the level of verbosity"),
-    )
-    .subcommand(SubCommand::with_name("repl").about("Opens an interactive shell"))
+    );
+
+    if cfg!(feature = "esplora") {
+        app = app.arg(
+            Arg::with_name("esplora")
+                .short("e")
+                .long("esplora")
+                .value_name("ESPLORA")
+                .help("Use the esplora server if given as parameter")
+                .takes_value(true),
+        );
+    }
+
+    if cfg!(feature = "electrum") {
+        app = app.arg(
+            Arg::with_name("server")
+                .short("s")
+                .long("server")
+                .value_name("SERVER:PORT")
+                .help("Sets the Electrum server to use")
+                .takes_value(true)
+                .default_value("ssl://electrum.blockstream.info:60002"),
+        );
+    }
+
+    app.subcommand(SubCommand::with_name("repl").about("Opens an interactive shell"))
 }
 
 #[maybe_async]

From 4d1617f4e0474cff2889efd09f731d7ca0310b50 Mon Sep 17 00:00:00 2001
From: Riccardo Casatta <riccardo@casatta.it>
Date: Tue, 17 Nov 2020 15:08:04 +0100
Subject: [PATCH 08/16] use proper type for EsploraHeader, make conversion to
 BlockHeader infallible

---
 src/blockchain/esplora.rs |  28 +++++------
 src/cli.rs                | 101 +++++++++++++++++++-------------------
 2 files changed, 62 insertions(+), 67 deletions(-)

diff --git a/src/blockchain/esplora.rs b/src/blockchain/esplora.rs
index 2b69d523..21fcf32b 100644
--- a/src/blockchain/esplora.rs
+++ b/src/blockchain/esplora.rs
@@ -48,7 +48,7 @@ use serde::Deserialize;
 use reqwest::{Client, StatusCode};
 
 use bitcoin::consensus::{deserialize, serialize};
-use bitcoin::hashes::hex::{FromHex, ToHex};
+use bitcoin::hashes::hex::ToHex;
 use bitcoin::hashes::{sha256, Hash};
 use bitcoin::{BlockHash, BlockHeader, Script, Transaction, TxMerkleNode, Txid};
 
@@ -57,7 +57,6 @@ use super::*;
 use crate::database::BatchDatabase;
 use crate::error::Error;
 use crate::FeeRate;
-use std::convert::TryInto;
 
 #[derive(Debug)]
 struct UrlClient {
@@ -192,7 +191,7 @@ impl UrlClient {
 
         let esplora_header = resp.json::<EsploraHeader>().await?;
 
-        Ok(esplora_header.try_into()?)
+        Ok(esplora_header.into())
     }
 
     async fn _broadcast(&self, transaction: &Transaction) -> Result<(), EsploraError> {
@@ -360,27 +359,23 @@ pub struct EsploraHeader {
     pub tx_count: u32,
     pub size: u32,
     pub weight: u32,
-    pub merkle_root: String,
-    pub previousblockhash: String,
+    pub merkle_root: TxMerkleNode,
+    pub previousblockhash: BlockHash,
     pub nonce: u32,
     pub bits: u32,
     pub difficulty: u32,
 }
 
-impl TryInto<BlockHeader> for EsploraHeader {
-    type Error = EsploraError;
-
-    fn try_into(self) -> Result<BlockHeader, esplora::EsploraError> {
-        Ok(BlockHeader {
+impl Into<BlockHeader> for EsploraHeader {
+    fn into(self) -> BlockHeader {
+        BlockHeader {
             version: self.version,
-            prev_blockhash: BlockHash::from_hex(&self.previousblockhash)
-                .map_err(|_| EsploraError::HeaderParseFail)?,
-            merkle_root: TxMerkleNode::from_hex(&self.merkle_root)
-                .map_err(|_| EsploraError::HeaderParseFail)?,
+            prev_blockhash: self.previousblockhash,
+            merkle_root: self.merkle_root,
             time: self.timestamp,
             bits: self.bits,
             nonce: self.nonce,
-        })
+        }
     }
 }
 
@@ -449,13 +444,12 @@ mod test {
     use crate::blockchain::esplora::EsploraHeader;
     use bitcoin::hashes::hex::FromHex;
     use bitcoin::{BlockHash, BlockHeader};
-    use std::convert::TryInto;
 
     #[test]
     fn test_esplora_header() {
         let json_str = r#"{"id":"00000000b873e79784647a6c82962c70d228557d24a747ea4d1b8bbe878e1206","height":1,"version":1,"timestamp":1296688928,"tx_count":1,"size":190,"weight":760,"merkle_root":"f0315ffc38709d70ad5647e22048358dd3745f3ce3874223c80a7c92fab0c8ba","previousblockhash":"000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943","nonce":1924588547,"bits":486604799,"difficulty":1}"#;
         let json: EsploraHeader = serde_json::from_str(&json_str).unwrap();
-        let header: BlockHeader = json.try_into().unwrap();
+        let header: BlockHeader = json.into();
         assert_eq!(
             header.block_hash(),
             BlockHash::from_hex("00000000b873e79784647a6c82962c70d228557d24a747ea4d1b8bbe878e1206")
diff --git a/src/cli.rs b/src/cli.rs
index de56e87e..06c11d46 100644
--- a/src/cli.rs
+++ b/src/cli.rs
@@ -309,56 +309,57 @@ pub fn make_cli_subcommands<'a, 'b>() -> App<'a, 'b> {
 }
 
 pub fn add_global_flags<'a, 'b>(app: App<'a, 'b>) -> App<'a, 'b> {
-    let mut app = app.arg(
-        Arg::with_name("network")
-            .short("n")
-            .long("network")
-            .value_name("NETWORK")
-            .help("Sets the network")
-            .takes_value(true)
-            .default_value("testnet")
-            .possible_values(&["testnet", "regtest"]),
-    )
-    .arg(
-        Arg::with_name("wallet")
-            .short("w")
-            .long("wallet")
-            .value_name("WALLET_NAME")
-            .help("Selects the wallet to use")
-            .takes_value(true)
-            .default_value("main"),
-    )
-    .arg(
-        Arg::with_name("proxy")
-            .short("p")
-            .long("proxy")
-            .value_name("SERVER:PORT")
-            .help("Sets the SOCKS5 proxy for the Electrum client")
-            .takes_value(true),
-    )
-    .arg(
-        Arg::with_name("descriptor")
-            .short("d")
-            .long("descriptor")
-            .value_name("DESCRIPTOR")
-            .help("Sets the descriptor to use for the external addresses")
-            .required(true)
-            .takes_value(true),
-    )
-    .arg(
-        Arg::with_name("change_descriptor")
-            .short("c")
-            .long("change_descriptor")
-            .value_name("DESCRIPTOR")
-            .help("Sets the descriptor to use for internal addresses")
-            .takes_value(true),
-    )
-    .arg(
-        Arg::with_name("v")
-            .short("v")
-            .multiple(true)
-            .help("Sets the level of verbosity"),
-    );
+    let mut app = app
+        .arg(
+            Arg::with_name("network")
+                .short("n")
+                .long("network")
+                .value_name("NETWORK")
+                .help("Sets the network")
+                .takes_value(true)
+                .default_value("testnet")
+                .possible_values(&["testnet", "regtest"]),
+        )
+        .arg(
+            Arg::with_name("wallet")
+                .short("w")
+                .long("wallet")
+                .value_name("WALLET_NAME")
+                .help("Selects the wallet to use")
+                .takes_value(true)
+                .default_value("main"),
+        )
+        .arg(
+            Arg::with_name("proxy")
+                .short("p")
+                .long("proxy")
+                .value_name("SERVER:PORT")
+                .help("Sets the SOCKS5 proxy for the Electrum client")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name("descriptor")
+                .short("d")
+                .long("descriptor")
+                .value_name("DESCRIPTOR")
+                .help("Sets the descriptor to use for the external addresses")
+                .required(true)
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name("change_descriptor")
+                .short("c")
+                .long("change_descriptor")
+                .value_name("DESCRIPTOR")
+                .help("Sets the descriptor to use for internal addresses")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name("v")
+                .short("v")
+                .multiple(true)
+                .help("Sets the level of verbosity"),
+        );
 
     if cfg!(feature = "esplora") {
         app = app.arg(

From 7a58d3dd7a3ee2db2ee4e6ec17421a01f218c9af Mon Sep 17 00:00:00 2001
From: Riccardo Casatta <riccardo@casatta.it>
Date: Tue, 17 Nov 2020 15:14:24 +0100
Subject: [PATCH 09/16] Use filter_map instead of filter and map

---
 src/blockchain/utils.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/blockchain/utils.rs b/src/blockchain/utils.rs
index bcfa19ba..73db08ce 100644
--- a/src/blockchain/utils.rs
+++ b/src/blockchain/utils.rs
@@ -95,8 +95,7 @@ pub trait ElectrumLikeSync {
                 let max_index = call_result
                     .iter()
                     .enumerate()
-                    .filter(|(_, v)| !v.is_empty())
-                    .map(|(i, _)| i as u32)
+                    .filter_map(|(i, v)| v.first().map(|_| i as u32))
                     .max();
                 if let Some(max) = max_index {
                     max_indexes.insert(script_type, max + (i * chunk_size) as u32);

From 2844ddec63c3e16ef2e029676ce486e3d867005f Mon Sep 17 00:00:00 2001
From: Riccardo Casatta <riccardo@casatta.it>
Date: Tue, 17 Nov 2020 15:19:09 +0100
Subject: [PATCH 10/16] avoid a max() call by checking minus or equal

---
 src/blockchain/utils.rs | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/blockchain/utils.rs b/src/blockchain/utils.rs
index 73db08ce..79c9c9f0 100644
--- a/src/blockchain/utils.rs
+++ b/src/blockchain/utils.rs
@@ -111,11 +111,10 @@ pub trait ElectrumLikeSync {
                     // el.height = -1 means unconfirmed with unconfirmed parents
                     // el.height =  0 means unconfirmed with confirmed parents
                     // but we threat those tx the same
-                    let height = el.height.max(0);
-                    if height == 0 {
+                    if el.height <= 0 {
                         txid_height.insert(el.tx_hash, None);
                     } else {
-                        txid_height.insert(el.tx_hash, Some(height as u32));
+                        txid_height.insert(el.tx_hash, Some(el.height as u32));
                     }
                     history_txs_id.insert(el.tx_hash);
                 }

From 16fde66c6af92b4a093f73181375646d3228f59f Mon Sep 17 00:00:00 2001
From: Riccardo Casatta <riccardo@casatta.it>
Date: Tue, 17 Nov 2020 15:24:26 +0100
Subject: [PATCH 11/16] use flatten instead of unwrap_or

---
 src/blockchain/utils.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/blockchain/utils.rs b/src/blockchain/utils.rs
index 79c9c9f0..dd5de887 100644
--- a/src/blockchain/utils.rs
+++ b/src/blockchain/utils.rs
@@ -159,7 +159,7 @@ pub trait ElectrumLikeSync {
 
         // save any tx details not in db but in history_txs_id or with different height/timestamp
         for txid in history_txs_id.iter() {
-            let height = *txid_height.get(txid).unwrap_or(&None);
+            let height = txid_height.get(txid).cloned().flatten();
             let timestamp = *new_timestamps.get(txid).unwrap_or(&0u64);
             if let Some(tx_details) = txs_details_in_db.get(txid) {
                 // check if height matches, otherwise updates it

From 02c0ad2fcad295068a08f57d2e3832dedd75a953 Mon Sep 17 00:00:00 2001
From: Riccardo Casatta <riccardo@casatta.it>
Date: Tue, 17 Nov 2020 15:50:15 +0100
Subject: [PATCH 12/16] eagerly unwrap height option, save one collect

---
 src/blockchain/utils.rs | 30 ++++++++++++++----------------
 1 file changed, 14 insertions(+), 16 deletions(-)

diff --git a/src/blockchain/utils.rs b/src/blockchain/utils.rs
index dd5de887..4fa51e42 100644
--- a/src/blockchain/utils.rs
+++ b/src/blockchain/utils.rs
@@ -251,31 +251,29 @@ pub trait ElectrumLikeSync {
         chunk_size: usize,
     ) -> Result<HashMap<Txid, u64>, Error> {
         let mut txid_timestamp = HashMap::new();
-        let needed_txid_height: HashMap<&Txid, &Option<u32>> = txid_height
+        let needed_txid_height: HashMap<&Txid, u32> = txid_height
             .iter()
-            .filter(|(txid, _)| txs_details_in_db.get(*txid).is_none())
+            .filter(|(t, _)| txs_details_in_db.get(*t).is_none())
+            .filter_map(|(t, o)| o.map(|h| (t, h)))
             .collect();
-        let needed_heights: HashSet<u32> =
-            needed_txid_height.iter().filter_map(|(_, b)| **b).collect();
+        let needed_heights: HashSet<u32> = needed_txid_height.values().cloned().collect();
         if !needed_heights.is_empty() {
             info!("{} headers to download for timestamp", needed_heights.len());
             let mut height_timestamp: HashMap<u32, u64> = HashMap::new();
             for chunk in ChunksIterator::new(needed_heights.into_iter(), chunk_size) {
                 let call_result: Vec<BlockHeader> =
                     maybe_await!(self.els_batch_block_header(chunk.clone()))?;
-                let vec: Vec<(u32, u64)> = chunk
-                    .into_iter()
-                    .zip(call_result.iter().map(|h| h.time as u64))
-                    .collect();
-                height_timestamp.extend(vec);
+                height_timestamp.extend(
+                    chunk
+                        .into_iter()
+                        .zip(call_result.iter().map(|h| h.time as u64)),
+                );
             }
-            for (txid, height_opt) in needed_txid_height {
-                if let Some(height) = height_opt {
-                    let timestamp = height_timestamp
-                        .get(height)
-                        .ok_or_else(|| Error::Generic("timestamp missing".to_string()))?;
-                    txid_timestamp.insert(*txid, *timestamp);
-                }
+            for (txid, height) in needed_txid_height {
+                let timestamp = height_timestamp
+                    .get(&height)
+                    .ok_or_else(|| Error::Generic("timestamp missing".to_string()))?;
+                txid_timestamp.insert(*txid, *timestamp);
             }
         }
 

From 42480ea37bd2b15024ba4f2f500bda84ef9f786e Mon Sep 17 00:00:00 2001
From: Riccardo Casatta <riccardo@casatta.it>
Date: Tue, 17 Nov 2020 16:29:03 +0100
Subject: [PATCH 13/16] Bring less data around

---
 src/blockchain/utils.rs | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/blockchain/utils.rs b/src/blockchain/utils.rs
index 4fa51e42..4b3e232f 100644
--- a/src/blockchain/utils.rs
+++ b/src/blockchain/utils.rs
@@ -308,7 +308,7 @@ fn save_transaction_details_and_utxos<D: BatchDatabase>(
     timestamp: u64,
     height: Option<u32>,
     updates: &mut dyn BatchOperations,
-    utxo_deps: &HashMap<OutPoint, UTXO>,
+    utxo_deps: &HashMap<OutPoint, OutPoint>,
 ) -> Result<(), Error> {
     let tx = db
         .get_raw_tx(txid)?
@@ -343,8 +343,8 @@ fn save_transaction_details_and_utxos<D: BatchDatabase>(
         }
 
         // removes conflicting UTXO if any (generated from same inputs, like for example RBF)
-        if let Some(utxo) = utxo_deps.get(&input.previous_output) {
-            updates.del_utxo(&utxo.outpoint)?;
+        if let Some(outpoint) = utxo_deps.get(&input.previous_output) {
+            updates.del_utxo(&outpoint)?;
         }
     }
 
@@ -386,7 +386,7 @@ fn save_transaction_details_and_utxos<D: BatchDatabase>(
 fn utxos_deps<D: BatchDatabase>(
     db: &mut D,
     tx_raw_in_db: &HashMap<Txid, Transaction>,
-) -> Result<HashMap<OutPoint, UTXO>, Error> {
+) -> Result<HashMap<OutPoint, OutPoint>, Error> {
     let utxos = db.iter_utxos()?;
     let mut utxos_deps = HashMap::new();
     for utxo in utxos {
@@ -394,7 +394,7 @@ fn utxos_deps<D: BatchDatabase>(
             .get(&utxo.outpoint.txid)
             .ok_or_else(|| Error::TransactionNotFound)?;
         for input in from_tx.input.iter() {
-            utxos_deps.insert(input.previous_output, utxo.clone());
+            utxos_deps.insert(input.previous_output, utxo.outpoint);
         }
     }
     Ok(utxos_deps)

From 8dfbbf2763f4f5d5d4bfe315f3a3d61635525350 Mon Sep 17 00:00:00 2001
From: Riccardo Casatta <riccardo@casatta.it>
Date: Tue, 17 Nov 2020 16:47:58 +0100
Subject: [PATCH 14/16] Require esplora feature for repl example

---
 Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Cargo.toml b/Cargo.toml
index ae590c50..5f5421ae 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -64,7 +64,7 @@ env_logger = "0.7"
 
 [[example]]
 name = "repl"
-required-features = ["cli-utils"]
+required-features = ["cli-utils", "esplora"]
 [[example]]
 name = "parse_descriptor"
 [[example]]

From b79fa27aa4ee6cca33b900336a9cbc209189e28e Mon Sep 17 00:00:00 2001
From: Riccardo Casatta <riccardo@casatta.it>
Date: Tue, 17 Nov 2020 18:54:34 +0100
Subject: [PATCH 15/16] Remove unused varaint HeaderParseFail

---
 src/blockchain/esplora.rs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/blockchain/esplora.rs b/src/blockchain/esplora.rs
index 21fcf32b..e2da5b80 100644
--- a/src/blockchain/esplora.rs
+++ b/src/blockchain/esplora.rs
@@ -409,8 +409,6 @@ pub enum EsploraError {
     HeaderHeightNotFound(u32),
     /// Header hash not found
     HeaderHashNotFound(BlockHash),
-    /// EsploraHeader cannot be converted in BlockHeader
-    HeaderParseFail,
 }
 
 impl fmt::Display for EsploraError {

From ae16c8b602183261c524ad6a87d0843537548c81 Mon Sep 17 00:00:00 2001
From: Riccardo Casatta <riccardo@casatta.it>
Date: Wed, 18 Nov 2020 09:27:01 +0100
Subject: [PATCH 16/16] fix typo

---
 src/blockchain/utils.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/blockchain/utils.rs b/src/blockchain/utils.rs
index 4b3e232f..d2e8f990 100644
--- a/src/blockchain/utils.rs
+++ b/src/blockchain/utils.rs
@@ -110,7 +110,7 @@ pub trait ElectrumLikeSync {
                 for el in flattened {
                     // el.height = -1 means unconfirmed with unconfirmed parents
                     // el.height =  0 means unconfirmed with confirmed parents
-                    // but we threat those tx the same
+                    // but we treat those tx the same
                     if el.height <= 0 {
                         txid_height.insert(el.tx_hash, None);
                     } else {