diff --git a/.gitignore b/.gitignore index 8b75e7bb..4dc9a2ff 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ Cargo.lock *.swp +.idea diff --git a/Cargo.toml b/Cargo.toml index ae590c50..5f5421ae 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,7 +64,7 @@ env_logger = "0.7" [[example]] name = "repl" -required-features = ["cli-utils"] +required-features = ["cli-utils", "esplora"] [[example]] name = "parse_descriptor" [[example]] diff --git a/examples/repl.rs b/examples/repl.rs index dc6a688e..69eca920 100644 --- a/examples/repl.rs +++ b/examples/repl.rs @@ -37,13 +37,15 @@ use log::{debug, error, info, trace, LevelFilter}; use bitcoin::Network; use bdk::bitcoin; -use bdk::blockchain::ConfigurableBlockchain; -use bdk::blockchain::ElectrumBlockchain; -use bdk::blockchain::ElectrumBlockchainConfig; +use bdk::blockchain::{ + AnyBlockchain, AnyBlockchainConfig, ConfigurableBlockchain, ElectrumBlockchainConfig, +}; use bdk::cli; use bdk::sled; use bdk::Wallet; +use bdk::blockchain::esplora::EsploraBlockchainConfig; + fn prepare_home_dir() -> PathBuf { let mut dir = PathBuf::new(); dir.push(&dirs::home_dir().unwrap()); @@ -90,19 +92,25 @@ fn main() { .unwrap(); debug!("database opened successfully"); - let blockchain_config = ElectrumBlockchainConfig { - url: matches.value_of("server").unwrap().to_string(), - socks5: matches.value_of("proxy").map(ToString::to_string), + let config = match matches.value_of("esplora") { + Some(base_url) => AnyBlockchainConfig::Esplora(EsploraBlockchainConfig { + base_url: base_url.to_string(), + }), + None => AnyBlockchainConfig::Electrum(ElectrumBlockchainConfig { + url: matches.value_of("server").unwrap().to_string(), + socks5: matches.value_of("proxy").map(ToString::to_string), + }), }; - let wallet = Wallet::new( - descriptor, - change_descriptor, - network, - tree, - ElectrumBlockchain::from_config(&blockchain_config).unwrap(), - ) - .unwrap(); - let wallet = Arc::new(wallet); + let wallet = Arc::new( + Wallet::new( + descriptor, + change_descriptor, + network, + tree, + AnyBlockchain::from_config(&config).unwrap(), + ) + .unwrap(), + ); if let Some(_sub_matches) = matches.subcommand_matches("repl") { let mut rl = Editor::<()>::new(); diff --git a/src/blockchain/electrum.rs b/src/blockchain/electrum.rs index 9655ebbb..a134158c 100644 --- a/src/blockchain/electrum.rs +++ b/src/blockchain/electrum.rs @@ -42,11 +42,11 @@ use std::collections::HashSet; #[allow(unused_imports)] use log::{debug, error, info, trace}; -use bitcoin::{Script, Transaction, Txid}; +use bitcoin::{BlockHeader, Script, Transaction, Txid}; use electrum_client::{Client, ElectrumApi}; -use self::utils::{ELSGetHistoryRes, ELSListUnspentRes, ElectrumLikeSync}; +use self::utils::{ELSGetHistoryRes, ElectrumLikeSync}; use super::*; use crate::database::BatchDatabase; use crate::error::Error; @@ -141,36 +141,18 @@ impl ElectrumLikeSync for Client { .map_err(Error::Electrum) } - fn els_batch_script_list_unspent<'s, I: IntoIterator>( + fn els_batch_transaction_get<'s, I: IntoIterator>( &self, - scripts: I, - ) -> Result>, Error> { - self.batch_script_list_unspent(scripts) - .map(|v| { - v.into_iter() - .map(|v| { - v.into_iter() - .map( - |electrum_client::ListUnspentRes { - height, - tx_hash, - tx_pos, - .. - }| ELSListUnspentRes { - height, - tx_hash, - tx_pos, - }, - ) - .collect() - }) - .collect() - }) - .map_err(Error::Electrum) + txids: I, + ) -> Result, Error> { + self.batch_transaction_get(txids).map_err(Error::Electrum) } - fn els_transaction_get(&self, txid: &Txid) -> Result { - self.transaction_get(txid).map_err(Error::Electrum) + fn els_batch_block_header>( + &self, + heights: I, + ) -> Result, Error> { + self.batch_block_header(heights).map_err(Error::Electrum) } } diff --git a/src/blockchain/esplora.rs b/src/blockchain/esplora.rs index ef649d0b..e2da5b80 100644 --- a/src/blockchain/esplora.rs +++ b/src/blockchain/esplora.rs @@ -50,9 +50,9 @@ use reqwest::{Client, StatusCode}; use bitcoin::consensus::{deserialize, serialize}; use bitcoin::hashes::hex::ToHex; use bitcoin::hashes::{sha256, Hash}; -use bitcoin::{Script, Transaction, Txid}; +use bitcoin::{BlockHash, BlockHeader, Script, Transaction, TxMerkleNode, Txid}; -use self::utils::{ELSGetHistoryRes, ELSListUnspentRes, ElectrumLikeSync}; +use self::utils::{ELSGetHistoryRes, ElectrumLikeSync}; use super::*; use crate::database::BatchDatabase; use crate::error::Error; @@ -161,6 +161,39 @@ impl UrlClient { Ok(Some(deserialize(&resp.error_for_status()?.bytes().await?)?)) } + async fn _get_tx_no_opt(&self, txid: &Txid) -> Result { + match self._get_tx(txid).await { + Ok(Some(tx)) => Ok(tx), + Ok(None) => Err(EsploraError::TransactionNotFound(*txid)), + Err(e) => Err(e), + } + } + + async fn _get_header(&self, block_height: u32) -> Result { + let resp = self + .client + .get(&format!("{}/block-height/{}", self.url, block_height)) + .send() + .await?; + + if let StatusCode::NOT_FOUND = resp.status() { + return Err(EsploraError::HeaderHeightNotFound(block_height)); + } + let bytes = resp.bytes().await?; + let hash = std::str::from_utf8(&bytes) + .map_err(|_| EsploraError::HeaderHeightNotFound(block_height))?; + + let resp = self + .client + .get(&format!("{}/block/{}", self.url, hash)) + .send() + .await?; + + let esplora_header = resp.json::().await?; + + Ok(esplora_header.into()) + } + async fn _broadcast(&self, transaction: &Transaction) -> Result<(), EsploraError> { self.client .post(&format!("{}/tx", self.url)) @@ -249,31 +282,6 @@ impl UrlClient { Ok(result) } - async fn _script_list_unspent( - &self, - script: &Script, - ) -> Result, EsploraError> { - Ok(self - .client - .get(&format!( - "{}/scripthash/{}/utxo", - self.url, - Self::script_to_scripthash(script) - )) - .send() - .await? - .error_for_status()? - .json::>() - .await? - .into_iter() - .map(|x| ELSListUnspentRes { - tx_hash: x.txid, - height: x.status.block_height.unwrap_or(0), - tx_pos: x.vout, - }) - .collect()) - } - async fn _get_fee_estimates(&self) -> Result, EsploraError> { Ok(self .client @@ -302,13 +310,13 @@ impl ElectrumLikeSync for UrlClient { await_or_block!(future) } - fn els_batch_script_list_unspent<'s, I: IntoIterator>( + fn els_batch_transaction_get<'s, I: IntoIterator>( &self, - scripts: I, - ) -> Result>, Error> { + txids: I, + ) -> Result, Error> { let future = async { - Ok(stream::iter(scripts) - .then(|script| self._script_list_unspent(&script)) + Ok(stream::iter(txids) + .then(|txid| self._get_tx_no_opt(&txid)) .try_collect() .await?) }; @@ -316,9 +324,18 @@ impl ElectrumLikeSync for UrlClient { await_or_block!(future) } - fn els_transaction_get(&self, txid: &Txid) -> Result { - Ok(await_or_block!(self._get_tx(txid))? - .ok_or_else(|| EsploraError::TransactionNotFound(*txid))?) + fn els_batch_block_header>( + &self, + heights: I, + ) -> Result, Error> { + let future = async { + Ok(stream::iter(heights) + .then(|h| self._get_header(h)) + .try_collect() + .await?) + }; + + await_or_block!(future) } } @@ -333,11 +350,33 @@ struct EsploraGetHistory { status: EsploraGetHistoryStatus, } -#[derive(Deserialize)] -struct EsploraListUnspent { - txid: Txid, - vout: usize, - status: EsploraGetHistoryStatus, +#[derive(Default, Debug, Clone, PartialEq, Deserialize)] +pub struct EsploraHeader { + pub id: String, + pub height: u32, + pub version: i32, + pub timestamp: u32, + pub tx_count: u32, + pub size: u32, + pub weight: u32, + pub merkle_root: TxMerkleNode, + pub previousblockhash: BlockHash, + pub nonce: u32, + pub bits: u32, + pub difficulty: u32, +} + +impl Into for EsploraHeader { + fn into(self) -> BlockHeader { + BlockHeader { + version: self.version, + prev_blockhash: self.previousblockhash, + merkle_root: self.merkle_root, + time: self.timestamp, + bits: self.bits, + nonce: self.nonce, + } + } } /// Configuration for an [`EsploraBlockchain`] @@ -366,6 +405,10 @@ pub enum EsploraError { /// Transaction not found TransactionNotFound(Txid), + /// Header height not found + HeaderHeightNotFound(u32), + /// Header hash not found + HeaderHashNotFound(BlockHash), } impl fmt::Display for EsploraError { @@ -393,3 +436,22 @@ impl From for EsploraError { EsploraError::BitcoinEncoding(other) } } + +#[cfg(test)] +mod test { + use crate::blockchain::esplora::EsploraHeader; + use bitcoin::hashes::hex::FromHex; + use bitcoin::{BlockHash, BlockHeader}; + + #[test] + fn test_esplora_header() { + let json_str = r#"{"id":"00000000b873e79784647a6c82962c70d228557d24a747ea4d1b8bbe878e1206","height":1,"version":1,"timestamp":1296688928,"tx_count":1,"size":190,"weight":760,"merkle_root":"f0315ffc38709d70ad5647e22048358dd3745f3ce3874223c80a7c92fab0c8ba","previousblockhash":"000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943","nonce":1924588547,"bits":486604799,"difficulty":1}"#; + let json: EsploraHeader = serde_json::from_str(&json_str).unwrap(); + let header: BlockHeader = json.into(); + assert_eq!( + header.block_hash(), + BlockHash::from_hex("00000000b873e79784647a6c82962c70d228557d24a747ea4d1b8bbe878e1206") + .unwrap() + ); + } +} diff --git a/src/blockchain/mod.rs b/src/blockchain/mod.rs index 83b6903d..6711e51e 100644 --- a/src/blockchain/mod.rs +++ b/src/blockchain/mod.rs @@ -40,6 +40,7 @@ use crate::database::BatchDatabase; use crate::error::Error; use crate::FeeRate; +#[cfg(any(feature = "electrum", feature = "esplora"))] pub(crate) mod utils; #[cfg(any(feature = "electrum", feature = "esplora", feature = "compact_filters"))] diff --git a/src/blockchain/utils.rs b/src/blockchain/utils.rs index bf8ff6cb..d2e8f990 100644 --- a/src/blockchain/utils.rs +++ b/src/blockchain/utils.rs @@ -22,19 +22,20 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -use std::cmp; -use std::collections::{HashSet, VecDeque}; -use std::convert::TryFrom; +use std::collections::{HashMap, HashSet}; #[allow(unused_imports)] use log::{debug, error, info, trace}; +use rand::seq::SliceRandom; +use rand::thread_rng; -use bitcoin::{Address, Network, OutPoint, Script, Transaction, Txid}; +use bitcoin::{BlockHeader, OutPoint, Script, Transaction, Txid}; use super::*; use crate::database::{BatchDatabase, BatchOperations, DatabaseUtils}; use crate::error::Error; use crate::types::{ScriptType, TransactionDetails, UTXO}; +use crate::wallet::time::Instant; use crate::wallet::utils::ChunksIterator; #[derive(Debug)] @@ -43,13 +44,6 @@ pub struct ELSGetHistoryRes { pub tx_hash: Txid, } -#[derive(Debug)] -pub struct ELSListUnspentRes { - pub height: usize, - pub tx_hash: Txid, - pub tx_pos: usize, -} - /// Implements the synchronization logic for an Electrum-like client. #[maybe_async] pub trait ElectrumLikeSync { @@ -58,306 +52,350 @@ pub trait ElectrumLikeSync { scripts: I, ) -> Result>, Error>; - fn els_batch_script_list_unspent<'s, I: IntoIterator>( + fn els_batch_transaction_get<'s, I: IntoIterator>( &self, - scripts: I, - ) -> Result>, Error>; + txids: I, + ) -> Result, Error>; - fn els_transaction_get(&self, txid: &Txid) -> Result; + fn els_batch_block_header>( + &self, + heights: I, + ) -> Result, Error>; // Provided methods down here... fn electrum_like_setup( &self, stop_gap: Option, - database: &mut D, + db: &mut D, _progress_update: P, ) -> Result<(), Error> { // TODO: progress + let start = Instant::new(); + debug!("start setup"); let stop_gap = stop_gap.unwrap_or(20); - let batch_query_size = 20; + let chunk_size = stop_gap; - // check unconfirmed tx, delete so they are retrieved later - let mut del_batch = database.begin_batch(); - for tx in database.iter_txs(false)? { - if tx.height.is_none() { - del_batch.del_tx(&tx.txid, false)?; - } - } - database.commit_batch(del_batch)?; + let mut history_txs_id = HashSet::new(); + let mut txid_height = HashMap::new(); + let mut max_indexes = HashMap::new(); - // maximum derivation index for a change address that we've seen during sync - let mut change_max_deriv = None; + let mut wallet_chains = vec![ScriptType::Internal, ScriptType::External]; + // shuffling improve privacy, the server doesn't know my first request is from my internal or external addresses + wallet_chains.shuffle(&mut thread_rng()); + // download history of our internal and external script_pubkeys + for script_type in wallet_chains.iter() { + let script_iter = db.iter_script_pubkeys(Some(*script_type))?.into_iter(); - let mut already_checked: HashSet