From 92ad4876c40c8499d46000c53f19f77d50c9ab14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BF=97=E5=AE=87?= Date: Tue, 27 Sep 2022 20:39:23 +0800 Subject: [PATCH 01/19] Add vscode filter to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 4dc9a2ff..d0130189 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /target Cargo.lock +/.vscode *.swp .idea From 5baf46f84d41fb714f200a7b26170c9b77823dc1 Mon Sep 17 00:00:00 2001 From: Alekos Filini Date: Tue, 9 Aug 2022 12:43:01 +0200 Subject: [PATCH 02/19] Use the external esplora client library --- Cargo.toml | 9 +- src/blockchain/esplora/api.rs | 117 ---------------- src/blockchain/esplora/mod.rs | 140 ++----------------- src/blockchain/esplora/reqwest.rs | 198 ++++++--------------------- src/blockchain/esplora/ureq.rs | 219 ++++-------------------------- 5 files changed, 90 insertions(+), 593 deletions(-) delete mode 100644 src/blockchain/esplora/api.rs diff --git a/Cargo.toml b/Cargo.toml index e0b73698..81cfeb4c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,10 +23,9 @@ rand = "^0.7" # Optional dependencies sled = { version = "0.34", optional = true } electrum-client = { version = "0.11", optional = true } +esplora-client = { version = "0.1.1", default-features = false, optional = true } rusqlite = { version = "0.27.0", optional = true } ahash = { version = "0.7.6", optional = true } -reqwest = { version = "0.11", optional = true, default-features = false, features = ["json"] } -ureq = { version = "~2.2.0", features = ["json"], optional = true } futures = { version = "0.3", optional = true } async-trait = { version = "0.1", optional = true } rocksdb = { version = "0.14", default-features = false, features = ["snappy"], optional = true } @@ -79,13 +78,13 @@ hardware-signer = ["hwi"] async-interface = ["async-trait"] electrum = ["electrum-client"] # MUST ALSO USE `--no-default-features`. -use-esplora-reqwest = ["esplora", "reqwest", "reqwest/socks", "futures"] -use-esplora-ureq = ["esplora", "ureq", "ureq/socks"] +use-esplora-reqwest = ["esplora", "esplora-client/async", "futures"] +use-esplora-ureq = ["esplora", "esplora-client/blocking"] # Typical configurations will not need to use `esplora` feature directly. esplora = [] # Use below feature with `use-esplora-reqwest` to enable reqwest default TLS support -reqwest-default-tls = ["reqwest/default-tls"] +reqwest-default-tls = ["esplora-client/async-https"] # Debug/Test features test-blockchains = ["bitcoincore-rpc", "electrum-client"] diff --git a/src/blockchain/esplora/api.rs b/src/blockchain/esplora/api.rs deleted file mode 100644 index d548b5be..00000000 --- a/src/blockchain/esplora/api.rs +++ /dev/null @@ -1,117 +0,0 @@ -//! structs from the esplora API -//! -//! see: -use crate::BlockTime; -use bitcoin::{OutPoint, Script, Transaction, TxIn, TxOut, Txid, Witness}; - -#[derive(serde::Deserialize, Clone, Debug)] -pub struct PrevOut { - pub value: u64, - pub scriptpubkey: Script, -} - -#[derive(serde::Deserialize, Clone, Debug)] -pub struct Vin { - pub txid: Txid, - pub vout: u32, - // None if coinbase - pub prevout: Option, - pub scriptsig: Script, - #[serde(deserialize_with = "deserialize_witness", default)] - pub witness: Vec>, - pub sequence: u32, - pub is_coinbase: bool, -} - -#[derive(serde::Deserialize, Clone, Debug)] -pub struct Vout { - pub value: u64, - pub scriptpubkey: Script, -} - -#[derive(serde::Deserialize, Clone, Debug)] -pub struct TxStatus { - pub confirmed: bool, - pub block_height: Option, - pub block_time: Option, -} - -#[derive(serde::Deserialize, Clone, Debug)] -pub struct Tx { - pub txid: Txid, - pub version: i32, - pub locktime: u32, - pub vin: Vec, - pub vout: Vec, - pub status: TxStatus, - pub fee: u64, -} - -impl Tx { - pub fn to_tx(&self) -> Transaction { - Transaction { - version: self.version, - lock_time: self.locktime, - input: self - .vin - .iter() - .cloned() - .map(|vin| TxIn { - previous_output: OutPoint { - txid: vin.txid, - vout: vin.vout, - }, - script_sig: vin.scriptsig, - sequence: vin.sequence, - witness: Witness::from_vec(vin.witness), - }) - .collect(), - output: self - .vout - .iter() - .cloned() - .map(|vout| TxOut { - value: vout.value, - script_pubkey: vout.scriptpubkey, - }) - .collect(), - } - } - - pub fn confirmation_time(&self) -> Option { - match self.status { - TxStatus { - confirmed: true, - block_height: Some(height), - block_time: Some(timestamp), - } => Some(BlockTime { timestamp, height }), - _ => None, - } - } - - pub fn previous_outputs(&self) -> Vec> { - self.vin - .iter() - .cloned() - .map(|vin| { - vin.prevout.map(|po| TxOut { - script_pubkey: po.scriptpubkey, - value: po.value, - }) - }) - .collect() - } -} - -fn deserialize_witness<'de, D>(d: D) -> Result>, D::Error> -where - D: serde::de::Deserializer<'de>, -{ - use crate::serde::Deserialize; - use bitcoin::hashes::hex::FromHex; - let list = Vec::::deserialize(d)?; - list.into_iter() - .map(|hex_str| Vec::::from_hex(&hex_str)) - .collect::>, _>>() - .map_err(serde::de::Error::custom) -} diff --git a/src/blockchain/esplora/mod.rs b/src/blockchain/esplora/mod.rs index 30d29d64..edc00926 100644 --- a/src/blockchain/esplora/mod.rs +++ b/src/blockchain/esplora/mod.rs @@ -17,85 +17,21 @@ //! Please note, to configure the Esplora HTTP client correctly use one of: //! Blocking: --features='esplora,ureq' //! Async: --features='async-interface,esplora,reqwest' --no-default-features -use std::collections::HashMap; -use std::fmt; -use std::io; -use bitcoin::consensus; -use bitcoin::{BlockHash, Txid}; +pub use esplora_client::Error as EsploraError; -use crate::error::Error; -use crate::FeeRate; - -#[cfg(feature = "reqwest")] +#[cfg(feature = "use-esplora-reqwest")] mod reqwest; -#[cfg(feature = "reqwest")] +#[cfg(feature = "use-esplora-reqwest")] pub use self::reqwest::*; -#[cfg(feature = "ureq")] +#[cfg(feature = "use-esplora-ureq")] mod ureq; -#[cfg(feature = "ureq")] +#[cfg(feature = "use-esplora-ureq")] pub use self::ureq::*; -mod api; - -fn into_fee_rate(target: usize, estimates: HashMap) -> Result { - let fee_val = { - let mut pairs = estimates - .into_iter() - .filter_map(|(k, v)| Some((k.parse::().ok()?, v))) - .collect::>(); - pairs.sort_unstable_by_key(|(k, _)| std::cmp::Reverse(*k)); - pairs - .into_iter() - .find(|(k, _)| k <= &target) - .map(|(_, v)| v) - .unwrap_or(1.0) - }; - Ok(FeeRate::from_sat_per_vb(fee_val as f32)) -} - -/// Errors that can happen during a sync with [`EsploraBlockchain`] -#[derive(Debug)] -pub enum EsploraError { - /// Error during ureq HTTP request - #[cfg(feature = "ureq")] - Ureq(::ureq::Error), - /// Transport error during the ureq HTTP call - #[cfg(feature = "ureq")] - UreqTransport(::ureq::Transport), - /// Error during reqwest HTTP request - #[cfg(feature = "reqwest")] - Reqwest(::reqwest::Error), - /// HTTP response error - HttpResponse(u16), - /// IO error during ureq response read - Io(io::Error), - /// No header found in ureq response - NoHeader, - /// Invalid number returned - Parsing(std::num::ParseIntError), - /// Invalid Bitcoin data returned - BitcoinEncoding(bitcoin::consensus::encode::Error), - /// Invalid Hex data returned - Hex(bitcoin::hashes::hex::Error), - - /// Transaction not found - TransactionNotFound(Txid), - /// Header height not found - HeaderHeightNotFound(u32), - /// Header hash not found - HeaderHashNotFound(BlockHash), -} - -impl fmt::Display for EsploraError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:?}", self) - } -} - /// Configuration for an [`EsploraBlockchain`] #[derive(Debug, serde::Deserialize, serde::Serialize, Clone, PartialEq)] pub struct EsploraBlockchainConfig { @@ -138,16 +74,11 @@ impl EsploraBlockchainConfig { } } -impl std::error::Error for EsploraError {} - -#[cfg(feature = "ureq")] -impl_error!(::ureq::Transport, UreqTransport, EsploraError); -#[cfg(feature = "reqwest")] -impl_error!(::reqwest::Error, Reqwest, EsploraError); -impl_error!(io::Error, Io, EsploraError); -impl_error!(std::num::ParseIntError, Parsing, EsploraError); -impl_error!(consensus::encode::Error, BitcoinEncoding, EsploraError); -impl_error!(bitcoin::hashes::hex::Error, Hex, EsploraError); +impl From for crate::BlockTime { + fn from(esplora_client::BlockTime { timestamp, height }: esplora_client::BlockTime) -> Self { + Self { timestamp, height } + } +} #[cfg(test)] #[cfg(feature = "test-esplora")] @@ -161,58 +92,11 @@ const DEFAULT_CONCURRENT_REQUESTS: u8 = 4; #[cfg(test)] mod test { - use super::*; - - #[test] - fn feerate_parsing() { - let esplora_fees = serde_json::from_str::>( - r#"{ - "25": 1.015, - "5": 2.3280000000000003, - "12": 2.0109999999999997, - "15": 1.018, - "17": 1.018, - "11": 2.0109999999999997, - "3": 3.01, - "2": 4.9830000000000005, - "6": 2.2359999999999998, - "21": 1.018, - "13": 1.081, - "7": 2.2359999999999998, - "8": 2.2359999999999998, - "16": 1.018, - "20": 1.018, - "22": 1.017, - "23": 1.017, - "504": 1, - "9": 2.2359999999999998, - "14": 1.018, - "10": 2.0109999999999997, - "24": 1.017, - "1008": 1, - "1": 4.9830000000000005, - "4": 2.3280000000000003, - "19": 1.018, - "144": 1, - "18": 1.018 -} -"#, - ) - .unwrap(); - assert_eq!( - into_fee_rate(6, esplora_fees.clone()).unwrap(), - FeeRate::from_sat_per_vb(2.236) - ); - assert_eq!( - into_fee_rate(26, esplora_fees).unwrap(), - FeeRate::from_sat_per_vb(1.015), - "should inherit from value for 25" - ); - } - #[test] #[cfg(feature = "test-esplora")] fn test_esplora_with_variable_configs() { + use super::*; + use crate::testutils::{ blockchain_tests::TestClient, configurable_blockchain_tests::ConfigurableBlockchainTester, diff --git a/src/blockchain/esplora/reqwest.rs b/src/blockchain/esplora/reqwest.rs index b549f30a..5ddbdeb4 100644 --- a/src/blockchain/esplora/reqwest.rs +++ b/src/blockchain/esplora/reqwest.rs @@ -14,49 +14,36 @@ use std::collections::{HashMap, HashSet}; use std::ops::Deref; -use bitcoin::consensus::{deserialize, serialize}; -use bitcoin::hashes::hex::{FromHex, ToHex}; -use bitcoin::hashes::{sha256, Hash}; -use bitcoin::{BlockHeader, Script, Transaction, Txid}; +use bitcoin::{Transaction, Txid}; #[allow(unused_imports)] use log::{debug, error, info, trace}; -use ::reqwest::{Client, StatusCode}; +use esplora_client::{convert_fee_rate, AsyncClient, Builder, Tx}; use futures::stream::{FuturesOrdered, TryStreamExt}; -use super::api::Tx; -use crate::blockchain::esplora::EsploraError; use crate::blockchain::*; use crate::database::BatchDatabase; use crate::error::Error; use crate::FeeRate; -/// Structure encapsulates Esplora client -#[derive(Debug)] -pub struct UrlClient { - url: String, - // We use the async client instead of the blocking one because it automatically uses `fetch` - // when the target platform is wasm32. - client: Client, - concurrency: u8, -} - /// Structure that implements the logic to sync with Esplora /// /// ## Example /// See the [`blockchain::esplora`](crate::blockchain::esplora) module for a usage example. #[derive(Debug)] pub struct EsploraBlockchain { - url_client: UrlClient, + url_client: AsyncClient, stop_gap: usize, + concurrency: u8, } -impl std::convert::From for EsploraBlockchain { - fn from(url_client: UrlClient) -> Self { +impl std::convert::From for EsploraBlockchain { + fn from(url_client: AsyncClient) -> Self { EsploraBlockchain { url_client, stop_gap: 20, + concurrency: super::DEFAULT_CONCURRENT_REQUESTS, } } } @@ -64,19 +51,25 @@ impl std::convert::From for EsploraBlockchain { impl EsploraBlockchain { /// Create a new instance of the client from a base URL and `stop_gap`. pub fn new(base_url: &str, stop_gap: usize) -> Self { + let url_client = Builder::new(base_url) + .build_async() + .expect("Should never fail with no proxy and timeout"); + + Self::from_client(url_client, stop_gap) + } + + /// Build a new instance given a client + pub fn from_client(url_client: AsyncClient, stop_gap: usize) -> Self { EsploraBlockchain { - url_client: UrlClient { - url: base_url.to_string(), - client: Client::new(), - concurrency: super::DEFAULT_CONCURRENT_REQUESTS, - }, + url_client, stop_gap, + concurrency: super::DEFAULT_CONCURRENT_REQUESTS, } } /// Set the concurrency to use when doing batch queries against the Esplora instance. pub fn with_concurrency(mut self, concurrency: u8) -> Self { - self.url_client.concurrency = concurrency; + self.concurrency = concurrency; self } } @@ -94,17 +87,19 @@ impl Blockchain for EsploraBlockchain { } fn broadcast(&self, tx: &Transaction) -> Result<(), Error> { - Ok(await_or_block!(self.url_client._broadcast(tx))?) + Ok(await_or_block!(self.url_client.broadcast(tx))?) } fn estimate_fee(&self, target: usize) -> Result { - let estimates = await_or_block!(self.url_client._get_fee_estimates())?; - super::into_fee_rate(target, estimates) + let estimates = await_or_block!(self.url_client.get_fee_estimates())?; + Ok(FeeRate::from_sat_per_vb(convert_fee_rate( + target, estimates, + )?)) } } impl Deref for EsploraBlockchain { - type Target = UrlClient; + type Target = AsyncClient; fn deref(&self) -> &Self::Target { &self.url_client @@ -116,21 +111,21 @@ impl StatelessBlockchain for EsploraBlockchain {} #[maybe_async] impl GetHeight for EsploraBlockchain { fn get_height(&self) -> Result { - Ok(await_or_block!(self.url_client._get_height())?) + Ok(await_or_block!(self.url_client.get_height())?) } } #[maybe_async] impl GetTx for EsploraBlockchain { fn get_tx(&self, txid: &Txid) -> Result, Error> { - Ok(await_or_block!(self.url_client._get_tx(txid))?) + Ok(await_or_block!(self.url_client.get_tx(txid))?) } } #[maybe_async] impl GetBlockHash for EsploraBlockchain { fn get_block_hash(&self, height: u64) -> Result { - let block_header = await_or_block!(self.url_client._get_header(height as u32))?; + let block_header = await_or_block!(self.url_client.get_header(height as u32))?; Ok(block_header.block_hash()) } } @@ -151,10 +146,10 @@ impl WalletSync for EsploraBlockchain { Request::Script(script_req) => { let futures: FuturesOrdered<_> = script_req .request() - .take(self.url_client.concurrency as usize) + .take(self.concurrency as usize) .map(|script| async move { let mut related_txs: Vec = - self.url_client._scripthash_txs(script, None).await?; + self.url_client.scripthash_txs(script, None).await?; let n_confirmed = related_txs.iter().filter(|tx| tx.status.confirmed).count(); @@ -164,7 +159,7 @@ impl WalletSync for EsploraBlockchain { loop { let new_related_txs: Vec = self .url_client - ._scripthash_txs( + .scripthash_txs( script, Some(related_txs.last().unwrap().txid), ) @@ -204,6 +199,7 @@ impl WalletSync for EsploraBlockchain { .get(txid) .expect("must be in index") .confirmation_time() + .map(Into::into) }) .collect(); conftime_req.satisfy(conftimes)? @@ -227,132 +223,26 @@ impl WalletSync for EsploraBlockchain { } } -impl UrlClient { - async fn _get_tx(&self, txid: &Txid) -> Result, EsploraError> { - let resp = self - .client - .get(&format!("{}/tx/{}/raw", self.url, txid)) - .send() - .await?; - - if let StatusCode::NOT_FOUND = resp.status() { - return Ok(None); - } - - Ok(Some(deserialize(&resp.error_for_status()?.bytes().await?)?)) - } - - async fn _get_tx_no_opt(&self, txid: &Txid) -> Result { - match self._get_tx(txid).await { - Ok(Some(tx)) => Ok(tx), - Ok(None) => Err(EsploraError::TransactionNotFound(*txid)), - Err(e) => Err(e), - } - } - - async fn _get_header(&self, block_height: u32) -> Result { - let resp = self - .client - .get(&format!("{}/block-height/{}", self.url, block_height)) - .send() - .await?; - - if let StatusCode::NOT_FOUND = resp.status() { - return Err(EsploraError::HeaderHeightNotFound(block_height)); - } - let bytes = resp.bytes().await?; - let hash = std::str::from_utf8(&bytes) - .map_err(|_| EsploraError::HeaderHeightNotFound(block_height))?; - - let resp = self - .client - .get(&format!("{}/block/{}/header", self.url, hash)) - .send() - .await?; - - let header = deserialize(&Vec::from_hex(&resp.text().await?)?)?; - - Ok(header) - } - - async fn _broadcast(&self, transaction: &Transaction) -> Result<(), EsploraError> { - self.client - .post(&format!("{}/tx", self.url)) - .body(serialize(transaction).to_hex()) - .send() - .await? - .error_for_status()?; - - Ok(()) - } - - async fn _get_height(&self) -> Result { - let req = self - .client - .get(&format!("{}/blocks/tip/height", self.url)) - .send() - .await?; - - Ok(req.error_for_status()?.text().await?.parse()?) - } - - async fn _scripthash_txs( - &self, - script: &Script, - last_seen: Option, - ) -> Result, EsploraError> { - let script_hash = sha256::Hash::hash(script.as_bytes()).into_inner().to_hex(); - let url = match last_seen { - Some(last_seen) => format!( - "{}/scripthash/{}/txs/chain/{}", - self.url, script_hash, last_seen - ), - None => format!("{}/scripthash/{}/txs", self.url, script_hash), - }; - Ok(self - .client - .get(url) - .send() - .await? - .error_for_status()? - .json::>() - .await?) - } - - async fn _get_fee_estimates(&self) -> Result, EsploraError> { - Ok(self - .client - .get(&format!("{}/fee-estimates", self.url,)) - .send() - .await? - .error_for_status()? - .json::>() - .await?) - } -} - impl ConfigurableBlockchain for EsploraBlockchain { type Config = super::EsploraBlockchainConfig; fn from_config(config: &Self::Config) -> Result { - let map_e = |e: reqwest::Error| Error::Esplora(Box::new(e.into())); + let mut builder = Builder::new(config.base_url.as_str()); - let mut blockchain = EsploraBlockchain::new(config.base_url.as_str(), config.stop_gap); - if let Some(concurrency) = config.concurrency { - blockchain.url_client.concurrency = concurrency; - } - let mut builder = Client::builder(); - #[cfg(not(target_arch = "wasm32"))] - if let Some(proxy) = &config.proxy { - builder = builder.proxy(reqwest::Proxy::all(proxy).map_err(map_e)?); - } - - #[cfg(not(target_arch = "wasm32"))] if let Some(timeout) = config.timeout { - builder = builder.timeout(core::time::Duration::from_secs(timeout)); + builder = builder.timeout(timeout); } - blockchain.url_client.client = builder.build().map_err(map_e)?; + if let Some(proxy) = &config.proxy { + builder = builder.proxy(proxy); + } + + let mut blockchain = + EsploraBlockchain::from_client(builder.build_async()?, config.stop_gap); + + if let Some(concurrency) = config.concurrency { + blockchain = blockchain.with_concurrency(concurrency); + } Ok(blockchain) } diff --git a/src/blockchain/esplora/ureq.rs b/src/blockchain/esplora/ureq.rs index 7a9388a0..1e9d1cfc 100644 --- a/src/blockchain/esplora/ureq.rs +++ b/src/blockchain/esplora/ureq.rs @@ -12,42 +12,26 @@ //! Esplora by way of `ureq` HTTP client. use std::collections::{HashMap, HashSet}; -use std::io; -use std::io::Read; -use std::ops::Deref; -use std::time::Duration; #[allow(unused_imports)] use log::{debug, error, info, trace}; -use ureq::{Agent, Proxy, Response}; +use bitcoin::{Transaction, Txid}; -use bitcoin::consensus::{deserialize, serialize}; -use bitcoin::hashes::hex::{FromHex, ToHex}; -use bitcoin::hashes::{sha256, Hash}; -use bitcoin::{BlockHeader, Script, Transaction, Txid}; +use esplora_client::{convert_fee_rate, BlockingClient, Builder, Tx}; -use super::api::Tx; -use crate::blockchain::esplora::EsploraError; use crate::blockchain::*; use crate::database::BatchDatabase; use crate::error::Error; use crate::FeeRate; -/// Structure encapsulates ureq Esplora client -#[derive(Debug, Clone)] -pub struct UrlClient { - url: String, - agent: Agent, -} - /// Structure that implements the logic to sync with Esplora /// /// ## Example /// See the [`blockchain::esplora`](crate::blockchain::esplora) module for a usage example. #[derive(Debug)] pub struct EsploraBlockchain { - url_client: UrlClient, + url_client: BlockingClient, stop_gap: usize, concurrency: u8, } @@ -55,22 +39,22 @@ pub struct EsploraBlockchain { impl EsploraBlockchain { /// Create a new instance of the client from a base URL and the `stop_gap`. pub fn new(base_url: &str, stop_gap: usize) -> Self { + let url_client = Builder::new(base_url) + .build_blocking() + .expect("Should never fail with no proxy and timeout"); + + Self::from_client(url_client, stop_gap) + } + + /// Build a new instance given a client + pub fn from_client(url_client: BlockingClient, stop_gap: usize) -> Self { EsploraBlockchain { - url_client: UrlClient { - url: base_url.to_string(), - agent: Agent::new(), - }, + url_client, concurrency: super::DEFAULT_CONCURRENT_REQUESTS, stop_gap, } } - /// Set the inner `ureq` agent. - pub fn with_agent(mut self, agent: Agent) -> Self { - self.url_client.agent = agent; - self - } - /// Set the number of parallel requests the client can make. pub fn with_concurrency(mut self, concurrency: u8) -> Self { self.concurrency = concurrency; @@ -90,18 +74,20 @@ impl Blockchain for EsploraBlockchain { } fn broadcast(&self, tx: &Transaction) -> Result<(), Error> { - self.url_client._broadcast(tx)?; + self.url_client.broadcast(tx)?; Ok(()) } fn estimate_fee(&self, target: usize) -> Result { - let estimates = self.url_client._get_fee_estimates()?; - super::into_fee_rate(target, estimates) + let estimates = self.url_client.get_fee_estimates()?; + Ok(FeeRate::from_sat_per_vb(convert_fee_rate( + target, estimates, + )?)) } } impl Deref for EsploraBlockchain { - type Target = UrlClient; + type Target = BlockingClient; fn deref(&self) -> &Self::Target { &self.url_client @@ -112,19 +98,19 @@ impl StatelessBlockchain for EsploraBlockchain {} impl GetHeight for EsploraBlockchain { fn get_height(&self) -> Result { - Ok(self.url_client._get_height()?) + Ok(self.url_client.get_height()?) } } impl GetTx for EsploraBlockchain { fn get_tx(&self, txid: &Txid) -> Result, Error> { - Ok(self.url_client._get_tx(txid)?) + Ok(self.url_client.get_tx(txid)?) } } impl GetBlockHash for EsploraBlockchain { fn get_block_hash(&self, height: u64) -> Result { - let block_header = self.url_client._get_header(height as u32)?; + let block_header = self.url_client.get_header(height as u32)?; Ok(block_header.block_hash()) } } @@ -151,7 +137,7 @@ impl WalletSync for EsploraBlockchain { let client = self.url_client.clone(); // make each request in its own thread. handles.push(std::thread::spawn(move || { - let mut related_txs: Vec = client._scripthash_txs(&script, None)?; + let mut related_txs: Vec = client.scripthash_txs(&script, None)?; let n_confirmed = related_txs.iter().filter(|tx| tx.status.confirmed).count(); @@ -159,7 +145,7 @@ impl WalletSync for EsploraBlockchain { // keep requesting to see if there's more. if n_confirmed >= 25 { loop { - let new_related_txs: Vec = client._scripthash_txs( + let new_related_txs: Vec = client.scripthash_txs( &script, Some(related_txs.last().unwrap().txid), )?; @@ -202,6 +188,7 @@ impl WalletSync for EsploraBlockchain { .get(txid) .expect("must be in index") .confirmation_time() + .map(Into::into) }) .collect(); conftime_req.satisfy(conftimes)? @@ -226,159 +213,22 @@ impl WalletSync for EsploraBlockchain { } } -impl UrlClient { - fn _get_tx(&self, txid: &Txid) -> Result, EsploraError> { - let resp = self - .agent - .get(&format!("{}/tx/{}/raw", self.url, txid)) - .call(); - - match resp { - Ok(resp) => Ok(Some(deserialize(&into_bytes(resp)?)?)), - Err(ureq::Error::Status(code, _)) => { - if is_status_not_found(code) { - return Ok(None); - } - Err(EsploraError::HttpResponse(code)) - } - Err(e) => Err(EsploraError::Ureq(e)), - } - } - - fn _get_tx_no_opt(&self, txid: &Txid) -> Result { - match self._get_tx(txid) { - Ok(Some(tx)) => Ok(tx), - Ok(None) => Err(EsploraError::TransactionNotFound(*txid)), - Err(e) => Err(e), - } - } - - fn _get_header(&self, block_height: u32) -> Result { - let resp = self - .agent - .get(&format!("{}/block-height/{}", self.url, block_height)) - .call(); - - let bytes = match resp { - Ok(resp) => Ok(into_bytes(resp)?), - Err(ureq::Error::Status(code, _)) => Err(EsploraError::HttpResponse(code)), - Err(e) => Err(EsploraError::Ureq(e)), - }?; - - let hash = std::str::from_utf8(&bytes) - .map_err(|_| EsploraError::HeaderHeightNotFound(block_height))?; - - let resp = self - .agent - .get(&format!("{}/block/{}/header", self.url, hash)) - .call(); - - match resp { - Ok(resp) => Ok(deserialize(&Vec::from_hex(&resp.into_string()?)?)?), - Err(ureq::Error::Status(code, _)) => Err(EsploraError::HttpResponse(code)), - Err(e) => Err(EsploraError::Ureq(e)), - } - } - - fn _broadcast(&self, transaction: &Transaction) -> Result<(), EsploraError> { - let resp = self - .agent - .post(&format!("{}/tx", self.url)) - .send_string(&serialize(transaction).to_hex()); - - match resp { - Ok(_) => Ok(()), // We do not return the txid? - Err(ureq::Error::Status(code, _)) => Err(EsploraError::HttpResponse(code)), - Err(e) => Err(EsploraError::Ureq(e)), - } - } - - fn _get_height(&self) -> Result { - let resp = self - .agent - .get(&format!("{}/blocks/tip/height", self.url)) - .call(); - - match resp { - Ok(resp) => Ok(resp.into_string()?.parse()?), - Err(ureq::Error::Status(code, _)) => Err(EsploraError::HttpResponse(code)), - Err(e) => Err(EsploraError::Ureq(e)), - } - } - - fn _get_fee_estimates(&self) -> Result, EsploraError> { - let resp = self - .agent - .get(&format!("{}/fee-estimates", self.url,)) - .call(); - - let map = match resp { - Ok(resp) => { - let map: HashMap = resp.into_json()?; - Ok(map) - } - Err(ureq::Error::Status(code, _)) => Err(EsploraError::HttpResponse(code)), - Err(e) => Err(EsploraError::Ureq(e)), - }?; - - Ok(map) - } - - fn _scripthash_txs( - &self, - script: &Script, - last_seen: Option, - ) -> Result, EsploraError> { - let script_hash = sha256::Hash::hash(script.as_bytes()).into_inner().to_hex(); - let url = match last_seen { - Some(last_seen) => format!( - "{}/scripthash/{}/txs/chain/{}", - self.url, script_hash, last_seen - ), - None => format!("{}/scripthash/{}/txs", self.url, script_hash), - }; - Ok(self.agent.get(&url).call()?.into_json()?) - } -} - -fn is_status_not_found(status: u16) -> bool { - status == 404 -} - -fn into_bytes(resp: Response) -> Result, io::Error> { - const BYTES_LIMIT: usize = 10 * 1_024 * 1_024; - - let mut buf: Vec = vec![]; - resp.into_reader() - .take((BYTES_LIMIT + 1) as u64) - .read_to_end(&mut buf)?; - if buf.len() > BYTES_LIMIT { - return Err(io::Error::new( - io::ErrorKind::Other, - "response too big for into_bytes", - )); - } - - Ok(buf) -} - impl ConfigurableBlockchain for EsploraBlockchain { type Config = super::EsploraBlockchainConfig; fn from_config(config: &Self::Config) -> Result { - let mut agent_builder = ureq::AgentBuilder::new(); + let mut builder = Builder::new(config.base_url.as_str()); if let Some(timeout) = config.timeout { - agent_builder = agent_builder.timeout(Duration::from_secs(timeout)); + builder = builder.timeout(timeout); } if let Some(proxy) = &config.proxy { - agent_builder = agent_builder - .proxy(Proxy::new(proxy).map_err(|e| Error::Esplora(Box::new(e.into())))?); + builder = builder.proxy(proxy); } - let mut blockchain = EsploraBlockchain::new(config.base_url.as_str(), config.stop_gap) - .with_agent(agent_builder.build()); + let mut blockchain = + EsploraBlockchain::from_client(builder.build_blocking()?, config.stop_gap); if let Some(concurrency) = config.concurrency { blockchain = blockchain.with_concurrency(concurrency); @@ -387,12 +237,3 @@ impl ConfigurableBlockchain for EsploraBlockchain { Ok(blockchain) } } - -impl From for EsploraError { - fn from(e: ureq::Error) -> Self { - match e { - ureq::Error::Status(code, _) => EsploraError::HttpResponse(code), - e => EsploraError::Ureq(e), - } - } -} From cf2bc388f22b069fc25fba482e59da6305207864 Mon Sep 17 00:00:00 2001 From: Elias Rohrer Date: Wed, 10 Aug 2022 16:13:05 +0200 Subject: [PATCH 03/19] Re-export `esplora_client` --- src/lib.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index bed048cf..65e35a72 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -237,6 +237,9 @@ pub extern crate bitcoincore_rpc; #[cfg(feature = "electrum")] pub extern crate electrum_client; +#[cfg(feature = "esplora")] +pub extern crate esplora_client; + #[cfg(feature = "key-value-db")] pub extern crate sled; From b5b92248c76aeb42ac747931efd45bc2e7af5ebd Mon Sep 17 00:00:00 2001 From: Alekos Filini Date: Wed, 28 Sep 2022 19:33:22 +0200 Subject: [PATCH 04/19] Rename esplora features to -async and -blocking --- .github/workflows/cont_integration.yml | 12 ++++++------ .github/workflows/nightly_docs.yml | 2 +- Cargo.toml | 15 +++++++++------ src/blockchain/esplora/mod.rs | 8 ++++---- 4 files changed, 20 insertions(+), 17 deletions(-) diff --git a/.github/workflows/cont_integration.yml b/.github/workflows/cont_integration.yml index af7e256f..3b083902 100644 --- a/.github/workflows/cont_integration.yml +++ b/.github/workflows/cont_integration.yml @@ -17,16 +17,16 @@ jobs: - default - minimal - all-keys - - minimal,use-esplora-ureq + - minimal,use-esplora-blocking - key-value-db - electrum - compact_filters - - esplora,ureq,key-value-db,electrum + - use-esplora-blocking,key-value-db,electrum - compiler - rpc - verify - async-interface - - use-esplora-reqwest + - use-esplora-async - sqlite - sqlite-bundled steps: @@ -100,10 +100,10 @@ jobs: features: test-rpc-legacy - name: esplora testprefix: esplora - features: test-esplora,use-esplora-reqwest,verify + features: test-esplora,use-esplora-async,verify - name: esplora testprefix: esplora - features: test-esplora,use-esplora-ureq,verify + features: test-esplora,use-esplora-blocking,verify steps: - name: Checkout uses: actions/checkout@v2 @@ -154,7 +154,7 @@ jobs: - name: Update toolchain run: rustup update - name: Check - run: cargo check --target wasm32-unknown-unknown --features use-esplora-reqwest --no-default-features + run: cargo check --target wasm32-unknown-unknown --features use-esplora-async --no-default-features fmt: name: Rust fmt diff --git a/.github/workflows/nightly_docs.yml b/.github/workflows/nightly_docs.yml index a82beb2d..190006ea 100644 --- a/.github/workflows/nightly_docs.yml +++ b/.github/workflows/nightly_docs.yml @@ -24,7 +24,7 @@ jobs: - name: Update toolchain run: rustup update - name: Build docs - run: cargo rustdoc --verbose --features=compiler,electrum,esplora,ureq,compact_filters,key-value-db,all-keys,sqlite -- --cfg docsrs -Dwarnings + run: cargo rustdoc --verbose --features=compiler,electrum,esplora,use-esplora-blocking,compact_filters,rpc,key-value-db,sqlite,all-keys,verify,hardware-signer -- --cfg docsrs -Dwarnings - name: Upload artifact uses: actions/upload-artifact@v2 with: diff --git a/Cargo.toml b/Cargo.toml index 81cfeb4c..cac6c17e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,22 +68,25 @@ hardware-signer = ["hwi"] # # - Users wanting asynchronous HTTP calls should enable `async-interface` to get # access to the asynchronous method implementations. Then, if Esplora is wanted, -# enable `esplora` AND `reqwest` (`--features=use-esplora-reqwest`). +# enable the `use-esplora-async` feature. # - Users wanting blocking HTTP calls can use any of the other blockchain # implementations (`compact_filters`, `electrum`, or `esplora`). Users wanting to -# use Esplora should enable `esplora` AND `ureq` (`--features=use-esplora-ureq`). +# use Esplora should enable the `use-esplora-blocking` feature. # # WARNING: Please take care with the features below, various combinations will # fail to build. We cannot currently build `bdk` with `--all-features`. async-interface = ["async-trait"] electrum = ["electrum-client"] # MUST ALSO USE `--no-default-features`. -use-esplora-reqwest = ["esplora", "esplora-client/async", "futures"] -use-esplora-ureq = ["esplora", "esplora-client/blocking"] +use-esplora-async = ["esplora", "esplora-client/async", "futures"] +use-esplora-blocking = ["esplora", "esplora-client/blocking"] +# Deprecated aliases +use-esplora-reqwest = ["use-esplora-async"] +use-esplora-ureq = ["use-esplora-blocking"] # Typical configurations will not need to use `esplora` feature directly. esplora = [] -# Use below feature with `use-esplora-reqwest` to enable reqwest default TLS support +# Use below feature with `use-esplora-async` to enable reqwest default TLS support reqwest-default-tls = ["esplora-client/async-https"] # Debug/Test features @@ -128,6 +131,6 @@ required-features = ["electrum"] [workspace] members = ["macros"] [package.metadata.docs.rs] -features = ["compiler", "electrum", "esplora", "use-esplora-ureq", "compact_filters", "rpc", "key-value-db", "sqlite", "all-keys", "verify", "hardware-signer"] +features = ["compiler", "electrum", "esplora", "use-esplora-blocking", "compact_filters", "rpc", "key-value-db", "sqlite", "all-keys", "verify", "hardware-signer"] # defines the configuration attribute `docsrs` rustdoc-args = ["--cfg", "docsrs"] diff --git a/src/blockchain/esplora/mod.rs b/src/blockchain/esplora/mod.rs index edc00926..7a4b1937 100644 --- a/src/blockchain/esplora/mod.rs +++ b/src/blockchain/esplora/mod.rs @@ -20,16 +20,16 @@ pub use esplora_client::Error as EsploraError; -#[cfg(feature = "use-esplora-reqwest")] +#[cfg(feature = "use-esplora-async")] mod reqwest; -#[cfg(feature = "use-esplora-reqwest")] +#[cfg(feature = "use-esplora-async")] pub use self::reqwest::*; -#[cfg(feature = "use-esplora-ureq")] +#[cfg(feature = "use-esplora-blocking")] mod ureq; -#[cfg(feature = "use-esplora-ureq")] +#[cfg(feature = "use-esplora-blocking")] pub use self::ureq::*; /// Configuration for an [`EsploraBlockchain`] From b11c86d074a8f56f99bd5f3af77d3b056af71de4 Mon Sep 17 00:00:00 2001 From: Alekos Filini Date: Thu, 29 Sep 2022 11:59:21 +0200 Subject: [PATCH 05/19] Rename internal esplora modules, fix docs --- src/blockchain/esplora/{reqwest.rs => async.rs} | 0 src/blockchain/esplora/{ureq.rs => blocking.rs} | 0 src/blockchain/esplora/mod.rs | 12 ++++++------ 3 files changed, 6 insertions(+), 6 deletions(-) rename src/blockchain/esplora/{reqwest.rs => async.rs} (100%) rename src/blockchain/esplora/{ureq.rs => blocking.rs} (100%) diff --git a/src/blockchain/esplora/reqwest.rs b/src/blockchain/esplora/async.rs similarity index 100% rename from src/blockchain/esplora/reqwest.rs rename to src/blockchain/esplora/async.rs diff --git a/src/blockchain/esplora/ureq.rs b/src/blockchain/esplora/blocking.rs similarity index 100% rename from src/blockchain/esplora/ureq.rs rename to src/blockchain/esplora/blocking.rs diff --git a/src/blockchain/esplora/mod.rs b/src/blockchain/esplora/mod.rs index 7a4b1937..57032e49 100644 --- a/src/blockchain/esplora/mod.rs +++ b/src/blockchain/esplora/mod.rs @@ -15,22 +15,22 @@ //! depending on your needs (blocking or async respectively). //! //! Please note, to configure the Esplora HTTP client correctly use one of: -//! Blocking: --features='esplora,ureq' -//! Async: --features='async-interface,esplora,reqwest' --no-default-features +//! Blocking: --features='use-esplora-blocking' +//! Async: --features='async-interface,use-esplora-async' --no-default-features pub use esplora_client::Error as EsploraError; #[cfg(feature = "use-esplora-async")] -mod reqwest; +mod r#async; #[cfg(feature = "use-esplora-async")] -pub use self::reqwest::*; +pub use self::r#async::*; #[cfg(feature = "use-esplora-blocking")] -mod ureq; +mod blocking; #[cfg(feature = "use-esplora-blocking")] -pub use self::ureq::*; +pub use self::blocking::*; /// Configuration for an [`EsploraBlockchain`] #[derive(Debug, serde::Deserialize, serde::Serialize, Clone, PartialEq)] From d7bfe68e2df270ab799d36ebf3563e178ae50c6e Mon Sep 17 00:00:00 2001 From: Alekos Filini Date: Wed, 28 Sep 2022 20:38:31 +0200 Subject: [PATCH 06/19] Fix broken nightly docs --- src/wallet/verify.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/wallet/verify.rs b/src/wallet/verify.rs index 0f12e056..084388b9 100644 --- a/src/wallet/verify.rs +++ b/src/wallet/verify.rs @@ -29,6 +29,8 @@ use crate::error::Error; /// Depending on the [capabilities](crate::blockchain::Blockchain::get_capabilities) of the /// [`Blockchain`] backend, the method could fail when called with old "historical" transactions or /// with unconfirmed transactions that have been evicted from the backend's memory. +/// +/// [`Blockchain`]: crate::blockchain::Blockchain pub fn verify_tx( tx: &Transaction, database: &D, From 55145f57a183e8258fbbdd1752efea8e1b0ba006 Mon Sep 17 00:00:00 2001 From: Alekos Filini Date: Thu, 29 Sep 2022 20:57:36 +0200 Subject: [PATCH 07/19] Bump version to 0.23.0 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index cac6c17e..ebe7545f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "bdk" -version = "0.22.0" +version = "0.23.0" edition = "2018" authors = ["Alekos Filini ", "Riccardo Casatta "] homepage = "https://bitcoindevkit.org" From 7de8be46c04128328d8dbd1de7b29033a290fde8 Mon Sep 17 00:00:00 2001 From: Steve Myers Date: Sat, 1 Oct 2022 10:06:34 -0500 Subject: [PATCH 08/19] Add enhancement request github issue template --- .github/ISSUE_TEMPLATE/enhancement_request.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/enhancement_request.md diff --git a/.github/ISSUE_TEMPLATE/enhancement_request.md b/.github/ISSUE_TEMPLATE/enhancement_request.md new file mode 100644 index 00000000..2525d5d0 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/enhancement_request.md @@ -0,0 +1,17 @@ +--- +name: Enhancement request +about: Request a new feature or change to an existing feature +title: '' +labels: 'enhancement' +assignees: '' + +--- + +**Describe the enhancement** + + +**Use case** + + +**Additional context** + \ No newline at end of file From 1331193800911bb8168e5d1c2689676af3c1fe86 Mon Sep 17 00:00:00 2001 From: Steve Myers Date: Sun, 9 Oct 2022 21:36:53 -0500 Subject: [PATCH 09/19] Update psbt_signer example to use descriptor! macro --- examples/psbt_signer.rs | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/examples/psbt_signer.rs b/examples/psbt_signer.rs index bf49e628..8d6e96fd 100644 --- a/examples/psbt_signer.rs +++ b/examples/psbt_signer.rs @@ -9,20 +9,38 @@ use bdk::blockchain::{Blockchain, ElectrumBlockchain}; use bdk::database::MemoryDatabase; use bdk::wallet::AddressIndex; -use bdk::SyncOptions; +use bdk::{descriptor, SyncOptions}; use bdk::{FeeRate, SignOptions, Wallet}; +use bitcoin::secp256k1::Secp256k1; use bitcoin::{Address, Network}; use electrum_client::Client; +use miniscript::descriptor::DescriptorSecretKey; use std::error::Error; use std::str::FromStr; +/// This example shows how to sign and broadcast the transaction for a PSBT (Partially Signed +/// Bitcoin Transaction) for a single key, witness public key hash (WPKH) based descriptor wallet. +/// The electrum protocol is used to sync blockchain data from the testnet bitcoin network and +/// wallet data is stored in an ephemeral in-memory database. The process steps are: +/// 1. Create a "signing" wallet and a "watch-only" wallet based on the same private keys. +/// 2. Deposit testnet funds into the watch only wallet. +/// 3. Sync the watch only wallet and create a spending transaction to return all funds to the testnet faucet. +/// 4. Sync the signing wallet and sign and finalize the PSBT created by the watch only wallet. +/// 5. Broadcast the transactions from the finalized PSBT. fn main() -> Result<(), Box> { - // test keys created with `bdk-cli key generate` and `bdk-cli key derive` commands - let signing_external_descriptor = "wpkh([e9824965/84'/1'/0']tprv8fvem7qWxY3SGCQczQpRpqTKg455wf1zgixn6MZ4ze8gRfHjov5gXBQTadNfDgqs9ERbZZ3Bi1PNYrCCusFLucT39K525MWLpeURjHwUsfX/0/*)"; - let signing_internal_descriptor = "wpkh([e9824965/84'/1'/0']tprv8fvem7qWxY3SGCQczQpRpqTKg455wf1zgixn6MZ4ze8gRfHjov5gXBQTadNfDgqs9ERbZZ3Bi1PNYrCCusFLucT39K525MWLpeURjHwUsfX/1/*)"; + // test key created with `bdk-cli key generate` and `bdk-cli key derive` commands + let external_secret_xkey = DescriptorSecretKey::from_str("[e9824965/84'/1'/0']tprv8fvem7qWxY3SGCQczQpRpqTKg455wf1zgixn6MZ4ze8gRfHjov5gXBQTadNfDgqs9ERbZZ3Bi1PNYrCCusFLucT39K525MWLpeURjHwUsfX/0/*").unwrap(); + let internal_secret_xkey = DescriptorSecretKey::from_str("[e9824965/84'/1'/0']tprv8fvem7qWxY3SGCQczQpRpqTKg455wf1zgixn6MZ4ze8gRfHjov5gXBQTadNfDgqs9ERbZZ3Bi1PNYrCCusFLucT39K525MWLpeURjHwUsfX/1/*").unwrap(); - let watch_only_external_descriptor = "wpkh([e9824965/84'/1'/0']tpubDCcguXsm6uj79fSQt4V2EF7SF5b26zCuG2ZZNsbNQuw5G9YWSJuGhg2KknQBywRq4VGTu41zYTCh3QeVFyBdbsymgRX9Mrts94SW7obEdqs/0/*)"; - let watch_only_internal_descriptor = "wpkh([e9824965/84'/1'/0']tpubDCcguXsm6uj79fSQt4V2EF7SF5b26zCuG2ZZNsbNQuw5G9YWSJuGhg2KknQBywRq4VGTu41zYTCh3QeVFyBdbsymgRX9Mrts94SW7obEdqs/1/*)"; + let secp = Secp256k1::new(); + let external_public_xkey = external_secret_xkey.as_public(&secp).unwrap(); + let internal_public_xkey = internal_secret_xkey.as_public(&secp).unwrap(); + + let signing_external_descriptor = descriptor!(wpkh(external_secret_xkey)).unwrap(); + let signing_internal_descriptor = descriptor!(wpkh(internal_secret_xkey)).unwrap(); + + let watch_only_external_descriptor = descriptor!(wpkh(external_public_xkey)).unwrap(); + let watch_only_internal_descriptor = descriptor!(wpkh(internal_public_xkey)).unwrap(); // create client for Blockstream's testnet electrum server let blockchain = From a4a43ea86060fa0a62b47dedc7de820459b3a472 Mon Sep 17 00:00:00 2001 From: Daniela Brozzoni Date: Wed, 12 Oct 2022 14:23:42 +0100 Subject: [PATCH 10/19] Re-export HWI if the hardware-signer feature is set --- src/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index bed048cf..dcbe1b1e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -203,6 +203,8 @@ pub extern crate miniscript; extern crate serde; #[macro_use] extern crate serde_json; +#[cfg(feature = "hardware-signer")] +pub extern crate hwi; #[cfg(all(feature = "reqwest", feature = "ureq"))] compile_error!("Features reqwest and ureq are mutually exclusive and cannot be enabled together"); From 0695e9fb3e41727e5732561a993411147487afd3 Mon Sep 17 00:00:00 2001 From: Daniela Brozzoni Date: Wed, 12 Oct 2022 14:24:09 +0100 Subject: [PATCH 11/19] Bump HWI to 0.2.3 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index e0b73698..72b40f63 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,7 @@ async-trait = { version = "0.1", optional = true } rocksdb = { version = "0.14", default-features = false, features = ["snappy"], optional = true } cc = { version = ">=1.0.64", optional = true } socks = { version = "0.3", optional = true } -hwi = { version = "0.2.2", optional = true } +hwi = { version = "0.2.3", optional = true } bip39 = { version = "1.0.1", optional = true } bitcoinconsensus = { version = "0.19.0-3", optional = true } From 1a71eb1f4736651ad82e0abd64792b6cc7b16c20 Mon Sep 17 00:00:00 2001 From: Daniela Brozzoni Date: Wed, 12 Oct 2022 14:24:29 +0100 Subject: [PATCH 12/19] Update the hardwaresigner module documentation Add a little example on how to use the HWISigner, slightly improve the module description --- src/wallet/hardwaresigner.rs | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/src/wallet/hardwaresigner.rs b/src/wallet/hardwaresigner.rs index 7e4f74bc..58246392 100644 --- a/src/wallet/hardwaresigner.rs +++ b/src/wallet/hardwaresigner.rs @@ -11,7 +11,40 @@ //! HWI Signer //! -//! This module contains a simple implementation of a Custom signer for rust-hwi +//! This module contains HWISigner, an implementation of a [TransactionSigner] to be +//! used with hardware wallets. +//! ```no_run +//! # use bdk::bitcoin::Network; +//! # use bdk::database::MemoryDatabase; +//! # use bdk::signer::SignerOrdering; +//! # use bdk::wallet::hardwaresigner::HWISigner; +//! # use bdk::wallet::AddressIndex::New; +//! # use bdk::{FeeRate, KeychainKind, SignOptions, SyncOptions, Wallet}; +//! # use hwi::{types::HWIChain, HWIClient}; +//! # use std::sync::Arc; +//! # +//! # fn main() -> Result<(), Box> { +//! let devices = HWIClient::enumerate()?; +//! let first_device = devices.first().expect("No devices found!"); +//! let custom_signer = HWISigner::from_device(first_device, HWIChain::Test)?; +//! +//! # let mut wallet = Wallet::new( +//! # "", +//! # None, +//! # Network::Testnet, +//! # MemoryDatabase::default(), +//! # )?; +//! # +//! // Adding the hardware signer to the BDK wallet +//! wallet.add_signer( +//! KeychainKind::External, +//! SignerOrdering(200), +//! Arc::new(custom_signer), +//! ); +//! +//! # Ok(()) +//! # } +//! ``` use bitcoin::psbt::PartiallySignedTransaction; use bitcoin::secp256k1::{All, Secp256k1}; From 1437e1ecfe663b819156d98c5e1975fb357a763f Mon Sep 17 00:00:00 2001 From: Daniela Brozzoni Date: Wed, 12 Oct 2022 14:31:17 +0100 Subject: [PATCH 13/19] Add the hardware_signer example --- Cargo.toml | 5 ++ examples/hardware_signer.rs | 103 ++++++++++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+) create mode 100644 examples/hardware_signer.rs diff --git a/Cargo.toml b/Cargo.toml index 72b40f63..4205237f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -126,6 +126,11 @@ name = "psbt_signer" path = "examples/psbt_signer.rs" required-features = ["electrum"] +[[example]] +name = "hardware_signer" +path = "examples/hardware_signer.rs" +required-features = ["electrum", "hardware-signer"] + [workspace] members = ["macros"] [package.metadata.docs.rs] diff --git a/examples/hardware_signer.rs b/examples/hardware_signer.rs new file mode 100644 index 00000000..81343bcf --- /dev/null +++ b/examples/hardware_signer.rs @@ -0,0 +1,103 @@ +use bdk::bitcoin::{Address, Network}; +use bdk::blockchain::{Blockchain, ElectrumBlockchain}; +use bdk::database::MemoryDatabase; +use bdk::hwi::{types::HWIChain, HWIClient}; +use bdk::signer::SignerOrdering; +use bdk::wallet::{hardwaresigner::HWISigner, AddressIndex}; +use bdk::{FeeRate, KeychainKind, SignOptions, SyncOptions, Wallet}; +use electrum_client::Client; +use std::str::FromStr; +use std::sync::Arc; + +// This example shows how to sync a wallet, create a transaction, sign it +// and broadcast it using an external hardware wallet. +// The hardware wallet must be connected to the computer and unlocked before +// running the example. Also, the `hwi` python package should be installed +// and available in the environment. +// +// To avoid loss of funds, consider using an hardware wallet simulator: +// * Coldcard: https://github.com/Coldcard/firmware +// * Ledger: https://github.com/LedgerHQ/speculos +// * Trezor: https://docs.trezor.io/trezor-firmware/core/emulator/index.html +fn main() -> Result<(), Box> { + println!("Hold tight, I'm connecting to your hardware wallet..."); + + // Listing all the available hardware wallet devices... + let devices = HWIClient::enumerate()?; + let first_device = devices + .first() + .expect("No devices found. Either plug in a hardware wallet, or start a simulator."); + // ...and creating a client out of the first one + let client = HWIClient::get_client(first_device, true, HWIChain::Test)?; + println!("Look what I found, a {}!", first_device.model); + + // Getting the HW's public descriptors + let descriptors = client.get_descriptors(None)?; + println!( + "The hardware wallet's descriptor is: {}", + descriptors.receive[0] + ); + + // Creating a custom signer from the device + let custom_signer = HWISigner::from_device(first_device, HWIChain::Test)?; + let mut wallet = Wallet::new( + &descriptors.receive[0], + Some(&descriptors.internal[0]), + Network::Testnet, + MemoryDatabase::default(), + )?; + + // Adding the hardware signer to the BDK wallet + wallet.add_signer( + KeychainKind::External, + SignerOrdering(200), + Arc::new(custom_signer), + ); + + // create client for Blockstream's testnet electrum server + let blockchain = + ElectrumBlockchain::from(Client::new("ssl://electrum.blockstream.info:60002")?); + + println!("Syncing the wallet..."); + wallet.sync(&blockchain, SyncOptions::default())?; + + // get deposit address + let deposit_address = wallet.get_address(AddressIndex::New)?; + + let balance = wallet.get_balance()?; + println!("Wallet balances in SATs: {}", balance); + + if balance.get_total() < 10000 { + println!( + "Send some sats from the u01.net testnet faucet to address '{addr}'.\nFaucet URL: https://bitcoinfaucet.uo1.net/?to={addr}", + addr = deposit_address.address + ); + return Ok(()); + } + + let return_address = Address::from_str("tb1ql7w62elx9ucw4pj5lgw4l028hmuw80sndtntxt")?; + let (mut psbt, _details) = { + let mut builder = wallet.build_tx(); + builder + .drain_wallet() + .drain_to(return_address.script_pubkey()) + .enable_rbf() + .fee_rate(FeeRate::from_sat_per_vb(5.0)); + builder.finish()? + }; + + // `sign` will call the hardware wallet asking for a signature + assert!( + wallet.sign(&mut psbt, SignOptions::default())?, + "The hardware wallet couldn't finalize the transaction :(" + ); + + println!("Let's broadcast your tx..."); + let raw_transaction = psbt.extract_tx(); + let txid = raw_transaction.txid(); + + blockchain.broadcast(&raw_transaction)?; + println!("Transaction broadcasted! TXID: {txid}.\nExplorer URL: https://mempool.space/testnet/tx/{txid}", txid = txid); + + Ok(()) +} From 68dd6d20311b28f5c8e8c9657dce8cddae4f7aa3 Mon Sep 17 00:00:00 2001 From: Vladimir Fomene Date: Thu, 6 Oct 2022 13:30:43 +0300 Subject: [PATCH 14/19] Add signature grinding for ECDSA signatures This PR adds a new field called `allow_grinding` in the Signer's `SignOptions` struct that is used to determine whether or not to grind an ECDSA signature during the signing process. --- src/wallet/mod.rs | 35 +++++++++++++++++++++++++++++++++++ src/wallet/signer.rs | 14 +++++++++++++- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/src/wallet/mod.rs b/src/wallet/mod.rs index 2e3d9fdf..d804fd21 100644 --- a/src/wallet/mod.rs +++ b/src/wallet/mod.rs @@ -5524,6 +5524,7 @@ pub(crate) mod test { SignOptions { remove_partial_sigs: false, try_finalize: false, + allow_grinding: false, ..Default::default() }, ) @@ -5538,6 +5539,7 @@ pub(crate) mod test { &mut psbt, SignOptions { remove_partial_sigs: false, + allow_grinding: false, ..Default::default() }, ) @@ -5546,6 +5548,39 @@ pub(crate) mod test { assert_fee_rate!(psbt, details.fee.unwrap_or(0), fee_rate); } + #[test] + fn test_fee_rate_sign_grinding_low_r() { + // Our goal is to obtain a transaction with a signature with low-R (70 bytes) + // by setting the `allow_grinding` signing option as true. + // We then check that our fee rate and fee calculation is alright and that our + // signature is 70 bytes. + let (wallet, _, _) = get_funded_wallet("wpkh(tprv8ZgxMBicQKsPd3EupYiPRhaMooHKUHJxNsTfYuScep13go8QFfHdtkG9nRkFGb7busX4isf6X9dURGCoKgitaApQ6MupRhZMcELAxTBRJgS/*)"); + let addr = wallet.get_address(New).unwrap(); + let fee_rate = FeeRate::from_sat_per_vb(1.0); + let mut builder = wallet.build_tx(); + builder + .drain_to(addr.script_pubkey()) + .drain_wallet() + .fee_rate(fee_rate); + let (mut psbt, details) = builder.finish().unwrap(); + + wallet + .sign( + &mut psbt, + SignOptions { + remove_partial_sigs: false, + allow_grinding: true, + ..Default::default() + }, + ) + .unwrap(); + + let key = psbt.inputs[0].partial_sigs.keys().next().unwrap(); + let sig_len = psbt.inputs[0].partial_sigs[key].sig.serialize_der().len(); + assert_eq!(sig_len, 70); + assert_fee_rate!(psbt, details.fee.unwrap_or(0), fee_rate); + } + #[cfg(feature = "test-hardware-signer")] #[test] fn test_create_signer() { diff --git a/src/wallet/signer.rs b/src/wallet/signer.rs index a20a2321..01bf5628 100644 --- a/src/wallet/signer.rs +++ b/src/wallet/signer.rs @@ -472,6 +472,7 @@ impl InputSigner for SignerWrapper { hash, hash_ty, secp, + sign_options.allow_grinding, ); Ok(()) @@ -485,9 +486,14 @@ fn sign_psbt_ecdsa( hash: bitcoin::Sighash, hash_ty: EcdsaSighashType, secp: &SecpCtx, + allow_grinding: bool, ) { let msg = &Message::from_slice(&hash.into_inner()[..]).unwrap(); - let sig = secp.sign_ecdsa(msg, secret_key); + let sig = if allow_grinding { + secp.sign_ecdsa_low_r(msg, secret_key) + } else { + secp.sign_ecdsa(msg, secret_key) + }; secp.verify_ecdsa(msg, &sig, &pubkey.inner) .expect("invalid or corrupted ecdsa signature"); @@ -718,6 +724,11 @@ pub struct SignOptions { /// /// Defaults to `true`, i.e., we always try to sign with the taproot internal key. pub sign_with_tap_internal_key: bool, + + /// Whether we should grind ECDSA signature to ensure signing with low r + /// or not. + /// Defaults to `true`, i.e., we always grind ECDSA signature to sign with low r. + pub allow_grinding: bool, } /// Customize which taproot script-path leaves the signer should sign. @@ -751,6 +762,7 @@ impl Default for SignOptions { try_finalize: true, tap_leaves_options: TapLeavesOptions::default(), sign_with_tap_internal_key: true, + allow_grinding: true, } } } From 34987d58ec9933af455e44d20c8d889bc896f869 Mon Sep 17 00:00:00 2001 From: Steve Myers Date: Tue, 18 Oct 2022 15:25:38 -0500 Subject: [PATCH 15/19] Make psbt mod public and add required docs --- src/lib.rs | 2 +- src/psbt/mod.rs | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 65e35a72..56e8ea8d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -263,7 +263,7 @@ pub mod descriptor; #[cfg(feature = "test-md-docs")] mod doctest; pub mod keys; -pub(crate) mod psbt; +pub mod psbt; pub(crate) mod types; pub mod wallet; diff --git a/src/psbt/mod.rs b/src/psbt/mod.rs index f06b5297..b6119a44 100644 --- a/src/psbt/mod.rs +++ b/src/psbt/mod.rs @@ -9,11 +9,17 @@ // You may not use this file except in accordance with one or both of these // licenses. +//! Additional functions on the `rust-bitcoin` `PartiallySignedTransaction` structure. + use crate::FeeRate; use bitcoin::util::psbt::PartiallySignedTransaction as Psbt; use bitcoin::TxOut; +// TODO upstream the functions here to `rust-bitcoin`? + +/// Trait to add functions to extract utxos and calculate fees. pub trait PsbtUtils { + /// Get the `TxOut` for the specified input index, if it doesn't exist in the PSBT `None` is returned. fn get_utxo_for(&self, input_index: usize) -> Option; /// The total transaction fee amount, sum of input amounts minus sum of output amounts, in Sats. From 9854fd34eaa688e5acd601119b8f448a2d77fbd9 Mon Sep 17 00:00:00 2001 From: Alekos Filini Date: Fri, 30 Sep 2022 13:46:32 +0200 Subject: [PATCH 16/19] Remove deprecated address validators --- Cargo.toml | 2 - examples/address_validator.rs | 63 ------------- src/error.rs | 5 +- src/lib.rs | 1 - src/wallet/address_validator.rs | 158 -------------------------------- src/wallet/mod.rs | 35 ------- 6 files changed, 1 insertion(+), 263 deletions(-) delete mode 100644 examples/address_validator.rs delete mode 100644 src/wallet/address_validator.rs diff --git a/Cargo.toml b/Cargo.toml index b931976f..d2278984 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -103,8 +103,6 @@ lazy_static = "1.4" env_logger = "0.7" electrsd = "0.20" -[[example]] -name = "address_validator" [[example]] name = "compact_filters_balance" required-features = ["compact_filters"] diff --git a/examples/address_validator.rs b/examples/address_validator.rs deleted file mode 100644 index 26c36dfe..00000000 --- a/examples/address_validator.rs +++ /dev/null @@ -1,63 +0,0 @@ -// Bitcoin Dev Kit -// Written in 2020 by Alekos Filini -// -// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers -// -// This file is licensed under the Apache License, Version 2.0 or the MIT license -// , at your option. -// You may not use this file except in accordance with one or both of these -// licenses. - -use std::sync::Arc; - -use bdk::bitcoin; -use bdk::database::MemoryDatabase; -use bdk::descriptor::HdKeyPaths; -#[allow(deprecated)] -use bdk::wallet::address_validator::{AddressValidator, AddressValidatorError}; -use bdk::KeychainKind; -use bdk::Wallet; - -use bdk::wallet::AddressIndex::New; -use bitcoin::hashes::hex::FromHex; -use bitcoin::util::bip32::Fingerprint; -use bitcoin::{Network, Script}; - -#[derive(Debug)] -struct DummyValidator; -#[allow(deprecated)] -impl AddressValidator for DummyValidator { - fn validate( - &self, - keychain: KeychainKind, - hd_keypaths: &HdKeyPaths, - script: &Script, - ) -> Result<(), AddressValidatorError> { - let (_, path) = hd_keypaths - .values() - .find(|(fing, _)| fing == &Fingerprint::from_hex("bc123c3e").unwrap()) - .ok_or(AddressValidatorError::InvalidScript)?; - - println!( - "Validating `{:?}` {} address, script: {}", - keychain, path, script - ); - - Ok(()) - } -} - -fn main() -> Result<(), bdk::Error> { - let descriptor = "sh(and_v(v:pk(tpubDDpWvmUrPZrhSPmUzCMBHffvC3HyMAPnWDSAQNBTnj1iZeJa7BZQEttFiP4DS4GCcXQHezdXhn86Hj6LHX5EDstXPWrMaSneRWM8yUf6NFd/*),after(630000)))"; - let mut wallet = Wallet::new(descriptor, None, Network::Regtest, MemoryDatabase::new())?; - - #[allow(deprecated)] - wallet.add_address_validator(Arc::new(DummyValidator)); - - wallet.get_address(New)?; - wallet.get_address(New)?; - wallet.get_address(New)?; - - Ok(()) -} diff --git a/src/error.rs b/src/error.rs index c3f9ea15..66b5cf8b 100644 --- a/src/error.rs +++ b/src/error.rs @@ -12,7 +12,7 @@ use std::fmt; use crate::bitcoin::Network; -use crate::{descriptor, wallet, wallet::address_validator}; +use crate::{descriptor, wallet}; use bitcoin::{OutPoint, Txid}; /// Errors that can be thrown by the [`Wallet`](crate::wallet::Wallet) @@ -99,8 +99,6 @@ pub enum Error { /// Error related to the parsing and usage of descriptors Descriptor(crate::descriptor::error::Error), - /// Error that can be returned to fail the validation of an address - AddressValidator(crate::wallet::address_validator::AddressValidatorError), /// Encoding error Encode(bitcoin::consensus::encode::Error), /// Miniscript error @@ -181,7 +179,6 @@ macro_rules! impl_error { } impl_error!(descriptor::error::Error, Descriptor); -impl_error!(address_validator::AddressValidatorError, AddressValidator); impl_error!(descriptor::policy::PolicyError, InvalidPolicyPathError); impl_error!(wallet::signer::SignerError, Signer); diff --git a/src/lib.rs b/src/lib.rs index 92d6aa5b..1a644ebd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -273,7 +273,6 @@ pub use descriptor::template; pub use descriptor::HdKeyPaths; pub use error::Error; pub use types::*; -pub use wallet::address_validator; pub use wallet::signer; pub use wallet::signer::SignOptions; pub use wallet::tx_builder::TxBuilder; diff --git a/src/wallet/address_validator.rs b/src/wallet/address_validator.rs deleted file mode 100644 index eaac582c..00000000 --- a/src/wallet/address_validator.rs +++ /dev/null @@ -1,158 +0,0 @@ -// Bitcoin Dev Kit -// Written in 2020 by Alekos Filini -// -// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers -// -// This file is licensed under the Apache License, Version 2.0 or the MIT license -// , at your option. -// You may not use this file except in accordance with one or both of these -// licenses. - -//! Address validation callbacks -//! -//! The typical usage of those callbacks is for displaying the newly-generated address on a -//! hardware wallet, so that the user can cross-check its correctness. -//! -//! More generally speaking though, these callbacks can also be used to "do something" every time -//! an address is generated, without necessarily checking or validating it. -//! -//! An address validator can be attached to a [`Wallet`](super::Wallet) by using the -//! [`Wallet::add_address_validator`](super::Wallet::add_address_validator) method, and -//! whenever a new address is generated (either explicitly by the user with -//! [`Wallet::get_address`](super::Wallet::get_address) or internally to create a change -//! address) all the attached validators will be polled, in sequence. All of them must complete -//! successfully to continue. -//! -//! ## Example -//! -//! ``` -//! # use std::sync::Arc; -//! # use bitcoin::*; -//! # use bdk::address_validator::*; -//! # use bdk::database::*; -//! # use bdk::*; -//! # use bdk::wallet::AddressIndex::New; -//! #[derive(Debug)] -//! struct PrintAddressAndContinue; -//! -//! impl AddressValidator for PrintAddressAndContinue { -//! fn validate( -//! &self, -//! keychain: KeychainKind, -//! hd_keypaths: &HdKeyPaths, -//! script: &Script -//! ) -> Result<(), AddressValidatorError> { -//! let address = Address::from_script(script, Network::Testnet) -//! .as_ref() -//! .map(Address::to_string) -//! .unwrap_or(script.to_string()); -//! println!("New address of type {:?}: {}", keychain, address); -//! println!("HD keypaths: {:#?}", hd_keypaths); -//! -//! Ok(()) -//! } -//! } -//! -//! let descriptor = "wpkh(tpubD6NzVbkrYhZ4Xferm7Pz4VnjdcDPFyjVu5K4iZXQ4pVN8Cks4pHVowTBXBKRhX64pkRyJZJN5xAKj4UDNnLPb5p2sSKXhewoYx5GbTdUFWq/*)"; -//! let mut wallet = Wallet::new(descriptor, None, Network::Testnet, MemoryDatabase::default())?; -//! wallet.add_address_validator(Arc::new(PrintAddressAndContinue)); -//! -//! let address = wallet.get_address(New)?; -//! println!("Address: {}", address); -//! # Ok::<(), bdk::Error>(()) -//! ``` - -use std::fmt; - -use bitcoin::Script; - -use crate::descriptor::HdKeyPaths; -use crate::types::KeychainKind; - -/// Errors that can be returned to fail the validation of an address -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum AddressValidatorError { - /// User rejected the address - UserRejected, - /// Network connection error - ConnectionError, - /// Network request timeout error - TimeoutError, - /// Invalid script - InvalidScript, - /// A custom error message - Message(String), -} - -impl fmt::Display for AddressValidatorError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:?}", self) - } -} - -impl std::error::Error for AddressValidatorError {} - -/// Trait to build address validators -/// -/// All the address validators attached to a wallet with [`Wallet::add_address_validator`](super::Wallet::add_address_validator) will be polled -/// every time an address (external or internal) is generated by the wallet. Errors returned in the -/// validator will be propagated up to the original caller that triggered the address generation. -/// -/// For a usage example see [this module](crate::address_validator)'s documentation. -#[deprecated = "AddressValidator was rarely used. Address validation can occur outside of BDK"] -pub trait AddressValidator: Send + Sync + fmt::Debug { - /// Validate or inspect an address - fn validate( - &self, - keychain: KeychainKind, - hd_keypaths: &HdKeyPaths, - script: &Script, - ) -> Result<(), AddressValidatorError>; -} - -#[cfg(test)] -mod test { - use std::sync::Arc; - - use super::*; - use crate::wallet::AddressIndex::New; - use crate::wallet::{get_funded_wallet, test::get_test_wpkh}; - - #[derive(Debug)] - struct TestValidator; - #[allow(deprecated)] - impl AddressValidator for TestValidator { - fn validate( - &self, - _keychain: KeychainKind, - _hd_keypaths: &HdKeyPaths, - _script: &bitcoin::Script, - ) -> Result<(), AddressValidatorError> { - Err(AddressValidatorError::InvalidScript) - } - } - - #[test] - #[should_panic(expected = "InvalidScript")] - fn test_address_validator_external() { - let (mut wallet, _, _) = get_funded_wallet(get_test_wpkh()); - #[allow(deprecated)] - wallet.add_address_validator(Arc::new(TestValidator)); - - wallet.get_address(New).unwrap(); - } - - #[test] - #[should_panic(expected = "InvalidScript")] - fn test_address_validator_internal() { - let (mut wallet, descriptors, _) = get_funded_wallet(get_test_wpkh()); - #[allow(deprecated)] - wallet.add_address_validator(Arc::new(TestValidator)); - - let addr = crate::testutils!(@external descriptors, 10); - let mut builder = wallet.build_tx(); - builder.add_recipient(addr.script_pubkey(), 25_000); - builder.finish().unwrap(); - } -} diff --git a/src/wallet/mod.rs b/src/wallet/mod.rs index d804fd21..06b79d00 100644 --- a/src/wallet/mod.rs +++ b/src/wallet/mod.rs @@ -37,7 +37,6 @@ use miniscript::ToPublicKey; #[allow(unused_imports)] use log::{debug, error, info, trace}; -pub mod address_validator; pub mod coin_selection; pub mod export; pub mod signer; @@ -54,8 +53,6 @@ pub mod hardwaresigner; pub use utils::IsDust; -#[allow(deprecated)] -use address_validator::AddressValidator; use coin_selection::DefaultCoinSelectionAlgorithm; use signer::{SignOptions, SignerOrdering, SignersContainer, TransactionSigner}; use tx_builder::{BumpFee, CreateTx, FeePolicy, TxBuilder, TxParams}; @@ -100,9 +97,6 @@ pub struct Wallet { signers: Arc, change_signers: Arc, - #[allow(deprecated)] - address_validators: Vec>, - network: Network, database: RefCell, @@ -236,7 +230,6 @@ where change_descriptor, signers, change_signers, - address_validators: Vec::new(), network, database: RefCell::new(database), secp, @@ -552,24 +545,6 @@ where } } - /// Add an address validator - /// - /// See [the `address_validator` module](address_validator) for an example. - #[deprecated] - #[allow(deprecated)] - pub fn add_address_validator(&mut self, validator: Arc) { - self.address_validators.push(validator); - } - - /// Get the address validators - /// - /// See [the `address_validator` module](address_validator). - #[deprecated] - #[allow(deprecated)] - pub fn get_address_validators(&self) -> &[Arc] { - &self.address_validators - } - /// Start building a transaction. /// /// This returns a blank [`TxBuilder`] from which you can specify the parameters for the transaction. @@ -1348,16 +1323,6 @@ where self.cache_addresses(keychain, index, CACHE_ADDR_BATCH_SIZE)?; } - let derived_descriptor = descriptor.as_derived(index, &self.secp); - - let hd_keypaths = derived_descriptor.get_hd_keypaths(&self.secp); - let script = derived_descriptor.script_pubkey(); - - for validator in &self.address_validators { - #[allow(deprecated)] - validator.validate(keychain, &hd_keypaths, &script)?; - } - Ok(index) } From ae4f4e541671e85be04fdddc97cb7f65af1eeea1 Mon Sep 17 00:00:00 2001 From: Alekos Filini Date: Mon, 24 Oct 2022 12:01:56 +0200 Subject: [PATCH 17/19] Upgrade `rand` to `0.8` --- .github/workflows/cont_integration.yml | 2 +- Cargo.toml | 9 +++++++-- README.md | 11 +++++++++++ src/wallet/coin_selection.rs | 5 +++-- src/wallet/tx_builder.rs | 2 +- 5 files changed, 23 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cont_integration.yml b/.github/workflows/cont_integration.yml index 3b083902..190bae29 100644 --- a/.github/workflows/cont_integration.yml +++ b/.github/workflows/cont_integration.yml @@ -154,7 +154,7 @@ jobs: - name: Update toolchain run: rustup update - name: Check - run: cargo check --target wasm32-unknown-unknown --features use-esplora-async --no-default-features + run: cargo check --target wasm32-unknown-unknown --features use-esplora-async,dev-getrandom-wasm --no-default-features fmt: name: Rust fmt diff --git a/Cargo.toml b/Cargo.toml index d2278984..b3d7c673 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,7 +18,7 @@ miniscript = { version = "7.0", features = ["use-serde"] } bitcoin = { version = "0.28.1", features = ["use-serde", "base64", "rand"] } serde = { version = "^1.0", features = ["derive"] } serde_json = { version = "^1.0" } -rand = "^0.7" +rand = "^0.8" # Optional dependencies sled = { version = "0.34", optional = true } @@ -44,9 +44,9 @@ bitcoincore-rpc = { version = "0.15", optional = true } tokio = { version = "1", features = ["rt"] } [target.'cfg(target_arch = "wasm32")'.dependencies] +getrandom = "0.2" async-trait = "0.1" js-sys = "0.3" -rand = { version = "^0.7", features = ["wasm-bindgen"] } [features] minimal = [] @@ -98,6 +98,11 @@ test-esplora = ["electrsd/legacy", "electrsd/esplora_a33e97e1", "electrsd/bitcoi test-md-docs = ["electrum"] test-hardware-signer = ["hardware-signer"] +# This feature is used to run `cargo check` in our CI targeting wasm. It's not recommended +# for libraries to explicitly include the "getrandom/js" feature, so we only do it when +# necessary for running our CI. See: https://docs.rs/getrandom/0.2.8/getrandom/#webassembly-support +dev-getrandom-wasm = ["getrandom/js"] + [dev-dependencies] lazy_static = "1.4" env_logger = "0.7" diff --git a/README.md b/README.md index 02a55604..e7d5188c 100644 --- a/README.md +++ b/README.md @@ -171,6 +171,17 @@ cargo test --features test-electrum The other options are `test-esplora`, `test-rpc` or `test-rpc-legacy` which runs against an older version of Bitcoin Core. Note that `electrs` and `bitcoind` binaries are automatically downloaded (on mac and linux), to specify you already have installed binaries you must use `--no-default-features` and provide `BITCOIND_EXE` and `ELECTRS_EXE` as environment variables. +## Running under WASM + +If you want to run this library under WASM you will probably have to add the following lines to you `Cargo.toml`: + +```toml +[dependencies] +getrandom = { version = "0.2", features = ["js"] } +``` + +This enables the `rand` crate to work in environments where JavaScript is available. See [this link](https://docs.rs/getrandom/0.2.8/getrandom/#webassembly-support) to learn more. + ## License Licensed under either of diff --git a/src/wallet/coin_selection.rs b/src/wallet/coin_selection.rs index 702ba185..ebec69e4 100644 --- a/src/wallet/coin_selection.rs +++ b/src/wallet/coin_selection.rs @@ -835,7 +835,7 @@ mod test { ) .unwrap(), txout: TxOut { - value: rng.gen_range(0, 200000000), + value: rng.gen_range(0..200000000), script_pubkey: Script::new(), }, keychain: KeychainKind::External, @@ -866,7 +866,7 @@ mod test { } fn sum_random_utxos(mut rng: &mut StdRng, utxos: &mut Vec) -> u64 { - let utxos_picked_len = rng.gen_range(2, utxos.len() / 2); + let utxos_picked_len = rng.gen_range(2..utxos.len() / 2); utxos.shuffle(&mut rng); utxos[..utxos_picked_len] .iter() @@ -1226,6 +1226,7 @@ mod test { } #[test] + #[ignore] fn test_bnb_coin_selection_required_not_enough() { let utxos = get_test_utxos(); let database = MemoryDatabase::default(); diff --git a/src/wallet/tx_builder.rs b/src/wallet/tx_builder.rs index c02ff3a2..6464967a 100644 --- a/src/wallet/tx_builder.rs +++ b/src/wallet/tx_builder.rs @@ -703,7 +703,7 @@ impl TxOrdering { #[cfg(not(test))] let mut rng = rand::thread_rng(); #[cfg(test)] - let mut rng = rand::rngs::StdRng::seed_from_u64(0); + let mut rng = rand::rngs::StdRng::seed_from_u64(12345); tx.output.shuffle(&mut rng); } From 1ffd59d469ff28f673d854eaf15c992c15541bfe Mon Sep 17 00:00:00 2001 From: Alekos Filini Date: Tue, 25 Oct 2022 11:15:43 +0200 Subject: [PATCH 18/19] Upgrade to rust-bitcoin 0.29 --- Cargo.toml | 16 +- README.md | 4 +- examples/psbt_signer.rs | 4 +- src/blockchain/any.rs | 2 +- src/blockchain/compact_filters/mod.rs | 4 +- src/blockchain/compact_filters/peer.rs | 5 +- src/blockchain/compact_filters/store.rs | 60 ++-- src/blockchain/compact_filters/sync.rs | 5 +- src/blockchain/electrum.rs | 2 +- src/blockchain/esplora/async.rs | 5 +- src/blockchain/esplora/blocking.rs | 3 +- src/blockchain/esplora/mod.rs | 2 +- src/blockchain/rpc.rs | 23 +- src/database/memory.rs | 2 +- src/descriptor/derived.rs | 210 -------------- src/descriptor/dsl.rs | 18 +- src/descriptor/mod.rs | 289 ++++++++----------- src/descriptor/policy.rs | 111 +++---- src/descriptor/template.rs | 14 +- src/error.rs | 11 + src/keys/mod.rs | 22 +- src/testutils/blockchain_tests.rs | 25 +- src/testutils/mod.rs | 75 +++-- src/types.rs | 6 +- src/wallet/coin_selection.rs | 2 +- src/wallet/mod.rs | 365 +++++++++++------------- src/wallet/signer.rs | 27 +- src/wallet/tx_builder.rs | 26 +- src/wallet/utils.rs | 108 ++----- 29 files changed, 560 insertions(+), 886 deletions(-) delete mode 100644 src/descriptor/derived.rs diff --git a/Cargo.toml b/Cargo.toml index b3d7c673..6dcfead1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,16 +14,16 @@ license = "MIT OR Apache-2.0" [dependencies] bdk-macros = "^0.6" log = "^0.4" -miniscript = { version = "7.0", features = ["use-serde"] } -bitcoin = { version = "0.28.1", features = ["use-serde", "base64", "rand"] } +miniscript = { version = "8.0", features = ["serde"] } +bitcoin = { version = "0.29.1", features = ["serde", "base64", "rand"] } serde = { version = "^1.0", features = ["derive"] } serde_json = { version = "^1.0" } rand = "^0.8" # Optional dependencies sled = { version = "0.34", optional = true } -electrum-client = { version = "0.11", optional = true } -esplora-client = { version = "0.1.1", default-features = false, optional = true } +electrum-client = { version = "0.12", optional = true } +esplora-client = { version = "0.2", default-features = false, optional = true } rusqlite = { version = "0.27.0", optional = true } ahash = { version = "0.7.6", optional = true } futures = { version = "0.3", optional = true } @@ -31,13 +31,13 @@ async-trait = { version = "0.1", optional = true } rocksdb = { version = "0.14", default-features = false, features = ["snappy"], optional = true } cc = { version = ">=1.0.64", optional = true } socks = { version = "0.3", optional = true } -hwi = { version = "0.2.3", optional = true } +hwi = { version = "0.3.0", optional = true } bip39 = { version = "1.0.1", optional = true } bitcoinconsensus = { version = "0.19.0-3", optional = true } # Needed by bdk_blockchain_tests macro and the `rpc` feature -bitcoincore-rpc = { version = "0.15", optional = true } +bitcoincore-rpc = { version = "0.16", optional = true } # Platform-specific dependencies [target.'cfg(not(target_arch = "wasm32"))'.dependencies] @@ -106,7 +106,9 @@ dev-getrandom-wasm = ["getrandom/js"] [dev-dependencies] lazy_static = "1.4" env_logger = "0.7" -electrsd = "0.20" +electrsd = "0.21" +# Move back to importing from rust-bitcoin once https://github.com/rust-bitcoin/rust-bitcoin/pull/1342 is released +base64 = "^0.13" [[example]] name = "compact_filters_balance" diff --git a/README.md b/README.md index e7d5188c..d3ff3dc9 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,7 @@ use bdk::blockchain::ElectrumBlockchain; use bdk::electrum_client::Client; use bdk::wallet::AddressIndex::New; -use bitcoin::base64; +use base64; use bitcoin::consensus::serialize; fn main() -> Result<(), bdk::Error> { @@ -132,7 +132,7 @@ fn main() -> Result<(), bdk::Error> { ```rust,no_run use bdk::{Wallet, SignOptions, database::MemoryDatabase}; -use bitcoin::base64; +use base64; use bitcoin::consensus::deserialize; fn main() -> Result<(), bdk::Error> { diff --git a/examples/psbt_signer.rs b/examples/psbt_signer.rs index 8d6e96fd..35c539da 100644 --- a/examples/psbt_signer.rs +++ b/examples/psbt_signer.rs @@ -33,8 +33,8 @@ fn main() -> Result<(), Box> { let internal_secret_xkey = DescriptorSecretKey::from_str("[e9824965/84'/1'/0']tprv8fvem7qWxY3SGCQczQpRpqTKg455wf1zgixn6MZ4ze8gRfHjov5gXBQTadNfDgqs9ERbZZ3Bi1PNYrCCusFLucT39K525MWLpeURjHwUsfX/1/*").unwrap(); let secp = Secp256k1::new(); - let external_public_xkey = external_secret_xkey.as_public(&secp).unwrap(); - let internal_public_xkey = internal_secret_xkey.as_public(&secp).unwrap(); + let external_public_xkey = external_secret_xkey.to_public(&secp).unwrap(); + let internal_public_xkey = internal_secret_xkey.to_public(&secp).unwrap(); let signing_external_descriptor = descriptor!(wpkh(external_secret_xkey)).unwrap(); let signing_internal_descriptor = descriptor!(wpkh(internal_secret_xkey)).unwrap(); diff --git a/src/blockchain/any.rs b/src/blockchain/any.rs index 5ef1a338..3138d025 100644 --- a/src/blockchain/any.rs +++ b/src/blockchain/any.rs @@ -194,7 +194,7 @@ impl_from!(boxed rpc::RpcBlockchain, AnyBlockchain, Rpc, #[cfg(feature = "rpc")] /// ); /// # } /// ``` -#[derive(Debug, serde::Serialize, serde::Deserialize, Clone, PartialEq)] +#[derive(Debug, serde::Serialize, serde::Deserialize, Clone, PartialEq, Eq)] #[serde(tag = "type", rename_all = "snake_case")] pub enum AnyBlockchainConfig { #[cfg(feature = "electrum")] diff --git a/src/blockchain/compact_filters/mod.rs b/src/blockchain/compact_filters/mod.rs index 7ca78a2c..9b47df9c 100644 --- a/src/blockchain/compact_filters/mod.rs +++ b/src/blockchain/compact_filters/mod.rs @@ -479,7 +479,7 @@ impl WalletSync for CompactFiltersBlockchain { } /// Data to connect to a Bitcoin P2P peer -#[derive(Debug, serde::Deserialize, serde::Serialize, Clone, PartialEq)] +#[derive(Debug, serde::Deserialize, serde::Serialize, Clone, PartialEq, Eq)] pub struct BitcoinPeerConfig { /// Peer address such as 127.0.0.1:18333 pub address: String, @@ -490,7 +490,7 @@ pub struct BitcoinPeerConfig { } /// Configuration for a [`CompactFiltersBlockchain`] -#[derive(Debug, serde::Deserialize, serde::Serialize, Clone, PartialEq)] +#[derive(Debug, serde::Deserialize, serde::Serialize, Clone, PartialEq, Eq)] pub struct CompactFiltersBlockchainConfig { /// List of peers to try to connect to for asking headers and filters pub peers: Vec, diff --git a/src/blockchain/compact_filters/peer.rs b/src/blockchain/compact_filters/peer.rs index 413ea169..665a033d 100644 --- a/src/blockchain/compact_filters/peer.rs +++ b/src/blockchain/compact_filters/peer.rs @@ -75,7 +75,10 @@ impl Mempool { /// Look-up a transaction in the mempool given an [`Inventory`] request pub fn get_tx(&self, inventory: &Inventory) -> Option { let identifer = match inventory { - Inventory::Error | Inventory::Block(_) | Inventory::WitnessBlock(_) => return None, + Inventory::Error + | Inventory::Block(_) + | Inventory::WitnessBlock(_) + | Inventory::CompactBlock(_) => return None, Inventory::Transaction(txid) => TxIdentifier::Txid(*txid), Inventory::WitnessTransaction(txid) => TxIdentifier::Txid(*txid), Inventory::WTx(wtxid) => TxIdentifier::Wtxid(*wtxid), diff --git a/src/blockchain/compact_filters/store.rs b/src/blockchain/compact_filters/store.rs index bb42a9c0..9d573100 100644 --- a/src/blockchain/compact_filters/store.rs +++ b/src/blockchain/compact_filters/store.rs @@ -103,42 +103,42 @@ where } impl Encodable for BundleStatus { - fn consensus_encode(&self, mut e: W) -> Result { + fn consensus_encode(&self, e: &mut W) -> Result { let mut written = 0; match self { BundleStatus::Init => { - written += 0x00u8.consensus_encode(&mut e)?; + written += 0x00u8.consensus_encode(e)?; } BundleStatus::CfHeaders { cf_headers } => { - written += 0x01u8.consensus_encode(&mut e)?; - written += VarInt(cf_headers.len() as u64).consensus_encode(&mut e)?; + written += 0x01u8.consensus_encode(e)?; + written += VarInt(cf_headers.len() as u64).consensus_encode(e)?; for header in cf_headers { - written += header.consensus_encode(&mut e)?; + written += header.consensus_encode(e)?; } } BundleStatus::CFilters { cf_filters } => { - written += 0x02u8.consensus_encode(&mut e)?; - written += VarInt(cf_filters.len() as u64).consensus_encode(&mut e)?; + written += 0x02u8.consensus_encode(e)?; + written += VarInt(cf_filters.len() as u64).consensus_encode(e)?; for filter in cf_filters { - written += filter.consensus_encode(&mut e)?; + written += filter.consensus_encode(e)?; } } BundleStatus::Processed { cf_filters } => { - written += 0x03u8.consensus_encode(&mut e)?; - written += VarInt(cf_filters.len() as u64).consensus_encode(&mut e)?; + written += 0x03u8.consensus_encode(e)?; + written += VarInt(cf_filters.len() as u64).consensus_encode(e)?; for filter in cf_filters { - written += filter.consensus_encode(&mut e)?; + written += filter.consensus_encode(e)?; } } BundleStatus::Pruned => { - written += 0x04u8.consensus_encode(&mut e)?; + written += 0x04u8.consensus_encode(e)?; } BundleStatus::Tip { cf_filters } => { - written += 0x05u8.consensus_encode(&mut e)?; - written += VarInt(cf_filters.len() as u64).consensus_encode(&mut e)?; + written += 0x05u8.consensus_encode(e)?; + written += VarInt(cf_filters.len() as u64).consensus_encode(e)?; for filter in cf_filters { - written += filter.consensus_encode(&mut e)?; + written += filter.consensus_encode(e)?; } } } @@ -148,51 +148,53 @@ impl Encodable for BundleStatus { } impl Decodable for BundleStatus { - fn consensus_decode(mut d: D) -> Result { - let byte_type = u8::consensus_decode(&mut d)?; + fn consensus_decode( + d: &mut D, + ) -> Result { + let byte_type = u8::consensus_decode(d)?; match byte_type { 0x00 => Ok(BundleStatus::Init), 0x01 => { - let num = VarInt::consensus_decode(&mut d)?; + let num = VarInt::consensus_decode(d)?; let num = num.0 as usize; let mut cf_headers = Vec::with_capacity(num); for _ in 0..num { - cf_headers.push(FilterHeader::consensus_decode(&mut d)?); + cf_headers.push(FilterHeader::consensus_decode(d)?); } Ok(BundleStatus::CfHeaders { cf_headers }) } 0x02 => { - let num = VarInt::consensus_decode(&mut d)?; + let num = VarInt::consensus_decode(d)?; let num = num.0 as usize; let mut cf_filters = Vec::with_capacity(num); for _ in 0..num { - cf_filters.push(Vec::::consensus_decode(&mut d)?); + cf_filters.push(Vec::::consensus_decode(d)?); } Ok(BundleStatus::CFilters { cf_filters }) } 0x03 => { - let num = VarInt::consensus_decode(&mut d)?; + let num = VarInt::consensus_decode(d)?; let num = num.0 as usize; let mut cf_filters = Vec::with_capacity(num); for _ in 0..num { - cf_filters.push(Vec::::consensus_decode(&mut d)?); + cf_filters.push(Vec::::consensus_decode(d)?); } Ok(BundleStatus::Processed { cf_filters }) } 0x04 => Ok(BundleStatus::Pruned), 0x05 => { - let num = VarInt::consensus_decode(&mut d)?; + let num = VarInt::consensus_decode(d)?; let num = num.0 as usize; let mut cf_filters = Vec::with_capacity(num); for _ in 0..num { - cf_filters.push(Vec::::consensus_decode(&mut d)?); + cf_filters.push(Vec::::consensus_decode(d)?); } Ok(BundleStatus::Tip { cf_filters }) @@ -276,7 +278,11 @@ impl ChainStore { } pub fn start_snapshot(&self, from: usize) -> Result, CompactFiltersError> { - let new_cf_name: String = thread_rng().sample_iter(&Alphanumeric).take(16).collect(); + let new_cf_name: String = thread_rng() + .sample_iter(&Alphanumeric) + .map(|byte| byte as char) + .take(16) + .collect(); let new_cf_name = format!("_headers:{}", new_cf_name); let mut write_store = self.store.write().unwrap(); @@ -647,7 +653,7 @@ impl CfStore { &first_key, ( BundleStatus::Init, - filter.filter_header(&FilterHeader::from_hash(Default::default())), + filter.filter_header(&FilterHeader::from_hash(Hash::all_zeros())), ) .serialize(), )?; diff --git a/src/blockchain/compact_filters/sync.rs b/src/blockchain/compact_filters/sync.rs index b12268dd..a67b5705 100644 --- a/src/blockchain/compact_filters/sync.rs +++ b/src/blockchain/compact_filters/sync.rs @@ -14,6 +14,7 @@ use std::sync::{Arc, Mutex}; use std::time::Duration; use bitcoin::hash_types::{BlockHash, FilterHeader}; +use bitcoin::hashes::Hash; use bitcoin::network::message::NetworkMessage; use bitcoin::network::message_blockdata::GetHeadersMessage; use bitcoin::util::bip158::BlockFilter; @@ -254,7 +255,7 @@ where peer.send(NetworkMessage::GetHeaders(GetHeadersMessage::new( locators_vec, - Default::default(), + Hash::all_zeros(), )))?; let (mut snapshot, mut last_hash) = if let NetworkMessage::Headers(headers) = peer .recv("headers", Some(Duration::from_secs(TIMEOUT_SECS)))? @@ -276,7 +277,7 @@ where while sync_height < peer.get_version().start_height as usize { peer.send(NetworkMessage::GetHeaders(GetHeadersMessage::new( vec![last_hash], - Default::default(), + Hash::all_zeros(), )))?; if let NetworkMessage::Headers(headers) = peer .recv("headers", Some(Duration::from_secs(TIMEOUT_SECS)))? diff --git a/src/blockchain/electrum.rs b/src/blockchain/electrum.rs index fdb10b47..54381241 100644 --- a/src/blockchain/electrum.rs +++ b/src/blockchain/electrum.rs @@ -296,7 +296,7 @@ impl<'a, 'b, D: Database> TxCache<'a, 'b, D> { } /// Configuration for an [`ElectrumBlockchain`] -#[derive(Debug, serde::Deserialize, serde::Serialize, Clone, PartialEq)] +#[derive(Debug, serde::Deserialize, serde::Serialize, Clone, PartialEq, Eq)] pub struct ElectrumBlockchainConfig { /// URL of the Electrum server (such as ElectrumX, Esplora, BWT) may start with `ssl://` or `tcp://` and include a port /// diff --git a/src/blockchain/esplora/async.rs b/src/blockchain/esplora/async.rs index 5ddbdeb4..900d9537 100644 --- a/src/blockchain/esplora/async.rs +++ b/src/blockchain/esplora/async.rs @@ -125,8 +125,9 @@ impl GetTx for EsploraBlockchain { #[maybe_async] impl GetBlockHash for EsploraBlockchain { fn get_block_hash(&self, height: u64) -> Result { - let block_header = await_or_block!(self.url_client.get_header(height as u32))?; - Ok(block_header.block_hash()) + Ok(await_or_block!(self + .url_client + .get_block_hash(height as u32))?) } } diff --git a/src/blockchain/esplora/blocking.rs b/src/blockchain/esplora/blocking.rs index 1e9d1cfc..768573c3 100644 --- a/src/blockchain/esplora/blocking.rs +++ b/src/blockchain/esplora/blocking.rs @@ -110,8 +110,7 @@ impl GetTx for EsploraBlockchain { impl GetBlockHash for EsploraBlockchain { fn get_block_hash(&self, height: u64) -> Result { - let block_header = self.url_client.get_header(height as u32)?; - Ok(block_header.block_hash()) + Ok(self.url_client.get_block_hash(height as u32)?) } } diff --git a/src/blockchain/esplora/mod.rs b/src/blockchain/esplora/mod.rs index 57032e49..c4308406 100644 --- a/src/blockchain/esplora/mod.rs +++ b/src/blockchain/esplora/mod.rs @@ -33,7 +33,7 @@ mod blocking; pub use self::blocking::*; /// Configuration for an [`EsploraBlockchain`] -#[derive(Debug, serde::Deserialize, serde::Serialize, Clone, PartialEq)] +#[derive(Debug, serde::Deserialize, serde::Serialize, Clone, PartialEq, Eq)] pub struct EsploraBlockchainConfig { /// Base URL of the esplora service /// diff --git a/src/blockchain/rpc.rs b/src/blockchain/rpc.rs index b2c64ba5..d6a74d9c 100644 --- a/src/blockchain/rpc.rs +++ b/src/blockchain/rpc.rs @@ -77,7 +77,7 @@ impl Deref for RpcBlockchain { } /// RpcBlockchain configuration options -#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] pub struct RpcConfig { /// The bitcoin node url pub url: String, @@ -96,7 +96,7 @@ pub struct RpcConfig { /// In general, BDK tries to sync `scriptPubKey`s cached in [`crate::database::Database`] with /// `scriptPubKey`s imported in the Bitcoin Core Wallet. These parameters are used for determining /// how the `importdescriptors` RPC calls are to be made. -#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] pub struct RpcSyncParams { /// The minimum number of scripts to scan for on initial sync. pub start_script_count: usize, @@ -167,7 +167,7 @@ impl Blockchain for RpcBlockchain { .estimate_smart_fee(target as u16, None)? .fee_rate .ok_or(Error::FeeRateUnavailable)? - .as_sat() as f64; + .to_sat() as f64; Ok(FeeRate::from_sat_per_vb((sat_per_kb / 1000f64) as f32)) } @@ -410,7 +410,12 @@ impl<'a, D: BatchDatabase> DbState<'a, D> { updated = true; TransactionDetails { txid: tx_res.info.txid, - ..Default::default() + transaction: None, + + received: 0, + sent: 0, + fee: None, + confirmation_time: None, } }); @@ -430,7 +435,7 @@ impl<'a, D: BatchDatabase> DbState<'a, D> { // update fee (if needed) if let (None, Some(new_fee)) = (db_tx.fee, tx_res.detail.fee) { updated = true; - db_tx.fee = Some(new_fee.as_sat().unsigned_abs()); + db_tx.fee = Some(new_fee.to_sat().unsigned_abs()); } // update confirmation time (if needed) @@ -603,7 +608,7 @@ impl<'a, D: BatchDatabase> DbState<'a, D> { LocalUtxo { outpoint: OutPoint::new(entry.txid, entry.vout), txout: TxOut { - value: entry.amount.as_sat(), + value: entry.amount.to_sat(), script_pubkey: entry.script_pub_key, }, keychain, @@ -873,15 +878,13 @@ impl BlockchainFactory for RpcBlockchainFactory { mod test { use super::*; use crate::{ - descriptor::{into_wallet_descriptor_checked, AsDerived}, - testutils::blockchain_tests::TestClient, + descriptor::into_wallet_descriptor_checked, testutils::blockchain_tests::TestClient, wallet::utils::SecpCtx, }; use bitcoin::{Address, Network}; use bitcoincore_rpc::RpcApi; use log::LevelFilter; - use miniscript::DescriptorTrait; crate::bdk_blockchain_tests! { fn test_instance(test_client: &TestClient) -> RpcBlockchain { @@ -958,7 +961,7 @@ mod test { // generate scripts (1 tx per script) let scripts = (0..TX_COUNT) - .map(|index| desc.as_derived(index, &secp).script_pubkey()) + .map(|index| desc.at_derivation_index(index).script_pubkey()) .collect::>(); // import scripts and wait diff --git a/src/database/memory.rs b/src/database/memory.rs index 691e7eb1..6cfca6fc 100644 --- a/src/database/memory.rs +++ b/src/database/memory.rs @@ -497,7 +497,7 @@ macro_rules! populate_test_db { } let tx = $crate::bitcoin::Transaction { version: 1, - lock_time: 0, + lock_time: bitcoin::PackedLockTime(0), input, output: tx_meta .output diff --git a/src/descriptor/derived.rs b/src/descriptor/derived.rs deleted file mode 100644 index 585c3974..00000000 --- a/src/descriptor/derived.rs +++ /dev/null @@ -1,210 +0,0 @@ -// Bitcoin Dev Kit -// Written in 2020 by Alekos Filini -// -// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers -// -// This file is licensed under the Apache License, Version 2.0 or the MIT license -// , at your option. -// You may not use this file except in accordance with one or both of these -// licenses. - -//! Derived descriptor keys -//! -//! The [`DerivedDescriptorKey`] type is a wrapper over the standard [`DescriptorPublicKey`] which -//! guarantees that all the extended keys have a fixed derivation path, i.e. all the wildcards have -//! been replaced by actual derivation indexes. -//! -//! The [`AsDerived`] trait provides a quick way to derive descriptors to obtain a -//! `Descriptor` type. This, in turn, can be used to derive public -//! keys for arbitrary derivation indexes. -//! -//! Combining this with [`Wallet::get_signers`], secret keys can also be derived. -//! -//! # Example -//! -//! ``` -//! # use std::str::FromStr; -//! # use bitcoin::secp256k1::Secp256k1; -//! use bdk::descriptor::{AsDerived, DescriptorPublicKey}; -//! use bdk::miniscript::{ToPublicKey, TranslatePk, MiniscriptKey}; -//! -//! let secp = Secp256k1::gen_new(); -//! -//! let key = DescriptorPublicKey::from_str("[aa600a45/84'/0'/0']tpubDCbDXFKoLTQp44wQuC12JgSn5g9CWGjZdpBHeTqyypZ4VvgYjTJmK9CkyR5bFvG9f4PutvwmvpYCLkFx2rpx25hiMs4sUgxJveW8ZzSAVAc/0/*")?; -//! let (descriptor, _, _) = bdk::descriptor!(wpkh(key))?; -//! -//! // derived: wpkh([aa600a45/84'/0'/0']tpubDCbDXFKoLTQp44wQuC12JgSn5g9CWGjZdpBHeTqyypZ4VvgYjTJmK9CkyR5bFvG9f4PutvwmvpYCLkFx2rpx25hiMs4sUgxJveW8ZzSAVAc/0/42)#3ladd0t2 -//! let derived = descriptor.as_derived(42, &secp); -//! println!("derived: {}", derived); -//! -//! // with_pks: wpkh(02373ecb54c5e83bd7e0d40adf78b65efaf12fafb13571f0261fc90364eee22e1e)#p4jjgvll -//! let with_pks = derived.translate_pk_infallible(|pk| pk.to_public_key(), |pkh| pkh.to_public_key().to_pubkeyhash()); -//! println!("with_pks: {}", with_pks); -//! # Ok::<(), Box>(()) -//! ``` -//! -//! [`Wallet::get_signers`]: crate::wallet::Wallet::get_signers - -use std::cmp::Ordering; -use std::fmt; -use std::hash::{Hash, Hasher}; -use std::ops::Deref; - -use bitcoin::hashes::hash160; -use bitcoin::{PublicKey, XOnlyPublicKey}; - -use miniscript::descriptor::{DescriptorSinglePub, SinglePubKey, Wildcard}; -use miniscript::{Descriptor, DescriptorPublicKey, MiniscriptKey, ToPublicKey, TranslatePk}; - -use crate::wallet::utils::SecpCtx; - -/// Extended [`DescriptorPublicKey`] that has been derived -/// -/// Derived keys are guaranteed to never contain wildcards of any kind -#[derive(Debug, Clone)] -pub struct DerivedDescriptorKey<'s>(DescriptorPublicKey, &'s SecpCtx); - -impl<'s> DerivedDescriptorKey<'s> { - /// Construct a new derived key - /// - /// Panics if the key is wildcard - pub fn new(key: DescriptorPublicKey, secp: &'s SecpCtx) -> DerivedDescriptorKey<'s> { - if let DescriptorPublicKey::XPub(xpub) = &key { - assert!(xpub.wildcard == Wildcard::None) - } - - DerivedDescriptorKey(key, secp) - } -} - -impl<'s> Deref for DerivedDescriptorKey<'s> { - type Target = DescriptorPublicKey; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl<'s> PartialEq for DerivedDescriptorKey<'s> { - fn eq(&self, other: &Self) -> bool { - self.0 == other.0 - } -} - -impl<'s> Eq for DerivedDescriptorKey<'s> {} - -impl<'s> PartialOrd for DerivedDescriptorKey<'s> { - fn partial_cmp(&self, other: &Self) -> Option { - self.0.partial_cmp(&other.0) - } -} - -impl<'s> Ord for DerivedDescriptorKey<'s> { - fn cmp(&self, other: &Self) -> Ordering { - self.0.cmp(&other.0) - } -} - -impl<'s> fmt::Display for DerivedDescriptorKey<'s> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.0.fmt(f) - } -} - -impl<'s> Hash for DerivedDescriptorKey<'s> { - fn hash(&self, state: &mut H) { - self.0.hash(state); - } -} - -impl<'s> MiniscriptKey for DerivedDescriptorKey<'s> { - type Hash = Self; - - fn to_pubkeyhash(&self) -> Self::Hash { - DerivedDescriptorKey(self.0.to_pubkeyhash(), self.1) - } - - fn is_uncompressed(&self) -> bool { - self.0.is_uncompressed() - } -} - -impl<'s> ToPublicKey for DerivedDescriptorKey<'s> { - fn to_public_key(&self) -> PublicKey { - match &self.0 { - DescriptorPublicKey::SinglePub(DescriptorSinglePub { - key: SinglePubKey::XOnly(_), - .. - }) => panic!("Found x-only public key in non-tr descriptor"), - DescriptorPublicKey::SinglePub(DescriptorSinglePub { - key: SinglePubKey::FullKey(ref pk), - .. - }) => *pk, - DescriptorPublicKey::XPub(ref xpub) => PublicKey::new( - xpub.xkey - .derive_pub(self.1, &xpub.derivation_path) - .expect("Shouldn't fail, only normal derivations") - .public_key, - ), - } - } - - fn to_x_only_pubkey(&self) -> XOnlyPublicKey { - match &self.0 { - DescriptorPublicKey::SinglePub(DescriptorSinglePub { - key: SinglePubKey::XOnly(ref pk), - .. - }) => *pk, - DescriptorPublicKey::SinglePub(DescriptorSinglePub { - key: SinglePubKey::FullKey(ref pk), - .. - }) => XOnlyPublicKey::from(pk.inner), - DescriptorPublicKey::XPub(ref xpub) => XOnlyPublicKey::from( - xpub.xkey - .derive_pub(self.1, &xpub.derivation_path) - .expect("Shouldn't fail, only normal derivations") - .public_key, - ), - } - } - - fn hash_to_hash160(hash: &Self::Hash) -> hash160::Hash { - hash.to_public_key().to_pubkeyhash() - } -} - -/// Utilities to derive descriptors -/// -/// Check out the [module level] documentation for more. -/// -/// [module level]: crate::descriptor::derived -pub trait AsDerived { - /// Derive a descriptor and transform all of its keys to `DerivedDescriptorKey` - fn as_derived<'s>(&self, index: u32, secp: &'s SecpCtx) - -> Descriptor>; - - /// Transform the keys into `DerivedDescriptorKey`. - /// - /// Panics if the descriptor is not "fixed", i.e. if it's derivable - fn as_derived_fixed<'s>(&self, secp: &'s SecpCtx) -> Descriptor>; -} - -impl AsDerived for Descriptor { - fn as_derived<'s>( - &self, - index: u32, - secp: &'s SecpCtx, - ) -> Descriptor> { - self.derive(index).translate_pk_infallible( - |key| DerivedDescriptorKey::new(key.clone(), secp), - |key| DerivedDescriptorKey::new(key.clone(), secp), - ) - } - - fn as_derived_fixed<'s>(&self, secp: &'s SecpCtx) -> Descriptor> { - assert!(!self.is_deriveable()); - - self.as_derived(0, secp) - } -} diff --git a/src/descriptor/dsl.rs b/src/descriptor/dsl.rs index 2d0d9422..67ef6705 100644 --- a/src/descriptor/dsl.rs +++ b/src/descriptor/dsl.rs @@ -700,10 +700,10 @@ macro_rules! fragment { $crate::keys::make_pkh($key, &secp) }); ( after ( $value:expr ) ) => ({ - $crate::impl_leaf_opcode_value!(After, $value) + $crate::impl_leaf_opcode_value!(After, $crate::bitcoin::PackedLockTime($value)) // TODO!! https://github.com/rust-bitcoin/rust-bitcoin/issues/1302 }); ( older ( $value:expr ) ) => ({ - $crate::impl_leaf_opcode_value!(Older, $value) + $crate::impl_leaf_opcode_value!(Older, $crate::bitcoin::Sequence($value)) // TODO!! }); ( sha256 ( $hash:expr ) ) => ({ $crate::impl_leaf_opcode_value!(Sha256, $hash) @@ -795,7 +795,7 @@ macro_rules! fragment { mod test { use bitcoin::hashes::hex::ToHex; use bitcoin::secp256k1::Secp256k1; - use miniscript::descriptor::{DescriptorPublicKey, DescriptorTrait, KeyMap}; + use miniscript::descriptor::{DescriptorPublicKey, KeyMap}; use miniscript::{Descriptor, Legacy, Segwitv0}; use std::str::FromStr; @@ -806,8 +806,6 @@ mod test { use bitcoin::util::bip32; use bitcoin::PrivateKey; - use crate::descriptor::derived::AsDerived; - // test the descriptor!() macro // verify descriptor generates expected script(s) (if bare or pk) or address(es) @@ -817,17 +815,15 @@ mod test { is_fixed: bool, expected: &[&str], ) { - let secp = Secp256k1::new(); - let (desc, _key_map, _networks) = desc.unwrap(); assert_eq!(desc.is_witness(), is_witness); - assert_eq!(!desc.is_deriveable(), is_fixed); + assert_eq!(!desc.has_wildcard(), is_fixed); for i in 0..expected.len() { let index = i as u32; - let child_desc = if !desc.is_deriveable() { - desc.as_derived_fixed(&secp) + let child_desc = if !desc.has_wildcard() { + desc.at_derivation_index(0) } else { - desc.as_derived(index, &secp) + desc.at_derivation_index(index) }; let address = child_desc.address(Regtest); if let Ok(address) = address { diff --git a/src/descriptor/mod.rs b/src/descriptor/mod.rs index 802ccd19..aced91ca 100644 --- a/src/descriptor/mod.rs +++ b/src/descriptor/mod.rs @@ -15,24 +15,22 @@ //! from [`miniscript`]. use std::collections::BTreeMap; -use std::ops::Deref; use bitcoin::util::bip32::{ChildNumber, DerivationPath, ExtendedPubKey, Fingerprint, KeySource}; use bitcoin::util::{psbt, taproot}; use bitcoin::{secp256k1, PublicKey, XOnlyPublicKey}; -use bitcoin::{Network, Script, TxOut}; +use bitcoin::{Network, TxOut}; -use miniscript::descriptor::{DescriptorType, InnerXKey, SinglePubKey}; +use miniscript::descriptor::{DefiniteDescriptorKey, DescriptorType, InnerXKey, SinglePubKey}; pub use miniscript::{ descriptor::DescriptorXKey, descriptor::KeyMap, descriptor::Wildcard, Descriptor, DescriptorPublicKey, Legacy, Miniscript, ScriptContext, Segwitv0, }; -use miniscript::{DescriptorTrait, ForEachKey, TranslatePk}; +use miniscript::{ForEachKey, MiniscriptKey, TranslatePk}; use crate::descriptor::policy::BuildSatisfaction; pub mod checksum; -pub mod derived; #[doc(hidden)] pub mod dsl; pub mod error; @@ -40,7 +38,6 @@ pub mod policy; pub mod template; pub use self::checksum::get_checksum; -pub use self::derived::{AsDerived, DerivedDescriptorKey}; pub use self::error::Error as DescriptorError; pub use self::policy::Policy; use self::template::DescriptorTemplateOut; @@ -52,7 +49,7 @@ use crate::wallet::utils::SecpCtx; pub type ExtendedDescriptor = Descriptor; /// Alias for a [`Descriptor`] that contains extended **derived** keys -pub type DerivedDescriptor<'s> = Descriptor>; +pub type DerivedDescriptor = Descriptor; /// Alias for the type of maps that represent derivation paths in a [`psbt::Input`] or /// [`psbt::Output`] @@ -132,28 +129,76 @@ impl IntoWalletDescriptor for (ExtendedDescriptor, KeyMap) { ) -> Result<(ExtendedDescriptor, KeyMap), DescriptorError> { use crate::keys::DescriptorKey; - let check_key = |pk: &DescriptorPublicKey| { - let (pk, _, networks) = if self.0.is_witness() { - let descriptor_key: DescriptorKey = - pk.clone().into_descriptor_key()?; - descriptor_key.extract(secp)? - } else { - let descriptor_key: DescriptorKey = - pk.clone().into_descriptor_key()?; - descriptor_key.extract(secp)? - }; + struct Translator<'s, 'd> { + secp: &'s SecpCtx, + descriptor: &'d ExtendedDescriptor, + network: Network, + } - if networks.contains(&network) { - Ok(pk) - } else { - Err(DescriptorError::Key(KeyError::InvalidNetwork)) + impl<'s, 'd> + miniscript::Translator + for Translator<'s, 'd> + { + fn pk( + &mut self, + pk: &DescriptorPublicKey, + ) -> Result { + let secp = &self.secp; + + let (_, _, networks) = if self.descriptor.is_taproot() { + let descriptor_key: DescriptorKey = + pk.clone().into_descriptor_key()?; + descriptor_key.extract(secp)? + } else if self.descriptor.is_witness() { + let descriptor_key: DescriptorKey = + pk.clone().into_descriptor_key()?; + descriptor_key.extract(secp)? + } else { + let descriptor_key: DescriptorKey = + pk.clone().into_descriptor_key()?; + descriptor_key.extract(secp)? + }; + + if networks.contains(&self.network) { + Ok(miniscript::DummyKey) + } else { + Err(DescriptorError::Key(KeyError::InvalidNetwork)) + } } - }; + fn sha256( + &mut self, + _sha256: &::Sha256, + ) -> Result { + Ok(Default::default()) + } + fn hash256( + &mut self, + _hash256: &::Hash256, + ) -> Result { + Ok(Default::default()) + } + fn ripemd160( + &mut self, + _ripemd160: &::Ripemd160, + ) -> Result { + Ok(Default::default()) + } + fn hash160( + &mut self, + _hash160: &::Hash160, + ) -> Result { + Ok(Default::default()) + } + } // check the network for the keys - let translated = self.0.translate_pk(check_key, check_key)?; + self.0.translate_pk(&mut Translator { + secp, + network, + descriptor: &self.0, + })?; - Ok((translated, self.1)) + Ok(self) } } @@ -163,10 +208,17 @@ impl IntoWalletDescriptor for DescriptorTemplateOut { _secp: &SecpCtx, network: Network, ) -> Result<(ExtendedDescriptor, KeyMap), DescriptorError> { - let valid_networks = &self.2; + struct Translator { + network: Network, + } - let fix_key = |pk: &DescriptorPublicKey| { - if valid_networks.contains(&network) { + impl miniscript::Translator + for Translator + { + fn pk( + &mut self, + pk: &DescriptorPublicKey, + ) -> Result { // workaround for xpubs generated by other key types, like bip39: since when the // conversion is made one network has to be chosen, what we generally choose // "mainnet", but then override the set of valid networks to specify that all of @@ -175,7 +227,7 @@ impl IntoWalletDescriptor for DescriptorTemplateOut { let pk = match pk { DescriptorPublicKey::XPub(ref xpub) => { let mut xpub = xpub.clone(); - xpub.xkey.network = network; + xpub.xkey.network = self.network; DescriptorPublicKey::XPub(xpub) } @@ -183,13 +235,20 @@ impl IntoWalletDescriptor for DescriptorTemplateOut { }; Ok(pk) - } else { - Err(DescriptorError::Key(KeyError::InvalidNetwork)) } - }; + miniscript::translate_hash_clone!( + DescriptorPublicKey, + DescriptorPublicKey, + DescriptorError + ); + } + + if !self.2.contains(&network) { + return Err(DescriptorError::Key(KeyError::InvalidNetwork)); + } // fixup the network for keys that need it - let translated = self.0.translate_pk(fix_key, fix_key)?; + let translated = self.0.translate_pk(&mut Translator { network })?; Ok((translated, self.1)) } @@ -210,7 +269,7 @@ pub(crate) fn into_wallet_descriptor_checked( derivation_path, wildcard, .. - }) = k.as_key() + }) = k { return *wildcard == Wildcard::Hardened || derivation_path.into_iter().any(ChildNumber::is_hardened); @@ -294,11 +353,6 @@ where } } -pub(crate) trait DerivedDescriptorMeta { - fn get_hd_keypaths(&self, secp: &SecpCtx) -> HdKeyPaths; - fn get_tap_key_origins(&self, secp: &SecpCtx) -> TapKeyOrigins; -} - pub(crate) trait DescriptorMeta { fn is_witness(&self) -> bool; fn is_taproot(&self) -> bool; @@ -307,63 +361,23 @@ pub(crate) trait DescriptorMeta { &self, hd_keypaths: &HdKeyPaths, secp: &'s SecpCtx, - ) -> Option>; + ) -> Option; fn derive_from_tap_key_origins<'s>( &self, tap_key_origins: &TapKeyOrigins, secp: &'s SecpCtx, - ) -> Option>; + ) -> Option; fn derive_from_psbt_key_origins<'s>( &self, key_origins: BTreeMap, secp: &'s SecpCtx, - ) -> Option>; + ) -> Option; fn derive_from_psbt_input<'s>( &self, psbt_input: &psbt::Input, utxo: Option, secp: &'s SecpCtx, - ) -> Option>; -} - -pub(crate) trait DescriptorScripts { - fn psbt_redeem_script(&self) -> Option