diff --git a/src/blockchain/compact_filters/mod.rs b/src/blockchain/compact_filters/mod.rs index 4b294046..ab775198 100644 --- a/src/blockchain/compact_filters/mod.rs +++ b/src/blockchain/compact_filters/mod.rs @@ -22,6 +22,42 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. +//! Compact Filters +//! +//! This module contains a multithreaded implementation of an [`OnlineBlockchain`] backend that +//! uses BIP157 (aka "Neutrino") to populate the wallet's [database](crate::database::Database) +//! by downloading compact filters from the P2P network. +//! +//! Since there are currently very few peers "in the wild" that advertise the required service +//! flag, this implementation requires that one or more known peers are provided by the user. +//! No dns or other kinds of peer discovery are done internally. +//! +//! Moreover, this module doesn't currently support detecting and resolving conflicts between +//! messages received by different peers. Thus, it's recommended to use this module by only +//! connecting to a single peer at a time, optionally by opening multiple connections if it's +//! desirable to use multiple threads at once to sync in parallel. +//! +//! ## Example +//! +//! ```no_run +//! # use std::sync::Arc; +//! # use bitcoin::*; +//! # use magical_bitcoin_wallet::*; +//! # use magical_bitcoin_wallet::blockchain::compact_filters::*; +//! let num_threads = 4; +//! +//! let mempool = Arc::new(Mempool::default()); +//! let peers = (0..num_threads) +//! .map(|_| Peer::connect( +//! "btcd-mainnet.lightning.computer:8333", +//! Arc::clone(&mempool), +//! Network::Bitcoin, +//! )) +//! .collect::>()?; +//! let blockchain = CompactFiltersBlockchain::new(peers, "./wallet-filters", Some(500_000))?; +//! # Ok::<(), magical_bitcoin_wallet::error::Error>(()) +//! ``` + use std::collections::HashSet; use std::fmt; use std::path::Path; @@ -56,10 +92,22 @@ const SYNC_HEADERS_COST: f32 = 1.0; const SYNC_FILTERS_COST: f32 = 11.6 * 1_000.0; const PROCESS_BLOCKS_COST: f32 = 20_000.0; +/// Structure implementing the required blockchain traits +/// +/// ## Example +/// See the [`blockchain::compact_filters`](crate::blockchain::compact_filters) module for a usage example. #[derive(Debug)] pub struct CompactFiltersBlockchain(Option); impl CompactFiltersBlockchain { + /// Construct a new instance given a list of peers, a path to store headers and block + /// filters downloaded during the sync and optionally a number of blocks to ignore starting + /// from the genesis while scanning for the wallet's outputs. + /// + /// For each [`Peer`] specified a new thread will be spawned to download and verify the filters + /// in parallel. It's currently recommended to only connect to a single peer to avoid + /// inconsistencies in the data returned, optionally with multiple connections in parallel to + /// speed-up the sync process. pub fn new>( peers: Vec, storage_dir: P, @@ -73,6 +121,7 @@ impl CompactFiltersBlockchain { } } +/// Internal struct that contains the state of a [`CompactFiltersBlockchain`] #[derive(Debug)] struct CompactFilters { peers: Vec>, @@ -81,6 +130,7 @@ struct CompactFilters { } impl CompactFilters { + /// Constructor, see [`CompactFiltersBlockchain::new`] for the documentation pub fn new>( peers: Vec, storage_dir: P, @@ -117,6 +167,8 @@ impl CompactFilters { }) } + /// Process a transaction by looking for inputs that spend from a UTXO in the database or + /// outputs that send funds to a know script_pubkey. fn process_tx( &self, database: &mut D, @@ -439,25 +491,40 @@ impl OnlineBlockchain for CompactFiltersBlockchain { } } +/// An error that can occur during sync with a [`CompactFiltersBlockchain`] #[derive(Debug)] pub enum CompactFiltersError { + /// A peer sent an invalid or unexpected response InvalidResponse, + /// The headers returned are invalid InvalidHeaders, + /// The compact filter headers returned are invalid InvalidFilterHeader, + /// The compact filter returned is invalid InvalidFilter, + /// The peer is missing a block in the valid chain MissingBlock, + /// The data stored in the block filters storage are corrupted DataCorruption, + /// A peer is not connected NotConnected, + /// A peer took too long to reply to one of our messages Timeout, + /// No peers have been specified NoPeers, + /// Internal database error DB(rocksdb::Error), + /// Internal I/O error IO(std::io::Error), + /// Invalid BIP158 filter BIP158(bitcoin::util::bip158::Error), + /// Internal system time error Time(std::time::SystemTimeError), + /// Wrapper for [`crate::error::Error`] Global(Box), } diff --git a/src/blockchain/compact_filters/peer.rs b/src/blockchain/compact_filters/peer.rs index a317b3ea..eb2cb220 100644 --- a/src/blockchain/compact_filters/peer.rs +++ b/src/blockchain/compact_filters/peer.rs @@ -48,18 +48,27 @@ use super::CompactFiltersError; type ResponsesMap = HashMap<&'static str, Arc<(Mutex>, Condvar)>>; -pub(crate) const TIMEOUT_SECS: u64 = 10; +pub(crate) const TIMEOUT_SECS: u64 = 30; +/// Container for unconfirmed, but valid Bitcoin transactions +/// +/// It is normally shared between [`Peer`]s with the use of [`Arc`], so that transactions are not +/// duplicated in memory. #[derive(Debug, Default)] pub struct Mempool { txs: RwLock>, } impl Mempool { + /// Add a transaction to the mempool + /// + /// Note that this doesn't propagate the transaction to other + /// peers. To do that, [`broadcast`](crate::blockchain::OnlineBlockchain::broadcast) should be used. pub fn add_tx(&self, tx: Transaction) { self.txs.write().unwrap().insert(tx.txid(), tx); } + /// Look-up a transaction in the mempool given an [`Inventory`] request pub fn get_tx(&self, inventory: &Inventory) -> Option { let txid = match inventory { Inventory::Error | Inventory::Block(_) | Inventory::WitnessBlock(_) => return None, @@ -69,15 +78,18 @@ impl Mempool { self.txs.read().unwrap().get(&txid).cloned() } + /// Return whether or not the mempool contains a transaction with a given txid pub fn has_tx(&self, txid: &Txid) -> bool { self.txs.read().unwrap().contains_key(txid) } + /// Return the list of transactions contained in the mempool pub fn iter_txs(&self) -> Vec { self.txs.read().unwrap().values().cloned().collect() } } +/// A Bitcoin peer #[derive(Debug)] pub struct Peer { writer: Arc>, @@ -93,7 +105,11 @@ pub struct Peer { } impl Peer { - pub fn new( + /// Connect to a peer over a plaintext TCP connection + /// + /// This function internally spawns a new thread that will monitor incoming messages from the + /// peer, and optionally reply to some of them transparently, like [pings](NetworkMessage::Ping) + pub fn connect( address: A, mempool: Arc, network: Network, @@ -103,7 +119,12 @@ impl Peer { Peer::from_stream(stream, mempool, network) } - pub fn new_proxy( + /// Connect to a peer through a SOCKS5 proxy, optionally by using some credentials, specified + /// as a tuple of `(username, password)` + /// + /// This function internally spawns a new thread that will monitor incoming messages from the + /// peer, and optionally reply to some of them transparently, like [pings](NetworkMessage::Ping) + pub fn connect_proxy( target: T, proxy: P, credentials: Option<(&str, &str)>, @@ -119,6 +140,7 @@ impl Peer { Peer::from_stream(socks_stream.into_inner(), mempool, network) } + /// Create a [`Peer`] from an already connected TcpStream fn from_stream( stream: TcpStream, mempool: Arc, @@ -194,6 +216,7 @@ impl Peer { }) } + /// Send a Bitcoin network message fn _send( writer: &mut TcpStream, magic: u32, @@ -210,6 +233,7 @@ impl Peer { Ok(()) } + /// Wait for a specific incoming Bitcoin message, optionally with a timeout fn _recv( responses: &Arc>, wait_for: &'static str, @@ -241,22 +265,27 @@ impl Peer { Ok(messages.pop()) } + /// Return the [`VersionMessage`] sent by the peer pub fn get_version(&self) -> &VersionMessage { &self.version } + /// Return the Bitcoin [`Network`] in use pub fn get_network(&self) -> Network { self.network } + /// Return the mempool used by this peer pub fn get_mempool(&self) -> Arc { Arc::clone(&self.mempool) } + /// Return whether or not the peer is still connected pub fn is_connected(&self) -> bool { *self.connected.read().unwrap() } + /// Internal function called once the `reader_thread` is spawned fn reader_thread( network: Network, connection: TcpStream, @@ -341,11 +370,13 @@ impl Peer { } } + /// Send a raw Bitcoin message to the peer pub fn send(&self, payload: NetworkMessage) -> Result<(), CompactFiltersError> { let mut writer = self.writer.lock().unwrap(); Self::_send(&mut writer, self.network.magic(), payload) } + /// Waits for a specific incoming Bitcoin message, optionally with a timeout pub fn recv( &self, wait_for: &'static str, diff --git a/src/blockchain/electrum.rs b/src/blockchain/electrum.rs index 6dc8683c..f65454c8 100644 --- a/src/blockchain/electrum.rs +++ b/src/blockchain/electrum.rs @@ -22,6 +22,21 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. +//! Electrum +//! +//! This module defines an [`OnlineBlockchain`] struct that wraps an [`electrum_client::Client`] +//! and implements the logic required to populate the wallet's [database](crate::database::Database) by +//! querying the inner client. +//! +//! ## Example +//! +//! ```no_run +//! # use magical_bitcoin_wallet::blockchain::electrum::ElectrumBlockchain; +//! let client = electrum_client::Client::new("ssl://electrum.blockstream.info:50002", None)?; +//! let blockchain = ElectrumBlockchain::from(client); +//! # Ok::<(), magical_bitcoin_wallet::error::Error>(()) +//! ``` + use std::collections::HashSet; #[allow(unused_imports)] @@ -37,6 +52,10 @@ use crate::database::BatchDatabase; use crate::error::Error; use crate::FeeRate; +/// Wrapper over an Electrum Client that implements the required blockchain traits +/// +/// ## Example +/// See the [`blockchain::electrum`](crate::blockchain::electrum) module for a usage example. pub struct ElectrumBlockchain(Option); #[cfg(test)] diff --git a/src/blockchain/esplora.rs b/src/blockchain/esplora.rs index 65c2a340..13f05f20 100644 --- a/src/blockchain/esplora.rs +++ b/src/blockchain/esplora.rs @@ -22,6 +22,19 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. +//! Esplora +//! +//! This module defines an [`OnlineBlockchain`] struct that can query an Esplora backend +//! populate the wallet's [database](crate::database::Database) by +//! +//! ## Example +//! +//! ```no_run +//! # use magical_bitcoin_wallet::blockchain::esplora::EsploraBlockchain; +//! let blockchain = EsploraBlockchain::new("https://blockstream.info/testnet/"); +//! # Ok::<(), magical_bitcoin_wallet::error::Error>(()) +//! ``` + use std::collections::{HashMap, HashSet}; use std::fmt; @@ -41,18 +54,22 @@ use bitcoin::{Script, Transaction, Txid}; use self::utils::{ELSGetHistoryRes, ELSListUnspentRes, ElectrumLikeSync}; use super::*; -use crate::database::{BatchDatabase, DatabaseUtils}; +use crate::database::BatchDatabase; use crate::error::Error; use crate::FeeRate; #[derive(Debug)] -pub struct UrlClient { +struct UrlClient { url: String, // We use the async client instead of the blocking one because it automatically uses `fetch` // when the target platform is wasm32. client: Client, } +/// Structure that implements the logic to sync with Esplora +/// +/// ## Example +/// See the [`blockchain::esplora`](crate::blockchain::esplora) module for a usage example. #[derive(Debug)] pub struct EsploraBlockchain(Option); @@ -63,6 +80,9 @@ impl std::convert::From for EsploraBlockchain { } impl EsploraBlockchain { + /// Create a new instance of the client from a base URL + /// + /// The client internally adds the `/api` prefix to `base_url` before making any requests pub fn new(base_url: &str) -> Self { EsploraBlockchain(Some(UrlClient { url: base_url.to_string(), @@ -350,12 +370,17 @@ struct EsploraListUnspent { status: EsploraGetHistoryStatus, } +/// Errors that can happen during a sync with [`EsploraBlockchain`] #[derive(Debug)] pub enum EsploraError { + /// Error with the HTTP call Reqwest(reqwest::Error), + /// Invalid number returned Parsing(std::num::ParseIntError), + /// Invalid Bitcoin data returned BitcoinEncoding(bitcoin::consensus::encode::Error), + /// Transaction not found TransactionNotFound(Txid), } diff --git a/src/blockchain/mod.rs b/src/blockchain/mod.rs index 98b2161d..dd78bfa6 100644 --- a/src/blockchain/mod.rs +++ b/src/blockchain/mod.rs @@ -22,6 +22,22 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. +//! Blockchain backends +//! +//! This module provides the implementation of a few commonly-used backends like +//! [Electrum](crate::blockchain::electrum), [Esplora](crate::blockchain::esplora) and +//! [Compact Filters/Neutrino](crate::blockchain::compact_filters), along with two generalized +//! traits [`Blockchain`] and [`OnlineBlockchain`] that can be implemented to build customized +//! backends. +//! +//! Types that only implement the [`Blockchain`] trait can be used as backends for [`Wallet`](crate::wallet::Wallet)s, but any +//! action that requires interacting with the blockchain won't be available ([`Wallet::sync`](crate::wallet::Wallet::sync) and +//! [`Wallet::broadcast`](crate::wallet::Wallet::broadcast)). This allows the creation of physically air-gapped wallets, that have no +//! ability to contact the outside world. An example of an offline-only client is [`OfflineBlockchain`]. +//! +//! Types that also implement [`OnlineBlockchain`] will make the two aforementioned actions +//! available. + use std::collections::HashSet; use std::ops::Deref; use std::sync::mpsc::{channel, Receiver, Sender}; @@ -53,19 +69,40 @@ pub mod compact_filters; #[cfg(feature = "compact_filters")] pub use self::compact_filters::CompactFiltersBlockchain; +/// Capabilities that can be supported by an [`OnlineBlockchain`] backend #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum Capability { + /// Can recover the full history of a wallet and not only the set of currently spendable UTXOs FullHistory, + /// Can fetch any historical transaction given its txid GetAnyTx, + /// Can compute accurate fees for the transactions found during sync AccurateFees, } +/// Base trait for a blockchain backend +/// +/// This trait is always required, even for "air-gapped" backends that don't actually make any +/// external call. Clients that have the ability to make external calls must also implement `OnlineBlockchain`. pub trait Blockchain { + /// Return whether or not the client has the ability to fullfill requests + /// + /// This should always be `false` for offline-only types, and can be true for types that also + /// implement [`OnlineBlockchain`], if they have the ability to fullfill requests. fn is_online(&self) -> bool; + /// Create a new instance of the client that is offline-only + /// + /// For types that also implement [`OnlineBlockchain`], this means creating an instance that + /// returns [`Error::OfflineClient`](crate::error::Error::OfflineClient) if any of the "online" + /// methods are called. + /// + /// This is generally implemented by wrapping the client in an [`Option`] that has [`Option::None`] value + /// when created with this method, and is [`Option::Some`] if properly instantiated. fn offline() -> Self; } +/// Type that only implements [`Blockchain`] and is always offline pub struct OfflineBlockchain; impl Blockchain for OfflineBlockchain { fn offline() -> Self { @@ -77,16 +114,47 @@ impl Blockchain for OfflineBlockchain { } } +/// Trait that defines the actions that must be supported by an online [`Blockchain`] #[maybe_async] pub trait OnlineBlockchain: Blockchain { + /// Return the set of [`Capability`] supported by this backend fn get_capabilities(&self) -> HashSet; + /// Setup the backend and populate the internal database for the first time + /// + /// This method is the equivalent of [`OnlineBlockchain::sync`], but it's guaranteed to only be + /// called once, at the first [`Wallet::sync`](crate::wallet::Wallet::sync). + /// + /// The rationale behind the distinction between `sync` and `setup` is that some custom backends + /// might need to perform specific actions only the first time they are synced. + /// + /// For types that do not have that distinction, only this method can be implemented, since + /// [`OnlineBlockchain::sync`] defaults to calling this internally if not overridden. fn setup( &self, stop_gap: Option, database: &mut D, progress_update: P, ) -> Result<(), Error>; + /// Populate the internal database with transactions and UTXOs + /// + /// If not overridden, it defaults to calling [`OnlineBlockchain::setup`] internally. + /// + /// This method should implement the logic required to iterate over the list of the wallet's + /// script_pubkeys using [`Database::iter_script_pubkeys`] and look for relevant transactions + /// in the blockchain to populate the database with [`BatchOperations::set_tx`] and + /// [`BatchOperations::set_utxo`]. + /// + /// This method should also take care of removing UTXOs that are seen as spent in the + /// blockchain, using [`BatchOperations::del_utxo`]. + /// + /// The `progress_update` object can be used to give the caller updates about the progress by using + /// [`Progress::update`]. + /// + /// [`Database::iter_script_pubkeys`]: crate::database::Database::iter_script_pubkeys + /// [`BatchOperations::set_tx`]: crate::database::BatchOperations::set_tx + /// [`BatchOperations::set_utxo`]: crate::database::BatchOperations::set_utxo + /// [`BatchOperations::del_utxo`]: crate::database::BatchOperations::del_utxo fn sync( &self, stop_gap: Option, @@ -96,19 +164,31 @@ pub trait OnlineBlockchain: Blockchain { maybe_await!(self.setup(stop_gap, database, progress_update)) } + /// Fetch a transaction from the blockchain given its txid fn get_tx(&self, txid: &Txid) -> Result, Error>; + /// Broadcast a transaction fn broadcast(&self, tx: &Transaction) -> Result<(), Error>; + /// Return the current height fn get_height(&self) -> Result; + /// Estimate the fee rate required to confirm a transaction in a given `target` of blocks fn estimate_fee(&self, target: usize) -> Result; } +/// Data sent with a progress update over a [`channel`] pub type ProgressData = (f32, Option); +/// Trait for types that can receive and process progress updates during [`OnlineBlockchain::sync`] and +/// [`OnlineBlockchain::setup`] pub trait Progress: Send { + /// Send a new progress update + /// + /// The `progress` value should be in the range 0.0 - 100.0, and the `message` value is an + /// optional text message that can be displayed to the user. fn update(&self, progress: f32, message: Option) -> Result<(), Error>; } +/// Shortcut to create a [`channel`] (pair of [`Sender`] and [`Receiver`]) that can transport [`ProgressData`] pub fn progress() -> (Sender, Receiver) { channel() } @@ -124,9 +204,11 @@ impl Progress for Sender { } } +/// Type that implements [`Progress`] and drops every update received #[derive(Clone)] pub struct NoopProgress; +/// Create a new instance of [`NoopProgress`] pub fn noop_progress() -> NoopProgress { NoopProgress } @@ -137,9 +219,11 @@ impl Progress for NoopProgress { } } +/// Type that implements [`Progress`] and logs at level `INFO` every update received #[derive(Clone)] pub struct LogProgress; +/// Create a nwe instance of [`LogProgress`] pub fn log_progress() -> LogProgress { LogProgress } diff --git a/src/lib.rs b/src/lib.rs index 433a201d..0437088b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -80,9 +80,3 @@ pub use wallet::address_validator; pub use wallet::signer; pub use wallet::tx_builder::TxBuilder; pub use wallet::{OfflineWallet, Wallet}; - -#[cfg(feature = "esplora")] -pub use blockchain::esplora::EsploraBlockchain; - -#[cfg(feature = "electrum")] -pub use blockchain::electrum::ElectrumBlockchain;