From bfef2e3cfe3069338a570bde55ef7bc744960004 Mon Sep 17 00:00:00 2001 From: Riccardo Casatta Date: Mon, 17 May 2021 17:20:32 +0200 Subject: [PATCH 01/11] Implements RPC Backend --- .github/workflows/cont_integration.yml | 4 + Cargo.toml | 7 +- run_blockchain_tests.sh | 6 +- src/blockchain/mod.rs | 8 + src/blockchain/rpc.rs | 664 +++++++++++++++++++++++++ src/database/memory.rs | 4 +- src/error.rs | 16 +- src/lib.rs | 3 + src/testutils/blockchain_tests.rs | 195 ++++---- src/types.rs | 2 +- src/wallet/mod.rs | 4 + 11 files changed, 820 insertions(+), 93 deletions(-) create mode 100644 src/blockchain/rpc.rs diff --git a/.github/workflows/cont_integration.yml b/.github/workflows/cont_integration.yml index fe4a97e1..d9d25094 100644 --- a/.github/workflows/cont_integration.yml +++ b/.github/workflows/cont_integration.yml @@ -22,6 +22,7 @@ jobs: - compact_filters - esplora,key-value-db,electrum - compiler + - rpc steps: - name: checkout uses: actions/checkout@v2 @@ -85,6 +86,9 @@ jobs: - name: esplora container: bitcoindevkit/esplora start: /root/electrs --network regtest -vvv --cookie admin:passw --jsonrpc-import --electrum-rpc-addr=0.0.0.0:60401 --http-addr 0.0.0.0:3002 + - name: rpc + container: bitcoindevkit/electrs + start: /root/electrs --network regtest --jsonrpc-import container: ${{ matrix.blockchain.container }} env: BDK_RPC_AUTH: USER_PASS diff --git a/Cargo.toml b/Cargo.toml index 78ddb116..8c79559b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,7 +33,7 @@ lazy_static = { version = "1.4", optional = true } tiny-bip39 = { version = "^0.8", optional = true } # Needed by bdk_blockchain_tests macro -bitcoincore-rpc = { version = "0.13", optional = true } +bitcoincore-rpc = { version = "0.13", optional = true } serial_test = { version = "0.4", optional = true } # Platform-specific dependencies @@ -56,9 +56,13 @@ key-value-db = ["sled"] async-interface = ["async-trait"] all-keys = ["keys-bip39"] keys-bip39 = ["tiny-bip39"] +rpc = ["bitcoincore-rpc"] + # Debug/Test features test-blockchains = ["bitcoincore-rpc", "electrum-client"] +test-electrum = ["electrum"] +test-rpc = ["rpc"] test-md-docs = ["electrum"] [dev-dependencies] @@ -67,6 +71,7 @@ env_logger = "0.7" base64 = "^0.11" clap = "2.33" serial_test = "0.4" +bitcoind = "0.9.0" [[example]] name = "address_validator" diff --git a/run_blockchain_tests.sh b/run_blockchain_tests.sh index ce87f26f..0ee3eb76 100755 --- a/run_blockchain_tests.sh +++ b/run_blockchain_tests.sh @@ -4,7 +4,7 @@ usage() { cat <<'EOF' Script for running the bdk blockchain tests for a specific blockchain by starting up the backend in docker. -Usage: ./run_blockchain_tests.sh [esplora|electrum] [test name]. +Usage: ./run_blockchain_tests.sh [esplora|electrum|rpc] [test name]. EOF } @@ -37,6 +37,10 @@ case "$blockchain" in id="$(docker run -d -p 127.0.0.1:18443-18444:18443-18444/tcp -p 127.0.0.1:60401:60401/tcp -p 127.0.0.1:3002:3002/tcp bitcoindevkit/esplora)" export BDK_ESPLORA_URL=http://127.0.0.1:3002 ;; + rpc) + eprintln "starting electrs docker container" + id="$(docker run -d -p 127.0.0.1:18443-18444:18443-18444/tcp -p 127.0.0.1:60401:60401/tcp bitcoindevkit/electrs)" + ;; *) usage; exit 1; diff --git a/src/blockchain/mod.rs b/src/blockchain/mod.rs index 48e6e6c5..d7443aa6 100644 --- a/src/blockchain/mod.rs +++ b/src/blockchain/mod.rs @@ -43,6 +43,13 @@ pub use self::electrum::ElectrumBlockchain; #[cfg(feature = "electrum")] pub use self::electrum::ElectrumBlockchainConfig; +#[cfg(feature = "rpc")] +pub mod rpc; +#[cfg(feature = "rpc")] +pub use self::rpc::RpcBlockchain; +#[cfg(feature = "rpc")] +pub use self::rpc::RpcConfig; + #[cfg(feature = "esplora")] #[cfg_attr(docsrs, doc(cfg(feature = "esplora")))] pub mod esplora; @@ -52,6 +59,7 @@ pub use self::esplora::EsploraBlockchain; #[cfg(feature = "compact_filters")] #[cfg_attr(docsrs, doc(cfg(feature = "compact_filters")))] pub mod compact_filters; + #[cfg(feature = "compact_filters")] pub use self::compact_filters::CompactFiltersBlockchain; diff --git a/src/blockchain/rpc.rs b/src/blockchain/rpc.rs new file mode 100644 index 00000000..85071f9c --- /dev/null +++ b/src/blockchain/rpc.rs @@ -0,0 +1,664 @@ +// Bitcoin Dev Kit +// Written in 2021 by Riccardo Casatta +// +// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers +// +// This file is licensed under the Apache License, Version 2.0 or the MIT license +// , at your option. +// You may not use this file except in accordance with one or both of these +// licenses. + +//! Rpc Blockchain +//! +//! Backend that gets blockchain data from Bitcoin Core RPC +//! +//! ## Example +//! +//! ```no_run +//! # use bdk::blockchain::{RpcConfig, RpcBlockchain, ConfigurableBlockchain}; +//! let config = RpcConfig { +//! url: "127.0.0.1:18332".to_string(), +//! auth: bitcoincore_rpc::Auth::CookieFile("/home/user/.bitcoin/.cookie".into()), +//! network: bdk::bitcoin::Network::Testnet, +//! wallet_name: "wallet_name".to_string(), +//! skip_blocks: None, +//! }; +//! let blockchain = RpcBlockchain::from_config(&config); +//! ``` + +use crate::bitcoin::consensus::deserialize; +use crate::bitcoin::{Address, Network, OutPoint, Transaction, TxOut, Txid}; +use crate::blockchain::{Blockchain, Capability, ConfigurableBlockchain, Progress}; +use crate::database::{BatchDatabase, DatabaseUtils}; +use crate::descriptor::{get_checksum, IntoWalletDescriptor}; +use crate::wallet::utils::SecpCtx; +use crate::{Error, FeeRate, KeychainKind, LocalUtxo, TransactionDetails}; +use bitcoincore_rpc::json::{ + GetAddressInfoResultLabel, ImportMultiOptions, ImportMultiRequest, + ImportMultiRequestScriptPubkey, ImportMultiRescanSince, +}; +use bitcoincore_rpc::jsonrpc::serde_json::Value; +use bitcoincore_rpc::{Auth, Client, RpcApi}; +use log::debug; +use serde::Deserialize; +use std::collections::{HashMap, HashSet}; +use std::str::FromStr; + +/// The main struct for RPC backend implementing the [crate::blockchain::Blockchain] trait +#[derive(Debug)] +pub struct RpcBlockchain { + /// Rpc client to the node, includes the wallet name + client: Client, + /// Network used + network: Network, + /// Blockchain capabilities, cached here at startup + capabilities: HashSet, + /// Skip this many blocks of the blockchain at the first rescan, if None the rescan is done from the genesis block + skip_blocks: Option, + + /// This is a fixed Address used as a hack key to store information on the node + _satoshi_address: Address, +} + +/// RpcBlockchain configuration options +#[derive(Debug)] +pub struct RpcConfig { + /// The bitcoin node url + pub url: String, + /// The bitcoin node authentication mechanism + pub auth: Auth, + /// The network we are using (it will be checked the bitcoin node network matches this) + pub network: Network, + /// The wallet name in the bitcoin node, consider using [wallet_name_from_descriptor] for this + pub wallet_name: String, + /// Skip this many blocks of the blockchain at the first rescan, if None the rescan is done from the genesis block + pub skip_blocks: Option, +} + +impl RpcBlockchain { + fn get_node_synced_height(&self) -> Result { + let info = self.client.get_address_info(&self._satoshi_address)?; + if let Some(GetAddressInfoResultLabel::Simple(label)) = info.labels.first() { + Ok(label + .parse::() + .unwrap_or_else(|_| self.skip_blocks.unwrap_or(0))) + } else { + Ok(self.skip_blocks.unwrap_or(0)) + } + } + + /// Set the synced height in the core node by using a label of a fixed address so that + /// another client with the same descriptor doesn't rescan the blockchain + fn set_node_synced_height(&self, height: u32) -> Result<(), Error> { + Ok(self + .client + .set_label(&self._satoshi_address, &height.to_string())?) + } +} + +impl Blockchain for RpcBlockchain { + fn get_capabilities(&self) -> HashSet { + self.capabilities.clone() + } + + fn setup( + &self, + stop_gap: Option, + database: &mut D, + progress_update: P, + ) -> Result<(), Error> { + let mut scripts_pubkeys = database.iter_script_pubkeys(Some(KeychainKind::External))?; + scripts_pubkeys.extend(database.iter_script_pubkeys(Some(KeychainKind::Internal))?); + debug!( + "importing {} script_pubkeys (some maybe already imported)", + scripts_pubkeys.len() + ); + let requests: Vec<_> = scripts_pubkeys + .iter() + .map(|s| ImportMultiRequest { + timestamp: ImportMultiRescanSince::Timestamp(0), + script_pubkey: Some(ImportMultiRequestScriptPubkey::Script(&s)), + watchonly: Some(true), + ..Default::default() + }) + .collect(); + let options = ImportMultiOptions { + rescan: Some(false), + }; + // Note we use import_multi because as of bitcoin core 0.21.0 many descriptors are not supported + // https://bitcoindevkit.org/descriptors/#compatibility-matrix + //TODO maybe convenient using import_descriptor for compatible descriptor and import_multi as fallback + self.client.import_multi(&requests, Some(&options))?; + self.sync(stop_gap, database, progress_update) + } + + fn sync( + &self, + _stop_gap: Option, + db: &mut D, + progress_update: P, + ) -> Result<(), Error> { + let current_height = self.get_height()?; + + // min because block invalidate may cause height to go down + let node_synced = self.get_node_synced_height()?.min(current_height); + + let mut indexes = HashMap::new(); + for keykind in &[KeychainKind::External, KeychainKind::Internal] { + indexes.insert(*keykind, db.get_last_index(*keykind)?.unwrap_or(0)); + } + + //TODO call rescan in chunks (updating node_synced_height) so that in case of + // interruption work can be partially recovered + debug!( + "rescan_blockchain from:{} to:{}", + node_synced, current_height + ); + self.client + .rescan_blockchain(Some(node_synced as usize), Some(current_height as usize))?; + progress_update.update(1.0, None)?; + + let mut known_txs: HashMap<_, _> = db + .iter_txs(true)? + .into_iter() + .map(|tx| (tx.txid, tx)) + .collect(); + let known_utxos: HashSet<_> = db.iter_utxos()?.into_iter().collect(); + + //TODO list_since_blocks would be more efficient + let current_utxo = self + .client + .list_unspent(Some(0), None, None, Some(true), None)?; + debug!("current_utxo len {}", current_utxo.len()); + + //TODO supported up to 1_000 txs, should use since_blocks or do paging + let list_txs = self + .client + .list_transactions(None, Some(1_000), None, Some(true))?; + let mut list_txs_ids = HashSet::new(); + + for tx_result in list_txs.iter().filter(|t| { + // list_txs returns all conflicting tx we want to + // filter out replaced tx => unconfirmed and not in the mempool + t.info.confirmations > 0 || self.client.get_mempool_entry(&t.info.txid).is_ok() + }) { + let txid = tx_result.info.txid; + list_txs_ids.insert(txid); + if let Some(mut known_tx) = known_txs.get_mut(&txid) { + if tx_result.info.blockheight != known_tx.height { + // reorg may change tx height + debug!( + "updating tx({}) height to: {:?}", + txid, tx_result.info.blockheight + ); + known_tx.height = tx_result.info.blockheight; + db.set_tx(&known_tx)?; + } + } else { + //TODO check there is already the raw tx in db? + let tx_result = self.client.get_transaction(&txid, Some(true))?; + let tx: Transaction = deserialize(&tx_result.hex)?; + let mut received = 0u64; + let mut sent = 0u64; + for output in tx.output.iter() { + if let Ok(Some((kind, index))) = + db.get_path_from_script_pubkey(&output.script_pubkey) + { + if index > *indexes.get(&kind).unwrap() { + indexes.insert(kind, index); + } + received += output.value; + } + } + + for input in tx.input.iter() { + if let Some(previous_output) = db.get_previous_output(&input.previous_output)? { + sent += previous_output.value; + } + } + + let td = TransactionDetails { + transaction: Some(tx), + txid: tx_result.info.txid, + timestamp: tx_result.info.time, + received, + sent, + fees: tx_result.fee.map(|f| f.as_sat().abs() as u64).unwrap_or(0), //TODO + height: tx_result.info.blockheight, + }; + debug!( + "saving tx: {} tx_result.fee:{:?} td.fees:{:?}", + td.txid, tx_result.fee, td.fees + ); + db.set_tx(&td)?; + } + } + + for known_txid in known_txs.keys() { + if !list_txs_ids.contains(known_txid) { + debug!("removing tx: {}", known_txid); + db.del_tx(known_txid, false)?; + } + } + + let current_utxos: HashSet = current_utxo + .into_iter() + .map(|u| LocalUtxo { + outpoint: OutPoint::new(u.txid, u.vout), + txout: TxOut { + value: u.amount.as_sat(), + script_pubkey: u.script_pub_key, + }, + keychain: KeychainKind::External, + }) + .collect(); + + let spent: HashSet<_> = known_utxos.difference(¤t_utxos).collect(); + for s in spent { + debug!("removing utxo: {:?}", s); + db.del_utxo(&s.outpoint)?; + } + let received: HashSet<_> = current_utxos.difference(&known_utxos).collect(); + for s in received { + debug!("adding utxo: {:?}", s); + db.set_utxo(s)?; + } + + for (keykind, index) in indexes { + debug!("{:?} max {}", keykind, index); + db.set_last_index(keykind, index)?; + } + + self.set_node_synced_height(current_height)?; + Ok(()) + } + + fn get_tx(&self, txid: &Txid) -> Result, Error> { + if self.capabilities.contains(&Capability::FullHistory) { + Ok(Some(self.client.get_raw_transaction(txid, None)?)) + } else { + Ok(None) + } + } + + fn broadcast(&self, tx: &Transaction) -> Result<(), Error> { + Ok(self.client.send_raw_transaction(tx).map(|_| ())?) + } + + fn get_height(&self) -> Result { + Ok(self.client.get_blockchain_info().map(|i| i.blocks as u32)?) + } + + fn estimate_fee(&self, target: usize) -> Result { + let sat_per_kb = self + .client + .estimate_smart_fee(target as u16, None)? + .fee_rate + .ok_or(Error::FeeRateUnavailable)? + .as_sat() as f64; + + Ok(FeeRate::from_sat_per_vb((sat_per_kb / 1000f64) as f32)) + } +} + +impl ConfigurableBlockchain for RpcBlockchain { + type Config = RpcConfig; + + /// Returns RpcBlockchain backend creating an RPC client to a specific wallet named as the descriptor's checksum + /// if it's the first time it creates the wallet in the node and upon return is granted the wallet is loaded + fn from_config(config: &Self::Config) -> Result { + let wallet_name = config.wallet_name.clone(); + let wallet_url = format!("{}/wallet/{}", config.url, &wallet_name); + debug!("connecting to {} auth:{:?}", wallet_url, config.auth); + + let client = Client::new(wallet_url, config.auth.clone())?; + let loaded_wallets = client.list_wallets()?; + if loaded_wallets.contains(&wallet_name) { + debug!("wallet already loaded {:?}", wallet_name); + } else { + let existing_wallets = list_wallet_dir(&client)?; + if existing_wallets.contains(&wallet_name) { + client.load_wallet(&wallet_name)?; + debug!("wallet loaded {:?}", wallet_name); + } else { + client.create_wallet(&wallet_name, Some(true), None, None, None)?; + debug!("wallet created {:?}", wallet_name); + } + } + + let blockchain_info = client.get_blockchain_info()?; + let network = match blockchain_info.chain.as_str() { + "main" => Network::Bitcoin, + "test" => Network::Testnet, + "regtest" => Network::Regtest, + _ => return Err(Error::Generic("Invalid network".to_string())), + }; + if network != config.network { + return Err(Error::InvalidNetwork { + requested: config.network, + found: network, + }); + } + + let mut capabilities: HashSet<_> = vec![Capability::FullHistory].into_iter().collect(); + let rpc_version = client.version()?; + if rpc_version >= 210_000 { + let info: HashMap = client.call("getindexinfo", &[]).unwrap(); + if info.contains_key("txindex") { + capabilities.insert(Capability::GetAnyTx); + capabilities.insert(Capability::AccurateFees); + } + } + + // this is just a fixed address used only to store a label containing the synced height in the node + let mut satoshi_address = Address::from_str("1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa").unwrap(); + satoshi_address.network = network; + + Ok(RpcBlockchain { + client, + network, + capabilities, + _satoshi_address: satoshi_address, + skip_blocks: config.skip_blocks, + }) + } +} + +/// Deterministically generate a unique name given the descriptors defining the wallet +pub fn wallet_name_from_descriptor( + descriptor: T, + change_descriptor: Option, + network: Network, + secp: &SecpCtx, +) -> Result +where + T: IntoWalletDescriptor, +{ + //TODO check descriptors contains only public keys + let descriptor = descriptor + .into_wallet_descriptor(&secp, network)? + .0 + .to_string(); + let mut wallet_name = get_checksum(&descriptor[..descriptor.find('#').unwrap()])?; + if let Some(change_descriptor) = change_descriptor { + let change_descriptor = change_descriptor + .into_wallet_descriptor(&secp, network)? + .0 + .to_string(); + wallet_name.push_str( + get_checksum(&change_descriptor[..change_descriptor.find('#').unwrap()])?.as_str(), + ); + } + + Ok(wallet_name) +} + +/// return the wallets available in default wallet directory +//TODO use bitcoincore_rpc method when PR #179 lands +fn list_wallet_dir(client: &Client) -> Result, Error> { + #[derive(Deserialize)] + struct Name { + name: String, + } + #[derive(Deserialize)] + struct Result { + wallets: Vec, + } + + let result: Result = client.call("listwalletdir", &[])?; + Ok(result.wallets.into_iter().map(|n| n.name).collect()) +} + +#[cfg(feature = "test-blockchains")] +crate::bdk_blockchain_tests! { + + fn test_instance() -> RpcBlockchain { + let url = std::env::var("BDK_RPC_URL").unwrap_or_else(|_| "127.0.0.1:18443".to_string()); + let url = format!("http://{}", url); + + // TODO same code in `fn get_auth` in testutils, make it public there + let auth = match std::env::var("BDK_RPC_AUTH").as_ref().map(String::as_ref) { + Ok("USER_PASS") => Auth::UserPass( + std::env::var("BDK_RPC_USER").unwrap(), + std::env::var("BDK_RPC_PASS").unwrap(), + ), + _ => Auth::CookieFile(std::path::PathBuf::from( + std::env::var("BDK_RPC_COOKIEFILE") + .unwrap_or_else(|_| "/home/user/.bitcoin/regtest/.cookie".to_string()), + )), + }; + let config = RpcConfig { + url, + auth, + network: Network::Regtest, + wallet_name: format!("client-wallet-test-{:?}", std::time::SystemTime::now() ), + skip_blocks: None, + }; + RpcBlockchain::from_config(&config).unwrap() + } +} + +#[cfg(feature = "test-rpc")] +#[cfg(test)] +mod test { + use super::{RpcBlockchain, RpcConfig}; + use crate::bitcoin::consensus::deserialize; + use crate::bitcoin::{Address, Amount, Network, Transaction}; + use crate::blockchain::rpc::wallet_name_from_descriptor; + use crate::blockchain::{noop_progress, Blockchain, Capability, ConfigurableBlockchain}; + use crate::database::MemoryDatabase; + use crate::wallet::AddressIndex; + use crate::Wallet; + use bitcoin::secp256k1::Secp256k1; + use bitcoin::Txid; + use bitcoincore_rpc::json::CreateRawTransactionInput; + use bitcoincore_rpc::RawTx; + use bitcoincore_rpc::{Auth, RpcApi}; + use bitcoind::BitcoinD; + use std::collections::HashMap; + + fn create_rpc( + bitcoind: &BitcoinD, + desc: &str, + network: Network, + ) -> Result { + let secp = Secp256k1::new(); + let wallet_name = wallet_name_from_descriptor(desc, None, network, &secp).unwrap(); + + let config = RpcConfig { + url: bitcoind.rpc_url(), + auth: Auth::CookieFile(bitcoind.cookie_file.clone()), + network, + wallet_name, + skip_blocks: None, + }; + RpcBlockchain::from_config(&config) + } + fn create_bitcoind(args: Vec) -> BitcoinD { + let exe = std::env::var("BITCOIND_EXE").unwrap(); + bitcoind::BitcoinD::with_args(exe, args, false, bitcoind::P2P::No).unwrap() + } + + const DESCRIPTOR_PUB: &'static str = "wpkh(tpubD6NzVbkrYhZ4X2yy78HWrr1M9NT8dKeWfzNiQqDdMqqa9UmmGztGGz6TaLFGsLfdft5iu32gxq1T4eMNxExNNWzVCpf9Y6JZi5TnqoC9wJq/*)"; + const DESCRIPTOR_PRIV: &'static str = "wpkh(tprv8ZgxMBicQKsPdZxBDUcvTSMEaLwCTzTc6gmw8KBKwa3BJzWzec4g6VUbQBHJcutDH6mMEmBeVyN27H1NF3Nu8isZ1Sts4SufWyfLE6Mf1MB/*)"; + + #[test] + fn test_rpc_wallet_setup() { + env_logger::try_init().unwrap(); + let bitcoind = create_bitcoind(vec![]); + let node_address = bitcoind.client.get_new_address(None, None).unwrap(); + let blockchain = create_rpc(&bitcoind, DESCRIPTOR_PUB, Network::Regtest).unwrap(); + let db = MemoryDatabase::new(); + let wallet = Wallet::new(DESCRIPTOR_PRIV, None, Network::Regtest, db, blockchain).unwrap(); + + wallet.sync(noop_progress(), None).unwrap(); + generate(&bitcoind, 101); + wallet.sync(noop_progress(), None).unwrap(); + let address = wallet.get_address(AddressIndex::New).unwrap(); + let expected_address = "bcrt1q8dyvgt4vhr8ald4xuwewcxhdjha9a5k78wxm5t"; + assert_eq!(expected_address, address.to_string()); + send_to_address(&bitcoind, &address, 100_000); + wallet.sync(noop_progress(), None).unwrap(); + assert_eq!(wallet.get_balance().unwrap(), 100_000); + + let mut builder = wallet.build_tx(); + builder.add_recipient(node_address.script_pubkey(), 50_000); + let (mut psbt, details) = builder.finish().unwrap(); + let finalized = wallet.sign(&mut psbt, Default::default()).unwrap(); + assert!(finalized, "Cannot finalize transaction"); + let tx = psbt.extract_tx(); + wallet.broadcast(tx).unwrap(); + wallet.sync(noop_progress(), None).unwrap(); + assert_eq!( + wallet.get_balance().unwrap(), + 100_000 - 50_000 - details.fees + ); + drop(wallet); + + // test skip_blocks + generate(&bitcoind, 5); + let config = RpcConfig { + url: bitcoind.rpc_url(), + auth: Auth::CookieFile(bitcoind.cookie_file.clone()), + network: Network::Regtest, + wallet_name: "another-name".to_string(), + skip_blocks: Some(103), + }; + let blockchain_skip = RpcBlockchain::from_config(&config).unwrap(); + let db = MemoryDatabase::new(); + let wallet_skip = + Wallet::new(DESCRIPTOR_PRIV, None, Network::Regtest, db, blockchain_skip).unwrap(); + wallet_skip.sync(noop_progress(), None).unwrap(); + send_to_address(&bitcoind, &address, 100_000); + wallet_skip.sync(noop_progress(), None).unwrap(); + assert_eq!(wallet_skip.get_balance().unwrap(), 100_000); + } + + #[test] + fn test_rpc_from_config() { + let bitcoind = create_bitcoind(vec![]); + let blockchain = create_rpc(&bitcoind, DESCRIPTOR_PUB, Network::Regtest); + assert!(blockchain.is_ok()); + let blockchain = create_rpc(&bitcoind, DESCRIPTOR_PUB, Network::Testnet); + assert!(blockchain.is_err(), "wrong network doesn't error"); + } + + #[test] + fn test_rpc_capabilities_get_tx() { + let bitcoind = create_bitcoind(vec![]); + let rpc = create_rpc(&bitcoind, DESCRIPTOR_PUB, Network::Regtest).unwrap(); + let capabilities = rpc.get_capabilities(); + assert!(capabilities.contains(&Capability::FullHistory) && capabilities.len() == 1); + let bitcoind_indexed = create_bitcoind(vec!["-txindex".to_string()]); + let rpc_indexed = create_rpc(&bitcoind_indexed, DESCRIPTOR_PUB, Network::Regtest).unwrap(); + assert_eq!(rpc_indexed.get_capabilities().len(), 3); + let address = generate(&bitcoind_indexed, 101); + let txid = send_to_address(&bitcoind_indexed, &address, 100_000); + assert!(rpc_indexed.get_tx(&txid).unwrap().is_some()); + assert!(rpc.get_tx(&txid).is_err()); + } + + #[test] + fn test_rpc_estimate_fee_get_height() { + let bitcoind = create_bitcoind(vec![]); + let rpc = create_rpc(&bitcoind, DESCRIPTOR_PUB, Network::Regtest).unwrap(); + let result = rpc.estimate_fee(2); + assert!(result.is_err()); + let address = generate(&bitcoind, 100); + // create enough tx so that core give some fee estimation + for _ in 0..15 { + let _ = bitcoind.client.generate_to_address(1, &address).unwrap(); + for _ in 0..2 { + send_to_address(&bitcoind, &address, 100_000); + } + } + let result = rpc.estimate_fee(2); + assert!(result.is_ok()); + assert_eq!(rpc.get_height().unwrap(), 115); + } + + #[test] + fn test_rpc_node_synced_height() { + let bitcoind = create_bitcoind(vec![]); + let rpc = create_rpc(&bitcoind, DESCRIPTOR_PUB, Network::Regtest).unwrap(); + let synced_height = rpc.get_node_synced_height().unwrap(); + + assert_eq!(synced_height, 0); + rpc.set_node_synced_height(1).unwrap(); + + let synced_height = rpc.get_node_synced_height().unwrap(); + assert_eq!(synced_height, 1); + } + + #[test] + fn test_rpc_broadcast() { + let bitcoind = create_bitcoind(vec![]); + let rpc = create_rpc(&bitcoind, DESCRIPTOR_PUB, Network::Regtest).unwrap(); + let address = generate(&bitcoind, 101); + let utxo = bitcoind + .client + .list_unspent(None, None, None, None, None) + .unwrap(); + let input = CreateRawTransactionInput { + txid: utxo[0].txid, + vout: utxo[0].vout, + sequence: None, + }; + + let out: HashMap<_, _> = vec![( + address.to_string(), + utxo[0].amount - Amount::from_sat(100_000), + )] + .into_iter() + .collect(); + let tx = bitcoind + .client + .create_raw_transaction(&[input], &out, None, None) + .unwrap(); + let signed_tx = bitcoind + .client + .sign_raw_transaction_with_wallet(tx.raw_hex(), None, None) + .unwrap(); + let parsed_tx: Transaction = deserialize(&signed_tx.hex).unwrap(); + rpc.broadcast(&parsed_tx).unwrap(); + assert!(bitcoind + .client + .get_raw_mempool() + .unwrap() + .contains(&tx.txid())); + } + + #[test] + fn test_rpc_wallet_name() { + let secp = Secp256k1::new(); + let name = + wallet_name_from_descriptor(DESCRIPTOR_PUB, None, Network::Regtest, &secp).unwrap(); + assert_eq!("tmg7aqay", name); + } + + fn generate(bitcoind: &BitcoinD, blocks: u64) -> Address { + let address = bitcoind.client.get_new_address(None, None).unwrap(); + bitcoind + .client + .generate_to_address(blocks, &address) + .unwrap(); + address + } + + fn send_to_address(bitcoind: &BitcoinD, address: &Address, amount: u64) -> Txid { + bitcoind + .client + .send_to_address( + &address, + Amount::from_sat(amount), + None, + None, + None, + None, + None, + None, + ) + .unwrap() + } +} diff --git a/src/database/memory.rs b/src/database/memory.rs index adf4e20f..2f624a3f 100644 --- a/src/database/memory.rs +++ b/src/database/memory.rs @@ -429,8 +429,8 @@ impl BatchDatabase for MemoryDatabase { } fn commit_batch(&mut self, mut batch: Self::Batch) -> Result<(), Error> { - for key in batch.deleted_keys { - self.map.remove(&key); + for key in batch.deleted_keys.iter() { + self.map.remove(key); } self.map.append(&mut batch.map); Ok(()) diff --git a/src/error.rs b/src/error.rs index 6430e9fd..e06066a3 100644 --- a/src/error.rs +++ b/src/error.rs @@ -11,6 +11,7 @@ use std::fmt; +use crate::bitcoin::Network; use crate::{descriptor, wallet, wallet::address_validator}; use bitcoin::OutPoint; @@ -64,6 +65,8 @@ pub enum Error { /// Required fee absolute value (satoshi) required: u64, }, + /// Node doesn't have data to estimate a fee rate + FeeRateUnavailable, /// In order to use the [`TxBuilder::add_global_xpubs`] option every extended /// key in the descriptor must either be a master key itself (having depth = 0) or have an /// explicit origin provided @@ -80,7 +83,13 @@ pub enum Error { InvalidPolicyPathError(crate::descriptor::policy::PolicyError), /// Signing error Signer(crate::wallet::signer::SignerError), - + /// Invalid network + InvalidNetwork { + /// requested network, for example what is given as bdk-cli option + requested: Network, + /// found network, for example the network of the bitcoin node + found: Network, + }, /// Progress value must be between `0.0` (included) and `100.0` (included) InvalidProgressValue(f32), /// Progress update error (maybe the channel has been closed) @@ -126,6 +135,9 @@ pub enum Error { #[cfg(feature = "key-value-db")] /// Sled database error Sled(sled::Error), + #[cfg(feature = "rpc")] + /// Rpc client error + Rpc(bitcoincore_rpc::Error), } impl fmt::Display for Error { @@ -179,6 +191,8 @@ impl_error!(electrum_client::Error, Electrum); impl_error!(crate::blockchain::esplora::EsploraError, Esplora); #[cfg(feature = "key-value-db")] impl_error!(sled::Error, Sled); +#[cfg(feature = "rpc")] +impl_error!(bitcoincore_rpc::Error, Rpc); #[cfg(feature = "compact_filters")] impl From for Error { diff --git a/src/lib.rs b/src/lib.rs index 77cbef61..9208d8e1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -219,6 +219,9 @@ extern crate bdk_macros; #[cfg(feature = "compact_filters")] extern crate lazy_static; +#[cfg(feature = "rpc")] +pub extern crate bitcoincore_rpc; + #[cfg(feature = "electrum")] pub extern crate electrum_client; diff --git a/src/testutils/blockchain_tests.rs b/src/testutils/blockchain_tests.rs index 3f7402cd..cd40e560 100644 --- a/src/testutils/blockchain_tests.rs +++ b/src/testutils/blockchain_tests.rs @@ -346,7 +346,6 @@ macro_rules! bdk_blockchain_tests { use $crate::database::MemoryDatabase; use $crate::types::KeychainKind; use $crate::{Wallet, FeeRate}; - use $crate::wallet::AddressIndex::New; use $crate::testutils; use $crate::serial_test::serial; @@ -370,6 +369,10 @@ macro_rules! bdk_blockchain_tests { let test_client = TestClient::default(); let wallet = get_wallet_from_descriptors(&descriptors); + // rpc need to call import_multi before receiving any tx, otherwise will not see tx in the mempool + #[cfg(feature = "rpc")] + wallet.sync(noop_progress(), None).unwrap(); + (wallet, descriptors, test_client) } @@ -386,14 +389,14 @@ macro_rules! bdk_blockchain_tests { wallet.sync(noop_progress(), None).unwrap(); - assert_eq!(wallet.get_balance().unwrap(), 50_000); - assert_eq!(wallet.list_unspent().unwrap()[0].keychain, KeychainKind::External); + assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance"); + assert_eq!(wallet.list_unspent().unwrap()[0].keychain, KeychainKind::External, "incorrect keychain kind"); let list_tx_item = &wallet.list_transactions(false).unwrap()[0]; - assert_eq!(list_tx_item.txid, txid); - assert_eq!(list_tx_item.received, 50_000); - assert_eq!(list_tx_item.sent, 0); - assert_eq!(list_tx_item.height, None); + assert_eq!(list_tx_item.txid, txid, "incorrect txid"); + assert_eq!(list_tx_item.received, 50_000, "incorrect received"); + assert_eq!(list_tx_item.sent, 0, "incorrect sent"); + assert_eq!(list_tx_item.height, None, "incorrect height"); } #[test] @@ -410,8 +413,8 @@ macro_rules! bdk_blockchain_tests { wallet.sync(noop_progress(), None).unwrap(); - assert_eq!(wallet.get_balance().unwrap(), 100_000); - assert_eq!(wallet.list_transactions(false).unwrap().len(), 2); + assert_eq!(wallet.get_balance().unwrap(), 100_000, "incorrect balance"); + assert_eq!(wallet.list_transactions(false).unwrap().len(), 2, "incorrect number of txs"); } #[test] @@ -428,8 +431,8 @@ macro_rules! bdk_blockchain_tests { wallet.sync(noop_progress(), None).unwrap(); - assert_eq!(wallet.get_balance().unwrap(), 50_000); - assert_eq!(wallet.list_transactions(false).unwrap().len(), 1); + assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance"); + assert_eq!(wallet.list_transactions(false).unwrap().len(), 1, "incorrect number of txs"); } #[test] @@ -443,15 +446,15 @@ macro_rules! bdk_blockchain_tests { wallet.sync(noop_progress(), None).unwrap(); - assert_eq!(wallet.get_balance().unwrap(), 105_000); - assert_eq!(wallet.list_transactions(false).unwrap().len(), 1); - assert_eq!(wallet.list_unspent().unwrap().len(), 3); + assert_eq!(wallet.get_balance().unwrap(), 105_000, "incorrect balance"); + assert_eq!(wallet.list_transactions(false).unwrap().len(), 1, "incorrect number of txs"); + assert_eq!(wallet.list_unspent().unwrap().len(), 3, "incorrect number of unspents"); let list_tx_item = &wallet.list_transactions(false).unwrap()[0]; - assert_eq!(list_tx_item.txid, txid); - assert_eq!(list_tx_item.received, 105_000); - assert_eq!(list_tx_item.sent, 0); - assert_eq!(list_tx_item.height, None); + assert_eq!(list_tx_item.txid, txid, "incorrect txid"); + assert_eq!(list_tx_item.received, 105_000, "incorrect received"); + assert_eq!(list_tx_item.sent, 0, "incorrect sent"); + assert_eq!(list_tx_item.height, None, "incorrect height"); } #[test] @@ -468,9 +471,9 @@ macro_rules! bdk_blockchain_tests { wallet.sync(noop_progress(), None).unwrap(); - assert_eq!(wallet.get_balance().unwrap(), 75_000); - assert_eq!(wallet.list_transactions(false).unwrap().len(), 2); - assert_eq!(wallet.list_unspent().unwrap().len(), 2); + assert_eq!(wallet.get_balance().unwrap(), 75_000, "incorrect balance"); + assert_eq!(wallet.list_transactions(false).unwrap().len(), 2, "incorrect number of txs"); + assert_eq!(wallet.list_unspent().unwrap().len(), 2, "incorrect number of unspent"); } #[test] @@ -490,7 +493,7 @@ macro_rules! bdk_blockchain_tests { }); wallet.sync(noop_progress(), None).unwrap(); - assert_eq!(wallet.get_balance().unwrap(), 75_000); + assert_eq!(wallet.get_balance().unwrap(), 75_000, "incorrect balance"); } #[test] @@ -504,29 +507,29 @@ macro_rules! bdk_blockchain_tests { wallet.sync(noop_progress(), None).unwrap(); - assert_eq!(wallet.get_balance().unwrap(), 50_000); - assert_eq!(wallet.list_transactions(false).unwrap().len(), 1); - assert_eq!(wallet.list_unspent().unwrap().len(), 1); + assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance"); + assert_eq!(wallet.list_transactions(false).unwrap().len(), 1, "incorrect number of txs"); + assert_eq!(wallet.list_unspent().unwrap().len(), 1, "incorrect unspent"); let list_tx_item = &wallet.list_transactions(false).unwrap()[0]; - assert_eq!(list_tx_item.txid, txid); - assert_eq!(list_tx_item.received, 50_000); - assert_eq!(list_tx_item.sent, 0); - assert_eq!(list_tx_item.height, None); + assert_eq!(list_tx_item.txid, txid, "incorrect txid"); + assert_eq!(list_tx_item.received, 50_000, "incorrect received"); + assert_eq!(list_tx_item.sent, 0, "incorrect sent"); + assert_eq!(list_tx_item.height, None, "incorrect height"); let new_txid = test_client.bump_fee(&txid); wallet.sync(noop_progress(), None).unwrap(); - assert_eq!(wallet.get_balance().unwrap(), 50_000); - assert_eq!(wallet.list_transactions(false).unwrap().len(), 1); - assert_eq!(wallet.list_unspent().unwrap().len(), 1); + assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance after bump"); + assert_eq!(wallet.list_transactions(false).unwrap().len(), 1, "incorrect number of txs after bump"); + assert_eq!(wallet.list_unspent().unwrap().len(), 1, "incorrect unspent after bump"); let list_tx_item = &wallet.list_transactions(false).unwrap()[0]; - assert_eq!(list_tx_item.txid, new_txid); - assert_eq!(list_tx_item.received, 50_000); - assert_eq!(list_tx_item.sent, 0); - assert_eq!(list_tx_item.height, None); + assert_eq!(list_tx_item.txid, new_txid, "incorrect txid after bump"); + assert_eq!(list_tx_item.received, 50_000, "incorrect received after bump"); + assert_eq!(list_tx_item.sent, 0, "incorrect sent after bump"); + assert_eq!(list_tx_item.height, None, "incorrect height after bump"); } // FIXME: I would like this to be cfg_attr(not(feature = "test-esplora"), ignore) but it @@ -543,24 +546,24 @@ macro_rules! bdk_blockchain_tests { wallet.sync(noop_progress(), None).unwrap(); - assert_eq!(wallet.get_balance().unwrap(), 50_000); - assert_eq!(wallet.list_transactions(false).unwrap().len(), 1); - assert_eq!(wallet.list_unspent().unwrap().len(), 1); + assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance"); + assert_eq!(wallet.list_transactions(false).unwrap().len(), 1, "incorrect number of txs"); + assert_eq!(wallet.list_unspent().unwrap().len(), 1, "incorrect number of unspents"); let list_tx_item = &wallet.list_transactions(false).unwrap()[0]; - assert_eq!(list_tx_item.txid, txid); - assert!(list_tx_item.height.is_some()); + assert_eq!(list_tx_item.txid, txid, "incorrect txid"); + assert!(list_tx_item.height.is_some(), "incorrect height"); // Invalidate 1 block test_client.invalidate(1); wallet.sync(noop_progress(), None).unwrap(); - assert_eq!(wallet.get_balance().unwrap(), 50_000); + assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance after invalidate"); let list_tx_item = &wallet.list_transactions(false).unwrap()[0]; - assert_eq!(list_tx_item.txid, txid); - assert_eq!(list_tx_item.height, None); + assert_eq!(list_tx_item.txid, txid, "incorrect txid after invalidate"); + assert_eq!(list_tx_item.height, None, "incorrect height after invalidate"); } #[test] @@ -575,7 +578,7 @@ macro_rules! bdk_blockchain_tests { }); wallet.sync(noop_progress(), None).unwrap(); - assert_eq!(wallet.get_balance().unwrap(), 50_000); + assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance"); let mut builder = wallet.build_tx(); builder.add_recipient(node_addr.script_pubkey(), 25_000); @@ -587,10 +590,10 @@ macro_rules! bdk_blockchain_tests { wallet.broadcast(tx).unwrap(); wallet.sync(noop_progress(), None).unwrap(); - assert_eq!(wallet.get_balance().unwrap(), details.received); + assert_eq!(wallet.get_balance().unwrap(), details.received, "incorrect balance after send"); - assert_eq!(wallet.list_transactions(false).unwrap().len(), 2); - assert_eq!(wallet.list_unspent().unwrap().len(), 1); + assert_eq!(wallet.list_transactions(false).unwrap().len(), 2, "incorrect number of txs"); + assert_eq!(wallet.list_unspent().unwrap().len(), 1, "incorrect number of unspents"); } #[test] @@ -598,38 +601,41 @@ macro_rules! bdk_blockchain_tests { fn test_sync_outgoing_from_scratch() { let (wallet, descriptors, mut test_client) = init_single_sig(); let node_addr = test_client.get_node_address(None); - let received_txid = test_client.receive(testutils! { @tx ( (@external descriptors, 0) => 50_000 ) }); wallet.sync(noop_progress(), None).unwrap(); - assert_eq!(wallet.get_balance().unwrap(), 50_000); + assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance"); let mut builder = wallet.build_tx(); builder.add_recipient(node_addr.script_pubkey(), 25_000); let (mut psbt, details) = builder.finish().unwrap(); + let finalized = wallet.sign(&mut psbt, Default::default()).unwrap(); assert!(finalized, "Cannot finalize transaction"); let sent_txid = wallet.broadcast(psbt.extract_tx()).unwrap(); wallet.sync(noop_progress(), None).unwrap(); - assert_eq!(wallet.get_balance().unwrap(), details.received); + assert_eq!(wallet.get_balance().unwrap(), details.received, "incorrect balance after receive"); // empty wallet let wallet = get_wallet_from_descriptors(&descriptors); - wallet.sync(noop_progress(), None).unwrap(); + #[cfg(feature = "rpc")] // rpc cannot see mempool tx before importmulti + test_client.generate(1, Some(node_addr)); + + wallet.sync(noop_progress(), None).unwrap(); let tx_map = wallet.list_transactions(false).unwrap().into_iter().map(|tx| (tx.txid, tx)).collect::>(); let received = tx_map.get(&received_txid).unwrap(); - assert_eq!(received.received, 50_000); - assert_eq!(received.sent, 0); + assert_eq!(received.received, 50_000, "incorrect received from receiver"); + assert_eq!(received.sent, 0, "incorrect sent from receiver"); let sent = tx_map.get(&sent_txid).unwrap(); - assert_eq!(sent.received, details.received); - assert_eq!(sent.sent, details.sent); - assert_eq!(sent.fees, details.fees); + assert_eq!(sent.received, details.received, "incorrect received from sender"); + assert_eq!(sent.sent, details.sent, "incorrect sent from sender"); + assert_eq!(sent.fees, details.fees, "incorrect fees from sender"); } #[test] @@ -643,7 +649,7 @@ macro_rules! bdk_blockchain_tests { }); wallet.sync(noop_progress(), None).unwrap(); - assert_eq!(wallet.get_balance().unwrap(), 50_000); + assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance"); let mut total_sent = 0; for _ in 0..5 { @@ -660,17 +666,23 @@ macro_rules! bdk_blockchain_tests { } wallet.sync(noop_progress(), None).unwrap(); - assert_eq!(wallet.get_balance().unwrap(), 50_000 - total_sent); + assert_eq!(wallet.get_balance().unwrap(), 50_000 - total_sent, "incorrect balance after chain"); // empty wallet + let wallet = get_wallet_from_descriptors(&descriptors); + + #[cfg(feature = "rpc")] // rpc cannot see mempool tx before importmulti + test_client.generate(1, Some(node_addr)); + wallet.sync(noop_progress(), None).unwrap(); - assert_eq!(wallet.get_balance().unwrap(), 50_000 - total_sent); + assert_eq!(wallet.get_balance().unwrap(), 50_000 - total_sent, "incorrect balance empty wallet"); + } #[test] #[serial] - fn test_sync_bump_fee() { + fn test_sync_bump_fee_basic() { let (wallet, descriptors, mut test_client) = init_single_sig(); let node_addr = test_client.get_node_address(None); @@ -679,7 +691,7 @@ macro_rules! bdk_blockchain_tests { }); wallet.sync(noop_progress(), None).unwrap(); - assert_eq!(wallet.get_balance().unwrap(), 50_000); + assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance"); let mut builder = wallet.build_tx(); builder.add_recipient(node_addr.script_pubkey().clone(), 5_000).enable_rbf(); @@ -688,8 +700,8 @@ macro_rules! bdk_blockchain_tests { assert!(finalized, "Cannot finalize transaction"); wallet.broadcast(psbt.extract_tx()).unwrap(); wallet.sync(noop_progress(), None).unwrap(); - assert_eq!(wallet.get_balance().unwrap(), 50_000 - details.fees - 5_000); - assert_eq!(wallet.get_balance().unwrap(), details.received); + assert_eq!(wallet.get_balance().unwrap(), 50_000 - details.fees - 5_000, "incorrect balance from fees"); + assert_eq!(wallet.get_balance().unwrap(), details.received, "incorrect balance from received"); let mut builder = wallet.build_fee_bump(details.txid).unwrap(); builder.fee_rate(FeeRate::from_sat_per_vb(2.1)); @@ -698,10 +710,10 @@ macro_rules! bdk_blockchain_tests { assert!(finalized, "Cannot finalize transaction"); wallet.broadcast(new_psbt.extract_tx()).unwrap(); wallet.sync(noop_progress(), None).unwrap(); - assert_eq!(wallet.get_balance().unwrap(), 50_000 - new_details.fees - 5_000); - assert_eq!(wallet.get_balance().unwrap(), new_details.received); + assert_eq!(wallet.get_balance().unwrap(), 50_000 - new_details.fees - 5_000, "incorrect balance from fees after bump"); + assert_eq!(wallet.get_balance().unwrap(), new_details.received, "incorrect balance from received after bump"); - assert!(new_details.fees > details.fees); + assert!(new_details.fees > details.fees, "incorrect fees"); } #[test] @@ -715,7 +727,7 @@ macro_rules! bdk_blockchain_tests { }); wallet.sync(noop_progress(), None).unwrap(); - assert_eq!(wallet.get_balance().unwrap(), 50_000); + assert_eq!(wallet.get_balance().unwrap(), 50_000, "incorrect balance"); let mut builder = wallet.build_tx(); builder.add_recipient(node_addr.script_pubkey().clone(), 49_000).enable_rbf(); @@ -724,8 +736,8 @@ macro_rules! bdk_blockchain_tests { assert!(finalized, "Cannot finalize transaction"); wallet.broadcast(psbt.extract_tx()).unwrap(); wallet.sync(noop_progress(), None).unwrap(); - assert_eq!(wallet.get_balance().unwrap(), 1_000 - details.fees); - assert_eq!(wallet.get_balance().unwrap(), details.received); + assert_eq!(wallet.get_balance().unwrap(), 1_000 - details.fees, "incorrect balance after send"); + assert_eq!(wallet.get_balance().unwrap(), details.received, "incorrect received after send"); let mut builder = wallet.build_fee_bump(details.txid).unwrap(); builder.fee_rate(FeeRate::from_sat_per_vb(5.0)); @@ -734,10 +746,10 @@ macro_rules! bdk_blockchain_tests { assert!(finalized, "Cannot finalize transaction"); wallet.broadcast(new_psbt.extract_tx()).unwrap(); wallet.sync(noop_progress(), None).unwrap(); - assert_eq!(wallet.get_balance().unwrap(), 0); - assert_eq!(new_details.received, 0); + assert_eq!(wallet.get_balance().unwrap(), 0, "incorrect balance after change removal"); + assert_eq!(new_details.received, 0, "incorrect received after change removal"); - assert!(new_details.fees > details.fees); + assert!(new_details.fees > details.fees, "incorrect fees"); } #[test] @@ -751,7 +763,7 @@ macro_rules! bdk_blockchain_tests { }); wallet.sync(noop_progress(), None).unwrap(); - assert_eq!(wallet.get_balance().unwrap(), 75_000); + assert_eq!(wallet.get_balance().unwrap(), 75_000, "incorrect balance"); let mut builder = wallet.build_tx(); builder.add_recipient(node_addr.script_pubkey().clone(), 49_000).enable_rbf(); @@ -760,8 +772,8 @@ macro_rules! bdk_blockchain_tests { assert!(finalized, "Cannot finalize transaction"); wallet.broadcast(psbt.extract_tx()).unwrap(); wallet.sync(noop_progress(), None).unwrap(); - assert_eq!(wallet.get_balance().unwrap(), 26_000 - details.fees); - assert_eq!(details.received, 1_000 - details.fees); + assert_eq!(wallet.get_balance().unwrap(), 26_000 - details.fees, "incorrect balance after send"); + assert_eq!(details.received, 1_000 - details.fees, "incorrect received after send"); let mut builder = wallet.build_fee_bump(details.txid).unwrap(); builder.fee_rate(FeeRate::from_sat_per_vb(10.0)); @@ -770,8 +782,8 @@ macro_rules! bdk_blockchain_tests { assert!(finalized, "Cannot finalize transaction"); wallet.broadcast(new_psbt.extract_tx()).unwrap(); wallet.sync(noop_progress(), None).unwrap(); - assert_eq!(new_details.sent, 75_000); - assert_eq!(wallet.get_balance().unwrap(), new_details.received); + assert_eq!(new_details.sent, 75_000, "incorrect sent"); + assert_eq!(wallet.get_balance().unwrap(), new_details.received, "incorrect balance after add input"); } #[test] @@ -785,7 +797,7 @@ macro_rules! bdk_blockchain_tests { }); wallet.sync(noop_progress(), None).unwrap(); - assert_eq!(wallet.get_balance().unwrap(), 75_000); + assert_eq!(wallet.get_balance().unwrap(), 75_000, "incorrect balance"); let mut builder = wallet.build_tx(); builder.add_recipient(node_addr.script_pubkey().clone(), 49_000).enable_rbf(); @@ -794,8 +806,8 @@ macro_rules! bdk_blockchain_tests { assert!(finalized, "Cannot finalize transaction"); wallet.broadcast(psbt.extract_tx()).unwrap(); wallet.sync(noop_progress(), None).unwrap(); - assert_eq!(wallet.get_balance().unwrap(), 26_000 - details.fees); - assert_eq!(details.received, 1_000 - details.fees); + assert_eq!(wallet.get_balance().unwrap(), 26_000 - details.fees, "incorrect balance after send"); + assert_eq!(details.received, 1_000 - details.fees, "incorrect received after send"); let mut builder = wallet.build_fee_bump(details.txid).unwrap(); builder.fee_rate(FeeRate::from_sat_per_vb(123.0)); @@ -806,24 +818,33 @@ macro_rules! bdk_blockchain_tests { assert!(finalized, "Cannot finalize transaction"); wallet.broadcast(new_psbt.extract_tx()).unwrap(); wallet.sync(noop_progress(), None).unwrap(); - assert_eq!(new_details.sent, 75_000); - assert_eq!(wallet.get_balance().unwrap(), 0); - assert_eq!(new_details.received, 0); + assert_eq!(new_details.sent, 75_000, "incorrect sent"); + assert_eq!(wallet.get_balance().unwrap(), 0, "incorrect balance after add input"); + assert_eq!(new_details.received, 0, "incorrect received after add input"); } #[test] #[serial] fn test_sync_receive_coinbase() { let (wallet, _, mut test_client) = init_single_sig(); - let wallet_addr = wallet.get_address(New).unwrap().address; + + let wallet_addr = wallet.get_address($crate::wallet::AddressIndex::New).unwrap().address; wallet.sync(noop_progress(), None).unwrap(); - assert_eq!(wallet.get_balance().unwrap(), 0); + assert_eq!(wallet.get_balance().unwrap(), 0, "incorrect balance"); test_client.generate(1, Some(wallet_addr)); + #[cfg(feature = "rpc")] + { + // rpc consider coinbase only when mature (100 blocks) + let node_addr = test_client.get_node_address(None); + test_client.generate(100, Some(node_addr)); + } + + wallet.sync(noop_progress(), None).unwrap(); - assert!(wallet.get_balance().unwrap() > 0); + assert!(wallet.get_balance().unwrap() > 0, "incorrect balance after receiving coinbase"); } } } diff --git a/src/types.rs b/src/types.rs index 780a6521..926f926e 100644 --- a/src/types.rs +++ b/src/types.rs @@ -80,7 +80,7 @@ impl std::default::Default for FeeRate { /// An unspent output owned by a [`Wallet`]. /// /// [`Wallet`]: crate::Wallet -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Hash)] pub struct LocalUtxo { /// Reference to a transaction output pub outpoint: OutPoint, diff --git a/src/wallet/mod.rs b/src/wallet/mod.rs index 0ec2a95e..767db40c 100644 --- a/src/wallet/mod.rs +++ b/src/wallet/mod.rs @@ -1489,12 +1489,14 @@ where false => 0, true => max_address_param.unwrap_or(CACHE_ADDR_BATCH_SIZE), }; + debug!("max_address {}", max_address); if self .database .borrow() .get_script_pubkey_from_path(KeychainKind::External, max_address.saturating_sub(1))? .is_none() { + debug!("caching external addresses"); run_setup = true; self.cache_addresses(KeychainKind::External, 0, max_address)?; } @@ -1511,11 +1513,13 @@ where .get_script_pubkey_from_path(KeychainKind::Internal, max_address.saturating_sub(1))? .is_none() { + debug!("caching internal addresses"); run_setup = true; self.cache_addresses(KeychainKind::Internal, 0, max_address)?; } } + debug!("run_setup: {}", run_setup); // TODO: what if i generate an address first and cache some addresses? // TODO: we should sync if generating an address triggers a new batch to be stored if run_setup { From 0b969657cdb6957a9e64f7f1b478fe882cf62971 Mon Sep 17 00:00:00 2001 From: Riccardo Casatta Date: Mon, 31 May 2021 13:35:58 +0200 Subject: [PATCH 02/11] update changelog with rpc feature --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 60d0241f..9b2f1d87 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added an option that must be explicitly enabled to allow signing using non-`SIGHASH_ALL` sighashes (#350) #### Changed `get_address` now returns an `AddressInfo` struct that includes the index and derefs to `Address`. +#### Added +- Bitcoin core RPC added as blockchain backend ## [v0.7.0] - [v0.6.0] From dffb753ce3cfcb67ce2736e5e0bd9bf3ffb9264e Mon Sep 17 00:00:00 2001 From: Riccardo Casatta Date: Tue, 1 Jun 2021 14:15:46 +0200 Subject: [PATCH 03/11] match also on signet --- src/blockchain/rpc.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/blockchain/rpc.rs b/src/blockchain/rpc.rs index 85071f9c..e944e47b 100644 --- a/src/blockchain/rpc.rs +++ b/src/blockchain/rpc.rs @@ -332,6 +332,7 @@ impl ConfigurableBlockchain for RpcBlockchain { "main" => Network::Bitcoin, "test" => Network::Testnet, "regtest" => Network::Regtest, + "signet" => Network::Signet, _ => return Err(Error::Generic("Invalid network".to_string())), }; if network != config.network { From 9b7ed08891ac0aa6404a975a10fd16010ff6c455 Mon Sep 17 00:00:00 2001 From: Riccardo Casatta Date: Tue, 1 Jun 2021 14:17:37 +0200 Subject: [PATCH 04/11] rename struct to CallResult --- src/blockchain/rpc.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/blockchain/rpc.rs b/src/blockchain/rpc.rs index e944e47b..f272246d 100644 --- a/src/blockchain/rpc.rs +++ b/src/blockchain/rpc.rs @@ -403,11 +403,11 @@ fn list_wallet_dir(client: &Client) -> Result, Error> { name: String, } #[derive(Deserialize)] - struct Result { + struct CallResult { wallets: Vec, } - let result: Result = client.call("listwalletdir", &[])?; + let result: CallResult = client.call("listwalletdir", &[])?; Ok(result.wallets.into_iter().map(|n| n.name).collect()) } From e1b037a921472b57e397aac7014869c4d315e98c Mon Sep 17 00:00:00 2001 From: Riccardo Casatta Date: Tue, 1 Jun 2021 14:18:40 +0200 Subject: [PATCH 05/11] change feature to execute sync from rpc to test-rpc --- .github/workflows/cont_integration.yml | 2 +- Cargo.toml | 1 + run_blockchain_tests.sh | 2 +- src/testutils/blockchain_tests.rs | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cont_integration.yml b/.github/workflows/cont_integration.yml index d9d25094..f45d00b6 100644 --- a/.github/workflows/cont_integration.yml +++ b/.github/workflows/cont_integration.yml @@ -124,7 +124,7 @@ jobs: - name: start ${{ matrix.blockchain.name }} run: nohup ${{ matrix.blockchain.start }} & sleep 5 - name: Test - run: $HOME/.cargo/bin/cargo test --features ${{ matrix.blockchain.name }},test-blockchains --no-default-features ${{ matrix.blockchain.name }}::bdk_blockchain_tests + run: $HOME/.cargo/bin/cargo test --features test-${{ matrix.blockchain.name }},test-blockchains --no-default-features ${{ matrix.blockchain.name }}::bdk_blockchain_tests check-wasm: name: Check WASM diff --git a/Cargo.toml b/Cargo.toml index 8c79559b..446f6d3e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -63,6 +63,7 @@ rpc = ["bitcoincore-rpc"] test-blockchains = ["bitcoincore-rpc", "electrum-client"] test-electrum = ["electrum"] test-rpc = ["rpc"] +test-esplora = ["esplora"] test-md-docs = ["electrum"] [dev-dependencies] diff --git a/run_blockchain_tests.sh b/run_blockchain_tests.sh index 0ee3eb76..cd094f5c 100755 --- a/run_blockchain_tests.sh +++ b/run_blockchain_tests.sh @@ -65,4 +65,4 @@ while ! cli getwalletinfo >/dev/null; do sleep 1; done # sleep again for good measure! sleep 1; -cargo test --features "test-blockchains,$blockchain" --no-default-features "$blockchain::bdk_blockchain_tests::$test_name" +cargo test --features "test-blockchains,test-$blockchain" --no-default-features "$blockchain::bdk_blockchain_tests::$test_name" diff --git a/src/testutils/blockchain_tests.rs b/src/testutils/blockchain_tests.rs index cd40e560..0b2b559e 100644 --- a/src/testutils/blockchain_tests.rs +++ b/src/testutils/blockchain_tests.rs @@ -370,7 +370,7 @@ macro_rules! bdk_blockchain_tests { let wallet = get_wallet_from_descriptors(&descriptors); // rpc need to call import_multi before receiving any tx, otherwise will not see tx in the mempool - #[cfg(feature = "rpc")] + #[cfg(feature = "test-rpc")] wallet.sync(noop_progress(), None).unwrap(); (wallet, descriptors, test_client) From 81851190f037d81680c8b6a6e235cea10a4cb79c Mon Sep 17 00:00:00 2001 From: Riccardo Casatta Date: Tue, 1 Jun 2021 16:19:32 +0200 Subject: [PATCH 06/11] correctly initialize UTXO keychain kind --- src/blockchain/rpc.rs | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/blockchain/rpc.rs b/src/blockchain/rpc.rs index f272246d..ea5b8dd1 100644 --- a/src/blockchain/rpc.rs +++ b/src/blockchain/rpc.rs @@ -242,17 +242,22 @@ impl Blockchain for RpcBlockchain { } } - let current_utxos: HashSet = current_utxo + let current_utxos: HashSet<_> = current_utxo .into_iter() - .map(|u| LocalUtxo { - outpoint: OutPoint::new(u.txid, u.vout), - txout: TxOut { - value: u.amount.as_sat(), - script_pubkey: u.script_pub_key, - }, - keychain: KeychainKind::External, + .map(|u| { + Ok(LocalUtxo { + outpoint: OutPoint::new(u.txid, u.vout), + keychain: db + .get_path_from_script_pubkey(&u.script_pub_key)? + .ok_or(Error::TransactionNotFound)? + .0, + txout: TxOut { + value: u.amount.as_sat(), + script_pubkey: u.script_pub_key, + }, + }) }) - .collect(); + .collect::>()?; let spent: HashSet<_> = known_utxos.difference(¤t_utxos).collect(); for s in spent { From ab982831596c1f555a079b44348eaadc10bd9231 Mon Sep 17 00:00:00 2001 From: Riccardo Casatta Date: Wed, 2 Jun 2021 10:06:05 +0200 Subject: [PATCH 07/11] always ask node for tx no matter capabilities --- src/blockchain/rpc.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/blockchain/rpc.rs b/src/blockchain/rpc.rs index ea5b8dd1..4572dbe0 100644 --- a/src/blockchain/rpc.rs +++ b/src/blockchain/rpc.rs @@ -280,11 +280,7 @@ impl Blockchain for RpcBlockchain { } fn get_tx(&self, txid: &Txid) -> Result, Error> { - if self.capabilities.contains(&Capability::FullHistory) { - Ok(Some(self.client.get_raw_transaction(txid, None)?)) - } else { - Ok(None) - } + Ok(Some(self.client.get_raw_transaction(txid, None)?)) } fn broadcast(&self, tx: &Transaction) -> Result<(), Error> { From ae5aa06586cdaefae0fb84abf95200060cda625e Mon Sep 17 00:00:00 2001 From: Riccardo Casatta Date: Thu, 3 Jun 2021 11:06:24 +0200 Subject: [PATCH 08/11] use storage address instead of satoshi's --- src/blockchain/rpc.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/blockchain/rpc.rs b/src/blockchain/rpc.rs index 4572dbe0..e943ba9e 100644 --- a/src/blockchain/rpc.rs +++ b/src/blockchain/rpc.rs @@ -58,7 +58,7 @@ pub struct RpcBlockchain { skip_blocks: Option, /// This is a fixed Address used as a hack key to store information on the node - _satoshi_address: Address, + _storage_address: Address, } /// RpcBlockchain configuration options @@ -78,7 +78,7 @@ pub struct RpcConfig { impl RpcBlockchain { fn get_node_synced_height(&self) -> Result { - let info = self.client.get_address_info(&self._satoshi_address)?; + let info = self.client.get_address_info(&self._storage_address)?; if let Some(GetAddressInfoResultLabel::Simple(label)) = info.labels.first() { Ok(label .parse::() @@ -93,7 +93,7 @@ impl RpcBlockchain { fn set_node_synced_height(&self, height: u32) -> Result<(), Error> { Ok(self .client - .set_label(&self._satoshi_address, &height.to_string())?) + .set_label(&self._storage_address, &height.to_string())?) } } @@ -354,14 +354,15 @@ impl ConfigurableBlockchain for RpcBlockchain { } // this is just a fixed address used only to store a label containing the synced height in the node - let mut satoshi_address = Address::from_str("1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa").unwrap(); - satoshi_address.network = network; + let mut storage_address = + Address::from_str("bc1qst0rewf0wm4kw6qn6kv0e5tc56nkf9yhcxlhqv").unwrap(); + storage_address.network = network; Ok(RpcBlockchain { client, network, capabilities, - _satoshi_address: satoshi_address, + _storage_address: storage_address, skip_blocks: config.skip_blocks, }) } From ab54a17eb7580127cc6c1f7f74baa064064710ee Mon Sep 17 00:00:00 2001 From: Riccardo Casatta Date: Thu, 3 Jun 2021 11:07:39 +0200 Subject: [PATCH 09/11] update bitcoind dep --- Cargo.toml | 2 +- src/blockchain/rpc.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 446f6d3e..e6249222 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,7 +72,7 @@ env_logger = "0.7" base64 = "^0.11" clap = "2.33" serial_test = "0.4" -bitcoind = "0.9.0" +bitcoind = "0.10.0" [[example]] name = "address_validator" diff --git a/src/blockchain/rpc.rs b/src/blockchain/rpc.rs index e943ba9e..32c2d82e 100644 --- a/src/blockchain/rpc.rs +++ b/src/blockchain/rpc.rs @@ -471,7 +471,7 @@ mod test { let config = RpcConfig { url: bitcoind.rpc_url(), - auth: Auth::CookieFile(bitcoind.cookie_file.clone()), + auth: Auth::CookieFile(bitcoind.config.cookie_file.clone()), network, wallet_name, skip_blocks: None, @@ -523,7 +523,7 @@ mod test { generate(&bitcoind, 5); let config = RpcConfig { url: bitcoind.rpc_url(), - auth: Auth::CookieFile(bitcoind.cookie_file.clone()), + auth: Auth::CookieFile(bitcoind.config.cookie_file.clone()), network: Network::Regtest, wallet_name: "another-name".to_string(), skip_blocks: Some(103), From 1639984b5658f22002f1478cc93791db963bca4e Mon Sep 17 00:00:00 2001 From: Riccardo Casatta Date: Thu, 3 Jun 2021 15:10:31 +0200 Subject: [PATCH 10/11] move scan in setup --- src/blockchain/rpc.rs | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/src/blockchain/rpc.rs b/src/blockchain/rpc.rs index 32c2d82e..1c4c6cf6 100644 --- a/src/blockchain/rpc.rs +++ b/src/blockchain/rpc.rs @@ -130,25 +130,12 @@ impl Blockchain for RpcBlockchain { // https://bitcoindevkit.org/descriptors/#compatibility-matrix //TODO maybe convenient using import_descriptor for compatible descriptor and import_multi as fallback self.client.import_multi(&requests, Some(&options))?; - self.sync(stop_gap, database, progress_update) - } - fn sync( - &self, - _stop_gap: Option, - db: &mut D, - progress_update: P, - ) -> Result<(), Error> { let current_height = self.get_height()?; // min because block invalidate may cause height to go down let node_synced = self.get_node_synced_height()?.min(current_height); - let mut indexes = HashMap::new(); - for keykind in &[KeychainKind::External, KeychainKind::Internal] { - indexes.insert(*keykind, db.get_last_index(*keykind)?.unwrap_or(0)); - } - //TODO call rescan in chunks (updating node_synced_height) so that in case of // interruption work can be partially recovered debug!( @@ -159,6 +146,22 @@ impl Blockchain for RpcBlockchain { .rescan_blockchain(Some(node_synced as usize), Some(current_height as usize))?; progress_update.update(1.0, None)?; + self.set_node_synced_height(current_height)?; + + self.sync(stop_gap, database, progress_update) + } + + fn sync( + &self, + _stop_gap: Option, + db: &mut D, + _progress_update: P, + ) -> Result<(), Error> { + let mut indexes = HashMap::new(); + for keykind in &[KeychainKind::External, KeychainKind::Internal] { + indexes.insert(*keykind, db.get_last_index(*keykind)?.unwrap_or(0)); + } + let mut known_txs: HashMap<_, _> = db .iter_txs(true)? .into_iter() @@ -275,7 +278,6 @@ impl Blockchain for RpcBlockchain { db.set_last_index(keykind, index)?; } - self.set_node_synced_height(current_height)?; Ok(()) } From ba2e3042cca6ba3dd4cf2b58880bdc759c3607a3 Mon Sep 17 00:00:00 2001 From: Riccardo Casatta Date: Fri, 4 Jun 2021 15:05:35 +0200 Subject: [PATCH 11/11] add details to TODO, format doc example --- src/blockchain/rpc.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/blockchain/rpc.rs b/src/blockchain/rpc.rs index 1c4c6cf6..40ec0ae6 100644 --- a/src/blockchain/rpc.rs +++ b/src/blockchain/rpc.rs @@ -18,12 +18,12 @@ //! ```no_run //! # use bdk::blockchain::{RpcConfig, RpcBlockchain, ConfigurableBlockchain}; //! let config = RpcConfig { -//! url: "127.0.0.1:18332".to_string(), -//! auth: bitcoincore_rpc::Auth::CookieFile("/home/user/.bitcoin/.cookie".into()), -//! network: bdk::bitcoin::Network::Testnet, -//! wallet_name: "wallet_name".to_string(), -//! skip_blocks: None, -//! }; +//! url: "127.0.0.1:18332".to_string(), +//! auth: bitcoincore_rpc::Auth::CookieFile("/home/user/.bitcoin/.cookie".into()), +//! network: bdk::bitcoin::Network::Testnet, +//! wallet_name: "wallet_name".to_string(), +//! skip_blocks: None, +//! }; //! let blockchain = RpcBlockchain::from_config(&config); //! ``` @@ -227,7 +227,9 @@ impl Blockchain for RpcBlockchain { timestamp: tx_result.info.time, received, sent, - fees: tx_result.fee.map(|f| f.as_sat().abs() as u64).unwrap_or(0), //TODO + //TODO it could happen according to the node situation/configuration that the + // fee is not known [TransactionDetails:fee] should be made [Option] + fees: tx_result.fee.map(|f| f.as_sat().abs() as u64).unwrap_or(0), height: tx_result.info.blockheight, }; debug!(